From 1b9a735b893e879ce154036e414099fb46610b9d Mon Sep 17 00:00:00 2001 From: ffgan Date: Tue, 12 Aug 2025 14:40:14 +0800 Subject: [PATCH 01/10] containerize Co-authored by: nijincheng@iscas.ac.cn; --- deploy/container/.gitignore | 5 + deploy/container/README.md | 66 ++++ deploy/container/chacra_image/Dockerfile | 53 +++ deploy/container/chacra_image/README.md | 12 + .../chacra_image/conf/alembic-prod.ini | 68 ++++ deploy/container/chacra_image/conf/nginx.conf | 68 ++++ .../chacra_image/conf/nginx_site.conf | 51 +++ deploy/container/chacra_image/conf/prod.py | 370 ++++++++++++++++++ .../chacra_image/conf/prod_api_creds.py | 3 + deploy/container/chacra_image/conf/prod_db.py | 9 + .../container/chacra_image/init-chacra-db.sh | 38 ++ deploy/container/chacra_image/main.sh | 18 + deploy/container/compose.yml | 63 +++ 13 files changed, 824 insertions(+) create mode 100644 deploy/container/.gitignore create mode 100644 deploy/container/README.md create mode 100644 deploy/container/chacra_image/Dockerfile create mode 100644 deploy/container/chacra_image/README.md create mode 100644 deploy/container/chacra_image/conf/alembic-prod.ini create mode 100644 deploy/container/chacra_image/conf/nginx.conf create mode 100644 deploy/container/chacra_image/conf/nginx_site.conf create mode 100644 deploy/container/chacra_image/conf/prod.py create mode 100644 deploy/container/chacra_image/conf/prod_api_creds.py create mode 100644 deploy/container/chacra_image/conf/prod_db.py create mode 100755 deploy/container/chacra_image/init-chacra-db.sh create mode 100755 deploy/container/chacra_image/main.sh create mode 100644 deploy/container/compose.yml diff --git a/deploy/container/.gitignore b/deploy/container/.gitignore new file mode 100644 index 00000000..c6df9f63 --- /dev/null +++ b/deploy/container/.gitignore @@ -0,0 +1,5 @@ +bin/** +repos/** +pgdata/** +logs/** +*.deb \ No newline at end of file diff --git a/deploy/container/README.md b/deploy/container/README.md new file mode 100644 index 00000000..c902e1b0 --- /dev/null +++ b/deploy/container/README.md @@ -0,0 +1,66 @@ +## 1. what's this + +Deploy chacra to a container. + +This is essentially done by following an Ansible script, using Docker Compose to start RabbitMQ and PostgreSQL. Then, run chacra and Nginx inside the container. + +It's in a pretty **raw** state right now, just enough to start and test chacra. + + +## 2. start + +### 1. build chacra image +```shell +cd chacra_image/ +sudo docker build -t chacra . +cd .. +``` + +### 2. start +```shell +sudo docker-compose up -d +sleep 30 +``` +please wait about 30s. + + +### 3. confiugre chacractl and test chacra + +install chacractl first +```shell +pip install chacractl +``` + +set below txt to ~/.chacractl + +```text +# This file was automatically generated by the chacractl CLI +# make sure to update it with the correct user and key to talk to the API + +url = "http://127.0.0.1/" +user = "admin" +key = "secret" +ssl_verify = False +``` + + +### 4. test + +```shell +wget https://chacra.ceph.com/r/libboost/master/9ea1fb8bdad548a88004db87761f173aa50dcc85/ubuntu/jammy/flavors/default/pool/main/b/boost1.87/ceph-libboost-atomic1.87-dev_1.87.0-1.1_amd64.deb + +ls *.deb | chacractl binary create libboost/master/9ea1fb8bdad548a88004db87761f173aa50dcc85/ubuntu/jammy/amd64/flavors/default + +rm -f ./ceph-libboost-atomic1.87-dev_1.87.0-1.1_amd64.deb +# wget http://127.0.0.1/binaries/libboost/master/9ea1fb8bdad548a88004db87761f173aa50dcc85/ubuntu/jammy/amd64/flavors/default/ceph-libboost-atomic1.87-dev_1.87.0-1.1_amd64.deb +``` + + +### 5. clean service +```shell +rm -f ./ceph-libboost-atomic1.87-dev_1.87.0-1.1_amd64.deb +sudo docker compose down +``` + +## 3. override conf without rebuild image +please see compose.yml:50 . \ No newline at end of file diff --git a/deploy/container/chacra_image/Dockerfile b/deploy/container/chacra_image/Dockerfile new file mode 100644 index 00000000..9f1d2a9c --- /dev/null +++ b/deploy/container/chacra_image/Dockerfile @@ -0,0 +1,53 @@ +FROM ubuntu:20.04 + +ARG APP_NAME="chacra" +# please do not pass env to container while start container. +# those env is valid on build image,not run time. +ENV APP_NAME=${APP_NAME} +ENV APP_HOME="/opt/${APP_NAME}" +ENV DEBIAN_FRONTEND=noninteractive + +# In Ubuntu 20.04, createrepo cannot be installed. In Ubuntu 22.04, we may be able to use createrepo-c instead of createrepo, but in Ubuntu 22.04, chacra has some Python problems(install celery[librabbitmq] will crash). Since the current chacra works fine in Ubuntu 20.04, we'll just ignore createrepo for now. +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + virtualenv \ + librabbitmq-dev \ + libffi-dev \ + pkg-config \ + postgresql-client \ + sudo \ + git cmake libtool autoconf nginx reprepro \ + && mkdir -p /var/log/celery /etc/nginx/sites-available /etc/nginx/sites-enabled \ + && rm -f /etc/nginx/sites-enabled/default \ + && mkdir -p /var/lib/nginx/body /var/log/nginx \ + && useradd --system --no-create-home --shell /bin/false --user-group nginx \ + && chown -R nginx:nginx /var/lib/nginx /var/log/nginx \ + && chmod -R 755 /var/lib/nginx /var/log/nginx + +# In Ubuntu 20.04, the Python version is 3.8. For chacra, we need to make some changes to satisfy Python 3.8 compatibility. So I use my repository instead of ceph/chakra. +RUN virtualenv ${APP_HOME} \ + && . ${APP_HOME}/bin/activate \ + && pip install --upgrade pip setuptools\ + && pip install -I -e git+https://github.com/ceph/chacra@main#egg=chacra \ + && pip install librabbitmq && pip install -r ${APP_HOME}/src/${APP_NAME}/requirements.txt + +COPY ./py3.8.patch ${APP_HOME}/src/${APP_NAME}/py3.8.patch + +RUN cd ${APP_HOME}/src/${APP_NAME} && git apply py3.8.patch + +COPY ./main.sh /bin/main.sh +COPY ./init-chacra-db.sh ${APP_HOME}/init-chacra-db.sh + +COPY ./conf/*.py ${APP_HOME}/src/${APP_NAME}/ +COPY ./conf/alembic-prod.ini ${APP_HOME}/src/${APP_NAME}/alembic-prod.ini + +COPY ./conf/nginx.conf /etc/nginx/nginx.conf +COPY ./conf/nginx_site.conf /etc/nginx/sites-available/${APP_NAME}.conf + +RUN ln -s /etc/nginx/sites-available/${APP_NAME}.conf /etc/nginx/sites-enabled/${APP_NAME}.conf + +ENV PECAN_CONFIG=${APP_HOME}/src/${APP_NAME}/prod.py + + +CMD ["/bin/sh","-c","/bin/main.sh"] \ No newline at end of file diff --git a/deploy/container/chacra_image/README.md b/deploy/container/chacra_image/README.md new file mode 100644 index 00000000..5a8b7112 --- /dev/null +++ b/deploy/container/chacra_image/README.md @@ -0,0 +1,12 @@ +## 1. APP_NAME + +The default APP_NAME is chacra. + +If you need to modify APP_NAME, please manually modify files including but not limited to nginx.conf,nginx_site.conf, etc. The reason one-click modification is not supported is that there are too many highly customized areas, and automatic modification would result in loss of universality. Therefore, if modification is needed, it can be implemented manually, which is actually not troublesome. Finally, pass this parameter when building. + +Please note that passing this parameter when starting the container is ineffective. + + +## 2. how to start this container + +This container cannot be started independently. You need to use docker compose to start postgresql and rabbitmq, and then the chacra container needs to be able to access the other two services via localhost. \ No newline at end of file diff --git a/deploy/container/chacra_image/conf/alembic-prod.ini b/deploy/container/chacra_image/conf/alembic-prod.ini new file mode 100644 index 00000000..8dbf5479 --- /dev/null +++ b/deploy/container/chacra_image/conf/alembic-prod.ini @@ -0,0 +1,68 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = /opt/chacra/src/chacra/alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat alembic/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = postgresql://postgres:example@127.0.0.1/chacra + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/deploy/container/chacra_image/conf/nginx.conf b/deploy/container/chacra_image/conf/nginx.conf new file mode 100644 index 00000000..912dd773 --- /dev/null +++ b/deploy/container/chacra_image/conf/nginx.conf @@ -0,0 +1,68 @@ +# {{ ansible_managed }} +user nginx; +worker_processes 20; +worker_rlimit_nofile 8192; + +pid /var/run/nginx.pid; + +events { + worker_connections 1024; + # multi_accept on; +} + +http { + + ## + # Basic Settings + ## + + #sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + server_tokens off; + + # server_names_hash_bucket_size 64; + # server_name_in_redirect off; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + ## + # Logging Settings + ## + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + ## + # Gzip Settings + ## + + gzip on; + gzip_disable "msie6"; + + # gzip_vary on; + # gzip_proxied any; + # gzip_comp_level 6; + # gzip_buffers 16 8k; + # gzip_http_version 1.1; + # gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; + + ## + # If HTTPS, then set a variable so it can be passed along. + ## + + map $scheme $server_https { + default off; + https on; + } + + ## + # Virtual Host Configs + ## + + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; +} diff --git a/deploy/container/chacra_image/conf/nginx_site.conf b/deploy/container/chacra_image/conf/nginx_site.conf new file mode 100644 index 00000000..aca164a5 --- /dev/null +++ b/deploy/container/chacra_image/conf/nginx_site.conf @@ -0,0 +1,51 @@ +# server { +# server_name localhost; +# location '/.well-known/acme-challenge' { +# default_type "text/plain"; +# root /var/www/localhost; +# } +# location / { +# add_header Strict-Transport-Security max-age=31536000; +# return 301 https://$server_name$request_uri; +# } +# } + +server { + listen 80 default_server; + server_name localhost; + +# {% if ssl_cert_exists.stat.exists == true %} +# ssl_certificate {{ nginx_ssl_cert_path }}; +# ssl_certificate_key {{ nginx_ssl_key_path }}; +# {% endif %} + # ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + # ssl_ciphers 'EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH'; + # ssl_prefer_server_ciphers on; + add_header Strict-Transport-Security "max-age=31536000"; + + access_log /var/log/nginx/chacra-access.log; + error_log /var/log/nginx/chacra-error.log; + + # Some binaries are gigantic + client_max_body_size 4096m; + + location / { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_pass http://127.0.0.1:8000; + proxy_read_timeout 5000; + } + + location /r/ { + autoindex on; + alias /opt/chacra/repos/; + } + + location /b/ { + internal; + alias /opt/chacra/binaries/; + } +} diff --git a/deploy/container/chacra_image/conf/prod.py b/deploy/container/chacra_image/conf/prod.py new file mode 100644 index 00000000..961f51ea --- /dev/null +++ b/deploy/container/chacra_image/conf/prod.py @@ -0,0 +1,370 @@ +from pecan.hooks import TransactionHook +from chacra import models +from chacra import hooks +from prod_db import sqlalchemy +from prod_api_creds import api_key, api_user + + +# Server Specific Configurations +server = {"port": "8000", "host": "0.0.0.0"} + +# Pecan Application Configurations +app = { + "root": "chacra.controllers.root.RootController", + "modules": ["chacra"], + "guess_content_type_from_ext": False, + "template_path": "%(confdir)s/chacra/templates", + "hooks": [ + TransactionHook( + models.start, + models.start_read_only, + models.commit, + models.rollback, + models.clear, + ), + hooks.CustomErrorHook(), + ], + "debug": False, +} + +logging = { + "disable_existing_loggers": False, + "loggers": { + "root": {"level": "INFO", "handlers": ["console"]}, + "chacra": {"level": "DEBUG", "handlers": ["console"]}, + "pecan": {"level": "WARNING", "handlers": ["console"]}, + "pecan.commands.serve": {"level": "DEBUG", "handlers": ["console"]}, + "py.warnings": {"handlers": ["console"]}, + "__force_dict__": True, + }, + "handlers": { + "console": { + "level": "DEBUG", + "class": "logging.StreamHandler", + "formatter": "simple", + } + }, + "formatters": { + "simple": { + "format": ( + "%(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s" + ) + }, + "color": { + "()": "pecan.log.ColorFormatter", + "format": ( + "%(asctime)s [%(padded_color_levelname)s] [%(name)s]" + "[%(threadName)s] %(message)s" + ), + "__force_dict__": True, + }, + }, +} + +# graphite/statsd settings +graphite_api_key = "xxxxxx666666" +# this value will be used when sending metrics to graphite +short_hostname = "ubuntu" + +# When True it will set the headers so that Nginx can serve the download +# instead of Pecan. +delegate_downloads = True + +# location for storing uploaded binaries +binary_root = "/opt/chacra/binaries" +repos_root = "/opt/chacra/repos" +distributions_root = "%(confdir)s/distributions" + +# Celery options +# How often (in seconds) the database should be queried for repos that need to +# be rebuilt +polling_cycle = 120 + +# {% if purge_repos is defined %} +# purge_repos = {{ purge_repos }} + +# purge_rotation = { +# 'ceph-medic': { +# 'ref': { +# 'main': { +# 'keep_minimum': 5 +# }, +# }, +# }, +# 'kernel': { +# 'ref': { +# 'testing': { +# 'keep_minimum': 10 +# }, +# 'main': { +# 'keep_minimum': 5 +# }, +# 'for-linus': { +# 'keep_minimum': 5 +# }, +# 'nightly_pre-single-major': { +# 'keep_minimum': 1 +# }, +# 'ceph-iscsi-stable': { +# 'keep_minimum': 1 +# }, +# 'ceph-iscsi-test': { +# 'keep_minimum': 1 +# }, +# }, +# }, +# 'calamari': { +# 'ref': { +# 'main': { +# 'keep_minimum': 5 +# }, +# '1.5': { +# 'keep_minimum': 5 +# }, +# }, +# }, +# 'ceph': { +# 'ref': { +# 'reef-nvmeof': { +# 'keep_minimum': 2 +# }, +# 'squid-nvmeof': { +# 'keep_minimum': 2 +# }, +# 'squid-nvmeof-8.0': { +# 'keep_minimum': 2 +# }, +# 'hammer': { +# 'days': 30, +# 'keep_minimum': 5 +# }, +# 'infernalis': { +# 'days': 30, +# 'keep_minimum': 5 +# }, +# 'jewel': { +# 'days': 14, +# 'keep_minimum': 10 +# }, +# 'kraken': { +# 'days': 14, +# 'keep_minimum': 10 +# }, +# 'luminous': { +# 'days': 14, +# 'keep_minimum': 10 +# }, +# 'mimic': { +# 'days': 14, +# 'keep_minimum': 10 +# }, +# 'nautilus': { +# 'days': 14, +# 'keep_minimum': 10 +# }, +# }, +# }, +# 'ceph-installer': { +# 'ref': { +# 'main': { +# 'keep_minimum': 10 +# }, +# }, +# }, +# 'ceph-ansible': { +# 'ref': { +# 'main': { +# 'keep_minimum': 5 +# }, +# 'stable-2.1': { +# 'keep_minimum': 5 +# }, +# }, +# }, +# 'python-rtslib': { +# 'ref': { +# 'main': { +# 'keep_minimum': 5 +# }, +# }, +# }, +# 'tcmu-runner': { +# 'ref': { +# 'main': { +# 'keep_minimum': 5 +# }, +# }, +# }, +# 'ceph-iscsi': { +# 'ref': { +# 'main': { +# 'keep_minimum': 5 +# }, +# }, +# }, +# 'ceph-iscsi-config': { +# 'ref': { +# 'main': { +# 'keep_minimum': 5 +# }, +# }, +# }, +# 'ceph-iscsi-cli': { +# 'ref': { +# 'main': { +# 'keep_minimum': 5 +# }, +# }, +# }, +# 'ceph-iscsi-tools': { +# 'ref': { +# 'main': { +# 'keep_minimum': 5 +# }, +# }, +# }, +# 'nfs-ganesha': { +# 'flavor': { +# 'ceph_mimic': { +# 'keep_minimum': 2 +# }, +# 'ceph_luminous': { +# 'keep_minimum': 2 +# }, +# 'ceph_kraken': { +# 'keep_minimum': 2 +# }, +# 'ceph_jewel': { +# 'keep_minimum': 2 +# }, +# }, +# }, +# '__force_dict__': True, +# } + +# {% endif %} + +# Once a "create repo" task is called, how many seconds (if any) to wait before actually +# creating the repository +quiet_time = 20 + +repos = { + "ceph": { + # 'automatic': False, + # ceph-deploy production builds should go into the "ref" main + # because otherwise we would be forced to list every "vN.N.N" ref + # here to avoid getting cruft like "test" builds + "all": { + "ceph-medic": ["main"], + }, + "hammer": { + "ceph-release": ["hammer"], + "ceph-deploy": ["1.5"], + }, + "infernalis": { + "ceph-release": ["infernalis"], + "ceph-deploy": ["1.5"], + }, + "jewel": { + "ceph-release": ["jewel"], + "ceph-deploy": ["1.5"], + }, + "kraken": { + "ceph-release": ["kraken"], + "ceph-deploy": ["1.5"], + }, + "luminous": { + "ceph-release": ["luminous"], + "ceph-deploy": ["main"], + }, + "mimic": { + "ceph-release": ["mimic"], + "ceph-deploy": ["main"], + }, + "nautilus": { + "ceph-release": ["nautilus"], + "ceph-deploy": ["main"], + }, + "octopus": { + "ceph-release": ["octopus"], + "ceph-deploy": ["main"], + }, + "pacific": { + "ceph-release": ["pacific"], + "ceph-deploy": ["main"], + }, + "quincy": { + "ceph-release": ["quincy"], + "ceph-deploy": ["main"], + }, + "reef": { + "ceph-release": ["reef"], + "ceph-deploy": ["main"], + }, + "squid": { + "ceph-release": ["squid"], + "ceph-deploy": ["main"], + }, + # when more 'testing' refs are built, we need to add them here as well + "testing": { + "ceph": ["jewel-rc"], + }, + # {% if combine_deb_repos|default(True) %} + # note: 'universal' binaries will be included to all these distro + # versions since they do not belong to any one in particular. + "combined": [ + "bookworm", + "bullseye", + "buster", + "stretch", + "wheezy", + "trusty", + "precise", + "jessie", + "xenial", + "bionic", + "focal", + "jammy", + "noble", + ], + # {% endif %} + }, + "libboost": {"master": {}}, + "cephmetrics": { + "main": { + "cephmetrics-deps": ["main"], + }, + # For testing cephmetrics changes against existing deps + "develop": { + "cephmetrics-deps": ["main"], + }, + # Release! + "1.0": { + "cephmetrics-deps": ["1.0"], + }, + }, + "__force_dict__": True, +} + + +# Use this to define how distributions files will be created per project +distributions = { + "defaults": { + "DebIndices": "Packages Release . .gz .bz2", + "DscIndices": "Sources Release .gz .bz2", + "Contents": ".gz .bz2", + "Origin": "ceph.com", + "Description": "", + "Architectures": "amd64 arm64 armhf i386 source", + "Suite": "stable", + "Components": "main", + }, + "ceph": { + "Description": "Ceph distributed file system", + }, +} + +# if this file exists the check at /health/ will fail +fail_check_trigger_path = "/tmp/fail_check" + +# production database configurations are imported from prod_db.py diff --git a/deploy/container/chacra_image/conf/prod_api_creds.py b/deploy/container/chacra_image/conf/prod_api_creds.py new file mode 100644 index 00000000..40107100 --- /dev/null +++ b/deploy/container/chacra_image/conf/prod_api_creds.py @@ -0,0 +1,3 @@ +# Basic HTTP Auth credentials +api_user = "admin" +api_key = "secret" diff --git a/deploy/container/chacra_image/conf/prod_db.py b/deploy/container/chacra_image/conf/prod_db.py new file mode 100644 index 00000000..abc7dbf7 --- /dev/null +++ b/deploy/container/chacra_image/conf/prod_db.py @@ -0,0 +1,9 @@ +# {{ ansible_managed }} + +sqlalchemy = { + 'url': 'postgresql+psycopg2://postgres:example@127.0.0.1/chacra', + 'echo': False, + 'echo_pool': False, + 'pool_recycle': 3600, + 'encoding': 'utf-8' +} diff --git a/deploy/container/chacra_image/init-chacra-db.sh b/deploy/container/chacra_image/init-chacra-db.sh new file mode 100755 index 00000000..72a81374 --- /dev/null +++ b/deploy/container/chacra_image/init-chacra-db.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# env like APP_NAME,APP_HOME is pass by dockerfile +echo $APP_NAME +echo $APP_HOME +echo "init chacra database..." + +psql -v ON_ERROR_STOP=1 -h 127.0.0.1 -p 5432 --username postgres --dbname postgres -tc \ + "SELECT 1 FROM pg_database WHERE datname = 'chacra'" | grep -q 1 || \ +psql -v ON_ERROR_STOP=1 -h 127.0.0.1 -p 5432 --username postgres --dbname postgres -c \ + "CREATE DATABASE chacra;" + +echo "init success" +echo "chacra database is prepare done." +echo "check chacra database weather to be fill data..." + +res=$(psql -v ON_ERROR_STOP=1 -h 127.0.0.1 -p 5432 --username postgres --dbname "chacra" -t -c "SELECT COUNT(*) FROM projects;") +DATABASE_CHECK_RC=$? + +echo "check result: code=$DATABASE_CHECK_RC" + +if [ $DATABASE_CHECK_RC -ne 0 ] || [ "$res" = "0" ]; then + echo "need to fill data for ${APP_NAME}..." + + if [ -f "${APP_HOME}/bin/pecan" ] && [ -f "${APP_HOME}/src/${APP_NAME}/prod.py" ]; then + echo "fill data is started..." + export ALEMBIC_CONFIG="${APP_HOME}/src/${APP_NAME}/alembic-prod.ini" + if ${APP_HOME}/bin/pecan populate ${APP_HOME}/src/${APP_NAME}/prod.py; then + echo "fill data done" + else + echo "warn: fill ddta fail,but start container continue" + fi + else + echo "warn: pecan isn't exist" + exit 1 + fi +else + echo "skip fill data" +fi \ No newline at end of file diff --git a/deploy/container/chacra_image/main.sh b/deploy/container/chacra_image/main.sh new file mode 100755 index 00000000..cd21bb5f --- /dev/null +++ b/deploy/container/chacra_image/main.sh @@ -0,0 +1,18 @@ +#! /bin/sh +# env is pass by dockerfile +echo $APP_NAME +echo $APP_HOME +cd ${APP_HOME}/src/${APP_NAME}/${APP_NAME} + +sleep 10 +${APP_HOME}/bin/celery multi start 5 -Q:1,2 poll_repos,celery -Q:3-5 build_repos -A asynch --logfile=/var/log/celery/%n%I.log + +${APP_HOME}/bin/celery -A asynch beat --loglevel=info --logfile=/var/log/celery/beat.log & + +/usr/sbin/nginx -g 'daemon off;' & + + +sleep 10 +env APP_HOME=${APP_HOME} APP_NAME=${APP_NAME} ${APP_HOME}/init-chacra-db.sh + +${APP_HOME}/bin/gunicorn_pecan -w 10 -t 1200 ${APP_HOME}/src/${APP_NAME}/prod.py \ No newline at end of file diff --git a/deploy/container/compose.yml b/deploy/container/compose.yml new file mode 100644 index 00000000..82e55501 --- /dev/null +++ b/deploy/container/compose.yml @@ -0,0 +1,63 @@ +services: + + db: + image: postgres + restart: always + # set shared memory limit when using docker compose + shm_size: 128mb + # or set shared memory limit when deploy via swarm stack + #volumes: + # - type: tmpfs + # target: /dev/shm + # tmpfs: + # size: 134217728 # 128*2^20 bytes = 128Mb + environment: + POSTGRES_PASSWORD: example + POSTGRES_DB: chacra + PGDATA: /var/lib/postgresql/data/pgdata + volumes: + - ./pgdata:/var/lib/postgresql/data + + ports: + - 5432:5432 # PostgreSQL + - 5672:5672 # RabbitMQ + - 15672:15672 # RabbitMQ WebUI + - 8000:8000 # chacra raw restful API + - 80:80 + + rabbitmq: + image: rabbitmq + restart: always + network_mode: "service:db" + + services: + # image is build from ./chacra_image, with tag latest or other else + image: chacra + restart: always + depends_on: + - db + - rabbitmq + network_mode: "service:db" + + volumes: + - ./bin:/opt/chacra/binaries + - ./repos:/opt/chacra/repos + + # we can modify below mount location to override the origin conf,like prod_api_creds. + # this doesn't need to rebuild the image + + # below file is fine to override. like src:dest, u can modify src. + # - ./chacra_image/conf/prod_api_creds.py:/opt/chacra/src/chacra/prod_api_creds.py + # - ./chacra_image/conf/prod.py:/opt/chacra/src/chacra/prod.py + # - ./chacra_image/conf/nginx.conf:/etc/nginx/nginx.conf + # - ./chacra_image/conf/nginx_site.conf:/etc/nginx/sites-available/chacra.conf + # - ./chacra_image/conf/alembic-prod.ini:/opt/chacra/src/chacra/alembic-prod.ini + + - ./logs/celery/:/var/log/celery/ + + # SQL manager WebUI,optional. + # adminer: + # image: adminer + # restart: always + # ports: + # - 8080:8080 From bccba46207bae1cbae4867203022c1947d5df23d Mon Sep 17 00:00:00 2001 From: ffgan Date: Tue, 12 Aug 2025 14:52:07 +0800 Subject: [PATCH 02/10] add patch for python 3.8 while running in container --- deploy/container/chacra_image/py3.8.patch | 65 +++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 deploy/container/chacra_image/py3.8.patch diff --git a/deploy/container/chacra_image/py3.8.patch b/deploy/container/chacra_image/py3.8.patch new file mode 100644 index 00000000..f4ce1888 --- /dev/null +++ b/deploy/container/chacra_image/py3.8.patch @@ -0,0 +1,65 @@ +diff --git a/chacra/controllers/util.py b/chacra/controllers/util.py +index 389a1f8..91084db 100644 +--- a/chacra/controllers/util.py ++++ b/chacra/controllers/util.py +@@ -1,4 +1,7 @@ +-from datetime import datetime, timedelta, UTC ++from datetime import datetime, timedelta, timezone ++ ++UTC = timezone.utc ++ + import logging + from pecan import conf + +diff --git a/chacra/models/__init__.py b/chacra/models/__init__.py +index 0f610bc..ae3d0e1 100644 +--- a/chacra/models/__init__.py ++++ b/chacra/models/__init__.py +@@ -54,7 +54,7 @@ def update_timestamp(mapper, connection, target): + """ + Automate the 'modified' attribute when a model changes + """ +- target.modified = datetime.datetime.now(datetime.UTC) ++ target.modified = datetime.datetime.now(datetime.timezone.utc) + + + # Utilities: +diff --git a/chacra/models/binaries.py b/chacra/models/binaries.py +index 6fcbd08..fd2626f 100644 +--- a/chacra/models/binaries.py ++++ b/chacra/models/binaries.py +@@ -50,7 +50,7 @@ class Binary(Base): + def __init__(self, name, project, repo=None, **kw): + self.name = name + self.project = project +- now = datetime.datetime.now(datetime.UTC) ++ now = datetime.datetime.now(datetime.timezone.utc) + self.created = now + self.modified = now + self.sha1 = kw.get('sha1', 'head') +diff --git a/chacra/models/repos.py b/chacra/models/repos.py +index 8a236e6..1d298df 100644 +--- a/chacra/models/repos.py ++++ b/chacra/models/repos.py +@@ -37,7 +37,7 @@ class Repo(Base): + self.ref = ref + self.distro = distro + self.distro_version = distro_version +- self.modified = datetime.datetime.now(datetime.UTC) ++ self.modified = datetime.datetime.now(datetime.timezone.utc) + self.sha1 = kwargs.get('sha1', 'head') + self.flavor = kwargs.get('flavor', 'default') + +diff --git a/chacra/tests/async/test_recurring.py b/chacra/tests/async/test_recurring.py +index 4b8f133..e48baed 100644 +--- a/chacra/tests/async/test_recurring.py ++++ b/chacra/tests/async/test_recurring.py +@@ -34,7 +34,7 @@ class TestPurgeRepos(object): + distro_version='7', + ) + +- self.now = datetime.datetime.now(datetime.UTC) ++ self.now = datetime.datetime.now(datetime.timezone.utc) + # slightly old + self.one_minute = self.now - datetime.timedelta(minutes=1) + # really old From 8f921e456cf017cccb9d79bd79d1ace4f40b45f8 Mon Sep 17 00:00:00 2001 From: ffgan Date: Tue, 12 Aug 2025 14:55:56 +0800 Subject: [PATCH 03/10] use github action to test chacra container Co-authored by: nijincheng@iscas.ac.cn; --- .github/workflows/container_test.yml | 61 ++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 .github/workflows/container_test.yml diff --git a/.github/workflows/container_test.yml b/.github/workflows/container_test.yml new file mode 100644 index 00000000..ad3d54c2 --- /dev/null +++ b/.github/workflows/container_test.yml @@ -0,0 +1,61 @@ +name: build and tests + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build the Docker image + working-directory: deploy/container/chacra_image + run: docker build -t chacra . + + - name: Set up Docker Compose + uses: docker/setup-compose-action@v1 + + - name: docker compose up -d + working-directory: deploy/container + run: | + docker compose up -d + sleep 30 + + - name: setup chacractl + run: | + pip install chacractl + cat > ~/.chacractl << 'EOF' + # This file was automatically generated by the chacractl CLI + # make sure to update it with the correct user and key to talk to the API + url = "http://127.0.0.1/" + user = "admin" + key = "secret" + ssl_verify = False + EOF + + - name: view container log + run: docker logs --tail 1000 chacra-containerize-services-1 + + - name: test + run: | + wget https://chacra.ceph.com/r/libboost/master/9ea1fb8bdad548a88004db87761f173aa50dcc85/ubuntu/jammy/flavors/default/pool/main/b/boost1.87/ceph-libboost-atomic1.87-dev_1.87.0-1.1_amd64.deb + + ls *.deb | chacractl binary create libboost/master/9ea1fb8bdad548a88004db87761f173aa50dcc85/ubuntu/jammy/amd64/flavors/default + + rm -f ./ceph-libboost-atomic1.87-dev_1.87.0-1.1_amd64.deb + + ls -l + + wget http://127.0.0.1/binaries/libboost/master/9ea1fb8bdad548a88004db87761f173aa50dcc85/ubuntu/jammy/amd64/flavors/default/ceph-libboost-atomic1.87-dev_1.87.0-1.1_amd64.deb + + ls -l + + - name: clean + working-directory: deploy/container + run: | + rm -f ./ceph-libboost-atomic1.87-dev_1.87.0-1.1_amd64.deb + sudo docker compose down \ No newline at end of file From 39503be90268cab907cb112fbf2d569d219793ff Mon Sep 17 00:00:00 2001 From: ffgan Date: Tue, 12 Aug 2025 14:57:18 +0800 Subject: [PATCH 04/10] modify trigger for container test --- .github/workflows/container_test.yml | 102 +++++++++++++-------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/.github/workflows/container_test.yml b/.github/workflows/container_test.yml index ad3d54c2..440fef11 100644 --- a/.github/workflows/container_test.yml +++ b/.github/workflows/container_test.yml @@ -2,60 +2,60 @@ name: build and tests on: push: - branches: [ "main" ] + branches: ["main", "feature/containerize"] pull_request: - branches: [ "main" ] + branches: ["main"] jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - - name: Build the Docker image - working-directory: deploy/container/chacra_image - run: docker build -t chacra . - - - name: Set up Docker Compose - uses: docker/setup-compose-action@v1 - - - name: docker compose up -d - working-directory: deploy/container - run: | - docker compose up -d - sleep 30 - - - name: setup chacractl - run: | - pip install chacractl - cat > ~/.chacractl << 'EOF' - # This file was automatically generated by the chacractl CLI - # make sure to update it with the correct user and key to talk to the API - url = "http://127.0.0.1/" - user = "admin" - key = "secret" - ssl_verify = False - EOF - - - name: view container log - run: docker logs --tail 1000 chacra-containerize-services-1 - - - name: test - run: | - wget https://chacra.ceph.com/r/libboost/master/9ea1fb8bdad548a88004db87761f173aa50dcc85/ubuntu/jammy/flavors/default/pool/main/b/boost1.87/ceph-libboost-atomic1.87-dev_1.87.0-1.1_amd64.deb - - ls *.deb | chacractl binary create libboost/master/9ea1fb8bdad548a88004db87761f173aa50dcc85/ubuntu/jammy/amd64/flavors/default - - rm -f ./ceph-libboost-atomic1.87-dev_1.87.0-1.1_amd64.deb - - ls -l - - wget http://127.0.0.1/binaries/libboost/master/9ea1fb8bdad548a88004db87761f173aa50dcc85/ubuntu/jammy/amd64/flavors/default/ceph-libboost-atomic1.87-dev_1.87.0-1.1_amd64.deb - - ls -l - - - name: clean - working-directory: deploy/container - run: | - rm -f ./ceph-libboost-atomic1.87-dev_1.87.0-1.1_amd64.deb - sudo docker compose down \ No newline at end of file + - uses: actions/checkout@v4 + + - name: Build the Docker image + working-directory: deploy/container/chacra_image + run: docker build -t chacra . + + - name: Set up Docker Compose + uses: docker/setup-compose-action@v1 + + - name: docker compose up -d + working-directory: deploy/container + run: | + docker compose up -d + sleep 30 + + - name: setup chacractl + run: | + pip install chacractl + cat > ~/.chacractl << 'EOF' + # This file was automatically generated by the chacractl CLI + # make sure to update it with the correct user and key to talk to the API + url = "http://127.0.0.1/" + user = "admin" + key = "secret" + ssl_verify = False + EOF + + - name: view container log + run: docker logs --tail 1000 chacra-containerize-services-1 + + - name: test + run: | + wget https://chacra.ceph.com/r/libboost/master/9ea1fb8bdad548a88004db87761f173aa50dcc85/ubuntu/jammy/flavors/default/pool/main/b/boost1.87/ceph-libboost-atomic1.87-dev_1.87.0-1.1_amd64.deb + + ls *.deb | chacractl binary create libboost/master/9ea1fb8bdad548a88004db87761f173aa50dcc85/ubuntu/jammy/amd64/flavors/default + + rm -f ./ceph-libboost-atomic1.87-dev_1.87.0-1.1_amd64.deb + + ls -l + + wget http://127.0.0.1/binaries/libboost/master/9ea1fb8bdad548a88004db87761f173aa50dcc85/ubuntu/jammy/amd64/flavors/default/ceph-libboost-atomic1.87-dev_1.87.0-1.1_amd64.deb + + ls -l + + - name: clean + working-directory: deploy/container + run: | + rm -f ./ceph-libboost-atomic1.87-dev_1.87.0-1.1_amd64.deb + sudo docker compose down From ad675cb353317513c7ed6ef3bb2a3055607146bc Mon Sep 17 00:00:00 2001 From: ffgan Date: Tue, 12 Aug 2025 15:00:25 +0800 Subject: [PATCH 05/10] modify `view container log` container name --- .github/workflows/container_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_test.yml b/.github/workflows/container_test.yml index 440fef11..752de1fd 100644 --- a/.github/workflows/container_test.yml +++ b/.github/workflows/container_test.yml @@ -38,7 +38,7 @@ jobs: EOF - name: view container log - run: docker logs --tail 1000 chacra-containerize-services-1 + run: docker logs --tail 1000 container-services-1 - name: test run: | From ca73a4fce4477359517a106501aa64ea15705958 Mon Sep 17 00:00:00 2001 From: ffgan Date: Thu, 14 Aug 2025 11:34:49 +0800 Subject: [PATCH 06/10] upgrade python version to 3.12 --- deploy/container/chacra_image/Dockerfile | 19 +++--- deploy/container/chacra_image/conf/prod.py | 4 +- .../container/chacra_image/init-chacra-db.sh | 1 + deploy/container/chacra_image/main.sh | 7 +- deploy/container/chacra_image/py3.8.patch | 65 ------------------- 5 files changed, 15 insertions(+), 81 deletions(-) delete mode 100644 deploy/container/chacra_image/py3.8.patch diff --git a/deploy/container/chacra_image/Dockerfile b/deploy/container/chacra_image/Dockerfile index 9f1d2a9c..15cc1a35 100644 --- a/deploy/container/chacra_image/Dockerfile +++ b/deploy/container/chacra_image/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:24.04 ARG APP_NAME="chacra" # please do not pass env to container while start container. @@ -7,7 +7,6 @@ ENV APP_NAME=${APP_NAME} ENV APP_HOME="/opt/${APP_NAME}" ENV DEBIAN_FRONTEND=noninteractive -# In Ubuntu 20.04, createrepo cannot be installed. In Ubuntu 22.04, we may be able to use createrepo-c instead of createrepo, but in Ubuntu 22.04, chacra has some Python problems(install celery[librabbitmq] will crash). Since the current chacra works fine in Ubuntu 20.04, we'll just ignore createrepo for now. RUN apt-get update && apt-get install -y \ python3 \ python3-pip \ @@ -17,7 +16,7 @@ RUN apt-get update && apt-get install -y \ pkg-config \ postgresql-client \ sudo \ - git cmake libtool autoconf nginx reprepro \ + git cmake libtool autoconf nginx reprepro createrepo-c\ && mkdir -p /var/log/celery /etc/nginx/sites-available /etc/nginx/sites-enabled \ && rm -f /etc/nginx/sites-enabled/default \ && mkdir -p /var/lib/nginx/body /var/log/nginx \ @@ -25,27 +24,25 @@ RUN apt-get update && apt-get install -y \ && chown -R nginx:nginx /var/lib/nginx /var/log/nginx \ && chmod -R 755 /var/lib/nginx /var/log/nginx -# In Ubuntu 20.04, the Python version is 3.8. For chacra, we need to make some changes to satisfy Python 3.8 compatibility. So I use my repository instead of ceph/chakra. RUN virtualenv ${APP_HOME} \ && . ${APP_HOME}/bin/activate \ && pip install --upgrade pip setuptools\ && pip install -I -e git+https://github.com/ceph/chacra@main#egg=chacra \ - && pip install librabbitmq && pip install -r ${APP_HOME}/src/${APP_NAME}/requirements.txt - -COPY ./py3.8.patch ${APP_HOME}/src/${APP_NAME}/py3.8.patch - -RUN cd ${APP_HOME}/src/${APP_NAME} && git apply py3.8.patch + && pip install -r ${APP_HOME}/src/${APP_NAME}/requirements.txt COPY ./main.sh /bin/main.sh COPY ./init-chacra-db.sh ${APP_HOME}/init-chacra-db.sh -COPY ./conf/*.py ${APP_HOME}/src/${APP_NAME}/ +COPY ./conf/prod.py ${APP_HOME}/src/${APP_NAME}/prod.py +COPY ./conf/prod_db.py ${APP_HOME}/src/${APP_NAME}/${APP_NAME}/prod_db.py +COPY ./conf/prod_api_creds.py ${APP_HOME}/src/${APP_NAME}/${APP_NAME}/prod_api_creds.py + COPY ./conf/alembic-prod.ini ${APP_HOME}/src/${APP_NAME}/alembic-prod.ini COPY ./conf/nginx.conf /etc/nginx/nginx.conf COPY ./conf/nginx_site.conf /etc/nginx/sites-available/${APP_NAME}.conf -RUN ln -s /etc/nginx/sites-available/${APP_NAME}.conf /etc/nginx/sites-enabled/${APP_NAME}.conf +RUN ln -s /etc/nginx/sites-available/${APP_NAME}.conf /etc/nginx/sites-enabled/${APP_NAME}.conf ENV PECAN_CONFIG=${APP_HOME}/src/${APP_NAME}/prod.py diff --git a/deploy/container/chacra_image/conf/prod.py b/deploy/container/chacra_image/conf/prod.py index 961f51ea..47af11a4 100644 --- a/deploy/container/chacra_image/conf/prod.py +++ b/deploy/container/chacra_image/conf/prod.py @@ -1,8 +1,8 @@ from pecan.hooks import TransactionHook from chacra import models from chacra import hooks -from prod_db import sqlalchemy -from prod_api_creds import api_key, api_user +from chacra.prod_db import sqlalchemy +from chacra.prod_api_creds import api_key, api_user # Server Specific Configurations diff --git a/deploy/container/chacra_image/init-chacra-db.sh b/deploy/container/chacra_image/init-chacra-db.sh index 72a81374..8a571c9f 100755 --- a/deploy/container/chacra_image/init-chacra-db.sh +++ b/deploy/container/chacra_image/init-chacra-db.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash # env like APP_NAME,APP_HOME is pass by dockerfile +set -x echo $APP_NAME echo $APP_HOME echo "init chacra database..." diff --git a/deploy/container/chacra_image/main.sh b/deploy/container/chacra_image/main.sh index cd21bb5f..dd5490f0 100755 --- a/deploy/container/chacra_image/main.sh +++ b/deploy/container/chacra_image/main.sh @@ -1,13 +1,14 @@ #! /bin/sh # env is pass by dockerfile +set -x echo $APP_NAME echo $APP_HOME -cd ${APP_HOME}/src/${APP_NAME}/${APP_NAME} +cd ${APP_HOME}/src/${APP_NAME} sleep 10 -${APP_HOME}/bin/celery multi start 5 -Q:1,2 poll_repos,celery -Q:3-5 build_repos -A asynch --logfile=/var/log/celery/%n%I.log +${APP_HOME}/bin/celery multi start 5 -Q:1,2 poll_repos,celery -Q:3-5 build_repos -A chacra.asynch --logfile=/var/log/celery/%n%I.log -${APP_HOME}/bin/celery -A asynch beat --loglevel=info --logfile=/var/log/celery/beat.log & +${APP_HOME}/bin/celery -A chacra.asynch beat --loglevel=info --logfile=/var/log/celery/beat.log & /usr/sbin/nginx -g 'daemon off;' & diff --git a/deploy/container/chacra_image/py3.8.patch b/deploy/container/chacra_image/py3.8.patch deleted file mode 100644 index f4ce1888..00000000 --- a/deploy/container/chacra_image/py3.8.patch +++ /dev/null @@ -1,65 +0,0 @@ -diff --git a/chacra/controllers/util.py b/chacra/controllers/util.py -index 389a1f8..91084db 100644 ---- a/chacra/controllers/util.py -+++ b/chacra/controllers/util.py -@@ -1,4 +1,7 @@ --from datetime import datetime, timedelta, UTC -+from datetime import datetime, timedelta, timezone -+ -+UTC = timezone.utc -+ - import logging - from pecan import conf - -diff --git a/chacra/models/__init__.py b/chacra/models/__init__.py -index 0f610bc..ae3d0e1 100644 ---- a/chacra/models/__init__.py -+++ b/chacra/models/__init__.py -@@ -54,7 +54,7 @@ def update_timestamp(mapper, connection, target): - """ - Automate the 'modified' attribute when a model changes - """ -- target.modified = datetime.datetime.now(datetime.UTC) -+ target.modified = datetime.datetime.now(datetime.timezone.utc) - - - # Utilities: -diff --git a/chacra/models/binaries.py b/chacra/models/binaries.py -index 6fcbd08..fd2626f 100644 ---- a/chacra/models/binaries.py -+++ b/chacra/models/binaries.py -@@ -50,7 +50,7 @@ class Binary(Base): - def __init__(self, name, project, repo=None, **kw): - self.name = name - self.project = project -- now = datetime.datetime.now(datetime.UTC) -+ now = datetime.datetime.now(datetime.timezone.utc) - self.created = now - self.modified = now - self.sha1 = kw.get('sha1', 'head') -diff --git a/chacra/models/repos.py b/chacra/models/repos.py -index 8a236e6..1d298df 100644 ---- a/chacra/models/repos.py -+++ b/chacra/models/repos.py -@@ -37,7 +37,7 @@ class Repo(Base): - self.ref = ref - self.distro = distro - self.distro_version = distro_version -- self.modified = datetime.datetime.now(datetime.UTC) -+ self.modified = datetime.datetime.now(datetime.timezone.utc) - self.sha1 = kwargs.get('sha1', 'head') - self.flavor = kwargs.get('flavor', 'default') - -diff --git a/chacra/tests/async/test_recurring.py b/chacra/tests/async/test_recurring.py -index 4b8f133..e48baed 100644 ---- a/chacra/tests/async/test_recurring.py -+++ b/chacra/tests/async/test_recurring.py -@@ -34,7 +34,7 @@ class TestPurgeRepos(object): - distro_version='7', - ) - -- self.now = datetime.datetime.now(datetime.UTC) -+ self.now = datetime.datetime.now(datetime.timezone.utc) - # slightly old - self.one_minute = self.now - datetime.timedelta(minutes=1) - # really old From 899570ac55eecd42aad18769df6c4455ba47d468 Mon Sep 17 00:00:00 2001 From: ffgan Date: Thu, 14 Aug 2025 12:19:21 +0800 Subject: [PATCH 07/10] run tox on github action --- .github/workflows/container_test.yml | 3 + deploy/container/chacra_image/Dockerfile | 7 +- .../container/chacra_image/conf/conftest.py | 254 ++++++++++++++++++ .../container/chacra_image/conf/test-conf.py | 103 +++++++ .../container/chacra_image/init-chacra-db.sh | 6 + 5 files changed, 372 insertions(+), 1 deletion(-) create mode 100644 deploy/container/chacra_image/conf/conftest.py create mode 100644 deploy/container/chacra_image/conf/test-conf.py diff --git a/.github/workflows/container_test.yml b/.github/workflows/container_test.yml index 752de1fd..dc3210f9 100644 --- a/.github/workflows/container_test.yml +++ b/.github/workflows/container_test.yml @@ -54,6 +54,9 @@ jobs: ls -l + - name: tox run + run: docker exec container-services-1 bash -c "source /opt/chacra/bin/activate && cd /opt/chacra/src/chacra && tox run" + - name: clean working-directory: deploy/container run: | diff --git a/deploy/container/chacra_image/Dockerfile b/deploy/container/chacra_image/Dockerfile index 15cc1a35..4a67245d 100644 --- a/deploy/container/chacra_image/Dockerfile +++ b/deploy/container/chacra_image/Dockerfile @@ -28,7 +28,8 @@ RUN virtualenv ${APP_HOME} \ && . ${APP_HOME}/bin/activate \ && pip install --upgrade pip setuptools\ && pip install -I -e git+https://github.com/ceph/chacra@main#egg=chacra \ - && pip install -r ${APP_HOME}/src/${APP_NAME}/requirements.txt + && pip install -r ${APP_HOME}/src/${APP_NAME}/requirements.txt \ + && pip install tox COPY ./main.sh /bin/main.sh COPY ./init-chacra-db.sh ${APP_HOME}/init-chacra-db.sh @@ -37,6 +38,10 @@ COPY ./conf/prod.py ${APP_HOME}/src/${APP_NAME}/prod.py COPY ./conf/prod_db.py ${APP_HOME}/src/${APP_NAME}/${APP_NAME}/prod_db.py COPY ./conf/prod_api_creds.py ${APP_HOME}/src/${APP_NAME}/${APP_NAME}/prod_api_creds.py +# override origin conf.py to pass tox +COPY ./conf/test-conf.py ${APP_HOME}/src/${APP_NAME}/${APP_NAME}/tests/config.py +COPY ./conf/conftest.py ${APP_HOME}/src/${APP_NAME}/${APP_NAME}/tests/conftest.py + COPY ./conf/alembic-prod.ini ${APP_HOME}/src/${APP_NAME}/alembic-prod.ini COPY ./conf/nginx.conf /etc/nginx/nginx.conf diff --git a/deploy/container/chacra_image/conf/conftest.py b/deploy/container/chacra_image/conf/conftest.py new file mode 100644 index 00000000..dea40c84 --- /dev/null +++ b/deploy/container/chacra_image/conf/conftest.py @@ -0,0 +1,254 @@ +from __future__ import print_function +import os +from pecan.testing import load_test_app + +import subprocess + +from copy import deepcopy +from pecan import conf +from pecan import configuration +from sqlalchemy import create_engine +from sqlalchemy.pool import NullPool + +from chacra import models as _db +from chacra.tests import util +import pytest + + +DBNAME = 'chacratest' +BIND = 'postgresql+psycopg2://postgres:example@127.0.0.1' +# for sqlite, use something like this (DBNAME is the name of the file) +# DBNAME = 'chacratest.db' +# BIND = 'sqlite://' + + +def config_file(): + here = os.path.abspath(os.path.dirname(__file__)) + return os.path.join(here, 'config.py') + + +def reload_config(): + from pecan import configuration + config = configuration.conf_from_file(config_file()).to_dict() + + # Add the appropriate connection string to the app config. + config['sqlalchemy'] = { + 'url': '%s/%s' % (BIND, DBNAME), + 'encoding': 'utf-8', + 'poolclass': NullPool + } + + configuration.set_config( + config, + overwrite=True + ) + _db.init_model() + + +@pytest.fixture +def fake(): + class Fake(object): + def __init__(self, *a, **kw): + for k, v, in kw.items(): + setattr(self, k, v) + return Fake + +@pytest.fixture +def recorder(): + class Recorder(object): + def __init__(self, *a, **kw): + self.recorder_init_call = [] + self.recorder_calls = [] + for k, v, in kw.items(): + setattr(self, k, v) + self.recorder_init_call.append( + {'args': a, 'kwargs': kw} + ) + def __call__(self, *a, **kw): + for k, v, in kw.items(): + setattr(self, k, v) + self.recorder_calls.append( + {'args': a, 'kwargs': kw} + ) + + return Recorder + +def pytest_collectstart(collector): + import os + os.environ['PECAN_CONFIG'] = config_file() + + +@pytest.fixture(scope='session') +def app(request): + config = configuration.conf_from_file(config_file()).to_dict() + + # Add the appropriate connection string to the app config. + config['sqlalchemy'] = { + 'url': '%s/%s' % (BIND, DBNAME), + 'encoding': 'utf-8', + 'poolclass': NullPool + } + + # Set up a fake app + app = TestApp(load_test_app(config)) + return app + + +@pytest.fixture(scope='session') +def connection(app, request): + """Session-wide test database.""" + # Connect and create the temporary database + print("=" * 80) + print("CREATING TEMPORARY DATABASE FOR TESTS") + print("=" * 80) + if BIND.startswith('postgresql'): + subprocess.call(['dropdb', DBNAME]) + subprocess.call(['createdb', DBNAME]) + + # Bind and create the database tables + _db.clear() + engine_url = '%s/%s' % (BIND, DBNAME) + + db_engine = create_engine( + engine_url, + encoding='utf-8', + poolclass=NullPool) + + # AKA models.start() + _db.Session.bind = db_engine + _db.metadata.bind = _db.Session.bind + + _db.Base.metadata.create_all(db_engine) + _db.commit() + _db.clear() + + def teardown(): + _db.Base.metadata.drop_all(db_engine) + + request.addfinalizer(teardown) + + # Slap our test app on it + _db.app = app + return _db + + +@pytest.fixture(scope='function') +def session(connection, request): + """Creates a new database session for a test.""" + _config = configuration.conf_from_file(config_file()).to_dict() + config = deepcopy(_config) + + # Add the appropriate connection string to the app config. + config['sqlalchemy'] = { + 'url': '%s/%s' % (BIND, DBNAME), + 'encoding': 'utf-8', + 'poolclass': NullPool + } + + connection.start() + + def teardown(): + + # Tear down and dispose the DB binding + connection.clear() + + # start a transaction + engine = conf.sqlalchemy.engine + conn = engine.connect() + trans = conn.begin() + + + # gather all data first before dropping anything. + # some DBs lock after things have been dropped in + # a transaction. + if BIND.startswith('postgresql'): + conn.execute("TRUNCATE TABLE %s RESTART IDENTITY CASCADE" % ( + ', '.join(engine.table_names()) + )) + elif BIND.startswith('sqlite'): + for table in engine.table_names(): + conn.execute("DELETE FROM %s" % table) + + trans.commit() + conn.close() + + request.addfinalizer(teardown) + return connection + + +class TestApp(object): + """ + A controller test starts a database transaction and creates a fake + WSGI app. + """ + + __headers__ = {} + + def __init__(self, app): + self.app = app + + def _do_request(self, url, method='GET', **kwargs): + methods = { + 'GET': self.app.get, + 'POST': self.app.post, + 'POSTJ': self.app.post_json, + 'PUT': self.app.put, + 'HEAD': self.app.head, + 'DELETE': self.app.delete + } + kwargs.setdefault('headers', {}).update(self.__headers__) + return methods.get(method, self.app.get)(str(url), **kwargs) + + def post_json(self, url, **kwargs): + """ + @param (string) url - The URL to emulate a POST request to + @returns (paste.fixture.TestResponse) + """ + # support automatic, correct authentication if not specified otherwise + if not kwargs.get('headers'): + kwargs['headers'] = {'Authorization': util.make_credentials()} + return self._do_request(url, 'POSTJ', **kwargs) + + def post(self, url, **kwargs): + """ + @param (string) url - The URL to emulate a POST request to + @returns (paste.fixture.TestResponse) + """ + # support automatic, correct authentication if not specified otherwise + if not kwargs.get('headers'): + kwargs['headers'] = {'Authorization': util.make_credentials()} + return self._do_request(url, 'POST', **kwargs) + + def get(self, url, **kwargs): + """ + @param (string) url - The URL to emulate a GET request to + @returns (paste.fixture.TestResponse) + """ + return self._do_request(url, 'GET', **kwargs) + + def put(self, url, **kwargs): + """ + @param (string) url - The URL to emulate a PUT request to + @returns (paste.fixture.TestResponse) + """ + if not kwargs.get('headers'): + kwargs['headers'] = {'Authorization': util.make_credentials()} + return self._do_request(url, 'PUT', **kwargs) + + def delete(self, url, **kwargs): + """ + @param (string) url - The URL to emulate a DELETE request to + @returns (paste.fixture.TestResponse) + """ + if not kwargs.get('headers'): + kwargs['headers'] = {'Authorization': util.make_credentials()} + return self._do_request(url, 'DELETE', **kwargs) + + def head(self, url, **kwargs): + """ + @param (string) url - The URL to emulate a HEAD request to + @returns (paste.fixture.TestResponse) + """ + if not kwargs.get('headers'): + kwargs['headers'] = {'Authorization': util.make_credentials()} + return self._do_request(url, 'HEAD', **kwargs) diff --git a/deploy/container/chacra_image/conf/test-conf.py b/deploy/container/chacra_image/conf/test-conf.py new file mode 100644 index 00000000..ffd7df7c --- /dev/null +++ b/deploy/container/chacra_image/conf/test-conf.py @@ -0,0 +1,103 @@ +from pecan.hooks import TransactionHook +from chacra import models + + +# Server Specific Configurations +server = { + 'port': '8080', + 'host': '0.0.0.0' +} + +# Pecan Application Configurations +app = { + 'root': 'chacra.controllers.root.RootController', + 'modules': ['chacra'], + 'static_root': '%(confdir)s/public', +# 'default_renderer': 'json', + 'guess_content_type_from_ext': False, + 'template_path': '%(confdir)s/../templates', + 'hooks': [ + TransactionHook( + models.start, + models.start_read_only, + models.commit, + models.rollback, + models.clear + ), + ], + 'debug': False, + #'errors': { + # 404: '/error/404', + # '__force_dict__': True + #} +} + +logging = { + 'loggers': { + 'root': {'level': 'INFO', 'handlers': ['console']}, + 'chacra': {'level': 'DEBUG', 'handlers': ['console']}, + 'pecan.commands.serve': {'level': 'DEBUG', 'handlers': ['console']}, + 'py.warnings': {'handlers': ['console']}, + '__force_dict__': True + }, + 'handlers': { + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'color' + } + }, + 'formatters': { + 'simple': { + 'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]' + '[%(threadName)s] %(message)s') + }, + 'color': { + '()': 'pecan.log.ColorFormatter', + 'format': ('%(asctime)s [%(padded_color_levelname)s] [%(name)s]' + '[%(threadName)s] %(message)s'), + '__force_dict__': True + } + } +} + +sqlalchemy = { + # You may use SQLite for testing + # 'url': 'sqlite:///dev.db', + # When you set up PostreSQL, it will look more like: + # 'url': 'postgresql+psycopg2://USER:PASSWORD@DB_HOST/DB_NAME', + 'url': 'postgresql+psycopg2://postgres:example@127.0.0.1/chacra', + 'echo': True, + 'echo_pool': True, + 'pool_recycle': 3600, + 'encoding': 'utf-8' +} + +binary_root = '/tmp/' +distributions_root = '/tmp/' + +# When True it will set the headers so that Nginx can serve the download +# instead of Pecan. +delegate_downloads = False + +api_user = 'admin' +api_key = 'secret' + +polling_cycle = 30 + +# Use this to define how distributions files will be created per project +distributions = { + "defaults": { + "DebIndices": "Packages Release . .gz .bz2", + "DscIndices": "Sources Release .gz .bz2", + "Contents": ".gz .bz2", + "Origin": "ceph.com", + "Description": "", + "Architectures": "amd64 armhf i386 source", + "Suite": "stable", + "Components": "main", + }, + "ceph": { + "Description": "Ceph distributed file system", + }, +} diff --git a/deploy/container/chacra_image/init-chacra-db.sh b/deploy/container/chacra_image/init-chacra-db.sh index 8a571c9f..184bf65f 100755 --- a/deploy/container/chacra_image/init-chacra-db.sh +++ b/deploy/container/chacra_image/init-chacra-db.sh @@ -10,6 +10,12 @@ psql -v ON_ERROR_STOP=1 -h 127.0.0.1 -p 5432 --username postgres --dbname postgr psql -v ON_ERROR_STOP=1 -h 127.0.0.1 -p 5432 --username postgres --dbname postgres -c \ "CREATE DATABASE chacra;" + +psql -v ON_ERROR_STOP=1 -h 127.0.0.1 -p 5432 --username postgres --dbname postgres -tc \ + "SELECT 1 FROM pg_database WHERE datname = 'chacratest'" | grep -q 1 || \ +psql -v ON_ERROR_STOP=1 -h 127.0.0.1 -p 5432 --username postgres --dbname postgres -c \ + "CREATE DATABASE chacratest;" + echo "init success" echo "chacra database is prepare done." echo "check chacra database weather to be fill data..." From 382410543298627232ae819c6e5856b29721df9d Mon Sep 17 00:00:00 2001 From: ffgan Date: Thu, 14 Aug 2025 12:29:28 +0800 Subject: [PATCH 08/10] add python3.11 for tox Co-authored by: nijincheng@iscas.ac.cn; --- deploy/container/chacra_image/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/container/chacra_image/Dockerfile b/deploy/container/chacra_image/Dockerfile index 4a67245d..60db2ebb 100644 --- a/deploy/container/chacra_image/Dockerfile +++ b/deploy/container/chacra_image/Dockerfile @@ -29,7 +29,8 @@ RUN virtualenv ${APP_HOME} \ && pip install --upgrade pip setuptools\ && pip install -I -e git+https://github.com/ceph/chacra@main#egg=chacra \ && pip install -r ${APP_HOME}/src/${APP_NAME}/requirements.txt \ - && pip install tox + && pip install tox uv \ + && uv python install 3.11 3.12 COPY ./main.sh /bin/main.sh COPY ./init-chacra-db.sh ${APP_HOME}/init-chacra-db.sh From c4a3fe7d7662726d520d6c9cb6c08432bc57c077 Mon Sep 17 00:00:00 2001 From: ffgan Date: Thu, 14 Aug 2025 12:33:45 +0800 Subject: [PATCH 09/10] add python 3.13 for tox Co-authored by: nijincheng@iscas.ac.cn; --- deploy/container/chacra_image/Dockerfile | 2 +- requirements.txt | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/container/chacra_image/Dockerfile b/deploy/container/chacra_image/Dockerfile index 60db2ebb..1946230f 100644 --- a/deploy/container/chacra_image/Dockerfile +++ b/deploy/container/chacra_image/Dockerfile @@ -30,7 +30,7 @@ RUN virtualenv ${APP_HOME} \ && pip install -I -e git+https://github.com/ceph/chacra@main#egg=chacra \ && pip install -r ${APP_HOME}/src/${APP_NAME}/requirements.txt \ && pip install tox uv \ - && uv python install 3.11 3.12 + && uv python install 3.11 3.12 3.13 COPY ./main.sh /bin/main.sh COPY ./init-chacra-db.sh ${APP_HOME}/init-chacra-db.sh diff --git a/requirements.txt b/requirements.txt index 711d4bac..2eada713 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -psycopg2-binary==2.9.9 +psycopg2-binary==2.9.10 gunicorn<20.1.0 pecan sqlalchemy==1.3.0 diff --git a/setup.py b/setup.py index 7bfa3bc5..c5f90276 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ install_requires=[ "pecan", "sqlalchemy==1.3.0", - "psycopg2-binary==2.9.9", + "psycopg2-binary==2.9.10", "pecan-notario", "python-statsd", "requests", From 706cff47af5d784fa0747b17302d4de251bb3e04 Mon Sep 17 00:00:00 2001 From: ffgan Date: Thu, 14 Aug 2025 13:20:38 +0800 Subject: [PATCH 10/10] fix python3.13 tox error --- deploy/container/chacra_image/Dockerfile | 5 +- .../container/chacra_image/conf/test-conf.py | 103 ------------------ 2 files changed, 3 insertions(+), 105 deletions(-) delete mode 100644 deploy/container/chacra_image/conf/test-conf.py diff --git a/deploy/container/chacra_image/Dockerfile b/deploy/container/chacra_image/Dockerfile index 1946230f..379ba77b 100644 --- a/deploy/container/chacra_image/Dockerfile +++ b/deploy/container/chacra_image/Dockerfile @@ -40,7 +40,6 @@ COPY ./conf/prod_db.py ${APP_HOME}/src/${APP_NAME}/${APP_NAME}/prod_db.py COPY ./conf/prod_api_creds.py ${APP_HOME}/src/${APP_NAME}/${APP_NAME}/prod_api_creds.py # override origin conf.py to pass tox -COPY ./conf/test-conf.py ${APP_HOME}/src/${APP_NAME}/${APP_NAME}/tests/config.py COPY ./conf/conftest.py ${APP_HOME}/src/${APP_NAME}/${APP_NAME}/tests/conftest.py COPY ./conf/alembic-prod.ini ${APP_HOME}/src/${APP_NAME}/alembic-prod.ini @@ -48,7 +47,9 @@ COPY ./conf/alembic-prod.ini ${APP_HOME}/src/${APP_NAME}/alembic-prod.ini COPY ./conf/nginx.conf /etc/nginx/nginx.conf COPY ./conf/nginx_site.conf /etc/nginx/sites-available/${APP_NAME}.conf -RUN ln -s /etc/nginx/sites-available/${APP_NAME}.conf /etc/nginx/sites-enabled/${APP_NAME}.conf +RUN ln -s /etc/nginx/sites-available/${APP_NAME}.conf /etc/nginx/sites-enabled/${APP_NAME}.conf \ + && sed -i 's/2\.9\.9/2.9.10/g' ${APP_HOME}/src/${APP_NAME}/setup.py \ + && sed -i 's/2\.9\.9/2.9.10/g' ${APP_HOME}/src/${APP_NAME}/requirements.txt ENV PECAN_CONFIG=${APP_HOME}/src/${APP_NAME}/prod.py diff --git a/deploy/container/chacra_image/conf/test-conf.py b/deploy/container/chacra_image/conf/test-conf.py deleted file mode 100644 index ffd7df7c..00000000 --- a/deploy/container/chacra_image/conf/test-conf.py +++ /dev/null @@ -1,103 +0,0 @@ -from pecan.hooks import TransactionHook -from chacra import models - - -# Server Specific Configurations -server = { - 'port': '8080', - 'host': '0.0.0.0' -} - -# Pecan Application Configurations -app = { - 'root': 'chacra.controllers.root.RootController', - 'modules': ['chacra'], - 'static_root': '%(confdir)s/public', -# 'default_renderer': 'json', - 'guess_content_type_from_ext': False, - 'template_path': '%(confdir)s/../templates', - 'hooks': [ - TransactionHook( - models.start, - models.start_read_only, - models.commit, - models.rollback, - models.clear - ), - ], - 'debug': False, - #'errors': { - # 404: '/error/404', - # '__force_dict__': True - #} -} - -logging = { - 'loggers': { - 'root': {'level': 'INFO', 'handlers': ['console']}, - 'chacra': {'level': 'DEBUG', 'handlers': ['console']}, - 'pecan.commands.serve': {'level': 'DEBUG', 'handlers': ['console']}, - 'py.warnings': {'handlers': ['console']}, - '__force_dict__': True - }, - 'handlers': { - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'color' - } - }, - 'formatters': { - 'simple': { - 'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]' - '[%(threadName)s] %(message)s') - }, - 'color': { - '()': 'pecan.log.ColorFormatter', - 'format': ('%(asctime)s [%(padded_color_levelname)s] [%(name)s]' - '[%(threadName)s] %(message)s'), - '__force_dict__': True - } - } -} - -sqlalchemy = { - # You may use SQLite for testing - # 'url': 'sqlite:///dev.db', - # When you set up PostreSQL, it will look more like: - # 'url': 'postgresql+psycopg2://USER:PASSWORD@DB_HOST/DB_NAME', - 'url': 'postgresql+psycopg2://postgres:example@127.0.0.1/chacra', - 'echo': True, - 'echo_pool': True, - 'pool_recycle': 3600, - 'encoding': 'utf-8' -} - -binary_root = '/tmp/' -distributions_root = '/tmp/' - -# When True it will set the headers so that Nginx can serve the download -# instead of Pecan. -delegate_downloads = False - -api_user = 'admin' -api_key = 'secret' - -polling_cycle = 30 - -# Use this to define how distributions files will be created per project -distributions = { - "defaults": { - "DebIndices": "Packages Release . .gz .bz2", - "DscIndices": "Sources Release .gz .bz2", - "Contents": ".gz .bz2", - "Origin": "ceph.com", - "Description": "", - "Architectures": "amd64 armhf i386 source", - "Suite": "stable", - "Components": "main", - }, - "ceph": { - "Description": "Ceph distributed file system", - }, -}