From 9179fdb2fabfd7b03476c37b915857514111b49f Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Fri, 17 Jan 2025 10:08:44 +0100 Subject: [PATCH 1/5] =?UTF-8?q?=F0=9F=94=A7(compose)=20rename=20legacy=20d?= =?UTF-8?q?ocker-compose.yml=20in=20compose.yaml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The compose configuration preferred filename is now compose.yaml docker-compose.yml is the legacy one used by docker-compose v1 --- README.md | 10 ++++------ bin/_config.sh | 2 +- docker-compose.yml => compose.yaml | 0 3 files changed, 5 insertions(+), 7 deletions(-) rename docker-compose.yml => compose.yaml (100%) diff --git a/README.md b/README.md index 27b1af11f..26d57bbdc 100644 --- a/README.md +++ b/README.md @@ -52,12 +52,10 @@ Make sure you have a recent version of Docker and [Docker Compose](https://docs. ```shellscript $ docker -v + Docker version 27.4.1, build b9d17ea -Docker version 20.10.2, build 2291f61 - -$ docker compose -v - -docker compose version 1.27.4, build 40524192 +$ docker compose version + Docker Compose version v2.32.1 ``` > ⚠️ You may need to run the following commands with sudo but this can be avoided by assigning your user to the `docker` group. @@ -170,4 +168,4 @@ docs Impress is built on top of [Django Rest Framework](https://www.django-rest-framework.org/), [Next.js](https://nextjs.org/), [MinIO](https://min.io/) and [BlocNote.js](https://www.blocknotejs.org/) ### States ❀️ open source -Docs is the result of a joint effort lead by the French πŸ‡«πŸ‡·πŸ₯– ([DINUM](https://www.numerique.gouv.fr/dinum/)) and German πŸ‡©πŸ‡ͺπŸ₯¨ government ([ZenDiS](https://zendis.de/)). We are always looking for new public partners feel free to reach out if you are interested in using or contributing to docs. \ No newline at end of file +Docs is the result of a joint effort lead by the French πŸ‡«πŸ‡·πŸ₯– ([DINUM](https://www.numerique.gouv.fr/dinum/)) and German πŸ‡©πŸ‡ͺπŸ₯¨ government ([ZenDiS](https://zendis.de/)). We are always looking for new public partners feel free to reach out if you are interested in using or contributing to docs. diff --git a/bin/_config.sh b/bin/_config.sh index 5ec03c465..dd2a5c29c 100644 --- a/bin/_config.sh +++ b/bin/_config.sh @@ -6,7 +6,7 @@ REPO_DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd)" UNSET_USER=0 TERRAFORM_DIRECTORY="./env.d/terraform" -COMPOSE_FILE="${REPO_DIR}/docker-compose.yml" +COMPOSE_FILE="${REPO_DIR}/compose.yaml" COMPOSE_PROJECT="docs" diff --git a/docker-compose.yml b/compose.yaml similarity index 100% rename from docker-compose.yml rename to compose.yaml From 24328b5d6b62a3befd6bbd3f39d1213d0ef856e6 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Fri, 17 Jan 2025 10:24:53 +0100 Subject: [PATCH 2/5] =?UTF-8?q?=E2=99=BB=EF=B8=8F(compose)=20remove=20usag?= =?UTF-8?q?e=20of=20dockerize?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compose is able to manage service status to be healthy in its dependecies. Configuring a healthcheck on services then defining a condition on the depends_on settings allow to remove dockerize --- Makefile | 5 ----- compose.yaml | 49 +++++++++++++++++++++++++++++++++++++------------ 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 0e81087b3..996662465 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,6 @@ COMPOSE_EXEC_APP = $(COMPOSE_EXEC) app-dev COMPOSE_RUN = $(COMPOSE) run --rm COMPOSE_RUN_APP = $(COMPOSE_RUN) app-dev COMPOSE_RUN_CROWDIN = $(COMPOSE_RUN) crowdin crowdin -WAIT_DB = @$(COMPOSE_RUN) dockerize -wait tcp://$(DB_HOST):$(DB_PORT) -timeout 60s # -- Backend MANAGE = $(COMPOSE_RUN_APP) python manage.py @@ -124,8 +123,6 @@ run: ## start the wsgi (production) and development server @$(COMPOSE) up --force-recreate -d celery-dev @$(COMPOSE) up --force-recreate -d y-provider @$(COMPOSE) up --force-recreate -d nginx - @echo "Wait for postgresql to be up..." - @$(WAIT_DB) .PHONY: run run-with-frontend: ## Start all the containers needed (backend to frontend) @@ -188,14 +185,12 @@ test-back-parallel: ## run all back-end tests in parallel makemigrations: ## run django makemigrations for the impress project. @echo "$(BOLD)Running makemigrations$(RESET)" @$(COMPOSE) up -d postgresql - @$(WAIT_DB) @$(MANAGE) makemigrations .PHONY: makemigrations migrate: ## run django migrations for the impress project. @echo "$(BOLD)Running migrations$(RESET)" @$(COMPOSE) up -d postgresql - @$(WAIT_DB) @$(MANAGE) migrate .PHONY: migrate diff --git a/compose.yaml b/compose.yaml index 74c90c085..0126c6443 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,6 +1,11 @@ services: postgresql: image: postgres:16 + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + interval: 1s + timeout: 2s + retries: 300 env_file: - env.d/development/postgresql ports: @@ -23,6 +28,11 @@ services: ports: - '9000:9000' - '9001:9001' + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 1s + timeout: 20s + retries: 300 entrypoint: "" command: minio server --console-address :9001 /data volumes: @@ -31,7 +41,9 @@ services: createbuckets: image: minio/mc depends_on: - - minio + minio: + condition: service_healthy + restart: true entrypoint: > sh -c " /usr/bin/mc alias set impress http://minio:9000 impress password && \ @@ -59,10 +71,15 @@ services: - ./src/backend:/app - ./data/static:/data/static depends_on: - - postgresql - - mailcatcher - - redis - - createbuckets + postgresql: + condition: service_healthy + restart: true + mailcatcher: + condition: service_started + redis: + condition: service_started + createbuckets: + condition: service_started celery-dev: user: ${DOCKER_USER:-1000} @@ -93,9 +110,13 @@ services: - env.d/development/common - env.d/development/postgresql depends_on: - - postgresql - - redis - - minio + postgresql: + condition: service_healthy + restart: true + redis: + condition: service_started + minio: + condition: service_started celery: user: ${DOCKER_USER:-1000} @@ -135,9 +156,6 @@ services: ports: - "3000:3000" - dockerize: - image: jwilder/dockerize - crowdin: image: crowdin/cli:3.16.0 volumes: @@ -169,6 +187,11 @@ services: kc_postgresql: image: postgres:14.3 + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + interval: 1s + timeout: 2s + retries: 300 ports: - "5433:5432" env_file: @@ -200,4 +223,6 @@ services: ports: - "8080:8080" depends_on: - - kc_postgresql + kc_postgresql: + condition: service_healthy + restart: true From 6f62d8ec2a30a3c5f5be9b9c9c74dc9f74b55385 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Wed, 22 Jan 2025 09:35:19 +0100 Subject: [PATCH 3/5] =?UTF-8?q?=E2=9C=A8(compose)=20a=20config=20to=20depl?= =?UTF-8?q?oy=20docs=20in=20production=20using=20compose?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To ease the deployment of the application on infrastructure not running k8s we created a compose file for production. --- .gitignore | 1 + Makefile | 42 ++++- bin/_config.sh | 9 +- bin/update_app_cacert.sh | 17 ++ compose.production.yaml | 167 ++++++++++++++++++ compose.yaml | 2 + .../production/etc/nginx/conf.d/default.conf | 132 ++++++++++++++ env.d/production.dist/backend | 58 ++++++ env.d/production.dist/kc_postgresql | 9 + env.d/production.dist/keycloak | 9 + env.d/production.dist/minio | 2 + env.d/production.dist/postgresql | 11 ++ env.d/production.dist/yprovider | 5 + 13 files changed, 458 insertions(+), 6 deletions(-) create mode 100755 bin/update_app_cacert.sh create mode 100644 compose.production.yaml create mode 100644 docker/files/production/etc/nginx/conf.d/default.conf create mode 100644 env.d/production.dist/backend create mode 100644 env.d/production.dist/kc_postgresql create mode 100644 env.d/production.dist/keycloak create mode 100644 env.d/production.dist/minio create mode 100644 env.d/production.dist/postgresql create mode 100644 env.d/production.dist/yprovider diff --git a/.gitignore b/.gitignore index aa5573815..c73156108 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ ENV/ env.bak/ venv.bak/ env.d/development/* +env.d/production/* !env.d/development/*.dist env.d/terraform diff --git a/Makefile b/Makefile index 996662465..d477ccecf 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,8 @@ DB_PORT = 5432 DOCKER_UID = $(shell id -u) DOCKER_GID = $(shell id -g) DOCKER_USER = $(DOCKER_UID):$(DOCKER_GID) -COMPOSE = DOCKER_USER=$(DOCKER_USER) docker compose +COMPOSE = DOCKER_USER=$(DOCKER_USER) ./bin/compose +COMPOSE_PRODUCTION = DOCKER_USER=$(DOCKER_USER) COMPOSE_FILE=compose.production.yaml ./bin/compose COMPOSE_EXEC = $(COMPOSE) exec COMPOSE_EXEC_APP = $(COMPOSE_EXEC) app-dev COMPOSE_RUN = $(COMPOSE) run --rm @@ -64,6 +65,19 @@ data/media: data/static: @mkdir -p data/static +# -- production volumes +data/production/media: + @mkdir -p data/production/media + +data/production/certs: + @mkdir -p data/production/certs + +data/production/databases/backend: + @mkdir -p data/production/databases/backend + +data/production/databases/keycloak: + @mkdir -p data/production/databases/keycloak + # -- Project create-env-files: ## Copy the dist env files to env files @@ -88,6 +102,27 @@ bootstrap: \ mails-build .PHONY: bootstrap +bootstrap-production: ## Prepare project to run in production mode using docker compose +bootstrap-production: \ + env.d/production \ + data/production/media \ + data/production/certs \ + data/production/databases/backend \ + data/production/databases/keycloak +bootstrap-production: + @echo 'Environment files created in env.d/production' + @echo 'Edit them to set good value for your production environment' +.PHONY: bootstrap-production + +run-production: ## Run compose project in production mode + @$(COMPOSE_PRODUCTION) up -d ingress +.PHONY: run-production + +stop-production: ## Stop compose project in production mode + @$(COMPOSE_PRODUCTION) stop +.PHONY: stop-production + + # -- Docker/compose build: cache ?= --no-cache build: ## build the project containers @@ -224,6 +259,8 @@ resetdb: ## flush database and create a superuser "admin" @${MAKE} superuser .PHONY: resetdb +# -- Environment variable files + env.d/development/common: cp -n env.d/development/common.dist env.d/development/common @@ -233,6 +270,9 @@ env.d/development/postgresql: env.d/development/kc_postgresql: cp -n env.d/development/kc_postgresql.dist env.d/development/kc_postgresql +env.d/production: + cp -rnf env.d/production.dist env.d/production + # -- Internationalization env.d/development/crowdin: diff --git a/bin/_config.sh b/bin/_config.sh index dd2a5c29c..28a3c2b49 100644 --- a/bin/_config.sh +++ b/bin/_config.sh @@ -6,9 +6,9 @@ REPO_DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd)" UNSET_USER=0 TERRAFORM_DIRECTORY="./env.d/terraform" -COMPOSE_FILE="${REPO_DIR}/compose.yaml" -COMPOSE_PROJECT="docs" - +if [ -z ${COMPOSE_FILE+x} ]; then + COMPOSE_FILE="${REPO_DIR}/compose.yaml" +fi # _set_user: set (or unset) default user id used to run docker commands # @@ -40,9 +40,8 @@ function _set_user() { # ARGS : docker compose command arguments function _docker_compose() { - echo "🐳(compose) project: '${COMPOSE_PROJECT}' file: '${COMPOSE_FILE}'" + echo "🐳(compose) project, file: '${COMPOSE_FILE}'" docker compose \ - -p "${COMPOSE_PROJECT}" \ -f "${COMPOSE_FILE}" \ --project-directory "${REPO_DIR}" \ "$@" diff --git a/bin/update_app_cacert.sh b/bin/update_app_cacert.sh new file mode 100755 index 000000000..702ff8eea --- /dev/null +++ b/bin/update_app_cacert.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -o errexit + +# The script is pretty simple. It downloads the latest cacert.pem file from the certifi package and appends the root certificate from mkcert to it. Then it copies the updated cacert.pem file to the container. +# The script is executed with the following command: +# $ bin/update_app_cacert.sh docs-production-backend-1 + +CONTAINER_NAME=${1:-"docs-production-backend-1"} + +echo "updating cacert.pem for certifi package in ${CONTAINER_NAME}" + + +curl --create-dirs https://raw.githubusercontent.com/certifi/python-certifi/refs/heads/master/certifi/cacert.pem -o /tmp/certifi/cacert.pem +cat "$(mkcert -CAROOT)/rootCA.pem" >> /tmp/certifi/cacert.pem +docker cp /tmp/certifi/cacert.pem ${CONTAINER_NAME}:/usr/local/lib/python3.12/site-packages/certifi/cacert.pem + +echo "end patching cacert.pem in ${CONTAINER_NAME}" diff --git a/compose.production.yaml b/compose.production.yaml new file mode 100644 index 000000000..ffeee0436 --- /dev/null +++ b/compose.production.yaml @@ -0,0 +1,167 @@ +name: docs-production + +services: + postgresql: + image: postgres:16 + healthcheck: + test: ["CMD", "pg_isready", "-q", "-U", "docs", "-d", "docs"] + interval: 1s + timeout: 2s + retries: 300 + env_file: + - env.d/production/postgresql + environment: + - PGDATA=/var/lib/postgresql/data/pgdata + volumes: + - ./data/production/databases/backend:/var/lib/postgresql/data/pgdata + + redis: + image: redis:5 + + backend-migration: + image: lasuite/impress-backend:latest + user: ${DOCKER_USER:-1000} + command: ["python", "manage.py", "migrate", "--noinput"] + environment: + - DJANGO_CONFIGURATION=Production + env_file: + - env.d/production/backend + - env.d/production/postgresql + - env.d/production/yprovider + depends_on: + postgresql: + condition: service_healthy + restart: true + redis: + condition: service_started + minio: + condition: service_started + + backend: + image: lasuite/impress-backend:latest + user: ${DOCKER_USER:-1000} + restart: always + environment: + - DJANGO_CONFIGURATION=Production + env_file: + - env.d/production/backend + - env.d/production/postgresql + - env.d/production/yprovider + healthcheck: + test: ["CMD", "python", "manage.py", "check"] + interval: 15s + timeout: 30s + retries: 20 + start_period: 10s + depends_on: + postgresql: + condition: service_healthy + restart: true + backend-migration: + condition: service_completed_successfully + redis: + condition: service_started + minio: + condition: service_started + minio-bootstrap: + condition: service_completed_successfully + + celery: + user: ${DOCKER_USER:-1000} + image: lasuite/impress-backend:latest + command: ["celery", "-A", "impress.celery_app", "worker", "-l", "INFO"] + environment: + - DJANGO_CONFIGURATION=Production + env_file: + - env.d/production/backend + - env.d/production/postgresql + - env.d/production/yprovider + depends_on: + - backend + + frontend: + image: lasuite/impress-frontend:latest + user: ${DOCKER_USER:-1000} + + y-provider: + image: lasuite/impress-y-provider:latest + user: ${DOCKER_USER:-1000} + env_file: + - env.d/production/yprovider + + kc_postgresql: + image: postgres:16 + healthcheck: + test: ["CMD", "pg_isready", "-q", "-U", "keycloak", "-d", "keycloak"] + interval: 1s + timeout: 2s + retries: 300 + env_file: + - env.d/production/kc_postgresql + environment: + - PGDATA=/var/lib/postgresql/data/pgdata + volumes: + - ./data/production/databases/keycloak:/var/lib/postgresql/data/pgdata + + keycloak: + image: quay.io/keycloak/keycloak:26.1.0 + command: ["start"] + env_file: + - env.d/production/keycloak + - env.d/production/kc_postgresql + ports: + - "8443:8443" + volumes: + - ${DOCS_PROD_KEYCLOAK_CERT_FOLDER:-./data/production/certs}:/etc/ssl/certs:ro + depends_on: + kc_postgresql: + condition: service_healthy + restart: true + + minio-bootstrap: + image: minio/mc + env_file: + - env.d/production/minio + depends_on: + minio: + condition: service_healthy + restart: true + entrypoint: > + sh -c " + /usr/bin/mc alias set docs http://minio:9000 $${MINIO_ROOT_USER} $${MINIO_ROOT_PASSWORD} && \ + /usr/bin/mc mb --ignore-existing docs/docs-media-storage && \ + /usr/bin/mc version enable docs/docs-media-storage && \ + exit 0;" + + minio: + user: ${DOCKER_USER:-1000} + image: minio/minio + env_file: + - env.d/production/minio + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 1s + timeout: 20s + retries: 300 + entrypoint: "" + command: minio server /data + volumes: + - ./data/production/media:/data + + ingress: + image: nginx:1.27 + ports: + - "${DOCS_PROD_NGING_PORT:-443}:8083" + volumes: + - ./docker/files/production/etc/nginx/conf.d:/etc/nginx/conf.d:ro + - ${DOCS_PROD_NGINX_CERT_FOLDER:-./data/production/certs}:/etc/nginx/ssl:ro + depends_on: + frontend: + condition: service_started + y-provider: + condition: service_started + keycloak: + condition: service_started + backend: + condition: service_healthy + restart: true diff --git a/compose.yaml b/compose.yaml index 0126c6443..1490a30cc 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,3 +1,5 @@ +name: docs + services: postgresql: image: postgres:16 diff --git a/docker/files/production/etc/nginx/conf.d/default.conf b/docker/files/production/etc/nginx/conf.d/default.conf new file mode 100644 index 000000000..9f426ead5 --- /dev/null +++ b/docker/files/production/etc/nginx/conf.d/default.conf @@ -0,0 +1,132 @@ +upstream docs_backend { + server backend:8000 fail_timeout=0; +} + +upstream docs_frontend { + server frontend:8080 fail_timeout=0; +} + +server { + listen 8083 ssl; + server_name localhost; + + # Disables server version feedback on pages and in headers + server_tokens off; + + ssl_certificate /etc/nginx/ssl/fullchain.pem; + ssl_certificate_key /etc/nginx/ssl/privkey.pem; + + location @proxy_to_docs_backend { + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_redirect off; + proxy_pass http://docs_backend; + } + + location @proxy_to_docs_frontend { + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_redirect off; + proxy_pass http://docs_frontend; + } + + location / { + try_files $uri @proxy_to_docs_frontend; + } + + location /api { + try_files $uri @proxy_to_docs_backend; + } + + location /admin { + try_files $uri @proxy_to_docs_backend; + } + + # Proxy auth for collaboration server + location /collaboration/ws/ { + # Collaboration Auth request configuration + auth_request /collaboration-auth; + proxy_set_header X-Forwarded-Proto https; + auth_request_set $authHeader $upstream_http_authorization; + auth_request_set $canEdit $upstream_http_x_can_edit; + auth_request_set $userId $upstream_http_x_user_id; + + # Pass specific headers from the auth response + proxy_set_header Authorization $authHeader; + proxy_set_header X-Can-Edit $canEdit; + proxy_set_header X-User-Id $userId; + + # Ensure WebSocket upgrade + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + + # Collaboration server + proxy_pass http://y-provider:4444; + + # Set appropriate timeout for WebSocket + proxy_read_timeout 86400; + proxy_send_timeout 86400; + + # Preserve original host and additional headers + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Origin $http_origin; + proxy_set_header Host $host; + } + + location /collaboration-auth { + proxy_pass http://docs_backend/api/v1.0/documents/collaboration-auth/; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Original-URL $request_uri; + + # Prevent the body from being passed + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-Method $request_method; + } + + location /collaboration/api/ { + # Collaboration server + proxy_pass http://y-provider:4444; + proxy_set_header Host $host; + } + + # Proxy auth for media + location /media/ { + # Auth request configuration + auth_request /media-auth; + auth_request_set $authHeader $upstream_http_authorization; + auth_request_set $authDate $upstream_http_x_amz_date; + auth_request_set $authContentSha256 $upstream_http_x_amz_content_sha256; + + # Pass specific headers from the auth response + proxy_set_header Authorization $authHeader; + proxy_set_header X-Amz-Date $authDate; + proxy_set_header X-Amz-Content-SHA256 $authContentSha256; + + # Get resource from Minio + proxy_pass http://minio:9000/docs-media-storage/; + proxy_set_header Host minio:9000; + } + + location /media-auth { + proxy_pass http://docs_backend/api/v1.0/documents/media-auth/; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Original-URL $request_uri; + + # Prevent the body from being passed + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-Method $request_method; + } +} diff --git a/env.d/production.dist/backend b/env.d/production.dist/backend new file mode 100644 index 000000000..f49db5582 --- /dev/null +++ b/env.d/production.dist/backend @@ -0,0 +1,58 @@ +## Django +DJANGO_ALLOWED_HOSTS=impress.127.0.0.1.nip.io,keycloack.127.0.0.1.nip.io +DJANGO_SECRET_KEY=ThisIsAnExampleKeyForDevPurposeOnly +DJANGO_SETTINGS_MODULE=impress.settings +DJANGO_SUPERUSER_PASSWORD=ThisIsAnExamplePassword + +# Logging +# Set to DEBUG level for dev only +LOGGING_LEVEL_HANDLERS_CONSOLE=ERROR +LOGGING_LEVEL_LOGGERS_ROOT=INFO +LOGGING_LEVEL_LOGGERS_APP=INFO + +# Python +PYTHONPATH=/app + +# impress settings + +# Mail +DJANGO_EMAIL_BRAND_NAME="La Suite NumΓ©rique" +DJANGO_EMAIL_HOST="mailcatcher" +DJANGO_EMAIL_LOGO_IMG="https://impress.127.0.0.1.nip.io/assets/logo-suite-numerique.png" +DJANGO_EMAIL_PORT=1025 + +# Media +STORAGES_STATICFILES_BACKEND=django.contrib.staticfiles.storage.StaticFilesStorage +AWS_S3_ENDPOINT_URL=http://minio:9000 +AWS_S3_ACCESS_KEY_ID= +AWS_S3_SECRET_ACCESS_KEY= +AWS_STORAGE_BUCKET_NAME=docs-media-storage +MEDIA_BASE_URL=impress.127.0.0.1.nip.io + +# OIDC +USER_OIDC_FIELD_TO_SHORTNAME="given_name" +USER_OIDC_FIELDS_TO_FULLNAME="given_name,usual_name" +OIDC_OP_JWKS_ENDPOINT=https://impress.127.0.0.1.nip.io/realms/impress/protocol/openid-connect/certs +OIDC_OP_AUTHORIZATION_ENDPOINT=https://impress.127.0.0.1.nip.io/realms/impress/protocol/openid-connect/auth +OIDC_OP_TOKEN_ENDPOINT=https://impress.127.0.0.1.nip.io/realms/impress/protocol/openid-connect/token +OIDC_OP_USER_ENDPOINT=https://impress.127.0.0.1.nip.io/realms/impress/protocol/openid-connect/userinfo +OIDC_OP_LOGOUT_ENDPOINT=https://impress.127.0.0.1.nip.io/realms/impress/protocol/openid-connect/logout +OIDC_RP_CLIENT_ID=impress +OIDC_RP_CLIENT_SECRETThisIsAnExampleKeyForDevPurposeOnly +OIDC_RP_SIGN_ALGO=RS256 +OIDC_RP_SCOPES="openid email" + +LOGIN_REDIRECT_URL=https://impress.127.0.0.1.nip.io +LOGIN_REDIRECT_URL_FAILURE=https://impress.127.0.0.1.nip.io +LOGOUT_REDIRECT_URL=https://impress.127.0.0.1.nip.io + +OIDC_REDIRECT_ALLOWED_HOSTS=["https://impress.127.0.0.1.nip.io"] +OIDC_AUTH_REQUEST_EXTRA_PARAMS={"acr_values": "eidas1"} + +# AI +AI_BASE_URL=https://openaiendpoint.com +AI_API_KEY=password +AI_MODEL=llama + +# Frontend +FRONTEND_THEME=dsfr diff --git a/env.d/production.dist/kc_postgresql b/env.d/production.dist/kc_postgresql new file mode 100644 index 000000000..8caaf2673 --- /dev/null +++ b/env.d/production.dist/kc_postgresql @@ -0,0 +1,9 @@ +# Postgresql db container configuration +POSTGRES_DB=keycloak +POSTGRES_USER=keycloak +POSTGRES_PASSWORD= + +# Keycloak database configuration +KC_DB_URL_DATABASE=keycloak +KC_DB_USERNAME=keycloak +KC_DB_PASSWORD= diff --git a/env.d/production.dist/keycloak b/env.d/production.dist/keycloak new file mode 100644 index 000000000..1320daed1 --- /dev/null +++ b/env.d/production.dist/keycloak @@ -0,0 +1,9 @@ +KC_BOOTSTRAP_ADMIN_USERNAME= +KC_BOOTSTRAP_ADMIN_PASSWORD= +KC_DB=postgres +KC_DB_URL_HOST=kc_postgresql +KC_DB_SCHEMA=public +PROXY_ADDRESS_FORWARDING='true' +KC_HOSTNAME=http://localhost:8083 +KC_HTTPS_CERTIFICATE_FILE=/etc/ssl/certs/docs.crt +KC_HTTPS_CERTIFICATE_KEY_FILE=/etc/ssl/private/docs.key diff --git a/env.d/production.dist/minio b/env.d/production.dist/minio new file mode 100644 index 000000000..18d384cb6 --- /dev/null +++ b/env.d/production.dist/minio @@ -0,0 +1,2 @@ +MINIO_ROOT_USER= +MINIO_ROOT_PASSWORD= diff --git a/env.d/production.dist/postgresql b/env.d/production.dist/postgresql new file mode 100644 index 000000000..232ea9165 --- /dev/null +++ b/env.d/production.dist/postgresql @@ -0,0 +1,11 @@ +# Postgresql db container configuration +POSTGRES_DB=docs +POSTGRES_USER=docs +POSTGRES_PASSWORD= + +# App database configuration +DB_HOST=postgresql +DB_NAME=docs +DB_USER=docs +DB_PASSWORD= +DB_PORT=5432 diff --git a/env.d/production.dist/yprovider b/env.d/production.dist/yprovider new file mode 100644 index 000000000..244bf287a --- /dev/null +++ b/env.d/production.dist/yprovider @@ -0,0 +1,5 @@ +COLLABORATION_LOGGING=true +Y_PROVIDER_API_KEY= +COLLABORATION_API_URL=https://impress.127.0.0.1.nip.io/collaboration/api/ +COLLABORATION_SERVER_ORIGIN=https://impress.127.0.0.1.nip.io +COLLABORATION_SERVER_SECRET= From 350643a4c82f43caeb6509871c4b0a32fb99455b Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Tue, 28 Jan 2025 12:34:25 +0100 Subject: [PATCH 4/5] =?UTF-8?q?=F0=9F=93=9D(doc)=20add=20documentation=20t?= =?UTF-8?q?o=20install=20using=20compose?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Write the documentation related to the docker compose installation. --- docs/installation/compose.md | 66 +++++++++++++++++++ docs/{installation.md => installation/k8s.md} | 0 2 files changed, 66 insertions(+) create mode 100644 docs/installation/compose.md rename docs/{installation.md => installation/k8s.md} (100%) diff --git a/docs/installation/compose.md b/docs/installation/compose.md new file mode 100644 index 000000000..13eb599dc --- /dev/null +++ b/docs/installation/compose.md @@ -0,0 +1,66 @@ +# Installation with docker compose + +We provide a configuration for running docs in production using docker compose. This configuration is experimental, the official way to deploy docs in production is to use [k8s](docs/installation/k8s.md) + +## Requirements + +- A modern version of docker and its compose plugin. +- SSL certificates for docs domain and keycloak. +- Two domain name. One for the docs application and an other one for keycloak. Both can be a subdomain of a common domain. (example: docs.domain.tld and keycloak.domain.tld) + +## Installation + +- Clone this repository: `git clone https://github.com/suitenumerique/docs.git` +- Then in the clone directory you can run the following command: `make bootsrap-production` + +## Configure your ssl certificates + +You have to provide the ssl certificates. The easiest way is to use [certbot](https://certbot.eff.org/), generate the certificates with it (both for docs and keycloak) and then mount them in ingress and keycloak containers. Two environment variables can be used for that: +- `DOCS_PROD_NGINX_CERT_FOLDER` path to the folder containing the certificates for docs. This folder will be mounted in `/etc/nginx/ssl` in the container. You have to adapt the certificates name in the file `docker/files/production/etc/nginx/conf.d/default.conf` accordingly with the certificates name you have (see `ssl_certificate` and `ssl_certificate_key` directives). +- `DOCS_PROD_KEYCLOAK_CERT_FOLDER` path to the folder containing the certificates for keycloak. This folder will be mounted in `/etc/ssl/certs` in the container. You have to adapt the certificates name in the configuration file in `env.d/production/keycloak` to add the correct path for environment variables `KC_HTTPS_CERTIFICATE_FILE` and `KC_HTTPS_CERTIFICATE_KEY_FILE`. + +### Configuration + +All the configuration files are in the directory `env.d/production`. You have to edit all the files to complete them. For the OIDC information you will have them once keycloak will be running and you will have configure your own realm on it. + +#### env.d/production/minio + +All the settings related to minio. You have to a username and a password to manage the minio cluster. You will need them later in the `env.d/production/backend file + +#### env.d/production/postgresql + +All the settings related to the postgresql database used by the Django application + +#### env.d/production/yprovider + +All the settings related to the collaboration server. All the secret and api key must be generated. + +#### env.d/production/kc_postgresql + +All the settings related to the postgresql database used by keycloak + +#### env.d/production/keycloak + +All the settings related to the keycloak application + +#### env.d/production/backend + +All the settings related to the Django application. Only the settings you don't have for now are all the one related to OIDC. You will have them once the compose started and you can access to keycloak. + +## Run the compose configuration + +The compose configuration can be run with the following command: `make run-production`. The first start can be a little bit long, lots of things are created. Once started you can check thant everything is running with the following command: `COMPOSE_FILE=compose.production.yaml ./bin/compose ps` + +## Configure keycloak + +You have to create a new realm in your keycloak and once created you have to create a new OIDC client in it. You will use this client to configure the OIDC part in `env.d/production/backend`. This is the last missing to complete the django application configuration. +Once the client information are set in `env.d/production/backend` you have to start the containers again by running the commande `make run-production`. The command will recreate the containers with the good configuration. + +### Helpers + +there is a helper script to control the `docker compose` command. You can export the variable `COMPOSE_FILE` with the compose filename (`export COMPOSE_FILE=compose.production.yaml`). After you can run `./bin/compose` to run the docker compose command line. + +Makefile commands available: +- `make bootstrap-production`: create the configuration files in `env.d/production`, create the directories : `data/production`. Both directories must be backup, if you loose them you loose all the data related to the application. +- `make run-production`: up the ingress containers. Will start all the containers needed in cascade. +- `make stop-production`: stop all the containers. diff --git a/docs/installation.md b/docs/installation/k8s.md similarity index 100% rename from docs/installation.md rename to docs/installation/k8s.md From fed3ad6a8136468dd27a58c9a7c0ac873b66a749 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Tue, 28 Jan 2025 16:31:49 +0100 Subject: [PATCH 5/5] =?UTF-8?q?fixup!=20=F0=9F=93=9D(doc)=20add=20document?= =?UTF-8?q?ation=20to=20install=20using=20compose?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/installation/compose.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/installation/compose.md b/docs/installation/compose.md index 13eb599dc..d6bbbfc8e 100644 --- a/docs/installation/compose.md +++ b/docs/installation/compose.md @@ -1,12 +1,12 @@ # Installation with docker compose -We provide a configuration for running docs in production using docker compose. This configuration is experimental, the official way to deploy docs in production is to use [k8s](docs/installation/k8s.md) +We provide a configuration for running Docs in production using docker compose. This configuration is experimental, the official way to deploy Docs in production is to use [k8s](docs/installation/k8s.md) ## Requirements -- A modern version of docker and its compose plugin. -- SSL certificates for docs domain and keycloak. -- Two domain name. One for the docs application and an other one for keycloak. Both can be a subdomain of a common domain. (example: docs.domain.tld and keycloak.domain.tld) +- A modern version of Docker and its Compose plugin. +- SSL certificates for Docs domain and Keycloak. +- Two domain name. One for the Docs application and an other one for Keycloak. Both can be a subdomain of a common domain. (example: docs.domain.tld and keycloak.domain.tld) ## Installation @@ -15,21 +15,21 @@ We provide a configuration for running docs in production using docker compose. ## Configure your ssl certificates -You have to provide the ssl certificates. The easiest way is to use [certbot](https://certbot.eff.org/), generate the certificates with it (both for docs and keycloak) and then mount them in ingress and keycloak containers. Two environment variables can be used for that: -- `DOCS_PROD_NGINX_CERT_FOLDER` path to the folder containing the certificates for docs. This folder will be mounted in `/etc/nginx/ssl` in the container. You have to adapt the certificates name in the file `docker/files/production/etc/nginx/conf.d/default.conf` accordingly with the certificates name you have (see `ssl_certificate` and `ssl_certificate_key` directives). -- `DOCS_PROD_KEYCLOAK_CERT_FOLDER` path to the folder containing the certificates for keycloak. This folder will be mounted in `/etc/ssl/certs` in the container. You have to adapt the certificates name in the configuration file in `env.d/production/keycloak` to add the correct path for environment variables `KC_HTTPS_CERTIFICATE_FILE` and `KC_HTTPS_CERTIFICATE_KEY_FILE`. +You have to provide the ssl certificates. The easiest way is to use [certbot](https://certbot.eff.org/), generate the certificates with it (both for Docs and Keycloak) and then mount them in ingress and keycloak containers. Two environment variables can be used for that: +- `DOCS_PROD_NGINX_CERT_FOLDER` path to the folder containing the certificates for Docs. This folder will be mounted in `/etc/nginx/ssl` in the container. You have to adapt the certificates name in the file `docker/files/production/etc/nginx/conf.d/default.conf` accordingly with the certificates name you have (see `ssl_certificate` and `ssl_certificate_key` directives). +- `DOCS_PROD_KEYCLOAK_CERT_FOLDER` path to the folder containing the certificates for Keycloak. This folder will be mounted in `/etc/ssl/certs` in the container. You have to adapt the certificates name in the configuration file in `env.d/production/keycloak` to add the correct path for environment variables `KC_HTTPS_CERTIFICATE_FILE` and `KC_HTTPS_CERTIFICATE_KEY_FILE`. ### Configuration -All the configuration files are in the directory `env.d/production`. You have to edit all the files to complete them. For the OIDC information you will have them once keycloak will be running and you will have configure your own realm on it. +All the configuration files are in the directory `env.d/production`. You have to edit all the files to complete them. For the OIDC information you will have them once Keycloak will be running and you have configured your own realm on it. #### env.d/production/minio -All the settings related to minio. You have to a username and a password to manage the minio cluster. You will need them later in the `env.d/production/backend file +All the settings related to Minio. You have to set a username and a password to manage the minio cluster. You will need them later in the `env.d/production/backend` file. #### env.d/production/postgresql -All the settings related to the postgresql database used by the Django application +All the settings related to the Postgresql database used by the Django application. #### env.d/production/yprovider @@ -37,24 +37,24 @@ All the settings related to the collaboration server. All the secret and api key #### env.d/production/kc_postgresql -All the settings related to the postgresql database used by keycloak +All the settings related to the Postgresql database used by keycloak. #### env.d/production/keycloak -All the settings related to the keycloak application +All the settings related to the Keycloak application. #### env.d/production/backend -All the settings related to the Django application. Only the settings you don't have for now are all the one related to OIDC. You will have them once the compose started and you can access to keycloak. +All the settings related to the Django application. Only the settings you don't have for now are all the one related to OIDC. You will have them once the compose started and you can access to Keycloak. ## Run the compose configuration -The compose configuration can be run with the following command: `make run-production`. The first start can be a little bit long, lots of things are created. Once started you can check thant everything is running with the following command: `COMPOSE_FILE=compose.production.yaml ./bin/compose ps` +The compose configuration can be run with the following command: `make run-production`. The first start can be a little bit long, lots of things are created. Once started you can check that everything is running with the following command: `COMPOSE_FILE=compose.production.yaml ./bin/compose ps` ## Configure keycloak -You have to create a new realm in your keycloak and once created you have to create a new OIDC client in it. You will use this client to configure the OIDC part in `env.d/production/backend`. This is the last missing to complete the django application configuration. -Once the client information are set in `env.d/production/backend` you have to start the containers again by running the commande `make run-production`. The command will recreate the containers with the good configuration. +You have to create a new realm in your Keycloak and once created you have to create a new OIDC client in it. You will use this client to configure the OIDC part in `env.d/production/backend`. This is the last missing part to complete the Django application configuration. +Once the client information set in `env.d/production/backend` you have to start the containers again by running the commande `make run-production`. The command will recreate the containers with the good configuration. ### Helpers