From 30a85b49d237add78f988b7b38e5c84be6450bb7 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 23 Sep 2021 13:59:19 +0200 Subject: [PATCH 01/18] file is not included if missing --- .../src/dy_static_file_server/inputs_to_outputs.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/services/dy-static-file-server/src/dy_static_file_server/inputs_to_outputs.py b/services/dy-static-file-server/src/dy_static_file_server/inputs_to_outputs.py index 7cc2dd6f..82cc6978 100644 --- a/services/dy-static-file-server/src/dy_static_file_server/inputs_to_outputs.py +++ b/services/dy-static-file-server/src/dy_static_file_server/inputs_to_outputs.py @@ -39,8 +39,9 @@ def remap_input_to_output(input_dir: Path, output_dir: Path) -> None: # move file to correct path input_file: Path = input_dir / "file_input" / "test_file" output_file_path: Path = output_dir / "file_output" / "test_file" - output_file_path.parent.mkdir(parents=True, exist_ok=True) - output_file_path.write_bytes(input_file.read_bytes()) + if input_file.is_file(): + output_file_path.parent.mkdir(parents=True, exist_ok=True) + output_file_path.write_bytes(input_file.read_bytes()) # rewrite key_values.json inputs_key_values_file = input_dir / "key_values.json" @@ -48,7 +49,8 @@ def remap_input_to_output(input_dir: Path, output_dir: Path) -> None: outputs_key_values = { k.replace("_input", "_output"): v["value"] for k, v in inputs_key_values.items() } - outputs_key_values["file_output"] = f"{output_file_path}" + if input_file.is_file(): + outputs_key_values["file_output"] = f"{output_file_path}" outputs_key_values_file = output_dir / "key_values.json" outputs_key_values_file.write_text( @@ -125,10 +127,6 @@ def main() -> None: inputs_objserver = InputsObserver(input_dir, output_dir) inputs_objserver.start() - - # manually trigger once when it starts - # remaps inputs to outputs before continuing - remap_input_to_output(input_dir=input_dir, output_dir=output_dir) inputs_objserver.join() logger.info("%s main exited", InputsObserver.__name__) From 982e59f1b71708480c57ae2086bee17cbf63d7d0 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 23 Sep 2021 13:59:27 +0200 Subject: [PATCH 02/18] fix nginx boot --- .../docker-compose-meta.yml | 6 ++--- .../docker/custom/Dockerfile | 27 ++++++++++++++++++- .../docker/custom/boot.sh | 8 ++++-- .../docker/custom/default.conf | 20 ++++++++++++++ 4 files changed, 55 insertions(+), 6 deletions(-) create mode 100644 services/dy-static-file-server/docker/custom/default.conf diff --git a/services/dy-static-file-server/docker-compose-meta.yml b/services/dy-static-file-server/docker-compose-meta.yml index 341b32d6..52a7f282 100644 --- a/services/dy-static-file-server/docker-compose-meta.yml +++ b/services/dy-static-file-server/docker-compose-meta.yml @@ -20,7 +20,7 @@ services: org.label-schema.vcs-url: ${VCS_URL} simcore.service.settings: '[{"name": "resources", "type": "Resources", "value": {"mem_limit":17179869184, "cpu_limit": 1000000000}}, {"name": "ports", "type": - "int", "value": 80}, {"name": "constraints", "type": "string", "value": + "int", "value": 8000}, {"name": "constraints", "type": "string", "value": ["node.platform.os == linux"]}]' dy-static-file-server-dynamic-sidecar: build: @@ -64,7 +64,7 @@ services: "/www/inputs", "state_paths": ["/workdir/generated-data"]}' simcore.service.settings: '[{"name": "resources", "type": "Resources", "value": {"mem_limit":17179869184, "cpu_limit": 1000000000}}, {"name": "ports", "type": - "int", "value": 80}, {"name": "constraints", "type": "string", "value": + "int", "value": 8000}, {"name": "constraints", "type": "string", "value": ["node.platform.os == linux"]}]' dy-static-file-server-dynamic-sidecar-compose-spec: build: @@ -111,6 +111,6 @@ services: "/www/inputs", "state_paths": ["/workdir/generated-data"]}' simcore.service.settings: '[{"name": "resources", "type": "Resources", "value": {"mem_limit":17179869184, "cpu_limit": 1000000000}}, {"name": "ports", "type": - "int", "value": 80}, {"name": "constraints", "type": "string", "value": + "int", "value": 8000}, {"name": "constraints", "type": "string", "value": ["node.platform.os == linux"]}]' version: '3.7' diff --git a/services/dy-static-file-server/docker/custom/Dockerfile b/services/dy-static-file-server/docker/custom/Dockerfile index f6bf8715..a148178a 100644 --- a/services/dy-static-file-server/docker/custom/Dockerfile +++ b/services/dy-static-file-server/docker/custom/Dockerfile @@ -7,13 +7,21 @@ FROM nginx:1.21.0-alpine as production # docker run dy-static-file-server:prod # +# making nginx user simialar => # simcore-user uid=8004(scu) gid=8004(scu) groups=8004(scu) +# shadow required for groupmod and usermod +RUN apk --no-cache add shadow && \ + groupmod -g 8004 nginx && \ + usermod -u 8004 -g 8004 nginx + ARG PYTHON_VERSION="3.8.10-r0" ARG INPUT_DIR="/www/inputs" ARG OUTPUT_DIR="/www/outputs" ARG WORKDIR="/workdir" +ARG USER="nginx" LABEL maintainer=GitHK COPY docker/custom/nginx.conf /etc/nginx/nginx.conf +COPY docker/custom/default.conf /etc/nginx/conf.d/default.conf COPY docker/custom/boot.sh /boot.sh RUN apk add --update --no-cache \ @@ -30,16 +38,33 @@ RUN python3 -m venv /venv COPY requirements/base.txt /tmp/requirements.txt RUN /venv/bin/pip3 install -r /tmp/requirements.txt -COPY static-content/hello-world.txt /www/hello-world.txt RUN mkdir -p ${INPUT_DIR} && \ mkdir -p ${OUTPUT_DIR} && \ mkdir -p ${WORKDIR} +COPY static-content/hello-world.txt /www/hello-world.txt COPY src/dy_static_file_server ${WORKDIR}/dy_static_file_server +## add permissions +RUN chown -R ${USER}:${USER} /boot.sh && \ + chown -R ${USER}:${USER} /var/cache/nginx && \ + chown -R ${USER}:${USER} /var/log/nginx && \ + chown -R ${USER}:${USER} /etc/nginx/conf.d && \ + chown -R ${USER}:${USER} /etc/nginx/conf.d/default.conf && \ + chown -R ${USER}:${USER} ${INPUT_DIR} && \ + chown -R ${USER}:${USER} ${OUTPUT_DIR} && \ + chown -R ${USER}:${USER} ${WORKDIR} + +RUN touch /var/run/nginx.pid && \ + chown -R ${USER}:${USER} /var/run/nginx.pid + +## switch to non-root user +USER ${USER} + ENV NGINX_SERVER_ROOT="/www" WORKDIR ${WORKDIR}/dy_static_file_server +EXPOSE 8080 CMD [ "/bin/sh", "/boot.sh" ] \ No newline at end of file diff --git a/services/dy-static-file-server/docker/custom/boot.sh b/services/dy-static-file-server/docker/custom/boot.sh index b8c2cb0d..322f47c5 100644 --- a/services/dy-static-file-server/docker/custom/boot.sh +++ b/services/dy-static-file-server/docker/custom/boot.sh @@ -24,9 +24,13 @@ else echo "Nginx will serve from ${NGINX_SERVER_ROOT}, nothing to do" fi + +# display workdir content +ls -lah + # ensure some random data is created in /workdir /venv/bin/python ensure_random_workdir_data.py -# keep mirroring running in the background -exec /venv/bin/python folder_mirror.py & +# keep moving data from inputs to outputs +exec /venv/bin/python inputs_to_outputs.py & exec nginx -g "daemon off;" \ No newline at end of file diff --git a/services/dy-static-file-server/docker/custom/default.conf b/services/dy-static-file-server/docker/custom/default.conf new file mode 100644 index 00000000..ef9edf57 --- /dev/null +++ b/services/dy-static-file-server/docker/custom/default.conf @@ -0,0 +1,20 @@ +server { + listen 8000; + server_name localhost; + + #access_log /var/log/nginx/host.access.log main; + + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } + + #error_page 404 /404.html; + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} \ No newline at end of file From 94f3da00c6e4d2ac4537980c2667226ccc3a999f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 23 Sep 2021 13:59:50 +0200 Subject: [PATCH 03/18] bumping service version --- services/dy-static-file-server/.cookiecutterrc | 2 +- services/dy-static-file-server/VERSION | 2 +- services/dy-static-file-server/docker-compose-meta.yml | 6 +++--- .../metadata/metadata-dynamic-sidecar-compose-spec.yml | 2 +- .../metadata/metadata-dynamic-sidecar.yml | 2 +- services/dy-static-file-server/metadata/metadata.yml | 2 +- services/dy-static-file-server/versioning/service.cfg | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/services/dy-static-file-server/.cookiecutterrc b/services/dy-static-file-server/.cookiecutterrc index d73f2e2f..7e89d36b 100644 --- a/services/dy-static-file-server/.cookiecutterrc +++ b/services/dy-static-file-server/.cookiecutterrc @@ -35,4 +35,4 @@ default_context: project_slug: 'dy-static-file-server' project_type: 'computational' release_date: '2021' - version: '1.0.7' + version: '1.0.8' diff --git a/services/dy-static-file-server/VERSION b/services/dy-static-file-server/VERSION index 238d6e88..b0f3d96f 100644 --- a/services/dy-static-file-server/VERSION +++ b/services/dy-static-file-server/VERSION @@ -1 +1 @@ -1.0.7 +1.0.8 diff --git a/services/dy-static-file-server/docker-compose-meta.yml b/services/dy-static-file-server/docker-compose-meta.yml index 52a7f282..8f977dcb 100644 --- a/services/dy-static-file-server/docker-compose-meta.yml +++ b/services/dy-static-file-server/docker-compose-meta.yml @@ -13,7 +13,7 @@ services: io.simcore.name: '{"name": "dy-static-file-server"}' io.simcore.outputs: '{"outputs": {}}' io.simcore.type: '{"type": "dynamic"}' - io.simcore.version: '{"version": "1.0.7"}' + io.simcore.version: '{"version": "1.0.8"}' org.label-schema.build-date: ${BUILD_DATE} org.label-schema.schema-version: '1.0' org.label-schema.vcs-ref: ${VCS_REF} @@ -55,7 +55,7 @@ services: {"displayOrder": 5, "label": "File output", "description": "File from input", "type": "data:*/*", "fileToKeyMap": {"test_file": "file_output"}}}}' io.simcore.type: '{"type": "dynamic"}' - io.simcore.version: '{"version": "1.0.7"}' + io.simcore.version: '{"version": "1.0.8"}' org.label-schema.build-date: ${BUILD_DATE} org.label-schema.schema-version: '1.0' org.label-schema.vcs-ref: ${VCS_REF} @@ -100,7 +100,7 @@ services: {"displayOrder": 5, "label": "File output", "description": "File from input", "type": "data:*/*", "fileToKeyMap": {"test_file": "file_output"}}}}' io.simcore.type: '{"type": "dynamic"}' - io.simcore.version: '{"version": "1.0.7"}' + io.simcore.version: '{"version": "1.0.8"}' org.label-schema.build-date: ${BUILD_DATE} org.label-schema.schema-version: '1.0' org.label-schema.vcs-ref: ${VCS_REF} diff --git a/services/dy-static-file-server/metadata/metadata-dynamic-sidecar-compose-spec.yml b/services/dy-static-file-server/metadata/metadata-dynamic-sidecar-compose-spec.yml index c5847c82..4889a253 100644 --- a/services/dy-static-file-server/metadata/metadata-dynamic-sidecar-compose-spec.yml +++ b/services/dy-static-file-server/metadata/metadata-dynamic-sidecar-compose-spec.yml @@ -2,7 +2,7 @@ name: dy-static-file-server-dynamic-sidecar-compose-spec key: simcore/services/dynamic/dy-static-file-server-dynamic-sidecar-compose-spec type: dynamic integration-version: 1.0.0 -version: 1.0.7 +version: 1.0.8 description: Modern test dynamic service providing a docker-compose specification file (with dynamic sidecar and compose-spec). Changes to the inputs will be forwarded to the outputs. The /workdir/generated-data directory is populated if no content is present. contact: anderegg@itis.swiss authors: diff --git a/services/dy-static-file-server/metadata/metadata-dynamic-sidecar.yml b/services/dy-static-file-server/metadata/metadata-dynamic-sidecar.yml index a97fd0ae..deec22c8 100644 --- a/services/dy-static-file-server/metadata/metadata-dynamic-sidecar.yml +++ b/services/dy-static-file-server/metadata/metadata-dynamic-sidecar.yml @@ -2,7 +2,7 @@ name: dy-static-file-server-dynamic-sidecar key: simcore/services/dynamic/dy-static-file-server-dynamic-sidecar type: dynamic integration-version: 1.0.0 -version: 1.0.7 +version: 1.0.8 description: Modern test dynamic service (with dynamic sidecar). Changes to the inputs will be forwarded to the outputs. The /workdir/generated-data directory is populated if no content is present. contact: anderegg@itis.swiss authors: diff --git a/services/dy-static-file-server/metadata/metadata.yml b/services/dy-static-file-server/metadata/metadata.yml index 5aafc7db..a119d05b 100644 --- a/services/dy-static-file-server/metadata/metadata.yml +++ b/services/dy-static-file-server/metadata/metadata.yml @@ -2,7 +2,7 @@ name: dy-static-file-server key: simcore/services/dynamic/dy-static-file-server type: dynamic integration-version: 1.0.0 -version: 1.0.7 +version: 1.0.8 description: Legacy test dynamic service (starts using original director-v0). The /workdir/generated-data directory is populated if no content is present. contact: anderegg@itis.swiss authors: diff --git a/services/dy-static-file-server/versioning/service.cfg b/services/dy-static-file-server/versioning/service.cfg index 765c1b0b..d841faf0 100644 --- a/services/dy-static-file-server/versioning/service.cfg +++ b/services/dy-static-file-server/versioning/service.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.7 +current_version = 1.0.8 commit = False message = service/kernel version: {current_version} → {new_version} tag = False From e4b32dd6247418eae770aaeafa9b77f9ac7b2eab Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 24 Sep 2021 17:01:04 +0200 Subject: [PATCH 04/18] added some debug printing --- .../ensure_random_workdir_data.py | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/services/dy-static-file-server/src/dy_static_file_server/ensure_random_workdir_data.py b/services/dy-static-file-server/src/dy_static_file_server/ensure_random_workdir_data.py index 4753d149..f0cceb4e 100644 --- a/services/dy-static-file-server/src/dy_static_file_server/ensure_random_workdir_data.py +++ b/services/dy-static-file-server/src/dy_static_file_server/ensure_random_workdir_data.py @@ -1,17 +1,18 @@ -import logging import random from pathlib import Path from typing import List import uuid +import grp, pwd +import getpass +import os TARGET_DIRECTORY = Path("/workdir/generated-data") -logger = logging.getLogger(__name__) - def make_random_file(target_dir: Path) -> None: file_path = target_dir / f"{uuid.uuid4()}.txt" file_path.write_text("no random data here") + print(f"Created {file_path}") def get_files_in_directory(directory: Path) -> List[Path]: @@ -23,14 +24,26 @@ def is_content_present(directory: Path) -> bool: return len(get_files_in_directory(directory)) > 0 +def print_user_and_directory_info() -> None: + user = getpass.getuser() + groups = [g.gr_name for g in grp.getgrall() if user in g.gr_mem] + gid = pwd.getpwnam(user).pw_gid + groups.append(grp.getgrgid(gid).gr_name) + + print(f"User {user}, groups {groups}") + os.system("ls -lah /workdir") + + def ensure_random_data(target_dir: Path) -> None: target_dir.mkdir(parents=True, exist_ok=True) + print_user_and_directory_info() + + print(f"Creating {target_dir} if missing") + if is_content_present(target_dir): - logger.info( - "Skipping content genration. Already detected %s", - get_files_in_directory(target_dir), - ) + files = get_files_in_directory(target_dir) + print(f"Skipping content genration. Already detected: {files}") return for _ in range(random.randint(1, 10)): @@ -38,7 +51,6 @@ def ensure_random_data(target_dir: Path) -> None: def main() -> None: - TARGET_DIRECTORY.mkdir(parents=True, exist_ok=True) ensure_random_data(TARGET_DIRECTORY) From d375e0c8de434ef6e654520c831bed075758e827 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 24 Sep 2021 17:01:18 +0200 Subject: [PATCH 05/18] migrated to diffrente webserver --- .../docker/custom/Dockerfile | 66 ++++++-------- .../docker/custom/boot.sh | 32 +++---- .../docker/custom/default.conf | 20 ----- .../docker/custom/entrypoint.sh | 89 +++++++++++++++++++ .../docker/custom/index.html | 0 .../docker/custom/nginx.conf | 24 ----- 6 files changed, 126 insertions(+), 105 deletions(-) mode change 100644 => 100755 services/dy-static-file-server/docker/custom/boot.sh delete mode 100644 services/dy-static-file-server/docker/custom/default.conf create mode 100755 services/dy-static-file-server/docker/custom/entrypoint.sh create mode 100644 services/dy-static-file-server/docker/custom/index.html delete mode 100644 services/dy-static-file-server/docker/custom/nginx.conf diff --git a/services/dy-static-file-server/docker/custom/Dockerfile b/services/dy-static-file-server/docker/custom/Dockerfile index a148178a..77365b9b 100644 --- a/services/dy-static-file-server/docker/custom/Dockerfile +++ b/services/dy-static-file-server/docker/custom/Dockerfile @@ -1,5 +1,5 @@ # TODO: Please set your custom image here and adapt the Dockerfile/entrypoint.sh accordingly -FROM nginx:1.21.0-alpine as production +FROM joseluisq/static-web-server:2.0.2-alpine as production # # USAGE: # cd services/dy-static-file-server @@ -7,26 +7,29 @@ FROM nginx:1.21.0-alpine as production # docker run dy-static-file-server:prod # -# making nginx user simialar => # simcore-user uid=8004(scu) gid=8004(scu) groups=8004(scu) -# shadow required for groupmod and usermod -RUN apk --no-cache add shadow && \ - groupmod -g 8004 nginx && \ - usermod -u 8004 -g 8004 nginx - ARG PYTHON_VERSION="3.8.10-r0" -ARG INPUT_DIR="/www/inputs" -ARG OUTPUT_DIR="/www/outputs" ARG WORKDIR="/workdir" -ARG USER="nginx" + +ENV SC_BUILD_TARGET=production +ENV SERVER_ROOT="/www" +ENV INPUT_FOLDER="${SERVER_ROOT}/inputs" +ENV OUTPUT_FOLDER="${SERVER_ROOT}/outputs" +ENV SC_USER_ID=101 +ENV SERVER_PORT=8080 +ENV SERVER_LOG_LEVEL=debug + +# creating own project's user +ENV SC_USER_ID 9004 +ENV SC_USER_NAME scudy +RUN adduser -D -u ${SC_USER_ID} -s /bin/sh -h /home/${SC_USER_NAME} ${SC_USER_NAME} + LABEL maintainer=GitHK -COPY docker/custom/nginx.conf /etc/nginx/nginx.conf -COPY docker/custom/default.conf /etc/nginx/conf.d/default.conf -COPY docker/custom/boot.sh /boot.sh RUN apk add --update --no-cache \ "python3=${PYTHON_VERSION}" \ - py3-pip + py3-pip \ + su-exec RUN pip3 install --upgrade \ pip \ virtualenv @@ -34,37 +37,22 @@ RUN pip3 install --upgrade \ RUN mkdir -p /venv RUN python3 -m venv /venv +# add additional directories +RUN mkdir -p ${WORKDIR} && chown ${SC_USER_NAME}:${SC_USER_NAME} ${WORKDIR} && \ + mkdir -p /docker && chown ${SC_USER_NAME}:${SC_USER_NAME} /docker + +COPY --chown=${SC_USER_NAME}:${SC_USER_NAME} docker/custom/*.sh /docker + # add python app requirements COPY requirements/base.txt /tmp/requirements.txt RUN /venv/bin/pip3 install -r /tmp/requirements.txt +COPY --chown=${SC_USER_NAME}:${SC_USER_NAME} static-content/hello-world.txt /www/hello-world.txt +COPY --chown=${SC_USER_NAME}:${SC_USER_NAME} src/dy_static_file_server ${WORKDIR}/dy_static_file_server -RUN mkdir -p ${INPUT_DIR} && \ - mkdir -p ${OUTPUT_DIR} && \ - mkdir -p ${WORKDIR} - -COPY static-content/hello-world.txt /www/hello-world.txt -COPY src/dy_static_file_server ${WORKDIR}/dy_static_file_server - -## add permissions -RUN chown -R ${USER}:${USER} /boot.sh && \ - chown -R ${USER}:${USER} /var/cache/nginx && \ - chown -R ${USER}:${USER} /var/log/nginx && \ - chown -R ${USER}:${USER} /etc/nginx/conf.d && \ - chown -R ${USER}:${USER} /etc/nginx/conf.d/default.conf && \ - chown -R ${USER}:${USER} ${INPUT_DIR} && \ - chown -R ${USER}:${USER} ${OUTPUT_DIR} && \ - chown -R ${USER}:${USER} ${WORKDIR} - -RUN touch /var/run/nginx.pid && \ - chown -R ${USER}:${USER} /var/run/nginx.pid - -## switch to non-root user -USER ${USER} - -ENV NGINX_SERVER_ROOT="/www" WORKDIR ${WORKDIR}/dy_static_file_server EXPOSE 8080 -CMD [ "/bin/sh", "/boot.sh" ] \ No newline at end of file +ENTRYPOINT ["/bin/sh", "/docker/entrypoint.sh"] +CMD ["/docker/boot.sh"] \ No newline at end of file diff --git a/services/dy-static-file-server/docker/custom/boot.sh b/services/dy-static-file-server/docker/custom/boot.sh old mode 100644 new mode 100755 index 322f47c5..4252e32c --- a/services/dy-static-file-server/docker/custom/boot.sh +++ b/services/dy-static-file-server/docker/custom/boot.sh @@ -8,29 +8,17 @@ echo User : "$(id "$(whoami)")" echo Workdir : "$(pwd)" echo Env : "$(env)" - -if [ -n "${SIMCORE_NODE_BASEPATH+set}" ] -then - echo - echo moving website to "${NGINX_SERVER_ROOT}${SIMCORE_NODE_BASEPATH}"... - echo - - mkdir -p "${NGINX_SERVER_ROOT}${SIMCORE_NODE_BASEPATH}" - mv "${NGINX_SERVER_ROOT}"/*.txt "${NGINX_SERVER_ROOT}${SIMCORE_NODE_BASEPATH}" - - echo "Nginx will serve from ${NGINX_SERVER_ROOT}${SIMCORE_NODE_BASEPATH}" - -else - echo "Nginx will serve from ${NGINX_SERVER_ROOT}, nothing to do" -fi - - -# display workdir content +echo "/workdir content" ls -lah -# ensure some random data is created in /workdir +echo "ensure some random data is created in in /workdir/generated-data content" /venv/bin/python ensure_random_workdir_data.py -# keep moving data from inputs to outputs -exec /venv/bin/python inputs_to_outputs.py & -exec nginx -g "daemon off;" \ No newline at end of file +echo "/workdir/generated-data content" +ls -lah /workdir/generated-data/ + +echo "starting background inputs->ouputs transformation" +/venv/bin/python inputs_to_outputs.py & + +echo "booting static-web-server" +exec static-web-server \ No newline at end of file diff --git a/services/dy-static-file-server/docker/custom/default.conf b/services/dy-static-file-server/docker/custom/default.conf deleted file mode 100644 index ef9edf57..00000000 --- a/services/dy-static-file-server/docker/custom/default.conf +++ /dev/null @@ -1,20 +0,0 @@ -server { - listen 8000; - server_name localhost; - - #access_log /var/log/nginx/host.access.log main; - - location / { - root /usr/share/nginx/html; - index index.html index.htm; - } - - #error_page 404 /404.html; - - # redirect server error pages to the static page /50x.html - # - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } -} \ No newline at end of file diff --git a/services/dy-static-file-server/docker/custom/entrypoint.sh b/services/dy-static-file-server/docker/custom/entrypoint.sh new file mode 100755 index 00000000..8596920d --- /dev/null +++ b/services/dy-static-file-server/docker/custom/entrypoint.sh @@ -0,0 +1,89 @@ +#!/bin/sh +set -o errexit +set -o nounset + +IFS=$(printf '\n\t') +# This entrypoint script: +# +# - Executes *inside* of the container upon start as --user [default root] +# - Notice that the container *starts* as --user [default root] but +# *runs* as non-root user [$SC_USER_NAME] +# +echo Entrypoint for stage "${SC_BUILD_TARGET}" ... +echo User : "$(id "$(whoami)")" +echo Workdir : "$(pwd)" + + +# expect input/output folders to be mounted +stat "${INPUT_FOLDER}" > /dev/null 2>&1 || \ + (echo "ERROR: You must mount '${INPUT_FOLDER}' to deduce user and group ids" && exit 1) +stat "${OUTPUT_FOLDER}" > /dev/null 2>&1 || \ + (echo "ERROR: You must mount '${OUTPUT_FOLDER}' to deduce user and group ids" && exit 1) + +# NOTE: expects docker run ... -v /path/to/input/folder:${INPUT_FOLDER} +# check input/output folders are owned by the same user +if [ "$(stat -c %u "${INPUT_FOLDER}")" -ne "$(stat -c %u "${OUTPUT_FOLDER}")" ] +then + echo "ERROR: '${INPUT_FOLDER}' and '${OUTPUT_FOLDER}' have different user id's. not allowed" && exit 1 +fi +# check input/outputfolders are owned by the same group +if [ "$(stat -c %g "${INPUT_FOLDER}")" -ne "$(stat -c %g "${OUTPUT_FOLDER}")" ] +then + echo "ERROR: '${INPUT_FOLDER}' and '${OUTPUT_FOLDER}' have different group id's. not allowed" && exit 1 +fi + +echo "listing inputs folder" +ls -lah "${INPUT_FOLDER}" +echo "listing outputs folder" +ls -lah "${OUTPUT_FOLDER}" + +echo "setting correct user id/group id..." +HOST_USERID=$(stat -c %u "${INPUT_FOLDER}") +HOST_GROUPID=$(stat -c %g "${INPUT_FOLDER}") +CONT_GROUPNAME=$(getent group "${HOST_GROUPID}" | cut -d: -f1) +if [ "$HOST_USERID" -eq 0 ] +then + echo "Warning: Folder mounted owned by root user... adding $SC_USER_NAME to root..." + addgroup "$SC_USER_NAME" root +else + echo "Folder mounted owned by user $HOST_USERID:$HOST_GROUPID-'$CONT_GROUPNAME'..." + # take host's credentials in $SC_USER_NAME + if [ -z "$CONT_GROUPNAME" ] + then + echo "Creating new group my$SC_USER_NAME" + CONT_GROUPNAME=my$SC_USER_NAME + addgroup -g "$HOST_GROUPID" "$CONT_GROUPNAME" + else + echo "group already exists" + fi + + echo "changing $SC_USER_NAME $SC_USER_ID:$SC_USER_ID to $HOST_USERID:$HOST_GROUPID" + # in alpine there is no such thing as usermod... so we delete the user and re-create it as part of $CONT_GROUPNAME + deluser "$SC_USER_NAME" > /dev/null 2>&1 + adduser -u "$HOST_USERID" -G "$CONT_GROUPNAME" -D -s /bin/sh "$SC_USER_NAME" + + echo "Changing group properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" + find / -path /var/log/nginx -prune -o -group "$SC_USER_ID" -print + find / -path /var/log/nginx -prune -o -group "$SC_USER_ID" -exec chgrp -h "$CONT_GROUPNAME" {} \; + # change user property of files already around + echo "Changing ownership properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" + find / -path /var/log/nginx -prune -o -user "$SC_USER_ID" -exec chown -h "$SC_USER_NAME" {} \; + find / -path /var/log/nginx -prune -o -user "$SC_USER_ID" -print +fi + +echo "Starting $* ..." +echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" +echo " local dir : $(ls -al)" +echo " input dir : $(ls -al "${INPUT_FOLDER}")" +echo " output dir : $(ls -al "${OUTPUT_FOLDER}")" + + +# from original etrypoint +set -e + +# Check if incomming command contains flags. +if [ "${1#-}" != "$1" ]; then + set -- static-web-server "$@" +fi + +su-exec "$SC_USER_NAME" "$@" diff --git a/services/dy-static-file-server/docker/custom/index.html b/services/dy-static-file-server/docker/custom/index.html new file mode 100644 index 00000000..e69de29b diff --git a/services/dy-static-file-server/docker/custom/nginx.conf b/services/dy-static-file-server/docker/custom/nginx.conf deleted file mode 100644 index 16ac91fa..00000000 --- a/services/dy-static-file-server/docker/custom/nginx.conf +++ /dev/null @@ -1,24 +0,0 @@ -pid /tmp/nginx.pid; - -events { - worker_connections 1024; -} - -http { - client_body_temp_path /tmp/client_temp; - proxy_temp_path /tmp/proxy_temp_path; - fastcgi_temp_path /tmp/fastcgi_temp; - uwsgi_temp_path /tmp/uwsgi_temp; - scgi_temp_path /tmp/scgi_temp; - - server { - # download - autoindex on; # enable directory listing output - autoindex_exact_size off; # output file sizes rounded to kilobytes, megabytes, and gigabytes - autoindex_localtime on; # output local times in the directory - - location / { - root /www/; - } - } -} \ No newline at end of file From 0a39e03d2c09064958a2bb73139a39bdf05938ae Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 24 Sep 2021 17:01:37 +0200 Subject: [PATCH 06/18] added index generation base --- .../index_html_generator.py | 18 ++++++++ .../inputs_to_outputs.py | 12 +++++- .../tests/unit/conftest.py | 17 +++++++- ...static_file_server_index_html_generator.py | 42 +++++++++++++++++++ ...dy_static_file_server_inputs_to_outputs.py | 1 + .../tests/unit/test_folder_structure.py | 1 - 6 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py create mode 100644 services/dy-static-file-server/tests/unit/test_dy_static_file_server_index_html_generator.py diff --git a/services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py b/services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py new file mode 100644 index 00000000..d3d3f78a --- /dev/null +++ b/services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py @@ -0,0 +1,18 @@ +from pathlib import Path +import os + + +def _get_server_root() -> Path: + return Path(os.environ["SERVER_ROOT"]) + + +def get_index_path() -> Path: + return Path(f"{_get_server_root()}/index.html") + + +def _get_index_content() -> str: + return "Hello" + + +def generate_index() -> None: + get_index_path().write_text(_get_index_content()) \ No newline at end of file diff --git a/services/dy-static-file-server/src/dy_static_file_server/inputs_to_outputs.py b/services/dy-static-file-server/src/dy_static_file_server/inputs_to_outputs.py index 82cc6978..64ca8dd7 100644 --- a/services/dy-static-file-server/src/dy_static_file_server/inputs_to_outputs.py +++ b/services/dy-static-file-server/src/dy_static_file_server/inputs_to_outputs.py @@ -1,4 +1,3 @@ -# TODO: have a watcher doing all the watching import logging import json import os @@ -10,6 +9,12 @@ from watchdog.events import DirModifiedEvent, FileSystemEventHandler from watchdog.observers import Observer +# at runtime dy_static_file_server is not a module +try: + from index_html_generator import generate_index +except ModuleNotFoundError: + from .index_html_generator import generate_index + logger = logging.getLogger(__name__) @@ -22,6 +27,8 @@ def __init__(self, input_dir: Path, output_dir: Path): def on_any_event(self, event: DirModifiedEvent): super().on_any_event(event) remap_input_to_output(self.input_dir, self.output_dir) + # alway regenerate index + generate_index() def _list_files_in_dir(path: Path) -> List[Path]: @@ -125,6 +132,9 @@ def main() -> None: if input_dir == output_dir: raise ValueError(f"Inputs and outputs directories match {input_dir}") + # make sure index exists before the monitor starts + generate_index() + inputs_objserver = InputsObserver(input_dir, output_dir) inputs_objserver.start() inputs_objserver.join() diff --git a/services/dy-static-file-server/tests/unit/conftest.py b/services/dy-static-file-server/tests/unit/conftest.py index 5fd3716d..f1664ebc 100644 --- a/services/dy-static-file-server/tests/unit/conftest.py +++ b/services/dy-static-file-server/tests/unit/conftest.py @@ -1,16 +1,31 @@ # pytest: disable = redefined-outer-name +# pylint: disable=redefined-outer-name import pytest from pathlib import Path from types import ModuleType import sys +from _pytest.monkeypatch import MonkeyPatch + @pytest.fixture def tmp_dir(tmp_path) -> Path: return Path(tmp_path) +@pytest.fixture +def server_root(tmp_dir) -> Path: + path = Path(tmp_dir) / "server_root" + path.mkdir(parents=True, exist_ok=True) + return path + + +@pytest.fixture +def env_server_root(monkeypatch: MonkeyPatch, server_root: Path) -> None: + monkeypatch.setenv("SERVER_ROOT", f"{server_root}") + + @pytest.fixture def dy_static_file_server(src_dir: Path) -> ModuleType: # allow to search for the module @@ -19,4 +34,4 @@ def dy_static_file_server(src_dir: Path) -> ModuleType: import dy_static_file_server - return dy_static_file_server \ No newline at end of file + return dy_static_file_server diff --git a/services/dy-static-file-server/tests/unit/test_dy_static_file_server_index_html_generator.py b/services/dy-static-file-server/tests/unit/test_dy_static_file_server_index_html_generator.py new file mode 100644 index 00000000..3d29bae6 --- /dev/null +++ b/services/dy-static-file-server/tests/unit/test_dy_static_file_server_index_html_generator.py @@ -0,0 +1,42 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +from types import ModuleType +from unittest.mock import patch + + +import pytest + +# UTILS + + +# FIXTURES + + +@pytest.fixture +def index_html_content() -> str: + return "some random content here" + + +# TESTS + + +def test_can_import_module( + dy_static_file_server: ModuleType, index_html_content: str +) -> None: + from dy_static_file_server import index_html_generator + + assert type(index_html_generator) == ModuleType + + +def test_generate_index_html( + dy_static_file_server: ModuleType, env_server_root: None, index_html_content: str +) -> None: + from dy_static_file_server import index_html_generator + + with patch.object( + index_html_generator, "_get_index_content", return_value=index_html_content + ): + index_html_generator.generate_index() + + index_html = index_html_generator.get_index_path() + assert index_html.read_text() == index_html_content diff --git a/services/dy-static-file-server/tests/unit/test_dy_static_file_server_inputs_to_outputs.py b/services/dy-static-file-server/tests/unit/test_dy_static_file_server_inputs_to_outputs.py index efe8e478..7649245b 100644 --- a/services/dy-static-file-server/tests/unit/test_dy_static_file_server_inputs_to_outputs.py +++ b/services/dy-static-file-server/tests/unit/test_dy_static_file_server_inputs_to_outputs.py @@ -163,6 +163,7 @@ def test_folder_mirror( output_dir: Path, create_files_in_input: Callable, key_values_json_outputs_content: str, + env_server_root: None, ) -> None: from dy_static_file_server.inputs_to_outputs import InputsObserver diff --git a/services/dy-static-file-server/tests/unit/test_folder_structure.py b/services/dy-static-file-server/tests/unit/test_folder_structure.py index 56604508..421c883c 100644 --- a/services/dy-static-file-server/tests/unit/test_folder_structure.py +++ b/services/dy-static-file-server/tests/unit/test_folder_structure.py @@ -15,7 +15,6 @@ "metadata/metadata-dynamic-sidecar-compose-spec.yml", "metadata/metadata-dynamic-sidecar.yml", "metadata/metadata.yml", - "docker/custom:nginx.conf", "docker/custom:boot.sh", "docker/custom:Dockerfile", "tools:update_compose_labels.py", From b5d64174346602056c1ce923384bc0dac49b8769 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 24 Sep 2021 17:01:50 +0200 Subject: [PATCH 07/18] service is not exposed on port 8080 --- services/dy-static-file-server/docker-compose-meta.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/dy-static-file-server/docker-compose-meta.yml b/services/dy-static-file-server/docker-compose-meta.yml index 8f977dcb..65a1ada7 100644 --- a/services/dy-static-file-server/docker-compose-meta.yml +++ b/services/dy-static-file-server/docker-compose-meta.yml @@ -20,7 +20,7 @@ services: org.label-schema.vcs-url: ${VCS_URL} simcore.service.settings: '[{"name": "resources", "type": "Resources", "value": {"mem_limit":17179869184, "cpu_limit": 1000000000}}, {"name": "ports", "type": - "int", "value": 8000}, {"name": "constraints", "type": "string", "value": + "int", "value": 8080}, {"name": "constraints", "type": "string", "value": ["node.platform.os == linux"]}]' dy-static-file-server-dynamic-sidecar: build: @@ -64,7 +64,7 @@ services: "/www/inputs", "state_paths": ["/workdir/generated-data"]}' simcore.service.settings: '[{"name": "resources", "type": "Resources", "value": {"mem_limit":17179869184, "cpu_limit": 1000000000}}, {"name": "ports", "type": - "int", "value": 8000}, {"name": "constraints", "type": "string", "value": + "int", "value": 8080}, {"name": "constraints", "type": "string", "value": ["node.platform.os == linux"]}]' dy-static-file-server-dynamic-sidecar-compose-spec: build: @@ -111,6 +111,6 @@ services: "/www/inputs", "state_paths": ["/workdir/generated-data"]}' simcore.service.settings: '[{"name": "resources", "type": "Resources", "value": {"mem_limit":17179869184, "cpu_limit": 1000000000}}, {"name": "ports", "type": - "int", "value": 8000}, {"name": "constraints", "type": "string", "value": + "int", "value": 8080}, {"name": "constraints", "type": "string", "value": ["node.platform.os == linux"]}]' version: '3.7' From 2e09a4bfe9a894b4742203adf42e57800482568d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 24 Sep 2021 17:06:17 +0200 Subject: [PATCH 08/18] file is generated on the fly --- services/dy-static-file-server/docker/custom/index.html | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 services/dy-static-file-server/docker/custom/index.html diff --git a/services/dy-static-file-server/docker/custom/index.html b/services/dy-static-file-server/docker/custom/index.html deleted file mode 100644 index e69de29b..00000000 From eb2b46b8705bed00971afe7b5b3f7122ce0685a4 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 24 Sep 2021 18:01:25 +0200 Subject: [PATCH 09/18] added more usable index --- .../index_html_generator.py | 49 +++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py b/services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py index d3d3f78a..75a20559 100644 --- a/services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py +++ b/services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py @@ -1,5 +1,14 @@ -from pathlib import Path import os +from typing import List +from textwrap import dedent +from datetime import datetime +from pathlib import Path + + +def _get_dir_files(dir_path: Path) -> List[str]: + return [ + str(x).replace(str(dir_path), "") for x in dir_path.rglob("*") if x.is_file() + ] def _get_server_root() -> Path: @@ -11,8 +20,42 @@ def get_index_path() -> Path: def _get_index_content() -> str: - return "Hello" + """ + Generates index.html content. + - lists all the files inside SERVER_ROOT + - reloads every second to be updated + """ + + files = _get_dir_files(_get_server_root()) + + rendered_file_list = "\n".join([f'{x}
' for x in files]) + + utc_time_stamp = datetime.utcnow().strftime("%m/%d/%Y, %H:%M:%S") + + refres_interval: int = 5 + + rendered_page = f""" + + + + + +

Listing files


+ + {rendered_file_list} + +
+
* Last recreated {utc_time_stamp} +
* Content will be created if there is a change in the ports or the status directory. +
* Page is refreshed every {refres_interval} seconds. + + + """ + return dedent(rendered_page) def generate_index() -> None: - get_index_path().write_text(_get_index_content()) \ No newline at end of file + index_html_path = get_index_path() + + index_html_path.write_text(_get_index_content()) + print(f"Regenerated {index_html_path}") \ No newline at end of file From b14d6072a9b129af9c2e95196bd639a4f244039e Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 24 Sep 2021 18:07:10 +0200 Subject: [PATCH 10/18] fixed descirption --- .../src/dy_static_file_server/index_html_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py b/services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py index 75a20559..f9aacbc1 100644 --- a/services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py +++ b/services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py @@ -46,7 +46,7 @@ def _get_index_content() -> str:

* Last recreated {utc_time_stamp} -
* Content will be created if there is a change in the ports or the status directory. +
* Content is recrated if there is a change in the inputs directory.
* Page is refreshed every {refres_interval} seconds. From 16f4876f3f0e68f2d95e105c48a08823ae7efbed Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 27 Sep 2021 10:43:34 +0200 Subject: [PATCH 11/18] updated redme --- .../src/dy_static_file_server/inputs_to_outputs.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/services/dy-static-file-server/src/dy_static_file_server/inputs_to_outputs.py b/services/dy-static-file-server/src/dy_static_file_server/inputs_to_outputs.py index 64ca8dd7..549f55c3 100644 --- a/services/dy-static-file-server/src/dy_static_file_server/inputs_to_outputs.py +++ b/services/dy-static-file-server/src/dy_static_file_server/inputs_to_outputs.py @@ -9,7 +9,8 @@ from watchdog.events import DirModifiedEvent, FileSystemEventHandler from watchdog.observers import Observer -# at runtime dy_static_file_server is not a module +# when not testing `dy_static_file_server` directory is not detected +# as a module; relative imports will not work try: from index_html_generator import generate_index except ModuleNotFoundError: @@ -120,6 +121,10 @@ def join(self) -> None: raise RuntimeError(f"{self.__class__.__name__} was not started") +def is_boot_mode_legacy() -> bool: + return False + + def main() -> None: logging.basicConfig( level=logging.INFO, From 9fd40a9522c4da78665ec71f078025fc37dfa29a Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 27 Sep 2021 10:43:51 +0200 Subject: [PATCH 12/18] enhanced html generation --- .../index_html_generator.py | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py b/services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py index f9aacbc1..6d73d7bb 100644 --- a/services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py +++ b/services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py @@ -4,6 +4,10 @@ from datetime import datetime from pathlib import Path +DATETIME_FORMAT = "%d/%m/%Y %H:%M:%S" + +REFRESH_INTERVAL: int = 5 + def _get_dir_files(dir_path: Path) -> List[str]: return [ @@ -19,6 +23,12 @@ def get_index_path() -> Path: return Path(f"{_get_server_root()}/index.html") +def get_last_change_timestamp(str_path: str) -> str: + return datetime.fromtimestamp(Path(str_path).stat().st_mtime).strftime( + DATETIME_FORMAT + ) + + def _get_index_content() -> str: """ Generates index.html content. @@ -28,26 +38,31 @@ def _get_index_content() -> str: files = _get_dir_files(_get_server_root()) - rendered_file_list = "\n".join([f'{x}
' for x in files]) + rendered_file_list = "\n".join( + [ + f'
  • {get_last_change_timestamp(x)} {x}
  • ' + for x in files + ] + ) - utc_time_stamp = datetime.utcnow().strftime("%m/%d/%Y, %H:%M:%S") - - refres_interval: int = 5 + utc_time_stamp = datetime.utcnow().strftime(DATETIME_FORMAT) rendered_page = f""" - +

    Listing files


    - - {rendered_file_list} + +
      + {rendered_file_list} +


    * Last recreated {utc_time_stamp}
    * Content is recrated if there is a change in the inputs directory. -
    * Page is refreshed every {refres_interval} seconds. +
    * Page is refreshed every {REFRESH_INTERVAL} seconds. """ From 6fb76a89c29306021fa712065f7706393097abe9 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 27 Sep 2021 11:12:02 +0200 Subject: [PATCH 13/18] fixed tests --- .../src/dy_static_file_server/index_html_generator.py | 7 ++++--- .../unit/test_dy_static_file_server_inputs_to_outputs.py | 8 +++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py b/services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py index 6d73d7bb..42f71272 100644 --- a/services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py +++ b/services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py @@ -3,6 +3,7 @@ from textwrap import dedent from datetime import datetime from pathlib import Path +from functools import lru_cache DATETIME_FORMAT = "%d/%m/%Y %H:%M:%S" @@ -15,6 +16,7 @@ def _get_dir_files(dir_path: Path) -> List[str]: ] +@lru_cache() def _get_server_root() -> Path: return Path(os.environ["SERVER_ROOT"]) @@ -24,9 +26,8 @@ def get_index_path() -> Path: def get_last_change_timestamp(str_path: str) -> str: - return datetime.fromtimestamp(Path(str_path).stat().st_mtime).strftime( - DATETIME_FORMAT - ) + file_path = _get_server_root() / str_path.strip("/") + return datetime.fromtimestamp(file_path.stat().st_mtime).strftime(DATETIME_FORMAT) def _get_index_content() -> str: diff --git a/services/dy-static-file-server/tests/unit/test_dy_static_file_server_inputs_to_outputs.py b/services/dy-static-file-server/tests/unit/test_dy_static_file_server_inputs_to_outputs.py index 7649245b..c26e6263 100644 --- a/services/dy-static-file-server/tests/unit/test_dy_static_file_server_inputs_to_outputs.py +++ b/services/dy-static-file-server/tests/unit/test_dy_static_file_server_inputs_to_outputs.py @@ -130,6 +130,12 @@ def _callable(input_dir: Path) -> None: return _callable +@pytest.fixture +def ensure_index_html(env_server_root: Path) -> None: + index_html = env_server_root / "index.html" + index_html.write_text("index.html") + + # TESTS @@ -163,7 +169,7 @@ def test_folder_mirror( output_dir: Path, create_files_in_input: Callable, key_values_json_outputs_content: str, - env_server_root: None, + ensure_index_html: None, ) -> None: from dy_static_file_server.inputs_to_outputs import InputsObserver From 23093092bf4842a51f288f54749809a38ae02ef8 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 27 Sep 2021 11:12:13 +0200 Subject: [PATCH 14/18] changed fixture --- services/dy-static-file-server/tests/unit/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/dy-static-file-server/tests/unit/conftest.py b/services/dy-static-file-server/tests/unit/conftest.py index f1664ebc..c9ea9907 100644 --- a/services/dy-static-file-server/tests/unit/conftest.py +++ b/services/dy-static-file-server/tests/unit/conftest.py @@ -22,8 +22,9 @@ def server_root(tmp_dir) -> Path: @pytest.fixture -def env_server_root(monkeypatch: MonkeyPatch, server_root: Path) -> None: +def env_server_root(monkeypatch: MonkeyPatch, server_root: Path) -> Path: monkeypatch.setenv("SERVER_ROOT", f"{server_root}") + return server_root @pytest.fixture From 245d123e872d28c1289687bbba795c361c3783cf Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 27 Sep 2021 11:12:26 +0200 Subject: [PATCH 15/18] updated readme --- .../dy_static_file_server/inputs_to_outputs.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/services/dy-static-file-server/src/dy_static_file_server/inputs_to_outputs.py b/services/dy-static-file-server/src/dy_static_file_server/inputs_to_outputs.py index 549f55c3..a044c578 100644 --- a/services/dy-static-file-server/src/dy_static_file_server/inputs_to_outputs.py +++ b/services/dy-static-file-server/src/dy_static_file_server/inputs_to_outputs.py @@ -9,7 +9,7 @@ from watchdog.events import DirModifiedEvent, FileSystemEventHandler from watchdog.observers import Observer -# when not testing `dy_static_file_server` directory is not detected +# when not testing `dy_static_file_server` directory is not detected # as a module; relative imports will not work try: from index_html_generator import generate_index @@ -121,10 +121,6 @@ def join(self) -> None: raise RuntimeError(f"{self.__class__.__name__} was not started") -def is_boot_mode_legacy() -> bool: - return False - - def main() -> None: logging.basicConfig( level=logging.INFO, @@ -132,8 +128,14 @@ def main() -> None: datefmt="%Y-%m-%d %H:%M:%S", ) - input_dir = get_path_from_env("DY_SIDECAR_PATH_INPUTS") - output_dir = get_path_from_env("DY_SIDECAR_PATH_OUTPUTS") + is_legacy = os.environ.get("SIMCORE_NODE_BASEPATH", None) is not None + + input_dir = get_path_from_env( + "INPUT_FOLDER" if is_legacy else "DY_SIDECAR_PATH_INPUTS" + ) + output_dir = get_path_from_env( + "OUTPUT_FOLDER" if is_legacy else "DY_SIDECAR_PATH_OUTPUTS" + ) if input_dir == output_dir: raise ValueError(f"Inputs and outputs directories match {input_dir}") From 92cae0e887e19af651cc84760b8789e6c5bf21fd Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 27 Sep 2021 11:46:55 +0200 Subject: [PATCH 16/18] slight refactor adapted for LEGACY boot mode --- .../docker/custom/Dockerfile | 10 +++++++--- .../dy-static-file-server/docker/custom/boot.sh | 8 ++++---- .../docker/custom/entrypoint.sh | 17 +++++++++++++++++ 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/services/dy-static-file-server/docker/custom/Dockerfile b/services/dy-static-file-server/docker/custom/Dockerfile index 77365b9b..d1c2d2aa 100644 --- a/services/dy-static-file-server/docker/custom/Dockerfile +++ b/services/dy-static-file-server/docker/custom/Dockerfile @@ -34,8 +34,12 @@ RUN pip3 install --upgrade \ pip \ virtualenv -RUN mkdir -p /venv -RUN python3 -m venv /venv + +# create and activate virtual environment +RUN mkdir -p /venv && \ + python3 -m venv /opt/venv +ENV PATH="/venv/bin:$PATH" + # add additional directories RUN mkdir -p ${WORKDIR} && chown ${SC_USER_NAME}:${SC_USER_NAME} ${WORKDIR} && \ @@ -45,7 +49,7 @@ COPY --chown=${SC_USER_NAME}:${SC_USER_NAME} docker/custom/*.sh /docker # add python app requirements COPY requirements/base.txt /tmp/requirements.txt -RUN /venv/bin/pip3 install -r /tmp/requirements.txt +RUN pip3 install -r /tmp/requirements.txt COPY --chown=${SC_USER_NAME}:${SC_USER_NAME} static-content/hello-world.txt /www/hello-world.txt COPY --chown=${SC_USER_NAME}:${SC_USER_NAME} src/dy_static_file_server ${WORKDIR}/dy_static_file_server diff --git a/services/dy-static-file-server/docker/custom/boot.sh b/services/dy-static-file-server/docker/custom/boot.sh index 4252e32c..041f02fe 100755 --- a/services/dy-static-file-server/docker/custom/boot.sh +++ b/services/dy-static-file-server/docker/custom/boot.sh @@ -11,14 +11,14 @@ echo Env : "$(env)" echo "/workdir content" ls -lah -echo "ensure some random data is created in in /workdir/generated-data content" -/venv/bin/python ensure_random_workdir_data.py +echo "ensure some random data is created in /workdir/generated-data content" +python3 ensure_random_workdir_data.py echo "/workdir/generated-data content" ls -lah /workdir/generated-data/ -echo "starting background inputs->ouputs transformation" -/venv/bin/python inputs_to_outputs.py & +echo "starting background inputs->ouputs mapping when inputs change" +python3 inputs_to_outputs.py & echo "booting static-web-server" exec static-web-server \ No newline at end of file diff --git a/services/dy-static-file-server/docker/custom/entrypoint.sh b/services/dy-static-file-server/docker/custom/entrypoint.sh index 8596920d..3e7092e0 100755 --- a/services/dy-static-file-server/docker/custom/entrypoint.sh +++ b/services/dy-static-file-server/docker/custom/entrypoint.sh @@ -14,7 +14,24 @@ echo User : "$(id "$(whoami)")" echo Workdir : "$(pwd)" +# adapt to be compatible for legacy boot mode +if [ -n "$SIMCORE_NODE_BASEPATH" ]; then + echo "Boot mode: LEGACY" + echo "Creating ${INPUT_FOLDER} and ${OUTPUT_FOLDER}" + mkdir -p "${INPUT_FOLDER}" + mkdir -p "${OUTPUT_FOLDER}" + echo "SERVER_ROOT: ${SERVER_ROOT}" + echo "SIMCORE_NODE_BASEPATH: ${SIMCORE_NODE_BASEPATH}" + echo "Creating ${SERVER_ROOT}${SIMCORE_NODE_BASEPATH} and changin ownership to $SC_USER_NAME" + mkdir -p "${SERVER_ROOT}${SIMCORE_NODE_BASEPATH}" + chown -R "$SC_USER_NAME" "${SERVER_ROOT}${SIMCORE_NODE_BASEPATH}" +else + echo "Boot mode: DYNAMIC-SIDECAR" +fi + + # expect input/output folders to be mounted +#TODO: determine if legacy boot more and based on that do stuff like creating stat "${INPUT_FOLDER}" > /dev/null 2>&1 || \ (echo "ERROR: You must mount '${INPUT_FOLDER}' to deduce user and group ids" && exit 1) stat "${OUTPUT_FOLDER}" > /dev/null 2>&1 || \ From c78a71e04d578faf0abd630eb249428fc730d95f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 27 Sep 2021 11:47:18 +0200 Subject: [PATCH 17/18] legacy boot mode compatibility --- .../src/dy_static_file_server/index_html_generator.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py b/services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py index 42f71272..59a0fc17 100644 --- a/services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py +++ b/services/dy-static-file-server/src/dy_static_file_server/index_html_generator.py @@ -18,7 +18,12 @@ def _get_dir_files(dir_path: Path) -> List[str]: @lru_cache() def _get_server_root() -> Path: - return Path(os.environ["SERVER_ROOT"]) + if os.environ.get("SIMCORE_NODE_BASEPATH", None) is None: + return Path(os.environ["SERVER_ROOT"]) + + # when in legacy boot mode + node_base_path = os.environ["SIMCORE_NODE_BASEPATH"].strip("/") + return Path(os.environ["SERVER_ROOT"]) / node_base_path def get_index_path() -> Path: From 5f7cb8fcb4144edfc63ed5ea8c5809942bf8cef6 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 27 Sep 2021 11:48:47 +0200 Subject: [PATCH 18/18] bumping version --- services/dy-static-file-server/.cookiecutterrc | 2 +- services/dy-static-file-server/VERSION | 2 +- services/dy-static-file-server/docker-compose-meta.yml | 6 +++--- .../metadata/metadata-dynamic-sidecar-compose-spec.yml | 2 +- .../metadata/metadata-dynamic-sidecar.yml | 2 +- services/dy-static-file-server/metadata/metadata.yml | 2 +- services/dy-static-file-server/versioning/service.cfg | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/services/dy-static-file-server/.cookiecutterrc b/services/dy-static-file-server/.cookiecutterrc index 7e89d36b..c637bf02 100644 --- a/services/dy-static-file-server/.cookiecutterrc +++ b/services/dy-static-file-server/.cookiecutterrc @@ -35,4 +35,4 @@ default_context: project_slug: 'dy-static-file-server' project_type: 'computational' release_date: '2021' - version: '1.0.8' + version: '2.0.0' diff --git a/services/dy-static-file-server/VERSION b/services/dy-static-file-server/VERSION index b0f3d96f..227cea21 100644 --- a/services/dy-static-file-server/VERSION +++ b/services/dy-static-file-server/VERSION @@ -1 +1 @@ -1.0.8 +2.0.0 diff --git a/services/dy-static-file-server/docker-compose-meta.yml b/services/dy-static-file-server/docker-compose-meta.yml index 65a1ada7..2eeaa2fd 100644 --- a/services/dy-static-file-server/docker-compose-meta.yml +++ b/services/dy-static-file-server/docker-compose-meta.yml @@ -13,7 +13,7 @@ services: io.simcore.name: '{"name": "dy-static-file-server"}' io.simcore.outputs: '{"outputs": {}}' io.simcore.type: '{"type": "dynamic"}' - io.simcore.version: '{"version": "1.0.8"}' + io.simcore.version: '{"version": "2.0.0"}' org.label-schema.build-date: ${BUILD_DATE} org.label-schema.schema-version: '1.0' org.label-schema.vcs-ref: ${VCS_REF} @@ -55,7 +55,7 @@ services: {"displayOrder": 5, "label": "File output", "description": "File from input", "type": "data:*/*", "fileToKeyMap": {"test_file": "file_output"}}}}' io.simcore.type: '{"type": "dynamic"}' - io.simcore.version: '{"version": "1.0.8"}' + io.simcore.version: '{"version": "2.0.0"}' org.label-schema.build-date: ${BUILD_DATE} org.label-schema.schema-version: '1.0' org.label-schema.vcs-ref: ${VCS_REF} @@ -100,7 +100,7 @@ services: {"displayOrder": 5, "label": "File output", "description": "File from input", "type": "data:*/*", "fileToKeyMap": {"test_file": "file_output"}}}}' io.simcore.type: '{"type": "dynamic"}' - io.simcore.version: '{"version": "1.0.8"}' + io.simcore.version: '{"version": "2.0.0"}' org.label-schema.build-date: ${BUILD_DATE} org.label-schema.schema-version: '1.0' org.label-schema.vcs-ref: ${VCS_REF} diff --git a/services/dy-static-file-server/metadata/metadata-dynamic-sidecar-compose-spec.yml b/services/dy-static-file-server/metadata/metadata-dynamic-sidecar-compose-spec.yml index 4889a253..6b030a86 100644 --- a/services/dy-static-file-server/metadata/metadata-dynamic-sidecar-compose-spec.yml +++ b/services/dy-static-file-server/metadata/metadata-dynamic-sidecar-compose-spec.yml @@ -2,7 +2,7 @@ name: dy-static-file-server-dynamic-sidecar-compose-spec key: simcore/services/dynamic/dy-static-file-server-dynamic-sidecar-compose-spec type: dynamic integration-version: 1.0.0 -version: 1.0.8 +version: 2.0.0 description: Modern test dynamic service providing a docker-compose specification file (with dynamic sidecar and compose-spec). Changes to the inputs will be forwarded to the outputs. The /workdir/generated-data directory is populated if no content is present. contact: anderegg@itis.swiss authors: diff --git a/services/dy-static-file-server/metadata/metadata-dynamic-sidecar.yml b/services/dy-static-file-server/metadata/metadata-dynamic-sidecar.yml index deec22c8..f86df4d5 100644 --- a/services/dy-static-file-server/metadata/metadata-dynamic-sidecar.yml +++ b/services/dy-static-file-server/metadata/metadata-dynamic-sidecar.yml @@ -2,7 +2,7 @@ name: dy-static-file-server-dynamic-sidecar key: simcore/services/dynamic/dy-static-file-server-dynamic-sidecar type: dynamic integration-version: 1.0.0 -version: 1.0.8 +version: 2.0.0 description: Modern test dynamic service (with dynamic sidecar). Changes to the inputs will be forwarded to the outputs. The /workdir/generated-data directory is populated if no content is present. contact: anderegg@itis.swiss authors: diff --git a/services/dy-static-file-server/metadata/metadata.yml b/services/dy-static-file-server/metadata/metadata.yml index a119d05b..5019f7b4 100644 --- a/services/dy-static-file-server/metadata/metadata.yml +++ b/services/dy-static-file-server/metadata/metadata.yml @@ -2,7 +2,7 @@ name: dy-static-file-server key: simcore/services/dynamic/dy-static-file-server type: dynamic integration-version: 1.0.0 -version: 1.0.8 +version: 2.0.0 description: Legacy test dynamic service (starts using original director-v0). The /workdir/generated-data directory is populated if no content is present. contact: anderegg@itis.swiss authors: diff --git a/services/dy-static-file-server/versioning/service.cfg b/services/dy-static-file-server/versioning/service.cfg index d841faf0..95accbc8 100644 --- a/services/dy-static-file-server/versioning/service.cfg +++ b/services/dy-static-file-server/versioning/service.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.8 +current_version = 2.0.0 commit = False message = service/kernel version: {current_version} → {new_version} tag = False