Is1585/cleanup storage

.gitignore

@@ -151,3 +151,7 @@ prof/
 
 # outputs from make
 .stack-*.yml
+
+
+# Copies
+services/**/.codeclimate.yml

scripts/common.Makefile

@@ -94,7 +94,12 @@ info: ## displays basic info
 .PHONY: autoformat
 autoformat: ## runs black python formatter on this service's code. Use AFTER make install-*
 	# sort imports
-	@python3 -m isort --atomic -rc $(CURDIR)
+	@python3 -m isort --verbose \
+		--atomic \
+		--recursive \
+		--skip-glob */client-sdk/* \
+		--skip-glob */migration/* \
+		$(CURDIR)
 	# auto formatting with black
 	@python3 -m black --verbose \
 		--exclude "/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|\.svn|_build|buck-out|build|dist|migration|client-sdk|generated_code)/" \
@@ -104,6 +109,18 @@ autoformat: ## runs black python formatter on this service's code. Use AFTER mak
 mypy: $(REPO_BASE_DIR)/scripts/mypy.bash $(REPO_BASE_DIR)/mypy.ini ## runs mypy python static type checker on this services's code. Use AFTER make install-*
 	@$(REPO_BASE_DIR)/scripts/mypy.bash src
 
+.PHONY: code-analysis
+code-analysis: $(REPO_BASE_DIR)/.codeclimate.yml ## runs code-climate analysis
+	# Copying config
+	cp $(REPO_BASE_DIR)/.codeclimate.yml $(CURDIR)/.codeclimate.yml
+	# Validates $< at ${PWD}
+	$(REPO_BASE_DIR)/scripts/code-climate.bash validate-config
+	# Running analysis
+	$(REPO_BASE_DIR)/scripts/code-climate.bash analyze
+	# Removing tmp config
+	@-rm $(CURDIR)/.codeclimate.yml
+
+
 .PHONY: version-patch version-minor version-major
 version-patch: ## commits version with bug fixes not affecting the cookiecuter config
 	$(_bumpversion)

services/storage/Dockerfile

@@ -1,5 +1,5 @@
-#FROM python:3.6-alpine as base
-FROM itisfoundation/python-with-pandas:3.6-alpine as base
+ARG PYTHON_VERSION="3.6.10"
+FROM python:${PYTHON_VERSION}-slim-buster as base
 #
 #  USAGE:
 #     cd sercices/storage
@@ -8,26 +8,50 @@ FROM itisfoundation/python-with-pandas:3.6-alpine as base
 #
 #  REQUIRED: context expected at ``osparc-simcore/`` folder because we need access to osparc-simcore/packages
 
-
 LABEL maintainer=mguidon
 
+RUN set -eux; \
+  apt-get update; \
+  apt-get install -y gosu; \
+  rm -rf /var/lib/apt/lists/*; \
+  # verify that the binary works
+  gosu nobody true
+
+
 # simcore-user uid=8004(scu) gid=8004(scu) groups=8004(scu)
-RUN adduser -D -u 8004 -s /bin/sh -h /home/scu scu
+ENV SC_USER_ID=8004 \
+  SC_USER_NAME=scu \
+  SC_BUILD_TARGET=base \
+  SC_BOOT_MODE=default
+
+RUN adduser \
+  --uid ${SC_USER_ID} \
+  --disabled-password \
+  --gecos "" \
+  --shell /bin/sh \
+  --home /home/${SC_USER_NAME} \
+  ${SC_USER_NAME}
 
-RUN apk add --no-cache \
-      su-exec
 
+# Sets utf-8 encoding for Python et al
+ENV LANG=C.UTF-8
+
+# Turns off writing .pyc files; superfluous on an ephemeral container.
+ENV PYTHONDONTWRITEBYTECODE=1 \
+  VIRTUAL_ENV=/home/scu/.venv
+
+# Ensures that the python and pip executables used
+# in the image will be those from our virtualenv.
+ENV PATH="${VIRTUAL_ENV}/bin:$PATH"
 
-ENV PATH "/home/scu/.local/bin:$PATH"
-ENV HOME /home/scu
 
-# All SC_ variables are customized
-ENV SC_PIP pip3 --no-cache-dir
-ENV SC_PIP2 pip2 --no-cache-dir
 ENV SC_BUILD_TARGET base
 ENV SC_BOOT_MODE default
 
 EXPOSE 8080
+# -------------------------- -------------------------------
+
+
 
 # -------------------------- Build stage -------------------
 # Installs build/package management tools and third party dependencies
@@ -37,57 +61,30 @@ EXPOSE 8080
 
 FROM base as build
 
-# Installing client libraries and any other package you need
-#
-#  libpq: client library for PostgreSQL https://www.postgresql.org/docs/9.5/libpq.html
-#  libstdc++: needed in ujson https://github.com/kohlschutter/junixsocket/issues/33
-#
-RUN apk update && \
-    apk add --no-cache \
-      libpq \
-      libstdc++
-
-RUN apk add --no-cache \
-      postgresql-dev \
-      gcc \
-      g++ \
-      libc-dev \
-      libffi-dev \
-      linux-headers
-
-RUN $SC_PIP install --upgrade \
-      pip~=20.1.1  \
-      wheel \
-      setuptools
+ENV SC_BUILD_TARGET build
 
-WORKDIR /build
+RUN apt-get update &&\
+  apt-get install -y --no-install-recommends \
+  build-essential
 
-# install base 3rd party dependencies
-COPY  --chown=scu:scu services/storage/requirements/*.txt \
-      tmp/storage/requirements/
 
-RUN $SC_PIP install \
-      -r tmp/storage/requirements/_base.txt
+# NOTE: python virtualenv is used here such that installed
+# packages may be moved to production image easily by copying the venv
+RUN python -m venv ${VIRTUAL_ENV}
 
-# --------------------------Development stage -------------------
-# Source code accessible in host but runs in container
-# Runs as scu with same gid/uid as host
-#
-#  + /devel         WORKDIR
-#    + packages  (mounted volume)
-#    + services  (mounted volume)
-#
-FROM build as development
+RUN pip --no-cache-dir install --upgrade \
+  pip~=20.1.1  \
+  wheel \
+  setuptools
 
-ENV SC_BUILD_TARGET development
-ENV SC_BOOT_MODE debug-ptvsd
+WORKDIR /build
+
+# install only base 3rd party dependencies
+COPY  --chown=scu:scu services/storage/requirements/_base.txt requirements_base.txt
+RUN pip --no-cache-dir --quiet install -r requirements_base.txt
+# -------------------------- -------------------------------
 
-WORKDIR /devel
-VOLUME  /devel/packages
-VOLUME  /devel/services/storage/
 
-ENTRYPOINT [ "/bin/sh", "services/storage/docker/entrypoint.sh" ]
-CMD ["/bin/sh", "services/storage/docker/boot.sh"]
 
 
 # --------------------------Cache stage -------------------
@@ -100,36 +97,73 @@ FROM build as cache
 
 ENV SC_BUILD_TARGET cache
 
+# 2nd party packages
 COPY --chown=scu:scu packages /build/packages
 COPY --chown=scu:scu services/storage /build/services/storage
 
 WORKDIR /build/services/storage
 
-RUN $SC_PIP install -r requirements/prod.txt &&\
-    $SC_PIP list -v
+RUN pip --no-cache-dir --quiet install -r requirements/prod.txt
+# -------------------------- -------------------------------
+
+
 
 # --------------------------Production stage -------------------
 # Final cleanup up to reduce image size and startup setup
 #
-FROM cache as production
+FROM base as production
+
+ENV SC_BUILD_TARGET=production \
+  SC_BOOT_MODE=production \
+  SC_HEALTHCHECK_INTERVAL=30 \
+  SC_HEALTHCHECK_RETRY=3
 
-ENV SC_BUILD_TARGET production
 ENV PYTHONOPTIMIZE=TRUE
 
 WORKDIR /home/scu
 
-RUN mkdir -p services/storage && chown scu:scu services/storage && \
-    mv /build/services/storage/docker services/storage/docker && \
-    rm -rf /build
+# bring installed package without build tools
+COPY --from=cache --chown=scu:scu ${VIRTUAL_ENV} ${VIRTUAL_ENV}
 
-RUN apk del --no-cache\
-      gcc
+# copy docker entrypoint and boot scripts
+COPY --chown=scu:scu services/storage/docker services/storage/docker
+RUN chmod +x services/storage/docker/*.sh
 
 HEALTHCHECK --interval=30s \
-            --timeout=120s \
-            --start-period=30s \
-            --retries=3 \
-            CMD ["python3", "/home/scu/services/storage/docker/healthcheck.py", "http://localhost:8080/v0/"]
+  --timeout=120s \
+  --start-period=30s \
+  --retries=3 \
+  CMD ["python3", "/home/scu/services/storage/docker/healthcheck.py", "http://localhost:8080/v0/"]
+
+
+ENTRYPOINT [ "/bin/sh", "services/storage/docker/entrypoint.sh" ]
+CMD ["/bin/sh", "services/storage/docker/boot.sh"]
+# -------------------------- -------------------------------
+
+
+
+
+# --------------------------Development stage -------------------
+# Source code accessible in host but runs in container
+# Runs as scu with same gid/uid as host
+#
+#  + /devel         WORKDIR
+#    + packages  (mounted volume)
+#    + services  (mounted volume)
+#
+FROM build as development
+
+ENV SC_BUILD_TARGET=development \
+  SC_DEVEL_MOUNT=/devel/services/storage/
+
+WORKDIR /devel
+
+RUN chown -R scu:scu ${VIRTUAL_ENV}
+
+# NOTE: declaring VOLUMEs here makes troubles mounting
+#       the client's output folder to /devel/services/web/client.
+#       The latter ls no files
 
 ENTRYPOINT [ "/bin/sh", "services/storage/docker/entrypoint.sh" ]
 CMD ["/bin/sh", "services/storage/docker/boot.sh"]
+# -------------------------- -------------------------------

services/storage/Makefile

@@ -3,7 +3,7 @@
 #
 include ../../scripts/common.Makefile
 
-APP_NAME     := $(notdir $(CURDIR))
+APP_NAME := $(notdir $(CURDIR))
 
 .PHONY: openapi-specs
 openapi-specs: ## updates and validates openapi specifications
@@ -22,6 +22,6 @@ tests: ## runs unit tests
 	@pytest -vv --exitfirst --failed-first --durations=10 --pdb $(CURDIR)/tests
 
 
-.PHONY: build
-build: openapi-specs ## builds docker image (using main services/docker-compose-build.yml)
+.PHONY: build build-devel
+build build-devel: openapi-specs ## builds docker image (using main services/docker-compose-build.yml)
 	@$(MAKE) -C ${REPO_BASE_DIR} $@ target=${APP_NAME}

services/storage/client-sdk/Makefile

@@ -0,0 +1,72 @@
+# AUTO-GENERATION of python client SDK
+#
+#
+# SEE https://openapi-generator.tech/docs/usage#generate
+# SEE https://openapi-generator.tech/docs/generators/python
+#
+
+#
+# WARNING: DO NOT USE until codegen.sh is deprecated!
+# TODO: Current generator will NOT produce the same client-sdk code since
+# bundling openapi.yaml did not preserve e.g. the same schema models !!
+#
+
+.DEFAULT_GOAL := generator-help
+
+
+PACKAGE_NAME := simcore_service_storage_sdk
+
+GIT_USER_ID := ITISFoundation
+GIT_REPO_ID := osparc-simcore
+
+GENERATOR_NAME := python
+
+REPO_BASE_DIR := $(shell git rev-parse --show-toplevel)
+SCRIPTS_DIR   := $(abspath $(REPO_BASE_DIR)/scripts)
+OAS_PATH      := $(abspath $(CURDIR)/../src/simcore_service_storage/api/v0/openapi.yaml)
+REL_CURDIR    := $(subst $(REPO_BASE_DIR)/,,$(CURDIR))
+FRAGMENT      := \#egg=$(PACKAGE_NAME)&subdirectory=$(REL_CURDIR)
+
+ADDITIONAL_PROPS := \
+	generateSourceCodeOnly=false\
+	hideGenerationTimestamp=true\
+	library=asyncio\
+	packageName=$(PACKAGE_NAME)\
+	packageUrl="https://github.com/$(GIT_USER_ID)/${GIT_REPO_ID}.git$(FRAGMENT)"\
+	packageVersion=$(APP_VERSION)\
+	projectName=simcore-service-storage-sdk\
+	projectDescription="Data storage manager service client's SDK"\
+
+ADDITIONAL_PROPS := $(foreach prop,$(ADDITIONAL_PROPS),$(strip $(prop)))
+
+null  :=
+space := $(null) #
+comma := ,
+
+
+.PHONY: python-client generator-help
+
+openapi.yaml: $(OAS_PATH)
+	cp $< $@
+
+python-client: openapi.yaml ## runs python client generator
+	cd $(CURDIR); \
+	$(SCRIPTS_DIR)/openapi-generator-cli.bash generate \
+		--generator-name=$(GENERATOR_NAME) \
+		--git-user-id=$(GIT_USER_ID)\
+		--git-repo-id=$(GIT_REPO_ID)\
+		--http-user-agent="$(PACKAGE_NAME)/{packageVersion}/{language}"\
+		--input-spec=/local/$< \
+		--output=/local/$@ \
+		--additional-properties=$(subst $(space),$(comma),$(strip $(ADDITIONAL_PROPS)))\
+		--package-name=$(PACKAGE_NAME)\
+		--release-note="Updated to $(APP_VERSION)"
+	# deleting $<
+	-@rm $<
+
+
+generator-help: ## help on client-api generator
+	# generate help
+	@$(SCRIPTS_DIR)/openapi-generator-cli.bash help generate
+	# generator config help
+	@$(SCRIPTS_DIR)/openapi-generator-cli.bash config-help -g $(GENERATOR_NAME)

services/storage/client-sdk/codegen.sh

@@ -1,9 +1,9 @@
-#/bin/bash
+#! /bin/bash
 
 ../../../scripts/openapi/openapi_codegen.sh \
-    -i ../src/simcore_service_storage/oas3/v0/openapi.yaml \
-    -o . \
-    -g python \
-    -c ./codegen_config.json
+  -i ../src/simcore_service_storage/api/v0/openapi.yaml \
+  -o . \
+  -g python \
+  -c ./codegen_config.json
 
-# rm -f output.yaml
+# rm -f output.yaml