Fine Grained access rights

Makefile

@@ -189,8 +189,8 @@ endif
 
 
 up-devel: .stack-simcore-development.yml .init-swarm $(CLIENT_WEB_OUTPUT) ## Deploys local development stack, qx-compile+watch and ops stack (pass 'make ops_disabled=1 up-...' to disable)
-	# Start compile+watch front-end container [front-end]
-	$(MAKE_C) services/web/client compile-dev flags=--watch
+	# Start compile+watch front-end container [front-end]	
+	$(MAKE_C) services/web/client down compile-dev flags=--watch
 	# Deploy stack $(SWARM_STACK_NAME) [back-end]
 	@docker stack deploy -c $< $(SWARM_STACK_NAME)
 	$(MAKE) .deploy-ops

api/specs/common/schemas/project-v0.0.1-converted.yaml

@@ -7,6 +7,7 @@ required:
   - name
   - description
   - prjOwner
+  - accessRights
   - creationDate
   - lastChangeDate
   - thumbnail
@@ -26,7 +27,13 @@ properties:
     example: Dabbling in temporal transitions ...
   prjOwner:
     type: string
-    description: user uuid
+    format: idn-email
+    description: user email
+  accessRights:
+    type: object
+    description: >-
+      object containing the GroupID as key and read/write/execution permissions
+      as value
   creationDate:
     type: string
     description: project creation date

api/specs/common/schemas/project-v0.0.1.json

@@ -10,6 +10,7 @@
     "name",
     "description",
     "prjOwner",
+    "accessRights",
     "creationDate",
     "lastChangeDate",
     "thumbnail",
@@ -40,7 +41,12 @@
     },
     "prjOwner": {
       "type": "string",
-      "description": "user uuid"
+      "format": "idn-email",
+      "description": "user email"
+    },
+    "accessRights": {
+      "type": "object",
+      "description": "object containing the GroupID as key and read/write/execution permissions as value"
     },
     "creationDate": {
       "type": "string",
@@ -101,7 +107,7 @@
               "type": "string"
             },
             "progress": {
-              "type":"number",
+              "type": "number",
               "maximum": 100,
               "minimum": 0,
               "description": "the node progress value"
@@ -153,7 +159,10 @@
                       ],
                       "properties": {
                         "store": {
-                          "type": ["string", "integer"]
+                          "type": [
+                            "string",
+                            "integer"
+                          ]
                         },
                         "dataset": {
                           "type": "string"
@@ -176,7 +185,11 @@
               "patternProperties": {
                 "^[-_a-zA-Z0-9]+$": {
                   "type": "string",
-                  "enum": ["Invisible", "ReadOnly", "ReadAndWrite"],
+                  "enum": [
+                    "Invisible",
+                    "ReadOnly",
+                    "ReadAndWrite"
+                  ],
                   "default": "ReadAndWrite",
                   "examples": [
                     "ReadOnly"
@@ -219,7 +232,10 @@
                       ],
                       "properties": {
                         "store": {
-                          "type": ["string", "integer"]
+                          "type": [
+                            "string",
+                            "integer"
+                          ]
                         },
                         "dataset": {
                           "type": "string"
@@ -253,7 +269,10 @@
               ]
             },
             "parent": {
-              "type": ["string", "null"],
+              "type": [
+                "string",
+                "null"
+              ],
               "pattern": "^\\S+$",
               "description": "Parent's (group-nodes') node ID s.",
               "examples": [
@@ -288,4 +307,4 @@
       }
     }
   }
-}
+}

api/specs/webserver/components/schemas/me.yaml

@@ -27,6 +27,17 @@ ProfileOutput:
         format: email
       role:
         type: string
+      groups:
+        type: object
+        properties:
+          me:
+            $ref: '#/UsersGroup'
+          organizations:
+            type: array
+            items:
+              $ref: '#/UsersGroup'
+          all:
+            $ref: '#/UsersGroup'
       gravatar_id:
         type: string
   example:
@@ -48,6 +59,26 @@ ProfileEnveloped:
       default: null
 
 
+UsersGroup:
+  type: object
+  properties:
+    gid:
+      type: string
+    label:
+      type: string
+    description:
+      type: string
+  example:
+    - gid: '27'
+      label: 'A user'
+      description: 'A very special user'
+    - gid: '1'
+      label: 'ITIS Foundation'
+      description: 'The Foundation for Research on Information Technologies in Society'
+    - gid: '0'
+      label: 'All'
+      description: 'Open to all users'
+
 Token:
   description: api keys for third party services
   type: object

api/tests/test_repo_data.py

@@ -16,8 +16,8 @@
 from utils import current_repo_dir
 
 SYNCED_VERSIONS_SUFFIX = [
-    ".json",                 # json-schema specs file
-    "-converted.yaml"        # equivalent openapi specs file (see scripts/json-schema-to-openapi-schema)
+    ".json",  # json-schema specs file
+    "-converted.yaml",  # equivalent openapi specs file (see scripts/json-schema-to-openapi-schema)
 ]
 
 
@@ -38,29 +38,33 @@
 # ./tests/integration/computation/workbench_sleeper_dag_adjacency_list.json
 # ./tests/integration/computation/workbench_sleeper_payload.json
 
+
 def _load_data(fpath: Path):
     with open(fpath) as fh:
         try:
             data = json.load(fh)
         except json.JSONDecodeError:
             fh.seek(0)
-            data = yaml.load(fh)
+            data = yaml.safe_load(fh)
     return data
 
 
 @pytest.fixture(
     scope="module",
-    params= [ str(schema_path)
+    params=[
+        str(schema_path)
         for suffix in SYNCED_VERSIONS_SUFFIX
         for schema_path in current_repo_dir.rglob(f"schemas/project*{suffix}")
-        ]
+    ],
 )
 def project_schema(request, api_specs_dir):
     schema_path = Path(request.param)
     return _load_data(schema_path)
 
+
 # TESTS --------------------------------------------------
 
+
 @pytest.mark.parametrize("data_path", PROJECTS_PATHS)
 def test_project_against_schema(data_path, project_schema, this_repo_root_dir):
     """
@@ -80,13 +84,16 @@ def test_project_against_schema(data_path, project_schema, this_repo_root_dir):
             "creationDate": "8715-11-30T9:1:51.388Z",
             "lastChangeDate": "0944-02-31T5:1:7.795Z",
             "thumbnail": "labore incid",
-            "workbench": data["workbench"]
+            "accessRights": {},
+            "workbench": data["workbench"],
         }
         data = prj
 
     assert any(isinstance(data, _type) for _type in [List, Dict])
     if isinstance(data, Dict):
-        data = [data,]
+        data = [
+            data,
+        ]
 
     for project_data in data:
         jsonschema.validate(project_data, project_schema)

packages/postgres-database/Makefile

@@ -21,6 +21,23 @@ tests: ## runs unit tests
 	@pytest -vv --exitfirst --failed-first --durations=10 --pdb $(CURDIR)/tests
 
 
+.PHONY: import-db
+import-db: scripts/copy_database_volume.sh guard-SOURCE_HOST guard-SOURCE_DATA_VOLUME guard-TARGET_DATA_VOLUME ## copy volume $(SOURCE_DATA_VOLUME) from $(SOURCE_HOST) into local $(TARGET_DATA_VOLUME)
+	./scripts/copy_database_volume.sh
+
+guard-%:
+	@ if [ "${${*}}" = "" ]; then \
+		echo "Environment variable $* not set"; \
+		exit 1; \
+	fi
+
+.PHONY: setup-prod
+export POSTGRES_DATA_VOLUME = $(TARGET_DATA_VOLUME)
+setup-prod: install-dev up-prod  ## sets up a database using an external postgres volume to test migration
+	# discovering
+	sc-pg --help
+	@echo "To test migration, sc-pg discover -u USER -p PASSWORD, then sc-pg upgrade"
+
 .PHONY: setup-commit
 setup-commit: install-dev up-pg ## sets up a database to create a new commit into migration history
 	# discovering
@@ -46,11 +63,12 @@ migrate: $(DOT_ENV_FILE) ## basic migration update (use ONLY for development pur
 
 
 
-.PHONY: up-pg down-pg
-DOCKER_COMPOSE_CONFIG := tests/docker-compose.yml
+.PHONY: up-pg down-pg up-prod down-prod
+docker-compose-configs = $(wildcard tests/docker-compose*.yml)
+up-pg up-prod: $(docker-compose-configs) ## starts pg server
+	docker-compose -f tests/docker-compose.yml $(if $(findstring -prod,$@),-f tests/docker-compose.prod.yml,) up -d
+
+down-pg down-prod: $(docker-compose-configs) ## stops pg server
+	docker-compose -f tests/docker-compose.yml  $(if $(findstring -prod,$@),-f tests/docker-compose.prod.yml,) down
 
-up-pg: $(DOCKER_COMPOSE_CONFIG) ## starts pg server
-	docker-compose -f $< up -d
 
-down-pg: $(DOCKER_COMPOSE_CONFIG) ## stops pg server
-	docker-compose -f $< down

packages/postgres-database/docker/Dockerfile

@@ -1,31 +1,54 @@
-FROM python:3.6-alpine
+FROM python:3.6-slim as base
+
+LABEL maintainer=sanderegg
+
+# Sets utf-8 encoding for Python et al
+ENV LANG=C.UTF-8
+# Turns off writing .pyc files; superfluous on an ephemeral container.
+ENV PYTHONDONTWRITEBYTECODE=1 \
+  VIRTUAL_ENV=/home/scu/.venv
+# Ensures that the python and pip executables used
+# in the image will be those from our virtualenv.
+ENV PATH="${VIRTUAL_ENV}/bin:$PATH"
+
+
+FROM base as build
+
+RUN apt-get update &&\
+  apt-get install -y --no-install-recommends \
+  build-essential \
+  git
+
+# NOTE: python virtualenv is used here such that installed packages may be moved to production image easily by copying the venv
+RUN python -m venv ${VIRTUAL_ENV}
+
+RUN pip --no-cache-dir install --upgrade \
+  pip~=20.0.2  \
+  wheel \
+  setuptools
 
 ARG GIT_BRANCH
 ARG GIT_REPOSITORY
 
-RUN apk add --no-cache --virtual .build-deps \
-    gcc \
-    git \
-    python3-dev \
-    musl-dev \
-    postgresql-dev \
-    && git clone --single-branch --branch ${GIT_BRANCH} ${GIT_REPOSITORY} \
-    && pip install osparc-simcore/packages/postgres-database[migration] \
-    && apk del --no-cache .build-deps
-
-# runtime dependency
-RUN apk add --no-cache \
-    bash \
-    postgresql
+RUN git clone --single-branch --branch ${GIT_BRANCH} ${GIT_REPOSITORY} \
+  && pip install osparc-simcore/packages/postgres-database[migration]
+
+FROM base as production
+
+ENV PYTHONOPTIMIZE=TRUE
+
+WORKDIR /home/scu
 
+# bring installed package without build tools
+COPY --from=build ${VIRTUAL_ENV} ${VIRTUAL_ENV}
 COPY entrypoint.bash /home/entrypoint.bash
 
 
 ENV POSTGRES_USER=scu \
-    POSTGRES_PASSWORD=adminadmin \
-    POSTGRES_HOST=postgres \
-    POSTGRES_PORT=5432 \
-    POSTGRES_DB=simcoredb
+  POSTGRES_PASSWORD=adminadmin \
+  POSTGRES_HOST=postgres \
+  POSTGRES_PORT=5432 \
+  POSTGRES_DB=simcoredb
 
 ENTRYPOINT [ "/bin/sh", "/home/entrypoint.bash" ]
 CMD [ "sc-pg", "upgrade" ]

packages/postgres-database/scripts/64614dc0fada_add_groups_to_existing_users.sql

@@ -0,0 +1,15 @@
+CREATE OR REPLACE FUNCTION upgrade_user_table_with_groups() RETURNS VOID AS $$
+DECLARE
+    group_id BIGINT;
+    temprow RECORD;
+BEGIN
+    FOR temprow IN SELECT id, name FROM "users" WHERE "primary_gid" IS NULL
+    LOOP
+        INSERT INTO "groups" ("name", "description") VALUES (temprow.name, 'primary group') RETURNING gid INTO group_id;
+        INSERT INTO "user_to_groups" ("uid", "gid") VALUES (temprow.id, group_id);
+        UPDATE "users" SET "primary_gid" = group_id WHERE "id" = temprow.id;
+    END LOOP;
+END; $$ LANGUAGE 'plpgsql';
+
+SELECT upgrade_user_table_with_groups()
+DROP FUNCTION upgrade_user_table_with_groups()

packages/postgres-database/scripts/copy_database_volume.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -o errexit
+set -o nounset
+
+IFS=$(printf '\n\t')
+
+ssh "${SOURCE_HOST}" \
+    "docker run --rm -v ${SOURCE_DATA_VOLUME}:/from alpine ash -c 'cd /from ; tar -cf - . '" \
+    | \
+    docker run --rm -i -v "${TARGET_DATA_VOLUME}":/to alpine ash -c "cd /to ; tar -xpvf - "