Upgrade pyyaml to 6.0.1 (#445)

* Upgrade pyyaml to 6.0.1
Previous versions are incompatible with Cython 3.0.0, which was released
just the previous week.
See yaml/pyyaml#724

* Upgrade package versions, from dependabot warnings.
sqlparse to 0.4.4
requests to 2.31.0
django to 3.2.20
cryptography to 41.0.2

* Fix CI.
The new machine runs a new version of Docker. We require changes.
The new machine runs python 3.10, we require changes.

Author: juagargi

.circleci/Dockerfile-coord

@@ -1,12 +1,12 @@
-FROM python:3.6
+FROM python:3.10
 ENV PYTHONUNBUFFERED 1
 
-RUN pip install appdeps
+RUN pip3 install appdeps
 
 RUN mkdir /scionlab
 WORKDIR /scionlab
 COPY requirements.txt dev-requirements.txt /scionlab/
-RUN pip install -r requirements.txt -r dev-requirements.txt
+RUN pip3 install -r requirements.txt -r dev-requirements.txt
 COPY . /scionlab/
 
 # Fixup django settings for the integration tests:

.circleci/Dockerfile-scionHost

@@ -1,7 +1,13 @@
-FROM ubuntu:bionic
+FROM ubuntu:jammy
+# New docker versions seem __NOT__ to create /.dockerenv during __image creation__. But we need it.
+RUN touch /.dockerenv
 
 ARG package_repo=packages.netsec.inf.ethz.ch
 
+# Force debconf (called by apt-get) to be noninteractive
+ENV DEBIAN_FRONTEND=noninteractive
+RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
+
 # Install base
 RUN apt-get update && apt-get install --assume-yes \
   systemd \
@@ -11,7 +17,7 @@ RUN apt-get update && apt-get install --assume-yes \
   curl \
   moreutils
 
-# systemd
+# systemd configuration. Note that the cgroup volume is no longer needed, nor works like documented below.
 # Based on: https://developers.redhat.com/blog/2014/05/05/running-systemd-within-docker-container/
 #  - converted to ubuntu, i.e. fixed some paths and removed unnecessary cleanup
 #  - keep systemd-user-sessions.service, to allow login through SSH (login disabled on startup until this is run)
@@ -23,7 +29,6 @@ rm -f /lib/systemd/system/local-fs.target.wants/*; \
 rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
 rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
 rm -f /lib/systemd/system/basic.target.wants/*;
-VOLUME [ "/sys/fs/cgroup" ]
 
 # Install SCION
 # XXX(matzf): install testing/prod based on branch name???

.circleci/config.yml

@@ -72,7 +72,7 @@ commands:
 jobs:
   style-check:
     docker:
-      - image: cimg/python:3.6.13
+      - image: cimg/python:3.10
     steps:
       - install_git_lfs
       - cached_checkout
@@ -91,7 +91,7 @@ jobs:
 
   unit-tests:
     docker:
-      - image: cimg/python:3.6.13
+      - image: cimg/python:3.10
     steps:
       - install_git_lfs
       - cached_checkout
@@ -111,7 +111,7 @@ jobs:
 
   integration-tests:
     machine:
-      image: ubuntu-2004:202107-02
+      image: ubuntu-2204:2023.07.2
       docker_layer_caching: true
     environment:
       COMPOSE_FILE: ".circleci/docker-compose.yml"
@@ -120,10 +120,23 @@ jobs:
 
       - cached_checkout
 
+      - run:
+          name: Python defaults
+          command: |
+            set -x
+
+            # Use python3 as default
+            VERSION=$(pyenv versions --bare | grep -e "^3." | tail -n 1)
+            echo "Python3 version is: $VERSION"
+            pyenv global $VERSION
+
+            pip install --upgrade pip setuptools wheel install
+
       - run:
           name: Build and start containers for coordinator and ASes
           command: |
             set -x
+
             pip install pyyaml
 
             .circleci/setup/build-containers.sh --build-arg package_repo=packages-test.netsec.inf.ethz.ch
@@ -199,10 +212,12 @@ jobs:
           name: OLD -- Checkout
           command: |
             git checkout --force -B master --track origin/master
+
       - run:
           name: OLD -- Build and start containers for coordinator and ASes
           command: |
             set -x
+
             pip install pyyaml
 
             .circleci/setup/build-containers.sh --build-arg package_repo=packages.netsec.inf.ethz.ch # start with released packages
@@ -237,7 +252,7 @@ jobs:
 
   production-stack-tests:
     machine:
-      image: ubuntu-2004:202107-02
+      image: ubuntu-2204:2023.07.2
       docker_layer_caching: true
     environment:
       COMPOSE_FILE: "docker-compose.yml:.circleci/docker-compose.test-prod-db.yaml"

.circleci/docker-compose.yml

@@ -53,8 +53,6 @@ services:
         ipv4_address: 172.31.0.110
     env_file: /tmp/as1301.env
     privileged: true
-    volumes:
-      - /sys/fs/cgroup:/sys/fs/cgroup:ro
 
   as1303:
     build:
@@ -65,8 +63,6 @@ services:
         ipv4_address: 172.31.0.111
     env_file: /tmp/as1303.env
     privileged: true
-    volumes:
-      - /sys/fs/cgroup:/sys/fs/cgroup:ro
 
   as1305:
     build:
@@ -77,8 +73,6 @@ services:
         ipv4_address: 172.31.0.112
     env_file: /tmp/as1305.env
     privileged: true
-    volumes:
-      - /sys/fs/cgroup:/sys/fs/cgroup:ro
 
   as1401:
     build:
@@ -89,8 +83,6 @@ services:
         ipv4_address: 172.31.0.113
     env_file: /tmp/as1401.env
     privileged: true
-    volumes:
-      - /sys/fs/cgroup:/sys/fs/cgroup:ro
 
   as1405:
     build:
@@ -101,8 +93,6 @@ services:
         ipv4_address: 172.31.0.114
     env_file: /tmp/as1405.env
     privileged: true
-    volumes:
-      - /sys/fs/cgroup:/sys/fs/cgroup:ro
 
   useras4:
     build:
@@ -113,8 +103,6 @@ services:
         ipv4_address: 172.31.0.2
     env_file: /tmp/as4.env
     privileged: true
-    volumes:
-      - /sys/fs/cgroup:/sys/fs/cgroup:ro
 
 volumes:
   pgdata:

.circleci/setup/generate-host-envs.py

@@ -3,7 +3,6 @@
 # Write the information to /tmp/asXY.env files, so it can easily be consumed in the docker-compose
 # setup.
 
-from __future__ import print_function
 import yaml
 
 import os

.circleci/setup/init-ases.sh

@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-for c in $(docker-compose ps --services egrep -x '(user)?as[0-9]+'); do
+for c in $(docker-compose ps --services | egrep -x '(user)?as[0-9]+'); do
   docker-compose exec -T "$c" /bin/bash -c 'scionlab-config --host-id ${SCIONLAB_HOST_ID} --host-secret ${SCIONLAB_HOST_SECRET} --url http://coord:8000'
 done

README.md

@@ -42,9 +42,9 @@ source venv/bin/activate
 # Install Python requirements (Django, libraries, etc.)
 pip install --require-hashes -r requirements.txt -r dev-requirements.txt
 
-# NOTE: the 'scrypt' package may fail to build if libssl is not installed
+# NOTE: Some packages may fail to build. To ensure a complete dev setup is ready, in ubuntu you would run:
+#         apt install gcc libpq-dev libpython3-dev libffi-dev
 #       on your machine; install and try again.
-#         apt install libssl1.0
 ```
 
 To render the topology graph, `graphviz` needs to be installed additionally to the python dependencies. On ubuntu:

deploy/Dockerfile-django

@@ -1,4 +1,4 @@
-FROM python:3.6
+FROM python:3.10
 ENV PYTHONUNBUFFERED 1
 
 # Graphviz is used to render the topology (see scionlab.views.topology); the python
@@ -8,12 +8,12 @@ RUN apt-get update && apt-get install -y graphviz
 # Appdeps is used by the entrypoint script to wait for the DB services.
 # Not included in requirements because it's a requirement of the
 # docker-compose plumbing, not the application.
-RUN pip install appdeps
+RUN pip3 install appdeps
 
 RUN mkdir /scionlab
 WORKDIR /scionlab
 COPY requirements.txt /scionlab/
-RUN pip install -r requirements.txt
+RUN pip3 install -r requirements.txt
 COPY . /scionlab/
 
 

requirements.in

@@ -1,5 +1,5 @@
-Django>=3.2.18,<4.0
-cryptography>=39.0.1
+Django>=3.2.20,<4.0
+cryptography>=41.0.2
 django-crispy-forms>=1.9.0
 django-extensions
 django-maintenance-mode>=0.14.0
@@ -10,6 +10,10 @@ gunicorn
 jinja2
 psycopg2-binary  # PostgreSQL backend
 pynacl
-pyyaml>=4.2b1  # for fixtures and config generation; high severity vulnerability before 4.1 (CVE-2017-18342)
+pyyaml>=6.0.1  # for fixtures and config generation; high severity vulnerability before 4.1 (CVE-2017-18342), Cython>=3.0.0 (released in 2023) incompatible before 6.0.1
 scrypt
 toml
+
+
+# Sub-dependencies of above packages which have security implications, and thus require specific prerequisites:
+requests>=2.31.0