Improve support for streaming

.github/workflows/build_and_test.yml

@@ -15,8 +15,8 @@ jobs:
 
     steps:
       # Checks-out your repository
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.10'
 

.github/workflows/python-publish.yml

@@ -13,9 +13,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.10'
     - name: Install dependencies

Makefile

@@ -56,7 +56,8 @@ run-pre-commit: setup-pre-commit
 update: down Pipfile.lock setup-pre-commit  ## Updates all the packages using Pipfile
 	docker compose run --rm --name spf_pipenv dev pipenv sync --dev && \
 	make devdocker && \
-	make pipenv-setup
+	make pipenv-setup && \
+	make up
 
 .PHONY:tests
 tests:

Pipfile

@@ -43,9 +43,9 @@ bounded-pool-executor = ">=0.0.3"
 # fastjsonschema is needed for validating JSON
 fastjsonschema= ">=2.18.0"
 # helix.fhir.client.sdk is needed for interacting with FHIR servers
-"helix.fhir.client.sdk" = ">=2.0.8"
+"helix.fhir.client.sdk" = ">=2.0.15"
 # opensearch-py is needed for interacting with OpenSearch
-opensearch-py= ">=1.1.0"
+opensearch-py= { extras = ['async'], version = ">=2.6.0" }
 # pyathena is needed for interacting with Athena in AWS
 pyathena = ">2.14.0"
 # spark-nlp is needed for natural language processing
@@ -60,6 +60,14 @@ pandas = ">=2.2.2"
 numexpr = ">=2.8.4"
 # bottleneck is needed for working with numerical data. pandas requires this minimum version.
 bottleneck = ">=1.3.6"
+# structlog is needed for structured logging
+structlog = ">=23.1.0"
+# usaddress is needed for parsing street addresses
+"usaddress"=">=0.5.10" # for parsing street addresses
+# usaddress-scourgify is needed for normalizing addresses
+"usaddress-scourgify"=">=0.6.0" # for normalizing addresses
+# aiohttp is needed for making HTTP requests asynchronously
+aiohttp = ">=3"
 
 [dev-packages]
 # setuptools is needed for building the package
@@ -73,11 +81,11 @@ pre-commit=">=3.7.1"
 # autoflake is needed for removing unused imports
 autoflake=">=2.3.1"
 # mypy is needed for type checking
-mypy = ">=1.10.1"
+mypy = ">=1.11.1"
 # pytest is needed for running tests
-pytest = ">=8.2.2"
+pytest = ">=8.3.2"
 # black is needed for formatting code
-black = ">=24.4.2"
+black = ">=24.8.0"
 # pygments is needed for syntax highlighting
 pygments=">=2.8.1" # not directly required, pinned by Snyk to avoid a vulnerability
 # Sphinx is needed for generating documentation
@@ -101,13 +109,13 @@ sparkdataframecomparer = ">=2.0.2"
 # pytest-ayncio is needed for running async tests
 pytest-asyncio = ">=0.23.8"
 # helix-mockserver-client is needed for mocking servers
-helix-mockserver-client=">=1.2.1"
+helix-mockserver-client=">=1.3.0"
 # sparkfhirschemas is needed for FHIR schemas
 sparkfhirschemas = ">=1.0.17"
 # types-boto3 is needed for type hints for boto3
 types-boto3 = ">=1.0.2"
 # moto is needed for mocking AWS services
-moto = { extras = ['all'], version = ">=5.0.11" }
+moto = { extras = ['all'], version = ">=5.0.12" }
 # types-requests is needed for type hints for requests
 types-requests=">=2.31.0"
 # types-PyMySQL is needed for type hints for PyMySQL
@@ -120,6 +128,8 @@ types-python-dateutil=">=2.8.19.14"
 pandas-stubs=">=2.2.2"
 # types-pytz is needed for type hints for pytz
 types-pytz=">=2024.1.0"
+# pydantic is needed for data class loading
+pydantic=">=2.8.2"
 
 # These dependencies are required for pipenv-setup.  They conflict with ones above, so we install these
 # only when running pipenv-setup

docker-compose.yml

@@ -1,6 +1,10 @@
 version: '3.5'
 services:
   dev:
+    depends_on:
+      - fhir
+      - warehouse
+      - elasticsearch
     user: root
     build:
       dockerfile: spark.Dockerfile
@@ -15,14 +19,13 @@ services:
     volumes:
       - ./:/SparkpipelineFramework/
 #      - ~/.local/share/virtualenvs:/root/.local/share/virtualenvs:rw
-#      - ../helix.fhir.client.sdk/helix_fhir_client_sdk:/usr/local/lib/python3.7/dist-packages/helix_fhir_client_sdk
+#      - ../helix.fhir.client.sdk/helix_fhir_client_sdk:/usr/local/lib/python3.10/dist-packages/helix_fhir_client_sdk
+#      - ../mockserver_client/mockserver_client:/usr/local/lib/python3.10/dist-packages/mockserver_client
     container_name: spf_dev
     working_dir: /SparkpipelineFramework
 
   warehouse:
     image: mysql:8.0.39
-    volumes:
-      - mysql_data:/var/lib/mysql
     environment:
       MYSQL_ROOT_PASSWORD: root_password
       MYSQL_DATABASE: fhir_rpt
@@ -51,7 +54,8 @@ services:
   fhir:
     depends_on:
       - mongo
-    image: imranq2/node-fhir-server-mongo:5.3.8
+      - keycloak
+    image: imranq2/node-fhir-server-mongo:5.3.16
     # To use local fhir code, comment above line and uncomment below
 #    build:
 #      dockerfile: Dockerfile
@@ -110,8 +114,6 @@ services:
       - '27017:27017'
     environment:
       - ALLOW_EMPTY_PASSWORD=yes
-    volumes:
-      - mongo_data:/data/db
     healthcheck:
       test: echo 'db.runCommand("ping").ok' | mongo mongo:27017/test --quiet
 
@@ -138,9 +140,6 @@ services:
       nofile:
         soft: 65536 # maximum number of open files for the Elasticsearch user, set to at least 65536 on modern systems
         hard: 65536
-    volumes:
-      - es_data:/usr/share/opensearch/data:delegated
-#      - ./conf/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:cached
     ports:
       - '9200:9200'
       - '9600:9600' # required for Performance Analyzer
@@ -194,8 +193,3 @@ services:
     command: ["start-dev", "--import-realm", "--verbose"]
     volumes:
       - ./keycloak-config/realm-import.json:/opt/keycloak/data/import/realm-import.json
-
-volumes:
-  mysql_data:
-  mongo_data:
-  es_data:

pytest.ini

@@ -1,2 +1,5 @@
 [pytest]
 norecursedirs = tests/**/temp/*
+addopts =
+; all discovered async tests are considered asyncio-driven
+asyncio_mode = auto

setup.cfg

@@ -1,5 +1,3 @@
-[tool:pytest]
-addopts =
 [mypy]
 python_version = 3.10
 warn_return_any = True
@@ -18,6 +16,8 @@ ignore_missing_imports = True
 ignore_missing_imports = True
 [mypy-py4j.protocol.*]
 ignore_missing_imports = True
+[mypy-usaddress.*]
+ignore_missing_imports = True
 [flake8]
 ignore = E501, W503, W504, E126, E123
 exclude = venv/

setup.py

@@ -74,15 +74,19 @@ def violation(operation: Any, *args: Any, **_: Any) -> None:
         "sqlparse>=0.4.4",
         "bounded-pool-executor>=0.0.3",
         "fastjsonschema>=2.18.0",
-        "helix.fhir.client.sdk>=2.0.8",
-        "opensearch-py>=1.1.0",
+        "helix.fhir.client.sdk>=2.0.15",
+        "opensearch-py[async]>=2.6.0",
         "pyathena>2.14.0",
         "spark-nlp>=4.2.3",
         "pymongo>=4.8.0",
         "more-itertools>=9.1.0",
         "pandas>=2.2.2",
         "numexpr>=2.8.4",
         "bottleneck>=1.3.6",
+        "structlog>=23.1.0",
+        "usaddress>=0.5.10",
+        "usaddress-scourgify>=0.6.0",
+        "aiohttp>=3",
     ],
     classifiers=[
         "Development Status :: 4 - Beta",