use structured logging

Makefile

@@ -31,6 +31,7 @@ up: Pipfile.lock
 	if [ "`docker inspect --format {{.State.Health.Status}} sparkpipelineframework-fhir-1`" != "healthy" ]; then docker ps && docker logs sparkpipelineframework-fhir-1 && printf "========== ERROR: sparkpipelineframework-fhir-1 did not start. Run docker logs sparkpipelineframework-fhir-1 =========\n" && exit 1; fi
 	@echo MockServer dashboard: http://localhost:1080/mockserver/dashboard
 	@echo Spark dashboard: http://localhost:8080/
+	@echo Seq Log dashboard http://localhost:8085/
 	@echo Fhir server dashboard http://localhost:3000/
 	@echo Keycloak OAuth dashboard http://admin:password@localhost:8080/
 

Pipfile

@@ -11,51 +11,56 @@ protobuf=">=3"
 # PySpark has to match the version of Spark we use for testing
 pyspark = "==3.3.0"
 # pyarrow is needed for Pandas UDF and mapInPandas
-pyarrow=">=4.0.0" # https://spark.apache.org/docs/latest/api/python/user_guide/sql/arrow_pandas.html#recommended-pandas-and-pyarrow-versions
-# pandas is needed for Pandas UDF and mapInPandas
-pandas = ">=1.0.5"
+pyarrow=">=17.0.0" # https://spark.apache.org/docs/latest/api/python/user_guide/sql/arrow_pandas.html#recommended-pandas-and-pyarrow-versions
 # delta-spark is needed for Delta Lake
 delta-spark="==2.3.0"
 # sparkautomapper is needed for mapping data in Spark
 sparkautomapper = ">=2.0.7"
 # pymysql is needed for connecting to MySQL
-pymysql=">=1.0.2"
+pymysql=">=1.0.3"
 # furl is needed for parsing URLs
-furl = "*"
+furl = ">=2.1.3"
 # requests is needed for making HTTP requests
-requests = "*"
+requests = ">=2.31.0"
 # boto3 is needed for interacting with AWS services
-boto3 = "*"
+boto3 = ">=1.34.140"
 # chardet is needed for detecting character encoding
 chardet="*"
 # slack_sdk is needed for sending messages to Slack
-slack_sdk = "*"
+slack_sdk = ">=3.22.0"
 # smart_open is needed for reading and writing files
 smart_open = { extras = ['s3'], version = ">=6.3.0" }
 # mlflow is needed for tracking experiments in MLFlow
-mlflow-skinny = ">=2.6.0"
+mlflow-skinny = ">=2.15.0"
 # sqlalchemy is needed for interacting with databases
 SQLAlchemy = ">=1.4.37"
 # alembic is needed for database migrations
-alembic = ">=1.8.0"
+alembic = ">=1.10.0"
 # sqlparse is needed for parsing SQL
-sqlparse = ">=0.4.2"
+sqlparse = ">=0.4.4"
 # bounded-pool-executor is needed for creating a bounded thread pool to run in parallel
 bounded-pool-executor = ">=0.0.3"
 # fastjsonschema is needed for validating JSON
-fastjsonschema= "*"
+fastjsonschema= ">=2.18.0"
 # helix.fhir.client.sdk is needed for interacting with FHIR servers
 "helix.fhir.client.sdk" = ">=2.0.8"
 # opensearch-py is needed for interacting with OpenSearch
-opensearch-py= "*"
+opensearch-py= ">=1.1.0"
 # pyathena is needed for interacting with Athena in AWS
-pyathena = "*"
+pyathena = ">2.14.0"
 # spark-nlp is needed for natural language processing
-spark-nlp = ">=4.2.2"
+spark-nlp = ">=4.2.3"
 # pymongo is needed for interacting with MongoDB
 pymongo=">=4.8.0"
 # more-itertools is needed for additional iterators
 more-itertools = ">=9.1.0"
+# pandas is needed for Pandas UDF and mapInPandas
+pandas = ">=2.2.2"
+# numexpr is needed for working with numerical expressions.  pandas requires this minimum version.
+numexpr = ">=2.8.4"
+# bottleneck is needed for working with numerical data. pandas requires this minimum version.
+bottleneck = ">=1.3.6"
+structlog = ">=22.1.0"
 
 [dev-packages]
 # setuptools is needed for building the package
@@ -91,7 +96,7 @@ py4j = "==0.10.9.5" # https://spark.apache.org/docs/latest/api/python/getting_st
 # pyspark is needed for running Spark jobs
 pyspark="==3.3.0"  # should match the version of spark we use for testing
 # Deprecated is needed for marking deprecated functions
-Deprecated = ">=1.2.12"
+Deprecated = ">=1.2.13"
 # sparkdataframecomparer is needed for comparing Spark DataFrames
 sparkdataframecomparer = ">=2.0.2"
 # pytest-ayncio is needed for running async tests
@@ -100,14 +105,17 @@ pytest-asyncio = ">=0.23.8"
 helix-mockserver-client=">=1.2.1"
 # sparkfhirschemas is needed for FHIR schemas
 sparkfhirschemas = ">=1.0.17"
+pygelf = ">=0.4.2"
+seqlog = ">=0.3.20"
+better-exceptions = ">=0.3.3"
 # types-boto3 is needed for type hints for boto3
 types-boto3 = ">=1.0.2"
 # moto is needed for mocking AWS services
 moto = { extras = ['all'], version = ">=5.0.11" }
 # types-requests is needed for type hints for requests
-types-requests=">=0.1.11"
+types-requests=">=2.31.0"
 # types-PyMySQL is needed for type hints for PyMySQL
-types-PyMySQL=">=0.1.5"
+types-PyMySQL=">=0.1.6"
 # types-urllib3 is needed for type hints for urllib3
 types-urllib3=">=1.26.0"
 # types-python-dateutil is needed for type hints for python-dateutil