refactor: code aws s3 bucket for remote storage of custom configs

Author: lpm0073

Makefile

@@ -14,7 +14,7 @@ PIP := $(PYTHON) -m pip
 ifneq ("$(wildcard .env)","")
     include .env
 else
-    $(shell echo -e "OPENAI_API_ORGANIZATION=PLEASE-ADD-ME\nOPENAI_API_KEY=PLEASE-ADD-ME\nPINECONE_API_KEY=PLEASE-ADD-ME\nPINECONE_ENVIRONMENT=gcp-starter\nGOOGLE_MAPS_API_KEY=PLEASE-ADD-ME\nDEBUG_MODE=True\SETTINGS_AWS_S3_BUCKET=PLEASE-ADD-ME\n" >> .env)
+    $(shell echo -e "OPENAI_API_ORGANIZATION=PLEASE-ADD-ME\nOPENAI_API_KEY=PLEASE-ADD-ME\nPINECONE_API_KEY=PLEASE-ADD-ME\nPINECONE_ENVIRONMENT=gcp-starter\nGOOGLE_MAPS_API_KEY=PLEASE-ADD-ME\nDEBUG_MODE=True\n" >> .env)
 endif
 
 .PHONY: analyze pre-commit api-init api-activate api-lint api-clean api-test client-init client-lint client-update client-run client-build client-release

api/terraform/python/openai_api/common/aws.py

@@ -158,9 +158,13 @@ def domain_exists(self) -> bool:
 
     def get_bucket_by_prefix(self, bucket_prefix) -> str:
         """Return the bucket name given the bucket prefix."""
-        for bucket in settings.aws_s3_client.buckets.all():
-            if bucket.name.startswith(bucket_prefix):
-                return f"arn:aws:s3:::{bucket.name}"
+        try:
+            for bucket in settings.aws_s3_client.list_buckets()["Buckets"]:
+                if bucket["Name"].startswith(bucket_prefix):
+                    return f"arn:aws:s3:::{bucket['Name']}"
+        except TypeError:
+            # TypeError: startswith first arg must be str or a tuple of str, not NoneType
+            pass
         return None
 
     def bucket_exists(self, bucket_prefix) -> bool:

api/terraform/python/openai_api/common/conf.py

@@ -98,9 +98,9 @@ class Services:
     AWS_IAM = ("iam", True)
     AWS_LAMBDA = ("lambda", True)
     AWS_ROUTE53 = ("route53", True)
+    AWS_S3 = ("s3", True)
 
     # disabled
-    AWS_S3 = ("s3", False)
     AWS_SNS = ("sns", False)
     AWS_SQS = ("sqs", False)
     AWS_SES = ("ses", False)
@@ -161,16 +161,15 @@ class SettingsDefaults:
     AWS_REGION = TFVARS.get("aws_region", "us-east-1")
 
     # aws api gateway defaults
-    AWS_APIGATEWAY_CREATE_CUSTOM_DOMAIN = TFVARS.get("aws_apigateway_create_custom_domaim", False)
-    AWS_APIGATEWAY_ROOT_DOMAIN = TFVARS.get("aws_apigateway_root_domain", None)
+    AWS_APIGATEWAY_CREATE_CUSTOM_DOMAIN = TFVARS.get("create_custom_domain", False)
+    AWS_APIGATEWAY_ROOT_DOMAIN = TFVARS.get("root_domain", None)
     AWS_APIGATEWAY_READ_TIMEOUT: int = TFVARS.get("aws_apigateway_read_timeout", 70)
     AWS_APIGATEWAY_CONNECT_TIMEOUT: int = TFVARS.get("aws_apigateway_connect_timeout", 70)
     AWS_APIGATEWAY_MAX_ATTEMPTS: int = TFVARS.get("aws_apigateway_max_attempts", 10)
 
     GOOGLE_MAPS_API_KEY: str = TFVARS.get("google_maps_api_key", None)
 
     LANGCHAIN_MEMORY_KEY = "chat_history"
-    SETTINGS_AWS_S3_BUCKET: str = None
     OPENAI_API_ORGANIZATION: str = None
     OPENAI_API_KEY = SecretStr(None)
     OPENAI_ENDPOINT_IMAGE_N = 4
@@ -358,7 +357,6 @@ def __init__(self, **data: Any):  # noqa: C901
         env="GOOGLE_MAPS_API_KEY",
     )
     langchain_memory_key: Optional[str] = Field(SettingsDefaults.LANGCHAIN_MEMORY_KEY, env="LANGCHAIN_MEMORY_KEY")
-    settings_aws_bucket: Optional[str] = Field(SettingsDefaults.SETTINGS_AWS_S3_BUCKET, env="SETTINGS_AWS_S3_BUCKET")
     openai_api_organization: Optional[str] = Field(
         SettingsDefaults.OPENAI_API_ORGANIZATION, env="OPENAI_API_ORGANIZATION"
     )
@@ -466,9 +464,12 @@ def aws_dynamodb_client(self):
         return self._aws_dynamodb_client
 
     @property
-    def aws_s3_bucket_name(self) -> str:
-        """Return the S3 bucket name."""
-        return self.aws_account_id + "-" + self.shared_resource_identifier
+    def aws_s3_client(self):
+        """S3 client"""
+        Services.raise_error_on_disabled(Services.AWS_S3)
+        if not self._aws_s3_client:
+            self._aws_s3_client = self.aws_session.client("s3")
+        return self._aws_s3_client
 
     @property
     def aws_apigateway_name(self) -> str:
@@ -488,6 +489,13 @@ def aws_apigateway_domain_name(self) -> str:
                 return f"{api_id}.execute-api.{settings.aws_region}.amazonaws.com"
         return None
 
+    @property
+    def aws_s3_bucket_name(self) -> str:
+        """Return the S3 bucket name."""
+        if self.shared_resource_identifier and self.aws_apigateway_root_domain:
+            return "api." + self.shared_resource_identifier + "." + self.aws_apigateway_root_domain
+        return None
+
     @property
     def is_using_dotenv_file(self) -> bool:
         """Is the dotenv file being used?"""
@@ -566,7 +574,7 @@ def get_installed_packages():
                 "google_maps_api_key": self.google_maps_api_key,
             },
             "openai_api": {
-                "settings_aws_bucket": self.settings_aws_bucket,
+                "aws_s3_bucket_name": self.aws_s3_bucket_name,
                 "langchain_memory_key": self.langchain_memory_key,
                 "openai_endpoint_image_n": self.openai_endpoint_image_n,
                 "openai_endpoint_image_size": self.openai_endpoint_image_size,
@@ -691,13 +699,6 @@ def check_langchain_memory_key(cls, v) -> str:
             return SettingsDefaults.LANGCHAIN_MEMORY_KEY
         return v
 
-    @field_validator("settings_aws_bucket")
-    def check_lambda_openai_function_config_url(cls, v) -> str:
-        """Check settings_aws_bucket"""
-        if v in [None, ""]:
-            return SettingsDefaults.SETTINGS_AWS_S3_BUCKET
-        return v
-
     @field_validator("openai_api_organization")
     def check_openai_api_organization(cls, v) -> str:
         """Check openai_api_organization"""

api/terraform/python/openai_api/common/tests/test_configuration.py

@@ -306,8 +306,8 @@ def test_settings_dynamodb(self):
     def test_settings_aws_s3_bucket_name(self):
         """Test that the S3 bucket name is valid."""
         mock_settings = Settings(init_info="test_settings_aws_s3_bucket_name()")
-        self.assertIsNotNone(mock_settings.aws_s3_bucket_name)
-        self.assertTrue(mock_settings.aws_s3_bucket_name.startswith(mock_settings.aws_account_id))
+        if mock_settings.aws_apigateway_create_custom_domaim:
+            self.assertIsNotNone(mock_settings.aws_s3_bucket_name)
 
     def test_settings_aws_apigateway_domain_name(self):
         """Test that the API Gateway domain name is valid."""

api/terraform/python/openai_api/lambda_openai_function/refers_to.py renamed to api/terraform/python/openai_api/lambda_openai_function/custom_config.py

@@ -263,7 +263,7 @@ class CustomConfigs:
 
     _custom_configs: list[CustomConfig] = None
     _aws_bucket_name: str = None
-    _aws_bucket_path: str = "/aws_openai/lambda_openai_function/custom_configs/"
+    _aws_bucket_path: str = "aws_openai/lambda_openai_function/custom_configs/"
     _aws_bucket_path_validated: bool = False
 
     def __init__(self, config_path: str = None, aws_s3_bucket_name: str = None):
@@ -285,21 +285,14 @@ def __init__(self, config_path: str = None, aws_s3_bucket_name: str = None):
             s3 = settings.aws_session.resource("s3")
             bucket = s3.Bucket(self._aws_bucket_name)
 
-            for obj in bucket.objects.filter(Prefix=self._aws_bucket_path):
-                i += 1
-                file_content = obj.get()["Body"].read().decode("utf-8")
-                config_json = yaml.safe_load(file_content)
-                custom_config = CustomConfig(config_json=config_json, index=i)
-                self._custom_configs.append(custom_config)
-
-    def list_yaml_files(bucket_name):
-        """List all the YAML files in the AWS S3 bucket"""
-        s3 = settings.aws_session.resource("s3")
-        bucket = s3.Bucket(bucket_name)
-
-        for obj in bucket.objects.all():
-            if obj.key.endswith(".yaml") or obj.key.endswith(".yml"):
-                print("Found YAML file:", obj.key)
+            for obj in bucket.objects.filter(Prefix=self.aws_bucket_path):
+                if obj.key.endswith(".yaml") or obj.key.endswith(".yml"):
+                    i += 1
+                    file_content = obj.get()["Body"].read().decode("utf-8")
+                    config_json = yaml.safe_load(file_content)
+                    if config_json:
+                        custom_config = CustomConfig(config_json=config_json, index=i)
+                        self._custom_configs.append(custom_config)
 
     @property
     def valid_configs(self) -> list[CustomConfig]:
@@ -311,38 +304,36 @@ def invalid_configs(self) -> list[CustomConfig]:
         """Return a list of invalid configs"""
         return [config for config in self._custom_configs if not config.is_valid]
 
+    @property
+    def aws_bucket_path(self) -> str:
+        """Return the remote host"""
+        return self._aws_bucket_path
+
     @property
     def aws_bucket_path_validated(self) -> bool:
         """Return True if the remote host is valid"""
         return self._aws_bucket_path_validated
 
-    @property
-    def aws_bucket_full_path(self) -> str:
-        """Return the remote host"""
-        if self.aws_bucket_path_validated:
-            return self._aws_bucket_name + self._aws_bucket_path
-        return None
-
     def verify_bucket(self, bucket_name: str):
         """Verify that the remote host is valid"""
+        if not bucket_name:
+            return
+
         s3 = settings.aws_session.resource("s3")
-        bucket = s3.Bucket(bucket_name)
-        folder_path = self._aws_bucket_path
         try:
             # Check if bucket exists
             s3.meta.client.head_bucket(Bucket=bucket_name)
         # pylint: disable=broad-exception-caught
-        except Exception:
+        except Exception as e:
+            log.warning("Bucket %s does not exist: %s", bucket_name, e)
             return
 
-        try:
-            # Create any missing folders
-            if not any(s3_object.key.startswith(folder_path) for s3_object in bucket.objects.all()):
-                s3.Object(bucket_name, folder_path).put()
-            self._aws_bucket_path_validated = True
-        # pylint: disable=broad-exception-caught
-        except Exception:
-            pass
+        # Create any missing folders
+        bucket = s3.Bucket(bucket_name)
+        if not any(s3_object.key.startswith(self.aws_bucket_path) for s3_object in bucket.objects.all()):
+            print(f"Creating folder {self.aws_bucket_path} in bucket {bucket_name}")
+            s3.Object(bucket_name, self.aws_bucket_path).put()
+        self._aws_bucket_path_validated = True
 
     def to_json(self) -> json:
         """Return the _custom_configs list as a JSON object"""
@@ -370,4 +361,9 @@ def custom_configs(self) -> CustomConfigs:
         return self._custom_configs
 
 
-config = SingletonCustomConfigs().custom_configs.valid_configs
+_custom_configs = SingletonCustomConfigs().custom_configs
+config = _custom_configs.valid_configs
+if len(_custom_configs.invalid_configs) > 0:
+    invalid_configurations = list(_custom_configs.invalid_configs.file_name)
+    invalid_configurations = ", ".join(invalid_configurations)
+    log.error("Invalid custom config files: %s", invalid_configurations)

api/terraform/python/openai_api/lambda_openai_function/function_refers_to.py

@@ -6,9 +6,9 @@
 import json
 
 from openai_api.common.const import PYTHON_ROOT
+from openai_api.lambda_openai_function.custom_config import CustomConfig
+from openai_api.lambda_openai_function.custom_config import config as refers_to_config
 from openai_api.lambda_openai_function.natural_language_processing import does_refer_to
-from openai_api.lambda_openai_function.refers_to import CustomConfig
-from openai_api.lambda_openai_function.refers_to import config as refers_to_config
 
 
 def search_terms_are_in_messages(messages: list, search_terms: list = None, search_pairs: list = None) -> bool:

api/terraform/python/openai_api/lambda_openai_function/lambda_handler.py

@@ -38,6 +38,7 @@
     validate_completion_request,
     validate_item,
 )
+from openai_api.lambda_openai_function.custom_config import config as refers_to_config
 from openai_api.lambda_openai_function.function_refers_to import (
     customized_prompt,
     get_additional_info,
@@ -50,7 +51,6 @@
     get_current_weather,
     weather_tool_factory,
 )
-from openai_api.lambda_openai_function.refers_to import config as refers_to_config
 
 
 openai.organization = settings.openai_api_organization

api/terraform/python/openai_api/lambda_openai_function/tests/test_refers_to.py renamed to api/terraform/python/openai_api/lambda_openai_function/tests/test_custom_config.py

@@ -21,7 +21,7 @@
 
 
 # pylint: disable=no-name-in-module
-from openai_api.lambda_openai_function.refers_to import (
+from openai_api.lambda_openai_function.custom_config import (
     AdditionalInformation,
     CustomConfig,
     SearchTerms,

api/terraform/python/openai_api/lambda_openai_function/tests/test_lambda_openai_refers_to.py renamed to api/terraform/python/openai_api/lambda_openai_function/tests/test_lambda_openai_custom_config.py

@@ -20,12 +20,13 @@
     sys.path.append(PYTHON_ROOT)  # noqa: E402
 
 
+from openai_api.lambda_openai_function.custom_config import CustomConfig
+
 # pylint: disable=no-name-in-module
 from openai_api.lambda_openai_function.function_refers_to import (
     get_additional_info,
     info_tool_factory,
 )
-from openai_api.lambda_openai_function.refers_to import CustomConfig
 from openai_api.lambda_openai_function.tests.test_setup import get_test_file_path
 
 

api/terraform/python/openai_api/lambda_openai_function/tests/test_lambda_openai_function.py

@@ -18,12 +18,12 @@
     sys.path.append(PYTHON_ROOT)  # noqa: E402
 
 
+from openai_api.lambda_openai_function.custom_config import CustomConfig
 from openai_api.lambda_openai_function.lambda_handler import (  # noqa: E402; handler,
     handler,
     search_terms_are_in_messages,
 )
 from openai_api.lambda_openai_function.natural_language_processing import does_refer_to
-from openai_api.lambda_openai_function.refers_to import CustomConfig
 from openai_api.lambda_openai_function.tests.test_setup import (
     get_test_file,
     get_test_file_path,

api/terraform/s3.tf

@@ -0,0 +1,35 @@
+
+locals {
+  s3_bucket_name = "api.${var.shared_resource_identifier}.${var.root_domain}"
+}
+
+resource "aws_s3_bucket" "openai" {
+  bucket = local.s3_bucket_name
+  tags   = var.tags
+}
+
+data "aws_iam_policy_document" "s3_bucket_policy" {
+  statement {
+    actions = [
+      "s3:PutObject",
+      "s3:GetObject",
+      "s3:ListBucket",
+      "s3:DeleteObject",
+    ]
+
+    principals {
+      type        = "AWS"
+      identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:user/mcdaniel"]
+    }
+
+    resources = [
+      "${aws_s3_bucket.openai.arn}/*",
+      "${aws_s3_bucket.openai.arn}"
+    ]
+  }
+}
+
+resource "aws_s3_bucket_policy" "openai_bucket_policy" {
+  bucket = aws_s3_bucket.openai.id
+  policy = data.aws_iam_policy_document.s3_bucket_policy.json
+}