diff --git a/.github/workflows/cortex-cpp-quality-gate.yml b/.github/workflows/cortex-cpp-quality-gate.yml index 2918840b6..39c5e7b42 100644 --- a/.github/workflows/cortex-cpp-quality-gate.yml +++ b/.github/workflows/cortex-cpp-quality-gate.yml @@ -247,6 +247,13 @@ jobs: cd engine make package + - name: Upload E2E Log + if: failure() + uses: actions/upload-artifact@v4 + with: + name: e2e-log-${{ matrix.os }}-${{ matrix.name }} + path: ./engine/e2e-test/logs + - name: Upload Artifact uses: actions/upload-artifact@v4 with: diff --git a/engine/e2e-test/api/files/blank.txt b/engine/e2e-test/api/files/blank.txt new file mode 100644 index 000000000..e69de29bb diff --git a/engine/e2e-test/api/files/test_api_create_file.py b/engine/e2e-test/api/files/test_api_create_file.py new file mode 100644 index 000000000..7c7226f50 --- /dev/null +++ b/engine/e2e-test/api/files/test_api_create_file.py @@ -0,0 +1,64 @@ +import pytest +import requests +from utils.test_runner import start_server, stop_server +import os +import platform +import jsonschema +from utils.logger import log_response +from utils.assertion import assert_equal +import fnmatch + + +class TestApiCreateFile: + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + # Setup + success = start_server() + if not success: + raise Exception("Failed to start server") + + yield + + # Teardown + stop_server() + + @pytest.mark.skipif(platform.system() != "Linux", reason="Todo: fix later on Mac and Window") + def test_api_create_file_successfully(self): + # Define file path + file_path_rel = os.path.join("e2e-test", "api", "files", "blank.txt") + file_path = os.path.join(os.getcwd(), file_path_rel) + log_response(file_path, "test_api_create_file_successfully") + + post_file_url = "http://127.0.0.1:3928/v1/files" + with open(file_path, "rb") as file: + files = {"file": ("blank.txt", file, "text/plain")} + data = {"purpose": "assistants"} + response = requests.post(post_file_url, files=files, data=data) + log_response(response.text, "test_api_create_file_successfully") + log_response(response.status_code, "test_api_create_file_successfully") + + json_data = response.json() + log_response(json_data, "test_api_create_file_successfully") + assert_equal(response.status_code, 200) + + # Schema to validate + schema = { + "type": "object", + "properties": { + "bytes": {"type": "integer"}, + "created_at": {"type": "integer"}, + "filename": {"type": "string"}, + "id": {"type": "string"}, + "object": {"type": "string"}, + "purpose": {"type": "string"} + }, + "required": ["bytes", "created_at", "filename", "id", "object", "purpose"] + } + + # Validate response schema + jsonschema.validate(instance=json_data, schema=schema) + + # Assert content + assert (fnmatch.fnmatch(json_data["filename"], "blank_*.txt") or json_data["filename"] == "blank.txt"), f"Filename {json_data['filename']} does not match pattern blank_*.txt or blank.txt" + assert_equal(json_data["purpose"], "assistants") \ No newline at end of file diff --git a/engine/e2e-test/api/files/test_api_delete_file.py b/engine/e2e-test/api/files/test_api_delete_file.py new file mode 100644 index 000000000..9cd651833 --- /dev/null +++ b/engine/e2e-test/api/files/test_api_delete_file.py @@ -0,0 +1,85 @@ +import pytest +import requests +from utils.test_runner import start_server, stop_server +import os +import jsonschema +from utils.logger import log_response +from utils.assertion import assert_equal +import platform + + +class TestApiDeleteFile: + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + # Setup + success = start_server() + if not success: + raise Exception("Failed to start server") + + yield + + # Teardown + stop_server() + + @pytest.mark.skipif(platform.system() != "Linux", reason="Todo: fix later on Mac and Window") + def test_api_del_file_successfully(self): + # Define file path + file_path = os.path.join("e2e-test", "api", "files", "blank.txt") + + # Upload file first + files = { + "file": ("blank.txt", open(file_path, "rb"), "text/plain") + } + data = { + "purpose": "assistants" + } + + file_url = "http://127.0.0.1:3928/v1/files" + response = requests.post(file_url, files=files, data=data) + + json_data = response.json() + log_response(json_data, "test_api_del_file_successfully") + assert_equal(response.status_code, 200) + + file_id=json_data["id"] + + # Delete message with id + file_id_url = f"http://127.0.0.1:3928/v1/files/{file_id}" + file_response = requests.delete(file_id_url) + json_data_file = file_response.json() + log_response(json_data_file, "test_api_del_file_successfully") + assert_equal(file_response.status_code,200) + + + # Schema to validate + schema = { + "properties": { + "deleted": { + "description": "Indicates if the file was successfully deleted", + "type": "boolean" + }, + "id": { + "description": "The ID of the deleted file", + "type": "string" + }, + "object": { + "description": "Type of object, always 'file'", + "type": "string" + } + }, + "required": [ + "deleted", + "id", + "object" + ], + "type": "object" + } + + # Validate response schema + jsonschema.validate(instance=json_data_file, schema=schema) + + # Assert content + assert_equal(json_data_file["deleted"], True) + assert_equal(json_data_file["id"], file_id) + assert_equal(json_data_file["object"], "file") \ No newline at end of file diff --git a/engine/e2e-test/api/files/test_api_get_file.py b/engine/e2e-test/api/files/test_api_get_file.py new file mode 100644 index 000000000..28ec38dda --- /dev/null +++ b/engine/e2e-test/api/files/test_api_get_file.py @@ -0,0 +1,105 @@ +import pytest +import requests +from utils.test_runner import start_server, stop_server +import platform +import os +import jsonschema +from utils.logger import log_response +from utils.assertion import assert_equal +import fnmatch + + +class TestApiGetFile: + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + # Setup + success = start_server() + if not success: + raise Exception("Failed to start server") + + yield + + # Teardown + stop_server() + + @pytest.mark.skipif(platform.system() != "Linux", reason="Todo: fix later on Mac and Window") + def test_api_get_file_successfully(self): + # Define file path + file_path = os.path.join("e2e-test", "api", "files", "blank.txt") + + # Upload file first + files = { + "file": ("blank.txt", open(file_path, "rb"), "text/plain") + } + data = { + "purpose": "assistants" + } + + file_url = "http://127.0.0.1:3928/v1/files" + response = requests.post(file_url, files=files, data=data) + log_response(response.text, "test_api_get_file_successfully") + + json_data = response.json() + log_response(json_data, "test_api_get_file_successfully") + assert_equal(response.status_code, 200) + + file_id=json_data["id"] + + # Get message with id + file_id_url = f"http://127.0.0.1:3928/v1/files/{file_id}" + file_response = requests.get(file_id_url) + json_data_file = file_response.json() + log_response(json_data_file, "test_api_get_file_successfully") + assert_equal(file_response.status_code,200) + + + # Schema to validate + schema = { + "properties": { + "bytes": { + "type": "integer", + "examples": [ + 3211109 + ] + }, + "created_at": { + "type": "integer", + "examples": [ + 1733942093 + ] + }, + "filename": { + "type": "string", + "examples": [ + "Enterprise_Application_Infrastructure_v2_20140903_toCTC_v1.0.pdf" + ] + }, + "id": { + "type": "string", + "examples": [ + "file-0001KNKPTDDAQSDVEQGRBTCTNJ" + ] + }, + "object": { + "type": "string", + "examples": [ + "file" + ] + }, + "purpose": { + "type": "string", + "examples": [ + "assistants" + ] + } + }, + "type": "object" + } + + # Validate response schema + jsonschema.validate(instance=json_data_file, schema=schema) + + # Assert content + assert (fnmatch.fnmatch(json_data["filename"], "blank_*.txt") or json_data["filename"] == "blank.txt"), f"Filename {json_data['filename']} does not match pattern blank_*.txt or blank.txt" + assert_equal(json_data_file["id"], file_id) \ No newline at end of file diff --git a/engine/e2e-test/api/files/test_api_get_list_file.py b/engine/e2e-test/api/files/test_api_get_list_file.py new file mode 100644 index 000000000..151a17837 --- /dev/null +++ b/engine/e2e-test/api/files/test_api_get_list_file.py @@ -0,0 +1,115 @@ +import pytest +import requests +from utils.test_runner import start_server, stop_server +import os +import platform +import jsonschema +from utils.logger import log_response +from utils.assertion import assert_equal +import fnmatch + + +class TestApiGetListFile: + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + # Setup + success = start_server() + if not success: + raise Exception("Failed to start server") + + yield + + # Teardown + stop_server() + + @pytest.mark.skipif(platform.system() != "Linux", reason="Todo: fix later on Mac and Window") + def test_api_get_list_file_successfully(self): + # Define file path + file_path = os.path.join("e2e-test", "api", "files", "blank.txt") + + # Upload file first + files = { + "file": ("blank.txt", open(file_path, "rb"), "text/plain") + } + data = { + "purpose": "assistants" + } + + file_url = "http://127.0.0.1:3928/v1/files" + response = requests.post(file_url, files=files, data=data) + log_response(response.text, "test_api_get_list_file_successfully") + + json_data = response.json() + log_response(json_data, "test_api_get_list_file_successfully") + assert_equal(response.status_code, 200) + + # Get list message + list_file_response = requests.get(file_url) + json_data_list_file = list_file_response.json() + log_response(json_data_list_file, "test_api_get_list_file_successfully") + assert_equal(list_file_response.status_code,200) + + + # Schema to validate + schema = { + "properties": { + "data": { + "items": { + "properties": { + "bytes": { + "type": "integer", + "examples": [ + 3211109 + ] + }, + "created_at": { + "type": "integer", + "examples": [ + 1733942093 + ] + }, + "filename": { + "type": "string", + "examples": [ + "Enterprise_Application_Infrastructure_v2_20140903_toCTC_v1.0.pdf" + ] + }, + "id": { + "type": "string", + "examples": [ + "file-0001KNKPTDDAQSDVEQGRBTCTNJ" + ] + }, + "object": { + "type": "string", + "examples": [ + "file" + ] + }, + "purpose": { + "type": "string", + "examples": [ + "assistants" + ] + } + }, + "type": "object" + }, + "type": "array" + }, + "object": { + "type": "string", + "examples": [ + "list" + ] + } + }, + "type": "object" + } + + # Validate response schema + jsonschema.validate(instance=json_data_list_file, schema=schema) + + # Assert content + assert (fnmatch.fnmatch(json_data["filename"], "blank_*.txt") or json_data["filename"] == "blank.txt"), f"Filename {json_data['filename']} does not match pattern blank_*.txt or blank.txt" \ No newline at end of file diff --git a/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py b/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py index 3afbbae9f..e78baf951 100644 --- a/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py +++ b/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py @@ -24,6 +24,10 @@ from test_api_post_default_engine import TestApiSetDefaultEngine from api.model.test_api_model import TestApiModel from api.model.test_api_model_import import TestApiModelImport +from api.files.test_api_create_file import TestApiCreateFile +from api.files.test_api_get_file import TestApiGetFile +from api.files.test_api_get_list_file import TestApiGetListFile +from api.files.test_api_delete_file import TestApiDeleteFile from api.message.test_api_get_message import TestApiGetMessage from api.message.test_api_get_list_message import TestApiGetListMessage from api.message.test_api_create_message import TestApiCreateMessage diff --git a/engine/e2e-test/runner/main.py b/engine/e2e-test/runner/main.py index c80c96972..3327625d2 100644 --- a/engine/e2e-test/runner/main.py +++ b/engine/e2e-test/runner/main.py @@ -24,6 +24,10 @@ from test_api_post_default_engine import TestApiSetDefaultEngine from api.model.test_api_model import TestApiModel from api.model.test_api_model_import import TestApiModelImport +from api.files.test_api_create_file import TestApiCreateFile +from api.files.test_api_get_file import TestApiGetFile +from api.files.test_api_get_list_file import TestApiGetListFile +from api.files.test_api_delete_file import TestApiDeleteFile from api.message.test_api_get_message import TestApiGetMessage from api.message.test_api_get_list_message import TestApiGetListMessage from api.message.test_api_create_message import TestApiCreateMessage