From 38c09a05431bf90be9b7b9e595ed59ea1d5d901d Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 10 Jul 2024 09:20:30 +0200 Subject: [PATCH 1/9] integrate grafana dashboard for visualisation --- tests/performance/Dockerfile | 1 + tests/performance/Makefile | 24 ++++++++++++++----- tests/performance/docker-compose.yml | 18 ++++++++++---- .../locust_settings/_locust_settings.py | 19 +++++++++++---- 4 files changed, 46 insertions(+), 16 deletions(-) diff --git a/tests/performance/Dockerfile b/tests/performance/Dockerfile index 2addc8c9294..fab828ca6f4 100644 --- a/tests/performance/Dockerfile +++ b/tests/performance/Dockerfile @@ -7,6 +7,7 @@ RUN pip3 --version && \ pip3 install \ faker \ locust-plugins \ + locust-plugins[dashboards] \ pydantic \ pydantic-settings \ tenacity &&\ diff --git a/tests/performance/Makefile b/tests/performance/Makefile index 16086c0eea4..31d719dd858 100644 --- a/tests/performance/Makefile +++ b/tests/performance/Makefile @@ -6,6 +6,9 @@ include ../../scripts/common.Makefile LOCUST_VERSION=2.29.1 export LOCUST_VERSION +ENV_FILE=$(shell pwd)/.env +export ENV_FILE + # UTILS get_my_ip := $(shell (hostname --all-ip-addresses || hostname -i) 2>/dev/null | cut --delimiter=" " --fields=1) @@ -36,14 +39,23 @@ down: ## stops and removes osparc locust containers docker compose --file docker-compose.yml down .PHONY: test -test: ## runs osparc locust. locust and test confiuration are specified in .env file next to target file - @$(call check_defined, target, please define target file when calling $@ - e.g. ```make $@ target=MY_LOCUST_FILE.py```) - @export LOCUST_FILE=$(target); \ - export ENV_FILE=$$(realpath .env); \ +test: ## runs osparc locust. Locust and test configuration are specified in ENV_FILE if [ ! -f $${ENV_FILE} ]; then echo "You must generate a .env file before running tests!!! See the README..." && exit 1; fi; \ docker compose --file docker-compose.yml up --scale worker=4 --exit-code-from=master +.PHONY: install-ci install-dev install + +install-dev: install +install-ci: install -.PHONY: install-dev -install-dev: _check_venv_active ## install python tools +install: _check_venv_active ## install python tools @uv pip install -r requirements.txt + + +.PHONY: dashboards-up dashboards-down + +dashboards-up: ## Create Grafana dashboard for inspecting locust results. See dashboard on localhost:3000 + locust-compose up --exit-code-from=grafana + +dashboards-down: + locust-compose down diff --git a/tests/performance/docker-compose.yml b/tests/performance/docker-compose.yml index 7dfe2d4d5b7..8b47734933c 100644 --- a/tests/performance/docker-compose.yml +++ b/tests/performance/docker-compose.yml @@ -4,18 +4,26 @@ services: ports: - "8089:8089" volumes: - - ./locust_files:/mnt/locust_files - - ./locust_report:/mnt/locust_report + - ./locust_files:/home/locust/locust_files command: > - -f /mnt/${LOCUST_FILE} --html /mnt/locust_report/locust_html.html --master + --master env_file: - ${ENV_FILE} + networks: + - dashboards_timenet worker: image: itisfoundation/locust:${LOCUST_VERSION} volumes: - - ./locust_files/:/mnt/locust_files + - ./locust_files/:/home/locust/locust_files command: > - -f /mnt/${LOCUST_FILE} --worker --master-host master + --worker --master-host master env_file: - ${ENV_FILE} + networks: + - dashboards_timenet + +networks: + dashboards_timenet : + external: true + name: dashboards_timenet diff --git a/tests/performance/settings/locust_settings/_locust_settings.py b/tests/performance/settings/locust_settings/_locust_settings.py index 39cf05a3f3e..be31b8ffe46 100644 --- a/tests/performance/settings/locust_settings/_locust_settings.py +++ b/tests/performance/settings/locust_settings/_locust_settings.py @@ -3,11 +3,13 @@ # pylint: disable=no-name-in-module from datetime import timedelta +from pathlib import Path from parse import Result, parse from pydantic import ( AnyHttpUrl, Field, + NonNegativeInt, PositiveFloat, PositiveInt, SerializationInfo, @@ -22,16 +24,23 @@ class LocustSettings(BaseSettings): model_config = SettingsConfigDict(cli_parse_args=True) + LOCUST_CHECK_AVG_RESPONSE_TIME: PositiveInt = Field(default=200) + LOCUST_CHECK_FAIL_RATIO: PositiveFloat = Field(default=0.01, ge=0.0, le=1.0) + LOCUST_HEADLESS: bool = Field(default=True) LOCUST_HOST: AnyHttpUrl = Field( default=..., examples=["https://api.osparc-master.speag.com"] ) - LOCUST_USERS: PositiveInt = Field(default=...) - LOCUST_HEADLESS: bool = Field(default=True) + LOCUST_LOCUSTFILE: Path = Field(default=...) LOCUST_PRINT_STATS: bool = Field(default=True) - LOCUST_SPAWN_RATE: PositiveInt = Field(default=20) LOCUST_RUN_TIME: timedelta = Field(default=...) - LOCUST_CHECK_AVG_RESPONSE_TIME: PositiveInt = Field(default=200) - LOCUST_CHECK_FAIL_RATIO: PositiveFloat = Field(default=0.01, ge=0.0, le=1.0) + LOCUST_SPAWN_RATE: PositiveInt = Field(default=20) + LOCUST_TIMESCALE: NonNegativeInt = Field(default=1, ge=0, le=1) + LOCUST_USERS: PositiveInt = Field(default=...) + + PGHOST: str = Field(default="postgres") + PGPASSWORD: str = Field(default="password") + PGPORT: int = Field(default=5432) + PGUSER: str = Field(default="postgres") @field_validator("LOCUST_RUN_TIME", mode="before") @classmethod From 8123fda258158b0fdca8f9313ff0f4f7a5566a9d Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 10 Jul 2024 09:21:10 +0200 Subject: [PATCH 2/9] minor correction --- tests/performance/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/Makefile b/tests/performance/Makefile index 31d719dd858..6c89af79727 100644 --- a/tests/performance/Makefile +++ b/tests/performance/Makefile @@ -55,7 +55,7 @@ install: _check_venv_active ## install python tools .PHONY: dashboards-up dashboards-down dashboards-up: ## Create Grafana dashboard for inspecting locust results. See dashboard on localhost:3000 - locust-compose up --exit-code-from=grafana + locust-compose up dashboards-down: locust-compose down From 00fbe1069966b727d684c5d07719ab8c8cc1780b Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 10 Jul 2024 09:24:32 +0200 Subject: [PATCH 3/9] cosmetics --- tests/performance/Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/performance/Makefile b/tests/performance/Makefile index 6c89af79727..39089a59be7 100644 --- a/tests/performance/Makefile +++ b/tests/performance/Makefile @@ -55,7 +55,8 @@ install: _check_venv_active ## install python tools .PHONY: dashboards-up dashboards-down dashboards-up: ## Create Grafana dashboard for inspecting locust results. See dashboard on localhost:3000 - locust-compose up + @echo "View your dashboard on localhost:3000" + @locust-compose up dashboards-down: - locust-compose down + @locust-compose down From da7d73fd2bdf1751e97890859cb133c8516d4b8f Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 10 Jul 2024 09:25:18 +0200 Subject: [PATCH 4/9] remove report folder --- tests/performance/locust_report/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/performance/locust_report/.gitkeep diff --git a/tests/performance/locust_report/.gitkeep b/tests/performance/locust_report/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 From 510233aaabef5a9b43291fdc34bf598a4046083c Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 10 Jul 2024 09:42:38 +0200 Subject: [PATCH 5/9] add locust file to settings --- tests/performance/locust_files/catalog_services.py | 7 ++++++- tests/performance/locust_files/director_services.py | 7 ++++++- tests/performance/locust_files/metamodeling/workflow.py | 8 +++++++- tests/performance/locust_files/platform_ping_test.py | 7 ++++++- tests/performance/locust_files/user_basic_calls.py | 7 ++++++- tests/performance/locust_files/webserver_services.py | 7 ++++++- 6 files changed, 37 insertions(+), 6 deletions(-) diff --git a/tests/performance/locust_files/catalog_services.py b/tests/performance/locust_files/catalog_services.py index 20418496aac..e821f8499bc 100644 --- a/tests/performance/locust_files/catalog_services.py +++ b/tests/performance/locust_files/catalog_services.py @@ -3,6 +3,7 @@ # import logging +from pathlib import Path from time import time import faker @@ -60,4 +61,8 @@ def on_stop(self): if __name__ == "__main__": from locust_settings import LocustSettings, dump_dotenv - dump_dotenv(LocustSettings()) + dump_dotenv( + LocustSettings( + LOCUST_LOCUSTFILE=Path(__file__).relative_to(Path(__file__).parent.parent) + ) + ) diff --git a/tests/performance/locust_files/director_services.py b/tests/performance/locust_files/director_services.py index e5d96ada8ab..651eed429f0 100644 --- a/tests/performance/locust_files/director_services.py +++ b/tests/performance/locust_files/director_services.py @@ -3,6 +3,7 @@ # import logging +from pathlib import Path from locust import task from locust.contrib.fasthttp import FastHttpUser @@ -34,4 +35,8 @@ def on_stop(self): # pylint: disable=no-self-use if __name__ == "__main__": from locust_settings import LocustSettings, dump_dotenv - dump_dotenv(LocustSettings()) + dump_dotenv( + LocustSettings( + LOCUST_LOCUSTFILE=Path(__file__).relative_to(Path(__file__).parent.parent) + ) + ) diff --git a/tests/performance/locust_files/metamodeling/workflow.py b/tests/performance/locust_files/metamodeling/workflow.py index 995f9e9298c..87741c1a476 100644 --- a/tests/performance/locust_files/metamodeling/workflow.py +++ b/tests/performance/locust_files/metamodeling/workflow.py @@ -171,4 +171,10 @@ def upload_file(self, file: Path) -> UUID: class MetaModelingSettings(UserSettings, LocustSettings): pass - dump_dotenv(MetaModelingSettings()) + dump_dotenv( + MetaModelingSettings( + LOCUST_LOCUSTFILE=Path(__file__).relative_to( + Path(__file__).parent.parent.parent + ) + ) + ) diff --git a/tests/performance/locust_files/platform_ping_test.py b/tests/performance/locust_files/platform_ping_test.py index a07218582cb..8bef6b01127 100644 --- a/tests/performance/locust_files/platform_ping_test.py +++ b/tests/performance/locust_files/platform_ping_test.py @@ -3,6 +3,7 @@ # import logging +from pathlib import Path import locust_plugins from locust import task @@ -58,4 +59,8 @@ def on_stop(self): # pylint: disable=no-self-use class LoadTestSettings(LocustAuth, LocustSettings): pass - dump_dotenv(LoadTestSettings()) + dump_dotenv( + LoadTestSettings( + LOCUST_LOCUSTFILE=Path(__file__).relative_to(Path(__file__).parent.parent) + ) + ) diff --git a/tests/performance/locust_files/user_basic_calls.py b/tests/performance/locust_files/user_basic_calls.py index 51780c33a06..6434a8410fa 100644 --- a/tests/performance/locust_files/user_basic_calls.py +++ b/tests/performance/locust_files/user_basic_calls.py @@ -4,6 +4,7 @@ import logging import time +from pathlib import Path import faker from locust import task @@ -96,4 +97,8 @@ def on_stop(self): if __name__ == "__main__": from locust_settings import LocustSettings, dump_dotenv - dump_dotenv(LocustSettings()) + dump_dotenv( + LocustSettings( + LOCUST_LOCUSTFILE=Path(__file__).relative_to(Path(__file__).parent.parent) + ) + ) diff --git a/tests/performance/locust_files/webserver_services.py b/tests/performance/locust_files/webserver_services.py index 6655be64958..6e8c12551d4 100644 --- a/tests/performance/locust_files/webserver_services.py +++ b/tests/performance/locust_files/webserver_services.py @@ -3,6 +3,7 @@ # import logging +from pathlib import Path import faker from dotenv import load_dotenv @@ -54,4 +55,8 @@ def on_stop(self): if __name__ == "__main__": from locust_settings import LocustSettings, dump_dotenv - dump_dotenv(LocustSettings()) + dump_dotenv( + LocustSettings( + LOCUST_LOCUSTFILE=Path(__file__).relative_to(Path(__file__).parent.parent) + ) + ) From e4b10890b8b561f29e1024cc70f9f4acb8a590d6 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 10 Jul 2024 09:50:04 +0200 Subject: [PATCH 6/9] minor change --- tests/performance/locust_files/locustfile.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/performance/locust_files/locustfile.py b/tests/performance/locust_files/locustfile.py index 82d69e2b74c..00d8a05c407 100644 --- a/tests/performance/locust_files/locustfile.py +++ b/tests/performance/locust_files/locustfile.py @@ -3,6 +3,7 @@ # import logging +from pathlib import Path from uuid import UUID import faker @@ -82,4 +83,8 @@ def on_stop(self): class LoadTestSettings(TemplateSettings, LocustSettings): pass - dump_dotenv(LoadTestSettings()) + dump_dotenv( + LoadTestSettings( + LOCUST_LOCUSTFILE=Path(__file__).relative_to(Path(__file__).parent.parent) + ) + ) From f63522f15ced0e0986e3ca0b6d3210ffe9c674f9 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 10 Jul 2024 10:01:23 +0200 Subject: [PATCH 7/9] ensure docker network is created/deleted --- tests/performance/Makefile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/performance/Makefile b/tests/performance/Makefile index 39089a59be7..cf37636c2be 100644 --- a/tests/performance/Makefile +++ b/tests/performance/Makefile @@ -9,6 +9,8 @@ export LOCUST_VERSION ENV_FILE=$(shell pwd)/.env export ENV_FILE +NETWORK_NAME=dashboards_timenet + # UTILS get_my_ip := $(shell (hostname --all-ip-addresses || hostname -i) 2>/dev/null | cut --delimiter=" " --fields=1) @@ -40,7 +42,11 @@ down: ## stops and removes osparc locust containers .PHONY: test test: ## runs osparc locust. Locust and test configuration are specified in ENV_FILE - if [ ! -f $${ENV_FILE} ]; then echo "You must generate a .env file before running tests!!! See the README..." && exit 1; fi; \ + @if [ ! -f $${ENV_FILE} ]; then echo "You must generate a .env file before running tests!!! See the README..." && exit 1; fi; + @if ! docker network ls | grep -q $(NETWORK_NAME); then \ + docker network create $(NETWORK_NAME); \ + echo "Created docker network $(NETWORK_NAME)"; \ + fi docker compose --file docker-compose.yml up --scale worker=4 --exit-code-from=master .PHONY: install-ci install-dev install @@ -56,6 +62,10 @@ install: _check_venv_active ## install python tools dashboards-up: ## Create Grafana dashboard for inspecting locust results. See dashboard on localhost:3000 @echo "View your dashboard on localhost:3000" + @if docker network ls | grep -q $(NETWORK_NAME); then \ + docker network rm $(NETWORK_NAME); \ + echo "Removed docker network $(NETWORK_NAME)"; \ + fi @locust-compose up dashboards-down: From ea8da8294615c7ecdcdc3436a94e2da7fcf52525 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 10 Jul 2024 10:16:45 +0200 Subject: [PATCH 8/9] document setup --- tests/performance/README.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/performance/README.md b/tests/performance/README.md index ff504817111..a9c06f63ee6 100644 --- a/tests/performance/README.md +++ b/tests/performance/README.md @@ -8,12 +8,22 @@ In the [locust_files] folder are located the test files. ## Usage -1. Generate a `.env` file with setting for your test. After running `make install-dev` you can execute your test script in python and set settings as arguments. Once your settings are validated you pipe them to `.env`, e.g. +1. Generate a `.env` file with setting for your test. After running `make install-dev` you can execute your test script in python and set settings as arguments. Once your settings are validated you pipe them to `.env`. E.g. if your testscript is `locust_files/platform_ping_test.py` you could run ```bash python locust_files/platform_ping_test.py --LOCUST_HOST=https://api.osparc-master.speag.com \ --LOCUST_USERS=100 --LOCUST_RUN_TIME=0:10:00 --SC_USER_NAME=myname --SC_PASSWORD=mypassword > .env ``` -2. Run your test script using the Make `test` recipe, setting the correct `target`, e.g. +2. Run your test script using the Make `test` recipe, e.g. ``` -make test target=locust_files/platform_ping_test.py +make test ``` + +## Dashboards for visualization +- You can visualize the results of your tests (in real time) in a collection of beautiful [Grafana dashboards](https://github.com/SvenskaSpel/locust-plugins/tree/master/locust_plugins/dashboards). +- To do this, run `make dashboards-up` and go to `localhost:3000` to view the dashboards. The way you tell locust to send test results to the database/grafana is by ensuring `LOCUST_TIMESCALE=1` (see how to generate settings in [usage](#usage)) +- When you are done you run `make dashboards-down` to clean up. +- If you are using VPN you will need to forward port 300 to your local machine to view the dashboard. + + +## Tricky settings 🚨 +- `LOCUST_TIMESCALE` tells locust whether or not to send data to the database associated with visualizing the results. If you are not using the Grafana [dashboards](#dashboards-for-visualization) you should set `LOCUST_TIMESCALE=0`. From 9a79629d597afdd47d93e79a63549d7b4e88cf6f Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 10 Jul 2024 10:59:04 +0200 Subject: [PATCH 9/9] ensure html file is created --- tests/performance/docker-compose.yml | 3 ++- tests/performance/locust_report/.gitkeep | 0 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 tests/performance/locust_report/.gitkeep diff --git a/tests/performance/docker-compose.yml b/tests/performance/docker-compose.yml index 8b47734933c..683f73d5898 100644 --- a/tests/performance/docker-compose.yml +++ b/tests/performance/docker-compose.yml @@ -5,8 +5,9 @@ services: - "8089:8089" volumes: - ./locust_files:/home/locust/locust_files + - ./locust_report:/home/locust/locust_report command: > - --master + --master --html=locust_report/locust_html.html env_file: - ${ENV_FILE} networks: diff --git a/tests/performance/locust_report/.gitkeep b/tests/performance/locust_report/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d