feat: video record and upload for standalone and dynamic grid (#2362)

* test: add test for video uploader with RCLONE

* Update Video/video_gridUrl.sh

---------

Signed-off-by: Viet Nguyen Duc <[email protected]>

Author: VietND96

.circleci/config.yml

@@ -81,6 +81,12 @@ workflows:
           use-random-user: false
           platforms: linux/arm64
           machine-type: ubuntu2204arm64
+      - docker-test:
+          name: "Docker test - Video recording standalone"
+          test-strategy: test_video_standalone
+          use-random-user: false
+          platforms: linux/arm64
+          machine-type: ubuntu2204arm64
       - docker-test:
           name: "Docker test - Dynamic Grid"
           test-strategy: test_node_docker

.github/workflows/docker-test.yml

@@ -50,6 +50,10 @@ jobs:
             use-random-user: false
             test-video: true
             build-all: false
+          - test-strategy: test_video_standalone
+            use-random-user: false
+            test-video: true
+            build-all: false
           - test-strategy: test_node_docker
             use-random-user: false
             test-video: true

Makefile

@@ -67,12 +67,14 @@ build: all
 
 ci: build test
 
-gen_certs:
+prepare_resources:
 	rm -rf ./Base/configs/node && mkdir -p ./Base/configs/node && cp -r ./charts/selenium-grid/configs/node ./Base/configs
+
+gen_certs:
 	rm -rf ./Base/certs && cp -r ./charts/selenium-grid/certs ./Base
 	./Base/certs/gen-cert-helper.sh -d ./Base/certs
 
-base: gen_certs
+base: prepare_resources gen_certs
 	cd ./Base && docker buildx build --platform $(PLATFORMS) $(BUILD_ARGS) --build-arg VERSION=$(BASE_VERSION) --build-arg RELEASE=$(BASE_RELEASE) --build-arg AUTHORS=$(AUTHORS) -t $(NAME)/base:$(TAG_VERSION) .
 
 base_nightly:
@@ -580,24 +582,31 @@ test_parallel: hub chrome firefox edge chromium
 			echo SELENIUM_GRID_PROTOCOL=https >> .env ; \
 			echo CHART_CERT_PATH=$$(readlink -f ./videos/certs/tls.crt) >> .env ; \
 			export $$(cat .env | xargs) ; \
-			DOCKER_DEFAULT_PLATFORM=$(PLATFORMS) docker compose --profile $(PLATFORMS) -f docker-compose-v3-test-parallel.yml up -d --no-log-prefix ; \
+			DOCKER_DEFAULT_PLATFORM=$(PLATFORMS) docker compose --profile $(PLATFORMS) -f docker-compose-v3-test-parallel.yml up -d --remove-orphans --no-log-prefix ; \
 			RUN_IN_DOCKER_COMPOSE=true bash ./bootstrap.sh $$node ; \
 	done ; \
 	docker compose -f docker-compose-v3-test-parallel.yml down
 
+test_video_standalone: standalone_chrome standalone_chromium standalone_firefox standalone_edge
+	DOCKER_COMPOSE_FILE=docker-compose-v3-test-standalone.yml make test_video
+
 test_video_dynamic_name:
-	VIDEO_FILE_NAME=auto TEST_DELAY_AFTER_TEST=0 \
+	VIDEO_FILE_NAME=auto \
 	make test_video
 
 # This should run on its own CI job. There is no need to combine it with the other tests.
 # Its main purpose is to check that a video file was generated.
 test_video: video hub chrome firefox edge chromium
 	sudo rm -rf ./tests/tests
-	sudo rm -rf ./tests/videos; mkdir -p ./tests/videos
+	sudo rm -rf ./tests/videos; mkdir -p ./tests/videos/upload
+	sudo chmod -R 777 ./tests/videos
+	docker_compose_file=$(or $(DOCKER_COMPOSE_FILE), docker-compose-v3-test-video.yml) ; \
+	list_of_tests_amd64=$(or $(LIST_OF_TESTS_AMD64), "NodeChrome NodeChromium NodeFirefox NodeEdge") ; \
+	list_of_tests_arm64=$(or $(LIST_OF_TESTS_ARM64), "NodeChromium NodeFirefox") ; \
 	if [ "$(PLATFORMS)" = "linux/amd64" ]; then \
-			list_nodes="NodeChrome NodeChromium NodeFirefox NodeEdge" ; \
+			list_nodes="$${list_of_tests_amd64}" ; \
 	else \
-			list_nodes="NodeChromium NodeFirefox" ; \
+			list_nodes="${list_of_tests_arm64}" ; \
 	fi; \
 	for node in $${list_nodes}; do \
 			cd ./tests || true ; \
@@ -607,6 +616,10 @@ test_video: video hub chrome firefox edge chromium
 			echo UID=$$(id -u) >> .env ; \
 			echo BINDING_VERSION=$(BINDING_VERSION) >> .env ; \
 			echo TEST_DELAY_AFTER_TEST=$(or $(TEST_DELAY_AFTER_TEST), 0) >> .env ; \
+			echo SELENIUM_ENABLE_MANAGED_DOWNLOADS=$(or $(SELENIUM_ENABLE_MANAGED_DOWNLOADS), "true") >> .env ; \
+			echo BASIC_AUTH_USERNAME=$(or $(BASIC_AUTH_USERNAME), "admin") >> .env ; \
+			echo BASIC_AUTH_PASSWORD=$(or $(BASIC_AUTH_PASSWORD), "admin") >> .env ; \
+			echo SUB_PATH=$(or $(SUB_PATH), "/selenium") >> .env ; \
 			if [ $$node = "NodeChrome" ] ; then \
 					echo BROWSER=chrome >> .env ; \
 					echo VIDEO_FILE_NAME=$${VIDEO_FILE_NAME:-"chrome_video.mp4"} >> .env ; \
@@ -627,7 +640,7 @@ test_video: video hub chrome firefox edge chromium
 					echo VIDEO_FILE_NAME=$${VIDEO_FILE_NAME:-"firefox_video.mp4"} >> .env ; \
 					echo VIDEO_FILE_NAME_SUFFIX=$${VIDEO_FILE_NAME_SUFFIX:-"true"} >> .env ; \
 			fi ; \
-			DOCKER_DEFAULT_PLATFORM=$(PLATFORMS) docker compose -f docker-compose-v3-test-video.yml up --abort-on-container-exit ; \
+			DOCKER_DEFAULT_PLATFORM=$(PLATFORMS) docker compose -f $${docker_compose_file} up --remove-orphans --build  --exit-code-from tests ; \
 	done
 	make test_video_integrity
 
@@ -676,18 +689,27 @@ test_node_relay: hub node_base standalone_firefox
 			fi ; \
 			export $$(cat .env | xargs) ; \
 			envsubst < relay_config.toml > ./videos/relay_config.toml ; \
-			DOCKER_DEFAULT_PLATFORM=$(PLATFORMS) docker compose --profile $$node -f docker-compose-v3-test-node-relay.yml up --no-log-prefix --exit-code-from tests ; \
+			DOCKER_DEFAULT_PLATFORM=$(PLATFORMS) docker compose --profile $$node -f docker-compose-v3-test-node-relay.yml up --remove-orphans --no-log-prefix --build --exit-code-from tests ; \
 			if [ $$? -ne 0 ]; then exit 1; fi ; \
 	done
 
+test_standalone_docker: standalone_docker
+	DOCKER_COMPOSE_FILE=docker-compose-v3-test-standalone-docker.yaml CONFIG_FILE=standalone_docker_config.toml \
+	RECORD_STANDALONE=true GRID_URL=http://0.0.0.0:4444 LIST_OF_TESTS_AMD64="DeploymentAutoscaling" TEST_PARALLEL_HARDENING=true \
+	SELENIUM_ENABLE_MANAGED_DOWNLOADS=true LOG_LEVEL=SEVERE SKIP_CHECK_DOWNLOADS_VOLUME=true make test_node_docker
+
 test_node_docker: hub standalone_docker standalone_chrome standalone_firefox standalone_edge standalone_chromium video
 	sudo rm -rf ./tests/tests
-	sudo rm -rf ./tests/videos; mkdir -p ./tests/videos/Downloads
+	sudo rm -rf ./tests/videos; mkdir -p ./tests/videos/Downloads; mkdir -p ./tests/videos/upload
 	sudo chmod -R 777 ./tests/videos
+	docker_compose_file=$(or $(DOCKER_COMPOSE_FILE), docker-compose-v3-test-node-docker.yaml) ; \
+	config_file=$(or $(CONFIG_FILE), config.toml) ; \
+	list_of_tests_amd64=$(or $(LIST_OF_TESTS_AMD64), "NodeChrome NodeChromium NodeFirefox NodeEdge") ; \
+	list_of_tests_arm64=$(or $(LIST_OF_TESTS_ARM64), "NodeChromium NodeFirefox") ; \
 	if [ "$(PLATFORMS)" = "linux/amd64" ]; then \
-			list_nodes="NodeChrome NodeChromium NodeFirefox NodeEdge" ; \
+			list_nodes="$${list_of_tests_amd64}" ; \
 	else \
-			list_nodes="NodeChromium NodeFirefox" ; \
+			list_nodes="$${list_of_tests_arm64}" ; \
 	fi; \
 	for node in $${list_nodes} ; do \
 			cd tests || true ; \
@@ -702,6 +724,8 @@ test_node_docker: hub standalone_docker standalone_chrome standalone_firefox sta
 			echo REQUEST_TIMEOUT=$(or $(REQUEST_TIMEOUT), 300) >> .env ; \
 			echo SELENIUM_ENABLE_MANAGED_DOWNLOADS=$(or $(SELENIUM_ENABLE_MANAGED_DOWNLOADS), "false") >> .env ; \
 			echo TEST_DELAY_AFTER_TEST=$(or $(TEST_DELAY_AFTER_TEST), 0) >> .env ; \
+			echo RECORD_STANDALONE=$(or $(RECORD_STANDALONE), "true") >> .env ; \
+			echo GRID_URL=$(or $(GRID_URL), "") >> .env ; \
 			echo NODE=$$node >> .env ; \
 			echo UID=$$(id -u) >> .env ; \
 			echo BINDING_VERSION=$(BINDING_VERSION) >> .env ; \
@@ -716,64 +740,20 @@ test_node_docker: hub standalone_docker standalone_chrome standalone_firefox sta
 			fi ; \
 			if [ $$node = "NodeChromium" ] ; then \
 					echo NODE_CHROME=chromium >> .env ; \
+			else \
+					echo NODE_CHROME=chromium >> .env ; \
 			fi ; \
 			export $$(cat .env | xargs) ; \
-			envsubst < config.toml > ./videos/config.toml ; \
-			DOCKER_DEFAULT_PLATFORM=$(PLATFORMS) docker compose -f docker-compose-v3-test-node-docker.yaml up --no-log-prefix --exit-code-from tests ; \
+			envsubst < $${config_file} > ./videos/config.toml ; \
+			DOCKER_DEFAULT_PLATFORM=$(PLATFORMS) docker compose -f $${docker_compose_file} up --remove-orphans --no-log-prefix --build --exit-code-from tests ; \
 			if [ $$? -ne 0 ]; then exit 1; fi ; \
-			if [ -d "$$DOWNLOADS_DIR" ] && [ $$(ls -1q $$DOWNLOADS_DIR | wc -l) -eq 0 ]; then \
+			if [ "$$SKIP_CHECK_DOWNLOADS_VOLUME" != "true" ] && [ -d "$$DOWNLOADS_DIR" ] && [ $$(ls -1q $$DOWNLOADS_DIR | wc -l) -eq 0 ]; then \
 					echo "Mounted downloads directory is empty. Downloaded files could not be retrieved!" ; \
 					exit 1 ; \
 			fi ; \
 	done
 	make test_video_integrity
 
-test_standalone_docker: standalone_docker standalone_chrome standalone_firefox standalone_edge standalone_chromium video
-	sudo rm -rf ./tests/tests
-	sudo rm -rf ./tests/videos; mkdir -p ./tests/videos/Downloads
-	sudo chmod -R 777 ./tests/videos
-	if [ "$(PLATFORMS)" = "linux/amd64" ]; then \
-			list_nodes="DeploymentAutoscaling" ; \
-	else \
-			list_nodes="NodeChromium NodeFirefox" ; \
-	fi; \
-	for node in $${list_nodes} ; do \
-			cd tests || true ; \
-			DOWNLOADS_DIR="./videos/Downloads" ; \
-			sudo rm -rf $$DOWNLOADS_DIR/* ; \
-			echo NAMESPACE=$(NAME) > .env ; \
-			echo TAG=$(TAG_VERSION) >> .env ; \
-			echo VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) >> .env ; \
-			echo TEST_DRAIN_AFTER_SESSION_COUNT=$(or $(TEST_DRAIN_AFTER_SESSION_COUNT), 0) >> .env ; \
-			echo TEST_PARALLEL_HARDENING=$(or $(TEST_PARALLEL_HARDENING), "true") >> .env ; \
-			echo LOG_LEVEL=$(or $(LOG_LEVEL), "INFO") >> .env ; \
-			echo REQUEST_TIMEOUT=$(or $(REQUEST_TIMEOUT), 300) >> .env ; \
-			echo SELENIUM_ENABLE_MANAGED_DOWNLOADS=$(or $(SELENIUM_ENABLE_MANAGED_DOWNLOADS), "true") >> .env ; \
-			echo TEST_DELAY_AFTER_TEST=$(or $(TEST_DELAY_AFTER_TEST), 0) >> .env ; \
-			echo NODE=$$node >> .env ; \
-			echo UID=$$(id -u) >> .env ; \
-			echo BINDING_VERSION=$(BINDING_VERSION) >> .env ; \
-			echo HOST_IP=$$(hostname -I | awk '{print $$1}') >> .env ; \
-			if [ "$(PLATFORMS)" = "linux/amd64" ]; then \
-					echo NODE_EDGE=edge >> .env ; \
-			else \
-					echo NODE_EDGE=chromium >> .env ; \
-			fi; \
-			if [ $$node = "NodeChrome" ] ; then \
-					echo NODE_CHROME=chrome >> .env ; \
-			fi ; \
-			if [ $$node = "NodeChromium" ] ; then \
-					echo NODE_CHROME=chromium >> .env ; \
-			else \
-					echo NODE_CHROME=chromium >> .env ; \
-			fi ; \
-			export $$(cat .env | xargs) ; \
-			envsubst < standalone_docker_config.toml > ./videos/config.toml ; \
-			DOCKER_DEFAULT_PLATFORM=$(PLATFORMS) docker compose -f docker-compose-v3-test-standalone-docker.yaml up --no-log-prefix --build --exit-code-from tests ; \
-			if [ $$? -ne 0 ]; then exit 1; fi ; \
-	done
-	make test_video_integrity
-
 test_custom_ca_cert:
 	VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/customCACert/bootstrap.sh
 
@@ -800,7 +780,7 @@ test_video_integrity:
 	# Using ffmpeg to verify file integrity
 	# https://superuser.com/questions/100288/how-can-i-check-the-integrity-of-a-video-file-avi-mpeg-mp4
 	list_files=$$(find ./tests/videos -type f -name "*.mp4"); \
-	echo "Number of video files: $$(echo $$list_files | wc -w)"; \
+	echo "::warning:: Number of video files: $$(echo $$list_files | wc -w)"; \
 	number_corrupted_files=0; \
 	if [ -z "$$list_files" ]; then \
 		echo "No video files found"; \
@@ -827,8 +807,8 @@ chart_render_template:
 	RENDER_HELM_TEMPLATE_ONLY=true make chart_test_autoscaling_disabled chart_test_autoscaling_deployment_https chart_test_autoscaling_deployment chart_test_autoscaling_job_https chart_test_autoscaling_job_hostname chart_test_autoscaling_job
 
 chart_test_autoscaling_disabled:
-	PLATFORMS=$(PLATFORMS) TEST_CHROMIUM=true RELEASE_NAME=selenium SELENIUM_GRID_AUTOSCALING=false TEST_DELAY_AFTER_TEST=0 CHART_ENABLE_TRACING=true \
-	SECURE_INGRESS_ONLY_GENERATE=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=$$(hostname -i) SELENIUM_GRID_PORT=443 \
+	PLATFORMS=$(PLATFORMS) TEST_CHROMIUM=true RELEASE_NAME=selenium SELENIUM_GRID_AUTOSCALING=false CHART_ENABLE_TRACING=true \
+	SECURE_INGRESS_ONLY_GENERATE=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=$$(hostname -i) SELENIUM_GRID_PORT=443 EXTERNAL_UPLOADER_CONFIG=true \
 	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) \
 	TEMPLATE_OUTPUT_FILENAME="k8s_nodeChromium_enableTracing_secureIngress_generateCerts_ingressPublicIP_subPath.yaml" \
 	./tests/charts/make/chart_test.sh NoAutoscaling
@@ -852,7 +832,7 @@ chart_test_autoscaling_deployment:
 chart_test_autoscaling_job_https:
 	PLATFORMS=$(PLATFORMS) TEST_EXISTING_KEDA=true RELEASE_NAME=selenium CHART_ENABLE_BASIC_AUTH=true \
 	SECURE_CONNECTION_SERVER=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 SUB_PATH=/ \
-	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) \
+	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) EXTERNAL_UPLOADER_CONFIG=true \
 	TEMPLATE_OUTPUT_FILENAME="k8s_prefixSelenium_basicAuth_secureServer_autoScaling_scaledJob_existingKEDA.yaml" \
 	./tests/charts/make/chart_test.sh JobAutoscaling
 

NodeFirefox/Dockerfile

@@ -31,7 +31,7 @@ RUN if [ "$(dpkg --print-architecture)" = "amd64" ]; then \
 # GeckoDriver
 #============
 ARG GECKODRIVER_VERSION=latest
-RUN LATEST_VERSION=$(curl -s https://api.github.com/repos/mozilla/geckodriver/releases/latest | jq -r '.tag_name') \
+RUN LATEST_VERSION=$(curl -sk https://api.github.com/repos/mozilla/geckodriver/releases/latest | jq -r '.tag_name') \
   && DRIVER_ARCH=$(if [ "$(dpkg --print-architecture)" = "amd64" ]; then echo "linux64"; else echo "linux-aarch64"; fi) \
   && GK_VERSION=$(if [ ${GECKODRIVER_VERSION:-latest} = "latest" ]; then echo "${LATEST_VERSION}"; else echo $GECKODRIVER_VERSION; fi) \
   && echo "Using GeckoDriver version: "$GK_VERSION \

README.md

@@ -685,16 +685,30 @@ services:
 ```
 
 `SE_VIDEO_FILE_NAME=auto` will use the session id as the video file name. This ensures that the video file name is unique to upload.
+Video file name construction automatically works based on Node endpoint `/status` (and optional GraphQL endpoint) to get session ID, capabilities.
+
+|                                          | Hub/Nodes         | Standalone roles | Dynamic Grid   |
+|------------------------------------------|-------------------|------------------|----------------|
+| `SE_VIDEO_RECORD_STANDALONE` (mandatory) | `false` (default) | `true`           | user input     |
+| `DISPLAY_CONTAINER_NAME` (mandatory)     | user input        | user input       | (not required) |
+| `SE_NODE_PORT` (optional)                | `5555`            | `4444`           | (not required) |
+| `SE_NODE_GRID_URL` (optional)            | user input        | (not required)   | (not required) |
 
 `SE_VIDEO_UPLOAD_ENABLED=true` will enable the video upload feature. In the background, it will create a pipefile with file and destination for uploader to consume and proceed.
 
 `SE_VIDEO_INTERNAL_UPLOAD=true` will use RCLONE installed in the container for upload. If you want to use another container for upload, set it to `false`.
 
 For environment variables with prefix `RCLONE_` is used to pass remote configuration to RCLONE. You can find more information about RCLONE configuration [here](https://rclone.org/docs/).
+When using in Dynamic Grid, those variables should be combined with the prefix `SE_`, for example `SE_RCLONE_`. See below reference for more details.
+
+### Reference
+- Configure video recording and uploading for Hub and Nodes: [docker-compose-v3-video-upload.yml](docker-compose-v3-video-upload.yml)
+
+- Configure video recording and uploading for Standalone roles: [docker-compose-v3-video-upload-standalone.yml](docker-compose-v3-video-upload-standalone.yml)
 
-[`docker-compose-v3-video-upload.yml`](docker-compose-v3-video-upload.yml)
+- Configure video recording and uploading for Dynamic Grid (node-docker): [docker-compose-v3-video-upload-dynamic-grid.yml](docker-compose-v3-video-upload-dynamic-grid.yml)
 
-Note that upload function is not supported for Dynamic Grid. If you want it, please create a feature request.
+- Configure video recording and uploading for Dynamic Grid standalone (standalone-docker): [tests/docker-compose-v3-test-standalone-docker.yaml](tests/docker-compose-v3-test-standalone-docker.yaml)
 
 ___
 

Video/Dockerfile

@@ -72,7 +72,7 @@ RUN groupadd ${SEL_GROUP} \
 # Add Supervisor configuration files
 #======================================
 COPY supervisord.conf /etc
-COPY --chown="${SEL_UID}:${SEL_GID}" entry_point.sh video.sh video_ready.py video_graphQLQuery.sh /opt/bin/
+COPY --chown="${SEL_UID}:${SEL_GID}" entry_point.sh video.sh video_ready.py video_graphQLQuery.sh video_gridUrl.sh /opt/bin/
 
 #======================================
 # Add RCLONE for uploading videos

Video/supervisord.conf

@@ -28,6 +28,8 @@ stdout_logfile_maxbytes=0
 [program:video-ready]
 priority=5
 command=python3 /opt/bin/video_ready.py
+stopasgroup = true
+killasgroup=true
 autostart=true
 autorestart=true
 
@@ -38,7 +40,9 @@ stdout_logfile_maxbytes=0
 
 [program:video-upload]
 priority=10
-command=bash -c "if [ ${SE_VIDEO_INTERNAL_UPLOAD} = "true" ]; then /opt/bin/upload.sh; fi"
+command=/opt/bin/upload.sh
+stopasgroup = true
+killasgroup=true
 autostart=%(ENV_SE_VIDEO_INTERNAL_UPLOAD)s
 autorestart=%(ENV_SE_VIDEO_INTERNAL_UPLOAD)s