Skip to content

Commit 6f25d80

Browse files
authored
K8s: Add default values for multiple nodes platform and version (#2543)
1 parent 02ea1a7 commit 6f25d80

File tree

9 files changed

+263
-47
lines changed

9 files changed

+263
-47
lines changed

Diff for: Makefile

+5-5
Original file line numberDiff line numberDiff line change
@@ -948,15 +948,15 @@ chart_test_autoscaling_deployment:
948948
./tests/charts/make/chart_test.sh DeploymentAutoscaling
949949

950950
chart_test_autoscaling_job_https:
951-
PLATFORMS=$(PLATFORMS) TEST_EXISTING_KEDA=true RELEASE_NAME=selenium CHART_ENABLE_BASIC_AUTH=true SELENIUM_GRID_MONITORING=false TEST_PATCHED_KEDA=$(TEST_PATCHED_KEDA) \
951+
PLATFORMS=$(PLATFORMS) TEST_EXISTING_KEDA=true RELEASE_NAME=selenium CHART_ENABLE_BASIC_AUTH=true SELENIUM_GRID_MONITORING=false TEST_PATCHED_KEDA=$(TEST_PATCHED_KEDA) TEST_MULTIPLE_PLATFORMS=true \
952952
SECURE_CONNECTION_SERVER=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 SUB_PATH=/ \
953953
MAX_SESSIONS_FIREFOX=1 MAX_SESSIONS_EDGE=2 MAX_SESSIONS_CHROME=3 TEST_NAME_OVERRIDE=true \
954954
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) EXTERNAL_UPLOADER_CONFIG=true \
955955
TEMPLATE_OUTPUT_FILENAME="k8s_prefixSelenium_basicAuth_secureServer_autoScaling_scaledJob_existingKEDA.yaml" \
956956
./tests/charts/make/chart_test.sh JobAutoscaling
957957

958958
chart_test_autoscaling_job_hostname:
959-
PLATFORMS=$(PLATFORMS) CHART_ENABLE_TRACING=true CHART_ENABLE_BASIC_AUTH=true BASIC_AUTH_EMBEDDED_URL=true TEST_PATCHED_KEDA=$(TEST_PATCHED_KEDA) \
959+
PLATFORMS=$(PLATFORMS) CHART_ENABLE_TRACING=true CHART_ENABLE_BASIC_AUTH=true BASIC_AUTH_EMBEDDED_URL=true TEST_PATCHED_KEDA=$(TEST_PATCHED_KEDA) TEST_MULTIPLE_PLATFORMS=true \
960960
SECURE_INGRESS_ONLY_DEFAULT=true SECURE_USE_EXTERNAL_CERT=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=$$(hostname -I | cut -d' ' -f1) SELENIUM_GRID_PORT=443 \
961961
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) \
962962
TEMPLATE_OUTPUT_FILENAME="k8s_enableTracing_basicAuth_secureIngress_externalCerts_ingressPublicIP_autoScaling_originKEDA_scaledJob_subPath.yaml" \
@@ -977,7 +977,7 @@ chart_test_autoscaling_job:
977977
./tests/charts/make/chart_test.sh JobAutoscaling
978978

979979
chart_test_autoscaling_playwright_connect_grid:
980-
PLATFORMS=$(PLATFORMS) CHART_FULL_DISTRIBUTED_MODE=true CHART_ENABLE_BASIC_AUTH=true TEST_EXTERNAL_DATASTORE=redis MATRIX_TESTS=CDPTests TEST_PATCHED_KEDA=$(TEST_PATCHED_KEDA) \
980+
PLATFORMS=$(PLATFORMS) CHART_FULL_DISTRIBUTED_MODE=true CHART_ENABLE_BASIC_AUTH=true TEST_EXTERNAL_DATASTORE=redis MATRIX_TESTS=CDPTests TEST_PATCHED_KEDA=$(TEST_PATCHED_KEDA) TEST_MULTIPLE_PLATFORMS=true \
981981
BASIC_AUTH_USERNAME=docker-selenium BASIC_AUTH_PASSWORD=2NMI4jdBi6k7bENoeUfV25295VvzwAE9chM24a+2VL95uOHozo \
982982
SECURE_INGRESS_ONLY_DEFAULT=true SECURE_USE_EXTERNAL_CERT=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=$$(hostname -I | cut -d' ' -f1) SELENIUM_GRID_PORT=443 \
983983
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) \
@@ -993,7 +993,7 @@ test_k8s_autoscaling_job_count_strategy_default_with_node_max_sessions:
993993
make test_k8s_autoscaling_job_count_strategy_default
994994

995995
test_k8s_autoscaling_job_count_strategy_default:
996-
MATRIX_TESTS=$(or $(MATRIX_TESTS), "AutoscalingTestsScaleUp") SCALING_STRATEGY=$(or $(SCALING_STRATEGY), "default") \
996+
MATRIX_TESTS=$(or $(MATRIX_TESTS), "AutoscalingTestsScaleUp") SCALING_STRATEGY=$(or $(SCALING_STRATEGY), "default") TEST_MULTIPLE_PLATFORMS=true \
997997
PLATFORMS=$(PLATFORMS) RELEASE_NAME=selenium TEST_PATCHED_KEDA=$(TEST_PATCHED_KEDA) SELENIUM_GRID_PROTOCOL=http SELENIUM_GRID_HOST=localhost SELENIUM_GRID_PORT=80 \
998998
SELENIUM_GRID_MONITORING=false CLEAR_POD_HISTORY=true SET_MAX_REPLICAS=100 ENABLE_VIDEO_RECORDER=false \
999999
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) \
@@ -1008,7 +1008,7 @@ test_k8s_autoscaling_deployment_count_with_node_max_sessions:
10081008
make test_k8s_autoscaling_deployment_count
10091009

10101010
test_k8s_autoscaling_deployment_count:
1011-
MATRIX_TESTS=$(or $(MATRIX_TESTS), "AutoscalingTestsScaleUp") \
1011+
MATRIX_TESTS=$(or $(MATRIX_TESTS), "AutoscalingTestsScaleUp") TEST_MULTIPLE_PLATFORMS=true \
10121012
PLATFORMS=$(PLATFORMS) RELEASE_NAME=selenium TEST_PATCHED_KEDA=$(TEST_PATCHED_KEDA) SELENIUM_GRID_PROTOCOL=http SELENIUM_GRID_HOST=localhost SELENIUM_GRID_PORT=80 \
10131013
SELENIUM_GRID_MONITORING=false CLEAR_POD_HISTORY=true SET_MAX_REPLICAS=100 ENABLE_VIDEO_RECORDER=false \
10141014
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) \

Diff for: charts/selenium-grid/CONFIGURATION.md

+12-12
Original file line numberDiff line numberDiff line change
@@ -423,10 +423,10 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
423423
| chromeNode.scaledOptions | string | `nil` | Override the scaled options for chrome nodes |
424424
| chromeNode.scaledJobOptions | string | `nil` | Override the scaledJobOptions for chrome nodes |
425425
| chromeNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for chrome nodes |
426-
| chromeNode.hpa.browserName | string | `"chrome"` | browserName from the capability |
426+
| chromeNode.hpa.browserName | string | `"chrome"` | browserName should match with Node stereotype and request capability is scaled by this scaler |
427427
| chromeNode.hpa.sessionBrowserName | string | `"chrome"` | sessionBrowserName if the browserName is different from the sessionBrowserName |
428-
| chromeNode.hpa.browserVersion | string | `""` | browserVersion from the capability |
429-
| chromeNode.hpa.platformName | string | `"Linux"` | platformName from the capability |
428+
| chromeNode.hpa.browserVersion | string | `""` | browserVersion should match with Node stereotype and request capability is scaled by this scaler |
429+
| chromeNode.hpa.platformName | string | `""` | platformName should match with Node stereotype and request capability is scaled by this scaler |
430430
| chromeNode.hpa.unsafeSsl | string | `"{{ template \"seleniumGrid.graphqlURL.unsafeSsl\" . }}"` | Skip check SSL when connecting to the Graphql endpoint |
431431
| chromeNode.initContainers | list | `[]` | It is used to add initContainers in the same pod of the browser node. It should be set using the --set-json option |
432432
| chromeNode.sidecars | list | `[]` | It is used to add sidecars proxy in the same pod of the browser node. It means it will add a new container to the deployment itself. It should be set using the --set-json option |
@@ -475,10 +475,10 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
475475
| firefoxNode.scaledOptions | string | `nil` | Override the scaled options for firefox nodes |
476476
| firefoxNode.scaledJobOptions | string | `nil` | Override the scaledJobOptions for firefox nodes |
477477
| firefoxNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for firefox nodes |
478-
| firefoxNode.hpa.browserName | string | `"firefox"` | browserName from the capability |
478+
| firefoxNode.hpa.browserName | string | `"firefox"` | browserName should match with Node stereotype and request capability is scaled by this scaler |
479479
| firefoxNode.hpa.sessionBrowserName | string | `"firefox"` | sessionBrowserName if the browserName is different from the sessionBrowserName |
480-
| firefoxNode.hpa.browserVersion | string | `""` | browserVersion from the capability |
481-
| firefoxNode.hpa.platformName | string | `"Linux"` | platformName from the capability |
480+
| firefoxNode.hpa.browserVersion | string | `""` | browserVersion should match with Node stereotype and request capability is scaled by this scaler |
481+
| firefoxNode.hpa.platformName | string | `""` | platformName should match with Node stereotype and request capability is scaled by this scaler |
482482
| firefoxNode.hpa.unsafeSsl | string | `"{{ template \"seleniumGrid.graphqlURL.unsafeSsl\" . }}"` | Skip check SSL when connecting to the Graphql endpoint |
483483
| firefoxNode.initContainers | list | `[]` | It is used to add initContainers in the same pod of the browser node. It should be set using the --set-json option |
484484
| firefoxNode.sidecars | list | `[]` | It is used to add sidecars proxy in the same pod of the browser node. It means it will add a new container to the deployment itself. It should be set using the --set-json option |
@@ -527,10 +527,10 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
527527
| edgeNode.scaledOptions | string | `nil` | Override the scaled options for edge nodes |
528528
| edgeNode.scaledJobOptions | string | `nil` | Override the scaledJobOptions for edge nodes |
529529
| edgeNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for edge nodes |
530-
| edgeNode.hpa.browserName | string | `"MicrosoftEdge"` | browserName from the capability |
530+
| edgeNode.hpa.browserName | string | `"MicrosoftEdge"` | browserName should match with Node stereotype and request capability is scaled by this scaler |
531531
| edgeNode.hpa.sessionBrowserName | string | `"msedge"` | sessionBrowserName if the browserName is different from the sessionBrowserName |
532-
| edgeNode.hpa.browserVersion | string | `""` | browserVersion from the capability |
533-
| edgeNode.hpa.platformName | string | `"Linux"` | platformName from the capability |
532+
| edgeNode.hpa.browserVersion | string | `""` | browserVersion should match with Node stereotype and request capability is scaled by this scaler |
533+
| edgeNode.hpa.platformName | string | `""` | platformName should match with Node stereotype and request capability is scaled by this scaler |
534534
| edgeNode.hpa.unsafeSsl | string | `"{{ template \"seleniumGrid.graphqlURL.unsafeSsl\" . }}"` | Skip check SSL when connecting to the Graphql endpoint |
535535
| edgeNode.initContainers | list | `[]` | It is used to add initContainers in the same pod of the browser node. It should be set using the --set-json option |
536536
| edgeNode.sidecars | list | `[]` | It is used to add sidecars proxy in the same pod of the browser node. It means it will add a new container to the deployment itself. It should be set using the --set-json option |
@@ -579,10 +579,10 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
579579
| relayNode.scaledOptions | string | `nil` | Override the scaled options for relay nodes |
580580
| relayNode.scaledJobOptions | string | `nil` | Override the scaledJobOptions for relay nodes |
581581
| relayNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for relay nodes |
582-
| relayNode.hpa.browserName | string | `"chrome"` | browserName from the capability |
582+
| relayNode.hpa.browserName | string | `"chrome"` | browserName should match with Node stereotype and request capability is scaled by this scaler |
583583
| relayNode.hpa.sessionBrowserName | string | `""` | sessionBrowserName if the browserName is different from the sessionBrowserName |
584-
| relayNode.hpa.platformName | string | `"Android"` | platformName from the capability |
585-
| relayNode.hpa.browserVersion | string | `""` | browserVersion from the capability |
584+
| relayNode.hpa.browserVersion | string | `""` | browserVersion should match with Node stereotype and request capability is scaled by this scaler |
585+
| relayNode.hpa.platformName | string | `"Android"` | platformName should match with Node stereotype and request capability is scaled by this scaler |
586586
| relayNode.hpa.unsafeSsl | string | `"{{ template \"seleniumGrid.graphqlURL.unsafeSsl\" . }}"` | Skip check SSL when connecting to the Graphql endpoint |
587587
| relayNode.initContainers | list | `[]` | It is used to add initContainers in the same pod of the browser node. It should be set using the --set-json option |
588588
| relayNode.sidecars | list | `[]` | It is used to add sidecars proxy in the same pod of the browser node. It means it will add a new container to the deployment itself. It should be set using the --set-json option |

Diff for: charts/selenium-grid/README.md

+136
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ This chart enables the creation of a Selenium Grid Server in Kubernetes.
1515
* [Settings common for both `job` and `deployment` scalingType](#settings-common-for-both-job-and-deployment-scalingtype)
1616
* [Settings when scalingType with `deployment`](#settings-when-scalingtype-with-deployment-)
1717
* [Settings when scalingType with `job`](#settings-when-scalingtype-with-job)
18+
* [Scaler trigger configuration](#scaler-trigger-configuration)
1819
* [Settings fixed-sized thread pool for the Distributor to create new sessions](#settings-fixed-sized-thread-pool-for-the-distributor-to-create-new-sessions)
1920
* [Updating Selenium-Grid release](#updating-selenium-grid-release)
2021
* [Uninstalling Selenium Grid release](#uninstalling-selenium-grid-release)
@@ -210,6 +211,141 @@ autoscaling:
210211

211212
Settings that KEDA [ScaledJob spec](https://keda.sh/docs/latest/concepts/scaling-jobs/#scaledjob-spec) supports can be set via `autoscaling.scaledJobOptions`.
212213

214+
Expected that with default configuration in KEDA resource, autoscaling behavior should be correct. Hence, in chart values, we keep the config key `autoscaling.scaledJobOptions.scalingStrategy.strategy` is `default`.
215+
216+
### Scaler trigger configuration
217+
218+
From KEDA core `v2.16.1+`, the trigger metadata `browserVersion`, `platformName` is recommended to be set explicitly to have the correct scaling behavior (especially when your Grid includes autoscaling Nodes, non-autoscaling Nodes, relay Nodes, etc.). Besides that, in client binding, it is also recommended to set the `browserVersion`, `platformName` to align with the trigger metadata. Please see below examples for more details.
219+
220+
Understand list trigger parameters
221+
222+
- `url` - Graphql url of your Selenium Grid. If endpoint requires authentication, you can use `TriggerAuthentication` to provide the credentials instead of embedding in the URL.
223+
- `browserName` - browserName should match with Node stereotype and request capability is scaled by this scaler. (Default: ``, Optional)
224+
- `sessionBrowserName` - sessionBrowserName if the browserName is different from the sessionBrowserName. (Default: ``, Optional)
225+
- `browserVersion` - browserVersion should match with Node stereotype and request capability is scaled by this scaler. (Default: ``, Optional)
226+
- `platformName` - platformName should match with Node stereotype and request capability is scaled by this scaler. (Default: ``, Optional)
227+
- `unsafeSsl` - Skip certificate validation when connecting over HTTPS. (Default: `false`, Optional)
228+
- `activationThreshold` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional)
229+
- `nodeMaxSessions` - Number of maximum sessions that can run in parallel on a Node. Update this parameter align with node config `--max-sessions` (`SE_NODE_MAX_SESSIONS`) to have the correct scaling behavior. (Default: `1`, Optional).
230+
231+
Understand list trigger authentication
232+
233+
- `username` - Username for basic authentication in GraphQL endpoint instead of embedding in the URL. (Optional)
234+
- `password` - Password for basic authentication in GraphQL endpoint instead of embedding in the URL. (Optional)
235+
- `authType` - Type of authentication to be used. This can be set to `Bearer` or `OAuth2` in case Selenium Grid behind an Ingress proxy with other authentication types. (Optional)
236+
- `accessToken` - Access token. This is required when `authType` is set a value. (Optional)
237+
238+
In each Node, trigger parameters value will be set under config key `hpa`. In template, those will be added spec of ScaledObject/ScaledJob.
239+
240+
In chart values, by default, `browserName`, `sessionBrowserName` are set for corresponding node browser. Parameters `browserVersion`, `platformName` are not set, leave them as empty by default. The default scaler metadata looks like
241+
242+
```yaml
243+
triggers:
244+
- type: selenium-grid
245+
metadata:
246+
url: 'http://selenium-hub:4444/graphql'
247+
browserName: 'chrome'
248+
browserVersion: ''
249+
platformName: ''
250+
```
251+
252+
In this case, the scaler will be triggered by below request (example in Python client, common use case that most users get started)
253+
254+
```python
255+
options = ChromeOptions()
256+
driver = webdriver.Remote(options=options, command_executor=SELENIUM_GRID_URL)
257+
```
258+
259+
With above script, the request is sent to Grid. Via GraphQL response, it looks like
260+
261+
```json
262+
{
263+
"data": {
264+
"grid": {
265+
"sessionCount": 0,
266+
"maxSession": 0,
267+
"totalSlots": 0
268+
},
269+
"nodesInfo": {
270+
"nodes": []
271+
},
272+
"sessionsInfo": {
273+
"sessionQueueRequests": [
274+
"{\"browserName\": \"chrome\"}"
275+
]
276+
}
277+
}
278+
}
279+
```
280+
281+
Scaler will trigger to scale up the Node with stereotypes matched to pick up the request in the queue. Via GraphQL response, it looks like
282+
283+
```json
284+
{
285+
"data": {
286+
"grid": {
287+
"sessionCount": 0,
288+
"maxSession": 1,
289+
"totalSlots": 1
290+
},
291+
"nodesInfo": {
292+
"nodes": [
293+
{
294+
"id": "UUID",
295+
"status": "UP",
296+
"sessionCount": 0,
297+
"maxSession": 1,
298+
"slotCount": 1,
299+
"stereotypes": "[{\"slots\": 1, \"stereotype\": {\"browserName\": \"chrome\", \"browserVersion\": \"\", \"platformName\": \"\"}}]",
300+
"sessions": []
301+
}
302+
]
303+
},
304+
"sessionsInfo": {
305+
"sessionQueueRequests": [
306+
"{\"browserName\": \"chrome\"}"
307+
]
308+
}
309+
}
310+
}
311+
```
312+
313+
In Node deployment spec, there is environment variable `SE_NODE_BROWSER_VERSION` which is able to unset `browserVersion` in Node stereotypes (it is setting short browser build number by default e.g `131.0`) or any custom value is up to you, which is expected to match with the request capabilities in queue and scaler trigger metadata.
314+
Similarly, `SE_NODE_PLATFORM_NAME` is used to unset the `platformName` in Node stereotypes if needed. Noted, update to newer image tag if these 2 env variables doesn't take effect for you.
315+
316+
For another example, where your Grid with multiple scalers have different metadata, one of them looks like
317+
318+
```yaml
319+
triggers:
320+
- type: selenium-grid
321+
metadata:
322+
url: 'http://selenium-hub:4444/graphql'
323+
browserName: 'chrome'
324+
browserVersion: '131.0'
325+
platformName: 'Linux'
326+
```
327+
328+
329+
The request to trigger this corresponds to the following Python script
330+
331+
```python
332+
options = ChromeOptions()
333+
options.set_capability('platformName', 'Linux')
334+
options.set_capability('browserVersion', '131.0')
335+
driver = webdriver.Remote(options=options, command_executor=SELENIUM_GRID_URL)
336+
```
337+
338+
### Define multiple scalers with different trigger parameters.
339+
When deploying the chart, you can define multiple scalers with different trigger parameters to scale up different Node stereotypes against different request capabilities.
340+
341+
Under config key `crossBrowsers`, in corresponding browser node, you can define array of item with structure same as that node, via `nameOverride` to set unique name for each scaler to avoid resources collision.
342+
343+
For example [multiple-nodes-platform.yaml](./multiple-nodes-platform.yaml) file, it defines 2 scalers per browser node to scale against requests with and without `platformName` capability.
344+
345+
For example [multiple-nodes-platform-version.yaml](./multiple-nodes-platform-version.yaml) file, it defines multiple scalers with `platformName: 'Linux'` and last few previous stable versions per browser node to scale against requests with `browserVersion` and `platformName` capabilities.
346+
347+
While deploying the chart, you can quickly use these extra values files by passing the file via `--values` flag to apply.
348+
213349
### Settings fixed-sized thread pool for the Distributor to create new sessions
214350

215351
When enabling autoscaling, the Distributor might be under a high workload with parallelism tests, which are many requests incoming and nodes scaling up simultaneously. (Refer to: [SeleniumHQ/selenium#13723](https://github.com/SeleniumHQ/selenium/issues/13723)).

0 commit comments

Comments
 (0)