Nits and stuff (#3565)

* Correct jenkins/Makefile header comment
* Add default test types to pbench-fio usage text
* Remove non-existent files from the linters' attentions
* Simplify upload content-length handling
* Fix setValueFromPath() docstring
* Make contrib pbench script use 'latest' container label
* Update Legacy test gold files

Author: webbnh

.flake8

@@ -2,12 +2,8 @@
 doctests = True
 filename =
         *.py
-        *server/bin/pbench-backup-tarballs
-        *server/bin/pbench-cull-unpacked-tarballs
         *server/bin/pbench-index
         *server/bin/pbench-reindex
-        *server/bin/pbench-unpack-tarballs
-        *server/bin/pbench-verify-backup-tarballs
 exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,*web-server/*,
         *agent/bench-scripts/test-bin/fio-histo-log-pctiles.py,
         *agent/bench-scripts/tests/pbench-trafficgen/test-39.trafficgen/*,

agent/bench-scripts/pbench-fio

@@ -35,7 +35,8 @@ nr_samples=5
 maxstddevpct=5 # maximum allowable standard deviation in percent
 max_failures=6 # after N failed attempts to hit below ${maxstddevpct}, move on to the next test
 supported_test_types="read,write,rw,randread,randwrite,randrw"
-test_types="read,randread"		# default is -non- destructive
+default_test_types="read,randread"		# default is -non- destructive
+test_types=${default_test_types}
 pre_check="n"
 export config=""
 rate_iops=""
@@ -71,7 +72,7 @@ function usage {
 	printf "The following options are available:\n"
 	printf "\n"
 	printf -- "\t-t str[,str...] --test-types=str[,str...]\n"
-	printf "\t\tone or more of %s\n" $supported_test_types
+	printf "\t\tone or more of %s (default is %s)\n" $supported_test_types $default_test_types
 	printf "\n"
 	printf -- "\t--direct=[0/1]\n"
 	printf "\t\t1 = O_DIRECT enabled (default), 0 = O_DIRECT disabled\n"

agent/bench-scripts/tests/pbench-fio/test-14.txt

@@ -3,7 +3,7 @@
 The following options are available:
 
 	-t str[,str...] --test-types=str[,str...]
-		one or more of read,write,rw,randread,randwrite,randrw
+		one or more of read,write,rw,randread,randwrite,randrw (default is read,randread)
 
 	--direct=[0/1]
 		1 = O_DIRECT enabled (default), 0 = O_DIRECT disabled

agent/bench-scripts/tests/pbench-fio/test-17.txt

@@ -3,7 +3,7 @@
 The following options are available:
 
 	-t str[,str...] --test-types=str[,str...]
-		one or more of read,write,rw,randread,randwrite,randrw
+		one or more of read,write,rw,randread,randwrite,randrw (default is read,randread)
 
 	--direct=[0/1]
 		1 = O_DIRECT enabled (default), 0 = O_DIRECT disabled

agent/bench-scripts/tests/pbench-fio/test-18.txt

@@ -6,7 +6,7 @@ pbench-fio --
 The following options are available:
 
 	-t str[,str...] --test-types=str[,str...]
-		one or more of read,write,rw,randread,randwrite,randrw
+		one or more of read,write,rw,randread,randwrite,randrw (default is read,randread)
 
 	--direct=[0/1]
 		1 = O_DIRECT enabled (default), 0 = O_DIRECT disabled

agent/bench-scripts/tests/pbench-fio/test-21.txt

@@ -6,7 +6,7 @@ pbench-fio --
 The following options are available:
 
 	-t str[,str...] --test-types=str[,str...]
-		one or more of read,write,rw,randread,randwrite,randrw
+		one or more of read,write,rw,randread,randwrite,randrw (default is read,randread)
 
 	--direct=[0/1]
 		1 = O_DIRECT enabled (default), 0 = O_DIRECT disabled

agent/bench-scripts/tests/pbench-fio/test-22.txt

@@ -1,5 +1,5 @@
 +++ Running test-22 pbench-fio
-[error][1900-01-01T00:00:00.000000] [pbench-fio] fio failed reporting its version: '/var/tmp/pbench-test-bench/opt/pbench-agent/bench-scripts/pbench-fio: line 477: fio: command not found'
+[error][1900-01-01T00:00:00.000000] [pbench-fio] fio failed reporting its version: '/var/tmp/pbench-test-bench/opt/pbench-agent/bench-scripts/pbench-fio: line 478: fio: command not found'
 --- Finished test-22 pbench-fio (status=1)
 +++ pbench tree state
 /var/tmp/pbench-test-bench/pbench-agent
@@ -15,5 +15,5 @@
 [debug][1900-01-01T00:00:00.000000] fio_pre_check(targets="/tmp/fio", clients="", ver="3.21", match="gte")
 [debug][1900-01-01T00:00:00.000000] Running pre-check locally
 [debug][1900-01-01T00:00:00.000000] local_pre_check(devs="", ver="3.21", match="gte")
-[error][1900-01-01T00:00:00.000000] [pbench-fio] fio failed reporting its version: '/var/tmp/pbench-test-bench/opt/pbench-agent/bench-scripts/pbench-fio: line 477: fio: command not found'
+[error][1900-01-01T00:00:00.000000] [pbench-fio] fio failed reporting its version: '/var/tmp/pbench-test-bench/opt/pbench-agent/bench-scripts/pbench-fio: line 478: fio: command not found'
 --- pbench.log file contents

agent/bench-scripts/tests/pbench-fio/test-26.txt

@@ -6,7 +6,7 @@ pbench-fio bar,zab --
 The following options are available:
 
 	-t str[,str...] --test-types=str[,str...]
-		one or more of read,write,rw,randread,randwrite,randrw
+		one or more of read,write,rw,randread,randwrite,randrw (default is read,randread)
 
 	--direct=[0/1]
 		1 = O_DIRECT enabled (default), 0 = O_DIRECT disabled

agent/bench-scripts/tests/pbench-fio/test-27.txt

@@ -6,7 +6,7 @@ pbench-fio /var/tmp/pbench-test-bench/tmp/test-27_clients.file --
 The following options are available:
 
 	-t str[,str...] --test-types=str[,str...]
-		one or more of read,write,rw,randread,randwrite,randrw
+		one or more of read,write,rw,randread,randwrite,randrw (default is read,randread)
 
 	--direct=[0/1]
 		1 = O_DIRECT enabled (default), 0 = O_DIRECT disabled

agent/bench-scripts/tests/pbench-fio/test-29.txt

@@ -6,7 +6,7 @@ pbench-fio --
 The following options are available:
 
 	-t str[,str...] --test-types=str[,str...]
-		one or more of read,write,rw,randread,randwrite,randrw
+		one or more of read,write,rw,randread,randwrite,randrw (default is read,randread)
 
 	--direct=[0/1]
 		1 = O_DIRECT enabled (default), 0 = O_DIRECT disabled

agent/bench-scripts/tests/test-benchmark-clis/test-CL.txt

@@ -5,7 +5,7 @@ Bench Script: pbench-fio --help
 The following options are available:
 
 	-t str[,str...] --test-types=str[,str...]
-		one or more of read,write,rw,randread,randwrite,randrw
+		one or more of read,write,rw,randread,randwrite,randrw (default is read,randread)
 
 	--direct=[0/1]
 		1 = O_DIRECT enabled (default), 0 = O_DIRECT disabled
@@ -100,7 +100,7 @@ Bench Script: pbench-fio --tool-group=bad --sysinfo=bad
 The following options are available:
 
 	-t str[,str...] --test-types=str[,str...]
-		one or more of read,write,rw,randread,randwrite,randrw
+		one or more of read,write,rw,randread,randwrite,randrw (default is read,randread)
 
 	--direct=[0/1]
 		1 = O_DIRECT enabled (default), 0 = O_DIRECT disabled
@@ -195,7 +195,7 @@ pbench-fio --bad-to-the-bone
 The following options are available:
 
 	-t str[,str...] --test-types=str[,str...]
-		one or more of read,write,rw,randread,randwrite,randrw
+		one or more of read,write,rw,randread,randwrite,randrw (default is read,randread)
 
 	--direct=[0/1]
 		1 = O_DIRECT enabled (default), 0 = O_DIRECT disabled

build.sh

@@ -13,7 +13,7 @@ python3 -m pip install --user -r lint-requirements.txt
 # match the owner of the Git checkout, then Git issues an error; these config
 # settings avoid the problem.
 GITTOP=$(git rev-parse --show-toplevel 2>&1 | head -n 1)
-if [[ ${GITTOP} =~ "fatal: unsafe repository ('/home/root/pbench'" ]] ; then
+if [[ ${GITTOP} = "fatal: unsafe repository ('/home/root/pbench'"* ]] ; then
 	git config --global --add safe.directory /home/root/pbench
 	GITTOP=$(git rev-parse --show-toplevel)
 fi
@@ -45,4 +45,4 @@ make -C agent/rpm ci
 make -C dashboard build
 
 # Display our victory
-ls -l ${HOME}/rpmbuild*/RPMS/noarch/*
+ls -l "${HOME}"/rpmbuild*/RPMS/noarch/*

contrib/containerized-pbench/pbench

@@ -32,7 +32,7 @@
 #    contrib/containerized-pbench/pbench pbench-results-move \
 #      --server https://<server>:8443 --token <api-token>
 
-image_name=${PB_AGENT_IMAGE_NAME:-quay.io/pbench/pbench-agent-all-centos-8:main}
+image_name=${PB_AGENT_IMAGE_NAME:-quay.io/pbench/pbench-agent-all-centos-8:latest}
 pbench_run_dir=${PB_AGENT_RUN_DIR:-/var/tmp/${USER}/pbench-agent/run}
 ca=${PB_AGENT_CA:-${REQUESTS_CA_BUNDLE}}
 if [[ ${ca} ]]; then

dashboard/src/actions/overviewActions.js

@@ -549,7 +549,7 @@ const isServerInternal = (string) =>
  * @function
  * @param {String} path - nested key to update
  * @param {Object} obj - nested object
- * @param {String} obj - new value to be updated in the object
+ * @param {String} value - new value to be updated in the object
  * @return {Object} - updated object with new value
  */
 

jenkins/Makefile

@@ -1,5 +1,5 @@
 #
-# A simple Makefile for building and pushing the various pbench-devel-*
+# A simple Makefile for building and pushing the various pbench-ci-fedora
 # and pbench-rpmbuild container images.
 #
 # This makefile defines the following targets:

lib/pbench/server/api/resources/intake_base.py

@@ -245,11 +245,13 @@ def _intake(
 
         audit: Optional[Audit] = None
         intake_dir: Optional[Path] = None
-        notes = []
+        notes: list[str] = []
 
         prefix = current_app.server_config.rest_uri
         origin = f"{self._get_uri_base(request).host}{prefix}/datasets/"
 
+        dataset: Optional[Dataset] = None
+
         try:
             try:
                 authorized_user = Auth.token_auth.current_user()
@@ -488,7 +490,7 @@ def _intake(
                     )
                     attributes["missing_metadata"] = True
                 else:
-                    p = metalog.get("pbench")
+                    p: Optional[JSONOBJECT] = metalog.get("pbench")
                     if p:
                         benchmark = p.get("script", Metadata.SERVER_BENCHMARK_UNKNOWN)
                     else:

lib/pbench/server/api/resources/upload.py

@@ -105,26 +105,16 @@ def _stream(self, intake: Intake, request: Request) -> Access:
         Raises:
             APIAbort on failure
         """
-        try:
-            length_string = request.headers["Content-Length"]
-            content_length = int(length_string)
-        except KeyError as e:
-            # NOTE: Werkzeug is "smart" about header access, and knows that
-            # Content-Length is an integer. Therefore, a non-integer value
-            # will raise KeyError. It's virtually impossible to report the
-            # actual incorrect value as we'd just get a KeyError again.
-            raise APIAbort(
-                HTTPStatus.LENGTH_REQUIRED,
-                "Missing or invalid 'Content-Length' header",
-            ) from e
-        except ValueError as e:
-            # NOTE: Because of the way Werkzeug works, this should not be
-            # possible: if Content-Length isn't an integer, we'll see the
-            # KeyError. This however serves as a clarifying backup case.
-            raise APIAbort(
-                HTTPStatus.BAD_REQUEST,
-                f"Invalid 'Content-Length' header, not an integer ({length_string})",
-            ) from e
+        # Werkzeug returns an integer or None; either way, a false-y value is
+        # bad, so report it as an error.
+        content_length = request.content_length
+        if not content_length:
+            cl_val = request.headers.get("Content-Length")
+            if cl_val is None:
+                msg = "Missing or invalid 'Content-Length' header"
+            else:
+                msg = f"Invalid 'Content-Length' header: {cl_val}"
+            raise APIAbort(HTTPStatus.LENGTH_REQUIRED, msg)
         return Access(content_length, request.stream)
 
     def _put(self, args: ApiParams, request: Request, context: ApiContext) -> Response:

lib/pbench/test/unit/server/test_upload.py

@@ -418,7 +418,7 @@ def test_empty_upload(
         filename = "tmp.tar.xz"
         datafile = tmp_path / filename
         datafile.touch()
-        expected_message = "'Content-Length' 0 must be greater than 0"
+        expected_message = "Invalid 'Content-Length' header: 0"
         with datafile.open("rb") as data_fp:
             response = client.put(
                 self.gen_uri(server_config, filename),
@@ -429,7 +429,7 @@ def test_empty_upload(
                     pbench_drb_token, "d41d8cd98f00b204e9800998ecf8427e"
                 ),
             )
-        assert response.status_code == HTTPStatus.BAD_REQUEST
+        assert response.status_code == HTTPStatus.LENGTH_REQUIRED
         assert response.json.get("message") == expected_message
         self.verify_logs(caplog)
         assert not self.cachemanager_created