diff --git a/agent/util-scripts/gold/test-start-stop-tool-meister/test-51.txt b/agent/util-scripts/gold/test-start-stop-tool-meister/test-51.txt index aa5007d6e7..a625769b07 100644 --- a/agent/util-scripts/gold/test-start-stop-tool-meister/test-51.txt +++ b/agent/util-scripts/gold/test-start-stop-tool-meister/test-51.txt @@ -125,7 +125,7 @@ install_check_output = perf: perf is installed +++ mock-run/tm/pbench-tool-data-sink.err file contents DEBUG pbench-tool-data-sink daemon -- re-constructing Redis server object DEBUG pbench-tool-data-sink daemon -- reconstructed Redis server object -DEBUG pbench-tool-data-sink driver -- params_key (tds-default): {'benchmark_run_dir': '/var/tmp/pbench-test-utils/pbench/mock-run', 'bind_hostname': 'localhost', 'channel_prefix': 'pbench-agent-cli', 'group': 'default', 'optional_md': {'config': '', 'date': '1900-01-01T00:00:00', 'script': 'fake-bm', 'ssh_opts': '-o BatchMode=yes -o StrictHostKeyChecking=no'}, 'port': 8080, 'tool_metadata': {'persistent': {'dcgm': {'collector': 'prometheus', 'port': '9400'}, 'node-exporter': {'collector': 'prometheus', 'port': '9100'}, 'pcp': {'collector': 'pcp', 'port': '44321'}}, 'transient': {'blktrace': None, 'bpftrace': None, 'cpuacct': None, 'disk': None, 'dm-cache': None, 'docker': None, 'docker-info': None, 'external-data-source': None, 'haproxy-ocp': None, 'iostat': None, 'jmap': None, 'jstack': None, 'kvm-spinlock': None, 'kvmstat': None, 'kvmtrace': None, 'lockstat': None, 'mpstat': None, 'numastat': None, 'oc': None, 'openvswitch': None, 'pcp-transient': None, 'perf': None, 'pidstat': None, 'pprof': None, 'proc-interrupts': None, 'proc-sched_debug': None, 'proc-vmstat': None, 'prometheus-metrics': None, 'qemu-migrate': None, 'rabbit': None, 'sar': None, 'strace': None, 'sysfs': None, 'systemtap': None, 'tcpdump': None, 'turbostat': None, 'user-tool': None, 'virsh-migrate': None, 'vmstat': None}}, 'tool_trigger': None, 'tools': {'testhost.example.com': {'mpstat': '', 'perf': '--record-opts="-a -freq=100 -g --event=branch-misses --event=cache-misses --event=instructions" --report-opts="-I -g"'}}} +DEBUG pbench-tool-data-sink driver -- params_key (tds-default): {'benchmark_run_dir': '/var/tmp/pbench-test-utils/pbench/mock-run', 'bind_hostname': 'localhost', 'channel_prefix': 'pbench-agent-cli', 'optional_md': {'config': '', 'date': '1900-01-01T00:00:00', 'script': 'fake-bm', 'ssh_opts': '-o BatchMode=yes -o StrictHostKeyChecking=no'}, 'port': 8080, 'tool_group': 'default', 'tool_metadata': {'persistent': {'dcgm': {'collector': 'prometheus', 'port': '9400'}, 'node-exporter': {'collector': 'prometheus', 'port': '9100'}, 'pcp': {'collector': 'pcp', 'port': '44321'}}, 'transient': {'blktrace': None, 'bpftrace': None, 'cpuacct': None, 'disk': None, 'dm-cache': None, 'docker': None, 'docker-info': None, 'external-data-source': None, 'haproxy-ocp': None, 'iostat': None, 'jmap': None, 'jstack': None, 'kvm-spinlock': None, 'kvmstat': None, 'kvmtrace': None, 'lockstat': None, 'mpstat': None, 'numastat': None, 'oc': None, 'openvswitch': None, 'pcp-transient': None, 'perf': None, 'pidstat': None, 'pprof': None, 'proc-interrupts': None, 'proc-sched_debug': None, 'proc-vmstat': None, 'prometheus-metrics': None, 'qemu-migrate': None, 'rabbit': None, 'sar': None, 'strace': None, 'sysfs': None, 'systemtap': None, 'tcpdump': None, 'turbostat': None, 'user-tool': None, 'virsh-migrate': None, 'vmstat': None}}, 'tool_trigger': None, 'tools': {'testhost.example.com': {'mpstat': '', 'perf': '--record-opts="-a -freq=100 -g --event=branch-misses --event=cache-misses --event=instructions" --report-opts="-I -g"'}}} INFO pbench-tool-data-sink web_server_run -- Running Bottle web server ... Bottle v#.##.## server starting up (using DataSinkWsgiServer(handler_class=.DataSinkWsgiRequestHandler'>))... Listening on http://localhost:8080/ @@ -209,7 +209,7 @@ port 17001 +++ mock-run/tm/tm.err file contents DEBUG pbench-tool-meister daemon -- re-constructing Redis server object DEBUG pbench-tool-meister daemon -- re-constructed Redis server object -DEBUG pbench-tool-meister driver -- params_key (tm-default-testhost.example.com): {'benchmark_run_dir': '/var/tmp/pbench-test-utils/pbench/mock-run', 'channel_prefix': 'pbench-agent-cli', 'controller': 'testhost.example.com', 'group': 'default', 'hostname': 'testhost.example.com', 'label': '', 'tds_hostname': 'localhost', 'tds_port': 8080, 'tool_metadata': {'persistent': {'dcgm': {'collector': 'prometheus', 'port': '9400'}, 'node-exporter': {'collector': 'prometheus', 'port': '9100'}, 'pcp': {'collector': 'pcp', 'port': '44321'}}, 'transient': {'blktrace': None, 'bpftrace': None, 'cpuacct': None, 'disk': None, 'dm-cache': None, 'docker': None, 'docker-info': None, 'external-data-source': None, 'haproxy-ocp': None, 'iostat': None, 'jmap': None, 'jstack': None, 'kvm-spinlock': None, 'kvmstat': None, 'kvmtrace': None, 'lockstat': None, 'mpstat': None, 'numastat': None, 'oc': None, 'openvswitch': None, 'pcp-transient': None, 'perf': None, 'pidstat': None, 'pprof': None, 'proc-interrupts': None, 'proc-sched_debug': None, 'proc-vmstat': None, 'prometheus-metrics': None, 'qemu-migrate': None, 'rabbit': None, 'sar': None, 'strace': None, 'sysfs': None, 'systemtap': None, 'tcpdump': None, 'turbostat': None, 'user-tool': None, 'virsh-migrate': None, 'vmstat': None}}, 'tools': {'mpstat': '', 'perf': '--record-opts="-a -freq=100 -g --event=branch-misses --event=cache-misses --event=instructions" --report-opts="-I -g"'}} +DEBUG pbench-tool-meister driver -- params_key (tm-default-testhost.example.com): {'benchmark_run_dir': '/var/tmp/pbench-test-utils/pbench/mock-run', 'channel_prefix': 'pbench-agent-cli', 'controller': 'testhost.example.com', 'group': 'default', 'hostname': 'testhost.example.com', 'label': '', 'tds_hostname': 'localhost', 'tds_port': 8080, 'tool_metadata': "{'persistent': {'dcgm': {'collector': 'prometheus', 'port': '9400'}, 'node-exporter': {'collector': 'prometheus', 'port': '9100'}, 'pcp': {'collector': 'pcp', 'port': '44321'}}, 'transient': {'blktrace': None, 'bpftrace': None, 'cpuacct': None, 'disk': None, 'dm-cache': None, 'docker': None, 'docker-info': None, 'external-data-source': None, 'haproxy-ocp': None, 'iostat': None, 'jmap': None, 'jstack': None, 'kvm-spinlock': None, 'kvmstat': None, 'kvmtrace': None, 'lockstat': None, 'mpstat': None, 'numastat': None, 'oc': None, 'openvswitch': None, 'pcp-transient': None, 'perf': None, 'pidstat': None, 'pprof': None, 'proc-interrupts': None, 'proc-sched_debug': None, 'proc-vmstat': None, 'prometheus-metrics': None, 'qemu-migrate': None, 'rabbit': None, 'sar': None, 'strace': None, 'sysfs': None, 'systemtap': None, 'tcpdump': None, 'turbostat': None, 'user-tool': None, 'virsh-migrate': None, 'vmstat': None}}", 'tools': {'mpstat': '', 'perf': '--record-opts="-a -freq=100 -g --event=branch-misses --event=cache-misses --event=instructions" --report-opts="-I -g"'}} DEBUG pbench-tool-meister __enter__ -- publish pbench-agent-cli-from-tms DEBUG pbench-tool-meister __enter__ -- published pbench-agent-cli-from-tms DEBUG pbench-tool-meister driver -- waiting ... @@ -240,7 +240,7 @@ DEBUG pbench-tool-meister _send_client_status -- publish pbench-agent-cli-from-t --- mock-run/tm/tm.err file contents +++ mock-run/tm/tm.logs file contents pbench-tool-meister-start - verify logging channel up -testhost.example.com 0000 DEBUG pbench-tool-meister driver -- params_key (tm-default-testhost.example.com): {'benchmark_run_dir': '/var/tmp/pbench-test-utils/pbench/mock-run', 'channel_prefix': 'pbench-agent-cli', 'controller': 'testhost.example.com', 'group': 'default', 'hostname': 'testhost.example.com', 'label': '', 'tds_hostname': 'localhost', 'tds_port': 8080, 'tool_metadata': {'persistent': {'dcgm': {'collector': 'prometheus', 'port': '9400'}, 'node-exporter': {'collector': 'prometheus', 'port': '9100'}, 'pcp': {'collector': 'pcp', 'port': '44321'}}, 'transient': {'blktrace': None, 'bpftrace': None, 'cpuacct': None, 'disk': None, 'dm-cache': None, 'docker': None, 'docker-info': None, 'external-data-source': None, 'haproxy-ocp': None, 'iostat': None, 'jmap': None, 'jstack': None, 'kvm-spinlock': None, 'kvmstat': None, 'kvmtrace': None, 'lockstat': None, 'mpstat': None, 'numastat': None, 'oc': None, 'openvswitch': None, 'pcp-transient': None, 'perf': None, 'pidstat': None, 'pprof': None, 'proc-interrupts': None, 'proc-sched_debug': None, 'proc-vmstat': None, 'prometheus-metrics': None, 'qemu-migrate': None, 'rabbit': None, 'sar': None, 'strace': None, 'sysfs': None, 'systemtap': None, 'tcpdump': None, 'turbostat': None, 'user-tool': None, 'virsh-migrate': None, 'vmstat': None}}, 'tools': {'mpstat': '', 'perf': '--record-opts="-a -freq=100 -g --event=branch-misses --event=cache-misses --event=instructions" --report-opts="-I -g"'}} +testhost.example.com 0000 DEBUG pbench-tool-meister driver -- params_key (tm-default-testhost.example.com): {'benchmark_run_dir': '/var/tmp/pbench-test-utils/pbench/mock-run', 'channel_prefix': 'pbench-agent-cli', 'controller': 'testhost.example.com', 'group': 'default', 'hostname': 'testhost.example.com', 'label': '', 'tds_hostname': 'localhost', 'tds_port': 8080, 'tool_metadata': "{'persistent': {'dcgm': {'collector': 'prometheus', 'port': '9400'}, 'node-exporter': {'collector': 'prometheus', 'port': '9100'}, 'pcp': {'collector': 'pcp', 'port': '44321'}}, 'transient': {'blktrace': None, 'bpftrace': None, 'cpuacct': None, 'disk': None, 'dm-cache': None, 'docker': None, 'docker-info': None, 'external-data-source': None, 'haproxy-ocp': None, 'iostat': None, 'jmap': None, 'jstack': None, 'kvm-spinlock': None, 'kvmstat': None, 'kvmtrace': None, 'lockstat': None, 'mpstat': None, 'numastat': None, 'oc': None, 'openvswitch': None, 'pcp-transient': None, 'perf': None, 'pidstat': None, 'pprof': None, 'proc-interrupts': None, 'proc-sched_debug': None, 'proc-vmstat': None, 'prometheus-metrics': None, 'qemu-migrate': None, 'rabbit': None, 'sar': None, 'strace': None, 'sysfs': None, 'systemtap': None, 'tcpdump': None, 'turbostat': None, 'user-tool': None, 'virsh-migrate': None, 'vmstat': None}}", 'tools': {'mpstat': '', 'perf': '--record-opts="-a -freq=100 -g --event=branch-misses --event=cache-misses --event=instructions" --report-opts="-I -g"'}} testhost.example.com 0001 DEBUG pbench-tool-meister __enter__ -- publish pbench-agent-cli-from-tms testhost.example.com 0002 DEBUG pbench-tool-meister __enter__ -- published pbench-agent-cli-from-tms testhost.example.com 0003 DEBUG pbench-tool-meister driver -- waiting ... diff --git a/agent/util-scripts/gold/test-start-stop-tool-meister/test-52.txt b/agent/util-scripts/gold/test-start-stop-tool-meister/test-52.txt index a568de34de..c5a45b868e 100644 --- a/agent/util-scripts/gold/test-start-stop-tool-meister/test-52.txt +++ b/agent/util-scripts/gold/test-start-stop-tool-meister/test-52.txt @@ -125,7 +125,7 @@ install_check_output = perf: perf is installed +++ mock-run/tm/pbench-tool-data-sink.err file contents DEBUG pbench-tool-data-sink daemon -- re-constructing Redis server object DEBUG pbench-tool-data-sink daemon -- reconstructed Redis server object -DEBUG pbench-tool-data-sink driver -- params_key (tds-mygroup): {'benchmark_run_dir': '/var/tmp/pbench-test-utils/pbench/mock-run', 'bind_hostname': 'localhost', 'channel_prefix': 'pbench-agent-cli', 'group': 'mygroup', 'optional_md': {'config': '', 'date': '1900-01-01T00:00:00', 'script': 'fake-bm', 'ssh_opts': '-o BatchMode=yes -o StrictHostKeyChecking=no'}, 'port': 8080, 'tool_metadata': {'persistent': {'dcgm': {'collector': 'prometheus', 'port': '9400'}, 'node-exporter': {'collector': 'prometheus', 'port': '9100'}, 'pcp': {'collector': 'pcp', 'port': '44321'}}, 'transient': {'blktrace': None, 'bpftrace': None, 'cpuacct': None, 'disk': None, 'dm-cache': None, 'docker': None, 'docker-info': None, 'external-data-source': None, 'haproxy-ocp': None, 'iostat': None, 'jmap': None, 'jstack': None, 'kvm-spinlock': None, 'kvmstat': None, 'kvmtrace': None, 'lockstat': None, 'mpstat': None, 'numastat': None, 'oc': None, 'openvswitch': None, 'pcp-transient': None, 'perf': None, 'pidstat': None, 'pprof': None, 'proc-interrupts': None, 'proc-sched_debug': None, 'proc-vmstat': None, 'prometheus-metrics': None, 'qemu-migrate': None, 'rabbit': None, 'sar': None, 'strace': None, 'sysfs': None, 'systemtap': None, 'tcpdump': None, 'turbostat': None, 'user-tool': None, 'virsh-migrate': None, 'vmstat': None}}, 'tool_trigger': None, 'tools': {'testhost.example.com': {'mpstat': '', 'perf': '--record-opts="-a -freq=100 -g --event=branch-misses --event=cache-misses --event=instructions" --report-opts="-I -g"'}}} +DEBUG pbench-tool-data-sink driver -- params_key (tds-mygroup): {'benchmark_run_dir': '/var/tmp/pbench-test-utils/pbench/mock-run', 'bind_hostname': 'localhost', 'channel_prefix': 'pbench-agent-cli', 'optional_md': {'config': '', 'date': '1900-01-01T00:00:00', 'script': 'fake-bm', 'ssh_opts': '-o BatchMode=yes -o StrictHostKeyChecking=no'}, 'port': 8080, 'tool_group': 'mygroup', 'tool_metadata': {'persistent': {'dcgm': {'collector': 'prometheus', 'port': '9400'}, 'node-exporter': {'collector': 'prometheus', 'port': '9100'}, 'pcp': {'collector': 'pcp', 'port': '44321'}}, 'transient': {'blktrace': None, 'bpftrace': None, 'cpuacct': None, 'disk': None, 'dm-cache': None, 'docker': None, 'docker-info': None, 'external-data-source': None, 'haproxy-ocp': None, 'iostat': None, 'jmap': None, 'jstack': None, 'kvm-spinlock': None, 'kvmstat': None, 'kvmtrace': None, 'lockstat': None, 'mpstat': None, 'numastat': None, 'oc': None, 'openvswitch': None, 'pcp-transient': None, 'perf': None, 'pidstat': None, 'pprof': None, 'proc-interrupts': None, 'proc-sched_debug': None, 'proc-vmstat': None, 'prometheus-metrics': None, 'qemu-migrate': None, 'rabbit': None, 'sar': None, 'strace': None, 'sysfs': None, 'systemtap': None, 'tcpdump': None, 'turbostat': None, 'user-tool': None, 'virsh-migrate': None, 'vmstat': None}}, 'tool_trigger': None, 'tools': {'testhost.example.com': {'mpstat': '', 'perf': '--record-opts="-a -freq=100 -g --event=branch-misses --event=cache-misses --event=instructions" --report-opts="-I -g"'}}} INFO pbench-tool-data-sink web_server_run -- Running Bottle web server ... Bottle v#.##.## server starting up (using DataSinkWsgiServer(handler_class=.DataSinkWsgiRequestHandler'>))... Listening on http://localhost:8080/ @@ -209,7 +209,7 @@ port 17001 +++ mock-run/tm/tm.err file contents DEBUG pbench-tool-meister daemon -- re-constructing Redis server object DEBUG pbench-tool-meister daemon -- re-constructed Redis server object -DEBUG pbench-tool-meister driver -- params_key (tm-mygroup-testhost.example.com): {'benchmark_run_dir': '/var/tmp/pbench-test-utils/pbench/mock-run', 'channel_prefix': 'pbench-agent-cli', 'controller': 'testhost.example.com', 'group': 'mygroup', 'hostname': 'testhost.example.com', 'label': '', 'tds_hostname': 'localhost', 'tds_port': 8080, 'tool_metadata': {'persistent': {'dcgm': {'collector': 'prometheus', 'port': '9400'}, 'node-exporter': {'collector': 'prometheus', 'port': '9100'}, 'pcp': {'collector': 'pcp', 'port': '44321'}}, 'transient': {'blktrace': None, 'bpftrace': None, 'cpuacct': None, 'disk': None, 'dm-cache': None, 'docker': None, 'docker-info': None, 'external-data-source': None, 'haproxy-ocp': None, 'iostat': None, 'jmap': None, 'jstack': None, 'kvm-spinlock': None, 'kvmstat': None, 'kvmtrace': None, 'lockstat': None, 'mpstat': None, 'numastat': None, 'oc': None, 'openvswitch': None, 'pcp-transient': None, 'perf': None, 'pidstat': None, 'pprof': None, 'proc-interrupts': None, 'proc-sched_debug': None, 'proc-vmstat': None, 'prometheus-metrics': None, 'qemu-migrate': None, 'rabbit': None, 'sar': None, 'strace': None, 'sysfs': None, 'systemtap': None, 'tcpdump': None, 'turbostat': None, 'user-tool': None, 'virsh-migrate': None, 'vmstat': None}}, 'tools': {'mpstat': '', 'perf': '--record-opts="-a -freq=100 -g --event=branch-misses --event=cache-misses --event=instructions" --report-opts="-I -g"'}} +DEBUG pbench-tool-meister driver -- params_key (tm-mygroup-testhost.example.com): {'benchmark_run_dir': '/var/tmp/pbench-test-utils/pbench/mock-run', 'channel_prefix': 'pbench-agent-cli', 'controller': 'testhost.example.com', 'group': 'mygroup', 'hostname': 'testhost.example.com', 'label': '', 'tds_hostname': 'localhost', 'tds_port': 8080, 'tool_metadata': "{'persistent': {'dcgm': {'collector': 'prometheus', 'port': '9400'}, 'node-exporter': {'collector': 'prometheus', 'port': '9100'}, 'pcp': {'collector': 'pcp', 'port': '44321'}}, 'transient': {'blktrace': None, 'bpftrace': None, 'cpuacct': None, 'disk': None, 'dm-cache': None, 'docker': None, 'docker-info': None, 'external-data-source': None, 'haproxy-ocp': None, 'iostat': None, 'jmap': None, 'jstack': None, 'kvm-spinlock': None, 'kvmstat': None, 'kvmtrace': None, 'lockstat': None, 'mpstat': None, 'numastat': None, 'oc': None, 'openvswitch': None, 'pcp-transient': None, 'perf': None, 'pidstat': None, 'pprof': None, 'proc-interrupts': None, 'proc-sched_debug': None, 'proc-vmstat': None, 'prometheus-metrics': None, 'qemu-migrate': None, 'rabbit': None, 'sar': None, 'strace': None, 'sysfs': None, 'systemtap': None, 'tcpdump': None, 'turbostat': None, 'user-tool': None, 'virsh-migrate': None, 'vmstat': None}}", 'tools': {'mpstat': '', 'perf': '--record-opts="-a -freq=100 -g --event=branch-misses --event=cache-misses --event=instructions" --report-opts="-I -g"'}} DEBUG pbench-tool-meister __enter__ -- publish pbench-agent-cli-from-tms DEBUG pbench-tool-meister __enter__ -- published pbench-agent-cli-from-tms DEBUG pbench-tool-meister driver -- waiting ... @@ -240,7 +240,7 @@ DEBUG pbench-tool-meister _send_client_status -- publish pbench-agent-cli-from-t --- mock-run/tm/tm.err file contents +++ mock-run/tm/tm.logs file contents pbench-tool-meister-start - verify logging channel up -testhost.example.com 0000 DEBUG pbench-tool-meister driver -- params_key (tm-mygroup-testhost.example.com): {'benchmark_run_dir': '/var/tmp/pbench-test-utils/pbench/mock-run', 'channel_prefix': 'pbench-agent-cli', 'controller': 'testhost.example.com', 'group': 'mygroup', 'hostname': 'testhost.example.com', 'label': '', 'tds_hostname': 'localhost', 'tds_port': 8080, 'tool_metadata': {'persistent': {'dcgm': {'collector': 'prometheus', 'port': '9400'}, 'node-exporter': {'collector': 'prometheus', 'port': '9100'}, 'pcp': {'collector': 'pcp', 'port': '44321'}}, 'transient': {'blktrace': None, 'bpftrace': None, 'cpuacct': None, 'disk': None, 'dm-cache': None, 'docker': None, 'docker-info': None, 'external-data-source': None, 'haproxy-ocp': None, 'iostat': None, 'jmap': None, 'jstack': None, 'kvm-spinlock': None, 'kvmstat': None, 'kvmtrace': None, 'lockstat': None, 'mpstat': None, 'numastat': None, 'oc': None, 'openvswitch': None, 'pcp-transient': None, 'perf': None, 'pidstat': None, 'pprof': None, 'proc-interrupts': None, 'proc-sched_debug': None, 'proc-vmstat': None, 'prometheus-metrics': None, 'qemu-migrate': None, 'rabbit': None, 'sar': None, 'strace': None, 'sysfs': None, 'systemtap': None, 'tcpdump': None, 'turbostat': None, 'user-tool': None, 'virsh-migrate': None, 'vmstat': None}}, 'tools': {'mpstat': '', 'perf': '--record-opts="-a -freq=100 -g --event=branch-misses --event=cache-misses --event=instructions" --report-opts="-I -g"'}} +testhost.example.com 0000 DEBUG pbench-tool-meister driver -- params_key (tm-mygroup-testhost.example.com): {'benchmark_run_dir': '/var/tmp/pbench-test-utils/pbench/mock-run', 'channel_prefix': 'pbench-agent-cli', 'controller': 'testhost.example.com', 'group': 'mygroup', 'hostname': 'testhost.example.com', 'label': '', 'tds_hostname': 'localhost', 'tds_port': 8080, 'tool_metadata': "{'persistent': {'dcgm': {'collector': 'prometheus', 'port': '9400'}, 'node-exporter': {'collector': 'prometheus', 'port': '9100'}, 'pcp': {'collector': 'pcp', 'port': '44321'}}, 'transient': {'blktrace': None, 'bpftrace': None, 'cpuacct': None, 'disk': None, 'dm-cache': None, 'docker': None, 'docker-info': None, 'external-data-source': None, 'haproxy-ocp': None, 'iostat': None, 'jmap': None, 'jstack': None, 'kvm-spinlock': None, 'kvmstat': None, 'kvmtrace': None, 'lockstat': None, 'mpstat': None, 'numastat': None, 'oc': None, 'openvswitch': None, 'pcp-transient': None, 'perf': None, 'pidstat': None, 'pprof': None, 'proc-interrupts': None, 'proc-sched_debug': None, 'proc-vmstat': None, 'prometheus-metrics': None, 'qemu-migrate': None, 'rabbit': None, 'sar': None, 'strace': None, 'sysfs': None, 'systemtap': None, 'tcpdump': None, 'turbostat': None, 'user-tool': None, 'virsh-migrate': None, 'vmstat': None}}", 'tools': {'mpstat': '', 'perf': '--record-opts="-a -freq=100 -g --event=branch-misses --event=cache-misses --event=instructions" --report-opts="-I -g"'}} testhost.example.com 0001 DEBUG pbench-tool-meister __enter__ -- publish pbench-agent-cli-from-tms testhost.example.com 0002 DEBUG pbench-tool-meister __enter__ -- published pbench-agent-cli-from-tms testhost.example.com 0003 DEBUG pbench-tool-meister driver -- waiting ... diff --git a/lib/pbench/__init__.py b/lib/pbench/__init__.py index 505774d456..6edf4c5f9d 100644 --- a/lib/pbench/__init__.py +++ b/lib/pbench/__init__.py @@ -3,6 +3,7 @@ """ from configparser import ConfigParser, NoSectionError, NoOptionError +from typing import NamedTuple from pbench.common import configtools from pbench.common.exceptions import BadConfig @@ -71,3 +72,35 @@ def __init__(self, cfg_name): def get(self, *args, **kwargs): return self.conf.get(*args, **kwargs) + + +class PbenchNamedTuple(NamedTuple): + """A simple extension of a NamedTuple to provide a deterministic string + representation. + """ + + def __str__(self): + """A string containing a deterministic representation of a NamedTuple + + Convert the field names to dict keys, produce them in sorted order, and + render their object values. For most field types, this is done in the + usual way by Python. In the case of object references, use the object's + __str__() method if it has one other than the one inherited from the + fundamental "object" (that prints the object's address, which varies + from run to run). For other objects, print out the object's class name. + If anything goes wrong, produce a default value. + """ + ret_val = {} + for k, v in sorted(self._asdict().items()): + try: + if isinstance( + v, (dict, list, tuple, str, int, float, complex, bool, type(None)) + ): + ret_val[k] = v + elif v.__class__.__str__ != object.__str__: + ret_val[k] = str(v) + else: + ret_val[k] = f"<{v.__class__.__name__} object>" + except Exception: + ret_val[k] = "" + return str(ret_val) diff --git a/lib/pbench/agent/tool_data_sink.py b/lib/pbench/agent/tool_data_sink.py index c21379223c..adfbd83f52 100644 --- a/lib/pbench/agent/tool_data_sink.py +++ b/lib/pbench/agent/tool_data_sink.py @@ -22,7 +22,7 @@ import sys import tempfile from threading import Thread, Lock, Condition -from typing import Any, Dict, List, NamedTuple, Tuple +from typing import Any, Dict, List, Tuple from bottle import Bottle, ServerAdapter, request, abort from configparser import DuplicateSectionError @@ -32,6 +32,7 @@ import redis from wsgiref.simple_server import WSGIRequestHandler, make_server +from pbench import PbenchNamedTuple from pbench.agent.constants import ( tm_allowed_actions, tm_channel_suffix_from_client, @@ -728,13 +729,31 @@ def validate(self, directory): return local_dir -class ToolDataSinkParams(NamedTuple): +class ExternalEnvironment(PbenchNamedTuple): + """Encapsulation of the various external environment parameters needed by + the operation of the Tool Data Sink. + """ + + cp_path: str + hostname: str + pbench_bin: Path + pbench_run: str + PROG: str + tar_path: str + + +class ToolDataSinkParams(PbenchNamedTuple): + """Encapsulation of the parameter set provided to the Tool Data Sink by + the orchestrator. + """ + benchmark_run_dir: str bind_hostname: str - port: str channel_prefix: str + optional_md: Dict[str, str] + port: str tool_group: str - tool_metadata: ToolMetadata + tool_metadata: Dict[str, str] tool_trigger: str tools: Dict[str, str] @@ -748,39 +767,30 @@ class ToolDataSink(Bottle): _data_actions = frozenset(("send", "sysinfo")) @staticmethod - def fetch_params(params, pbench_run): + def fetch_params(params: Dict[str, Any]) -> ToolDataSinkParams: try: - _benchmark_run_dir = BenchmarkRunDir( - params["benchmark_run_dir"], pbench_run - ) - tdsp = ToolDataSinkParams( - benchmark_run_dir=_benchmark_run_dir, + return ToolDataSinkParams( + benchmark_run_dir=params["benchmark_run_dir"], bind_hostname=params["bind_hostname"], - port=params["port"], channel_prefix=params["channel_prefix"], + optional_md=params.get("optional_md", dict()), + port=params["port"], tool_group=params["group"], - tool_metadata=ToolMetadata.tool_md_from_dict(params["tool_metadata"]), + tool_metadata=params["tool_metadata"], tool_trigger=params["tool_trigger"], tools=params["tools"], ) except KeyError as exc: raise ToolDataSinkError(f"Invalid parameter block, missing key {exc}") - else: - return tdsp def __init__( self, - pbench_bin, - pbench_run, - hostname, - tar_path, - cp_path, - redis_server, - redis_host, - redis_port, - params, - optional_md, - logger, + ext_env: ExternalEnvironment, + redis_server: redis.Redis, + redis_host: str, + redis_port: int, + tdsp: ToolDataSinkParams, + logger: logging.Logger, ): """Constructor for the Tool Data Sink object - responsible for recording parameters, and setting up initial state. @@ -788,15 +798,19 @@ def __init__( """ super(ToolDataSink, self).__init__() # Save external state - self.pbench_bin = pbench_bin - self.hostname = hostname - self.tar_path = tar_path - self.cp_path = cp_path + self.pbench_bin = ext_env.pbench_bin + self.hostname = ext_env.hostname + self.tar_path = ext_env.tar_path + self.cp_path = ext_env.cp_path self.redis_server = redis_server self.redis_host = redis_host self.redis_port = redis_port - self.params = self.fetch_params(params, pbench_run) - self.optional_md = optional_md + self.benchmark_run_dir = BenchmarkRunDir( + tdsp.benchmark_run_dir, ext_env.pbench_run + ) + self.tool_metadata = ToolMetadata.tool_md_from_dict(tdsp.tool_metadata) + self.optional_md = tdsp.optional_md + self.params = tdsp self.logger = logger # Initialize internal state self.action = None @@ -933,7 +947,7 @@ def tm_log_capture(self): # logs from remote Tool Meisters. logger = logging.getLogger("tm_log_capture_thread") logger.setLevel(logging.WARNING) - tm_log_file = self.params.benchmark_run_dir.local / "tm" / "tm.logs" + tm_log_file = self.benchmark_run_dir.local / "tm" / "tm.logs" with tm_log_file.open("w") as fp: try: with self._lock: @@ -1025,8 +1039,8 @@ def record_tms(self, tms): The second thing we do is record all the data and metadata about the Tool Meisters in the ${benchmark_run_dir}/metadata.log file. """ - persistent_tools_l = self.params.tool_metadata.getPersistentTools() - transient_tools_l = self.params.tool_metadata.getTransientTools() + persistent_tools_l = self.tool_metadata.getPersistentTools() + transient_tools_l = self.tool_metadata.getTransientTools() for host, tm in tms.items(): assert tm["kind"] == "tm", f"what? {tm!r}" assert "tools" in tm, f"what? {tm!r}" @@ -1077,7 +1091,7 @@ def record_tms(self, tms): home = os.environ.get("HOME", "") if home: src = str(Path(home) / ".ssh" / "config") - dst = str(self.params.benchmark_run_dir.local / "ssh.config") + dst = str(self.benchmark_run_dir.local / "ssh.config") try: shutil.copyfile(src, dst) except FileNotFoundError: @@ -1087,7 +1101,7 @@ def record_tms(self, tms): # cp -L /etc/ssh/ssh_config ${dir}/ > /dev/null 2>&1 etc_ssh = Path("/etc") / "ssh" src = str(etc_ssh / "ssh_config") - dst = str(self.params.benchmark_run_dir.local / "ssh_config") + dst = str(self.benchmark_run_dir.local / "ssh_config") try: shutil.copyfile(src, dst) except FileNotFoundError: @@ -1124,14 +1138,14 @@ def record_tms(self, tms): self.cp_path, "-rL", "/etc/ssh/ssh_config.d", - f"{self.params.benchmark_run_dir.local}/", + f"{self.benchmark_run_dir.local}/", ], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) - mdlog_name = self.params.benchmark_run_dir.local / "metadata.log" + mdlog_name = self.benchmark_run_dir.local / "metadata.log" mdlog = MetadataLog() try: with mdlog_name.open("r") as fp: @@ -1142,22 +1156,22 @@ def record_tms(self, tms): section = "pbench" mdlog.add_section(section) - mdlog.set(section, "config", self.optional_md["config"]) - mdlog.set(section, "date", self.optional_md["date"]) - mdlog.set(section, "name", self.params.benchmark_run_dir.local.name) + mdlog.set(section, "config", self.optional_md.get("config", "")) + mdlog.set(section, "date", self.optional_md.get("date", "")) + mdlog.set(section, "name", self.benchmark_run_dir.local.name) version, seqno, sha1, hostdata = collect_local_info(self.pbench_bin) rpm_version = f"v{version}-{seqno}g{sha1}" mdlog.set(section, "rpm-version", rpm_version) rpm_versions = dict() rpm_versions[rpm_version] = 1 - mdlog.set(section, "script", self.optional_md["script"]) + mdlog.set(section, "script", self.optional_md.get("script", "")) section = "controller" mdlog.add_section(section) mdlog.set(section, "hostname", self.hostname) for hd_key, hd_val in sorted(hostdata.items()): mdlog.set(section, f"hostname-{hd_key}", hd_val) - mdlog.set(section, "ssh_opts", self.optional_md["ssh_opts"]) + mdlog.set(section, "ssh_opts", self.optional_md.get("ssh_opts", "")) section = "run" mdlog.add_section(section) @@ -1435,7 +1449,7 @@ def execute_action(self, action, directory_str, args, data): # the caller wants to report that it is stopping all the Tool # Meisters due to an interruption (SIGINT or otherwise). # - mdlog_name = self.params.benchmark_run_dir.local / "metadata.log" + mdlog_name = self.benchmark_run_dir.local / "metadata.log" mdlog = MetadataLog() try: with (mdlog_name).open("r") as fp: @@ -1454,7 +1468,7 @@ def execute_action(self, action, directory_str, args, data): if args["interrupt"]: # args["interrupt"] == True ==> run / run_interrupted mdlog.set(section, "run_interrupted", "true") - iterations = self.params.benchmark_run_dir.local / ".iterations" + iterations = self.benchmark_run_dir.local / ".iterations" try: iterations_val = iterations.read_text() except FileNotFoundError: @@ -1480,17 +1494,17 @@ def execute_action(self, action, directory_str, args, data): return try: - local_dir = self.params.benchmark_run_dir.validate(directory_str) - except self.params.benchmark_run_dir.Prefix: + local_dir = self.benchmark_run_dir.validate(directory_str) + except self.benchmark_run_dir.Prefix: self.logger.error( "action '%s' with invalid directory, '%s' (not a sub-directory of '%s')", action, directory_str, - self.params.benchmark_run_dir, + self.benchmark_run_dir, ) self._send_client_status(action, "directory not a sub-dir of run directory") return - except self.params.benchmark_run_dir.Exists: + except self.benchmark_run_dir.Exists: self.logger.error( "action '%s' with invalid directory, '%s' (does not exist)", action, @@ -1517,7 +1531,7 @@ def execute_action(self, action, directory_str, args, data): pcp_tools = [] persist_tools = self._tm_tracking[tm]["persistent_tools"] for tool in persist_tools: - tool_data = self.params.tool_metadata.getProperties(tool) + tool_data = self.tool_metadata.getProperties(tool) if tool_data["collector"] == "prometheus": prom_tools.append(tool) elif tool_data["collector"] == "pcp": @@ -1544,10 +1558,10 @@ def execute_action(self, action, directory_str, args, data): if prom_tool_dict: self._prom_server = PromCollector( self.pbench_bin, - self.params.benchmark_run_dir, + self.benchmark_run_dir, self.params.tool_group, prom_tool_dict, - self.params.tool_metadata, + self.tool_metadata, self.tar_path, logger=self.logger, ) @@ -1559,10 +1573,10 @@ def execute_action(self, action, directory_str, args, data): ) self._pcp_server = PcpCollector( self.pbench_bin, - self.params.benchmark_run_dir, + self.benchmark_run_dir, self.params.tool_group, pcp_tool_dict, - self.params.tool_metadata, + self.tool_metadata, self.tar_path, redis_host=self.redis_host, redis_port=self.redis_port, @@ -1864,7 +1878,7 @@ def get_logger(PROG: str, daemon: bool = False, level: str = "info") -> logging. return logger -class Arguments(NamedTuple): +class Arguments(PbenchNamedTuple): host: str port: int key: str @@ -1873,36 +1887,25 @@ class Arguments(NamedTuple): def driver( - PROG: str, + ext_env: ExternalEnvironment, redis_server: redis.Redis, parsed: Arguments, - pbench_bin: str, - pbench_run: str, - hostname: str, - tar_path: str, - cp_path: str, - params: Dict[str, Any], - optional_md: Dict[str, Any], + tdsp: ToolDataSinkParams, logger: logging.Logger = None, ): """Create and drive a Tool Data Sink instance""" if logger is None: - logger = get_logger(PROG, level=parsed.level) + logger = get_logger(ext_env.PROG, level=parsed.level) - logger.debug("params_key (%s): %r", parsed.key, params) + logger.debug("params_key (%s): %s", parsed.key, tdsp) try: with ToolDataSink( - pbench_bin, - pbench_run, - hostname, - tar_path, - cp_path, + ext_env, redis_server, parsed.host, parsed.port, - params, - optional_md, + tdsp, logger, ) as tds_app: tds_app.execute() @@ -1910,8 +1913,8 @@ def driver( if exc.errno == errno.EADDRINUSE: logger.error( "ERROR - tool data sink failed to start, %s:%s already in use", - params["bind_hostname"], - params["port"], + tdsp.bind_hostname, + tdsp.port, ) ret_val = 8 else: @@ -1926,16 +1929,10 @@ def driver( def daemon( - PROG: str, + ext_env: ExternalEnvironment, redis_server: redis.Redis, parsed: Arguments, - pbench_bin: str, - pbench_run: str, - hostname: str, - tar_path: str, - cp_path: str, - params: Dict[str, Any], - optional_md: Dict[str, Any], + tdsp: ToolDataSinkParams, ): """Daemonize a Tool Data Sink instance""" # Disconnect any existing connections to the Redis server. @@ -1946,10 +1943,10 @@ def daemon( sys.stderr.flush() sys.stdout.flush() - pidfile_name = f"{PROG}.pid" + pidfile_name = f"{ext_env.PROG}.pid" pfctx = pidfile.PIDFile(pidfile_name) - with open(f"{PROG}.out", "w") as sofp, open( - f"{PROG}.err", "w" + with open(f"{ext_env.PROG}.out", "w") as sofp, open( + f"{ext_env.PROG}.err", "w" ) as sefp, DaemonContext( stdout=sofp, stderr=sefp, @@ -1957,7 +1954,7 @@ def daemon( umask=0o022, pidfile=pfctx, ): - logger = get_logger(PROG, daemon=True, level=parsed.level) + logger = get_logger(ext_env.PROG, daemon=True, level=parsed.level) # We have to re-open the connection to the redis server now that we # are "daemonized". @@ -1975,16 +1972,10 @@ def daemon( else: logger.debug("reconstructed Redis server object") return driver( - PROG, + ext_env, redis_server, parsed, - pbench_bin, - pbench_run, - hostname, - tar_path, - cp_path, - params, - optional_md, + tdsp, logger=logger, ) @@ -2060,7 +2051,7 @@ def start(prog: Path, parsed: Arguments): # E.g. params = '{ "channel_prefix": "some-prefix", # "benchmark_run_dir": "/loo/goo" }' params = json.loads(params_str) - ToolDataSink.fetch_params(params, pbench_run) + tdsp = ToolDataSink.fetch_params(params) except Exception as ex: print( f"Unable to fetch and decode parameter key, {parsed.key}: {ex}", @@ -2068,20 +2059,21 @@ def start(prog: Path, parsed: Arguments): ) return 6 - optional_md = params.get("optional_md", dict()) + ext_env = ExternalEnvironment( + cp_path=cp_path, + hostname=hostname, + pbench_bin=pbench_bin, + pbench_run=pbench_run, + PROG=PROG, + tar_path=tar_path, + ) func = daemon if parsed.daemonize else driver ret_val = func( - PROG, + ext_env, redis_server, parsed, - pbench_bin, - pbench_run, - hostname, - tar_path, - cp_path, - params, - optional_md, + tdsp, ) return ret_val diff --git a/lib/pbench/agent/tool_meister.py b/lib/pbench/agent/tool_meister.py index 63829b65a9..da8c52f920 100644 --- a/lib/pbench/agent/tool_meister.py +++ b/lib/pbench/agent/tool_meister.py @@ -50,13 +50,14 @@ import tempfile import threading import time -from typing import Any, Dict, List, NamedTuple, Tuple +from typing import Any, Dict, List, Tuple from daemon import DaemonContext from pathlib import Path import pidfile import redis +from pbench import PbenchNamedTuple from pbench.agent.constants import ( tm_allowed_actions, tm_channel_suffix_to_tms, @@ -95,7 +96,7 @@ class object methods. pass -class InstallationResult(NamedTuple): +class InstallationResult(PbenchNamedTuple): returncode: int output: str @@ -669,7 +670,7 @@ class ToolMeisterError(Exception): pass -class ToolMeisterParams(NamedTuple): +class ToolMeisterParams(PbenchNamedTuple): benchmark_run_dir: str channel_prefix: str tds_hostname: str @@ -791,7 +792,7 @@ def __init__( tmp_dir: Path, tar_path: str, sysinfo_dump: str, - params: Dict[str, Any], + tm_params: ToolMeisterParams, redis_server: redis.Redis, logger: logging.Logger, ): @@ -810,7 +811,7 @@ def __init__( self._tmp_dir = tmp_dir self.tar_path = tar_path self.sysinfo_dump = sysinfo_dump - self._params = self.fetch_params(params) + self._params = tm_params self._rs = redis_server self.logger = logger self._usable_tools = dict() @@ -1798,7 +1799,7 @@ def get_logger(PROG: str, daemon: bool = False, level: str = "info") -> logging. return logger -class Arguments(NamedTuple): +class Arguments(PbenchNamedTuple): host: str port: int key: str @@ -1813,7 +1814,7 @@ def driver( pbench_install_dir: Path, tmp_dir: Path, parsed: Arguments, - params: Dict[str, Any], + tm_params: ToolMeisterParams, redis_server: redis.Redis, logger: logging.Logger = None, ): @@ -1823,10 +1824,10 @@ def driver( # Add a logging handler to send logs back to the Redis server, with each # log entry prepended with the given hostname parameter. - channel_prefix = params["channel_prefix"] + channel_prefix = tm_params.channel_prefix rh = RedisHandler( channel=f"{channel_prefix}-{tm_channel_suffix_to_logging}", - hostname=params["hostname"], + hostname=tm_params.hostname, redis_client=redis_server, ) redis_fmtstr = fmtstr_ut if os.environ.get("_PBENCH_UNIT_TESTS") else fmtstr @@ -1834,7 +1835,7 @@ def driver( rh.setFormatter(rhf) logger.addHandler(rh) - logger.debug("params_key (%s): %r", parsed.key, params) + logger.debug("params_key (%s): %s", parsed.key, tm_params) # FIXME: we should establish signal handlers that do the following: # a. handle graceful termination (TERM, INT, QUIT) @@ -1847,7 +1848,7 @@ def driver( tmp_dir, tar_path, sysinfo_dump, - params, + tm_params, redis_server, logger, ) as tm: @@ -1884,7 +1885,7 @@ def daemon( pbench_install_dir: Path, tmp_dir: Path, parsed: Arguments, - params: Dict[str, Any], + tm_params: ToolMeisterParams, redis_server: redis.Redis, ): """Daemonize a Tool Meister instance""" @@ -1897,7 +1898,7 @@ def daemon( sys.stderr.flush() sys.stdout.flush() - if params["hostname"] != params["controller"]: + if tm_params.hostname != tm_params.controller: working_dir = tmp_dir else: working_dir = Path(".") @@ -1943,7 +1944,7 @@ def daemon( pbench_install_dir, tmp_dir, parsed, - params, + tm_params, redis_server, logger=logger, ) @@ -2029,10 +2030,7 @@ def start(prog: Path, parsed: Arguments) -> int: # Wait for the key to show up with a value. params_str = wait_for_conn_and_key(redis_server, parsed.key, PROG) params = json.loads(params_str) - # Validate the Tool Meister parameters without constructing an object - # just yet, as we want to make sure we can talk to the redis server - # before we go through the trouble of daemonizing below. - ToolMeister.fetch_params(params) + tm_params = ToolMeister.fetch_params(params) except Exception as exc: print( f"{PROG}: Unable to fetch and decode parameter key, '{parsed.key}': {exc}", @@ -2048,7 +2046,7 @@ def start(prog: Path, parsed: Arguments) -> int: pbench_install_dir, tmp_dir, parsed, - params, + tm_params, redis_server, ) diff --git a/lib/pbench/agent/toolmetadata.py b/lib/pbench/agent/toolmetadata.py index 3c27055e13..b671511ab4 100644 --- a/lib/pbench/agent/toolmetadata.py +++ b/lib/pbench/agent/toolmetadata.py @@ -106,3 +106,6 @@ def getProperties(self, tool): except KeyError: tool_prop = None return tool_prop + + def __str__(self): + return str(self.getFullData()) diff --git a/lib/pbench/test/unit/agent/test_tool_meister.py b/lib/pbench/test/unit/agent/test_tool_meister.py index b39be12c3f..f28d4ca9ef 100644 --- a/lib/pbench/test/unit/agent/test_tool_meister.py +++ b/lib/pbench/test/unit/agent/test_tool_meister.py @@ -1054,7 +1054,7 @@ def tool_meister(): tmp_dir=MockedPath(), tar_path="tar_path", sysinfo_dump=None, - params=tm_params, + tm_params=ToolMeister.fetch_params(tm_params), redis_server=None, logger=logging.getLogger(), )