diff --git a/_topic_maps/_topic_map.yml b/_topic_maps/_topic_map.yml index a79c5a54db3f..c3aef93c0b0c 100644 --- a/_topic_maps/_topic_map.yml +++ b/_topic_maps/_topic_map.yml @@ -2901,14 +2901,21 @@ Topics: File: what-huge-pages-do-and-how-they-are-consumed-by-apps Distros: openshift-origin,openshift-enterprise - Name: Low latency tuning - File: cnf-low-latency-tuning + Dir: low_latency_tuning Distros: openshift-origin,openshift-enterprise -- Name: Performing latency tests for platform verification - File: cnf-performing-platform-verification-latency-tests + Topics: + - Name: Understanding low latency + File: cnf-understanding-low-latency + - Name: Tuning nodes for low latency with the performance profile + File: cnf-tuning-low-latency-nodes-with-perf-profile + - Name: Provisioning real-time and low latency workloads + File: cnf-provisioning-low-latency-workloads + - Name: Debugging low latency tuning + File: cnf-debugging-low-latency-tuning-status + - Name: Performing latency tests for platform verification + File: cnf-performing-platform-verification-latency-tests - Name: Improving cluster stability in high latency environments using worker latency profiles File: scaling-worker-latency-profiles -- Name: Creating a performance profile - File: cnf-create-performance-profiles Distros: openshift-origin,openshift-enterprise - Name: Workload partitioning File: enabling-workload-partitioning diff --git a/installing/installing-preparing.adoc b/installing/installing-preparing.adoc index 15fe1790edc1..edb33f9b9696 100644 --- a/installing/installing-preparing.adoc +++ b/installing/installing-preparing.adoc @@ -112,7 +112,7 @@ For a production cluster, you must configure the following integrations: [id="installing-preparing-cluster-for-workloads"] == Preparing your cluster for workloads -Depending on your workload needs, you might need to take extra steps before you begin deploying applications. For example, after you prepare infrastructure to support your application xref:../cicd/builds/build-strategies.adoc#build-strategies[build strategy], you might need to make provisions for xref:../scalability_and_performance/cnf-low-latency-tuning.adoc#cnf-low-latency-tuning[low-latency] workloads or to xref:../nodes/pods/nodes-pods-secrets.adoc#nodes-pods-secrets[protect sensitive workloads]. You can also configure xref:../observability/monitoring/enabling-monitoring-for-user-defined-projects.adoc#enabling-monitoring-for-user-defined-projects[monitoring] for application workloads. +Depending on your workload needs, you might need to take extra steps before you begin deploying applications. For example, after you prepare infrastructure to support your application xref:../cicd/builds/build-strategies.adoc#build-strategies[build strategy], you might need to make provisions for xref:../scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc#cnf-low-latency-perf-profile[low-latency] workloads or to xref:../nodes/pods/nodes-pods-secrets.adoc#nodes-pods-secrets[protect sensitive workloads]. You can also configure xref:../observability/monitoring/enabling-monitoring-for-user-defined-projects.adoc#enabling-monitoring-for-user-defined-projects[monitoring] for application workloads. If you plan to run xref:../windows_containers/enabling-windows-container-workloads.adoc#enabling-windows-container-workloads[Windows workloads], you must enable xref:../networking/ovn_kubernetes_network_provider/configuring-hybrid-networking.adoc#configuring-hybrid-networking[hybrid networking with OVN-Kubernetes] during the installation process; hybrid networking cannot be enabled after your cluster is installed. [id="supported-installation-methods-for-different-platforms"] diff --git a/installing/installing_openstack/installing-openstack-nfv-preparing.adoc b/installing/installing_openstack/installing-openstack-nfv-preparing.adoc index 71a1417c86c4..41d7fe222733 100644 --- a/installing/installing_openstack/installing-openstack-nfv-preparing.adoc +++ b/installing/installing_openstack/installing-openstack-nfv-preparing.adoc @@ -42,4 +42,5 @@ After you perform preinstallation tasks, install your cluster by following the m * Consult the following references after you deploy your cluster to improve its performance: ** xref:../../networking/hardware_networks/using-dpdk-and-rdma.adoc#nw-openstack-ovs-dpdk-testpmd-pod_using-dpdk-and-rdma[A test pod template for clusters that use OVS-DPDK on OpenStack]. ** xref:../../networking/hardware_networks/add-pod.adoc#nw-openstack-sr-iov-testpmd-pod_add-pod[A test pod template for clusters that use SR-IOV on OpenStack]. -** xref:../../scalability_and_performance/cnf-create-performance-profiles.adoc#installation-openstack-ovs-dpdk-performance-profile_cnf-create-performance-profiles[A performance profile template for clusters that use OVS-DPDK on OpenStack]. +** xref:../../scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc#installation-openstack-ovs-dpdk-performance-profile_cnf-low-latency-perf-profile[A performance profile template for clusters that use OVS-DPDK on OpenStack] +. diff --git a/modules/cnf-about-hyperthreading-for-low-latency-and-real-time-applications.adoc b/modules/cnf-about-hyperthreading-for-low-latency-and-real-time-applications.adoc new file mode 100644 index 000000000000..f906e5e0f1c6 --- /dev/null +++ b/modules/cnf-about-hyperthreading-for-low-latency-and-real-time-applications.adoc @@ -0,0 +1,16 @@ +// Module included in the following assemblies: +// +// * scalability_and_performance/low_latency_tuning/cnf-understanding-low-latency.adoc + +:_mod-docs-content-type: CONCEPT +[id="cnf-about-hyper-threading-for-low-latency-and-real-time-applications_{context}"] += About Hyper-Threading for low latency and real-time applications + +Hyper-Threading is an Intel processor technology that allows a physical CPU processor core to function as two logical cores, executing two independent threads simultaneously. Hyper-Threading allows for better system throughput for certain workload types where parallel processing is beneficial. The default {product-title} configuration expects Hyper-Threading to be enabled. + +For telecommunications applications, it is important to design your application infrastructure to minimize latency as much as possible. Hyper-Threading can slow performance times and negatively affect throughput for compute-intensive workloads that require low latency. Disabling Hyper-Threading ensures predictable performance and can decrease processing times for these workloads. + +[NOTE] +==== +Hyper-Threading implementation and configuration differs depending on the hardware you are running {product-title} on. Consult the relevant host hardware tuning information for more details of the Hyper-Threading implementation specific to that hardware. Disabling Hyper-Threading can increase the cost per core of the cluster. +==== diff --git a/modules/cnf-about-irq-affinity-setting.adoc b/modules/cnf-about-irq-affinity-setting.adoc index 451004ef5bce..315302608a6a 100644 --- a/modules/cnf-about-irq-affinity-setting.adoc +++ b/modules/cnf-about-irq-affinity-setting.adoc @@ -1,10 +1,11 @@ // Module included in the following assemblies: // -// scalability_and_performance/cnf-low-latency-tuning.adoc +// * scalability_and_performance/cnf-low-latency-tuning.adoc +// * scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc :_mod-docs-content-type: CONCEPT [id="about_irq_affinity_setting_{context}"] -= About support of IRQ affinity setting += Finding the effective IRQ affinity setting for a node Some IRQ controllers lack support for IRQ affinity setting and will always expose all online CPUs as the IRQ mask. These IRQ controllers effectively run on CPU 0. @@ -60,4 +61,4 @@ $ find /proc/irq -name effective_affinity -printf "%p: " -exec cat {} \; /proc/irq/34/effective_affinity: 2 ---- -Some drivers use `managed_irqs`, whose affinity is managed internally by the kernel and userspace cannot change the affinity. In some cases, these IRQs might be assigned to isolated CPUs. For more information about `managed_irqs`, see link:https://access.redhat.com/solutions/4819541[Affinity of managed interrupts cannot be changed even if they target isolated CPU]. \ No newline at end of file +Some drivers use `managed_irqs`, whose affinity is managed internally by the kernel and userspace cannot change the affinity. In some cases, these IRQs might be assigned to isolated CPUs. For more information about `managed_irqs`, see link:https://access.redhat.com/solutions/4819541[Affinity of managed interrupts cannot be changed even if they target isolated CPU]. diff --git a/modules/cnf-about-the-profile-creator-tool.adoc b/modules/cnf-about-the-profile-creator-tool.adoc index 3ff072199913..9a42f66ea8fb 100644 --- a/modules/cnf-about-the-profile-creator-tool.adoc +++ b/modules/cnf-about-the-profile-creator-tool.adoc @@ -1,6 +1,6 @@ // Module included in the following assemblies: -// Epic CNF-792 (4.8) -// * scalability_and_performance/cnf-create-performance-profiles.adoc +// +// * scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc :_mod-docs-content-type: CONCEPT [id="cnf-about-the-profile-creator-tool_{context}"] diff --git a/modules/cnf-about_hyperthreading_for_low_latency_and_real_time_applications.adoc b/modules/cnf-about_hyperthreading_for_low_latency_and_real_time_applications.adoc deleted file mode 100644 index bf42fa0e49a0..000000000000 --- a/modules/cnf-about_hyperthreading_for_low_latency_and_real_time_applications.adoc +++ /dev/null @@ -1,16 +0,0 @@ -// Module included in the following assemblies: -// -// scalability_and_performance/cnf-low-latency-tuning.adoc - -:_mod-docs-content-type: CONCEPT -[id="about_hyperthreading_for_low_latency_and_real_time_applications_{context}"] -= About hyperthreading for low latency and real-time applications - -Hyperthreading is an Intel processor technology that allows a physical CPU processor core to function as two logical cores, executing two independent threads simultaneously. Hyperthreading allows for better system throughput for certain workload types where parallel processing is beneficial. The default {product-title} configuration expects hyperthreading to be enabled by default. - -For telecommunications applications, it is important to design your application infrastructure to minimize latency as much as possible. Hyperthreading can slow performance times and negatively affect throughput for compute intensive workloads that require low latency. Disabling hyperthreading ensures predictable performance and can decrease processing times for these workloads. - -[NOTE] -==== -Hyperthreading implementation and configuration differs depending on the hardware you are running {product-title} on. Consult the relevant host hardware tuning information for more details of the hyperthreading implementation specific to that hardware. Disabling hyperthreading can increase the cost per core of the cluster. -==== diff --git a/modules/cnf-adjusting-nic-queues-with-the-performance-profile.adoc b/modules/cnf-adjusting-nic-queues-with-the-performance-profile.adoc index bdae2c69dc91..63ae9237549a 100644 --- a/modules/cnf-adjusting-nic-queues-with-the-performance-profile.adoc +++ b/modules/cnf-adjusting-nic-queues-with-the-performance-profile.adoc @@ -1,6 +1,6 @@ // Module included in the following assemblies: -//CNF-1483 (4.8) -// * scalability_and_performance/low-latency-tuning.adoc +// +// * scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc :_mod-docs-content-type: PROCEDURE [id="adjusting-nic-queues-with-the-performance-profile_{context}"] @@ -165,4 +165,4 @@ spec: [source,terminal] ---- $ oc apply -f .yaml ----- \ No newline at end of file +---- diff --git a/modules/cnf-allocating-multiple-huge-page-sizes.adoc b/modules/cnf-allocating-multiple-huge-page-sizes.adoc index e79e3da910f2..543c11ad5187 100644 --- a/modules/cnf-allocating-multiple-huge-page-sizes.adoc +++ b/modules/cnf-allocating-multiple-huge-page-sizes.adoc @@ -1,7 +1,7 @@ -// CNF-538 Promote Multiple Huge Pages Sizes for Pods and Containers to beta // Module included in the following assemblies: // -// *scalability_and_performance/cnf-low-latency-tuning.adoc +// * scalability_and_performance/cnf-low-latency-tuning.adoc +// * scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc [id="cnf-allocating-multiple-huge-page-sizes_{context}"] = Allocating multiple huge page sizes @@ -22,4 +22,4 @@ spec: - count: 4 node: 1 size: 1G ----- \ No newline at end of file +---- diff --git a/modules/cnf-collecting-low-latency-tuning-debugging-data-for-red-hat-support.adoc b/modules/cnf-collecting-low-latency-tuning-debugging-data-for-red-hat-support.adoc index e52c523cdd4a..07dd7ca884ad 100644 --- a/modules/cnf-collecting-low-latency-tuning-debugging-data-for-red-hat-support.adoc +++ b/modules/cnf-collecting-low-latency-tuning-debugging-data-for-red-hat-support.adoc @@ -1,7 +1,7 @@ -// CNF-643 Support and debugging tools for CNF // Module included in the following assemblies: // -// *scalability_and_performance/cnf-low-latency-tuning.adoc +// * scalability_and_performance/cnf-low-latency-tuning.adoc +// * scalability_and_performance/low_latency_tuning/cnf-debugging-low-latency-tuning-status.adoc :_mod-docs-content-type: PROCEDURE [id="cnf-collecting-low-latency-tuning-debugging-data-for-red-hat-support_{context}"] diff --git a/modules/cnf-configure_for_irq_dynamic_load_balancing.adoc b/modules/cnf-configure_for_irq_dynamic_load_balancing.adoc index f751153763ad..8e138b603f38 100644 --- a/modules/cnf-configure_for_irq_dynamic_load_balancing.adoc +++ b/modules/cnf-configure_for_irq_dynamic_load_balancing.adoc @@ -176,4 +176,4 @@ find /proc/irq/ -name smp_affinity_list -exec sh -c 'i="$1"; mask=$(cat $i); fil /proc/irq/28/smp_affinity_list: 1 /proc/irq/29/smp_affinity_list: 0 /proc/irq/30/smp_affinity_list: 0-5 ----- \ No newline at end of file +---- diff --git a/modules/cnf-configuring-high-priority-workload-pods.adoc b/modules/cnf-configuring-high-priority-workload-pods.adoc new file mode 100644 index 000000000000..19e932243165 --- /dev/null +++ b/modules/cnf-configuring-high-priority-workload-pods.adoc @@ -0,0 +1,55 @@ +// Module included in the following assemblies: +// +// * scalability_and_performance/low_latency_tuning/cnf-provisioning-low-latency-workloads.adoc + +:_mod-docs-content-type: PROCEDURE +[id="cnf-configuring-high-priority-workload-pods_{context}"] += Disabling power saving mode for high priority pods + +You can configure pods to ensure that high priority workloads are unaffected when you configure power saving for the node that the workloads run on. + +When you configure a node with a power saving configuration, you must configure high priority workloads with performance configuration at the pod level, which means that the configuration applies to all the cores used by the pod. + +By disabling P-states and C-states at the pod level, you can configure high priority workloads for best performance and lowest latency. + +.Configuration for high priority workloads +[cols="1,2,3", options="header"] + +|=== +| Annotation | Possible Values | Description + +|`cpu-c-states.crio.io:` a| * `"enable"` +* `"disable"` +* `"max_latency:microseconds"` | This annotation allows you to enable or disable C-states for each CPU. Alternatively, you can also specify a maximum latency in microseconds for the C-states. For example, enable C-states with a maximum latency of 10 microseconds with the setting `cpu-c-states.crio.io`: `"max_latency:10"`. Set the value to `"disable"` to provide the best performance for a pod. + +| `cpu-freq-governor.crio.io:` | Any supported `cpufreq governor`. | Sets the `cpufreq` governor for each CPU. The `"performance"` governor is recommended for high priority workloads. +|=== + +.Prerequisites + +* You have configured power saving in the performance profile for the node where the high priority workload pods are scheduled. + +.Procedure + +. Add the required annotations to your high priority workload pods. The annotations override the `default` settings. ++ +.Example high priority workload annotation +[source,yaml] +---- +apiVersion: v1 +kind: Pod +metadata: + #... + annotations: + #... + cpu-c-states.crio.io: "disable" + cpu-freq-governor.crio.io: "performance" + #... + #... +spec: + #... + runtimeClassName: performance- + #... +---- + +. Restart the pods to apply the annotation. diff --git a/modules/cnf-configuring-huge-pages.adoc b/modules/cnf-configuring-huge-pages.adoc index 6b631d888f23..896221ab42e7 100644 --- a/modules/cnf-configuring-huge-pages.adoc +++ b/modules/cnf-configuring-huge-pages.adoc @@ -1,6 +1,7 @@ // Module included in the following assemblies: -//CNF-78 (4.4) +// // * scalability_and_performance/cnf-low-latency-tuning.adoc +// * scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc [id="cnf-configuring-huge-pages_{context}"] = Configuring huge pages diff --git a/modules/configuring_hyperthreading_for_a_cluster.adoc b/modules/cnf-configuring-hyperthreading-for-a-cluster.adoc similarity index 65% rename from modules/configuring_hyperthreading_for_a_cluster.adoc rename to modules/cnf-configuring-hyperthreading-for-a-cluster.adoc index 47b335b84ed6..d4229958c213 100644 --- a/modules/configuring_hyperthreading_for_a_cluster.adoc +++ b/modules/cnf-configuring-hyperthreading-for-a-cluster.adoc @@ -1,21 +1,21 @@ // Module included in the following assemblies: // -// scalability_and_performance/cnf-low-latency-tuning.adoc +// * scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc :_mod-docs-content-type: PROCEDURE -[id="configuring_hyperthreading_for_a_cluster_{context}"] -= Configuring hyperthreading for a cluster +[id="cnf-configuring-hyperthreading-for-a-cluster_{context}"] += Configuring Hyper-Threading for a cluster -To configure hyperthreading for an {product-title} cluster, set the CPU threads in the performance profile to the same cores that are configured for the reserved or isolated CPU pools. +To configure Hyper-Threading for an {product-title} cluster, set the CPU threads in the performance profile to the same cores that are configured for the reserved or isolated CPU pools. [NOTE] ==== -If you configure a performance profile, and subsequently change the hyperthreading configuration for the host, ensure that you update the CPU `isolated` and `reserved` fields in the `PerformanceProfile` YAML to match the new configuration. +If you configure a performance profile, and subsequently change the Hyper-Threading configuration for the host, ensure that you update the CPU `isolated` and `reserved` fields in the `PerformanceProfile` YAML to match the new configuration. ==== [WARNING] ==== -Disabling a previously enabled host hyperthreading configuration can cause the CPU core IDs listed in the `PerformanceProfile` YAML to be incorrect. This incorrect configuration can cause the node to become unavailable because the listed CPUs can no longer be found. +Disabling a previously enabled host Hyper-Threading configuration can cause the CPU core IDs listed in the `PerformanceProfile` YAML to be incorrect. This incorrect configuration can cause the node to become unavailable because the listed CPUs can no longer be found. ==== .Prerequisites @@ -51,7 +51,7 @@ CPU NODE SOCKET CORE L1d:L1i:L2:L3 ONLINE MAXMHZ MINMHZ + In this example, there are eight logical CPU cores running on four physical CPU cores. CPU0 and CPU4 are running on physical Core0, CPU1 and CPU5 are running on physical Core 1, and so on. + -Alternatively, to view the threads that are set for a particular physical CPU core (`cpu0` in the example below), open a command prompt and run the following: +Alternatively, to view the threads that are set for a particular physical CPU core (`cpu0` in the example below), open a shell prompt and run the following: + [source,terminal] ---- @@ -83,20 +83,23 @@ The reserved and isolated CPU pools must not overlap and together must span all [IMPORTANT] ==== -Hyperthreading is enabled by default on most Intel processors. If you enable hyperthreading, all threads processed by a particular core must be isolated or processed on the same core. +Hyper-Threading is enabled by default on most Intel processors. If you enable Hyper-Threading, all threads processed by a particular core must be isolated or processed on the same core. + +When Hyper-Threading is enabled, all guaranteed pods must use multiples of the simultaneous multi-threading (SMT) level to avoid a "noisy neighbor" situation that can cause the pod to fail. +See link:https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/#static-policy-options[Static policy options] for more information. ==== [id="disabling_hyperthreading_for_low_latency_applications_{context}"] -== Disabling hyperthreading for low latency applications +== Disabling Hyper-Threading for low latency applications -When configuring clusters for low latency processing, consider whether you want to disable hyperthreading before you deploy the cluster. To disable hyperthreading, do the following: +When configuring clusters for low latency processing, consider whether you want to disable Hyper-Threading before you deploy the cluster. To disable Hyper-Threading, perform the following steps: . Create a performance profile that is appropriate for your hardware and topology. . Set `nosmt` as an additional kernel argument. The following example performance profile illustrates this setting: + [source,yaml] ---- -apiVersion: performance.openshift.io/v2 +apiVersion: performance.openshift.io/v2 kind: PerformanceProfile metadata: name: example-performanceprofile diff --git a/modules/cnf-configuring-power-saving-for-nodes.adoc b/modules/cnf-configuring-power-saving-for-nodes.adoc new file mode 100644 index 000000000000..a3ff8cff405a --- /dev/null +++ b/modules/cnf-configuring-power-saving-for-nodes.adoc @@ -0,0 +1,77 @@ +// Module included in the following assemblies: +// +// * scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc + +:_mod-docs-content-type: PROCEDURE +[id="cnf-configuring-power-saving-for-nodes_{context}"] += Configuring power saving for nodes that run colocated high and low priority workloads + +You can enable power savings for a node that has low priority workloads that are colocated with high priority workloads without impacting the latency or throughput of the high priority workloads. Power saving is possible without modifications to the workloads themselves. + +[IMPORTANT] +==== +The feature is supported on Intel Ice Lake and later generations of Intel CPUs. The capabilities of the processor might impact the latency and throughput of the high priority workloads. +==== + +.Prerequisites + +* You enabled C-states and operating system controlled P-states in the BIOS + +.Procedure + +. Generate a `PerformanceProfile` with the `per-pod-power-management` argument set to `true`: ++ +[source,terminal,subs="attributes+"] +---- +$ podman run --entrypoint performance-profile-creator -v \ +/must-gather:/must-gather:z registry.redhat.io/openshift4/ose-cluster-node-tuning-operator:v{product-version} \ +--mcp-name=worker-cnf --reserved-cpu-count=20 --rt-kernel=true \ +--split-reserved-cpus-across-numa=false --topology-manager-policy=single-numa-node \ +--must-gather-dir-path /must-gather --power-consumption-mode=low-latency \ <1> +--per-pod-power-management=true > my-performance-profile.yaml +---- +<1> The `power-consumption-mode` argument must be `default` or `low-latency` when the `per-pod-power-management` argument is set to `true`. + ++ +.Example `PerformanceProfile` with `perPodPowerManagement` + +[source,yaml] +---- +apiVersion: performance.openshift.io/v2 +kind: PerformanceProfile +metadata: + name: performance +spec: + [.....] + workloadHints: + realTime: true + highPowerConsumption: false + perPodPowerManagement: true +---- + +. Set the default `cpufreq` governor as an additional kernel argument in the `PerformanceProfile` custom resource (CR): ++ +[source,yaml] +---- +apiVersion: performance.openshift.io/v2 +kind: PerformanceProfile +metadata: + name: performance +spec: + ... + additionalKernelArgs: + - cpufreq.default_governor=schedutil <1> +---- +<1> Using the `schedutil` governor is recommended, however, you can use other governors such as the `ondemand` or `powersave` governors. + +. Set the maximum CPU frequency in the `TunedPerformancePatch` CR: ++ +[source,yaml] +---- +spec: + profile: + - data: | + [sysfs] + /sys/devices/system/cpu/intel_pstate/max_perf_pct = <1> +---- +<1> The `max_perf_pct` controls the maximum frequency that the `cpufreq` driver is allowed to set as a percentage of the maximum supported cpu frequency. This value applies to all CPUs. You can check the maximum supported frequency in `/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq`. As a starting point, you can use a percentage that caps all CPUs at the `All Cores Turbo` frequency. The `All Cores Turbo` frequency is the frequency that all cores will run at when the cores are all fully occupied. diff --git a/modules/cnf-configuring-workload-hints.adoc b/modules/cnf-configuring-workload-hints.adoc index 4c2b1c78bfeb..0cff6d4e1927 100644 --- a/modules/cnf-configuring-workload-hints.adoc +++ b/modules/cnf-configuring-workload-hints.adoc @@ -1,10 +1,11 @@ // Module included in the following assemblies: // -// scalability_and_performance/cnf-low-latency-tuning.adoc +// * scalability_and_performance/cnf-low-latency-tuning.adoc +// * scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc :_mod-docs-content-type: CONCEPT [id="configuring-workload-hints_{context}"] -= Configuring workload hints manually += Configuring node power consumption and realtime processing with workload hints .Procedure @@ -30,4 +31,53 @@ [NOTE] ==== When the `realTime` workload hint flag is set to `true` in a performance profile, add the `cpu-quota.crio.io: disable` annotation to every guaranteed pod with pinned CPUs. This annotation is necessary to prevent the degradation of the process performance within the pod. If the `realTime` workload hint is not explicitly set then it defaults to `true`. -==== \ No newline at end of file +==== + +The following table describes how combinations of power consumption and real-time settings impact latency. + +.Impact of combinations of power consumption and real-time settings on latency +[cols="1,1,1,1",options="header"] +|=== +|Performance Profile creator setting |Hint |Environment |Description + +|Default +a|[source,terminal] +---- +workloadHints: +highPowerConsumption: false +realTime: false +---- +|High throughput cluster without latency requirements +|Performance achieved through CPU partitioning only. + +|Low-latency +a|[source,terminal] +---- +workloadHints: +highPowerConsumption: false +realTime: true +---- +|Regional data-centers +|Both energy savings and low-latency are desirable: compromise between power management, latency and throughput. + +|Ultra-low-latency +a|[source,terminal] +---- +workloadHints: +highPowerConsumption: true +realTime: true +---- +|Far edge clusters, latency critical workloads +|Optimized for absolute minimal latency and maximum determinism at the cost of increased power consumption. + +|Per-pod power management +a|[source,terminal] +---- +workloadHints: +realTime: true +highPowerConsumption: false +perPodPowerManagement: true +---- +|Critical and non-critical workloads +|Allows for power management per pod. +|=== diff --git a/modules/cnf-cpu-infra-container.adoc b/modules/cnf-cpu-infra-container.adoc index 71f55c08d876..3c061e155ac1 100644 --- a/modules/cnf-cpu-infra-container.adoc +++ b/modules/cnf-cpu-infra-container.adoc @@ -1,6 +1,7 @@ // Module included in the following assemblies: // -// scalability_and_performance/cnf-low-latency-tuning.adoc +// * scalability_and_performance/cnf-low-latency-tuning.adoc +// * scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc :_mod-docs-content-type: PROCEDURE [id="cnf-cpu-infra-container_{context}"] diff --git a/modules/cnf-debugging-low-latency-cnf-tuning-status.adoc b/modules/cnf-debugging-low-latency-cnf-tuning-status.adoc index 4dc6937eb0d0..139224fa529e 100644 --- a/modules/cnf-debugging-low-latency-cnf-tuning-status.adoc +++ b/modules/cnf-debugging-low-latency-cnf-tuning-status.adoc @@ -1,9 +1,7 @@ // Module included in the following assemblies: -// Epic CNF-303 (4.5) -// scalability_and_performance/cnf-low-latency-tuning.adoc -//CNF-303 Performance add-ons status CNF-372 -//Performance Addon Operator Detailed Status -//See: https://issues.redhat.com/browse/CNF-379 (Yanir Quinn) +// +// * scalability_and_performance/cnf-low-latency-tuning.adoc +// * scalability_and_performance/low_latency_tuning/cnf-debugging-low-latency-tuning-status.adoc [id="cnf-debugging-low-latency-cnf-tuning-status_{context}"] = Debugging low latency CNF tuning status diff --git a/modules/cnf-disabling-cpu-cfs-quota.adoc b/modules/cnf-disabling-cpu-cfs-quota.adoc new file mode 100644 index 000000000000..894ee5511a2c --- /dev/null +++ b/modules/cnf-disabling-cpu-cfs-quota.adoc @@ -0,0 +1,26 @@ +// Module included in the following assemblies: +// +// * scalability_and_performance/low_latency_tuning/cnf-provisioning-low-latency-workloads.adoc + +[id="cnf-disabling-cpu-cfs-quota_{context}"] += Disabling CPU CFS quota + +To eliminate CPU throttling for pinned pods, create a pod with the `cpu-quota.crio.io: "disable"` annotation. This annotation disables the CPU completely fair scheduler (CFS) quota when the pod runs. + +.Example pod specification with `cpu-quota.crio.io` disabled +[source,yaml] +---- +apiVersion: v1 +kind: Pod +metadata: + annotations: + cpu-quota.crio.io: "disable" +spec: + runtimeClassName: performance- +#... +---- + +[NOTE] +==== +Only disable CPU CFS quota when the CPU manager static policy is enabled and for pods with guaranteed QoS that use whole CPUs. For example, pods that contain CPU-pinned containers. Otherwise, disabling CPU CFS quota can affect the performance of other containers in the cluster. +==== diff --git a/modules/cnf-disabling-interrupt-processing-for-individual-pods.adoc b/modules/cnf-disabling-interrupt-processing-for-individual-pods.adoc new file mode 100644 index 000000000000..ac8eb8893885 --- /dev/null +++ b/modules/cnf-disabling-interrupt-processing-for-individual-pods.adoc @@ -0,0 +1,24 @@ +// Module included in the following assemblies: +// +// * scalability_and_performance/low_latency_tuning/cnf-provisioning-low-latency-workloads.adoc + +:_mod-docs-content-type: PROCEDURE +[id="cnf-disabling-interrupt-processing-for-individual-pods_{context}"] += Disabling interrupt processing for CPUs where pinned containers are running + +To achieve low latency for workloads, some containers require that the CPUs they are pinned to do not process device interrupts. A pod annotation, `irq-load-balancing.crio.io`, is used to define whether device interrupts are processed or not on the CPUs where the pinned containers are running. When configured, CRI-O disables device interrupts where the pod containers are running. + +To disable interrupt processing for CPUs where containers belonging to individual pods are pinned, ensure that `globallyDisableIrqLoadBalancing` is set to `false` in the performance profile. Then, in the pod specification, set the `irq-load-balancing.crio.io` pod annotation to `disable`. + +The following pod specification contains this annotation: +[source,yaml] +---- +apiVersion: performance.openshift.io/v2 +kind: Pod +metadata: + annotations: + irq-load-balancing.crio.io: "disable" +spec: + runtimeClassName: performance- +... +---- diff --git a/modules/cnf-gathering-data-about-cluster-using-must-gather.adoc b/modules/cnf-gathering-data-about-cluster-using-must-gather.adoc index 97897f419cd1..a4232b6b6458 100644 --- a/modules/cnf-gathering-data-about-cluster-using-must-gather.adoc +++ b/modules/cnf-gathering-data-about-cluster-using-must-gather.adoc @@ -1,6 +1,6 @@ // Module included in the following assemblies: -// Epic CNF-792 (4.8) -// * scalability_and_performance/cnf-create-performance-profiles.adoc +// +// * scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc :_mod-docs-content-type: PROCEDURE [id="gathering-data-about-your-cluster-using-must-gather_{context}"] diff --git a/modules/cnf-how-run-podman-to-create-profile.adoc b/modules/cnf-how-run-podman-to-create-profile.adoc index ede62cb5a310..20d2b0e4ba10 100644 --- a/modules/cnf-how-run-podman-to-create-profile.adoc +++ b/modules/cnf-how-run-podman-to-create-profile.adoc @@ -1,9 +1,9 @@ // Module included in the following assemblies: -// Epic CNF-792 (4.8) -// * scalability_and_performance/cnf-create-performance-profiles.adoc +// +// * scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc [id="how-to-run-podman-to-create-a-profile_{context}"] -= How to run `podman` to create a performance profile += How to run podman to create a performance profile The following example illustrates how to run `podman` to create a performance profile with 20 reserved CPUs that are to be split across the NUMA nodes. diff --git a/modules/cnf-logging-associated-with-adjusting-nic-queues.adoc b/modules/cnf-logging-associated-with-adjusting-nic-queues.adoc index a28052fb7c6a..a66cfb5014e4 100644 --- a/modules/cnf-logging-associated-with-adjusting-nic-queues.adoc +++ b/modules/cnf-logging-associated-with-adjusting-nic-queues.adoc @@ -1,6 +1,6 @@ // Module included in the following assemblies: -//CNF-1483 (4.8) -// * scalability_and_performance/cnf-low-latency-tuning.adoc +// +// * scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc [id="logging-associated-with-adjusting-nic-queues_{context}"] = Logging associated with adjusting NIC queues diff --git a/modules/cnf-managing-device-interrupt-processing-for-guaranteed-pod-isolated-cpus.adoc b/modules/cnf-managing-device-interrupt-processing-for-guaranteed-pod-isolated-cpus.adoc index aecf8fdeee1e..a01b09130108 100644 --- a/modules/cnf-managing-device-interrupt-processing-for-guaranteed-pod-isolated-cpus.adoc +++ b/modules/cnf-managing-device-interrupt-processing-for-guaranteed-pod-isolated-cpus.adoc @@ -1,7 +1,6 @@ -// CNF-802 Infrastructure-provided interrupt processing for guaranteed pod CPUs // Module included in the following assemblies: // -// *cnf-low-latency-tuning.adoc +// * scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc [id="managing-device-interrupt-processing-for-guaranteed-pod-isolated-cpus_{context}"] = Managing device interrupt processing for guaranteed pod isolated CPUs @@ -11,62 +10,3 @@ The Node Tuning Operator can manage host CPUs by dividing them into reserved CPU Device interrupts are load balanced between all isolated and reserved CPUs to avoid CPUs being overloaded, with the exception of CPUs where there is a guaranteed pod running. Guaranteed pod CPUs are prevented from processing device interrupts when the relevant annotations are set for the pod. In the performance profile, `globallyDisableIrqLoadBalancing` is used to manage whether device interrupts are processed or not. For certain workloads, the reserved CPUs are not always sufficient for dealing with device interrupts, and for this reason, device interrupts are not globally disabled on the isolated CPUs. By default, Node Tuning Operator does not disable device interrupts on isolated CPUs. - -To achieve low latency for workloads, some (but not all) pods require the CPUs they are running on to not process device interrupts. A pod annotation, `irq-load-balancing.crio.io`, is used to define whether device interrupts are processed or not. When configured, CRI-O disables device interrupts only as long as the pod is running. - -[id="disabling-cpu-cfs-quota_{context}"] -== Disabling CPU CFS quota - -To reduce CPU throttling for individual guaranteed pods, create a pod specification with the annotation `cpu-quota.crio.io: "disable"`. This annotation disables the CPU completely fair scheduler (CFS) quota at the pod run time. The following pod specification contains this annotation: - -[source,yaml] ----- -apiVersion: v1 -kind: Pod -metadata: - annotations: - cpu-quota.crio.io: "disable" -spec: - runtimeClassName: performance- -... ----- - -[NOTE] -==== -Only disable CPU CFS quota when the CPU manager static policy is enabled and for pods with guaranteed QoS that use whole CPUs. Otherwise, disabling CPU CFS quota can affect the performance of other containers in the cluster. -==== - -[id="configuring-global-device-interrupts-handling-for-isolated-cpus_{context}"] -== Disabling global device interrupts handling in Node Tuning Operator - -To configure Node Tuning Operator to disable global device interrupts for the isolated CPU set, set the `globallyDisableIrqLoadBalancing` field in the performance profile to `true`. When `true`, conflicting pod annotations are ignored. When `false`, IRQ loads are balanced across all CPUs. - -A performance profile snippet illustrates this setting: - -[source,yaml] ----- -apiVersion: performance.openshift.io/v2 -kind: PerformanceProfile -metadata: - name: manual -spec: - globallyDisableIrqLoadBalancing: true -... ----- - -[id="disabling_interrupt_processing_for_individual_pods_{context}"] -== Disabling interrupt processing for individual pods - -To disable interrupt processing for individual pods, ensure that `globallyDisableIrqLoadBalancing` is set to `false` in the performance profile. Then, in the pod specification, set the `irq-load-balancing.crio.io` pod annotation to `disable`. The following pod specification contains this annotation: - -[source,yaml] ----- -apiVersion: performance.openshift.io/v2 -kind: Pod -metadata: - annotations: - irq-load-balancing.crio.io: "disable" -spec: - runtimeClassName: performance- -... ----- diff --git a/modules/cnf-measuring-latency.adoc b/modules/cnf-measuring-latency.adoc index 2046cb308d4c..bfa0610cafe6 100644 --- a/modules/cnf-measuring-latency.adoc +++ b/modules/cnf-measuring-latency.adoc @@ -1,6 +1,6 @@ // Module included in the following assemblies: // -// * scalability_and_performance/cnf-performing-platform-verification-latency-tests.adoc +// * scalability_and_performance/low_latency_tuning/cnf-performing-platform-verification-latency-tests.adoc :_mod-docs-content-type: CONCEPT [id="cnf-measuring-latency_{context}"] diff --git a/modules/cnf-node-tuning-operator-creating-pod-with-guaranteed-qos-class.adoc b/modules/cnf-node-tuning-operator-creating-pod-with-guaranteed-qos-class.adoc new file mode 100644 index 000000000000..76eac7dae920 --- /dev/null +++ b/modules/cnf-node-tuning-operator-creating-pod-with-guaranteed-qos-class.adoc @@ -0,0 +1,71 @@ +// Module included in the following assemblies: +// +// * scalability_and_performance/low_latency_tuning/cnf-provisioning-low-latency-workloads.adoc + +:_mod-docs-content-type: PROCEDURE +[id="cnf-node-tuning-operator-creating-pod-with-guaranteed-qos-class_{context}"] += Creating a pod with a guaranteed QoS class + +Keep the following in mind when you create a pod that is given a QoS class of `Guaranteed`: + +* Every container in the pod must have a memory limit and a memory request, and they must be the same. +* Every container in the pod must have a CPU limit and a CPU request, and they must be the same. + +The following example shows the configuration file for a pod that has one container. The container has a memory limit and a memory request, both equal to 200 MiB. The container has a CPU limit and a CPU request, both equal to 1 CPU. + +[source,yaml] +---- +apiVersion: v1 +kind: Pod +metadata: + name: qos-demo + namespace: qos-example +spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: qos-demo-ctr + image: + resources: + limits: + memory: "200Mi" + cpu: "1" + requests: + memory: "200Mi" + cpu: "1" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] +---- + +. Create the pod: ++ +[source,terminal] +---- +$ oc apply -f qos-pod.yaml --namespace=qos-example +---- + +. View detailed information about the pod: ++ +[source,terminal] +---- +$ oc get pod qos-demo --namespace=qos-example --output=yaml +---- ++ +.Example output +[source,yaml] +---- +spec: + containers: + ... +status: + qosClass: Guaranteed +---- ++ +[NOTE] +==== +If you specify a memory limit for a container, but do not specify a memory request, {product-title} automatically assigns a memory request that matches the limit. Similarly, if you specify a CPU limit for a container, but do not specify a CPU request, {product-title} automatically assigns a CPU request that matches the limit. +==== diff --git a/modules/cnf-node-tuning-operator-disabling-cpu-load-balancing-for-dpdk.adoc b/modules/cnf-node-tuning-operator-disabling-cpu-load-balancing-for-dpdk.adoc new file mode 100644 index 000000000000..81a8c1fb95ba --- /dev/null +++ b/modules/cnf-node-tuning-operator-disabling-cpu-load-balancing-for-dpdk.adoc @@ -0,0 +1,47 @@ +// Module included in the following assemblies: +// +// * scalability_and_performance/low_latency_tuning/cnf-provisioning-low-latency-workloads.adoc + +:_mod-docs-content-type: PROCEDURE +[id="cnf-node-tuning-operator-disabling-cpu-load-balancing-for-dpdk_{context}"] += Disabling CPU load balancing in a Pod + +Functionality to disable or enable CPU load balancing is implemented on the CRI-O level. The code under the CRI-O disables or enables CPU load balancing only when the following requirements are met. + +* The pod must use the `performance-` runtime class. You can get the proper name by looking at the status of the performance profile, as shown here: ++ +[source,yaml] +---- +apiVersion: performance.openshift.io/v2 +kind: PerformanceProfile +... +status: + ... + runtimeClass: performance-manual +---- + +The Node Tuning Operator is responsible for the creation of the high-performance runtime handler config snippet under relevant nodes and for creation of the high-performance runtime class under the cluster. It will have the same content as the default runtime handler except that it enables the CPU load balancing configuration functionality. + +To disable the CPU load balancing for the pod, the `Pod` specification must include the following fields: + +[source,yaml] +---- +apiVersion: v1 +kind: Pod +metadata: + #... + annotations: + #... + cpu-load-balancing.crio.io: "disable" + #... + #... +spec: + #... + runtimeClassName: performance- + #... +---- + +[NOTE] +==== +Only disable CPU load balancing when the CPU manager static policy is enabled and for pods with guaranteed QoS that use whole CPUs. Otherwise, disabling CPU load balancing can affect the performance of other containers in the cluster. +==== diff --git a/modules/cnf-performance-profile-creator-arguments.adoc b/modules/cnf-performance-profile-creator-arguments.adoc index 3d45d481d32f..e83293a017bd 100644 --- a/modules/cnf-performance-profile-creator-arguments.adoc +++ b/modules/cnf-performance-profile-creator-arguments.adoc @@ -1,6 +1,6 @@ // Module included in the following assemblies: -// Epic CNF-792 (4.8) -// * scalability_and_performance/cnf-create-performance-profiles.adoc +// +// * scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc [id="performance-profile-creator-arguments_{context}"] @@ -122,4 +122,4 @@ Default: `restricted`. Possible values: `true` or `false`. Default: `false`. -|=== \ No newline at end of file +|=== diff --git a/modules/cnf-performing-end-to-end-tests-disconnected-mode.adoc b/modules/cnf-performing-end-to-end-tests-disconnected-mode.adoc index 920a63031e58..53fcf6c482ea 100644 --- a/modules/cnf-performing-end-to-end-tests-disconnected-mode.adoc +++ b/modules/cnf-performing-end-to-end-tests-disconnected-mode.adoc @@ -1,6 +1,6 @@ // Module included in the following assemblies: // -// * scalability_and_performance/cnf-performing-platform-verification-latency-tests.adoc +// * scalability_and_performance/low_latency_tuning/cnf-performing-platform-verification-latency-tests.adoc :_mod-docs-content-type: PROCEDURE [id="cnf-performing-end-to-end-tests-disconnected-mode_{context}"] diff --git a/modules/cnf-performing-end-to-end-tests-running-cyclictest.adoc b/modules/cnf-performing-end-to-end-tests-running-cyclictest.adoc index ce6d0ece4ee1..bcee262e7c9b 100644 --- a/modules/cnf-performing-end-to-end-tests-running-cyclictest.adoc +++ b/modules/cnf-performing-end-to-end-tests-running-cyclictest.adoc @@ -1,6 +1,6 @@ // Module included in the following assemblies: // -// * scalability_and_performance/cnf-performing-platform-verification-latency-tests.adoc +// * scalability_and_performance/low_latency_tuning/cnf-performing-platform-verification-latency-tests.adoc :_mod-docs-content-type: PROCEDURE [id="cnf-performing-end-to-end-tests-running-cyclictest_{context}"] @@ -39,7 +39,7 @@ registry.redhat.io/openshift4/cnf-tests-rhel8:v{product-version} \ /usr/bin/test-run.sh -ginkgo.v -ginkgo.focus="cyclictest" --ginkgo.timeout="24h" ---- + -The command runs the `cyclictest` tool for 10 minutes (600 seconds). The test runs successfully when the maximum observed latency is lower than `MAXIMUM_LATENCY` (in this example, 20 μs). Latency spikes of 20 μs and above are generally not acceptable for {rds} workloads. +The command runs the `cyclictest` tool for 10 minutes (600 seconds). The test runs successfully when the maximum observed latency is lower than `MAXIMUM_LATENCY` (in this example, 20 μs). Latency spikes of 20 μs and above are generally not acceptable for telco RAN workloads. + If the results exceed the latency threshold, the test fails. + diff --git a/modules/cnf-performing-end-to-end-tests-running-hwlatdetect.adoc b/modules/cnf-performing-end-to-end-tests-running-hwlatdetect.adoc index f9bb96de2476..d3471dd46259 100644 --- a/modules/cnf-performing-end-to-end-tests-running-hwlatdetect.adoc +++ b/modules/cnf-performing-end-to-end-tests-running-hwlatdetect.adoc @@ -1,6 +1,6 @@ // Module included in the following assemblies: // -// * scalability_and_performance/cnf-performing-platform-verification-latency-tests.adoc +// * scalability_and_performance/low_latency_tuning/cnf-performing-platform-verification-latency-tests.adoc :_mod-docs-content-type: CONCEPT [id="cnf-performing-end-to-end-tests-running-hwlatdetect_{context}"] diff --git a/modules/cnf-performing-end-to-end-tests-running-in-single-node-cluster.adoc b/modules/cnf-performing-end-to-end-tests-running-in-single-node-cluster.adoc index 6f2e0f375f11..51305e837df8 100644 --- a/modules/cnf-performing-end-to-end-tests-running-in-single-node-cluster.adoc +++ b/modules/cnf-performing-end-to-end-tests-running-in-single-node-cluster.adoc @@ -1,6 +1,6 @@ // Module included in the following assemblies: // -// * scalability_and_performance/cnf-performing-platform-verification-latency-tests.adoc +// * scalability_and_performance/low_latency_tuning/cnf-performing-platform-verification-latency-tests.adoc :_mod-docs-content-type: PROCEDURE [id="cnf-performing-end-to-end-tests-running-in-single-node-cluster_{context}"] diff --git a/modules/cnf-performing-end-to-end-tests-running-oslat.adoc b/modules/cnf-performing-end-to-end-tests-running-oslat.adoc index 656de4468d7a..8cbbdd7c6452 100644 --- a/modules/cnf-performing-end-to-end-tests-running-oslat.adoc +++ b/modules/cnf-performing-end-to-end-tests-running-oslat.adoc @@ -1,6 +1,6 @@ // Module included in the following assemblies: // -// * scalability_and_performance/cnf-performing-platform-verification-latency-tests.adoc +// * scalability_and_performance/low_latency_tuning/cnf-performing-platform-verification-latency-tests.adoc :_mod-docs-content-type: PROCEDURE [id="cnf-performing-end-to-end-tests-running-oslat_{context}"] diff --git a/modules/cnf-performing-end-to-end-tests-test-failure-report.adoc b/modules/cnf-performing-end-to-end-tests-test-failure-report.adoc index 3627d84e463a..14c833d9bd4e 100644 --- a/modules/cnf-performing-end-to-end-tests-test-failure-report.adoc +++ b/modules/cnf-performing-end-to-end-tests-test-failure-report.adoc @@ -1,6 +1,6 @@ // Module included in the following assemblies: // -// * scalability_and_performance/cnf-performing-platform-verification-latency-tests.adoc +// * scalability_and_performance/low_latency_tuning/cnf-performing-platform-verification-latency-tests.adoc :_mod-docs-content-type: PROCEDURE [id="cnf-performing-end-to-end-tests-test-failure-report_{context}"] diff --git a/modules/cnf-performing-end-to-end-tests-troubleshooting.adoc b/modules/cnf-performing-end-to-end-tests-troubleshooting.adoc index e076dd23aed4..6b8deace8fb4 100644 --- a/modules/cnf-performing-end-to-end-tests-troubleshooting.adoc +++ b/modules/cnf-performing-end-to-end-tests-troubleshooting.adoc @@ -1,6 +1,6 @@ // Module included in the following assemblies: // -// * scalability_and_performance/cnf-performing-platform-verification-latency-tests.adoc +// * scalability_and_performance/low_latency_tuning/cnf-performing-platform-verification-latency-tests.adoc :_mod-docs-content-type: PROCEDURE [id="cnf-performing-end-to-end-tests-troubleshooting_{context}"] diff --git a/modules/cnf-provisioning-real-time-and-low-latency-workloads.adoc b/modules/cnf-provisioning-real-time-and-low-latency-workloads.adoc deleted file mode 100644 index ca387e0b3b9b..000000000000 --- a/modules/cnf-provisioning-real-time-and-low-latency-workloads.adoc +++ /dev/null @@ -1,445 +0,0 @@ -// CNF-489 Real time and low latency workload provisioning -// Module included in the following assemblies: -// -// *cnf-low-latency-tuning.adoc - -:_mod-docs-content-type: PROCEDURE -[id="cnf-provisioning-real-time-and-low-latency-workloads_{context}"] -= Provisioning real-time and low latency workloads - -Many industries and organizations need extremely high performance computing and might require low and predictable latency, especially in the financial and telecommunications industries. For these industries, with their unique requirements, {product-title} provides the Node Tuning Operator to implement automatic tuning to achieve low latency performance and consistent response time for {product-title} applications. - -The cluster administrator can use this performance profile configuration to make these changes in a more reliable way. The administrator can specify whether to update the kernel to kernel-rt (real-time), reserve CPUs for cluster and operating system housekeeping duties, including pod infra containers, isolate CPUs for application containers to run the workloads, and disable unused CPUs to reduce power consumption. - -[WARNING] -==== -The usage of execution probes in conjunction with applications that require guaranteed CPUs can cause latency spikes. It is recommended to use other probes, such as a properly configured set of network probes, as an alternative. -==== - -[NOTE] -==== -In earlier versions of {product-title}, the Performance Addon Operator was used to implement automatic tuning to achieve low latency performance for OpenShift applications. In {product-title} 4.11 and later, these functions are part of the Node Tuning Operator. -==== - -[id="node-tuning-operator-known-limitations-for-real-time_{context}"] -== Known limitations for real-time - -[NOTE] -==== -In most deployments, kernel-rt is supported only on worker nodes when you use a standard cluster with three control plane nodes and three worker nodes. There are exceptions for compact and single nodes on {product-title} deployments. For installations on a single node, kernel-rt is supported on the single control plane node. -==== - -To fully utilize the real-time mode, the containers must run with elevated privileges. -See link:https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-capabilities-for-a-container[Set capabilities for a Container] for information on granting privileges. - -{product-title} restricts the allowed capabilities, so you might need to create a `SecurityContext` as well. - -[NOTE] -==== -This procedure is fully supported with bare metal installations using {op-system-first} systems. -==== - -Establishing the right performance expectations refers to the fact that the real-time kernel is not a panacea. Its objective is consistent, low-latency determinism offering predictable response times. There is some additional kernel overhead associated with the real-time kernel. This is due primarily to handling hardware interruptions in separately scheduled threads. The increased overhead in some workloads results in some degradation in overall throughput. The exact amount of degradation is very workload dependent, ranging from 0% to 30%. However, it is the cost of determinism. - -[id="node-tuning-operator-provisioning-worker-with-real-time-capabilities_{context}"] -== Provisioning a worker with real-time capabilities - -. Optional: Add a node to the {product-title} cluster. -See link:https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux_for_real_time/8/html/optimizing_rhel_8_for_real_time_for_low_latency_operation/setting-bios-parameters-for-system-tuning_optimizing-rhel8-for-real-time-for-low-latency-operation[Setting BIOS parameters for system tuning]. - -. Add the label `worker-rt` to the worker nodes that require the real-time capability by using the `oc` command. - -. Create a new machine config pool for real-time nodes: -+ -[source,yaml] ----- -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfigPool -metadata: - name: worker-rt - labels: - machineconfiguration.openshift.io/role: worker-rt -spec: - machineConfigSelector: - matchExpressions: - - { - key: machineconfiguration.openshift.io/role, - operator: In, - values: [worker, worker-rt], - } - paused: false - nodeSelector: - matchLabels: - node-role.kubernetes.io/worker-rt: "" ----- -Note that a machine config pool worker-rt is created for group of nodes that have the label `worker-rt`. - -. Add the node to the proper machine config pool by using node role labels. -+ -[NOTE] -==== -You must decide which nodes are configured with real-time workloads. You could configure all of the nodes in the cluster, or a subset of the nodes. The Node Tuning Operator that expects all of the nodes are part of a dedicated machine config pool. If you use all of the nodes, you must point the Node Tuning Operator to the worker node role label. If you use a subset, you must group the nodes into a new machine config pool. -==== -. Create the `PerformanceProfile` with the proper set of housekeeping cores and `realTimeKernel: enabled: true`. - -. You must set `machineConfigPoolSelector` in `PerformanceProfile`: -+ -[source,yaml] ----- - apiVersion: performance.openshift.io/v2 - kind: PerformanceProfile - metadata: - name: example-performanceprofile - spec: - ... - realTimeKernel: - enabled: true - nodeSelector: - node-role.kubernetes.io/worker-rt: "" - machineConfigPoolSelector: - machineconfiguration.openshift.io/role: worker-rt ----- -. Verify that a matching machine config pool exists with a label: -+ -[source,terminal] ----- -$ oc describe mcp/worker-rt ----- -+ -.Example output -[source,yaml] ----- -Name: worker-rt -Namespace: -Labels: machineconfiguration.openshift.io/role=worker-rt ----- - -. {product-title} will start configuring the nodes, which might involve multiple reboots. Wait for the nodes to settle. This can take a long time depending on the specific hardware you use, but 20 minutes per node is expected. - -. Verify everything is working as expected. - -[id="node-tuning-operator-verifying-real-time-kernel-installation_{context}"] -== Verifying the real-time kernel installation - -Use this command to verify that the real-time kernel is installed: - -[source,terminal] ----- -$ oc get node -o wide ----- - -Note the worker with the role `worker-rt` that contains the string `4.18.0-305.30.1.rt7.102.el8_4.x86_64 cri-o://1.27.3-99.rhaos4.10.gitc3131de.el8`: - -[source,terminal] ----- -NAME STATUS ROLES AGE VERSION INTERNAL-IP -EXTERNAL-IP OS-IMAGE KERNEL-VERSION -CONTAINER-RUNTIME -rt-worker-0.example.com Ready worker,worker-rt 5d17h v1.27.3 -128.66.135.107 Red Hat Enterprise Linux CoreOS 46.82.202008252340-0 (Ootpa) -4.18.0-305.30.1.rt7.102.el8_4.x86_64 cri-o://1.27.3-99.rhaos4.10.gitc3131de.el8 -[...] ----- - -[id="node-tuning-operator-creating-workload-that-works-in-real-time_{context}"] -== Creating a workload that works in real-time - -Use the following procedures for preparing a workload that will use real-time capabilities. - -.Procedure - -. Create a pod with a QoS class of `Guaranteed`. -. Optional: Disable CPU load balancing for DPDK. -. Assign a proper node selector. - -When writing your applications, follow the general recommendations described in -link:https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux_for_real_time/8/html-single/tuning_guide/index#chap-Application_Tuning_and_Deployment[Application tuning and deployment]. - -[id="node-tuning-operator-creating-pod-with-guaranteed-qos-class_{context}"] -== Creating a pod with a QoS class of `Guaranteed` - -Keep the following in mind when you create a pod that is given a QoS class of `Guaranteed`: - -* Every container in the pod must have a memory limit and a memory request, and they must be the same. -* Every container in the pod must have a CPU limit and a CPU request, and they must be the same. - -The following example shows the configuration file for a pod that has one container. The container has a memory limit and a memory request, both equal to 200 MiB. The container has a CPU limit and a CPU request, both equal to 1 CPU. - -[source,yaml] ----- -apiVersion: v1 -kind: Pod -metadata: - name: qos-demo - namespace: qos-example -spec: - containers: - - name: qos-demo-ctr - image: - resources: - limits: - memory: "200Mi" - cpu: "1" - requests: - memory: "200Mi" - cpu: "1" ----- - -. Create the pod: -+ -[source,terminal] ----- -$ oc apply -f qos-pod.yaml --namespace=qos-example ----- - -. View detailed information about the pod: -+ -[source,terminal] ----- -$ oc get pod qos-demo --namespace=qos-example --output=yaml ----- -+ -.Example output -[source,yaml] ----- -spec: - containers: - ... -status: - qosClass: Guaranteed ----- -+ -[NOTE] -==== -If a container specifies its own memory limit, but does not specify a memory request, {product-title} automatically assigns a memory request that matches the limit. Similarly, if a container specifies its own CPU limit, but does not specify a CPU request, {product-title} automatically assigns a CPU request that matches the limit. -==== - -[id="node-tuning-operator-disabling-cpu-load-balancing-for-dpdk_{context}"] -== Optional: Disabling CPU load balancing for DPDK - -Functionality to disable or enable CPU load balancing is implemented on the CRI-O level. The code under the CRI-O disables or enables CPU load balancing only when the following requirements are met. - -* The pod must use the `performance-` runtime class. You can get the proper name by looking at the status of the performance profile, as shown here: -+ -[source,yaml] ----- -apiVersion: performance.openshift.io/v2 -kind: PerformanceProfile -... -status: - ... - runtimeClass: performance-manual ----- - -[NOTE] -==== -Currently, disabling CPU load balancing is not supported with cgroup v2. -==== - -The Node Tuning Operator is responsible for the creation of the high-performance runtime handler config snippet under relevant nodes and for creation of the high-performance runtime class under the cluster. It will have the same content as default runtime handler except it enables the CPU load balancing configuration functionality. - -To disable the CPU load balancing for the pod, the `Pod` specification must include the following fields: - -[source,yaml] ----- -apiVersion: v1 -kind: Pod -metadata: - ... - annotations: - ... - cpu-load-balancing.crio.io: "disable" - ... - ... -spec: - ... - runtimeClassName: performance- - ... ----- - -[NOTE] -==== -Only disable CPU load balancing when the CPU manager static policy is enabled and for pods with guaranteed QoS that use whole CPUs. Otherwise, disabling CPU load balancing can affect the performance of other containers in the cluster. -==== - -[id="node-tuning-operator-assigning-proper-node-selector_{context}"] -== Assigning a proper node selector - -The preferred way to assign a pod to nodes is to use the same node selector the performance profile used, as shown here: - -[source,yaml] ----- -apiVersion: v1 -kind: Pod -metadata: - name: example -spec: - # ... - nodeSelector: - node-role.kubernetes.io/worker-rt: "" ----- - -For more information, see link:https://access.redhat.com/documentation/en-us/openshift_container_platform/4.5/html-single/nodes/index#nodes-scheduler-node-selectors[Placing pods on specific nodes using node selectors]. - -[id="node-tuning-operator-scheduling-workload-onto-worker-with-real-time-capabilities_{context}"] -== Scheduling a workload onto a worker with real-time capabilities - -Use label selectors that match the nodes attached to the machine config pool that was configured for low latency by the Node Tuning Operator. For more information, see link:https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/[Assigning pods to nodes]. - -[id="node-tuning-operator-disabling-CPUs-for-power-consumption_{context}"] -== Reducing power consumption by taking CPUs offline - -You can generally anticipate telecommunication workloads. When not all of the CPU resources are required, the Node Tuning Operator allows you take unused CPUs offline to reduce power consumption by manually updating the performance profile. - -To take unused CPUs offline, you must perform the following tasks: - -. Set the offline CPUs in the performance profile and save the contents of the YAML file: -+ -.Example performance profile with offlined CPUs -[source,yaml] ----- -apiVersion: performance.openshift.io/v2 -kind: PerformanceProfile -metadata: - name: performance -spec: - additionalKernelArgs: - - nmi_watchdog=0 - - audit=0 - - mce=off - - processor.max_cstate=1 - - intel_idle.max_cstate=0 - - idle=poll - cpu: - isolated: "2-23,26-47" - reserved: "0,1,24,25" - offlined: "48-59" <1> - nodeSelector: - node-role.kubernetes.io/worker-cnf: "" - numa: - topologyPolicy: single-numa-node - realTimeKernel: - enabled: true ----- -<1> Optional. You can list CPUs in the `offlined` field to take the specified CPUs offline. - -. Apply the updated profile by running the following command: -+ -[source,terminal] ----- -$ oc apply -f my-performance-profile.yaml ----- - -[id="node-tuning-operator-pod-power-saving-config_{context}"] -== Optional: Power saving configurations - -You can enable power savings for a node that has low priority workloads that are colocated with high priority workloads without impacting the latency or throughput of the high priority workloads. Power saving is possible without modifications to the workloads themselves. - -[IMPORTANT] -==== -The feature is supported on Intel Ice Lake and later generations of Intel CPUs. The capabilities of the processor might impact the latency and throughput of the high priority workloads. -==== - -When you configure a node with a power saving configuration, you must configure high priority workloads with performance configuration at the pod level, which means that the configuration applies to all the cores used by the pod. - -By disabling P-states and C-states at the pod level, you can configure high priority workloads for best performance and lowest latency. - -.Configuration for high priority workloads -[cols="1,2,3", options="header"] - -|=== -| Annotation | Possible Values | Description - -|`cpu-c-states.crio.io:` a| * `"enable"` -* `"disable"` -* `"max_latency:microseconds"` | This annotation allows you to enable or disable C-states for each CPU. Alternatively, you can also specify a maximum latency in microseconds for the C-states. For example, enable C-states with a maximum latency of 10 microseconds with the setting `cpu-c-states.crio.io`: `"max_latency:10"`. Set the value to `"disable"` to provide the best performance for a pod. - -| `cpu-freq-governor.crio.io:` | Any supported `cpufreq governor`. | Sets the `cpufreq` governor for each CPU. The `"performance"` governor is recommended for high priority workloads. - -|=== - -.Prerequisites - -* You enabled C-states and OS-controlled P-states in the BIOS - -.Procedure - -. Generate a `PerformanceProfile` with `per-pod-power-management` set to `true`: -+ -[source,terminal,subs="attributes+"] ----- -$ podman run --entrypoint performance-profile-creator -v \ -/must-gather:/must-gather:z registry.redhat.io/openshift4/ose-cluster-node-tuning-operator:v{product-version} \ ---mcp-name=worker-cnf --reserved-cpu-count=20 --rt-kernel=true \ ---split-reserved-cpus-across-numa=false --topology-manager-policy=single-numa-node \ ---must-gather-dir-path /must-gather -power-consumption-mode=low-latency \ <1> ---per-pod-power-management=true > my-performance-profile.yaml ----- -<1> The `power-consumption-mode` must be `default` or `low-latency` when the `per-pod-power-management` is set to `true`. - -+ -.Example `PerformanceProfile` with `perPodPowerManagement` - -[source,yaml] ----- -apiVersion: performance.openshift.io/v2 -kind: PerformanceProfile -metadata: - name: performance -spec: - [.....] - workloadHints: - realTime: true - highPowerConsumption: false - perPodPowerManagement: true ----- - -. Set the default `cpufreq` governor as an additional kernel argument in the `PerformanceProfile` custom resource (CR): -+ -[source,yaml] ----- -apiVersion: performance.openshift.io/v2 -kind: PerformanceProfile -metadata: - name: performance -spec: - ... - additionalKernelArgs: - - cpufreq.default_governor=schedutil <1> ----- -<1> Using the `schedutil` governor is recommended, however, you can use other governors such as the `ondemand` or `powersave` governors. - -. Set the maximum CPU frequency in the `TunedPerformancePatch` CR: -+ -[source,yaml] ----- -spec: - profile: - - data: | - [sysfs] - /sys/devices/system/cpu/intel_pstate/max_perf_pct = <1> ----- -<1> The `max_perf_pct` controls the maximum frequency the `cpufreq` driver is allowed to set as a percentage of the maximum supported cpu frequency. This value applies to all CPUs. You can check the maximum supported frequency in `/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq`. As a starting point, you can use a percentage that caps all CPUs at the `All Cores Turbo` frequency. The `All Cores Turbo` frequency is the frequency that all cores will run at when the cores are all fully occupied. - -. Add the desired annotations to your high priority workload pods. The annotations override the `default` settings. -+ -.Example high priority workload annotation -[source,yaml] ----- -apiVersion: v1 -kind: Pod -metadata: - ... - annotations: - ... - cpu-c-states.crio.io: "disable" - cpu-freq-governor.crio.io: "performance" - ... - ... -spec: - ... - runtimeClassName: performance- - ... ----- - -. Restart the pods. diff --git a/modules/cnf-reducing-netqueues-using-nto.adoc b/modules/cnf-reducing-netqueues-using-nto.adoc deleted file mode 100644 index f1c5e8e01977..000000000000 --- a/modules/cnf-reducing-netqueues-using-nto.adoc +++ /dev/null @@ -1,8 +0,0 @@ -// Module included in the following assemblies: -//CNF-1483 (4.8) -// * scalability_and_performance/low-latency-tuning.adoc - -[id="reducing-nic-queues-using-the-node-tuning-operator_{context}"] -= Reducing NIC queues using the Node Tuning Operator - -The Node Tuning Operator allows you to adjust the network interface controller (NIC) queue count for each network device. By using a PerformanceProfile, the amount of queues can be reduced to the number of reserved CPUs. \ No newline at end of file diff --git a/modules/cnf-running-the-performance-creator-profile-offline.adoc b/modules/cnf-running-the-performance-creator-profile-offline.adoc index aaedc51abc09..741342f9d687 100644 --- a/modules/cnf-running-the-performance-creator-profile-offline.adoc +++ b/modules/cnf-running-the-performance-creator-profile-offline.adoc @@ -1,6 +1,6 @@ // Module included in the following assemblies: -// Epic CNF-792 (4.8) -// * scalability_and_performance/cnf-create-performance-profiles.adoc +// +// * scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc :_mod-docs-content-type: PROCEDURE [id="running-the-performance-profile-creator-wrapper-script_{context}"] diff --git a/modules/cnf-running-the-performance-creator-profile.adoc b/modules/cnf-running-the-performance-creator-profile.adoc index de0080407e39..370af13f59b6 100644 --- a/modules/cnf-running-the-performance-creator-profile.adoc +++ b/modules/cnf-running-the-performance-creator-profile.adoc @@ -1,10 +1,10 @@ // Module included in the following assemblies: -// Epic CNF-792 (4.8) -// * scalability_and_performance/cnf-create-performance-profiles.adoc +// +// * scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc :_mod-docs-content-type: PROCEDURE [id="running-the-performance-profile-profile-cluster-using-podman_{context}"] -= Running the Performance Profile Creator using podman += Running the Performance Profile Creator using Podman As a cluster administrator, you can run `podman` and the Performance Profile Creator to create a performance profile. @@ -82,10 +82,11 @@ Flags: + [NOTE] ==== -Discovery mode inspects your cluster using the output from `must-gather`. The output produced includes information on: +Discovery mode inspects your cluster by using the output from `must-gather`. +The output produced includes information on the following conditions: * The NUMA cell partitioning with the allocated CPU ids -* Whether hyperthreading is enabled +* Whether Hyper-Threading is enabled Using this information you can set appropriate values for some of the arguments supplied to the Performance Profile Creator tool. ==== @@ -99,10 +100,10 @@ $ podman run --entrypoint performance-profile-creator -v /m ==== This command uses the performance profile creator as a new entry point to `podman`. It maps the `must-gather` data for the host into the container image and invokes the required user-supplied profile arguments to produce the `my-performance-profile.yaml` file. -The `-v` option can be the path to either: +The `-v` option can be the path to either of the following components: * The `must-gather` output directory -* An existing directory containing the `must-gather` decompressed tarball +* An existing directory containing the `must-gather` decompressed .tar file The `info` option requires a value which specifies the output format. Possible values are log and JSON. The JSON format is reserved for debugging. ==== diff --git a/modules/cnf-scheduling-workload-onto-worker-with-real-time-capabilities.adoc b/modules/cnf-scheduling-workload-onto-worker-with-real-time-capabilities.adoc new file mode 100644 index 000000000000..38afc4500bc4 --- /dev/null +++ b/modules/cnf-scheduling-workload-onto-worker-with-real-time-capabilities.adoc @@ -0,0 +1,180 @@ +// Module included in the following assemblies: +// +// * scalability_and_performance/low_latency_tuning/cnf-provisioning-low-latency-workloads.adoc + +[id="cnf-scheduling-workload-onto-worker-with-real-time-capabilities_{context}"] += Scheduling a low latency workload onto a worker with real-time capabilities + +You can schedule low latency workloads onto a worker node where a performance profile that configures real-time capabilities is applied. + +[NOTE] +==== +To schedule the workload on specific nodes, use label selectors in the `Pod` custom resource (CR). +The label selectors must match the nodes that are attached to the machine config pool that was configured for low latency by the Node Tuning Operator. +==== + +.Prerequisites + +* You have installed the OpenShift CLI (`oc`). + +* You have logged in as a user with `cluster-admin` privileges. + +* You have applied a performance profile in the cluster that tunes worker nodes for low latency workloads. + +.Procedure + +. Create a `Pod` CR for the low latency workload and apply it in the cluster, for example: ++ +.Example `Pod` spec configured to use real-time processing +[source,yaml,subs="attributes+"] +---- +apiVersion: v1 +kind: Pod +metadata: + name: dynamic-low-latency-pod + annotations: + cpu-quota.crio.io: "disable" <1> + cpu-load-balancing.crio.io: "disable" <2> + irq-load-balancing.crio.io: "disable" <3> +spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: dynamic-low-latency-pod + image: "registry.redhat.io/openshift4/cnf-tests-rhel8:v{product-version}" + command: ["sleep", "10h"] + resources: + requests: + cpu: 2 + memory: "200M" + limits: + cpu: 2 + memory: "200M" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + nodeSelector: + node-role.kubernetes.io/worker-cnf: "" <4> + runtimeClassName: performance-dynamic-low-latency-profile <5> +# ... +---- +<1> Disables the CPU completely fair scheduler (CFS) quota at the pod run time. +<2> Disables CPU load balancing. +<3> Opts the pod out of interrupt handling on the node. +<4> The `nodeSelector` label must match the label that you specify in the `Node` CR. +<5> `runtimeClassName` must match the name of the performance profile configured in the cluster. + +. Enter the pod `runtimeClassName` in the form performance-, where is the `name` from the `PerformanceProfile` YAML. +In the previous example, the `name` is `performance-dynamic-low-latency-profile`. + +. Ensure the pod is running correctly. Status should be `running`, and the correct cnf-worker node should be set: ++ +[source,terminal] +---- +$ oc get pod -o wide +---- ++ +.Expected output ++ +[source,terminal] +---- +NAME READY STATUS RESTARTS AGE IP NODE +dynamic-low-latency-pod 1/1 Running 0 5h33m 10.131.0.10 cnf-worker.example.com +---- + +. Get the CPUs that the pod configured for IRQ dynamic load balancing runs on: ++ +[source,terminal] +---- +$ oc exec -it dynamic-low-latency-pod -- /bin/bash -c "grep Cpus_allowed_list /proc/self/status | awk '{print $2}'" +---- ++ +.Expected output ++ +[source,terminal] +---- +Cpus_allowed_list: 2-3 +---- + +.Verification + +Ensure the node configuration is applied correctly. + +. Log in to the node to verify the configuration. ++ +[source,terminal] +---- +$ oc debug node/ +---- + +. Verify that you can use the node file system: ++ +[source,terminal] +---- +sh-4.4# chroot /host +---- ++ +.Expected output ++ +[source,terminal] +---- +sh-4.4# +---- + +. Ensure the default system CPU affinity mask does not include the `dynamic-low-latency-pod` CPUs, for example, CPUs 2 and 3. ++ +[source,terminal] +---- +sh-4.4# cat /proc/irq/default_smp_affinity +---- ++ +.Example output ++ +[source,terminal] +---- +33 +---- + +. Ensure the system IRQs are not configured to run on the `dynamic-low-latency-pod` CPUs: ++ +[source,terminal] +---- +sh-4.4# find /proc/irq/ -name smp_affinity_list -exec sh -c 'i="$1"; mask=$(cat $i); file=$(echo $i); echo $file: $mask' _ {} \; +---- ++ +.Example output ++ +[source,terminal] +---- +/proc/irq/0/smp_affinity_list: 0-5 +/proc/irq/1/smp_affinity_list: 5 +/proc/irq/2/smp_affinity_list: 0-5 +/proc/irq/3/smp_affinity_list: 0-5 +/proc/irq/4/smp_affinity_list: 0 +/proc/irq/5/smp_affinity_list: 0-5 +/proc/irq/6/smp_affinity_list: 0-5 +/proc/irq/7/smp_affinity_list: 0-5 +/proc/irq/8/smp_affinity_list: 4 +/proc/irq/9/smp_affinity_list: 4 +/proc/irq/10/smp_affinity_list: 0-5 +/proc/irq/11/smp_affinity_list: 0 +/proc/irq/12/smp_affinity_list: 1 +/proc/irq/13/smp_affinity_list: 0-5 +/proc/irq/14/smp_affinity_list: 1 +/proc/irq/15/smp_affinity_list: 0 +/proc/irq/24/smp_affinity_list: 1 +/proc/irq/25/smp_affinity_list: 1 +/proc/irq/26/smp_affinity_list: 1 +/proc/irq/27/smp_affinity_list: 5 +/proc/irq/28/smp_affinity_list: 1 +/proc/irq/29/smp_affinity_list: 0 +/proc/irq/30/smp_affinity_list: 0-5 +---- + +[WARNING] +==== +When you tune nodes for low latency, the usage of execution probes in conjunction with applications that require guaranteed CPUs can cause latency spikes. Use other probes, such as a properly configured set of network probes, as an alternative. +==== diff --git a/modules/cnf-telco-core-reference-design-performance-profile-template.adoc b/modules/cnf-telco-core-reference-design-performance-profile-template.adoc new file mode 100644 index 000000000000..87e6ddcb5092 --- /dev/null +++ b/modules/cnf-telco-core-reference-design-performance-profile-template.adoc @@ -0,0 +1,15 @@ +// Module included in the following assemblies: +// +// * scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc + +:_mod-docs-content-type: REFERENCE +[id="cnf-telco-core-reference-design-performance-profile-template_{context}"] += Telco core reference design performance profile template + +The following performance profile configures node-level performance settings for {product-title} clusters on commodity hardware to host telco core workloads. + +.Telco core reference design performance profile +[source,yaml] +---- +include::snippets/ztp_PerformanceProfile.yaml[] +---- diff --git a/modules/cnf-telco-ran-reference-design-performance-profile-template.adoc b/modules/cnf-telco-ran-reference-design-performance-profile-template.adoc new file mode 100644 index 000000000000..0d07b593a4ae --- /dev/null +++ b/modules/cnf-telco-ran-reference-design-performance-profile-template.adoc @@ -0,0 +1,15 @@ +// Module included in the following assemblies: +// +// * scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc + +:_mod-docs-content-type: REFERENCE +[id="cnf-telco-ran-reference-design-performance-profile-template_{context}"] += Telco RAN DU reference design performance profile template + +The following performance profile configures node-level performance settings for {product-title} clusters on commodity hardware to host telco RAN DU workloads. + +.Telco RAN DU reference design performance profile +[source,yaml] +---- +include::snippets/ztp_PerformanceProfile.yaml[] +---- diff --git a/modules/cnf-understanding-low-latency.adoc b/modules/cnf-understanding-low-latency.adoc index a8c06194b73f..1a85a36933f7 100644 --- a/modules/cnf-understanding-low-latency.adoc +++ b/modules/cnf-understanding-low-latency.adoc @@ -1,14 +1,11 @@ // Module included in the following assemblies: -// Epic CNF-78 (4.4) +// // * scalability_and_performance/cnf-low-latency-tuning.adoc +// * scalability_and_performance/low_latency_tuning/cnf-understanding-low-latency.adoc :_mod-docs-content-type: CONCEPT [id="cnf-understanding-low-latency_{context}"] -= Understanding low latency - -The emergence of Edge computing in the area of Telco / 5G plays a key role in reducing latency and congestion problems and improving application performance. - -Simply put, latency determines how fast data (packets) moves from the sender to receiver and returns to the sender after processing by the receiver. Maintaining a network architecture with the lowest possible delay of latency speeds is key for meeting the network performance requirements of 5G. Compared to 4G technology, with an average latency of 50 ms, 5G is targeted to reach latency numbers of 1 ms or less. This reduction in latency boosts wireless throughput by a factor of 10. += About low latency Many of the deployed applications in the Telco space require low latency that can only tolerate zero packet loss. Tuning for zero packet loss helps mitigate the inherent issues that degrade network performance. For more information, see link:https://www.redhat.com/en/blog/tuning-zero-packet-loss-red-hat-openstack-platform-part-1[Tuning for Zero Packet Loss in {rh-openstack-first}]. @@ -40,7 +37,6 @@ Workload hints simplify the fine-tuning of performance to industry sector settin * Real-time capability * Efficient use of power -In an ideal world, all of those would be prioritized: in real life, some come at the expense of others. The Node Tuning Operator is now aware of the workload expectations and better able to meet the demands of the workload. The cluster admin can now specify into which use case that workload falls. The Node Tuning Operator uses the `PerformanceProfile` to fine tune the performance settings for the workload. +Ideally, all of the previously listed items are prioritized. Some of these items come at the expense of others however. The Node Tuning Operator is now aware of the workload expectations and better able to meet the demands of the workload. The cluster admin can now specify into which use case that workload falls. The Node Tuning Operator uses the `PerformanceProfile` to fine tune the performance settings for the workload. The environment in which an application is operating influences its behavior. For a typical data center with no strict latency requirements, only minimal default tuning is needed that enables CPU partitioning for some high performance workload pods. For data centers and workloads where latency is a higher priority, measures are still taken to optimize power consumption. The most complicated cases are clusters close to latency-sensitive equipment such as manufacturing machinery and software-defined radios. This last class of deployment is often referred to as Far edge. For Far edge deployments, ultra-low latency is the ultimate priority, and is achieved at the expense of power management. - diff --git a/modules/cnf-understanding-workload-hints.adoc b/modules/cnf-understanding-workload-hints.adoc deleted file mode 100644 index f92b8b43bf1d..000000000000 --- a/modules/cnf-understanding-workload-hints.adoc +++ /dev/null @@ -1,64 +0,0 @@ -// Module included in the following assemblies: -// -// scalability_and_performance/cnf-low-latency-tuning.adoc - -:_mod-docs-content-type: CONCEPT -[id="cnf-understanding-workload-hints_{context}"] -= Understanding workload hints - -The following table describes how combinations of power consumption and real-time settings impact on latency. -[NOTE] -==== -The following workload hints can be configured manually. You can also work with workload hints using the Performance Profile Creator. For more information about the performance profile, see the "Creating a performance profile" section. -If the workload hint is configured manually and the `realTime` workload hint is not explicitly set then it defaults to `true`. -==== - -[cols="1,1,1,1",options="header"] -|=== - | Performance Profile creator setting| Hint | Environment | Description - - | Default - a|[source,terminal] ----- -workloadHints: -highPowerConsumption: false -realTime: false ----- - | High throughput cluster without latency requirements - | Performance achieved through CPU partitioning only. - - - - | Low-latency - a|[source,terminal] ----- -workloadHints: -highPowerConsumption: false -realTime: true ----- - | Regional datacenters - | Both energy savings and low-latency are desirable: compromise between power management, latency and throughput. - - - | Ultra-low-latency - a|[source,terminal] ----- -workloadHints: -highPowerConsumption: true -realTime: true ----- - | Far edge clusters, latency critical workloads - | Optimized for absolute minimal latency and maximum determinism at the cost of increased power consumption. - - | Per-pod power management - a|[source,terminal] ----- -workloadHints: -realTime: true -highPowerConsumption: false -perPodPowerManagement: true ----- - | Critical and non-critical workloads - | Allows for power management per pod. - -|=== diff --git a/modules/cnf-use-device-interrupt-processing-for-isolated-cpus.adoc b/modules/cnf-use-device-interrupt-processing-for-isolated-cpus.adoc index f09231fed01d..d0abc35acc3d 100644 --- a/modules/cnf-use-device-interrupt-processing-for-isolated-cpus.adoc +++ b/modules/cnf-use-device-interrupt-processing-for-isolated-cpus.adoc @@ -1,10 +1,16 @@ -// CNF-802 Infrastructure-provided interrupt processing for guaranteed pod CPUs // Module included in the following assemblies: // -// *cnf-low-latency-tuning.adoc +// * scalability_and_performance/low_latency_tuning/cnf-provisioning-low-latency-workloads.adoc +// * scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc +[id="nto_supported_api_versions_{context}"] += Supported performance profile API versions + +The Node Tuning Operator supports `v2`, `v1`, and `v1alpha1` for the performance profile `apiVersion` field. The v1 and v1alpha1 APIs are identical. The v2 API includes an optional boolean field `globallyDisableIrqLoadBalancing` with a default value of `false`. + +[discrete] [id="use-device-interrupt-processing-for-isolated-cpus_{context}"] -= Upgrading the performance profile to use device interrupt processing +== Upgrading the performance profile to use device interrupt processing When you upgrade the Node Tuning Operator performance profile custom resource definition (CRD) from v1 or v1alpha1 to v2, `globallyDisableIrqLoadBalancing` is set to `true` on existing profiles. @@ -13,16 +19,13 @@ When you upgrade the Node Tuning Operator performance profile custom resource de `globallyDisableIrqLoadBalancing` toggles whether IRQ load balancing will be disabled for the Isolated CPU set. When the option is set to `true` it disables IRQ load balancing for the Isolated CPU set. Setting the option to `false` allows the IRQs to be balanced across all CPUs. ==== -[id="nto_supported_api_versions_{context}"] -== Supported API Versions - -The Node Tuning Operator supports `v2`, `v1`, and `v1alpha1` for the performance profile `apiVersion` field. The v1 and v1alpha1 APIs are identical. The v2 API includes an optional boolean field `globallyDisableIrqLoadBalancing` with a default value of `false`. - +[discrete] [id="upgrading_nto_api_from_v1alpha1_to_v1_{context}"] === Upgrading Node Tuning Operator API from v1alpha1 to v1 When upgrading Node Tuning Operator API version from v1alpha1 to v1, the v1alpha1 performance profiles are converted on-the-fly using a "None" Conversion strategy and served to the Node Tuning Operator with API version v1. +[discrete] [id="upgrading_nto_api_from_v1alpha1_to_v1_or_v2_{context}"] === Upgrading Node Tuning Operator API from v1alpha1 or v1 to v2 diff --git a/modules/cnf-verifying-queue-status.adoc b/modules/cnf-verifying-queue-status.adoc index 06a90dc73ffe..25b2748f9d08 100644 --- a/modules/cnf-verifying-queue-status.adoc +++ b/modules/cnf-verifying-queue-status.adoc @@ -1,6 +1,6 @@ // Module included in the following assemblies: -//CNF-1483 (4.8) -// * scalability_and_performance/cnf-low-latency-tuning.adoc +// +// * scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc [id="verifying-queue-status_{context}"] = Verifying the queue status diff --git a/modules/installation-openstack-nfv-requirements.adoc b/modules/installation-openstack-nfv-requirements.adoc index 877976f50489..9a5b75caec87 100644 --- a/modules/installation-openstack-nfv-requirements.adoc +++ b/modules/installation-openstack-nfv-requirements.adoc @@ -1,6 +1,6 @@ // Module included in the following assemblies: // -// * installing/installing_openstack/installing-openstack-nfv-preparing +// * installing/installing_openstack/installing-openstack-nfv-preparing.adoc :_mod-docs-content-type: REFERENCE [id="installation-openstack-nfv-requirements_{context}"] diff --git a/modules/installation-openstack-ovs-dpdk-performance-profile.adoc b/modules/installation-openstack-ovs-dpdk-performance-profile.adoc index 063f81c6eb87..c3b8717bc9b0 100644 --- a/modules/installation-openstack-ovs-dpdk-performance-profile.adoc +++ b/modules/installation-openstack-ovs-dpdk-performance-profile.adoc @@ -1,17 +1,16 @@ // Module included in the following assemblies: // -// * scalability_and_performance/cnf-create-performance-profiles.adoc -// TODO: ^^^ at the moment +// * scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc :_mod-docs-content-type: REFERENCE [id="installation-openstack-ovs-dpdk-performance-profile_{context}"] -= A performance profile template for clusters that use OVS-DPDK on OpenStack += Performance profile template for clusters that use OVS-DPDK on OpenStack To maximize machine performance in a cluster that uses Open vSwitch with the Data Plane Development Kit (OVS-DPDK) on {rh-openstack-first}, you can use a performance profile. You can use the following performance profile template to create a profile for your deployment. -.A performance profile template for clusters that use OVS-DPDK +.Performance profile template for clusters that use OVS-DPDK [source,yaml] ---- apiVersion: performance.openshift.io/v2 @@ -46,5 +45,3 @@ spec: ---- Insert values that are appropriate for your configuration for the `CPU_ISOLATED`, `CPU_RESERVED`, and `HUGEPAGES_COUNT` keys. - -To learn how to create and use performance profiles, see the "Creating a performance profile" page in the "Scalability and performance" section of the {product-title} documentation. \ No newline at end of file diff --git a/modules/installation-openstack-ovs-dpdk-requirements.adoc b/modules/installation-openstack-ovs-dpdk-requirements.adoc index be9f09ce8257..df6fd060afad 100644 --- a/modules/installation-openstack-ovs-dpdk-requirements.adoc +++ b/modules/installation-openstack-ovs-dpdk-requirements.adoc @@ -1,6 +1,6 @@ // Module included in the following assemblies: // -// * installing/installing_openstack/installing-openstack-nfv-preparing +// * installing/installing_openstack/installing-openstack-nfv-preparing.adoc :_mod-docs-content-type: REFERENCE [id="installation-openstack-ovs-dpdk-requirements_{context}"] diff --git a/modules/installation-openstack-sr-iov-requirements.adoc b/modules/installation-openstack-sr-iov-requirements.adoc index ee2af95cb516..dbb2984cb66a 100644 --- a/modules/installation-openstack-sr-iov-requirements.adoc +++ b/modules/installation-openstack-sr-iov-requirements.adoc @@ -1,6 +1,6 @@ // Module included in the following assemblies: // -// * installing/installing_openstack/installing-openstack-nfv-preparing +// * installing/installing_openstack/installing-openstack-nfv-preparing.adoc :_mod-docs-content-type: REFERENCE [id="installation-openstack-sr-iov-requirements_{context}"] diff --git a/modules/installation-osp-configuring-sr-iov.adoc b/modules/installation-osp-configuring-sr-iov.adoc index ede1f49e9754..176c34f92ab8 100644 --- a/modules/installation-osp-configuring-sr-iov.adoc +++ b/modules/installation-osp-configuring-sr-iov.adoc @@ -1,5 +1,7 @@ // Module included in the following assemblies: // +// * installing/installing_openstack/installing-openstack-installer-ovs-dpdk.adoc +// * installing/installing_openstack/installing-openstack-nfv-preparing.adoc // * installing/installing_openstack/installing-openstack-user-sr-iov.adoc // * installing/installing_openstack/installing-openstack-user-sr-iov-kuryr.adoc diff --git a/networking/hardware_networks/using-dpdk-and-rdma.adoc b/networking/hardware_networks/using-dpdk-and-rdma.adoc index 5bee135988ad..7b9ed974b5fe 100644 --- a/networking/hardware_networks/using-dpdk-and-rdma.adoc +++ b/networking/hardware_networks/using-dpdk-and-rdma.adoc @@ -21,7 +21,7 @@ include::modules/nw-running-dpdk-rootless-tap.adoc[leveloffset=+1] * xref:../../networking/multiple_networks/configuring-additional-network.adoc#nw-multus-enable-container_use_devices_configuring-additional-network[Enabling the container_use_devices boolean] -* xref:../../scalability_and_performance/cnf-create-performance-profiles.adoc#cnf-create-performance-profiles[Creating a performance profile] +* xref:../../scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc#cnf-create-performance-profiles[Creating a performance profile] * xref:../../networking/hardware_networks/configuring-sriov-device.adoc#configuring-sriov-device[Configuring an SR-IOV network device] @@ -54,13 +54,13 @@ include::modules/nw-openstack-hw-offload-testpmd-pod.adoc[leveloffset=+1] [id="additional-resources_using-dpdk-and-rdma"] == Additional resources -* xref:../../scalability_and_performance/cnf-create-performance-profiles.adoc#cnf-about-the-profile-creator-tool_cnf-create-performance-profiles[Creating a performance profile] -* xref:../../scalability_and_performance/cnf-low-latency-tuning.adoc#adjusting-nic-queues-with-the-performance-profile_cnf-master[Reducing NIC queues using the Node Tuning Operator] -* xref:../../scalability_and_performance/cnf-low-latency-tuning.adoc#node-tuning-operator-provisioning-worker-with-real-time-capabilities_cnf-master[Provisioning a worker with real-time capabilities] +* xref:../../scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc#cnf-create-performance-profiles[Creating a performance profile] +* xref:../../scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc#adjusting-nic-queues-with-the-performance-profile_cnf-low-latency-perf-profile[Adjusting the NIC queues with the performance profile] +* xref:../../scalability_and_performance/low_latency_tuning/cnf-provisioning-low-latency-workloads.adoc#cnf-provisioning-low-latency-workloads[Provisioning real-time and low latency workloads] * xref:../../networking/hardware_networks/installing-sriov-operator.adoc#installing-sr-iov-operator_installing-sriov-operator[Installing the SR-IOV Network Operator] * xref:../../networking/hardware_networks/configuring-sriov-device.adoc#nw-sriov-networknodepolicy-object_configuring-sriov-device[Configuring an SR-IOV network device] * xref:../../networking/multiple_networks/configuring-additional-network.adoc#nw-multus-whereabouts_configuring-additional-network[Dynamic IP address assignment configuration with Whereabouts] -* xref:../../scalability_and_performance/cnf-low-latency-tuning.adoc#disabling_interrupt_processing_for_individual_pods_cnf-master[Disabling interrupt processing for individual pods] +* xref:../../scalability_and_performance/low_latency_tuning/cnf-provisioning-low-latency-workloads.adoc#disabling-interrupt-processing-for-individual-pods_cnf-provisioning-low-latency[Disabling interrupt processing for individual pods] * xref:../../networking/hardware_networks/configuring-sriov-net-attach.adoc#configuring-sriov-net-attach[Configuring an SR-IOV Ethernet network attachment] * The xref:../../networking/hardware_networks/about-sriov.adoc#nw-sriov-app-netutil_about-sriov[app-netutil library] provides several API methods for gathering network information about a container's parent pod. diff --git a/operators/operator-reference.adoc b/operators/operator-reference.adoc index 45ccce118cc8..7caf9178c630 100644 --- a/operators/operator-reference.adoc +++ b/operators/operator-reference.adoc @@ -130,7 +130,7 @@ include::modules/node-tuning-operator.adoc[leveloffset=+1] [role="_additional-resources"] [id="cluster-operators-ref-nto-addtl-resources"] === Additional resources -* xref:../scalability_and_performance/cnf-low-latency-tuning.adoc#cnf-understanding-low-latency_cnf-master[Low latency tuning of OCP nodes] +* xref:../scalability_and_performance/low_latency_tuning/cnf-understanding-low-latency.adoc#cnf-understanding-low-latency_cnf-understanding-low-latency[About low latency] include::modules/openshift-apiserver-operator.adoc[leveloffset=+1] diff --git a/release_notes/ocp-4-14-release-notes.adoc b/release_notes/ocp-4-14-release-notes.adoc index d7476e8b9090..a3c20e551593 100644 --- a/release_notes/ocp-4-14-release-notes.adoc +++ b/release_notes/ocp-4-14-release-notes.adoc @@ -1240,7 +1240,8 @@ The Network Observability Operator releases updates independently from the {prod ==== PAO must-gather image added to default must-gather image With this release, the Performance Addon Operator (PAO) must-gather image is no longer required as an argument for the `must-gather` command to capture debugging data related to low-latency tuning. The functions of the PAO must-gather image are now under the default plugin image used by the `must-gather` command without any image arguments. -For further information about gathering debugging information relating to low-latency tuning, see xref:../scalability_and_performance/cnf-low-latency-tuning.adoc#cnf-collecting-low-latency-tuning-debugging-data-for-red-hat-support_cnf-master[Collecting low latency tuning debugging data for Red Hat Support]. +For further information about gathering debugging information relating to low-latency tuning, see xref:../scalability_and_performance/low_latency_tuning/cnf-debugging-low-latency-tuning-status.adoc#cnf-collecting-low-latency-tuning-debugging-data-for-red-hat-support_cnf-debugging-low-latency[Collecting low latency tuning debugging data for Red Hat Support] +. [id="ocp-4-14-NRO-image-must-gather"] ==== Collecting data for the NUMA Resources Operator with the must-gather image of the Operator @@ -1250,7 +1251,7 @@ In this release, the `must-gather` tool is updated to collect the data of the NU [id="ocp-4-14-additional-power-savings-control"] ==== Enabling more control over the C-states for each pod -With this release, you have more control over the C-states for your pods. Now, instead of disabling C-states completely, you can specify a maximum latency in microseconds for C-states. You can configure this option in the `cpu-c-states.crio.io` annotation. This helps to optimize power savings in high-priority applications by enabling some of the shallower C-states instead of disabling them completely. For further information about controlling pod C-states, see xref:../scalability_and_performance/cnf-low-latency-tuning.adoc#node-tuning-operator-pod-power-saving-config_cnf-master[Optional: Power saving configurations]. +With this release, you have more control over the C-states for your pods. Now, instead of disabling C-states completely, you can specify a maximum latency in microseconds for C-states. You can configure this option in the `cpu-c-states.crio.io` annotation. This helps to optimize power savings in high-priority applications by enabling some of the shallower C-states instead of disabling them completely. For further information about controlling pod C-states, see xref:../scalability_and_performance/low_latency_tuning/cnf-provisioning-low-latency-workloads.adoc#cnf-configuring-high-priority-workload-pods_cnf-provisioning-low-latency[Disabling power saving mode for high priority pods]. [id="ocp-4-14-nw-ipv6-spoke-cluster-support"] ==== Support for provisioning IPv6 spoke clusters from dual-stack hub clusters diff --git a/scalability_and_performance/cnf-create-performance-profiles.adoc b/scalability_and_performance/cnf-create-performance-profiles.adoc deleted file mode 100644 index 2caae812a162..000000000000 --- a/scalability_and_performance/cnf-create-performance-profiles.adoc +++ /dev/null @@ -1,37 +0,0 @@ -:_mod-docs-content-type: ASSEMBLY -[id="cnf-create-performance-profiles"] -= Creating a performance profile -include::_attributes/common-attributes.adoc[] -:context: cnf-create-performance-profiles - -toc::[] - -Learn about the Performance Profile Creator (PPC) and how you can use it to create a performance profile. - -[NOTE] -==== -Currently, disabling CPU load balancing is not supported by cgroup v2. As a result, you might not get the desired behavior from performance profiles if you have cgroup v2 enabled. Enabling cgroup v2 is not recommended if you are using performance profiles. -==== - -include::modules/cnf-about-the-profile-creator-tool.adoc[leveloffset=+1] - -include::modules/cnf-gathering-data-about-cluster-using-must-gather.adoc[leveloffset=+2] - -include::modules/cnf-running-the-performance-creator-profile.adoc[leveloffset=+2] - -include::modules/cnf-how-run-podman-to-create-profile.adoc[leveloffset=+3] - -include::modules/cnf-running-the-performance-creator-profile-offline.adoc[leveloffset=+2] - -include::modules/cnf-performance-profile-creator-arguments.adoc[leveloffset=+2] - -[id="cnf-create-performance-profiles-reference"] -== Reference performance profiles - -include::modules/installation-openstack-ovs-dpdk-performance-profile.adoc[leveloffset=+2] - -[id="{context}-additional-resources"] -[role="_additional-resources"] -== Additional resources -* For more information about the `must-gather` tool, -see xref:../support/gathering-cluster-data.adoc#nodes-nodes-managing[Gathering data about your cluster]. diff --git a/scalability_and_performance/cnf-low-latency-tuning.adoc b/scalability_and_performance/cnf-low-latency-tuning.adoc deleted file mode 100644 index 539df10e73cb..000000000000 --- a/scalability_and_performance/cnf-low-latency-tuning.adoc +++ /dev/null @@ -1,101 +0,0 @@ -:_mod-docs-content-type: ASSEMBLY -[id="cnf-low-latency-tuning"] -= Low latency tuning -include::_attributes/common-attributes.adoc[] -:context: cnf-master - -toc::[] - -include::modules/cnf-understanding-low-latency.adoc[leveloffset=+1] - -include::modules/cnf-about_hyperthreading_for_low_latency_and_real_time_applications.adoc[leveloffset=+2] - -[role="_additional-resources"] -.Additional resources - -* xref:../scalability_and_performance/cnf-low-latency-tuning.adoc#configuring_hyperthreading_for_a_cluster_{context}[Configuring hyperthreading for a cluster] - -include::modules/cnf-provisioning-real-time-and-low-latency-workloads.adoc[leveloffset=+1] - -[role="_additional-resources"] -.Additional resources - -* For more information about recommended firmware configuration, see xref:../scalability_and_performance/ztp_far_edge/ztp-vdu-validating-cluster-tuning.adoc#ztp-du-firmware-config-reference_vdu-config-ref[Recommended firmware configuration for vDU cluster hosts]. - -include::modules/cnf-managing-device-interrupt-processing-for-guaranteed-pod-isolated-cpus.adoc[leveloffset=+2] - -include::modules/cnf-use-device-interrupt-processing-for-isolated-cpus.adoc[leveloffset=+2] - -include::modules/cnf-tuning-nodes-for-low-latency-via-performanceprofile.adoc[leveloffset=+1] - -[role="_additional-resources"] -.Additional resources - -* For information on using the Performance Profile Creator (PPC) to generate a performance profile, see xref:../scalability_and_performance/cnf-create-performance-profiles.adoc#cnf-create-performance-profiles[Creating a performance profile]. - -include::modules/cnf-configuring-huge-pages.adoc[leveloffset=+2] - -include::modules/cnf-allocating-multiple-huge-page-sizes.adoc[leveloffset=+2] - -include::modules/cnf-configure_for_irq_dynamic_load_balancing.adoc[leveloffset=+2] - -include::modules/cnf-about-irq-affinity-setting.adoc[leveloffset=+2] - -include::modules/configuring_hyperthreading_for_a_cluster.adoc[leveloffset=+2] - -include::modules/cnf-understanding-workload-hints.adoc[leveloffset=+2] - -[role="_additional-resources"] -.Additional resources - -* For information about using the Performance Profile Creator (PPC) to generate a performance profile, see xref:../scalability_and_performance/cnf-create-performance-profiles.adoc#cnf-create-performance-profiles[Creating a performance profile]. - -include::modules/cnf-configuring-workload-hints.adoc[leveloffset=+2] - -[role="_additional-resources"] -.Additional resources - -* For information about reducing CPU throttling for individual guaranteed pods, see xref:../scalability_and_performance/cnf-low-latency-tuning.adoc#disabling-cpu-cfs-quota_cnf-master[Disabling CPU CFS quota]. - -include::modules/cnf-cpu-infra-container.adoc[leveloffset=+2] - -[role="_additional-resources"] -.Additional resources - -* xref:../scalability_and_performance/cnf-low-latency-tuning.adoc#managing-device-interrupt-processing-for-guaranteed-pod-isolated-cpus_{context}[Managing device interrupt processing for guaranteed pod isolated CPUs] - -* link:https://kubernetes.io/docs/tasks/configure-pod-container/quality-service-pod/#create-a-pod-that-gets-assigned-a-qos-class-of-guaranteed[Create a pod that gets assigned a QoS class of Guaranteed] - -include::modules/cnf-reducing-netqueues-using-nto.adoc[leveloffset=+1] - -include::modules/cnf-adjusting-nic-queues-with-the-performance-profile.adoc[leveloffset=+2] - -[role="_additional-resources"] -.Additional resources - -* xref:../scalability_and_performance/cnf-create-performance-profiles.adoc#cnf-create-performance-profiles[Creating a performance profile]. - -include::modules/cnf-verifying-queue-status.adoc[leveloffset=+2] - -include::modules/cnf-logging-associated-with-adjusting-nic-queues.adoc[leveloffset=+2] - -include::modules/cnf-debugging-low-latency-cnf-tuning-status.adoc[leveloffset=+1] - -include::modules/cnf-collecting-low-latency-tuning-debugging-data-for-red-hat-support.adoc[leveloffset=+1] - -[role="_additional-resources"] -.Additional resources - -* For more information about the `must-gather` tool, see xref:../support/gathering-cluster-data.adoc#gathering-cluster-data[Gathering data about your cluster] - -* For more information about MachineConfig and KubeletConfig, -see xref:../nodes/nodes/nodes-nodes-managing.adoc#nodes-nodes-managing[Managing nodes]. - -* For more information about the Node Tuning Operator, -see xref:../scalability_and_performance/using-node-tuning-operator.adoc#using-node-tuning-operator[Using the Node Tuning Operator]. - -* For more information about the PerformanceProfile, -see xref:../scalability_and_performance/what-huge-pages-do-and-how-they-are-consumed-by-apps.adoc#configuring-huge-pages_huge-pages[Configuring huge pages]. - -* For more information about consuming huge pages from your containers, -see xref:../scalability_and_performance/what-huge-pages-do-and-how-they-are-consumed-by-apps.adoc#how-huge-pages-are-consumed-by-apps_huge-pages[How huge pages are consumed by apps]. diff --git a/scalability_and_performance/cnf-numa-aware-scheduling.adoc b/scalability_and_performance/cnf-numa-aware-scheduling.adoc index 11996d852d9a..56690f7a94c8 100644 --- a/scalability_and_performance/cnf-numa-aware-scheduling.adoc +++ b/scalability_and_performance/cnf-numa-aware-scheduling.adoc @@ -35,7 +35,7 @@ include::modules/cnf-deploying-the-numa-aware-scheduler.adoc[leveloffset=+2] .Additional resources -* xref:../scalability_and_performance/cnf-create-performance-profiles.adoc#cnf-about-the-profile-creator-tool_cnf-create-performance-profiles[About the Performance Profile Creator]. +* xref:../scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc#cnf-about-the-profile-creator-tool_cnf-low-latency-perf-profile[About the Performance Profile Creator] include::modules/cnf-scheduling-numa-aware-workloads.adoc[leveloffset=+2] diff --git a/scalability_and_performance/low_latency_tuning/_attributes b/scalability_and_performance/low_latency_tuning/_attributes new file mode 120000 index 000000000000..20cc1dcb77bf --- /dev/null +++ b/scalability_and_performance/low_latency_tuning/_attributes @@ -0,0 +1 @@ +../../_attributes/ \ No newline at end of file diff --git a/scalability_and_performance/low_latency_tuning/cnf-debugging-low-latency-tuning-status.adoc b/scalability_and_performance/low_latency_tuning/cnf-debugging-low-latency-tuning-status.adoc new file mode 100644 index 000000000000..297cfd533f77 --- /dev/null +++ b/scalability_and_performance/low_latency_tuning/cnf-debugging-low-latency-tuning-status.adoc @@ -0,0 +1,26 @@ +:_mod-docs-content-type: ASSEMBLY +[id="cnf-debugging-low-latency-tuning-status"] += Debugging low latency node tuning status +include::_attributes/common-attributes.adoc[] +:context: cnf-debugging-low-latency + +toc::[] + +Use the `PerformanceProfile` custom resource (CR) status fields for reporting tuning status and debugging latency issues in the cluster node. + +include::modules/cnf-debugging-low-latency-cnf-tuning-status.adoc[leveloffset=+1] + +include::modules/cnf-collecting-low-latency-tuning-debugging-data-for-red-hat-support.adoc[leveloffset=+1] + +[role="_additional-resources"] +.Additional resources + +* xref:../../support/gathering-cluster-data.adoc#gathering-cluster-data[Gathering data about your cluster with the `must-gather` tool] + +* xref:../../nodes/nodes/nodes-nodes-managing.adoc#nodes-nodes-managing[Managing nodes with MachineConfig and KubeletConfig CRs] + +* xref:../../scalability_and_performance/using-node-tuning-operator.adoc#using-node-tuning-operator[Using the Node Tuning Operator] + +* xref:../../scalability_and_performance/what-huge-pages-do-and-how-they-are-consumed-by-apps.adoc#configuring-huge-pages_huge-pages[Configuring huge pages at boot time] + +* xref:../../scalability_and_performance/what-huge-pages-do-and-how-they-are-consumed-by-apps.adoc#how-huge-pages-are-consumed-by-apps_huge-pages[How huge pages are consumed by apps] diff --git a/scalability_and_performance/cnf-performing-platform-verification-latency-tests.adoc b/scalability_and_performance/low_latency_tuning/cnf-performing-platform-verification-latency-tests.adoc similarity index 89% rename from scalability_and_performance/cnf-performing-platform-verification-latency-tests.adoc rename to scalability_and_performance/low_latency_tuning/cnf-performing-platform-verification-latency-tests.adoc index 4a22d01b0d97..8360d03c865d 100644 --- a/scalability_and_performance/cnf-performing-platform-verification-latency-tests.adoc +++ b/scalability_and_performance/low_latency_tuning/cnf-performing-platform-verification-latency-tests.adoc @@ -29,7 +29,7 @@ Your cluster must meet the following requirements before you can run the latency [role="_additional-resources"] .Additional resources -* For more information about creating the cluster performance profile, see xref:../scalability_and_performance/cnf-low-latency-tuning.adoc#node-tuning-operator-provisioning-worker-with-real-time-capabilities_cnf-master[Provisioning a worker with real-time capabilities]. +* xref:../../scalability_and_performance/low_latency_tuning/cnf-provisioning-low-latency-workloads.adoc#cnf-scheduling-workload-onto-worker-with-real-time-capabilities_cnf-provisioning-low-latency[Scheduling a workload onto a worker with real-time capabilities] include::modules/cnf-latency-tests-discovery-mode.adoc[leveloffset=+1] diff --git a/scalability_and_performance/low_latency_tuning/cnf-provisioning-low-latency-workloads.adoc b/scalability_and_performance/low_latency_tuning/cnf-provisioning-low-latency-workloads.adoc new file mode 100644 index 000000000000..44ddeb7bce12 --- /dev/null +++ b/scalability_and_performance/low_latency_tuning/cnf-provisioning-low-latency-workloads.adoc @@ -0,0 +1,57 @@ +:_mod-docs-content-type: ASSEMBLY +[id="cnf-provisioning-low-latency-workloads"] += Provisioning real-time and low latency workloads +include::_attributes/common-attributes.adoc[] +:context: cnf-provisioning-low-latency + +toc::[] + +Many organizations need high performance computing and low, predictable latency, especially in the financial and telecommunications industries. + +{product-title} provides the Node Tuning Operator to implement automatic tuning to achieve low latency performance and consistent response time for {product-title} applications. +You use the performance profile configuration to make these changes. +You can update the kernel to kernel-rt, reserve CPUs for cluster and operating system housekeeping duties, including pod infra containers, isolate CPUs for application containers to run the workloads, and disable unused CPUs to reduce power consumption. + +[NOTE] +==== +When writing your applications, follow the general recommendations described in link:https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux_for_real_time/9/html-single/understanding_rhel_for_real_time/index#assembly_rhel-for-real-time-processes-and-threads_understanding-RHEL-for-Real-Time-core-concepts[RHEL for Real Time processes and threads]. +==== + +[role="_additional-resources"] +.Additional resources + +* xref:../../scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc#cnf-tuning-low-latency-nodes-with-perf-profile[Tuning nodes for low latency with the performance profile] + +include::modules/cnf-scheduling-workload-onto-worker-with-real-time-capabilities.adoc[leveloffset=+1] + +[role="_additional-resources"] +.Additional resources + +* xref:../../nodes/scheduling/nodes-scheduler-node-selectors.adoc#nodes-pods-node-selectors[Placing pods on specific nodes using node selectors] + +* link:https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node[Assigning pods to nodes] + +include::modules/cnf-node-tuning-operator-creating-pod-with-guaranteed-qos-class.adoc[leveloffset=+1] + +include::modules/cnf-node-tuning-operator-disabling-cpu-load-balancing-for-dpdk.adoc[leveloffset=+1] + +include::modules/cnf-configuring-high-priority-workload-pods.adoc[leveloffset=+1] + +[role="_additional-resources"] +.Additional resources + +xref:../../scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc#cnf-configuring-power-saving-for-nodes_cnf-low-latency-perf-profile[Configuring power saving for nodes that run colocated high and low priority workloads] + +include::modules/cnf-disabling-cpu-cfs-quota.adoc[leveloffset=+1] + +[role="_additional-resources"] +.Additional resources + +* xref:../../scalability_and_performance/ztp_far_edge/ztp-vdu-validating-cluster-tuning.adoc#ztp-du-firmware-config-reference_vdu-config-ref[Recommended firmware configuration for vDU cluster hosts] + +include::modules/cnf-disabling-interrupt-processing-for-individual-pods.adoc[leveloffset=+1] + +[role="_additional-resources"] +.Additional resources + +* xref:../../scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile#managing-device-interrupt-processing-for-guaranteed-pod-isolated-cpus_cnf-low-latency-perf-profile[Managing device interrupt processing for guaranteed pod isolated CPUs] diff --git a/scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc b/scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc new file mode 100644 index 000000000000..1371c2fcd8e9 --- /dev/null +++ b/scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc @@ -0,0 +1,93 @@ +:_mod-docs-content-type: ASSEMBLY +[id="cnf-tuning-low-latency-nodes-with-perf-profile"] += Tuning nodes for low latency with the performance profile +include::_attributes/common-attributes.adoc[] +:context: cnf-low-latency-perf-profile + +toc::[] + +Tune nodes for low latency by using the cluster performance profile. +You can restrict CPUs for infra and application containers, configure huge pages, Hyper-Threading, and configure CPU partitions for latency-sensitive processes. + +[role="_additional-resources"] +.Additional resources + +* xref:../../scalability_and_performance/low_latency_tuning/cnf-provisioning-low-latency-workloads.adoc#cnf-provisioning-low-latency-workloads[Provisioning real-time and low latency workloads] + +[id="cnf-create-performance-profiles"] +== Creating a performance profile + +Learn about the Performance Profile Creator (PPC) and how you can use it to create a performance profile. + +include::modules/cnf-about-the-profile-creator-tool.adoc[leveloffset=+2] + +include::modules/cnf-gathering-data-about-cluster-using-must-gather.adoc[leveloffset=+2] + +include::modules/cnf-running-the-performance-creator-profile.adoc[leveloffset=+2] + +[role="_additional-resources"] +.Additional resources + +* For more information about the `must-gather` tool, +see xref:../../support/gathering-cluster-data.adoc#nodes-nodes-managing[Gathering data about your cluster]. + +include::modules/cnf-how-run-podman-to-create-profile.adoc[leveloffset=+3] + +include::modules/cnf-running-the-performance-creator-profile-offline.adoc[leveloffset=+3] + +include::modules/cnf-performance-profile-creator-arguments.adoc[leveloffset=+3] + +[id="cnf-create-performance-profiles-reference"] +=== Reference performance profiles + +Use the following reference performance profiles as the basis to develop your own custom profiles. + +include::modules/installation-openstack-ovs-dpdk-performance-profile.adoc[leveloffset=+3] + +include::modules/cnf-telco-ran-reference-design-performance-profile-template.adoc[leveloffset=+3] + +include::modules/cnf-telco-core-reference-design-performance-profile-template.adoc[leveloffset=+3] + +include::modules/cnf-use-device-interrupt-processing-for-isolated-cpus.adoc[leveloffset=+1] + +include::modules/cnf-configuring-workload-hints.adoc[leveloffset=+1] + +include::modules/cnf-configuring-power-saving-for-nodes.adoc[leveloffset=+1] + +[role="_additional-resources"] +.Additional resources + +* xref:../../scalability_and_performance/low_latency_tuning/cnf-provisioning-low-latency-workloads.adoc#cnf-configuring-high-priority-workload-pods_cnf-provisioning-low-latency[Disabling power saving mode for high priority pods] + +* xref:../../scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc#managing-device-interrupt-processing-for-guaranteed-pod-isolated-cpus_cnf-low-latency-perf-profile[Managing device interrupt processing for guaranteed pod isolated CPUs] + +include::modules/cnf-cpu-infra-container.adoc[leveloffset=+1] + +include::modules/cnf-configuring-hyperthreading-for-a-cluster.adoc[leveloffset=+1] + +include::modules/cnf-managing-device-interrupt-processing-for-guaranteed-pod-isolated-cpus.adoc[leveloffset=+1] + +include::modules/cnf-about-irq-affinity-setting.adoc[leveloffset=+2] + +include::modules/cnf-configure_for_irq_dynamic_load_balancing.adoc[leveloffset=+2] + +include::modules/cnf-configuring-huge-pages.adoc[leveloffset=+1] + +include::modules/cnf-allocating-multiple-huge-page-sizes.adoc[leveloffset=+2] + +[id="cnf-reducing-nic-queues-with-nto"] +== Reducing NIC queues using the Node Tuning Operator + +The Node Tuning Operator facilitates reducing NIC queues for enhanced performance. +Adjustments are made using the performance profile, allowing customization of queues for different network devices. + +include::modules/cnf-adjusting-nic-queues-with-the-performance-profile.adoc[leveloffset=+2] + +[role="_additional-resources"] +.Additional resources + +* xref:../../scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc#cnf-create-performance-profiles[Creating a performance profile]. + +include::modules/cnf-verifying-queue-status.adoc[leveloffset=+2] + +include::modules/cnf-logging-associated-with-adjusting-nic-queues.adoc[leveloffset=+2] diff --git a/scalability_and_performance/low_latency_tuning/cnf-understanding-low-latency.adoc b/scalability_and_performance/low_latency_tuning/cnf-understanding-low-latency.adoc new file mode 100644 index 000000000000..e528aee325b7 --- /dev/null +++ b/scalability_and_performance/low_latency_tuning/cnf-understanding-low-latency.adoc @@ -0,0 +1,20 @@ +:_mod-docs-content-type: ASSEMBLY +[id="cnf-understanding-low-latency"] += Understanding low latency tuning for cluster nodes +include::_attributes/common-attributes.adoc[] +:context: cnf-understanding-low-latency + +toc::[] + +Edge computing has a key role in reducing latency and congestion problems and improving application performance for telco and 5G network applications. +Maintaining a network architecture with the lowest possible latency is key for meeting the network performance requirements of 5G. +Compared to 4G technology, with an average latency of 50 ms, 5G is targeted to reach latency of 1 ms or less. This reduction in latency boosts wireless throughput by a factor of 10. + +include::modules/cnf-understanding-low-latency.adoc[leveloffset=+1] + +include::modules/cnf-about-hyperthreading-for-low-latency-and-real-time-applications.adoc[leveloffset=+1] + +[role="_additional-resources"] +.Additional resources + +* xref:../../scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc#cnf-configuring-hyperthreading-for-a-cluster_cnf-low-latency-perf-profile[Configuring Hyper-Threading for a cluster] diff --git a/scalability_and_performance/low_latency_tuning/images b/scalability_and_performance/low_latency_tuning/images new file mode 120000 index 000000000000..847b03ed0541 --- /dev/null +++ b/scalability_and_performance/low_latency_tuning/images @@ -0,0 +1 @@ +../../images/ \ No newline at end of file diff --git a/scalability_and_performance/low_latency_tuning/modules b/scalability_and_performance/low_latency_tuning/modules new file mode 120000 index 000000000000..36719b9de743 --- /dev/null +++ b/scalability_and_performance/low_latency_tuning/modules @@ -0,0 +1 @@ +../../modules/ \ No newline at end of file diff --git a/scalability_and_performance/low_latency_tuning/snippets b/scalability_and_performance/low_latency_tuning/snippets new file mode 120000 index 000000000000..5a3f5add140e --- /dev/null +++ b/scalability_and_performance/low_latency_tuning/snippets @@ -0,0 +1 @@ +../../snippets/ \ No newline at end of file diff --git a/scalability_and_performance/ztp_far_edge/ztp-advanced-policy-config.adoc b/scalability_and_performance/ztp_far_edge/ztp-advanced-policy-config.adoc index 017e55101872..52db8f4461b0 100644 --- a/scalability_and_performance/ztp_far_edge/ztp-advanced-policy-config.adoc +++ b/scalability_and_performance/ztp_far_edge/ztp-advanced-policy-config.adoc @@ -33,9 +33,7 @@ include::modules/ztp-using-pgt-to-configure-power-states.adoc[leveloffset=+1] [role="_additional-resources"] .Additional resources -* xref:../../scalability_and_performance/cnf-low-latency-tuning.adoc#cnf-understanding-workload-hints_cnf-master[Understanding workload hints] - -* xref:../../scalability_and_performance/cnf-low-latency-tuning.adoc#configuring-workload-hints_cnf-master[Configuring workload hints manually] +* xref:../../scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc#configuring-workload-hints_cnf-low-latency-perf-profile[Configuring node power consumption and realtime processing with workload hints] include::modules/ztp-using-pgt-to-configure-performance-mode.adoc[leveloffset=+2] @@ -46,7 +44,7 @@ include::modules/ztp-using-pgt-to-configure-power-saving-mode.adoc[leveloffset=+ [role="_additional-resources"] .Additional resources -* xref:../../scalability_and_performance/cnf-low-latency-tuning.adoc#node-tuning-operator-pod-power-saving-config_cnf-master[Enabling critical workloads for power saving configurations] +* xref:../../scalability_and_performance/low_latency_tuning/cnf-tuning-low-latency-nodes-with-perf-profile.adoc#cnf-configuring-power-saving-for-nodes_cnf-low-latency-perf-profile[Configuring power saving for nodes that run colocated high and low priority workloads] * xref:../../scalability_and_performance/ztp_far_edge/ztp-reference-cluster-configuration-for-vdu.adoc#ztp-du-configuring-host-firmware-requirements_sno-configure-for-vdu[Configuring host firmware for low latency and high performance]