wait in test until metrics are available before running tests that rely on the values.

up until now, the metrics go routine ran in tests with time.Sleep(0), which means metrics were avaiable immediately.
while in tests in might be acceptable to wait few seconds using sleep, in the actual code (not tests) it's a bad practice to use
sleep which was replaced with a ticker (to perform periodic task in an endless loop).

Signed-off-by: Nir Rozenbaum <[email protected]>

Author: nirrozenbaum

test/integration/epp/hermetic_test.go

@@ -1496,6 +1496,7 @@ func TestFullDuplexStreamed_KubeInferenceModelRequest(t *testing.T) {
 			if len(test.wantMetrics) != 0 {
 				for metricName, value := range test.wantMetrics {
 					if err := metricsutils.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(value), metricName); err != nil {
+						t.Log(metricName, value)
 						t.Error(err)
 					}
 				}
@@ -1548,6 +1549,8 @@ func setUpHermeticServer(t *testing.T, podAndMetrics map[backendmetrics.Pod]*bac
 		}
 	}()
 
+	time.Sleep(serverRunner.RefreshPrometheusMetricsInterval) // wait for metrics to get published before running tests that rely on these metrics
+
 	// check if all pods are synced to datastore
 	assert.EventuallyWithT(t, func(t *assert.CollectT) {
 		assert.Len(t, serverRunner.Datastore.PodGetAll(), len(podAndMetrics), "Datastore not synced")