[ML][TEST] Fix randomly failing HLRC test (#40973)

Made changes to ensure that unique IDs are generated for model snapshots
used by the deleteExpiredDataTest test in the MachineLearningIT suite.

Previously a sleep of 1s was performed between jobs under the assumption
that this would be sufficient to guarantee that the timestamps used in
the composition of the snapshot IDs would be different.

The new approach is to wait on the condition that the old and new
timestamps are in fact different (to 1s resolution).

Author: edsavage

client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java

@@ -878,6 +878,18 @@ private  String createExpiredData(String jobId) throws Exception {
 
         waitForJobToClose(jobId);
 
+        long prevJobTimeStamp = System.currentTimeMillis() / 1000;
+
+        // Check that the current timestamp component, in seconds, differs from previously.
+        // Note that we used to use an 'awaitBusy(() -> false, 1, TimeUnit.SECONDS);'
+        // for the same purpose but the new approach...
+        // a) explicitly checks that the timestamps, in seconds, are actually different and
+        // b) is slightly more efficient since we may not need to wait an entire second for the timestamp to increment
+        assertBusy(() -> {
+            long timeNow = System.currentTimeMillis() / 1000;
+            assertFalse(prevJobTimeStamp >= timeNow);
+        });
+
         // Update snapshot timestamp to force it out of snapshot retention window
         long oneDayAgo = nowMillis - TimeValue.timeValueHours(24).getMillis() - 1;
         updateModelSnapshotTimestamp(jobId, String.valueOf(oneDayAgo));
@@ -1418,6 +1430,7 @@ private void startDatafeed(String datafeedId, String start, String end) throws E
     }
 
     private void updateModelSnapshotTimestamp(String jobId, String timestamp) throws Exception {
+
         MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
 
         GetModelSnapshotsRequest getModelSnapshotsRequest = new GetModelSnapshotsRequest(jobId);
@@ -1435,9 +1448,6 @@ private void updateModelSnapshotTimestamp(String jobId, String timestamp) throws
         UpdateRequest updateSnapshotRequest = new UpdateRequest(".ml-anomalies-" + jobId, "_doc", documentId);
         updateSnapshotRequest.doc(snapshotUpdate.getBytes(StandardCharsets.UTF_8), XContentType.JSON);
         highLevelClient().update(updateSnapshotRequest, RequestOptions.DEFAULT);
-
-        // Wait a second to ensure subsequent model snapshots will have a different ID (it depends on epoch seconds)
-        awaitBusy(() -> false, 1, TimeUnit.SECONDS);
     }