[Test] Ensure test pass with explicit counter and timer (elastic#74666)

ApiKeyServiceTests#testApiKeyAuthCacheWillLogWarningOnPossibleThrashing
is a new tested added as part of elastic#74388. But it fails intermittently on CI. The likely
reason is that either the counter or the timer is not the expected value when the
code runs.
This PR explicitly configures both the counter and timer needed for triggering
the warning message so that the warning message should always fire as expected.

Resolves: elastic#74586

Author: ywangd

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java

@@ -1149,6 +1149,11 @@ LongAdder getEvictionCounter() {
         return evictionCounter;
     }
 
+    // package private for test
+    AtomicLong getLastEvictionCheckedAt() {
+        return lastEvictionCheckedAt;
+    }
+
     /**
      * Returns realm name for the authenticated user.
      * If the user is authenticated by realm type {@value API_KEY_REALM_TYPE}

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java

@@ -123,6 +123,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
@@ -778,39 +779,57 @@ public void testApiKeyCacheWillNotTraceLogOnEvictionDueToCacheTtl() throws Illeg
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/74586")
     public void testApiKeyAuthCacheWillLogWarningOnPossibleThrashing() throws Exception {
         ApiKeyService service = createApiKeyService(
-            Settings.builder().put("xpack.security.authc.api_key.cache.max_keys", 1).build());
+            Settings.builder().put("xpack.security.authc.api_key.cache.max_keys", 2).build());
         final Cache<String, ListenableFuture<CachedApiKeyHashResult>> apiKeyAuthCache = service.getApiKeyAuthCache();
 
         // Fill the cache
-        final String apiKeyId = randomAlphaOfLength(22);
-        apiKeyAuthCache.put(apiKeyId, new ListenableFuture<>());
+        apiKeyAuthCache.put(randomAlphaOfLength(20), new ListenableFuture<>());
+        apiKeyAuthCache.put(randomAlphaOfLength(21), new ListenableFuture<>());
         final Logger logger = LogManager.getLogger(ApiKeyService.class);
-        Loggers.setLevel(logger, Level.WARN);
+        Loggers.setLevel(logger, Level.TRACE);
         final MockLogAppender appender = new MockLogAppender();
         Loggers.addAppender(logger, appender);
         appender.start();
 
         try {
             // Prepare the warning logging to trigger
             service.getEvictionCounter().add(4500);
+            final long thrashingCheckIntervalInSeconds = 300L;
+            final long secondsToNanoSeconds = 1_000_000_000L;
+            // Calculate the last thrashing check time to ensure that the elapsed time is longer than the
+            // thrashing checking interval (300 seconds). Also add another 10 seconds to counter any
+            // test flakiness.
+            final long lastCheckedAt = System.nanoTime() - (thrashingCheckIntervalInSeconds + 10L) * secondsToNanoSeconds;
+            service.getLastEvictionCheckedAt().set(lastCheckedAt);
             // Ensure the counter is updated
             assertBusy(() -> assertThat(service.getEvictionCounter().longValue() >= 4500, is(true)));
+            appender.addExpectation(new MockLogAppender.SeenEventExpectation(
+                "evict", ApiKeyService.class.getName(), Level.TRACE,
+                "API key with ID [*] was evicted from the authentication cache*"
+            ));
             appender.addExpectation(new MockLogAppender.SeenEventExpectation(
                 "thrashing", ApiKeyService.class.getName(), Level.WARN,
                 "Possible thrashing for API key authentication cache,*"
             ));
-            apiKeyAuthCache.put(randomAlphaOfLength(23), new ListenableFuture<>());
+            apiKeyAuthCache.put(randomAlphaOfLength(22), new ListenableFuture<>());
             appender.assertAllExpectationsMatched();
 
+            // Counter and timer should be reset
+            assertThat(service.getLastEvictionCheckedAt().get(), lessThanOrEqualTo(System.nanoTime()));
+            assertBusy(() -> assertThat(service.getEvictionCounter().longValue(), equalTo(0L)));
+
             // Will not log warning again for the next eviction because of throttling
+            appender.addExpectation(new MockLogAppender.SeenEventExpectation(
+                "evict-again", ApiKeyService.class.getName(), Level.TRACE,
+                "API key with ID [*] was evicted from the authentication cache*"
+            ));
             appender.addExpectation(new MockLogAppender.UnseenEventExpectation(
                 "throttling", ApiKeyService.class.getName(), Level.WARN,
                 "Possible thrashing for API key authentication cache,*"
             ));
-            apiKeyAuthCache.put(randomAlphaOfLength(24), new ListenableFuture<>());
+            apiKeyAuthCache.put(randomAlphaOfLength(23), new ListenableFuture<>());
             appender.assertAllExpectationsMatched();
         } finally {
             appender.stop();