add eviction metrics to caches (#5523)

Author: trinity-1686a

quickwit/quickwit-storage/src/cache/byte_range_cache.rs

@@ -314,6 +314,8 @@ impl<T: 'static + ToOwned + ?Sized + Ord> NeedMutByteRangeCache<T> {
         self.num_bytes -= num_bytes as u64;
         self.cache_counters.in_cache_count.dec();
         self.cache_counters.in_cache_num_bytes.sub(num_bytes as i64);
+        self.cache_counters.evict_num_items.inc();
+        self.cache_counters.evict_num_bytes.inc_by(num_bytes as u64);
     }
 }
 

quickwit/quickwit-storage/src/cache/memory_sized_cache.rs

@@ -74,6 +74,7 @@ struct NeedMutMemorySizedCache<K: Hash + Eq> {
 
 impl<K: Hash + Eq> Drop for NeedMutMemorySizedCache<K> {
     fn drop(&mut self) {
+        // we don't count this toward evicted entries, as we are clearing the whole cache
         self.cache_counters
             .in_cache_count
             .sub(self.num_items as i64);
@@ -110,6 +111,8 @@ impl<K: Hash + Eq> NeedMutMemorySizedCache<K> {
         self.num_bytes -= num_bytes;
         self.cache_counters.in_cache_count.dec();
         self.cache_counters.in_cache_num_bytes.sub(num_bytes as i64);
+        self.cache_counters.evict_num_items.inc();
+        self.cache_counters.evict_num_bytes.inc_by(num_bytes);
     }
 
     pub fn get<Q>(&mut self, cache_key: &Q) -> Option<OwnedBytes>

quickwit/quickwit-storage/src/file_descriptor_cache.rs

@@ -119,6 +119,9 @@ impl FileDescriptorCache {
         self.fd_cache_metrics
             .in_cache_count
             .set(fd_cache_lock.len() as i64);
+        self.fd_cache_metrics
+            .evict_num_items
+            .inc_by(split_ids.len() as u64);
     }
 
     pub async fn get_or_open_split_file(

quickwit/quickwit-storage/src/metrics.rs

@@ -120,6 +120,8 @@ pub struct CacheMetrics {
     pub hits_num_items: IntCounter,
     pub hits_num_bytes: IntCounter,
     pub misses_num_items: IntCounter,
+    pub evict_num_items: IntCounter,
+    pub evict_num_bytes: IntCounter,
 }
 
 impl CacheMetrics {
@@ -157,6 +159,18 @@ impl CacheMetrics {
                 CACHE_METRICS_NAMESPACE,
                 &[("component_name", component_name)],
             ),
+            evict_num_items: new_counter_with_labels(
+                "cache_evict_total",
+                "Number of cache entry evicted by component",
+                CACHE_METRICS_NAMESPACE,
+                &[("component_name", component_name)],
+            ),
+            evict_num_bytes: new_counter_with_labels(
+                "cache_evict_bytes",
+                "Number of cache entry evicted in bytes by component",
+                CACHE_METRICS_NAMESPACE,
+                &[("component_name", component_name)],
+            ),
         }
     }
 }

quickwit/quickwit-storage/src/split_cache/split_table.rs

@@ -165,6 +165,14 @@ impl SplitTable {
                     .searcher_split_cache
                     .in_cache_num_bytes
                     .sub(num_bytes as i64);
+                crate::metrics::STORAGE_METRICS
+                    .searcher_split_cache
+                    .evict_num_items
+                    .inc();
+                crate::metrics::STORAGE_METRICS
+                    .searcher_split_cache
+                    .evict_num_bytes
+                    .inc_by(num_bytes);
                 &mut self.on_disk_splits
             }
         };