[DE-561] ArangoSearch column cache (#492)

* inverted index: primaryKeyCache

* updated test docker images

* primaryKeyCache test fix

* inverted index: primarySort.cache

* inverted index: StoredValue.cache

* inverted index: cache

* inverted index: InvertedIndexField.cache

* test fixes

* arangosearch view: FieldLink.cache

* arangosearch view: primarySortCache & primaryKeyCache

Author: rashtao

.github/workflows/native.yml

@@ -14,7 +14,7 @@ jobs:
       fail-fast: false
       matrix:
         docker-img:
-          - docker.io/arangodb/enterprise:3.10.1
+          - docker.io/arangodb/enterprise:3.10.5
         topology:
           - single
         java-version:
@@ -53,7 +53,7 @@ jobs:
       fail-fast: false
       matrix:
         docker-img:
-          - docker.io/arangodb/enterprise:3.10.1
+          - docker.io/arangodb/enterprise:3.10.5
         topology:
           - single
         java-version:

.github/workflows/test.yml

@@ -25,12 +25,10 @@ jobs:
       fail-fast: false
       matrix:
         docker-img:
-          - docker.io/arangodb/arangodb:3.8.8
-          - docker.io/arangodb/arangodb:3.9.9
-          - docker.io/arangodb/arangodb:3.10.4
-          - docker.io/arangodb/enterprise:3.8.8
-          - docker.io/arangodb/enterprise:3.9.9
-          - docker.io/arangodb/enterprise:3.10.4
+          - docker.io/arangodb/arangodb:3.9.10
+          - docker.io/arangodb/arangodb:3.10.5
+          - docker.io/arangodb/enterprise:3.9.10
+          - docker.io/arangodb/enterprise:3.10.5
         topology:
           - single
           - cluster
@@ -81,7 +79,7 @@ jobs:
       fail-fast: false
       matrix:
         docker-img:
-          - docker.io/arangodb/arangodb:3.10.4
+          - docker.io/arangodb/arangodb:3.10.5
         topology:
           - single
         java-version:
@@ -131,7 +129,7 @@ jobs:
       fail-fast: false
       matrix:
         docker-img:
-          - docker.io/arangodb/enterprise:3.10.4
+          - docker.io/arangodb/enterprise:3.10.5
         topology:
           - single
           - cluster
@@ -184,7 +182,7 @@ jobs:
           - 2.11.4
           - 2.10.5
         docker-img:
-          - docker.io/arangodb/arangodb:3.10.4
+          - docker.io/arangodb/arangodb:3.10.5
         topology:
           - single
         db-ext-names:
@@ -222,7 +220,7 @@ jobs:
       fail-fast: false
       matrix:
         docker-img:
-          - docker.io/arangodb/arangodb:3.10.4
+          - docker.io/arangodb/arangodb:3.10.5
         topology:
           - single
           - cluster
@@ -267,7 +265,7 @@ jobs:
       fail-fast: false
       matrix:
         docker-img:
-          - docker.io/arangodb/enterprise:3.10.4
+          - docker.io/arangodb/enterprise:3.10.5
         topology:
           - cluster
         db-ext-names:

core/src/main/java/com/arangodb/entity/InvertedIndexEntity.java

@@ -57,6 +57,8 @@ public final class InvertedIndexEntity {
     private Long writebufferIdle;
     private Long writebufferActive;
     private Long writebufferSizeMax;
+    private Boolean cache;
+    private Boolean primaryKeyCache;
 
     public String getId() {
         return id;
@@ -149,4 +151,12 @@ public Long getWritebufferActive() {
     public Long getWritebufferSizeMax() {
         return writebufferSizeMax;
     }
+
+    public Boolean getCache() {
+        return cache;
+    }
+
+    public Boolean getPrimaryKeyCache() {
+        return primaryKeyCache;
+    }
 }

core/src/main/java/com/arangodb/entity/InvertedIndexField.java

@@ -15,6 +15,7 @@ public final class InvertedIndexField {
     private Boolean includeAllFields;
     private Boolean searchField;
     private Boolean trackListPositions;
+    private Boolean cache;
     private final Set<AnalyzerFeature> features = new HashSet<>();
     private Collection<InvertedIndexField> nested;
 
@@ -103,6 +104,23 @@ public InvertedIndexField trackListPositions(Boolean trackListPositions) {
         return this;
     }
 
+    public Boolean getCache() {
+        return cache;
+    }
+
+    /**
+     * @param cache Enable this option to always cache the field normalization values in memory for this specific field.
+     *              This can improve the performance of scoring and ranking queries. Otherwise, these values are
+     *              memory-mapped and it is up to the operating system to load them from disk into memory and to evict
+     *              them from memory. (Enterprise Edition only)
+     * @return this
+     * @since ArangoDB 3.10.2
+     */
+    public InvertedIndexField cache(Boolean cache) {
+        this.cache = cache;
+        return this;
+    }
+
     public Set<AnalyzerFeature> getFeatures() {
         return features;
     }
@@ -139,11 +157,11 @@ public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         InvertedIndexField that = (InvertedIndexField) o;
-        return Objects.equals(name, that.name) && Objects.equals(analyzer, that.analyzer) && Objects.equals(includeAllFields, that.includeAllFields) && Objects.equals(searchField, that.searchField) && Objects.equals(trackListPositions, that.trackListPositions) && Objects.equals(features, that.features) && Objects.equals(nested, that.nested);
+        return Objects.equals(name, that.name) && Objects.equals(analyzer, that.analyzer) && Objects.equals(includeAllFields, that.includeAllFields) && Objects.equals(searchField, that.searchField) && Objects.equals(trackListPositions, that.trackListPositions) && Objects.equals(cache, that.cache) && Objects.equals(features, that.features) && Objects.equals(nested, that.nested);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(name, analyzer, includeAllFields, searchField, trackListPositions, features, nested);
+        return Objects.hash(name, analyzer, includeAllFields, searchField, trackListPositions, cache, features, nested);
     }
 }

core/src/main/java/com/arangodb/entity/InvertedIndexPrimarySort.java

@@ -17,6 +17,7 @@
 public final class InvertedIndexPrimarySort {
     private final List<Field> fields = new ArrayList<>();
     private ArangoSearchCompression compression;
+    private Boolean cache;
 
     public List<Field> getFields() {
         return fields;
@@ -44,17 +45,34 @@ public InvertedIndexPrimarySort compression(ArangoSearchCompression compression)
         return this;
     }
 
+    public Boolean getCache() {
+        return cache;
+    }
+
+    /**
+     * @param cache If you enable this option, then the primary sort columns are always cached in memory. This can
+     *              improve the performance of queries that utilize the primary sort order. Otherwise, these values are
+     *              memory-mapped and it is up to the operating system to load them from disk into memory and to evict
+     *              them from memory (Enterprise Edition only).
+     * @return this
+     * @since ArangoDB 3.10.2
+     */
+    public InvertedIndexPrimarySort cache(Boolean cache) {
+        this.cache = cache;
+        return this;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         InvertedIndexPrimarySort that = (InvertedIndexPrimarySort) o;
-        return Objects.equals(fields, that.fields) && compression == that.compression;
+        return Objects.equals(fields, that.fields) && compression == that.compression && Objects.equals(cache, that.cache);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(fields, compression);
+        return Objects.hash(fields, compression, cache);
     }
 
     public static class Field {

core/src/main/java/com/arangodb/entity/arangosearch/ArangoSearchPropertiesEntity.java

@@ -41,6 +41,8 @@ public final class ArangoSearchPropertiesEntity extends ViewEntity {
     private Collection<CollectionLink> links;
     private ArangoSearchCompression primarySortCompression;
     private Collection<StoredValue> storedValues;
+    private Boolean primarySortCache;
+    private Boolean primaryKeyCache;
 
     /**
      * @return Wait at least this many milliseconds between committing view data store changes and making documents
@@ -119,4 +121,11 @@ public Collection<StoredValue> getStoredValues() {
         return storedValues;
     }
 
+    public Boolean getPrimarySortCache() {
+        return primarySortCache;
+    }
+
+    public Boolean getPrimaryKeyCache() {
+        return primaryKeyCache;
+    }
 }

core/src/main/java/com/arangodb/entity/arangosearch/FieldLink.java

@@ -21,6 +21,7 @@ public final class FieldLink {
     private Collection<FieldLink> fields;
     private Collection<FieldLink> nested;
     private Boolean inBackground;
+    private Boolean cache;
 
     private FieldLink(final String name) {
         super();
@@ -112,6 +113,19 @@ public FieldLink inBackground(final Boolean inBackground) {
         return this;
     }
 
+    /**
+     * @param cache If you enable this option, then field normalization values are always cached in memory. This can
+     *              improve the performance of scoring and ranking queries. Otherwise, these values are memory-mapped
+     *              and it is up to the operating system to load them from disk into memory and to evict them from
+     *              memory.
+     * @return link
+     * @since ArangoDB 3.9.5, Enterprise Edition only
+     */
+    public FieldLink cache(final Boolean cache) {
+        this.cache = cache;
+        return this;
+    }
+
     @JsonIgnore
     public String getName() {
         return name;
@@ -146,4 +160,8 @@ public Collection<FieldLink> getNested() {
     public Boolean getInBackground() {
         return inBackground;
     }
+
+    public Boolean getCache() {
+        return cache;
+    }
 }

core/src/main/java/com/arangodb/model/InvertedIndexOptions.java

@@ -51,6 +51,8 @@ public final class InvertedIndexOptions extends IndexOptions<InvertedIndexOption
     private Long writebufferIdle;
     private Long writebufferActive;
     private Long writebufferSizeMax;
+    private Boolean cache;
+    private Boolean primaryKeyCache;
 
     public InvertedIndexOptions() {
         super();
@@ -347,16 +349,52 @@ public InvertedIndexOptions writebufferSizeMax(Long writebufferSizeMax) {
         return this;
     }
 
+    public Boolean getCache() {
+        return cache;
+    }
+
+    /**
+     * @param cache Enable this option to always cache the field normalization values in memory for all fields by
+     *              default. This can improve the performance of scoring and ranking queries. Otherwise, these values
+     *              are memory-mapped and it is up to the operating system to load them from disk into memory and to
+     *              evict them from memory.
+     *              <p/>
+     *              Default: `false`. (Enterprise Edition only)
+     * @return this
+     * @since ArangoDB 3.10.2
+     */
+    public InvertedIndexOptions cache(Boolean cache) {
+        this.cache = cache;
+        return this;
+    }
+
+    public Boolean getPrimaryKeyCache() {
+        return primaryKeyCache;
+    }
+
+    /**
+     * @param primaryKeyCache If you enable this option, then the primary key columns are always cached in memory. This
+     *                        can improve the performance of queries that return many documents. Otherwise, these values
+     *                        are memory-mapped and it is up to the operating system to load them from disk into memory
+     *                        and to evict them from memory (Enterprise Edition only). (default: false)
+     * @return this
+     * @since ArangoDB 3.10.2
+     */
+    public InvertedIndexOptions primaryKeyCache(Boolean primaryKeyCache) {
+        this.primaryKeyCache = primaryKeyCache;
+        return this;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         InvertedIndexOptions that = (InvertedIndexOptions) o;
-        return type == that.type && Objects.equals(parallelism, that.parallelism) && Objects.equals(primarySort, that.primarySort) && Objects.equals(storedValues, that.storedValues) && Objects.equals(analyzer, that.analyzer) && Objects.equals(features, that.features) && Objects.equals(includeAllFields, that.includeAllFields) && Objects.equals(trackListPositions, that.trackListPositions) && Objects.equals(searchField, that.searchField) && Objects.equals(fields, that.fields) && Objects.equals(consolidationIntervalMsec, that.consolidationIntervalMsec) && Objects.equals(commitIntervalMsec, that.commitIntervalMsec) && Objects.equals(cleanupIntervalStep, that.cleanupIntervalStep) && Objects.equals(consolidationPolicy, that.consolidationPolicy) && Objects.equals(writebufferIdle, that.writebufferIdle) && Objects.equals(writebufferActive, that.writebufferActive) && Objects.equals(writebufferSizeMax, that.writebufferSizeMax);
+        return type == that.type && Objects.equals(parallelism, that.parallelism) && Objects.equals(primarySort, that.primarySort) && Objects.equals(storedValues, that.storedValues) && Objects.equals(analyzer, that.analyzer) && Objects.equals(features, that.features) && Objects.equals(includeAllFields, that.includeAllFields) && Objects.equals(trackListPositions, that.trackListPositions) && Objects.equals(searchField, that.searchField) && Objects.equals(fields, that.fields) && Objects.equals(consolidationIntervalMsec, that.consolidationIntervalMsec) && Objects.equals(commitIntervalMsec, that.commitIntervalMsec) && Objects.equals(cleanupIntervalStep, that.cleanupIntervalStep) && Objects.equals(consolidationPolicy, that.consolidationPolicy) && Objects.equals(writebufferIdle, that.writebufferIdle) && Objects.equals(writebufferActive, that.writebufferActive) && Objects.equals(writebufferSizeMax, that.writebufferSizeMax) && Objects.equals(cache, that.cache) && Objects.equals(primaryKeyCache, that.primaryKeyCache);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(type, parallelism, primarySort, storedValues, analyzer, features, includeAllFields, trackListPositions, searchField, fields, consolidationIntervalMsec, commitIntervalMsec, cleanupIntervalStep, consolidationPolicy, writebufferIdle, writebufferActive, writebufferSizeMax);
+        return Objects.hash(type, parallelism, primarySort, storedValues, analyzer, features, includeAllFields, trackListPositions, searchField, fields, consolidationIntervalMsec, commitIntervalMsec, cleanupIntervalStep, consolidationPolicy, writebufferIdle, writebufferActive, writebufferSizeMax, cache, primaryKeyCache);
     }
 }

core/src/main/java/com/arangodb/model/arangosearch/ArangoSearchCreateOptions.java

@@ -43,6 +43,8 @@ public final class ArangoSearchCreateOptions {
     private Collection<PrimarySort> primarySorts;
     private ArangoSearchCompression primarySortCompression;
     private Collection<StoredValue> storedValues;
+    private Boolean primarySortCache;
+    private Boolean primaryKeyCache;
 
     public ArangoSearchCreateOptions() {
         super();
@@ -162,6 +164,32 @@ public ArangoSearchCreateOptions storedValues(final StoredValue... storedValues)
         return this;
     }
 
+    /**
+     * @param primarySortCache If you enable this option, then the primary sort columns are always cached in memory.
+     *                         This can improve the performance of queries that utilize the primary sort order.
+     *                         Otherwise, these values are memory-mapped and it is up to the operating system to load
+     *                         them from disk into memory and to evict them from memory.
+     * @return options
+     * @since ArangoDB 3.9.6, Enterprise Edition only
+     */
+    public ArangoSearchCreateOptions primarySortCache(final Boolean primarySortCache) {
+        this.primarySortCache = primarySortCache;
+        return this;
+    }
+
+    /**
+     * @param primaryKeyCache If you enable this option, then the primary key columns are always cached in memory. This
+     *                        can improve the performance of queries that return many documents. Otherwise, these values
+     *                        are memory-mapped and it is up to the operating system to load them from disk into memory
+     *                        and to evict them from memory.
+     * @return options
+     * @since ArangoDB 3.9.6, Enterprise Edition only
+     */
+    public ArangoSearchCreateOptions primaryKeyCache(final Boolean primaryKeyCache) {
+        this.primaryKeyCache = primaryKeyCache;
+        return this;
+    }
+
     public String getName() {
         return name;
     }
@@ -203,4 +231,12 @@ public Collection<StoredValue> getStoredValues() {
         return storedValues;
     }
 
+    public Boolean getPrimarySortCache() {
+        return primarySortCache;
+    }
+
+    public Boolean getPrimaryKeyCache() {
+        return primaryKeyCache;
+    }
+
 }