From a76cb4d4a02b5f715a94da002384befa715ed171 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Wed, 31 Jul 2024 11:44:06 -0700 Subject: [PATCH 01/15] Make completion postings format extensible --- server/src/main/java/module-info.java | 6 ++++- .../org/elasticsearch/env/BuildVersion.java | 2 ++ .../index/codec/PerFieldFormatSupplier.java | 7 +++--- .../index/mapper/CompletionFieldMapper.java | 19 ++++++++++++++-- .../index/mapper/MappingLookup.java | 17 -------------- .../internal/PostingsFormatExtension.java | 22 +++++++++++++++++++ 6 files changed, 50 insertions(+), 23 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/internal/PostingsFormatExtension.java diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 2c3f3c20abeb4..7f66c7d01d7dd 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -8,6 +8,7 @@ import org.elasticsearch.index.codec.Elasticsearch814Codec; import org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat; +import org.elasticsearch.internal.PostingsFormatExtension; import org.elasticsearch.plugins.internal.RestExtension; /** The Elasticsearch Server Module. */ @@ -289,7 +290,8 @@ to org.elasticsearch.serverless.version, org.elasticsearch.serverless.buildinfo, - org.elasticsearch.serverless.constants; + org.elasticsearch.serverless.constants, + org.elasticsearch.serverless.codec; exports org.elasticsearch.lucene.analysis.miscellaneous; exports org.elasticsearch.lucene.grouping; exports org.elasticsearch.lucene.queries; @@ -395,6 +397,7 @@ org.elasticsearch.stateless, org.elasticsearch.settings.secure, org.elasticsearch.serverless.constants, + org.elasticsearch.serverless.codec, org.elasticsearch.serverless.apifiltering, org.elasticsearch.internal.security; @@ -415,6 +418,7 @@ uses org.elasticsearch.node.internal.TerminationHandlerProvider; uses org.elasticsearch.internal.VersionExtension; uses org.elasticsearch.internal.BuildExtension; + uses PostingsFormatExtension; uses org.elasticsearch.features.FeatureSpecification; uses org.elasticsearch.plugins.internal.LoggingDataProvider; uses org.elasticsearch.cluster.metadata.DataStreamFactoryRetention; diff --git a/server/src/main/java/org/elasticsearch/env/BuildVersion.java b/server/src/main/java/org/elasticsearch/env/BuildVersion.java index e19ad87932e7f..d6b9ad8f44f14 100644 --- a/server/src/main/java/org/elasticsearch/env/BuildVersion.java +++ b/server/src/main/java/org/elasticsearch/env/BuildVersion.java @@ -100,8 +100,10 @@ private static class CurrentExtensionHolder { private static BuildExtension findExtension() { return ExtensionLoader.loadSingleton(ServiceLoader.load(BuildExtension.class)).orElse(new DefaultBuildExtension()); } + } + private static class DefaultBuildExtension implements BuildExtension { @Override public Build getCurrentBuild() { diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java index 0b4bb9dfc10ae..34707c1767895 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java +++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.codec.bloomfilter.ES87BloomFilterPostingsFormat; import org.elasticsearch.index.codec.postings.ES812PostingsFormat; import org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat; +import org.elasticsearch.index.mapper.CompletionFieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; @@ -58,9 +59,9 @@ public PostingsFormat getPostingsFormatForField(String field) { private PostingsFormat internalGetPostingsFormatForField(String field) { if (mapperService != null) { - final PostingsFormat format = mapperService.mappingLookup().getPostingsFormat(field); - if (format != null) { - return format; + Mapper mapper = mapperService.mappingLookup().getMapper(field); + if (mapper instanceof CompletionFieldMapper) { + return CompletionFieldMapper.postingsFormat(); } } // return our own posting format using PFOR diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index 23272fbd354f3..7ddc985906698 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -27,6 +27,8 @@ import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.internal.PostingsFormatExtension; +import org.elasticsearch.plugins.ExtensionLoader; import org.elasticsearch.search.suggest.completion.CompletionSuggester; import org.elasticsearch.search.suggest.completion.context.ContextMapping; import org.elasticsearch.search.suggest.completion.context.ContextMappings; @@ -49,6 +51,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.ServiceLoader; import java.util.Set; /** @@ -369,8 +372,20 @@ public CompletionFieldType fieldType() { return (CompletionFieldType) super.fieldType(); } - static PostingsFormat postingsFormat() { - return PostingsFormat.forName("Completion99"); + public static PostingsFormat postingsFormat() { + return PostingsFormatHolder.POSTINGS_FORMAT; + } + + private static class PostingsFormatHolder { + private static final PostingsFormat POSTINGS_FORMAT = getPostingsFormat(); + + private static PostingsFormat getPostingsFormat() { + String defaultName = "Completion99"; + String codecName = ExtensionLoader.loadSingleton(ServiceLoader.load(PostingsFormatExtension.class)) + .map(PostingsFormatExtension::getCompletionPostingsFormatName) + .orElse(defaultName); + return PostingsFormat.forName(codecName); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index ffb32f2c9b2a4..d0170af15dffa 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -8,7 +8,6 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.codecs.PostingsFormat; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.InferenceFieldMetadata; import org.elasticsearch.index.IndexSettings; @@ -20,7 +19,6 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -57,7 +55,6 @@ private CacheKey() {} private final Map indexAnalyzersMap; private final List indexTimeScriptMappers; private final Mapping mapping; - private final Set completionFields; private final int totalFieldsCount; /** @@ -160,7 +157,6 @@ private MappingLookup( this.nestedLookup = NestedLookup.build(nestedMappers); final Map indexAnalyzersMap = new HashMap<>(); - final Set completionFields = new HashSet<>(); final List indexTimeScriptMappers = new ArrayList<>(); for (FieldMapper mapper : mappers) { if (objects.containsKey(mapper.fullPath())) { @@ -173,9 +169,6 @@ private MappingLookup( if (mapper.hasScript()) { indexTimeScriptMappers.add(mapper); } - if (mapper instanceof CompletionFieldMapper) { - completionFields.add(mapper.fullPath()); - } } for (FieldAliasMapper aliasMapper : aliasMappers) { @@ -210,7 +203,6 @@ private MappingLookup( this.objectMappers = Map.copyOf(objects); this.runtimeFieldMappersCount = runtimeFields.size(); this.indexAnalyzersMap = Map.copyOf(indexAnalyzersMap); - this.completionFields = Set.copyOf(completionFields); this.indexTimeScriptMappers = List.copyOf(indexTimeScriptMappers); runtimeFields.stream().flatMap(RuntimeField::asMappedFieldTypes).map(MappedFieldType::name).forEach(this::validateDoesNotShadow); @@ -284,15 +276,6 @@ public Iterable fieldMappers() { return fieldMappers.values(); } - /** - * Gets the postings format for a particular field - * @param field the field to retrieve a postings format for - * @return the postings format for the field, or {@code null} if the default format should be used - */ - public PostingsFormat getPostingsFormat(String field) { - return completionFields.contains(field) ? CompletionFieldMapper.postingsFormat() : null; - } - void checkLimits(IndexSettings settings) { checkFieldLimit(settings.getMappingTotalFieldsLimit()); checkObjectDepthLimit(settings.getMappingDepthLimit()); diff --git a/server/src/main/java/org/elasticsearch/internal/PostingsFormatExtension.java b/server/src/main/java/org/elasticsearch/internal/PostingsFormatExtension.java new file mode 100644 index 0000000000000..9ae8f602b94b9 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/internal/PostingsFormatExtension.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.internal; + +import org.apache.lucene.search.suggest.document.CompletionPostingsFormat; + +/** + * Allows plugging-in the Postings Format. + */ +public interface PostingsFormatExtension { + + /** + * Returns the name of the {@link CompletionPostingsFormat} that Elasticsearch should use. + */ + String getCompletionPostingsFormatName(); +} From 3a317a8dedc71f0995b12704db8c11b82816c483 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Wed, 31 Jul 2024 11:48:23 -0700 Subject: [PATCH 02/15] Spotless --- server/src/main/java/org/elasticsearch/env/BuildVersion.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/env/BuildVersion.java b/server/src/main/java/org/elasticsearch/env/BuildVersion.java index d6b9ad8f44f14..dfd2168d14bb6 100644 --- a/server/src/main/java/org/elasticsearch/env/BuildVersion.java +++ b/server/src/main/java/org/elasticsearch/env/BuildVersion.java @@ -103,7 +103,6 @@ private static BuildExtension findExtension() { } - private static class DefaultBuildExtension implements BuildExtension { @Override public Build getCurrentBuild() { From f62a7a70f3076dc5fbb7c86f34356ef654b3329a Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Wed, 31 Jul 2024 12:44:54 -0700 Subject: [PATCH 03/15] Spotless --- server/src/main/java/org/elasticsearch/env/BuildVersion.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/env/BuildVersion.java b/server/src/main/java/org/elasticsearch/env/BuildVersion.java index dfd2168d14bb6..e19ad87932e7f 100644 --- a/server/src/main/java/org/elasticsearch/env/BuildVersion.java +++ b/server/src/main/java/org/elasticsearch/env/BuildVersion.java @@ -100,7 +100,6 @@ private static class CurrentExtensionHolder { private static BuildExtension findExtension() { return ExtensionLoader.loadSingleton(ServiceLoader.load(BuildExtension.class)).orElse(new DefaultBuildExtension()); } - } private static class DefaultBuildExtension implements BuildExtension { From c664739f64f758ae3ad9ca83f0db4df1044da3e0 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Wed, 31 Jul 2024 15:13:46 -0700 Subject: [PATCH 04/15] Update docs/changelog/111494.yaml --- docs/changelog/111494.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/111494.yaml diff --git a/docs/changelog/111494.yaml b/docs/changelog/111494.yaml new file mode 100644 index 0000000000000..6aba41f93dc17 --- /dev/null +++ b/docs/changelog/111494.yaml @@ -0,0 +1,5 @@ +pr: 111494 +summary: Extensible Completion Postings Formats +area: "Search, Mapping" +type: enhancement +issues: [] From 4343eea3210e0be45d7d8fa9f36622ac699bbda0 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Fri, 2 Aug 2024 09:50:53 -0700 Subject: [PATCH 05/15] Fix docs --- docs/changelog/111494.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/111494.yaml b/docs/changelog/111494.yaml index 6aba41f93dc17..6c7b84bb04798 100644 --- a/docs/changelog/111494.yaml +++ b/docs/changelog/111494.yaml @@ -1,5 +1,5 @@ pr: 111494 summary: Extensible Completion Postings Formats -area: "Search, Mapping" +area: "Suggesters" type: enhancement issues: [] From a088ae935792f62b2f6fcbc923a0f721009d900a Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Wed, 31 Jul 2024 11:44:06 -0700 Subject: [PATCH 06/15] Make completion postings format extensible --- server/src/main/java/org/elasticsearch/env/BuildVersion.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/env/BuildVersion.java b/server/src/main/java/org/elasticsearch/env/BuildVersion.java index e19ad87932e7f..d6b9ad8f44f14 100644 --- a/server/src/main/java/org/elasticsearch/env/BuildVersion.java +++ b/server/src/main/java/org/elasticsearch/env/BuildVersion.java @@ -100,8 +100,10 @@ private static class CurrentExtensionHolder { private static BuildExtension findExtension() { return ExtensionLoader.loadSingleton(ServiceLoader.load(BuildExtension.class)).orElse(new DefaultBuildExtension()); } + } + private static class DefaultBuildExtension implements BuildExtension { @Override public Build getCurrentBuild() { From 322d1768ed69060d096fed2de73b1e751fdfe9ca Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Wed, 31 Jul 2024 11:48:23 -0700 Subject: [PATCH 07/15] Spotless --- server/src/main/java/org/elasticsearch/env/BuildVersion.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/env/BuildVersion.java b/server/src/main/java/org/elasticsearch/env/BuildVersion.java index d6b9ad8f44f14..dfd2168d14bb6 100644 --- a/server/src/main/java/org/elasticsearch/env/BuildVersion.java +++ b/server/src/main/java/org/elasticsearch/env/BuildVersion.java @@ -103,7 +103,6 @@ private static BuildExtension findExtension() { } - private static class DefaultBuildExtension implements BuildExtension { @Override public Build getCurrentBuild() { From a88ead09477afb3fc7131f759d2ccaf9f292bdc0 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Wed, 31 Jul 2024 12:44:54 -0700 Subject: [PATCH 08/15] Spotless --- server/src/main/java/org/elasticsearch/env/BuildVersion.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/env/BuildVersion.java b/server/src/main/java/org/elasticsearch/env/BuildVersion.java index dfd2168d14bb6..e19ad87932e7f 100644 --- a/server/src/main/java/org/elasticsearch/env/BuildVersion.java +++ b/server/src/main/java/org/elasticsearch/env/BuildVersion.java @@ -100,7 +100,6 @@ private static class CurrentExtensionHolder { private static BuildExtension findExtension() { return ExtensionLoader.loadSingleton(ServiceLoader.load(BuildExtension.class)).orElse(new DefaultBuildExtension()); } - } private static class DefaultBuildExtension implements BuildExtension { From 4a80fe3802b426493c7bdb74bec6b749bebdcd24 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Tue, 27 Aug 2024 11:09:55 -0700 Subject: [PATCH 09/15] Allow PostingsFormatExtension to be configured by serverless --- server/src/main/java/module-info.java | 6 +++--- .../org/elasticsearch/internal/PostingsFormatExtension.java | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 6b1cf826dcbdc..fcfcb64409b28 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -8,7 +8,6 @@ import org.elasticsearch.index.codec.Elasticsearch814Codec; import org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat; -import org.elasticsearch.internal.PostingsFormatExtension; import org.elasticsearch.plugins.internal.RestExtension; /** The Elasticsearch Server Module. */ @@ -291,7 +290,8 @@ org.elasticsearch.serverless.version, org.elasticsearch.serverless.buildinfo, org.elasticsearch.serverless.constants, - org.elasticsearch.serverless.codec; + org.elasticsearch.serverless.codec, + org.elasticsearch.stateless; exports org.elasticsearch.lucene.analysis.miscellaneous; exports org.elasticsearch.lucene.grouping; exports org.elasticsearch.lucene.queries; @@ -419,7 +419,7 @@ uses org.elasticsearch.node.internal.TerminationHandlerProvider; uses org.elasticsearch.internal.VersionExtension; uses org.elasticsearch.internal.BuildExtension; - uses PostingsFormatExtension; + uses org.elasticsearch.internal.PostingsFormatExtension; uses org.elasticsearch.features.FeatureSpecification; uses org.elasticsearch.plugins.internal.LoggingDataProvider; uses org.elasticsearch.cluster.metadata.DataStreamFactoryRetention; diff --git a/server/src/main/java/org/elasticsearch/internal/PostingsFormatExtension.java b/server/src/main/java/org/elasticsearch/internal/PostingsFormatExtension.java index 9ae8f602b94b9..b16f76597dc1f 100644 --- a/server/src/main/java/org/elasticsearch/internal/PostingsFormatExtension.java +++ b/server/src/main/java/org/elasticsearch/internal/PostingsFormatExtension.java @@ -19,4 +19,6 @@ public interface PostingsFormatExtension { * Returns the name of the {@link CompletionPostingsFormat} that Elasticsearch should use. */ String getCompletionPostingsFormatName(); + + void setFeatureEnabled(boolean isFeatureEnabled); } From f243893baba727f5e58fbbb0d9d2f6272361b869 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Fri, 6 Sep 2024 15:12:24 -0700 Subject: [PATCH 10/15] Simplify names and logic --- server/src/main/java/module-info.java | 3 ++- .../index/mapper/CompletionFieldMapper.java | 6 +++--- ...sion.java => CompletionsPostingsFormatExtension.java} | 9 ++++++--- 3 files changed, 11 insertions(+), 7 deletions(-) rename server/src/main/java/org/elasticsearch/internal/{PostingsFormatExtension.java => CompletionsPostingsFormatExtension.java} (69%) diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 1befa25bb4f99..859e8891aa997 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -8,6 +8,7 @@ import org.elasticsearch.index.codec.Elasticsearch814Codec; import org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat; +import org.elasticsearch.internal.CompletionsPostingsFormatExtension; import org.elasticsearch.plugins.internal.RestExtension; /** The Elasticsearch Server Module. */ @@ -420,7 +421,7 @@ uses org.elasticsearch.node.internal.TerminationHandlerProvider; uses org.elasticsearch.internal.VersionExtension; uses org.elasticsearch.internal.BuildExtension; - uses org.elasticsearch.internal.PostingsFormatExtension; + uses CompletionsPostingsFormatExtension; uses org.elasticsearch.features.FeatureSpecification; uses org.elasticsearch.plugins.internal.LoggingDataProvider; uses org.elasticsearch.cluster.metadata.DataStreamFactoryRetention; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index 7ddc985906698..32e87ecaf1549 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -27,7 +27,7 @@ import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.internal.PostingsFormatExtension; +import org.elasticsearch.internal.CompletionsPostingsFormatExtension; import org.elasticsearch.plugins.ExtensionLoader; import org.elasticsearch.search.suggest.completion.CompletionSuggester; import org.elasticsearch.search.suggest.completion.context.ContextMapping; @@ -381,8 +381,8 @@ private static class PostingsFormatHolder { private static PostingsFormat getPostingsFormat() { String defaultName = "Completion99"; - String codecName = ExtensionLoader.loadSingleton(ServiceLoader.load(PostingsFormatExtension.class)) - .map(PostingsFormatExtension::getCompletionPostingsFormatName) + String codecName = ExtensionLoader.loadSingleton(ServiceLoader.load(CompletionsPostingsFormatExtension.class)) + .map(CompletionsPostingsFormatExtension::getFormatName) .orElse(defaultName); return PostingsFormat.forName(codecName); } diff --git a/server/src/main/java/org/elasticsearch/internal/PostingsFormatExtension.java b/server/src/main/java/org/elasticsearch/internal/CompletionsPostingsFormatExtension.java similarity index 69% rename from server/src/main/java/org/elasticsearch/internal/PostingsFormatExtension.java rename to server/src/main/java/org/elasticsearch/internal/CompletionsPostingsFormatExtension.java index b16f76597dc1f..ec82faed20e06 100644 --- a/server/src/main/java/org/elasticsearch/internal/PostingsFormatExtension.java +++ b/server/src/main/java/org/elasticsearch/internal/CompletionsPostingsFormatExtension.java @@ -13,12 +13,15 @@ /** * Allows plugging-in the Postings Format. */ -public interface PostingsFormatExtension { +public interface CompletionsPostingsFormatExtension { /** * Returns the name of the {@link CompletionPostingsFormat} that Elasticsearch should use. */ - String getCompletionPostingsFormatName(); + String getFormatName(); - void setFeatureEnabled(boolean isFeatureEnabled); + /** + * Sets whether this extension is enabled. If the extension is not enabled, {@link #getFormatName()} should return null. + */ + void setExtensionEnabled(boolean isExtensionEnabled); } From 40fb72f708cc0b196daae385c39c78bd4b330926 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Thu, 10 Oct 2024 13:31:59 -0700 Subject: [PATCH 11/15] Update license header --- .../internal/CompletionsPostingsFormatExtension.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/internal/CompletionsPostingsFormatExtension.java b/server/src/main/java/org/elasticsearch/internal/CompletionsPostingsFormatExtension.java index ec82faed20e06..4048f649ba629 100644 --- a/server/src/main/java/org/elasticsearch/internal/CompletionsPostingsFormatExtension.java +++ b/server/src/main/java/org/elasticsearch/internal/CompletionsPostingsFormatExtension.java @@ -1,9 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ package org.elasticsearch.internal; From c91c23dfd7993e4846bd6870f0fd387834be983f Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Fri, 11 Oct 2024 15:02:12 -0700 Subject: [PATCH 12/15] Add javadoc --- .../internal/CompletionsPostingsFormatExtension.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/internal/CompletionsPostingsFormatExtension.java b/server/src/main/java/org/elasticsearch/internal/CompletionsPostingsFormatExtension.java index 4048f649ba629..8c3faf9aaacb0 100644 --- a/server/src/main/java/org/elasticsearch/internal/CompletionsPostingsFormatExtension.java +++ b/server/src/main/java/org/elasticsearch/internal/CompletionsPostingsFormatExtension.java @@ -17,7 +17,8 @@ public interface CompletionsPostingsFormatExtension { /** - * Returns the name of the {@link CompletionPostingsFormat} that Elasticsearch should use. + * Returns the name of the {@link CompletionPostingsFormat} that Elasticsearch should use. Should return null if the extension + * is not enabled. */ String getFormatName(); From ade1546f8364627ad09c41a9c5457ea1666bb38d Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Tue, 15 Oct 2024 13:30:33 -0700 Subject: [PATCH 13/15] Improve javadoc --- .../internal/CompletionsPostingsFormatExtension.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/internal/CompletionsPostingsFormatExtension.java b/server/src/main/java/org/elasticsearch/internal/CompletionsPostingsFormatExtension.java index 8c3faf9aaacb0..b3f0203c903be 100644 --- a/server/src/main/java/org/elasticsearch/internal/CompletionsPostingsFormatExtension.java +++ b/server/src/main/java/org/elasticsearch/internal/CompletionsPostingsFormatExtension.java @@ -12,7 +12,7 @@ import org.apache.lucene.search.suggest.document.CompletionPostingsFormat; /** - * Allows plugging-in the Postings Format. + * Allows plugging-in the Completions Postings Format. */ public interface CompletionsPostingsFormatExtension { @@ -24,6 +24,8 @@ public interface CompletionsPostingsFormatExtension { /** * Sets whether this extension is enabled. If the extension is not enabled, {@link #getFormatName()} should return null. + *

+ * This allows all nodes to be upgraded to a version that supports the extension before it is enabled. */ void setExtensionEnabled(boolean isExtensionEnabled); } From 35f66e8a297e10e68514d6c71c57bd25a00d7f29 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Wed, 30 Oct 2024 10:53:19 -0700 Subject: [PATCH 14/15] PR Feedback: remove feature --- .../internal/CompletionsPostingsFormatExtension.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/internal/CompletionsPostingsFormatExtension.java b/server/src/main/java/org/elasticsearch/internal/CompletionsPostingsFormatExtension.java index b3f0203c903be..bb28d4dd6c901 100644 --- a/server/src/main/java/org/elasticsearch/internal/CompletionsPostingsFormatExtension.java +++ b/server/src/main/java/org/elasticsearch/internal/CompletionsPostingsFormatExtension.java @@ -19,13 +19,10 @@ public interface CompletionsPostingsFormatExtension { /** * Returns the name of the {@link CompletionPostingsFormat} that Elasticsearch should use. Should return null if the extension * is not enabled. - */ - String getFormatName(); - - /** - * Sets whether this extension is enabled. If the extension is not enabled, {@link #getFormatName()} should return null. *

- * This allows all nodes to be upgraded to a version that supports the extension before it is enabled. + * Note that the name must match a codec that is available on all nodes in the cluster, otherwise IndexCorruptionExceptions will occur. + * A feature can be used to protect against this scenario, or alternatively, the codec code can be rolled out prior to its usage by this + * extension. */ - void setExtensionEnabled(boolean isExtensionEnabled); + String getFormatName(); } From 70f2143560a77ef83e80381724ae632468c09c0e Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Thu, 31 Oct 2024 13:37:04 -0700 Subject: [PATCH 15/15] PR Feedback: remove export and move format --- server/src/main/java/module-info.java | 3 +-- .../index/codec/PerFieldFormatSupplier.java | 19 +++++++++++++++++- .../index/mapper/CompletionFieldMapper.java | 20 ------------------- 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 0c40add9ee83f..c04c7ef077f20 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -289,8 +289,7 @@ org.elasticsearch.serverless.version, org.elasticsearch.serverless.buildinfo, org.elasticsearch.serverless.constants, - org.elasticsearch.serverless.codec, - org.elasticsearch.stateless; + org.elasticsearch.serverless.codec; exports org.elasticsearch.lucene.analysis.miscellaneous; exports org.elasticsearch.lucene.grouping; exports org.elasticsearch.lucene.queries; diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java index 82079fe566467..4d3d37ab4f3af 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java +++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java @@ -25,6 +25,10 @@ import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.elasticsearch.internal.CompletionsPostingsFormatExtension; +import org.elasticsearch.plugins.ExtensionLoader; + +import java.util.ServiceLoader; /** * Class that encapsulates the logic of figuring out the most appropriate file format for a given field, across postings, doc values and @@ -56,13 +60,26 @@ private PostingsFormat internalGetPostingsFormatForField(String field) { if (mapperService != null) { Mapper mapper = mapperService.mappingLookup().getMapper(field); if (mapper instanceof CompletionFieldMapper) { - return CompletionFieldMapper.postingsFormat(); + return PostingsFormatHolder.POSTINGS_FORMAT; } } // return our own posting format using PFOR return es812PostingsFormat; } + private static class PostingsFormatHolder { + private static final PostingsFormat POSTINGS_FORMAT = getPostingsFormat(); + + private static PostingsFormat getPostingsFormat() { + String defaultName = "Completion912"; // Caution: changing this name will result in exceptions if a field is created during a + // rolling upgrade and the new codec (specified by the name) is not available on all nodes in the cluster. + String codecName = ExtensionLoader.loadSingleton(ServiceLoader.load(CompletionsPostingsFormatExtension.class)) + .map(CompletionsPostingsFormatExtension::getFormatName) + .orElse(defaultName); + return PostingsFormat.forName(codecName); + } + } + boolean useBloomFilter(String field) { if (mapperService == null) { return false; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index e9851ff5fb013..bb229c795a83e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -8,7 +8,6 @@ */ package org.elasticsearch.index.mapper; -import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; @@ -26,8 +25,6 @@ import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.internal.CompletionsPostingsFormatExtension; -import org.elasticsearch.plugins.ExtensionLoader; import org.elasticsearch.search.suggest.completion.CompletionSuggester; import org.elasticsearch.search.suggest.completion.context.ContextMapping; import org.elasticsearch.search.suggest.completion.context.ContextMappings; @@ -50,7 +47,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.ServiceLoader; import java.util.Set; /** @@ -347,22 +343,6 @@ public CompletionFieldType fieldType() { return (CompletionFieldType) super.fieldType(); } - public static PostingsFormat postingsFormat() { - return PostingsFormatHolder.POSTINGS_FORMAT; - } - - private static class PostingsFormatHolder { - private static final PostingsFormat POSTINGS_FORMAT = getPostingsFormat(); - - private static PostingsFormat getPostingsFormat() { - String defaultName = "Completion912"; - String codecName = ExtensionLoader.loadSingleton(ServiceLoader.load(CompletionsPostingsFormatExtension.class)) - .map(CompletionsPostingsFormatExtension::getFormatName) - .orElse(defaultName); - return PostingsFormat.forName(codecName); - } - } - @Override public boolean parsesArrayValue() { return true;