Skip to content

Don't require DocumentMapper as an argument when parsing a document #66780

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Jan 4, 2021
Merged

Don't require DocumentMapper as an argument when parsing a document #66780

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5e8ca79
Don't require DocumentMapper as an argument when parsing a document
javanna Dec 23, 2020
e7fe9a7
iter
javanna Dec 23, 2020
ffe6637
Merge branch 'master' into refactoring/untangle_doc_parser_doc_mapper
javanna Jan 4, 2021
42f0b09
Restore imports order
javanna Jan 4, 2021
bc2acf4
Merge branch 'master' into refactoring/untangle_doc_parser_doc_mapper
javanna Jan 4, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1113,7 +1113,8 @@ private void duelRun(PercolateQuery.QueryStore queryStore, MemoryIndex memoryInd
}

private void addQuery(Query query, List<ParseContext.Document> docs) {
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper.mapping(),
documentMapper.mappers(), mapperService.getIndexSettings(), mapperService.getIndexAnalyzers(), null, null, null, null);
fieldMapper.processQuery(query, parseContext);
ParseContext.Document queryDocument = parseContext.doc();
// Add to string representation of the query to make debugging easier:
Expand Down
18 changes: 12 additions & 6 deletions ...les/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,8 @@ public void testExtractTerms() throws Exception {

DocumentMapper documentMapper = mapperService.documentMapper();
PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName);
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper.mapping(),
documentMapper.mappers(), mapperService.getIndexSettings(), mapperService.getIndexAnalyzers(), null, null, null, null);
fieldMapper.processQuery(bq.build(), parseContext);
ParseContext.Document document = parseContext.doc();

Expand All @@ -202,7 +203,8 @@ public void testExtractTerms() throws Exception {
bq.add(termQuery1, Occur.MUST);
bq.add(termQuery2, Occur.MUST);

parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
parseContext = new ParseContext.InternalParseContext(documentMapper.mapping(),
documentMapper.mappers(), mapperService.getIndexSettings(), mapperService.getIndexAnalyzers(), null, null, null, null);
fieldMapper.processQuery(bq.build(), parseContext);
document = parseContext.doc();

Expand Down Expand Up @@ -231,7 +233,8 @@ public void testExtractRanges() throws Exception {

DocumentMapper documentMapper = mapperService.documentMapper();
PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName);
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper.mapping(),
documentMapper.mappers(), mapperService.getIndexSettings(), mapperService.getIndexAnalyzers(), null, null, null, null);
fieldMapper.processQuery(bq.build(), parseContext);
ParseContext.Document document = parseContext.doc();

Expand All @@ -256,7 +259,8 @@ public void testExtractRanges() throws Exception {
.rangeQuery(15, 20, true, true, null, null, null, context);
bq.add(rangeQuery2, Occur.MUST);

parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
parseContext = new ParseContext.InternalParseContext(documentMapper.mapping(),
documentMapper.mappers(), mapperService.getIndexSettings(), mapperService.getIndexAnalyzers(), null, null, null, null);
fieldMapper.processQuery(bq.build(), parseContext);
document = parseContext.doc();

Expand All @@ -279,7 +283,8 @@ public void testExtractTermsAndRanges_failed() throws Exception {
TermRangeQuery query = new TermRangeQuery("field1", new BytesRef("a"), new BytesRef("z"), true, true);
DocumentMapper documentMapper = mapperService.documentMapper();
PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName);
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper.mapping(),
documentMapper.mappers(), mapperService.getIndexSettings(), mapperService.getIndexAnalyzers(), null, null, null, null);
fieldMapper.processQuery(query, parseContext);
ParseContext.Document document = parseContext.doc();

Expand All @@ -293,7 +298,8 @@ public void testExtractTermsAndRanges_partial() throws Exception {
PhraseQuery phraseQuery = new PhraseQuery("field", "term");
DocumentMapper documentMapper = mapperService.documentMapper();
PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName);
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper.mapping(),
documentMapper.mappers(), mapperService.getIndexSettings(), mapperService.getIndexAnalyzers(), null, null, null, null);
fieldMapper.processQuery(phraseQuery, parseContext);
ParseContext.Document document = parseContext.doc();

Expand Down
17 changes: 5 additions & 12 deletions server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@
import java.util.Objects;
import java.util.stream.Stream;


public class DocumentMapper implements ToXContentFragment {

public static final class Builder {
Expand Down Expand Up @@ -133,14 +132,6 @@ private DocumentMapper(IndexSettings indexSettings,
.filter(field -> noopTombstoneMetadataFields.contains(field.name())).toArray(MetadataFieldMapper[]::new);
}

IndexSettings indexSettings() {
return indexSettings;
}

IndexAnalyzers indexAnalyzers() {
return indexAnalyzers;
}

public Mapping mapping() {
return mapping;
}
Expand Down Expand Up @@ -194,18 +185,20 @@ public MappingLookup mappers() {
}

public ParsedDocument parse(SourceToParse source) throws MapperParsingException {
return documentParser.parseDocument(source, mapping.metadataMappers, this);
return documentParser.parseDocument(source, mapping, fieldMappers, indexSettings, indexAnalyzers);
}

public ParsedDocument createDeleteTombstoneDoc(String index, String id) throws MapperParsingException {
final SourceToParse emptySource = new SourceToParse(index, id, new BytesArray("{}"), XContentType.JSON);
return documentParser.parseDocument(emptySource, deleteTombstoneMetadataFieldMappers, this).toTombstone();
return documentParser.parseDocument(emptySource, mapping, deleteTombstoneMetadataFieldMappers, fieldMappers,
indexSettings, indexAnalyzers).toTombstone();
}

public ParsedDocument createNoopTombstoneDoc(String index, String reason) throws MapperParsingException {
final String id = ""; // _id won't be used.
final SourceToParse sourceToParse = new SourceToParse(index, id, new BytesArray("{}"), XContentType.JSON);
final ParsedDocument parsedDoc = documentParser.parseDocument(sourceToParse, noopTombstoneMetadataFieldMappers, this).toTombstone();
final ParsedDocument parsedDoc = documentParser.parseDocument(sourceToParse, mapping, noopTombstoneMetadataFieldMappers,
fieldMappers, indexSettings, indexAnalyzers).toTombstone();
// Store the reason of a noop as a raw string in the _source field
final BytesRef byteRef = new BytesRef(reason);
parsedDoc.rootDoc().add(new StoredField(SourceFieldMapper.NAME, byteRef.bytes, byteRef.offset, byteRef.length));
Expand Down
43 changes: 30 additions & 13 deletions server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.query.QueryShardContext;

/** A parser for documents, given mappings from a DocumentMapper */
Expand All @@ -58,18 +60,33 @@ final class DocumentParser {
}

ParsedDocument parseDocument(SourceToParse source,
MetadataFieldMapper[] metadataFieldsMappers,
DocumentMapper docMapper) throws MapperParsingException {
Mapping mapping,
MappingLookup mappingLookup,
IndexSettings indexSettings,
IndexAnalyzers indexAnalyzers) throws MapperParsingException {
return parseDocument(source, mapping, mapping.metadataMappers, mappingLookup, indexSettings, indexAnalyzers);
}

final Mapping mapping = docMapper.mapping();
ParsedDocument parseDocument(SourceToParse source,
Mapping mapping,
MetadataFieldMapper[] metadataFieldsMappers,
MappingLookup mappingLookup,
IndexSettings indexSettings,
IndexAnalyzers indexAnalyzers) throws MapperParsingException {
final ParseContext.InternalParseContext context;
final XContentType xContentType = source.getXContentType();

try (XContentParser parser = XContentHelper.createParser(xContentRegistry,
LoggingDeprecationHandler.INSTANCE, source.source(), xContentType)) {
context = new ParseContext.InternalParseContext(docMapper, dateParserContext, dynamicRuntimeFieldsBuilder, source, parser);
context = new ParseContext.InternalParseContext(mapping,
mappingLookup,
indexSettings,
indexAnalyzers,
dateParserContext,
dynamicRuntimeFieldsBuilder,
source,
parser);
validateStart(parser);
internalParseDocument(mapping, metadataFieldsMappers, context, parser);
internalParseDocument(mapping.root(), metadataFieldsMappers, context, parser);
validateEnd(parser);
} catch (Exception e) {
throw wrapInMapperParsingException(source, e);
Expand All @@ -81,7 +98,7 @@ ParsedDocument parseDocument(SourceToParse source,

context.postParse();

return parsedDocument(source, context, createDynamicUpdate(mapping, docMapper.mappers(),
return parsedDocument(source, context, createDynamicUpdate(mapping, mappingLookup,
context.getDynamicMappers(), context.getDynamicRuntimeFields()));
}

Expand All @@ -99,19 +116,19 @@ private static boolean containsDisabledObjectMapper(ObjectMapper objectMapper, S
return false;
}

private static void internalParseDocument(Mapping mapping, MetadataFieldMapper[] metadataFieldsMappers,
private static void internalParseDocument(RootObjectMapper root, MetadataFieldMapper[] metadataFieldsMappers,
ParseContext context, XContentParser parser) throws IOException {
final boolean emptyDoc = isEmptyDoc(mapping, parser);
final boolean emptyDoc = isEmptyDoc(root, parser);

for (MetadataFieldMapper metadataMapper : metadataFieldsMappers) {
metadataMapper.preParse(context);
}

if (mapping.root.isEnabled() == false) {
if (root.isEnabled() == false) {
// entire type is disabled
parser.skipChildren();
} else if (emptyDoc == false) {
parseObjectOrNested(context, mapping.root);
parseObjectOrNested(context, root);
}

for (MetadataFieldMapper metadataMapper : metadataFieldsMappers) {
Expand All @@ -137,8 +154,8 @@ private static void validateEnd(XContentParser parser) throws IOException {
}
}

private static boolean isEmptyDoc(Mapping mapping, XContentParser parser) throws IOException {
if (mapping.root.isEnabled()) {
private static boolean isEmptyDoc(RootObjectMapper root, XContentParser parser) throws IOException {
if (root.isEnabled()) {
final XContentParser.Token token = parser.nextToken();
if (token == XContentParser.Token.END_OBJECT) {
// empty doc, we can handle it...
Expand Down
15 changes: 9 additions & 6 deletions server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -336,23 +336,26 @@ public static class InternalParseContext extends ParseContext {
private long numNestedDocs;
private boolean docsReversed = false;

public InternalParseContext(DocumentMapper docMapper,
public InternalParseContext(Mapping mapping,
MappingLookup mappingLookup,
IndexSettings indexSettings,
IndexAnalyzers indexAnalyzers,
Function<DateFormatter, Mapper.TypeParser.ParserContext> parserContextFunction,
DynamicRuntimeFieldsBuilder dynamicRuntimeFieldsBuilder,
SourceToParse source,
XContentParser parser) {
this.mapping = docMapper.mapping();
this.mappingLookup = docMapper.mappers();
this.indexSettings = docMapper.indexSettings();
this.indexAnalyzers = docMapper.indexAnalyzers();
this.mapping = mapping;
this.mappingLookup = mappingLookup;
this.indexSettings = indexSettings;
this.indexAnalyzers = indexAnalyzers;
this.parserContextFunction = parserContextFunction;
this.dynamicRuntimeFieldsBuilder = dynamicRuntimeFieldsBuilder;
this.parser = parser;
this.document = new Document();
this.documents.add(document);
this.version = null;
this.sourceToParse = source;
this.maxAllowedNumNestedDocs = docMapper.indexSettings().getMappingNestedDocsLimit();
this.maxAllowedNumNestedDocs = indexSettings.getMappingNestedDocsLimit();
this.numNestedDocs = 0L;
}

Expand Down
4 changes: 3 additions & 1 deletion server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -617,7 +617,9 @@ MapperService createMapperService() throws Exception {

// creates an object mapper, which is about 100x harder than it should be....
ObjectMapper createObjectMapper(MapperService mapperService, String name) {
ParseContext context = new ParseContext.InternalParseContext(mapperService.documentMapper(), null, null, null, null);
DocumentMapper docMapper = mapperService.documentMapper();
ParseContext context = new ParseContext.InternalParseContext(docMapper.mapping(), docMapper.mappers(),
mapperService.getIndexSettings(), mapperService.getIndexAnalyzers(), null, null, null, null);
String[] nameParts = name.split("\\.");
for (int i = 0; i < nameParts.length - 1; ++i) {
context.path().add(nameParts[i]);
Expand Down