From 50c1746108a03d1a7038ecc803261420c93a7e08 Mon Sep 17 00:00:00 2001 From: iverase Date: Wed, 7 Oct 2020 17:53:18 +0200 Subject: [PATCH 01/18] TwoPhaseDateRangeQuery --- .../document/TwoPhaseDatePointMillis.java | 59 +++ .../document/TwoPhaseDatePointNanos.java | 61 +++ .../queries/TwoPhaseDateRangeQuery.java | 452 ++++++++++++++++++ .../index/mapper/DateFieldMapper.java | 126 ++++- .../lucene/queries/TwoPhaseDateTests.java | 99 ++++ 5 files changed, 785 insertions(+), 12 deletions(-) create mode 100644 server/src/main/java/org/apache/lucene/document/TwoPhaseDatePointMillis.java create mode 100644 server/src/main/java/org/apache/lucene/document/TwoPhaseDatePointNanos.java create mode 100644 server/src/main/java/org/apache/lucene/queries/TwoPhaseDateRangeQuery.java create mode 100644 server/src/test/java/org/apache/lucene/queries/TwoPhaseDateTests.java diff --git a/server/src/main/java/org/apache/lucene/document/TwoPhaseDatePointMillis.java b/server/src/main/java/org/apache/lucene/document/TwoPhaseDatePointMillis.java new file mode 100644 index 0000000000000..310e90283659a --- /dev/null +++ b/server/src/main/java/org/apache/lucene/document/TwoPhaseDatePointMillis.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.document; + +import org.apache.lucene.queries.TwoPhaseDateRangeQuery; +import org.apache.lucene.search.Query; + +import java.time.Instant; + + +public final class TwoPhaseDatePointMillis { + + + public static Field[] createIndexableFields(String name, Instant... instants) { + Field[] fields = new Field[2*instants.length]; + for (int i = 0; i < instants.length; i++) { + fields[2*i] = new LongPoint(name, instants[i].getEpochSecond()); + fields[2*i+1] = new SortedNumericDocValuesField(name, instants[i].toEpochMilli()); + } + return fields; + } + + // static methods for generating queries + + + public static Query newExactQuery(String field, Instant instant) { + return newRangeQuery(field, instant, instant); + } + + + public static Query newRangeQuery(String field, Instant lowerValue, Instant upperValue) { + return new TwoPhaseDateRangeQuery(field, lowerValue, upperValue) { + + @Override + protected long toApproxPrecision(Instant instant) { + return instant.getEpochSecond(); + } + + @Override + protected long toExactPrecision(Instant instant) { + return instant.toEpochMilli(); + } + }; + } +} diff --git a/server/src/main/java/org/apache/lucene/document/TwoPhaseDatePointNanos.java b/server/src/main/java/org/apache/lucene/document/TwoPhaseDatePointNanos.java new file mode 100644 index 0000000000000..c4c4a76cc840c --- /dev/null +++ b/server/src/main/java/org/apache/lucene/document/TwoPhaseDatePointNanos.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.document; + +import org.apache.lucene.queries.TwoPhaseDateRangeQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.common.time.DateUtils; + +import java.time.Instant; + + +public final class TwoPhaseDatePointNanos { + + + public static Field[] createIndexableFields(String name, Instant... instants) { + Field[] fields = new Field[2*instants.length]; + for (int i = 0; i < instants.length; i++) { + fields[2*i] = new LongPoint(name, instants[i].getEpochSecond()); + fields[2*i+1] = new SortedNumericDocValuesField(name, DateUtils.toLong(instants[i])); + } + return fields; + } + + + // static methods for generating queries + + public static Query newExactQuery(String field, Instant instant) { + return newRangeQuery(field, instant, instant); + } + + + public static Query newRangeQuery(String field, Instant lowerValue, Instant upperValue) { + return new TwoPhaseDateRangeQuery(field, lowerValue, upperValue) { + + @Override + protected long toApproxPrecision(Instant instant) { + return instant.getEpochSecond(); + } + + @Override + protected long toExactPrecision(Instant instant) { + return DateUtils.toLong(instant); + } + }; + } + +} diff --git a/server/src/main/java/org/apache/lucene/queries/TwoPhaseDateRangeQuery.java b/server/src/main/java/org/apache/lucene/queries/TwoPhaseDateRangeQuery.java new file mode 100644 index 0000000000000..719954d1b05f7 --- /dev/null +++ b/server/src/main/java/org/apache/lucene/queries/TwoPhaseDateRangeQuery.java @@ -0,0 +1,452 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.queries; + +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.PointValues; +import org.apache.lucene.index.PointValues.IntersectVisitor; +import org.apache.lucene.index.PointValues.Relation; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.BitSetIterator; +import org.apache.lucene.util.DocIdSetBuilder; +import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.common.time.DateUtils; + +import java.io.IOException; +import java.time.Instant; +import java.util.Arrays; +import java.util.Objects; + +/** + * Abstract class for range queries against single or multidimensional points such as + * {@link IntPoint}. + *

+ * This is for subclasses and works on the underlying binary encoding: to + * create range queries for lucene's standard {@code Point} types, refer to factory + * methods on those classes, e.g. {@link IntPoint#newRangeQuery IntPoint.newRangeQuery()} for + * fields indexed with {@link IntPoint}. + *

+ * For a single-dimensional field this query is a simple range query; in a multi-dimensional field it's a box shape. + * @see PointValues + * @lucene.experimental + */ +public abstract class TwoPhaseDateRangeQuery extends Query { + final String field; + final Instant lower; + final Instant upper; + + /** + * Expert: create a multidimensional range query for point values. + * + * @param field field name. must not be {@code null}. + * @param lower lower portion of the range (inclusive). + * @param upper upper portion of the range (inclusive). + * @throws IllegalArgumentException if {@code field} is null, or if {@code lowerValue.length != upperValue.length} + */ + public TwoPhaseDateRangeQuery(String field, Instant lower, Instant upper) { + this.field = field; + this.lower = lower; + this.upper = upper; + } + + @Override + public void visit(QueryVisitor visitor) { + if (visitor.acceptField(field)) { + visitor.visitLeaf(this); + } + } + + protected abstract long toApproxPrecision(Instant instant); + + protected abstract long toExactPrecision(Instant instant); + + @Override + public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + + // We don't use RandomAccessWeight here: it's no good to approximate with "match all docs". + // This is an inverted structure and should be used in the first pass: + long l = toApproxPrecision(lower); + long u = toApproxPrecision(upper); + byte[] lowerInc = new byte[Long.BYTES]; + NumericUtils.longToSortableBytes(l + 1, lowerInc, 0); + byte[] lowerExc = new byte[Long.BYTES]; + NumericUtils.longToSortableBytes(l, lowerExc, 0); + byte[] upperInc = new byte[Long.BYTES]; + NumericUtils.longToSortableBytes(u - 1, upperInc, 0); + byte[] upperExc = new byte[Long.BYTES]; + NumericUtils.longToSortableBytes(u, upperExc, 0); + + final ApproximatedQuery phase1 = new ApproximatedQuery(lowerInc, lowerExc, upperInc, upperExc); + + return new ConstantScoreWeight(this, boost) { + + + @Override + public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException { + LeafReader reader = context.reader(); + + PointValues values = reader.getPointValues(field); + if (values == null) { + // No docs in this segment/field indexed any points + return null; + } + + if (values.getNumIndexDimensions() != 1) { + throw new IllegalArgumentException("field=\"" + field + "\" was indexed with numIndexDimensions=" + values.getNumIndexDimensions() + " but this query has numDims=" + 1); + } + if (Long.BYTES != values.getBytesPerDimension()) { + throw new IllegalArgumentException("field=\"" + field + "\" was indexed with bytesPerDim=" + values.getBytesPerDimension() + " but this query has bytesPerDim=" + Long.BYTES); + } + final Weight weight = this; + if (values.getDocCount() == reader.maxDoc()) { + final byte[] fieldPackedLower = values.getMinPackedValue(); + final byte[] fieldPackedUpper = values.getMaxPackedValue(); + // equals?? + if (Arrays.compareUnsigned(lowerExc, 0, Long.BYTES, fieldPackedLower, 0, Long.BYTES) <= 0 + && Arrays.compareUnsigned(upperExc, 0, Long.BYTES, fieldPackedUpper, 0, Long.BYTES) >= 0) { + return new ScorerSupplier() { + @Override + public Scorer get(long leadCost) { + return new ConstantScoreScorer(weight, score(), scoreMode, DocIdSetIterator.all(reader.maxDoc())); + } + + @Override + public long cost() { + return reader.maxDoc(); + } + }; + } + } + final DocIdSetIterator exactIterator; + final DocIdSetIterator approxDISI; + if (values.getDocCount() == reader.maxDoc() + && values.getDocCount() == values.size() + && phase1.cost(values) > reader.maxDoc() / 2) { + // If all docs have exactly one value and the cost is greater + // than half the leaf size then maybe we can make things faster + // by computing the set of documents that do NOT match the range + final FixedBitSet result = new FixedBitSet(reader.maxDoc()); + final FixedBitSet approx = new FixedBitSet(reader.maxDoc()); + result.set(0, reader.maxDoc()); + approx.set(0, reader.maxDoc()); + int[] cost = new int[] { reader.maxDoc() }; + phase1.execute(values, result, approx, cost); + exactIterator = new BitSetIterator(result, cost[0]);; + approxDISI = new BitSetIterator(approx, cost[0]); + } else { + DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values, field); + DocIdSetBuilder approximation = new DocIdSetBuilder(reader.maxDoc(), values, field); + phase1.execute(values, result, approximation); + exactIterator = result.build().iterator(); + approxDISI = approximation.build().iterator(); + } + + final SortedNumericDocValues docValues = reader.getSortedNumericDocValues(field); + final long upperLong = toExactPrecision(upper); + final long lowerLong = toExactPrecision(lower); + + final TwoPhaseIterator twoPhaseIterator = new TwoPhaseIterator(approxDISI) { + + @Override + public boolean matches() throws IOException { + final int doc = approxDISI.docID(); + if (exactIterator != null) { + if (exactIterator.docID() < doc) { + exactIterator.advance(doc); + } + if (exactIterator.docID() == doc) { + return true; + } + } + return matches(doc); + } + + private boolean matches(int docId) throws IOException { + if (docValues.advanceExact(docId)) { + int count = docValues.docValueCount(); + for (int i = 0; i < count; i++) { + final long value = docValues.nextValue(); + if (value <= upperLong && value >= lowerLong) { + return true; + } + } + } + return false; + } + + @Override + public float matchCost() { + return 100; // TODO: use cost of exactIterator.advance() and predFuncValues.cost() + } + }; + return new ScorerSupplier() { + @Override + public Scorer get(long leadCost) { + return new ConstantScoreScorer(weight, score(), scoreMode, twoPhaseIterator); + } + + @Override + public long cost() { + return exactIterator.cost(); + } + }; + + } + + @Override + public Scorer scorer(LeafReaderContext context) throws IOException { + ScorerSupplier scorerSupplier = scorerSupplier(context); + if (scorerSupplier == null) { + return null; + } + return scorerSupplier.get(Long.MAX_VALUE); + } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return true; + } + + }; + } + + + public String getField() { + return field; + } + + @Override + public final int hashCode() { + int hash = classHash(); + hash = 31 * hash + field.hashCode(); + hash = 31 * hash + Long.hashCode(toExactPrecision(upper)); + hash = 31 * hash + Long.hashCode(toExactPrecision(lower)); + return hash; + } + + @Override + public final boolean equals(Object o) { + return sameClassAs(o) && + equalsTo(getClass().cast(o)); + } + + private boolean equalsTo(TwoPhaseDateRangeQuery other) { + return Objects.equals(field, other.field) && + Objects.equals(lower, other.lower) && Objects.equals(upper, other.upper); + } + + @Override + public final String toString(String field) { + final StringBuilder sb = new StringBuilder(); + if (this.field.equals(field) == false) { + sb.append(this.field); + sb.append(':'); + } + sb.append('['); + sb.append(toExactPrecision(lower)); + sb.append(" TO "); + sb.append(toExactPrecision(upper)); + sb.append(']'); + return sb.toString(); + } + + private static class ApproximatedQuery { + private final byte[] lowerPointInc; + private final byte[] lowerPointExc; + private final byte[] upperPointInc; + private final byte[] upperPointExc; + private long cost; + + ApproximatedQuery(byte[] lowerPointInc, byte[] lowerPointExc, byte[] upperPointInc, byte[] upperPointExc) { + this.lowerPointInc = lowerPointInc; + this.lowerPointExc = lowerPointExc; + this.upperPointInc = upperPointInc; + this.upperPointExc = upperPointExc; + cost = -1; + } + + void execute(PointValues values, DocIdSetBuilder result, DocIdSetBuilder approximation) throws IOException { + final IntersectVisitor visitor = getIntersectVisitor(result, approximation); + values.intersect(visitor); + } + + void execute(PointValues values, FixedBitSet result, FixedBitSet approximation, int[] cost) throws IOException { + final IntersectVisitor visitor = getInverseIntersectVisitor(result, approximation, cost); + values.intersect(visitor); + } + + long cost(PointValues values) { + if (cost == -1) { + // Computing the cost may be expensive, so only do it if necessary + cost = values.estimateDocCount(getIntersectVisitor(null, null)); + assert cost >= 0; + } + return cost; + } + + private IntersectVisitor getIntersectVisitor(DocIdSetBuilder result, DocIdSetBuilder approx) { + return new IntersectVisitor() { + + DocIdSetBuilder.BulkAdder adderExact; + DocIdSetBuilder.BulkAdder adderApprox; + + @Override + public void grow(int count) { + adderExact = result.grow(count); + adderApprox = approx.grow(count); + } + + @Override + public void visit(int docID) { + adderExact.add(docID); + adderApprox.add(docID); + } + + @Override + public void visit(int docID, byte[] packedValue) { + if (matches(packedValue)) { + adderApprox.add(docID); + if (addToExact(packedValue)) { + adderExact.add(docID); + } + } + } + + @Override + public void visit(DocIdSetIterator iterator, byte[] packedValue) throws IOException { + if (matches(packedValue)) { + boolean addExact = addToExact(packedValue); + int docID; + while ((docID = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + adderApprox.add(docID); + if (addExact) { + adderExact.add(docID); + } + } + } + } + + @Override + public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { + return relate(minPackedValue, maxPackedValue); + } + }; + } + + private IntersectVisitor getInverseIntersectVisitor(FixedBitSet exact, FixedBitSet approx, int[] cost) { + return new IntersectVisitor() { + + @Override + public void visit(int docID) { + exact.clear(docID); + approx.clear(docID); + cost[0]--; + } + + @Override + public void visit(int docID, byte[] packedValue) { + if (addToExact(packedValue) == false) { + exact.clear(docID); + if (matches(packedValue) == false) { + approx.clear(docID); + } + } + } + + @Override + public void visit(DocIdSetIterator iterator, byte[] packedValue) throws IOException { + if (addToExact(packedValue) == false) { + boolean matches = matches(packedValue); + int docID; + while ((docID = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + cost[0]--; + exact.clear(docID); + if (matches == false) { + approx.clear(docID); + } + } + } + } + + @Override + public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { + Relation relation = relate(minPackedValue, maxPackedValue); + switch (relation) { + case CELL_INSIDE_QUERY: + // all points match, skip this subtree + return Relation.CELL_OUTSIDE_QUERY; + case CELL_OUTSIDE_QUERY: + // none of the points match, clear all documents + return Relation.CELL_INSIDE_QUERY; + default: + return relation; + } + } + }; + } + + private boolean matches(byte[] packedValue) { + if (Arrays.compareUnsigned(packedValue, 0, Long.BYTES, lowerPointExc, 0, Long.BYTES) < 0) { + // Doc's value is too low + return false; + } + if (Arrays.compareUnsigned(packedValue, 0, Long.BYTES, upperPointExc, 0, Long.BYTES) > 0) { + // Doc's value is too high, in this dimension + return false; + } + return true; + } + + private boolean addToExact(byte[] packedValue) { + if (Arrays.compareUnsigned(packedValue, 0, Long.BYTES, lowerPointInc, 0, Long.BYTES) < 0) { + // Doc's value is too low + return false; + } + if (Arrays.compareUnsigned(packedValue, 0, Long.BYTES, upperPointInc, 0, Long.BYTES) > 0) { + // Doc's value is too high, in this dimension + return false; + } + return true; + } + + private Relation relate(byte[] minPackedValue, byte[] maxPackedValue) { + if (Arrays.compareUnsigned(minPackedValue, 0, Long.BYTES, upperPointExc, 0, Long.BYTES) > 0 || + Arrays.compareUnsigned(maxPackedValue, 0, Long.BYTES, lowerPointExc, 0, Long.BYTES) < 0) { + return Relation.CELL_OUTSIDE_QUERY; + } else if (Arrays.compareUnsigned(minPackedValue, 0, Long.BYTES, lowerPointInc, 0, Long.BYTES) < 0 || + Arrays.compareUnsigned(maxPackedValue, 0, Long.BYTES, upperPointInc, 0, Long.BYTES) > 0) { + return Relation.CELL_CROSSES_QUERY; + } else { + return Relation.CELL_INSIDE_QUERY; + } + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 10f918149cb5d..51a0cd998711f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -19,9 +19,12 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.document.Field; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.TwoPhaseDatePointMillis; +import org.apache.lucene.document.TwoPhaseDatePointNanos; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.PointValues; import org.apache.lucene.search.IndexOrDocValuesQuery; @@ -100,6 +103,40 @@ public long parsePointAsMillis(byte[] value) { protected Query distanceFeatureQuery(String field, float boost, long origin, TimeValue pivot) { return LongPoint.newDistanceFeatureQuery(field, boost, origin, pivot.getMillis()); } + + @Override + public void index(ParseContext.Document doc, String name, long value, boolean hasDocValues) { + if (hasDocValues) { + Field[] fields = TwoPhaseDatePointMillis.createIndexableFields(name, toInstant(value)); + for (Field field : fields) { + doc.add(field); + } + } else { + doc.add(new LongPoint(name, value)); + } + } + + @Override + public Query rangeQuery(String field, long min, long max, boolean hasDocValues) { + if (hasDocValues) { + Instant minInstant = toInstant(min); + Instant maxInstant = toInstant(max); + if ((min == Long.MIN_VALUE || min == convert(Instant.ofEpochSecond(minInstant.getEpochSecond()))) && + (max == Long.MAX_VALUE || max == convert(Instant.ofEpochSecond(maxInstant.getEpochSecond())))) { + return LongPoint.newRangeQuery(field, minInstant.getEpochSecond(), maxInstant.getEpochSecond()); + } else { + return TwoPhaseDatePointMillis.newRangeQuery(field, minInstant, maxInstant); + } + } else { + return LongPoint.newRangeQuery(field, min, max); + } + } + + @Override + public long toApprox(long exact) { + return toInstant(exact).getEpochSecond(); + } + }, NANOSECONDS(DATE_NANOS_CONTENT_TYPE, NumericType.DATE_NANOSECONDS) { @Override @@ -126,6 +163,39 @@ public long parsePointAsMillis(byte[] value) { protected Query distanceFeatureQuery(String field, float boost, long origin, TimeValue pivot) { return LongPoint.newDistanceFeatureQuery(field, boost, origin, pivot.getNanos()); } + + @Override + public void index(ParseContext.Document doc, String name, long value, boolean hasDocValues) { + if (hasDocValues) { + Field[] fields = TwoPhaseDatePointNanos.createIndexableFields(name, toInstant(value)); + for (Field field : fields) { + doc.add(field); + } + } else { + doc.add(new LongPoint(name, value)); + } + } + + @Override + public Query rangeQuery(String field, long min, long max, boolean hasDocValues) { + if (hasDocValues) { + Instant minInstant = toInstant(min); + Instant maxInstant = toInstant(max); + if (min == convert(Instant.ofEpochSecond(minInstant.getEpochSecond())) && + max == convert(Instant.ofEpochSecond(maxInstant.getEpochSecond()))) { + return LongPoint.newRangeQuery(field, minInstant.getEpochSecond(), maxInstant.getEpochSecond()); + } else { + return TwoPhaseDatePointNanos.newRangeQuery(field, minInstant, maxInstant); + } + } else { + return LongPoint.newRangeQuery(field, min, max); + } + } + + @Override + public long toApprox(long exact) { + return toInstant(exact).getEpochSecond(); + } }; private final String type; @@ -165,6 +235,21 @@ NumericType numericType() { */ public abstract long parsePointAsMillis(byte[] value); + /** + * Add index fields to doc + */ + public abstract void index(ParseContext.Document doc, String name, long value, boolean hasDocValues); + + /** + * Build range query + */ + public abstract Query rangeQuery(String field, long min, long max, boolean hasDocValues); + + /** + * Build range query + */ + public abstract long toApprox(long exact); + public static Resolution ofOrdinal(int ord) { for (Resolution resolution : values()) { if (ord == resolution.ordinal()) { @@ -352,8 +437,9 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower DateMathParser parser = forcedDateParser == null ? dateMathParser : forcedDateParser; + return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> { - Query query = LongPoint.newRangeQuery(name(), l, u); + Query query = resolution.rangeQuery(name(), l, u, hasDocValues()); if (hasDocValues()) { Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); query = new IndexOrDocValuesQuery(query, dvQuery); @@ -442,6 +528,12 @@ public Query distanceFeatureQuery(Object origin, String pivot, float boost, Quer public Relation isFieldWithinQuery(IndexReader reader, Object from, Object to, boolean includeLower, boolean includeUpper, ZoneId timeZone, DateMathParser dateParser, QueryRewriteContext context) throws IOException { + + if (PointValues.size(reader, name()) == 0) { + // no points, so nothing matches + return Relation.DISJOINT; + } + if (dateParser == null) { dateParser = this.dateMathParser; } @@ -468,20 +560,29 @@ public Relation isFieldWithinQuery(IndexReader reader, } } - if (PointValues.size(reader, name()) == 0) { - // no points, so nothing matches - return Relation.DISJOINT; - } - long minValue = LongPoint.decodeDimension(PointValues.getMinPackedValue(reader, name()), 0); long maxValue = LongPoint.decodeDimension(PointValues.getMaxPackedValue(reader, name()), 0); - if (minValue >= fromInclusive && maxValue <= toInclusive) { - return Relation.WITHIN; - } else if (maxValue < fromInclusive || minValue > toInclusive) { - return Relation.DISJOINT; + long approxFromInclusive; + long approxToInclusive; + if (hasDocValues()) { + approxFromInclusive = resolution.toApprox(fromInclusive); + approxToInclusive = resolution.toApprox(toInclusive); + if (minValue >= approxFromInclusive + 1 && maxValue <= approxToInclusive - 1) { + return Relation.WITHIN; + } else if (maxValue < approxFromInclusive || minValue > approxToInclusive) { + return Relation.DISJOINT; + } else { + return Relation.INTERSECTS; + } } else { - return Relation.INTERSECTS; + if (minValue >= fromInclusive && maxValue <= toInclusive) { + return Relation.WITHIN; + } else if (maxValue < fromInclusive || minValue > toInclusive) { + return Relation.DISJOINT; + } else { + return Relation.INTERSECTS; + } } } @@ -613,13 +714,14 @@ protected void parseCreateField(ParseContext context) throws IOException { } if (indexed) { - context.doc().add(new LongPoint(fieldType().name(), timestamp)); + resolution.index(context.doc(), fieldType().name(), timestamp, hasDocValues); } if (hasDocValues) { context.doc().add(new SortedNumericDocValuesField(fieldType().name(), timestamp)); } else if (store || indexed) { createFieldNamesField(context); } + if (store) { context.doc().add(new StoredField(fieldType().name(), timestamp)); } diff --git a/server/src/test/java/org/apache/lucene/queries/TwoPhaseDateTests.java b/server/src/test/java/org/apache/lucene/queries/TwoPhaseDateTests.java new file mode 100644 index 0000000000000..91a5334b433c4 --- /dev/null +++ b/server/src/test/java/org/apache/lucene/queries/TwoPhaseDateTests.java @@ -0,0 +1,99 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lucene.queries; + +import com.carrotsearch.randomizedtesting.generators.RandomNumbers; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.TwoPhaseDatePointMillis; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.SerialMergeScheduler; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.LuceneTestCase; + +import java.time.Instant; + +public class TwoPhaseDateTests extends LuceneTestCase { + + public void testIndexMillis() throws Exception { + IndexWriterConfig iwc = newIndexWriterConfig(); + // Else seeds may not reproduce: + iwc.setMergeScheduler(new SerialMergeScheduler()); + // Else we can get O(N^2) merging: + iwc.setMaxBufferedDocs(10); + Directory dir = newDirectory(); + // RandomIndexWriter is too slow here: + int numPoints = random().nextInt(10000); + IndexWriter w = new IndexWriter(dir, iwc); + for (int id = 0; id < numPoints; id++) { + Document doc = new Document(); + Instant instant = randomInstant(); + doc.add(new LongPoint("exact", instant.toEpochMilli())); + Field[] fields = TwoPhaseDatePointMillis.createIndexableFields("approx", instant); + for (int i =0; i < fields.length; i++) { + doc.add(fields[i]); + } + w.addDocument(doc); + } + + if (random().nextBoolean()) { + w.forceMerge(1); + } + final IndexReader r = DirectoryReader.open(w); + w.close(); + + IndexSearcher s = newSearcher(r); + for ( int i = 0; i < 100; i++) { + Instant instant1 = randomInstant(); + Instant instant2 = randomInstant(); + Query q1; + Query q2; + if (instant1.toEpochMilli() >= instant2.toEpochMilli()) { + q1 = LongPoint.newRangeQuery("exact", instant2.toEpochMilli(), instant1.toEpochMilli()); + q2 = TwoPhaseDatePointMillis.newRangeQuery("approx", instant2, instant1); + } else { + q1 = LongPoint.newRangeQuery("exact", instant1.toEpochMilli(), instant2.toEpochMilli()); + q2 = TwoPhaseDatePointMillis.newRangeQuery("approx", instant1, instant2); + } + assertEquals(s.count(q1), s.count(q2)); + } + + IOUtils.close(r, dir); + } + + /** + * @return a random instant between 1970 and ca 2065 + */ + protected Instant randomInstant() { + //return Instant.ofEpochSecond(randomLongBetween(0, 3000000000L), randomLongBetween(0, 999999999)); + return Instant.ofEpochSecond(randomLongBetween(0, 30000L), randomLongBetween(0, 999999999)); + } + + public static long randomLongBetween(long min, long max) { + return RandomNumbers.randomLongBetween(random(), min, max); + } +} From 4b71097acc6cd045297593176231c94f7bcd6f3f Mon Sep 17 00:00:00 2001 From: iverase Date: Thu, 8 Oct 2020 15:30:01 +0200 Subject: [PATCH 02/18] don't duplicate doc values --- .../index/mapper/DateFieldMapper.java | 64 ++++++++++++++----- 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 51a0cd998711f..7d074e1d645ed 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -27,6 +27,7 @@ import org.apache.lucene.document.TwoPhaseDatePointNanos; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.PointValues; +import org.apache.lucene.queries.TwoPhaseDateRangeQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.Query; @@ -84,6 +85,11 @@ public long convert(Instant instant) { return instant.toEpochMilli(); } + @Override + public long convertApprox(Instant instant) { + return instant.getEpochSecond(); + } + @Override public Instant toInstant(long value) { return Instant.ofEpochMilli(value); @@ -107,10 +113,7 @@ protected Query distanceFeatureQuery(String field, float boost, long origin, Tim @Override public void index(ParseContext.Document doc, String name, long value, boolean hasDocValues) { if (hasDocValues) { - Field[] fields = TwoPhaseDatePointMillis.createIndexableFields(name, toInstant(value)); - for (Field field : fields) { - doc.add(field); - } + doc.add(new LongPoint(name, toInstant(value).getEpochSecond())); } else { doc.add(new LongPoint(name, value)); } @@ -121,11 +124,22 @@ public Query rangeQuery(String field, long min, long max, boolean hasDocValues) if (hasDocValues) { Instant minInstant = toInstant(min); Instant maxInstant = toInstant(max); - if ((min == Long.MIN_VALUE || min == convert(Instant.ofEpochSecond(minInstant.getEpochSecond()))) && - (max == Long.MAX_VALUE || max == convert(Instant.ofEpochSecond(maxInstant.getEpochSecond())))) { - return LongPoint.newRangeQuery(field, minInstant.getEpochSecond(), maxInstant.getEpochSecond()); + if ((min == Long.MIN_VALUE || min == convert(Instant.ofEpochSecond(convertApprox(minInstant)))) && + (max == Long.MAX_VALUE || max == convert(Instant.ofEpochSecond(convertApprox(maxInstant))))) { + return LongPoint.newRangeQuery(field, convertApprox(minInstant), convertApprox(maxInstant)); } else { - return TwoPhaseDatePointMillis.newRangeQuery(field, minInstant, maxInstant); + return new TwoPhaseDateRangeQuery(field, minInstant, maxInstant) { + + @Override + protected long toApproxPrecision(Instant instant) { + return convertApprox(instant); + } + + @Override + protected long toExactPrecision(Instant instant) { + return convert(instant); + } + }; } } else { return LongPoint.newRangeQuery(field, min, max); @@ -144,6 +158,11 @@ public long convert(Instant instant) { return toLong(instant); } + @Override + public long convertApprox(Instant instant) { + return instant.getEpochSecond(); + } + @Override public Instant toInstant(long value) { return DateUtils.toInstant(value); @@ -167,10 +186,7 @@ protected Query distanceFeatureQuery(String field, float boost, long origin, Tim @Override public void index(ParseContext.Document doc, String name, long value, boolean hasDocValues) { if (hasDocValues) { - Field[] fields = TwoPhaseDatePointNanos.createIndexableFields(name, toInstant(value)); - for (Field field : fields) { - doc.add(field); - } + doc.add(new LongPoint(name, toInstant(value).getEpochSecond())); } else { doc.add(new LongPoint(name, value)); } @@ -181,11 +197,22 @@ public Query rangeQuery(String field, long min, long max, boolean hasDocValues) if (hasDocValues) { Instant minInstant = toInstant(min); Instant maxInstant = toInstant(max); - if (min == convert(Instant.ofEpochSecond(minInstant.getEpochSecond())) && - max == convert(Instant.ofEpochSecond(maxInstant.getEpochSecond()))) { - return LongPoint.newRangeQuery(field, minInstant.getEpochSecond(), maxInstant.getEpochSecond()); + if ((min == Long.MIN_VALUE || min == convert(Instant.ofEpochSecond(convertApprox(minInstant)))) && + (max == Long.MAX_VALUE || max == convert(Instant.ofEpochSecond(convertApprox(maxInstant))))) { + return LongPoint.newRangeQuery(field, convertApprox(minInstant), convertApprox(maxInstant)); } else { - return TwoPhaseDatePointNanos.newRangeQuery(field, minInstant, maxInstant); + return new TwoPhaseDateRangeQuery(field, minInstant, maxInstant) { + + @Override + protected long toApproxPrecision(Instant instant) { + return convertApprox(instant); + } + + @Override + protected long toExactPrecision(Instant instant) { + return convert(instant); + } + }; } } else { return LongPoint.newRangeQuery(field, min, max); @@ -219,6 +246,11 @@ NumericType numericType() { */ public abstract long convert(Instant instant); + /** + * Convert an {@linkplain Instant} into a long value in the approximated resolution. + */ + public abstract long convertApprox(Instant instant); + /** * Convert a long value in this resolution into an instant. */ From 40e6e2d81dc59b4594f46e638058c68467b67bc4 Mon Sep 17 00:00:00 2001 From: iverase Date: Thu, 8 Oct 2020 16:30:51 +0200 Subject: [PATCH 03/18] cleanup --- .../index/mapper/DateFieldMapper.java | 142 +++++------------- 1 file changed, 40 insertions(+), 102 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 7d074e1d645ed..c395a13186250 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -19,12 +19,9 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.document.Field; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; -import org.apache.lucene.document.TwoPhaseDatePointMillis; -import org.apache.lucene.document.TwoPhaseDatePointNanos; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.PointValues; import org.apache.lucene.queries.TwoPhaseDateRangeQuery; @@ -96,8 +93,8 @@ public Instant toInstant(long value) { } @Override - public Instant clampToValidRange(Instant instant) { - return instant; + public Instant toInstantFromApprox(long value) { + return Instant.ofEpochSecond(value); } @Override @@ -109,48 +106,6 @@ public long parsePointAsMillis(byte[] value) { protected Query distanceFeatureQuery(String field, float boost, long origin, TimeValue pivot) { return LongPoint.newDistanceFeatureQuery(field, boost, origin, pivot.getMillis()); } - - @Override - public void index(ParseContext.Document doc, String name, long value, boolean hasDocValues) { - if (hasDocValues) { - doc.add(new LongPoint(name, toInstant(value).getEpochSecond())); - } else { - doc.add(new LongPoint(name, value)); - } - } - - @Override - public Query rangeQuery(String field, long min, long max, boolean hasDocValues) { - if (hasDocValues) { - Instant minInstant = toInstant(min); - Instant maxInstant = toInstant(max); - if ((min == Long.MIN_VALUE || min == convert(Instant.ofEpochSecond(convertApprox(minInstant)))) && - (max == Long.MAX_VALUE || max == convert(Instant.ofEpochSecond(convertApprox(maxInstant))))) { - return LongPoint.newRangeQuery(field, convertApprox(minInstant), convertApprox(maxInstant)); - } else { - return new TwoPhaseDateRangeQuery(field, minInstant, maxInstant) { - - @Override - protected long toApproxPrecision(Instant instant) { - return convertApprox(instant); - } - - @Override - protected long toExactPrecision(Instant instant) { - return convert(instant); - } - }; - } - } else { - return LongPoint.newRangeQuery(field, min, max); - } - } - - @Override - public long toApprox(long exact) { - return toInstant(exact).getEpochSecond(); - } - }, NANOSECONDS(DATE_NANOS_CONTENT_TYPE, NumericType.DATE_NANOSECONDS) { @Override @@ -169,8 +124,8 @@ public Instant toInstant(long value) { } @Override - public Instant clampToValidRange(Instant instant) { - return DateUtils.clampToNanosRange(instant); + public Instant toInstantFromApprox(long value) { + return Instant.ofEpochSecond(value); } @Override @@ -182,47 +137,6 @@ public long parsePointAsMillis(byte[] value) { protected Query distanceFeatureQuery(String field, float boost, long origin, TimeValue pivot) { return LongPoint.newDistanceFeatureQuery(field, boost, origin, pivot.getNanos()); } - - @Override - public void index(ParseContext.Document doc, String name, long value, boolean hasDocValues) { - if (hasDocValues) { - doc.add(new LongPoint(name, toInstant(value).getEpochSecond())); - } else { - doc.add(new LongPoint(name, value)); - } - } - - @Override - public Query rangeQuery(String field, long min, long max, boolean hasDocValues) { - if (hasDocValues) { - Instant minInstant = toInstant(min); - Instant maxInstant = toInstant(max); - if ((min == Long.MIN_VALUE || min == convert(Instant.ofEpochSecond(convertApprox(minInstant)))) && - (max == Long.MAX_VALUE || max == convert(Instant.ofEpochSecond(convertApprox(maxInstant))))) { - return LongPoint.newRangeQuery(field, convertApprox(minInstant), convertApprox(maxInstant)); - } else { - return new TwoPhaseDateRangeQuery(field, minInstant, maxInstant) { - - @Override - protected long toApproxPrecision(Instant instant) { - return convertApprox(instant); - } - - @Override - protected long toExactPrecision(Instant instant) { - return convert(instant); - } - }; - } - } else { - return LongPoint.newRangeQuery(field, min, max); - } - } - - @Override - public long toApprox(long exact) { - return toInstant(exact).getEpochSecond(); - } }; private final String type; @@ -257,10 +171,9 @@ NumericType numericType() { public abstract Instant toInstant(long value); /** - * Return the instant that this range can represent that is closest to - * the provided instant. + * Convert a long value in this resolution into an instant. */ - public abstract Instant clampToValidRange(Instant instant); + public abstract Instant toInstantFromApprox(long value); /** * Decode the points representation of this field as milliseconds. @@ -270,17 +183,42 @@ NumericType numericType() { /** * Add index fields to doc */ - public abstract void index(ParseContext.Document doc, String name, long value, boolean hasDocValues); - - /** - * Build range query - */ - public abstract Query rangeQuery(String field, long min, long max, boolean hasDocValues); + public void index(ParseContext.Document doc, String name, long value, boolean hasDocValues) { + if (hasDocValues) { + doc.add(new LongPoint(name, convertApprox(toInstant(value)))); + } else { + doc.add(new LongPoint(name, value)); + } + } /** * Build range query */ - public abstract long toApprox(long exact); + public Query rangeQuery(String field, long min, long max, boolean hasDocValues) { + if (hasDocValues) { + Instant minInstant = toInstant(min); + Instant maxInstant = toInstant(max); + if ((min == Long.MIN_VALUE || min == convert(toInstantFromApprox(convertApprox(minInstant)))) && + (max == Long.MAX_VALUE || max == convert(toInstantFromApprox(convertApprox(maxInstant))))) { + return LongPoint.newRangeQuery(field, convertApprox(minInstant), convertApprox(maxInstant)); + } else { + return new TwoPhaseDateRangeQuery(field, minInstant, maxInstant) { + + @Override + protected long toApproxPrecision(Instant instant) { + return convertApprox(instant); + } + + @Override + protected long toExactPrecision(Instant instant) { + return convert(instant); + } + }; + } + } else { + return LongPoint.newRangeQuery(field, min, max); + } + } public static Resolution ofOrdinal(int ord) { for (Resolution resolution : values()) { @@ -598,8 +536,8 @@ public Relation isFieldWithinQuery(IndexReader reader, long approxFromInclusive; long approxToInclusive; if (hasDocValues()) { - approxFromInclusive = resolution.toApprox(fromInclusive); - approxToInclusive = resolution.toApprox(toInclusive); + approxFromInclusive = resolution.convertApprox(resolution.toInstant(fromInclusive)); + approxToInclusive = resolution.convertApprox(resolution.toInstant(toInclusive)); if (minValue >= approxFromInclusive + 1 && maxValue <= approxToInclusive - 1) { return Relation.WITHIN; } else if (maxValue < approxFromInclusive || minValue > approxToInclusive) { From b98ff24011dab4da08cdf997c90274cc0f55867e Mon Sep 17 00:00:00 2001 From: iverase Date: Fri, 9 Oct 2020 11:52:07 +0200 Subject: [PATCH 04/18] iter --- .../search/query/QueryStringIT.java | 25 + .../document/TwoPhaseDatePointMillis.java | 59 --- .../document/TwoPhaseDatePointNanos.java | 61 --- .../TwoPhaseLongDistanceFeatureQuery.java | 454 ++++++++++++++++++ ...Query.java => TwoPhaseLongRangeQuery.java} | 106 ++-- .../index/mapper/DateFieldMapper.java | 111 +++-- ...TwoPhaseLongDistanceFeatureQueryTests.java | 117 +++++ ....java => TwoPhaseLongRangeQueryTests.java} | 43 +- 8 files changed, 742 insertions(+), 234 deletions(-) delete mode 100644 server/src/main/java/org/apache/lucene/document/TwoPhaseDatePointMillis.java delete mode 100644 server/src/main/java/org/apache/lucene/document/TwoPhaseDatePointNanos.java create mode 100644 server/src/main/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQuery.java rename server/src/main/java/org/apache/lucene/queries/{TwoPhaseDateRangeQuery.java => TwoPhaseLongRangeQuery.java} (85%) create mode 100644 server/src/test/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQueryTests.java rename server/src/test/java/org/apache/lucene/queries/{TwoPhaseDateTests.java => TwoPhaseLongRangeQueryTests.java} (69%) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java index 57f888c9a4d33..833bcad05d454 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java @@ -43,6 +43,8 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery; +import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; @@ -118,6 +120,29 @@ public void testWithDate() throws Exception { assertHitCount(resp, 2L); } + public void testExactDate() throws Exception { + List reqs = new ArrayList<>(); + reqs.add(client().prepareIndex("test2").setId("1").setSource("f1", "foo", "f_date", "2015-09-02T00:00:00.000")); + reqs.add(client().prepareIndex("test2").setId("2").setSource("f1", "bar", "f_date", "2015-09-02T00:00:00.001")); + reqs.add(client().prepareIndex("test2").setId("3").setSource("f1", "bar", "f_date", "2015-09-01T23:59:59.999")); + indexRandom(true, false, reqs); + + SearchResponse resp = client().prepareSearch("test2").setQuery(rangeQuery("f_date") + .from("2015-09-02T00:00:00.000").to("2015-09-02T00:00:00.000").includeLower(true).includeUpper(true)).get(); + assertHits(resp.getHits(), "1"); + assertHitCount(resp, 1L); + resp = client().prepareSearch("test2").setQuery(rangeQuery("f_date") + .from("2015-09-02T00:00:00.000").to("2015-09-02T00:00:00.001").includeLower(true).includeUpper(false)).get(); + assertHits(resp.getHits(), "1"); + resp = client().prepareSearch("test2").setQuery(rangeQuery("f_date") + .from("2015-09-02T00:00:00.000").to("2015-09-02T00:00:00.001").includeLower(false).includeUpper(true)).get(); + assertHits(resp.getHits(), "2"); + resp = client().prepareSearch("test2").setQuery(rangeQuery("f_date") + .from("2015-09-02T00:00:00.000").to("2015-09-02T00:00:00.001").includeLower(true).includeUpper(true)).get(); + assertHits(resp.getHits(), "1", "2"); + assertHitCount(resp, 2L); + } + public void testWithLotsOfTypes() throws Exception { List reqs = new ArrayList<>(); reqs.add(client().prepareIndex("test").setId("1").setSource("f1", "foo", diff --git a/server/src/main/java/org/apache/lucene/document/TwoPhaseDatePointMillis.java b/server/src/main/java/org/apache/lucene/document/TwoPhaseDatePointMillis.java deleted file mode 100644 index 310e90283659a..0000000000000 --- a/server/src/main/java/org/apache/lucene/document/TwoPhaseDatePointMillis.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.document; - -import org.apache.lucene.queries.TwoPhaseDateRangeQuery; -import org.apache.lucene.search.Query; - -import java.time.Instant; - - -public final class TwoPhaseDatePointMillis { - - - public static Field[] createIndexableFields(String name, Instant... instants) { - Field[] fields = new Field[2*instants.length]; - for (int i = 0; i < instants.length; i++) { - fields[2*i] = new LongPoint(name, instants[i].getEpochSecond()); - fields[2*i+1] = new SortedNumericDocValuesField(name, instants[i].toEpochMilli()); - } - return fields; - } - - // static methods for generating queries - - - public static Query newExactQuery(String field, Instant instant) { - return newRangeQuery(field, instant, instant); - } - - - public static Query newRangeQuery(String field, Instant lowerValue, Instant upperValue) { - return new TwoPhaseDateRangeQuery(field, lowerValue, upperValue) { - - @Override - protected long toApproxPrecision(Instant instant) { - return instant.getEpochSecond(); - } - - @Override - protected long toExactPrecision(Instant instant) { - return instant.toEpochMilli(); - } - }; - } -} diff --git a/server/src/main/java/org/apache/lucene/document/TwoPhaseDatePointNanos.java b/server/src/main/java/org/apache/lucene/document/TwoPhaseDatePointNanos.java deleted file mode 100644 index c4c4a76cc840c..0000000000000 --- a/server/src/main/java/org/apache/lucene/document/TwoPhaseDatePointNanos.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.document; - -import org.apache.lucene.queries.TwoPhaseDateRangeQuery; -import org.apache.lucene.search.Query; -import org.elasticsearch.common.time.DateUtils; - -import java.time.Instant; - - -public final class TwoPhaseDatePointNanos { - - - public static Field[] createIndexableFields(String name, Instant... instants) { - Field[] fields = new Field[2*instants.length]; - for (int i = 0; i < instants.length; i++) { - fields[2*i] = new LongPoint(name, instants[i].getEpochSecond()); - fields[2*i+1] = new SortedNumericDocValuesField(name, DateUtils.toLong(instants[i])); - } - return fields; - } - - - // static methods for generating queries - - public static Query newExactQuery(String field, Instant instant) { - return newRangeQuery(field, instant, instant); - } - - - public static Query newRangeQuery(String field, Instant lowerValue, Instant upperValue) { - return new TwoPhaseDateRangeQuery(field, lowerValue, upperValue) { - - @Override - protected long toApproxPrecision(Instant instant) { - return instant.getEpochSecond(); - } - - @Override - protected long toExactPrecision(Instant instant) { - return DateUtils.toLong(instant); - } - }; - } - -} diff --git a/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQuery.java b/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQuery.java new file mode 100644 index 0000000000000..157b88a8f98c5 --- /dev/null +++ b/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQuery.java @@ -0,0 +1,454 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.queries; + +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.PointValues; +import org.apache.lucene.index.PointValues.IntersectVisitor; +import org.apache.lucene.index.PointValues.Relation; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.DocIdSetBuilder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; +import java.util.Set; + +public abstract class TwoPhaseLongDistanceFeatureQuery extends Query { + + private final String field; + private final long originExact; + private final long pivotDistanceExact; + private final long originApprox; + + public TwoPhaseLongDistanceFeatureQuery(String field, long originExact, long pivotDistanceExact, long originApprox) { + this.field = Objects.requireNonNull(field); + this.originExact = originExact; + if (pivotDistanceExact <= 0) { + throw new IllegalArgumentException("pivotDistance must be > 0, got " + pivotDistanceExact); + } + this.pivotDistanceExact = pivotDistanceExact; + this.originApprox = originApprox; + } + + public abstract long convertDistance(long distanceExact); + + @Override + public final boolean equals(Object o) { + return sameClassAs(o) && + equalsTo(getClass().cast(o)); + } + + private boolean equalsTo(TwoPhaseLongDistanceFeatureQuery other) { + return Objects.equals(field, other.field) && + originExact == other.originExact && + pivotDistanceExact == other.pivotDistanceExact && + originApprox == other.originApprox; + } + + @Override + public int hashCode() { + int h = classHash(); + h = 31 * h + field.hashCode(); + h = 31 * h + Long.hashCode(originExact); + h = 31 * h + Long.hashCode(pivotDistanceExact); + h = 31 * h + Long.hashCode(originApprox); + return h; + } + + @Override + public void visit(QueryVisitor visitor) { + if (visitor.acceptField(field)) { + visitor.visitLeaf(this); + } + } + + @Override + public String toString(String field) { + return getClass().getSimpleName() + "(field=" + field + ",origin=" + originExact + ",pivotDistance=" + pivotDistanceExact + ")"; + } + + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + return new Weight(this) { + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return false; + } + + @Override + public void extractTerms(Set terms) { + } + + @Override + public Explanation explain(LeafReaderContext context, int doc) throws IOException { + SortedNumericDocValues multiDocValues = DocValues.getSortedNumeric(context.reader(), field); + if (multiDocValues.advanceExact(doc) == false) { + return Explanation.noMatch("Document " + doc + " doesn't have a value for field " + field); + } + long value = selectValue(multiDocValues); + long distance = Math.max(value, originExact) - Math.min(value, originExact); + if (distance < 0) { + // underflow, treat as MAX_VALUE + distance = Long.MAX_VALUE; + } + float score = (float) (boost * (pivotDistanceExact / (pivotDistanceExact + (double) distance))); + return Explanation.match(score, "Distance score, computed as weight * pivotDistance / (pivotDistance + abs(value - origin)) from:", + Explanation.match(boost, "weight"), + Explanation.match(pivotDistanceExact, "pivotDistance"), + Explanation.match(originExact, "origin"), + Explanation.match(value, "current value")); + } + + private long selectValue(SortedNumericDocValues multiDocValues) throws IOException { + int count = multiDocValues.docValueCount(); + + long next = multiDocValues.nextValue(); + if (count == 1 || next >= originExact) { + return next; + } + long previous = next; + for (int i = 1; i < count; ++i) { + next = multiDocValues.nextValue(); + if (next >= originExact) { + // Unsigned comparison because of underflows + if (Long.compareUnsigned(originExact - previous, next - originExact) < 0) { + return previous; + } else { + return next; + } + } + previous = next; + } + + assert next < originExact; + return next; + } + + private NumericDocValues selectValues(SortedNumericDocValues multiDocValues) { + final NumericDocValues singleton = DocValues.unwrapSingleton(multiDocValues); + if (singleton != null) { + return singleton; + } + return new NumericDocValues() { + + long value; + + @Override + public long longValue() throws IOException { + return value; + } + + @Override + public boolean advanceExact(int target) throws IOException { + if (multiDocValues.advanceExact(target)) { + value = selectValue(multiDocValues); + return true; + } else { + return false; + } + } + + @Override + public int docID() { + return multiDocValues.docID(); + } + + @Override + public int nextDoc() throws IOException { + return multiDocValues.nextDoc(); + } + + @Override + public int advance(int target) throws IOException { + return multiDocValues.advance(target); + } + + @Override + public long cost() { + return multiDocValues.cost(); + } + + }; + } + + @Override + public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException { + PointValues pointValues = context.reader().getPointValues(field); + if (pointValues == null) { + // No data on this segment + return null; + } + final SortedNumericDocValues multiDocValues = DocValues.getSortedNumeric(context.reader(), field); + final NumericDocValues docValues = selectValues(multiDocValues); + + final Weight weight = this; + return new ScorerSupplier() { + + @Override + public Scorer get(long leadCost) throws IOException { + return new DistanceScorer(weight, context.reader().maxDoc(), leadCost, boost, pointValues, docValues); + } + + @Override + public long cost() { + return docValues.cost(); + } + }; + } + + @Override + public Scorer scorer(LeafReaderContext context) throws IOException { + ScorerSupplier scorerSupplier = scorerSupplier(context); + if (scorerSupplier == null) { + return null; + } + return scorerSupplier.get(Long.MAX_VALUE); + } + + }; + } + + private class DistanceScorer extends Scorer { + + private final int maxDoc; + private DocIdSetIterator it; + private int doc = -1; + private final long leadCost; + private final float boost; + private final PointValues pointValues; + private final NumericDocValues docValues; + private long maxDistance = Long.MAX_VALUE; + + protected DistanceScorer(Weight weight, int maxDoc, long leadCost, float boost, + PointValues pointValues, NumericDocValues docValues) { + super(weight); + this.maxDoc = maxDoc; + this.leadCost = leadCost; + this.boost = boost; + this.pointValues = pointValues; + this.docValues = docValues; + // initially use doc values in order to iterate all documents that have + // a value for this field + this.it = docValues; + } + + @Override + public int docID() { + return doc; + } + + private float score(double distance) { + return (float) (boost * (pivotDistanceExact / (pivotDistanceExact + distance))); + } + + /** + * Inverting the score computation is very hard due to all potential + * rounding errors, so we binary search the maximum distance. + */ + private long computeMaxDistance(float minScore, long previousMaxDistance) { + assert score(0) >= minScore; + if (score(previousMaxDistance) >= minScore) { + // minScore did not decrease enough to require an update to the max distance + return previousMaxDistance; + } + assert score(previousMaxDistance) < minScore; + long min = 0, max = previousMaxDistance; + // invariant: score(min) >= minScore && score(max) < minScore + while (max - min > 1) { + long mid = (min + max) >>> 1; + float score = score(mid); + if (score >= minScore) { + min = mid; + } else { + max = mid; + } + } + assert score(min) >= minScore; + assert min == Long.MAX_VALUE || score(min + 1) < minScore; + return min; + } + + @Override + public float score() throws IOException { + if (docValues.advanceExact(docID()) == false) { + return 0; + } + long v = docValues.longValue(); + // note: distance is unsigned + long distance = Math.max(v, originExact) - Math.min(v, originExact); + if (distance < 0) { + // underflow + // treat distances that are greater than MAX_VALUE as MAX_VALUE + distance = Long.MAX_VALUE; + } + return score(distance); + } + + @Override + public DocIdSetIterator iterator() { + // add indirection so that if 'it' is updated then it will + // be taken into account + return new DocIdSetIterator() { + + @Override + public int nextDoc() throws IOException { + return doc = it.nextDoc(); + } + + @Override + public int docID() { + return doc; + } + + @Override + public long cost() { + return it.cost(); + } + + @Override + public int advance(int target) throws IOException { + return doc = it.advance(target); + } + }; + } + + @Override + public float getMaxScore(int upTo) { + return boost; + } + + private int setMinCompetitiveScoreCounter = 0; + + @Override + public void setMinCompetitiveScore(float minScore) throws IOException { + if (minScore > boost) { + it = DocIdSetIterator.empty(); + return; + } + + // Start sampling if we get called too much + setMinCompetitiveScoreCounter++; + if (setMinCompetitiveScoreCounter > 256 && (setMinCompetitiveScoreCounter & 0x1f) != 0x1f) { + return; + } + + long previousMaxDistance = maxDistance; + maxDistance = computeMaxDistance(minScore, maxDistance); + if (maxDistance == previousMaxDistance) { + // nothing to update + return; + } + long minValue = originApprox - convertDistance(maxDistance) - 1; + if (minValue > originApprox) { + // underflow + minValue = Long.MIN_VALUE; + } + long maxValue = originApprox + convertDistance(maxDistance) + 1; + if (maxValue < originApprox) { + // overflow + maxValue = Long.MAX_VALUE; + } + + final byte[] minValueAsBytes = new byte[Long.BYTES]; + LongPoint.encodeDimension(minValue, minValueAsBytes, 0); + final byte[] maxValueAsBytes = new byte[Long.BYTES]; + LongPoint.encodeDimension(maxValue, maxValueAsBytes, 0); + + DocIdSetBuilder result = new DocIdSetBuilder(maxDoc); + final int doc = docID(); + IntersectVisitor visitor = new IntersectVisitor() { + + DocIdSetBuilder.BulkAdder adder; + + @Override + public void grow(int count) { + adder = result.grow(count); + } + + @Override + public void visit(int docID) { + if (docID <= doc) { + // Already visited or skipped + return; + } + adder.add(docID); + } + + @Override + public void visit(int docID, byte[] packedValue) { + if (docID <= doc) { + // Already visited or skipped + return; + } + if (Arrays.compareUnsigned(packedValue, 0, Long.BYTES, minValueAsBytes, 0, Long.BYTES) < 0) { + // Doc's value is too low, in this dimension + return; + } + if (Arrays.compareUnsigned(packedValue, 0, Long.BYTES, maxValueAsBytes, 0, Long.BYTES) > 0) { + // Doc's value is too high, in this dimension + return; + } + + // Doc is in-bounds + adder.add(docID); + } + + @Override + public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { + if (Arrays.compareUnsigned(minPackedValue, 0, Long.BYTES, maxValueAsBytes, 0, Long.BYTES) > 0 || + Arrays.compareUnsigned(maxPackedValue, 0, Long.BYTES, minValueAsBytes, 0, Long.BYTES) < 0) { + return Relation.CELL_OUTSIDE_QUERY; + } + + if (Arrays.compareUnsigned(minPackedValue, 0, Long.BYTES, minValueAsBytes, 0, Long.BYTES) < 0 || + Arrays.compareUnsigned(maxPackedValue, 0, Long.BYTES, maxValueAsBytes, 0, Long.BYTES) > 0) { + return Relation.CELL_CROSSES_QUERY; + } + + return Relation.CELL_INSIDE_QUERY; + } + }; + + final long currentQueryCost = Math.min(leadCost, it.cost()); + final long threshold = currentQueryCost >>> 3; + long estimatedNumberOfMatches = pointValues.estimatePointCount(visitor); // runs in O(log(numPoints)) + // TODO: what is the right factor compared to the current disi? Is 8 optimal? + if (estimatedNumberOfMatches >= threshold) { + // the new range is not selective enough to be worth materializing + return; + } + pointValues.intersect(visitor); + it = result.build().iterator(); + } + + } +} diff --git a/server/src/main/java/org/apache/lucene/queries/TwoPhaseDateRangeQuery.java b/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongRangeQuery.java similarity index 85% rename from server/src/main/java/org/apache/lucene/queries/TwoPhaseDateRangeQuery.java rename to server/src/main/java/org/apache/lucene/queries/TwoPhaseLongRangeQuery.java index 719954d1b05f7..2228ff81d6a7c 100644 --- a/server/src/main/java/org/apache/lucene/queries/TwoPhaseDateRangeQuery.java +++ b/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongRangeQuery.java @@ -38,10 +38,8 @@ import org.apache.lucene.util.DocIdSetBuilder; import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.NumericUtils; -import org.elasticsearch.common.time.DateUtils; import java.io.IOException; -import java.time.Instant; import java.util.Arrays; import java.util.Objects; @@ -56,25 +54,28 @@ *

* For a single-dimensional field this query is a simple range query; in a multi-dimensional field it's a box shape. * @see PointValues - * @lucene.experimental */ -public abstract class TwoPhaseDateRangeQuery extends Query { +public class TwoPhaseLongRangeQuery extends Query { final String field; - final Instant lower; - final Instant upper; + final long lowerExact; + final long upperExact; + final long lowerApprox; + final long upperApprox; /** * Expert: create a multidimensional range query for point values. * * @param field field name. must not be {@code null}. - * @param lower lower portion of the range (inclusive). - * @param upper upper portion of the range (inclusive). + * @param lowerExact lower portion of the range (inclusive). + * @param upperExact upper portion of the range (inclusive). * @throws IllegalArgumentException if {@code field} is null, or if {@code lowerValue.length != upperValue.length} */ - public TwoPhaseDateRangeQuery(String field, Instant lower, Instant upper) { - this.field = field; - this.lower = lower; - this.upper = upper; + public TwoPhaseLongRangeQuery(String field, long lowerExact, long upperExact, long lowerApprox, long upperApprox) { + this.field = field; + this.lowerExact = lowerExact; + this.upperExact = upperExact; + this.lowerApprox = lowerApprox; + this.upperApprox = upperApprox; } @Override @@ -84,25 +85,16 @@ public void visit(QueryVisitor visitor) { } } - protected abstract long toApproxPrecision(Instant instant); - - protected abstract long toExactPrecision(Instant instant); - @Override - public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { - - // We don't use RandomAccessWeight here: it's no good to approximate with "match all docs". - // This is an inverted structure and should be used in the first pass: - long l = toApproxPrecision(lower); - long u = toApproxPrecision(upper); + public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) { byte[] lowerInc = new byte[Long.BYTES]; - NumericUtils.longToSortableBytes(l + 1, lowerInc, 0); + NumericUtils.longToSortableBytes(lowerApprox + 1, lowerInc, 0); byte[] lowerExc = new byte[Long.BYTES]; - NumericUtils.longToSortableBytes(l, lowerExc, 0); + NumericUtils.longToSortableBytes(lowerApprox, lowerExc, 0); byte[] upperInc = new byte[Long.BYTES]; - NumericUtils.longToSortableBytes(u - 1, upperInc, 0); + NumericUtils.longToSortableBytes(upperApprox - 1, upperInc, 0); byte[] upperExc = new byte[Long.BYTES]; - NumericUtils.longToSortableBytes(u, upperExc, 0); + NumericUtils.longToSortableBytes(upperApprox, upperExc, 0); final ApproximatedQuery phase1 = new ApproximatedQuery(lowerInc, lowerExc, upperInc, upperExc); @@ -129,9 +121,8 @@ public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOExcepti if (values.getDocCount() == reader.maxDoc()) { final byte[] fieldPackedLower = values.getMinPackedValue(); final byte[] fieldPackedUpper = values.getMaxPackedValue(); - // equals?? - if (Arrays.compareUnsigned(lowerExc, 0, Long.BYTES, fieldPackedLower, 0, Long.BYTES) <= 0 - && Arrays.compareUnsigned(upperExc, 0, Long.BYTES, fieldPackedUpper, 0, Long.BYTES) >= 0) { + if (Arrays.compareUnsigned(lowerExc, 0, Long.BYTES, fieldPackedLower, 0, Long.BYTES) < 0 + && Arrays.compareUnsigned(upperExc, 0, Long.BYTES, fieldPackedUpper, 0, Long.BYTES) > 0) { return new ScorerSupplier() { @Override public Scorer get(long leadCost) { @@ -159,20 +150,32 @@ public long cost() { approx.set(0, reader.maxDoc()); int[] cost = new int[] { reader.maxDoc() }; phase1.execute(values, result, approx, cost); - exactIterator = new BitSetIterator(result, cost[0]);; - approxDISI = new BitSetIterator(approx, cost[0]); + exactIterator = new BitSetIterator(result, cost[0]); + approxDISI = phase1.hasApproximated ? new BitSetIterator(approx, cost[0]) : null; } else { DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values, field); DocIdSetBuilder approximation = new DocIdSetBuilder(reader.maxDoc(), values, field); phase1.execute(values, result, approximation); exactIterator = result.build().iterator(); - approxDISI = approximation.build().iterator(); + approxDISI = phase1.hasApproximated ? approximation.build().iterator() : null; } - final SortedNumericDocValues docValues = reader.getSortedNumericDocValues(field); - final long upperLong = toExactPrecision(upper); - final long lowerLong = toExactPrecision(lower); + if (approxDISI == null) { + return new ScorerSupplier() { + + @Override + public Scorer get(long leadCost) { + return new ConstantScoreScorer(weight, score(), scoreMode, exactIterator); + } + @Override + public long cost() { + return exactIterator.cost(); + } + }; + } + + final SortedNumericDocValues docValues = reader.getSortedNumericDocValues(field); final TwoPhaseIterator twoPhaseIterator = new TwoPhaseIterator(approxDISI) { @Override @@ -194,7 +197,7 @@ private boolean matches(int docId) throws IOException { int count = docValues.docValueCount(); for (int i = 0; i < count; i++) { final long value = docValues.nextValue(); - if (value <= upperLong && value >= lowerLong) { + if (value <= upperExact && value >= lowerExact) { return true; } } @@ -238,18 +241,19 @@ public boolean isCacheable(LeafReaderContext ctx) { }; } - public String getField() { return field; } @Override public final int hashCode() { - int hash = classHash(); - hash = 31 * hash + field.hashCode(); - hash = 31 * hash + Long.hashCode(toExactPrecision(upper)); - hash = 31 * hash + Long.hashCode(toExactPrecision(lower)); - return hash; + int hash = classHash(); + hash = 31 * hash + field.hashCode(); + hash = 31 * hash + Long.hashCode(upperExact); + hash = 31 * hash + Long.hashCode(lowerExact); + hash = 31 * hash + Long.hashCode(upperApprox); + hash = 31 * hash + Long.hashCode(lowerApprox); + return hash; } @Override @@ -258,9 +262,10 @@ public final boolean equals(Object o) { equalsTo(getClass().cast(o)); } - private boolean equalsTo(TwoPhaseDateRangeQuery other) { + private boolean equalsTo(TwoPhaseLongRangeQuery other) { return Objects.equals(field, other.field) && - Objects.equals(lower, other.lower) && Objects.equals(upper, other.upper); + upperExact == other.upperExact && lowerExact == other.lowerExact && + upperApprox == other.upperApprox && lowerApprox == other.lowerApprox; } @Override @@ -271,9 +276,9 @@ public final String toString(String field) { sb.append(':'); } sb.append('['); - sb.append(toExactPrecision(lower)); + sb.append(lowerExact); sb.append(" TO "); - sb.append(toExactPrecision(upper)); + sb.append(upperExact); sb.append(']'); return sb.toString(); } @@ -283,6 +288,7 @@ private static class ApproximatedQuery { private final byte[] lowerPointExc; private final byte[] upperPointInc; private final byte[] upperPointExc; + private boolean hasApproximated; private long cost; ApproximatedQuery(byte[] lowerPointInc, byte[] lowerPointExc, byte[] upperPointInc, byte[] upperPointExc) { @@ -336,6 +342,8 @@ public void visit(int docID, byte[] packedValue) { adderApprox.add(docID); if (addToExact(packedValue)) { adderExact.add(docID); + } else { + hasApproximated = true; } } } @@ -349,6 +357,8 @@ public void visit(DocIdSetIterator iterator, byte[] packedValue) throws IOExcept adderApprox.add(docID); if (addExact) { adderExact.add(docID); + } else { + hasApproximated = true; } } } @@ -377,6 +387,8 @@ public void visit(int docID, byte[] packedValue) { exact.clear(docID); if (matches(packedValue) == false) { approx.clear(docID); + } else { + hasApproximated = true; } } } @@ -391,6 +403,8 @@ public void visit(DocIdSetIterator iterator, byte[] packedValue) throws IOExcept exact.clear(docID); if (matches == false) { approx.clear(docID); + } else { + hasApproximated = true; } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index c395a13186250..e2605c5cba867 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -24,7 +24,9 @@ import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.PointValues; -import org.apache.lucene.queries.TwoPhaseDateRangeQuery; +import org.apache.lucene.queries.TwoPhaseLongDistanceFeatureQuery; +import org.apache.lucene.queries.TwoPhaseLongRangeQuery; +import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.Query; @@ -82,29 +84,28 @@ public long convert(Instant instant) { return instant.toEpochMilli(); } - @Override - public long convertApprox(Instant instant) { - return instant.getEpochSecond(); + public long convert(TimeValue timeValue) { + return timeValue.getMillis(); } @Override - public Instant toInstant(long value) { - return Instant.ofEpochMilli(value); + public long convertApprox(long value) { + return value / 1000L; } @Override - public Instant toInstantFromApprox(long value) { - return Instant.ofEpochSecond(value); + public long convertExact(long value) { + return value * 1000L; } @Override - public long parsePointAsMillis(byte[] value) { - return LongPoint.decodeDimension(value, 0); + public Instant toInstant(long value) { + return Instant.ofEpochMilli(value); } @Override - protected Query distanceFeatureQuery(String field, float boost, long origin, TimeValue pivot) { - return LongPoint.newDistanceFeatureQuery(field, boost, origin, pivot.getMillis()); + public long parsePointAsMillis(byte[] value) { + return LongPoint.decodeDimension(value, 0); } }, NANOSECONDS(DATE_NANOS_CONTENT_TYPE, NumericType.DATE_NANOSECONDS) { @@ -113,19 +114,23 @@ public long convert(Instant instant) { return toLong(instant); } + public long convert(TimeValue timeValue) { + return timeValue.getNanos(); + } + @Override - public long convertApprox(Instant instant) { - return instant.getEpochSecond(); + public long convertApprox(long value) { + return value / 1000000000L; } @Override - public Instant toInstant(long value) { - return DateUtils.toInstant(value); + public long convertExact(long value) { + return value * 1000000000L; } @Override - public Instant toInstantFromApprox(long value) { - return Instant.ofEpochSecond(value); + public Instant toInstant(long value) { + return DateUtils.toInstant(value); } @Override @@ -133,10 +138,6 @@ public long parsePointAsMillis(byte[] value) { return DateUtils.toMilliSeconds(LongPoint.decodeDimension(value, 0)); } - @Override - protected Query distanceFeatureQuery(String field, float boost, long origin, TimeValue pivot) { - return LongPoint.newDistanceFeatureQuery(field, boost, origin, pivot.getNanos()); - } }; private final String type; @@ -161,9 +162,9 @@ NumericType numericType() { public abstract long convert(Instant instant); /** - * Convert an {@linkplain Instant} into a long value in the approximated resolution. + * Convert an {@linkplain TimeValue} into a long value in this resolution. */ - public abstract long convertApprox(Instant instant); + public abstract long convert(TimeValue timeValue); /** * Convert a long value in this resolution into an instant. @@ -171,9 +172,14 @@ NumericType numericType() { public abstract Instant toInstant(long value); /** - * Convert a long value in this resolution into an instant. + * Convert an exact long value into a long value in the approximated resolution. */ - public abstract Instant toInstantFromApprox(long value); + public abstract long convertApprox(long value); + + /** + * Convert an exact approximated value into a long value in the exact resolution. + */ + public abstract long convertExact(long value); /** * Decode the points representation of this field as milliseconds. @@ -185,7 +191,7 @@ NumericType numericType() { */ public void index(ParseContext.Document doc, String name, long value, boolean hasDocValues) { if (hasDocValues) { - doc.add(new LongPoint(name, convertApprox(toInstant(value)))); + doc.add(new LongPoint(name, convertApprox(value))); } else { doc.add(new LongPoint(name, value)); } @@ -196,24 +202,13 @@ public void index(ParseContext.Document doc, String name, long value, boolean ha */ public Query rangeQuery(String field, long min, long max, boolean hasDocValues) { if (hasDocValues) { - Instant minInstant = toInstant(min); - Instant maxInstant = toInstant(max); - if ((min == Long.MIN_VALUE || min == convert(toInstantFromApprox(convertApprox(minInstant)))) && - (max == Long.MAX_VALUE || max == convert(toInstantFromApprox(convertApprox(maxInstant))))) { - return LongPoint.newRangeQuery(field, convertApprox(minInstant), convertApprox(maxInstant)); + long minApprox = convertApprox(min); + long maxApprox = convertApprox(max); + if ((min == Long.MIN_VALUE || min == convertExact(convertApprox(minApprox))) && + (max == Long.MAX_VALUE || max + 1 == convertExact(convertApprox(maxApprox))) ) { + return LongPoint.newRangeQuery(field, minApprox, maxApprox); } else { - return new TwoPhaseDateRangeQuery(field, minInstant, maxInstant) { - - @Override - protected long toApproxPrecision(Instant instant) { - return convertApprox(instant); - } - - @Override - protected long toExactPrecision(Instant instant) { - return convert(instant); - } - }; + return new TwoPhaseLongRangeQuery(field, min, max, minApprox, maxApprox); } } else { return LongPoint.newRangeQuery(field, min, max); @@ -229,7 +224,23 @@ public static Resolution ofOrdinal(int ord) { throw new IllegalArgumentException("unknown resolution ordinal [" + ord + "]"); } - protected abstract Query distanceFeatureQuery(String field, float boost, long origin, TimeValue pivot); + protected Query distanceFeatureQuery(String field, float boost, long origin, TimeValue pivot, boolean hasDocValues) { + if (hasDocValues) { + long originApprox = convertApprox(origin); + Query query = new TwoPhaseLongDistanceFeatureQuery(field, origin, convert(pivot), originApprox) { + @Override + public long convertDistance(long distanceExact) { + return convertApprox(distanceExact); + } + }; + if (boost != 1f) { + query = new BoostQuery(query, boost); + } + return query; + } else { + return LongPoint.newDistanceFeatureQuery(field, boost, origin, pivot.getNanos()); + } + } } private static DateFieldMapper toType(FieldMapper in) { @@ -491,7 +502,7 @@ public static long parseToLong( public Query distanceFeatureQuery(Object origin, String pivot, float boost, QueryShardContext context) { long originLong = parseToLong(origin, true, null, null, context::nowInMillis); TimeValue pivotTime = TimeValue.parseTimeValue(pivot, "distance_feature.pivot"); - return resolution.distanceFeatureQuery(name(), boost, originLong, pivotTime); + return resolution.distanceFeatureQuery(name(), boost, originLong, pivotTime, hasDocValues()); } @Override @@ -536,11 +547,11 @@ public Relation isFieldWithinQuery(IndexReader reader, long approxFromInclusive; long approxToInclusive; if (hasDocValues()) { - approxFromInclusive = resolution.convertApprox(resolution.toInstant(fromInclusive)); - approxToInclusive = resolution.convertApprox(resolution.toInstant(toInclusive)); - if (minValue >= approxFromInclusive + 1 && maxValue <= approxToInclusive - 1) { + approxFromInclusive = resolution.convertApprox(fromInclusive); + approxToInclusive = resolution.convertApprox(toInclusive); + if (minValue > approxFromInclusive + 1 && maxValue <= approxToInclusive) { return Relation.WITHIN; - } else if (maxValue < approxFromInclusive || minValue > approxToInclusive) { + } else if (maxValue < approxFromInclusive - 1 || minValue > approxToInclusive) { return Relation.DISJOINT; } else { return Relation.INTERSECTS; diff --git a/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQueryTests.java b/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQueryTests.java new file mode 100644 index 0000000000000..ca273c6bcc05e --- /dev/null +++ b/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQueryTests.java @@ -0,0 +1,117 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lucene.queries; + +import com.carrotsearch.randomizedtesting.generators.RandomNumbers; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.SerialMergeScheduler; +import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.LuceneTestCase; + +import java.time.Instant; + +public class TwoPhaseLongDistanceFeatureQueryTests extends LuceneTestCase { + + public void testIndexAndQuerySmall() throws Exception { + dotestIndexAndQuery(random().nextInt(10)); + } + + public void testIndexAndQueryMedium() throws Exception { + dotestIndexAndQuery(random().nextInt(1000)); + } + + public void testIndexAndQueryBig() throws Exception { + dotestIndexAndQuery(random().nextInt(10000)); + } + + public void dotestIndexAndQuery(int numPoints) throws Exception { + IndexWriterConfig iwc = newIndexWriterConfig(); + // Else seeds may not reproduce: + iwc.setMergeScheduler(new SerialMergeScheduler()); + // Else we can get O(N^2) merging: + iwc.setMaxBufferedDocs(10); + Directory dir = newDirectory(); + // RandomIndexWriter is too slow here: + IndexWriter w = new IndexWriter(dir, iwc); + for (int id = 0; id < numPoints; id++) { + Document doc = new Document(); + Instant instant = randomInstant(); + doc.add(new LongPoint("exact", instant.toEpochMilli())); + doc.add(new SortedNumericDocValuesField("exact", instant.toEpochMilli())); + doc.add(new LongPoint("approx", instant.getEpochSecond())); + doc.add(new SortedNumericDocValuesField("approx", instant.toEpochMilli())); + w.addDocument(doc); + } + + if (random().nextBoolean()) { + w.forceMerge(1); + } + final IndexReader r = DirectoryReader.open(w); + w.close(); + + IndexSearcher s = newSearcher(r); + for ( int i = 0; i < 100; i++) { + Instant origin = randomInstant(); + long distance = randomLongBetween(0, 10000); + + Query q1 = LongPoint.newDistanceFeatureQuery("exact", 2f, origin.toEpochMilli(), distance); + Query q2 = new TwoPhaseLongDistanceFeatureQuery("approx", origin.toEpochMilli(), distance, origin.getEpochSecond()) { + @Override + public long convertDistance(long distanceExact) { + return distanceExact/1000; + } + }; + q2 = new BoostQuery(q2, 2f); + assertEquals(s.count(q1), s.count(q2)); + TopDocs topDocs1 = s.search(q1, 10); + TopDocs topDocs2 = s.search(q2, 10); + assertEquals(topDocs1.totalHits, topDocs1.totalHits); + assertEquals(topDocs1.scoreDocs.length, topDocs2.scoreDocs.length); + for (int j = 0; j < topDocs1.scoreDocs.length; j++) { + assertEquals(topDocs1.scoreDocs[j].doc, topDocs2.scoreDocs[j].doc); + assertEquals(topDocs1.scoreDocs[j].score, topDocs2.scoreDocs[j].score, 0.0); + } + } + + IOUtils.close(r, dir); + } + + /** + * @return a random instant between 1970 and ca 2065 + */ + protected Instant randomInstant() { + return Instant.ofEpochMilli(randomLongBetween(0, 3000L)); + } + + public static long randomLongBetween(long min, long max) { + return RandomNumbers.randomLongBetween(random(), min, max); + } +} diff --git a/server/src/test/java/org/apache/lucene/queries/TwoPhaseDateTests.java b/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongRangeQueryTests.java similarity index 69% rename from server/src/test/java/org/apache/lucene/queries/TwoPhaseDateTests.java rename to server/src/test/java/org/apache/lucene/queries/TwoPhaseLongRangeQueryTests.java index 91a5334b433c4..df1c23567e7d6 100644 --- a/server/src/test/java/org/apache/lucene/queries/TwoPhaseDateTests.java +++ b/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongRangeQueryTests.java @@ -21,9 +21,8 @@ import com.carrotsearch.randomizedtesting.generators.RandomNumbers; import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; import org.apache.lucene.document.LongPoint; -import org.apache.lucene.document.TwoPhaseDatePointMillis; +import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; @@ -37,9 +36,21 @@ import java.time.Instant; -public class TwoPhaseDateTests extends LuceneTestCase { +public class TwoPhaseLongRangeQueryTests extends LuceneTestCase { - public void testIndexMillis() throws Exception { + public void testIndexAndQuerySmall() throws Exception { + dotestIndexAndQuery(random().nextInt(10)); + } + + public void testIndexAndQueryMedium() throws Exception { + dotestIndexAndQuery(random().nextInt(1000)); + } + + public void testIndexAndQueryBig() throws Exception { + dotestIndexAndQuery(random().nextInt(10000)); + } + + public void dotestIndexAndQuery(int numPoints) throws Exception { IndexWriterConfig iwc = newIndexWriterConfig(); // Else seeds may not reproduce: iwc.setMergeScheduler(new SerialMergeScheduler()); @@ -47,16 +58,13 @@ public void testIndexMillis() throws Exception { iwc.setMaxBufferedDocs(10); Directory dir = newDirectory(); // RandomIndexWriter is too slow here: - int numPoints = random().nextInt(10000); IndexWriter w = new IndexWriter(dir, iwc); for (int id = 0; id < numPoints; id++) { Document doc = new Document(); Instant instant = randomInstant(); doc.add(new LongPoint("exact", instant.toEpochMilli())); - Field[] fields = TwoPhaseDatePointMillis.createIndexableFields("approx", instant); - for (int i =0; i < fields.length; i++) { - doc.add(fields[i]); - } + doc.add(new LongPoint("approx", instant.getEpochSecond())); + doc.add(new SortedNumericDocValuesField("approx", instant.toEpochMilli())); w.addDocument(doc); } @@ -70,15 +78,14 @@ public void testIndexMillis() throws Exception { for ( int i = 0; i < 100; i++) { Instant instant1 = randomInstant(); Instant instant2 = randomInstant(); - Query q1; - Query q2; - if (instant1.toEpochMilli() >= instant2.toEpochMilli()) { - q1 = LongPoint.newRangeQuery("exact", instant2.toEpochMilli(), instant1.toEpochMilli()); - q2 = TwoPhaseDatePointMillis.newRangeQuery("approx", instant2, instant1); - } else { - q1 = LongPoint.newRangeQuery("exact", instant1.toEpochMilli(), instant2.toEpochMilli()); - q2 = TwoPhaseDatePointMillis.newRangeQuery("approx", instant1, instant2); + if (instant1.toEpochMilli() > instant2.toEpochMilli()) { + Instant tmp = instant1; + instant1 = instant2; + instant2 = tmp; } + Query q1 = LongPoint.newRangeQuery("exact", instant2.toEpochMilli(), instant1.toEpochMilli()); + Query q2 = new TwoPhaseLongRangeQuery("approx", instant2.toEpochMilli(), instant1.toEpochMilli(), + instant2.getEpochSecond(), instant1.getEpochSecond()); assertEquals(s.count(q1), s.count(q2)); } @@ -90,7 +97,7 @@ public void testIndexMillis() throws Exception { */ protected Instant randomInstant() { //return Instant.ofEpochSecond(randomLongBetween(0, 3000000000L), randomLongBetween(0, 999999999)); - return Instant.ofEpochSecond(randomLongBetween(0, 30000L), randomLongBetween(0, 999999999)); + return Instant.ofEpochMilli(randomLongBetween(0, 3000L)); } public static long randomLongBetween(long min, long max) { From 10f3aee6824c28376224539779bc629659664a46 Mon Sep 17 00:00:00 2001 From: iverase Date: Fri, 9 Oct 2020 12:21:18 +0200 Subject: [PATCH 05/18] iter --- .../index/mapper/DateFieldMapper.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index e2605c5cba867..d2ec3755c9bee 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -204,8 +204,8 @@ public Query rangeQuery(String field, long min, long max, boolean hasDocValues) if (hasDocValues) { long minApprox = convertApprox(min); long maxApprox = convertApprox(max); - if ((min == Long.MIN_VALUE || min == convertExact(convertApprox(minApprox))) && - (max == Long.MAX_VALUE || max + 1 == convertExact(convertApprox(maxApprox))) ) { + if ((min == Long.MIN_VALUE || min == convertExact(minApprox)) && + (max == Long.MAX_VALUE || max == convertExact(maxApprox))) { // +1? return LongPoint.newRangeQuery(field, minApprox, maxApprox); } else { return new TwoPhaseLongRangeQuery(field, min, max, minApprox, maxApprox); @@ -420,6 +420,19 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower : forcedDateParser; return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> { + // if l <= min value on the index or u >= max value on the index, we should change it to Long.MAX_VALUE / Long.MIN_VALUE + try { + long minValue = LongPoint.decodeDimension(PointValues.getMinPackedValue(context.getIndexReader(), name()), 0); + if (l < resolution.convertExact(minValue)) { + l = Long.MIN_VALUE; + } + long maxValue = LongPoint.decodeDimension(PointValues.getMaxPackedValue(context.getIndexReader(), name()), 0); + if (u > resolution.convertExact(maxValue)) { + u = Long.MAX_VALUE; + } + } catch (IOException e) { + // let's ignore for now + } Query query = resolution.rangeQuery(name(), l, u, hasDocValues()); if (hasDocValues()) { Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); From da6895dcb33a9752935fa427b6b0a6c2cb991318 Mon Sep 17 00:00:00 2001 From: iverase Date: Fri, 9 Oct 2020 14:19:05 +0200 Subject: [PATCH 06/18] iter --- .../elasticsearch/index/mapper/DateFieldMapper.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index d2ec3755c9bee..f49034f639fc9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -205,7 +205,7 @@ public Query rangeQuery(String field, long min, long max, boolean hasDocValues) long minApprox = convertApprox(min); long maxApprox = convertApprox(max); if ((min == Long.MIN_VALUE || min == convertExact(minApprox)) && - (max == Long.MAX_VALUE || max == convertExact(maxApprox))) { // +1? + (max == Long.MAX_VALUE || max + 1 == convertExact(maxApprox))) { // + 1, should not be included return LongPoint.newRangeQuery(field, minApprox, maxApprox); } else { return new TwoPhaseLongRangeQuery(field, min, max, minApprox, maxApprox); @@ -431,6 +431,7 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower u = Long.MAX_VALUE; } } catch (IOException e) { + throw new IllegalArgumentException(e); // let's ignore for now } Query query = resolution.rangeQuery(name(), l, u, hasDocValues()); @@ -523,11 +524,6 @@ public Relation isFieldWithinQuery(IndexReader reader, Object from, Object to, boolean includeLower, boolean includeUpper, ZoneId timeZone, DateMathParser dateParser, QueryRewriteContext context) throws IOException { - if (PointValues.size(reader, name()) == 0) { - // no points, so nothing matches - return Relation.DISJOINT; - } - if (dateParser == null) { dateParser = this.dateMathParser; } @@ -554,6 +550,11 @@ public Relation isFieldWithinQuery(IndexReader reader, } } + if (PointValues.size(reader, name()) == 0) { + // no points, so nothing matches + return Relation.DISJOINT; + } + long minValue = LongPoint.decodeDimension(PointValues.getMinPackedValue(reader, name()), 0); long maxValue = LongPoint.decodeDimension(PointValues.getMaxPackedValue(reader, name()), 0); From 720b5da155273be75fd97e28d86bc064297f528f Mon Sep 17 00:00:00 2001 From: iverase Date: Fri, 9 Oct 2020 16:37:07 +0200 Subject: [PATCH 07/18] iter --- .../index/mapper/DateFieldMapper.java | 125 +++++++++--------- 1 file changed, 59 insertions(+), 66 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index f49034f639fc9..88de2e3ce3147 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -89,12 +89,12 @@ public long convert(TimeValue timeValue) { } @Override - public long convertApprox(long value) { + public long convertToIndex(long value) { return value / 1000L; } @Override - public long convertExact(long value) { + public long convertToDocValue(long value) { return value * 1000L; } @@ -119,12 +119,12 @@ public long convert(TimeValue timeValue) { } @Override - public long convertApprox(long value) { + public long convertToIndex(long value) { return value / 1000000000L; } @Override - public long convertExact(long value) { + public long convertToDocValue(long value) { return value * 1000000000L; } @@ -172,26 +172,35 @@ NumericType numericType() { public abstract Instant toInstant(long value); /** - * Convert an exact long value into a long value in the approximated resolution. + * Decode the points representation of this field as milliseconds. */ - public abstract long convertApprox(long value); + public abstract long parsePointAsMillis(byte[] value); + + public static Resolution ofOrdinal(int ord) { + for (Resolution resolution : values()) { + if (ord == resolution.ordinal()) { + return resolution; + } + } + throw new IllegalArgumentException("unknown resolution ordinal [" + ord + "]"); + } /** - * Convert an exact approximated value into a long value in the exact resolution. + * Convert the given long value in this resolution into the index resolution */ - public abstract long convertExact(long value); + public abstract long convertToIndex(long value); /** - * Decode the points representation of this field as milliseconds. + * Convert the given long value in the index resolution into this resolution */ - public abstract long parsePointAsMillis(byte[] value); + public abstract long convertToDocValue(long value); /** - * Add index fields to doc + * Add index fields in the provided doc */ public void index(ParseContext.Document doc, String name, long value, boolean hasDocValues) { if (hasDocValues) { - doc.add(new LongPoint(name, convertApprox(value))); + doc.add(new LongPoint(name, convertToIndex(value))); } else { doc.add(new LongPoint(name, value)); } @@ -200,12 +209,13 @@ public void index(ParseContext.Document doc, String name, long value, boolean ha /** * Build range query */ - public Query rangeQuery(String field, long min, long max, boolean hasDocValues) { + protected Query rangeQuery(String field, long min, long max, boolean hasDocValues) { if (hasDocValues) { - long minApprox = convertApprox(min); - long maxApprox = convertApprox(max); - if ((min == Long.MIN_VALUE || min == convertExact(minApprox)) && - (max == Long.MAX_VALUE || max + 1 == convertExact(maxApprox))) { // + 1, should not be included + final long minApprox = convertToIndex(min); + final long maxApprox = convertToIndex(max); + // I have my doubts here, probably the upper bound must be unbounded?? + if ((min == Long.MIN_VALUE || min == convertToDocValue(minApprox)) && + (max == Long.MAX_VALUE || max + 1 == convertToDocValue(maxApprox))) { // + 1, should not be included return LongPoint.newRangeQuery(field, minApprox, maxApprox); } else { return new TwoPhaseLongRangeQuery(field, min, max, minApprox, maxApprox); @@ -215,22 +225,16 @@ public Query rangeQuery(String field, long min, long max, boolean hasDocValues) } } - public static Resolution ofOrdinal(int ord) { - for (Resolution resolution : values()) { - if (ord == resolution.ordinal()) { - return resolution; - } - } - throw new IllegalArgumentException("unknown resolution ordinal [" + ord + "]"); - } - + /** + * Build Distance feature query + */ protected Query distanceFeatureQuery(String field, float boost, long origin, TimeValue pivot, boolean hasDocValues) { if (hasDocValues) { - long originApprox = convertApprox(origin); + long originApprox = convertToIndex(origin); Query query = new TwoPhaseLongDistanceFeatureQuery(field, origin, convert(pivot), originApprox) { @Override public long convertDistance(long distanceExact) { - return convertApprox(distanceExact); + return convertToIndex(distanceExact); } }; if (boost != 1f) { @@ -241,6 +245,28 @@ public long convertDistance(long distanceExact) { return LongPoint.newDistanceFeatureQuery(field, boost, origin, pivot.getNanos()); } } + + protected MappedFieldType.Relation isWithin(long fromInclusive, long toInclusive, long minValue, long maxValue, boolean hasDocValues) { + if (hasDocValues) { + final long approxFromInclusive = convertToIndex(fromInclusive); + final long approxToInclusive = convertToIndex(toInclusive); + if (minValue > approxFromInclusive + 1 && maxValue <= approxToInclusive) { + return MappedFieldType.Relation.WITHIN; + } else if (maxValue < approxFromInclusive - 1 || minValue > approxToInclusive) { + return MappedFieldType.Relation.DISJOINT; + } else { + return MappedFieldType.Relation.INTERSECTS; + } + } else { + if (minValue >= fromInclusive && maxValue <= toInclusive) { + return MappedFieldType.Relation.WITHIN; + } else if (maxValue < fromInclusive || minValue > toInclusive) { + return MappedFieldType.Relation.DISJOINT; + } else { + return MappedFieldType.Relation.INTERSECTS; + } + } + } } private static DateFieldMapper toType(FieldMapper in) { @@ -420,20 +446,8 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower : forcedDateParser; return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> { - // if l <= min value on the index or u >= max value on the index, we should change it to Long.MAX_VALUE / Long.MIN_VALUE - try { - long minValue = LongPoint.decodeDimension(PointValues.getMinPackedValue(context.getIndexReader(), name()), 0); - if (l < resolution.convertExact(minValue)) { - l = Long.MIN_VALUE; - } - long maxValue = LongPoint.decodeDimension(PointValues.getMaxPackedValue(context.getIndexReader(), name()), 0); - if (u > resolution.convertExact(maxValue)) { - u = Long.MAX_VALUE; - } - } catch (IOException e) { - throw new IllegalArgumentException(e); - // let's ignore for now - } + // if l < min value on the index or u > max value on the index, we should change it to Long.MAX_VALUE / Long.MIN_VALUE. + // This would help in choosing to use a cheaper query?? Query query = resolution.rangeQuery(name(), l, u, hasDocValues()); if (hasDocValues()) { Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); @@ -555,30 +569,9 @@ public Relation isFieldWithinQuery(IndexReader reader, return Relation.DISJOINT; } - long minValue = LongPoint.decodeDimension(PointValues.getMinPackedValue(reader, name()), 0); - long maxValue = LongPoint.decodeDimension(PointValues.getMaxPackedValue(reader, name()), 0); - - long approxFromInclusive; - long approxToInclusive; - if (hasDocValues()) { - approxFromInclusive = resolution.convertApprox(fromInclusive); - approxToInclusive = resolution.convertApprox(toInclusive); - if (minValue > approxFromInclusive + 1 && maxValue <= approxToInclusive) { - return Relation.WITHIN; - } else if (maxValue < approxFromInclusive - 1 || minValue > approxToInclusive) { - return Relation.DISJOINT; - } else { - return Relation.INTERSECTS; - } - } else { - if (minValue >= fromInclusive && maxValue <= toInclusive) { - return Relation.WITHIN; - } else if (maxValue < fromInclusive || minValue > toInclusive) { - return Relation.DISJOINT; - } else { - return Relation.INTERSECTS; - } - } + final long minValue = LongPoint.decodeDimension(PointValues.getMinPackedValue(reader, name()), 0); + final long maxValue = LongPoint.decodeDimension(PointValues.getMaxPackedValue(reader, name()), 0); + return resolution.isWithin(fromInclusive, toInclusive, minValue, maxValue, hasDocValues()); } @Override From 0a44f0e6de7e3e3add2e7341b09d81fec0162c48 Mon Sep 17 00:00:00 2001 From: iverase Date: Mon, 12 Oct 2020 09:01:22 +0200 Subject: [PATCH 08/18] RangeQueryBuilder --- .../index/mapper/DateFieldMapper.java | 107 ++++++++++++++---- .../index/mapper/MappedFieldType.java | 38 ++++++- .../index/query/RangeQueryBuilder.java | 56 ++++++--- .../index/mapper/DateFieldTypeTests.java | 26 +++-- .../index/query/RangeQueryBuilderTests.java | 29 ++++- .../index/query/RangeQueryRewriteTests.java | 9 +- 6 files changed, 205 insertions(+), 60 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 88de2e3ce3147..6ed6693ef3b3a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -246,21 +246,63 @@ public long convertDistance(long distanceExact) { } } - protected MappedFieldType.Relation isWithin(long fromInclusive, long toInclusive, long minValue, long maxValue, boolean hasDocValues) { +// protected MappedFieldType.Relation isWithin(long fromInclusive, long toInclusive, long minValue, long maxValue, boolean hasDocValues) { +// if (hasDocValues) { +// final long approxFromInclusive = convertToIndex(fromInclusive); +// final long approxToInclusive = convertToIndex(toInclusive); +// if (minValue > approxFromInclusive + 1 && maxValue <= approxToInclusive) { +// return MappedFieldType.Relation.WITHIN; +// } else if (maxValue < approxFromInclusive - 1 || minValue > approxToInclusive) { +// return MappedFieldType.Relation.DISJOINT; +// } else { +// return MappedFieldType.Relation.INTERSECTS; +// } +// } else { +// if (minValue >= fromInclusive && maxValue <= toInclusive) { +// return MappedFieldType.Relation.WITHIN; +// } else if (maxValue < fromInclusive || minValue > toInclusive) { +// return MappedFieldType.Relation.DISJOINT; +// } else { +// return MappedFieldType.Relation.INTERSECTS; +// } +// } +// } + + protected MappedFieldType.Relation isToWithin(long toInclusive, long minValue, long maxValue, boolean hasDocValues) { if (hasDocValues) { - final long approxFromInclusive = convertToIndex(fromInclusive); final long approxToInclusive = convertToIndex(toInclusive); - if (minValue > approxFromInclusive + 1 && maxValue <= approxToInclusive) { + if (maxValue < approxToInclusive - 1) { + return MappedFieldType.Relation.WITHIN; + } else if (minValue > approxToInclusive + 1) { + return MappedFieldType.Relation.DISJOINT; + } else { + return MappedFieldType.Relation.INTERSECTS; + } + } else { + if (maxValue <= toInclusive) { + return MappedFieldType.Relation.WITHIN; + } else if (minValue > toInclusive) { + return MappedFieldType.Relation.DISJOINT; + } else { + return MappedFieldType.Relation.INTERSECTS; + } + } + } + + protected MappedFieldType.Relation isFromWithin(long fromInclusive, long minValue, long maxValue, boolean hasDocValues) { + if (hasDocValues) { + final long approxFromInclusive = convertToIndex(fromInclusive); + if (minValue >= approxFromInclusive) { return MappedFieldType.Relation.WITHIN; - } else if (maxValue < approxFromInclusive - 1 || minValue > approxToInclusive) { + } else if (maxValue < approxFromInclusive) { return MappedFieldType.Relation.DISJOINT; } else { return MappedFieldType.Relation.INTERSECTS; } } else { - if (minValue >= fromInclusive && maxValue <= toInclusive) { + if (minValue >= fromInclusive) { return MappedFieldType.Relation.WITHIN; - } else if (maxValue < fromInclusive || minValue > toInclusive) { + } else if (maxValue < fromInclusive) { return MappedFieldType.Relation.DISJOINT; } else { return MappedFieldType.Relation.INTERSECTS; @@ -439,11 +481,11 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower failIfNotIndexed(); if (relation == ShapeRelation.DISJOINT) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + - "] does not support DISJOINT ranges"); + "] does not support DISJOINT ranges"); } DateMathParser parser = forcedDateParser == null - ? dateMathParser - : forcedDateParser; + ? dateMathParser + : forcedDateParser; return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> { // if l < min value on the index or u > max value on the index, we should change it to Long.MAX_VALUE / Long.MIN_VALUE. @@ -534,10 +576,9 @@ public Query distanceFeatureQuery(Object origin, String pivot, float boost, Quer } @Override - public Relation isFieldWithinQuery(IndexReader reader, - Object from, Object to, boolean includeLower, boolean includeUpper, - ZoneId timeZone, DateMathParser dateParser, QueryRewriteContext context) throws IOException { - + public Relation isFieldMinWithinQuery(IndexReader reader, + Object from, boolean includeLower, + ZoneId timeZone, DateMathParser dateParser, QueryRewriteContext context) throws IOException { if (dateParser == null) { dateParser = this.dateMathParser; } @@ -553,6 +594,25 @@ public Relation isFieldWithinQuery(IndexReader reader, } } + if (PointValues.size(reader, name()) == 0) { + // no points, so nothing matches + return Relation.DISJOINT; + } + + long minValue = LongPoint.decodeDimension(PointValues.getMinPackedValue(reader, name()), 0); + long maxValue = LongPoint.decodeDimension(PointValues.getMaxPackedValue(reader, name()), 0); + + return resolution.isFromWithin(fromInclusive, minValue, maxValue, hasDocValues()); + } + + @Override + public Relation isFieldMaxWithinQuery(IndexReader reader, + Object to, boolean includeUpper, + ZoneId timeZone, DateMathParser dateParser, QueryRewriteContext context) throws IOException { + if (dateParser == null) { + dateParser = this.dateMathParser; + } + long toInclusive = Long.MAX_VALUE; if (to != null) { toInclusive = parseToLong(to, includeUpper, timeZone, dateParser, context::nowInMillis, resolution); @@ -569,9 +629,10 @@ public Relation isFieldWithinQuery(IndexReader reader, return Relation.DISJOINT; } - final long minValue = LongPoint.decodeDimension(PointValues.getMinPackedValue(reader, name()), 0); - final long maxValue = LongPoint.decodeDimension(PointValues.getMaxPackedValue(reader, name()), 0); - return resolution.isWithin(fromInclusive, toInclusive, minValue, maxValue, hasDocValues()); + long minValue = LongPoint.decodeDimension(PointValues.getMinPackedValue(reader, name()), 0); + long maxValue = LongPoint.decodeDimension(PointValues.getMaxPackedValue(reader, name()), 0); + + return resolution.isToWithin(toInclusive, minValue, maxValue, hasDocValues()); } @Override @@ -627,13 +688,13 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { private final Version indexCreatedVersion; private DateFieldMapper( - String simpleName, - MappedFieldType mappedFieldType, - MultiFields multiFields, - CopyTo copyTo, - Long nullValue, - Resolution resolution, - Builder builder) { + String simpleName, + MappedFieldType mappedFieldType, + MultiFields multiFields, + CopyTo copyTo, + Long nullValue, + Resolution resolution, + Builder builder) { super(simpleName, mappedFieldType, multiFields, copyTo); this.store = builder.store.getValue(); this.indexed = builder.index.getValue(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 60574cd3017f4..ca3c597a53f2f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -305,15 +305,45 @@ public enum Relation { DISJOINT } + /** Return whether all values of the given {@link IndexReader} are within the min range, + * outside the min range or cross the min range. The default implementation returns + * {@link Relation#INTERSECTS}, which is always fine to return when there is + * no way to check whether values are actually within bounds. */ + public Relation isFieldMaxWithinQuery( + IndexReader reader, Object to, boolean includeUpper, + ZoneId timeZone, DateMathParser dateMathParser, QueryRewriteContext context) throws IOException { + return Relation.INTERSECTS; + } + /** Return whether all values of the given {@link IndexReader} are within the range, * outside the range or cross the range. The default implementation returns * {@link Relation#INTERSECTS}, which is always fine to return when there is * no way to check whether values are actually within bounds. */ public Relation isFieldWithinQuery( - IndexReader reader, - Object from, Object to, - boolean includeLower, boolean includeUpper, - ZoneId timeZone, DateMathParser dateMathParser, QueryRewriteContext context) throws IOException { + IndexReader reader, + Object from, Object to, + boolean includeLower, boolean includeUpper, + ZoneId timeZone, DateMathParser dateMathParser, QueryRewriteContext context) throws IOException { + + final Relation minRelation = isFieldMinWithinQuery(reader, from, includeLower, timeZone, dateMathParser, context); + final Relation maxRelation = isFieldMaxWithinQuery(reader, to, includeUpper, timeZone, dateMathParser, context); + if (minRelation == Relation.DISJOINT || maxRelation == Relation.DISJOINT) { + return Relation.DISJOINT; + } else if (minRelation == Relation.WITHIN && maxRelation == Relation.WITHIN) { + return Relation.WITHIN; + } else { + return Relation.INTERSECTS; + } + } + + /** Return whether all values of the given {@link IndexReader} are within the max range, + * outside the max range or cross the max range. The default implementation returns + * {@link Relation#INTERSECTS}, which is always fine to return when there is + * no way to check whether values are actually within bounds. */ + public Relation isFieldMinWithinQuery( + IndexReader reader, + Object from, boolean includeLower, + ZoneId timeZone, DateMathParser dateMathParser, QueryRewriteContext context) throws IOException { return Relation.INTERSECTS; } diff --git a/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java index 2c3cc5797fca3..162ec3aed6bcf 100644 --- a/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.query; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; @@ -427,7 +428,7 @@ public String getWriteableName() { } // Overridable for testing only - protected MappedFieldType.Relation getRelation(QueryRewriteContext queryRewriteContext) throws IOException { + protected MappedFieldType.Relation getMinRelation(QueryRewriteContext queryRewriteContext) throws IOException { QueryShardContext shardContext = queryRewriteContext.convertToShardContext(); if (shardContext != null) { final MappedFieldType fieldType = shardContext.getFieldType(fieldName); @@ -438,10 +439,30 @@ protected MappedFieldType.Relation getRelation(QueryRewriteContext queryRewriteC // No reader, this may happen e.g. for percolator queries. return MappedFieldType.Relation.INTERSECTS; } + DateMathParser dateMathParser = getForceDateParser(); + return fieldType.isFieldMinWithinQuery(shardContext.getIndexReader(), from, includeLower, + timeZone, dateMathParser, queryRewriteContext); + } + + // Not on the shard, we have no way to know what the relation is. + return MappedFieldType.Relation.INTERSECTS; + } + // Overridable for testing only + protected MappedFieldType.Relation getMaxRelation(QueryRewriteContext queryRewriteContext) throws IOException { + QueryShardContext shardContext = queryRewriteContext.convertToShardContext(); + if (shardContext != null) { + final MappedFieldType fieldType = shardContext.getFieldType(fieldName); + if (fieldType == null) { + return MappedFieldType.Relation.DISJOINT; + } + if (shardContext.getIndexReader() == null) { + // No reader, this may happen e.g. for percolator queries. + return MappedFieldType.Relation.INTERSECTS; + } DateMathParser dateMathParser = getForceDateParser(); - return fieldType.isFieldWithinQuery(shardContext.getIndexReader(), from, to, includeLower, - includeUpper, timeZone, dateMathParser, queryRewriteContext); + return fieldType.isFieldMaxWithinQuery(shardContext.getIndexReader(), to, includeUpper, + timeZone, dateMathParser, queryRewriteContext); } // Not on the shard, we have no way to know what the relation is. @@ -450,11 +471,9 @@ protected MappedFieldType.Relation getRelation(QueryRewriteContext queryRewriteC @Override protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException { - final MappedFieldType.Relation relation = getRelation(queryRewriteContext); - switch (relation) { - case DISJOINT: - return new MatchNoneQueryBuilder(); - case WITHIN: + final MappedFieldType.Relation minRelation = getMinRelation(queryRewriteContext); + final MappedFieldType.Relation maxRelation = getMaxRelation(queryRewriteContext); + if (minRelation == MappedFieldType.Relation.WITHIN && maxRelation == MappedFieldType.Relation.WITHIN) { if (from != null || to != null || format != null || timeZone != null) { RangeQueryBuilder newRangeQuery = new RangeQueryBuilder(fieldName); newRangeQuery.from(null); @@ -462,14 +481,23 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws newRangeQuery.format = null; newRangeQuery.timeZone = null; return newRangeQuery; - } else { - return this; } - case INTERSECTS: - return this; - default: - throw new AssertionError(); + } else if (minRelation == MappedFieldType.Relation.WITHIN || maxRelation == MappedFieldType.Relation.WITHIN) { + Object from = (minRelation == MappedFieldType.Relation.WITHIN) ? null : this.from; + Object to = (maxRelation == MappedFieldType.Relation.WITHIN) ? null : this.to; + if (from != this.from || to != this.from) { + RangeQueryBuilder newRangeQuery = new RangeQueryBuilder(fieldName); + newRangeQuery.from(from); + newRangeQuery.to(to); + newRangeQuery.format = format; + newRangeQuery.timeZone = timeZone; + return newRangeQuery; + } + } else if (minRelation == MappedFieldType.Relation.DISJOINT || maxRelation == MappedFieldType.Relation.DISJOINT) { + return new MatchNoneQueryBuilder(); } + return this; + } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index 95e09f229256c..46f159b8442a0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -85,11 +85,15 @@ public void isFieldWithinRangeTestCase(DateFieldType ft) throws IOException { Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null)); - Document doc = new Document(); - LongPoint field = new LongPoint("my_date", ft.parse("2015-10-12")); - doc.add(field); + //Document doc = new Document(); + ParseContext.Document doc = new ParseContext.Document(); + ft.resolution().index(doc, "my_date", ft.parse("2015-10-12"), true); + //LongPoint field = new LongPoint("my_date", ft.parse("2015-10-12")); + //doc.add(field); w.addDocument(doc); - field.setLongValue(ft.parse("2016-04-03")); + doc = new ParseContext.Document(); + ft.resolution().index(doc, "my_date", ft.parse("2016-04-03"), true); + //field.setLongValue(ft.parse("2016-04-03")); w.addDocument(doc); DirectoryReader reader = DirectoryReader.open(w); @@ -167,11 +171,11 @@ public void testTermQuery() { new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null); - MappedFieldType ft = new DateFieldType("field"); + DateFieldType ft = new DateFieldType("field"); String date = "2015-10-12T14:10:55"; long instant = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)).toInstant().toEpochMilli(); Query expected = new IndexOrDocValuesQuery( - LongPoint.newRangeQuery("field", instant, instant + 999), + ft.resolution().rangeQuery("field", instant, instant + 999, true), SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant + 999)); assertEquals(expected, ft.termQuery(date, context)); @@ -189,14 +193,14 @@ public void testRangeQuery() throws IOException { new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null); - MappedFieldType ft = new DateFieldType("field"); + DateFieldType ft = new DateFieldType("field"); String date1 = "2015-10-12T14:10:55"; String date2 = "2016-04-28T11:33:52"; long instant1 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date1)).toInstant().toEpochMilli(); long instant2 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date2)).toInstant().toEpochMilli() + 999; Query expected = new IndexOrDocValuesQuery( - LongPoint.newRangeQuery("field", instant1, instant2), + ft.resolution().rangeQuery("field", instant1, instant2, true), SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2)); assertEquals(expected, ft.rangeQuery(date1, date2, true, true, null, null, null, context).rewrite(new MultiReader())); @@ -204,7 +208,7 @@ BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry instant1 = nowInMillis; instant2 = instant1 + 100; expected = new DateRangeIncludingNowQuery(new IndexOrDocValuesQuery( - LongPoint.newRangeQuery("field", instant1, instant2), + ft.resolution().rangeQuery("field", instant1, instant2, true), SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2) )); assertEquals(expected, @@ -234,13 +238,13 @@ public void testRangeQueryWithIndexSort() { BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> 0L, null, null, () -> true, null); - MappedFieldType ft = new DateFieldType("field"); + DateFieldType ft = new DateFieldType("field"); String date1 = "2015-10-12T14:10:55"; String date2 = "2016-04-28T11:33:52"; long instant1 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date1)).toInstant().toEpochMilli(); long instant2 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date2)).toInstant().toEpochMilli() + 999; - Query pointQuery = LongPoint.newRangeQuery("field", instant1, instant2); + Query pointQuery = ft.resolution().rangeQuery("field", instant1, instant2, true); Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2); Query expected = new IndexSortSortedNumericDocValuesRangeQuery("field", instant1, instant2, new IndexOrDocValuesQuery(pointQuery, dvQuery)); diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java index 7df7eae86b55c..ab3d079e74641 100644 --- a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java @@ -393,7 +393,12 @@ public void testRewriteDateToMatchAll() throws IOException { String fieldName = DATE_FIELD_NAME; RangeQueryBuilder query = new RangeQueryBuilder(fieldName) { @Override - protected MappedFieldType.Relation getRelation(QueryRewriteContext queryRewriteContext) { + protected MappedFieldType.Relation getMinRelation(QueryRewriteContext queryRewriteContext) { + return Relation.WITHIN; + } + + @Override + protected MappedFieldType.Relation getMaxRelation(QueryRewriteContext queryRewriteContext) { return Relation.WITHIN; } }; @@ -428,7 +433,12 @@ public void testRewriteDateToMatchAllWithTimezoneAndFormat() throws IOException String fieldName = DATE_FIELD_NAME; RangeQueryBuilder query = new RangeQueryBuilder(fieldName) { @Override - protected MappedFieldType.Relation getRelation(QueryRewriteContext queryRewriteContext) { + protected MappedFieldType.Relation getMinRelation(QueryRewriteContext queryRewriteContext) { + return Relation.WITHIN; + } + + @Override + protected MappedFieldType.Relation getMaxRelation(QueryRewriteContext queryRewriteContext) { return Relation.WITHIN; } }; @@ -453,7 +463,7 @@ public void testRewriteDateToMatchNone() throws IOException { String fieldName = randomAlphaOfLengthBetween(1, 20); RangeQueryBuilder query = new RangeQueryBuilder(fieldName) { @Override - protected MappedFieldType.Relation getRelation(QueryRewriteContext queryRewriteContext) { + protected MappedFieldType.Relation getMinRelation(QueryRewriteContext queryRewriteContext) { return Relation.DISJOINT; } }; @@ -470,7 +480,12 @@ public void testRewriteDateToSame() throws IOException { String fieldName = randomAlphaOfLengthBetween(1, 20); RangeQueryBuilder query = new RangeQueryBuilder(fieldName) { @Override - protected MappedFieldType.Relation getRelation(QueryRewriteContext queryRewriteContext) { + protected MappedFieldType.Relation getMinRelation(QueryRewriteContext queryRewriteContext) { + return Relation.INTERSECTS; + } + + @Override + protected MappedFieldType.Relation getMaxRelation(QueryRewriteContext queryRewriteContext) { return Relation.INTERSECTS; } }; @@ -487,7 +502,11 @@ public void testRewriteOpenBoundsToSame() throws IOException { String fieldName = randomAlphaOfLengthBetween(1, 20); RangeQueryBuilder query = new RangeQueryBuilder(fieldName) { @Override - protected MappedFieldType.Relation getRelation(QueryRewriteContext queryRewriteContext) { + protected MappedFieldType.Relation getMinRelation(QueryRewriteContext queryRewriteContext) { + return Relation.INTERSECTS; + } + @Override + protected MappedFieldType.Relation getMaxRelation(QueryRewriteContext queryRewriteContext) { return Relation.INTERSECTS; } }; diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java index 263a9eb95942f..18c97fc708fd5 100644 --- a/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java @@ -43,7 +43,8 @@ public void testRewriteMissingField() throws Exception { null, null, indexService.mapperService(), null, null, xContentRegistry(), writableRegistry(), null, new IndexSearcher(reader), null, null, null, () -> true, null); RangeQueryBuilder range = new RangeQueryBuilder("foo"); - assertEquals(Relation.DISJOINT, range.getRelation(context)); + assertEquals(Relation.DISJOINT, range.getMinRelation(context)); + assertEquals(Relation.DISJOINT, range.getMaxRelation(context)); } public void testRewriteMissingReader() throws Exception { @@ -62,7 +63,8 @@ public void testRewriteMissingReader() throws Exception { null, null, null, null, null, () -> true, null); RangeQueryBuilder range = new RangeQueryBuilder("foo"); // can't make assumptions on a missing reader, so it must return INTERSECT - assertEquals(Relation.INTERSECTS, range.getRelation(context)); + assertEquals(Relation.INTERSECTS, range.getMinRelation(context)); + assertEquals(Relation.INTERSECTS, range.getMaxRelation(context)); } public void testRewriteEmptyReader() throws Exception { @@ -82,6 +84,7 @@ public void testRewriteEmptyReader() throws Exception { null, new IndexSearcher(reader), null, null, null, () -> true, null); RangeQueryBuilder range = new RangeQueryBuilder("foo"); // no values -> DISJOINT - assertEquals(Relation.DISJOINT, range.getRelation(context)); + assertEquals(Relation.DISJOINT, range.getMinRelation(context)); + assertEquals(Relation.DISJOINT, range.getMaxRelation(context)); } } From 519678f53de8b3b2ab8374b0d928c450286aa25d Mon Sep 17 00:00:00 2001 From: iverase Date: Mon, 12 Oct 2020 14:34:35 +0200 Subject: [PATCH 09/18] sort optimization using distance feature query --- .../queries/TwoPhaseLongRangeQuery.java | 2 +- .../index/mapper/DateFieldMapper.java | 41 +++++++++---------- .../search/query/QueryPhase.java | 8 +++- .../queries/TwoPhaseLongRangeQueryTests.java | 2 +- 4 files changed, 29 insertions(+), 24 deletions(-) diff --git a/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongRangeQuery.java b/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongRangeQuery.java index 2228ff81d6a7c..74d527d46b586 100644 --- a/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongRangeQuery.java +++ b/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongRangeQuery.java @@ -121,7 +121,7 @@ public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOExcepti if (values.getDocCount() == reader.maxDoc()) { final byte[] fieldPackedLower = values.getMinPackedValue(); final byte[] fieldPackedUpper = values.getMaxPackedValue(); - if (Arrays.compareUnsigned(lowerExc, 0, Long.BYTES, fieldPackedLower, 0, Long.BYTES) < 0 + if (Arrays.compareUnsigned(lowerExc, 0, Long.BYTES, fieldPackedLower, 0, Long.BYTES) <= 0 && Arrays.compareUnsigned(upperExc, 0, Long.BYTES, fieldPackedUpper, 0, Long.BYTES) > 0) { return new ScorerSupplier() { @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 6ed6693ef3b3a..c925c48ae7591 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -246,27 +246,26 @@ public long convertDistance(long distanceExact) { } } -// protected MappedFieldType.Relation isWithin(long fromInclusive, long toInclusive, long minValue, long maxValue, boolean hasDocValues) { -// if (hasDocValues) { -// final long approxFromInclusive = convertToIndex(fromInclusive); -// final long approxToInclusive = convertToIndex(toInclusive); -// if (minValue > approxFromInclusive + 1 && maxValue <= approxToInclusive) { -// return MappedFieldType.Relation.WITHIN; -// } else if (maxValue < approxFromInclusive - 1 || minValue > approxToInclusive) { -// return MappedFieldType.Relation.DISJOINT; -// } else { -// return MappedFieldType.Relation.INTERSECTS; -// } -// } else { -// if (minValue >= fromInclusive && maxValue <= toInclusive) { -// return MappedFieldType.Relation.WITHIN; -// } else if (maxValue < fromInclusive || minValue > toInclusive) { -// return MappedFieldType.Relation.DISJOINT; -// } else { -// return MappedFieldType.Relation.INTERSECTS; -// } -// } -// } + /** + * Build Distance feature query from index values + */ + public Query distanceFeatureQuery(String field, float boost, long origin, long pivotFromIndex, boolean hasDocValues) { + if (hasDocValues) { + long originApprox = convertToIndex(origin); + Query query = new TwoPhaseLongDistanceFeatureQuery(field, origin, convertToDocValue(pivotFromIndex), originApprox) { + @Override + public long convertDistance(long distanceExact) { + return convertToIndex(distanceExact); + } + }; + if (boost != 1f) { + query = new BoostQuery(query, boost); + } + return query; + } else { + return LongPoint.newDistanceFeatureQuery(field, boost, origin, pivotFromIndex); + } + } protected MappedFieldType.Relation isToWithin(long toInclusive, long minValue, long maxValue, boolean hasDocValues) { if (hasDocValues) { diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 4c5d6946ca63a..7436bedbb6f2a 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -471,6 +471,7 @@ private static Query tryRewriteLongSort(SearchContext searchContext, IndexReader if (shortcutTotalHitCount(reader, query) == -1) return null; } + byte[] minValueBytes = PointValues.getMinPackedValue(reader, fieldName); byte[] maxValueBytes = PointValues.getMaxPackedValue(reader, fieldName); if ((maxValueBytes == null) || (minValueBytes == null)) return null; @@ -487,7 +488,12 @@ private static Query tryRewriteLongSort(SearchContext searchContext, IndexReader if (pivotDistance == 0) { // 0 if maxValue = (minValue + 1) pivotDistance = 1; } - rewrittenQuery = LongPoint.newDistanceFeatureQuery(sortField.getField(), 1, origin, pivotDistance); + if (fieldType instanceof DateFieldType) { + rewrittenQuery = ((DateFieldType)fieldType).resolution() + .distanceFeatureQuery(sortField.getField(), 1, origin, pivotDistance, fieldType.hasDocValues()); + } else { + rewrittenQuery = LongPoint.newDistanceFeatureQuery(sortField.getField(), 1, origin, pivotDistance); + } } rewrittenQuery = new BooleanQuery.Builder() .add(query, BooleanClause.Occur.FILTER) // filter for original query diff --git a/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongRangeQueryTests.java b/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongRangeQueryTests.java index df1c23567e7d6..e8fc3549cc18c 100644 --- a/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongRangeQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongRangeQueryTests.java @@ -97,7 +97,7 @@ public void dotestIndexAndQuery(int numPoints) throws Exception { */ protected Instant randomInstant() { //return Instant.ofEpochSecond(randomLongBetween(0, 3000000000L), randomLongBetween(0, 999999999)); - return Instant.ofEpochMilli(randomLongBetween(0, 3000L)); + return Instant.ofEpochMilli(randomLongBetween(2000000L, 2100000L)); } public static long randomLongBetween(long min, long max) { From 758d0afe0f1ce140bad1dd0709352784a3ebc37d Mon Sep 17 00:00:00 2001 From: iverase Date: Mon, 12 Oct 2020 16:38:00 +0200 Subject: [PATCH 10/18] sort optimization using distance feature query --- .../index/mapper/DateFieldMapper.java | 52 +++++++------------ .../search/query/QueryPhase.java | 2 +- 2 files changed, 20 insertions(+), 34 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index c925c48ae7591..bf5745ffcfc84 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -226,45 +226,30 @@ protected Query rangeQuery(String field, long min, long max, boolean hasDocValue } /** - * Build Distance feature query + * Build Distance feature query. Field must have doc values */ - protected Query distanceFeatureQuery(String field, float boost, long origin, TimeValue pivot, boolean hasDocValues) { - if (hasDocValues) { - long originApprox = convertToIndex(origin); - Query query = new TwoPhaseLongDistanceFeatureQuery(field, origin, convert(pivot), originApprox) { - @Override - public long convertDistance(long distanceExact) { - return convertToIndex(distanceExact); - } - }; - if (boost != 1f) { - query = new BoostQuery(query, boost); - } - return query; - } else { - return LongPoint.newDistanceFeatureQuery(field, boost, origin, pivot.getNanos()); - } + protected Query distanceFeatureQuery(String field, float boost, long origin, TimeValue pivot) { + return distanceFeatureQuery(field, boost, origin, convertToIndex(origin), convert(pivot)); } /** - * Build Distance feature query from index values + * Build Distance feature query from index values. Field must have doc values */ - public Query distanceFeatureQuery(String field, float boost, long origin, long pivotFromIndex, boolean hasDocValues) { - if (hasDocValues) { - long originApprox = convertToIndex(origin); - Query query = new TwoPhaseLongDistanceFeatureQuery(field, origin, convertToDocValue(pivotFromIndex), originApprox) { - @Override - public long convertDistance(long distanceExact) { - return convertToIndex(distanceExact); - } - }; - if (boost != 1f) { - query = new BoostQuery(query, boost); + public Query distanceFeatureQuery(String field, float boost, long origin, long pivotFromIndex) { + return distanceFeatureQuery(field, boost, origin, convertToIndex(origin), convertToDocValue(pivotFromIndex)); + } + + private Query distanceFeatureQuery(String field, float boost, long origin, long originApprox, long pivot) { + Query query = new TwoPhaseLongDistanceFeatureQuery(field, origin, pivot, originApprox) { + @Override + public long convertDistance(long distanceExact) { + return convertToIndex(distanceExact); } - return query; - } else { - return LongPoint.newDistanceFeatureQuery(field, boost, origin, pivotFromIndex); + }; + if (boost != 1f) { + query = new BoostQuery(query, boost); } + return query; } protected MappedFieldType.Relation isToWithin(long toInclusive, long minValue, long maxValue, boolean hasDocValues) { @@ -571,7 +556,8 @@ public static long parseToLong( public Query distanceFeatureQuery(Object origin, String pivot, float boost, QueryShardContext context) { long originLong = parseToLong(origin, true, null, null, context::nowInMillis); TimeValue pivotTime = TimeValue.parseTimeValue(pivot, "distance_feature.pivot"); - return resolution.distanceFeatureQuery(name(), boost, originLong, pivotTime, hasDocValues()); + assert hasDocValues(); + return resolution.distanceFeatureQuery(name(), boost, originLong, pivotTime); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 7436bedbb6f2a..0bc6e515d00eb 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -490,7 +490,7 @@ private static Query tryRewriteLongSort(SearchContext searchContext, IndexReader } if (fieldType instanceof DateFieldType) { rewrittenQuery = ((DateFieldType)fieldType).resolution() - .distanceFeatureQuery(sortField.getField(), 1, origin, pivotDistance, fieldType.hasDocValues()); + .distanceFeatureQuery(sortField.getField(), 1, origin, pivotDistance); } else { rewrittenQuery = LongPoint.newDistanceFeatureQuery(sortField.getField(), 1, origin, pivotDistance); } From a484617c7efb7b103b7cac9144d22841025a5a12 Mon Sep 17 00:00:00 2001 From: iverase Date: Mon, 12 Oct 2020 17:02:03 +0200 Subject: [PATCH 11/18] 1 min precision --- .../java/org/elasticsearch/index/mapper/DateFieldMapper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index bf5745ffcfc84..40e1c5dfe9d88 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -90,12 +90,12 @@ public long convert(TimeValue timeValue) { @Override public long convertToIndex(long value) { - return value / 1000L; + return value / 60000L; } @Override public long convertToDocValue(long value) { - return value * 1000L; + return value * 60000L; } @Override From b0b2f7cbaf74a6569a180ca127f94cfa1db2ac6e Mon Sep 17 00:00:00 2001 From: iverase Date: Mon, 12 Oct 2020 17:02:13 +0200 Subject: [PATCH 12/18] 1 min precision --- .../TwoPhaseLongDistanceFeatureQuery.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQuery.java b/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQuery.java index 157b88a8f98c5..592f653ade7b8 100644 --- a/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQuery.java +++ b/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQuery.java @@ -1,3 +1,22 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with From 0978b804f70b6ca5391c6ec59d9f9bae122a4bca Mon Sep 17 00:00:00 2001 From: iverase Date: Mon, 12 Oct 2020 17:54:49 +0200 Subject: [PATCH 13/18] fix queries --- .../queries/TwoPhaseLongRangeQuery.java | 2 +- .../queries/TwoPhaseLongRangeQueryTests.java | 30 +++++++------------ 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongRangeQuery.java b/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongRangeQuery.java index 74d527d46b586..2228ff81d6a7c 100644 --- a/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongRangeQuery.java +++ b/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongRangeQuery.java @@ -121,7 +121,7 @@ public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOExcepti if (values.getDocCount() == reader.maxDoc()) { final byte[] fieldPackedLower = values.getMinPackedValue(); final byte[] fieldPackedUpper = values.getMaxPackedValue(); - if (Arrays.compareUnsigned(lowerExc, 0, Long.BYTES, fieldPackedLower, 0, Long.BYTES) <= 0 + if (Arrays.compareUnsigned(lowerExc, 0, Long.BYTES, fieldPackedLower, 0, Long.BYTES) < 0 && Arrays.compareUnsigned(upperExc, 0, Long.BYTES, fieldPackedUpper, 0, Long.BYTES) > 0) { return new ScorerSupplier() { @Override diff --git a/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongRangeQueryTests.java b/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongRangeQueryTests.java index e8fc3549cc18c..ac772b0bb6c32 100644 --- a/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongRangeQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongRangeQueryTests.java @@ -61,10 +61,10 @@ public void dotestIndexAndQuery(int numPoints) throws Exception { IndexWriter w = new IndexWriter(dir, iwc); for (int id = 0; id < numPoints; id++) { Document doc = new Document(); - Instant instant = randomInstant(); - doc.add(new LongPoint("exact", instant.toEpochMilli())); - doc.add(new LongPoint("approx", instant.getEpochSecond())); - doc.add(new SortedNumericDocValuesField("approx", instant.toEpochMilli())); + long value = randomLongBetween(2000000L, 3000000L); + doc.add(new LongPoint("exact", value)); + doc.add(new LongPoint("approx", approximate(value))); + doc.add(new SortedNumericDocValuesField("approx", value)); w.addDocument(doc); } @@ -76,28 +76,18 @@ public void dotestIndexAndQuery(int numPoints) throws Exception { IndexSearcher s = newSearcher(r); for ( int i = 0; i < 100; i++) { - Instant instant1 = randomInstant(); - Instant instant2 = randomInstant(); - if (instant1.toEpochMilli() > instant2.toEpochMilli()) { - Instant tmp = instant1; - instant1 = instant2; - instant2 = tmp; - } - Query q1 = LongPoint.newRangeQuery("exact", instant2.toEpochMilli(), instant1.toEpochMilli()); - Query q2 = new TwoPhaseLongRangeQuery("approx", instant2.toEpochMilli(), instant1.toEpochMilli(), - instant2.getEpochSecond(), instant1.getEpochSecond()); + long value1 = randomLongBetween(2000000L, 3000000L); + long value2 = randomLongBetween(value1, 3000000L); + Query q1 = LongPoint.newRangeQuery("exact", value1, value2); + Query q2 = new TwoPhaseLongRangeQuery("approx", value1, value2, approximate(value1), approximate(value2)); assertEquals(s.count(q1), s.count(q2)); } IOUtils.close(r, dir); } - /** - * @return a random instant between 1970 and ca 2065 - */ - protected Instant randomInstant() { - //return Instant.ofEpochSecond(randomLongBetween(0, 3000000000L), randomLongBetween(0, 999999999)); - return Instant.ofEpochMilli(randomLongBetween(2000000L, 2100000L)); + protected long approximate(long value) { + return value / 60000L; } public static long randomLongBetween(long min, long max) { From d10957a08830bca009102585736f27b525bd1797 Mon Sep 17 00:00:00 2001 From: iverase Date: Mon, 12 Oct 2020 18:26:52 +0200 Subject: [PATCH 14/18] fix queries --- ...TwoPhaseLongDistanceFeatureQueryTests.java | 25 +++++++++---------- .../queries/TwoPhaseLongRangeQueryTests.java | 2 -- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQueryTests.java b/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQueryTests.java index ca273c6bcc05e..46f887b0c6663 100644 --- a/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQueryTests.java @@ -36,7 +36,6 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; -import java.time.Instant; public class TwoPhaseLongDistanceFeatureQueryTests extends LuceneTestCase { @@ -63,11 +62,11 @@ public void dotestIndexAndQuery(int numPoints) throws Exception { IndexWriter w = new IndexWriter(dir, iwc); for (int id = 0; id < numPoints; id++) { Document doc = new Document(); - Instant instant = randomInstant(); - doc.add(new LongPoint("exact", instant.toEpochMilli())); - doc.add(new SortedNumericDocValuesField("exact", instant.toEpochMilli())); - doc.add(new LongPoint("approx", instant.getEpochSecond())); - doc.add(new SortedNumericDocValuesField("approx", instant.toEpochMilli())); + long value = randomLongBetween(2000000L, 3000000L); + doc.add(new LongPoint("exact",value)); + doc.add(new SortedNumericDocValuesField("exact", value)); + doc.add(new LongPoint("approx", approximate(value))); + doc.add(new SortedNumericDocValuesField("approx", value)); w.addDocument(doc); } @@ -79,14 +78,14 @@ public void dotestIndexAndQuery(int numPoints) throws Exception { IndexSearcher s = newSearcher(r); for ( int i = 0; i < 100; i++) { - Instant origin = randomInstant(); - long distance = randomLongBetween(0, 10000); + long origin = randomLongBetween(2000000L, 3000000L); + long distance = randomLongBetween(1, 10000); - Query q1 = LongPoint.newDistanceFeatureQuery("exact", 2f, origin.toEpochMilli(), distance); - Query q2 = new TwoPhaseLongDistanceFeatureQuery("approx", origin.toEpochMilli(), distance, origin.getEpochSecond()) { + Query q1 = LongPoint.newDistanceFeatureQuery("exact", 2f, origin, distance); + Query q2 = new TwoPhaseLongDistanceFeatureQuery("approx", origin, distance, approximate(origin)) { @Override public long convertDistance(long distanceExact) { - return distanceExact/1000; + return approximate(distanceExact); } }; q2 = new BoostQuery(q2, 2f); @@ -107,8 +106,8 @@ public long convertDistance(long distanceExact) { /** * @return a random instant between 1970 and ca 2065 */ - protected Instant randomInstant() { - return Instant.ofEpochMilli(randomLongBetween(0, 3000L)); + protected long approximate(long value) { + return value / 60000L; } public static long randomLongBetween(long min, long max) { diff --git a/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongRangeQueryTests.java b/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongRangeQueryTests.java index ac772b0bb6c32..4464bb3e68ed6 100644 --- a/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongRangeQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongRangeQueryTests.java @@ -34,8 +34,6 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; -import java.time.Instant; - public class TwoPhaseLongRangeQueryTests extends LuceneTestCase { public void testIndexAndQuerySmall() throws Exception { From 6dffe9c1c0bb23fbf24a75020cf31419f8c00f86 Mon Sep 17 00:00:00 2001 From: iverase Date: Tue, 13 Oct 2020 07:41:03 +0200 Subject: [PATCH 15/18] fix queries --- .../java/org/elasticsearch/index/mapper/DateFieldMapper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 40e1c5dfe9d88..3921641bb2ea7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -254,7 +254,7 @@ public long convertDistance(long distanceExact) { protected MappedFieldType.Relation isToWithin(long toInclusive, long minValue, long maxValue, boolean hasDocValues) { if (hasDocValues) { - final long approxToInclusive = convertToIndex(toInclusive); + final long approxToInclusive = toInclusive;//convertToIndex(toInclusive); if (maxValue < approxToInclusive - 1) { return MappedFieldType.Relation.WITHIN; } else if (minValue > approxToInclusive + 1) { @@ -275,7 +275,7 @@ protected MappedFieldType.Relation isToWithin(long toInclusive, long minValue, l protected MappedFieldType.Relation isFromWithin(long fromInclusive, long minValue, long maxValue, boolean hasDocValues) { if (hasDocValues) { - final long approxFromInclusive = convertToIndex(fromInclusive); + final long approxFromInclusive = fromInclusive;//convertToIndex(fromInclusive); if (minValue >= approxFromInclusive) { return MappedFieldType.Relation.WITHIN; } else if (maxValue < approxFromInclusive) { From 2ac73c991e6a578a0defe35e05fd1f3849ac90ef Mon Sep 17 00:00:00 2001 From: iverase Date: Tue, 13 Oct 2020 07:52:14 +0200 Subject: [PATCH 16/18] fix sort --- .../index/mapper/DateFieldMapper.java | 24 +++++++++---------- .../index/mapper/MappedFieldType.java | 10 ++++---- .../index/query/RangeQueryBuilder.java | 4 ++-- .../search/sort/FieldSortBuilder.java | 2 +- .../index/mapper/DateFieldTypeTests.java | 24 +++++++++---------- .../index/mapper/KeywordFieldTypeTests.java | 2 +- .../index/mapper/NumberFieldTypeTests.java | 2 +- 7 files changed, 34 insertions(+), 34 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 3921641bb2ea7..9f8738ca8ece7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -252,9 +252,10 @@ public long convertDistance(long distanceExact) { return query; } - protected MappedFieldType.Relation isToWithin(long toInclusive, long minValue, long maxValue, boolean hasDocValues) { + protected MappedFieldType.Relation isToWithin(long toInclusive, long minValue, long maxValue, + boolean hasDocValues, boolean fromIndex) { if (hasDocValues) { - final long approxToInclusive = toInclusive;//convertToIndex(toInclusive); + final long approxToInclusive = fromIndex ? toInclusive : convertToIndex(toInclusive); if (maxValue < approxToInclusive - 1) { return MappedFieldType.Relation.WITHIN; } else if (minValue > approxToInclusive + 1) { @@ -273,9 +274,10 @@ protected MappedFieldType.Relation isToWithin(long toInclusive, long minValue, l } } - protected MappedFieldType.Relation isFromWithin(long fromInclusive, long minValue, long maxValue, boolean hasDocValues) { + protected MappedFieldType.Relation isFromWithin(long fromInclusive, long minValue, long maxValue, + boolean hasDocValues, boolean fromIndex) { if (hasDocValues) { - final long approxFromInclusive = fromInclusive;//convertToIndex(fromInclusive); + final long approxFromInclusive = fromIndex ? fromInclusive : convertToIndex(fromInclusive); if (minValue >= approxFromInclusive) { return MappedFieldType.Relation.WITHIN; } else if (maxValue < approxFromInclusive) { @@ -561,9 +563,8 @@ public Query distanceFeatureQuery(Object origin, String pivot, float boost, Quer } @Override - public Relation isFieldMinWithinQuery(IndexReader reader, - Object from, boolean includeLower, - ZoneId timeZone, DateMathParser dateParser, QueryRewriteContext context) throws IOException { + public Relation isFieldMinWithinQuery(IndexReader reader, Object from, boolean includeLower, ZoneId timeZone, + DateMathParser dateParser, QueryRewriteContext context, boolean fromIndex) throws IOException { if (dateParser == null) { dateParser = this.dateMathParser; } @@ -587,13 +588,12 @@ public Relation isFieldMinWithinQuery(IndexReader reader, long minValue = LongPoint.decodeDimension(PointValues.getMinPackedValue(reader, name()), 0); long maxValue = LongPoint.decodeDimension(PointValues.getMaxPackedValue(reader, name()), 0); - return resolution.isFromWithin(fromInclusive, minValue, maxValue, hasDocValues()); + return resolution.isFromWithin(fromInclusive, minValue, maxValue, hasDocValues(), fromIndex); } @Override - public Relation isFieldMaxWithinQuery(IndexReader reader, - Object to, boolean includeUpper, - ZoneId timeZone, DateMathParser dateParser, QueryRewriteContext context) throws IOException { + public Relation isFieldMaxWithinQuery(IndexReader reader, Object to, boolean includeUpper, ZoneId timeZone, + DateMathParser dateParser, QueryRewriteContext context, boolean fromIndex) throws IOException { if (dateParser == null) { dateParser = this.dateMathParser; } @@ -617,7 +617,7 @@ public Relation isFieldMaxWithinQuery(IndexReader reader, long minValue = LongPoint.decodeDimension(PointValues.getMinPackedValue(reader, name()), 0); long maxValue = LongPoint.decodeDimension(PointValues.getMaxPackedValue(reader, name()), 0); - return resolution.isToWithin(toInclusive, minValue, maxValue, hasDocValues()); + return resolution.isToWithin(toInclusive, minValue, maxValue, hasDocValues(), fromIndex); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index ca3c597a53f2f..a2dc1dc07ce1e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -311,7 +311,7 @@ public enum Relation { * no way to check whether values are actually within bounds. */ public Relation isFieldMaxWithinQuery( IndexReader reader, Object to, boolean includeUpper, - ZoneId timeZone, DateMathParser dateMathParser, QueryRewriteContext context) throws IOException { + ZoneId timeZone, DateMathParser dateMathParser, QueryRewriteContext context, boolean fromIndex) throws IOException { return Relation.INTERSECTS; } @@ -323,10 +323,10 @@ public Relation isFieldWithinQuery( IndexReader reader, Object from, Object to, boolean includeLower, boolean includeUpper, - ZoneId timeZone, DateMathParser dateMathParser, QueryRewriteContext context) throws IOException { + ZoneId timeZone, DateMathParser dateMathParser, QueryRewriteContext context, boolean fromIndex) throws IOException { - final Relation minRelation = isFieldMinWithinQuery(reader, from, includeLower, timeZone, dateMathParser, context); - final Relation maxRelation = isFieldMaxWithinQuery(reader, to, includeUpper, timeZone, dateMathParser, context); + final Relation minRelation = isFieldMinWithinQuery(reader, from, includeLower, timeZone, dateMathParser, context, fromIndex); + final Relation maxRelation = isFieldMaxWithinQuery(reader, to, includeUpper, timeZone, dateMathParser, context, fromIndex); if (minRelation == Relation.DISJOINT || maxRelation == Relation.DISJOINT) { return Relation.DISJOINT; } else if (minRelation == Relation.WITHIN && maxRelation == Relation.WITHIN) { @@ -343,7 +343,7 @@ public Relation isFieldWithinQuery( public Relation isFieldMinWithinQuery( IndexReader reader, Object from, boolean includeLower, - ZoneId timeZone, DateMathParser dateMathParser, QueryRewriteContext context) throws IOException { + ZoneId timeZone, DateMathParser dateMathParser, QueryRewriteContext context, boolean fromIndex) throws IOException { return Relation.INTERSECTS; } diff --git a/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java index 162ec3aed6bcf..4f6e124a2a4ff 100644 --- a/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java @@ -441,7 +441,7 @@ protected MappedFieldType.Relation getMinRelation(QueryRewriteContext queryRewri } DateMathParser dateMathParser = getForceDateParser(); return fieldType.isFieldMinWithinQuery(shardContext.getIndexReader(), from, includeLower, - timeZone, dateMathParser, queryRewriteContext); + timeZone, dateMathParser, queryRewriteContext, false); } // Not on the shard, we have no way to know what the relation is. @@ -462,7 +462,7 @@ protected MappedFieldType.Relation getMaxRelation(QueryRewriteContext queryRewri } DateMathParser dateMathParser = getForceDateParser(); return fieldType.isFieldMaxWithinQuery(shardContext.getIndexReader(), to, includeUpper, - timeZone, dateMathParser, queryRewriteContext); + timeZone, dateMathParser, queryRewriteContext, false); } // Not on the shard, we have no way to know what the relation is. diff --git a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java index 694188094a041..a377c3835dec6 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java @@ -401,7 +401,7 @@ public boolean isBottomSortShardDisjoint(QueryShardContext context, SearchSortVa Object maxValue = order() == SortOrder.DESC ? null : bottomSortValue; try { MappedFieldType.Relation relation = fieldType.isFieldWithinQuery(context.getIndexReader(), minValue, maxValue, - true, true, null, dateMathParser, context); + true, true, null, dateMathParser, context, true); return relation == MappedFieldType.Relation.DISJOINT; } catch (ElasticsearchParseException exc) { // can happen if the sort field is mapped differently in another search index diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index 46f159b8442a0..e2400e53b9c57 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -68,7 +68,7 @@ public void testIsFieldWithinRangeEmptyReader() throws IOException { IndexReader reader = new MultiReader(); DateFieldType ft = new DateFieldType("my_date"); assertEquals(Relation.DISJOINT, ft.isFieldWithinQuery(reader, "2015-10-12", "2016-04-03", - randomBoolean(), randomBoolean(), null, null, context)); + randomBoolean(), randomBoolean(), null, null, context, false)); } public void testIsFieldWithinQueryDateMillis() throws IOException { @@ -108,7 +108,7 @@ public void isFieldWithinRangeTestCase(DateFieldType ft) throws IOException { // Fields with no value indexed. DateFieldType ft2 = new DateFieldType("my_date2"); - assertEquals(Relation.DISJOINT, ft2.isFieldWithinQuery(reader, "2015-10-09", "2016-01-02", false, false, null, null, context)); + assertEquals(Relation.DISJOINT, ft2.isFieldWithinQuery(reader, "2015-10-09", "2016-01-02", false, false, null, null, context, false)); IOUtils.close(reader, w, dir); } @@ -117,25 +117,25 @@ private void doTestIsFieldWithinQuery(DateFieldType ft, DirectoryReader reader, DateTimeZone zone, DateMathParser alternateFormat) throws IOException { QueryRewriteContext context = new QueryRewriteContext(xContentRegistry(), writableRegistry(), null, () -> nowInMillis); assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2015-10-09", "2016-01-02", - randomBoolean(), randomBoolean(), null, null, context)); + randomBoolean(), randomBoolean(), null, null, context, false)); assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2016-01-02", "2016-06-20", - randomBoolean(), randomBoolean(), null, null, context)); + randomBoolean(), randomBoolean(), null, null, context, false)); assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2016-01-02", "2016-02-12", - randomBoolean(), randomBoolean(), null, null, context)); + randomBoolean(), randomBoolean(), null, null, context, false)); assertEquals(Relation.DISJOINT, ft.isFieldWithinQuery(reader, "2014-01-02", "2015-02-12", - randomBoolean(), randomBoolean(), null, null, context)); + randomBoolean(), randomBoolean(), null, null, context, false)); assertEquals(Relation.DISJOINT, ft.isFieldWithinQuery(reader, "2016-05-11", "2016-08-30", - randomBoolean(), randomBoolean(), null, null, context)); + randomBoolean(), randomBoolean(), null, null, context, false)); assertEquals(Relation.WITHIN, ft.isFieldWithinQuery(reader, "2015-09-25", "2016-05-29", - randomBoolean(), randomBoolean(), null, null, context)); + randomBoolean(), randomBoolean(), null, null, context, false)); assertEquals(Relation.WITHIN, ft.isFieldWithinQuery(reader, "2015-10-12", "2016-04-03", - true, true, null, null, context)); + true, true, null, null, context, false)); assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2015-10-12", "2016-04-03", - false, false, null, null, context)); + false, false, null, null, context, false)); assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2015-10-12", "2016-04-03", - false, true, null, null, context)); + false, true, null, null, context, false)); assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2015-10-12", "2016-04-03", - true, false, null, null, context)); + true, false, null, null, context, false)); } public void testValueFormat() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java index 2632753ae1b58..d7c6d96dbbd85 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java @@ -69,7 +69,7 @@ public void testIsFieldWithinQuery() throws IOException { assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(null, RandomStrings.randomAsciiLettersOfLengthBetween(random(), 0, 5), RandomStrings.randomAsciiLettersOfLengthBetween(random(), 0, 5), - randomBoolean(), randomBoolean(), null, null, null)); + randomBoolean(), randomBoolean(), null, null, null, false)); } public void testTermQuery() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index 6f40d9443dc77..876c00f816c33 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -94,7 +94,7 @@ public void testIsFieldWithinQuery() throws IOException { MappedFieldType ft = new NumberFieldType("field", NumberType.INTEGER); // current impl ignores args and should always return INTERSECTS assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(null, randomDouble(), randomDouble(), - randomBoolean(), randomBoolean(), null, null, null)); + randomBoolean(), randomBoolean(), null, null, null, false)); } public void testIntegerTermsQueryWithDecimalPart() { From 68e7e973e16054e039a078b2f486b52ccddef682 Mon Sep 17 00:00:00 2001 From: iverase Date: Fri, 6 Nov 2020 10:56:27 +0100 Subject: [PATCH 17/18] compile, style issues --- .../index/mapper/CollationFieldTypeTests.java | 2 +- .../search/query/QueryStringIT.java | 1 - .../TwoPhaseLongDistanceFeatureQuery.java | 22 ++---------- .../queries/TwoPhaseLongRangeQuery.java | 36 +++++++++++-------- .../index/mapper/DateFieldMapper.java | 6 ++-- .../index/query/RangeQueryBuilder.java | 1 - ...TwoPhaseLongDistanceFeatureQueryTests.java | 2 +- .../queries/TwoPhaseLongRangeQueryTests.java | 2 +- .../index/mapper/DateFieldTypeTests.java | 4 +-- .../RewriteCachingDirectoryReaderTests.java | 6 ++-- 10 files changed, 37 insertions(+), 45 deletions(-) diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/CollationFieldTypeTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/CollationFieldTypeTests.java index 14fefdf4382d3..96cd66ba4e52b 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/CollationFieldTypeTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/CollationFieldTypeTests.java @@ -51,7 +51,7 @@ public void testIsFieldWithinQuery() throws IOException { assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(null, RandomStrings.randomAsciiOfLengthBetween(random(), 0, 5), RandomStrings.randomAsciiOfLengthBetween(random(), 0, 5), - randomBoolean(), randomBoolean(), null, null, null)); + randomBoolean(), randomBoolean(), null, null, null, false)); } public void testTermQuery() { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java index 833bcad05d454..85a799711920b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java @@ -44,7 +44,6 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery; import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; diff --git a/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQuery.java b/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQuery.java index 592f653ade7b8..974ea1de86443 100644 --- a/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQuery.java +++ b/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQuery.java @@ -7,7 +7,7 @@ * not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -16,23 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package org.apache.lucene.queries; import org.apache.lucene.document.LongPoint; @@ -140,7 +123,8 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio distance = Long.MAX_VALUE; } float score = (float) (boost * (pivotDistanceExact / (pivotDistanceExact + (double) distance))); - return Explanation.match(score, "Distance score, computed as weight * pivotDistance / (pivotDistance + abs(value - origin)) from:", + return Explanation.match(score, + "Distance score, computed as weight * pivotDistance / (pivotDistance + abs(value - origin)) from:", Explanation.match(boost, "weight"), Explanation.match(pivotDistanceExact, "pivotDistance"), Explanation.match(originExact, "origin"), diff --git a/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongRangeQuery.java b/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongRangeQuery.java index 2228ff81d6a7c..fbd2517ae3fa8 100644 --- a/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongRangeQuery.java +++ b/server/src/main/java/org/apache/lucene/queries/TwoPhaseLongRangeQuery.java @@ -1,18 +1,20 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.apache.lucene.queries; @@ -112,10 +114,16 @@ public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOExcepti } if (values.getNumIndexDimensions() != 1) { - throw new IllegalArgumentException("field=\"" + field + "\" was indexed with numIndexDimensions=" + values.getNumIndexDimensions() + " but this query has numDims=" + 1); + throw new IllegalArgumentException( + "field=\"" + field + "\" was indexed with numIndexDimensions=" + values.getNumIndexDimensions() + + " but this query has numDims=" + 1 + ); } if (Long.BYTES != values.getBytesPerDimension()) { - throw new IllegalArgumentException("field=\"" + field + "\" was indexed with bytesPerDim=" + values.getBytesPerDimension() + " but this query has bytesPerDim=" + Long.BYTES); + throw new IllegalArgumentException( + "field=\"" + field + "\" was indexed with bytesPerDim=" + values.getBytesPerDimension() + + " but this query has bytesPerDim=" + Long.BYTES + ); } final Weight weight = this; if (values.getDocCount() == reader.maxDoc()) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index b88c1e7d2e00c..2f41579de0df5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -564,7 +564,8 @@ public Query distanceFeatureQuery(Object origin, String pivot, QueryShardContext @Override public Relation isFieldMinWithinQuery(IndexReader reader, Object from, boolean includeLower, ZoneId timeZone, - DateMathParser dateParser, QueryRewriteContext context, boolean fromIndex) throws IOException { + DateMathParser dateParser, QueryRewriteContext context, boolean fromIndex + ) throws IOException { if (dateParser == null) { dateParser = this.dateMathParser; } @@ -593,7 +594,8 @@ public Relation isFieldMinWithinQuery(IndexReader reader, Object from, boolean i @Override public Relation isFieldMaxWithinQuery(IndexReader reader, Object to, boolean includeUpper, ZoneId timeZone, - DateMathParser dateParser, QueryRewriteContext context, boolean fromIndex) throws IOException { + DateMathParser dateParser, QueryRewriteContext context, boolean fromIndex + ) throws IOException { if (dateParser == null) { dateParser = this.dateMathParser; } diff --git a/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java index 4f6e124a2a4ff..40b3a42ab0aa2 100644 --- a/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.query; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; diff --git a/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQueryTests.java b/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQueryTests.java index 46f887b0c6663..be5f23c0f70b4 100644 --- a/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongDistanceFeatureQueryTests.java @@ -33,8 +33,8 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; -import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.core.internal.io.IOUtils; public class TwoPhaseLongDistanceFeatureQueryTests extends LuceneTestCase { diff --git a/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongRangeQueryTests.java b/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongRangeQueryTests.java index 4464bb3e68ed6..f25c0282d6716 100644 --- a/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongRangeQueryTests.java +++ b/server/src/test/java/org/apache/lucene/queries/TwoPhaseLongRangeQueryTests.java @@ -31,8 +31,8 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; -import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.core.internal.io.IOUtils; public class TwoPhaseLongRangeQueryTests extends LuceneTestCase { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index e2400e53b9c57..2bd6199cd5803 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.index.mapper; -import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.DirectoryReader; @@ -108,7 +107,8 @@ public void isFieldWithinRangeTestCase(DateFieldType ft) throws IOException { // Fields with no value indexed. DateFieldType ft2 = new DateFieldType("my_date2"); - assertEquals(Relation.DISJOINT, ft2.isFieldWithinQuery(reader, "2015-10-09", "2016-01-02", false, false, null, null, context, false)); + assertEquals(Relation.DISJOINT, + ft2.isFieldWithinQuery(reader, "2015-10-09", "2016-01-02", false, false, null, null, context, false)); IOUtils.close(reader, w, dir); } diff --git a/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/RewriteCachingDirectoryReaderTests.java b/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/RewriteCachingDirectoryReaderTests.java index 4367f5a085cda..21ddf93f4197e 100644 --- a/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/RewriteCachingDirectoryReaderTests.java +++ b/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/RewriteCachingDirectoryReaderTests.java @@ -90,15 +90,15 @@ public void testIsWithinQuery() throws IOException { DateFieldMapper.DateFieldType dateFieldType = new DateFieldMapper.DateFieldType("test"); QueryRewriteContext context = new QueryRewriteContext(xContentRegistry(), writableRegistry(), null, () -> 0); MappedFieldType.Relation relation = dateFieldType.isFieldWithinQuery(cachingDirectoryReader, 0, 10, - true, true, ZoneOffset.UTC, null, context); + true, true, ZoneOffset.UTC, null, context, false); assertEquals(relation, MappedFieldType.Relation.WITHIN); relation = dateFieldType.isFieldWithinQuery(cachingDirectoryReader, 3, 11, - true, true, ZoneOffset.UTC, null, context); + true, true, ZoneOffset.UTC, null, context, false); assertEquals(relation, MappedFieldType.Relation.INTERSECTS); relation = dateFieldType.isFieldWithinQuery(cachingDirectoryReader, 10, 11, - false, true, ZoneOffset.UTC, null, context); + false, true, ZoneOffset.UTC, null, context, false); assertEquals(relation, MappedFieldType.Relation.DISJOINT); } } From cc7b75f74b7a04b1396a82ac34be97b7c56d32d1 Mon Sep 17 00:00:00 2001 From: iverase Date: Sun, 8 Nov 2020 10:21:38 +0100 Subject: [PATCH 18/18] disable reading data from index in agg --- .../java/org/elasticsearch/index/mapper/DateFieldMapper.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 2f41579de0df5..3b9966f083ca2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -624,9 +624,7 @@ public Relation isFieldMaxWithinQuery(IndexReader reader, Object to, boolean inc @Override public Function pointReaderIfPossible() { - if (isSearchable()) { - return resolution()::parsePointAsMillis; - } + // cannot resolve with the index return null; }