From 081d6ac60ed97e73c776b7a489d63dd885dd0704 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 27 Mar 2018 17:15:26 +0200 Subject: [PATCH] Fix NDCG for empty search results Fixes and edge case where DiscountedCumulativeGain can return NaN as result of the quality metric calculation. This can happen when the search result set is empty and normalization is used. We should return 0 in this case. Also adding related unit tests to the other two metrics. --- .../rankeval/DiscountedCumulativeGain.java | 9 ++++--- .../index/rankeval/MeanReciprocalRank.java | 4 +++ .../DiscountedCumulativeGainTests.java | 26 +++++++++++++++++++ .../rankeval/MeanReciprocalRankTests.java | 7 +++++ .../index/rankeval/PrecisionAtKTests.java | 8 ++++++ 5 files changed, 51 insertions(+), 3 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java index edb69fcb93523..3019532779800 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java @@ -140,9 +140,12 @@ public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, if (normalize) { Collections.sort(allRatings, Comparator.nullsLast(Collections.reverseOrder())); - double idcg = computeDCG( - allRatings.subList(0, Math.min(ratingsInSearchHits.size(), allRatings.size()))); - dcg = dcg / idcg; + double idcg = computeDCG(allRatings.subList(0, Math.min(ratingsInSearchHits.size(), allRatings.size()))); + if (idcg > 0) { + dcg = dcg / idcg; + } else { + dcg = 0; + } } EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, dcg); evalQueryQuality.addHitsAndRatings(ratedHits); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java index ef510b399d409..0f51f6d5d6369 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java @@ -228,6 +228,10 @@ public String getWriteableName() { return NAME; } + /** + * the ranking of the first relevant document, or -1 if no relevant document was + * found + */ int getFirstRelevantRank() { return firstRelevantRank; } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java index ea14e51512b24..22c3542c0fab4 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java @@ -205,6 +205,32 @@ public void testDCGAtFourMoreRatings() { assertEquals(12.392789260714371 / 13.347184833073591, dcg.evaluate("id", hits, ratedDocs).getQualityLevel(), DELTA); } + /** + * test that metric returns 0.0 when there are no search results + */ + public void testNoResults() throws Exception { + Integer[] relevanceRatings = new Integer[] { 3, 2, 3, null, 1, null }; + List ratedDocs = new ArrayList<>(); + for (int i = 0; i < 6; i++) { + if (i < relevanceRatings.length) { + if (relevanceRatings[i] != null) { + ratedDocs.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i])); + } + } + } + SearchHit[] hits = new SearchHit[0]; + DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); + EvalQueryQuality result = dcg.evaluate("id", hits, ratedDocs); + assertEquals(0.0d, result.getQualityLevel(), DELTA); + assertEquals(0, filterUnknownDocuments(result.getHitsAndRatings()).size()); + + // also check normalized + dcg = new DiscountedCumulativeGain(true, null, 10); + result = dcg.evaluate("id", hits, ratedDocs); + assertEquals(0.0d, result.getQualityLevel(), DELTA); + assertEquals(0, filterUnknownDocuments(result.getHitsAndRatings()).size()); + } + public void testParseFromXContent() throws IOException { assertParsedCorrect("{ \"unknown_doc_rating\": 2, \"normalize\": true, \"k\" : 15 }", 2, true, 15); assertParsedCorrect("{ \"normalize\": false, \"k\" : 15 }", null, false, 15); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java index 8ab4f146ff724..6604dbc74a065 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java @@ -158,6 +158,13 @@ public void testEvaluationNoRelevantInResults() { assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE); } + public void testNoResults() throws Exception { + SearchHit[] hits = new SearchHit[0]; + EvalQueryQuality evaluated = (new MeanReciprocalRank()).evaluate("id", hits, Collections.emptyList()); + assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001); + assertEquals(-1, ((MeanReciprocalRank.Breakdown) evaluated.getMetricDetails()).getFirstRelevantRank()); + } + public void testXContentRoundtrip() throws IOException { MeanReciprocalRank testItem = createTestItem(); XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java index a6d18c3457fa1..aa3dd5a0b7e32 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java @@ -142,6 +142,14 @@ public void testNoRatedDocs() throws Exception { assertEquals(0, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } + public void testNoResults() throws Exception { + SearchHit[] hits = new SearchHit[0]; + EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", hits, Collections.emptyList()); + assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001); + assertEquals(0, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(0, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRetrieved()); + } + public void testParseFromXContent() throws IOException { String xContent = " {\n" + " \"relevant_rating_threshold\" : 2" + "}"; try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {