CircuitBreakerServiceIT.testParentChecking failures

[jpountz]

https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+6.1+multijob-unix-compatibility/os=opensuse/191/consoleFull

The failure is not reproducible but seems to only affect 6.1.

12:35:56   1> org.elasticsearch.transport.RemoteTransportException: [node_td3][127.0.0.1:39190][indices:data/read/search[phase/query/id]]
12:35:56   1> Caused by: org.elasticsearch.search.query.QueryPhaseExecutionException: Query Failed [Failed to execute main query]
12:35:56   1> 	at org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:298) ~[main/:?]
12:35:56   1> 	at org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:110) ~[main/:?]
12:35:56   1> 	at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:420) ~[main/:?]
12:35:56   1> 	at org.elasticsearch.action.search.SearchTransportService$7.messageReceived(SearchTransportService.java:404) ~[main/:?]
12:35:56   1> 	at org.elasticsearch.action.search.SearchTransportService$7.messageReceived(SearchTransportService.java:401) ~[main/:?]
12:35:56   1> 	at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) ~[main/:?]
12:35:56   1> 	at org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1554) ~[main/:?]
12:35:56   1> 	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:637) ~[main/:?]
12:35:56   1> 	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[main/:?]
12:35:56   1> 	at org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41) ~[main/:?]
12:35:56   1> 	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[main/:?]
12:35:56   1> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_151]
12:35:56   1> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_151]
12:35:56   1> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_151]
12:35:56   1> Caused by: org.elasticsearch.ElasticsearchException: java.util.concurrent.ExecutionException: CircuitBreakingException[[fielddata] Data too large, data for [test] would be [1758/1.7kb], which is larger than the limit of [10/10b]]
12:35:56   1> 	at org.elasticsearch.index.fielddata.plain.AbstractIndexFieldData.load(AbstractIndexFieldData.java:74) ~[main/:?]
12:35:56   1> 	at org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource$1.getSortedDocValues(BytesRefFieldComparatorSource.java:87) ~[main/:?]
12:35:56   1> 	at org.apache.lucene.search.FieldComparator$TermOrdValComparator.getLeafComparator(FieldComparator.java:714) ~[lucene-core-7.1.0.jar:7.1.0 84c90ad2c0218156c840e19a64d72b8a38550659 - ubuntu - 2017-10-13 16:12:42]
12:35:56   1> 	at org.apache.lucene.search.FieldValueHitQueue.getComparators(FieldValueHitQueue.java:180) ~[lucene-core-7.1.0.jar:7.1.0 84c90ad2c0218156c840e19a64d72b8a38550659 - ubuntu - 2017-10-13 16:12:42]
12:35:56   1> 	at org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector.getLeafCollector(TopFieldCollector.java:100) ~[lucene-core-7.1.0.jar:7.1.0 84c90ad2c0218156c840e19a64d72b8a38550659 - ubuntu - 2017-10-13 16:12:42]
12:35:56   1> 	at org.apache.lucene.search.FilterCollector.getLeafCollector(FilterCollector.java:40) ~[lucene-core-7.1.0.jar:7.1.0 84c90ad2c0218156c840e19a64d72b8a38550659 - ubuntu - 2017-10-13 16:12:42]
12:35:56   1> 	at org.elasticsearch.search.query.CancellableCollector.getLeafCollector(CancellableCollector.java:51) ~[main/:?]
12:35:56   1> 	at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:649) ~[lucene-core-7.1.0.jar:7.1.0 84c90ad2c0218156c840e19a64d72b8a38550659 - ubuntu - 2017-10-13 16:12:42]
12:35:56   1> 	at org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:186) ~[main/:?]
12:35:56   1> 	at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:462) ~[lucene-core-7.1.0.jar:7.1.0 84c90ad2c0218156c840e19a64d72b8a38550659 - ubuntu - 2017-10-13 16:12:42]
12:35:56   1> 	at org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:272) ~[main/:?]
12:35:56   1> 	at org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:110) ~[main/:?]
12:35:56   1> 	at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:420) ~[main/:?]
12:35:56   1> 	at org.elasticsearch.action.search.SearchTransportService$7.messageReceived(SearchTransportService.java:404) ~[main/:?]
12:35:56   1> 	at org.elasticsearch.action.search.SearchTransportService$7.messageReceived(SearchTransportService.java:401) ~[main/:?]
12:35:56   1> 	at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) ~[main/:?]
12:35:56   1> 	at org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1554) ~[main/:?]
12:35:56   1> 	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:637) ~[main/:?]
12:35:56   1> 	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[main/:?]
12:35:56   1> 	at org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41) ~[main/:?]
12:35:56   1> 	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[main/:?]
12:35:56   1> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_151]
12:35:56   1> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_151]
12:35:56   1> 	at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_151]

12:36:01   2> REPRODUCE WITH: ./gradlew :core:integTest -Dtests.seed=32BD17E618B23F85 -Dtests.class=org.elasticsearch.indices.memory.breaker.CircuitBreakerServiceIT -Dtests.method="testParentChecking" -Dtests.security.manager=true -Dtests.locale=it-CH -Dtests.timezone=Asia/Beirut

[dakrone]

I looked into this, the failure isn't that a circuit breaker is being tripped (that's expected), it's that there are non-closed contexts:

FAILURE 21.1s J2 | CircuitBreakerServiceIT.testParentChecking <<< FAILURES!
   > Throwable #1: java.lang.AssertionError: There are still [2] in-flight contexts. The first one's creation site is listed as the cause of this exception.
   > 	at org.elasticsearch.search.MockSearchService.assertNoInFlightContext(MockSearchService.java:51)
   > 	at org.elasticsearch.test.ESTestCase.lambda$ensureAllSearchContextsReleased$4(ESTestCase.java:419)
   > 	at org.elasticsearch.test.ESTestCase.assertBusy(ESTestCase.java:726)
   > 	at org.elasticsearch.test.ESTestCase.assertBusy(ESTestCase.java:700)
   > 	at org.elasticsearch.test.ESTestCase.ensureAllSearchContextsReleased(ESTestCase.java:419)
   > 	at org.elasticsearch.test.ESIntegTestCase.cleanUpCluster(ESIntegTestCase.java:2068)
   > 	at java.lang.Thread.run(Thread.java:748)
   > 	Suppressed: java.lang.AssertionError: There are still [2] in-flight contexts. The first one's creation site is listed as the cause of this exception.
   > 		at org.elasticsearch.search.MockSearchService.assertNoInFlightContext(MockSearchService.java:51)
   > 		at org.elasticsearch.test.ESTestCase.lambda$ensureAllSearchContextsReleased$4(ESTestCase.java:419)
   > 		at org.elasticsearch.test.ESTestCase.assertBusy(ESTestCase.java:714)
   > 		... 37 more
   > 	Caused by: java.lang.RuntimeException: [dZ_96OWrR4qirRgFGrID6w][cb-test][6]searchType=[DFS_QUERY_THEN_FETCH] query=[*:*]
   > 		at org.elasticsearch.search.MockSearchService.addActiveContext(MockSearchService.java:59)
   > 		at org.elasticsearch.search.MockSearchService.putContext(MockSearchService.java:78)
   > 		at org.elasticsearch.search.SearchService.createAndPutContext(SearchService.java:531)
   > 		at org.elasticsearch.search.SearchService.executeDfsPhase(SearchService.java:275)
   > 		at org.elasticsearch.search.SearchService.access$000(SearchService.java:113)
   > 		at org.elasticsearch.search.SearchService$1.onResponse(SearchService.java:261)
   > 		at org.elasticsearch.search.SearchService$1.onResponse(SearchService.java:257)
   > 		at org.elasticsearch.search.SearchService$3.doRun(SearchService.java:996)
   > 		at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:637)
   > 		at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
   > 		at org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
   > 		at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
   > 		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   > 		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   > 		... 1 more
   > 	Suppressed: java.lang.AssertionError: There are still [2] in-flight contexts. The first one's creation site is listed as the cause of this exception.
   > 		at org.elasticsearch.search.MockSearchService.assertNoInFlightContext(MockSearchService.java:51)
   > 		at org.elasticsearch.test.ESTestCase.lambda$ensureAllSearchContextsReleased$4(ESTestCase.java:419)
   > 		at org.elasticsearch.test.ESTestCase.assertBusy(ESTestCase.java:714)
   > 		... 37 more

I wasn't able to find a cause or reproduce the issue though :(