Skip to content

Move the terms index of _id off-heap. #52405

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Feb 19, 2020
Merged

Move the terms index of _id off-heap. #52405

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5ddb043
Move the terms index of `_id` off-heap.
jpountz Feb 17, 2020
10c42d3
Register the setting.
jpountz Feb 17, 2020
9da21a4
typo
jpountz Feb 19, 2020
8ab9764
Merge branch 'master' into enhancement/offheap_terms_dict
jpountz Feb 19, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
IndexSettings.DEFAULT_PIPELINE,
IndexSettings.FINAL_PIPELINE,
MetaDataIndexStateService.VERIFIED_BEFORE_CLOSE_SETTING,
IndexSettings.ON_HEAP_ID_TERMS_INDEX,

// validate that built-in similarities don't get redefined
Setting.groupSetting(
Expand Down
3 changes: 3 additions & 0 deletions server/src/main/java/org/elasticsearch/index/IndexSettings.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,9 @@ public final class IndexSettings {
"[true, false, checksum] but was: " + s);
}
}, Property.IndexScope);
// This setting is undocumented as it is considered as an escape hatch.
public static final Setting<Boolean> ON_HEAP_ID_TERMS_INDEX =
Setting.boolSetting("index.force_memory_id_terms_dictinary", false, Property.IndexScope);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: dictinary -> dictionary

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for catching this, you were not nit-picking!


/**
* Index setting describing the maximum value of from + size on a query.
Expand Down
18 changes: 11 additions & 7 deletions server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@
import java.io.Closeable;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
Expand Down Expand Up @@ -2074,21 +2075,24 @@ IndexWriter createWriter(Directory directory, IndexWriterConfig iwc) throws IOEx
}
}

static Map<String, String> getReaderAttributes(Directory directory) {
static Map<String, String> getReaderAttributes(Directory directory, IndexSettings indexSettings) {
Directory unwrap = FilterDirectory.unwrap(directory);
boolean defaultOffHeap = FsDirectoryFactory.isHybridFs(unwrap) || unwrap instanceof MMapDirectory;
return Map.of(
BlockTreeTermsReader.FST_MODE_KEY, // if we are using MMAP for term dics we force all off heap unless it's the ID field
defaultOffHeap ? FSTLoadMode.OFF_HEAP.name() : FSTLoadMode.ON_HEAP.name()
, BlockTreeTermsReader.FST_MODE_KEY + "." + IdFieldMapper.NAME, // always force ID field on-heap for fast updates
FSTLoadMode.ON_HEAP.name());
Map<String, String> attributes = new HashMap<>();
attributes.put(BlockTreeTermsReader.FST_MODE_KEY, defaultOffHeap ? FSTLoadMode.OFF_HEAP.name() : FSTLoadMode.ON_HEAP.name());
if (IndexSettings.ON_HEAP_ID_TERMS_INDEX.exists(indexSettings.getSettings())) {
final boolean idOffHeap = IndexSettings.ON_HEAP_ID_TERMS_INDEX.get(indexSettings.getSettings()) == false;
attributes.put(BlockTreeTermsReader.FST_MODE_KEY + "." + IdFieldMapper.NAME,
idOffHeap ? FSTLoadMode.OFF_HEAP.name() : FSTLoadMode.ON_HEAP.name());
}
return Collections.unmodifiableMap(attributes);
}

private IndexWriterConfig getIndexWriterConfig() {
final IndexWriterConfig iwc = new IndexWriterConfig(engineConfig.getAnalyzer());
iwc.setCommitOnClose(false); // we by default don't commit on close
iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND);
iwc.setReaderAttributes(getReaderAttributes(store.directory()));
iwc.setReaderAttributes(getReaderAttributes(store.directory(), engineConfig.getIndexSettings()));
iwc.setIndexDeletionPolicy(combinedDeletionPolicy);
// with tests.verbose, lucene sets this up: plumb to align with filesystem stream
boolean verbose = false;
Expand Down
56 changes: 37 additions & 19 deletions server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5538,53 +5538,71 @@ public void testRefreshAndCloseEngineConcurrently() throws Exception {
}

public void testGetReaderAttributes() throws IOException {
Settings.Builder settingsBuilder = Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
Settings settings = settingsBuilder.build();
IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);

try(BaseDirectoryWrapper dir = newFSDirectory(createTempDir())) {
Directory unwrap = FilterDirectory.unwrap(dir);
boolean isMMap = unwrap instanceof MMapDirectory;
Map<String, String> readerAttributes = InternalEngine.getReaderAttributes(dir);
assertEquals(2, readerAttributes.size());
assertEquals("ON_HEAP", readerAttributes.get("blocktree.terms.fst._id"));
if (isMMap) {
assertEquals("OFF_HEAP", readerAttributes.get("blocktree.terms.fst"));
} else {
assertEquals("ON_HEAP", readerAttributes.get("blocktree.terms.fst"));
}
Map<String, String> readerAttributes = InternalEngine.getReaderAttributes(dir, indexSettings);
assertEquals(Collections.singletonMap("blocktree.terms.fst", isMMap ? "OFF_HEAP" : "ON_HEAP"), readerAttributes);
}

try(MMapDirectory dir = new MMapDirectory(createTempDir())) {
Map<String, String> readerAttributes =
InternalEngine.getReaderAttributes(randomBoolean() ? dir :
new MockDirectoryWrapper(random(), dir));
assertEquals(2, readerAttributes.size());
assertEquals("ON_HEAP", readerAttributes.get("blocktree.terms.fst._id"));
assertEquals("OFF_HEAP", readerAttributes.get("blocktree.terms.fst"));
new MockDirectoryWrapper(random(), dir), indexSettings);
assertEquals(Collections.singletonMap("blocktree.terms.fst", "OFF_HEAP"), readerAttributes);
}

Settings.Builder settingsBuilder = Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
Settings settings = settingsBuilder.build();
IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
FsDirectoryFactory service = new FsDirectoryFactory();
Path tempDir = createTempDir().resolve(indexSettings.getUUID()).resolve("0");
ShardPath path = new ShardPath(false, tempDir, tempDir, new ShardId(indexSettings.getIndex(), 0));
try (Directory directory = service.newDirectory(indexSettings, path)) {

Map<String, String> readerAttributes =
InternalEngine.getReaderAttributes(randomBoolean() ? directory :
new MockDirectoryWrapper(random(), directory));
assertEquals(2, readerAttributes.size());
new MockDirectoryWrapper(random(), directory), indexSettings);
assertEquals(1, readerAttributes.size());

switch (IndexModule.defaultStoreType(true)) {
case HYBRIDFS:
case MMAPFS:
assertEquals("ON_HEAP", readerAttributes.get("blocktree.terms.fst._id"));
assertEquals("OFF_HEAP", readerAttributes.get("blocktree.terms.fst"));
break;
case NIOFS:
case SIMPLEFS:
case FS:
assertEquals("ON_HEAP", readerAttributes.get("blocktree.terms.fst"));
break;
default:
fail("unknownw type");
}
}

settingsBuilder.put(IndexSettings.ON_HEAP_ID_TERMS_INDEX.getKey(), true);
settings = settingsBuilder.build();
indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
try (Directory directory = service.newDirectory(indexSettings, path)) {

Map<String, String> readerAttributes =
InternalEngine.getReaderAttributes(randomBoolean() ? directory :
new MockDirectoryWrapper(random(), directory), indexSettings);
assertEquals(2, readerAttributes.size());

switch (IndexModule.defaultStoreType(true)) {
case HYBRIDFS:
case MMAPFS:
assertEquals("OFF_HEAP", readerAttributes.get("blocktree.terms.fst"));
assertEquals("ON_HEAP", readerAttributes.get("blocktree.terms.fst._id"));
break;
case NIOFS:
case SIMPLEFS:
case FS:
assertEquals("ON_HEAP", readerAttributes.get("blocktree.terms.fst"));
assertEquals("ON_HEAP", readerAttributes.get("blocktree.terms.fst._id"));
break;
default:
fail("unknownw type");
Expand Down
4 changes: 4 additions & 0 deletions test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,10 @@ protected Settings.Builder setRandomIndexSettings(Random random, Settings.Builde
RandomNumbers.randomIntBetween(random, 1, 15) + "ms");
}

if (random.nextBoolean()) {
builder.put(IndexSettings.ON_HEAP_ID_TERMS_INDEX.getKey(), random.nextBoolean());
}

return builder;
}

Expand Down
1 change: 1 addition & 0 deletions ...gin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -401,6 +401,7 @@ static String[] extractLeaderShardHistoryUUIDs(Map<String, String> ccrIndexMetaD
IndexSettings.INDEX_FLUSH_AFTER_MERGE_THRESHOLD_SIZE_SETTING,
IndexSettings.INDEX_GC_DELETES_SETTING,
IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD,
IndexSettings.ON_HEAP_ID_TERMS_INDEX,
IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING,
BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING,
SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING,
Expand Down