Skip to content

Commit d5c9142

Browse files
dnhatndnhatn
committed
move bitset to common
1 parent 0f6f3b8 commit d5c9142

File tree

3 files changed

+213
-47
lines changed

3 files changed

+213
-47
lines changed

‎core/src/main/java/org/elasticsearch/common/util/CountedBitSet.java

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.common.util;
21+
22+
import org.apache.lucene.util.BitSet;
23+
import org.apache.lucene.util.FixedBitSet;
24+
25+
/**
26+
* A {@link CountedBitSet} wraps a {@link FixedBitSet} but automatically releases the internal bitset
27+
* when all bits are set to reduce memory usage. This structure can work well for sequence numbers
28+
* from translog as these numbers are likely to form contiguous ranges (eg. filling all bits).
29+
*/
30+
public final class CountedBitSet extends BitSet {
31+
private short onBits; // Number of bits are set.
32+
private FixedBitSet bitset;
33+
34+
public CountedBitSet(short numBits) {
35+
assert numBits > 0;
36+
this.onBits = 0;
37+
this.bitset = new FixedBitSet(numBits);
38+
}
39+
40+
@Override
41+
public boolean get(int index) {
42+
assert index >= 0;
43+
assert bitset == null || onBits < bitset.length() : "Bitset should be released when all bits are set";
44+
45+
return bitset == null ? true : bitset.get(index);
46+
}
47+
48+
@Override
49+
public void set(int index) {
50+
assert index >= 0;
51+
assert bitset == null || onBits < bitset.length() : "Bitset should be released when all bits are set";
52+
53+
// Ignore set when bitset is full.
54+
if (bitset != null) {
55+
boolean wasOn = bitset.getAndSet(index);
56+
if (wasOn == false) {
57+
onBits++;
58+
// Once all bits are set, we can simply just return YES for all indexes.
59+
// This allows us to clear the internal bitset and use null check as the guard.
60+
if (onBits == bitset.length()) {
61+
bitset = null;
62+
}
63+
}
64+
}
65+
}
66+
67+
@Override
68+
public void clear(int startIndex, int endIndex) {
69+
throw new UnsupportedOperationException("Not implemented yet");
70+
}
71+
72+
@Override
73+
public void clear(int index) {
74+
throw new UnsupportedOperationException("Not implemented yet");
75+
}
76+
77+
@Override
78+
public int cardinality() {
79+
return onBits;
80+
}
81+
82+
@Override
83+
public int length() {
84+
return bitset == null ? onBits : bitset.length();
85+
}
86+
87+
@Override
88+
public int prevSetBit(int index) {
89+
throw new UnsupportedOperationException("Not implemented yet");
90+
}
91+
92+
@Override
93+
public int nextSetBit(int index) {
94+
throw new UnsupportedOperationException("Not implemented yet");
95+
}
96+
97+
@Override
98+
public long ramBytesUsed() {
99+
throw new UnsupportedOperationException("Not implemented yet");
100+
}
101+
102+
// Exposed for testing
103+
boolean isInternalBitsetReleased() {
104+
return bitset == null;
105+
}
106+
}

‎core/src/main/java/org/elasticsearch/index/translog/MultiSnapshot.java

Lines changed: 10 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@
2121

2222
import com.carrotsearch.hppc.LongObjectHashMap;
2323
import com.carrotsearch.hppc.cursors.ObjectCursor;
24-
import org.apache.lucene.util.FixedBitSet;
24+
import org.apache.lucene.util.BitSet;
25+
import org.elasticsearch.common.util.CountedBitSet;
2526
import org.elasticsearch.index.seqno.SequenceNumbers;
2627

2728
import java.io.Closeable;
@@ -83,70 +84,32 @@ public void close() throws IOException {
8384
onClose.close();
8485
}
8586

86-
/**
87-
* A {@link CountedBitSet} wraps a {@link FixedBitSet} but automatically releases the internal bitset
88-
* when all bits are set to reduce memory usage. This structure can work well for sequence numbers
89-
* from translog as these numbers are likely to form contiguous ranges (eg. filling all bits).
90-
*/
91-
private static final class CountedBitSet {
92-
private short onBits; // Number of bits are set.
93-
private FixedBitSet bitset;
94-
95-
CountedBitSet(short numBits) {
96-
assert numBits > 0;
97-
this.onBits = 0;
98-
this.bitset = new FixedBitSet(numBits);
99-
}
100-
101-
boolean getAndSet(int index) {
102-
assert index >= 0;
103-
assert bitset == null || onBits < bitset.length() : "Bitset should be cleared when all bits are set";
104-
105-
// A null bitset means all bits are set.
106-
if (bitset == null) {
107-
return true;
108-
}
109-
110-
boolean wasOn = bitset.getAndSet(index);
111-
if (wasOn == false) {
112-
onBits++;
113-
// Once all bits are set, we can simply just return YES for all indexes.
114-
// This allows us to clear the internal bitset and use null check as the guard.
115-
if (onBits == bitset.length()) {
116-
bitset = null;
117-
}
118-
}
119-
return wasOn;
120-
}
121-
122-
boolean hasAllBitsOn() {
123-
return bitset == null;
124-
}
125-
}
126-
12787
static final class SeqNoSet {
12888
static final short BIT_SET_SIZE = 1024;
129-
private final LongObjectHashMap<CountedBitSet> bitSets = new LongObjectHashMap<>();
89+
private final LongObjectHashMap<BitSet> bitSets = new LongObjectHashMap<>();
13090

13191
/**
13292
* Marks this sequence number and returns <tt>true</tt> if it is seen before.
13393
*/
13494
boolean getAndSet(long value) {
13595
assert value >= 0;
13696
final long key = value / BIT_SET_SIZE;
137-
CountedBitSet bitset = bitSets.get(key);
97+
BitSet bitset = bitSets.get(key);
13898
if (bitset == null) {
13999
bitset = new CountedBitSet(BIT_SET_SIZE);
140100
bitSets.put(key, bitset);
141101
}
142-
return bitset.getAndSet(Math.toIntExact(value % BIT_SET_SIZE));
102+
final int index = Math.toIntExact(value % BIT_SET_SIZE);
103+
final boolean wasOn = bitset.get(index);
104+
bitset.set(index);
105+
return wasOn;
143106
}
144107

145108
// For testing
146109
long completeSetsSize() {
147110
int completedBitSets = 0;
148-
for (ObjectCursor<CountedBitSet> bitset : bitSets.values()) {
149-
if (bitset.value.hasAllBitsOn()) {
111+
for (ObjectCursor<BitSet> bitset : bitSets.values()) {
112+
if (bitset.value.cardinality() == bitset.value.length()) {
150113
completedBitSets++;
151114
}
152115
}

‎core/src/test/java/org/elasticsearch/common/util/CountedBitSetTests.java

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.common.util;
21+
22+
import org.apache.lucene.util.FixedBitSet;
23+
import org.elasticsearch.test.ESTestCase;
24+
25+
import java.util.List;
26+
import java.util.stream.Collectors;
27+
import java.util.stream.IntStream;
28+
29+
import static org.hamcrest.Matchers.equalTo;
30+
31+
public class CountedBitSetTests extends ESTestCase {
32+
33+
public void testCompareToFixedBitset() {
34+
int numBits = (short) randomIntBetween(8, 4096);
35+
final FixedBitSet fixedBitSet = new FixedBitSet(numBits);
36+
final CountedBitSet countedBitSet = new CountedBitSet((short) numBits);
37+
38+
final int iterations = iterations(1000, 20000);
39+
for (int i = 0; i < iterations; i++) {
40+
final int key = randomInt(numBits - 1);
41+
assertThat(countedBitSet.get(key), equalTo(fixedBitSet.get(key)));
42+
if (frequently()) {
43+
countedBitSet.set(key);
44+
fixedBitSet.set(key);
45+
assertThat(countedBitSet.get(key), equalTo(fixedBitSet.get(key)));
46+
assertThat(countedBitSet.length(), equalTo(fixedBitSet.length()));
47+
assertThat(countedBitSet.cardinality(), equalTo(fixedBitSet.cardinality()));
48+
}
49+
}
50+
}
51+
52+
public void testReleaseInternalBitSet() {
53+
int numBits = (short) randomIntBetween(8, 4096);
54+
final CountedBitSet countedBitSet = new CountedBitSet((short) numBits);
55+
final List<Integer> values = IntStream.range(0, numBits).boxed().collect(Collectors.toList());
56+
57+
for (int i = 1; i < numBits; i++) {
58+
final int value = values.get(i);
59+
assertThat(countedBitSet.get(value), equalTo(false));
60+
assertThat(countedBitSet.isInternalBitsetReleased(), equalTo(false));
61+
62+
countedBitSet.set(value);
63+
64+
assertThat(countedBitSet.get(value), equalTo(true));
65+
assertThat(countedBitSet.isInternalBitsetReleased(), equalTo(false));
66+
assertThat(countedBitSet.length(), equalTo(numBits));
67+
assertThat(countedBitSet.cardinality(), equalTo(i));
68+
}
69+
70+
// The missing piece to fill all bits.
71+
{
72+
final int value = values.get(0);
73+
assertThat(countedBitSet.get(value), equalTo(false));
74+
assertThat(countedBitSet.isInternalBitsetReleased(), equalTo(false));
75+
76+
countedBitSet.set(value);
77+
78+
assertThat(countedBitSet.get(value), equalTo(true));
79+
assertThat(countedBitSet.isInternalBitsetReleased(), equalTo(true));
80+
assertThat(countedBitSet.length(), equalTo(numBits));
81+
assertThat(countedBitSet.cardinality(), equalTo(numBits));
82+
}
83+
84+
// Tests with released internal bitset.
85+
final int iterations = iterations(1000, 10000);
86+
for (int i = 0; i < iterations; i++) {
87+
final int value = randomInt(numBits - 1);
88+
assertThat(countedBitSet.get(value), equalTo(true));
89+
assertThat(countedBitSet.isInternalBitsetReleased(), equalTo(true));
90+
assertThat(countedBitSet.length(), equalTo(numBits));
91+
assertThat(countedBitSet.cardinality(), equalTo(numBits));
92+
if (frequently()) {
93+
assertThat(countedBitSet.get(value), equalTo(true));
94+
}
95+
}
96+
}
97+
}

0 commit comments

Comments
 (0)