Skip to content

Commit 67d86de

Browse files
kimchykimchy
committed
Gateway: Store cluster meta data in JSON (and not binary), closes #36.
1 parent 575250e commit 67d86de

File tree

5 files changed

+253
-12
lines changed

5 files changed

+253
-12
lines changed

‎modules/elasticsearch/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java

+76-4
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,14 @@
2020
package org.elasticsearch.cluster.metadata;
2121

2222
import com.google.common.collect.ImmutableMap;
23+
import org.codehaus.jackson.JsonParser;
24+
import org.codehaus.jackson.JsonToken;
2325
import org.elasticsearch.util.MapBuilder;
26+
import org.elasticsearch.util.Nullable;
2427
import org.elasticsearch.util.Preconditions;
2528
import org.elasticsearch.util.concurrent.Immutable;
29+
import org.elasticsearch.util.json.JsonBuilder;
30+
import org.elasticsearch.util.json.ToJson;
2631
import org.elasticsearch.util.settings.ImmutableSettings;
2732
import org.elasticsearch.util.settings.Settings;
2833

@@ -52,8 +57,8 @@ public class IndexMetaData {
5257
private transient final int totalNumberOfShards;
5358

5459
private IndexMetaData(String index, Settings settings, ImmutableMap<String, String> mappings) {
55-
Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_SHARDS, -1) != -1, "must specify numberOfShards");
56-
Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, -1) != -1, "must specify numberOfReplicas");
60+
Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_SHARDS, -1) != -1, "must specify numberOfShards for index [" + index + "]");
61+
Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, -1) != -1, "must specify numberOfReplicas for index [" + index + "]");
5762
this.index = index;
5863
this.settings = settings;
5964
this.mappings = mappings;
@@ -115,7 +120,7 @@ public String index() {
115120
}
116121

117122
public Builder numberOfShards(int numberOfShards) {
118-
settings = ImmutableSettings.settingsBuilder().putAll(settings).putInt(SETTING_NUMBER_OF_SHARDS, numberOfShards).build();
123+
settings = settingsBuilder().putAll(settings).putInt(SETTING_NUMBER_OF_SHARDS, numberOfShards).build();
119124
return this;
120125
}
121126

@@ -124,14 +129,19 @@ public int numberOfShards() {
124129
}
125130

126131
public Builder numberOfReplicas(int numberOfReplicas) {
127-
settings = ImmutableSettings.settingsBuilder().putAll(settings).putInt(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas).build();
132+
settings = settingsBuilder().putAll(settings).putInt(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas).build();
128133
return this;
129134
}
130135

131136
public int numberOfReplicas() {
132137
return settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, -1);
133138
}
134139

140+
public Builder settings(Settings.Builder settings) {
141+
this.settings = settings.build();
142+
return this;
143+
}
144+
135145
public Builder settings(Settings settings) {
136146
this.settings = settings;
137147
return this;
@@ -151,6 +161,68 @@ public IndexMetaData build() {
151161
return new IndexMetaData(index, settings, mappings.immutableMap());
152162
}
153163

164+
public static void toJson(IndexMetaData indexMetaData, JsonBuilder builder, ToJson.Params params) throws IOException {
165+
builder.startObject(indexMetaData.index());
166+
167+
builder.startObject("settings");
168+
for (Map.Entry<String, String> entry : indexMetaData.settings().getAsMap().entrySet()) {
169+
builder.field(entry.getKey(), entry.getValue());
170+
}
171+
builder.endObject();
172+
173+
builder.startObject("mappings");
174+
for (Map.Entry<String, String> entry : indexMetaData.mappings().entrySet()) {
175+
builder.startObject(entry.getKey());
176+
builder.field("source", entry.getValue());
177+
builder.endObject();
178+
}
179+
builder.endObject();
180+
181+
builder.endObject();
182+
}
183+
184+
public static IndexMetaData fromJson(JsonParser jp, @Nullable Settings globalSettings) throws IOException {
185+
Builder builder = new Builder(jp.getCurrentName());
186+
187+
String currentFieldName = null;
188+
JsonToken token = jp.nextToken();
189+
while ((token = jp.nextToken()) != JsonToken.END_OBJECT) {
190+
if (token == JsonToken.FIELD_NAME) {
191+
currentFieldName = jp.getCurrentName();
192+
} else if (token == JsonToken.START_OBJECT) {
193+
if ("settings".equals(currentFieldName)) {
194+
ImmutableSettings.Builder settingsBuilder = settingsBuilder().globalSettings(globalSettings);
195+
while ((token = jp.nextToken()) != JsonToken.END_OBJECT) {
196+
String key = jp.getCurrentName();
197+
token = jp.nextToken();
198+
String value = jp.getText();
199+
settingsBuilder.put(key, value);
200+
}
201+
builder.settings(settingsBuilder.build());
202+
} else if ("mappings".equals(currentFieldName)) {
203+
while ((token = jp.nextToken()) != JsonToken.END_OBJECT) {
204+
String mappingType = jp.getCurrentName();
205+
String mappingSource = null;
206+
while ((token = jp.nextToken()) != JsonToken.END_OBJECT) {
207+
if (token == JsonToken.FIELD_NAME) {
208+
if ("source".equals(jp.getCurrentName())) {
209+
jp.nextToken();
210+
mappingSource = jp.getText();
211+
}
212+
}
213+
}
214+
if (mappingSource == null) {
215+
// crap, no mapping source, warn?
216+
} else {
217+
builder.putMapping(mappingType, mappingSource);
218+
}
219+
}
220+
}
221+
}
222+
}
223+
return builder.build();
224+
}
225+
154226
public static IndexMetaData readFrom(DataInput in, Settings globalSettings) throws ClassNotFoundException, IOException {
155227
Builder builder = new Builder(in.readUTF());
156228
builder.settings(readSettingsFromStream(in, globalSettings));

‎modules/elasticsearch/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java

+49
Original file line numberDiff line numberDiff line change
@@ -21,16 +21,21 @@
2121

2222
import com.google.common.collect.ImmutableMap;
2323
import com.google.common.collect.UnmodifiableIterator;
24+
import org.codehaus.jackson.JsonParser;
25+
import org.codehaus.jackson.JsonToken;
2426
import org.elasticsearch.util.MapBuilder;
2527
import org.elasticsearch.util.Nullable;
2628
import org.elasticsearch.util.concurrent.Immutable;
29+
import org.elasticsearch.util.json.JsonBuilder;
30+
import org.elasticsearch.util.json.ToJson;
2731
import org.elasticsearch.util.settings.Settings;
2832

2933
import java.io.DataInput;
3034
import java.io.DataOutput;
3135
import java.io.IOException;
3236

3337
import static org.elasticsearch.util.MapBuilder.*;
38+
import static org.elasticsearch.util.json.JsonBuilder.*;
3439

3540
/**
3641
* @author kimchy (Shay Banon)
@@ -120,6 +125,50 @@ public MetaData build() {
120125
return new MetaData(indices.immutableMap(), maxNumberOfShardsPerNode);
121126
}
122127

128+
public static String toJson(MetaData metaData) throws IOException {
129+
JsonBuilder builder = jsonBuilder().prettyPrint();
130+
builder.startObject();
131+
toJson(metaData, builder, ToJson.EMPTY_PARAMS);
132+
builder.endObject();
133+
return builder.string();
134+
}
135+
136+
public static void toJson(MetaData metaData, JsonBuilder builder, ToJson.Params params) throws IOException {
137+
builder.startObject("meta-data");
138+
builder.field("maxNumberOfShardsPerNode", metaData.maxNumberOfShardsPerNode());
139+
140+
builder.startObject("indices");
141+
for (IndexMetaData indexMetaData : metaData) {
142+
IndexMetaData.Builder.toJson(indexMetaData, builder, params);
143+
}
144+
builder.endObject();
145+
146+
builder.endObject();
147+
}
148+
149+
public static MetaData fromJson(JsonParser jp, @Nullable Settings globalSettings) throws IOException {
150+
Builder builder = new Builder();
151+
152+
String currentFieldName = null;
153+
JsonToken token = jp.nextToken();
154+
while ((token = jp.nextToken()) != JsonToken.END_OBJECT) {
155+
if (token == JsonToken.FIELD_NAME) {
156+
currentFieldName = jp.getCurrentName();
157+
} else if (token == JsonToken.START_OBJECT) {
158+
if ("indices".equals(currentFieldName)) {
159+
while ((token = jp.nextToken()) != JsonToken.END_OBJECT) {
160+
builder.put(IndexMetaData.Builder.fromJson(jp, globalSettings));
161+
}
162+
}
163+
} else if (token == JsonToken.VALUE_NUMBER_INT) {
164+
if ("maxNumberOfShardsPerNode".equals(currentFieldName)) {
165+
builder.maxNumberOfShardsPerNode(jp.getIntValue());
166+
}
167+
}
168+
}
169+
return builder.build();
170+
}
171+
123172
public static MetaData readFrom(DataInput in, @Nullable Settings globalSettings) throws IOException, ClassNotFoundException {
124173
Builder builder = new Builder();
125174
builder.maxNumberOfShardsPerNode(in.readInt());

‎modules/elasticsearch/src/main/java/org/elasticsearch/gateway/fs/FsGateway.java

+19-8
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121

2222
import com.google.inject.Inject;
2323
import com.google.inject.Module;
24+
import org.codehaus.jackson.JsonEncoding;
25+
import org.codehaus.jackson.JsonParser;
2426
import org.elasticsearch.ElasticSearchException;
2527
import org.elasticsearch.cluster.ClusterName;
2628
import org.elasticsearch.cluster.metadata.MetaData;
@@ -30,12 +32,16 @@
3032
import org.elasticsearch.index.gateway.fs.FsIndexGatewayModule;
3133
import org.elasticsearch.util.component.AbstractComponent;
3234
import org.elasticsearch.util.component.Lifecycle;
33-
import org.elasticsearch.util.io.FastDataOutputStream;
3435
import org.elasticsearch.util.io.FileSystemUtils;
36+
import org.elasticsearch.util.json.Jackson;
37+
import org.elasticsearch.util.json.JsonBuilder;
38+
import org.elasticsearch.util.json.ToJson;
3539
import org.elasticsearch.util.settings.Settings;
3640

3741
import java.io.*;
3842

43+
import static org.elasticsearch.util.io.FileSystemUtils.*;
44+
3945
/**
4046
* @author kimchy (Shay Banon)
4147
*/
@@ -150,13 +156,17 @@ private static File createGatewayHome(String location, Environment environment,
150156
}
151157

152158
FileOutputStream fileStream = new FileOutputStream(file);
153-
FastDataOutputStream outStream = new FastDataOutputStream(fileStream);
154159

155-
MetaData.Builder.writeTo(metaData, outStream);
160+
JsonBuilder builder = new JsonBuilder(Jackson.defaultJsonFactory().createJsonGenerator(fileStream, JsonEncoding.UTF8));
161+
builder.prettyPrint();
162+
builder.startObject();
163+
MetaData.Builder.toJson(metaData, builder, ToJson.EMPTY_PARAMS);
164+
builder.endObject();
165+
builder.close();
156166

157-
outStream.close();
167+
fileStream.close();
158168

159-
FileSystemUtils.syncFile(file);
169+
syncFile(file);
160170

161171
currentIndex++;
162172

@@ -187,11 +197,12 @@ private static File createGatewayHome(String location, Environment environment,
187197
}
188198

189199
FileInputStream fileStream = new FileInputStream(file);
190-
DataInputStream inStream = new DataInputStream(fileStream);
191200

192-
MetaData metaData = MetaData.Builder.readFrom(inStream, settings);
201+
JsonParser jp = Jackson.defaultJsonFactory().createJsonParser(fileStream);
202+
MetaData metaData = MetaData.Builder.fromJson(jp, settings);
203+
jp.close();
193204

194-
inStream.close();
205+
fileStream.close();
195206

196207
return metaData;
197208

‎modules/elasticsearch/src/main/java/org/elasticsearch/util/json/JsonBuilder.java

+7
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import org.apache.lucene.util.UnicodeUtil;
2323
import org.codehaus.jackson.JsonFactory;
24+
import org.codehaus.jackson.JsonGenerator;
2425
import org.elasticsearch.ElasticSearchException;
2526
import org.elasticsearch.util.concurrent.NotThreadSafe;
2627
import org.elasticsearch.util.io.FastCharArrayWriter;
@@ -94,6 +95,12 @@ public JsonBuilder(JsonFactory factory) throws IOException {
9495
this.generator = factory.createJsonGenerator(writer);
9596
}
9697

98+
public JsonBuilder(JsonGenerator generator) throws IOException {
99+
this.writer = new FastCharArrayWriter();
100+
this.generator = generator;
101+
this.factory = null;
102+
}
103+
97104
public JsonBuilder prettyPrint() {
98105
generator.useDefaultPrettyPrinter();
99106
return this;

‎modules/elasticsearch/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java

+102
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
/*
2+
* Licensed to Elastic Search and Shay Banon under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. Elastic Search licenses this
6+
* file to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.cluster.metadata;
21+
22+
import org.elasticsearch.util.json.Jackson;
23+
import org.testng.annotations.Test;
24+
25+
import java.io.IOException;
26+
27+
import static org.elasticsearch.cluster.metadata.IndexMetaData.*;
28+
import static org.elasticsearch.cluster.metadata.MetaData.*;
29+
import static org.elasticsearch.util.settings.ImmutableSettings.*;
30+
import static org.hamcrest.MatcherAssert.*;
31+
import static org.hamcrest.Matchers.*;
32+
33+
/**
34+
* @author kimchy (shay.banon)
35+
*/
36+
@Test
37+
public class ToAndFromJsonMetaDataTests {
38+
39+
@Test
40+
public void testSimpleJsonFromAndTo() throws IOException {
41+
MetaData metaData = newMetaDataBuilder()
42+
.maxNumberOfShardsPerNode(2)
43+
.put(newIndexMetaDataBuilder("test1")
44+
.numberOfShards(1)
45+
.numberOfReplicas(2))
46+
.put(newIndexMetaDataBuilder("test2")
47+
.settings(settingsBuilder().put("setting1", "value1").put("setting2", "value2"))
48+
.numberOfShards(2)
49+
.numberOfReplicas(3))
50+
.put(newIndexMetaDataBuilder("test3")
51+
.numberOfShards(1)
52+
.numberOfReplicas(2)
53+
.putMapping("mapping1", MAPPING_SOURCE1))
54+
.put(newIndexMetaDataBuilder("test4")
55+
.settings(settingsBuilder().put("setting1", "value1").put("setting2", "value2"))
56+
.numberOfShards(1)
57+
.numberOfReplicas(2)
58+
.putMapping("mapping1", MAPPING_SOURCE1)
59+
.putMapping("mapping2", MAPPING_SOURCE2))
60+
.build();
61+
62+
String metaDataSource = MetaData.Builder.toJson(metaData);
63+
System.out.println("ToJson: " + metaDataSource);
64+
65+
MetaData parsedMetaData = MetaData.Builder.fromJson(Jackson.defaultJsonFactory().createJsonParser(metaDataSource), null);
66+
assertThat(parsedMetaData.maxNumberOfShardsPerNode(), equalTo(2));
67+
68+
IndexMetaData indexMetaData = metaData.index("test1");
69+
assertThat(indexMetaData.numberOfShards(), equalTo(1));
70+
assertThat(indexMetaData.numberOfReplicas(), equalTo(2));
71+
assertThat(indexMetaData.settings().getAsMap().size(), equalTo(2));
72+
assertThat(indexMetaData.mappings().size(), equalTo(0));
73+
74+
indexMetaData = metaData.index("test2");
75+
assertThat(indexMetaData.numberOfShards(), equalTo(2));
76+
assertThat(indexMetaData.numberOfReplicas(), equalTo(3));
77+
assertThat(indexMetaData.settings().getAsMap().size(), equalTo(4));
78+
assertThat(indexMetaData.settings().get("setting1"), equalTo("value1"));
79+
assertThat(indexMetaData.settings().get("setting2"), equalTo("value2"));
80+
assertThat(indexMetaData.mappings().size(), equalTo(0));
81+
82+
indexMetaData = metaData.index("test3");
83+
assertThat(indexMetaData.numberOfShards(), equalTo(1));
84+
assertThat(indexMetaData.numberOfReplicas(), equalTo(2));
85+
assertThat(indexMetaData.settings().getAsMap().size(), equalTo(2));
86+
assertThat(indexMetaData.mappings().size(), equalTo(1));
87+
assertThat(indexMetaData.mappings().get("mapping1"), equalTo(MAPPING_SOURCE1));
88+
89+
indexMetaData = metaData.index("test4");
90+
assertThat(indexMetaData.numberOfShards(), equalTo(1));
91+
assertThat(indexMetaData.numberOfReplicas(), equalTo(2));
92+
assertThat(indexMetaData.settings().getAsMap().size(), equalTo(4));
93+
assertThat(indexMetaData.settings().get("setting1"), equalTo("value1"));
94+
assertThat(indexMetaData.settings().get("setting2"), equalTo("value2"));
95+
assertThat(indexMetaData.mappings().size(), equalTo(2));
96+
assertThat(indexMetaData.mappings().get("mapping1"), equalTo(MAPPING_SOURCE1));
97+
assertThat(indexMetaData.mappings().get("mapping2"), equalTo(MAPPING_SOURCE2));
98+
}
99+
100+
private static final String MAPPING_SOURCE1 = "{ text1: { type : \"string\" } }";
101+
private static final String MAPPING_SOURCE2 = "{ text2: { type : \"string\" } }";
102+
}

0 commit comments

Comments
 (0)