Skip to content

Commit bc9ec5a

Browse files
dadoonetdadoonet
committed
Add some tokenizer and analyzer tests
Related to #9.
1 parent 4da84bb commit bc9ec5a

File tree

4 files changed

+157
-2
lines changed

4 files changed

+157
-2
lines changed

‎pom.xml

-2
Original file line numberDiff line numberDiff line change
@@ -114,8 +114,6 @@
114114
<goal>junit4</goal>
115115
</goals>
116116
<configuration>
117-
<!-- We don't have any test yet so we disable launch -->
118-
<skipTests>true</skipTests>
119117
<heartbeat>20</heartbeat>
120118
<jvmOutputAction>pipe,warn</jvmOutputAction>
121119
<leaveTemporary>true</leaveTemporary>

‎src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
/*
2+
* Licensed to ElasticSearch and Shay Banon under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. ElasticSearch licenses this
6+
* file to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.index.analysis;
21+
22+
import org.apache.lucene.analysis.Analyzer;
23+
import org.apache.lucene.analysis.pl.PolishAnalyzer;
24+
import org.elasticsearch.common.inject.Injector;
25+
import org.elasticsearch.common.inject.ModulesBuilder;
26+
import org.elasticsearch.common.settings.SettingsModule;
27+
import org.elasticsearch.env.Environment;
28+
import org.elasticsearch.env.EnvironmentModule;
29+
import org.elasticsearch.index.Index;
30+
import org.elasticsearch.index.IndexNameModule;
31+
import org.elasticsearch.index.analysis.pl.PolishAnalysisBinderProcessor;
32+
import org.elasticsearch.index.analysis.pl.PolishStemTokenFilterFactory;
33+
import org.elasticsearch.index.settings.IndexSettingsModule;
34+
import org.elasticsearch.indices.analysis.IndicesAnalysisModule;
35+
import org.elasticsearch.indices.analysis.IndicesAnalysisService;
36+
import org.elasticsearch.test.ElasticsearchTestCase;
37+
import org.hamcrest.MatcherAssert;
38+
import org.junit.Test;
39+
40+
import static org.elasticsearch.common.settings.ImmutableSettings.Builder.EMPTY_SETTINGS;
41+
import static org.hamcrest.Matchers.instanceOf;
42+
43+
/**
44+
*/
45+
public class PolishAnalysisTests extends ElasticsearchTestCase {
46+
47+
@Test
48+
public void testDefaultsPolishAnalysis() {
49+
Index index = new Index("test");
50+
51+
Injector parentInjector = new ModulesBuilder().add(new SettingsModule(EMPTY_SETTINGS), new EnvironmentModule(new Environment(EMPTY_SETTINGS)), new IndicesAnalysisModule()).createInjector();
52+
Injector injector = new ModulesBuilder().add(
53+
new IndexSettingsModule(index, EMPTY_SETTINGS),
54+
new IndexNameModule(index),
55+
new AnalysisModule(EMPTY_SETTINGS, parentInjector.getInstance(IndicesAnalysisService.class)).addProcessor(new PolishAnalysisBinderProcessor()))
56+
.createChildInjector(parentInjector);
57+
58+
AnalysisService analysisService = injector.getInstance(AnalysisService.class);
59+
60+
TokenFilterFactory tokenizerFactory = analysisService.tokenFilter("polish_stem");
61+
MatcherAssert.assertThat(tokenizerFactory, instanceOf(PolishStemTokenFilterFactory.class));
62+
63+
Analyzer analyzer = analysisService.analyzer("polish").analyzer();
64+
MatcherAssert.assertThat(analyzer, instanceOf(PolishAnalyzer.class));
65+
}
66+
}

‎src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java

+86
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
package org.elasticsearch.index.analysis;
2+
3+
import org.apache.lucene.analysis.Analyzer;
4+
import org.apache.lucene.analysis.TokenStream;
5+
import org.apache.lucene.analysis.core.KeywordTokenizer;
6+
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
7+
import org.elasticsearch.common.inject.Injector;
8+
import org.elasticsearch.common.inject.ModulesBuilder;
9+
import org.elasticsearch.common.settings.ImmutableSettings;
10+
import org.elasticsearch.common.settings.Settings;
11+
import org.elasticsearch.common.settings.SettingsModule;
12+
import org.elasticsearch.env.Environment;
13+
import org.elasticsearch.env.EnvironmentModule;
14+
import org.elasticsearch.index.Index;
15+
import org.elasticsearch.index.IndexNameModule;
16+
import org.elasticsearch.index.analysis.pl.PolishAnalysisBinderProcessor;
17+
import org.elasticsearch.index.settings.IndexSettingsModule;
18+
import org.elasticsearch.indices.analysis.IndicesAnalysisModule;
19+
import org.elasticsearch.indices.analysis.IndicesAnalysisService;
20+
import org.elasticsearch.test.ElasticsearchTestCase;
21+
import org.junit.Test;
22+
23+
import java.io.IOException;
24+
import java.io.StringReader;
25+
26+
import static org.hamcrest.Matchers.equalTo;
27+
28+
public class SimplePolishTokenFilterTests extends ElasticsearchTestCase {
29+
30+
@Test
31+
public void testBasicUsage() throws Exception {
32+
testToken("kwiaty", "kwć");
33+
testToken("canona", "ć");
34+
testToken("wirtualna", "wirtualny");
35+
testToken("polska", "polski");
36+
37+
testAnalyzer("wirtualna polska", "wirtualny", "polski");
38+
}
39+
40+
private void testToken(String source, String expected) throws IOException {
41+
Index index = new Index("test");
42+
Settings settings = ImmutableSettings.settingsBuilder()
43+
.put("index.analysis.filter.myStemmer.type", "polish_stem")
44+
.build();
45+
AnalysisService analysisService = createAnalysisService(index, settings);
46+
47+
TokenFilterFactory filterFactory = analysisService.tokenFilter("myStemmer");
48+
49+
TokenStream ts = filterFactory.create(new KeywordTokenizer(new StringReader(source)));
50+
51+
CharTermAttribute term1 = ts.addAttribute(CharTermAttribute.class);
52+
ts.reset();
53+
assertThat(ts.incrementToken(), equalTo(true));
54+
55+
assertThat(term1.toString(), equalTo(expected));
56+
}
57+
58+
private void testAnalyzer(String source, String... expected_terms) throws IOException {
59+
Index index = new Index("test");
60+
Settings settings = ImmutableSettings.settingsBuilder().build();
61+
AnalysisService analysisService = createAnalysisService(index, settings);
62+
63+
Analyzer analyzer = analysisService.analyzer("polish").analyzer();
64+
65+
TokenStream ts = analyzer.tokenStream("test", source);
66+
67+
CharTermAttribute term1 = ts.addAttribute(CharTermAttribute.class);
68+
ts.reset();
69+
70+
for (String expected : expected_terms) {
71+
assertThat(ts.incrementToken(), equalTo(true));
72+
assertThat(term1.toString(), equalTo(expected));
73+
}
74+
}
75+
76+
private AnalysisService createAnalysisService(Index index, Settings settings) {
77+
Injector parentInjector = new ModulesBuilder().add(new SettingsModule(settings), new EnvironmentModule(new Environment(settings)), new IndicesAnalysisModule()).createInjector();
78+
Injector injector = new ModulesBuilder().add(
79+
new IndexSettingsModule(index, settings),
80+
new IndexNameModule(index),
81+
new AnalysisModule(settings, parentInjector.getInstance(IndicesAnalysisService.class)).addProcessor(new PolishAnalysisBinderProcessor()))
82+
.createChildInjector(parentInjector);
83+
84+
return injector.getInstance(AnalysisService.class);
85+
}
86+
}

‎src/test/resources/log4j.properties

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
log4j.rootLogger=INFO, out
2+
3+
log4j.appender.out=org.apache.log4j.ConsoleAppender
4+
log4j.appender.out.layout=org.apache.log4j.PatternLayout
5+
log4j.appender.out.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c] %m%n

0 commit comments

Comments
 (0)