Add smartcn_word & smartcn_sentence back for backwards compatibility

In f4d0d27 the deprecated `smartcn_sentence` tokenizer and deprecated
`smartcn_word` token filter were removed as the new all in one
`smartcn_tokenizer` should be used instead. However for those with pre-existing
indices with mappings that reference the deprecated tokenizer and token filter
this changes causes indexing and search errors.

This change preserves the `smartcn_sentence` tokenizer name and aliases it to
the new `smartcn_tokenizer`. The change set also reintroduces a `smartcn_word`
token filter that is a noop filter. The result of these changes should allow
elasticsearch instances with the existing deprecated mappings to upgrade and
take advantage of the new tokenizer in lucene.

(cherry picked from commit 25a0071)

Closes #29

Author: xyu

src/main/java/org/elasticsearch/index/analysis/SmartChineseAnalysisBinderProcessor.java

@@ -31,5 +31,13 @@ public void processAnalyzers(AnalyzersBindings analyzersBindings) {
     @Override
     public void processTokenizers(TokenizersBindings tokenizersBindings) {
         tokenizersBindings.processTokenizer("smartcn_tokenizer", SmartChineseTokenizerTokenizerFactory.class);
+        // This is an alias to "smartcn_tokenizer"; it's here for backwards compat
+        tokenizersBindings.processTokenizer("smartcn_sentence", SmartChineseTokenizerTokenizerFactory.class);
+    }
+
+    @Override
+    public void processTokenFilters(TokenFiltersBindings tokenFiltersBindings) {
+        // This is a noop token filter; it's here for backwards compat before we had "smartcn_tokenizer"
+        tokenFiltersBindings.processTokenFilter("smartcn_word", SmartChineseNoOpTokenFilterFactory.class);
     }
 }

src/main/java/org/elasticsearch/index/analysis/SmartChineseNoOpTokenFilterFactory.java

@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.inject.assistedinject.Assisted;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.settings.IndexSettings;
+
+public class SmartChineseNoOpTokenFilterFactory extends AbstractTokenFilterFactory {
+
+    @Inject
+    public SmartChineseNoOpTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) {
+        super(index, indexSettings, name, settings);
+    }
+
+    @Override
+    public TokenStream create(TokenStream tokenStream) {
+        return tokenStream;
+    }
+}

src/main/java/org/elasticsearch/indices/analysis/smartcn/SmartChineseIndicesAnalysis.java

@@ -20,15 +20,13 @@
 package org.elasticsearch.indices.analysis.smartcn;
 
 import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.cn.smart.HMMChineseTokenizer;
 import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.analysis.AnalyzerScope;
-import org.elasticsearch.index.analysis.PreBuiltAnalyzerProviderFactory;
-import org.elasticsearch.index.analysis.PreBuiltTokenizerFactoryFactory;
-import org.elasticsearch.index.analysis.TokenizerFactory;
+import org.elasticsearch.index.analysis.*;
 import org.elasticsearch.indices.analysis.IndicesAnalysisService;
 
 import java.io.Reader;
@@ -59,5 +57,30 @@ public Tokenizer create(Reader reader) {
             }
         }));
 
+        // Register smartcn_sentence tokenizer -- for backwards compat an alias to smartcn_tokenizer
+        indicesAnalysisService.tokenizerFactories().put("smartcn_sentence", new PreBuiltTokenizerFactoryFactory(new TokenizerFactory() {
+            @Override
+            public String name() {
+                return "smartcn_sentence";
+            }
+
+            @Override
+            public Tokenizer create(Reader reader) {
+                return new HMMChineseTokenizer(reader);
+            }
+        }));
+
+        // Register smartcn_word token filter -- noop
+        indicesAnalysisService.tokenFilterFactories().put("smartcn_word", new PreBuiltTokenFilterFactoryFactory(new TokenFilterFactory() {
+            @Override
+            public String name() {
+                return "smartcn_word";
+            }
+
+            @Override
+            public TokenStream create(TokenStream tokenStream) {
+                return tokenStream;
+            }
+        }));
     }
 }