Merge pull request #1125 from elasticsearch/feature/keep-types-filtering

added support for the keep_types token filter fix #1103

Author: gmarz

src/Nest/Domain/Analysis/TokenFilter/KeepTypesTokenFilter.cs

@@ -0,0 +1,24 @@
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	/// <summary>
+	/// A token filter of type keep that only keeps tokens with text contained in a predefined set of words.
+	/// </summary>
+	public class KeepTypesTokenFilter : TokenFilterBase
+	{
+		public KeepTypesTokenFilter()
+			: base("keep_types")
+		{
+
+		}
+
+		/// <summary>
+		/// A list of types to keep.
+		/// </summary>
+		[JsonProperty("types")]
+		public IEnumerable<string> Types { get; set; }
+
+	}
+}

src/Nest/Domain/Analysis/TokenFilter/KeepWordsTokenFilter.cs

@@ -3,34 +3,34 @@
 
 namespace Nest
 {
-    /// <summary>
-    /// A token filter of type keep that only keeps tokens with text contained in a predefined set of words.
-    /// </summary>
-    public class KeepWordsTokenFilter : TokenFilterBase
-    {
-        public KeepWordsTokenFilter()
-            : base("keep")
-        {
+	/// <summary>
+	/// A token filter of type keep that only keeps tokens with text contained in a predefined set of words.
+	/// </summary>
+	public class KeepWordsTokenFilter : TokenFilterBase
+	{
+		public KeepWordsTokenFilter()
+			: base("keep")
+		{
 
-        }
+		}
 
-        /// <summary>
-        /// A list of words to keep.
-        /// </summary>
-        [JsonProperty("keep_words")]
-        public IEnumerable<string> KeepWords { get; set; }
+		/// <summary>
+		/// A list of words to keep.
+		/// </summary>
+		[JsonProperty("keep_words")]
+		public IEnumerable<string> KeepWords { get; set; }
 
-        /// <summary>
-        /// A path to a words file.
-        /// </summary>
-        [JsonProperty("rules_path")]
-        public string KeepWordsPath { get; set; }
+		/// <summary>
+		/// A path to a words file.
+		/// </summary>
+		[JsonProperty("rules_path")]
+		public string KeepWordsPath { get; set; }
 
-        /// <summary>
-        /// A boolean indicating whether to lower case the words.
-        /// </summary>
-        [JsonProperty("keep_words_case")]
+		/// <summary>
+		/// A boolean indicating whether to lower case the words.
+		/// </summary>
+		[JsonProperty("keep_words_case")]
 		public bool? KeepWordsCase { get; set; }
 
-    }
+	}
 }

src/Nest/Nest.csproj

@@ -134,6 +134,7 @@
     <Compile Include="Domain\Analysis\CharFilter\PatternReplaceCharFilter.cs" />
     <Compile Include="Domain\Analysis\TokenFilter\DelimitedPayloadTokenFilter.cs" />
     <Compile Include="Domain\Analysis\TokenFilter\CommonGramsTokenFilter.cs" />
+    <Compile Include="Domain\Analysis\TokenFilter\KeepTypesTokenFilter.cs" />
     <Compile Include="Domain\Analysis\TokenFilter\KeywordRepeatTokenFilter.cs" />
     <Compile Include="Domain\Analysis\TokenFilter\HunspellTokenFilter.cs" />
     <Compile Include="Domain\Analysis\TokenFilter\LimitTokenCountTokenFilter.cs" />

src/Tests/Nest.Tests.Unit/Core/Indices/Analysis/Analyzers/AnalyzerTests.cs

@@ -116,5 +116,17 @@ public void WhitespaceAnalyzerTest()
 
 			this.JsonEquals(result.ConnectionStatus.Request, MethodInfo.GetCurrentMethod());
 		}
+		
+		[Test]
+		public void KeepTypesTokenFilter()
+		{
+			var result = this.Analysis(a => a
+				.TokenFilters(tf => tf
+					.Add("keep", new KeepTypesTokenFilter {Types = new[] {"<NUM>"}})
+				)
+			);
+
+			this.JsonEquals(result.ConnectionStatus.Request, MethodInfo.GetCurrentMethod());
+		}
 	}
 }

src/Tests/Nest.Tests.Unit/Core/Indices/Analysis/Analyzers/KeepTypesTokenFilter.json

@@ -0,0 +1,14 @@
+{
+  "settings": {
+    "index": {
+      "analysis": {
+        "filter": {
+          "keep": {
+            "types" :  [ "<NUM>" ],
+            "type": "keep_types"
+          }
+        }
+      }
+    }
+  }
+}

src/Tests/Nest.Tests.Unit/Nest.Tests.Unit.csproj

@@ -148,6 +148,9 @@
     <None Include="Core\Indices\Analysis\Analyzers\LanguageAnalyzerTest.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Include="Core\Indices\Analysis\Analyzers\KeepTypesTokenFilter.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
     <None Include="Core\Indices\Analysis\Analyzers\WhitespaceAnalyzerTest.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>