Skip to content

add lenient to synonym and synonym_graph token filters #3428

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Oct 17, 2018
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions src/Nest/Analysis/TokenFilters/Synonym/SynonymGraphTokenFilter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ public interface ISynonymGraphTokenFilter : ITokenFilter

[JsonProperty("tokenizer")]
string Tokenizer { get; set; }

/// <inheritdoc cref="ISynonymTokenFilter.Lenient"/>
[JsonProperty("lenient")]
bool? Lenient { get; set; }
}

/// <inheritdoc/>
Expand All @@ -54,6 +58,9 @@ public SynonymGraphTokenFilter() : base("synonym_graph") { }
/// <inheritdoc/>
public bool? Expand { get; set; }

/// <inheritdoc cref="ISynonymTokenFilter.Lenient"/>
public bool? Lenient { get; set; }

/// <inheritdoc/>
public string Tokenizer { get; set; }
}
Expand All @@ -65,6 +72,7 @@ public class SynonymGraphTokenFilterDescriptor

bool? ISynonymGraphTokenFilter.IgnoreCase { get; set; }
bool? ISynonymGraphTokenFilter.Expand { get; set; }
bool? ISynonymGraphTokenFilter.Lenient { get; set; }
string ISynonymGraphTokenFilter.Tokenizer { get; set; }
string ISynonymGraphTokenFilter.SynonymsPath { get; set; }
SynonymFormat? ISynonymGraphTokenFilter.Format{ get; set; }
Expand All @@ -78,6 +86,10 @@ public class SynonymGraphTokenFilterDescriptor
///<inheritdoc/>
public SynonymGraphTokenFilterDescriptor Expand(bool? expand = true) => Assign(a => a.Expand = expand);

/// <inheritdoc cref="ISynonymTokenFilter.Lenient"/>
public SynonymGraphTokenFilterDescriptor Lenient(bool? lenient = true) => Assign(a => a.Lenient = lenient);


///<inheritdoc/>
public SynonymGraphTokenFilterDescriptor Tokenizer(string tokenizer) => Assign(a => a.Tokenizer = tokenizer);

Expand Down
14 changes: 14 additions & 0 deletions src/Nest/Analysis/TokenFilters/Synonym/SynonymTokenFilter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,13 @@ public interface ISynonymTokenFilter : ITokenFilter

[JsonProperty("tokenizer")]
string Tokenizer { get; set; }

/// <summary>
/// If `true` ignores exceptions while parsing the synonym configuration. It is important
// to note that only those synonym rules which cannot get parsed are ignored.
/// </summary>
[JsonProperty("lenient")]
bool? Lenient { get; set; }
}

/// <inheritdoc/>
Expand All @@ -53,6 +60,9 @@ public SynonymTokenFilter() : base("synonym") { }
/// <inheritdoc/>
public bool? Expand { get; set; }

/// <inheritdoc cref="ISynonymTokenFilter.Lenient"/>
public bool? Lenient { get; set; }

/// <inheritdoc/>
public string Tokenizer { get; set; }
}
Expand All @@ -64,6 +74,7 @@ public class SynonymTokenFilterDescriptor

bool? ISynonymTokenFilter.IgnoreCase { get; set; }
bool? ISynonymTokenFilter.Expand { get; set; }
bool? ISynonymTokenFilter.Lenient { get; set; }
string ISynonymTokenFilter.Tokenizer { get; set; }
string ISynonymTokenFilter.SynonymsPath { get; set; }
SynonymFormat? ISynonymTokenFilter.Format { get; set; }
Expand All @@ -76,6 +87,9 @@ public class SynonymTokenFilterDescriptor
///<inheritdoc/>
public SynonymTokenFilterDescriptor Expand(bool? expand = true) => Assign(a => a.Expand = expand);

/// <inheritdoc cref="ISynonymTokenFilter.Lenient"/>
public SynonymTokenFilterDescriptor Lenient(bool? lenient = true) => Assign(a => a.Lenient = lenient);

///<inheritdoc/>
public SynonymTokenFilterDescriptor Tokenizer(string tokenizer) => Assign(a => a.Tokenizer = tokenizer);

Expand Down
2 changes: 1 addition & 1 deletion src/Nest/Analysis/Tokenizers/Tokenizers.cs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ public Tokenizers(Dictionary<string, ITokenizer> container)
public void Add(string name, ITokenizer analyzer) => BackingDictionary.Add(name, analyzer);
}

public class TokenizersDescriptor :IsADictionaryDescriptorBase<TokenizersDescriptor, ITokenizers, string, ITokenizer>
public class TokenizersDescriptor : IsADictionaryDescriptorBase<TokenizersDescriptor, ITokenizers, string, ITokenizer>
{
public TokenizersDescriptor() : base(new Tokenizers()) { }

Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
using Elastic.Managed.Ephemeral.Plugins;
using Tests.Core.ManagedElasticsearch.NodeSeeders;
using Tests.Core.ManagedElasticsearch.NodeSeeders;
using static Elastic.Managed.Ephemeral.Plugins.ElasticsearchPlugin;

namespace Tests.Core.ManagedElasticsearch.Clusters
{
public class ReadOnlyCluster : ClientTestClusterBase
{
public ReadOnlyCluster() : base(ElasticsearchPlugin.MapperMurmur3) { }
public ReadOnlyCluster() : base(MapperMurmur3) { }

protected override void SeedCluster() => new DefaultSeeder(this.Client).SeedNode();
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,18 @@
using Elastic.Managed.Ephemeral.Plugins;
using Tests.Core.ManagedElasticsearch.NodeSeeders;
using Tests.Core.ManagedElasticsearch.NodeSeeders;
using static Elastic.Managed.Ephemeral.Plugins.ElasticsearchPlugin;

namespace Tests.Core.ManagedElasticsearch.Clusters
{
/// <summary> Use this cluster for api's that do writes. If they are however intrusive or long running consider IntrusiveOperationCluster instead. </summary>
public class WritableCluster : ClientTestClusterBase
{
public WritableCluster() : base(new ClientTestClusterConfiguration(
ElasticsearchPlugin.IngestGeoIp, ElasticsearchPlugin.IngestAttachment, ElasticsearchPlugin.AnalysisKuromoji, ElasticsearchPlugin.AnalysisIcu, ElasticsearchPlugin.AnalysisPhonetic, ElasticsearchPlugin.MapperMurmur3
IngestGeoIp,
IngestAttachment,
AnalysisKuromoji,
AnalysisIcu,
AnalysisPhonetic,
MapperMurmur3
)
{
MaxConcurrency = 4
Expand Down
2 changes: 1 addition & 1 deletion src/Tests/Tests.Core/Tests.Core.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Tests.Domain\Tests.Domain.csproj" />
<PackageReference Include="Elastic.Xunit" Version="0.1.0-ci20180902T153954" />
<PackageReference Include="Elastic.Xunit" Version="0.1.0-ci20180925T171717" />
<PackageReference Include="xunit" Version="2.3.1" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.5.0" />
<PackageReference Include="FluentAssertions" Version="4.19.2" />
Expand Down
2 changes: 1 addition & 1 deletion src/Tests/Tests.Domain/Tests.Domain.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Bogus" Version="22.1.2" />
<PackageReference Include="Elastic.Managed" Version="0.1.0-ci20180902T153954" />
<PackageReference Include="Elastic.Managed" Version="0.1.0-ci20180925T171717" />
<ProjectReference Include="..\Tests.Configuration\Tests.Configuration.csproj" />
</ItemGroup>
</Project>
82 changes: 82 additions & 0 deletions src/Tests/Tests/Analysis/AnalysisComponentTestBase.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Elastic.Xunit;
using Elastic.Xunit.XunitPlumbing;
using FluentAssertions;
using Nest;
using Tests.Core.Client;
using Tests.Core.ManagedElasticsearch.Clusters;
using Tests.Core.Serialization;
using Tests.Framework.Integration;

namespace Tests.Analysis
{
public interface IAnalysisAssertion
{
string Name { get; }
object Json { get; }
}
public interface IAnalysisAssertion<out TComponent, out TContainer, in TDescriptor> : IAnalysisAssertion
where TContainer : class
{
TComponent Initializer { get; }
Func<string, TDescriptor, IPromise<TContainer>> Fluent { get; }
}

[IntegrationTestCluster(typeof(WritableCluster))]
public abstract class AnalysisComponentTestBase<TAssertion, TComponent, TContainer, TDescriptor>
: IAnalysisAssertion<TComponent, TContainer, TDescriptor>
where TAssertion : AnalysisComponentTestBase<TAssertion, TComponent, TContainer, TDescriptor>, new()
where TContainer : class
{
private static readonly SingleEndpointUsage<ICreateIndexResponse> Usage = new SingleEndpointUsage<ICreateIndexResponse>
(
fluent: (s, c) => c.CreateIndex(s, AssertionSetup.FluentCall),
fluentAsync: (s, c) => c.CreateIndexAsync(s, AssertionSetup.FluentCall),
request: (s, c) => c.CreateIndex(AssertionSetup.InitializerCall(s)),
requestAsync: (s, c) => c.CreateIndexAsync(AssertionSetup.InitializerCall(s)),
valuePrefix: $"test-{typeof(TAssertion).Name.ToLowerInvariant()}"
)
{
OnAfterCall = c=> c.DeleteIndex(Usage.CallUniqueValues.Value)
};
protected static TAssertion AssertionSetup { get; } = new TAssertion();

protected AnalysisComponentTestBase()
{
this.Client = (ElasticXunitRunner.CurrentCluster as ReadOnlyCluster)?.Client ?? TestClient.DefaultInMemoryClient;
Usage.KickOffOnce(this.Client, oneRandomCall: true);
}

private IElasticClient Client { get; }

public abstract string Name { get; }
public abstract TComponent Initializer { get; }
public abstract Func<string, TDescriptor, IPromise<TContainer>> Fluent { get; }
public abstract object Json { get; }

private Func<CreateIndexDescriptor, ICreateIndexRequest> FluentCall => i =>i.Settings(s => s.Analysis(this.FluentAnalysis));
protected abstract IAnalysis FluentAnalysis(AnalysisDescriptor an);

private CreateIndexRequest InitializerCall(string index) => new CreateIndexRequest(index)
{
Settings = new IndexSettings { Analysis = this.InitializerAnalysis() }
};
protected abstract Nest.Analysis InitializerAnalysis();

[U] public virtual async Task TestPutSettingsRequest() => await Usage.AssertOnAllResponses(r =>
{
var json = new { settings = new { analysis = this.AnalysisJson } };
SerializationTestHelper.Expect(json).FromRequest(r);
});

protected abstract object AnalysisJson { get; }

[I] public virtual async Task TestPutSettingsResponse() => await Usage.AssertOnAllResponses(r =>
{
r.ApiCall.HttpStatusCode.Should().Be(200);
});

}
}
25 changes: 11 additions & 14 deletions src/Tests/Tests/Analysis/AnalysisCrudTests.cs
Original file line number Diff line number Diff line change
@@ -1,19 +1,16 @@
using System.Linq;
using Elastic.Xunit.XunitPlumbing;
using FluentAssertions;
using Nest;
using Tests.Analysis.Tokenizers;
using Tests.Core.Extensions;
using Tests.Core.ManagedElasticsearch.Clusters;
using Tests.Framework;
using Tests.Framework.Integration;
using Tests.Framework.ManagedElasticsearch.Clusters;
using Xunit;
using static Tests.Framework.Promisify;

namespace Tests.Analysis
{

[SkipVersion("<5.2.0", "This tests contains analyzers/tokenfilters not found in previous versions, need a clean way to seperate these out")]
public class AnalysisCrudTests
: CrudWithNoDeleteTestBase<ICreateIndexResponse, IGetIndexSettingsResponse, IUpdateIndexSettingsResponse>
{
Expand Down Expand Up @@ -46,21 +43,21 @@ protected override LazyResponses Create() => Calls<CreateIndexDescriptor, Create
{
Analysis = new Nest.Analysis
{
Analyzers = Analyzers.AnalyzerUsageTests.InitializerExample.Analysis.Analyzers,
CharFilters = CharFilters.CharFilterUsageTests.InitializerExample.Analysis.CharFilters,
Tokenizers = Tokenizers.TokenizerUsageTests.InitializerExample.Analysis.Tokenizers,
TokenFilters = TokenFilters.TokenFilterUsageTests.InitializerExample.Analysis.TokenFilters,
Analyzers = AnalysisUsageTests.AnalyzersInitializer.Analysis.Analyzers,
CharFilters = AnalysisUsageTests.CharFiltersInitializer.Analysis.CharFilters,
Tokenizers = AnalysisUsageTests.TokenizersInitializer.Analysis.Tokenizers,
TokenFilters = AnalysisUsageTests.TokenFiltersInitializer.Analysis.TokenFilters,
}
}
};

protected virtual ICreateIndexRequest CreateFluent(string indexName, CreateIndexDescriptor c) =>
c.Settings(s => s
.Analysis(a => a
.Analyzers(t => Promise(Analyzers.AnalyzerUsageTests.FluentExample(s).Value.Analysis.Analyzers))
.CharFilters(t => Promise(CharFilters.CharFilterUsageTests.FluentExample(s).Value.Analysis.CharFilters))
.Tokenizers(t => Promise(Tokenizers.TokenizerUsageTests.FluentExample(s).Value.Analysis.Tokenizers))
.TokenFilters(t => Promise(TokenFilters.TokenFilterUsageTests.FluentExample(s).Value.Analysis.TokenFilters))
.Analyzers(t => Promise(AnalysisUsageTests.AnalyzersFluent.Analysis.Analyzers))
.CharFilters(t => Promise(AnalysisUsageTests.CharFiltersFluent.Analysis.CharFilters))
.Tokenizers(t => Promise(AnalysisUsageTests.TokenizersFluent.Analysis.Tokenizers))
.TokenFilters(t => Promise(AnalysisUsageTests.TokenFiltersFluent.Analysis.TokenFilters))
)
);

Expand All @@ -82,7 +79,7 @@ protected override LazyResponses Read() => Calls<GetIndexSettingsDescriptor, Get

/**
* Here we assert over the response from `GetIndexSettings()` after the index creation to make sure our analysis chain did infact
* store our html char filter called `stripMe`
* store our html char filter called `htmls`
*/
protected override void ExpectAfterCreate(IGetIndexSettingsResponse response)
{
Expand All @@ -94,7 +91,7 @@ protected override void ExpectAfterCreate(IGetIndexSettingsResponse response)
indexSettings.Analysis.Should().NotBeNull();
indexSettings.Analysis.CharFilters.Should().NotBeNull();

var firstHtmlCharFilter = indexSettings.Analysis.CharFilters["stripMe"];
var firstHtmlCharFilter = indexSettings.Analysis.CharFilters["htmls"];
firstHtmlCharFilter.Should().NotBeNull();
}

Expand Down
Loading