Skip to content

Commit 9b42920

Browse files
Mubashir Kaziabbende
Mubashir Kazia
authored and
bbende
committed
NIFI-7269 - Upgrade solrj version to 7 in nifi-solr-processors
Remove unused imports Use the latest solrj version(8.4.1) Setup default schemaFactory for tests The default schemaFactory ManagedIndexSchemaFactory creates additional files in test's resources directory. Change it to ClassicIndexSchemaFactory for classic behavior. This closes apache#4152. Signed-off-by: Bryan Bende <[email protected]>
1 parent 6412097 commit 9b42920

File tree

9 files changed

+150
-109
lines changed

9 files changed

+150
-109
lines changed

‎NOTICE

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,9 +66,9 @@ This includes derived works from Apache Calcite available under Apache Software
6666
and nifi-nar-bundles/nifi-standard-nar/nifi-standard-processors/../FlowFileTableScan
6767

6868
This includes derived works from Apache Solr available under Apache Software License V2. Portions of the code found in
69-
https://github.com/apache/lucene-solr/blob/branch_6x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Krb5HttpClientConfigurer.java
70-
Copyright 2006-2018 The Apache Software Foundation
71-
The code can be found nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/kerberos/KerberosHttpClientConfigurer.java
69+
https://github.com/apache/lucene-solr/blob/branch_7x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Krb5HttpClientBuilder.java
70+
Copyright 2006-2020 The Apache Software Foundation
71+
The code can be found nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/kerberos/KerberosHttpClientBuilder.java
7272

7373
This includes derived works from Apache Hadoop available under Apache Software License V2. Portions of the code found in
7474
https://github.com/apache/hadoop/blob/release-2.7.3-RC2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java

‎nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
<artifactId>nifi-solr-processors</artifactId>
2424
<packaging>jar</packaging>
2525
<properties>
26-
<solr.version>6.6.6</solr.version>
26+
<solr.version>8.4.1</solr.version>
2727
</properties>
2828
<dependencies>
2929
<dependency>

‎nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/KerberosHttpClientBuilder.java

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.nifi.processors.solr;
18+
19+
import java.lang.invoke.MethodHandles;
20+
import java.security.Principal;
21+
import java.util.Optional;
22+
23+
import org.apache.http.HttpEntity;
24+
import org.apache.http.HttpEntityEnclosingRequest;
25+
import org.apache.http.HttpRequestInterceptor;
26+
import org.apache.http.auth.AuthSchemeProvider;
27+
import org.apache.http.auth.AuthScope;
28+
import org.apache.http.auth.Credentials;
29+
import org.apache.http.client.CredentialsProvider;
30+
import org.apache.http.client.config.AuthSchemes;
31+
import org.apache.http.config.Lookup;
32+
import org.apache.http.config.RegistryBuilder;
33+
import org.apache.http.cookie.CookieSpecProvider;
34+
import org.apache.http.entity.BufferedHttpEntity;
35+
import org.apache.http.impl.auth.SPNegoSchemeFactory;
36+
import org.apache.http.impl.client.BasicCredentialsProvider;
37+
import org.apache.solr.client.solrj.impl.HttpClientBuilderFactory;
38+
import org.apache.solr.client.solrj.impl.HttpClientUtil;
39+
import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder;
40+
import org.apache.solr.client.solrj.impl.SolrPortAwareCookieSpecFactory;
41+
import org.slf4j.Logger;
42+
import org.slf4j.LoggerFactory;
43+
44+
/**
45+
* This class is a modified version of Krb5HttpClientBuilder that is part of SolrJ.
46+
*
47+
* In our case we don't want to warn about the useSubjectCreds property since we know we are going to do a
48+
* login and will have subject creds, and we also don't want to mess the static JAAS configuration of the JVM.
49+
*/
50+
public class KerberosHttpClientBuilder implements HttpClientBuilderFactory {
51+
52+
private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
53+
54+
public KerberosHttpClientBuilder() {
55+
56+
}
57+
58+
public SolrHttpClientBuilder getBuilder() {
59+
return getBuilder(HttpClientUtil.getHttpClientBuilder());
60+
}
61+
62+
public void close() {
63+
HttpClientUtil.removeRequestInterceptor(bufferedEntityInterceptor);
64+
}
65+
66+
@Override
67+
public SolrHttpClientBuilder getHttpClientBuilder(Optional<SolrHttpClientBuilder> builder) {
68+
return builder.isPresent() ? getBuilder(builder.get()) : getBuilder();
69+
}
70+
71+
public SolrHttpClientBuilder getBuilder(SolrHttpClientBuilder builder) {
72+
73+
//Enable only SPNEGO authentication scheme.
74+
75+
builder.setAuthSchemeRegistryProvider(() -> {
76+
Lookup<AuthSchemeProvider> authProviders = RegistryBuilder.<AuthSchemeProvider>create()
77+
.register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true, false))
78+
.build();
79+
return authProviders;
80+
});
81+
// Get the credentials from the JAAS configuration rather than here
82+
Credentials useJaasCreds = new Credentials() {
83+
public String getPassword() {
84+
return null;
85+
}
86+
public Principal getUserPrincipal() {
87+
return null;
88+
}
89+
};
90+
91+
HttpClientUtil.setCookiePolicy(SolrPortAwareCookieSpecFactory.POLICY_NAME);
92+
93+
builder.setCookieSpecRegistryProvider(() -> {
94+
SolrPortAwareCookieSpecFactory cookieFactory = new SolrPortAwareCookieSpecFactory();
95+
96+
Lookup<CookieSpecProvider> cookieRegistry = RegistryBuilder.<CookieSpecProvider> create()
97+
.register(SolrPortAwareCookieSpecFactory.POLICY_NAME, cookieFactory).build();
98+
99+
return cookieRegistry;
100+
});
101+
102+
builder.setDefaultCredentialsProvider(() -> {
103+
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
104+
credentialsProvider.setCredentials(AuthScope.ANY, useJaasCreds);
105+
return credentialsProvider;
106+
});
107+
HttpClientUtil.addRequestInterceptor(bufferedEntityInterceptor);
108+
return builder;
109+
}
110+
111+
// Set a buffered entity based request interceptor
112+
private HttpRequestInterceptor bufferedEntityInterceptor = (request, context) -> {
113+
if(request instanceof HttpEntityEnclosingRequest) {
114+
HttpEntityEnclosingRequest enclosingRequest = ((HttpEntityEnclosingRequest) request);
115+
HttpEntity requestEntity = enclosingRequest.getEntity();
116+
enclosingRequest.setEntity(new BufferedHttpEntity(requestEntity));
117+
}
118+
};
119+
}

‎nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/KerberosHttpClientConfigurer.java

Lines changed: 0 additions & 92 deletions
This file was deleted.

‎nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/main/java/org/apache/nifi/processors/solr/SolrUtils.java

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -69,10 +69,12 @@
6969
import java.time.ZoneId;
7070
import java.time.format.DateTimeFormatter;
7171
import java.util.ArrayList;
72+
import java.util.Arrays;
7273
import java.util.HashMap;
7374
import java.util.LinkedHashMap;
7475
import java.util.List;
7576
import java.util.Map;
77+
import java.util.Optional;
7678
import java.util.SortedMap;
7779
import java.util.TreeMap;
7880
import java.util.concurrent.TimeUnit;
@@ -246,7 +248,7 @@ public static SolrClient createSolrClient(final PropertyContext context, final S
246248

247249
// has to happen before the client is created below so that correct configurer would be set if needed
248250
if (kerberosCredentialsService != null || (!StringUtils.isBlank(kerberosPrincipal) && !StringUtils.isBlank(kerberosPassword))) {
249-
HttpClientUtil.setConfigurer(new KerberosHttpClientConfigurer());
251+
HttpClientUtil.setHttpClientBuilder(new KerberosHttpClientBuilder().getHttpClientBuilder(Optional.empty()));
250252
}
251253

252254
final HttpClient httpClient = HttpClientUtil.createClient(params);
@@ -259,13 +261,21 @@ public static SolrClient createSolrClient(final PropertyContext context, final S
259261
}
260262

261263
if (SOLR_TYPE_STANDARD.getValue().equals(context.getProperty(SOLR_TYPE).getValue())) {
262-
return new HttpSolrClient(solrLocation, httpClient);
264+
return new HttpSolrClient.Builder(solrLocation).withHttpClient(httpClient).build();
263265
} else {
266+
// CloudSolrClient.Builder now requires a List of ZK addresses and znode for solr as separate parameters
267+
final String zk[] = solrLocation.split("/");
268+
final List zkList = Arrays.asList(zk[0].split(","));
269+
String zkRoot = "/";
270+
if (zk.length > 1 && ! zk[1].isEmpty()) {
271+
zkRoot += zk[1];
272+
}
273+
264274
final String collection = context.getProperty(COLLECTION).evaluateAttributeExpressions().getValue();
265275
final Integer zkClientTimeout = context.getProperty(ZK_CLIENT_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
266276
final Integer zkConnectionTimeout = context.getProperty(ZK_CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue();
267277

268-
CloudSolrClient cloudSolrClient = new CloudSolrClient(solrLocation, httpClient);
278+
CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder(zkList, Optional.of(zkRoot)).withHttpClient(httpClient).build();
269279
cloudSolrClient.setDefaultCollection(collection);
270280
cloudSolrClient.setZkClientTimeout(zkClientTimeout);
271281
cloudSolrClient.setZkConnectTimeout(zkConnectionTimeout);

‎nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/QuerySolrIT.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.apache.solr.client.solrj.SolrClient;
3131
import org.apache.solr.client.solrj.SolrServerException;
3232
import org.apache.solr.client.solrj.impl.CloudSolrClient;
33+
import org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider;
3334
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
3435
import org.apache.solr.common.SolrInputDocument;
3536
import org.junit.AfterClass;
@@ -82,7 +83,8 @@ public class QuerySolrIT {
8283
public static void setup() throws IOException, SolrServerException {
8384
CloudSolrClient solrClient = createSolrClient();
8485
Path currentDir = Paths.get(ZK_CONFIG_PATH);
85-
solrClient.uploadConfig(currentDir, ZK_CONFIG_NAME);
86+
ZkClientClusterStateProvider stateProvider = new ZkClientClusterStateProvider(SOLR_LOCATION);
87+
stateProvider.uploadConfig(currentDir, ZK_CONFIG_NAME);
8688
solrClient.setDefaultCollection(SOLR_COLLECTION);
8789

8890
if (!solrClient.getZkStateReader().getClusterState().hasCollection(SOLR_COLLECTION)) {

‎nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/TestGetSolr.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -355,7 +355,7 @@ public void testForValidXml() throws IOException, SolrServerException, Initializ
355355
runner.assertAllFlowFilesTransferred(GetSolr.REL_SUCCESS, 1);
356356
runner.assertAllFlowFilesContainAttribute(CoreAttributes.MIME_TYPE.key());
357357

358-
String expectedXml = "<docs><doc boost=\"1.0\"><field name=\"id\">doc1</field></doc></docs>";
358+
String expectedXml = "<docs><doc><field name=\"id\">doc1</field></doc></docs>";
359359
assertThat(expectedXml, CompareMatcher.isIdenticalTo(new String(runner.getContentAsByteArray(runner.getFlowFilesForRelationship(GetSolr.REL_SUCCESS).get(0)))));
360360
}
361361

‎nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/java/org/apache/nifi/processors/solr/TestQuerySolr.java

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -406,7 +406,7 @@ public void testExpressionLanguageForProperties() throws IOException {
406406
runner.run();
407407
runner.assertTransferCount(QuerySolr.RESULTS, 1);
408408

409-
String expectedXml = "<docs><doc boost=\"1.0\"><field name=\"id\">doc2</field></doc><doc boost=\"1.0\"><field name=\"id\">doc1</field></doc></docs>";
409+
String expectedXml = "<docs><doc><field name=\"id\">doc2</field></doc><doc><field name=\"id\">doc1</field></doc></docs>";
410410
assertThat(expectedXml, CompareMatcher.isIdenticalTo(new String(runner.getContentAsByteArray(runner.getFlowFilesForRelationship(QuerySolr.RESULTS).get(0)))));
411411

412412
solrClient.close();
@@ -425,7 +425,7 @@ public void testSingleFilterQuery() throws IOException {
425425
runner.run();
426426
runner.assertTransferCount(QuerySolr.RESULTS, 1);
427427

428-
String expectedXml = "<docs><doc boost=\"1.0\"><field name=\"id\">doc2</field></doc><doc boost=\"1.0\"><field name=\"id\">doc3</field></doc></docs>";
428+
String expectedXml = "<docs><doc><field name=\"id\">doc2</field></doc><doc><field name=\"id\">doc3</field></doc></docs>";
429429
assertThat(expectedXml, CompareMatcher.isIdenticalTo(new String(runner.getContentAsByteArray(runner.getFlowFilesForRelationship(QuerySolr.RESULTS).get(0)))));
430430

431431
solrClient.close();
@@ -447,7 +447,7 @@ public void testMultipleFilterQueries() throws IOException {
447447
runner.run();
448448
runner.assertTransferCount(QuerySolr.RESULTS, 1);
449449

450-
String expectedXml = "<docs><doc boost=\"1.0\"><field name=\"id\">doc2</field></doc><doc boost=\"1.0\"><field name=\"id\">doc3</field></doc></docs>";
450+
String expectedXml = "<docs><doc><field name=\"id\">doc2</field></doc><doc><field name=\"id\">doc3</field></doc></docs>";
451451
assertThat(expectedXml, CompareMatcher.isIdenticalTo(new String(runner.getContentAsByteArray(runner.getFlowFilesForRelationship(QuerySolr.RESULTS).get(0)))));
452452

453453
solrClient.close();
@@ -471,7 +471,7 @@ public void testStandardResponse() throws IOException {
471471
flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_SOLR_STATUS);
472472
flowFile.assertAttributeExists(QuerySolr.ATTRIBUTE_QUERY_TIME);
473473

474-
String expectedXml = "<docs><doc boost=\"1.0\"><field name=\"id\">doc1</field></doc><doc boost=\"1.0\"><field name=\"id\">doc0</field></doc></docs>";
474+
String expectedXml = "<docs><doc><field name=\"id\">doc1</field></doc><doc><field name=\"id\">doc0</field></doc></docs>";
475475
assertThat(expectedXml, CompareMatcher.isIdenticalTo(new String(runner.getContentAsByteArray(flowFile))));
476476

477477
solrClient.close();
@@ -492,7 +492,7 @@ public void testPreserveOriginalContent() throws IOException {
492492
runner.assertTransferCount(QuerySolr.RESULTS, 1);
493493
runner.assertTransferCount(QuerySolr.ORIGINAL, 1);
494494

495-
String expectedXml = "<docs><doc boost=\"1.0\"><field name=\"id\">doc0</field></doc></docs>";
495+
String expectedXml = "<docs><doc><field name=\"id\">doc0</field></doc></docs>";
496496
assertThat(expectedXml, CompareMatcher.isIdenticalTo(new String(runner.getContentAsByteArray(runner.getFlowFilesForRelationship(QuerySolr.RESULTS).get(0)))));
497497
assertEquals(content, new String(runner.getContentAsByteArray(runner.getFlowFilesForRelationship(QuerySolr.ORIGINAL).get(0))));
498498

@@ -526,9 +526,9 @@ public void testRetrievalOfFullResults() throws IOException {
526526
startParam += 2;
527527

528528
StringBuffer expectedXml = new StringBuffer()
529-
.append("<docs><doc boost=\"1.0\"><field name=\"id\">doc")
529+
.append("<docs><doc><field name=\"id\">doc")
530530
.append(documentCounter++)
531-
.append("</field></doc><doc boost=\"1.0\"><field name=\"id\">doc")
531+
.append("</field></doc><doc><field name=\"id\">doc")
532532
.append(documentCounter++)
533533
.append("</field></doc></docs>");
534534
assertThat(expectedXml.toString(), CompareMatcher.isIdenticalTo(new String(runner.getContentAsByteArray(flowFile))));

‎nifi-nar-bundles/nifi-solr-bundle/nifi-solr-processors/src/test/resources/solr/testCollection/conf/solrconfig.xml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,6 @@
1717
<requestHandler name="/select" class="solr.SearchHandler" />
1818
<requestHandler name="/update" class="solr.UpdateRequestHandler" />
1919

20-
</config>
20+
<schemaFactory class="ClassicIndexSchemaFactory" />
21+
22+
</config>

0 commit comments

Comments
 (0)