Skip to content

Commit 05ee848

Browse files
authored
Updates to use external resource files (#150)
1 parent 4cca17c commit 05ee848

File tree

12 files changed

+177
-40
lines changed

12 files changed

+177
-40
lines changed

.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,5 @@
99
.settings
1010
bin
1111
build
12+
li-hadoop-plugin/src/main/resources
1213
ligradle

VERSIONS.md

+4
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,10 @@ the License.
1717
Note that the LinkedIn build system occasionally requires that we skip a
1818
version bump, so you will see a few skipped version numbers in the list below.
1919

20+
0.11.23
21+
22+
* Updates to use external resource files
23+
2024
0.11.22
2125

2226
* LIHADOOP-28747 Hadoop plugin's test plugin should have functionality to add assertions on tests.

gradle.properties

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
org.gradle.daemon=true
2-
version=0.11.22
2+
version=0.11.23

hadoop-plugin/src/main/groovy/com/linkedin/gradle/azkaban/AzkabanFlowStatusTask.groovy

+4-4
Original file line numberDiff line numberDiff line change
@@ -252,12 +252,12 @@ class AzkabanFlowStatusTask extends DefaultTask {
252252
}
253253

254254
/**
255-
* Builds Dr. Elephant URL
255+
* Builds the URL to Dr. Elephant for a given flow.
256256
*
257-
* @param execUrl
258-
* @return DrElephantUrl
257+
* @param execUrl The flow execution URL
258+
* @return The URL to Dr. Elephant for the given flow
259259
*/
260-
public String getDrElephantURL(String execUrl) {
260+
String getDrElephantURL(String execUrl) {
261261
final String DR_ELEPHANT_URL = null;
262262
if (DR_ELEPHANT_URL != null) {
263263
return DR_ELEPHANT_URL + "/search?flow-exec-id=" + URLEncoder.encode(execUrl, "UTF-8").toString();

hadoop-plugin/src/main/groovy/com/linkedin/gradle/dependency/CheckDependencyTask.groovy

+1-1
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,7 @@ class CheckDependencyTask extends DefaultTask {
202202
String jsonTextToParse = null;
203203

204204
if (Thread.currentThread().getContextClassLoader().getResource(dependencyPatternFile) != null) {
205-
jsonTextToParse = Thread.currentThread().getContextClassLoader().getResource(getDependencyPatternFile(project)).text;
205+
jsonTextToParse = Thread.currentThread().getContextClassLoader().getResource(dependencyPatternFile).text;
206206
} else if (new File(dependencyPatternFile).exists()) {
207207
jsonTextToParse = new File(dependencyPatternFile).text;
208208
} else {

li-hadoop-plugin/src/main/groovy/com/linkedin/gradle/liazkaban/LiAzkabanFlowStatusTask.groovy

+30-21
Original file line numberDiff line numberDiff line change
@@ -13,37 +13,46 @@
1313
* License for the specific language governing permissions and limitations under
1414
* the License.
1515
*/
16-
package com.linkedin.gradle.liazkaban;
16+
package com.linkedin.gradle.liazkaban
1717

18-
import com.linkedin.gradle.azkaban.AzkabanFlowStatusTask;
18+
import com.linkedin.gradle.azkaban.AzkabanFlowStatusTask
19+
import com.linkedin.gradle.lihadoop.LiHadoopProperties
1920

2021
class LiAzkabanFlowStatusTask extends AzkabanFlowStatusTask {
2122

23+
private static final String DR_ELEPHANT_HLD_URL = LiHadoopProperties.get(LiHadoopProperties.DR_ELEPHANT_HLD_URL)
24+
private static final String DR_ELEPHANT_PKE_URL = LiHadoopProperties.get(LiHadoopProperties.DR_ELEPHANT_PKE_URL)
25+
private static final String DR_ELEPHANT_SPD_URL = LiHadoopProperties.get(LiHadoopProperties.DR_ELEPHANT_SPD_URL)
26+
private static final String DR_ELEPHANT_TRK_URL = LiHadoopProperties.get(LiHadoopProperties.DR_ELEPHANT_TRK_URL)
27+
private static final String DR_ELEPHANT_WAR_URL = LiHadoopProperties.get(LiHadoopProperties.DR_ELEPHANT_WAR_URL)
28+
private static final String HLD_GRID_NAME = LiHadoopProperties.get(LiHadoopProperties.HLD_GRID_NAME)
29+
private static final String PKE_GRID_NAME = LiHadoopProperties.get(LiHadoopProperties.PKE_GRID_NAME)
30+
private static final String SPD_GRID_NAME = LiHadoopProperties.get(LiHadoopProperties.SPD_GRID_NAME)
31+
private static final String TRK_GRID_NAME = LiHadoopProperties.get(LiHadoopProperties.TRK_GRID_NAME)
32+
private static final String WAR_GRID_NAME = LiHadoopProperties.get(LiHadoopProperties.WAR_GRID_NAME)
33+
2234
/**
23-
* Configuring to print Dr. Elephant URL
35+
* Builds the URL to Dr. Elephant for a given flow.
2436
*
25-
* @param execUrl Execution URL of the flow.
26-
* @return DrElephantUrl
37+
* @param execUrl The flow execution URL
38+
* @return The URL to Dr. Elephant for the given flow
2739
*/
2840
@Override
2941
String getDrElephantURL(String execUrl) {
30-
final String DR_ELEPHANT_HOLDEM_URL = "http://ltx1-holdemdre01.grid.linkedin.com:8080/new#/workflow?referrer=hadoopplugin&workflowid=";
31-
final String DR_ELEPHANT_WAR_URL = "http://lva1-wardre01.grid.linkedin.com:8080/new#/workflow?referrer=hadoopplugin&workflowid=";
32-
final String DR_ELEPHANT_TAROCK_URL = "http://lva1-tarockaz01.grid.linkedin.com:8080/search?referrer=hadoopplugin&flow-exec-id=";
33-
final String DR_ELEPHANT_SPADES_URL = "http://lva1-spadesaz01.grid.linkedin.com:8080/search?referrer=hadoopplugin&flow-exec-id=";
34-
final String DR_ELEPHANT_POKEMON_URL = "http://lva1-pokemonaz01.grid.linkedin.com:8080/new#/workflow?referrer=hadoopplugin&workflowid=";
42+
String drUrl = null
3543

36-
if (execUrl.contains("holdem")) {
37-
return DR_ELEPHANT_HOLDEM_URL + URLEncoder.encode(execUrl, "UTF-8").toString();
38-
} else if (execUrl.contains("war")) {
39-
return DR_ELEPHANT_WAR_URL + URLEncoder.encode(execUrl, "UTF-8").toString();
40-
} else if (execUrl.contains("tarock")) {
41-
return DR_ELEPHANT_TAROCK_URL + URLEncoder.encode(execUrl, "UTF-8").toString();
42-
} else if (execUrl.contains("spades")) {
43-
return DR_ELEPHANT_SPADES_URL + URLEncoder.encode(execUrl, "UTF-8").toString();
44-
} else if (execUrl.contains("pokemon")) {
45-
return DR_ELEPHANT_POKEMON_URL + URLEncoder.encode(execUrl, "UTF-8").toString();
44+
if (execUrl.contains(HLD_GRID_NAME)) {
45+
drUrl = DR_ELEPHANT_HLD_URL
46+
} else if (execUrl.contains(PKE_GRID_NAME)) {
47+
drUrl = DR_ELEPHANT_PKE_URL
48+
} else if (execUrl.contains(SPD_GRID_NAME)) {
49+
drUrl = DR_ELEPHANT_SPD_URL
50+
} else if (execUrl.contains(TRK_GRID_NAME)) {
51+
drUrl = DR_ELEPHANT_TRK_URL
52+
} else if (execUrl.contains(WAR_GRID_NAME)) {
53+
drUrl = DR_ELEPHANT_WAR_URL
4654
}
47-
return null;
55+
56+
return (drUrl == null) ? null : drUrl + URLEncoder.encode(execUrl, "UTF-8").toString()
4857
}
4958
}

li-hadoop-plugin/src/main/groovy/com/linkedin/gradle/liazkaban/LiAzkabanPlugin.groovy

+5-3
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import com.linkedin.gradle.azkaban.AzkabanDslCompiler;
1919
import com.linkedin.gradle.azkaban.AzkabanFlowStatusTask;
2020
import com.linkedin.gradle.azkaban.AzkabanPlugin;
2121
import com.linkedin.gradle.azkaban.AzkabanProject;
22+
import com.linkedin.gradle.lihadoop.LiHadoopProperties;
2223

2324
import org.gradle.api.Project;
2425

@@ -27,7 +28,8 @@ import org.gradle.api.Project;
2728
*/
2829
class LiAzkabanPlugin extends AzkabanPlugin {
2930

30-
private static final String MESSAGE_URL = "https://gitli.corp.linkedin.com/hadoop-dev/azkaban-linkedin-files/raw/li-hadoop-plugin_message.txt";
31+
private static final String AZKABAN_URL = LiHadoopProperties.get(LiHadoopProperties.AZKABAN_URL)
32+
private static final String MESSAGE_URL = LiHadoopProperties.get(LiHadoopProperties.MESSAGE_URL)
3133

3234
/**
3335
* Factory method to return the AzkabanFlowStatusTask class. Subclasses can override this method to
@@ -51,9 +53,9 @@ class LiAzkabanPlugin extends AzkabanPlugin {
5153
*/
5254
@Override
5355
AzkabanProject makeDefaultAzkabanProject(Project project) {
54-
AzkabanProject azkabanProject = makeAzkabanProject();
56+
AzkabanProject azkabanProject = makeAzkabanProject(project);
5557
azkabanProject.azkabanProjName = "";
56-
azkabanProject.azkabanUrl = "https://ltx1-holdemaz01.grid.linkedin.com:8443";
58+
azkabanProject.azkabanUrl = AZKABAN_URL;
5759
azkabanProject.azkabanUserName = System.getProperty("user.name");
5860
azkabanProject.azkabanValidatorAutoFix = "true";
5961
azkabanProject.azkabanZipTask = "";
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
/*
2+
* Copyright 2015 LinkedIn Corp.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
package com.linkedin.gradle.lihadoop
17+
18+
/**
19+
* Helper class for loading LinkedIn-specific properties from an external resource file.
20+
* <p>
21+
* For open-source builds, the file will be missing. For internal builds, we will copy in an
22+
* external resource file with the correct values.
23+
*/
24+
class LiHadoopProperties {
25+
26+
// The keys that should exist in this file
27+
public static final String ARTIFACTORY_REPO = "artifactoryRepo"
28+
public static final String AZKABAN_URL = "azkabanUrl"
29+
public static final String MESSAGE_URL = "messageUrl"
30+
public static final String HLD_GATEWAY_HOME = "hldGatewayHome"
31+
public static final String HLD_GATEWAY_NODE = "hldGatewayNode"
32+
public static final String HLD_NAME_NODE_HDFS = "hldNameNodeHdfs"
33+
public static final String HLD_NAME_NODE_WEBHDFS = "hldNameNodeWebHdfs"
34+
public static final String HLD_GRID_NAME = "hldGridName"
35+
public static final String PKE_GRID_NAME = "pkeGridName"
36+
public static final String SPD_GRID_NAME = "spdGridName"
37+
public static final String TRK_GRID_NAME = "trkGridName"
38+
public static final String WAR_GRID_NAME = "warGridName"
39+
public static final String DR_ELEPHANT_HLD_URL = "drElephantHldUrl"
40+
public static final String DR_ELEPHANT_PKE_URL = "drElephantPkeUrl"
41+
public static final String DR_ELEPHANT_SPD_URL = "drElephantSpdUrl"
42+
public static final String DR_ELEPHANT_TRK_URL = "drElephantTrkUrl"
43+
public static final String DR_ELEPHANT_WAR_URL = "drElephantWarUrl"
44+
public static final String OOZIE_URI = "oozieUri"
45+
public static final String REMOTE_PIG_COMMAND = "remotePigCommand"
46+
public static final String REMOTE_SPARK_COMMAND = "remoteSparkCommand"
47+
48+
// For open-source builds the resource file will be missing, which should not cause an error
49+
private static boolean missing = true
50+
51+
// Java properties object that will hold the key-value pairs from the embedded resource file
52+
private static Properties props = null
53+
54+
/**
55+
* Helper function to load the embedded resource property file. This method is synchronized, so
56+
* that the property file will be loaded only once even in the context of multiple threads.
57+
*
58+
* @throws Exception If the embedded resource property file cannot be read
59+
*/
60+
static synchronized void checkLoadResourceProperties() throws Exception {
61+
if (props == null) {
62+
props = new Properties()
63+
64+
// Reference to another class besides this one, since we are in the middle of initializing it
65+
InputStream inputStream = LiHadoopPlugin.class.getResourceAsStream("/linkedin.properties")
66+
if (inputStream != null) {
67+
props.load(inputStream)
68+
missing = false
69+
}
70+
}
71+
}
72+
73+
/**
74+
* Return the property value for the given property key.
75+
* <p>
76+
* Since the underlying Properties class returns null if the key is not found, we will explicitly
77+
* check if the properties object contains the key and throw an exception if it is not found
78+
* (unless this is an open-source build, in which case we return the empty string).
79+
*
80+
* @param key The property key
81+
* @return The property value for the given key
82+
* @throws IllegalArgumentException If the given property key does not exist
83+
*/
84+
static String get(String key) throws IllegalArgumentException {
85+
// Check if we need to load the embedded property file
86+
checkLoadResourceProperties()
87+
88+
// If this is an open-source build, the resource file will be missing. This should not cause an
89+
// error, so just return the empty string.
90+
if (missing) {
91+
return ""
92+
}
93+
94+
if (!props.containsKey(key)) {
95+
throw new IllegalArgumentException("The key " + key + " does not exist in the linkedin.properties file")
96+
} else {
97+
return props.getProperty(key)
98+
}
99+
}
100+
}

li-hadoop-plugin/src/main/groovy/com/linkedin/gradle/lihadoopValidator/LiHadoopValidatorPlugin.groovy

+6-2
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,17 @@ package com.linkedin.gradle.lihadoopValidator;
1818
import com.linkedin.gradle.hadoopValidator.HadoopValidatorPlugin;
1919
import com.linkedin.gradle.hadoopValidator.PigValidator.PigValidatorPlugin;
2020
import com.linkedin.gradle.hadoopValidator.ValidatorConstants;
21+
import com.linkedin.gradle.lihadoop.LiHadoopProperties;
2122
import com.linkedin.gradle.lihadoopValidator.liPigValidator.LiPigValidatorPlugin;
2223

2324
/**
2425
* LinkedIn-specific customizations to the Hadoop Validator Plugin.
2526
*/
2627
class LiHadoopValidatorPlugin extends HadoopValidatorPlugin {
2728

29+
private static final String ARTIFACTORY_REPO = LiHadoopProperties.get(LiHadoopProperties.ARTIFACTORY_REPO)
30+
private static final String HLD_NAME_NODE_HDFS = LiHadoopProperties.get(LiHadoopProperties.HLD_NAME_NODE_HDFS)
31+
2832
/**
2933
* Factory method to return the LiPigValidatorPlugin class. Subclasses can override this method to return
3034
* their own LiPigValidatorPlugin class.
@@ -43,7 +47,7 @@ class LiHadoopValidatorPlugin extends HadoopValidatorPlugin {
4347
*/
4448
@Override
4549
void setValidatorProperties(Properties properties) {
46-
properties.setProperty(ValidatorConstants.NAME_NODE, "hdfs://ltx1-holdemnn01.grid.linkedin.com:9000");
47-
properties.setProperty(ValidatorConstants.REPOSITORY_URL, "http://artifactory.corp.linkedin.com:8081/artifactory/ext-libraries");
50+
properties.setProperty(ValidatorConstants.NAME_NODE, HLD_NAME_NODE_HDFS)
51+
properties.setProperty(ValidatorConstants.REPOSITORY_URL, ARTIFACTORY_REPO)
4852
}
4953
}

li-hadoop-plugin/src/main/groovy/com/linkedin/gradle/lioozie/LiOoziePlugin.groovy

+7-2
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
*/
1616
package com.linkedin.gradle.lioozie;
1717

18+
import com.linkedin.gradle.lihadoop.LiHadoopProperties;
1819
import com.linkedin.gradle.oozie.OozieCommandTask;
1920
import com.linkedin.gradle.oozie.OoziePlugin;
2021
import com.linkedin.gradle.oozie.OozieProject;
@@ -26,6 +27,10 @@ import org.gradle.api.Project;
2627
* Linkedin specific customizations to the OoziePlugin.
2728
*/
2829
class LiOoziePlugin extends OoziePlugin {
30+
31+
private static final String HLD_NAME_NODE_WEBHDFS = LiHadoopProperties.get(LiHadoopProperties.HLD_NAME_NODE_WEBHDFS)
32+
private static final String OOZIE_URI = LiHadoopProperties.get(LiHadoopProperties.OOZIE_URI)
33+
2934
/**
3035
* Factory method to return the OozieUploadTask class. Subclasses can override this method to
3136
* return their own OozieUploadTask class.
@@ -59,8 +64,8 @@ class LiOoziePlugin extends OoziePlugin {
5964
@Override
6065
OozieProject makeDefaultOozieProject(Project project) {
6166
OozieProject oozieProject = makeOozieProject(project);
62-
oozieProject.clusterURI = "webhdfs://ltx1-holdemnn01.grid.linkedin.com:50070";
63-
oozieProject.oozieURI = "http://ltx1-holdemoz01.grid.linkedin.com:11000/oozie/";
67+
oozieProject.clusterURI = HLD_NAME_NODE_WEBHDFS;
68+
oozieProject.oozieURI = OOZIE_URI;
6469
oozieProject.oozieZipTask = "";
6570
oozieProject.projectName = "";
6671
oozieProject.uploadPath = "/user/${System.getProperty('user.name')}";

li-hadoop-plugin/src/main/groovy/com/linkedin/gradle/lipig/LiPigExtension.groovy

+9-3
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
*/
1616
package com.linkedin.gradle.lipig;
1717

18+
import com.linkedin.gradle.lihadoop.LiHadoopProperties;
1819
import com.linkedin.gradle.pig.PigExtension;
1920

2021
import org.gradle.api.Project;
@@ -24,6 +25,11 @@ import org.gradle.api.Project;
2425
* default property values necessary to run the Pig Plugin on our development Hadoop cluster.
2526
*/
2627
class LiPigExtension extends PigExtension {
28+
29+
private static final String HLD_GATEWAY_HOME = LiHadoopProperties.get(LiHadoopProperties.HLD_GATEWAY_HOME)
30+
private static final String HLD_GATEWAY_NODE = LiHadoopProperties.get(LiHadoopProperties.HLD_GATEWAY_NODE)
31+
private static final String REMOTE_PIG_COMMAND = LiHadoopProperties.get(LiHadoopProperties.REMOTE_PIG_COMMAND)
32+
2733
/**
2834
* Constructor for the LiPigExtension.
2935
*
@@ -41,9 +47,9 @@ class LiPigExtension extends PigExtension {
4147
}
4248

4349
this.pigCacheDir = "${System.getProperty('user.home')}/.hadoopPlugin";
44-
this.pigCommand = "/export/apps/pig/linkedin-pig-h2-0.11.1.li69-1/bin/pig";
45-
this.remoteHostName = "ltx1-holdemgw01.grid.linkedin.com";
46-
this.remoteCacheDir = "/export/home/${System.getProperty('user.name')}/.hadoopPlugin";
50+
this.pigCommand = REMOTE_PIG_COMMAND;
51+
this.remoteHostName = HLD_GATEWAY_NODE;
52+
this.remoteCacheDir = "${HLD_GATEWAY_HOME}/${System.getProperty('user.name')}/.hadoopPlugin";
4753
this.remoteSshOpts = "-q -K";
4854
}
4955
}

li-hadoop-plugin/src/main/groovy/com/linkedin/gradle/lispark/LiSparkExtension.groovy

+9-3
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
*/
1616
package com.linkedin.gradle.lispark;
1717

18+
import com.linkedin.gradle.lihadoop.LiHadoopProperties;
1819
import com.linkedin.gradle.spark.SparkExtension;
1920

2021
import org.gradle.api.Project;
@@ -24,6 +25,11 @@ import org.gradle.api.Project;
2425
* default property values necessary to run the Spark Plugin on our development Hadoop cluster.
2526
*/
2627
class LiSparkExtension extends SparkExtension {
28+
29+
private static final String HLD_GATEWAY_HOME = LiHadoopProperties.get(LiHadoopProperties.HLD_GATEWAY_HOME)
30+
private static final String HLD_GATEWAY_NODE = LiHadoopProperties.get(LiHadoopProperties.HLD_GATEWAY_NODE)
31+
private static final String REMOTE_SPARK_COMMAND = LiHadoopProperties.get(LiHadoopProperties.REMOTE_SPARK_COMMAND)
32+
2733
/**
2834
* Constructor for the LiSparkExtension.
2935
*
@@ -32,9 +38,9 @@ class LiSparkExtension extends SparkExtension {
3238
LiSparkExtension(Project project) {
3339
super(project);
3440
this.sparkCacheDir = "${System.getProperty('user.home')}/.hadoopPlugin";
35-
this.sparkCommand = "/export/apps/spark/latest/bin/spark-submit";
36-
this.remoteHostName = "ltx1-holdemgw01.grid.linkedin.com";
37-
this.remoteCacheDir = "/export/home/${System.getProperty('user.name')}/.hadoopPlugin";
41+
this.sparkCommand = REMOTE_SPARK_COMMAND;
42+
this.remoteHostName = HLD_GATEWAY_NODE;
43+
this.remoteCacheDir = "${HLD_GATEWAY_HOME}/${System.getProperty('user.name')}/.hadoopPlugin";
3844
this.remoteSshOpts = "-q -K";
3945
}
4046
}

0 commit comments

Comments
 (0)