simplifying build (HADOOP-160)

Author: Justin Lee

Diff for: build.gradle

@@ -8,46 +8,12 @@ apply from: 'gradle/functions.gradle'
 
 ext.configDir = new File(rootDir, 'config')
 ext.hadoopBinaries = "${System.getProperty('user.home')}/hadoop-binaries"
-ext.branchVersion = project.getProperties().get('clusterVersion', '2.4').toLowerCase()
-
-ext.hiveVersionMap = [
-        '1.0' : '0.10.0',
-        '1.1' : '0.10.0',
-        'cdh4': '0.10.0-cdh4.6.0',
-        'cdh5': '0.12.0-cdh5.0.0'
-].withDefault {
-    '0.12.0'
-}
-
-ext.hadoopClusters = [
-        '0.23': '0.23.10',
-        '1.0' : '1.0.4',
-        '1.1' : '1.1.2',
-        '1.2' : '1.2.1',
-        '2.2' : '2.2.0',
-        '2.3' : '2.3.0',
-        '2.4' : '2.4.0',
-        'cdh4': '2.0.0-cdh4.6.0',
-        'cdh5': '2.3.0-cdh5.0.0',
-].withDefault {
-    throw new GradleException("Unknown hadoop version: ${it}")
-}
-
-ext.pigVersionMap = [
-        'cdh4': '0.11.0-cdh4.6.0',
-        'cdh5': '0.12.0-cdh5.0.0'
-].withDefault {
-    '0.13.0'
-}
-
 ext.javaDriverVersion = '2.12.3'
-
-ext.clusterVersion = hadoopClusters[branchVersion]
-ext.hiveVersion = hiveVersionMap[branchVersion]
-ext.pigVersion = pigVersionMap[branchVersion]
+ext.hiveVersion = '0.12.0'
+ext.pigVersion = '0.13.0'
 ext.hadoopVersion = '2.4.1'
 
-ext.hadoopHome = "${hadoopBinaries}/hadoop-${clusterVersion}"
+ext.hadoopHome = "${hadoopBinaries}/hadoop-${hadoopVersion}"
 ext.hiveHome = "${hadoopBinaries}/hive-${hiveVersion}"
 ext.pigHome = "${hadoopBinaries}/pig-${pigVersion}"
 ext.dataHome = "${hadoopBinaries}/examples/data"
@@ -76,7 +42,11 @@ buildscript {
 }
 
 task wrapper(type: Wrapper) {
-    gradleVersion = '1.11'
+    gradleVersion = '1.12'
+}
+
+if(project.hasProperty("clusterVersion")) {
+    println "The cluster version property has been obsoleted.  See https://jira.mongodb.org/browse/HADOOP-160 for details."
 }
 
 configure(subprojects) {
@@ -146,8 +116,8 @@ configure(subprojects) {
     tasks.withType(Test) {
         maxParallelForks = 1
         systemProperty 'project.version', project.version
-        systemProperty 'cluster.version', hadoopClusters[branchVersion]
-        systemProperty 'hadoop.version', branchVersion
+        systemProperty 'cluster.version', hadoopVersion
+        systemProperty 'hadoop.version', hadoopVersion
         systemProperty 'hive.version', hiveVersion
         systemProperty 'pig.version', pigVersion
         systemProperties << System.getProperties()
@@ -158,7 +128,7 @@ configure(subprojects) {
 
         finalizedBy ':shutdownCluster'
     }
-
+    
     test.dependsOn ':configureCluster'
     compileJava.dependsOn ':generateScripts'
     clean.dependsOn ':cleanHadoop'
@@ -433,9 +403,6 @@ def hadoop(jar, className, args) {
     }
     println "Executing hadoop job:\n ${line.join(' \\\n\t')}"
     def hadoopEnv = [:]
-    if (clusterVersion.contains("cdh")) {
-        hadoopEnv.MAPRED_DIR = 'share/hadoop/mapreduce2'
-    }
     hadoopEnv.HADOOP_PREFIX=''
     exec() {
         environment << hadoopEnv

Diff for: core/src/test/java/com/mongodb/hadoop/testutils/BaseHadoopTest.java

@@ -42,8 +42,9 @@ public abstract class BaseHadoopTest {
 
     public static final String HADOOP_HOME;
     public static final String PROJECT_VERSION = loadProperty("project.version", "1.4.0-SNAPSHOT");
-    public static final String HADOOP_VERSION = loadProperty("hadoop.version", "2.4");
-    public static final String CLUSTER_VERSION = loadProperty("cluster.version", "2.4.0");
+    public static final String HADOOP_VERSION = loadProperty("hadoop.version", "2.4.1");
+    @Deprecated
+    public static final String CLUSTER_VERSION = loadProperty("cluster.version", HADOOP_VERSION);
     public static final String HIVE_VERSION = loadProperty("hive.version", "0.12.0");
 
     public static final String HIVE_HOME;
@@ -75,7 +76,7 @@ public abstract class BaseHadoopTest {
             HADOOP_BINARIES = new File(String.format("%s/hadoop-binaries/", System.getProperty("user.home"))).getCanonicalPath();
             EXAMPLE_DATA_HOME = new File(HADOOP_BINARIES, "examples/data").getCanonicalPath();
 
-            HADOOP_HOME = new File(HADOOP_BINARIES, String.format("hadoop-%s/", CLUSTER_VERSION)).getCanonicalPath();
+            HADOOP_HOME = new File(HADOOP_BINARIES, String.format("hadoop-%s/", HADOOP_VERSION)).getCanonicalPath();
             HIVE_HOME = new File(HADOOP_BINARIES, String.format("hive-%s/", HIVE_VERSION)).getCanonicalPath();
 
             File current = new File(".").getCanonicalFile();

Diff for: examples/treasury_yield/src/test/java/com/mongodb/hadoop/StreamingJob.java

@@ -25,12 +25,11 @@
 import java.util.TreeMap;
 import java.util.concurrent.TimeoutException;
 
+import static com.mongodb.hadoop.testutils.BaseHadoopTest.HADOOP_HOME;
+import static com.mongodb.hadoop.testutils.BaseHadoopTest.HADOOP_VERSION;
 import static com.mongodb.hadoop.util.MongoConfigUtil.INPUT_URI;
 import static com.mongodb.hadoop.util.MongoConfigUtil.OUTPUT_URI;
 import static java.lang.String.format;
-import static com.mongodb.hadoop.testutils.BaseHadoopTest.HADOOP_HOME;
-import static com.mongodb.hadoop.testutils.BaseHadoopTest.CLUSTER_VERSION;
-import static com.mongodb.hadoop.testutils.BaseHadoopTest.HADOOP_VERSION;
 
 public class StreamingJob {
     private static final String STREAMING_JAR;
@@ -77,11 +76,11 @@ public StreamingJob() {
         if (HADOOP_VERSION.startsWith("1.1")) {
             cmd.add(String.format("%s/contrib/streaming/hadoop-streaming-%s.jar",
                                   HADOOP_HOME,
-                                  CLUSTER_VERSION));
+                                  HADOOP_VERSION));
         } else {
             cmd.add(String.format("%s/share/hadoop/tools/lib/hadoop-streaming-%s.jar",
                                   HADOOP_HOME,
-                                  CLUSTER_VERSION));
+                                  HADOOP_VERSION));
         }
         //        add("-libjars", STREAMING_JAR);
         add("-io", "mongodb");

Diff for: gradle/functions.gradle

@@ -12,16 +12,16 @@ task generateScripts() << {
         filter ReplaceTokens, tokens: [
                 PROJECT_HOME   : project.rootDir.getAbsolutePath(),
 
-                HADOOP_HOME    : "${hadoopBinaries}/hadoop-${clusterVersion}".toString(),
+                HADOOP_HOME    : "${hadoopBinaries}/hadoop-${hadoopVersion}".toString(),
                 HIVE_HOME      : "${hadoopBinaries}/hive-${hiveVersion}".toString(),
                 PIG_HOME       : "${hadoopBinaries}/pig-${pigVersion}".toString(),
 
 
-                HADOOP_VERSION : clusterVersion.toString(),
+                HADOOP_VERSION : hadoopVersion.toString(),
                 HIVE_VERSION   : hiveVersion.toString(),
 
                 HADOOP_BINARIES: hadoopBinaries.toString(),
-                BIN            : branchVersion.startsWith("1.") ? 'hadoop' : 'hdfs',
+                BIN            : hadoopVersion.startsWith("1.") ? 'hadoop' : 'hdfs',
                 PYTHON_BIN     : python
         ]
         into "build/"
@@ -34,19 +34,8 @@ task generateScripts() << {
 task installHadoop() << {
     new File(hadoopHome).getParentFile().mkdirs()
 
-    def url
-    switch (branchVersion) {
-        case ("cdh4"):
-            url = "http://archive.cloudera.com/cdh4/cdh/4/hadoop-${clusterVersion}.tar.gz"
-            break
-        case ("cdh5"):
-            url = "http://archive.cloudera.com/cdh5/cdh/5/hadoop-${clusterVersion}.tar.gz"
-            break
-        default:
-            url = "http://archive.apache.org/dist/hadoop/common/hadoop-${clusterVersion}/hadoop-${clusterVersion}.tar.gz"
-            break
-    }
-    
+    def url = "http://archive.apache.org/dist/hadoop/common/hadoop-${hadoopVersion}/hadoop-${hadoopVersion}.tar.gz"
+
     download(url, hadoopBinaries, hadoopHome)
 }
 
@@ -85,50 +74,21 @@ def download(url, destination, target) {
 }
 
 task installHive() << {
-    def url
-    switch (branchVersion) {
-        case ("cdh4"):
-            url = "http://archive.cloudera.com/cdh4/cdh/4/hive-${hiveVersion}.tar.gz"
-            break
-        case ("cdh5"):
-            url = "http://archive.cloudera.com/cdh5/cdh/5/hive-${hiveVersion}.tar.gz"
-            break
-        default:
-            url = "https://archive.apache.org/dist/hive/hive-${hiveVersion}/hive-${hiveVersion}.tar.gz"
-            break
-    }
+    def url = "https://archive.apache.org/dist/hive/hive-${hiveVersion}/hive-${hiveVersion}.tar.gz"
 
     download(url, hadoopBinaries, hiveHome)
 
 }
 
 task installPig() << {
-    def url
-    switch (branchVersion) {
-        case ("cdh4"):
-            url = "http://archive.cloudera.com/cdh4/cdh/4/pig-${pigVersion}.tar.gz"
-            break
-        case ("cdh5"):
-            url = "http://archive.cloudera.com/cdh5/cdh/5/pig-${pigVersion}.tar.gz"
-            break
-        default:
-            url = "https://archive.apache.org/dist/pig/pig-${pigVersion}/pig-${pigVersion}.tar.gz"
-            break
-    }
+    def url = "https://archive.apache.org/dist/pig/pig-${pigVersion}/pig-${pigVersion}.tar.gz"
 
     download(url, hadoopBinaries, pigHome)
 }
 
 task copyFiles(dependsOn: [installHadoop, installHive, installPig]) << {
-    def hadoopEtc
-    def hadoopLib
-    if (clusterVersion.startsWith("1")) {
-        hadoopLib = "${hadoopHome}/lib"
-        hadoopEtc = "${hadoopHome}/conf"
-    } else {
-        hadoopLib = "${hadoopHome}/share/hadoop/common"
-        hadoopEtc = "${hadoopHome}/etc/hadoop"
-    }
+    def hadoopEtc = "${hadoopHome}/etc/hadoop"
+    def hadoopLib = "${hadoopHome}/share/hadoop/common"
 
     println "Updating mongo jars"
 
@@ -149,7 +109,7 @@ def findJar(String proj, String filter) {
 }
 
 task configureCluster(dependsOn: copyFiles) << {
-    println "Configuring ${branchVersion} hadoop cluster"
+    println "Configuring ${hadoopVersion} hadoop cluster"
     exec {
         commandLine 'build/hadoop.py'
         args 'start', '-format'

Diff for: gradle/wrapper/gradle-wrapper.jar

@@ -1,6 +1,6 @@
-#Sun Jul 27 12:12:18 CEST 2014
+#Thu Aug 28 09:21:33 EDT 2014
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=http\://services.gradle.org/distributions/gradle-1.11-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-1.12-bin.zip

Diff for: gradle/wrapper/gradle-wrapper.properties

@@ -2,42 +2,22 @@
 
 alias g="./gradlew --daemon"
 
-VERSIONS=(0.23 1.0 1.1 2.2 2.3 2.4 cdh4 cdh5)
 OPTS=test
-#VERSIONS=(0.23 cdh4 cdh5 2.2 2.3 2.4)
 
 while [ "$1" ]
 do
-	if [[ " ${VERSIONS[*]} " == *" $1 "* ]]
-	then
-		HV=$1
-	else 
-		case $1 in
-			"examples")
-				OPTS="historicalYield sensorData enronEmails"
-				;;
-			"all")
-				HV="all"
-				;;
-		esac
-	fi
+    case $1 in
+        "examples")
+            OPTS="historicalYield sensorData enronEmails"
+            ;;
+        "all")
+            HV="all"
+            ;;
+    esac
 	shift
 done
 
-echo Running \"$OPTS\" against \"${HV}\"
-
-function choose() {
-	if [ -z "${HV}" ]
-	then
-		echo Choose a hadoop version
-		select HV in ${VERSIONS[@]}
-		do
-			break
-		done
-	else 
-		shift
-	fi
-}
+echo Running \"$OPTS\"
 
 function browser() {
 	while [ "$1" ]
@@ -50,7 +30,7 @@ function browser() {
 function run() {
 	#g clean jar testJar configureCluster test historicalYield sensorData enronEmails \
 		#-PclusterVersion=${HV} 2>&1 | tee -a test-${HV}.out
-	g clean jar testJar configureCluster $OPTS -PclusterVersion=${HV} --stacktrace 2>&1 | tee -a test-${HV}.out
+	g clean jar testJar $OPTS --stacktrace 2>&1 | tee -a build/test.out
 
 
 	for i in "*/build/reports/tests/index.html"
@@ -69,18 +49,4 @@ function run() {
 	done
 }
 
-if [ "$HV" == "all" ]
-then
-	shift
-	for HV in ${VERSIONS[@]}
-	do
-		> test-${HV}.out
-	done
-	for HV in ${VERSIONS[@]}
-	do
-		run $*
-	done
-else
-	choose
-	run $*
-fi
+run