Replace more flaky "tests" with proper integration tests

  - Add integTest sourceSet and task to li-hadoop-plugin subproject
  - Remove unused test resources and li-hadoop-plugin-test subproject
  - Local builds should no longer freeze or hang

Signed-off-by: Kyle Moore <[email protected]>

Author: DPUkyle

CONTRIBUTORS.md

@@ -184,4 +184,5 @@ The following were contributed by Sayali Kadam.
 
 [Kyle Moore](https://www.linkedin.com/in/dpukyle/) ([DPUkyle](https://github.com/DPUkyle)) contributed the following:
 * `Refactored Hadoop DSL "tests" into proper integration tests using Gradle TestKit`
+* `Refactored li-hadoop-plugin "tests" into proper integration tests using Gradle TestKit`
 

README.md

@@ -93,8 +93,7 @@ The project structure is setup as follows:
   * `example-project`: Example project that uses the Hadoop Plugin and DSL to build an example Azkaban workflow
   * `hadoop-jobs`: Code for re-usable Hadoop jobs and implementations of Hadoop DSL job types 
   * `hadoop-plugin`: Code and tests for the various plugins that comprise the Hadoop Plugin
-  * `li-hadoop-plugin`: LinkedIn-specific extensions to the Hadoop Plugin
-  * `li-hadoop-plugin-test`: Test cases for the LinkedIn-specific extensions to the Hadoop Plugin
+  * `li-hadoop-plugin`: LinkedIn-specific extensions to the Hadoop Plugin, and tests
 
 Although the `li-hadoop-plugin` code is generally specific to LinkedIn, it is included in the
 project to show you how to use subclassing to extend the core functionality of the Hadoop Plugin for your
@@ -111,4 +110,4 @@ Unit tests are invoked by running `./gradlew :hadoop-plugin:test`.  Individual t
 
 ##### Integration tests
 
-Integration tests are invoked by running `./gradlew :hadoop-plugin:integTest`, or by running the `check` task.
+Integration tests are invoked by running `./gradlew :hadoop-plugin:integTest`, or by running the `check` task.

build.gradle

@@ -32,10 +32,6 @@ if (isLinkedInBuild) {
     }
   }
 }
-else {
-  project(':li-hadoop-plugin-test').ext['pluginTestDir'] = "${project(':hadoop-plugin').projectDir}/build/libs"
-  project(':li-hadoop-plugin-test').ext['liPluginTestDir'] = "${project(':li-hadoop-plugin').projectDir}/build/libs"
-}
 
 // Configure the Gradle wrapper
 wrapper {

li-hadoop-plugin-test/.crt

@@ -1,5 +1,6 @@
 plugins {
   id 'groovy'
+  id 'java-gradle-plugin'
   id 'eclipse'
   id 'idea'
 }
@@ -23,6 +24,12 @@ sourceSets {
     compileClasspath+=configurations.provided
     runtimeClasspath+=configurations.provided
   }
+  integTest {
+    groovy.srcDir file('src/integTest/groovy')
+    resources.srcDir file('src/integTest/resources')
+    compileClasspath += sourceSets.main.output + configurations.testRuntimeClasspath
+    runtimeClasspath += output + compileClasspath
+  }
 }
 
 dependencies {
@@ -34,6 +41,8 @@ dependencies {
 
   testCompile 'junit:junit:4.12'
   testCompile 'org.mockito:mockito-core:2.10.0'
+
+  integTestCompile 'org.spockframework:spock-core:1.3-groovy-2.4@jar'
 }
 
 task sourceJar(type: Jar) {
@@ -52,6 +61,25 @@ artifacts {
   archives groovydocJar, sourceJar
 }
 
+task integTest(type: Test) {
+  description = 'Runs the integration tests.'
+  group = 'verification'
+  testClassesDirs = sourceSets.integTest.output.classesDirs
+  classpath = sourceSets.integTest.runtimeClasspath
+  mustRunAfter test
+}
+
+check.dependsOn integTest
+
+gradlePlugin {
+  plugins {
+    liHadoopPlugin {
+      id = 'li-hadoop-plugin'
+      implementationClass = 'com.linkedin.gradle.lihadoop.LiHadoopPlugin'
+    }
+  }
+}
+
 // In LinkedIn internal builds, we pull in some company-specific resource files into this project.
 boolean isLinkedInBuild = project.hasProperty('overrideBuildEnvironment')
 

li-hadoop-plugin-test/build.gradle

@@ -0,0 +1,69 @@
+package com.linkedin.gradle
+
+
+import org.gradle.testkit.runner.BuildResult
+import org.gradle.testkit.runner.GradleRunner
+import org.gradle.testkit.runner.TaskOutcome
+import org.junit.Rule
+import org.junit.rules.TemporaryFolder
+import spock.lang.Specification
+
+import java.util.zip.ZipFile
+
+class BuildHadoopZips extends Specification {
+
+    @Rule
+    TemporaryFolder tmp = new TemporaryFolder()
+
+    def buildDotGradle
+    def settingsDotGradle
+    def gradleDotProperties
+    def dotCrt
+
+    def setup() {
+        tmp.create()
+        buildDotGradle = tmp.newFile('build.gradle')
+        settingsDotGradle = tmp.newFile('settings.gradle')
+        gradleDotProperties = tmp.newFile('gradle.properties')
+        dotCrt = tmp.newFile('.crt')
+    }
+
+    /**
+     * Integration test for including the sources zip and SCM metadata file in the Hadoop zip
+     */
+    def 'verify zip contents'() {
+        given:
+        def projectName = 'build-hadoop-zips'
+        def version = '1.0.0'
+        buildDotGradle << this.class.classLoader.getResource('buildZips/buildZipsCRT.gradle').text
+        settingsDotGradle << """rootProject.name='${projectName}'"""
+        gradleDotProperties << """version=${version}"""
+        dotCrt << '''Test .crt file\n'''
+        GradleRunner runner = GradleRunner.create()
+                .withProjectDir(tmp.root)
+                .withPluginClasspath()
+                .withArguments('buildHadoopZips', '-is')
+
+        when:
+        BuildResult result = runner.build()
+
+        then:
+        result.task(':azkabanHadoopZip').outcome == TaskOutcome.SUCCESS
+        result.task(':CRTHadoopZip').outcome == TaskOutcome.SUCCESS
+
+        // find CRTHadoopZip, assert contains .crt file (build/distributions/build-hadoop-zips-1.0.0.zip)
+        def crtHadoopZip = new File(tmp.root, "build/distributions/${projectName}-${version}.zip")
+        crtHadoopZip.exists()
+        def crtHadoopZipContents = new ZipFile(crtHadoopZip)
+        assert crtHadoopZipContents.getEntry('.crt') != null
+
+        // find azkabanHadoopZip, assert contents (build/distributions/build-hadoop-zips-1.0.0-azkaban.zip)
+        //  contents == build.gradle, buildMetadata.json, build-hadoop-zips-1.0.0-sources.zip
+        def azkabanZip = new File(tmp.root, "build/distributions/${projectName}-${version}-azkaban.zip")
+        azkabanZip.exists()
+        def zipFileContents = new ZipFile(azkabanZip)
+        zipFileContents.getEntry('build.gradle') != null
+        zipFileContents.getEntry('buildMetadata.json') != null
+        zipFileContents.getEntry("${projectName}-${version}-sources.zip") != null
+    }
+}

li-hadoop-plugin-test/buildZipsCRT.gradle

@@ -0,0 +1,14 @@
+plugins {
+  id 'distribution'
+  id 'li-hadoop-plugin'
+}
+
+hadoopZip {
+  base {
+    from "build.gradle"
+  }
+  zip("azkaban") {
+    from "buildZipsCRT.gradle"
+  }
+  CRT { }
+}

li-hadoop-plugin/build.gradle

@@ -17,4 +17,3 @@ include 'azkaban-client'
 include 'hadoop-jobs'
 include 'hadoop-plugin'
 include 'li-hadoop-plugin'
-include 'li-hadoop-plugin-test'