HADOOP-13614 apply to branch-2; test and verify that all appears well. Tune more tests, including ITestS3AMiniYarnCluster.

A big change here is fixing up the scale tests to work as subclasses of the S3AScaleTestBase indirect subclasses of AbstractFSContractTestBase, because that sets up the test timeout rule. Rather than have a field of the same name and hope that its timeout gets picked up, I've tuned how timeouts get set up, so the subclasses do it. All well and good, except those subclasses are being called during the initialization of the base class, that is: before the subclasses are full inited. I don't ever like doing that, though it is working here.

Author: steveloughran

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java

@@ -19,7 +19,7 @@
 package org.apache.hadoop.fs.contract.s3a;
 
 import static org.apache.hadoop.fs.s3a.Constants.*;
-import static org.apache.hadoop.fs.s3a.S3ATestConstants.SCALE_TEST_TIMEOUT;
+import static org.apache.hadoop.fs.s3a.S3ATestConstants.SCALE_TEST_TIMEOUT_MILLIS;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.tools.contract.AbstractContractDistCpTest;
@@ -35,7 +35,7 @@ public class ITestS3AContractDistCp extends AbstractContractDistCpTest {
 
   @Override
   protected int getTestTimeoutMillis() {
-    return SCALE_TEST_TIMEOUT;
+    return SCALE_TEST_TIMEOUT_MILLIS;
   }
 
   @Override

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java

@@ -26,6 +26,8 @@
 import org.apache.hadoop.fs.contract.s3a.S3AContract;
 import org.apache.hadoop.io.IOUtils;
 import org.junit.Before;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 
@@ -38,6 +40,9 @@
 public abstract class AbstractS3ATestBase extends AbstractFSContractTestBase
     implements S3ATestConstants {
 
+  protected static final Logger LOG =
+      LoggerFactory.getLogger(AbstractS3ATestBase.class);
+
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
     return new S3AContract(conf);
@@ -73,6 +78,17 @@ public S3AFileSystem getFileSystem() {
     return (S3AFileSystem) super.getFileSystem();
   }
 
+  /**
+   * Describe a test in the logs.
+   * @param text text to print
+   * @param args arguments to format in the printing
+   */
+  protected void describe(String text, Object... args) {
+    LOG.info("\n\n{}: {}\n",
+        methodName.getMethodName(),
+        String.format(text, args));
+  }
+
   /**
    * Write a file, read it back, validate the dataset. Overwrites the file
    * if it is present

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockingThreadPool.java

@@ -25,7 +25,10 @@
 import org.apache.hadoop.fs.contract.AbstractFSContract;
 import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
 import org.apache.hadoop.fs.contract.s3a.S3AContract;
+
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -36,16 +39,11 @@
 /**
  * S3A tests for configuring block size.
  */
-public class ITestS3ABlocksize extends AbstractFSContractTestBase {
+public class ITestS3ABlocksize extends AbstractS3ATestBase {
 
   private static final Logger LOG =
       LoggerFactory.getLogger(ITestS3ABlocksize.class);
 
-  @Override
-  protected AbstractFSContract createContract(Configuration conf) {
-    return new S3AContract(conf);
-  }
-
   @Test
   @SuppressWarnings("deprecation")
   public void testBlockSize() throws Exception {

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlocksize.java

@@ -66,34 +66,37 @@ public class ITestS3AConfiguration {
   private static final Logger LOG =
       LoggerFactory.getLogger(ITestS3AConfiguration.class);
 
-  private static final String TEST_ENDPOINT = "test.fs.s3a.endpoint";
-
   @Rule
-  public Timeout testTimeout = new Timeout(30 * 60 * 1000);
+  public Timeout testTimeout = new Timeout(
+      S3ATestConstants.S3A_TEST_TIMEOUT
+  );
 
   @Rule
   public final TemporaryFolder tempDir = new TemporaryFolder();
 
   /**
    * Test if custom endpoint is picked up.
    * <p>
-   * The test expects TEST_ENDPOINT to be defined in the Configuration
+   * The test expects {@link S3ATestConstants#CONFIGURATION_TEST_ENDPOINT}
+   * to be defined in the Configuration
    * describing the endpoint of the bucket to which TEST_FS_S3A_NAME points
    * (i.e. "s3-eu-west-1.amazonaws.com" if the bucket is located in Ireland).
    * Evidently, the bucket has to be hosted in the region denoted by the
    * endpoint for the test to succeed.
    * <p>
    * More info and the list of endpoint identifiers:
-   * http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
+   * @see <a href="http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region">endpoint list</a>.
    *
    * @throws Exception
    */
   @Test
   public void testEndpoint() throws Exception {
     conf = new Configuration();
-    String endpoint = conf.getTrimmed(TEST_ENDPOINT, "");
+    String endpoint = conf.getTrimmed(
+        S3ATestConstants.CONFIGURATION_TEST_ENDPOINT, "");
     if (endpoint.isEmpty()) {
-      LOG.warn("Custom endpoint test skipped as " + TEST_ENDPOINT + "config " +
+      LOG.warn("Custom endpoint test skipped as " +
+          S3ATestConstants.CONFIGURATION_TEST_ENDPOINT + "config " +
           "setting was not detected");
     } else {
       conf.set(Constants.ENDPOINT, endpoint);

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java

@@ -232,7 +232,8 @@ public void testFakeDirectoryDeletion() throws Throwable {
 
     int destDirDepth = directoriesInPath(destDir);
     directoriesCreated.assertDiffEquals(state, 1);
-/*  TODO: uncomment once HADOOP-13222 is in
+/*  TODO: uncomment once HADOOP-13222 "s3a.mkdirs() to delete empty fake parent directories"
+    is in
     deleteRequests.assertDiffEquals(state,1);
     directoriesDeleted.assertDiffEquals(state,0);
     fakeDirectoriesDeleted.assertDiffEquals(state,destDirDepth);

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java

@@ -134,18 +134,11 @@ public interface S3ATestConstants {
    */
   int DEFAULT_DIRECTORY_COUNT = 2;
 
-  /**
-   * Default scale test timeout in seconds: {@value}.
-   */
-  int DEFAULT_TEST_TIMEOUT = 30 * 60;
-
   /**
    * Default policy on scale tests: {@value}.
    */
   boolean DEFAULT_SCALE_TESTS_ENABLED = false;
 
-  String KEY_ENCRYPTION_TESTS = TEST_FS_S3A + "encryption.enabled";
-
   /**
    * Fork ID passed down from maven if the test is running in parallel.
    */
@@ -159,7 +152,15 @@ public interface S3ATestConstants {
   int S3A_TEST_TIMEOUT = 10 * 60 * 1000;
 
   /**
-   * Timeout in Milliseconds for Scale Tests: {@value}.
+   * Timeout in Seconds for Scale Tests: {@value}.
+   */
+  int SCALE_TEST_TIMEOUT_SECONDS = 30 * 60;
+
+  int SCALE_TEST_TIMEOUT_MILLIS = SCALE_TEST_TIMEOUT_SECONDS * 1000;
+  /**
+   * Optional custom endpoint for S3A configuration tests.
+   * This does <i>not</i> set the endpoint for s3 access elsewhere.
    */
-  int SCALE_TEST_TIMEOUT = 30 * 60 * 1000;
+  String CONFIGURATION_TEST_ENDPOINT =
+      "test.fs.s3a.endpoint";
 }

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java

@@ -59,7 +59,7 @@ public class S3ATestUtils {
    */
   public static S3AFileSystem createTestFileSystem(Configuration conf)
       throws IOException {
-    return createTestFileSystem(conf, true);
+    return createTestFileSystem(conf, false);
   }
 
   /**

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java

@@ -38,6 +38,7 @@
 import org.apache.hadoop.fs.StorageStatistics;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
 import org.apache.hadoop.fs.s3a.S3AFileStatus;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.apache.hadoop.fs.s3a.Statistic;
 import org.apache.hadoop.util.Progressable;
 
@@ -70,27 +71,22 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
   private int partitionSize;
 
   @Override
-  public void setUp() throws Exception {
-    super.setUp();
-
+  public void setup() throws Exception {
+    super.setup();
     final Path testPath = getTestPath();
     scaleTestDir = new Path(testPath, "scale");
     hugefile = new Path(scaleTestDir, "hugefile");
     hugefileRenamed = new Path(scaleTestDir, "hugefileRenamed");
   }
 
-  @Override
-  public void tearDown() throws Exception {
-    // do nothing. Specifically: do not delete the test dir
-  }
 
   /**
    * Note that this can get called before test setup.
    * @return the configuration to use.
    */
   @Override
-  protected Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
+  protected Configuration createScaleConfiguration() {
+    Configuration conf = super.createScaleConfiguration();
     partitionSize = (int)getTestPropertyBytes(conf,
         KEY_HUGE_PARTITION_SIZE,
         DEFAULT_PARTITION_SIZE);
@@ -155,6 +151,7 @@ public void test_010_CreateHugeFile() throws IOException {
     // perform the upload.
     // there's lots of logging here, so that a tail -f on the output log
     // can give a view of what is happening.
+    S3AFileSystem fs = getFileSystem();
     StorageStatistics storageStatistics = fs.getStorageStatistics();
     String putRequests = Statistic.OBJECT_PUT_REQUESTS.getSymbol();
     String putBytes = Statistic.OBJECT_PUT_BYTES.getSymbol();
@@ -286,12 +283,13 @@ private void verifyNoFailures(String operation) {
   }
 
   void assumeHugeFileExists() throws IOException {
+    S3AFileSystem fs = getFileSystem();
     ContractTestUtils.assertPathExists(fs, "huge file not created", hugefile);
     ContractTestUtils.assertIsFile(fs, hugefile);
   }
 
   private void logFSState() {
-    LOG.info("File System state after operation:\n{}", fs);
+    LOG.info("File System state after operation:\n{}", getFileSystem());
   }
 
   @Test
@@ -305,6 +303,7 @@ public void test_040_PositionedReadHugeFile() throws Throwable {
     }
     String filetype = encrypted ? "encrypted file" : "file";
     describe("Positioned reads of %s %s", filetype, hugefile);
+    S3AFileSystem fs = getFileSystem();
     S3AFileStatus status = fs.getFileStatus(hugefile);
     long filesize = status.getLen();
     int ops = 0;
@@ -344,6 +343,7 @@ public void test_040_PositionedReadHugeFile() throws Throwable {
   public void test_050_readHugeFile() throws Throwable {
     assumeHugeFileExists();
     describe("Reading %s", hugefile);
+    S3AFileSystem fs = getFileSystem();
     S3AFileStatus status = fs.getFileStatus(hugefile);
     long filesize = status.getLen();
     long blocks = filesize / uploadBlockSize;
@@ -369,6 +369,7 @@ public void test_050_readHugeFile() throws Throwable {
   public void test_100_renameHugeFile() throws Throwable {
     assumeHugeFileExists();
     describe("renaming %s to %s", hugefile, hugefileRenamed);
+    S3AFileSystem fs = getFileSystem();
     S3AFileStatus status = fs.getFileStatus(hugefile);
     long filesize = status.getLen();
     fs.delete(hugefileRenamed, false);
@@ -396,7 +397,7 @@ public void test_100_renameHugeFile() throws Throwable {
   public void test_999_DeleteHugeFiles() throws IOException {
     deleteHugeFile();
     ContractTestUtils.NanoTimer timer2 = new ContractTestUtils.NanoTimer();
-
+    S3AFileSystem fs = getFileSystem();
     fs.delete(hugefileRenamed, false);
     timer2.end("time to delete %s", hugefileRenamed);
     ContractTestUtils.rm(fs, getTestPath(), true, true);
@@ -405,7 +406,7 @@ public void test_999_DeleteHugeFiles() throws IOException {
   protected void deleteHugeFile() throws IOException {
     describe("Deleting %s", hugefile);
     NanoTimer timer = new NanoTimer();
-    fs.delete(hugefile, false);
+    getFileSystem().delete(hugefile, false);
     timer.end("time to delete %s", hugefile);
   }
 

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java

@@ -27,8 +27,8 @@
 public class ITestS3ADeleteFilesOneByOne extends ITestS3ADeleteManyFiles {
 
   @Override
-  protected Configuration createConfiguration() {
-    Configuration configuration = super.createConfiguration();
+  protected Configuration createScaleConfiguration() {
+    Configuration configuration = super.createScaleConfiguration();
     configuration.setBoolean(Constants.ENABLE_MULTI_DELETE, false);
     return configuration;
   }

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java

@@ -20,6 +20,8 @@
 
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -54,8 +56,8 @@ public void testBulkRenameAndDelete() throws Throwable {
     final Path srcDir = new Path(scaleTestDir, "src");
     final Path finalDir = new Path(scaleTestDir, "final");
     final long count = getOperationCount();
+    final S3AFileSystem fs = getFileSystem();
     ContractTestUtils.rm(fs, scaleTestDir, true, false);
-
     fs.mkdirs(srcDir);
     fs.mkdirs(finalDir);