test: address flaky system test (#364)

Author: bcoe

dataproc/instantiateInlineWorkflowTemplate.js

@@ -21,7 +21,7 @@
 
 /*eslint no-warning-comments: [0, { "terms": ["todo", "fixme"], "location": "anywhere" }]*/
 
-function main(projectId = 'YOUR_PROJECT_ID', region = 'YOUR_REGION') {
+async function main(projectId = 'YOUR_PROJECT_ID', region = 'YOUR_REGION') {
   // [START dataproc_instantiate_inline_workflow_template]
   const dataproc = require('@google-cloud/dataproc');
 
@@ -91,4 +91,7 @@ function main(projectId = 'YOUR_PROJECT_ID', region = 'YOUR_REGION') {
   instantiateInlineWorkflowTemplate();
 }
 
-main(...process.argv.slice(2));
+main(...process.argv.slice(2)).catch(err => {
+  console.error(err);
+  process.exitCode = 1;
+});

dataproc/system-test/instantiateInlineWorkflowTemplate.test.js

@@ -26,8 +26,12 @@ const execSync = cmd =>
     encoding: 'utf-8',
   });
 
+const {delay} = require('./util');
+
 describe('instantiate an inline workflow template', () => {
-  it('should instantiate an inline workflow template', async () => {
+  it('should instantiate an inline workflow template', async function () {
+    this.retries(4);
+    await delay(this.test);
     const stdout = execSync(
       `node instantiateInlineWorkflowTemplate.js "${projectId}" "${region}"`
     );

dataproc/system-test/util.js

@@ -0,0 +1,28 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// ML tests frequently run into concurrency and quota issues, for which
+// retrying with a backoff is a good strategy:
+module.exports = {
+  async delay(test) {
+    const retries = test.currentRetry();
+    if (retries === 0) return; // no retry on the first failure.
+    // see: https://cloud.google.com/storage/docs/exponential-backoff:
+    const ms = Math.pow(2, retries) * 1000 + Math.random() * 2000;
+    return new Promise(done => {
+      console.info(`retrying "${test.title}" in ${ms}ms`);
+      setTimeout(done, ms);
+    });
+  },
+};