test: increase timeout and number of retries for system and sample tests (#624)

Increasing instance deletion timeout to 2 hrs
Increasing instance creation retries to 8

Author: asthamohta

samples/samples/backup_sample.py

@@ -38,7 +38,7 @@ def create_backup(instance_id, database_id, backup_id, version_time):
     operation = backup.create()
 
     # Wait for backup operation to complete.
-    operation.result(1200)
+    operation.result(2100)
 
     # Verify that the backup is ready.
     backup.reload()
@@ -74,7 +74,7 @@ def create_backup_with_encryption_key(instance_id, database_id, backup_id, kms_k
     operation = backup.create()
 
     # Wait for backup operation to complete.
-    operation.result(1200)
+    operation.result(2100)
 
     # Verify that the backup is ready.
     backup.reload()

samples/samples/conftest.py

@@ -24,7 +24,7 @@
 import pytest
 from test_utils import retry
 
-INSTANCE_CREATION_TIMEOUT = 240  # seconds
+INSTANCE_CREATION_TIMEOUT = 560  # seconds
 
 retry_429 = retry.RetryErrors(exceptions.ResourceExhausted, delay=15)
 

tests/system/_helpers.py

@@ -34,10 +34,10 @@
 SKIP_BACKUP_TESTS = os.getenv(SKIP_BACKUP_TESTS_ENVVAR) is not None
 
 INSTANCE_OPERATION_TIMEOUT_IN_SECONDS = int(
-    os.getenv("SPANNER_INSTANCE_OPERATION_TIMEOUT_IN_SECONDS", 240)
+    os.getenv("SPANNER_INSTANCE_OPERATION_TIMEOUT_IN_SECONDS", 560)
 )
 DATABASE_OPERATION_TIMEOUT_IN_SECONDS = int(
-    os.getenv("SPANNER_DATABASE_OPERATION_TIMEOUT_IN_SECONDS", 60)
+    os.getenv("SPANNER_DATABASE_OPERATION_TIMEOUT_IN_SECONDS", 120)
 )
 
 USE_EMULATOR_ENVVAR = "SPANNER_EMULATOR_HOST"
@@ -57,7 +57,7 @@
 
 retry_503 = retry.RetryErrors(exceptions.ServiceUnavailable)
 retry_429_503 = retry.RetryErrors(
-    exceptions.TooManyRequests, exceptions.ServiceUnavailable,
+    exceptions.TooManyRequests, exceptions.ServiceUnavailable, 8
 )
 retry_mabye_aborted_txn = retry.RetryErrors(exceptions.ServerError, exceptions.Aborted)
 retry_mabye_conflict = retry.RetryErrors(exceptions.ServerError, exceptions.Conflict)
@@ -107,7 +107,7 @@ def scrub_instance_ignore_not_found(to_scrub):
 
 
 def cleanup_old_instances(spanner_client):
-    cutoff = int(time.time()) - 1 * 60 * 60  # one hour ago
+    cutoff = int(time.time()) - 2 * 60 * 60  # two hour ago
     instance_filter = "labels.python-spanner-systests:true"
 
     for instance_pb in spanner_client.list_instances(filter_=instance_filter):