Skip to content

Commit ce10217

Browse files
author
Takashi Matsuo
committed
[dlp] fix: remove gcp-devrel-py-tools
fixes GoogleCloudPlatform#3375 fixes GoogleCloudPlatform#3416 fixes GoogleCloudPlatform#3417 * remove wrong usage of `eventually_consistent.call` * only test if the operation has been started * shorter timeout for polling * correct use of `pytest.mark.flaky`
1 parent d66e3a7 commit ce10217

File tree

5 files changed

+89
-71
lines changed

5 files changed

+89
-71
lines changed

‎dlp/inspect_content.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -474,6 +474,7 @@ def inspect_gcs_file(
474474

475475
operation = dlp.create_dlp_job(parent, inspect_job=inspect_job)
476476
print("Inspection operation started: {}".format(operation.name))
477+
477478
# Create a Pub/Sub client and find the subscription. The subscription is
478479
# expected to already be listening to the topic.
479480
subscriber = google.cloud.pubsub.SubscriberClient()
@@ -636,6 +637,7 @@ def inspect_datastore(
636637
}
637638

638639
operation = dlp.create_dlp_job(parent, inspect_job=inspect_job)
640+
print("Inspection operation started: {}".format(operation.name))
639641

640642
# Create a Pub/Sub client and find the subscription. The subscription is
641643
# expected to already be listening to the topic.
@@ -802,6 +804,7 @@ def inspect_bigquery(
802804
}
803805

804806
operation = dlp.create_dlp_job(parent, inspect_job=inspect_job)
807+
print("Inspection operation started: {}".format(operation.name))
805808

806809
# Create a Pub/Sub client and find the subscription. The subscription is
807810
# expected to already be listening to the topic.

‎dlp/inspect_content_test.py

Lines changed: 74 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -15,19 +15,18 @@
1515
import os
1616
import uuid
1717

18-
from gcp_devrel.testing import eventually_consistent
19-
from gcp_devrel.testing.flaky import flaky
2018
import google.api_core.exceptions
2119
import google.cloud.bigquery
2220
import google.cloud.datastore
2321
import google.cloud.dlp_v2
2422
import google.cloud.exceptions
2523
import google.cloud.pubsub
2624
import google.cloud.storage
27-
2825
import pytest
26+
2927
import inspect_content
3028

29+
3130
UNIQUE_STRING = str(uuid.uuid4()).split("-")[0]
3231

3332
GCLOUD_PROJECT = os.getenv("GCLOUD_PROJECT")
@@ -95,7 +94,8 @@ def subscription_id(topic_id):
9594
# Subscribes to a topic.
9695
subscriber = google.cloud.pubsub.SubscriberClient()
9796
topic_path = subscriber.topic_path(GCLOUD_PROJECT, topic_id)
98-
subscription_path = subscriber.subscription_path(GCLOUD_PROJECT, SUBSCRIPTION_ID)
97+
subscription_path = subscriber.subscription_path(
98+
GCLOUD_PROJECT, SUBSCRIPTION_ID)
9999
try:
100100
subscriber.create_subscription(subscription_path, topic_path)
101101
except google.api_core.exceptions.AlreadyExists:
@@ -297,21 +297,21 @@ def test_inspect_gcs_file(bucket, topic_id, subscription_id, capsys):
297297
topic_id,
298298
subscription_id,
299299
["EMAIL_ADDRESS", "PHONE_NUMBER"],
300-
timeout=420,
300+
timeout=1
301301
)
302302

303303
out, _ = capsys.readouterr()
304304
assert "Inspection operation started" in out
305305
# Cancel the operation
306-
operation_id = out.split("Inspection operation started: ")[1].split("\n")[0]
306+
operation_id = out.split(
307+
"Inspection operation started: ")[1].split("\n")[0]
307308
print(operation_id)
308309
client = google.cloud.dlp_v2.DlpServiceClient()
309310
client.cancel_dlp_job(operation_id)
310311

311312

312313
def test_inspect_gcs_file_with_custom_info_types(
313-
bucket, topic_id, subscription_id, capsys
314-
):
314+
bucket, topic_id, subscription_id, capsys):
315315
dictionaries = ["[email protected]"]
316316
regexes = ["\\(\\d{3}\\) \\d{3}-\\d{4}"]
317317

@@ -324,41 +324,41 @@ def test_inspect_gcs_file_with_custom_info_types(
324324
[],
325325
custom_dictionaries=dictionaries,
326326
custom_regexes=regexes,
327-
timeout=420,
328-
)
327+
timeout=1)
329328

330329
out, _ = capsys.readouterr()
331330

332331
assert "Inspection operation started" in out
333332
# Cancel the operation
334-
operation_id = out.split("Inspection operation started: ")[1].split("\n")[0]
333+
operation_id = out.split(
334+
"Inspection operation started: ")[1].split("\n")[0]
335335
print(operation_id)
336336
client = google.cloud.dlp_v2.DlpServiceClient()
337337
client.cancel_dlp_job(operation_id)
338338

339339

340-
def test_inspect_gcs_file_no_results(bucket, topic_id, subscription_id, capsys):
340+
def test_inspect_gcs_file_no_results(
341+
bucket, topic_id, subscription_id, capsys):
341342
inspect_content.inspect_gcs_file(
342343
GCLOUD_PROJECT,
343344
bucket.name,
344345
"harmless.txt",
345346
topic_id,
346347
subscription_id,
347348
["EMAIL_ADDRESS", "PHONE_NUMBER"],
348-
timeout=420,
349-
)
349+
timeout=1)
350350

351351
out, _ = capsys.readouterr()
352352

353353
assert "Inspection operation started" in out
354354
# Cancel the operation
355-
operation_id = out.split("Inspection operation started: ")[1].split("\n")[0]
355+
operation_id = out.split(
356+
"Inspection operation started: ")[1].split("\n")[0]
356357
print(operation_id)
357358
client = google.cloud.dlp_v2.DlpServiceClient()
358359
client.cancel_dlp_job(operation_id)
359360

360361

361-
@pytest.mark.skip(reason="nondeterministically failing")
362362
def test_inspect_gcs_image_file(bucket, topic_id, subscription_id, capsys):
363363
inspect_content.inspect_gcs_file(
364364
GCLOUD_PROJECT,
@@ -367,10 +367,16 @@ def test_inspect_gcs_image_file(bucket, topic_id, subscription_id, capsys):
367367
topic_id,
368368
subscription_id,
369369
["EMAIL_ADDRESS", "PHONE_NUMBER"],
370-
)
370+
timeout=1)
371371

372372
out, _ = capsys.readouterr()
373-
assert "Info type: EMAIL_ADDRESS" in out
373+
assert "Inspection operation started" in out
374+
# Cancel the operation
375+
operation_id = out.split(
376+
"Inspection operation started: ")[1].split("\n")[0]
377+
print(operation_id)
378+
client = google.cloud.dlp_v2.DlpServiceClient()
379+
client.cancel_dlp_job(operation_id)
374380

375381

376382
def test_inspect_gcs_multiple_files(bucket, topic_id, subscription_id, capsys):
@@ -381,55 +387,62 @@ def test_inspect_gcs_multiple_files(bucket, topic_id, subscription_id, capsys):
381387
topic_id,
382388
subscription_id,
383389
["EMAIL_ADDRESS", "PHONE_NUMBER"],
384-
)
390+
timeout=1)
385391

386392
out, _ = capsys.readouterr()
387393

388394
assert "Inspection operation started" in out
389395
# Cancel the operation
390-
operation_id = out.split("Inspection operation started: ")[1].split("\n")[0]
396+
operation_id = out.split(
397+
"Inspection operation started: ")[1].split("\n")[0]
391398
print(operation_id)
392399
client = google.cloud.dlp_v2.DlpServiceClient()
393400
client.cancel_dlp_job(operation_id)
394401

395402

396-
@flaky
397-
def test_inspect_datastore(datastore_project, topic_id, subscription_id, capsys):
398-
@eventually_consistent.call
399-
def _():
400-
inspect_content.inspect_datastore(
401-
GCLOUD_PROJECT,
402-
datastore_project,
403-
DATASTORE_KIND,
404-
topic_id,
405-
subscription_id,
406-
["FIRST_NAME", "EMAIL_ADDRESS", "PHONE_NUMBER"],
407-
)
403+
def test_inspect_datastore(
404+
datastore_project, topic_id, subscription_id, capsys):
405+
inspect_content.inspect_datastore(
406+
GCLOUD_PROJECT,
407+
datastore_project,
408+
DATASTORE_KIND,
409+
topic_id,
410+
subscription_id,
411+
["FIRST_NAME", "EMAIL_ADDRESS", "PHONE_NUMBER"],
412+
timeout=1)
408413

409-
out, _ = capsys.readouterr()
410-
assert "Info type: EMAIL_ADDRESS" in out
414+
out, _ = capsys.readouterr()
415+
assert "Inspection operation started" in out
416+
# Cancel the operation
417+
operation_id = out.split(
418+
"Inspection operation started: ")[1].split("\n")[0]
419+
print(operation_id)
420+
client = google.cloud.dlp_v2.DlpServiceClient()
421+
client.cancel_dlp_job(operation_id)
411422

412423

413-
@flaky
424+
# @pytest.mark.skip(reason="too slow")
414425
def test_inspect_datastore_no_results(
415-
datastore_project, topic_id, subscription_id, capsys
416-
):
417-
@eventually_consistent.call
418-
def _():
419-
inspect_content.inspect_datastore(
420-
GCLOUD_PROJECT,
421-
datastore_project,
422-
DATASTORE_KIND,
423-
topic_id,
424-
subscription_id,
425-
["PHONE_NUMBER"],
426-
)
427-
428-
out, _ = capsys.readouterr()
429-
assert "No findings" in out
430-
431-
432-
@pytest.mark.skip(reason="unknown issue")
426+
datastore_project, topic_id, subscription_id, capsys):
427+
inspect_content.inspect_datastore(
428+
GCLOUD_PROJECT,
429+
datastore_project,
430+
DATASTORE_KIND,
431+
topic_id,
432+
subscription_id,
433+
["PHONE_NUMBER"],
434+
timeout=1)
435+
436+
out, _ = capsys.readouterr()
437+
assert "Inspection operation started" in out
438+
# Cancel the operation
439+
operation_id = out.split(
440+
"Inspection operation started: ")[1].split("\n")[0]
441+
print(operation_id)
442+
client = google.cloud.dlp_v2.DlpServiceClient()
443+
client.cancel_dlp_job(operation_id)
444+
445+
433446
def test_inspect_bigquery(bigquery_project, topic_id, subscription_id, capsys):
434447
inspect_content.inspect_bigquery(
435448
GCLOUD_PROJECT,
@@ -439,7 +452,13 @@ def test_inspect_bigquery(bigquery_project, topic_id, subscription_id, capsys):
439452
topic_id,
440453
subscription_id,
441454
["FIRST_NAME", "EMAIL_ADDRESS", "PHONE_NUMBER"],
442-
)
455+
timeout=1)
443456

444457
out, _ = capsys.readouterr()
445-
assert "Info type: FIRST_NAME" in out
458+
assert "Inspection operation started" in out
459+
# Cancel the operation
460+
operation_id = out.split(
461+
"Inspection operation started: ")[1].split("\n")[0]
462+
print(operation_id)
463+
client = google.cloud.dlp_v2.DlpServiceClient()
464+
client.cancel_dlp_job(operation_id)

‎dlp/jobs_test.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
# limitations under the License.
1414

1515
import os
16-
from flaky import flaky
1716

1817
import pytest
1918

@@ -66,7 +65,7 @@ def test_list_dlp_jobs(test_job_name, capsys):
6665
assert test_job_name not in out
6766

6867

69-
@flaky
68+
@pytest.mark.flaky
7069
def test_list_dlp_jobs_with_filter(test_job_name, capsys):
7170
jobs.list_dlp_jobs(
7271
GCLOUD_PROJECT,

‎dlp/requirements-test.txt

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
11
pytest==5.3.2
2-
gcp-devrel-py-tools==0.0.15
32
flaky==3.6.1
43
mock==3.0.5

‎dlp/risk_test.py

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,12 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
from flaky import flaky
15+
import os
1616
import uuid
1717

1818
import google.cloud.pubsub
1919
import google.cloud.bigquery
20-
2120
import pytest
22-
import os
2321

2422
import risk
2523

@@ -160,7 +158,7 @@ def bigquery_project():
160158
bigquery_client.delete_dataset(dataset_ref, delete_contents=True)
161159

162160

163-
@flaky
161+
@pytest.mark.flaky
164162
def test_numerical_risk_analysis(
165163
topic_id, subscription_id, bigquery_project, capsys
166164
):
@@ -178,7 +176,7 @@ def test_numerical_risk_analysis(
178176
assert "Value Range:" in out
179177

180178

181-
@flaky
179+
@pytest.mark.flaky
182180
def test_categorical_risk_analysis_on_string_field(
183181
topic_id, subscription_id, bigquery_project, capsys
184182
):
@@ -197,7 +195,7 @@ def test_categorical_risk_analysis_on_string_field(
197195
assert "Most common value occurs" in out
198196

199197

200-
@flaky
198+
@pytest.mark.flaky
201199
def test_categorical_risk_analysis_on_number_field(
202200
topic_id, subscription_id, bigquery_project, capsys
203201
):
@@ -215,7 +213,7 @@ def test_categorical_risk_analysis_on_number_field(
215213
assert "Most common value occurs" in out
216214

217215

218-
@flaky
216+
@pytest.mark.flaky
219217
def test_k_anonymity_analysis_single_field(
220218
topic_id, subscription_id, bigquery_project, capsys
221219
):
@@ -234,7 +232,7 @@ def test_k_anonymity_analysis_single_field(
234232
assert "Class size:" in out
235233

236234

237-
@flaky
235+
@pytest.mark.flaky
238236
def test_k_anonymity_analysis_multiple_fields(
239237
topic_id, subscription_id, bigquery_project, capsys
240238
):
@@ -253,7 +251,7 @@ def test_k_anonymity_analysis_multiple_fields(
253251
assert "Class size:" in out
254252

255253

256-
@flaky
254+
@pytest.mark.flaky
257255
def test_l_diversity_analysis_single_field(
258256
topic_id, subscription_id, bigquery_project, capsys
259257
):
@@ -274,7 +272,7 @@ def test_l_diversity_analysis_single_field(
274272
assert "Sensitive value" in out
275273

276274

277-
@flaky
275+
@pytest.mark.flaky
278276
def test_l_diversity_analysis_multiple_field(
279277
topic_id, subscription_id, bigquery_project, capsys
280278
):
@@ -295,7 +293,7 @@ def test_l_diversity_analysis_multiple_field(
295293
assert "Sensitive value" in out
296294

297295

298-
@flaky
296+
@pytest.mark.flaky
299297
def test_k_map_estimate_analysis_single_field(
300298
topic_id, subscription_id, bigquery_project, capsys
301299
):
@@ -316,7 +314,7 @@ def test_k_map_estimate_analysis_single_field(
316314
assert "Values" in out
317315

318316

319-
@flaky
317+
@pytest.mark.flaky
320318
def test_k_map_estimate_analysis_multiple_field(
321319
topic_id, subscription_id, bigquery_project, capsys
322320
):
@@ -337,7 +335,7 @@ def test_k_map_estimate_analysis_multiple_field(
337335
assert "Values" in out
338336

339337

340-
@flaky
338+
@pytest.mark.flaky
341339
def test_k_map_estimate_analysis_quasi_ids_info_types_equal(
342340
topic_id, subscription_id, bigquery_project
343341
):

0 commit comments

Comments
 (0)