Skip to content

Commit b4c3c59

Browse files
dinesh-crestdinesh-crest
authored
DLP: Added samples for inspect bigquery, datastore and GCS with SCC (#3359)
Added unit test cases for same
1 parent 3630f5a commit b4c3c59

File tree

5 files changed

+821
-0
lines changed

5 files changed

+821
-0
lines changed

‎dlp/inspectBigquerySendToScc.js

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
// Copyright 2023 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
'use strict';
16+
17+
// sample-metadata:
18+
// title: Create a Dlp Job with Big Query Data and send its findings to Scc
19+
// description: Uses the Data Loss Prevention API to Create a Dlp Job with Big Query Data and send its findings to Scc
20+
// usage: node inspectBigquerySendToScc.js my-project dataProjectId datasetId tableId
21+
async function main(projectId, dataProjectId, datasetId, tableId) {
22+
// [START dlp_inspect_bigquery_send_to_scc]
23+
// Imports the Google Cloud Data Loss Prevention library
24+
const DLP = require('@google-cloud/dlp');
25+
26+
// Instantiates a client
27+
const dlp = new DLP.DlpServiceClient();
28+
29+
// The project ID to run the API call under.
30+
// const projectId = "your-project-id";
31+
32+
// The project ID the table is stored under
33+
// This may or (for public datasets) may not equal the calling project ID
34+
// const dataProjectId = 'my-project';
35+
36+
// The ID of the dataset to inspect, e.g. 'my_dataset'
37+
// const datasetId = 'my_dataset';
38+
39+
// The ID of the table to inspect, e.g. 'my_table'
40+
// const tableId = 'my_table';
41+
42+
async function inspectBigQuerySendToScc() {
43+
// Specify the storage configuration object with big query table.
44+
const storageItem = {
45+
bigQueryOptions: {
46+
tableReference: {
47+
projectId: dataProjectId,
48+
datasetId: datasetId,
49+
tableId: tableId,
50+
},
51+
},
52+
};
53+
54+
// Specify the type of info the inspection will look for.
55+
const infoTypes = [
56+
{name: 'EMAIL_ADDRESS'},
57+
{name: 'PERSON_NAME'},
58+
{name: 'LOCATION'},
59+
{name: 'PHONE_NUMBER'},
60+
];
61+
62+
// Construct inspect configuration.
63+
const inspectConfig = {
64+
infoTypes: infoTypes,
65+
includeQuote: true,
66+
minLikelihood: DLP.protos.google.privacy.dlp.v2.Likelihood.UNLIKELY,
67+
limits: {
68+
maxFindingsPerItem: 100,
69+
},
70+
};
71+
72+
// Specify the action that is triggered when the job completes.
73+
const action = {
74+
publishSummaryToCscc: {
75+
enable: true,
76+
},
77+
};
78+
79+
// Configure the inspection job we want the service to perform.
80+
const inspectJobConfig = {
81+
inspectConfig: inspectConfig,
82+
storageConfig: storageItem,
83+
actions: [action],
84+
};
85+
86+
// Construct the job creation request to be sent by the client.
87+
const request = {
88+
parent: `projects/${projectId}/locations/global`,
89+
inspectJob: inspectJobConfig,
90+
};
91+
92+
// Send the job creation request and process the response.
93+
const [jobsResponse] = await dlp.createDlpJob(request);
94+
const jobName = jobsResponse.name;
95+
96+
// Waiting for a maximum of 15 minutes for the job to get complete.
97+
let job;
98+
let numOfAttempts = 30;
99+
while (numOfAttempts > 0) {
100+
// Fetch DLP Job status
101+
[job] = await dlp.getDlpJob({name: jobName});
102+
103+
// Check if the job has completed.
104+
if (job.state === 'DONE') {
105+
break;
106+
}
107+
if (job.state === 'FAILED') {
108+
console.log('Job Failed, Please check the configuration.');
109+
return;
110+
}
111+
// Sleep for a short duration before checking the job status again.
112+
await new Promise(resolve => {
113+
setTimeout(() => resolve(), 30000);
114+
});
115+
numOfAttempts -= 1;
116+
}
117+
118+
// Print out the results.
119+
const infoTypeStats = job.inspectDetails.result.infoTypeStats;
120+
if (infoTypeStats.length > 0) {
121+
infoTypeStats.forEach(infoTypeStat => {
122+
console.log(
123+
` Found ${infoTypeStat.count} instance(s) of infoType ${infoTypeStat.infoType.name}.`
124+
);
125+
});
126+
} else {
127+
console.log('No findings.');
128+
}
129+
}
130+
await inspectBigQuerySendToScc();
131+
// [END dlp_inspect_bigquery_send_to_scc]
132+
}
133+
134+
process.on('unhandledRejection', err => {
135+
console.error(err.message);
136+
process.exitCode = 1;
137+
});
138+
139+
// TODO(developer): Please uncomment below line before running sample
140+
// main(...process.argv.slice(2));
141+
142+
module.exports = main;

‎dlp/inspectDatastoreSendToScc.js

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
// Copyright 2023 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
'use strict';
16+
17+
// sample-metadata:
18+
// title: Create a Dlp Job with Datastore Data and send its findings to Scc
19+
// description: Uses the Data Loss Prevention API to Create a Dlp Job with Datastore Data and send its findings to Scc
20+
// usage: node inspectDatastoreSendToScc.js my-project datastoreNamespace, datastoreKind
21+
async function main(projectId, datastoreNamespace, datastoreKind) {
22+
// [START dlp_inspect_datastore_send_to_scc]
23+
// Imports the Google Cloud Data Loss Prevention library
24+
const DLP = require('@google-cloud/dlp');
25+
26+
// Instantiates a client
27+
const dlp = new DLP.DlpServiceClient();
28+
29+
// The project ID to run the API call under.
30+
// const projectId = "your-project-id";
31+
32+
// Datastore namespace
33+
// const datastoreNamespace = 'datastore-namespace';
34+
35+
// Datastore kind
36+
// const datastoreKind = 'datastore-kind';
37+
38+
async function inspectDatastoreSendToScc() {
39+
// Specify the storage configuration object with datastore.
40+
const storageConfig = {
41+
datastoreOptions: {
42+
kind: {
43+
name: datastoreKind,
44+
},
45+
partitionId: {
46+
projectId: projectId,
47+
namespaceId: datastoreNamespace,
48+
},
49+
},
50+
};
51+
52+
// Construct the info types to look for in the datastore.
53+
const infoTypes = [
54+
{name: 'EMAIL_ADDRESS'},
55+
{name: 'PERSON_NAME'},
56+
{name: 'LOCATION'},
57+
{name: 'PHONE_NUMBER'},
58+
];
59+
60+
// Construct the inspection configuration.
61+
const inspectConfig = {
62+
infoTypes: infoTypes,
63+
minLikelihood: DLP.protos.google.privacy.dlp.v2.Likelihood.UNLIKELY,
64+
limits: {
65+
maxFindingsPerItem: 100,
66+
},
67+
includeQuote: true,
68+
};
69+
70+
// Specify the action that is triggered when the job completes
71+
const action = {
72+
publishSummaryToCscc: {enable: true},
73+
};
74+
75+
// Configure the inspection job we want the service to perform.
76+
const inspectJobConfig = {
77+
inspectConfig: inspectConfig,
78+
storageConfig: storageConfig,
79+
actions: [action],
80+
};
81+
82+
// Construct the job creation request to be sent by the client.
83+
const request = {
84+
parent: `projects/${projectId}/locations/global`,
85+
inspectJob: inspectJobConfig,
86+
};
87+
88+
// Send the job creation request and process the response.
89+
const [jobsResponse] = await dlp.createDlpJob(request);
90+
const jobName = jobsResponse.name;
91+
92+
// Waiting for a maximum of 15 minutes for the job to get complete.
93+
let job;
94+
let numOfAttempts = 30;
95+
while (numOfAttempts > 0) {
96+
// Fetch DLP Job status
97+
[job] = await dlp.getDlpJob({name: jobName});
98+
99+
// Check if the job has completed.
100+
if (job.state === 'DONE') {
101+
break;
102+
}
103+
if (job.state === 'FAILED') {
104+
console.log('Job Failed, Please check the configuration.');
105+
return;
106+
}
107+
// Sleep for a short duration before checking the job status again.
108+
await new Promise(resolve => {
109+
setTimeout(() => resolve(), 30000);
110+
});
111+
numOfAttempts -= 1;
112+
}
113+
114+
// Print out the results.
115+
const infoTypeStats = job.inspectDetails.result.infoTypeStats;
116+
if (infoTypeStats.length > 0) {
117+
infoTypeStats.forEach(infoTypeStat => {
118+
console.log(
119+
`Found ${infoTypeStat.count} instance(s) of infoType ${infoTypeStat.infoType.name}.`
120+
);
121+
});
122+
} else {
123+
console.log('No findings.');
124+
}
125+
}
126+
await inspectDatastoreSendToScc();
127+
// [END dlp_inspect_datastore_send_to_scc]
128+
}
129+
130+
process.on('unhandledRejection', err => {
131+
console.error(err.message);
132+
process.exitCode = 1;
133+
});
134+
135+
// TODO(developer): Please uncomment below line before running sample
136+
// main(...process.argv.slice(2));
137+
138+
module.exports = main;

0 commit comments

Comments
 (0)