Merge branch 'main' into demosite-app-recaptcha

Author: Sita04

.github/workflows/dlp.yaml

@@ -0,0 +1,85 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: dlp
+on:
+  push:
+    branches:
+    - main
+    paths:
+    - 'dlp/**'
+    - '.github/workflows/dlp.yaml'
+  pull_request:
+    paths:
+    - 'dlp/**'
+    - '.github/workflows/dlp.yaml'
+  pull_request_target:
+    types: [labeled]
+    paths:
+    - 'dlp/**'
+    - '.github/workflows/dlp.yaml'
+  schedule:
+  - cron:  '0 0 * * 0'
+jobs:
+  test:
+    if: ${{ github.event.action != 'labeled' || github.event.label.name == 'actions:force-run' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    permissions:
+      contents: 'write'
+      pull-requests: 'write'
+      id-token: 'write'
+    steps:
+    - uses: actions/[email protected]
+      with:
+        ref: ${{github.event.pull_request.head.sha}}
+    - uses: 'google-github-actions/[email protected]'
+      with:
+        workload_identity_provider: 'projects/1046198160504/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider'
+        service_account: '[email protected]'
+        create_credentials_file: 'true'
+        access_token_lifetime: 600s
+    - uses: actions/[email protected]
+      with:
+        node-version: 16
+    - run: npm install
+      working-directory: dlp
+    - run: npm test
+      working-directory: dlp
+      env:
+        MOCHA_REPORTER_SUITENAME: dlp
+        MOCHA_REPORTER_OUTPUT: dlp_sponge_log.xml
+        MOCHA_REPORTER: xunit
+    - if: ${{ github.event.action == 'labeled' && github.event.label.name == 'actions:force-run' }}
+      uses: actions/github-script@v6
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        script: |
+          try {
+            await github.rest.issues.removeLabel({
+              name: 'actions:force-run',
+              owner: 'GoogleCloudPlatform',
+              repo: 'nodejs-docs-samples',
+              issue_number: context.payload.pull_request.number
+            });
+          } catch (e) {
+            if (!e.message.includes('Label does not exist')) {
+              throw e;
+            }
+          }
+    - if: ${{ github.event_name == 'schedule' && always() }}
+      run: |
+        curl https://github.com/googleapis/repo-automation-bots/releases/download/flakybot-1.1.0/flakybot -o flakybot -s -L
+        chmod +x ./flakybot
+        ./flakybot --repo GoogleCloudPlatform/nodejs-docs-samples --commit_hash ${{github.sha}} --build_url https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}

.github/workflows/workflows.json

@@ -42,6 +42,7 @@
   "datastore/functions",
   "dialogflow",
   "dialogflow-cx",
+  "dlp",
   "document-ai",
   "endpoints/getting-started",
   "endpoints/getting-started-grpc",

CODEOWNERS

@@ -33,6 +33,7 @@ secret-manager @GoogleCloudPlatform/dee-infra @GoogleCloudPlatform/nodejs-sample
 security-center @GoogleCloudPlatform/dee-infra @GoogleCloudPlatform/nodejs-samples-reviewers
 service-directory @GoogleCloudPlatform/dee-infra @GoogleCloudPlatform/nodejs-samples-reviewers
 webrisk @GoogleCloudPlatform/dee-infra @GoogleCloudPlatform/nodejs-samples-reviewers
+dlp @GoogleCloudPlatform/dee-infra @GoogleCloudPlatform/nodejs-samples-reviewers
 
 # DEE Platform Ops (DEEPO)
 container @GoogleCloudPlatform/dee-platform-ops @GoogleCloudPlatform/nodejs-samples-reviewers

appengine/hello-world/flexible/package.json

@@ -10,7 +10,7 @@
     "url": "https://github.com/GoogleCloudPlatform/nodejs-docs-samples.git"
   },
   "engines": {
-    "node": "=v16.19.0"
+    "node": "=v16.19.1"
   },
   "scripts": {
     "start": "node app.js",

asset/snippets/package.json

@@ -20,7 +20,7 @@
     "@google-cloud/compute": "^3.0.0",
     "@google-cloud/resource-manager": "^4.1.1",
     "@google-cloud/storage": "^6.0.0",
-    "yargs": "^16.0.0"
+    "yargs": "^17.0.0"
   },
   "devDependencies": {
     "chai": "^4.2.0",

automl/package.json

@@ -19,7 +19,7 @@
     "@google-cloud/automl": "^3.1.2",
     "csv": "^6.0.0",
     "mathjs": "^11.0.0",
-    "yargs": "^16.0.0"
+    "yargs": "^17.0.0"
   },
   "devDependencies": {
     "@google-cloud/storage": "^6.0.0",

cloud-language/package.json

@@ -18,7 +18,7 @@
     "mathjs": "^11.0.0",
     "@google-cloud/language": "^5.1.0",
     "@google-cloud/storage": "^6.0.0",
-    "yargs": "^16.0.0"
+    "yargs": "^17.0.0"
   },
   "devDependencies": {
     "@google-cloud/automl": "^3.0.0",

datacatalog/snippets/package.json

@@ -20,7 +20,7 @@
   },
   "devDependencies": {
     "chai": "^4.2.0",
-    "execa": "^6.0.0",
+    "execa": "^7.0.0",
     "mocha": "^10.0.0",
     "uuid": "^9.0.0"
   }

dialogflow/package.json

@@ -17,7 +17,7 @@
   "dependencies": {
     "@google-cloud/dialogflow": "^5.3.0",
     "pb-util": "^1.0.0",
-    "yargs": "^16.0.0"
+    "yargs": "^17.0.0"
   },
   "devDependencies": {
     "chai": "^4.2.0",

dlp/.eslintrc.yml

@@ -0,0 +1,5 @@
+---
+rules:
+  no-console: off
+  no-warning-comments: off
+  node/no-missing-require: off

dlp/.gitignore

@@ -0,0 +1,3 @@
+# Test outputs
+*.actual.png
+*.actual.csv

dlp/categoricalRiskAnalysis.js

@@ -0,0 +1,160 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict';
+
+// sample-metadata:
+//   title: Categorical Risk Analysis
+//   description: Computes risk metrics of a column of data in a Google BigQuery table.
+//   usage: node categoricalRiskAnalysis.js my-project nhtsa_traffic_fatalities accident_2015 state_name my-topic my-subscription bigquery-public-data
+
+function main(
+  projectId,
+  tableProjectId,
+  datasetId,
+  tableId,
+  columnName,
+  topicId,
+  subscriptionId
+) {
+  // [START dlp_categorical_stats]
+  // Import the Google Cloud client libraries
+  const DLP = require('@google-cloud/dlp');
+  const {PubSub} = require('@google-cloud/pubsub');
+
+  // Instantiates clients
+  const dlp = new DLP.DlpServiceClient();
+  const pubsub = new PubSub();
+
+  // The project ID to run the API call under
+  // const projectId = 'my-project';
+
+  // The project ID the table is stored under
+  // This may or (for public datasets) may not equal the calling project ID
+  // const tableProjectId = 'my-project';
+
+  // The ID of the dataset to inspect, e.g. 'my_dataset'
+  // const datasetId = 'my_dataset';
+
+  // The ID of the table to inspect, e.g. 'my_table'
+  // const tableId = 'my_table';
+
+  // The name of the Pub/Sub topic to notify once the job completes
+  // TODO(developer): create a Pub/Sub topic to use for this
+  // const topicId = 'MY-PUBSUB-TOPIC'
+
+  // The name of the Pub/Sub subscription to use when listening for job
+  // completion notifications
+  // TODO(developer): create a Pub/Sub subscription to use for this
+  // const subscriptionId = 'MY-PUBSUB-SUBSCRIPTION'
+
+  // The name of the column to compute risk metrics for, e.g. 'firstName'
+  // const columnName = 'firstName';
+  async function categoricalRiskAnalysis() {
+    const sourceTable = {
+      projectId: tableProjectId,
+      datasetId: datasetId,
+      tableId: tableId,
+    };
+
+    // Construct request for creating a risk analysis job
+    const request = {
+      parent: `projects/${projectId}/locations/global`,
+      riskJob: {
+        privacyMetric: {
+          categoricalStatsConfig: {
+            field: {
+              name: columnName,
+            },
+          },
+        },
+        sourceTable: sourceTable,
+        actions: [
+          {
+            pubSub: {
+              topic: `projects/${projectId}/topics/${topicId}`,
+            },
+          },
+        ],
+      },
+    };
+
+    // Create helper function for unpacking values
+    const getValue = obj => obj[Object.keys(obj)[0]];
+
+    // Run risk analysis job
+    const [topicResponse] = await pubsub.topic(topicId).get();
+    const subscription = await topicResponse.subscription(subscriptionId);
+    const [jobsResponse] = await dlp.createDlpJob(request);
+    const jobName = jobsResponse.name;
+    // Watch the Pub/Sub topic until the DLP job finishes
+    await new Promise((resolve, reject) => {
+      const messageHandler = message => {
+        if (message.attributes && message.attributes.DlpJobName === jobName) {
+          message.ack();
+          subscription.removeListener('message', messageHandler);
+          subscription.removeListener('error', errorHandler);
+          resolve(jobName);
+        } else {
+          message.nack();
+        }
+      };
+
+      const errorHandler = err => {
+        subscription.removeListener('message', messageHandler);
+        subscription.removeListener('error', errorHandler);
+        reject(err);
+      };
+
+      subscription.on('message', messageHandler);
+      subscription.on('error', errorHandler);
+    });
+    setTimeout(() => {
+      console.log(' Waiting for DLP job to fully complete');
+    }, 500);
+    const [job] = await dlp.getDlpJob({name: jobName});
+    const histogramBuckets =
+      job.riskDetails.categoricalStatsResult.valueFrequencyHistogramBuckets;
+    histogramBuckets.forEach((histogramBucket, histogramBucketIdx) => {
+      console.log(`Bucket ${histogramBucketIdx}:`);
+
+      // Print bucket stats
+      console.log(
+        `  Most common value occurs ${histogramBucket.valueFrequencyUpperBound} time(s)`
+      );
+      console.log(
+        `  Least common value occurs ${histogramBucket.valueFrequencyLowerBound} time(s)`
+      );
+
+      // Print bucket values
+      console.log(`${histogramBucket.bucketSize} unique values total.`);
+      histogramBucket.bucketValues.forEach(valueBucket => {
+        console.log(
+          `  Value ${getValue(valueBucket.value)} occurs ${
+            valueBucket.count
+          } time(s).`
+        );
+      });
+    });
+  }
+
+  categoricalRiskAnalysis();
+  // [END dlp_categorical_stats]
+}
+
+main(...process.argv.slice(2));
+process.on('unhandledRejection', err => {
+  console.error(err.message);
+  process.exitCode = 1;
+});