test(tracer): make e2e tests to follow the same convention in Logger and Metrics (#788)

* test(tracer): refactor to pass runtime info + extract code to add a Lambda function

* test(tracer): refactor to extract common used values out

* test: refactor to take manual test out in its own separated file

* test(tracer): refactor to move reusable code to functions

* test(tracer): add middy test + refactor to make it more readable

* test(tracer): Add two variant of middy case

* test(tracer): Add decorator cases

* test:(tracer): add async handler test cases

* test:(tracer) remove unused code

* test(tracer): update comment + fix incorrect test case name

* test(tracer): add targets to run test in both runtimes

* test(tracer): make import relative and put relative imports after

Author: ijemmy

Diff for: packages/commons/tests/utils/e2eUtils.ts

@@ -16,7 +16,7 @@ const lambdaClient = new AWS.Lambda();
 
 const testRuntimeKeys = [ 'nodejs12x', 'nodejs14x' ];
 export type TestRuntimesKey = typeof testRuntimeKeys[number];
-const TEST_RUNTIMES: Record<TestRuntimesKey, Runtime> = {
+export const TEST_RUNTIMES: Record<TestRuntimesKey, Runtime> = {
   nodejs12x: Runtime.NODEJS_12_X,
   nodejs14x: Runtime.NODEJS_14_X,
 };
@@ -32,6 +32,8 @@ export type StackWithLambdaFunctionOptions = {
   runtime: string
 };
 
+type FunctionPayload = {[key: string]: string | boolean | number};
+
 export const isValidRuntimeKey = (runtime: string): runtime is TestRuntimesKey => testRuntimeKeys.includes(runtime);
 
 export const createStackWithLambdaFunction = (params: StackWithLambdaFunctionOptions): Stack => {
@@ -57,14 +59,15 @@ export const createStackWithLambdaFunction = (params: StackWithLambdaFunctionOpt
 export const generateUniqueName = (name_prefix: string, uuid: string, runtime: string, testName: string): string => 
   `${name_prefix}-${runtime}-${testName}-${uuid}`.substring(0, 64);
 
-export const invokeFunction = async (functionName: string, times: number = 1, invocationMode: 'PARALLEL' | 'SEQUENTIAL' = 'PARALLEL'): Promise<InvocationLogs[]> => {
+export const invokeFunction = async (functionName: string, times: number = 1, invocationMode: 'PARALLEL' | 'SEQUENTIAL' = 'PARALLEL', payload: FunctionPayload = {}): Promise<InvocationLogs[]> => {
   const invocationLogs: InvocationLogs[] = [];
 
   const promiseFactory = (): Promise<void> => {
     const invokePromise = lambdaClient
       .invoke({
         FunctionName: functionName,
         LogType: 'Tail', // Wait until execution completes and return all logs
+        Payload: JSON.stringify(payload),
       })
       .promise()
       .then((response) => {

Diff for: packages/tracing/package.json

@@ -13,7 +13,9 @@
     "commit": "commit",
     "test": "npm run test:unit",
     "test:unit": "jest --group=unit --detectOpenHandles --coverage --verbose",
-    "test:e2e": "jest --group=e2e",
+    "test:e2e:nodejs12x": "RUNTIME=nodejs12x jest --group=e2e",
+    "test:e2e:nodejs14x": "RUNTIME=nodejs14x jest --group=e2e",
+    "test:e2e": "concurrently \"npm:test:e2e:nodejs12x\" \"npm:test:e2e:nodejs14x\"",
     "watch": "jest --watch",
     "build": "tsc",
     "lint": "eslint --ext .ts --fix --no-error-on-unmatched-pattern src tests",

Diff for: packages/tracing/tests/e2e/tracer.test.Decorator.ts renamed to packages/tracing/tests/e2e/allFeatures.decorator.test.functionCode.ts

@@ -0,0 +1,331 @@
+/**
+ * Test tracer in decorator setup
+ *
+ * @group e2e/tracer/decorator
+ */
+
+import { randomUUID } from 'crypto';
+import path from 'path';
+import { Table, AttributeType, BillingMode } from 'aws-cdk-lib/aws-dynamodb';
+import { App, Stack, RemovalPolicy } from 'aws-cdk-lib';
+import * as AWS from 'aws-sdk';
+import { deployStack, destroyStack } from '../../../commons/tests/utils/cdk-cli';
+import {
+  getTraces,
+  getInvocationSubsegment,
+  splitSegmentsByName,
+  invokeAllTestCases,
+  createTracerTestFunction,
+  getFunctionArn,
+  getFirstSubsegment,
+} from '../helpers/tracesUtils';
+import {
+  generateUniqueName,
+  isValidRuntimeKey,
+} from '../../../commons/tests/utils/e2eUtils';
+import { 
+  RESOURCE_NAME_PREFIX,
+  SETUP_TIMEOUT, 
+  TEARDOWN_TIMEOUT, 
+  TEST_CASE_TIMEOUT,
+  expectedCustomAnnotationKey, 
+  expectedCustomAnnotationValue, 
+  expectedCustomMetadataKey, 
+  expectedCustomMetadataValue, 
+  expectedCustomResponseValue, 
+  expectedCustomErrorMessage,
+} from './constants';
+import { 
+  assertAnnotation,
+  assertErrorAndFault,
+} from '../helpers/traceAssertions';
+
+const runtime: string = process.env.RUNTIME || 'nodejs14x';
+
+if (!isValidRuntimeKey(runtime)) {
+  throw new Error(`Invalid runtime key value: ${runtime}`);
+}
+
+/**
+ * We will create a stack with 3 Lambda functions:
+ * 1. With all flags enabled (capture both response and error)
+ * 2. Do not capture error or response
+ * 3. Do not enable tracer
+ * Each stack must use a unique `serviceName` as it's used to for retrieving the trace.
+ * Using the same one will result in traces from different test cases mixing up.
+ */
+const stackName = generateUniqueName(RESOURCE_NAME_PREFIX, randomUUID(), runtime, 'AllFeatures-Decorator');
+const lambdaFunctionCodeFile = 'allFeatures.decorator.test.functionCode.ts';
+let startTime: Date;
+
+/**
+ * Function #1 is with all flags enabled.
+ */
+const uuidFunction1 = randomUUID();
+const functionNameWithAllFlagsEnabled = generateUniqueName(RESOURCE_NAME_PREFIX, uuidFunction1, runtime, 'AllFeatures-Decoratory-AllFlagsEnabled');
+const serviceNameWithAllFlagsEnabled = functionNameWithAllFlagsEnabled; 
+
+/**
+ * Function #2 doesn't capture error or response
+ */
+const uuidFunction2 = randomUUID();
+const functionNameWithNoCaptureErrorOrResponse = generateUniqueName(RESOURCE_NAME_PREFIX, uuidFunction2, runtime, 'AllFeatures-Decorator-NoCaptureErrorOrResponse');
+const serviceNameWithNoCaptureErrorOrResponse = functionNameWithNoCaptureErrorOrResponse; 
+/**
+ * Function #3 disables tracer
+ */
+const uuidFunction3 = randomUUID();
+const functionNameWithTracerDisabled = generateUniqueName(RESOURCE_NAME_PREFIX, uuidFunction3, runtime, 'AllFeatures-Decorator-TracerDisabled');
+const serviceNameWithTracerDisabled = functionNameWithNoCaptureErrorOrResponse; 
+
+const xray = new AWS.XRay();
+const invocations = 3;
+
+const integTestApp = new App();
+let stack: Stack;
+
+describe(`Tracer E2E tests, all features with decorator instantiation for runtime: ${runtime}`, () => {
+
+  beforeAll(async () => {
+    
+    // Prepare
+    startTime = new Date();
+    const ddbTableName = stackName + '-table';
+    stack = new Stack(integTestApp, stackName);
+
+    const ddbTable = new Table(stack, 'Table', {
+      tableName: ddbTableName,
+      partitionKey: {
+        name: 'id',
+        type: AttributeType.STRING
+      },
+      billingMode: BillingMode.PAY_PER_REQUEST,
+      removalPolicy: RemovalPolicy.DESTROY
+    });
+
+    const entry = path.join(__dirname, lambdaFunctionCodeFile);
+    const functionWithAllFlagsEnabled = createTracerTestFunction({
+      stack,
+      functionName: functionNameWithAllFlagsEnabled,
+      entry,
+      expectedServiceName: serviceNameWithAllFlagsEnabled,
+      environmentParams: {
+        TEST_TABLE_NAME: ddbTableName,
+        POWERTOOLS_TRACER_CAPTURE_RESPONSE: 'true',
+        POWERTOOLS_TRACER_CAPTURE_ERROR: 'true',
+        POWERTOOLS_TRACE_ENABLED: 'true',
+      },
+      runtime
+    });
+    ddbTable.grantWriteData(functionWithAllFlagsEnabled);
+
+    const functionThatDoesNotCapturesErrorAndResponse = createTracerTestFunction({
+      stack,
+      functionName: functionNameWithNoCaptureErrorOrResponse,
+      entry,
+      expectedServiceName: serviceNameWithNoCaptureErrorOrResponse,
+      environmentParams: {
+        TEST_TABLE_NAME: ddbTableName,
+        POWERTOOLS_TRACER_CAPTURE_RESPONSE: 'false',
+        POWERTOOLS_TRACER_CAPTURE_ERROR: 'false',
+        POWERTOOLS_TRACE_ENABLED: 'true',
+      },
+      runtime
+    });
+    ddbTable.grantWriteData(functionThatDoesNotCapturesErrorAndResponse);
+
+    const functionWithTracerDisabled = createTracerTestFunction({
+      stack,
+      functionName: functionNameWithTracerDisabled,
+      entry,
+      expectedServiceName: serviceNameWithTracerDisabled,
+      environmentParams: {
+        TEST_TABLE_NAME: ddbTableName,
+        POWERTOOLS_TRACER_CAPTURE_RESPONSE: 'true',
+        POWERTOOLS_TRACER_CAPTURE_ERROR: 'true',
+        POWERTOOLS_TRACE_ENABLED: 'false',
+      },
+      runtime
+    });
+    ddbTable.grantWriteData(functionWithTracerDisabled);
+
+    await deployStack(integTestApp, stack);
+
+    // Act
+    await Promise.all([
+      invokeAllTestCases(functionNameWithAllFlagsEnabled),
+      invokeAllTestCases(functionNameWithNoCaptureErrorOrResponse),
+      invokeAllTestCases(functionNameWithTracerDisabled),
+    ]);
+    
+  }, SETUP_TIMEOUT);
+
+  afterAll(async () => {
+    if (!process.env.DISABLE_TEARDOWN) {
+      await destroyStack(integTestApp, stack);
+    }
+  }, TEARDOWN_TIMEOUT);
+
+  it('should generate all custom traces', async () => {
+    
+    const tracesWhenAllFlagsEnabled = await getTraces(xray, startTime, await getFunctionArn(functionNameWithAllFlagsEnabled), invocations, 5);
+    
+    expect(tracesWhenAllFlagsEnabled.length).toBe(invocations);
+
+    // Assess
+    for (let i = 0; i < invocations; i++) {
+      const trace = tracesWhenAllFlagsEnabled[i];
+
+      /**
+       * Expect the trace to have 5 segments:
+       * 1. Lambda Context (AWS::Lambda)
+       * 2. Lambda Function (AWS::Lambda::Function)
+       * 3. DynamoDB (AWS::DynamoDB)
+       * 4. DynamoDB Table (AWS::DynamoDB::Table)
+       * 5. Remote call (httpbin.org)
+       */
+      expect(trace.Segments.length).toBe(5);
+      const invocationSubsegment = getInvocationSubsegment(trace);
+      
+      /**
+       * Invocation subsegment should have a subsegment '## index.handler' (default behavior for PowerTool tracer)
+       * '## index.handler' subsegment should have 4 subsegments
+       * 1. DynamoDB (PutItem on the table)
+       * 2. DynamoDB (PutItem overhead)
+       * 3. httpbin.org (Remote call)
+       * 4. '### myMethod' (method decorator)
+       */
+      const handlerSubsegment = getFirstSubsegment(invocationSubsegment);
+      expect(handlerSubsegment.name).toBe('## index.handler');
+      expect(handlerSubsegment?.subsegments).toHaveLength(4);
+
+      if (!handlerSubsegment.subsegments) {
+        fail('"## index.handler" subsegment should have subsegments');
+      }
+      const subsegments = splitSegmentsByName(handlerSubsegment.subsegments, [ 'DynamoDB', 'httpbin.org', '### myMethod' ]);
+      expect(subsegments.get('DynamoDB')?.length).toBe(2);
+      expect(subsegments.get('httpbin.org')?.length).toBe(1);
+      expect(subsegments.get('### myMethod')?.length).toBe(1);
+      expect(subsegments.get('other')?.length).toBe(0);
+      
+      const shouldThrowAnError = (i === (invocations - 1));
+      if (shouldThrowAnError) {
+        assertErrorAndFault(invocationSubsegment, expectedCustomErrorMessage);
+      }
+    }
+
+  }, TEST_CASE_TIMEOUT);
+  
+  it('should have correct annotations and metadata', async () => {
+    const tracesWhenAllFlagsEnabled = await getTraces(xray, startTime, await getFunctionArn(functionNameWithAllFlagsEnabled), invocations, 5);
+
+    for (let i = 0; i < invocations; i++) {
+      const trace = tracesWhenAllFlagsEnabled[i];
+      const invocationSubsegment = getInvocationSubsegment(trace);
+      const handlerSubsegment = getFirstSubsegment(invocationSubsegment);
+      const { annotations, metadata } = handlerSubsegment;
+
+      const isColdStart = (i === 0);
+      assertAnnotation({
+        annotations,
+        isColdStart,
+        expectedServiceName: serviceNameWithAllFlagsEnabled,
+        expectedCustomAnnotationKey,
+        expectedCustomAnnotationValue,
+      });
+      
+      if (!metadata) {
+        fail('metadata is missing');
+      }
+      expect(metadata[serviceNameWithAllFlagsEnabled][expectedCustomMetadataKey])
+        .toEqual(expectedCustomMetadataValue);
+
+      const shouldThrowAnError = (i === (invocations - 1));
+      if (!shouldThrowAnError) {
+        // Assert that the metadata object contains the response
+        expect(metadata[serviceNameWithAllFlagsEnabled]['index.handler response'])
+          .toEqual(expectedCustomResponseValue);
+      }
+    }
+  }, TEST_CASE_TIMEOUT);
+
+  it('should not capture error nor response when the flags are false', async () => {
+    
+    const tracesWithNoCaptureErrorOrResponse = await getTraces(xray, startTime, await getFunctionArn(functionNameWithNoCaptureErrorOrResponse), invocations, 5);
+    
+    expect(tracesWithNoCaptureErrorOrResponse.length).toBe(invocations);
+
+    // Assess
+    for (let i = 0; i < invocations; i++) {
+      const trace = tracesWithNoCaptureErrorOrResponse[i];
+
+      /**
+       * Expect the trace to have 5 segments:
+       * 1. Lambda Context (AWS::Lambda)
+       * 2. Lambda Function (AWS::Lambda::Function)
+       * 3. DynamoDB (AWS::DynamoDB)
+       * 4. DynamoDB Table (AWS::DynamoDB::Table)
+       * 5. Remote call (httpbin.org)
+       */
+      expect(trace.Segments.length).toBe(5);
+      const invocationSubsegment = getInvocationSubsegment(trace);
+      
+      /**
+       * Invocation subsegment should have a subsegment '## index.handler' (default behavior for PowerTool tracer)
+       * '## index.handler' subsegment should have 4 subsegments
+       * 1. DynamoDB (PutItem on the table)
+       * 2. DynamoDB (PutItem overhead)
+       * 3. httpbin.org (Remote call)
+       * 4. '### myMethod' (method decorator)
+       */
+      const handlerSubsegment = getFirstSubsegment(invocationSubsegment);
+      expect(handlerSubsegment.name).toBe('## index.handler');
+      expect(handlerSubsegment?.subsegments).toHaveLength(4);
+
+      if (!handlerSubsegment.subsegments) {
+        fail('"## index.handler" subsegment should have subsegments');
+      }
+      const subsegments = splitSegmentsByName(handlerSubsegment.subsegments, [ 'DynamoDB', 'httpbin.org', '### myMethod' ]);
+      expect(subsegments.get('DynamoDB')?.length).toBe(2);
+      expect(subsegments.get('httpbin.org')?.length).toBe(1);
+      expect(subsegments.get('### myMethod')?.length).toBe(1);
+      expect(subsegments.get('other')?.length).toBe(0);
+      
+      const shouldThrowAnError = (i === (invocations - 1));
+      if (shouldThrowAnError) {
+        // Assert that the subsegment has the expected fault
+        expect(invocationSubsegment.error).toBe(true);
+        expect(handlerSubsegment.error).toBe(true);
+        // Assert that no error was captured on the subsegment
+        expect(handlerSubsegment.hasOwnProperty('cause')).toBe(false);
+      }
+    }
+
+  }, TEST_CASE_TIMEOUT);
+
+  it('should not capture any custom traces when disabled', async () => {
+    const expectedNoOfTraces = 2;
+    const tracesWithTracerDisabled = await getTraces(xray, startTime, await getFunctionArn(functionNameWithTracerDisabled), invocations, expectedNoOfTraces);
+    
+    expect(tracesWithTracerDisabled.length).toBe(invocations);
+
+    // Assess
+    for (let i = 0; i < invocations; i++) {
+      const trace = tracesWithTracerDisabled[i];
+      expect(trace.Segments.length).toBe(2);
+      
+      /**
+       * Expect no subsegment in the invocation
+       */
+      const invocationSubsegment = getInvocationSubsegment(trace);
+      expect(invocationSubsegment?.subsegments).toBeUndefined();
+      
+      const shouldThrowAnError = (i === (invocations - 1));
+      if (shouldThrowAnError) {
+        expect(invocationSubsegment.error).toBe(true);
+      }
+    }
+
+  }, TEST_CASE_TIMEOUT);
+});
+