feat(integ-tests): make assertions on deployed infrastructure (#20071)

This PR introduces a new group of constructs that allow you to make assertions against deployed infrastructure. They are not exported yet so we can work through the todo list in follow up PRs. TODO: - [ ] Add more assertion types (i.e. objectContaining) - [ ] Update integ-runner to collect the assertion results - [ ] Assertion custom resources should not(?) be part of the snapshot diff - [ ] Assertions need to be run on every deploy (i.e. update workflow) but that should not be part of the snapshot diff ---- ### All Submissions: * [ ] Have you followed the guidelines in our [Contributing guide?](https://github.com/aws/aws-cdk/blob/master/CONTRIBUTING.md) ### Adding new Unconventional Dependencies: * [ ] This PR adds new unconventional dependencies following the process described [here](https://github.com/aws/aws-cdk/blob/master/CONTRIBUTING.md/#adding-new-unconventional-dependencies) ### New Features * [ ] Have you added the new feature to an [integration test](https://github.com/aws/aws-cdk/blob/master/INTEGRATION_TESTS.md)? * [ ] Did you use `yarn integ` to deploy the infrastructure and generate the snapshot (i.e. `yarn integ` without `--dry-run`)? *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
aws · Apr 25, 2022 · 8362efe · 8362efe
1 parent fd306ee
commit 8362efe
Show file tree

Hide file tree

Showing 25 changed files with 2,718 additions and 7 deletions.
diff --git a/packages/@aws-cdk/integ-tests/lib/assertions/assertions.ts b/packages/@aws-cdk/integ-tests/lib/assertions/assertions.ts
@@ -0,0 +1,53 @@
+import { CustomResource } from '@aws-cdk/core';
+import { Construct } from 'constructs';
+import { IAssertion } from './deploy-assert';
+import { AssertionRequest, AssertionsProvider, ASSERT_RESOURCE_TYPE, AssertionType } from './providers';
+//
+// keep this import separate from other imports to reduce chance for merge conflicts with v2-main
+// eslint-disable-next-line no-duplicate-imports, import/order
+import { Construct as CoreConstruct } from '@aws-cdk/core';
+
+/**
+ * Options for an EqualsAssertion
+ */
+export interface EqualsAssertionProps {
+  /**
+   * The CustomResource that continains the "actual" results
+   */
+  readonly inputResource: CustomResource;
+
+  /**
+   * The CustomResource attribute that continains the "actual" results
+   */
+  readonly inputResourceAtt: string;
+
+  /**
+   * The expected result to assert
+   */
+  readonly expected: any;
+}
+
+/**
+ * Construct that creates a CustomResource to assert that two
+ * values are equal
+ */
+export class EqualsAssertion extends CoreConstruct implements IAssertion {
+  public readonly result: string;
+
+  constructor(scope: Construct, id: string, props: EqualsAssertionProps) {
+    super(scope, id);
+
+    const assertionProvider = new AssertionsProvider(this, 'AssertionProvider');
+    const properties: AssertionRequest = {
+      actual: props.inputResource.getAttString(props.inputResourceAtt),
+      expected: props.expected,
+      assertionType: AssertionType.EQUALS,
+    };
+    const resource = new CustomResource(this, 'Default', {
+      serviceToken: assertionProvider.serviceToken,
+      properties,
+      resourceType: ASSERT_RESOURCE_TYPE,
+    });
+    this.result = resource.getAttString('data');
+  }
+}
diff --git a/packages/@aws-cdk/integ-tests/lib/assertions/deploy-assert.ts b/packages/@aws-cdk/integ-tests/lib/assertions/deploy-assert.ts
@@ -0,0 +1,95 @@
+import { CfnOutput, CustomResource, Lazy } from '@aws-cdk/core';
+import { Construct, IConstruct, Node } from 'constructs';
+import { md5hash } from './private/hash';
+import { RESULTS_RESOURCE_TYPE, AssertionsProvider } from './providers';
+import { SdkQuery, SdkQueryOptions } from './sdk';
+
+const DEPLOY_ASSERT_SYMBOL = Symbol.for('@aws-cdk/integ-tests.DeployAssert');
+
+// keep this import separate from other imports to reduce chance for merge conflicts with v2-main
+// eslint-disable-next-line no-duplicate-imports, import/order
+import { Construct as CoreConstruct } from '@aws-cdk/core';
+
+/**
+ * Represents a deploy time assertion
+ */
+export interface IAssertion {
+  /**
+   * The result of the assertion
+   */
+  readonly result: string;
+}
+
+/**
+ * Options for DeployAssert
+ */
+export interface DeployAssertProps { }
+
+/**
+ * Construct that allows for registering a list of assertions
+ * that should be performed on a construct
+ */
+export class DeployAssert extends CoreConstruct {
+
+  /**
+   * Returns whether the construct is a DeployAssert construct
+   */
+  public static isDeployAssert(x: any): x is DeployAssert {
+    return x !== null && typeof(x) === 'object' && DEPLOY_ASSERT_SYMBOL in x;
+  }
+
+  /**
+   * Finds a DeployAssert construct in the given scope
+   */
+  public static of(construct: IConstruct): DeployAssert {
+    const scopes = Node.of(construct).scopes.reverse();
+    const deployAssert = scopes.find(s => DeployAssert.isDeployAssert(s));
+    if (!deployAssert) {
+      throw new Error('No DeployAssert construct found in scopes');
+    }
+    return deployAssert as DeployAssert;
+  }
+
+  /** @internal */
+  public readonly _assertions: IAssertion[];
+
+  constructor(scope: Construct) {
+    super(scope, 'DeployAssert');
+
+    Object.defineProperty(this, DEPLOY_ASSERT_SYMBOL, { value: true });
+    this._assertions = [];
+
+    const provider = new AssertionsProvider(this, 'ResultsProvider');
+
+    const resource = new CustomResource(this, 'ResultsCollection', {
+      serviceToken: provider.serviceToken,
+      properties: {
+        assertionResults: Lazy.list({
+          produce: () => this._assertions.map(a => a.result),
+        }),
+      },
+      resourceType: RESULTS_RESOURCE_TYPE,
+    });
+
+    // TODO: need to show/store this information
+    new CfnOutput(this, 'Results', {
+      value: `\n${resource.getAttString('message')}`,
+    }).overrideLogicalId('Results');
+  }
+
+  /**
+   * Query AWS using JavaScript SDK V2 API calls
+   */
+  public queryAws(options: SdkQueryOptions): SdkQuery {
+    const id = md5hash(options);
+    return new SdkQuery(this, `SdkQuery${id}`, options);
+  }
+
+  /**
+   * Register an assertion that should be run as part of the
+   * deployment
+   */
+  public registerAssertion(assertion: IAssertion) {
+    this._assertions.push(assertion);
+  }
+}
diff --git a/packages/@aws-cdk/integ-tests/lib/assertions/index.ts b/packages/@aws-cdk/integ-tests/lib/assertions/index.ts
@@ -0,0 +1,4 @@
+export * from './assertions';
+export * from './sdk';
+export * from './deploy-assert';
+export * from './providers';
diff --git a/packages/@aws-cdk/integ-tests/lib/assertions/private/hash.ts b/packages/@aws-cdk/integ-tests/lib/assertions/private/hash.ts
@@ -0,0 +1,10 @@
+import * as crypto from 'crypto';
+
+export function md5hash(obj: any): string {
+  if (!obj || (typeof(obj) === 'object' && Object.keys(obj).length === 0)) {
+    throw new Error('Cannot compute md5 hash for falsy object');
+  }
+  const hash = crypto.createHash('md5');
+  hash.update(JSON.stringify(obj));
+  return hash.digest('hex');
+}
diff --git a/packages/@aws-cdk/integ-tests/lib/assertions/providers/index.ts b/packages/@aws-cdk/integ-tests/lib/assertions/providers/index.ts
@@ -0,0 +1,2 @@
+export * from './lambda-handler/types';
+export * from './provider';
diff --git a/packages/@aws-cdk/integ-tests/lib/assertions/providers/lambda-handler/assertion.ts b/packages/@aws-cdk/integ-tests/lib/assertions/providers/lambda-handler/assertion.ts
@@ -0,0 +1,34 @@
+/* eslint-disable no-console */
+import * as assert from 'assert';
+import { CustomResourceHandler } from './base';
+import { AssertionRequest, AssertionResult } from './types';
+
+export class AssertionHandler extends CustomResourceHandler<AssertionRequest, AssertionResult> {
+  protected async processEvent(request: AssertionRequest): Promise<AssertionResult | undefined> {
+    let result: AssertionResult;
+    switch (request.assertionType) {
+      case 'equals':
+        console.log(`Testing equality between ${JSON.stringify(request.actual)} and ${JSON.stringify(request.expected)}`);
+        try {
+          assert.deepStrictEqual(request.actual, request.expected);
+          result = { data: { status: 'pass' } };
+        } catch (e) {
+          if (e instanceof assert.AssertionError) {
+            result = {
+              data: {
+                status: 'fail',
+                message: e.message,
+              },
+            };
+          } else {
+            throw e;
+          }
+        }
+        break;
+      default:
+        throw new Error(`Unsupported query type ${request.assertionType}`);
+    }
+
+    return result;
+  }
+}
diff --git a/packages/@aws-cdk/integ-tests/lib/assertions/providers/lambda-handler/base.ts b/packages/@aws-cdk/integ-tests/lib/assertions/providers/lambda-handler/base.ts
@@ -0,0 +1,100 @@
+/* eslint-disable no-console */
+import * as https from 'https';
+import * as url from 'url';
+
+interface HandlerResponse {
+  readonly status: 'SUCCESS' | 'FAILED';
+  readonly reason: 'OK' | string;
+  readonly data?: any;
+}
+
+// eslint-disable-next-line @typescript-eslint/ban-types
+export abstract class CustomResourceHandler<Request extends object, Response extends object> {
+  public readonly physicalResourceId: string;
+  private readonly timeout: NodeJS.Timeout;
+  private timedOut = false;
+
+  constructor(protected readonly event: AWSLambda.CloudFormationCustomResourceEvent, protected readonly context: AWSLambda.Context) {
+    this.timeout = setTimeout(async () => {
+      await this.respond({
+        status: 'FAILED',
+        reason: 'Lambda Function Timeout',
+        data: this.context.logStreamName,
+      });
+      this.timedOut = true;
+    }, context.getRemainingTimeInMillis() - 1200);
+    this.event = event;
+    this.physicalResourceId = extractPhysicalResourceId(event);
+  }
+
+  public async handle(): Promise<void> {
+    try {
+      console.log(`Event: ${JSON.stringify(this.event)}`);
+      const response = await this.processEvent(this.event.ResourceProperties as unknown as Request);
+      console.log(`Event output : ${JSON.stringify(response)}`);
+      await this.respond({
+        status: 'SUCCESS',
+        reason: 'OK',
+        data: response,
+      });
+    } catch (e) {
+      console.log(e);
+      await this.respond({
+        status: 'FAILED',
+        reason: e.message ?? 'Internal Error',
+      });
+    } finally {
+      clearTimeout(this.timeout);
+    }
+  }
+
+  protected abstract processEvent(request: Request): Promise<Response | undefined>;
+
+  private respond(response: HandlerResponse) {
+    if (this.timedOut) {
+      return;
+    }
+    const cfResponse: AWSLambda.CloudFormationCustomResourceResponse = {
+      Status: response.status,
+      Reason: response.reason,
+      PhysicalResourceId: this.physicalResourceId,
+      StackId: this.event.StackId,
+      RequestId: this.event.RequestId,
+      LogicalResourceId: this.event.LogicalResourceId,
+      NoEcho: false,
+      Data: response.data,
+    };
+    const responseBody = JSON.stringify(cfResponse);
+
+    console.log('Responding to CloudFormation', responseBody);
+
+    const parsedUrl = url.parse(this.event.ResponseURL);
+    const requestOptions = {
+      hostname: parsedUrl.hostname,
+      path: parsedUrl.path,
+      method: 'PUT',
+      headers: { 'content-type': '', 'content-length': responseBody.length },
+    };
+
+    return new Promise((resolve, reject) => {
+      try {
+        const request = https.request(requestOptions, resolve);
+        request.on('error', reject);
+        request.write(responseBody);
+        request.end();
+      } catch (e) {
+        reject(e);
+      }
+    });
+  }
+}
+
+function extractPhysicalResourceId(event: AWSLambda.CloudFormationCustomResourceEvent): string {
+  switch (event.RequestType) {
+    case 'Create':
+      return event.LogicalResourceId;
+    case 'Update':
+    case 'Delete':
+      return event.PhysicalResourceId;
+  }
+}
diff --git a/packages/@aws-cdk/integ-tests/lib/assertions/providers/lambda-handler/index.ts b/packages/@aws-cdk/integ-tests/lib/assertions/providers/lambda-handler/index.ts
@@ -0,0 +1,21 @@
+import { AssertionHandler } from './assertion';
+import { ResultsCollectionHandler } from './results';
+import { SdkHandler } from './sdk';
+import * as types from './types';
+
+export async function handler(event: AWSLambda.CloudFormationCustomResourceEvent, context: AWSLambda.Context) {
+  const provider = createResourceHandler(event, context);
+  await provider.handle();
+}
+
+function createResourceHandler(event: AWSLambda.CloudFormationCustomResourceEvent, context: AWSLambda.Context) {
+  if (event.ResourceType.startsWith(types.SDK_RESOURCE_TYPE_PREFIX)) {
+    return new SdkHandler(event, context);
+  }
+  switch (event.ResourceType) {
+    case types.ASSERT_RESOURCE_TYPE: return new AssertionHandler(event, context);
+    case types.RESULTS_RESOURCE_TYPE: return new ResultsCollectionHandler(event, context);
+    default:
+      throw new Error(`Unsupported resource type "${event.ResourceType}`);
+  }
+}
diff --git a/packages/@aws-cdk/integ-tests/lib/assertions/providers/lambda-handler/results.ts b/packages/@aws-cdk/integ-tests/lib/assertions/providers/lambda-handler/results.ts
@@ -0,0 +1,12 @@
+import { CustomResourceHandler } from './base';
+import { ResultsCollectionRequest, ResultsCollectionResult } from './types';
+
+export class ResultsCollectionHandler extends CustomResourceHandler<ResultsCollectionRequest, ResultsCollectionResult> {
+  protected async processEvent(request: ResultsCollectionRequest): Promise<ResultsCollectionResult | undefined> {
+    const reduced: string = request.assertionResults.reduce((agg, result, idx) => {
+      const msg = result.status === 'pass' ? 'pass' : `fail - ${result.message}`;
+      return `${agg}\nTest${idx}: ${msg}`;
+    }, '').trim();
+    return { message: reduced };
+  }
+}
diff --git a/packages/@aws-cdk/integ-tests/lib/assertions/providers/lambda-handler/sdk.ts b/packages/@aws-cdk/integ-tests/lib/assertions/providers/lambda-handler/sdk.ts
@@ -0,0 +1,59 @@
+/* eslint-disable no-console */
+import { CustomResourceHandler } from './base';
+import { SdkRequest, SdkResult } from './types';
+
+/**
+ * Flattens a nested object
+ *
+ * @param object the object to be flattened
+ * @returns a flat object with path as keys
+ */
+export function flatten(object: object): { [key: string]: any } {
+  return Object.assign(
+    {},
+    ...function _flatten(child: any, path: string[] = []): any {
+      return [].concat(...Object.keys(child)
+        .map(key => {
+          const childKey = Buffer.isBuffer(child[key]) ? child[key].toString('utf8') : child[key];
+          return typeof childKey === 'object' && childKey !== null
+            ? _flatten(childKey, path.concat([key]))
+            : ({ [path.concat([key]).join('.')]: childKey });
+        }));
+    }(object),
+  );
+}
+
+
+export class SdkHandler extends CustomResourceHandler<SdkRequest, SdkResult | { [key: string]: string }> {
+  protected async processEvent(request: SdkRequest): Promise<SdkResult | { [key: string]: string } | undefined> {
+    // eslint-disable-next-line
+    const AWS: any = require('aws-sdk');
+    console.log(`AWS SDK VERSION: ${AWS.VERSION}`);
+
+    const service = new AWS[request.service]();
+    const response = await service[request.api](request.parameters && decode(request.parameters)).promise();
+    console.log(`SDK response received ${JSON.stringify(response)}`);
+    delete response.ResponseMetadata;
+    const respond = {
+      apiCallResponse: response,
+    };
+    const flatData: { [key: string]: string } = {
+      ...flatten(respond),
+    };
+
+    return request.flattenResponse === 'true' ? flatData : respond;
+  }
+}
+
+function decode(object: Record<string, unknown>) {
+  return JSON.parse(JSON.stringify(object), (_k, v) => {
+    switch (v) {
+      case 'TRUE:BOOLEAN':
+        return true;
+      case 'FALSE:BOOLEAN':
+        return false;
+      default:
+        return v;
+    }
+  });
+}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		export * from './lambda-handler/types';
		export * from './provider';