feat(integ-runner): support custom --app commands (#22761)

To prepare for supporting other languages than TypeScript (see #22521) we need to be able to run test files with commands other than the current default `node`. With this PR, users can specify a custom `--app` command that will substitute the string `{filePath}` with the filename of each discovered test, and synth each test with that command. Example usage: `integ-runner --app '"node --no-warnings {filePath}"'` Until we also allow discovery of different file prefixes, this can't be used to run languages like Python, because the default prefix used now (`integ.`) contains a `.` which is not a valid character for Python module names. ---- ### All Submissions: * [x] Have you followed the guidelines in our [Contributing guide?](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md) ### Adding new Unconventional Dependencies: * [ ] This PR adds new unconventional dependencies following the process described [here](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md/#adding-new-unconventional-dependencies) ### New Features * [ ] Have you added the new feature to an [integration test](https://github.com/aws/aws-cdk/blob/main/INTEGRATION_TESTS.md)? * [ ] Did you use `yarn integ` to deploy the infrastructure and generate the snapshot (i.e. `yarn integ` without `--dry-run`)? *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
aws · Nov 7, 2022 · a7bb6e1 · a7bb6e1
1 parent df5830c
commit a7bb6e1
Show file tree

Hide file tree

Showing 7 changed files with 63 additions and 3 deletions.
diff --git a/packages/@aws-cdk/integ-runner/README.md b/packages/@aws-cdk/integ-runner/README.md
@@ -68,7 +68,8 @@ to be a self contained CDK app. The runner will execute the following for each f
   Read the list of tests from this file
 - `--disable-update-workflow` (default=`false`)
   If this is set to `true` then the [update workflow](#update-workflow) will be disabled
-
+- `--app`
+  The custom CLI command that will be used to run the test files. You can include {filePath} to specify where in the command the test file path should be inserted. Example: --app="python3.8 {filePath}".
 Example:
 
 ```bash

diff --git a/packages/@aws-cdk/integ-runner/lib/cli.ts b/packages/@aws-cdk/integ-runner/lib/cli.ts
@@ -30,6 +30,7 @@ async function main() {
     .options('from-file', { type: 'string', desc: 'Read TEST names from a file (one TEST per line)' })
     .option('inspect-failures', { type: 'boolean', desc: 'Keep the integ test cloud assembly if a failure occurs for inspection', default: false })
     .option('disable-update-workflow', { type: 'boolean', default: false, desc: 'If this is "true" then the stack update workflow will be disabled' })
+    .option('app', { type: 'string', default: undefined, desc: 'The custom CLI command that will be used to run the test files. You can include {filePath} to specify where in the command the test file path should be inserted. Example: --app="python3.8 {filePath}".' })
     .strict()
     .argv;
 
@@ -76,6 +77,7 @@ async function main() {
     failedSnapshots = await runSnapshotTests(pool, testsFromArgs, {
       retain: argv['inspect-failures'],
       verbose: Boolean(argv.verbose),
+      appCommand: argv.app,
     });
     for (const failure of failedSnapshots) {
       destructiveChanges.push(...failure.destructiveChanges ?? []);
@@ -99,6 +101,7 @@ async function main() {
         dryRun: argv['dry-run'],
         verbosity: argv.verbose,
         updateWorkflow: !argv['disable-update-workflow'],
+        appCommand: argv.app,
       });
       testsSucceeded = success;
 

diff --git a/packages/@aws-cdk/integ-runner/lib/runner/runner-base.ts b/packages/@aws-cdk/integ-runner/lib/runner/runner-base.ts
@@ -49,6 +49,14 @@ export interface IntegRunnerOptions {
    */
   readonly cdk?: ICdk;
 
+  /**
+   * You can specify a custom run command, and it will be applied to all test files.
+   * If it contains {filePath}, the test file names will be substituted at that place in the command for each run.
+   *
+   * @default - test run command will be `node {filePath}`
+   */
+  readonly appCommand?: string;
+
   /**
    * Show output from running integration tests
    *
@@ -150,7 +158,10 @@ export abstract class IntegRunner {
       },
     });
     this.cdkOutDir = options.integOutDir ?? this.test.temporaryOutputDir;
-    this.cdkApp = `node ${path.relative(this.directory, this.test.fileName)}`;
+
+    const testRunCommand = options.appCommand ?? 'node {filePath}';
+    this.cdkApp = testRunCommand.replace('{filePath}', path.relative(this.directory, this.test.fileName));
+
     this.profile = options.profile;
     if (this.hasSnapshot()) {
       this.expectedTestSuite = this.loadManifest();

diff --git a/packages/@aws-cdk/integ-runner/lib/workers/common.ts b/packages/@aws-cdk/integ-runner/lib/workers/common.ts
@@ -103,6 +103,13 @@ export interface SnapshotVerificationOptions {
    * @default false
    */
   readonly verbose?: boolean;
+
+  /**
+   * The CLI command used to run the test files.
+   *
+   * @default - test run command will be `node {filePath}`
+   */
+  readonly appCommand?: string;
 }
 
 /**
@@ -162,6 +169,13 @@ export interface IntegTestOptions {
    * @default true
    */
   readonly updateWorkflow?: boolean;
+
+  /**
+   * The CLI command used to run the test files.
+   *
+   * @default - test run command will be `node {filePath}`
+   */
+  readonly appCommand?: string;
 }
 
 /**

diff --git a/packages/@aws-cdk/integ-runner/lib/workers/extract/extract_worker.ts b/packages/@aws-cdk/integ-runner/lib/workers/extract/extract_worker.ts
@@ -27,6 +27,7 @@ export function integTestWorker(request: IntegTestBatchRequest): IntegTestWorker
         env: {
           AWS_REGION: request.region,
         },
+        appCommand: request.appCommand,
         showOutput: verbosity >= 2,
       }, testInfo.destructiveChanges);
 
@@ -105,7 +106,7 @@ export function snapshotTestWorker(testInfo: IntegTestInfo, options: SnapshotVer
   }, 60_000);
 
   try {
-    const runner = new IntegSnapshotRunner({ test });
+    const runner = new IntegSnapshotRunner({ test, appCommand: options.appCommand });
     if (!runner.hasSnapshot()) {
       workerpool.workerEmit({
         reason: DiagnosticReason.NO_SNAPSHOT,

diff --git a/packages/@aws-cdk/integ-runner/lib/workers/integ-test-worker.ts b/packages/@aws-cdk/integ-runner/lib/workers/integ-test-worker.ts
@@ -135,6 +135,7 @@ export async function runIntegrationTestsInParallel(
         dryRun: options.dryRun,
         verbosity: options.verbosity,
         updateWorkflow: options.updateWorkflow,
+        appCommand: options.appCommand,
       }], {
         on: printResults,
       });

diff --git a/packages/@aws-cdk/integ-runner/test/runner/integ-test-runner.test.ts b/packages/@aws-cdk/integ-runner/test/runner/integ-test-runner.test.ts
@@ -559,4 +559,33 @@ describe('IntegTest runIntegTests', () => {
       debug,
     }));
   });
+
+  test('with custom app run command for JavaScript', () => {
+    // WHEN
+    const integTest = new IntegTestRunner({
+      cdk: cdkMock.cdk,
+      test: new IntegTest({
+        fileName: 'test/test-data/xxxxx.test-with-snapshot.js',
+        discoveryRoot: 'test/test-data',
+      }),
+      appCommand: 'node --no-warnings {filePath}',
+    });
+    integTest.runIntegTestCase({
+      testCaseName: 'xxxxx.test-with-snapshot',
+    });
+
+    // THEN
+    expect(deployMock).toHaveBeenCalledTimes(2);
+    expect(destroyMock).toHaveBeenCalledTimes(1);
+    expect(synthFastMock).toHaveBeenCalledTimes(1);
+    expect(deployMock).toHaveBeenCalledWith(expect.objectContaining({
+      app: 'node --no-warnings xxxxx.test-with-snapshot.js',
+    }));
+    expect(synthFastMock).toHaveBeenCalledWith(expect.objectContaining({
+      execCmd: ['node', '--no-warnings', 'xxxxx.test-with-snapshot.js'],
+    }));
+    expect(destroyMock).toHaveBeenCalledWith(expect.objectContaining({
+      app: 'node --no-warnings xxxxx.test-with-snapshot.js',
+    }));
+  });
 });