feat(integ-runner): support config file

aws · mergify · Nov 17, 2022 · Nov 16, 2022 · Nov 16, 2022 · Nov 16, 2022
commit 1eb6c1214c4fcb8c83851e40a57cf12c68292b2a
diff --git a/packages/@aws-cdk/integ-runner/lib/cli.ts b/packages/@aws-cdk/integ-runner/lib/cli.ts
@@ -1,21 +1,26 @@
 // Exercise all integ stacks and if they deploy, update the expected synth files
-import { promises as fs } from 'fs';
+import * as fs from 'fs';
 import * as path from 'path';
 import * as chalk from 'chalk';
 import * as workerpool from 'workerpool';
 import * as logger from './logger';
-import { IntegrationTests, IntegTestInfo, IntegTest } from './runner/integration-tests';
+import { IntegrationTests, IntegTestInfo } from './runner/integration-tests';
 import { runSnapshotTests, runIntegrationTests, IntegRunnerMetrics, IntegTestWorkerConfig, DestructiveChange } from './workers';
 
 // https://github.com/yargs/yargs/issues/1929
 // https://github.com/evanw/esbuild/issues/1492
 // eslint-disable-next-line @typescript-eslint/no-require-imports
 const yargs = require('yargs');
 
-
-export async function main(args: string[]) {
+export function parseCliArgs(args: string[] = []) {
   const argv = yargs
     .usage('Usage: integ-runner [TEST...]')
+    .option('config', {
+      config: true,
+      configParser: IntegrationTests.configFromFile,
+      default: 'integ.config.json',
+      desc: 'Load options from a JSON config file. Options provided as CLI arguments take precedent.',
+    })
     .option('list', { type: 'boolean', default: false, desc: 'List tests instead of running them' })
     .option('clean', { type: 'boolean', default: true, desc: 'Skips stack clean up after test is completed (use --no-clean to negate)' })
     .option('verbose', { type: 'boolean', default: false, alias: 'v', count: true, desc: 'Verbose logs and metrics on integration tests durations (specify multiple times to increase verbosity)' })
@@ -35,61 +40,97 @@ export async function main(args: string[]) {
     .strict()
     .parse(args);
 
-  const pool = workerpool.pool(path.join(__dirname, '../lib/workers/extract/index.js'), {
-    maxWorkers: argv['max-workers'],
-  });
-
   // list of integration tests that will be executed
+  const tests: string[] = argv._;
+  const app: string | undefined = argv.app;
   const testRegex = arrayFromYargs(argv['test-regex']);
-  const testsToRun: IntegTestWorkerConfig[] = [];
-  const destructiveChanges: DestructiveChange[] = [];
-  const testsFromArgs: IntegTest[] = [];
   const parallelRegions = arrayFromYargs(argv['parallel-regions']);
   const testRegions: string[] = parallelRegions ?? ['us-east-1', 'us-east-2', 'us-west-2'];
   const profiles = arrayFromYargs(argv.profiles);
   const runUpdateOnFailed = argv['update-on-failed'] ?? false;
   const fromFile: string | undefined = argv['from-file'];
   const exclude: boolean = argv.exclude;
-  const app: string | undefined = argv.app;
+  const maxWorkers: number = argv['max-workers'];
+  const list: boolean = argv.list;
+  const directory: string = argv.directory;
+  const inspectFailures: boolean = argv['inspect-failures'];
+  const verbosity: number = argv.verbose;
+  const verbose: boolean = verbosity >= 1;
+
+  const numTests = testRegions.length * (profiles ?? [1]).length;
+  if (maxWorkers < numTests) {
+    logger.warning('You are attempting to run %s tests in parallel, but only have %s workers. Not all of your profiles+regions will be utilized', numTests, maxWorkers);
+  }
 
-  let failedSnapshots: IntegTestWorkerConfig[] = [];
-  if (argv['max-workers'] < testRegions.length * (profiles ?? [1]).length) {
-    logger.warning('You are attempting to run %s tests in parallel, but only have %s workers. Not all of your profiles+regions will be utilized', argv.profiles * argv['parallel-regions'], argv['max-workers']);
+  if (tests.length > 0 && fromFile) {
+    throw new Error('A list of tests cannot be provided if "--from-file" is provided');
   }
+  const requestedTests = fromFile
+    ? (fs.readFileSync(fromFile, { encoding: 'utf8' })).split('\n').filter(x => x)
+    : (tests.length > 0 ? tests : undefined); // 'undefined' means no request
+
+  return {
+    tests: requestedTests,
+    app,
+    testRegex,
+    testRegions,
+    profiles,
+    runUpdateOnFailed,
+    fromFile,
+    exclude,
+    maxWorkers,
+    list,
+    directory,
+    inspectFailures,
+    verbosity,
+    verbose,
+    clean: argv.clean as boolean,
+    force: argv.force as boolean,
+    dryRun: argv['dry-run'] as boolean,
+    disableUpdateWorkflow: argv['disable-update-workflow'] as boolean,
+  };
+}
 
-  let testsSucceeded = false;
-  try {
-    if (argv.list) {
-      const tests = await new IntegrationTests(argv.directory).fromCliArgs({ testRegex, app });
-      process.stdout.write(tests.map(t => t.discoveryRelativeFileName).join('\n') + '\n');
-      return;
-    }
 
-    if (argv._.length > 0 && fromFile) {
-      throw new Error('A list of tests cannot be provided if "--from-file" is provided');
-    }
-    const requestedTests = fromFile
-      ? (await fs.readFile(fromFile, { encoding: 'utf8' })).split('\n').filter(x => x)
-      : (argv._.length > 0 ? argv._ : undefined); // 'undefined' means no request
+export async function main(args: string[]) {
+  const options = parseCliArgs(args);
+
+  const testsFromArgs = await new IntegrationTests(path.resolve(options.directory)).fromCliArgs({
+    app: options.app,
+    testRegex: options.testRegex,
+    tests: options.tests,
+    exclude: options.exclude,
+  });
+
+  // List only prints the discoverd tests
+  if (options.list) {
+    process.stdout.write(testsFromArgs.map(t => t.discoveryRelativeFileName).join('\n') + '\n');
+    return;
+  }
+
 
-    testsFromArgs.push(...(await new IntegrationTests(path.resolve(argv.directory)).fromCliArgs({
-      app,
-      testRegex,
-      tests: requestedTests,
-      exclude,
-    })));
+  const pool = workerpool.pool(path.join(__dirname, '../lib/workers/extract/index.js'), {
+    maxWorkers: options.maxWorkers,
+  });
 
+
+  const testsToRun: IntegTestWorkerConfig[] = [];
+  const destructiveChanges: DestructiveChange[] = [];
+  let failedSnapshots: IntegTestWorkerConfig[] = [];
+  let testsSucceeded = false;
+
+  try {
     // always run snapshot tests, but if '--force' is passed then
     // run integration tests on all failed tests, not just those that
     // failed snapshot tests
     failedSnapshots = await runSnapshotTests(pool, testsFromArgs, {
-      retain: argv['inspect-failures'],
-      verbose: Boolean(argv.verbose),
+      retain: options.inspectFailures,
+      verbose: options.verbose,
     });
     for (const failure of failedSnapshots) {
       destructiveChanges.push(...failure.destructiveChanges ?? []);
     }
-    if (!argv.force) {
+    if (!options.force) {
       testsToRun.push(...failedSnapshots);
     } else {
       // if any of the test failed snapshot tests, keep those results
@@ -98,25 +139,25 @@ export async function main(args: string[]) {
     }
 
     // run integration tests if `--update-on-failed` OR `--force` is used
-    if (runUpdateOnFailed || argv.force) {
+    if (options.runUpdateOnFailed || options.force) {
       const { success, metrics } = await runIntegrationTests({
         pool,
         tests: testsToRun,
-        regions: testRegions,
-        profiles,
-        clean: argv.clean,
-        dryRun: argv['dry-run'],
-        verbosity: argv.verbose,
-        updateWorkflow: !argv['disable-update-workflow'],
+        regions: options.testRegions,
+        profiles: options.profiles,
+        clean: options.clean,
+        dryRun: options.dryRun,
+        verbosity: options.verbosity,
+        updateWorkflow: !options.disableUpdateWorkflow,
       });
       testsSucceeded = success;
 
 
-      if (argv.clean === false) {
+      if (options.clean === false) {
         logger.warning('Not cleaning up stacks since "--no-clean" was used');
       }
 
-      if (Boolean(argv.verbose)) {
+      if (Boolean(options.verbose)) {
         printMetrics(metrics);
       }
 
@@ -134,7 +175,7 @@ export async function main(args: string[]) {
   }
   if (failedSnapshots.length > 0) {
     let message = '';
-    if (!runUpdateOnFailed) {
+    if (!options.runUpdateOnFailed) {
       message = 'To re-run failed tests run: yarn integ-runner --update-on-failed';
     }
     if (!testsSucceeded) {

diff --git a/packages/@aws-cdk/integ-runner/lib/runner/integration-tests.ts b/packages/@aws-cdk/integ-runner/lib/runner/integration-tests.ts
@@ -175,32 +175,26 @@ export interface IntegrationTestsDiscoveryOptions {
 }
 
 
-/**
- * The list of tests to run can be provided in a file
- * instead of as command line arguments.
- */
-export interface IntegrationTestFileConfig extends IntegrationTestsDiscoveryOptions {
-  /**
-   * List of tests to include (or exclude if `exclude=true`)
-   */
-  readonly tests: string[];
-}
-
 /**
  * Discover integration tests
  */
 export class IntegrationTests {
-  constructor(private readonly directory: string) {
-  }
-
   /**
-   * Takes a file name of a file that contains a list of test
-   * to either run or exclude and returns a list of Integration Tests to run
+   * Return configuration options from a file
    */
-  public async fromFile(fileName: string): Promise<IntegTest[]> {
-    const file: IntegrationTestFileConfig = JSON.parse(fs.readFileSync(fileName, { encoding: 'utf-8' }));
+  public static configFromFile(fileName?: string): Record<string, any> {
+    if (!fileName) {
+      return {};
+    }
 
-    return this.discover(file);
+    try {
+      return JSON.parse(fs.readFileSync(fileName, { encoding: 'utf-8' }));
+    } catch {
+      return {};
+    }
+  }
+
+  constructor(private readonly directory: string) {
   }
 
   /**

diff --git a/packages/@aws-cdk/integ-runner/test/cli.test.ts b/packages/@aws-cdk/integ-runner/test/cli.test.ts
@@ -1,5 +1,18 @@
+import * as fs from 'fs';
+import * as os from 'os';
 import * as path from 'path';
-import { main } from '../lib/cli';
+import { main, parseCliArgs } from '../lib/cli';
+
+let stdoutMock: jest.SpyInstance;
+let stderrMock: jest.SpyInstance;
+beforeEach(() => {
+  stdoutMock = jest.spyOn(process.stdout, 'write').mockImplementation(() => { return true; });
+  stderrMock = jest.spyOn(process.stderr, 'write').mockImplementation(() => { return true; });
+});
+afterEach(() => {
+  stdoutMock.mockRestore();
+  stderrMock.mockRestore();
+});
 
 describe('CLI', () => {
   const currentCwd = process.cwd();
@@ -10,14 +23,6 @@ describe('CLI', () => {
     process.chdir(currentCwd);
   });
 
-  let stdoutMock: jest.SpyInstance;
-  beforeEach(() => {
-    stdoutMock = jest.spyOn(process.stdout, 'write').mockImplementation(() => { return true; });
-  });
-  afterEach(() => {
-    stdoutMock.mockRestore();
-  });
-
   test('find by default pattern', async () => {
     await main(['--list', '--directory=test/test-data']);
 
@@ -41,4 +46,41 @@ describe('CLI', () => {
       ].join('\n'),
     ]]);
   });
+
+  test('list only shows explicitly provided tests', async () => {
+    await main(['xxxxx.integ-test1.js', 'xxxxx.integ-test2.js', '--list', '--directory=test/test-data', '--test-regex="^xxxxx\..*\.js$"']);
+
+    expect(stdoutMock.mock.calls).toEqual([[
+      [
+        'xxxxx.integ-test1.js',
+        'xxxxx.integ-test2.js',
+        '',
+      ].join('\n'),
+    ]]);
+  });
 });
+
+describe('CLI config file', () => {
+  const configFile = 'integ.config.json';
+  const withConfig = (settings: any, fileName = configFile) => {
+    fs.writeFileSync(fileName, JSON.stringify(settings, null, 2), { encoding: 'utf-8' });
+  };
+
+  const currentCwd = process.cwd();
+  beforeEach(() => {
+    process.chdir(os.tmpdir());
+  });
+  afterEach(() => {
+    process.chdir(currentCwd);
+  });
+
+  test('options are read from config file', async () => {
+    // WHEN
+    withConfig({ list: true, app: 'echo' });
+    const options = parseCliArgs();
+
+    // THEN
+    expect(options.list).toBe(true);
+    expect(options.app).toBe('echo');
+  });
+});