Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(integ-runner): support --language presets for JavaScript, TypeScript, Python and Go #22058

Merged
merged 5 commits into from
Jan 5, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/@aws-cdk/integ-runner/.eslintrc.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
const baseConfig = require('@aws-cdk/cdk-build-tools/config/eslintrc');
baseConfig.parserOptions.project = __dirname + '/tsconfig.json';
baseConfig.ignorePatterns = [...baseConfig.ignorePatterns, "test/language-tests/**/integ.*.ts"];
module.exports = baseConfig;
25 changes: 22 additions & 3 deletions packages/@aws-cdk/integ-runner/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,16 +70,35 @@ to be a self contained CDK app. The runner will execute the following for each f
If this is set to `true` then the [update workflow](#update-workflow) will be disabled
- `--app`
The custom CLI command that will be used to run the test files. You can include {filePath} to specify where in the command the test file path should be inserted. Example: --app="python3.8 {filePath}".

Use together with `--test-regex` to fully customize how tests are run, or use with a single `--language` preset to change the command used for this language.
- `--test-regex`
Detect integration test files matching this JavaScript regex pattern. If used multiple times, all files matching any one of the patterns are detected.


Use together with `--app` to fully customize how tests are run, or use with a single `--language` preset to change which files are detected for this language.
- `--language`
The language presets to use. You can discover and run tests written in multiple languages by passing this flag multiple times (`--language typescript --language python`). Defaults to all supported languages. Currently supported language presets are:
- `javascript`:
- File RegExp: `^integ\..*\.js$`
- App run command: `node {filePath}`
- `typescript`:\
Note that for TypeScript files compiled to JavaScript, the JS tests will take precedence and the TS ones won't be evaluated.
- File RegExp: `^integ\..*(?<!\.d)\.ts$`
- App run command: `node -r ts-node/register {filePath}`
- `python`:
- File RegExp: `^integ_.*\.py$`
- App run command: `python {filePath}`
- `go`:
- File RegExp: `^integ_.*\.go$`
- App run command: `go run {filePath}`

Example:

```bash
integ-runner --update-on-failed --parallel-regions us-east-1 --parallel-regions us-east-2 --parallel-regions us-west-2 --directory ./
integ-runner --update-on-failed --parallel-regions us-east-1 --parallel-regions us-east-2 --parallel-regions us-west-2 --directory ./ --language python
```

This will search for integration tests recursively from the current directory and then execute them in parallel across `us-east-1`, `us-east-2`, & `us-west-2`.
This will search for python integration tests recursively from the current directory and then execute them in parallel across `us-east-1`, `us-east-2`, & `us-west-2`.

If you are providing a list of tests to execute, either as CLI arguments or from a file, the name of the test needs to be relative to the `directory`.
For example, if there is a test `aws-iam/test/integ.policy.js` and the current working directory is `aws-iam` you would provide `integ.policy.js`
Expand Down
35 changes: 28 additions & 7 deletions packages/@aws-cdk/integ-runner/lib/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ export function parseCliArgs(args: string[] = []) {
.usage('Usage: integ-runner [TEST...]')
.option('config', {
config: true,
configParser: IntegrationTests.configFromFile,
configParser: configFromFile,
default: 'integ.config.json',
desc: 'Load options from a JSON config file. Options provided as CLI arguments take precedent.',
})
Expand All @@ -35,6 +35,13 @@ export function parseCliArgs(args: string[] = []) {
.options('from-file', { type: 'string', desc: 'Read TEST names from a file (one TEST per line)' })
.option('inspect-failures', { type: 'boolean', desc: 'Keep the integ test cloud assembly if a failure occurs for inspection', default: false })
.option('disable-update-workflow', { type: 'boolean', default: false, desc: 'If this is "true" then the stack update workflow will be disabled' })
.option('language', {
alias: 'l',
default: ['javascript', 'typescript', 'python', 'go'],
choices: ['javascript', 'typescript', 'python', 'go'],
type: 'array',
desc: 'Use these presets to run integration tests for the selected languages',
})
.option('app', { type: 'string', default: undefined, desc: 'The custom CLI command that will be used to run the test files. You can include {filePath} to specify where in the command the test file path should be inserted. Example: --app="python3.8 {filePath}".' })
.option('test-regex', { type: 'array', desc: 'Detect integration test files matching this JavaScript regex pattern. If used multiple times, all files matching any one of the patterns are detected.', default: [] })
.strict()
Expand Down Expand Up @@ -80,19 +87,15 @@ export function parseCliArgs(args: string[] = []) {
force: argv.force as boolean,
dryRun: argv['dry-run'] as boolean,
disableUpdateWorkflow: argv['disable-update-workflow'] as boolean,
language: arrayFromYargs(argv.language),
};
}


export async function main(args: string[]) {
const options = parseCliArgs(args);

const testsFromArgs = await new IntegrationTests(path.resolve(options.directory)).fromCliArgs({
app: options.app,
testRegex: options.testRegex,
tests: options.tests,
exclude: options.exclude,
});
const testsFromArgs = await new IntegrationTests(path.resolve(options.directory)).fromCliOptions(options);

// List only prints the discoverd tests
if (options.list) {
Expand Down Expand Up @@ -227,3 +230,21 @@ export function cli(args: string[] = process.argv.slice(2)) {
process.exitCode = 1;
});
}

/**
* Read CLI options from a config file if provided.
*
* @param fileName
* @returns parsed CLI config options
*/
function configFromFile(fileName?: string): Record<string, any> {
if (!fileName) {
return {};
}

try {
return JSON.parse(fs.readFileSync(fileName, { encoding: 'utf-8' }));
} catch {
return {};
}
}
161 changes: 119 additions & 42 deletions packages/@aws-cdk/integ-runner/lib/runner/integration-tests.ts
Original file line number Diff line number Diff line change
Expand Up @@ -159,42 +159,112 @@ export interface IntegrationTestsDiscoveryOptions {
readonly tests?: string[];

/**
* Detect integration test files matching any of these JavaScript regex patterns.
*
* @default
*/
readonly testRegex?: string[];

/**
* The CLI command used to run this test.
* If it contains {filePath}, the test file names will be substituted at that place in the command for each run.
* A map of of the app commands to run integration tests with,
* and the regex patterns matching the integration test files each app command.
*
* @default - test run command will be `node {filePath}`
* If the app command contains {filePath}, the test file names will be substituted at that place in the command for each run.
*/
readonly app?: string;
readonly testCases: {
[app: string]: string[]
}
}

/**
* Returns the name of the Python executable for the current OS
*/
function pythonExecutable() {
let python = 'python3';
if (process.platform === 'win32') {
python = 'python';
}
return python;
}

/**
* Discover integration tests
*/
export class IntegrationTests {
constructor(private readonly directory: string) {}

/**
* Return configuration options from a file
* Get integration tests discovery options from CLI options
*/
public static configFromFile(fileName?: string): Record<string, any> {
if (!fileName) {
return {};
public async fromCliOptions(options: {
app?: string;
exclude?: boolean,
language?: string[],
testRegex?: string[],
tests?: string[],
}): Promise<IntegTest[]> {
const baseOptions = {
tests: options.tests,
exclude: options.exclude,
};

// Explicitly set both, app and test-regex
if (options.app && options.testRegex) {
return this.discover({
testCases: {
[options.app]: options.testRegex,
},
...baseOptions,
});
}

// Use the selected presets
if (!options.app && !options.testRegex) {
// Only case with multiple languages, i.e. the only time we need to check the special case
const ignoreUncompiledTypeScript = options.language?.includes('javascript') && options.language?.includes('typescript');

return this.discover({
testCases: this.getLanguagePresets(options.language),
...baseOptions,
}, ignoreUncompiledTypeScript);
}

try {
return JSON.parse(fs.readFileSync(fileName, { encoding: 'utf-8' }));
} catch {
return {};
// Only one of app or test-regex is set, with a single preset selected
// => override either app or test-regex
if (options.language?.length === 1) {
const [presetApp, presetTestRegex] = this.getLanguagePreset(options.language[0]);
return this.discover({
testCases: {
[options.app ?? presetApp]: options.testRegex ?? presetTestRegex,
},
...baseOptions,
});
}

// Only one of app or test-regex is set, with multiple presets
// => impossible to resolve
const option = options.app ? '--app' : '--test-regex';
throw new Error(`Only a single "--language" can be used with "${option}". Alternatively provide both "--app" and "--test-regex" to fully customize the configuration.`);
}

/**
* Get the default configuration for a language
*/
private getLanguagePreset(language: string) {
const languagePresets: {
[language: string]: [string, string[]]
} = {
javascript: ['node {filePath}', ['^integ\\..*\\.js$']],
typescript: ['node -r ts-node/register {filePath}', ['^integ\\.(?!.*\\.d\\.ts$).*\\.ts$']],
python: [`${pythonExecutable()} {filePath}`, ['^integ_.*\\.py$']],
go: ['go run {filePath}', ['^integ_.*\\.go$']],
};

return languagePresets[language];
}

constructor(private readonly directory: string) {
/**
* Get the config for all selected languages
*/
private getLanguagePresets(languages: string[] = []) {
return Object.fromEntries(
languages
.map(language => this.getLanguagePreset(language))
.filter(Boolean),
);
}

/**
Expand All @@ -209,7 +279,6 @@ export class IntegrationTests {
return discoveredTests;
}


const allTests = discoveredTests.filter(t => {
const matches = requestedTests.some(pattern => t.matches(pattern));
return matches !== !!exclude; // Looks weird but is equal to (matches && !exclude) || (!matches && exclude)
Expand Down Expand Up @@ -237,33 +306,41 @@ export class IntegrationTests {
* @param tests Tests to include or exclude, undefined means include all tests.
* @param exclude Whether the 'tests' list is inclusive or exclusive (inclusive by default).
*/
public async fromCliArgs(options: IntegrationTestsDiscoveryOptions = {}): Promise<IntegTest[]> {
return this.discover(options);
}

private async discover(options: IntegrationTestsDiscoveryOptions): Promise<IntegTest[]> {
const patterns = options.testRegex ?? ['^integ\\..*\\.js$'];

private async discover(options: IntegrationTestsDiscoveryOptions, ignoreUncompiledTypeScript: boolean = false): Promise<IntegTest[]> {
const files = await this.readTree();
const integs = files.filter(fileName => patterns.some((p) => {
const regex = new RegExp(p);
return regex.test(fileName) || regex.test(path.basename(fileName));
}));

return this.request(integs, options);
}

private request(files: string[], options: IntegrationTestsDiscoveryOptions): IntegTest[] {
const discoveredTests = files.map(fileName => new IntegTest({
discoveryRoot: this.directory,
fileName,
appCommand: options.app,
}));

const testCases = Object.entries(options.testCases)
.flatMap(([appCommand, patterns]) => files
.filter(fileName => patterns.some((pattern) => {
const regex = new RegExp(pattern);
return regex.test(fileName) || regex.test(path.basename(fileName));
}))
.map(fileName => new IntegTest({
discoveryRoot: this.directory,
fileName,
appCommand,
})),
);

const discoveredTests = ignoreUncompiledTypeScript ? this.filterUncompiledTypeScript(testCases) : testCases;

return this.filterTests(discoveredTests, options.tests, options.exclude);
}

private filterUncompiledTypeScript(testCases: IntegTest[]): IntegTest[] {
const jsTestCases = testCases.filter(t => t.fileName.endsWith('.js'));

return testCases
// Remove all TypeScript test cases (ending in .ts)
// for which a compiled version is present (same name, ending in .js)
.filter((tsCandidate) => {
if (!tsCandidate.fileName.endsWith('.ts')) {
return true;
}
return jsTestCases.findIndex(jsTest => jsTest.testName === tsCandidate.testName) === -1;
});
}

private async readTree(): Promise<string[]> {
const ret = new Array<string>();

Expand Down
11 changes: 8 additions & 3 deletions packages/@aws-cdk/integ-runner/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"awslint": "cdk-awslint",
"pkglint": "pkglint -f",
"test": "cdk-test",
"integ": "integ-runner",
"watch": "cdk-watch",
"build+test": "yarn build && yarn test",
"build+test+package": "yarn build+test && yarn package",
Expand Down Expand Up @@ -52,15 +53,19 @@
"license": "Apache-2.0",
"devDependencies": {
"@aws-cdk/cdk-build-tools": "0.0.0",
"@types/mock-fs": "^4.13.1",
"mock-fs": "^4.14.0",
"@aws-cdk/core": "0.0.0",
"@aws-cdk/integ-tests": "0.0.0",
"@aws-cdk/pkglint": "0.0.0",
"@types/fs-extra": "^8.1.2",
"@types/jest": "^27.5.2",
"@types/mock-fs": "^4.13.1",
"@types/node": "^14.18.34",
"@types/workerpool": "^6.1.0",
"@types/yargs": "^15.0.14",
"jest": "^27.5.1"
"constructs": "^10.0.0",
"mock-fs": "^4.14.0",
"jest": "^27.5.1",
"ts-node": "^10.9.1"
},
"dependencies": {
"@aws-cdk/cloud-assembly-schema": "0.0.0",
Expand Down
Loading