fix(integ-runner): cannot pass test name after --language (#28922)

Current situation: yarn integ allows to run a TS integ file CI doesn’t allow it, which caused confusing issues. What we want instead: Both local integ-runner and CI don’t allow it, so that this issue won’t happen again. To do this, we need to update integ-runner to disable running TS locally. `integ-runner` package json should stay the same as it includes other tests in python and go. Had a previous attempt in this PR #28770, but reverted as it caused an issue, see detail in #28891. BREAKING CHANGE: Parsing of the cli input caused arguments passed after the first instance `--language <language>` to be interpreted as a language as well. This prevented passing a test name after providing cli options. To overcome this limitation, `integ-runner` now requires an explicit `--language` option for each language you want to include: `integ-runner --language javascript --language python`. This was already documented that way and always the intended way to use this feature. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
aws · Jan 30, 2024 · f9fbbb4 · f9fbbb4
1 parent e9a867a
commit f9fbbb4
Show file tree

Hide file tree

Showing 38 changed files with 54 additions and 38 deletions.
diff --git a/packages/@aws-cdk-testing/framework-integ/package.json b/packages/@aws-cdk-testing/framework-integ/package.json
@@ -9,7 +9,7 @@
     "lint": "cdk-lint",
     "pkglint": "pkglint -f",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "package": "cdk-package",
     "build+test": "yarn build && yarn test",
     "build+extract": "yarn build",

diff --git a/packages/@aws-cdk/app-staging-synthesizer-alpha/package.json b/packages/@aws-cdk/app-staging-synthesizer-alpha/package.json
@@ -48,7 +48,7 @@
     "watch": "cdk-watch",
     "lint": "cdk-lint",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "pkglint": "pkglint -f",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-amplify-alpha/package.json b/packages/@aws-cdk/aws-amplify-alpha/package.json
@@ -50,7 +50,7 @@
   "homepage": "https://github.com/aws/aws-cdk",
   "scripts": {
     "build": "cdk-build",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "lint": "cdk-lint",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-appconfig-alpha/package.json b/packages/@aws-cdk/aws-appconfig-alpha/package.json
@@ -52,7 +52,7 @@
     "watch": "cdk-watch",
     "lint": "cdk-lint",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "pkglint": "pkglint -f",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-apprunner-alpha/package.json b/packages/@aws-cdk/aws-apprunner-alpha/package.json
@@ -54,7 +54,7 @@
     "watch": "cdk-watch",
     "lint": "cdk-lint",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "pkglint": "pkglint -f",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-cloud9-alpha/package.json b/packages/@aws-cdk/aws-cloud9-alpha/package.json
@@ -52,7 +52,7 @@
     "watch": "cdk-watch",
     "lint": "cdk-lint",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "pkglint": "pkglint -f",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-codestar-alpha/package.json b/packages/@aws-cdk/aws-codestar-alpha/package.json
@@ -50,7 +50,7 @@
   "homepage": "https://github.com/aws/aws-cdk",
   "scripts": {
     "build": "cdk-build",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "lint": "cdk-lint",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-cognito-identitypool-alpha/package.json b/packages/@aws-cdk/aws-cognito-identitypool-alpha/package.json
@@ -51,7 +51,7 @@
     "watch": "cdk-watch",
     "lint": "cdk-lint",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "pkglint": "pkglint -f",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-gamelift-alpha/package.json b/packages/@aws-cdk/aws-gamelift-alpha/package.json
@@ -52,7 +52,7 @@
     "watch": "cdk-watch",
     "lint": "cdk-lint",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "pkglint": "pkglint -f",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-glue-alpha/package.json b/packages/@aws-cdk/aws-glue-alpha/package.json
@@ -52,7 +52,7 @@
     "watch": "cdk-watch",
     "lint": "cdk-lint",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "pkglint": "pkglint -f",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-iot-actions-alpha/package.json b/packages/@aws-cdk/aws-iot-actions-alpha/package.json
@@ -51,7 +51,7 @@
     "watch": "cdk-watch",
     "lint": "cdk-lint",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "pkglint": "pkglint -f",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-iot-alpha/package.json b/packages/@aws-cdk/aws-iot-alpha/package.json
@@ -52,7 +52,7 @@
     "watch": "cdk-watch",
     "lint": "cdk-lint",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "pkglint": "pkglint -f",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-iotevents-actions-alpha/package.json b/packages/@aws-cdk/aws-iotevents-actions-alpha/package.json
@@ -44,7 +44,7 @@
     "watch": "cdk-watch",
     "lint": "cdk-lint",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "pkglint": "pkglint -f",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-iotevents-alpha/package.json b/packages/@aws-cdk/aws-iotevents-alpha/package.json
@@ -50,7 +50,7 @@
   "homepage": "https://github.com/aws/aws-cdk",
   "scripts": {
     "build": "cdk-build",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "lint": "cdk-lint",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-ivs-alpha/package.json b/packages/@aws-cdk/aws-ivs-alpha/package.json
@@ -53,7 +53,7 @@
     "watch": "cdk-watch",
     "lint": "cdk-lint",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "pkglint": "pkglint -f",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-kinesisanalytics-flink-alpha/package.json b/packages/@aws-cdk/aws-kinesisanalytics-flink-alpha/package.json
@@ -51,7 +51,7 @@
     "watch": "cdk-watch",
     "lint": "cdk-lint",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "pkglint": "pkglint -f",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-kinesisfirehose-alpha/package.json b/packages/@aws-cdk/aws-kinesisfirehose-alpha/package.json
@@ -52,7 +52,7 @@
     "watch": "cdk-watch",
     "lint": "cdk-lint",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "pkglint": "pkglint -f",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-kinesisfirehose-destinations-alpha/package.json b/packages/@aws-cdk/aws-kinesisfirehose-destinations-alpha/package.json
@@ -51,7 +51,7 @@
     "watch": "cdk-watch",
     "lint": "cdk-lint",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "pkglint": "pkglint -f",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-lambda-go-alpha/package.json b/packages/@aws-cdk/aws-lambda-go-alpha/package.json
@@ -53,7 +53,7 @@
     "watch": "cdk-watch",
     "lint": "cdk-lint",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "pkglint": "pkglint -f",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-lambda-python-alpha/package.json b/packages/@aws-cdk/aws-lambda-python-alpha/package.json
@@ -52,7 +52,7 @@
     "watch": "cdk-watch",
     "lint": "cdk-lint",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "pkglint": "pkglint -f",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-location-alpha/package.json b/packages/@aws-cdk/aws-location-alpha/package.json
@@ -54,7 +54,7 @@
     "watch": "cdk-watch",
     "lint": "cdk-lint",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "pkglint": "pkglint -f",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-msk-alpha/package.json b/packages/@aws-cdk/aws-msk-alpha/package.json
@@ -50,7 +50,7 @@
   "homepage": "https://github.com/aws/aws-cdk",
   "scripts": {
     "build": "cdk-build",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "lint": "cdk-lint",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-neptune-alpha/package.json b/packages/@aws-cdk/aws-neptune-alpha/package.json
@@ -50,7 +50,7 @@
   "homepage": "https://github.com/aws/aws-cdk",
   "scripts": {
     "build": "cdk-build",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "lint": "cdk-lint",
     "package": "cdk-package",
     "pkglint": "pkglint -f",

diff --git a/packages/@aws-cdk/aws-redshift-alpha/package.json b/packages/@aws-cdk/aws-redshift-alpha/package.json
@@ -52,7 +52,7 @@
     "watch": "cdk-watch",
     "lint": "cdk-lint",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "pkglint": "pkglint -f",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-route53resolver-alpha/package.json b/packages/@aws-cdk/aws-route53resolver-alpha/package.json
@@ -50,7 +50,7 @@
   "homepage": "https://github.com/aws/aws-cdk",
   "scripts": {
     "build": "cdk-build",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "lint": "cdk-lint",
     "package": "cdk-package",
     "pkglint": "pkglint -f",

diff --git a/packages/@aws-cdk/aws-s3objectlambda-alpha/package.json b/packages/@aws-cdk/aws-s3objectlambda-alpha/package.json
@@ -54,7 +54,7 @@
     "watch": "cdk-watch",
     "lint": "cdk-lint",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "pkglint": "pkglint -f",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-sagemaker-alpha/package.json b/packages/@aws-cdk/aws-sagemaker-alpha/package.json
@@ -50,7 +50,7 @@
   "homepage": "https://github.com/aws/aws-cdk",
   "scripts": {
     "build": "cdk-build",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "lint": "cdk-lint",
     "package": "cdk-package",
     "pkglint": "pkglint -f",

diff --git a/packages/@aws-cdk/aws-scheduler-alpha/package.json b/packages/@aws-cdk/aws-scheduler-alpha/package.json
@@ -49,7 +49,7 @@
   "homepage": "https://github.com/aws/aws-cdk",
   "scripts": {
     "build": "cdk-build",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "lint": "cdk-lint",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-scheduler-targets-alpha/package.json b/packages/@aws-cdk/aws-scheduler-targets-alpha/package.json
@@ -49,7 +49,7 @@
   "homepage": "https://github.com/aws/aws-cdk",
   "scripts": {
     "build": "cdk-build",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "lint": "cdk-lint",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/aws-servicecatalogappregistry-alpha/package.json b/packages/@aws-cdk/aws-servicecatalogappregistry-alpha/package.json
@@ -54,7 +54,7 @@
     "watch": "cdk-watch",
     "lint": "cdk-lint",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "pkglint": "pkglint -f",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/cdk-cli-wrapper/package.json b/packages/@aws-cdk/cdk-cli-wrapper/package.json
@@ -10,7 +10,7 @@
     "watch": "cdk-watch",
     "lint": "cdk-lint",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "pkglint": "pkglint -f",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/custom-resource-handlers/package.json b/packages/@aws-cdk/custom-resource-handlers/package.json
@@ -5,7 +5,7 @@
   "version": "0.0.0",
   "scripts": {
     "build": "tsc -b && node scripts/generate.js",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "lint": "cdk-lint",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/example-construct-library/package.json b/packages/@aws-cdk/example-construct-library/package.json
@@ -48,7 +48,7 @@
     "watch": "cdk-watch",
     "lint": "cdk-lint",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "pkglint": "pkglint -f",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/packages/@aws-cdk/integ-runner/lib/cli.ts b/packages/@aws-cdk/integ-runner/lib/cli.ts
@@ -42,6 +42,7 @@ export function parseCliArgs(args: string[] = []) {
       default: ['javascript', 'typescript', 'python', 'go'],
       choices: ['javascript', 'typescript', 'python', 'go'],
       type: 'array',
+      nargs: 1,
       desc: 'Use these presets to run integration tests for the selected languages',
     })
     .option('app', { type: 'string', default: undefined, desc: 'The custom CLI command that will be used to run the test files. You can include {filePath} to specify where in the command the test file path should be inserted. Example: --app="python3.8 {filePath}".' })

diff --git a/packages/@aws-cdk/integ-tests-alpha/package.json b/packages/@aws-cdk/integ-tests-alpha/package.json
@@ -50,7 +50,7 @@
     "awslint": "cdk-awslint",
     "pkglint": "pkglint -f",
     "test": "cdk-test",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "watch": "cdk-watch",
     "build+test": "yarn build && yarn test",
     "build+test+package": "yarn build+test && yarn package",

diff --git a/packages/cdk-assets/package.json b/packages/cdk-assets/package.json
@@ -10,7 +10,7 @@
   },
   "scripts": {
     "build": "cdk-build",
-    "integ": "integ-runner",
+    "integ": "integ-runner --language javascript",
     "lint": "cdk-lint",
     "package": "cdk-package",
     "awslint": "cdk-awslint",

diff --git a/tools/@aws-cdk/pkglint/lib/rules.ts b/tools/@aws-cdk/pkglint/lib/rules.ts
@@ -1231,7 +1231,7 @@ export class MustHaveIntegCommand extends ValidationRule {
   public validate(pkg: PackageJson): void {
     if (!hasIntegTests(pkg)) { return; }
 
-    expectJSON(this.name, pkg, 'scripts.integ', 'integ-runner');
+    expectJSON(this.name, pkg, 'scripts.integ', /integ-runner/, undefined, false, true);
 
     // We can't ACTUALLY require cdk-build-tools/package.json here,
     // because WE don't depend on cdk-build-tools and we don't know if

diff --git a/tools/@aws-cdk/pkglint/lib/util.ts b/tools/@aws-cdk/pkglint/lib/util.ts
@@ -5,17 +5,32 @@ import { PackageJson, PKGLINT_IGNORES } from './packagejson';
 /**
  * Expect a particular JSON key to be a given value
  */
-export function expectJSON(ruleName: string, pkg: PackageJson, jsonPath: string, expected: any, ignore?: RegExp, caseInsensitive: boolean = false) {
+export function expectJSON(
+  ruleName: string,
+  pkg: PackageJson,
+  jsonPath: string,
+  expected: any,
+  ignore?: RegExp,
+  caseInsensitive: boolean = false,
+  regexMatch: boolean = false,
+) {
   const parts = jsonPath.split('.');
   const actual = deepGet(pkg.json, parts);
-  if (applyCaseInsensitive(applyIgnore(actual)) !== applyCaseInsensitive(applyIgnore(expected))) {
+  if (checkEquality()) {
     pkg.report({
       ruleName,
       message: `${jsonPath} should be ${JSON.stringify(expected)}${ignore ? ` (ignoring ${ignore})` : ''}, is ${JSON.stringify(actual)}`,
       fix: () => { deepSet(pkg.json, parts, expected); },
     });
   }
 
+  function checkEquality(): boolean {
+    if (regexMatch) {
+      return !expected.test(applyCaseInsensitive(applyIgnore(actual)));
+    }
+    return applyCaseInsensitive(applyIgnore(actual)) !== applyCaseInsensitive(applyIgnore(expected));
+  }
+
   function applyIgnore(val: any): string {
     if (!ignore || val == null) { return JSON.stringify(val); }
     const str = JSON.stringify(val);