Skip to content

Commit

Permalink
fix(runner): Make exit code configurable when tests are failing (#3116)
Browse files Browse the repository at this point in the history
Fixes #1300
  • Loading branch information
andreaspsson authored and johnjbarton committed Aug 30, 2018
1 parent eeadcf2 commit 74da748
Show file tree
Hide file tree
Showing 4 changed files with 112 additions and 10 deletions.
21 changes: 19 additions & 2 deletions lib/browser_collection.js
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,24 @@ class BrowserCollection {
return this.browsers.map((browser) => browser.serialize())
}

getResults () {
calculateExitCode (results, singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite) {
if (results.disconnected || singleRunBrowserNotCaptured) {
return 1
}
if (results.success + results.failed === 0 && !failOnEmptyTestSuite) {
return 0
}
if (results.error) {
return 1
}
if (failOnFailingTestSuite === false) {
// Tests executed without infrastructure error, exit with 0 independent of test status.
return 0
}
return results.failed ? 1 : 0
}

getResults (singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite) {
const results = this.browsers.reduce((previous, current) => {
previous.success += current.lastResult.success
previous.failed += current.lastResult.failed
Expand All @@ -56,7 +73,7 @@ class BrowserCollection {
}, {success: 0, failed: 0, error: false, disconnected: false, exitCode: 0})

// compute exit status code
results.exitCode = results.failed || results.error || results.disconnected ? 1 : 0
results.exitCode = this.calculateExitCode(results, singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite)

return results
}
Expand Down
6 changes: 6 additions & 0 deletions lib/cli.js
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ function processArgs (argv, options, fs, path) {
options.failOnEmptyTestSuite = options.failOnEmptyTestSuite === 'true'
}

if (helper.isString(options.failOnFailingTestSuite)) {
options.failOnFailingTestSuite = options.failOnFailingTestSuite === 'true'
}

if (helper.isString(options.formatError)) {
let required
try {
Expand Down Expand Up @@ -196,6 +200,8 @@ function describeStart () {
.describe('report-slower-than', '<integer> Report tests that are slower than given time [ms].')
.describe('fail-on-empty-test-suite', 'Fail on empty test suite.')
.describe('no-fail-on-empty-test-suite', 'Do not fail on empty test suite.')
.describe('fail-on-failing-test-suite', 'Fail on failing test suite.')
.describe('no-fail-on-failing-test-suite', 'Do not fail on failing test suite.')
.describe('help', 'Print usage and options.')
}

Expand Down
9 changes: 1 addition & 8 deletions lib/server.js
Original file line number Diff line number Diff line change
Expand Up @@ -247,14 +247,7 @@ class Server extends KarmaEventEmitter {

const emitRunCompleteIfAllBrowsersDone = () => {
if (Object.keys(singleRunDoneBrowsers).every((key) => singleRunDoneBrowsers[key])) {
const results = singleRunBrowsers.getResults()
if (singleRunBrowserNotCaptured) {
results.exitCode = 1
} else if (results.success + results.failed === 0 && !config.failOnEmptyTestSuite) {
results.exitCode = 0
this.log.warn('Test suite was empty.')
}
this.emit('run_complete', singleRunBrowsers, results)
this.emit('run_complete', singleRunBrowsers, singleRunBrowsers.getResults(singleRunBrowserNotCaptured, config.failOnEmptyTestSuite, config.failOnFailingTestSuite))
}
}

Expand Down
86 changes: 86 additions & 0 deletions test/unit/browser_collection.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -258,4 +258,90 @@ describe('BrowserCollection', () => {
})
})
})

// ============================================================================
// server.calculateExitCode
// ============================================================================
describe('calculateExitCode', () => {
const EXIT_CODE_ERROR = 1
const EXIT_CODE_SUCCESS = 0

describe('no tests', () => {
const results = {
success: 0,
failed: 0,
error: true
}
it('shall pass if failOnEmptyTestSuite not is set', () => {
const res = collection.calculateExitCode(results)
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
it('shall pass if failOnEmptyTestSuite is false', () => {
const res = collection.calculateExitCode(results, false, false)
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
it('shall fail if failOnEmptyTestSuite is true', () => {
const res = collection.calculateExitCode(results, false, true)
expect(res).to.be.equal(EXIT_CODE_ERROR)
})
it('shall fail if failOnFailingTestSuite is set', () => {
const res = collection.calculateExitCode(results, false, true, true)
expect(res).to.be.equal(EXIT_CODE_ERROR)
})
})
describe('all test passed, no errors', () => {
const results = {
success: 10,
failed: 0,
error: false
}
it('shall fail if singleRunBrowserNotCaptured is true', () => {
const res = collection.calculateExitCode(results, true)
expect(res).to.be.equal(EXIT_CODE_ERROR)
})
it('shall pass if failOnEmptyTestSuite not is set', () => {
const res = collection.calculateExitCode(results, false)
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
it('shall pass if failOnEmptyTestSuite not is set', () => {
const res = collection.calculateExitCode(results, false, false)
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
it('shall pass if failOnFailingTestSuite is true', () => {
const res = collection.calculateExitCode(results, false, true, true)
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
it('shall pass if failOnFailingTestSuite is false', () => {
const res = collection.calculateExitCode(results, false, true, false)
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
})
describe('all test passed, with error', () => {
const results = {
success: 10,
failed: 5,
error: false
}
it('shall fail if singleRunBrowserNotCaptured is true', () => {
const res = collection.calculateExitCode(results, true)
expect(res).to.be.equal(EXIT_CODE_ERROR)
})
it('shall fail if failOnEmptyTestSuite not is set', () => {
const res = collection.calculateExitCode(results, false)
expect(res).to.be.equal(EXIT_CODE_ERROR)
})
it('shall fail if failOnEmptyTestSuite not is set', () => {
const res = collection.calculateExitCode(results, false, false)
expect(res).to.be.equal(EXIT_CODE_ERROR)
})
it('shall fail if failOnFailingTestSuite is true', () => {
const res = collection.calculateExitCode(results, false, true, true)
expect(res).to.be.equal(EXIT_CODE_ERROR)
})
it('shall pass if failOnFailingTestSuite is false', () => {
const res = collection.calculateExitCode(results, false, true, false)
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
})
})
})

0 comments on commit 74da748

Please sign in to comment.