Skip to content

Commit 0f2dd6a

Browse files
dothebartdothebart
authored
QA-722: Run job as forked shell instead of server task (#22137)
* Run job as forked shell instead of server task * move functionality to spawn stress shells up into the client tools; use it in tests * use stress tool in tests instead of tasks * use stress tool in tests instead of tasks * lint * don't use string representations * fix sequence - we only want to end the transaction in the distant future.
1 parent 1a75040 commit 0f2dd6a

File tree

7 files changed

+286
-225
lines changed

7 files changed

+286
-225
lines changed

‎js/client/modules/@arangodb/testsuites/hotbackup.js‎

Lines changed: 98 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/* jshint strict: false, sub: true */
2-
/* global print, arango, db */
2+
/* global print, arango, db, idx */
33
'use strict';
44

55
// //////////////////////////////////////////////////////////////////////////////
@@ -253,24 +253,25 @@ function hotBackup_views (options) {
253253
let which = "hot_backup_views";
254254
return hotBackup_load_backend(options, which, {
255255
// Insert documents into test_collection:
256-
noiseScript: `
257-
let i=0;
258-
while(true) {
259-
console.log('Noise starts');
260-
try {
261-
db._useDatabase('test_view');
262-
for (; i < 10000; i++) {
263-
db.test_collection.insert({"number": i, "field1": "stone"});
264-
}
265-
print('done');
266-
break;
267-
}
268-
catch (ex) {
269-
console.log(ex);
270-
break;
271-
}
272-
}
273-
`,
256+
noiseScript: function () {
257+
let i=0;
258+
while(true) {
259+
console.log('Noise starts');
260+
try {
261+
db._useDatabase('test_view');
262+
for (; i < 10000; i++) {
263+
db.test_collection.insert({"number": i, "field1": "stone"});
264+
}
265+
print('done');
266+
break;
267+
}
268+
catch (ex) {
269+
console.log(ex);
270+
break;
271+
}
272+
return 0;
273+
}
274+
},
274275
noiseVolume: 1,
275276
// :/ making sure that something has happened from the
276277
// inserter thread
@@ -376,24 +377,25 @@ function hotBackup_aql (options) {
376377

377378
let which = "hot_backup_aql";
378379
return hotBackup_load_backend(options, which, {
379-
noiseScript: `
380-
const errors = require('internal').errors;
381-
while (true) {
382-
try {
383-
db._query("FOR i IN 1..1000 INSERT {thrd: @idx, i} INTO test_collection",
384-
bind_vars={"idx": \`\${idx}\`}).toArray();
385-
} catch (ex) {
386-
if (ex.errorNum === errors.ERROR_SHUTTING_DOWN.code ||
387-
ex.errorNum === errors.ERROR_SIMPLE_CLIENT_COULD_NOT_CONNECT.code) {
388-
break;
389-
}
390-
else {
391-
console.error(ex);
392-
throw ex;
393-
}
394-
}
395-
}
396-
`,
380+
noiseScript: function () {
381+
const errors = require('internal').errors;
382+
while (true) {
383+
try {
384+
db._query("FOR i IN 1..1000 INSERT {thrd: @idx, i} INTO test_collection",
385+
{"idx": `${idx}`}).toArray();
386+
} catch (ex) {
387+
if (ex.errorNum === errors.ERROR_SHUTTING_DOWN.code ||
388+
ex.errorNum === errors.ERROR_SIMPLE_CLIENT_COULD_NOT_CONNECT.code) {
389+
break;
390+
}
391+
else {
392+
console.error(ex);
393+
throw ex;
394+
}
395+
}
396+
}
397+
return 0;
398+
},
397399
noiseVolume: 20,
398400
noiseDuration: 15,
399401
preRestoreFn: function() {
@@ -433,41 +435,42 @@ function hotBackup_smart_graphs (options) {
433435

434436
let which = "hot_backup_load";
435437
return hotBackup_load_backend(options, which, {
436-
noiseScript: `
437-
db._useDatabase('test');
438-
const errors = require('internal').errors;
439-
function shuffleArray(array) {
440-
for (let i = array.length - 1; i > 0; i--) {
441-
const j = Math.floor(Math.random() * (i + 1));
442-
[array[i], array[j]] = [array[j], array[i]];
443-
}
444-
}
445-
let ids1 = [...Array(20).keys()];
446-
let ids2 = [...Array(20).keys()];
447-
while(true) {
448-
shuffleArray(ids1);
449-
shuffleArray(ids2);
450-
let edge_docs = [];
451-
for (let i = 0; i < 20; i++) {
452-
edge_docs.push({
453-
"_from": \`foo/\${ids1[i]}:v\${ids1[i]}\`,
454-
"_to": \`foo/\${ids2[i]}:v\${ids2[i]}\`,
455-
"idx": \`\${i}\`,
456-
"f": ids1[i],
457-
"t": ids2[i]
458-
});
459-
}
460-
try {
461-
db.is_foo.save(edge_docs);
462-
} catch (ex) {
463-
if (ex.errorNum === errors.ERROR_SHUTTING_DOWN.code ||
464-
ex.errorNum === errors.ERROR_SIMPLE_CLIENT_COULD_NOT_CONNECT.code) {
465-
break;
466-
}
467-
throw ex;
468-
}
469-
}
470-
`,
438+
noiseScript: function() {
439+
db._useDatabase('test');
440+
const errors = require('internal').errors;
441+
function shuffleArray(array) {
442+
for (let i = array.length - 1; i > 0; i--) {
443+
const j = Math.floor(Math.random() * (i + 1));
444+
[array[i], array[j]] = [array[j], array[i]];
445+
}
446+
}
447+
let ids1 = [...Array(20).keys()];
448+
let ids2 = [...Array(20).keys()];
449+
while(true) {
450+
shuffleArray(ids1);
451+
shuffleArray(ids2);
452+
let edge_docs = [];
453+
for (let i = 0; i < 20; i++) {
454+
edge_docs.push({
455+
"_from": `foo/${ids1[i]}:v${ids1[i]}`,
456+
"_to": `foo/${ids2[i]}:v${ids2[i]}`,
457+
"idx": `${i}`,
458+
"f": ids1[i],
459+
"t": ids2[i]
460+
});
461+
}
462+
try {
463+
db.is_foo.save(edge_docs);
464+
} catch (ex) {
465+
if (ex.errorNum === errors.ERROR_SHUTTING_DOWN.code ||
466+
ex.errorNum === errors.ERROR_SIMPLE_CLIENT_COULD_NOT_CONNECT.code) {
467+
break;
468+
}
469+
throw ex;
470+
}
471+
}
472+
return 0;
473+
},
471474
noiseVolume: 10,
472475
noiseDuration: 5,
473476
preRestoreFn: function() {
@@ -552,29 +555,29 @@ function hotBackup_el_cheapo (options) {
552555

553556
let which = "hot_backup_el_cheapo";
554557
return hotBackup_load_backend(options, which, {
555-
noiseScript: `
556-
const errors = require('internal').errors;
557-
let collections = ${JSON.stringify(collections)};
558-
let i = 0;
559-
while (true) {
560-
try {
561-
let txn = db._createTransaction({collections: { write: collections}});
562-
collections.forEach(col => {
563-
let trx_col = txn.collection(col);
564-
trx_col.insert({"trd": idx, "i": i});
565-
});
566-
txn.commit();
567-
i += 1;
568-
} catch (ex) {
569-
if (ex.errorNum === errors.ERROR_SHUTTING_DOWN.code ||
570-
ex.errorNum === errors.ERROR_SIMPLE_CLIENT_COULD_NOT_CONNECT.code) {
571-
// todo: write i
572-
break;
573-
}
574-
}
575-
}
576-
577-
`,
558+
noiseScript: function () {
559+
const errors = require('internal').errors;
560+
let collections = args.collections;
561+
let i = 0;
562+
while (true) {
563+
try {
564+
let txn = db._createTransaction({collections: { write: collections}});
565+
collections.forEach(col => {
566+
let trx_col = txn.collection(col);
567+
trx_col.insert({"trd": idx, "i": i});
568+
});
569+
txn.commit();
570+
i += 1;
571+
} catch (ex) {
572+
if (ex.errorNum === errors.ERROR_SHUTTING_DOWN.code ||
573+
ex.errorNum === errors.ERROR_SIMPLE_CLIENT_COULD_NOT_CONNECT.code) {
574+
// todo: write i
575+
break;
576+
}
577+
}
578+
}
579+
return 0;
580+
},
578581
noiseVolume: 10,
579582
noiseDuration: 1,
580583
args: args,

‎js/client/modules/@arangodb/testutils/client-tools.js‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
// //////////////////////////////////////////////////////////////////////////////
2828

2929
const internal = require('internal');
30+
const sleep = internal.sleep;
3031
const _ = require('lodash');
3132
const tu = require('@arangodb/testutils/test-utils');
3233
const pu = require('@arangodb/testutils/process-utils');
@@ -397,6 +398,42 @@ function launchPlainSnippetInBG (snippet, key) {
397398
return launchInShellBG(file);
398399
}
399400

401+
402+
function spawnStressArangoshInBG (arangoshList, snippet, key, volume) {
403+
let IM = global.instanceManager;
404+
let globalFn = fs.getTempFile();
405+
fs.write(globalFn, "x");
406+
let testFns = [];
407+
for (let i=0; i < volume; i++) {
408+
let testFn = fs.getTempFile() + `_${i}`;
409+
fs.write(testFn, "x");
410+
let mySnippet = `const fs = require('fs');
411+
fs.remove('${testFn}');
412+
let volume = ${volume};
413+
let idx = ${i};
414+
let endpoint = '${IM.endpoint}';
415+
let passvoid = '${IM.options.password}';
416+
while (fs.exists('${globalFn}')) {
417+
require('internal').sleep(0.1);
418+
}
419+
let testfunc = ${String(snippet)};
420+
testfunc();
421+
`;
422+
arangoshList.push(
423+
launchPlainSnippetInBG(mySnippet, key + `_${i}`)
424+
);
425+
}
426+
// wait for the spawned clients to reach the entry gate:
427+
testFns.forEach(testFn => {
428+
while (fs.exists(testFn)) {
429+
sleep(0.1);
430+
}
431+
});
432+
// GO!
433+
fs.remove(globalFn);
434+
return true;
435+
}
436+
400437
function launchSnippetInBG (options, snippet, key, cn, single=false) {
401438
let file = fs.getTempFile() + "-" + key;
402439
if (single) {
@@ -824,6 +861,7 @@ exports.run = {
824861
launchInShellBG: launchInShellBG,
825862
launchPlainSnippetInBG: launchPlainSnippetInBG,
826863
launchSnippetInBG: launchSnippetInBG,
864+
spawnStressArangoshInBG: spawnStressArangoshInBG,
827865
joinBGShells: joinBGShells,
828866
joinForceBGShells: joinForceBGShells,
829867
joinFinishedBGShells: joinFinishedBGShells,

‎js/client/modules/@arangodb/testutils/dump.js‎

Lines changed: 1 addition & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -766,35 +766,7 @@ class DumpRestoreHelper extends trs.runLocalInArangoshRunner {
766766

767767
spawnStressArangosh(snippet, key, volume) {
768768
global.instanceManager = this.instanceManager;
769-
let globalFn = fs.getTempFile();
770-
fs.write(globalFn, "x");
771-
let testFns = [];
772-
for (let i=0; i < volume; i++) {
773-
let testFn = fs.getTempFile() + `_${i}`;
774-
fs.write(testFn, "x");
775-
let mySnippet = `const fs = require('fs');
776-
fs.remove('${testFn}');
777-
let volume = ${volume};
778-
let idx = ${i};
779-
let endpoint = '${this.instanceManager.endpoint}';
780-
let passvoid = '${this.instanceManager.options.password}';
781-
while (fs.exists('${globalFn}')) {
782-
require('internal').sleep(0.1);
783-
}
784-
${snippet}`;
785-
this.clientInstances.push(
786-
ct.run.launchPlainSnippetInBG(mySnippet, key + `_${i}`)
787-
);
788-
}
789-
// wait for the spawned clients to reach the entry gate:
790-
testFns.forEach(testFn => {
791-
while (fs.exists(testFn)) {
792-
sleep(0.1);
793-
}
794-
});
795-
// GO!
796-
fs.remove(globalFn);
797-
return true;
769+
return ct.run.spawnStressArangoshInBG(this.clientInstances, snippet, key, volume);
798770
}
799771
stopStressArangosh() {
800772
try {

‎tests/js/client/aql/aql-view-arangosearch-ddl-combined.js‎

Lines changed: 56 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2289,7 +2289,7 @@ function IResearchFeatureDDLTestSuite2() {
22892289
////////////////////////////////////////////////////////////////////////////
22902290
/// @brief test creating link with 'inBackground' set to true
22912291
////////////////////////////////////////////////////////////////////////////
2292-
testCreateLinkInBackgroundMode: function () {
2292+
testCreateLinkInBackgroundModeSJS: function () {
22932293
if (SYS_IS_V8_BUILD) {
22942294
const colName = 'TestCollection';
22952295
const viewName = 'TestView';
@@ -2350,7 +2350,61 @@ function IResearchFeatureDDLTestSuite2() {
23502350
assertTrue(undefined === propertiesReturned.links[colName].inBackground);
23512351
}
23522352
},
2353-
2353+
testCreateLinkInBackgroundMode: function () {
2354+
const IM = global.instanceManager;
2355+
const ct = require('@arangodb/testutils/client-tools');
2356+
const colName = 'TestCollection';
2357+
const viewName = 'TestView';
2358+
const initialCount = 500;
2359+
const inTransCount = 1000;
2360+
const markerFileName = fs.join(fs.getTempPath(), "backgroundLinkMarker");
2361+
try { fs.remove(markerFileName); } catch (e) { }
2362+
db._useDatabase(dbName);
2363+
let col = db._create(colName);
2364+
col.ensureIndex(indexMetaGlobal);
2365+
let v = db._createView(viewName, 'arangosearch', {});
2366+
// some initial documents
2367+
for (let i = 0; i < initialCount; ++i) {
2368+
col.insert({ myField: 'test' + i, name_1: i.toString() });
2369+
}
2370+
let bgJob = ct.run.launchPlainSnippetInBG(`
2371+
var fs = require('fs');
2372+
var db = require('internal').db;
2373+
var db = require('internal').db;
2374+
let trx = db._createTransaction({collections: { write: ["${colName}"]}});
2375+
var c = trx.collection("${colName}");
2376+
fs.write("${markerFileName}", "TEST");
2377+
for (var i = 0; i < ${inTransCount}; ++i) {
2378+
c.insert({ myField: 'background' + i });
2379+
}
2380+
require('internal').sleep(20);
2381+
trx.commit();
2382+
`, 0);
2383+
while (!fs.exists(markerFileName)) {
2384+
require('internal').sleep(1); // give transaction some time to run
2385+
}
2386+
v.properties({ links: { [colName]: { includeAllFields: true, inBackground: true } } });
2387+
// check that all documents are visible
2388+
let docs = db._query("FOR doc IN " + viewName + " OPTIONS { waitForSync: true } RETURN doc").toArray();
2389+
assertEqual(initialCount + inTransCount, docs.length);
2390+
2391+
// inBackground should not be returned as part of index definition
2392+
let indexes = col.indexes(false, true);
2393+
assertEqual(3, indexes.length);
2394+
var index = indexes[1];
2395+
assertEqual("inverted", index.type);
2396+
assertTrue(undefined === index.inBackground);
2397+
var link = indexes[2];
2398+
assertEqual("arangosearch", link.type);
2399+
assertTrue(undefined === link.inBackground);
2400+
2401+
// inBackground should not be returned as part of link definition
2402+
let propertiesReturned = v.properties();
2403+
assertTrue(undefined === propertiesReturned.links[colName].inBackground);
2404+
if (!ct.run.joinForceBGShells(IM.options, [bgJob])) {
2405+
throw new Error(`failed to collect ${JSON.stringify(bgJob)}`);
2406+
}
2407+
},
23542408
testCachedColumns: function () {
23552409
const colName = 'TestCollectionCache';
23562410
const viewName = 'TestViewCache';

0 commit comments

Comments
 (0)