Skip to content

Commit cb57552

Browse files
rclarkrclark
committed
moving files around
1 parent 162b0f4 commit cb57552

15 files changed

Lines changed: 234 additions & 234 deletions

‎bin/backup-table.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/usr/bin/env node
22

3-
var backup = require('../backup');
3+
var backup = require('../lib/backup');
44
var fastlog = require('fastlog');
55
var args = require('minimist')(process.argv.slice(2));
66
var crypto = require('crypto');

‎bin/diff-tables.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
#!/usr/bin/env node
22

3-
var diff = require('../diff');
3+
var diff = require('../lib/diff');
44
var fastlog = require('fastlog');
55
var args = require('minimist')(process.argv.slice(2));
66
var crypto = require('crypto');
7-
var parse_location = require('../parse-location')
7+
var parse_location = require('../lib/parse-location');
88

99
function usage() {
1010
console.error('');

‎bin/incremental-backfill.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
var args = require('minimist')(process.argv.slice(2));
44
var s3urls = require('s3urls');
5-
var backfill = require('../s3-backfill');
5+
var backfill = require('../lib/s3-backfill');
66

77
function usage() {
88
console.error('');

‎bin/incremental-snapshot.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ var AWS = require('aws-sdk');
44
var args = require('minimist')(process.argv.slice(2));
55
var s3urls = require('s3urls');
66
var fastlog = require('fastlog');
7-
var snapshot = require('../s3-snapshot');
7+
var snapshot = require('../lib/s3-snapshot');
88
var fs = require('fs');
99
var path = require('path');
1010

‎index.js‎

Lines changed: 5 additions & 219 deletions
Original file line numberDiff line numberDiff line change
@@ -1,219 +1,5 @@
1-
var AWS = require('aws-sdk');
2-
var Dyno = require('dyno');
3-
var queue = require('queue-async');
4-
var crypto = require('crypto');
5-
var https = require('https');
6-
7-
module.exports.replicate = replicate;
8-
module.exports.backup = incrementalBackup;
9-
module.exports.snapshot = require('./s3-snapshot');
10-
module.exports.agent = new https.Agent({
11-
keepAlive: true,
12-
maxSockets: Math.ceil(require('os').cpus().length * 16),
13-
keepAliveMsecs: 60000
14-
})
15-
16-
function replicate(event, context, callback) {
17-
var replicaConfig = {
18-
accessKeyId: process.env.ReplicaAccessKeyId || undefined,
19-
secretAccessKey: process.env.ReplicaSecretAccessKey || undefined,
20-
table: process.env.ReplicaTable,
21-
region: process.env.ReplicaRegion,
22-
maxRetries: 1000,
23-
httpOptions: {
24-
timeout: 2000,
25-
agent: module.exports.agent
26-
}
27-
};
28-
29-
if (process.env.ReplicaEndpoint) replicaConfig.endpoint = process.env.ReplicaEndpoint;
30-
var replica = new Dyno(replicaConfig);
31-
32-
var keyAttrs = Object.keys(event.Records[0].dynamodb.Keys);
33-
34-
var filterer;
35-
if (process.env.TurnoverRole && process.env.TurnoverAt) {
36-
// Filterer function should return true if the record SHOULD be processed
37-
filterer = function(record) {
38-
var created = Number(record.dynamodb.ApproximateCreationDateTime + '000');
39-
var turnoverAt = Number(process.env.TurnoverAt);
40-
if (process.env.TurnoverRole === 'BEFORE') return created < turnoverAt;
41-
else if (process.env.TurnoverRole === 'AFTER') return created >= turnoverAt;
42-
else return true;
43-
};
44-
}
45-
46-
var count = 0;
47-
var allRecords = event.Records.reduce(function(allRecords, change) {
48-
if (filterer && !filterer(change)) return allRecords;
49-
var id = JSON.stringify(change.dynamodb.Keys);
50-
allRecords[id] = allRecords[id] || [];
51-
allRecords[id].push(change);
52-
count++;
53-
return allRecords;
54-
}, {});
55-
56-
if (count === 0) {
57-
console.log('No records replicated');
58-
return callback();
59-
}
60-
61-
var params = { RequestItems: {} };
62-
params.RequestItems[process.env.ReplicaTable] = Object.keys(allRecords).map(function(key) {
63-
var change = allRecords[key].pop();
64-
if (change.eventName === 'INSERT' || change.eventName === 'MODIFY') {
65-
return {
66-
PutRequest: { Item: Dyno.deserialize(JSON.stringify(change.dynamodb.NewImage)) }
67-
};
68-
} else if (change.eventName === 'REMOVE') {
69-
return {
70-
DeleteRequest: { Key: Dyno.deserialize(JSON.stringify(change.dynamodb.Keys)) }
71-
}
72-
}
73-
});
74-
75-
(function batchWrite(requestSet, attempts) {
76-
requestSet.forEach(function(req) {
77-
if (req) req.on('retry', function(res) {
78-
if (!res.error || !res.httpResponse || !res.httpResponse.headers) return;
79-
if (res.error.name === 'TimeoutError') res.error.retryable = true;
80-
console.log(
81-
'[failed-request] %s | request-id: %s | crc32: %s | items: %j',
82-
res.error.message,
83-
res.httpResponse.headers['x-amzn-requestid'],
84-
res.httpResponse.headers['x-amz-crc32'],
85-
req.params.RequestItems[process.env.ReplicaTable].map(function(req) {
86-
if (req.DeleteRequest) return req.DeleteRequest.Key;
87-
if (req.PutRequest) return keyAttrs.reduce(function(key, k) {
88-
key[k] = req.PutRequest.Item[k];
89-
return key;
90-
}, {});
91-
})
92-
);
93-
});
94-
});
95-
96-
requestSet.sendAll(100, function(errs, responses, unprocessed) {
97-
attempts++;
98-
99-
if (errs) {
100-
var messages = errs
101-
.filter(function(err) { return !!err; })
102-
.map(function(err) { return err.message; })
103-
.join(' | ');
104-
console.log('[error] %s', messages);
105-
return callback(errs);
106-
}
107-
108-
if (unprocessed) {
109-
console.log('[retry] attempt %s contained unprocessed items', attempts);
110-
return setTimeout(batchWrite, Math.pow(2, attempts), unprocessed, attempts);
111-
}
112-
113-
console.log('Replicated ' + count + ' records');
114-
callback();
115-
});
116-
})(replica.batchWriteItemRequests(params), 0);
117-
}
118-
119-
function incrementalBackup(event, context, callback) {
120-
var params = {
121-
maxRetries: 1000,
122-
httpOptions: {
123-
timeout: 1000,
124-
agent: module.exports.agent
125-
}
126-
};
127-
128-
if (process.env.BackupRegion) params.region = process.env.BackupRegion;
129-
130-
var s3 = new AWS.S3(params);
131-
132-
var filterer;
133-
if (process.env.TurnoverRole && process.env.TurnoverAt) {
134-
// Filterer function should return true if the record SHOULD be processed
135-
filterer = function(record) {
136-
var created = Number(record.dynamodb.ApproximateCreationDateTime + '000');
137-
var turnoverAt = Number(process.env.TurnoverAt);
138-
if (process.env.TurnoverRole === 'BEFORE') return created < turnoverAt;
139-
else if (process.env.TurnoverRole === 'AFTER') return created >= turnoverAt;
140-
else return true;
141-
};
142-
}
143-
144-
var count = 0;
145-
var allRecords = event.Records.reduce(function(allRecords, action) {
146-
if (filterer && !filterer(action)) return allRecords;
147-
148-
var id = JSON.stringify(action.dynamodb.Keys);
149-
150-
allRecords[id] = allRecords[id] || [];
151-
allRecords[id].push(action);
152-
count++;
153-
return allRecords;
154-
}, {});
155-
156-
if (count === 0) {
157-
console.log('No records backed up');
158-
return callback();
159-
}
160-
161-
var q = queue();
162-
163-
Object.keys(allRecords).forEach(function(key) {
164-
q.defer(backupRecord, allRecords[key]);
165-
});
166-
167-
q.awaitAll(function(err) {
168-
if (err) throw err;
169-
callback();
170-
});
171-
172-
function backupRecord(changes, callback) {
173-
var q = queue(1);
174-
175-
changes.forEach(function(change) {
176-
q.defer(function(next) {
177-
var id = crypto.createHash('md5')
178-
.update(JSON.stringify(change.dynamodb.Keys))
179-
.digest('hex');
180-
181-
var table = change.eventSourceARN.split('/')[1];
182-
183-
var params = {
184-
Bucket: process.env.BackupBucket,
185-
Key: [process.env.BackupPrefix, table, id].join('/')
186-
};
187-
188-
var req = change.eventName === 'REMOVE' ? 'deleteObject' : 'putObject';
189-
if (req === 'putObject') params.Body = JSON.stringify(change.dynamodb.NewImage);
190-
191-
s3[req](params, function(err) {
192-
if (err) console.log(
193-
'[error] %s | %s s3://%s/%s | %s',
194-
JSON.stringify(change.dynamodb.Keys),
195-
req, params.Bucket, params.Key,
196-
err.message
197-
);
198-
next(err);
199-
}).on('retry', function(res) {
200-
if (!res.error || !res.httpResponse || !res.httpResponse.headers) return;
201-
if (res.error.name === 'TimeoutError') res.error.retryable = true;
202-
console.log(
203-
'[failed-request] request-id: %s | id-2: %s | %s s3://%s/%s | %s',
204-
res.httpResponse.headers['x-amz-request-id'],
205-
res.httpResponse.headers['x-amz-id-2'],
206-
req, params.Bucket, params.Key,
207-
res.error
208-
);
209-
});
210-
});
211-
});
212-
213-
q.awaitAll(function(err) {
214-
if (err) return callback(err);
215-
console.log('Backed up ' + count + ' records')
216-
callback();
217-
});
218-
}
219-
}
1+
module.exports = {
2+
replicate: require('./lib/replicate'),
3+
backup: require('./lib/incremental-backup'),
4+
snapshot: require('./lib/s3-snapshot')
5+
};

backup.js renamed to lib/backup.js

File renamed without changes.

diff.js renamed to lib/diff.js

File renamed without changes.

‎lib/incremental-backup.js‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
var AWS = require('aws-sdk');
2+
var d3 = require('d3-queue');
3+
var crypto = require('crypto');
4+
var https = require('https');
5+
6+
module.exports = function incrementalBackup(event, context, callback) {
7+
var params = {
8+
maxRetries: 1000,
9+
httpOptions: {
10+
timeout: 1000,
11+
agent: new https.Agent({
12+
keepAlive: true,
13+
maxSockets: Math.ceil(require('os').cpus().length * 16),
14+
keepAliveMsecs: 60000
15+
})
16+
}
17+
};
18+
19+
if (process.env.BackupRegion) params.region = process.env.BackupRegion;
20+
21+
var s3 = new AWS.S3(params);
22+
23+
var filterer;
24+
if (process.env.TurnoverRole && process.env.TurnoverAt) {
25+
// Filterer function should return true if the record SHOULD be processed
26+
filterer = function(record) {
27+
var created = Number(record.dynamodb.ApproximateCreationDateTime + '000');
28+
var turnoverAt = Number(process.env.TurnoverAt);
29+
if (process.env.TurnoverRole === 'BEFORE') return created < turnoverAt;
30+
else if (process.env.TurnoverRole === 'AFTER') return created >= turnoverAt;
31+
else return true;
32+
};
33+
}
34+
35+
var count = 0;
36+
var allRecords = event.Records.reduce(function(allRecords, action) {
37+
if (filterer && !filterer(action)) return allRecords;
38+
39+
var id = JSON.stringify(action.dynamodb.Keys);
40+
41+
allRecords[id] = allRecords[id] || [];
42+
allRecords[id].push(action);
43+
count++;
44+
return allRecords;
45+
}, {});
46+
47+
if (count === 0) {
48+
console.log('No records backed up');
49+
return callback();
50+
}
51+
52+
var q = d3.queue();
53+
54+
Object.keys(allRecords).forEach(function(key) {
55+
q.defer(backupRecord, allRecords[key]);
56+
});
57+
58+
q.awaitAll(function(err) {
59+
if (err) throw err;
60+
callback();
61+
});
62+
63+
function backupRecord(changes, callback) {
64+
var q = d3.queue(1);
65+
66+
changes.forEach(function(change) {
67+
q.defer(function(next) {
68+
var id = crypto.createHash('md5')
69+
.update(JSON.stringify(change.dynamodb.Keys))
70+
.digest('hex');
71+
72+
var table = change.eventSourceARN.split('/')[1];
73+
74+
var params = {
75+
Bucket: process.env.BackupBucket,
76+
Key: [process.env.BackupPrefix, table, id].join('/')
77+
};
78+
79+
var req = change.eventName === 'REMOVE' ? 'deleteObject' : 'putObject';
80+
if (req === 'putObject') params.Body = JSON.stringify(change.dynamodb.NewImage);
81+
82+
s3[req](params, function(err) {
83+
if (err) console.log(
84+
'[error] %s | %s s3://%s/%s | %s',
85+
JSON.stringify(change.dynamodb.Keys),
86+
req, params.Bucket, params.Key,
87+
err.message
88+
);
89+
next(err);
90+
}).on('retry', function(res) {
91+
if (!res.error || !res.httpResponse || !res.httpResponse.headers) return;
92+
if (res.error.name === 'TimeoutError') res.error.retryable = true;
93+
console.log(
94+
'[failed-request] request-id: %s | id-2: %s | %s s3://%s/%s | %s',
95+
res.httpResponse.headers['x-amz-request-id'],
96+
res.httpResponse.headers['x-amz-id-2'],
97+
req, params.Bucket, params.Key,
98+
res.error
99+
);
100+
});
101+
});
102+
});
103+
104+
q.awaitAll(function(err) {
105+
if (err) return callback(err);
106+
console.log('Backed up ' + count + ' records')
107+
callback();
108+
});
109+
}
110+
}

parse-location.js renamed to lib/parse-location.js

File renamed without changes.

0 commit comments

Comments
 (0)