Skip to content

Commit 8fa39a8

Browse files
matchillingmatchilling
committed
Add CSV exporter
1 parent 6140a3e commit 8fa39a8

File tree

12 files changed

+291
-155
lines changed

12 files changed

+291
-155
lines changed

‎README.md‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -46,11 +46,11 @@ $ npm start collect facebook react
4646

4747
# Push data to graphite
4848
# npm start export <owner> <repository>
49-
$ npm start export facebook react | nc localhost 2003
49+
$ npm start export:graphite facebook react | nc localhost 2003
5050

5151
# The generated time series will be written to `stdout`.
52-
# github.github-user-name.repository-name.pull_requests.size_m.time_to_merge 3450 1554125772
53-
# github.github-user-name.repository-name.pull_requests.size_xxl.time_to_merge 935617 1553187544
52+
# github.facebook.react.pull_requests.size_m.time_to_merge 3450 1554125772
53+
# github.facebook.react.pull_requests.size_xxl.time_to_merge 935617 1553187544
5454
# ...
5555
```
5656

@@ -61,8 +61,6 @@ For example:
6161
```sh
6262
# Note that the `./data` directory is mounted to the docker container, to keep your data persistent place your sqlite database in here
6363
$ cat > .env <<EOL
64-
GITHUB_USER_NAME=facebook
65-
GITHUB_REPO_NAME=react
6664
GITHUB_TOKEN=create a new token here https://github.com/settings/tokens/new
6765
DATABASE_PATH=path to your sqlite3 database e.g. data/github.db
6866
EOL

‎package.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
"dependencies": {
77
"better-sqlite3": "^5.4.0",
88
"commander": "^2.20.0",
9+
"csv-writer": "^1.3.0",
910
"github-graphql-api": "^1.0.6",
1011
"moment": "^2.24.0"
1112
},
@@ -15,8 +16,8 @@
1516
},
1617
"scripts": {
1718
"env": "node src/print_env.js",
18-
"lint": "prettier --single-quote --trailing-comma=es5 --debug-check es5 src/**/*.js src/*.js README.md",
19-
"lint:fix": "prettier --single-quote --trailing-comma=es5 es5 --write src/**/*.js src/*.js README.md",
19+
"lint": "prettier --single-quote --trailing-comma=es5 --debug-check es5 'src/**/*.js' README.md",
20+
"lint:fix": "prettier --single-quote --trailing-comma=es5 es5 --write 'src/**/*.js' README.md",
2021
"start": "node src/cli.js",
2122
"test": "jest"
2223
},

‎src/cli.js‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
const { collector } = require('./collector');
2+
const { exporter: csvExporter } = require('./exporter/csv');
23
const { exporter: graphiteExporter } = require('./exporter/graphite');
34
const Database = require('better-sqlite3');
45
const { getOrFail } = require('./env');
@@ -38,7 +39,22 @@ program
3839

3940
program
4041
.version(pkg.version)
41-
.command('export:toGraphite <owner> <repository>')
42+
.command('export:csv')
43+
.action(async (owner, repository) => {
44+
const database = new Database(getOrFail('DATABASE_PATH'), {
45+
fileMustExist: true,
46+
});
47+
48+
await csvExporter(database);
49+
50+
if (database.open) {
51+
database.close();
52+
}
53+
});
54+
55+
program
56+
.version(pkg.version)
57+
.command('export:graphite <owner> <repository>')
4258
.action(async (owner, repository) => {
4359
const database = new Database(getOrFail('DATABASE_PATH'), {
4460
fileMustExist: true,

‎src/collector/index.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ const update = async (
5757
lastCursor: '',
5858
}
5959
) => {
60-
const repository = `${githubUserName}-${githubRepoName}`;
60+
const repository = `${githubUserName}/${githubRepoName}`;
6161
const stmt = database.prepare(
6262
`SELECT value FROM pull_request_cursor WHERE repository = ?;`
6363
);

‎src/exporter/csv/index.js‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
const createCsvStringifier = require('csv-writer').createObjectCsvStringifier;
2+
const moment = require('moment');
3+
const { sizeLabel } = require('../../lib/sizeLabel');
4+
5+
const fields = {
6+
OWNER: 'owner',
7+
REPOSITORY: 'repository',
8+
ID: 'id',
9+
SIZE: 'size',
10+
TIME_TO_MERGE: 'time_to_merge',
11+
CREATED_AT: 'created_at',
12+
CLOSED_AT: 'closed_at',
13+
UPDATED_AT: 'updated_at',
14+
MERGED_AT: 'merged_at',
15+
TITLE: 'title',
16+
URL: 'url',
17+
COMMENTS: 'comments',
18+
ADDITIONS: 'additions',
19+
CHANGED_FILES: 'changed_files',
20+
DELETIONS: 'deletions',
21+
};
22+
23+
const exporter = async database => {
24+
const stringifier = createCsvStringifier({
25+
header: Object.keys(fields).map(key => ({
26+
id: fields[key],
27+
title: fields[key],
28+
})),
29+
});
30+
31+
process.stdout.write(stringifier.getHeaderString());
32+
33+
await database
34+
.prepare('SELECT * FROM pull_request;')
35+
.all()
36+
.map(async row => {
37+
const { repository } = row;
38+
const owner = repository.split('/')[0];
39+
const repo = repository.split('/')[1];
40+
41+
const data = JSON.parse(row.data);
42+
const size = sizeLabel(data.additions + data.deletions);
43+
44+
if (size) {
45+
const createdAt = moment(data.createdAt);
46+
const mergedAt = moment(data.mergedAt);
47+
const timeToMerge = mergedAt.diff(createdAt, 's');
48+
49+
const record = {
50+
[fields.OWNER]: owner,
51+
[fields.REPOSITORY]: repo,
52+
[fields.ID]: data.id,
53+
[fields.SIZE]: size,
54+
[fields.TIME_TO_MERGE]: timeToMerge,
55+
[fields.CREATED_AT]: data.createdAt,
56+
[fields.CLOSED_AT]: data.closedAt,
57+
[fields.UPDATED_AT]: data.updatedAt,
58+
[fields.MERGED_AT]: data.mergedAt,
59+
[fields.TITLE]: data.title,
60+
[fields.URL]: data.url,
61+
[fields.COMMENTS]: data.comments.totalCount,
62+
[fields.ADDITIONS]: data.additions,
63+
[fields.CHANGED_FILES]: data.changedFiles,
64+
[fields.DELETIONS]: data.deletions,
65+
};
66+
67+
process.stdout.write(stringifier.stringifyRecords([record]));
68+
}
69+
});
70+
};
71+
72+
module.exports = {
73+
exporter,
74+
};

‎src/exporter/csv/index.test.js‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
const { exporter } = require('./index');
2+
3+
describe('#exporter', () => {
4+
const composeDatabaseMock = payload => {
5+
return (database = {
6+
prepare: () => {
7+
return {
8+
all: () => payload,
9+
};
10+
},
11+
});
12+
};
13+
14+
beforeEach(() => {
15+
process.stdout.write = jest.fn();
16+
});
17+
18+
test('writes pull request data in csv to std out', async () => {
19+
const database = composeDatabaseMock([
20+
{
21+
repository: 'bob/left-pad',
22+
data:
23+
'{"createdAt":"2019-04-03T23:55:58Z","closedAt":"2019-04-04T15:55:36Z","updatedAt":"2019-05-03T17:35:13Z","mergedAt":"2019-04-04T15:55:36Z","id":"MDExOlB1bGxSZXF1ZXN0MjY3MjUwMzMy","title":"React events: keyboard press, types, tests","url":"https://github.com/facebook/react/pull/15314","comments":{"totalCount":1},"additions":584,"changedFiles":6,"deletions":241}',
24+
},
25+
]);
26+
27+
await exporter(database);
28+
expect(process.stdout.write).toHaveBeenCalledTimes(2);
29+
expect(process.stdout.write.mock.calls).toEqual([
30+
[
31+
'owner,repository,id,size,time_to_merge,created_at,closed_at,updated_at,merged_at,title,url,comments,additions,changed_files,deletions\n',
32+
],
33+
[
34+
'bob,left-pad,MDExOlB1bGxSZXF1ZXN0MjY3MjUwMzMy,size_xl,57578,2019-04-03T23:55:58Z,2019-04-04T15:55:36Z,2019-05-03T17:35:13Z,2019-04-04T15:55:36Z,"React events: keyboard press, types, tests",https://github.com/facebook/react/pull/15314,1,584,6,241\n',
35+
],
36+
]);
37+
});
38+
});

‎src/exporter/graphite/index.js‎

Lines changed: 3 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,5 @@
1-
const moment = require('moment');
2-
3-
const toMetricTimeToMerge = (data, metricPath) => {
4-
const createdAt = moment(data.createdAt);
5-
const mergedAt = moment(data.mergedAt);
6-
const differenceInSeconds = mergedAt.diff(createdAt, 's');
7-
8-
return `${metricPath} ${differenceInSeconds} ${createdAt.unix()}`;
9-
};
10-
11-
const sizeLabel = lineCount => {
12-
const sizes = {
13-
s: 10,
14-
m: 30,
15-
l: 100,
16-
xl: 500,
17-
xxl: 1000,
18-
};
19-
20-
if (!Number.isInteger(lineCount)) {
21-
return undefined;
22-
} else if (lineCount < 0) {
23-
return undefined;
24-
} else if (lineCount < sizes.s) {
25-
return 'size_xs';
26-
} else if (lineCount < sizes.m) {
27-
return 'size_s';
28-
} else if (lineCount < sizes.l) {
29-
return 'size_m';
30-
} else if (lineCount < sizes.xl) {
31-
return 'size_l';
32-
} else if (lineCount < sizes.xxl) {
33-
return 'size_xl';
34-
}
35-
36-
return 'size_xxl';
37-
};
1+
const { timeToMerge } = require('../../lib/metric');
2+
const { sizeLabel } = require('../../lib/sizeLabel');
383

394
const exporter = async (database, githubUserName, githubRepoName) => {
405
await database
@@ -45,7 +10,7 @@ const exporter = async (database, githubUserName, githubRepoName) => {
4510
const size = sizeLabel(data.additions + data.deletions);
4611

4712
if (size) {
48-
const metric = toMetricTimeToMerge(
13+
const metric = timeToMerge(
4914
data,
5015
`github.${githubUserName}.${githubRepoName}.pull_requests.${size}.time_to_merge`
5116
);
@@ -57,6 +22,4 @@ const exporter = async (database, githubUserName, githubRepoName) => {
5722

5823
module.exports = {
5924
exporter,
60-
sizeLabel,
61-
toMetricTimeToMerge,
6225
};

‎src/exporter/graphite/index.test.js‎

Lines changed: 1 addition & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
const { exporter, sizeLabel, toMetricTimeToMerge } = require('./index');
1+
const { exporter, toMetricTimeToMerge } = require('./index');
22

33
describe('#exporter', () => {
44
const composeDatabaseMock = payload => {
@@ -41,108 +41,3 @@ describe('#exporter', () => {
4141
expect(process.stdout.write).toHaveBeenCalledTimes(0);
4242
});
4343
});
44-
45-
describe('#sizeLabel', () => {
46-
test('returns the correct size label per line count', () => {
47-
const useCases = [
48-
{
49-
lineCount: NaN,
50-
expectedLabel: undefined,
51-
},
52-
{
53-
lineCount: -1,
54-
expectedLabel: undefined,
55-
},
56-
{
57-
lineCount: 0,
58-
expectedLabel: 'size_xs',
59-
},
60-
{
61-
lineCount: 9,
62-
expectedLabel: 'size_xs',
63-
},
64-
{
65-
lineCount: 10,
66-
expectedLabel: 'size_s',
67-
},
68-
{
69-
lineCount: 29,
70-
expectedLabel: 'size_s',
71-
},
72-
{
73-
lineCount: 30,
74-
expectedLabel: 'size_m',
75-
},
76-
{
77-
lineCount: 99,
78-
expectedLabel: 'size_m',
79-
},
80-
{
81-
lineCount: 100,
82-
expectedLabel: 'size_l',
83-
},
84-
{
85-
lineCount: 499,
86-
expectedLabel: 'size_l',
87-
},
88-
{
89-
lineCount: 500,
90-
expectedLabel: 'size_xl',
91-
},
92-
{
93-
lineCount: 999,
94-
expectedLabel: 'size_xl',
95-
},
96-
{
97-
lineCount: 1000,
98-
expectedLabel: 'size_xxl',
99-
},
100-
];
101-
102-
useCases.forEach(useCase => {
103-
expect(sizeLabel(useCase.lineCount)).toEqual(useCase.expectedLabel);
104-
});
105-
});
106-
});
107-
108-
describe('#toMetricTimeToMerge', () => {
109-
test('calculates the elapsed time between the creation of a pull request and the merging of the pull request', () => {
110-
const useCases = [
111-
{
112-
data: {
113-
createdAt: '2019-05-02T10:00:00Z',
114-
mergedAt: '2019-05-02T11:00:00Z',
115-
},
116-
differenceSeconds: 60 * 60, // one hour
117-
},
118-
{
119-
data: {
120-
createdAt: '2019-05-02T10:00:00Z',
121-
mergedAt: '2019-05-03T10:00:00Z',
122-
},
123-
differenceSeconds: 60 * 60 * 24, // one day
124-
},
125-
{
126-
data: {
127-
createdAt: '2019-05-02T10:00:00Z',
128-
mergedAt: '2019-05-09T10:00:00Z',
129-
},
130-
differenceSeconds: 60 * 60 * 24 * 7, // one week
131-
},
132-
{
133-
data: {
134-
createdAt: '2019-05-02T10:00:00Z',
135-
mergedAt: '2019-06-02T10:00:00Z',
136-
},
137-
differenceSeconds: 60 * 60 * 24 * 31, // a month of 31 days
138-
},
139-
];
140-
141-
useCases.forEach(useCase => {
142-
const result = toMetricTimeToMerge(useCase.data, 'path.to.metric');
143-
expect(result).toEqual(
144-
`path.to.metric ${useCase.differenceSeconds} 1556791200`
145-
);
146-
});
147-
});
148-
});

‎src/lib/metric.js‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
const moment = require('moment');
2+
3+
const timeToMerge = (data, metricPath) => {
4+
const createdAt = moment(data.createdAt);
5+
const mergedAt = moment(data.mergedAt);
6+
const differenceInSeconds = mergedAt.diff(createdAt, 's');
7+
8+
return `${metricPath} ${differenceInSeconds} ${createdAt.unix()}`;
9+
};
10+
11+
module.exports = {
12+
timeToMerge,
13+
};

0 commit comments

Comments
 (0)