Skip to content

Commit

Permalink
Add code linting and styleguide, and enforce them on npm scripts and CI
Browse files Browse the repository at this point in the history 
Fix #7
  • Loading branch information
@nihey
nihey committed Apr 7, 2019
1 parent 330d20a commit 6efcf2d
Show file tree
Hide file tree
Showing 8 changed files with 1,040 additions and 148 deletions.
11 changes: 11 additions & 0 deletions .eslintrc.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
env:
browser: true
commonjs: true
es6: true
extends: standard
globals:
Atomics: readonly
SharedArrayBuffer: readonly
parserOptions:
ecmaVersion: 2018
rules: {}
4 changes: 4 additions & 0 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
language: node_js
node_js:
- "node"
- "lts/*"
208 changes: 103 additions & 105 deletions bin/build.js
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,12 @@
const fs = require("fs");
const path = require("path");
const axios = require("axios");
const mkdirp = require("mkdirp");
const AdmZip = require("adm-zip");
const readline = require("readline");
const moment = require("moment");
const sequelize = require("../lib/db");
const util = require("../lib/util");
const { Quote } = sequelize;
const meow = require("meow");
const fs = require('fs')
const path = require('path')
const axios = require('axios')
const mkdirp = require('mkdirp')
const AdmZip = require('adm-zip')
const moment = require('moment')
const sequelize = require('../lib/db')
const util = require('../lib/util')
const meow = require('meow')
const cli = meow(`
Usage:
$ node bin/build.js [Options]
Expand All @@ -27,75 +25,75 @@ const cli = meow(`
`, {
flags: {
update: {
type: "boolean",
alias: "u",
type: 'boolean',
alias: 'u'
},
'no-cache': {
type: "boolean",
alias: "n",
},
},
});
type: 'boolean',
alias: 'n'
}
}
})

const download = async function(year, cache) {
const url = `http://bvmf.bmfbovespa.com.br/InstDados/SerHist/COTAHIST_A${year}.ZIP`;
mkdirp.sync(path.join(__dirname, '..', "_downloaded", "raw"))
const filepath = path.resolve(__dirname, '..', "_downloaded", "raw", `A${year}.zip`);
const download = async function (year, cache) {
const url = `http://bvmf.bmfbovespa.com.br/InstDados/SerHist/COTAHIST_A${year}.ZIP`
mkdirp.sync(path.join(__dirname, '..', '_downloaded', 'raw'))
const filepath = path.resolve(__dirname, '..', '_downloaded', 'raw', `A${year}.zip`)

// Use cached file, if it exists
if (fs.existsSync(filepath) && cache) {
console.log("Using cached", year);
return;
console.log('Using cached', year)
return
}

const writer = fs.createWriteStream(filepath);
const writer = fs.createWriteStream(filepath)

console.log("Downloading ", year)
console.log('Downloading ', year)
const response = await axios({
url,
method: "GET",
responseType: "stream",
});
method: 'GET',
responseType: 'stream'
})

response.data.pipe(writer);
response.data.pipe(writer)
return new Promise((resolve, reject) => {
writer.on("finish", resolve);
writer.on("error", reject);
});
writer.on('finish', resolve)
writer.on('error', reject)
})
}

const extract = async function(year, cache) {
const extractedPath = path.resolve(__dirname, '..', "_downloaded", "extracted");
mkdirp.sync(extractedPath);
const sourcePath = path.resolve(__dirname, '..', "_downloaded", "raw", `A${year}.zip`);
const destinationPath = path.resolve(__dirname, '..', "_downloaded", "extracted", `A${year}.txt`);
const extract = async function (year, cache) {
const extractedPath = path.resolve(__dirname, '..', '_downloaded', 'extracted')
mkdirp.sync(extractedPath)
const sourcePath = path.resolve(__dirname, '..', '_downloaded', 'raw', `A${year}.zip`)
const destinationPath = path.resolve(__dirname, '..', '_downloaded', 'extracted', `A${year}.txt`)

// Use cached file, if it exists
if (fs.existsSync(destinationPath) && cache) {
console.log("Using cached", year);
return;
console.log('Using cached', year)
return
}

const zip = new AdmZip(sourcePath);
const zip = new AdmZip(sourcePath)

console.log("Extracting", year)
console.log('Extracting', year)
const entries = zip.getEntries().map(e => e.entryName);
[`COTAHIST_A${year}.TXT`, `COTAHIST_A${year}`, `COTAHIST.A${year}`].forEach(filename => {
if (!entries.find(f => filename === f)) {
return;
return
}

zip.extractEntryTo(filename, extractedPath, false, true);
fs.renameSync(path.join(extractedPath, filename), destinationPath);
zip.extractEntryTo(filename, extractedPath, false, true)
fs.renameSync(path.join(extractedPath, filename), destinationPath)
})
}

const parse = async function(year, func) {
const source = path.resolve(__dirname, '..', "_downloaded", "extracted", `A${year}.txt`);
const lines = fs.readFileSync(source, "utf-8").split(/\r?\n/);
const parse = async function (year, func) {
const source = path.resolve(__dirname, '..', '_downloaded', 'extracted', `A${year}.txt`)
const lines = fs.readFileSync(source, 'utf-8').split(/\r?\n/)
for (const line of lines) {
// Ignore header and footer
if (line.slice(0, 2) !== "01") {
if (line.slice(0, 2) !== '01') {
continue
}

Expand Down Expand Up @@ -124,112 +122,112 @@ const parse = async function(year, func) {
fatcot: line.slice(210, 217),
ptoexe: line.slice(217, 230),
codisi: line.slice(230, 242),
dismes: line.slice(242, 245),
dismes: line.slice(242, 245)
};

// Convert dates
["day", "datven"].forEach(attr => {
data[attr] = moment(data[attr], "YYYYMMDD");
['day', 'datven'].forEach(attr => {
data[attr] = moment(data[attr], 'YYYYMMDD')
});

// Convert float values
["preabe", "premin", "premax", "premed", "preult", "preofc", "preofv", "preexe"].forEach(attr => {
['preabe', 'premin', 'premax', 'premed', 'preult', 'preofc', 'preofv', 'preexe'].forEach(attr => {
let v = data[attr]
data[attr] = parseFloat(v.slice(0, v.length - 2) + "." + v.slice(v.length - 2, v.length));
data[attr] = parseFloat(v.slice(0, v.length - 2) + '.' + v.slice(v.length - 2, v.length))
});

// Conver int values
["totneg", "quatot", "voltot"].forEach(attr => {
data[attr] = parseInt(data[attr]);
});
['totneg', 'quatot', 'voltot'].forEach(attr => {
data[attr] = parseInt(data[attr])
})

Object.entries(data).forEach(([k, v]) => {
if (typeof v === "string") {
data[k] = v.trim();
if (typeof v === 'string') {
data[k] = v.trim()
}
});
})

func(data);
func(data)
}
};
}

async function main() {
const update = cli.flags.update;
const cache = !cli.flags.n && !update;
async function main () {
const update = cli.flags.update
const cache = !cli.flags.n && !update

let years = [];
let years = []
for (let i = 1999; i < 2020; i++) {
years.push(i);
years.push(i)
}

if (update) {
years = [moment().format("YYYY")];
years = [moment().format('YYYY')]
}

for (const year of years) {
console.log("Downloading files...");
await download(year, cache);
console.log("Done\n");
console.log('Downloading files...')
await download(year, cache)
console.log('Done\n')

console.log("Extracting files...");
await extract(year, cache);
console.log("Done\n");
console.log('Extracting files...')
await extract(year, cache)
console.log('Done\n')

console.log("Parsing files");
let query = "INSERT INTO quote "
let columns;
let rows = [];
let set = new Set();
console.log('Parsing files')
let query = 'INSERT INTO quote '
let columns
let rows = []
let set = new Set()

if (update) {
set = await util.get.filled(moment({y: year}).startOf("year"));
set = await util.get.filled(moment({ y: year }).startOf('year'))
}

await parse(year, data => {
if (!columns) {
columns = Object.keys(data);
columns = Object.keys(data)
}

const key = data.day.format("YYYY-MM-DD") + "-" + data.codneg;
const key = data.day.format('YYYY-MM-DD') + '-' + data.codneg
if (set.has(key)) {
return;
return
}

console.log("Added:", key);
set.add(key);
console.log('Added:', key)
set.add(key)

rows.push(columns.map(column => {
const value = data[column];
if (typeof value === "string") {
return `'${value}'`;
const value = data[column]
if (typeof value === 'string') {
return `'${value}'`
}
if (value.format) {
return value.format("'YYYY-MM-DD'");
return value.format("'YYYY-MM-DD'")
}

return value;
}));
});
return value
}))
})

console.log("Inserting:", year);
console.log("Rows to go:", rows.length);
console.log('Inserting:', year)
console.log('Rows to go:', rows.length)
for (let i = 0; i < rows.length; i += 50000) {
console.log("Batch", i, i + 50000);
const rowSet = rows.slice(i, i + 50000);
let raw_query = query;
raw_query += " (" + columns.join(", ") + ") VALUES ";
raw_query += rowSet.map(x => `(${x})`).join(",\n") + ";";
await sequelize.query(raw_query);
console.log('Batch', i, i + 50000)
const rowSet = rows.slice(i, i + 50000)
let rawQuery = query
rawQuery += ' (' + columns.join(', ') + ') VALUES '
rawQuery += rowSet.map(x => `(${x})`).join(',\n') + ';'
await sequelize.query(rawQuery)
}

console.log("Done\n");
console.log('Done\n')
}

console.log("Finished");
sequelize.close();
process.exit(0);
console.log('Finished')
sequelize.close()
process.exit(0)
}

if (require.main === module) {
main();
main()
}
24 changes: 12 additions & 12 deletions bin/server.js
View file
Original file line number Diff line number Diff line change
@@ -1,23 +1,23 @@
const moment = require("moment");
const sequelize = require("../lib/db");
const { Quote } = sequelize;
const moment = require('moment')
const sequelize = require('../lib/db')
const { Quote } = sequelize

module.exports = async (fastify, options) => {
fastify.get("/api/quote/:code/:date", async (request, reply) => {
fastify.get('/api/quote/:code/:date', async (request, reply) => {
const quote = await Quote.findOne({
where: {
codneg: request.params.code.toUpperCase(),
day: moment(request.params.date),
},
});
day: moment(request.params.date)
}
})

if (!quote) {
reply.status(404);
reply.status(404)
return {
error: "NOT_FOUND",
};
error: 'NOT_FOUND'
}
}

return quote.dataValues;
});
return quote.dataValues
})
}
Loading

0 comments on commit 6efcf2d

Please sign in to comment.