Skip to content

Commit

Permalink
feat(file-list): Use glob.sync for better speed
Browse files Browse the repository at this point in the history
  • Loading branch information
dignifiedquire committed Jun 25, 2015
1 parent cc15dc4 commit 1b65cde
Show file tree
Hide file tree
Showing 3 changed files with 97 additions and 92 deletions.
68 changes: 35 additions & 33 deletions lib/file-list.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ require('core-js')
var from = require('core-js/library/fn/array/from')
var Promise = require('bluebird')
var mm = require('minimatch')
var Glob = require('glob').Glob
var fs = Promise.promisifyAll(require('fs'))

var File = require('./file')
Expand All @@ -25,7 +26,8 @@ var log = require('./logger').create('watcher')
var GLOB_OPTS = {
cwd: '/',
follow: true,
nodir: true
nodir: true,
sync: true
}

// Helper Functions
Expand Down Expand Up @@ -153,42 +155,42 @@ List.prototype._refresh = function () {
return Promise.resolve()
}

return helper.glob(pattern, GLOB_OPTS).spread(function (files, mg) {
buckets.set(pattern, new Set())
var mg = new Glob(pattern, GLOB_OPTS)
var files = mg.found
buckets.set(pattern, new Set())

if (_.isEmpty(files)) {
log.warn('Pattern "%s" does not match any file.', pattern)
return
if (_.isEmpty(files)) {
log.warn('Pattern "%s" does not match any file.', pattern)
return
}

return Promise.all(files.map(function (path) {
if (self._isExcluded(path)) {
log.debug('Excluded file "%s"', path)
return Promise.resolve()
}

return Promise.all(files.map(function (path) {
if (self._isExcluded(path)) {
log.debug('Excluded file "%s"', path)
return Promise.resolve()
}

var mtime = mg.statCache[path].mtime
var doNotCache = patternObject.nocache
var file = new File(path, mtime, doNotCache)

if (file.doNotCache) {
log.debug('Not preprocessing "%s" due to nocache')
return Promise.resolve(file)
}

return self._preprocess(file).then(function () {
return file
})
}))
.then(function (files) {
files = _.compact(files)

if (_.isEmpty(files)) {
log.warn('All files matched by "%s" were excluded.', pattern)
} else {
buckets.set(pattern, new Set(files))
}
var mtime = mg.statCache[path].mtime
var doNotCache = patternObject.nocache
var file = new File(path, mtime, doNotCache)

if (file.doNotCache) {
log.debug('Not preprocessing "%s" due to nocache')
return Promise.resolve(file)
}

return self._preprocess(file).then(function () {
return file
})
}))
.then(function (files) {
files = _.compact(files)

if (_.isEmpty(files)) {
log.warn('All files matched by "%s" were excluded.', pattern)
} else {
buckets.set(pattern, new Set(files))
}
})
}))
.cancellable()
Expand Down
10 changes: 0 additions & 10 deletions lib/helper.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ var path = require('path')
var _ = require('lodash')
var useragent = require('useragent')
var Promise = require('bluebird')
var Glob = require('glob').Glob

exports.browserFullNameToShort = function (fullName) {
var agent = useragent.parse(fullName)
Expand Down Expand Up @@ -105,14 +104,5 @@ exports.defer = function () {
}
}

exports.glob = function glob (pattern, opts) {
return new Promise(function (resolve, reject) {
var mg = new Glob(pattern, opts, function (err, res) {
if (err) return reject(err)
resolve([res, mg])
})
})
}

// export lodash
exports._ = _
111 changes: 62 additions & 49 deletions test/unit/file-list.spec.coffee
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,8 @@ mockFs = mocks.fs.create
'c.txt': mocks.fs.file 0
'a.js': mocks.fs.file '2012-01-01'

List = proxyquire('../../lib/file-list', {
helper: helper
fs: mockFs
})


describe 'FileList', ->
list = emitter = preprocess = patternList = mg = modified = null
describe.only 'FileList', ->
List = list = emitter = preprocess = patternList = mg = modified = glob = null

beforeEach ->

Expand All @@ -62,12 +56,14 @@ describe 'FileList', ->
preprocess = sinon.spy((file, done) -> process.nextTick done)
emitter = new EventEmitter()

sinon.stub(helper, 'glob', (pattern, opts) ->
Promise.resolve [patternList[pattern], mg]
)
glob = Glob: (pattern, opts) ->
{found: patternList[pattern], statCache: mg.statCache}

afterEach ->
helper.glob.restore()
List = proxyquire('../../lib/file-list', {
helper: helper
glob: glob
fs: mockFs
})

it 'returns a flat array of served files', ->
list = new List(
Expand Down Expand Up @@ -174,21 +170,25 @@ describe 'FileList', ->
beforeEach ->
preprocess = sinon.spy((file, done) -> process.nextTick done)
emitter = new EventEmitter()

glob = Glob: (pattern, opts) ->
{found: patternList[pattern], statCache: mg.statCache}

List = proxyquire('../../lib/file-list', {
helper: helper
glob: glob
fs: mockFs
})

list = new List(
patterns('/some/*.js', '*.txt'),
[],
emitter,
preprocess
)
sinon.stub(helper, 'glob', (pattern, opts) ->
Promise.resolve [PATTERN_LIST[pattern], MG]
)

list.refresh()

afterEach ->
helper.glob.restore()

it 'returns false when no match is found', ->
expect(list._exists('/some/s.js')).to.be.false
expect(list._exists('/hello/world.ex')).to.be.false
Expand All @@ -203,20 +203,23 @@ describe 'FileList', ->
mg = _.cloneDeep(MG)
preprocess = sinon.spy((file, done) -> process.nextTick done)
emitter = new EventEmitter()

glob = Glob: (pattern, opts) ->
{found: patternList[pattern], statCache: mg.statCache}

List = proxyquire('../../lib/file-list', {
helper: helper
glob: glob
fs: mockFs
})

list = new List(
patterns('/some/*.js', '*.txt'),
[],
emitter,
preprocess
)

sinon.stub(helper, 'glob', (pattern, opts) ->
Promise.resolve [patternList[pattern], mg]
)

afterEach ->
helper.glob.restore()

it 'resolves patterns', ->
list.refresh().then (files) ->
expect(list.buckets.size).to.equal 2
Expand Down Expand Up @@ -326,20 +329,23 @@ describe 'FileList', ->

preprocess = sinon.spy((file, done) -> process.nextTick done)
emitter = new EventEmitter()

glob = Glob: (pattern, opts) ->
{found: patternList[pattern], statCache: mg.statCache}

List = proxyquire('../../lib/file-list', {
helper: helper
glob: glob
fs: mockFs
})

list = new List(
patterns('/some/*.js', '*.txt'),
['/secret/*.txt'],
emitter,
preprocess
)

sinon.stub(helper, 'glob', (pattern, opts) ->
Promise.resolve [patternList[pattern], mg]
)

afterEach ->
helper.glob.restore()

it 'does not add excluded files', ->
list.refresh().then (before) ->
list.addFile('/secret/hello.txt').then (files) ->
Expand Down Expand Up @@ -419,19 +425,23 @@ describe 'FileList', ->

preprocess = sinon.spy((file, done) -> process.nextTick done)
emitter = new EventEmitter()
sinon.stub(helper, 'glob', (pattern, opts) ->
Promise.resolve [patternList[pattern], mg]
)


glob = Glob: (pattern, opts) ->
{found: patternList[pattern], statCache: mg.statCache}

List = proxyquire('../../lib/file-list', {
helper: helper
glob: glob
fs: mockFs
})

mockFs._touchFile '/some/a.js', '2012-04-04'
mockFs._touchFile '/some/b.js', '2012-05-05'

modified = sinon.stub()
emitter.on 'file_list_modified', modified

afterEach ->
helper.glob.restore()

it 'updates mtime and fires "file_list_modified"', ->
# MATCH: /some/a.js, /some/b.js
list = new List(patterns('/some/*.js', '/a.*'), [], emitter, preprocess)
Expand Down Expand Up @@ -484,16 +494,19 @@ describe 'FileList', ->

preprocess = sinon.spy((file, done) -> process.nextTick done)
emitter = new EventEmitter()
sinon.stub(helper, 'glob', (pattern, opts) ->
Promise.resolve [patternList[pattern], mg]
)

glob = Glob: (pattern, opts) ->
{found: patternList[pattern], statCache: mg.statCache}

List = proxyquire('../../lib/file-list', {
helper: helper
glob: glob
fs: mockFs
})

modified = sinon.stub()
emitter.on 'file_list_modified', modified

afterEach ->
helper.glob.restore()

it 'removes the file from the list and fires "file_list_modified"', ->
# MATCH: /some/a.js, /some/b.js, /a.txt
list = new List(patterns('/some/*.js', '/a.*'), [], emitter, preprocess)
Expand Down Expand Up @@ -525,9 +538,9 @@ describe 'FileList', ->

preprocess = sinon.spy((file, done) -> process.nextTick done)
emitter = new EventEmitter()
sinon.stub(helper, 'glob', (pattern, opts) ->
Promise.resolve [patternList[pattern], mg]
)

glob = Glob: (pattern, opts) ->
{found: patternList[pattern], statCache: mg.statCache}

modified = sinon.stub()
emitter.on 'file_list_modified', modified
Expand All @@ -538,11 +551,11 @@ describe 'FileList', ->
helper._ = _.runInContext()
List = proxyquire('../../lib/file-list', {
helper: helper
glob: glob
fs: mockFs
})

afterEach ->
helper.glob.restore()
clock.restore()

it 'batches multiple changes within an interval', (done) ->
Expand Down

0 comments on commit 1b65cde

Please sign in to comment.