-
Notifications
You must be signed in to change notification settings - Fork 2
/
data.lua
executable file
·51 lines (49 loc) · 1.89 KB
/
data.lua
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
-- Copyright 2018 Joel Janai, Fatma Güney, Anurag Ranjan and the Max Planck Gesellschaft.
-- All rights reserved.
-- This software is provided for research purposes only.
-- By using this software you agree to the terms of the license file
-- in the root folder.
-- For commercial use, please contact [email protected].
--
-- Copyright (c) 2014, Facebook, Inc.
-- All rights reserved.
--
-- This source code is licensed under the BSD-style license found in the
-- LICENSE file in the root directory of this source tree. An additional grant
-- of patent rights can be found in the PATENTS file in the same directory.
--
local ffi = require 'ffi'
local Threads = require 'threads'
Threads.serialization('threads.sharedserialize')
-- This script contains the logic to create K threads for parallel data-loading.
-- For the data-loading details, look at donkey.lua
-------------------------------------------------------------------------------
do -- start K datathreads (donkeys)
if opt.nDonkeys > 0 then
local options = opt -- make an upvalue to serialize over to donkey threads
donkeys = Threads(
opt.nDonkeys,
function()
require 'torch'
end,
function(idx)
opt = options -- pass to all donkeys via upvalue
tid = idx
local seed = opt.manualSeed + idx
torch.manualSeed(seed)
print(string.format('Starting donkey with id: %d seed: %d', tid, seed))
paths.dofile('donkey.lua')
end
);
else -- single threaded data loading. useful for debugging
paths.dofile('donkey.lua')
donkeys = {}
function donkeys:addjob(f1, f2) f2(f1()) end
function donkeys:synchronize() end
end
end
nTest = 0
donkeys:addjob(function() return testLoader:size() end, function(c) nTest = c end)
donkeys:synchronize()
assert(nTest > 0, "Failed to get nTest")
print('nTest: ', nTest)