Improve symbol loading system & add async request module

1. Improve symbol loading system. 2. Fix a bug in transposed tensor product 3. add async request module
suquark · Feb 20, 2017 · c781934 · c781934
1 parent fc24e0e
commit c781934
Show file tree

Hide file tree

Showing 7 changed files with 195 additions and 66 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,2 +1,4 @@
 test/
 tags
+
+.vscode/launch.json
diff --git a/backend/ops.js b/backend/ops.js
@@ -21,8 +21,10 @@ function TensorVectorProduct(ov, m, v) {
 
 /**
  * ov += m' * v
+ * @param { Tensor } m - tensor to be transposed
+ * @param { Tensor } v - right-hand-side tensor
  */
-function TransposedTensorVectorProductAdd(ov, m, v) {
+function TransposedTensorVectorAddAssign(ov, m, v) {
     let ncol = m.axis(-1) | 0;  
     let nrow = m.axis(-2) | 0;
     let new_shape = m.shape.slice();
@@ -41,7 +43,7 @@ function TransposedTensorVectorProductAdd(ov, m, v) {
         for (let i = 0; i < nrow; i++) {
             for (let j = 0; j < ncol; j++) {
                 // transposed order
-                ow[z * ncol + j] += mw[z * bs + i * ncol + j] * vw[i];
+                ow[z * ncol + j] += mw[z * bs + i * ncol + j] * vw[j];
             }
         }
     }
@@ -70,4 +72,4 @@ function HadmardProductAssign(o, x) {
     }
 }
 
-export { TensorConstantProduct, TensorVectorProduct, TransposedTensorVectorProductAdd, HadmardProductAssign };
+export { TensorConstantProduct, TensorVectorProduct, TransposedTensorVectorAddAssign, HadmardProductAssign };
diff --git a/backend/symbols.js b/backend/symbols.js
@@ -1,34 +1,110 @@
+/**
+ * 
+ * This module manages global symbols
+ * 
+ * A filesystem like scheme is designed for mapping data between memory and workspace 
+ * 
+ */
+
 import { Tensor } from 'backend/tensor.js';
+import { getBinary, getJSON } from 'util/request.js';
 var globals = {}
 
-function loadFile2Global(mapfile, rawfile, callback) {
-    $.getJSON(mapfile, (map) => {
-        $.getBinary(rawfile, (buf) => {
-            load2Global(map, buf);
-            callback();
-        });
+/**
+ * Fetch object by path
+ * @param {string} abspath - Absolute path for object
+ */
+function fetch(abspath) {
+    if (abspath.startsWith('/')) {
+        abspath = abspath.substring(1);  // OK, unix-style is just right
+    }
+    return fetchFrom(globals, abspath);
+}
+
+/**
+ * Fetch object within certain context by path
+ * @param {object} dir - top directory to find with
+ * @param {string} path - path to object
+ * @returns {object} - a directory or a value
+ */
+function fetchFrom(dir, path) {
+    if (!dir) throw "Path not found";
+    if (path === '') return dir;  // return a directory
+    let idx = path.indexOf('/') + 1;
+    if (idx === 0) return dir[path];  // return value
+    return fetchFrom(dir[path.substring(0, idx)], path.substring(idx));
+}
+
+function batchLoadFile2Global(pairs, callback) {
+    return Promise.all(pairs.map(p => loadFile2Global(p[0], p[1])));
+}
+
+function loadFile2Global(mapfile, rawfile) {
+    // create a pair of file access
+    let task_pair = [getJSON(mapfile), getBinary(rawfile)];
+    // wait for both of then to be done and return a `Promise`
+    return Promise.all(task_pair).then(pair => new Promise((resolve, reject) => {
+        if (pair && pair.length == 2) {
+            load2Global(pair[0], pair[1]);
+            resolve();
+        } else {
+            reject(new Error("Bad pair of mapfile & rawfile"));
+        }
+    })).catch(error => {
+        reject(error);
     });
 }
 
+/**
+ * load data in the buffer into global object by instruction of the map
+ * @param { object } dir - current treenode where we construct our data 
+ * @param { object } map - The map that tells the structure of data 
+ */
 function load2Global(map, buf) {
-    let ns = {};
-    let m = map.mapping;
-    let offset = 0;
-    for (let i in m) {
-        let t = m[i];
-        let length = _getVol(t.shape);
-        ns[t.id] = new Tensor(t.shape, new Float32Array(buf, offset, length));
-        offset += length;
+    return _loadDir(globals, map, buf, 0);
+}
+
+/**
+ * @param { object } dir - current treenode where we construct our data 
+ * @param { object } map - The map that tells the structure of data 
+ * @param { ArrayBuffer } buf - The ArrayBuffer contains data
+ * @param { number } offset - where should we point the reader to. offset is passed between functions, not as a global value to avoid async side-effects
+ */
+function _loadDir(dir, map, buf, offset) {
+    for (let i in map) {
+        let item = map[i];
+        let name = item.name;
+        if (item.nodes) {
+            dir[name] = {};
+            offset = _loadDir(dir[name], item.nodes, buf, offset);
+        } else {
+            [offset, dir[name]] = _loadValue(buf, offset, item);
+        }
+    }
+    return length;
+}
+
+
+function _loadValue(buf, offset, v) {
+    switch (v.type)
+    {
+        case 'tensor':
+            return _loadTensor(buf, offset, v);
+        default:
+            throw "Undefined type";
     }
-    globals[m.namespace] = ns;
 }
 
-function _getVol(shape) {
-    let m = 1;
-    for (let i = 0; i < shape.length; i++) {
-        m *= shape[i];
+function _loadTensor(buf, offset, t) {
+    let length = 1;
+    for (let i = 0; i < t.shape.length; i++) {
+        length *= t.shape[i];
     }
-    return m;
+    return [offset + length, new Tensor(t.shape, new Float32Array(buf, offset, length))];
 }
 
-export { loadFile2Global, globals };
+export { 
+    globals,
+    fetch, fetchFrom,
+    loadFile2Global, batchLoadFile2Global
+};
diff --git a/backend/tensor.js b/backend/tensor.js
@@ -54,6 +54,10 @@ class Tensor {
         return this._shape;
     }
 
+    get size() {
+        return this._size;
+    }
+
     /**
      * length of pixel
      */
@@ -133,10 +137,7 @@ class Tensor {
     cloneAndZero() { return new Tensor(this.shape); }
 
     clone() {
-        var V = new Tensor(this.shape);
-        var n = this.w.length;
-        V.w = this.w.slice();
-        return V;
+        return new Tensor(this.shape, this.w.slice());
     }
 
     zeros_like() {
@@ -164,7 +165,6 @@ class Vector extends Tensor {
     }
 }
 
-
 class Placeholder {
     constructor(shape=[null]) {
         this._shape = shape;

diff --git a/util.js b/util.js
@@ -28,6 +28,7 @@ function normalize_angle(angle) {
     return nangle;
 }
 
+
 class AvgWindow {
     // a window stores _size_ number of values
     // and returns averages. Useful for keeping running
@@ -54,24 +55,7 @@ class AvgWindow {
     }
 }
 
-// Load JSON text from server hosted file and return JSON parsed object
-function loadJSONSync(filePath) {
-  // Load json file;
-  var json = loadTextFileSync(filePath, "application/json");
-  // Parse json
-  return JSON.parse(json);
-}
 
-function loadTextFileSync(filePath, mimeType)
-{
-    var xmlhttp = new XMLHttpRequest();
-    xmlhttp.open("GET", filePath, false);
-    if (mimeType != null && xmlhttp.overrideMimeType) {
-        xmlhttp.overrideMimeType(mimeType);
-    }
-    xmlhttp.send();
-    return xmlhttp.status == 200 ? xmlhttp.responseText : null;
-}
 
 function b64ToUint6 (nChr) {
   return nChr > 64 && nChr < 91 ? nChr - 65 : 
@@ -99,24 +83,9 @@ function base64DecToArr (sBase64, nBlocksSize) {
   return taBytes;
 }
 
-if (!$) $ = {};
-
-$.getBinary = function(url, callback) {
-    var oReq = new XMLHttpRequest();
-    oReq.open("GET", url, true);
-    oReq.responseType = "arraybuffer";
-    oReq.onload = function (oEvent) {
-        var arrayBuffer = oReq.response; // Note: not oReq.responseText
-        callback(arrayBuffer);
-    };
-    oReq.send(null);
-}
-
-
 
 export {
-    getopt, clip, normalize_angle, AvgWindow,
-    loadJSONSync, loadTextFileSync
+    getopt, clip, normalize_angle, AvgWindow
 };
 
 export * from 'util/random.js';

diff --git a/util/request.js b/util/request.js
@@ -0,0 +1,80 @@
+/**
+ * This is the request module for web browser
+ * All async function are rewritten in `Promise` style
+ */
+
+function getJSON(url, async=true) {
+    if (async) {
+        var promise = new Promise(function(resolve, reject) {
+            getText(url, "application/json").then((json) => {
+                resolve(JSON.parse(json));
+            }).catch((error) => {
+                reject(error);
+            });
+        });
+        return promise;
+    } else {
+        var json = getTextSync(url, "application/json", false);
+        return JSON.parse(json);
+    }
+}
+
+function getBinary(url) {
+    var promise = new Promise(function(resolve, reject) {
+        var client = new XMLHttpRequest();
+        client.open("GET", url, true);
+        client.responseType = "arraybuffer";
+        client.onload = function (oEvent) {
+            if (client.readyState !== 4) {
+                return;
+            }
+            if (client.status === 200) {
+                resolve(client.response);
+            } else {
+                reject(new Error(client.statusText));
+            }
+        };
+        client.send();
+    });
+    return promise;
+}
+
+
+function getText(url, mimeType, async=true)
+{
+    var client = new XMLHttpRequest();
+    if (async) {
+        var promise = new Promise(function(resolve, reject){
+            var client = new XMLHttpRequest();
+            if (mimeType != null && client.overrideMimeType) {
+                client.overrideMimeType(mimeType);
+            }
+            client.open("GET", url);
+            client.onreadystatechange = () => {
+                if (client.readyState !== 4) {
+                    return;
+                }
+                if (client.status === 200) {
+                    resolve(client.response);
+                } else {
+                    reject(new Error(client.statusText));
+                }
+            };
+            client.send();
+        });
+        return promise;
+    } else {
+        client.open("GET", url, false);
+        if (mimeType != null && client.overrideMimeType) {
+            client.overrideMimeType(mimeType);
+        }
+        client.send();
+        if (client.status === 200) {
+            return client.responseText;
+        } else {
+            throw client.statusText;
+        }
+    }
+}
+
+export { getJSON, getBinary, getText };
diff --git a/visualize/input.js b/visualize/input.js
@@ -1,11 +1,11 @@
 import { Vector } from 'vol.js';
 
-function sliders2Vector(sliders, prep=x=>x) {
-    let arr = sliders.map(s => prep(s.value));
-    let N = sliders.length;
+function elements2Vector(elements, prep=x=>x.value) {
+    let arr = elements.map(prep);
+    let N = elements.length;
     let v = new Vector(N);  // TODO: after change into tensor, use `new Vector(N, arr)` instead
     for (let i = 0; i < N; i++) v.w[i] = arr[i];
     return v;
 }
 
-export { sliders2Vector };
+export { elements2Vector };