Improve storage ability

1. Fix bugs in loading/saving items 2. Add image buffer for efficient storage of pictures 3. replace old MNIST dataset with new one that we generates
suquark · Feb 22, 2017 · 659c9bb · 659c9bb
1 parent bc62cfd
commit 659c9bb
Show file tree

Hide file tree

Showing 11 changed files with 215 additions and 122 deletions.
diff --git a/util/buffer.js → backend/buffer.js b/util/buffer.js → backend/buffer.js
@@ -1,4 +1,5 @@
 import { getBinary, exportBinary } from 'util/request.js';
+import { checkTypeStrict } from 'util/assert.js';
 
 // FIXME: not very efficient
 function bufCopy(dst, src, start=0) {
@@ -18,20 +19,36 @@ class Buffer {
     align(len) {
         let new_offset = Math.ceil(this.offset / len) | 0;
         this.offset = new_offset * len;
-        return new_offset;
+        return this.offset;
     }
 
     read(count, type) {
         var value;
         switch (type) {
+            case 'Uint8Array':
+                value = new Uint8Array(this.buffer, this.offset, count);
+                this.offset += count;
+                break;
+            case 'Uint8ClampedArray':
+                value = new Uint8ClampedArray(this.buffer, this.offset, count);
+                this.offset += count;
+                break;
             case 'Int8Array':
                 value = new Int8Array(this.buffer, this.offset, count);
                 this.offset += count;
                 break;
+            case 'Uint16Array':
+                value = new Uint16Array(this.buffer, this.align(2), count);
+                this.offset += 2 * count;
+                break;
             case 'Int16Array':
                 value = new Int16Array(this.buffer, this.align(2), count);
                 this.offset += 2 * count;
                 break;
+            case 'Uint32Array':
+                value = new Uint32Array(this.buffer, this.align(4), count);
+                this.offset += 4 * count;
+                break;
             case 'Int32Array':
                 value = new Int32Array(this.buffer, this.align(4), count);
                 this.offset += 4 * count;
@@ -44,6 +61,8 @@ class Buffer {
                 value = new Float32Array(this.buffer, this.align(8), count);
                 this.offset += 8 * count;
                 break;
+            default:
+                throw 'unexpected type';
         }
         return value;
     }
@@ -60,6 +79,7 @@ class Buffer {
     }
 
     write(buffer) {
+        if (!checkTypeStrict(buffer, 'ArrayBuffer')) buffer = buffer.slice().buffer;
         if (this.offset + buffer.byteLength > this.byteLength) {
             this._refit(this.offset + buffer.byteLength);
         }
@@ -75,6 +95,13 @@ class Buffer {
         return this.buffer.byteLength;
     }
 
+    // Try to load typed array
+    static load(v, buf) {
+        return buf.read(v.length, v.type);
+    }
+
+    //getNativeType
+
     static fromURL(url) {
         return getBinary(url).then(buf => new Buffer(buf));
     }

diff --git a/backend/image.js b/backend/image.js
@@ -0,0 +1,70 @@
+import { Tensor } from 'backend/tensor.js';
+import { assert } from 'util/assert.js';
+import { clip_pixel, scale_shift } from 'backend/ops.js';
+
+/**
+ * This class provides a more uniform and general way to store and save images with same shape
+ */
+class ImageBuffer {
+    constructor() {}
+
+    /**
+     * @param { Array } tsArray - Array of Tensor
+     */
+    static fromTensors(tsArray) {
+        assert(tsArray.length > 0, 'empty array');
+        // we assume that all images have the same shape
+        let ib = new ImageBuffer();
+        ib.shape = tsArray[0].shape;
+        ib.images = [];
+        let size = tsArray[0].size;
+        for (let i of tsArray) {
+            let x = i.clone();
+            clip_pixel(x);
+            let ba = new Uint8ClampedArray(size); 
+            for (let j = 0; j < size; j++) {
+                ba[j] = x.w[j];
+            }
+            ib.images.push(ba);
+        }
+        return ib;
+    }
+
+    get tensors() {
+        return this.images.map(b => {
+            let t = new Tensor(this.shape);
+            let N = t.size;
+            for (let i = 0; i < N; i++) {
+                t.w[i] = (b[i] / 255.0) * 2.0 - 1.0;
+            }
+            return t;
+        });
+    }
+
+    /**
+     * Load images from buffer
+     * @param { object } t - Object that contains info about tensor we want to get
+     * @param { BufferReader } buf - The ArrayBuffer contains data
+     * @return { ImageBuffer } - the loaded images
+     */
+    static load(map, buf) {
+        let ib = new ImageBuffer();
+        ib.name = map.name;
+        ib.shape = map.shape;
+        ib.images = [];
+
+        let size = ib.shape[0] * ib.shape[1] * ib.shape[2];  // FIXME: may a more general way?
+        // OK, load values from buffer
+        for (let i = 0; i < map.count; i++) {
+            ib.images.push(buf.read(size, 'Uint8ClampedArray'));
+        }
+        return ib;
+    }
+
+    save(buf) {
+        for (let i of this.images) buf.write(i);
+        return {name: this.name, shape: this.shape, count: this.images.length, type: 'images'};
+    }
+}
+
+export { ImageBuffer };
diff --git a/backend/ops.js b/backend/ops.js
@@ -18,6 +18,8 @@ function normal_rand(t, mu, std) {
     scale_shift(t, std, mu);
 }
 
+
+
 function clip(x, min_value=-1.0, max_value=1.0) {
     let N = x.size, w = x.w;
     for (let i = 0; i < N; i++) {
@@ -30,7 +32,7 @@ function clip_pixel(x) {
     let N = x.size, w = x.w;
     for (let i = 0; i < N; i++) {
         let wi = w[i];
-        if (wi >= 1.0) w[i] = 255; else if (wi <= -1.0) w[i] = 0; else w[i] = Math.round(255 * (x + 1.0) / 2.0) | 0;
+        if (wi >= 1.0) w[i] = 255; else if (wi <= -1.0) w[i] = 0; else w[i] = Math.round(255 * (wi + 1.0) / 2.0) | 0;
     }
 }
 
@@ -96,6 +98,13 @@ function TensorConstantProduct(x, c) {
 }
 
 
+function negative(x) {
+    let N = x.size, xw = x.w;
+    for (let i = 0; i < N; i++) {
+         xw[i] = -xw[i];
+    }
+}
+
 function scale(x, c) {
     let N = x.size, xw = x.w;
     for (let i = 0; i < N; i++) {
@@ -132,7 +141,7 @@ function HadmardProductAssign(o, x) {
 
 export {
     clip, clip_pixel,
-    scale, shift, scale_shift,
+    scale, shift, scale_shift, negative,
     TensorVectorProduct, TransposedTensorVectorAddAssign, HadmardProductAssign,
     uniform_rand, normal_rand
 };
diff --git a/backend/symbols.js b/backend/symbols.js
@@ -7,8 +7,10 @@
  */
 
 import { Tensor } from 'backend/tensor.js';
-import { Buffer } from 'util/buffer.js';
+import { Buffer } from 'backend/buffer.js';
+import { ImageBuffer } from 'backend/image.js';
 import { getJSON, exportJSON } from 'util/request.js';
+import { getNativeType } from 'util/assert.js';
 var globals = {};
 
 
@@ -24,7 +26,6 @@ function fetch(abspath) {
     return fetchFrom(globals, abspath);
 }
 
-
 /**
  * Fetch object within certain context by path
  * @param {object} dir - top directory to find with
@@ -40,6 +41,46 @@ function fetchFrom(dir, path) {
 }
 
 
+/**
+ * Fetch object by path
+ * @param {string} abspath - Absolute path for object
+ * @returns {object} - object we get
+ */
+function store(abspath, value) {
+    if (abspath.startsWith('/')) {
+        abspath = abspath.substring(1);  // OK, unix-style is just right
+    }
+    return storeAt(globals, abspath, value);
+}
+
+/**
+ * Fetch object within certain context by path
+ * @param {object} dir - top directory to find with
+ * @param {string} path - path to object
+ * @returns {object} - a directory or a value
+ */
+function storeAt(dir, path, value) {
+    if (typeof dir == 'undefined') throw "unexpected error - given a bad object for looking-up?";
+    if (path === '') return;  // a directory exists. do nothing
+    let idx = path.indexOf('/') + 1;
+    if (idx === 0) {
+        // OK, store value just where
+        dir[path] = value;
+        return;  // never going down!
+    }
+
+    let dirname = path.substring(0, idx);
+    if (typeof dir[dirname] === 'undefined') {
+        // we build the path for user
+        if (dirname.endsWith('[]/')) {
+            dir[dirname] = [];
+        } else if (dirname.endsWith('/')) {
+            dir[dirname] = {};
+        }
+    }
+    return storeAt(dir[dirname], path.substring(idx), value);
+}
+
 
 //////  Load Part  //////
 
@@ -52,16 +93,7 @@ function loadFile2Global(mapfile, rawfile) {
     // create a pair of file access
     let task_pair = [getJSON(mapfile), Buffer.fromURL(rawfile)];
     // wait for both of then to be done and return a `Promise`
-    return Promise.all(task_pair).then(pair => new Promise((resolve, reject) => {
-        if (pair && pair.length == 2) {
-            load2Global(pair[0], pair[1]);
-            resolve();
-        } else {
-            reject(new Error("Bad pair of mapfile & rawfile"));
-        }
-    })).catch(error => {
-        reject(error);
-    });
+    return Promise.all(task_pair).then(pair => load2Global(pair[0], pair[1]));
 }
 
 
@@ -134,10 +166,10 @@ function _loadValue(v, buf) {
     {
         case 'tensor':
             return Tensor.load(v, buf);
-        // case 'int32array':
-        //     return 
+        case 'images':
+            return ImageBuffer.load(v, buf);
         default:
-            throw "Undefined type";
+            return Buffer.load(v, buf);
     }
 }
 
@@ -177,19 +209,29 @@ function _saveDir(dir, maplist, buf) {
             _saveDir(dir[name], packet.nodes, buf);
         } else {
             packet = _saveValue(dir[name], buf);
+            packet.name = name;  // override name
         }
         maplist.push(packet);
     }
 }
 
 function _saveValue(value, buf) {
+    // we use the simple way at present. 
     return value.save(buf)
 }
 
 
+// never use arrow function here!!!
+Object.getPrototypeOf(Int8Array.prototype).save = function(buf) {
+    // really ... hacking ways
+    buf.write(this);
+    return {type: getNativeType(this), name: this.name, length: this.length};
+}
+
+
 export { 
     globals,
-    fetch, fetchFrom,
+    fetch, fetchFrom, store, storeAt,
     loadFile2Global, batchLoadFile2Global,
     saveGlobal, saveDict
 };
diff --git a/backend/tensor.js b/backend/tensor.js
@@ -142,7 +142,7 @@ class Tensor {
     }
 
     save(buf) {
-        buf.write(this.buffer);
+        buf.write(this.w);
         return {name: this.name, shape: this.shape, type: 'tensor'};
     }
 

diff --git a/dataset/mnist2000.json b/dataset/mnist2000.json
@@ -0,0 +1 @@
+[{"name":"mnist/","nodes":[{"name":"x","shape":[28,28,1],"count":2000,"type":"images"},{"type":"Int32Array","name":"y","length":2000}]}]
diff --git a/dataset/mnist2000.raw b/dataset/mnist2000.raw
diff --git a/dataset/mnist_2000.js b/dataset/mnist_2000.js
diff --git a/dataset/mnist_raw2000.js b/dataset/mnist_raw2000.js
diff --git a/util/assert.js b/util/assert.js
@@ -16,6 +16,17 @@ function checkClass(instance, match) {
     return Object.prototype.toString.call(instance).match(match);
 }
 
+function checkTypeStrict(instance, name) {
+    return Object.prototype.toString.call(instance) === `[object ${name}]`;
+}
+
+function getNativeType(instance) {
+    // [object ***]
+    let typestr = Object.prototype.toString.call(instance);
+    assert(typestr.startsWith('[object ') && typestr.endsWith(']'), 'not a native type');
+    return typestr.substring(8, typestr.length - 1);
+}
+
 function isArray(arr) {
     return checkClass(arr, 'Array') && typeof arr.length === 'number' && arr.length === Math.floor(arr.length);
 }
@@ -36,4 +47,4 @@ function assertArray2D(array) {
     return [array.length, array[0].length];
 }
 
-export { assert, checkClass, assertSquare, assertArray2D, isArray };
+export { assert, checkClass, checkTypeStrict, getNativeType, assertSquare, assertArray2D, isArray };
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		[{"name":"mnist/","nodes":[{"name":"x","shape":[28,28,1],"count":2000,"type":"images"},{"type":"Int32Array","name":"y","length":2000}]}]