iOS (#109)

* ios * merge * Merge remote-tracking branch 'origin/master' into ios * ios * its working...? haha, not. * improve precision * fix accuracy on iOS * fix the faster / slightly less precise version * slight speedup * fix numerical issues on ios. (use resultUV instead of gl_FragCoord, and highp int) * merge master * actually merge * flag guard byte textures * Merge remote-tracking branch 'origin' into ios * Merge remote-tracking branch 'origin' into ios * more changes * merge * start pulling tests apart * tests * merge * get remaining tests to pass * Merge remote-tracking branch 'origin' into ios * remove comments, remove imagenet util change * imagenet * test_util commits * ndarray tests * test_util blank space * softmax underflow on mac, copy gpu test revert * revert _gpu_tests * remove console.log * fix lint errors * respond to comments
tensorflow · Oct 9, 2017 · f164662 · f164662
1 parent 61cdbe9
commit f164662
Show file tree

Hide file tree

Showing 34 changed files with 4,777 additions and 2,181 deletions.
diff --git a/demos/mnist/mnist.ts b/demos/mnist/mnist.ts
@@ -39,7 +39,7 @@ reader.getAllVariables().then(vars => {
         const probsVal = sess.eval(probs, [{tensor: input, data: inputData}]);
         console.log(`Item ${i}, probsVal ${probsVal.get()}.`);
         const label = data.labels[i];
-        const predictedLabel = probsVal.get();
+        const predictedLabel = Math.round(probsVal.get());
         if (label === predictedLabel) {
           numCorrect++;
         }
@@ -79,12 +79,10 @@ export function buildModelMathAPI(
 
   return (x: Array1D): Scalar => {
     return math.scope(() => {
-      const hidden1 =
-          math.relu(math.add(math.vectorTimesMatrix(x, hidden1W), hidden1B)) as
-          Array1D;
-      const hidden2 =
-          math.relu(math.add(
-              math.vectorTimesMatrix(hidden1, hidden2W), hidden2B)) as Array1D;
+      const hidden1 = math.relu(
+          math.add(math.vectorTimesMatrix(x, hidden1W), hidden1B)) as Array1D;
+      const hidden2 = math.relu(math.add(
+          math.vectorTimesMatrix(hidden1, hidden2W), hidden2B)) as Array1D;
       const logits =
           math.add(math.vectorTimesMatrix(hidden2, softmaxW), softmaxB);
       return math.argMax(logits);

diff --git a/demos/one_plus_one/one_plus_one.ts b/demos/one_plus_one/one_plus_one.ts
@@ -16,40 +16,12 @@
  */
 
 // tslint:disable-next-line:max-line-length
-import {Graph, NDArrayMath, NDArrayMathGPU, Scalar, Session, Tensor} from '../deeplearn';
+import {NDArrayMathGPU, Scalar} from '../deeplearn';
 
-class Adder {
-  inputTensorA: Tensor;
-  inputTensorB: Tensor;
-  sum: Tensor;
-  session: Session;
-  math: NDArrayMath = new NDArrayMathGPU();
-  setupSession(): void {
-    const graph = new Graph();
+const math = new NDArrayMathGPU();
+const a = Scalar.new(1);
+const b = Scalar.new(1);
 
-    this.inputTensorA = graph.placeholder('A', []);
-    this.inputTensorB = graph.placeholder('B', []);
-    this.sum = graph.add(this.inputTensorA, this.inputTensorB);
-    this.session = new Session(graph, this.math);
-  }
+const result = math.add(a, b).get();
 
-  computeSum(a: number, b: number): number {
-    const feeds = [
-      {tensor: this.inputTensorA, data: Scalar.new(a)},
-      {tensor: this.inputTensorB, data: Scalar.new(b)}
-    ];
-    let result;
-    this.math.scope(() => {
-      result = this.session.eval(this.sum, feeds).get();
-    });
-    return result;
-  }
-}
-
-const adder = new Adder();
-adder.setupSession();
-const result = adder.computeSum(1, 1);
-
-const outputEl = document.getElementById('output');
-if (!outputEl) throw new Error('output element not found');
-outputEl.innerText = String(result);
+document.getElementById('output').innerText = '' + result;
diff --git a/src/environment.ts b/src/environment.ts
@@ -31,12 +31,16 @@ export interface Features {
   'WEBGL_DISJOINT_QUERY_TIMER_EXTENSION_RELIABLE'?: boolean;
   // 0: No WebGL, 1: WebGL 1.0, 2: WebGL 2.0.
   'WEBGL_VERSION'?: number;
+  // Whether writing & reading floating point textures is enabled. When
+  // false, fall back to using unsigned byte textures.
+  'WEBGL_FLOAT_TEXTURE_ENABLED'?: boolean;
 }
 
 export const URL_PROPERTIES: URLProperty[] = [
   {name: 'WEBGL_DISJOINT_QUERY_TIMER_EXTENSION_ENABLED', type: Type.BOOLEAN},
   {name: 'WEBGL_DISJOINT_QUERY_TIMER_EXTENSION_RELIABLE', type: Type.BOOLEAN},
-  {name: 'WEBGL_VERSION', type: Type.NUMBER}
+  {name: 'WEBGL_VERSION', type: Type.NUMBER},
+  {name: 'WEBGL_FLOAT_TEXTURE_ENABLED', type: Type.BOOLEAN}
 ];
 
 export interface URLProperty {
@@ -91,6 +95,37 @@ function isWebGLDisjointQueryTimerEnabled(webGLVersion: number) {
   return isExtEnabled;
 }
 
+function isFloatTextureReadPixelsEnabled(webGLVersion: number): boolean {
+  if (webGLVersion === 0) {
+    return false;
+  }
+
+  if (webGLVersion === 2) {
+    // WebGL 2 has floating point textures enabled by default.
+    return true;
+  }
+
+  const gl = getWebGLRenderingContext(webGLVersion);
+  gl.getExtension('OES_texture_float');
+  gl.getExtension('WEBGL_color_buffer_float');
+
+  const frameBuffer = gl.createFramebuffer();
+  const texture = gl.createTexture();
+
+  gl.bindTexture(gl.TEXTURE_2D, texture);
+  gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 1, 1, 0, gl.RGBA, gl.FLOAT, null);
+  gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
+  gl.framebufferTexture2D(
+      gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0);
+
+  const frameBufferComplete =
+      (gl.checkFramebufferStatus(gl.FRAMEBUFFER) === gl.FRAMEBUFFER_COMPLETE);
+
+  loseContext(gl);
+
+  return frameBufferComplete;
+}
+
 export class Environment {
   private features: Features = {};
 
@@ -129,6 +164,8 @@ export class Environment {
         return 1;
       }
       return 0;
+    } else if (feature === 'WEBGL_FLOAT_TEXTURE_ENABLED') {
+      return isFloatTextureReadPixelsEnabled(this.get('WEBGL_VERSION'));
     }
     throw new Error(`Unknown feature ${feature}.`);
   }
@@ -139,7 +176,7 @@ const DEEPLEARNJS_FLAGS_PREFIX = 'dljsflags';
 function getFeaturesFromURL(): Features {
   const features: Features = {};
 
-  if(typeof window === 'undefined') {
+  if (typeof window === 'undefined') {
     return features;
   }
 

diff --git a/src/graph/ops/argmax_test.ts b/src/graph/ops/argmax_test.ts
@@ -17,6 +17,7 @@
 
 import {NDArrayMathCPU} from '../../math/math_cpu';
 import {Array1D, Array2D} from '../../math/ndarray';
+import * as test_util from '../../test_util';
 import {Tensor} from '../graph';
 import {TensorArrayMap} from '../tensor_array_map';
 
@@ -50,7 +51,7 @@ describe('Argmax oper', () => {
     const yVal = tensorArrayMap.get(y);
 
     expect(yVal.shape).toEqual([]);
-    expect(yVal.get()).toEqual(1);
+    test_util.expectNumbersClose(yVal.get(), 1);
   });
 
   it('argmax of Array2D', () => {
@@ -64,6 +65,6 @@ describe('Argmax oper', () => {
     const yVal = tensorArrayMap.get(y);
 
     expect(yVal.shape).toEqual([]);
-    expect(yVal.get()).toEqual(4);
+    test_util.expectNumbersClose(yVal.get(), 4);
   });
 });
diff --git a/src/index.ts b/src/index.ts
@@ -16,6 +16,7 @@
  */
 
 import * as xhr_dataset from './data/xhr-dataset';
+import * as environment from './environment';
 import * as conv_util from './math/conv_util';
 import * as gpgpu_util from './math/webgl/gpgpu_util';
 import * as render_ndarray_gpu_util from './math/webgl/render_ndarray_gpu_util';
@@ -28,7 +29,7 @@ export {DataStats, InMemoryDataset} from './data/dataset';
 // tslint:disable-next-line:max-line-length
 export {InCPUMemoryShuffledInputProviderBuilder, InGPUMemoryShuffledInputProviderBuilder, InputProvider} from './data/input_provider';
 export {XhrDataset, XhrDatasetConfig, XhrModelConfig} from './data/xhr-dataset';
-export {ENV, Features} from './environment';
+export {ENV, Environment, Features} from './environment';
 export {Graph, Tensor} from './graph/graph';
 export {AdadeltaOptimizer} from './graph/optimizers/adadelta_optimizer';
 export {AdagradOptimizer} from './graph/optimizers/adagrad_optimizer';
@@ -51,6 +52,7 @@ export {GPGPUContext} from './math/webgl/gpgpu_context';
 // Second level exports.
 export {
   conv_util,
+  environment,
   gpgpu_util,
   render_ndarray_gpu_util,
   test_util,

diff --git a/src/math/batchnorm_test.ts b/src/math/batchnorm_test.ts
@@ -0,0 +1,191 @@
+/**
+ * @license
+ * Copyright 2017 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+
+import * as test_util from '../test_util';
+import {MathTests} from '../test_util';
+
+import {Array1D, Array3D} from './ndarray';
+
+// math.batchNormalization3D
+{
+  // TODO(nsthorat): Fix the precision for byte-packed batchnorm.
+  const epsilon = 1e-1;
+  const tests: MathTests = it => {
+    it('simple batchnorm, no offset or scale, 2x1x2', math => {
+      const x = Array3D.new([2, 1, 2], new Float32Array([2, 100, 4, 400]));
+      const mean = Array1D.new([1, 2]);
+      const variance = Array1D.new([2, 3]);
+      const varianceEpsilon = .001;
+
+      const result = math.batchNormalization3D(
+          x, mean, variance, varianceEpsilon, undefined, undefined);
+
+      test_util.expectArraysClose(
+          result.getValues(), new Float32Array([
+            (x.get(0, 0, 0) - mean.get(0)) * 1 /
+                Math.sqrt(variance.get(0) + varianceEpsilon),
+            (x.get(0, 0, 1) - mean.get(1)) * 1 /
+                Math.sqrt(variance.get(1) + varianceEpsilon),
+            (x.get(1, 0, 0) - mean.get(0)) * 1 /
+                Math.sqrt(variance.get(0) + varianceEpsilon),
+            (x.get(1, 0, 1) - mean.get(1)) * 1 /
+                Math.sqrt(variance.get(1) + varianceEpsilon)
+          ]),
+          epsilon);
+
+      x.dispose();
+      mean.dispose();
+      variance.dispose();
+    });
+
+    it('simple batchnorm, no offset, 2x1x2', math => {
+      const x = Array3D.new([2, 1, 2], new Float32Array([2, 100, 4, 400]));
+      const mean = Array1D.new([1, 2]);
+      const variance = Array1D.new([2, 3]);
+      const scale = Array1D.new([4, 5]);
+      const varianceEpsilon = .001;
+
+      const result = math.batchNormalization3D(
+          x, mean, variance, varianceEpsilon, scale, undefined);
+
+      test_util.expectArraysClose(
+          result.getValues(), new Float32Array([
+            (x.get(0, 0, 0) - mean.get(0)) * scale.get(0) /
+                Math.sqrt(variance.get(0) + varianceEpsilon),
+            (x.get(0, 0, 1) - mean.get(1)) * scale.get(1) /
+                Math.sqrt(variance.get(1) + varianceEpsilon),
+            (x.get(1, 0, 0) - mean.get(0)) * scale.get(0) /
+                Math.sqrt(variance.get(0) + varianceEpsilon),
+            (x.get(1, 0, 1) - mean.get(1)) * scale.get(1) /
+                Math.sqrt(variance.get(1) + varianceEpsilon)
+          ]),
+          epsilon);
+
+      x.dispose();
+      mean.dispose();
+      variance.dispose();
+      scale.dispose();
+    });
+
+    it('simple batchnorm, no scale, 2x1x2', math => {
+      const x = Array3D.new([2, 1, 2], new Float32Array([2, 100, 4, 400]));
+      const mean = Array1D.new([1, 2]);
+      const variance = Array1D.new([2, 3]);
+      const offset = Array1D.new([4, 5]);
+
+      const varianceEpsilon = .001;
+
+      const result = math.batchNormalization3D(
+          x, mean, variance, varianceEpsilon, undefined, offset);
+
+      test_util.expectArraysClose(
+          result.getValues(), new Float32Array([
+            offset.get(0) +
+                (x.get(0, 0, 0) - mean.get(0)) * 1 /
+                    Math.sqrt(variance.get(0) + varianceEpsilon),
+            offset.get(1) +
+                (x.get(0, 0, 1) - mean.get(1)) * 1 /
+                    Math.sqrt(variance.get(1) + varianceEpsilon),
+            offset.get(0) +
+                (x.get(1, 0, 0) - mean.get(0)) * 1 /
+                    Math.sqrt(variance.get(0) + varianceEpsilon),
+            offset.get(1) +
+                (x.get(1, 0, 1) - mean.get(1)) * 1 /
+                    Math.sqrt(variance.get(1) + varianceEpsilon)
+          ]),
+          epsilon);
+      x.dispose();
+      mean.dispose();
+      variance.dispose();
+      offset.dispose();
+    });
+
+    it('simple batchnorm, 2x1x2', math => {
+      const x = Array3D.new([2, 1, 2], new Float32Array([2, 100, 4, 400]));
+      const mean = Array1D.new([1, 2]);
+      const variance = Array1D.new([2, 3]);
+      const offset = Array1D.new([3, 4]);
+      const scale = Array1D.new([4, 5]);
+
+      const varianceEpsilon = .001;
+
+      const result = math.batchNormalization3D(
+          x, mean, variance, varianceEpsilon, scale, offset);
+
+      test_util.expectArraysClose(
+          result.getValues(), new Float32Array([
+            offset.get(0) +
+                (x.get(0, 0, 0) - mean.get(0)) * scale.get(0) /
+                    Math.sqrt(variance.get(0) + varianceEpsilon),
+            offset.get(1) +
+                (x.get(0, 0, 1) - mean.get(1)) * scale.get(1) /
+                    Math.sqrt(variance.get(1) + varianceEpsilon),
+            offset.get(0) +
+                (x.get(1, 0, 0) - mean.get(0)) * scale.get(0) /
+                    Math.sqrt(variance.get(0) + varianceEpsilon),
+            offset.get(1) +
+                (x.get(1, 0, 1) - mean.get(1)) * scale.get(1) /
+                    Math.sqrt(variance.get(1) + varianceEpsilon)
+          ]),
+          epsilon);
+      x.dispose();
+      mean.dispose();
+      variance.dispose();
+      scale.dispose();
+      offset.dispose();
+    });
+
+    it('batchnorm matches tensorflow, 2x3x3', math => {
+      const x = Array3D.new(
+          [2, 3, 3], new Float32Array([
+            0.49955603, 0.04158615, -1.09440524, 2.03854165, -0.61578344,
+            2.87533573, 1.18105987, 0.807462, 1.87888837, 2.26563962,
+            -0.37040935, 1.35848753, -0.75347094, 0.15683117, 0.91925946,
+            0.34121279, 0.92717143, 1.89683965
+          ]));
+      const mean = Array1D.new([0.39745062, -0.48062894, 0.4847822]);
+      const variance = Array1D.new([0.32375343, 0.67117643, 1.08334653]);
+      const offset = Array1D.new([0.69398749, -1.29056387, 0.9429723]);
+      const scale = Array1D.new([-0.5607271, 0.9878457, 0.25181573]);
+      const varianceEpsilon = .001;
+
+      const result = math.batchNormalization3D(
+          x, mean, variance, varianceEpsilon, scale, offset);
+
+      test_util.expectArraysClose(
+          result.getValues(), new Float32Array([
+            0.59352049, -0.66135202, 0.5610874, -0.92077015, -1.45341019,
+            1.52106473, -0.07704776, 0.26144429, 1.28010017, -1.14422404,
+            -1.15776136, 1.15425493, 1.82644104, -0.52249442, 1.04803919,
+            0.74932291, 0.40568101, 1.2844412
+          ]));
+
+      x.dispose();
+      mean.dispose();
+      variance.dispose();
+      scale.dispose();
+      offset.dispose();
+    });
+  };
+
+  test_util.describeMathCPU('batchNormalization3D', [tests]);
+  test_util.describeMathGPU('batchNormalization3D', [tests], [
+    {'WEBGL_FLOAT_TEXTURE_ENABLED': true, 'WEBGL_VERSION': 1},
+    {'WEBGL_FLOAT_TEXTURE_ENABLED': true, 'WEBGL_VERSION': 2},
+    {'WEBGL_FLOAT_TEXTURE_ENABLED': false, 'WEBGL_VERSION': 1}
+  ]);
+}