Adding dropout as a graph operation (migueldeicaza#83)

cesarsouza · migueldeicaza · commit ccbe06af9157 · 2017-06-27T21:06:47.000-04:00
* migueldeicazaGH-82: Adding Dropout as a Graph operation. * Adding one more unit test to test matrix addition (other test was testing only single values) * Ignoring files specific to VS and related tools. * Fixing a formatting issue (tabs instead of spaces in the documentation).
diff --git a/.gitignore b/.gitignore
@@ -38,3 +38,6 @@ Thumbs.db
 
 # dotCover
 *.dotCover
+
+.vs
+*.GhostDoc.xml
diff --git a/TensorFlowSharp/OperationsExtras.cs b/TensorFlowSharp/OperationsExtras.cs
@@ -187,7 +187,7 @@ TFOutput ShapeTensorOutput (TFShape shape)
 		/// <param name="mean">The mean of the standard distribution.</param>
 		/// <param name="stddev">The standard deviation of the normal distribution.</param>
 		/// <param name="seed">Integer seed used for the random distribution, using the TensorFlow SetRandomSeed .</param>
-		/// <param name="operName">>Operation name, optional.</param>
+		/// <param name="operName">Operation name, optional.</param>
 		public TFOutput RandomNormal (TFShape shape, double mean = 0, double stddev = 1, int? seed = null, string operName = null)
 		{
 			var scopeName = MakeName ("RandomNormal", operName);
@@ -257,5 +257,67 @@ public void GetRandomSeeds (int? operationSeed, out int graphSeed, out int local
 				}					
 			}
 		}
-	}
+
+        /// <summary>
+        /// Computes dropout. 
+        /// </summary>
+        /// <param name="x">A tensor.</param>
+        /// <param name="keep_prob">A scalar Tensor with the same type as x. The probability that each element is kept.</param>
+        /// <param name="noise_shape">A 1-D Tensor of type int32, representing the shape for randomly generated keep/drop flags.</param>
+        /// <param name="seed">Integer seed used for the random distribution, using the TensorFlow SetRandomSeed .</param>
+        /// <param name="operName">Operation name, optional.</param>
+        /// <remarks>
+        /// With probability keep_prob, outputs the input element scaled up by 1 / keep_prob, 
+        /// otherwise outputs 0. The scaling is so that the expected sum is unchanged.
+        /// </remarks>
+        public TFOutput Dropout (TFOutput x, TFOutput keep_prob, TFShape noise_shape = null, int? seed= null, string operName= null)
+        {
+            var scopeName = MakeName("dropout", operName);
+
+            using (var newScope = WithScope(scopeName)) {
+                if (noise_shape == null)
+                    noise_shape = new TFShape(GetShape(x));
+
+                TFOutput shapeTensor = ShapeTensorOutput(noise_shape);
+
+                // uniform [keep_prob, 1.0 + keep_prob)
+                TFOutput random_tensor = keep_prob;
+                random_tensor = Add(random_tensor, RandomUniform(shapeTensor, seed: seed, dtype: x.OutputType));
+
+                // 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
+                TFOutput binary_tensor = Floor(random_tensor);
+                TFOutput ret = Mul(Div(x, keep_prob) , binary_tensor);
+                SetTensorShape(ret, GetShape(x));
+                return ret;
+            }
+        }
+
+        /// <summary>
+        /// Computes dropout. 
+        /// </summary>
+        /// <param name="x">A tensor.</param>
+        /// <param name="keep_prob">A scalar Tensor with the same type as x. The probability that each element is kept.</param>
+        /// <param name="noise_shape">A 1-D Tensor of type int32, representing the shape for randomly generated keep/drop flags.</param>
+        /// <param name="seed">Integer seed used for the random distribution, using the TensorFlow SetRandomSeed .</param>
+        /// <param name="operName">Operation name, optional.</param>
+        /// <remarks>
+        /// With probability keep_prob, outputs the input element scaled up by 1 / keep_prob, 
+        /// otherwise outputs 0. The scaling is so that the expected sum is unchanged.
+        /// </remarks>
+        public TFOutput Dropout (TFOutput x, double keep_prob, TFShape noise_shape = null, int? seed = null, string operName = null)
+        {
+            if (keep_prob < 0 || keep_prob >= 1)
+                throw new ArgumentOutOfRangeException("keep_prob must be a scalar tensor or a float in the range (0, 1], got " + keep_prob);
+
+            if (keep_prob == 1)
+                return x;
+
+            var scopeName = MakeName("dropout", operName);
+            using (var newScope = WithScope(scopeName)) {
+                var tkeep_prob = Const(keep_prob);
+                return Dropout(x, tkeep_prob, noise_shape, seed, operName);
+            }
+        }
+    }
+
 }
diff --git a/tests/TensorFlowSharp.Tests/ArithmeticOperationTests.fs b/tests/TensorFlowSharp.Tests/ArithmeticOperationTests.fs
@@ -27,3 +27,31 @@ module ArithmeticOperationTests =
         let mulValue = mulTensor.GetValue() :?> float32
 
         Assert.Equal(expected, mulValue)
+
+
+    [<Theory>]
+    let Should_EvaluateAddExpression_ForFloatDataType() =
+        use graph = new TFGraph()
+        use session = new TFSession(graph)
+
+        let a = graph.Placeholder(TFDataType.Float) // create symbolic variable a
+        let b = graph.Placeholder(TFDataType.Float) // create symbolic variable b
+
+        let y = graph.Mul(a, b) // multiply symbolic variables
+
+        let aValue   = Array2D.init 40 20 (fun i j -> i)
+        let bValue   = Array2D.init 40 20 (fun i j -> i + j)
+        let sValue   = Array2D.init 40 20 (fun i j -> 2*i + j) 
+
+        
+        // evaluate expression with parameters for a and b
+        let add = 
+            session.Run([| a; b |], 
+                        [| new TFTensor(aValue); new TFTensor(bValue) |], 
+                        [| y |])
+        
+        let addTensor = add.[0]
+        let addValue = addTensor.GetValue() :?> int[,]
+        let it = sValue = addValue : bool
+        Assert.True(it);
+        
diff --git a/tests/TensorFlowSharp.Tests/NeuralNetOperationTests.fs b/tests/TensorFlowSharp.Tests/NeuralNetOperationTests.fs
@@ -0,0 +1,38 @@
+&#65279;namespace TensorFlowSharp.Tests
+
+open TensorFlow
+open Xunit
+
+module NeuralNetOperationTests = 
+
+    [<Theory>]
+    [<InlineData(0.1f)>]
+    [<InlineData(0.5f)>]
+    [<InlineData(1.0f)>]
+    let Should_ApplyDropout_ForFloatDataType(keep_prob:float32) =
+        use graph = new TFGraph()
+        use session = new TFSession(graph)
+
+        let inputs = Array2D.init 4000 200 (fun i j -> float32(i+j+1))
+
+        let a = graph.Placeholder(TFDataType.Float, new TFShape(4000L, 200L)) // create symbolic variable x
+        let b = graph.Placeholder(TFDataType.Float, new TFShape(4000L, 200L)) // create symbolic variable keep_prob
+
+        let y = graph.Dropout(a, b) // apply dropout to symbolic variables
+        
+        // evaluate expression with parameters for a and b
+        let res = 
+            session.Run([| a; b |], 
+                        [| new TFTensor(inputs); new TFTensor(keep_prob) |], 
+                        [| y |])
+        
+        let resTensor = res.[0]
+        let resValue = resTensor.GetValue() :?> float32[,]
+
+        let countZeros arr = arr |> Seq.cast<float32> |> Seq.filter (fun x -> x = 0.0f) |> Seq.length
+        let numberOfOnes = countZeros resValue |> float32;
+        let totalLength = resValue |> Seq.cast<float32> |> Seq.length |> float32
+        let actualRatio = numberOfOnes / totalLength;
+        let expected = 1.0f - keep_prob : float32
+
+        Assert.True(System.Math.Abs(expected - actualRatio) < 0.05f)
