Skip to content

Commit 9c8d7aa

Browse files
cesarsouzamigueldeicaza
cesarsouza
authored and
migueldeicaza
committed
Adding clipping operations (migueldeicaza#117)
* Adding ClipByValue, ClipByNorm, GlobalNorm, ClipByAverageNorm, and Range. * Adding unit tests for the operations that have been just added. * Reformatting to follow the same style as the rest of the project.
1 parent caf962c commit 9c8d7aa

File tree

5 files changed

+755
-65
lines changed

5 files changed

+755
-65
lines changed

‎TensorFlowSharp/OperationsExtras.cs‎

Lines changed: 261 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ TFOutput ReduceDims (TFOutput input, TFOutput? axis = null)
3535
for (int i = 0; i < array.Length; i++)
3636
array [i] = i;
3737

38-
return this.Const (array, TFDataType.Int32);
38+
return this.Const (array, TFDataType.Int32);
3939
}
4040
return Range (Const (0), Const (shape.Length), Const (1));
4141
}
@@ -277,70 +277,266 @@ public void GetRandomSeeds (int? operationSeed, out int graphSeed, out int local
277277
localSeed = operationSeed.Value;
278278
} else {
279279
localSeed = 0;
280-
}
280+
}
281+
}
282+
}
283+
284+
/// <summary>
285+
/// Computes dropout.
286+
/// </summary>
287+
/// <param name="x">A tensor.</param>
288+
/// <param name="keep_prob">A scalar Tensor with the same type as x. The probability that each element is kept.</param>
289+
/// <param name="noise_shape">A 1-D Tensor of type int32, representing the shape for randomly generated keep/drop flags.</param>
290+
/// <param name="seed">Integer seed used for the random distribution, using the TensorFlow SetRandomSeed .</param>
291+
/// <param name="operName">Operation name, optional.</param>
292+
/// <remarks>
293+
/// With probability keep_prob, outputs the input element scaled up by 1 / keep_prob,
294+
/// otherwise outputs 0. The scaling is so that the expected sum is unchanged.
295+
/// </remarks>
296+
public TFOutput Dropout (TFOutput x, TFOutput keep_prob, TFShape noise_shape = null, int? seed = null, string operName = null)
297+
{
298+
var scopeName = MakeName ("dropout", operName);
299+
300+
using (var newScope = WithScope (scopeName)) {
301+
if (noise_shape == null)
302+
noise_shape = new TFShape (GetShape (x));
303+
304+
TFOutput shapeTensor = ShapeTensorOutput (noise_shape);
305+
306+
// uniform [keep_prob, 1.0 + keep_prob)
307+
TFOutput random_tensor = keep_prob;
308+
random_tensor = Add (random_tensor, RandomUniform (shapeTensor, seed: seed, dtype: x.OutputType));
309+
310+
// 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
311+
TFOutput binary_tensor = Floor (random_tensor);
312+
TFOutput ret = Mul (Div (x, keep_prob), binary_tensor);
313+
SetTensorShape (ret, GetShape (x));
314+
return ret;
315+
}
316+
}
317+
318+
/// <summary>
319+
/// Computes dropout.
320+
/// </summary>
321+
/// <param name="x">A tensor.</param>
322+
/// <param name="keep_prob">A scalar Tensor with the same type as x. The probability that each element is kept.</param>
323+
/// <param name="noise_shape">A 1-D Tensor of type int32, representing the shape for randomly generated keep/drop flags.</param>
324+
/// <param name="seed">Integer seed used for the random distribution, using the TensorFlow SetRandomSeed .</param>
325+
/// <param name="operName">Operation name, optional.</param>
326+
/// <remarks>
327+
/// With probability keep_prob, outputs the input element scaled up by 1 / keep_prob,
328+
/// otherwise outputs 0. The scaling is so that the expected sum is unchanged.
329+
/// </remarks>
330+
public TFOutput Dropout (TFOutput x, double keep_prob, TFShape noise_shape = null, int? seed = null, string operName = null)
331+
{
332+
if (keep_prob < 0 || keep_prob >= 1)
333+
throw new ArgumentOutOfRangeException ("keep_prob must be a scalar tensor or a float in the range (0, 1], got " + keep_prob);
334+
335+
if (keep_prob == 1)
336+
return x;
337+
338+
var scopeName = MakeName ("dropout", operName);
339+
using (var newScope = WithScope (scopeName)) {
340+
var tkeep_prob = Const (keep_prob);
341+
return Dropout (x, tkeep_prob, noise_shape, seed, operName);
342+
}
343+
}
344+
345+
346+
347+
/// <summary>
348+
/// Clips tensor values to a specified min and max.
349+
/// </summary>
350+
/// <remarks>
351+
/// Given a tensor <paramref name="x"/>, this operation returns a tensor of the same type and shape
352+
/// as <paramref name="x"/> with its values clipped to <paramref name="clip_value_min"/> and <paramref name="clip_value_max"/>.
353+
/// Any values less than <paramref name="clib_value_min"/> are set to <paramref name="clip_value_min"/>. Any values greater than
354+
/// <paramref name="clip_value_max"/> are set to <paramref name="clip_value_max"/>.
355+
/// </remarks>
356+
/// <param name="x">The tensor.</param>
357+
/// <param name="clip_value_min">The minimum value to clip by. A 0 - D(scalar) tensor, or a tensor with the same shape as <paramref name="x"/>.</param>
358+
/// <param name="clip_value_max">The minimum value to clip by. A 0 - D(scalar) tensor, or a tensor with the same shape as <paramref name="x"/>.</param>
359+
/// <param name="operName">Operation name, optional.</param>
360+
/// <returns>A clipped <see cref="TFOutput">tensor</see>.</returns>
361+
public TFOutput ClipByValue (TFOutput x, TFOutput clip_value_min, TFOutput clip_value_max, string operName = null)
362+
{
363+
// https://github.com/tensorflow/tensorflow/blob/r1.2/tensorflow/python/ops/clip_ops.py#L33
364+
var scopeName = MakeName ("ClipByValue", operName);
365+
using (var newScope = WithScope (scopeName)) {
366+
// Go through list of tensors, for each value in each tensor clip
367+
var t_min = Minimum (x, clip_value_max);
368+
var t_max = Maximum (t_min, clip_value_min, operName: operName);
369+
return t_max;
370+
}
371+
}
372+
373+
/// <summary>
374+
/// Clips tensor values to a maximum L2-norm.
375+
/// </summary>
376+
/// <remarks>
377+
/// <para>
378+
/// Given a tensor <paramref name="x"/>, and a maximum clip value <paramref name="clip_norm"/>, this operation normalizes
379+
/// <paramref name="x"/> so that its L2-norm is less than or equal to <paramref name="clip_norm"/>, along the dimensions
380+
/// given in <paramref name="axes"/>. Specifically, in the default case where all dimensions are used for calculation, if
381+
/// the L2-norm of <paramref name="x"/> is already less than or equal to <paramref name="clip_norm"/>, then <paramref name="x"/>
382+
/// is not modified. If the L2-norm is greater than <paramref name="clip_norm"/>, then this operation returns a tensor of
383+
/// the same type and shape as <paramref name="x"/> with its values set to: <c>t* clip_norm / l2norm(t)</c></para>
384+
/// </remarks>
385+
/// <param name="x">The tensor.</param>
386+
/// <param name="clip_norm">The minimum value to clip by. A 0 - D(scalar) tensor, or a tensor with the same shape as <paramref name="x"/>.</param>
387+
/// <param name="axes">The minimum value to clip by. A 0 - D(scalar) tensor, or a tensor with the same shape as <paramref name="x"/>.</param>
388+
/// <param name="operName">Operation name, optional.</param>
389+
/// <returns>A clipped <see cref="TFOutput">tensor</see>.</returns>
390+
public TFOutput ClipByNorm (TFOutput x, TFOutput clip_norm, TFOutput? axes = null, string operName = null)
391+
{
392+
// https://github.com/tensorflow/tensorflow/blob/r1.2/tensorflow/python/ops/clip_ops.py#L73
393+
var scopeName = MakeName ("ClipByNorm", operName);
394+
using (var newScope = WithScope (scopeName)) {
395+
// Calculate L2-norm, clip elements by ratio of clip_norm to L2-norm
396+
var l2norm_inv = Rsqrt (ReduceSum (Mul (x, x), axes, keep_dims: true));
397+
var intermediate = Mul (x, clip_norm);
398+
399+
var tclip = Identity (Mul (intermediate, Minimum (l2norm_inv, Div (Const (new TFTensor (1.0)), clip_norm), operName: operName)));
400+
401+
return tclip;
402+
}
403+
}
404+
405+
/// <summary>
406+
/// Computes the global norm of multiple tensors.
407+
/// </summary>
408+
/// <remarks>
409+
/// <para>
410+
/// Given a tuple or list of tensors <paramref name="tensors"/>, this operation returns the global norm of the elements in all tensors
411+
/// in <paramref name="tensors"/>. The global norm is computed as: <c>global_norm = sqrt(sum([l2norm(t)**2 for t in t_list]))</c>. Any
412+
/// entries in <paramref name="tensors"/> that are of type None are ignored.</para>
413+
/// </remarks>
414+
/// <param name="tensors">The input tensors.</param>
415+
/// <param name="operName">Operation name, optional.</param>
416+
/// <returns>A clipped <see cref="TFOutput">tensor</see>.</returns>
417+
public TFOutput GlobalNorm (TFOutput [] tensors, string operName = null)
418+
{
419+
// https://github.com/tensorflow/tensorflow/blob/r1.2/tensorflow/python/ops/clip_ops.py#L122
420+
var scopeName = MakeName ("GlobalNorm", operName);
421+
using (var newScope = WithScope (scopeName)) {
422+
TFOutput [] half_squared_norms = new TFOutput [tensors.Length];
423+
424+
for (int i = 0; i < half_squared_norms.Length; i++)
425+
half_squared_norms [i] = L2Loss (tensors [i]);
426+
427+
TFOutput half_squared_norm = ReduceSum (Stack (half_squared_norms));
428+
TFOutput norm = Sqrt (Mul (half_squared_norm, Const (2.0)), operName: "global_norm");
429+
return norm;
430+
}
431+
}
432+
433+
/// <summary>
434+
/// Clips tensor values to a maximum average L2-norm.
435+
/// </summary>
436+
/// <remarks>
437+
/// Given a tensor <paramref name="x"/>, and a maximum clip value <paramref name="clip_norm"/>, this operation
438+
/// normalizes <paramref name="x"/> so that its its average L2-norm is less than or equal to <paramref name="clip_norm"/>.
439+
/// Specifically, if the average L2-norm is already less than or equal to <paramref name="clip_norm"/>, then <paramref name="x"/>
440+
/// is not modified. If the average L2-norm is greater than <paramref name="clip_norm"/>, then this operation returns a tensor of the same
441+
/// type and shape as <paramref name="x"/> with its values set to: <c>t* clip_norm / l2norm_avg(t)</c>. In this case,
442+
/// the average L2-norm of the output tensor is <paramref name="clip_norm"/>.
443+
/// </remarks>
444+
/// <param name="x">The input tensor.</param>
445+
/// <param name="clip_norm">A maximum clipping value.</param>
446+
/// <param name="operName">Name of the oper.</param>
447+
public TFOutput ClipByAverageNorm (TFOutput x, TFOutput clip_norm, string operName = null)
448+
{
449+
// https://github.com/tensorflow/tensorflow/blob/r1.2/tensorflow/python/ops/clip_ops.py#L251
450+
var scopeName = MakeName ("ClipByAverageNorm", operName);
451+
using (var newScope = WithScope (scopeName)) {
452+
// Calculate L2-norm per element, clip elements by ratio of clip_norm to
453+
// L2-norm per element
454+
TFOutput n_element = Cast (Size (x), TFDataType.Float);
455+
TFOutput l2norm_inv = Rsqrt (ReduceSum (Mul (x, x), Range (Rank (x))));
456+
TFOutput tclip = Identity (Mul (Mul (x, clip_norm), Minimum (Mul (l2norm_inv, n_element), Div (Const (new TFTensor (1.0)), clip_norm)), operName: operName));
457+
458+
return tclip;
459+
}
460+
}
461+
462+
463+
464+
465+
466+
467+
468+
469+
470+
471+
/// <summary>
472+
/// Stacks a list of rank-`R` tensors into one rank-`(R+1)` tensor.
473+
/// </summary>
474+
/// <remarks>
475+
/// Packs the list of tensors in <paramref name="values"/> into a tensor with rank one higher than
476+
/// each tensor in <paramref name="values"/>, by packing them along the <paramref name="axis"/> dimension.
477+
/// Given a list of length <c>N</c> of tensors of shape </c>(A, B, C)</c>: if <c>axis == 0</c> then the
478+
/// <c>output</c> tensor will have the shape <c>(N, A, B, C)</c>; if <c>axis == 1<c> then the <c>output<c>
479+
/// tensor will have the shape <c>(A, N, B, C)<c>; etc.
480+
/// </remarks>
481+
///
482+
public TFOutput Stack (TFOutput [] values, int? axis = 0, string operName = "stack")
483+
{
484+
// https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/ops/array_ops.py#L804
485+
486+
int ndims = GetTensorNumDims (values [0]);
487+
488+
int expanded_num_dims = ndims + 1;
489+
if (axis < -expanded_num_dims || axis >= expanded_num_dims)
490+
throw new InvalidOperationException ($"axis = {axis} not in [{-expanded_num_dims}, {expanded_num_dims}]");
491+
492+
return Pack (values, axis: axis, operName: operName);
493+
}
494+
495+
/// <summary>
496+
/// Creates a sequence of numbers.
497+
/// </summary>
498+
/// <remarks>
499+
/// Creates a sequence of numbers that begins at `start` and extends by increments of `delta` up to but not including
500+
/// `limit`. The dtype of the resulting tensor is inferred from the inputs unless it is provided explicitly.
501+
/// </remarks>
502+
/// <param name="start">A 0 - D `Tensor` (scalar).Acts as first entry in the range if `limit` is not None; otherwise, acts as range limit and first entry defaults to 0.</param>
503+
/// <param name="limit">A 0 - D `Tensor` (scalar).Upper limit of sequence, exclusive. If None, defaults to the value of `start` while the first entry of the range defaults to 0.</param>
504+
/// <param name="delta">A 0 - D `Tensor` (scalar).Number that increments `start`. Defaults to 1.</param>
505+
/// <param name="dataType">The type of the elements of the resulting tensor.</param>
506+
/// <param name="operName">A name for the operation.Defaults to "range".</param>
507+
public TFOutput Range (TFOutput start, TFOutput? limit = null, TFOutput? delta = null, TFDataType? dataType = null, string operName = "range")
508+
{
509+
// https://github.com/tensorflow/tensorflow/blob/r1.2/tensorflow/python/ops/math_ops.py#L1156
510+
511+
if (limit == null) {
512+
limit = start;
513+
start = Cast (Const (new TFTensor (0.0)), start.OutputType); // TODO: Maybe add dataType as convenience in Const?
514+
}
515+
516+
if (delta == null)
517+
delta = Cast (Const (new TFTensor (1.0)), start.OutputType);
518+
519+
using (var newScope = WithScope (MakeName ("Range", operName))) {
520+
// infer dtype if not explicitly provided
521+
if (dataType == null) {
522+
var dtype_hierarchy = new [] { TFDataType.Int32, TFDataType.Int64, TFDataType.Float, TFDataType.Double };
523+
if (!dtype_hierarchy.Contains (start.OutputType)
524+
|| !dtype_hierarchy.Contains (limit.Value.OutputType)
525+
|| !dtype_hierarchy.Contains (delta.Value.OutputType))
526+
throw new ArgumentException ("Unexpected type");
527+
528+
TFDataType [] dtypes = new [] { start.OutputType, limit.Value.OutputType, delta.Value.OutputType };
529+
int imax = dtypes.Select (x => Array.IndexOf (dtype_hierarchy, x)).Max ();
530+
TFDataType inferred_dtype = dtype_hierarchy [imax];
531+
532+
start = Cast (start, inferred_dtype);
533+
limit = Cast (limit.Value, inferred_dtype);
534+
delta = Cast (delta.Value, inferred_dtype);
535+
}
536+
537+
return Range (start, limit.Value, delta.Value, operName: operName);
281538
}
282539
}
283540

284-
/// <summary>
285-
/// Computes dropout.
286-
/// </summary>
287-
/// <param name="x">A tensor.</param>
288-
/// <param name="keep_prob">A scalar Tensor with the same type as x. The probability that each element is kept.</param>
289-
/// <param name="noise_shape">A 1-D Tensor of type int32, representing the shape for randomly generated keep/drop flags.</param>
290-
/// <param name="seed">Integer seed used for the random distribution, using the TensorFlow SetRandomSeed .</param>
291-
/// <param name="operName">Operation name, optional.</param>
292-
/// <remarks>
293-
/// With probability keep_prob, outputs the input element scaled up by 1 / keep_prob,
294-
/// otherwise outputs 0. The scaling is so that the expected sum is unchanged.
295-
/// </remarks>
296-
public TFOutput Dropout (TFOutput x, TFOutput keep_prob, TFShape noise_shape = null, int? seed= null, string operName= null)
297-
{
298-
var scopeName = MakeName("dropout", operName);
299-
300-
using (var newScope = WithScope(scopeName)) {
301-
if (noise_shape == null)
302-
noise_shape = new TFShape(GetShape(x));
303-
304-
TFOutput shapeTensor = ShapeTensorOutput(noise_shape);
305-
306-
// uniform [keep_prob, 1.0 + keep_prob)
307-
TFOutput random_tensor = keep_prob;
308-
random_tensor = Add(random_tensor, RandomUniform(shapeTensor, seed: seed, dtype: x.OutputType));
309-
310-
// 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
311-
TFOutput binary_tensor = Floor(random_tensor);
312-
TFOutput ret = Mul(Div(x, keep_prob) , binary_tensor);
313-
SetTensorShape(ret, GetShape(x));
314-
return ret;
315-
}
316-
}
317-
318-
/// <summary>
319-
/// Computes dropout.
320-
/// </summary>
321-
/// <param name="x">A tensor.</param>
322-
/// <param name="keep_prob">A scalar Tensor with the same type as x. The probability that each element is kept.</param>
323-
/// <param name="noise_shape">A 1-D Tensor of type int32, representing the shape for randomly generated keep/drop flags.</param>
324-
/// <param name="seed">Integer seed used for the random distribution, using the TensorFlow SetRandomSeed .</param>
325-
/// <param name="operName">Operation name, optional.</param>
326-
/// <remarks>
327-
/// With probability keep_prob, outputs the input element scaled up by 1 / keep_prob,
328-
/// otherwise outputs 0. The scaling is so that the expected sum is unchanged.
329-
/// </remarks>
330-
public TFOutput Dropout (TFOutput x, double keep_prob, TFShape noise_shape = null, int? seed = null, string operName = null)
331-
{
332-
if (keep_prob < 0 || keep_prob >= 1)
333-
throw new ArgumentOutOfRangeException("keep_prob must be a scalar tensor or a float in the range (0, 1], got " + keep_prob);
334-
335-
if (keep_prob == 1)
336-
return x;
337-
338-
var scopeName = MakeName("dropout", operName);
339-
using (var newScope = WithScope(scopeName)) {
340-
var tkeep_prob = Const(keep_prob);
341-
return Dropout(x, tkeep_prob, noise_shape, seed, operName);
342-
}
343-
}
344-
}
345-
346-
}
541+
}
542+
}

0 commit comments

Comments
 (0)