Skip to content

Commit cebc074

Browse files
zeahmedmigueldeicaza
zeahmed
authored and
migueldeicaza
committed
Added support for model optimization using RMSProp algorithm. (migueldeicaza#413)
* Setting variable shared_name property to avoid variables sharing. * Added SGD and MomentumSGD optimizers together with relevant tests. * Tests added for momentum and Nesterov SGD with and without lr decay. * Added MNIST multilayer test. * Added MNIST GPU test in disabled mode. * Added support to place an operation on a specific device. * Disabled 'DevicePlacementTest' because it requires GPUs. * Added MNIST multilayer test. * Updated comments. * Disabled MnistGPU test. * Removed uncessary files. * Added Adagrad optimization algorithm. * Added RMSProp optimization algorithm.
1 parent 006361b commit cebc074

File tree

7 files changed

+513
-107
lines changed

7 files changed

+513
-107
lines changed

‎TensorFlowSharp/Optimizer.cs‎

Lines changed: 145 additions & 107 deletions
Large diffs are not rendered by default.

‎tests/TensorFlowSharp.Tests.CSharp/OptimizerTests.cs‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -669,5 +669,117 @@ public void LinearRegresionTrainingWithAdagradDecayTest()
669669
}
670670
}
671671
}
672+
673+
[Fact]
674+
public void LinearRegresionTrainingWithRMSPropTest()
675+
{
676+
Console.WriteLine("Linear regression");
677+
// Parameters
678+
var learning_rate = 0.01f;
679+
var training_epochs = 5;
680+
681+
// Training data
682+
var train_x = new float[] {
683+
3.3f, 4.4f, 5.5f, 6.71f, 6.93f, 4.168f, 9.779f, 6.182f, 7.59f, 2.167f,
684+
7.042f, 10.791f, 5.313f, 7.997f, 5.654f, 9.27f, 3.1f
685+
};
686+
var train_y = new float[] {
687+
1.7f, 2.76f,2.09f,3.19f,1.694f,1.573f,3.366f,2.596f,2.53f,1.221f,
688+
2.827f,3.465f,1.65f,2.904f,2.42f,2.94f,1.3f
689+
};
690+
var n_samples = train_x.Length;
691+
using (var graph = new TFGraph())
692+
{
693+
var rng = new Random(0);
694+
// tf Graph Input
695+
696+
var X = graph.Placeholder(TFDataType.Float, TFShape.Scalar);
697+
var Y = graph.Placeholder(TFDataType.Float, TFShape.Scalar);
698+
699+
var W = graph.Variable(graph.Const(0.1f), operName: "weight");
700+
var b = graph.Variable(graph.Const(0.1f), operName: "bias");
701+
var pred = graph.Add(graph.Mul(X, W.Read, "x_w"), b.Read);
702+
703+
var cost = graph.Div(graph.ReduceSum(graph.Pow(graph.Sub(pred, Y), graph.Const(2f))), graph.Mul(graph.Const(2f), graph.Const((float)n_samples), "2_n_samples"));
704+
705+
var sgd = new RMSProp(graph, learning_rate, initialAccumulatorValue: 1.0f);
706+
var updateOps = sgd.Minimize(cost);
707+
708+
using (var sesssion = new TFSession(graph))
709+
{
710+
sesssion.GetRunner().AddTarget(graph.GetGlobalVariablesInitializer()).Run();
711+
712+
var expectedLines = File.ReadAllLines(Path.Combine(_testDataPath, "RMSProp", "expected.txt"));
713+
for (int i = 0; i < training_epochs; i++)
714+
{
715+
for (int j = 0; j < n_samples; j++)
716+
{
717+
var tensors = sesssion.GetRunner()
718+
.AddInput(X, new TFTensor(train_x[j]))
719+
.AddInput(Y, new TFTensor(train_y[j]))
720+
.AddTarget(updateOps).Fetch(cost, W.Read, b.Read, pred).Run();
721+
var output = $"loss: {tensors[0].GetValue():F4}, W: {tensors[1].GetValue():F4}, b: {tensors[2].GetValue():F4}";
722+
Assert.Equal(expectedLines[i * n_samples + j], output);
723+
}
724+
}
725+
}
726+
}
727+
}
728+
729+
[Fact]
730+
public void LinearRegresionTrainingWithRMSPropDecayTest()
731+
{
732+
Console.WriteLine("Linear regression");
733+
// Parameters
734+
var learning_rate = 0.01f;
735+
var training_epochs = 5;
736+
737+
// Training data
738+
var train_x = new float[] {
739+
3.3f, 4.4f, 5.5f, 6.71f, 6.93f, 4.168f, 9.779f, 6.182f, 7.59f, 2.167f,
740+
7.042f, 10.791f, 5.313f, 7.997f, 5.654f, 9.27f, 3.1f
741+
};
742+
var train_y = new float[] {
743+
1.7f, 2.76f,2.09f,3.19f,1.694f,1.573f,3.366f,2.596f,2.53f,1.221f,
744+
2.827f,3.465f,1.65f,2.904f,2.42f,2.94f,1.3f
745+
};
746+
var n_samples = train_x.Length;
747+
using (var graph = new TFGraph())
748+
{
749+
var rng = new Random(0);
750+
// tf Graph Input
751+
752+
var X = graph.Placeholder(TFDataType.Float, TFShape.Scalar);
753+
var Y = graph.Placeholder(TFDataType.Float, TFShape.Scalar);
754+
755+
var W = graph.Variable(graph.Const(0.1f), operName: "weight");
756+
var b = graph.Variable(graph.Const(0.1f), operName: "bias");
757+
var pred = graph.Add(graph.Mul(X, W.Read, "x_w"), b.Read);
758+
759+
var cost = graph.Div(graph.ReduceSum(graph.Pow(graph.Sub(pred, Y), graph.Const(2f))), graph.Mul(graph.Const(2f), graph.Const((float)n_samples), "2_n_samples"));
760+
761+
var sgd = new RMSProp(graph, learning_rate, decay: 0.5f, initialAccumulatorValue: 1.0f);
762+
var updateOps = sgd.Minimize(cost);
763+
764+
using (var sesssion = new TFSession(graph))
765+
{
766+
sesssion.GetRunner().AddTarget(graph.GetGlobalVariablesInitializer()).Run();
767+
768+
var expectedLines = File.ReadAllLines(Path.Combine(_testDataPath, "RMSPropTimeDecay", "expected.txt"));
769+
for (int i = 0; i < training_epochs; i++)
770+
{
771+
for (int j = 0; j < n_samples; j++)
772+
{
773+
var tensors = sesssion.GetRunner()
774+
.AddInput(X, new TFTensor(train_x[j]))
775+
.AddInput(Y, new TFTensor(train_y[j]))
776+
.AddTarget(updateOps).Fetch(sgd.Iterations.Read, cost, W.Read, b.Read, sgd.LearningRate.Read).Run();
777+
var output = $"step: {tensors[0].GetValue():D}, loss: {tensors[1].GetValue():F4}, W: {tensors[2].GetValue():F4}, b: {tensors[3].GetValue():F4}, lr: {tensors[4].GetValue():F8}";
778+
Assert.Equal(expectedLines[i * n_samples + j], output);
779+
}
780+
}
781+
}
782+
}
783+
}
672784
}
673785
}

‎tests/TensorFlowSharp.Tests.CSharp/TensorFlowSharp.Tests.CSharp.csproj‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,12 @@
104104
<None Include="TestData\Momentum\optimizer_lr_test.py">
105105
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
106106
</None>
107+
<None Include="TestData\RMSPropTimeDecay\optimizer_lr_test.py">
108+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
109+
</None>
110+
<None Include="TestData\RMSProp\optimizer_lr_test.py">
111+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
112+
</None>
107113
<None Include="TestData\SGDMnist\optimizer_lr_test.py">
108114
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
109115
</None>
@@ -147,6 +153,12 @@
147153
<Content Include="TestData\Momentum\expected.txt">
148154
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
149155
</Content>
156+
<Content Include="TestData\RMSPropTimeDecay\expected.txt">
157+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
158+
</Content>
159+
<Content Include="TestData\RMSProp\expected.txt">
160+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
161+
</Content>
150162
<Content Include="TestData\SGDMnistGPU\expected.txt">
151163
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
152164
</Content>

‎tests/TensorFlowSharp.Tests.CSharp/TestData/RMSProp/expected.txt‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
loss: 0.0474, W: 0.1000, b: 0.1000
2+
loss: 0.1434, W: 0.1026, b: 0.1008
3+
loss: 0.0568, W: 0.1088, b: 0.1022
4+
loss: 0.1587, W: 0.1139, b: 0.1032
5+
loss: 0.0156, W: 0.1242, b: 0.1049
6+
loss: 0.0257, W: 0.1277, b: 0.1054
7+
loss: 0.1157, W: 0.1305, b: 0.1062
8+
loss: 0.0751, W: 0.1440, b: 0.1079
9+
loss: 0.0478, W: 0.1511, b: 0.1093
10+
loss: 0.0173, W: 0.1581, b: 0.1105
11+
loss: 0.0746, W: 0.1594, b: 0.1112
12+
loss: 0.0695, W: 0.1682, b: 0.1129
13+
loss: 0.0097, W: 0.1807, b: 0.1146
14+
loss: 0.0515, W: 0.1832, b: 0.1152
15+
loss: 0.0437, W: 0.1917, b: 0.1168
16+
loss: 0.0289, W: 0.1975, b: 0.1184
17+
loss: 0.0087, W: 0.2053, b: 0.1197
18+
loss: 0.0237, W: 0.2068, b: 0.1205
19+
loss: 0.0866, W: 0.2096, b: 0.1219
20+
loss: 0.0175, W: 0.2170, b: 0.1246
21+
loss: 0.0734, W: 0.2212, b: 0.1258
22+
loss: 0.0001, W: 0.2319, b: 0.1286
23+
loss: 0.0068, W: 0.2316, b: 0.1285
24+
loss: 0.0266, W: 0.2338, b: 0.1294
25+
loss: 0.0269, W: 0.2441, b: 0.1314
26+
loss: 0.0071, W: 0.2509, b: 0.1334
27+
loss: 0.0084, W: 0.2553, b: 0.1345
28+
loss: 0.0229, W: 0.2568, b: 0.1358
29+
loss: 0.0065, W: 0.2647, b: 0.1380
30+
loss: 0.0001, W: 0.2714, b: 0.1392
31+
loss: 0.0102, W: 0.2719, b: 0.1394
32+
loss: 0.0145, W: 0.2787, b: 0.1411
33+
loss: 0.0007, W: 0.2846, b: 0.1433
34+
loss: 0.0021, W: 0.2869, b: 0.1438
35+
loss: 0.0107, W: 0.2882, b: 0.1447
36+
loss: 0.0520, W: 0.2916, b: 0.1469
37+
loss: 0.0023, W: 0.3017, b: 0.1518
38+
loss: 0.0291, W: 0.3045, b: 0.1529
39+
loss: 0.0126, W: 0.3162, b: 0.1570
40+
loss: 0.0005, W: 0.3081, b: 0.1542
41+
loss: 0.0010, W: 0.3092, b: 0.1548
42+
loss: 0.0075, W: 0.3128, b: 0.1557
43+
loss: 0.0001, W: 0.3192, b: 0.1582
44+
loss: 0.0041, W: 0.3184, b: 0.1579
45+
loss: 0.0050, W: 0.3202, b: 0.1600
46+
loss: 0.0015, W: 0.3270, b: 0.1624
47+
loss: 0.0014, W: 0.3211, b: 0.1610
48+
loss: 0.0012, W: 0.3182, b: 0.1596
49+
loss: 0.0056, W: 0.3224, b: 0.1610
50+
loss: 0.0022, W: 0.3291, b: 0.1641
51+
loss: 0.0006, W: 0.3220, b: 0.1620
52+
loss: 0.0065, W: 0.3233, b: 0.1632
53+
loss: 0.0389, W: 0.3280, b: 0.1671
54+
loss: 0.0000, W: 0.3426, b: 0.1767
55+
loss: 0.0149, W: 0.3431, b: 0.1769
56+
loss: 0.0272, W: 0.3568, b: 0.1834
57+
loss: 0.0000, W: 0.3398, b: 0.1745
58+
loss: 0.0005, W: 0.3396, b: 0.1744
59+
loss: 0.0035, W: 0.3360, b: 0.1730
60+
loss: 0.0018, W: 0.3423, b: 0.1767
61+
loss: 0.0030, W: 0.3366, b: 0.1740
62+
loss: 0.0020, W: 0.3388, b: 0.1777
63+
loss: 0.0056, W: 0.3449, b: 0.1810
64+
loss: 0.0023, W: 0.3304, b: 0.1754
65+
loss: 0.0005, W: 0.3256, b: 0.1716
66+
loss: 0.0044, W: 0.3290, b: 0.1734
67+
loss: 0.0038, W: 0.3366, b: 0.1791
68+
loss: 0.0004, W: 0.3253, b: 0.1736
69+
loss: 0.0059, W: 0.3266, b: 0.1755
70+
loss: 0.0366, W: 0.3321, b: 0.1828
71+
loss: 0.0000, W: 0.3485, b: 0.1992
72+
loss: 0.0127, W: 0.3480, b: 0.1988
73+
loss: 0.0310, W: 0.3625, b: 0.2090
74+
loss: 0.0001, W: 0.3430, b: 0.1942
75+
loss: 0.0009, W: 0.3424, b: 0.1934
76+
loss: 0.0030, W: 0.3372, b: 0.1906
77+
loss: 0.0022, W: 0.3434, b: 0.1959
78+
loss: 0.0026, W: 0.3368, b: 0.1912
79+
loss: 0.0017, W: 0.3390, b: 0.1966
80+
loss: 0.0062, W: 0.3449, b: 0.2012
81+
loss: 0.0025, W: 0.3292, b: 0.1925
82+
loss: 0.0005, W: 0.3241, b: 0.1867
83+
loss: 0.0042, W: 0.3276, b: 0.1893
84+
loss: 0.0039, W: 0.3351, b: 0.1973
85+
loss: 0.0003, W: 0.3234, b: 0.1895

‎tests/TensorFlowSharp.Tests.CSharp/TestData/RMSProp/optimizer_lr_test.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# This script is used to create data file (expected.txt)
2+
# which is used to compare the output from TensorFlowSharp optimizer tests.
3+
4+
import tensorflow as tf
5+
6+
# Training data
7+
train_x =[
8+
3.3, 4.4, 5.5, 6.71, 6.93, 4.168, 9.779, 6.182, 7.59, 2.167,
9+
7.042, 10.791, 5.313, 7.997, 5.654, 9.27, 3.1
10+
]
11+
train_y = [
12+
1.7, 2.76,2.09,3.19,1.694,1.573,3.366,2.596,2.53,1.221,
13+
2.827,3.465,1.65,2.904,2.42,2.94,1.3
14+
]
15+
n_samples = len(train_x)
16+
learning_rate = 0.01
17+
X = tf.placeholder(tf.float32)
18+
Y = tf.placeholder(tf.float32)
19+
20+
W = tf.Variable(tf.constant(0.1), dtype=tf.float32)
21+
b = tf.Variable(tf.constant(0.1), dtype=tf.float32)
22+
23+
pred = tf.add(tf.multiply(X,W), b)
24+
25+
cost = tf.divide(tf.reduce_sum(tf.pow(tf.subtract(pred, Y), 2.0)), tf.multiply(2.0, n_samples))
26+
optimizer = tf.train.RMSPropOptimizer(learning_rate, momentum=0.0,epsilon=1e-7).minimize(cost, name = "RMSPropOptimizer")
27+
28+
init = tf.global_variables_initializer()
29+
with tf.Session() as session:
30+
session.run(init)
31+
for e in range(5):
32+
for i in range(n_samples):
33+
_, cost_v, W_v, b_v, pred_v = session.run([optimizer, cost, W, b, pred], feed_dict = {X: train_x[i], Y: train_y[i]})
34+
print(f"loss: {cost_v:.4f}, W: {W_v:.4f}, b: {b_v:.4f}")
35+
#print("Prediction: %f == Actual: %f" % (pred_v, train_y[i]))

‎tests/TensorFlowSharp.Tests.CSharp/TestData/RMSPropTimeDecay/expected.txt‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
step: 1, loss: 0.0474, W: 0.1000, b: 0.1000, lr: 0.01000000
2+
step: 2, loss: 0.1434, W: 0.1026, b: 0.1008, lr: 0.00666667
3+
step: 3, loss: 0.0578, W: 0.1067, b: 0.1017, lr: 0.00500000
4+
step: 4, loss: 0.1630, W: 0.1093, b: 0.1022, lr: 0.00400000
5+
step: 5, loss: 0.0190, W: 0.1135, b: 0.1029, lr: 0.00333333
6+
step: 6, loss: 0.0289, W: 0.1147, b: 0.1031, lr: 0.00285714
7+
step: 7, loss: 0.1337, W: 0.1156, b: 0.1033, lr: 0.00250000
8+
step: 8, loss: 0.0906, W: 0.1192, b: 0.1038, lr: 0.00222222
9+
step: 9, loss: 0.0669, W: 0.1209, b: 0.1041, lr: 0.00200000
10+
step: 10, loss: 0.0213, W: 0.1225, b: 0.1044, lr: 0.00181818
11+
step: 11, loss: 0.1015, W: 0.1227, b: 0.1046, lr: 0.00166667
12+
step: 12, loss: 0.1198, W: 0.1244, b: 0.1049, lr: 0.00153846
13+
step: 13, loss: 0.0223, W: 0.1267, b: 0.1052, lr: 0.00142857
14+
step: 14, loss: 0.0933, W: 0.1272, b: 0.1054, lr: 0.00133333
15+
step: 15, loss: 0.0741, W: 0.1286, b: 0.1057, lr: 0.00125000
16+
step: 16, loss: 0.0786, W: 0.1294, b: 0.1059, lr: 0.00117647
17+
step: 17, loss: 0.0183, W: 0.1307, b: 0.1062, lr: 0.00111111
18+
step: 18, loss: 0.0397, W: 0.1309, b: 0.1063, lr: 0.00105263
19+
step: 19, loss: 0.1268, W: 0.1313, b: 0.1065, lr: 0.00100000
20+
step: 20, loss: 0.0465, W: 0.1320, b: 0.1068, lr: 0.00095238
21+
step: 21, loss: 0.1415, W: 0.1326, b: 0.1070, lr: 0.00090909
22+
step: 22, loss: 0.0128, W: 0.1337, b: 0.1073, lr: 0.00086957
23+
step: 23, loss: 0.0242, W: 0.1340, b: 0.1075, lr: 0.00083333
24+
step: 24, loss: 0.1112, W: 0.1343, b: 0.1076, lr: 0.00080000
25+
step: 25, loss: 0.0801, W: 0.1356, b: 0.1079, lr: 0.00076923
26+
step: 26, loss: 0.0566, W: 0.1363, b: 0.1082, lr: 0.00074074
27+
step: 27, loss: 0.0196, W: 0.1369, b: 0.1084, lr: 0.00071429
28+
step: 28, loss: 0.0904, W: 0.1371, b: 0.1085, lr: 0.00068966
29+
step: 29, loss: 0.1027, W: 0.1378, b: 0.1088, lr: 0.00066667
30+
step: 30, loss: 0.0190, W: 0.1389, b: 0.1091, lr: 0.00064516
31+
step: 31, loss: 0.0832, W: 0.1391, b: 0.1093, lr: 0.00062500
32+
step: 32, loss: 0.0679, W: 0.1398, b: 0.1096, lr: 0.00060606
33+
step: 33, loss: 0.0688, W: 0.1403, b: 0.1098, lr: 0.00058824
34+
step: 34, loss: 0.0167, W: 0.1410, b: 0.1101, lr: 0.00057143
35+
step: 35, loss: 0.0372, W: 0.1411, b: 0.1102, lr: 0.00055556
36+
step: 36, loss: 0.1210, W: 0.1413, b: 0.1104, lr: 0.00054054
37+
step: 37, loss: 0.0423, W: 0.1417, b: 0.1108, lr: 0.00052632
38+
step: 38, loss: 0.1329, W: 0.1421, b: 0.1110, lr: 0.00051282
39+
step: 39, loss: 0.0104, W: 0.1427, b: 0.1114, lr: 0.00050000
40+
step: 40, loss: 0.0220, W: 0.1429, b: 0.1115, lr: 0.00048780
41+
step: 41, loss: 0.1012, W: 0.1431, b: 0.1117, lr: 0.00047619
42+
step: 42, loss: 0.0748, W: 0.1439, b: 0.1121, lr: 0.00046512
43+
step: 43, loss: 0.0514, W: 0.1443, b: 0.1124, lr: 0.00045455
44+
step: 44, loss: 0.0186, W: 0.1447, b: 0.1126, lr: 0.00044444
45+
step: 45, loss: 0.0844, W: 0.1448, b: 0.1128, lr: 0.00043478
46+
step: 46, loss: 0.0936, W: 0.1453, b: 0.1131, lr: 0.00042553
47+
step: 47, loss: 0.0170, W: 0.1460, b: 0.1135, lr: 0.00041667
48+
step: 48, loss: 0.0773, W: 0.1462, b: 0.1137, lr: 0.00040816
49+
step: 49, loss: 0.0642, W: 0.1466, b: 0.1140, lr: 0.00040000
50+
step: 50, loss: 0.0630, W: 0.1469, b: 0.1143, lr: 0.00039216
51+
step: 51, loss: 0.0156, W: 0.1474, b: 0.1146, lr: 0.00038462
52+
step: 52, loss: 0.0355, W: 0.1475, b: 0.1148, lr: 0.00037736
53+
step: 53, loss: 0.1171, W: 0.1476, b: 0.1150, lr: 0.00037037
54+
step: 54, loss: 0.0396, W: 0.1479, b: 0.1154, lr: 0.00036364
55+
step: 55, loss: 0.1273, W: 0.1482, b: 0.1156, lr: 0.00035714
56+
step: 56, loss: 0.0088, W: 0.1487, b: 0.1161, lr: 0.00035088
57+
step: 57, loss: 0.0206, W: 0.1488, b: 0.1162, lr: 0.00034483
58+
step: 58, loss: 0.0946, W: 0.1489, b: 0.1164, lr: 0.00033898
59+
step: 59, loss: 0.0711, W: 0.1495, b: 0.1167, lr: 0.00033333
60+
step: 60, loss: 0.0479, W: 0.1498, b: 0.1170, lr: 0.00032787
61+
step: 61, loss: 0.0178, W: 0.1501, b: 0.1173, lr: 0.00032258
62+
step: 62, loss: 0.0803, W: 0.1502, b: 0.1175, lr: 0.00031746
63+
step: 63, loss: 0.0873, W: 0.1505, b: 0.1178, lr: 0.00031250
64+
step: 64, loss: 0.0156, W: 0.1510, b: 0.1182, lr: 0.00030769
65+
step: 65, loss: 0.0731, W: 0.1512, b: 0.1183, lr: 0.00030303
66+
step: 66, loss: 0.0614, W: 0.1515, b: 0.1186, lr: 0.00029851
67+
step: 67, loss: 0.0588, W: 0.1517, b: 0.1189, lr: 0.00029412
68+
step: 68, loss: 0.0148, W: 0.1521, b: 0.1192, lr: 0.00028986
69+
step: 69, loss: 0.0342, W: 0.1522, b: 0.1193, lr: 0.00028571
70+
step: 70, loss: 0.1142, W: 0.1523, b: 0.1195, lr: 0.00028169
71+
step: 71, loss: 0.0376, W: 0.1525, b: 0.1199, lr: 0.00027778
72+
step: 72, loss: 0.1230, W: 0.1527, b: 0.1201, lr: 0.00027397
73+
step: 73, loss: 0.0077, W: 0.1531, b: 0.1205, lr: 0.00027027
74+
step: 74, loss: 0.0195, W: 0.1532, b: 0.1206, lr: 0.00026667
75+
step: 75, loss: 0.0897, W: 0.1533, b: 0.1208, lr: 0.00026316
76+
step: 76, loss: 0.0684, W: 0.1537, b: 0.1211, lr: 0.00025974
77+
step: 77, loss: 0.0452, W: 0.1539, b: 0.1214, lr: 0.00025641
78+
step: 78, loss: 0.0172, W: 0.1542, b: 0.1216, lr: 0.00025316
79+
step: 79, loss: 0.0771, W: 0.1542, b: 0.1217, lr: 0.00025000
80+
step: 80, loss: 0.0826, W: 0.1545, b: 0.1220, lr: 0.00024691
81+
step: 81, loss: 0.0146, W: 0.1549, b: 0.1223, lr: 0.00024390
82+
step: 82, loss: 0.0699, W: 0.1550, b: 0.1224, lr: 0.00024096
83+
step: 83, loss: 0.0592, W: 0.1553, b: 0.1227, lr: 0.00023810
84+
step: 84, loss: 0.0557, W: 0.1555, b: 0.1229, lr: 0.00023529
85+
step: 85, loss: 0.0142, W: 0.1558, b: 0.1232, lr: 0.00023256

‎tests/TensorFlowSharp.Tests.CSharp/TestData/RMSPropTimeDecay/optimizer_lr_test.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# This script is used to create data file (expected.txt)
2+
# which is used to compare the output from TensorFlowSharp optimizer tests.
3+
4+
import tensorflow as tf
5+
6+
# Training data
7+
train_x =[
8+
3.3, 4.4, 5.5, 6.71, 6.93, 4.168, 9.779, 6.182, 7.59, 2.167,
9+
7.042, 10.791, 5.313, 7.997, 5.654, 9.27, 3.1
10+
]
11+
train_y = [
12+
1.7, 2.76,2.09,3.19,1.694,1.573,3.366,2.596,2.53,1.221,
13+
2.827,3.465,1.65,2.904,2.42,2.94,1.3
14+
]
15+
n_samples = len(train_x)
16+
X = tf.placeholder(tf.float32)
17+
Y = tf.placeholder(tf.float32)
18+
19+
W = tf.Variable(tf.constant(0.1), dtype=tf.float32)
20+
b = tf.Variable(tf.constant(0.1), dtype=tf.float32)
21+
22+
pred = tf.add(tf.multiply(X,W), b)
23+
24+
global_step = tf.Variable(0, trainable=False)
25+
learning_rate = 0.01
26+
decay_rate = 0.5
27+
decayed_learning_rate = learning_rate * (1. / (1. + decay_rate * tf.cast(global_step, tf.float32)))
28+
29+
cost = tf.divide(tf.reduce_sum(tf.pow(tf.subtract(pred, Y), 2.0)), tf.multiply(2.0, n_samples))
30+
optimizer = tf.train.RMSPropOptimizer(decayed_learning_rate, momentum=0.0,epsilon=1e-7).minimize(cost, global_step=global_step, name = "AdagradOptimizer")
31+
32+
init = tf.global_variables_initializer()
33+
with tf.Session() as session:
34+
session.run(init)
35+
for e in range(5):
36+
for i in range(n_samples):
37+
_, cost_v, W_v, b_v, pred_v, lr_v, step_v = session.run([optimizer, cost, W, b, pred, decayed_learning_rate, global_step], feed_dict = {X: train_x[i], Y: train_y[i]})
38+
print(f"step: {step_v:d}, loss: {cost_v:.4f}, W: {W_v:.4f}, b: {b_v:.4f}, lr: {lr_v:.8f}")
39+
#print("Prediction: %f == Actual: %f" % (pred_v, train_y[i]))

0 commit comments

Comments
 (0)