Skip to content

Commit 1d655c0

Browse files
skynodeskynode
authored
Merge pull request #4 from migueldeicaza/master
PR to merge into skynode master branch
2 parents dc41fd2 + a457072 commit 1d655c0

89 files changed

Lines changed: 89227 additions & 41588 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.travis.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,5 +11,6 @@ script:
1111
- cp -R $TRAVIS_BUILD_DIR/native/*.dylib $TRAVIS_BUILD_DIR/tests/TensorFlowSharp.Tests.CSharp/bin/Release/
1212
- cd $TRAVIS_BUILD_DIR/packages/xunit.runner.console.2.2.0/tools
1313
- mono --arch=64 xunit.console.exe "$TRAVIS_BUILD_DIR/tests/TensorFlowSharp.Tests/bin/Release/TensorFlowSharp.Tests.dll" "$TRAVIS_BUILD_DIR/tests/TensorFlowSharp.Tests.CSharp/bin/Release/TensorFlowSharp.Tests.CSharp.dll"
14-
mono: "5.2.0"
14+
mono: "5.4.1"
15+
dotnet: "2.1.4"
1516
os: "osx"

‎Examples/ExampleCommon/ImageUtil.cs‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -13,20 +13,19 @@ public static TFTensor CreateTensorFromImageFile (string file, TFDataType destin
1313
// DecodeJpeg uses a scalar String-valued tensor as input.
1414
var tensor = TFTensor.CreateString (contents);
1515

16-
TFGraph graph;
1716
TFOutput input, output;
1817

1918
// Construct a graph to normalize the image
20-
ConstructGraphToNormalizeImage (out graph, out input, out output, destinationDataType);
21-
22-
// Execute that graph to normalize this one image
23-
using (var session = new TFSession (graph)) {
24-
var normalized = session.Run (
25-
inputs: new [] { input },
26-
inputValues: new [] { tensor },
27-
outputs: new [] { output });
28-
29-
return normalized [0];
19+
using (var graph = ConstructGraphToNormalizeImage (out input, out output, destinationDataType)){
20+
// Execute that graph to normalize this one image
21+
using (var session = new TFSession (graph)) {
22+
var normalized = session.Run (
23+
inputs: new [] { input },
24+
inputValues: new [] { tensor },
25+
outputs: new [] { output });
26+
27+
return normalized [0];
28+
}
3029
}
3130
}
3231

@@ -37,7 +36,7 @@ public static TFTensor CreateTensorFromImageFile (string file, TFDataType destin
3736
// This function constructs a graph of TensorFlow operations which takes as
3837
// input a JPEG-encoded string and returns a tensor suitable as input to the
3938
// inception model.
40-
private static void ConstructGraphToNormalizeImage (out TFGraph graph, out TFOutput input, out TFOutput output, TFDataType destinationDataType = TFDataType.Float)
39+
private static TFGraph ConstructGraphToNormalizeImage (out TFOutput input, out TFOutput output, TFDataType destinationDataType = TFDataType.Float)
4140
{
4241
// Some constants specific to the pre-trained model at:
4342
// https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip
@@ -51,7 +50,7 @@ private static void ConstructGraphToNormalizeImage (out TFGraph graph, out TFOut
5150
const float Mean = 117;
5251
const float Scale = 1;
5352

54-
graph = new TFGraph ();
53+
var graph = new TFGraph ();
5554
input = graph.Placeholder (TFDataType.String);
5655

5756
output = graph.Cast (graph.Div (
@@ -64,6 +63,8 @@ private static void ConstructGraphToNormalizeImage (out TFGraph graph, out TFOut
6463
size: graph.Const (new int [] { W, H }, "size")),
6564
y: graph.Const (Mean, "mean")),
6665
y: graph.Const (Scale, "scale")), destinationDataType);
66+
67+
return graph;
6768
}
6869
}
6970
}

‎OpGenerator/OpGenerator.cs‎

Lines changed: 180 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,109 @@
2222
using System.Runtime.InteropServices;
2323
using tensorflow;
2424

25+
class ApiDefMap : IDisposable
26+
{
27+
public class Status : IDisposable
28+
{
29+
[DllImport ("libtensorflow")]
30+
static extern unsafe IntPtr TF_NewStatus ();
31+
32+
[DllImport ("libtensorflow")]
33+
internal static extern unsafe void TF_DeleteStatus (IntPtr status);
34+
35+
[DllImport ("libtensorflow")]
36+
static extern unsafe int TF_GetCode (IntPtr s);
37+
38+
IntPtr handle;
39+
public Status ()
40+
{
41+
handle = TF_NewStatus ();
42+
}
43+
44+
void IDisposable.Dispose ()
45+
{
46+
TF_DeleteStatus (handle);
47+
handle = IntPtr.Zero;
48+
}
49+
50+
public bool Ok => TF_GetCode (handle) == 0;
51+
public bool Error => TF_GetCode (handle) != 0;
52+
53+
public static implicit operator IntPtr (Status s)
54+
{
55+
return s.handle;
56+
}
57+
}
58+
59+
[DllImport ("libtensorflow")]
60+
unsafe extern static IntPtr TF_NewApiDefMap (IntPtr buffer, IntPtr status);
61+
62+
[DllImport ("libtensorflow")]
63+
static extern void TF_DeleteApiDefMap (IntPtr handle);
64+
65+
[DllImport ("libtensorflow")]
66+
static extern void TF_ApiDefMapPut (IntPtr handle, string text, IntPtr textLen, IntPtr status);
67+
68+
[DllImport ("libtensorflow")]
69+
unsafe static extern OpGenerator.LLBuffer *TF_ApiDefMapGet (IntPtr handle, string name, IntPtr nameLen, IntPtr status);
70+
71+
IntPtr handle;
72+
73+
unsafe public ApiDefMap (OpGenerator.LLBuffer* buffer)
74+
{
75+
using (var status = new Status ()) {
76+
handle = TF_NewApiDefMap ((IntPtr)buffer, status);
77+
78+
if (status.Error)
79+
throw new ArgumentException ("Failure to call TF_NewApiDefMap");
80+
}
81+
}
82+
83+
void IDisposable.Dispose ()
84+
{
85+
Dispose (true);
86+
GC.SuppressFinalize (this);
87+
}
88+
89+
~ApiDefMap ()
90+
{
91+
Dispose (false);
92+
}
93+
94+
void Dispose (bool disposing)
95+
{
96+
if (disposing) {
97+
if (handle != IntPtr.Zero) {
98+
TF_DeleteApiDefMap (handle);
99+
handle = IntPtr.Zero;
100+
}
101+
}
102+
}
103+
104+
public unsafe ApiDef Get (string name)
105+
{
106+
using (var status = new Status ()) {
107+
var ptr = TF_ApiDefMapGet (handle, name, (IntPtr)name.Length, status);
108+
if (status.Error)
109+
return null;
110+
var ret = new byte [(int)ptr->length];
111+
Marshal.Copy (ptr->data, ret, 0, (int)ptr->length);
112+
var str = new MemoryStream (ret);
113+
return Serializer.Deserialize<ApiDef> (str);
114+
}
115+
}
116+
117+
public unsafe bool Put (string text)
118+
{
119+
using (var status = new Status ()) {
120+
TF_ApiDefMapPut (handle, text, (IntPtr)text.Length, status);
121+
if (status.Error)
122+
return false;
123+
return true;
124+
}
125+
}
126+
}
127+
25128
class OpGenerator
26129
{
27130
//
@@ -173,22 +276,64 @@ void Comment (string text)
173276
if (text == null || text == "")
174277
return;
175278
var lines = text.Split ('\n');
279+
var open = true;
280+
281+
string Quote (string input)
282+
{
283+
var p = input.IndexOf ('`');
284+
if (p == -1)
285+
return input;
286+
var res = new StringBuilder ();
287+
foreach (var c in input) {
288+
if (c == '`') {
289+
res.Append (open ? "<c>" : "</c>");
290+
open = !open;
291+
} else
292+
res.Append (c);
293+
}
294+
return res.ToString ();
295+
}
296+
297+
bool blockOpen = true;
176298
foreach (var line in lines) {
177-
var line2 = line.Replace ("<", "&lt;").Replace (">", "&gt;").Replace ("&", "&amp;");
178-
p ($"/// {line2}");
299+
if (line.IndexOf ("in image height coordinates.") != -1) {
300+
Console.WriteLine ("Hello");
301+
}
302+
303+
var line2 = line.Trim ().Replace ("<", "&lt;").Replace (">", "&gt;").Replace ("&", "&amp;");
304+
305+
if (line2.StartsWith ("```")){
306+
p ("/// " + (blockOpen ? "<code>" : "</code>"));
307+
blockOpen = !blockOpen;
308+
if (line2 == "```python" || line2 == "```c++" || line2 == "```")
309+
continue;
310+
// Handle some broken comments in the api specs, they sometimes missuse the
311+
312+
line2 = line2.Substring (3);
313+
if (line2.EndsWith ("```")){
314+
var line3 = line2.Substring (0, line2.Length - 3);
315+
p ($"/// {Quote (line3)}");
316+
p ("/// " + (blockOpen ? "<code>" : "</code>"));
317+
blockOpen = !blockOpen;
318+
continue;
319+
}
320+
}
321+
p ($"/// {Quote (line2)}");
322+
179323
}
180324
}
181325

182326

183327
// Produces the C# inline documentation
184328
void GenDocs (OpDef oper)
185329
{
330+
var api = apimap.Get (oper.name);
186331
p ("/// <summary>");
187-
Comment (oper.summary);
332+
Comment (api.Summary);
188333
p ("/// </summary>");
189-
foreach (var input in oper.input_arg) {
190-
p ($"/// <param name=\"{ParamMap (input.name)}\">");
191-
Comment (input.description);
334+
foreach (var input in api.InArgs) {
335+
p ($"/// <param name=\"{ParamMap (input.Name)}\">");
336+
Comment (input.Description);
192337
p ($"/// </param>");
193338
}
194339
#if DOCS
@@ -208,23 +353,25 @@ void GenDocs (OpDef oper)
208353
foreach (var attr in optional_attrs) {
209354
p ($"/// <param name=\"{ParamMap (attr.name)}\">");
210355
Comment ("Optional argument");
211-
Comment (attr.description);
356+
357+
Comment (api.Attrs.Where (x=>x.Name == attr.name).FirstOrDefault ().Description);
212358
p ($"/// </param>");
213359
}
214360
foreach (var attr in required_attrs) {
215361
p ($"/// <param name=\"{ParamMap (attr.name)}\">");
216-
Comment (attr.description);
362+
Comment (api.Attrs.Where (x=>x.Name == attr.name).FirstOrDefault ().Description);
217363
p ($"/// </param>");
218364
}
219365
p ($"/// <returns>");
220366
if (have_return_value) {
221367
if (oper.output_arg.Count == 1) {
222-
Comment (oper.output_arg.First ().description);
368+
Comment (api.OutArgs.First ().Description);
223369
Comment ("The TFOperation can be fetched from the resulting TFOutput, by fethching the Operation property from the result.");
224370
} else {
225371
Comment ("Returns a tuple with multiple values, as follows:");
226372
foreach (var arg in oper.output_arg) {
227-
Comment (ParamMap (arg.name) + ": " + arg.description);
373+
var oapi = api.OutArgs.Where (x => x.Name == arg.name).FirstOrDefault ();
374+
Comment (ParamMap (arg.name) + ": " + oapi.Description);
228375
}
229376

230377
Comment ("The TFOperation can be fetched from any of the TFOutputs returned in the tuple values, by fethching the Operation property.");
@@ -234,9 +381,9 @@ void GenDocs (OpDef oper)
234381
}
235382
p ($"/// </returns>");
236383

237-
if (!String.IsNullOrEmpty (oper.description)) {
384+
if (!String.IsNullOrEmpty (api.Description)) {
238385
p ("/// <remarks>");
239-
Comment (oper.description);
386+
Comment (api.Description);
240387
p ("/// </remarks>");
241388
}
242389
}
@@ -383,23 +530,36 @@ internal struct LLBuffer
383530

384531
[DllImport ("libtensorflow")]
385532
unsafe extern static LLBuffer *TF_GetAllOpList ();
533+
ApiDefMap apimap;
386534

387535
MemoryStream GetOpsList ()
388536
{
389537
unsafe
390538
{
391539
LLBuffer* ptr = TF_GetAllOpList ();
540+
apimap = new ApiDefMap (ptr);
392541
var ret = new byte [(int)ptr->length];
393542
Marshal.Copy (ptr->data, ret, 0, (int)ptr->length);
394543
return new MemoryStream (ret);
395544
}
396545
}
397546

398-
void Run ()
547+
// Incorporates out-of-band data into the API definitions that we pulled out of GetAllOpList
548+
void UpdateApis (string [] dirs)
549+
{
550+
foreach (var dir in dirs) {
551+
foreach (var f in Directory.GetFiles (dir)) {
552+
var s = File.ReadAllText (f);
553+
apimap.Put (s);
554+
}
555+
}
556+
}
557+
558+
void Run (string [] dirs)
399559
{
400-
401560
output = File.CreateText ("../../../TensorFlowSharp/Operations.g.cs");
402561
var operations = Serializer.Deserialize<List<OpDef>> (GetOpsList ());
562+
UpdateApis (dirs);
403563
p ("using System;\n");
404564

405565
pi ("namespace TensorFlow {");
@@ -434,9 +594,10 @@ void Run ()
434594
continue;
435595
}
436596
#endif
597+
var def = apimap.Get (oper.name);
437598

438599
// Undocumented operation, perhaps we should not surface
439-
if (oper.summary == "")
600+
if (def.Summary == "")
440601
continue;
441602

442603
Generate (oper);
@@ -478,6 +639,9 @@ public static void Main (string [] args)
478639
{
479640
if (Marshal.SizeOf (typeof (IntPtr)) != 8)
480641
throw new Exception ("Need to run in 64");
481-
new OpGenerator ().Run ();
642+
if (args.Length == 0)
643+
args = new string [] { "/cvs/tensorflow/tensorflow/core/api_def/base_api" };
644+
645+
new OpGenerator ().Run (args);
482646
}
483647
}

‎OpGenerator/OpGenerator.csproj‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
<WarningLevel>4</WarningLevel>
3030
<ExternalConsole>true</ExternalConsole>
3131
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
32+
<PlatformTarget>x64</PlatformTarget>
3233
</PropertyGroup>
3334
<ItemGroup>
3435
<Reference Include="System" />
@@ -44,6 +45,7 @@
4445
<Compile Include="OpGenerator.cs" />
4546
<Compile Include="Properties\AssemblyInfo.cs" />
4647
<Compile Include="Opdefs.cs" />
48+
<Compile Include="api_def.cs" />
4749
</ItemGroup>
4850
<ItemGroup>
4951
<Folder Include="New Folder\" />

0 commit comments

Comments
 (0)