Skip to content

Commit e060acd

Browse files
migueldeicazamigueldeicaza
committed
Upgraded generator to support the 1.6 style of API documentation, improve generated docs
1 parent 9288146 commit e060acd

2 files changed

Lines changed: 5201 additions & 3905 deletions

File tree

‎OpGenerator/OpGenerator.cs‎

Lines changed: 112 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,38 @@
2424

2525
class ApiDefMap : IDisposable
2626
{
27+
public class Status : IDisposable
28+
{
29+
[DllImport ("libtensorflow")]
30+
static extern unsafe IntPtr TF_NewStatus ();
31+
32+
[DllImport ("libtensorflow")]
33+
internal static extern unsafe void TF_DeleteStatus (IntPtr status);
34+
35+
[DllImport ("libtensorflow")]
36+
static extern unsafe int TF_GetCode (IntPtr s);
37+
38+
IntPtr handle;
39+
public Status ()
40+
{
41+
handle = TF_NewStatus ();
42+
}
43+
44+
void IDisposable.Dispose ()
45+
{
46+
TF_DeleteStatus (handle);
47+
handle = IntPtr.Zero;
48+
}
49+
50+
public bool Ok => TF_GetCode (handle) == 0;
51+
public bool Error => TF_GetCode (handle) != 0;
52+
53+
public static implicit operator IntPtr (Status s)
54+
{
55+
return s.handle;
56+
}
57+
}
58+
2759
[DllImport ("libtensorflow")]
2860
unsafe extern static IntPtr TF_NewApiDefMap (IntPtr buffer, IntPtr status);
2961

@@ -36,23 +68,16 @@ class ApiDefMap : IDisposable
3668
[DllImport ("libtensorflow")]
3769
unsafe static extern OpGenerator.LLBuffer *TF_ApiDefMapGet (IntPtr handle, string name, IntPtr nameLen, IntPtr status);
3870

39-
// extern TF_Status * TF_NewStatus ();
40-
[DllImport ("libtensorflow")]
41-
static extern unsafe IntPtr TF_NewStatus ();
42-
43-
[DllImport ("libtensorflow")]
44-
static extern unsafe int TF_GetCode (IntPtr s);
45-
4671
IntPtr handle;
47-
IntPtr status;
4872

4973
unsafe public ApiDefMap (OpGenerator.LLBuffer* buffer)
5074
{
51-
status = TF_NewStatus ();
75+
using (var status = new Status ()) {
76+
handle = TF_NewApiDefMap ((IntPtr)buffer, status);
5277

53-
handle = TF_NewApiDefMap ((IntPtr)buffer, status);
54-
if (TF_GetCode (status) != 0)
55-
throw new ArgumentException ("Failure to call TF_NewApiDefMap");
78+
if (status.Error)
79+
throw new ArgumentException ("Failure to call TF_NewApiDefMap");
80+
}
5681
}
5782

5883
void IDisposable.Dispose ()
@@ -78,15 +103,26 @@ void Dispose (bool disposing)
78103

79104
public unsafe ApiDef Get (string name)
80105
{
81-
var ptr = TF_ApiDefMapGet (handle, name, (IntPtr)name.Length, status);
82-
if (TF_GetCode (status) != 0)
83-
return null;
84-
var ret = new byte [(int)ptr->length];
85-
Marshal.Copy (ptr->data, ret, 0, (int)ptr->length);
86-
var str = new MemoryStream (ret);
87-
return Serializer.Deserialize<ApiDef> (str);
106+
using (var status = new Status ()) {
107+
var ptr = TF_ApiDefMapGet (handle, name, (IntPtr)name.Length, status);
108+
if (status.Error)
109+
return null;
110+
var ret = new byte [(int)ptr->length];
111+
Marshal.Copy (ptr->data, ret, 0, (int)ptr->length);
112+
var str = new MemoryStream (ret);
113+
return Serializer.Deserialize<ApiDef> (str);
114+
}
88115
}
89116

117+
public unsafe bool Put (string text)
118+
{
119+
using (var status = new Status ()) {
120+
TF_ApiDefMapPut (handle, text, (IntPtr)text.Length, status);
121+
if (status.Error)
122+
return false;
123+
return true;
124+
}
125+
}
90126
}
91127

92128
class OpGenerator
@@ -235,27 +271,52 @@ string FillArguments (OpDef def)
235271
return sb.ToString ();
236272
}
237273

274+
string Quote (string input)
275+
{
276+
var p = input.IndexOf ('`');
277+
if (p == -1)
278+
return input;
279+
var res = new StringBuilder ();
280+
bool open = true;
281+
foreach (var c in input) {
282+
if (c == '`') {
283+
res.Append (open ? "<code>" : "</code>");
284+
open = !open;
285+
} else
286+
res.Append (c);
287+
}
288+
return res.ToString ();
289+
}
290+
238291
void Comment (string text)
239292
{
240293
if (text == null || text == "")
241294
return;
242295
var lines = text.Split ('\n');
296+
bool open = true;
243297
foreach (var line in lines) {
244-
var line2 = line.Replace ("<", "&lt;").Replace (">", "&gt;").Replace ("&", "&amp;");
245-
p ($"/// {line2}");
298+
var line2 = line.Trim ().Replace ("<", "&lt;").Replace (">", "&gt;").Replace ("&", "&amp;");
299+
300+
if (line == "```") {
301+
p ("/// " + (open ? "<code>" : "</code>"));
302+
open = !open;
303+
} else {
304+
p ($"/// {Quote (line2)}");
305+
}
246306
}
247307
}
248308

249309

250310
// Produces the C# inline documentation
251311
void GenDocs (OpDef oper)
252312
{
313+
var api = apimap.Get (oper.name);
253314
p ("/// <summary>");
254-
Comment (oper.summary);
315+
Comment (api.Summary);
255316
p ("/// </summary>");
256-
foreach (var input in oper.input_arg) {
257-
p ($"/// <param name=\"{ParamMap (input.name)}\">");
258-
Comment (input.description);
317+
foreach (var input in api.InArgs) {
318+
p ($"/// <param name=\"{ParamMap (input.Name)}\">");
319+
Comment (input.Description);
259320
p ($"/// </param>");
260321
}
261322
#if DOCS
@@ -275,23 +336,25 @@ void GenDocs (OpDef oper)
275336
foreach (var attr in optional_attrs) {
276337
p ($"/// <param name=\"{ParamMap (attr.name)}\">");
277338
Comment ("Optional argument");
278-
Comment (attr.description);
339+
340+
Comment (api.Attrs.Where (x=>x.Name == attr.name).FirstOrDefault ().Description);
279341
p ($"/// </param>");
280342
}
281343
foreach (var attr in required_attrs) {
282344
p ($"/// <param name=\"{ParamMap (attr.name)}\">");
283-
Comment (attr.description);
345+
Comment (api.Attrs.Where (x=>x.Name == attr.name).FirstOrDefault ().Description);
284346
p ($"/// </param>");
285347
}
286348
p ($"/// <returns>");
287349
if (have_return_value) {
288350
if (oper.output_arg.Count == 1) {
289-
Comment (oper.output_arg.First ().description);
351+
Comment (api.OutArgs.First ().Description);
290352
Comment ("The TFOperation can be fetched from the resulting TFOutput, by fethching the Operation property from the result.");
291353
} else {
292354
Comment ("Returns a tuple with multiple values, as follows:");
293355
foreach (var arg in oper.output_arg) {
294-
Comment (ParamMap (arg.name) + ": " + arg.description);
356+
var oapi = api.OutArgs.Where (x => x.Name == arg.name).FirstOrDefault ();
357+
Comment (ParamMap (arg.name) + ": " + oapi.Description);
295358
}
296359

297360
Comment ("The TFOperation can be fetched from any of the TFOutputs returned in the tuple values, by fethching the Operation property.");
@@ -301,9 +364,9 @@ void GenDocs (OpDef oper)
301364
}
302365
p ($"/// </returns>");
303366

304-
if (!String.IsNullOrEmpty (oper.description)) {
367+
if (!String.IsNullOrEmpty (api.Description)) {
305368
p ("/// <remarks>");
306-
Comment (oper.description);
369+
Comment (api.Description);
307370
p ("/// </remarks>");
308371
}
309372
}
@@ -464,10 +527,22 @@ MemoryStream GetOpsList ()
464527
}
465528
}
466529

467-
void Run ()
530+
// Incorporates out-of-band data into the API definitions that we pulled out of GetAllOpList
531+
void UpdateApis (string [] dirs)
532+
{
533+
foreach (var dir in dirs) {
534+
foreach (var f in Directory.GetFiles (dir)) {
535+
var s = File.ReadAllText (f);
536+
apimap.Put (s);
537+
}
538+
}
539+
}
540+
541+
void Run (string [] dirs)
468542
{
469543
output = File.CreateText ("../../../TensorFlowSharp/Operations.g.cs");
470544
var operations = Serializer.Deserialize<List<OpDef>> (GetOpsList ());
545+
UpdateApis (dirs);
471546
p ("using System;\n");
472547

473548
pi ("namespace TensorFlow {");
@@ -505,7 +580,7 @@ void Run ()
505580
var def = apimap.Get (oper.name);
506581

507582
// Undocumented operation, perhaps we should not surface
508-
if (oper.summary == "")
583+
if (def.Summary == "")
509584
continue;
510585

511586
Generate (oper);
@@ -547,6 +622,9 @@ public static void Main (string [] args)
547622
{
548623
if (Marshal.SizeOf (typeof (IntPtr)) != 8)
549624
throw new Exception ("Need to run in 64");
550-
new OpGenerator ().Run ();
625+
if (args.Length == 0)
626+
args = new string [] { "/cvs/tensorflow/tensorflow/core/api_def/base_api" };
627+
628+
new OpGenerator ().Run (args);
551629
}
552630
}

0 commit comments

Comments
 (0)