Skip to content

Commit

Permalink
Improvements to image handling
Browse files Browse the repository at this point in the history
  • Loading branch information
@olokobayusuf
olokobayusuf committed Apr 14, 2024
1 parent 636e9f2 commit 558d2d1
Show file tree
Hide file tree
Showing 17 changed files with 153 additions and 131 deletions.
2 changes: 1 addition & 1 deletion Assets/Tests/Editor/EnvironmentTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ internal sealed class EnvironmentTest {
private Function fxn;

[SetUp]
public void Before () => fxn = FunctionUnity.Create(url: @"https://api.fxn.dev/graph");
public void Before () => fxn = FunctionUnity.Create(url: @"https://api.fxn.dev");

[Test(Description = @"Should list user environment variables")]
public async Task ListEnvironmentVariables () {
Expand Down
2 changes: 1 addition & 1 deletion Assets/Tests/Editor/PredictionTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ internal sealed class PredictionTest {
private Function fxn;

[SetUp]
public void Before () => fxn = FunctionUnity.Create(url: @"https://api.fxn.dev/graph");
public void Before () => fxn = FunctionUnity.Create(url: @"https://api.fxn.dev");

[Test(Description = @"Should create a cloud prediction")]
public async Task CreateCloudPrediction () {
Expand Down
2 changes: 1 addition & 1 deletion Assets/Tests/Editor/PredictorTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ internal sealed class PredictorTest {
private Function fxn;

[SetUp]
public void Before () => fxn = FunctionUnity.Create(url: @"https://api.fxn.dev/graph");
public void Before () => fxn = FunctionUnity.Create(url: @"https://api.fxn.dev");

[Test(Description = @"Should retrieve a valid predictor")]
public async Task RetrievePredictor () {
Expand Down
5 changes: 1 addition & 4 deletions Assets/Tests/Editor/StorageTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,18 +9,15 @@ namespace Function.Tests {
using System.IO;
using System.Threading.Tasks;
using NUnit.Framework;
using UnityEngine;
using UnityEngine.TestTools;
using Unity.Collections.LowLevel.Unsafe;
using Services;
using Types;

internal sealed class StorageTest {

private Function fxn;

[SetUp]
public void Before () => fxn = FunctionUnity.Create(url: @"https://api.fxn.dev/graph");
public void Before () => fxn = FunctionUnity.Create(url: @"https://api.fxn.dev");

[Test(Description = @"Should create an upload URL")]
public async Task CreateUploadURL () {
Expand Down
2 changes: 1 addition & 1 deletion Assets/Tests/Editor/UserTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ internal sealed class UserTest {
private Function fxn;

[SetUp]
public void Before () => fxn = FunctionUnity.Create(url: @"https://api.fxn.dev/graph");
public void Before () => fxn = FunctionUnity.Create(url: @"https://api.fxn.dev");

[Test(Description = @"Should retrieve the current user")]
public async Task RetrieveUser () {
Expand Down
2 changes: 1 addition & 1 deletion Assets/Tests/Runtime/EdgeFunctionTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ private async void Update () {
Debug.Log(JsonConvert.SerializeObject(prediction, Formatting.Indented));
}

private async void OnDestroy () {
private async void OnDisable () {
var deleted = await fxn.Predictions.Delete(Tag);
Debug.Log($"Deleted predictor: {deleted}");
}
Expand Down
7 changes: 3 additions & 4 deletions Assets/Tests/Runtime/ImageTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,11 @@ internal sealed class ImageTest : MonoBehaviour {

private async void Start () {
// Predict
var fxn = FunctionUnity.Create(url: @"https://api.fxn.dev/graph");
var fxn = FunctionUnity.Create(url: @"https://api.fxn.dev");
var prediction = await fxn.Predictions.Create(
"@natml/auto-test-v1",
"@yusuf/image-identity",
new () {
["image"] = await image.ToValue(),
["contrast"] = contrast
["image"] = image.ToImage(),
}
);
// Display
Expand Down
10 changes: 6 additions & 4 deletions Assets/Tests/Runtime/StreamingTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,21 @@

namespace Function.Tests {

using System.Collections.Generic;
using UnityEngine;
using Newtonsoft.Json;
using API;
using Internal;

internal sealed class StreamingTest : MonoBehaviour {

private async void Start () {
var fxn = FunctionUnity.Create("fxn-aVPMIDpsxTsr4of8dknUt", "https://api.fxn.dev");
var fxn = FunctionUnity.Create(
accessKey: FunctionSettings.Instance.accessKey,
url: @"https://api.fxn.dev"
);
var stream = fxn.Predictions.Stream(
tag: "@yusuf-delete/streaming",
inputs: new () {
["sentence"] = "Hello world"
["sentence"] = @"Hello world"
}
);
await foreach (var prediction in stream)
Expand Down
36 changes: 10 additions & 26 deletions Assets/Tests/Tests.unity
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ RenderSettings:
m_ReflectionIntensity: 1
m_CustomReflection: {fileID: 0}
m_Sun: {fileID: 705507994}
m_IndirectSpecularColor: {r: 0.44407493, g: 0.49331808, b: 0.5724013, a: 1}
m_IndirectSpecularColor: {r: 0.44657838, g: 0.49641228, b: 0.57481676, a: 1}
m_UseRadianceAmbientProbe: 0
--- !u!157 &3
LightmapSettings:
Expand Down Expand Up @@ -104,7 +104,7 @@ NavMeshSettings:
serializedVersion: 2
m_ObjectHideFlags: 0
m_BuildSettings:
serializedVersion: 3
serializedVersion: 2
agentTypeID: 0
agentRadius: 0.5
agentHeight: 2
Expand All @@ -117,7 +117,7 @@ NavMeshSettings:
cellSize: 0.16666667
manualTileSize: 0
tileSize: 256
buildHeightMesh: 0
accuratePlacement: 0
maxJobWorkers: 0
preserveTilesOutsideBounds: 0
debug:
Expand Down Expand Up @@ -201,7 +201,6 @@ Canvas:
m_SortingBucketNormalizedSize: 0
m_VertexColorAlwaysGammaSpace: 0
m_AdditionalShaderChannelsFlag: 0
m_UpdateRectTransformForStandalone: 0
m_SortingLayerID: 0
m_SortingOrder: 0
m_TargetDisplay: 0
Expand All @@ -219,6 +218,7 @@ RectTransform:
m_Children:
- {fileID: 1654635356}
m_Father: {fileID: 0}
m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_AnchorMin: {x: 0, y: 0}
m_AnchorMax: {x: 0, y: 0}
Expand Down Expand Up @@ -311,13 +311,13 @@ Transform:
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 705507993}
serializedVersion: 2
m_LocalRotation: {x: 0.40821788, y: -0.23456968, z: 0.10938163, w: 0.8754261}
m_LocalPosition: {x: 0, y: 3, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0
m_Children: []
m_Father: {fileID: 0}
m_RootOrder: 1
m_LocalEulerAnglesHint: {x: 50, y: -30, z: 0}
--- !u!1 &963194225
GameObject:
Expand Down Expand Up @@ -359,17 +359,9 @@ Camera:
m_projectionMatrixMode: 1
m_GateFitMode: 2
m_FOVAxisMode: 0
m_Iso: 200
m_ShutterSpeed: 0.005
m_Aperture: 16
m_FocusDistance: 10
m_FocalLength: 50
m_BladeCount: 5
m_Curvature: {x: 2, y: 11}
m_BarrelClipping: 0.25
m_Anamorphism: 0
m_SensorSize: {x: 36, y: 24}
m_LensShift: {x: 0, y: 0}
m_FocalLength: 50
m_NormalizedViewPortRect:
serializedVersion: 2
x: 0
Expand Down Expand Up @@ -403,13 +395,13 @@ Transform:
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 963194225}
serializedVersion: 2
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: 0, y: 1, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0
m_Children: []
m_Father: {fileID: 0}
m_RootOrder: 3
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!1 &1167683551
GameObject:
Expand Down Expand Up @@ -442,13 +434,13 @@ Transform:
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1167683551}
serializedVersion: 2
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0
m_Children: []
m_Father: {fileID: 0}
m_RootOrder: 4
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!114 &1167683554
MonoBehaviour:
Expand Down Expand Up @@ -610,13 +602,13 @@ Transform:
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1413509856}
serializedVersion: 2
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0
m_Children: []
m_Father: {fileID: 0}
m_RootOrder: 2
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!1 &1654635355
GameObject:
Expand Down Expand Up @@ -650,6 +642,7 @@ RectTransform:
m_ConstrainProportionsScale: 0
m_Children: []
m_Father: {fileID: 289723530}
m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_AnchorMin: {x: 0, y: 0}
m_AnchorMax: {x: 0, y: 0}
Expand Down Expand Up @@ -705,12 +698,3 @@ CanvasRenderer:
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1654635355}
m_CullTransparentMesh: 1
--- !u!1660057539 &9223372036854775807
SceneRoots:
m_ObjectHideFlags: 0
m_Roots:
- {fileID: 963194228}
- {fileID: 705507995}
- {fileID: 289723530}
- {fileID: 1413509859}
- {fileID: 1167683553}
2 changes: 2 additions & 0 deletions Changelog.md
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
## 0.0.13
+ Added support for Unity 2021 LTS.
+ Updated `FunctionUnity.ToImage` extension method to accept an optional buffer to avoid allocating memory.
+ Removed `FunctionUnity.ToValue(Texture2D)` extension method. Use `FunctionUnity.ToImage` method instead.

## 0.0.12
+ Added `Function.Types.Image` struct for making edge predictions on images.
Expand Down
13 changes: 13 additions & 0 deletions Packages/ai.fxn.fxn3d/Runtime/Internal/Function.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,19 @@ out IntPtr value
);
[DllImport(Assembly, EntryPoint = @"FXNValueCreateNull")]
public static extern Status CreateNullValue (out IntPtr value);
[DllImport(Assembly, EntryPoint = @"FXNValueCreateBySerializingValue")]
public static extern Status CreateSerializedValue (
this IntPtr value,
ValueFlags flags,
out IntPtr result
);
[DllImport(Assembly, EntryPoint = @"FXNValueCreateByDeserializingValue")]
public static extern Status CreateDeserializedValue (
this IntPtr value,
Dtype type,
ValueFlags flags,
out IntPtr result
);
#endregion


Expand Down
37 changes: 28 additions & 9 deletions Packages/ai.fxn.fxn3d/Runtime/Services/Prediction.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,13 +188,15 @@ public Task<bool> Delete (string tag) {
/// <param name="minUploadSize">Values larger than this size in bytes will be uploaded.</param>
/// <returns>Function value.</returns>
public async Task<Value> ToValue (
object value,
object? value,
string name,
Dtype? type = null,
int minUploadSize = 4096,
string? mime = null,
string? key = null
) => value switch {
Value x => x,
null => new Value { type = Dtype.Null },
float x => new Value { data = await storage.Upload(name, new [] { x }.ToStream(), UploadType.Value, dataUrlLimit: minUploadSize, key: key), type = Dtype.Float32, shape = new int[0] },
double x => new Value { data = await storage.Upload(name, new [] { x }.ToStream(), UploadType.Value, dataUrlLimit: minUploadSize, key: key), type = Dtype.Float64, shape = new int[0] },
sbyte x => new Value { data = await storage.Upload(name, new [] { x }.ToStream(), UploadType.Value, dataUrlLimit: minUploadSize, key: key), type = Dtype.Int8, shape = new int[0] },
Expand Down Expand Up @@ -227,12 +229,12 @@ public async Task<Value> ToValue (
Tensor<ushort> x => new Value { data = await storage.Upload(name, x.data.ToStream(), UploadType.Value, dataUrlLimit: minUploadSize, key: key), type = Dtype.Uint16, shape = x.shape },
Tensor<uint> x => new Value { data = await storage.Upload(name, x.data.ToStream(), UploadType.Value, dataUrlLimit: minUploadSize, key: key), type = Dtype.Uint32, shape = x.shape },
Tensor<ulong> x => new Value { data = await storage.Upload(name, x.data.ToStream(), UploadType.Value, dataUrlLimit: minUploadSize, key: key), type = Dtype.Uint64, shape = x.shape },
string x => new Value { data = await storage.Upload(name, x.ToStream(), UploadType.Value, dataUrlLimit: minUploadSize, key: key), type = Dtype.String },
Stream x => new Value { data = await storage.Upload(name, x, UploadType.Value, dataUrlLimit: minUploadSize, key: key), type = type ?? Dtype.Binary },
IList x => new Value { data = await storage.Upload(name, JsonConvert.SerializeObject(x).ToStream(), UploadType.Value, dataUrlLimit: minUploadSize, key: key), type = Dtype.List },
IDictionary x => new Value { data = await storage.Upload(name, JsonConvert.SerializeObject(x).ToStream(), UploadType.Value, dataUrlLimit: minUploadSize, key: key), type = Dtype.Dict },
null => new Value { type = Dtype.Null },
_ => throw new InvalidOperationException($"Cannot create a Function value from value '{value}' of type {value.GetType()}"),
string x => new Value { data = await storage.Upload(name, x.ToStream(), UploadType.Value, mime: @"text/plain", dataUrlLimit: minUploadSize, key: key), type = Dtype.String },
IList x => new Value { data = await storage.Upload(name, JsonConvert.SerializeObject(x).ToStream(), UploadType.Value, mime: @"application/json", dataUrlLimit: minUploadSize, key: key), type = Dtype.List },
IDictionary x => new Value { data = await storage.Upload(name, JsonConvert.SerializeObject(x).ToStream(), UploadType.Value, mime: @"application/json", dataUrlLimit: minUploadSize, key: key), type = Dtype.Dict },
Image x => await ToValue(x, name, minUploadSize: minUploadSize, key: key),
Stream x => new Value { data = await storage.Upload(name, x, UploadType.Value, mime: mime, dataUrlLimit: minUploadSize, key: key), type = type ?? Dtype.Binary },
_ => throw new InvalidOperationException($"Cannot create a Function value from value '{value}' of type `{value.GetType()}`"),
};
#endregion

Expand Down Expand Up @@ -489,18 +491,35 @@ private static unsafe IntPtr ToValue<T> (Tensor<T> tensor) where T : unmanaged {
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe IntPtr ToValue (Image image) {
private static unsafe IntPtr ToValue (Image image, bool forcePin = false) {
fixed (byte* data = image)
return Function.CreateImageValue(
data,
image.width,
image.height,
image.channels,
image.data != null ? ValueFlags.CopyData : ValueFlags.None,
!forcePin && image.data != null ? ValueFlags.CopyData : ValueFlags.None,
out var value
).Throw() == Status.Ok ? value : default;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe Task<Value> ToValue (
Image image,
string name,
int minUploadSize,
string? key
) {
fixed (byte* data = image) {
var imageValue = ToValue(image, forcePin: true); // zero copy even for managed arrays
imageValue.CreateSerializedValue(0, out var serializedValue).Throw();
var stream = ToObject(serializedValue) as Stream;
imageValue.ReleaseValue();
serializedValue.ReleaseValue();
return ToValue(stream, name, type: Dtype.Image, minUploadSize: minUploadSize, mime: @"image/png", key: key);
}
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static object ToObject<T> (MemoryStream stream, int[] shape) where T : unmanaged {
var data = stream.ToArray<T>();
Expand Down
22 changes: 15 additions & 7 deletions Packages/ai.fxn.fxn3d/Runtime/Types/Image.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public unsafe readonly struct Image {
/// <summary>
/// Create an image.
/// </summary>
/// <param name="data">Image pixel buffer.</param>
/// <param name="data">Pixel buffer. The pixel buffer format MUST be `R8`, `RGB888`, or `RGBA8888`.</param>
/// <param name="width">Image width.</param>
/// <param name="height">Image height.</param>
/// <param name="channels">Image channels.</param>
Expand All @@ -51,19 +51,27 @@ public Image (byte[] data, int width, int height, int channels) {
this.height = height;
this.channels = channels;
}
#endregion


#region --Operations--
private readonly byte* nativeData;

public unsafe Image (byte* data, int width, int height, int channels) { // Zero copy into `FXNValue`
/// <summary>
/// Create an image from a pixel buffer.
/// NOTE: DO NOT use this overload unless you absolutely know what you are doing.
/// </summary>
/// <param name="data">Pixel buffer. The pixel buffer format MUST be `R8`, `RGB888`, or `RGBA8888`.</param>
/// <param name="width">Image width.</param>
/// <param name="height">Image height</param>
/// <param name="channels">Image channels.</param>
public unsafe Image (byte* data, int width, int height, int channels) { // Enables zero copy into `FXNValue`
this.data = null!;
this.nativeData = data;
this.width = width;
this.height = height;
this.channels = channels;
}
#endregion


#region --Operations--
private readonly byte* nativeData;

public ref byte GetPinnableReference () => ref (nativeData == null ? ref data[0] : ref *nativeData);
#endregion
Expand Down
Loading

0 comments on commit 558d2d1

Please sign in to comment.