diff --git a/.media/hub.png b/.media/hub.png new file mode 100644 index 0000000..ad4ce41 Binary files /dev/null and b/.media/hub.png differ diff --git a/.media/mobilenet.gif b/.media/mobilenet.gif new file mode 100644 index 0000000..446191f Binary files /dev/null and b/.media/mobilenet.gif differ diff --git a/.media/wall.png b/.media/wall.png new file mode 100644 index 0000000..f6b3eb3 Binary files /dev/null and b/.media/wall.png differ diff --git a/Changelog.md b/Changelog.md new file mode 100644 index 0000000..03c9b79 --- /dev/null +++ b/Changelog.md @@ -0,0 +1,328 @@ +## 1.1.15 ++ Fixed `NatMLSettings` not being available to some clients at app start. + +## 1.1.14 ++ Minor updates. + +## 1.1.13 ++ Fixed `MLEdgeModel.Create` method overload that accepts an `MLModelData` instance raising `NullReferenceException`. + +## 1.1.12 ++ Fixed `JsonException` when creating `MLEdgeModel` due to high code stripping (#58). + +## 1.1.11 ++ Fixed access key not updating after creating a build. + +## 1.1.10 ++ Fixed `JsonException` when creating `MLEdgeModel` due to managed code stripping (#58). + +## 1.1.9 ++ Removed `MLCloudModel` class. Migrate to [Function AI](https://fxn.ai) to make cloud predictions. ++ Removed `Endpoint` class. ++ Removed `EndpointPrediction` class. ++ Removed `Feature` class. ++ Removed `FeatureInput` class. ++ Removed `NatMLClient.Endpoints` property. ++ Removed `NatMLClient.EndpointPredictions` property. ++ Removed `Predictor.endpoints` property. + +## 1.1.8 ++ Fixed `GraphFormat` error when creating models. ++ Fixed build errors when building for WebGL. ++ NatML now requires Unity 2022.3+. + +## 1.1.7 ++ Fixed `JsonException` when creating some models (#58). ++ Fixed crash when building on Unity Cloud Build with macOS worker (#56). + +## 1.1.6 ++ Fixed `DllNotFoundException` when building on Linux (#54). ++ Fixed `Illegal byte sequence encountered in the input` error when building on macOS and Windows (#55). ++ Refactored `PredictionSession` class to `EndpointPrediction`. ++ Refactored `NatML.PredictionSessions` field to `EndpointPredictions`. ++ Removed `MLAudioFeature.FromStreamingAssets` method. Use `MLUnityExtensions.StreamingAssetsToAbsolutePath` to get a path and create an audio feature. ++ Deprecated `PredictorSession` class. ++ Deprecated `NatML.PredictorSessions` field. + +## 1.1.5 ++ Added support for Safari 16.4, bringing NatML to 88% of browsers. ++ Added `MLArrayFeature.CopyTo` method overload that accepts a `Texture2D`. ++ Added `MLEdgeModel.Create` method overload that accepts a `NatMLClient` instance in place of an `accessKey`. ++ Added `MLCloudModel.Create` method overload that accepts a `NatMLClient` instance in place of an `accessKey`. ++ Added `Feature.dictValue` field for working with dictionary features. ++ Refactored `Dtype.List` enumeration member to `Dtype.List`. ++ Refactored `Dtype.Dictionary` enumeration member to `Dtype.Dict`. ++ Removed `MLCloudModel.Predict` method overload that accepts a `Dictionary`. ++ Removed `MLImageFeature.RegionOfInterest` method. + +## 1.1.4 ++ Added `NatML.API` namespace for accessing the full NatML web API from .NET. ++ Fixed `MLEdgeModel.Create` method blocking for a long time on Android and Windows (#49). ++ Fixed `MLEdgeModel.Create` method throwing error when called from non-Unity threads. ++ Updated default edge prediction configuration to use the CPU and neural processor (and not the GPU) on Android (#49). ++ Refactored `MLDepthFeature.ViewportToWorldPoint` method to `Unproject`. ++ Removed `MLEdgeModel.AudioFormat` struct. Use `NatML.API.Types.AudioFormat` class instead. ++ Removed `MLEdgeModel.Normalization` struct. Use `NatML.API.Types.Normalization` class instead. ++ Removed `MLImageFeature.AspectMode` enumeration. Use `NatML.API.Types.AspectMode` enumeration instead. ++ Removed `MLAudioType.FromAudioClip` utility method. ++ Removed `MLAudioType.FromVideoClip` utility method. ++ Removed `MLVideoType.FromVideoClip` utility method. + +## 1.1.3 ++ Added support for making edge predictions on the GPU on Android with `MLEdgeModel.ComputeTarget.GPU`. ++ Fixed `MLEdgeModel.Predict` sporadically crashing on Android. + +## 1.1.2 ++ Added support for CoreML models that consume pixel buffers (`CVPixelBufferRef`) instead of multi-arrays. ++ Fixed `MLEdgeModel.Create` ignoring `configuration` argument when model is fetched from NatML Hub. ++ Fixed `MLAsyncPredictor` accumulating massive amounts of memory on iOS and macOS. ++ Refactored `MLTextFeature` class to `MLStringFeature`. ++ Refactored `MLTextType` class to `MLStringType`. ++ Refactored `MLModelDataEmbed` attribute to `MLEdgeModel.Embed`. + +## 1.1.1 ++ Added model embedding support for encrypted models. + +## 1.1.0 ++ Added `MLCloudModel` class for making predictions with predictor endpoints. ++ Added `MLEdgeModel.Create` methods for creating an edge model from a predictor tag or an `MLModelData` instance. ++ Added `MLEdgeModel.ComputeTarget` enumeration for specifying the compute target for model predictions. ++ Added `MLEdgeModel.Configuration` data class for configuring the edge model creation process. ++ Added `MLEdgeModel.labels` property for inspecting classification labels required by the model. ++ Added `MLEdgeModel.aspectMode` property for inspecting the aspect mode required in image data consumed by the model. ++ Added `MLEdgeModel.audioFormat` property for inspecting the audio format required in audio data consumed by the model. ++ Added support for model encryption, protecting your valuable intellectual property. ++ Added support for pinning `MLArrayFeature` instances with `fixed` statements. ++ Added support for pinning `MLImageFeature` instances with `fixed` statements. ++ Added `MLArrayFeature` constructor from an `MLCloudFeature` for making predictions with predictor endpoints. ++ Added `MLImageFeature` constructor from an `MLCloudFeature` for making predictions with predictor endpoints. ++ Added `MLTextFeature` constructor from an `MLCloudFeature` for making predictions with predictor endpoints. ++ Added `MLArrayFeature.CopyTo(MLArrayFeature)` method for copying image data from one image feature to another. ++ Added `MLImageFeature.CopyTo(MLImageFeature)` method for copying image data from one image feature to another. ++ Added implicit conversion from `float` to `MLFeature`. ++ Added implicit conversion from `int` to `MLFeature`. ++ Added implicit conversion from `bool` to `MLFeature`. ++ Added implicit conversion from `int[]` to `MLFeature`. ++ Updated `MLImageFeature` class to be `sealed` thus preventing inheritance. ++ Updated `float[]` implicit conversion to `MLArrayFeature` to have a 1D shape instead of a `null` shape. ++ Refactored `MLModelData.ComputeTarget` enumeration to `MLEdgeModel.ComputeTarget`. ++ Refactored `MLModelData.Normalization` enumeration to `MLEdgeModel.Normalization`. ++ Refactored `MLModelData.AudioFormat` enumeration to `MLEdgeModel.AudioFormat`. ++ Deprecated `MLImageFeature.RegionOfInterest` method. Use `MLImageFeature.CopyTo` method instead. ++ Deprecated `MLModelData.ComputeTarget.CPUOnly` enumeration member. Use `ComputeTarget.CPU` instead. ++ Removed `MLEdgeModel` constructor. Use `MLEdgeModel.Create` method instead. ++ Removed `MLModelData.Deserialize` method. Use `MLEdgeModel.Create` method instead. ++ Removed `MLModelData.FromHub` method. Use `MLEdgeModel.Create` method instead. ++ Removed `MLModelData.FromFile` method. Use `MLEdgeModel.Create` method instead. ++ Removed `MLModelData.tag` property. ++ Removed `MLModelData.computeTarget` property. Use `MLEdgeModel.Configuration.computeTarget` property instead. ++ Removed `MLModelData.computeDevice` property. Use `MLEdgeModel.Configuration.computeDevice` property instead. ++ Removed `MLModelData.labels` property. Use `MLEdgeModel.labels` property instead. ++ Removed `MLModelData.normalization` property. Use `MLEdgeModel.normalization` property instead. ++ Removed `MLModelData.audioFormat` property. Use `MLEdgeModel.audioFormat` property instead. ++ Removed `MLModelData.ComputeTarget` enumeration. ++ Removed `MLArrayFeature.CopyTo(T[])` method overload. ++ Removed `MLArrayFeature.CopyTo(NativeArray)` method overload. ++ Removed `MLArrayFeature.CopyTo(T*)` method overload. ++ Removed `MLImageFeature.CopyTo(T[])` method overload. ++ Removed `MLImageFeature.CopyTo(NativeArray)` method overload. ++ Removed `MLImageFeature.CopyTo(T*)` method overload. ++ NatML now requires iOS 14+. + +## 1.0.19 ++ Added `MLEdgeModel` public constructor and moved the class to the top-level `NatML` namespace. ++ Fixed app storage size increasing every time that `MLModelData.Deserialize` was invoked on iOS. ++ Deprecated `MLModelData.Deserialize` method. Use `MLEdgeModel` constructor instead. + +## 1.0.18 ++ NatML now defaults to accelerating predictions on Windows by using the GPU. This results in much better performance and lower CPU usage for many models. ++ Added `MLModelData.computeTarget` property for specifying the compute target used for prediction. ++ Added `MLModelData.computeDevice` property for specifying the compute device used for prediction. ++ Added `MLImageFeature.CopyTo` overloads that accepted pixel buffers in managed and unmanaged memory. ++ Fixed `MLImageFeature` producing incorrect prediction results on WebGL. ++ Fixed rare crash when calling `MLModelData.Deserialize` on low-end Android devices. + +## 1.0.17 ++ Upgraded to Hub 1.0.12. + +## 1.0.16 ++ Refactored `MLDepthFeature.TransformPoint` method to `ViewportToWorldPoint`. ++ Removed `MLImageFeature.CopyTo` overloads that accepted pixel buffers. Use `Texture2D` overload instead. ++ Removed `MLImageFeature.ToTexture` method. Use `MLImageFeature.CopyTo` method instead. + +## 1.0.15 ++ Added initial support for WebGL! NatML can now be used in the browser. ++ Improved `MLImageFeature` texture constructor to avoid copying pixel buffers when possible. ++ Improved performance of `MLImageFeature.NonMaxSuppression` for large number of candidate boxes. ++ Changed `MLEdgeFeature.dataType` property type from `System.Type` to `NatML.DataType`. + +## 1.0.13 ++ Added support for rotated ROI in `MLImageFeature.RegionOfInterest` method. ++ Added `MLModelData.FromFile` method to load ML model data from model files. ++ Added `MLImageFeature.TransformPoint` method for transforming detection points from feature space to image space. ++ Added `MLDepthFeature.TransformPoint` method for projecting 2D points into 3D space using depth. ++ Added `MLEdgeFeature.dataType` property for inspecting the data type of Edge features. ++ Added `MLAsyncPredictor` class for making predictions on a background thread. ++ Added `MLPredictorExtensions.ToAsync` extension method for converting predictor to an async predictor. ++ Improved prediction performance on Android devices with dedicated neural processing units. ++ Changed `MLEdgeFeature` class to `readonly struct` to prevent GC pressure. ++ Fixed `MLImageFeature` not respecting `AspectMode.AspectFit` when making predictions. ++ Fixed sporadic `NullReferenceException` when `MLModelData.FromHub` is called on some Android devices. ++ Updated `MLDepthFeature.Sample` method to accept a `Vector2` point instead of individual coordinates. ++ Removed `IMLCloudFeature` interface as it is no longer supported by the NatML Hub API. ++ Removed `MLCloudFeature` class as it is no longer supported by the NatML Hub API. ++ Removed `MLCloudModel` class as it is no longer supported by the NatML Hub API. ++ Removed `IMLAsyncPredictor` interface. ++ Removed `MLFeature.CloudType` utility method as it is no longer supported by the NatML Hub API. ++ Removed `MLArrayFeature` constructor that accepted an `MLCloudFeature`. ++ Removed `MLAudioFeature` constructor that accepted an `MLCloudFeature`. ++ Removed `MLImageFeature` constructor that accepted an `MLCloudFeature`. ++ Removed `MLImageFeature` constructor that accepted an encoded image `byte[]`. ++ Removed `MLImageFeature.Contiguous` method. ++ Removed `MLTextFeature` constructor that accepted an `MLCloudFeature`. ++ Removed `MLEdgeFeature.ReleaseFeature` method as it has long been deprecated. + +## 1.0.12 ++ Upgraded to Hub 1.0.8. + +## 1.0.11 ++ Added `MLModelDataEmbed` attribute for embedding model data at build time, making models immediately available in builds without downloads. ++ Added `MLImageFeature.NonMaxSuppression` method for performing non-maximum suppression on detection proposals. ++ Added `MLImageFeature.TransformRect` method for transforming detection rectangles from feature space to image space. ++ Added custom icon for identifying ML model files imported by NatML. ++ Migrated `MLPredictorExtensions.ToAsync` extension method and `MLAsyncPredictor` class to NatMLX. ++ Removed `MLPredictorExtensions.RectifyAspect` method. Use `MLImageFeature.TransformRect` method instead. ++ Removed `MLPredictorExtensions.NonMaxSuppression` method. Use `MLImageFeature.NonMaxSuppression` method instead. ++ Refactored top-level namespace from `NatSuite.ML` to `NatML` for parity with our other API's. + +## 1.0.10 ++ Improved prediction performance on Windows systems with dedicated GPU's. ++ Added `MLArrayFeature` constructors that accept `NativeArray` native arrays to minimize memory copies. ++ Added setter accessors to `MLArrayFeature` indexers allowing for writing values to feature data. ++ Added `MLAudioFeature.Contiguous` method for decoding encoded audio feature into memory. ++ Added support for creating greyscale image `MLEdgeFeature` features from `MLImageFeature` features. ++ Added `MLImageFeature.RegionOfInterest` method for extracting an ROI from an image. ++ Added `MLImageFeature.Contiguous` method for decoding encoded image feature into memory. ++ Added `MLDepthFeature.width` convenience property for getting width of depth features. ++ Added `MLDepthFeature.height` convenience property for getting height of depth features. ++ Added `MLImageType.interleaved` property for checking whether image feature is interleaved or planar. ++ Added `IMLCloudFeature` interface to create cloud ML features for making cloud predictions. ++ Added "Clear Predictor Cache" menu item for clearing predictor cache in the editor. ++ Added `NatMLHub.Subscribe` method for making subscription requests to the NatML API. ++ Fixed memory leak when using certain vision predictors like Robust Video Matting. ++ Refactored `MLHubModel` class to `MLCloudModel`. ++ Refactored `MLHubFeature` class to `MLCloudFeature`. ++ Refactored `HubDataType` class to `DataType`. ++ Deprecated `IMLHubFeature` interface. ++ Removed `MLAudioFeature.ReadToEnd` method. Use `MLAudioFeature.Contiguous` method instead. + +## 1.0.9 ++ Added exclusive support for running CoreML graphs on iOS and macOS. ++ Added exclusive support for running TensorFlow Lite graphs on Android. ++ Added support for working with CoreML `.mlmodel` files in Unity projects. ++ Added support for working with TensorFlow Lite `.tflite` files in Unity projects. ++ Added support for Apple Silicon on macOS. ++ Added `MLVideoFeature` class for making ML predictions on video files. ++ Added `MLVideoType` feature type for inspecting video features. ++ Added `MLDepthFeature` abstract class for working with predictors that use depth data. ++ Added support for audio feature resampling in `MLAudioFeature` with `sampleRate` and `channelCount` fields. ++ Added `MLEdgeFeature` type for added type safety when authoring edge predictors. ++ Added `MLImageFeature` constructor which accepts a `NativeArray` pixel buffer. ++ Added `MLImageFeature.width` convenience property for getting width of image features. ++ Added `MLImageFeature.height` convenience property for getting height of image features. ++ Added `MLImageFeature.CopyTo` methods for copying pixel data from image feature. ++ Added `MLAudioFeature` constructor that accepts a `NativeArray` to minimize memory copies. ++ Added `MLAudioFeature` path constructor for reading audio features from audio and video files. ++ Added `MLAudioFeature.CopyTo` methods for copying audio data from audio feature. ++ Added `MLAudioFeature.FromStreamingAssets` method for creating an audio feature from an audio file in the `StreamingAssets` folder. ++ Added `MLAudioFeature.ToAudioClip` method for converting audio feature to an `AudioClip`. ++ Added `MLAudioFeature.ReadToEnd` method to read audio data into memory for audio features backed by an audio file. ++ Added `MLAudioType.FromFile` method for inspecting the audio type of a video or audio file. ++ Added `MLAudioType.FromAudioClip` method for inspecting the audio type of an audio clip. ++ Added `MLAudioType.FromVideoClip` method for inspecting the audio type of a video clip. ++ Added `MLAudioType.FromStreamingAssets` method for inspecting the audio type of a video file in the `StreamingAssets` folder. ++ Added `MLImageType` constructor that accepts image `channels`. ++ Fixed `MLImageFeature` type incorrectly reporting 3 channels instead of 4. ++ Fixed `MLImageFeature` default normalization standard deviation having `0` for alpha channel. ++ Removed `IMLModel` interface as it has long been deprecated. ++ Removed `IMLFeature` interface as it has long been deprecated. + +## 1.0.8 ++ Fixed `Cannot deserialize graph` exception when deserializing cached predictors. ++ Fixed `MLModelData` being cached for `DRAFT` predictors. + +## 1.0.7 ++ Added `MLArrayFeature` constructor that accepts an `MLHubFeature` for working with Hub predictors. ++ Added `MLAudioFeature` constructor that accepts an `MLHubFeature` for working with Hub predictors. ++ Added `MLImageFeature` constructor that accepts an `MLHubFeature` for working with Hub predictors. ++ Added `MLTextFeature` constructor that accepts an `MLHubFeature` for working with Hub predictors. ++ Fixed `DirectoryNotFoundException` when loading cached `MLModelData` on iOS. ++ Removed prediction analytics reporting to NatML Hub, relieving network bandwidth pressure. ++ Removed `cache` flag in `MLModelData.FromHub` method. + +## 1.0.6 ++ Introduced Hub Predictors, which make predictions using server-side processing on [NatML Hub](https://hub.natml.ai). ++ Added `MLHubModel` class for authoring predictors that make cloud-based predictions using NatML Hub. ++ Added `MLEdgeModel` class for authoring predictors that make edge (on-device) predictions. ++ Added `IMLHubFeature` interface for creating server-side features when making predictions with NatML Hub. ++ Added `IMLEdgeFeature` interface for creating native features when making edge (on-device) predictions. ++ Added `MLTextType` feature type for inspecting `MLTextFeature` features. ++ Added `MLModelData.tag` property to identify the predictor tag from [NatML Hub](https://hub.natml.ai). ++ Added `MLModel.metadata` dictionary for inspecting model metadata. ++ Added `MLArrayFeature.Squeeze` to remove singleton dimensions from an array feature. ++ Added `MLArrayFeature.Flatten` to flatten an array feature into one-dimensional array feature. ++ Added `MLArrayFeature.ToArray` to convert an array feature into a flattened primitive array. ++ Added `MLImageFeature.ToTexture` to convert an image feature into a `Texture2D`. ++ Added `MLImageType.FromType` static method for converting arbitrary feature types to image types. ++ Added implicit conversion from `MLFeatureType` to `bool` indicating if the type is non-`null`. ++ Added implicit conversion from `MLTextFeature` to `string`. ++ Fixed `MLImageType` image resolution constructor assuming planar format instead of interleaved format. ++ Moved `IMLPredictor` interface to the top-level `NatSuite.ML` namespace. ++ Moved `IMLAsyncPredictor` interface to the top-level `NatSuite.ML` namespace. ++ Deprecated `IMLModel` interface. Cast model to `MLEdgeModel` class instead. ++ Deprecated `IMLFeature` interface. Cast feature to `IMLEdgeFeature` interface instead. ++ Deprecated `MLPredictorExtensions.GetImageSize` static method. Use `MLImageType.FromType` instead. ++ Removed `MLModelData.FromFile` method. Use [NatML Hub](https://hub.natml.ai) instead. ++ Removed `MLModelData.FromStreamingAssets` method. Use [NatML Hub](https://hub.natml.ai) instead. ++ Removed `MLPredictorExtensions.SerializeAudio` method. ++ Removed `MLPredictorExtensions.SerializeImage` method. ++ Removed `MLModel` dictionary indexers. Use `MLModel.metadata` property instead. + +## 1.0.5 ++ Changed `MLImageFeature.mean` and `std` types to `Vector4` to support normalization for alpha channel. ++ Fixed bitcode not being generated for iOS `NatML.framework`. ++ Removed metadata accessors from `IMLModel` interface. Cast to `MLModel` instead. + +## 1.0.4 ++ Greatly improved performance and memory pressure when performing multi-indexing with `MLArrayFeature`. ++ Added `MLPredictorExtensions.RectifyAspect` extension method for correcting detection rects from aspect-scaled images. ++ Fixed crash when making predictions with recurrent models on previous state features. ++ Fixed crash when getting native array feature shape for `MLArrayFeature`. ++ Fixed memory leak when making predictions with image features on iOS and macOS. + +## 1.0.3 ++ Added `IMLAsyncPredictor` interface for making server-side ML predictions with NatML Hub. ++ Added `MLArrayFeature` constructor which accepts a native array feature for easy interop. ++ Added multi-indexing support to `MLArrayFeature` for post-processing native array features. ++ Added `MLArrayType.elementCount` property to get the total number of elements for an array type. ++ Added `MLArrayFeature.shape` property which returns the feature type's shape for convenience. ++ Added `MLArrayFeature.elementCount` property which returns the feature type's element count for convenience. ++ Added `MLArrayFeature.CopyTo` method to copy feature data into an array. ++ Added `MLArrayFeature.Permute` method to create a shallow array view with permuted dimensions. ++ Added `MLArrayFeature.View` method to create a shallow array view with a different shape. ++ Added `MLPredictorExtensions.NonMaxSuppression` method for working with detection models. ++ Added `MLPredictorExtensions.GetImageSize` method for making predictions with image features. ++ Added `MLPredictorExtensions.SerializeAudio` method for making Hub predictions with audio features. ++ Added `MLPredictorExtensions.SerializeImage` method for making Hub predictions with image features. ++ Fixed `MLAsyncPredictor` predictions never completing if backing predictor encountered exception. + +## 1.0.2 ++ Added `MLModelData.audioFormat` property for working with audio and speech ML models. ++ Added `MLTextFeature` for working with natural language processing models. ++ Added NatML menu items for fetching access key, viewing models, and more. ++ Exposed `mean` and `std` arrays in `MLModelData.Normalization` struct for models that require arbitrary normalization. ++ Removed generic `MLClassificationPredictor` and `MLDenseClassificationPredictor` predictors. ++ Removed ability to specify class labels for local `.onnx` file in project. Use NatML Hub instead. + +## 1.0.0 ++ First release. \ No newline at end of file diff --git a/Changelog.md.meta b/Changelog.md.meta new file mode 100644 index 0000000..1538219 --- /dev/null +++ b/Changelog.md.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 5a0b29da739df41b28508310315e9d15 +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor.meta b/Editor.meta new file mode 100644 index 0000000..f5a1905 --- /dev/null +++ b/Editor.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 31668ee3fd0b149b88c07f6e11755491 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/BuildEmbedHelper.cs b/Editor/BuildEmbedHelper.cs new file mode 100644 index 0000000..73b8d2c --- /dev/null +++ b/Editor/BuildEmbedHelper.cs @@ -0,0 +1,78 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Editor { + + using System; + using System.Collections.Generic; + using System.Linq; + using UnityEditor; + using UnityEditor.Build; + using UnityEditor.Build.Reporting; + using UnityEngine; + + /// + /// Lightweight utility for embedding scriptable objects into builds. + /// + public abstract class BuildEmbedHelper : IPreprocessBuildWithReport, IPostprocessBuildWithReport where T : ScriptableObject { + + #region --Client API-- + /// + /// Callback order for prioritization. + /// + public virtual int callbackOrder => 0; + + /// + /// Supported targets for this build embed. + /// If `null` the helper will embed for all build targets. + /// + protected virtual BuildTarget[] SupportedTargets => null; + + /// + /// Create embeds for the build. + /// + protected abstract T[] CreateEmbeds (BuildReport report); + + /// + /// Clear embeds after the build. + /// + protected virtual void ClearEmbeds (BuildReport report) => ClearEmbeds(); + + /// + /// Clear any existing data embedded in the build. + /// + protected static void ClearEmbeds () { + var assets = PlayerSettings.GetPreloadedAssets()?.ToList(); + if (assets == null) + return; + assets.RemoveAll(asset => asset && asset.GetType() == typeof(U)); + PlayerSettings.SetPreloadedAssets(assets.ToArray()); + } + #endregion + + + #region --Operations-- + + void IPreprocessBuildWithReport.OnPreprocessBuild (BuildReport report) { + // Clear + ClearEmbeds(); + // Check target + var targets = SupportedTargets; + if (!targets?.Contains(report.summary.platform) ?? false) + return; + // Create + var data = CreateEmbeds(report); + if (data == null) + return; + // Embed + var assets = PlayerSettings.GetPreloadedAssets()?.ToList() ?? new List(); + assets.AddRange(data); + PlayerSettings.SetPreloadedAssets(assets.ToArray()); + } + + void IPostprocessBuildWithReport.OnPostprocessBuild (BuildReport report) => ClearEmbeds(report); + #endregion + } +} \ No newline at end of file diff --git a/Editor/BuildEmbedHelper.cs.meta b/Editor/BuildEmbedHelper.cs.meta new file mode 100644 index 0000000..88c1f57 --- /dev/null +++ b/Editor/BuildEmbedHelper.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 3d1cf23ae55ae4941ae259913719f3d0 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/Importers.meta b/Editor/Importers.meta new file mode 100644 index 0000000..a55550a --- /dev/null +++ b/Editor/Importers.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: b787897db4564425d9e5f8259ce03f68 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/Importers/CoreMLImporter.cs b/Editor/Importers/CoreMLImporter.cs new file mode 100644 index 0000000..cff86e0 --- /dev/null +++ b/Editor/Importers/CoreMLImporter.cs @@ -0,0 +1,18 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Editor.Importers { + + using UnityEditor.AssetImporters; + using API.Types; + + [ScriptedImporter(1, @"mlmodel")] + internal sealed class CoreMLImporter : GraphImporter { + + protected override PredictorSession CreateSession () => new PredictorSession { + format = GraphFormat.CoreML + }; + } +} \ No newline at end of file diff --git a/Editor/Importers/CoreMLImporter.cs.meta b/Editor/Importers/CoreMLImporter.cs.meta new file mode 100644 index 0000000..1248576 --- /dev/null +++ b/Editor/Importers/CoreMLImporter.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 3e882272056fc4ddfa14de161aaba2ba +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/Importers/GraphImporter.cs b/Editor/Importers/GraphImporter.cs new file mode 100644 index 0000000..2f4f478 --- /dev/null +++ b/Editor/Importers/GraphImporter.cs @@ -0,0 +1,25 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Editor.Importers { + + using System.IO; + using UnityEngine; + using UnityEditor.AssetImporters; + using API.Types; + + public abstract class GraphImporter : ScriptedImporter { + + public sealed override void OnImportAsset (AssetImportContext ctx) { + var modelData = ScriptableObject.CreateInstance(); + modelData.session = CreateSession(); + modelData.graph = File.ReadAllBytes(ctx.assetPath); + ctx.AddObjectToAsset("MLModelData", modelData); + ctx.SetMainObject(modelData); + } + + protected abstract PredictorSession CreateSession (); + } +} \ No newline at end of file diff --git a/Editor/Importers/GraphImporter.cs.meta b/Editor/Importers/GraphImporter.cs.meta new file mode 100644 index 0000000..9ff8c0c --- /dev/null +++ b/Editor/Importers/GraphImporter.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 97e53583325ef403c9035dc020c4f97c +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/Importers/ONNXImporter.cs b/Editor/Importers/ONNXImporter.cs new file mode 100644 index 0000000..36fc64f --- /dev/null +++ b/Editor/Importers/ONNXImporter.cs @@ -0,0 +1,18 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Editor.Importers { + + using UnityEditor.AssetImporters; + using API.Types; + + [ScriptedImporter(1, @"onnx")] + internal sealed class ONNXImporter : GraphImporter { + + protected override PredictorSession CreateSession () => new PredictorSession { + format = GraphFormat.ONNX + }; + } +} \ No newline at end of file diff --git a/Editor/Importers/ONNXImporter.cs.meta b/Editor/Importers/ONNXImporter.cs.meta new file mode 100644 index 0000000..a2455f3 --- /dev/null +++ b/Editor/Importers/ONNXImporter.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 8264490bef67c46f2982e6dd3f5e46cd +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/Importers/TFLiteImporter.cs b/Editor/Importers/TFLiteImporter.cs new file mode 100644 index 0000000..f903521 --- /dev/null +++ b/Editor/Importers/TFLiteImporter.cs @@ -0,0 +1,18 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Editor.Importers { + + using UnityEditor.AssetImporters; + using API.Types; + + [ScriptedImporter(1, @"tflite")] + internal sealed class TFLiteImporter : GraphImporter { + + protected override PredictorSession CreateSession () => new PredictorSession { + format = GraphFormat.TFLite + }; + } +} \ No newline at end of file diff --git a/Editor/Importers/TFLiteImporter.cs.meta b/Editor/Importers/TFLiteImporter.cs.meta new file mode 100644 index 0000000..3666054 --- /dev/null +++ b/Editor/Importers/TFLiteImporter.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 48142632c930045a8afd7cb111e91c2f +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/NatML.Editor.asmdef b/Editor/NatML.Editor.asmdef new file mode 100644 index 0000000..af97a77 --- /dev/null +++ b/Editor/NatML.Editor.asmdef @@ -0,0 +1,17 @@ +{ + "name": "NatML.Editor", + "references": [ + "NatML.Runtime" + ], + "includePlatforms": [ + "Editor" + ], + "excludePlatforms": [], + "allowUnsafeCode": false, + "overrideReferences": false, + "precompiledReferences": [], + "autoReferenced": true, + "defineConstraints": [], + "versionDefines": [], + "noEngineReferences": false +} \ No newline at end of file diff --git a/Editor/NatML.Editor.asmdef.meta b/Editor/NatML.Editor.asmdef.meta new file mode 100644 index 0000000..6c38bec --- /dev/null +++ b/Editor/NatML.Editor.asmdef.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 61916ed8ff37f4233826fe350b41eaa5 +AssemblyDefinitionImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/NatMLMenu.cs b/Editor/NatMLMenu.cs new file mode 100644 index 0000000..e5f9399 --- /dev/null +++ b/Editor/NatMLMenu.cs @@ -0,0 +1,44 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Editor { + + using System.IO; + using UnityEditor; + using UnityEngine; + using Internal; + + internal static class NatMLMenu { + + private const int BasePriority = -50; + + [MenuItem(@"NatML/NatML " + NatML.Version, false, BasePriority)] + private static void Version () { } + + [MenuItem(@"NatML/NatML " + NatML.Version, true, BasePriority)] + private static bool EnableVersion () => false; + + [MenuItem(@"NatML/Get Access Key", false, BasePriority + 1)] + private static void OpenAccessKey () => Help.BrowseURL(@"https://hub.natml.ai/account/developers"); + + [MenuItem(@"NatML/Explore Predictors", false, BasePriority + 2)] + private static void OpenHub () => Help.BrowseURL(@"https://hub.natml.ai"); + + [MenuItem(@"NatML/Join Discord Community", false, BasePriority + 3)] + private static void OpenDiscord () => Help.BrowseURL(@"https://natml.ai/community"); + + [MenuItem(@"NatML/View NatML Docs", false, BasePriority + 4)] + private static void OpenDocs () => Help.BrowseURL(@"https://docs.natml.ai/unity"); + + [MenuItem(@"NatML/Open a NatML Issue", false, BasePriority + 5)] + private static void OpenIssue () => Help.BrowseURL(@"https://github.com/natmlx/natml-unity"); + + [MenuItem(@"NatML/Clear Predictor Cache", false, BasePriority + 6)] + private static void ClearCache () { + MLEdgeModel.ClearCache(); + Debug.Log("NatML: Cleared predictor cache"); + } + } +} diff --git a/Editor/NatMLMenu.cs.meta b/Editor/NatMLMenu.cs.meta new file mode 100644 index 0000000..d319f04 --- /dev/null +++ b/Editor/NatMLMenu.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: d7153ab015e1d48ab8531ee280ce0245 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/NatMLProjectSettings.cs b/Editor/NatMLProjectSettings.cs new file mode 100644 index 0000000..548a42f --- /dev/null +++ b/Editor/NatMLProjectSettings.cs @@ -0,0 +1,59 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Editor { + + using System.Threading.Tasks; + using UnityEngine; + using UnityEditor; + using Internal; + + [FilePath(@"ProjectSettings/NatMLHub.asset", FilePathAttribute.Location.ProjectFolder)] + internal sealed class NatMLProjectSettings : ScriptableSingleton { + + #region --Data-- + [SerializeField] + private string accessKey; + #endregion + + + #region --Client API-- + /// + /// NatML access key. + /// + internal string AccessKey { + get => accessKey; + set { + // Check + if (value == accessKey) + return; + // Update + accessKey = value; + Save(false); + NatMLSettings.Instance = CreateSettings(); + } + } + + /// + /// NatML settings from the current project settings. + /// + internal static NatMLSettings CreateSettings () { + var settings = ScriptableObject.CreateInstance(); + settings.accessKey = instance.AccessKey; + return settings; + } + #endregion + + + #region --Operations-- + + [InitializeOnLoadMethod] + private static void OnLoad () => NatMLSettings.Instance = CreateSettings(); + + [InitializeOnEnterPlayMode] + private static void OnEnterPlaymodeInEditor (EnterPlayModeOptions options) => NatMLSettings.Instance = CreateSettings(); + #endregion + } +} \ No newline at end of file diff --git a/Editor/NatMLProjectSettings.cs.meta b/Editor/NatMLProjectSettings.cs.meta new file mode 100644 index 0000000..6c8256d --- /dev/null +++ b/Editor/NatMLProjectSettings.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: c01f2b1efb70c41f2b6921c9af976839 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/NatMLSettingsEmbed.cs b/Editor/NatMLSettingsEmbed.cs new file mode 100644 index 0000000..d323d0e --- /dev/null +++ b/Editor/NatMLSettingsEmbed.cs @@ -0,0 +1,86 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Editor { + + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Text; + using System.Threading.Tasks; + using UnityEditor; + using UnityEditor.Build.Reporting; + using UnityEngine; + using API; + using API.Types; + using Internal; + + internal sealed class NatMLSettingsEmbed : BuildEmbedHelper { + + protected override BuildTarget[] SupportedTargets => new [] { + BuildTarget.Android, + BuildTarget.iOS, + BuildTarget.StandaloneOSX, + BuildTarget.StandaloneWindows, + BuildTarget.StandaloneWindows64, + BuildTarget.WebGL, + }; + private const string CachePath = @"Assets/NMLBuildCache"; + + protected override NatMLSettings[] CreateEmbeds (BuildReport report) { + // Check platform // We don't support embeds on Linux editor (#54) + if (Application.platform == RuntimePlatform.LinuxEditor) + return new NatMLSettings[0]; + // Create cache path + Directory.CreateDirectory(CachePath); + // Create settings + var settings = NatMLProjectSettings.CreateSettings(); + // Get embeds + var embeds = AppDomain.CurrentDomain.GetAssemblies() + .SelectMany(assembly => assembly + .GetTypes() + .SelectMany(type => Attribute.GetCustomAttributes(type, typeof(MLEdgeModel.EmbedAttribute))) + ) + .Cast() + .ToArray(); + var defaultAccessKey = NatMLSettings.Instance.accessKey; + var format = GetFormat(report.summary.platform); + var secret = MLEdgeModel.CreateSecret().Result; // this completes immediately everywhere except web + settings.embeds = Task.WhenAll(embeds.Select(e => Task.Run(async () => { + try { + var accessKey = !string.IsNullOrEmpty(e.accessKey) ? e.accessKey : defaultAccessKey; + var client = new NatMLClient(accessKey); + var session = await client.PredictorSessions.Create(e.tag, format, secret); + var graphStream = await client.Storage.Download(session.graph); + var graph = graphStream.ToArray(); + var embed = new NatMLSettings.Embed { fingerprint = session.fingerprint, data = graph }; + return embed; + } catch (Exception ex) { + Debug.LogWarning($"NatML: Failed to embed {e.tag} with error: {ex.Message}"); + return null; + } + }))).Result; + // Write settings + AssetDatabase.CreateAsset(settings, $"{CachePath}/NatML.asset"); + return new [] { settings }; + } + + protected override void ClearEmbeds (BuildReport report) { + base.ClearEmbeds(report); + AssetDatabase.DeleteAsset(CachePath); + } + + private static GraphFormat GetFormat (BuildTarget target) => target switch { + BuildTarget.Android => GraphFormat.TFLite, + BuildTarget.iOS => GraphFormat.CoreML, + BuildTarget.StandaloneOSX => GraphFormat.CoreML, + BuildTarget.StandaloneWindows => GraphFormat.ONNX, + BuildTarget.StandaloneWindows64 => GraphFormat.ONNX, + BuildTarget.WebGL => GraphFormat.ONNX, + _ => 0, + }; + } +} \ No newline at end of file diff --git a/Editor/NatMLSettingsEmbed.cs.meta b/Editor/NatMLSettingsEmbed.cs.meta new file mode 100644 index 0000000..618b0fc --- /dev/null +++ b/Editor/NatMLSettingsEmbed.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 5df429337aaa449b1b8938682aecf5ec +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/NatMLSettingsProvider.cs b/Editor/NatMLSettingsProvider.cs new file mode 100644 index 0000000..14aec8f --- /dev/null +++ b/Editor/NatMLSettingsProvider.cs @@ -0,0 +1,23 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Editor { + + using System.Collections.Generic; + using UnityEditor; + + internal static class NatMLSettingsProvider { + + [SettingsProvider] + public static SettingsProvider CreateProvider () => new SettingsProvider(@"Project/NatML", SettingsScope.Project) { + label = @"NatML", + guiHandler = searchContext => { + EditorGUILayout.LabelField(@"NatML Account", EditorStyles.boldLabel); + NatMLProjectSettings.instance.AccessKey = EditorGUILayout.TextField(@"Access Key", NatMLProjectSettings.instance.AccessKey); + }, + keywords = new HashSet(new[] { @"NatML", @"NatCorder", @"NatDevice", @"NatShare", @"NatML Hub" }), + }; + } +} \ No newline at end of file diff --git a/Editor/NatMLSettingsProvider.cs.meta b/Editor/NatMLSettingsProvider.cs.meta new file mode 100644 index 0000000..c76f51c --- /dev/null +++ b/Editor/NatMLSettingsProvider.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 188df6ed018974476bbc4bb738279dd3 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Editor/WebGLBuildHelper.cs b/Editor/WebGLBuildHelper.cs new file mode 100644 index 0000000..82cec41 --- /dev/null +++ b/Editor/WebGLBuildHelper.cs @@ -0,0 +1,29 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Editor { + + using UnityEditor; + using UnityEditor.Build; + using UnityEditor.Build.Reporting; + + internal sealed class WebGLBuildHelper : IPreprocessBuildWithReport { + + public int callbackOrder => 0; + private readonly string[] EM_ARGS = new [] { + @"--bind", + }; + + void IPreprocessBuildWithReport.OnPreprocessBuild (BuildReport report) { + if (report.summary.platform != BuildTarget.WebGL) + return; + foreach (var arg in EM_ARGS) { + var standaloneArg = $" {arg} "; + if (!PlayerSettings.WebGL.emscriptenArgs.Contains(standaloneArg)) + PlayerSettings.WebGL.emscriptenArgs += standaloneArg; + } + } + } +} \ No newline at end of file diff --git a/Editor/WebGLBuildHelper.cs.meta b/Editor/WebGLBuildHelper.cs.meta new file mode 100644 index 0000000..6ac51fe --- /dev/null +++ b/Editor/WebGLBuildHelper.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 69228c5e9c3af4fe6852f6eef98ac796 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..f49a4e1 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/LICENSE.md.meta b/LICENSE.md.meta new file mode 100644 index 0000000..02fd109 --- /dev/null +++ b/LICENSE.md.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: a77d378a5392848019d586ee4bec68b4 +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Plugins.meta b/Plugins.meta new file mode 100644 index 0000000..0becc72 --- /dev/null +++ b/Plugins.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 73f0bb1244d1e4c3f8a0ffa6f75fb696 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Plugins/Android.meta b/Plugins/Android.meta new file mode 100644 index 0000000..e8b4716 --- /dev/null +++ b/Plugins/Android.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: c942c1ca495ae4a9d94feca8fdcbc943 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Plugins/Android/NatML.aar b/Plugins/Android/NatML.aar new file mode 100644 index 0000000..a3ff33b Binary files /dev/null and b/Plugins/Android/NatML.aar differ diff --git a/Plugins/Android/NatML.aar.meta b/Plugins/Android/NatML.aar.meta new file mode 100644 index 0000000..5abe714 --- /dev/null +++ b/Plugins/Android/NatML.aar.meta @@ -0,0 +1,32 @@ +fileFormatVersion: 2 +guid: d605ab5dc3a9545b2a25a2591d551f2c +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + defineConstraints: [] + isPreloaded: 0 + isOverridable: 0 + isExplicitlyReferenced: 0 + validateReferences: 1 + platformData: + - first: + Android: Android + second: + enabled: 1 + settings: {} + - first: + Any: + second: + enabled: 0 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + DefaultValueInitialized: true + userData: + assetBundleName: + assetBundleVariant: diff --git a/Plugins/WebGL.meta b/Plugins/WebGL.meta new file mode 100644 index 0000000..b4e0bc2 --- /dev/null +++ b/Plugins/WebGL.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 1f6256a5f1c6248ffa3e3a1d07ee3083 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Plugins/WebGL/libNatML.a b/Plugins/WebGL/libNatML.a new file mode 100644 index 0000000..ae65c75 Binary files /dev/null and b/Plugins/WebGL/libNatML.a differ diff --git a/Plugins/WebGL/libNatML.a.meta b/Plugins/WebGL/libNatML.a.meta new file mode 100644 index 0000000..036786b --- /dev/null +++ b/Plugins/WebGL/libNatML.a.meta @@ -0,0 +1,86 @@ +fileFormatVersion: 2 +guid: c796fddd008ae47f59c2eac65b25d594 +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + defineConstraints: [] + isPreloaded: 0 + isOverridable: 0 + isExplicitlyReferenced: 0 + validateReferences: 1 + platformData: + - first: + : Any + second: + enabled: 0 + settings: + Exclude Android: 1 + Exclude Editor: 1 + Exclude Linux64: 1 + Exclude OSXUniversal: 1 + Exclude WebGL: 0 + Exclude Win: 1 + Exclude Win64: 1 + Exclude iOS: 1 + - first: + Android: Android + second: + enabled: 0 + settings: + CPU: ARMv7 + - first: + Any: + second: + enabled: 0 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + CPU: AnyCPU + DefaultValueInitialized: true + OS: AnyOS + - first: + Standalone: Linux64 + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: OSXUniversal + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: Win + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: Win64 + second: + enabled: 0 + settings: + CPU: None + - first: + WebGL: WebGL + second: + enabled: 1 + settings: {} + - first: + iPhone: iOS + second: + enabled: 0 + settings: + AddToEmbeddedBinaries: false + CPU: AnyCPU + CompileFlags: + FrameworkDependencies: + userData: + assetBundleName: + assetBundleVariant: diff --git a/Plugins/Windows.meta b/Plugins/Windows.meta new file mode 100644 index 0000000..7ff8f0a --- /dev/null +++ b/Plugins/Windows.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: a19a0f2ccc0234e7aa4580e8663cd3f0 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Plugins/Windows/DirectML.dll b/Plugins/Windows/DirectML.dll new file mode 100644 index 0000000..4a6e877 Binary files /dev/null and b/Plugins/Windows/DirectML.dll differ diff --git a/Plugins/Windows/DirectML.dll.meta b/Plugins/Windows/DirectML.dll.meta new file mode 100644 index 0000000..6ba59b7 --- /dev/null +++ b/Plugins/Windows/DirectML.dll.meta @@ -0,0 +1,63 @@ +fileFormatVersion: 2 +guid: 4ffa6b00fb7ae5447a0aeb05a86ff05d +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + defineConstraints: [] + isPreloaded: 1 + isOverridable: 0 + isExplicitlyReferenced: 0 + validateReferences: 1 + platformData: + - first: + : Any + second: + enabled: 0 + settings: + Exclude Editor: 0 + Exclude Linux64: 0 + Exclude OSXUniversal: 0 + Exclude Win: 1 + Exclude Win64: 0 + - first: + Any: + second: + enabled: 0 + settings: {} + - first: + Editor: Editor + second: + enabled: 1 + settings: + CPU: x86_64 + DefaultValueInitialized: true + OS: Windows + - first: + Standalone: Linux64 + second: + enabled: 1 + settings: + CPU: None + - first: + Standalone: OSXUniversal + second: + enabled: 1 + settings: + CPU: None + - first: + Standalone: Win + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: Win64 + second: + enabled: 1 + settings: + CPU: x86_64 + userData: + assetBundleName: + assetBundleVariant: diff --git a/Plugins/Windows/NatML.dll b/Plugins/Windows/NatML.dll new file mode 100644 index 0000000..5d559e7 Binary files /dev/null and b/Plugins/Windows/NatML.dll differ diff --git a/Plugins/Windows/NatML.dll.meta b/Plugins/Windows/NatML.dll.meta new file mode 100644 index 0000000..5a05a27 --- /dev/null +++ b/Plugins/Windows/NatML.dll.meta @@ -0,0 +1,63 @@ +fileFormatVersion: 2 +guid: ff8039324bc6445f88df9751d65523db +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + defineConstraints: [] + isPreloaded: 0 + isOverridable: 0 + isExplicitlyReferenced: 0 + validateReferences: 1 + platformData: + - first: + : Any + second: + enabled: 0 + settings: + Exclude Editor: 0 + Exclude Linux64: 0 + Exclude OSXUniversal: 0 + Exclude Win: 1 + Exclude Win64: 0 + - first: + Any: + second: + enabled: 0 + settings: {} + - first: + Editor: Editor + second: + enabled: 1 + settings: + CPU: x86_64 + DefaultValueInitialized: true + OS: Windows + - first: + Standalone: Linux64 + second: + enabled: 1 + settings: + CPU: None + - first: + Standalone: OSXUniversal + second: + enabled: 1 + settings: + CPU: None + - first: + Standalone: Win + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: Win64 + second: + enabled: 1 + settings: + CPU: x86_64 + userData: + assetBundleName: + assetBundleVariant: diff --git a/Plugins/iOS.meta b/Plugins/iOS.meta new file mode 100644 index 0000000..6fdb30b --- /dev/null +++ b/Plugins/iOS.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: b804d42cf4c3b4ef3b4d4f6e4197bb3b +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Plugins/iOS/NatML.framework.meta b/Plugins/iOS/NatML.framework.meta new file mode 100644 index 0000000..9b87247 --- /dev/null +++ b/Plugins/iOS/NatML.framework.meta @@ -0,0 +1,81 @@ +fileFormatVersion: 2 +guid: 57c582c395c394df8947a34b16a3aa52 +folderAsset: yes +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + defineConstraints: [] + isPreloaded: 0 + isOverridable: 0 + isExplicitlyReferenced: 0 + validateReferences: 1 + platformData: + - first: + : Any + second: + enabled: 0 + settings: + Exclude Android: 1 + Exclude Editor: 1 + Exclude Linux64: 1 + Exclude OSXUniversal: 1 + Exclude Win: 1 + Exclude Win64: 1 + Exclude iOS: 0 + - first: + Android: Android + second: + enabled: 0 + settings: + CPU: ARMv7 + - first: + Any: + second: + enabled: 0 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + CPU: AnyCPU + DefaultValueInitialized: true + OS: AnyOS + - first: + Standalone: Linux64 + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: OSXUniversal + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: Win + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: Win64 + second: + enabled: 0 + settings: + CPU: None + - first: + iPhone: iOS + second: + enabled: 1 + settings: + AddToEmbeddedBinaries: true + CPU: ARM64 + CompileFlags: + FrameworkDependencies: + userData: + assetBundleName: + assetBundleVariant: diff --git a/Plugins/iOS/NatML.framework/Headers/NMLFeature.h b/Plugins/iOS/NatML.framework/Headers/NMLFeature.h new file mode 100644 index 0000000..92b4815 --- /dev/null +++ b/Plugins/iOS/NatML.framework/Headers/NMLFeature.h @@ -0,0 +1,289 @@ +// +// NMLFeature.h +// NatML +// +// Created by Yusuf Olokoba on 3/14/2021. +// Copyright © 2023 NatML Inc. All rights reserved. +// + +#pragma once + +#include "NMLFeatureType.h" + +#pragma region --Enumerations-- +/*! + @enum NMLFeatureFlags + + @abstract Feature flags. + + @constant NML_FEATURE_FLAG_NONE + No flags. + + @constant NML_ARRAY_FLAG_COPY_DATA + Copy input tensor data when creating feature. + When this flag is not set, the feature data MUST remain valid for the lifetime of the created feature. + + @constant NML_IMAGE_FLAG_ASPECT_SCALE + Image feature is scaled to fit feature size. + + @constant NML_IMAGE_FLAG_ASPECT_FILL + Image feature is aspect-filled to the feature size. + + @constant NML_IMAGE_FLAG_ASPECT_FIT + Image feature is aspect-fit to the feature size. +*/ +enum NMLFeatureFlags { + NML_FEATURE_FLAG_NONE = 0, + // MLArrayFeature + NML_ARRAY_FLAG_COPY_DATA = 1, + // MLImageFeature::AspectMode + NML_IMAGE_FLAG_ASPECT_SCALE = 0, + NML_IMAGE_FLAG_ASPECT_FILL = 1, + NML_IMAGE_FLAG_ASPECT_FIT = 2, +}; +typedef enum NMLFeatureFlags NMLFeatureFlags; +#pragma endregion + + +#pragma region --Types-- +/*! + @struct NMLFeature + + @abstract ML model input or output feature. + + @discussion ML model input or output feature. + This is loosely based on `DLPack::DLTensor`. +*/ +struct NMLFeature; +typedef struct NMLFeature NMLFeature; +#pragma endregion + + +#pragma region --Operations-- +/*! + @function NMLReleaseFeature + + @abstract Release an ML feature. + + @discussion Release an ML feature. + + @param feature + ML feature. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLReleaseFeature (NMLFeature* feature); + +/*! + @function NMLFeatureGetType + + @abstract Get the feature type. + + @discussion Get the feature type. + + @param feature + ML feature. + + @param type + Output feature type. This type should be released once it is no longer in use. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLFeatureGetType ( + NMLFeature* feature, + NMLFeatureType** type +); + +/*! + @function NMLFeatureGetData + + @abstract Get the feature data. + + @discussion Get the feature data. + + @param feature + ML feature. + + @returns Opaque pointer to feature data. +*/ +NML_BRIDGE NML_EXPORT void* NML_API NMLFeatureGetData (NMLFeature* feature); +#pragma endregion + + +#pragma region --Constructors-- +/*! + @function NMLCreateArrayFeature + + @abstract Create an array feature from a data buffer. + + @discussion Create an array feature from a data buffer. + The data will not be released when the feature is released. + + @param data + Feature data. + + @param shape + Feature shape. + + @param dims + Number of dimensions in shape. + + @param dtype + Feature data type. + + @param flags + Feature creation flags. + + @param feature + Output ML feature. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLCreateArrayFeature ( + void* data, + const int32_t* shape, + int32_t dims, + NMLDataType dtype, + NMLFeatureFlags flags, + NMLFeature** feature +); + +/*! + @function NMLCreateImageFeature + + @abstract Create an image feature from a pixel buffer. + + @discussion Create an image feature from a pixel buffer. + The pixel buffer MUST have an RGBA8888 layout (32 bits per pixel). + + @param pixelBuffer + Pixel buffer to convert to a feature. + + @param width + Pixel buffer width. + + @param height + Pixel buffer height. + + @param shape + Feature shape. + + @param dtype + Feature data type. This MUST be `NML_DTYPE_FLOAT32`. + + @param mean + Per-channel normalization mean. + + @param std + Per-channel normalization standard deviation. + + @param flags + Feature creation flags. + + @param feature + Output ML feature. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLCreateImageFeature ( + const uint8_t* pixelBuffer, + int32_t width, + int32_t height, + const int32_t shape[4], + NMLDataType dtype, + const float* mean, + const float* std, + NMLFeatureFlags flags, + NMLFeature** feature +); + +/*! + @function NMLCreateAudioFeature + + @abstract Create an audio feature from a sample buffer. + + @discussion Create an audio feature from a sample buffer. + The sample buffer MUST be linear PCM interleaved by channel in range [-1.0, 1.0]. + + @param sampleBuffer + Sample buffer to convert to a feature. + + @param bufferSampleRate + Sample rate of the sample buffer. + + @param bufferShape + Tensor shape of the sample buffer. + This is always (1,F,C) where `F` is the frame count and `C` is the channel count. + + @param sampleRate + Feature sample rate. Sample buffer will be resampled if necessary. + + @param channelCount + Feature channel count. Sample buffer will be resampled if necessary. + + @param dtype + Feature data type. This MUST be `NML_DTYPE_FLOAT32`. + + @param mean + Per-channel normalization mean. + + @param std + Per-channel normalization standard deviation. + + @param flags + Feature creation flags. + + @param feature + Output ML feature. + This will always be planar with shape (1,C,F) where `C` is the feature channel count and `F` is the feature frame count. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLCreateAudioFeature ( + const float* sampleBuffer, + int32_t bufferSampleRate, + const int32_t bufferShape[3], + int32_t sampleRate, + int32_t channelCount, + NMLDataType dtype, + const float* mean, + const float* std, + NMLFeatureFlags flags, + NMLFeature** feature +); +#pragma endregion + + +#pragma region --Utilities-- +/*! + @function NMLImageFeatureCopyTo + + @abstract Copy image data from an image feature. + + @discussion Copy image data from an image feature. + + @param srcBuffer + Source pixel buffer. + This MUST have an `RGBA8888` layout. + + @param width + Source pixel buffer width. + + @param height + Source pixel buffer height. + + @param rect + Region of interest rectangle in pixel coordinates. + This MUST be in format (cx,cy,w,h). + + @param rotation + Region of interest clockwise rotation in degrees. + + @param background + Background color for out-of-bounds pixels. This is (R,G,B,A). + + @param dstBuffer + Destination pixel buffer. + This MUST have an `RGBA8888` layout. + This MUST be large enough to hold the enough pixel data for the given rectangle size. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLImageFeatureCopyTo ( + const uint8_t* srcBuffer, + int32_t width, + int32_t height, + const int32_t rect[4], + float rotation, + const uint8_t background[4], + uint8_t* dstBuffer +); +#pragma endregion \ No newline at end of file diff --git a/Plugins/iOS/NatML.framework/Headers/NMLFeatureReader.h b/Plugins/iOS/NatML.framework/Headers/NMLFeatureReader.h new file mode 100644 index 0000000..89443b5 --- /dev/null +++ b/Plugins/iOS/NatML.framework/Headers/NMLFeatureReader.h @@ -0,0 +1,156 @@ +// +// NMLMedia.h +// NatML +// +// Created by Yusuf Olokoba on 1/23/2022. +// Copyright © 2023 NatML Inc. All rights reserved. +// + +#include "NMLFeature.h" + +#pragma region --Types-- +/*! + @struct NMLFeatureReader + + @abstract Feature reader for reading feature data from media sources. + + @discussion Feature reader for reading feature data from media sources. +*/ +struct NMLFeatureReader; +typedef struct NMLFeatureReader NMLFeatureReader; +#pragma endregion + + +#pragma region --Inspection-- +/*! + @function NMLFeatureReaderGetVideoFormat + + @abstract Get the video format from a video file. + + @discussion Get the video format from a video file. + + @param path + Path to video file on the system. + + @param outWidth + Output width. + + @param outHeight + Output width. + + @param outFrames + Output frame count. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLFeatureReaderGetVideoFormat ( + const char* path, + int32_t* outWidth, + int32_t* outHeight, + int32_t* outFrames +); + +/*! + @function NMLFeatureReaderGetAudioFormat + + @abstract Get the audio format from a video or audio file. + + @discussion Get the audio format from a video or audio file. + + @param path + Path to video or audio file on the system. + + @param outSampleRate + Output sample rate. + + @param outChannelCount + Output channel count. + + @param outSampleCount + Output total sample count. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLFeatureReaderGetAudioFormat ( + const char* path, + int32_t* outSampleRate, + int32_t* outChannelCount, + int32_t* outSampleCount +); +#pragma endregion + + +#pragma region --Reader-- +/*! + @function NMLReleaseFeatureReader + + @abstract Release a feature reader. + + @discussion Release a feature reader. + + @param reader + ML feature reader. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLReleaseFeatureReader (NMLFeatureReader* reader); + +/*! + @function NMLFeatureReaderReadNextFeature + + @abstract Read the next feature from a feature reader. + + @discussion Read the next feature from a feature reader. + The feature must be released with `NMLReleaseFeature` when it is no longer needed. + If there are no more features available, the output timestamp will be `-1`. + + @param reader + ML feature reader. + + @param timestamp + Output feature timestamp in nanoseconds. + + @param feature + Output feature. The feature MUST be released when it is no longer needed. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLFeatureReaderReadNextFeature ( + NMLFeatureReader* reader, + int64_t* timestamp, + NMLFeature** feature +); +#pragma endregion + + +#pragma region --Constructors-- +/*! + @function NMLCreateImageFeatureReader + + @abstract Create an image feature reader. + + @discussion Create an image feature reader. + This currently supports reading from `.mp4` files. + + @param path + Path to video file. + + @param reader + Output ML feature reader. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLCreateImageFeatureReader ( + const char* path, + NMLFeatureReader** reader +); + +/*! + @function NMLCreateAudioFeatureReader + + @abstract Create an audio feature reader. + + @discussion Create an audio feature reader. + This currently supports reading from `.mp3` and `.mp4` files. + + @param path + Path to audio or video file. + + @param reader + Output ML feature reader. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLCreateAudioFeatureReader ( + const char* path, + NMLFeatureReader** reader +); +#pragma endregion \ No newline at end of file diff --git a/Plugins/iOS/NatML.framework/Headers/NMLFeatureType.h b/Plugins/iOS/NatML.framework/Headers/NMLFeatureType.h new file mode 100644 index 0000000..183eae5 --- /dev/null +++ b/Plugins/iOS/NatML.framework/Headers/NMLFeatureType.h @@ -0,0 +1,229 @@ +// +// NMLFeatureType.h +// NatML +// +// Created by Yusuf Olokoba on 3/14/2021. +// Copyright © 2023 NatML Inc. All rights reserved. +// + +#pragma once + +#include +#include "NMLTypes.h" + +#pragma region --Enumerations-- +/*! + @enum NMLDataType + + @abstract Feature data type. + + @constant NML_DTYPE_UNDEFINED + Type is undefined or invalid. + + @constant NML_DTYPE_INT8 + Type is a `int8_t` in C/C++ and `sbyte` in C#. + + @constant NML_DTYPE_INT16 + Type is `int16_t` in C/C++ and `short` in C#. + + @constant NML_DTYPE_INT32 + Type is `int32_t` in C/C++ and `int` in C#. + + @constant NML_DTYPE_INT64 + Type is `int64_t` in C/C++ and `long` in C#. + + @constant NML_DTYPE_UINT8 + Type is `uint8_t` in C/C++ and `byte` in C#. + + @constant NML_DTYPE_UINT16 + Type is a `uint16_t` in C/C++ and `ushort` in C#. + + @constant NML_DTYPE_UINT32 + Type is a `uint32_t` in C/C++ and `uint` in C#. + + @constant NML_DTYPE_UINT64 + Type is a `uint64_t` in C/C++ and `ulong` in C#. + + @constant NML_DTYPE_FLOAT16 + Type is a generic half-precision float. + + @constant NML_DTYPE_FLOAT32 + Type is `float` in C/C++/C#. + + @constant NML_DTYPE_FLOAT64 + Type is `double` in C/C++/C#. + + @constant NML_DTYPE_BOOL + Type is a `bool` in C/C++/C#. + + @constant NML_DTYPE_STRING + Type is `std::string` in C++ and `string` in C#. + + @constant NML_DTYPE_IMAGE + Type is an encoded image. + + @constant NML_DTYPE_BINARY + Type is a binary blob. + + @constant NML_DTYPE_LIST + Type is a sequence. + + @constant NML_DTYPE_DICT + Type is a dictionary. +*/ +enum NMLDataType { + NML_DTYPE_UNDEFINED = 0, + NML_DTYPE_INT8 = 10, + NML_DTYPE_INT16 = 2, + NML_DTYPE_INT32 = 3, + NML_DTYPE_INT64 = 4, + NML_DTYPE_UINT8 = 1, + NML_DTYPE_UINT16 = 11, + NML_DTYPE_UINT32 = 12, + NML_DTYPE_UINT64 = 13, + NML_DTYPE_FLOAT16 = 14, + NML_DTYPE_FLOAT32 = 5, + NML_DTYPE_FLOAT64 = 6, + NML_DTYPE_BOOL = 15, + NML_DTYPE_STRING = 7, + NML_DTYPE_IMAGE = 16, + NML_DTYPE_BINARY = 17, + NML_DTYPE_LIST = 8, + NML_DTYPE_DICT = 9, +}; +typedef enum NMLDataType NMLDataType; +#pragma endregion + + +#pragma region --Types-- +/*! + @struct NMLFeatureType + + @abstract Descriptor for an ML feature. + + @discussion Descriptor for an ML feature. +*/ +struct NMLFeatureType; +typedef struct NMLFeatureType NMLFeatureType; +#pragma endregion + + +#pragma region --Lifecycle-- +/*! + @function NMLCreateFeatureType + + @abstract Create an ML feature type. + + @discussion Create an ML feature type. + + @param name + Feature type name. + + @param dtype + Feature data type. + + @param dims + Feature dimensions. + Use `0` for scalar features and `-1` for unknown or undefined dims. + + @param shape + Feature shape. + This must contain at least `dims` integers when specified. + This can be `NULL` for feature types that do not describe tensors. + + @param type + Created ML feature type. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLCreateFeatureType ( // INCOMPLETE + const char* name, + NMLDataType dtype, + int32_t dims, + int32_t* shape, + NMLFeatureType** type +); + +/*! + @function NMLReleaseFeatureType + + @abstract Release an ML feature type. + + @discussion Release an ML feature type. + + @param type + ML feature type to release. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLReleaseFeatureType (NMLFeatureType* type); +#pragma endregion + + +#pragma region --Operations-- +/*! + @function NMLFeatureTypeName + + @abstract Get the name of a given feature type. + + @discussion Get the name of a given feature type. + + @param type + ML feature type. + + @param name + Destination UTF-8 string. + + @param size + Size of destination buffer in bytes. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLFeatureTypeGetName ( + NMLFeatureType* type, + char* name, + int32_t size +); + +/*! + @function NMLFeatureTypeDataType + + @abstract Get the data type of a given feature type. + + @discussion Get the data type of a given feature type. + + @param type + ML feature type. +*/ +NML_BRIDGE NML_EXPORT NMLDataType NML_API NMLFeatureTypeGetDataType (NMLFeatureType* type); + +/*! + @function NMLFeatureTypeDimensions + + @abstract Get the number of dimensions for a given feature type. + + @discussion Get the number of dimensions for a given feature type. + If the type is not a tensor, this function will return `0`. + + @param type + ML feature type. +*/ +NML_BRIDGE NML_EXPORT int32_t NML_API NMLFeatureTypeGetDimensions (NMLFeatureType* type); + +/*! + @function NMLFeatureTypeShape + + @abstract Get the shape of a given feature type. + + @discussion Get the shape of a given feature type. + The length of the shape array must be at least as large as the number of dimensions present in the type. + + @param type + ML feature type. + + @param shape + Destination shape array. + + @param shapeLen + Destination shape array length. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLFeatureTypeGetShape ( + NMLFeatureType* type, + int32_t* shape, + int32_t shapeLen +); +#pragma endregion diff --git a/Plugins/iOS/NatML.framework/Headers/NMLModel.h b/Plugins/iOS/NatML.framework/Headers/NMLModel.h new file mode 100644 index 0000000..cab0b0c --- /dev/null +++ b/Plugins/iOS/NatML.framework/Headers/NMLModel.h @@ -0,0 +1,251 @@ +// +// NMLModel.h +// NatML +// +// Created by Yusuf Olokoba on 3/14/2021. +// Copyright © 2023 NatML Inc. All rights reserved. +// + +#pragma once + +#include "NMLFeature.h" +#include "NMLModelConfiguration.h" + +#pragma region --Types-- +/*! + @struct NMLModel + + @abstract Opaque type for a NatML model. +*/ +struct NMLModel; +typedef struct NMLModel NMLModel; + +/*! + @abstract Callback invoked with created model. + + @param context + User context provided to the model creation function. + + @param model + ML model. If model creation fails for any reason, this will be `NULL`. +*/ +typedef void (*NMLModelCreationHandler) (void* context, NMLModel* model); +#pragma endregion + + +#pragma region --Lifecycle-- +/*! + @function NMLCreateModel + + @abstract Create an ML model. + + @discussion Create an ML model. + + @param buffer + ML model data. + The buffer can be released immediately after calling this function. + + @param bufferSize + ML model data size. + + @param configuration + ML model configuration. + This can be `NULL` in which case reasonable defaults will be used. + The configuration can be released immediately after calling this function. + + @param handler + Callback to receive created model. + + @param context + User context to pass to handler. Can be `NULL`. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLCreateModel ( + const uint8_t* buffer, + int64_t bufferSize, + NMLModelConfiguration* configuration, + NMLModelCreationHandler handler, + void* context +); + +/*! + @function NMLReleaseModel + + @abstract Release an ML model. + + @discussion Release an ML model. + + @param model + ML model. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLReleaseModel (NMLModel* model); +#pragma endregion + + +#pragma region --Metadata-- +/*! + @function NMLModelGetMetadataCount + + @abstract Get the number of metadata keys in a model. + + @discussion Get the number of metadata keys in a model. + + @param model + ML model. + + @returns Number of metadata keys in the model. +*/ +NML_BRIDGE NML_EXPORT int32_t NML_API NMLModelGetMetadataCount (NMLModel* model); + +/*! + @function NMLModelGetMetadataKey + + @abstract Get the metadata key in a model. + + @discussion Get the metadata key in a model. + + @param model + ML model. + + @param index + Metadata key index. + + @param key + Destination UTF-8 encoded key string. + + @param size + Size of destination buffer. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLModelGetMetadataKey ( + NMLModel* model, + int32_t index, + char* key, + int32_t size +); + +/*! + @function NMLModelGetMetadataValue + + @abstract Get the metadata value for a correspondig key in a model. + + @discussion Get the metadata value for a correspondig key in a model. + + @param model + ML model. + + @param key + Metadata key. + + @param value + Destination UTF-8 encoded value string. + + @param size + Size of destination buffer. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLModelGetMetadataValue ( + NMLModel* model, + const char* key, + char* value, + int32_t size +); +#pragma endregion + + +#pragma region --Inspection-- +/*! + @function NMLModelGetInputFeatureCount + + @abstract Get the number of input features in a model. + + @discussion Get the number of input features in a model. + + @param model + ML model. + + @returns Number of input features in the model. +*/ +NML_BRIDGE NML_EXPORT int32_t NML_API NMLModelGetInputFeatureCount (NMLModel* model); + +/*! + @function NMLModelGetInputFeatureType + + @abstract Get the input feature type in a model. + + @discussion Get the input feature type in a model. + + @param model + ML model. + + @param index + Input feature index. + + @param type + Output feature type. This type should be released once it is no longer in use. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLModelGetInputFeatureType ( + NMLModel* model, + int32_t index, + NMLFeatureType** type +); + +/*! + @function NMLModelGetOutputFeatureCount + + @abstract Get the number of output features in a model. + + @discussion Get the number of output features in a model. + + @param model + ML model. + + @returns Number of output features in the model. +*/ +NML_BRIDGE NML_EXPORT int32_t NML_API NMLModelGetOutputFeatureCount (NMLModel* model); + +/*! + @function NMLModelGetOutputFeatureType + + @abstract Get the output feature type in a model. + + @discussion Get the output feature type in a model. + + @param model + ML model. + + @param index + Output feature index. + + @param type + Output feature type. This type should be released once it is no longer in use. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLModelGetOutputFeatureType ( + NMLModel* model, + int32_t index, + NMLFeatureType** type +); +#pragma endregion + + +#pragma region --Prediction-- +/*! + @function NMLModelPredict + + @abstract Make a prediction with a model. + + @discussion Make a prediction with a model. + + @param model + ML model. + + @param inputs + Input features to run prediction on. The length of this array must match the model's input count. + + @param outputs + Output features. The length of this array must match the model's output count. + Each feature MUST be released when no longer needed. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLModelPredict ( + NMLModel* model, + NMLFeature * const * inputs, + NMLFeature** outputs +); +#pragma endregion \ No newline at end of file diff --git a/Plugins/iOS/NatML.framework/Headers/NMLModelConfiguration.h b/Plugins/iOS/NatML.framework/Headers/NMLModelConfiguration.h new file mode 100644 index 0000000..44d589b --- /dev/null +++ b/Plugins/iOS/NatML.framework/Headers/NMLModelConfiguration.h @@ -0,0 +1,190 @@ +// +// NMLModelConfiguration.h +// NatML +// +// Created by Yusuf Olokoba on 9/2/2022. +// Copyright © 2023 NatML Inc. All rights reserved. +// + +#pragma once + +#include +#include "NMLTypes.h" + +#pragma region --Enumerations-- +/*! + @enum NMLComputeTarget + + @abstract Compute target used for ML model predictions. + + @constant NML_COMPUTE_TARGET_DEFAULT + Use the default compute target for the given platform. + + @constant NML_COMPUTE_TARGET_CPU + Use the CPU. + + @constant NML_COMPUTE_TARGET_GPU + Use the GPU. + + @constant NML_COMPUTE_TARGET_NPU + Use the neural processing unit + + @constant NML_COMPUTE_TARGET_ALL + Use all available compute targets including the CPU, GPU, and neural processing units. +*/ +enum NMLComputeTarget { + NML_COMPUTE_TARGET_DEFAULT = 0, + NML_COMPUTE_TARGET_CPU = 1 << 0, + NML_COMPUTE_TARGET_GPU = 1 << 1, + NML_COMPUTE_TARGET_NPU = 1 << 2, + NML_COMPUTE_TARGET_ALL = NML_COMPUTE_TARGET_CPU | NML_COMPUTE_TARGET_GPU | NML_COMPUTE_TARGET_NPU +}; +typedef enum NMLComputeTarget NMLComputeTarget; +#pragma endregion + + +#pragma region --Types-- +/*! + @struct NMLModelConfiguration + + @abstract Opaque type for NatML edge model configuration. +*/ +struct NMLModelConfiguration; +typedef struct NMLModelConfiguration NMLModelConfiguration; + +/*! + @abstract Callback invoked with created secret. + + @param context + User context provided to the secret creation function. + + @param secret + Predictor session secret. If secret creation fails for any reason, this will be `NULL`. +*/ +typedef void (*NMLSecretCreationHandler) (void* context, const char* secret); +#pragma endregion + + +#pragma region --Lifecycle-- +/*! + @function NMLCreateModelConfiguration + + @abstract Create ML model configuration. + + @discussion Create ML model configuration. + + @param configuration + Destination configuration. Must not be `NULL`. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLCreateModelConfiguration (NMLModelConfiguration** configuration); + +/*! + @function NMLReleaseModelConfiguration + + @abstract Release ML model configuration. + + @discussion Release ML model configuration. + + @param configuration + ML model configuration. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLReleaseModelConfiguration (NMLModelConfiguration* configuration); +#pragma endregion + + +#pragma region --Configuration-- +/*! + @function NMLModelConfigurationSetComputeTarget + + @abstract Specify the compute target used for ML model predictions. + + @discussion Specify the compute target used for ML model predictions. + + @param configuration + ML model configuration. + + @param target + Compute target used for ML model predictions. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLModelConfigurationSetComputeTarget ( + NMLModelConfiguration* configuration, + NMLComputeTarget target +); + +/*! + @function NMLModelConfigurationSetComputeDevice + + @abstract Specify the compute device used for ML model predictions. + + @discussion Specify the compute device used for ML model predictions. + + @param configuration + ML model configuration. + + @param device + Compute device used for ML model predictions. + The type of this device is platform-dependent. + Pass `NULL` to use the default compute device. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLModelConfigurationSetComputeDevice ( + NMLModelConfiguration* configuration, + void* device +); + +/*! + @function NMLModelCofigurationSetFingerprint + + @abstract Set predictor session fingerprint. + + @discussion Set predictor session fingerprint. + + @param configuration + ML model configuration. + + @param fingerprint + Predictor session fingerprint. Can be `NULL`. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLModelConfigurationSetFingerprint ( + NMLModelConfiguration* configuration, + const char* fingerprint +); + +/*! + @function NMLModelConfigurationSetSecret + + @abstract Set predictor session secret. + + @discussion Set predictor session secret. + + @param configuration + ML model configuration. + + @param secret + Predictor session secret. Can be `NULL`. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLModelConfigurationSetSecret ( + NMLModelConfiguration* configuration, + const char* secret +); +#pragma endregion + + +#pragma region --Secret-- +/*! + @function NMLModelConfigurationCreateSecret + + @abstract Create a predictor session secret. + + @discussion Create a predictor session secret. + + @param handler + Callback to receive the created secret. + + @param context + User context to pass to handler. Can be `NULL`. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLModelConfigurationCreateSecret ( + NMLSecretCreationHandler handler, + void* context +); +#pragma endregion \ No newline at end of file diff --git a/Plugins/iOS/NatML.framework/Headers/NMLTypes.h b/Plugins/iOS/NatML.framework/Headers/NMLTypes.h new file mode 100644 index 0000000..b9e9119 --- /dev/null +++ b/Plugins/iOS/NatML.framework/Headers/NMLTypes.h @@ -0,0 +1,29 @@ +// +// NMLTypes.h +// NatML +// +// Created by Yusuf Olokoba on 3/14/2021. +// Copyright © 2023 NatML Inc. All rights reserved. +// + +#pragma once + +#ifdef __cplusplus + #define NML_BRIDGE extern "C" +#else + #define NML_BRIDGE extern +#endif + +#ifdef _WIN64 + #define NML_EXPORT __declspec(dllexport) +#else + #define NML_EXPORT +#endif + +#ifdef __EMSCRIPTEN__ + #define NML_API EMSCRIPTEN_KEEPALIVE +#elif defined(_WIN64) + #define NML_API APIENTRY +#else + #define NML_API +#endif \ No newline at end of file diff --git a/Plugins/iOS/NatML.framework/Headers/NatML.h b/Plugins/iOS/NatML.framework/Headers/NatML.h new file mode 100644 index 0000000..8ee3e10 --- /dev/null +++ b/Plugins/iOS/NatML.framework/Headers/NatML.h @@ -0,0 +1,17 @@ +// +// NatML.h +// NatML +// +// Created by Yusuf Olokoba on 10/02/2020. +// Copyright © 2023 NatML Inc. All rights reserved. +// + +#pragma once + +#include "NMLTypes.h" +#include "NMLFeatureType.h" +#include "NMLFeature.h" +#include "NMLModelConfiguration.h" +#include "NMLModel.h" +#include "NMLPredictor.h" +#include "NMLFeatureReader.h" \ No newline at end of file diff --git a/Plugins/iOS/NatML.framework/Info.plist b/Plugins/iOS/NatML.framework/Info.plist new file mode 100644 index 0000000..61d25b6 Binary files /dev/null and b/Plugins/iOS/NatML.framework/Info.plist differ diff --git a/Plugins/iOS/NatML.framework/Modules/module.modulemap b/Plugins/iOS/NatML.framework/Modules/module.modulemap new file mode 100644 index 0000000..65fd156 --- /dev/null +++ b/Plugins/iOS/NatML.framework/Modules/module.modulemap @@ -0,0 +1,6 @@ +framework module NatML { + umbrella header "NatML.h" + + export * + module * { export * } +} diff --git a/Plugins/iOS/NatML.framework/NatML b/Plugins/iOS/NatML.framework/NatML new file mode 100755 index 0000000..6b06b81 Binary files /dev/null and b/Plugins/iOS/NatML.framework/NatML differ diff --git a/Plugins/macOS.meta b/Plugins/macOS.meta new file mode 100644 index 0000000..e290886 --- /dev/null +++ b/Plugins/macOS.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 3fd8cc84f63d34e91b3e53944d0525eb +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Plugins/macOS/NatML.bundle.meta b/Plugins/macOS/NatML.bundle.meta new file mode 100644 index 0000000..e107d1a --- /dev/null +++ b/Plugins/macOS/NatML.bundle.meta @@ -0,0 +1,81 @@ +fileFormatVersion: 2 +guid: a7c59ac2086f546b6a57707c3001fc75 +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + defineConstraints: [] + isPreloaded: 0 + isOverridable: 0 + isExplicitlyReferenced: 0 + validateReferences: 1 + platformData: + - first: + : Any + second: + enabled: 0 + settings: + Exclude Android: 1 + Exclude Editor: 0 + Exclude Linux64: 1 + Exclude OSXUniversal: 0 + Exclude WebGL: 1 + Exclude Win: 1 + Exclude Win64: 1 + Exclude iOS: 1 + - first: + Android: Android + second: + enabled: 0 + settings: + CPU: ARMv7 + - first: + Any: + second: + enabled: 0 + settings: {} + - first: + Editor: Editor + second: + enabled: 1 + settings: + CPU: AnyCPU + DefaultValueInitialized: true + OS: OSX + - first: + Standalone: Linux64 + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: OSXUniversal + second: + enabled: 1 + settings: + CPU: AnyCPU + - first: + Standalone: Win + second: + enabled: 0 + settings: + CPU: x86 + - first: + Standalone: Win64 + second: + enabled: 0 + settings: + CPU: x86_64 + - first: + iPhone: iOS + second: + enabled: 0 + settings: + AddToEmbeddedBinaries: false + CPU: AnyCPU + CompileFlags: + FrameworkDependencies: + userData: + assetBundleName: + assetBundleVariant: diff --git a/Plugins/macOS/NatML.bundle/Contents/Headers/NMLFeature.h b/Plugins/macOS/NatML.bundle/Contents/Headers/NMLFeature.h new file mode 100644 index 0000000..92b4815 --- /dev/null +++ b/Plugins/macOS/NatML.bundle/Contents/Headers/NMLFeature.h @@ -0,0 +1,289 @@ +// +// NMLFeature.h +// NatML +// +// Created by Yusuf Olokoba on 3/14/2021. +// Copyright © 2023 NatML Inc. All rights reserved. +// + +#pragma once + +#include "NMLFeatureType.h" + +#pragma region --Enumerations-- +/*! + @enum NMLFeatureFlags + + @abstract Feature flags. + + @constant NML_FEATURE_FLAG_NONE + No flags. + + @constant NML_ARRAY_FLAG_COPY_DATA + Copy input tensor data when creating feature. + When this flag is not set, the feature data MUST remain valid for the lifetime of the created feature. + + @constant NML_IMAGE_FLAG_ASPECT_SCALE + Image feature is scaled to fit feature size. + + @constant NML_IMAGE_FLAG_ASPECT_FILL + Image feature is aspect-filled to the feature size. + + @constant NML_IMAGE_FLAG_ASPECT_FIT + Image feature is aspect-fit to the feature size. +*/ +enum NMLFeatureFlags { + NML_FEATURE_FLAG_NONE = 0, + // MLArrayFeature + NML_ARRAY_FLAG_COPY_DATA = 1, + // MLImageFeature::AspectMode + NML_IMAGE_FLAG_ASPECT_SCALE = 0, + NML_IMAGE_FLAG_ASPECT_FILL = 1, + NML_IMAGE_FLAG_ASPECT_FIT = 2, +}; +typedef enum NMLFeatureFlags NMLFeatureFlags; +#pragma endregion + + +#pragma region --Types-- +/*! + @struct NMLFeature + + @abstract ML model input or output feature. + + @discussion ML model input or output feature. + This is loosely based on `DLPack::DLTensor`. +*/ +struct NMLFeature; +typedef struct NMLFeature NMLFeature; +#pragma endregion + + +#pragma region --Operations-- +/*! + @function NMLReleaseFeature + + @abstract Release an ML feature. + + @discussion Release an ML feature. + + @param feature + ML feature. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLReleaseFeature (NMLFeature* feature); + +/*! + @function NMLFeatureGetType + + @abstract Get the feature type. + + @discussion Get the feature type. + + @param feature + ML feature. + + @param type + Output feature type. This type should be released once it is no longer in use. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLFeatureGetType ( + NMLFeature* feature, + NMLFeatureType** type +); + +/*! + @function NMLFeatureGetData + + @abstract Get the feature data. + + @discussion Get the feature data. + + @param feature + ML feature. + + @returns Opaque pointer to feature data. +*/ +NML_BRIDGE NML_EXPORT void* NML_API NMLFeatureGetData (NMLFeature* feature); +#pragma endregion + + +#pragma region --Constructors-- +/*! + @function NMLCreateArrayFeature + + @abstract Create an array feature from a data buffer. + + @discussion Create an array feature from a data buffer. + The data will not be released when the feature is released. + + @param data + Feature data. + + @param shape + Feature shape. + + @param dims + Number of dimensions in shape. + + @param dtype + Feature data type. + + @param flags + Feature creation flags. + + @param feature + Output ML feature. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLCreateArrayFeature ( + void* data, + const int32_t* shape, + int32_t dims, + NMLDataType dtype, + NMLFeatureFlags flags, + NMLFeature** feature +); + +/*! + @function NMLCreateImageFeature + + @abstract Create an image feature from a pixel buffer. + + @discussion Create an image feature from a pixel buffer. + The pixel buffer MUST have an RGBA8888 layout (32 bits per pixel). + + @param pixelBuffer + Pixel buffer to convert to a feature. + + @param width + Pixel buffer width. + + @param height + Pixel buffer height. + + @param shape + Feature shape. + + @param dtype + Feature data type. This MUST be `NML_DTYPE_FLOAT32`. + + @param mean + Per-channel normalization mean. + + @param std + Per-channel normalization standard deviation. + + @param flags + Feature creation flags. + + @param feature + Output ML feature. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLCreateImageFeature ( + const uint8_t* pixelBuffer, + int32_t width, + int32_t height, + const int32_t shape[4], + NMLDataType dtype, + const float* mean, + const float* std, + NMLFeatureFlags flags, + NMLFeature** feature +); + +/*! + @function NMLCreateAudioFeature + + @abstract Create an audio feature from a sample buffer. + + @discussion Create an audio feature from a sample buffer. + The sample buffer MUST be linear PCM interleaved by channel in range [-1.0, 1.0]. + + @param sampleBuffer + Sample buffer to convert to a feature. + + @param bufferSampleRate + Sample rate of the sample buffer. + + @param bufferShape + Tensor shape of the sample buffer. + This is always (1,F,C) where `F` is the frame count and `C` is the channel count. + + @param sampleRate + Feature sample rate. Sample buffer will be resampled if necessary. + + @param channelCount + Feature channel count. Sample buffer will be resampled if necessary. + + @param dtype + Feature data type. This MUST be `NML_DTYPE_FLOAT32`. + + @param mean + Per-channel normalization mean. + + @param std + Per-channel normalization standard deviation. + + @param flags + Feature creation flags. + + @param feature + Output ML feature. + This will always be planar with shape (1,C,F) where `C` is the feature channel count and `F` is the feature frame count. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLCreateAudioFeature ( + const float* sampleBuffer, + int32_t bufferSampleRate, + const int32_t bufferShape[3], + int32_t sampleRate, + int32_t channelCount, + NMLDataType dtype, + const float* mean, + const float* std, + NMLFeatureFlags flags, + NMLFeature** feature +); +#pragma endregion + + +#pragma region --Utilities-- +/*! + @function NMLImageFeatureCopyTo + + @abstract Copy image data from an image feature. + + @discussion Copy image data from an image feature. + + @param srcBuffer + Source pixel buffer. + This MUST have an `RGBA8888` layout. + + @param width + Source pixel buffer width. + + @param height + Source pixel buffer height. + + @param rect + Region of interest rectangle in pixel coordinates. + This MUST be in format (cx,cy,w,h). + + @param rotation + Region of interest clockwise rotation in degrees. + + @param background + Background color for out-of-bounds pixels. This is (R,G,B,A). + + @param dstBuffer + Destination pixel buffer. + This MUST have an `RGBA8888` layout. + This MUST be large enough to hold the enough pixel data for the given rectangle size. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLImageFeatureCopyTo ( + const uint8_t* srcBuffer, + int32_t width, + int32_t height, + const int32_t rect[4], + float rotation, + const uint8_t background[4], + uint8_t* dstBuffer +); +#pragma endregion \ No newline at end of file diff --git a/Plugins/macOS/NatML.bundle/Contents/Headers/NMLFeatureReader.h b/Plugins/macOS/NatML.bundle/Contents/Headers/NMLFeatureReader.h new file mode 100644 index 0000000..89443b5 --- /dev/null +++ b/Plugins/macOS/NatML.bundle/Contents/Headers/NMLFeatureReader.h @@ -0,0 +1,156 @@ +// +// NMLMedia.h +// NatML +// +// Created by Yusuf Olokoba on 1/23/2022. +// Copyright © 2023 NatML Inc. All rights reserved. +// + +#include "NMLFeature.h" + +#pragma region --Types-- +/*! + @struct NMLFeatureReader + + @abstract Feature reader for reading feature data from media sources. + + @discussion Feature reader for reading feature data from media sources. +*/ +struct NMLFeatureReader; +typedef struct NMLFeatureReader NMLFeatureReader; +#pragma endregion + + +#pragma region --Inspection-- +/*! + @function NMLFeatureReaderGetVideoFormat + + @abstract Get the video format from a video file. + + @discussion Get the video format from a video file. + + @param path + Path to video file on the system. + + @param outWidth + Output width. + + @param outHeight + Output width. + + @param outFrames + Output frame count. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLFeatureReaderGetVideoFormat ( + const char* path, + int32_t* outWidth, + int32_t* outHeight, + int32_t* outFrames +); + +/*! + @function NMLFeatureReaderGetAudioFormat + + @abstract Get the audio format from a video or audio file. + + @discussion Get the audio format from a video or audio file. + + @param path + Path to video or audio file on the system. + + @param outSampleRate + Output sample rate. + + @param outChannelCount + Output channel count. + + @param outSampleCount + Output total sample count. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLFeatureReaderGetAudioFormat ( + const char* path, + int32_t* outSampleRate, + int32_t* outChannelCount, + int32_t* outSampleCount +); +#pragma endregion + + +#pragma region --Reader-- +/*! + @function NMLReleaseFeatureReader + + @abstract Release a feature reader. + + @discussion Release a feature reader. + + @param reader + ML feature reader. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLReleaseFeatureReader (NMLFeatureReader* reader); + +/*! + @function NMLFeatureReaderReadNextFeature + + @abstract Read the next feature from a feature reader. + + @discussion Read the next feature from a feature reader. + The feature must be released with `NMLReleaseFeature` when it is no longer needed. + If there are no more features available, the output timestamp will be `-1`. + + @param reader + ML feature reader. + + @param timestamp + Output feature timestamp in nanoseconds. + + @param feature + Output feature. The feature MUST be released when it is no longer needed. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLFeatureReaderReadNextFeature ( + NMLFeatureReader* reader, + int64_t* timestamp, + NMLFeature** feature +); +#pragma endregion + + +#pragma region --Constructors-- +/*! + @function NMLCreateImageFeatureReader + + @abstract Create an image feature reader. + + @discussion Create an image feature reader. + This currently supports reading from `.mp4` files. + + @param path + Path to video file. + + @param reader + Output ML feature reader. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLCreateImageFeatureReader ( + const char* path, + NMLFeatureReader** reader +); + +/*! + @function NMLCreateAudioFeatureReader + + @abstract Create an audio feature reader. + + @discussion Create an audio feature reader. + This currently supports reading from `.mp3` and `.mp4` files. + + @param path + Path to audio or video file. + + @param reader + Output ML feature reader. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLCreateAudioFeatureReader ( + const char* path, + NMLFeatureReader** reader +); +#pragma endregion \ No newline at end of file diff --git a/Plugins/macOS/NatML.bundle/Contents/Headers/NMLFeatureType.h b/Plugins/macOS/NatML.bundle/Contents/Headers/NMLFeatureType.h new file mode 100644 index 0000000..183eae5 --- /dev/null +++ b/Plugins/macOS/NatML.bundle/Contents/Headers/NMLFeatureType.h @@ -0,0 +1,229 @@ +// +// NMLFeatureType.h +// NatML +// +// Created by Yusuf Olokoba on 3/14/2021. +// Copyright © 2023 NatML Inc. All rights reserved. +// + +#pragma once + +#include +#include "NMLTypes.h" + +#pragma region --Enumerations-- +/*! + @enum NMLDataType + + @abstract Feature data type. + + @constant NML_DTYPE_UNDEFINED + Type is undefined or invalid. + + @constant NML_DTYPE_INT8 + Type is a `int8_t` in C/C++ and `sbyte` in C#. + + @constant NML_DTYPE_INT16 + Type is `int16_t` in C/C++ and `short` in C#. + + @constant NML_DTYPE_INT32 + Type is `int32_t` in C/C++ and `int` in C#. + + @constant NML_DTYPE_INT64 + Type is `int64_t` in C/C++ and `long` in C#. + + @constant NML_DTYPE_UINT8 + Type is `uint8_t` in C/C++ and `byte` in C#. + + @constant NML_DTYPE_UINT16 + Type is a `uint16_t` in C/C++ and `ushort` in C#. + + @constant NML_DTYPE_UINT32 + Type is a `uint32_t` in C/C++ and `uint` in C#. + + @constant NML_DTYPE_UINT64 + Type is a `uint64_t` in C/C++ and `ulong` in C#. + + @constant NML_DTYPE_FLOAT16 + Type is a generic half-precision float. + + @constant NML_DTYPE_FLOAT32 + Type is `float` in C/C++/C#. + + @constant NML_DTYPE_FLOAT64 + Type is `double` in C/C++/C#. + + @constant NML_DTYPE_BOOL + Type is a `bool` in C/C++/C#. + + @constant NML_DTYPE_STRING + Type is `std::string` in C++ and `string` in C#. + + @constant NML_DTYPE_IMAGE + Type is an encoded image. + + @constant NML_DTYPE_BINARY + Type is a binary blob. + + @constant NML_DTYPE_LIST + Type is a sequence. + + @constant NML_DTYPE_DICT + Type is a dictionary. +*/ +enum NMLDataType { + NML_DTYPE_UNDEFINED = 0, + NML_DTYPE_INT8 = 10, + NML_DTYPE_INT16 = 2, + NML_DTYPE_INT32 = 3, + NML_DTYPE_INT64 = 4, + NML_DTYPE_UINT8 = 1, + NML_DTYPE_UINT16 = 11, + NML_DTYPE_UINT32 = 12, + NML_DTYPE_UINT64 = 13, + NML_DTYPE_FLOAT16 = 14, + NML_DTYPE_FLOAT32 = 5, + NML_DTYPE_FLOAT64 = 6, + NML_DTYPE_BOOL = 15, + NML_DTYPE_STRING = 7, + NML_DTYPE_IMAGE = 16, + NML_DTYPE_BINARY = 17, + NML_DTYPE_LIST = 8, + NML_DTYPE_DICT = 9, +}; +typedef enum NMLDataType NMLDataType; +#pragma endregion + + +#pragma region --Types-- +/*! + @struct NMLFeatureType + + @abstract Descriptor for an ML feature. + + @discussion Descriptor for an ML feature. +*/ +struct NMLFeatureType; +typedef struct NMLFeatureType NMLFeatureType; +#pragma endregion + + +#pragma region --Lifecycle-- +/*! + @function NMLCreateFeatureType + + @abstract Create an ML feature type. + + @discussion Create an ML feature type. + + @param name + Feature type name. + + @param dtype + Feature data type. + + @param dims + Feature dimensions. + Use `0` for scalar features and `-1` for unknown or undefined dims. + + @param shape + Feature shape. + This must contain at least `dims` integers when specified. + This can be `NULL` for feature types that do not describe tensors. + + @param type + Created ML feature type. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLCreateFeatureType ( // INCOMPLETE + const char* name, + NMLDataType dtype, + int32_t dims, + int32_t* shape, + NMLFeatureType** type +); + +/*! + @function NMLReleaseFeatureType + + @abstract Release an ML feature type. + + @discussion Release an ML feature type. + + @param type + ML feature type to release. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLReleaseFeatureType (NMLFeatureType* type); +#pragma endregion + + +#pragma region --Operations-- +/*! + @function NMLFeatureTypeName + + @abstract Get the name of a given feature type. + + @discussion Get the name of a given feature type. + + @param type + ML feature type. + + @param name + Destination UTF-8 string. + + @param size + Size of destination buffer in bytes. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLFeatureTypeGetName ( + NMLFeatureType* type, + char* name, + int32_t size +); + +/*! + @function NMLFeatureTypeDataType + + @abstract Get the data type of a given feature type. + + @discussion Get the data type of a given feature type. + + @param type + ML feature type. +*/ +NML_BRIDGE NML_EXPORT NMLDataType NML_API NMLFeatureTypeGetDataType (NMLFeatureType* type); + +/*! + @function NMLFeatureTypeDimensions + + @abstract Get the number of dimensions for a given feature type. + + @discussion Get the number of dimensions for a given feature type. + If the type is not a tensor, this function will return `0`. + + @param type + ML feature type. +*/ +NML_BRIDGE NML_EXPORT int32_t NML_API NMLFeatureTypeGetDimensions (NMLFeatureType* type); + +/*! + @function NMLFeatureTypeShape + + @abstract Get the shape of a given feature type. + + @discussion Get the shape of a given feature type. + The length of the shape array must be at least as large as the number of dimensions present in the type. + + @param type + ML feature type. + + @param shape + Destination shape array. + + @param shapeLen + Destination shape array length. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLFeatureTypeGetShape ( + NMLFeatureType* type, + int32_t* shape, + int32_t shapeLen +); +#pragma endregion diff --git a/Plugins/macOS/NatML.bundle/Contents/Headers/NMLModel.h b/Plugins/macOS/NatML.bundle/Contents/Headers/NMLModel.h new file mode 100644 index 0000000..cab0b0c --- /dev/null +++ b/Plugins/macOS/NatML.bundle/Contents/Headers/NMLModel.h @@ -0,0 +1,251 @@ +// +// NMLModel.h +// NatML +// +// Created by Yusuf Olokoba on 3/14/2021. +// Copyright © 2023 NatML Inc. All rights reserved. +// + +#pragma once + +#include "NMLFeature.h" +#include "NMLModelConfiguration.h" + +#pragma region --Types-- +/*! + @struct NMLModel + + @abstract Opaque type for a NatML model. +*/ +struct NMLModel; +typedef struct NMLModel NMLModel; + +/*! + @abstract Callback invoked with created model. + + @param context + User context provided to the model creation function. + + @param model + ML model. If model creation fails for any reason, this will be `NULL`. +*/ +typedef void (*NMLModelCreationHandler) (void* context, NMLModel* model); +#pragma endregion + + +#pragma region --Lifecycle-- +/*! + @function NMLCreateModel + + @abstract Create an ML model. + + @discussion Create an ML model. + + @param buffer + ML model data. + The buffer can be released immediately after calling this function. + + @param bufferSize + ML model data size. + + @param configuration + ML model configuration. + This can be `NULL` in which case reasonable defaults will be used. + The configuration can be released immediately after calling this function. + + @param handler + Callback to receive created model. + + @param context + User context to pass to handler. Can be `NULL`. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLCreateModel ( + const uint8_t* buffer, + int64_t bufferSize, + NMLModelConfiguration* configuration, + NMLModelCreationHandler handler, + void* context +); + +/*! + @function NMLReleaseModel + + @abstract Release an ML model. + + @discussion Release an ML model. + + @param model + ML model. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLReleaseModel (NMLModel* model); +#pragma endregion + + +#pragma region --Metadata-- +/*! + @function NMLModelGetMetadataCount + + @abstract Get the number of metadata keys in a model. + + @discussion Get the number of metadata keys in a model. + + @param model + ML model. + + @returns Number of metadata keys in the model. +*/ +NML_BRIDGE NML_EXPORT int32_t NML_API NMLModelGetMetadataCount (NMLModel* model); + +/*! + @function NMLModelGetMetadataKey + + @abstract Get the metadata key in a model. + + @discussion Get the metadata key in a model. + + @param model + ML model. + + @param index + Metadata key index. + + @param key + Destination UTF-8 encoded key string. + + @param size + Size of destination buffer. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLModelGetMetadataKey ( + NMLModel* model, + int32_t index, + char* key, + int32_t size +); + +/*! + @function NMLModelGetMetadataValue + + @abstract Get the metadata value for a correspondig key in a model. + + @discussion Get the metadata value for a correspondig key in a model. + + @param model + ML model. + + @param key + Metadata key. + + @param value + Destination UTF-8 encoded value string. + + @param size + Size of destination buffer. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLModelGetMetadataValue ( + NMLModel* model, + const char* key, + char* value, + int32_t size +); +#pragma endregion + + +#pragma region --Inspection-- +/*! + @function NMLModelGetInputFeatureCount + + @abstract Get the number of input features in a model. + + @discussion Get the number of input features in a model. + + @param model + ML model. + + @returns Number of input features in the model. +*/ +NML_BRIDGE NML_EXPORT int32_t NML_API NMLModelGetInputFeatureCount (NMLModel* model); + +/*! + @function NMLModelGetInputFeatureType + + @abstract Get the input feature type in a model. + + @discussion Get the input feature type in a model. + + @param model + ML model. + + @param index + Input feature index. + + @param type + Output feature type. This type should be released once it is no longer in use. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLModelGetInputFeatureType ( + NMLModel* model, + int32_t index, + NMLFeatureType** type +); + +/*! + @function NMLModelGetOutputFeatureCount + + @abstract Get the number of output features in a model. + + @discussion Get the number of output features in a model. + + @param model + ML model. + + @returns Number of output features in the model. +*/ +NML_BRIDGE NML_EXPORT int32_t NML_API NMLModelGetOutputFeatureCount (NMLModel* model); + +/*! + @function NMLModelGetOutputFeatureType + + @abstract Get the output feature type in a model. + + @discussion Get the output feature type in a model. + + @param model + ML model. + + @param index + Output feature index. + + @param type + Output feature type. This type should be released once it is no longer in use. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLModelGetOutputFeatureType ( + NMLModel* model, + int32_t index, + NMLFeatureType** type +); +#pragma endregion + + +#pragma region --Prediction-- +/*! + @function NMLModelPredict + + @abstract Make a prediction with a model. + + @discussion Make a prediction with a model. + + @param model + ML model. + + @param inputs + Input features to run prediction on. The length of this array must match the model's input count. + + @param outputs + Output features. The length of this array must match the model's output count. + Each feature MUST be released when no longer needed. +*/ +NML_BRIDGE NML_EXPORT void NML_API NMLModelPredict ( + NMLModel* model, + NMLFeature * const * inputs, + NMLFeature** outputs +); +#pragma endregion \ No newline at end of file diff --git a/Plugins/macOS/NatML.bundle/Contents/Headers/NMLTypes.h b/Plugins/macOS/NatML.bundle/Contents/Headers/NMLTypes.h new file mode 100644 index 0000000..b9e9119 --- /dev/null +++ b/Plugins/macOS/NatML.bundle/Contents/Headers/NMLTypes.h @@ -0,0 +1,29 @@ +// +// NMLTypes.h +// NatML +// +// Created by Yusuf Olokoba on 3/14/2021. +// Copyright © 2023 NatML Inc. All rights reserved. +// + +#pragma once + +#ifdef __cplusplus + #define NML_BRIDGE extern "C" +#else + #define NML_BRIDGE extern +#endif + +#ifdef _WIN64 + #define NML_EXPORT __declspec(dllexport) +#else + #define NML_EXPORT +#endif + +#ifdef __EMSCRIPTEN__ + #define NML_API EMSCRIPTEN_KEEPALIVE +#elif defined(_WIN64) + #define NML_API APIENTRY +#else + #define NML_API +#endif \ No newline at end of file diff --git a/Plugins/macOS/NatML.bundle/Contents/Headers/NatML.h b/Plugins/macOS/NatML.bundle/Contents/Headers/NatML.h new file mode 100644 index 0000000..8ee3e10 --- /dev/null +++ b/Plugins/macOS/NatML.bundle/Contents/Headers/NatML.h @@ -0,0 +1,17 @@ +// +// NatML.h +// NatML +// +// Created by Yusuf Olokoba on 10/02/2020. +// Copyright © 2023 NatML Inc. All rights reserved. +// + +#pragma once + +#include "NMLTypes.h" +#include "NMLFeatureType.h" +#include "NMLFeature.h" +#include "NMLModelConfiguration.h" +#include "NMLModel.h" +#include "NMLPredictor.h" +#include "NMLFeatureReader.h" \ No newline at end of file diff --git a/Plugins/macOS/NatML.bundle/Contents/Info.plist b/Plugins/macOS/NatML.bundle/Contents/Info.plist new file mode 100644 index 0000000..1c221c9 --- /dev/null +++ b/Plugins/macOS/NatML.bundle/Contents/Info.plist @@ -0,0 +1,46 @@ + + + + + BuildMachineOSBuild + 22F82 + CFBundleDevelopmentRegion + en + CFBundleExecutable + NatML + CFBundleIdentifier + ai.natml.natml + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + NatML + CFBundlePackageType + BNDL + CFBundleShortVersionString + 1.0 + CFBundleSupportedPlatforms + + MacOSX + + CFBundleVersion + 1 + DTCompiler + com.apple.compilers.llvm.clang.1_0 + DTPlatformBuild + + DTPlatformName + macosx + DTPlatformVersion + 13.3 + DTSDKBuild + 22E245 + DTSDKName + macosx13.3 + DTXcode + 1431 + DTXcodeBuild + 14E300c + LSMinimumSystemVersion + 10.15 + + diff --git a/Plugins/macOS/NatML.bundle/Contents/MacOS/NatML b/Plugins/macOS/NatML.bundle/Contents/MacOS/NatML new file mode 100755 index 0000000..46e6fd3 Binary files /dev/null and b/Plugins/macOS/NatML.bundle/Contents/MacOS/NatML differ diff --git a/README.md b/README.md new file mode 100644 index 0000000..070d19a --- /dev/null +++ b/README.md @@ -0,0 +1,89 @@ +# NatML + +![NatML](.media/wall.png) + +NatML allows developers to integrate machine learning into their Unity applications in under five lines of code with zero infrastructure. NatML completely removes the need to have any experience with machine learning in order to take advantage of the features it can provide. Features include: + +- **Universal Machine Learning**. With NatML, you can drop CoreML (`.mlmodel`), TensorFlow Lite (`.tflite`), and ONNX (`.onnx`) models directly into your Unity project and run them. + +- **Bare Metal Performance**. NatML takes advantage of hardware machine learning accelerators, like CoreML on iOS and macOS, NNAPI on Android, and DirectML on Windows. As a result, it is [multiple times faster](https://github.com/natmlx/ml-bench) than Unity's own Barracuda engine. + +- **Cross Platform**. NatML supports Android, iOS, macOS, WebGL, and Windows alike. As a result, you can build your app once, test it in the Editor, and deploy it various platforms and devices all in one seamless workflow. + +- **Extremely Easy to Use**. NatML exposes machine learning models with simple classes that return familiar data types. These are called "Predictors", and they handle all of the heavy lifting for you. No need to write pre-processing scripts or shaders, wrangle tensors, or anything of that sort. + +- **Growing Catalog**. NatML is designed with a singular focus on applications. As such, we maintain a growing catalog of predictors that developers can quickly discover and deploy in their applications. [Check out NatML Hub](https://hub.natml.ai). + +- **Lightweight Package**. NatML is distributed in a self-contained package, with no external dependencies. As a result, you can simply import the package and get going--no setup necessary. + +## Installing NatML +Add the following items to your Unity project's `Packages/manifest.json`: +```json +{ + "scopedRegistries": [ + { + "name": "NatML", + "url": "https://registry.npmjs.com", + "scopes": ["ai.natml"] + } + ], + "dependencies": { + "ai.natml.natml": "1.1.15" + } +} +``` + +## Using ML Models + +![drag and drop](.media/mobilenet.gif) + +If you have a CoreML, ONNX, or TensorFlow Lite model, you can simply drag and drop it into your project. [See the documentation for more details](https://docs.natml.ai/unity/workflows/models). + +> Note that specific model formats can only be used on specific platforms. CoreML models can only be used on iOS and macOS; ONNX can only be used on Windows; and TensorFlow Lite can only be used on Android. Use [NatML Hub](https://hub.natml.ai) to convert your model to different ML formats. + +## Discover ML Models on NatML Hub +**[Create an account on NatML Hub](https://hub.natml.ai/profile)** to find and download ML predictors to use in your project! + +![NatML Hub](.media/hub.png) + +You can also [upload your models to Hub](https://hub.natml.ai/create) and make them private or public. [Check out the documentation](https://docs.natml.ai/unity/api/mledgemodel) for information on writing predictors for your models. + +## Using ML Models in Two Simple Steps +You will always use NatML in two steps. First, create a **predictor** by fetching model data from [NatML Hub](https://hub.natml.ai) or by loading a local ML model file in your project (`.mlmodel`, `.tflite`, and `.onnx`): +```csharp +// Create the MobileNet v2 predictor +var predictor = await MobileNetv2Predictor.Create(); +``` + +Then make predictions with the predictor: +```csharp +// Make prediction on an image +Texture2D image = ...; +var (label, score) = predictor.Predict(image); +``` + +Different predictors accept and produce different data types, but the usage pattern will always be the same. + +___ + +## Requirements +- Unity 2022.3+ + +## Supported Platforms +- Android API Level 24+ +- iOS 14+ +- macOS 10.15+ (Apple Silicon and Intel) +- Windows 10+ (64-bit only) +- WebGL: + - Chrome 91+ + - Firefox 90+ + - Safari 16.4+ + +## Resources +- Join the [NatML community on Discord](https://natml.ai/community). +- See the [NatML documentation](https://docs.natml.ai/unity). +- Check out [NatML on GitHub](https://github.com/natmlx). +- Read the [NatML blog](https://blog.natml.ai/). +- Contact us at [hi@natml.ai](mailto:hi@natml.ai). + +Thank you very much! \ No newline at end of file diff --git a/README.md.meta b/README.md.meta new file mode 100644 index 0000000..e6820f4 --- /dev/null +++ b/README.md.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: b868032b23cee4a73ab74da9e0a49901 +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime.meta b/Runtime.meta new file mode 100644 index 0000000..4b8116e --- /dev/null +++ b/Runtime.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 9958c9fdda3e144fd8467d12c7f463ef +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API.meta b/Runtime/API.meta new file mode 100644 index 0000000..1969e12 --- /dev/null +++ b/Runtime/API.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 523bf3f4d6e45467999372d1e51b7961 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API/Graph.meta b/Runtime/API/Graph.meta new file mode 100644 index 0000000..930cab3 --- /dev/null +++ b/Runtime/API/Graph.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 5b48a06cfb07e4227ab5795027bc1f2d +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API/Graph/DotNetClient.cs b/Runtime/API/Graph/DotNetClient.cs new file mode 100644 index 0000000..a4bbef9 --- /dev/null +++ b/Runtime/API/Graph/DotNetClient.cs @@ -0,0 +1,101 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +#nullable enable + +namespace NatML.API.Graph { + + using System; + using System.Collections.Generic; + using System.IO; + using System.Net.Http; + using System.Net.Http.Headers; + using System.Text; + using System.Threading.Tasks; + using Newtonsoft.Json; + + /// + /// + public sealed class DotNetClient : IGraphClient { + + #region --Client API-- + /// + /// Create the .NET client. + /// + /// NatML Graph API URL. + /// NatML access key. + public DotNetClient (string url, string accessKey) { + this.url = url; + this.accessKey = accessKey; + } + + /// + /// Query the NatML Graph API. + /// + /// Graph query. + /// Query result key. + /// Query inputs. + public async Task Query (string query, string key, Dictionary? variables = default) { + // Serialize payload + var payload = new GraphRequest { + query = query, + variables = variables + }; + var serializationSettings = new JsonSerializerSettings { + NullValueHandling = NullValueHandling.Ignore + }; + var payloadStr = JsonConvert.SerializeObject(payload, serializationSettings); + // Create client + using var client = new HttpClient(); + using var content = new StringContent(payloadStr, Encoding.UTF8, @"application/json"); + // Add auth token + var authHeader = !string.IsNullOrEmpty(accessKey) ? new AuthenticationHeaderValue(@"Bearer", accessKey) : null; + client.DefaultRequestHeaders.Authorization = authHeader; + // Post + using var response = await client.PostAsync(url, content); + // Parse + var responseStr = await response.Content.ReadAsStringAsync(); + var responsePayload = JsonConvert.DeserializeObject>(responseStr); + // Check error + if (responsePayload.errors != null) + throw new InvalidOperationException(responsePayload.errors[0].message); + // Return + return responsePayload.data.TryGetValue(key, out var value) ? value : default; + } + + /// + /// Download a file. + /// + /// Data URL. + public async Task Download (string url) { + using var client = new HttpClient(); + using var dataStream = await client.GetStreamAsync(url); + using var memoryStream = new MemoryStream(); + await dataStream.CopyToAsync(memoryStream); + return memoryStream; + } + + /// + /// Upload a data stream. + /// + /// Data stream. + /// Upload URL. + /// MIME type. + public async Task Upload (MemoryStream stream, string url, string? mime = null) { + using var client = new HttpClient(); + using var content = new StreamContent(stream); + content.Headers.ContentType = new MediaTypeHeaderValue(mime ?? @"application/octet-stream"); + using var response = await client.PutAsync(url, content); + response.EnsureSuccessStatusCode(); + } + #endregion + + + #region --Operations-- + private readonly string url; + private readonly string accessKey; + #endregion + } +} \ No newline at end of file diff --git a/Runtime/API/Graph/DotNetClient.cs.meta b/Runtime/API/Graph/DotNetClient.cs.meta new file mode 100644 index 0000000..77770fb --- /dev/null +++ b/Runtime/API/Graph/DotNetClient.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: ced400b87f89d4f9da4330b532a3f58f +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API/Graph/IGraphClient.cs b/Runtime/API/Graph/IGraphClient.cs new file mode 100644 index 0000000..46ff991 --- /dev/null +++ b/Runtime/API/Graph/IGraphClient.cs @@ -0,0 +1,69 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +#nullable enable + +namespace NatML.API.Graph { + + using System; + using System.Collections.Generic; + using System.IO; + using System.Threading.Tasks; + + /// + /// NatML Graph API client. + /// + public interface IGraphClient { + + /// + /// Query the NatML Graph API. + /// + /// Graph query. + /// Query result key. + /// Query inputs. + Task Query (string query, string key, Dictionary? variables = default); + + /// + /// Download a file. + /// + /// URL + Task Download (string url); + + /// + /// Upload a data stream. + /// + /// Data stream. + /// Upload URL. + /// MIME type. + Task Upload (MemoryStream stream, string url, string? mime = null); + } + + /// + /// NatML graph API request. + /// + [Preserve] + public sealed class GraphRequest { + + public string query = string.Empty; + public Dictionary? variables; + } + + /// + /// NatML graph API response. + /// + [Preserve] + public sealed class GraphResponse { + + public Dictionary? data; + public Error[]? errors; + + public sealed class Error { + public string message; + } + } + + [AttributeUsage(AttributeTargets.All, Inherited = true, AllowMultiple = false)] + public sealed class PreserveAttribute : Attribute { } +} \ No newline at end of file diff --git a/Runtime/API/Graph/IGraphClient.cs.meta b/Runtime/API/Graph/IGraphClient.cs.meta new file mode 100644 index 0000000..d7a47a8 --- /dev/null +++ b/Runtime/API/Graph/IGraphClient.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: ea148d2242d284914b23dc1ef4312d0c +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API/NatMLClient.cs b/Runtime/API/NatMLClient.cs new file mode 100644 index 0000000..a6d1eda --- /dev/null +++ b/Runtime/API/NatMLClient.cs @@ -0,0 +1,78 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.API { + + using System; + using Graph; + using Services; + + /// + /// NatML API client. + /// + public sealed class NatMLClient { + + #region --Client API-- + /// + /// NatML graph API URL. + /// + public const string URL = @"https://api.natml.ai/graph"; + + /// + /// Manage users. + /// + public readonly UserService Users; + + /// + /// Manage predictors. + /// + public readonly PredictorService Predictors; + + /// + /// Manage predictor graphs. + /// + public readonly GraphService Graphs; + + /// + /// Upload and download files. + /// + public readonly StorageService Storage; + + /// + /// Create the NatML client. + /// + /// NatML access key. + /// NatML Graph API URL. + public NatMLClient ( + string accessKey = null, + string url = null + ) : this(new DotNetClient(url ?? URL, accessKey)) { } + + /// + /// Create the NatML client. + /// + /// NatML Graph API client. + public NatMLClient (IGraphClient client) { + this.client = client; + this.Users = new UserService(client); + this.Predictors = new PredictorService(client); + this.Graphs = new GraphService(client); + this.PredictorSessions = new PredictorSessionService(client); + this.Storage = new StorageService(client); + } + #endregion + + + #region --Operations-- + private readonly IGraphClient client; + #endregion + + + #region --DEPRECATED-- + [Obsolete(@"Deprecated in NatML 1.1.6.")] + public readonly PredictorSessionService PredictorSessions; + #endregion + } +} \ No newline at end of file diff --git a/Runtime/API/NatMLClient.cs.meta b/Runtime/API/NatMLClient.cs.meta new file mode 100644 index 0000000..1984dca --- /dev/null +++ b/Runtime/API/NatMLClient.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: bbe641e4385cc4a3096cb74a5aa6c080 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API/Services.meta b/Runtime/API/Services.meta new file mode 100644 index 0000000..12b8c11 --- /dev/null +++ b/Runtime/API/Services.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 57826e1c92505445bab273860fc92c9d +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API/Services/Graph.cs b/Runtime/API/Services/Graph.cs new file mode 100644 index 0000000..0ccc3cf --- /dev/null +++ b/Runtime/API/Services/Graph.cs @@ -0,0 +1,142 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +#nullable enable + +namespace NatML.API.Services { + + using System.Linq; + using System.Threading.Tasks; + using Newtonsoft.Json; + using Graph; + using Types; + + /// + /// Manage predictor graphs. + /// + public sealed class GraphService { + + #region --Client API-- + /// + /// Retrieve a predictor graph. + /// + /// Graph tag. If the tag does not contain a variant then the variant defaults to `main`. + /// Predictor graph. + public async Task Retrieve (string tag, GraphFormat format) { + // Ensure this is a predictor tag + Tag.TryParse(tag, out var parsedTag); + var variant = parsedTag.variant ?? @"main"; + var predictorTag = new Tag(parsedTag.username, parsedTag.name); + // There isn't a query to get a specific graph so just filter for now + var graphs = await List(predictorTag); + var graph = graphs?.FirstOrDefault(graph => graph.variant == variant && graph.format == format); + // Return + return graph; + } + + /// + /// List all predictor graphs. + /// + /// Predictor tag. This MUST NOT be a variant tag. + /// Predictor graphs. + public async Task List (string tag) { + var predictor = await client.Query( + @$"query ($input: PredictorInput!) {{ + predictor (input: $input) {{ + graphs {{ + {Fields} + }} + }} + }}", + @"predictor", + new () { + ["input"] = new PredictorInput { + tag = tag + } + } + ); + return predictor?.graphs; + } + + /// + /// Create a predictor graph. + /// + /// Graph tag. If the tag does not contain a variant then the variant defaults to `main`. + /// Graph URL. + /// Graph format. + public Task Create ( + string tag, + string graph, + GraphFormat format + ) => client.Query( + @$"mutation ($input: CreateGraphInput!) {{ + createGraph (input: $input) {{ + {Fields} + }} + }}", + @"createGraph", + new () { + ["input"] = new CreateGraphInput { + tag = tag, + graph = graph, + format = format + } + } + ); + + /// + /// Delete a predictor graph. + /// + /// Graph tag. If the tag does not contain a variant then the variant defaults to `main`. + /// Graph format. + /// Whether the graph was successfully deleted. + public Task Delete ( + string tag, + GraphFormat format + ) => client.Query( + @$"mutation ($input: DeleteGraphInput!) {{ + deleteGraph (input: $input) + }}", + @"deleteGraph", + new () { + ["input"] = new DeleteGraphInput { + tag = tag, + format = format + } + } + ); + #endregion + + + #region --Operations-- + private readonly IGraphClient client; + private const string Fields = @" + variant + format + status + encrypted + created + error + "; + + internal GraphService (IGraphClient client) => this.client = client; + #endregion + } + + #region --Types-- + + internal sealed class CreateGraphInput { + public string tag; + public string graph; + public GraphFormat format; + } + + internal sealed class DeleteGraphInput { + public string tag; + public string graph; + public GraphFormat format; + } + #endregion +} \ No newline at end of file diff --git a/Runtime/API/Services/Graph.cs.meta b/Runtime/API/Services/Graph.cs.meta new file mode 100644 index 0000000..2ec0ad0 --- /dev/null +++ b/Runtime/API/Services/Graph.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 737eb059893a24801b8bef4f7eecee23 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API/Services/Predictor.cs b/Runtime/API/Services/Predictor.cs new file mode 100644 index 0000000..7a03890 --- /dev/null +++ b/Runtime/API/Services/Predictor.cs @@ -0,0 +1,287 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +#nullable enable + +namespace NatML.API.Services { + + using System.Threading.Tasks; + using Newtonsoft.Json; + using Graph; + using Types; + + /// + /// Manage predictors. + /// + public sealed class PredictorService { + + #region --Client API-- + /// + /// Retrieve a predictor. + /// + /// Predictor tag. This MUST NOT be a variant tag. + public Task Retrieve (string tag) => client.Query( + @$"query ($input: PredictorInput!) {{ + predictor (input: $input) {{ + {Fields} + }} + }}", + @"predictor", + new () { + ["input"] = new PredictorInput { + tag = tag + } + } + ); + + /// + /// View available predictors. + /// + /// Fetch only predictors owned by me. + /// Predictor status. This only applies when `mine` is `true`. + /// Pagination offset. + /// Pagination count. + public async Task List ( + bool? mine = null, + PredictorStatus? status = null, + int? offset = null, + int? count = null + ) => await client.Query( + @$"query ($input: PredictorsInput!) {{ + predictors (input: $input) {{ + {Fields} + }} + }}", + @"predictors", + new () { + ["input"] = new PredictorsInput { + mine = mine, + status = status, + offset = offset, + count = count + } + } + ); + + /// + /// Search predictors. + /// + /// Search query. + /// Pagination offset. + /// Pagination count. + public Task Search ( + string query, + int? offset = null, + int? count = null + ) => client.Query( + @$"query ($input: PredictorsInput!) {{ + predictors (input: $input) {{ + {Fields} + }} + }}", + @"predictors", + new () { + ["input"] = new PredictorsInput { + query = query, + offset = offset, + count = count + } + } + ); + + /// + /// Create a predictor. + /// + /// Predictor tag. + /// Predictor description. This supports Markdown. + /// Predictor access mode. + public Task Create ( + string tag, + string? description = null, + AccessMode? access = null + ) => client.Query( + @$"mutation ($input: CreatePredictorInput!) {{ + createPredictor (input: $input) {{ + {Fields} + }} + }}", + @"createPredictor", + new () { + ["input"] = new CreatePredictorInput { + tag = tag, + description = description, + access = access + } + } + ); + + /// + /// Update a predictor. + /// + /// Predictor tag. + /// Predictor description. This supports Markdown. + /// Predictor access mode. + /// Predictor license. + /// Predictor topics. + /// Predictor media URL or path. + /// Classification labels. + /// Feature normalization. + /// Image aspect mode. + /// Audio format. + public Task Update ( + string tag, + string? description = null, + AccessMode? access = null, + string? license = null, + string[]? topics = null, + string? media = null, + string[]? labels = null, + Normalization? normalization = null, + AspectMode? aspectMode = null, + AudioFormat? audioFormat = null + ) => client.Query( + @$"mutation ($input: UpdatePredictorInput!) {{ + updatePredictor (input: $input) {{ + {Fields} + }} + }}", + @"updatePredictor", + new () { + ["input"] = new UpdatePredictorInput { + tag = tag, + description = description, + access = access, + license = license, + topics = topics, + media = media, + labels = labels, + normalization = normalization, + aspectMode = aspectMode, + audioFormat = audioFormat + } + } + ); + + /// + /// Delete a draft predictor. + /// + /// Predictor tag. + public Task Delete (string tag) => client.Query( + @$"mutation ($input: DeletePredictorInput!) {{ + deletePredictor (input: $input) + }}", + @"deletePredictor", + new () { + ["input"] = new DeletePredictorInput { + tag = tag + } + } + ); + + /// + /// Publish a draft predictor. + /// + /// Predictor tag. + public Task Publish (string tag) => client.Query( + @$"mutation ($input: PublishPredictorInput!) {{ + publishPredictor (input: $input) {{ + {Fields} + }} + }}", + @"publishPredictor", + new () { + ["input"] = new PublishPredictorInput { + tag = tag + } + } + ); + + /// + /// Archive a published predictor. + /// + /// Predictor tag. + public Task Archive (string tag) => client.Query( + @$"mutation ($input: ArchivePredictorInput!) {{ + archivePredictor (input: $input) {{ + {Fields} + }} + }}", + @"archivePredictor", + new () { + ["input"] = new ArchivePredictorInput { + tag = tag + } + } + ); + #endregion + + + #region --Operations-- + private readonly IGraphClient client; + private const string Fields = @" + tag + owner { + username + created + name + avatar + bio + website + github + } + name + description + status + access + license + topics + created + media + "; + + internal PredictorService (IGraphClient client) => this.client = client; + #endregion + } + + #region --Types-- + + internal class PredictorInput { + public string tag; + } + + internal sealed class PredictorsInput { + public bool? mine; + public PredictorStatus? status; + public string? query; + public int? offset; + public int? count; + } + + internal sealed class CreatePredictorInput { + public string tag; + public string? description; + public AccessMode? access; + } + + internal sealed class UpdatePredictorInput { + public string tag; + public string? description; + public AccessMode? access; + public string? license; + public string[]? topics; + public string? media; + public string[]? labels; + public Normalization? normalization; + public AspectMode? aspectMode; + public AudioFormat? audioFormat; + } + + internal sealed class DeletePredictorInput : PredictorInput { } + + internal sealed class PublishPredictorInput : PredictorInput { } + + internal sealed class ArchivePredictorInput : PredictorInput { } + #endregion +} \ No newline at end of file diff --git a/Runtime/API/Services/Predictor.cs.meta b/Runtime/API/Services/Predictor.cs.meta new file mode 100644 index 0000000..9ab0e4e --- /dev/null +++ b/Runtime/API/Services/Predictor.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: fdb5c0a8b1a91470580dce88e7bb241d +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API/Services/PredictorSession.cs b/Runtime/API/Services/PredictorSession.cs new file mode 100644 index 0000000..6b35458 --- /dev/null +++ b/Runtime/API/Services/PredictorSession.cs @@ -0,0 +1,98 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +#nullable enable + +namespace NatML.API.Services { + + using System; + using System.Threading.Tasks; + using Newtonsoft.Json; + using Graph; + using Types; + + /// + /// Create graph prediction sessions. + /// + [Obsolete(@"Deprecated in NatML 1.1.6")] + public sealed class PredictorSessionService { + + #region --Client API-- + /// + /// Create a graph prediction session. + /// + /// Graph tag. + /// Graph format. + public Task Create ( + string tag, + GraphFormat format, + string? secret = null, + string? device = null + ) => client.Query( + @$"mutation ($input: CreatePredictorSessionInput!) {{ + createPredictorSession (input: $input) {{ + id + predictor {{ + tag + owner {{ + username + }} + name + description + status + access + license + labels + normalization {{ + mean + std + }} + aspectMode + audioFormat {{ + sampleRate + channelCount + }} + }} + graph + format + flags + fingerprint + created + secret + }} + }} + ", + "createPredictorSession", + new () { + ["input"] = new CreatePredictorSessionInput { + tag = tag, + format = format, + secret = secret, + client = @"dotnet", + device = device + } + } + ); + #endregion + + + #region --Operations-- + private readonly IGraphClient client; + + internal PredictorSessionService (IGraphClient client) => this.client = client; + #endregion + } + + #region --Types-- + + internal sealed class CreatePredictorSessionInput { + public string tag; + public GraphFormat format; + public string? secret; + public string? client; + public string? device; + } + #endregion +} \ No newline at end of file diff --git a/Runtime/API/Services/PredictorSession.cs.meta b/Runtime/API/Services/PredictorSession.cs.meta new file mode 100644 index 0000000..425873a --- /dev/null +++ b/Runtime/API/Services/PredictorSession.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 9bfc526504ca94ec79087981a18b853f +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API/Services/Storage.cs b/Runtime/API/Services/Storage.cs new file mode 100644 index 0000000..35572b6 --- /dev/null +++ b/Runtime/API/Services/Storage.cs @@ -0,0 +1,109 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +#nullable enable + +namespace NatML.API.Services { + + using System; + using System.IO; + using System.Threading.Tasks; + using Newtonsoft.Json; + using Graph; + using Types; + + /// + /// Upload and download files. + /// + public sealed class StorageService { + + #region --Client API-- + /// + /// Download a file. + /// + /// Data URL. + public async Task Download (string url) { + // Handle data URL + if (url.StartsWith(@"data:")) { + var dataIdx = url.LastIndexOf(",") + 1; + var b64Data = url.Substring(dataIdx); + var data = Convert.FromBase64String(b64Data); + return new MemoryStream(data, 0, data.Length, false, false); + } + // Remote URL + return await client.Download(url); + } + + /// + /// Upload a data stream. + /// + /// File name. + /// Data stream. + /// Upload type. + /// Return a data URL if the provided stream is smaller than this limit (in bytes). + public async Task Upload ( + string name, + MemoryStream stream, + UploadType type, + string? mime = null, + int dataUrlLimit = 0 + ) { + mime ??= @"application/octet-stream"; + // Data URL + if (stream.Length < dataUrlLimit) { + var data = Convert.ToBase64String(stream.ToArray()); + var result = $"data:{mime};base64,{data}"; + return result; + } + // Upload + var url = await CreateUploadURL(name, type); + await client.Upload(stream, url, mime); + // Return + return url; + } + + /// + /// Create an upload URL. + /// + /// File name. + /// Upload type. + /// File key. This is useful for grouping related files. + public async Task CreateUploadURL ( + string name, + UploadType type, + string? key = null + ) => await client.Query( + @$"mutation ($input: CreateUploadURLInput!) {{ + createUploadURL (input: $input) + }}", + @"createUploadURL", + new () { + ["input"] = new CreateUploadURLInput { + name = name, + type = type, + key = key + } + } + ); + #endregion + + + #region --Operations-- + private readonly IGraphClient client; + + internal StorageService (IGraphClient client) => this.client = client; + #endregion + } + + + #region --Types-- + + internal sealed class CreateUploadURLInput { + public string name; + public UploadType type; + public string? key; + } + #endregion +} \ No newline at end of file diff --git a/Runtime/API/Services/Storage.cs.meta b/Runtime/API/Services/Storage.cs.meta new file mode 100644 index 0000000..057408e --- /dev/null +++ b/Runtime/API/Services/Storage.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 578bcdb5e81064f4597ecba4c685b3d4 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API/Services/User.cs b/Runtime/API/Services/User.cs new file mode 100644 index 0000000..6b09e38 --- /dev/null +++ b/Runtime/API/Services/User.cs @@ -0,0 +1,64 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +#nullable enable + +namespace NatML.API.Services { + + using System.Threading.Tasks; + using Graph; + using Types; + + /// + /// Manage users. + /// + public sealed class UserService { + + #region --Client API-- + /// + /// Retrieve a user. + /// + /// Username. If `null` then this will retrieve the currently authenticated user. + public async Task Retrieve (string? username = null) { + var profile = !string.IsNullOrEmpty(username); + var user = await client.Query( + @$"query {(profile ? "($input: UserInput)" : string.Empty)} {{ + user {(profile ? "(input: $input)" : "")} {{ + username + {(profile ? string.Empty : "email")} + created + name + avatar + bio + website + github + }} + }}", + @"user", + new () { + ["input"] = new UserInput { + username = username + } + } + ); + return user; + } + #endregion + + + #region --Operations-- + private readonly IGraphClient client; + + internal UserService (IGraphClient client) => this.client = client; + #endregion + } + + #region --Types-- + + internal sealed class UserInput { + public string username; + } + #endregion +} \ No newline at end of file diff --git a/Runtime/API/Services/User.cs.meta b/Runtime/API/Services/User.cs.meta new file mode 100644 index 0000000..adab9a1 --- /dev/null +++ b/Runtime/API/Services/User.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 40665cf80c8374ed283b2b1a5bd9bca2 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API/Types.meta b/Runtime/API/Types.meta new file mode 100644 index 0000000..2c6f2a3 --- /dev/null +++ b/Runtime/API/Types.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 17737e0a1ceb04fecbc44c32d5222f4f +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API/Types/AudioFormat.cs b/Runtime/API/Types/AudioFormat.cs new file mode 100644 index 0000000..eb17945 --- /dev/null +++ b/Runtime/API/Types/AudioFormat.cs @@ -0,0 +1,27 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.API.Types { + + using System; + using API.Graph; + + /// + /// Audio format. + /// + [Preserve, Serializable] + public sealed class AudioFormat { + + /// + /// Sample rate. + /// + public int sampleRate; + + /// + /// Channel count. + /// + public int channelCount; + } +} \ No newline at end of file diff --git a/Runtime/API/Types/AudioFormat.cs.meta b/Runtime/API/Types/AudioFormat.cs.meta new file mode 100644 index 0000000..35981c5 --- /dev/null +++ b/Runtime/API/Types/AudioFormat.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: ae1733e037ad94abfba46d012a159bc8 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API/Types/Dtype.cs b/Runtime/API/Types/Dtype.cs new file mode 100644 index 0000000..9b968b5 --- /dev/null +++ b/Runtime/API/Types/Dtype.cs @@ -0,0 +1,120 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.API.Types { + + using System; + using System.Collections; + using System.Runtime.Serialization; + using Newtonsoft.Json; + using Newtonsoft.Json.Converters; + + /// + /// Feature data type. + /// This follows `numpy` dtypes. + /// + [JsonConverter(typeof(StringEnumConverter))] + public enum Dtype : int { // CHECK // Must match `Function.h` + /// + /// Unknown or unsupported data type. + /// + Undefined = 0, + /// + /// Type is a `int8_t` in C/C++ and `sbyte` in C#. + /// + [EnumMember(Value = @"int8")] + Int8 = 10, + /// + /// Type is `int16_t` in C/C++ and `short` in C#. + /// + [EnumMember(Value = @"int16")] + Int16 = 2, + /// + /// Type is `int32_t` in C/C++ and `int` in C#. + /// + [EnumMember(Value = @"int32")] + Int32 = 3, + /// + /// Type is `int64_t` in C/C++ and `long` in C#. + /// + [EnumMember(Value = @"int64")] + Int64 = 4, + /// + /// Type is `uint8_t` in C/C++ and `byte` in C#. + /// + [EnumMember(Value = @"uint8")] + Uint8 = 1, + /// + /// Type is a `uint16_t` in C/C++ and `ushort` in C#. + /// + [EnumMember(Value = @"uint16")] + Uint16 = 11, + /// + /// Type is a `uint32_t` in C/C++ and `uint` in C#. + /// + [EnumMember(Value = @"uint32")] + Uint32 = 12, + /// + /// Type is a `uint64_t` in C/C++ and `ulong` in C#. + /// + [EnumMember(Value = @"uint64")] + Uint64 = 13, + /// + /// Type is a generic half-precision float. + /// + [EnumMember(Value = @"float16")] + Float16 = 14, + /// + /// Type is `float` in C/C++/C#. + /// + [EnumMember(Value = @"float32")] + Float32 = 5, + /// + /// Type is `double` in C/C++/C#. + /// + [EnumMember(Value = @"float64")] + Float64 = 6, + /// + /// Type is a `bool` in C/C++/C#. + /// + [EnumMember(Value = @"bool")] + Bool = 15, + /// + /// Type is `std::string` in C++ and `string` in C#. + /// + [EnumMember(Value = @"string")] + String = 7, + /// + /// Type is an encoded image. + /// + [EnumMember(Value = @"image")] + Image = 16, + /// + /// Type is an encoded audio. + /// + [EnumMember(Value = @"audio")] + Audio = 18, + /// + /// Type is an encoded video. + /// + [EnumMember(Value = @"video")] + Video = 19, + /// + /// Type is a binary blob. + /// + [EnumMember(Value = @"binary")] + Binary = 17, + /// + /// Type is a generic list. + /// + [EnumMember(Value = @"list")] + List = 8, + /// + /// Type is a generic dictionary. + /// + [EnumMember(Value = @"dict")] + Dict = 9, + } +} \ No newline at end of file diff --git a/Runtime/API/Types/Dtype.cs.meta b/Runtime/API/Types/Dtype.cs.meta new file mode 100644 index 0000000..badb32c --- /dev/null +++ b/Runtime/API/Types/Dtype.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: f852fa61d1ca546dc8ac01b925bece0b +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API/Types/Graph.cs b/Runtime/API/Types/Graph.cs new file mode 100644 index 0000000..510bf55 --- /dev/null +++ b/Runtime/API/Types/Graph.cs @@ -0,0 +1,98 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +#nullable enable +#pragma warning disable 8618 + +namespace NatML.API.Types { + + using System; + using System.Runtime.Serialization; + using Newtonsoft.Json; + using Newtonsoft.Json.Converters; + using API.Graph; + + /// + /// Predictor graph. + /// + [Preserve, Serializable] + public sealed class Graph { + + /// + /// Graph variant. + /// + public string variant; + + /// + /// Graph format. + /// + public GraphFormat format; + + /// + /// Graph status. + /// + public GraphStatus status; + + /// + /// Whether the graph is encrypted. + /// + public bool encrypted; + + /// + /// Date created. + /// + public string created; + + /// + /// Graph provisioning error. + /// This is populated when the graph is invalid. + /// + public string? error; + } + + /// + /// Graph format. + /// + [JsonConverter(typeof(StringEnumConverter))] + public enum GraphFormat { + /// + /// Apple CoreML. + /// + [EnumMember(Value = @"COREML")] + CoreML = 0, + /// + /// Open Neural Network Exchange. + /// + [EnumMember(Value = @"ONNX")] + ONNX = 1, + /// + /// TensorFlow Lite. + /// + [EnumMember(Value = @"TFLITE")] + TFLite = 2, + } + + /// + /// Graph status. + /// + [JsonConverter(typeof(StringEnumConverter))] + public enum GraphStatus { + /// + /// Graph is being provisioned. + /// + [EnumMember(Value = @"PROVISIONING")] + Provisioning = 0, + /// + /// Graph is active. + /// + [EnumMember(Value = @"ACTIVE")] + Active = 1, + /// + /// Graph is invalid. + /// + [EnumMember(Value = @"INVALID")] + Invalid = 2, + } +} \ No newline at end of file diff --git a/Runtime/API/Types/Graph.cs.meta b/Runtime/API/Types/Graph.cs.meta new file mode 100644 index 0000000..ba4aff6 --- /dev/null +++ b/Runtime/API/Types/Graph.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 6dc1866a4b8e841c5b82f3e9faf4bd3c +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API/Types/License.cs b/Runtime/API/Types/License.cs new file mode 100644 index 0000000..02ff7ae --- /dev/null +++ b/Runtime/API/Types/License.cs @@ -0,0 +1,49 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.API.Types { + + /// + /// Predictor license. + /// + public static class License { + public const string OpenRAIL = @"openrail"; + public const string Apache20 = @"apache-2.0"; + public const string BSD = @"bsd"; + public const string BSD2Clause = @"bsd-2-clause"; + public const string BSD3Clause = @"bsd-3-clause"; + public const string AGPLv3 = @"agpl-3.0"; + public const string GPL = @"gpl"; + public const string GPLv2 = @"gpl-2.0"; + public const string GPLv3 = @"gpl-3.0"; + public const string LGPL = @"lgpl"; + public const string LGPLv21 = @"lgpl-2.1"; + public const string LGPLv3 = @"lgpl-3.0"; + public const string ISC = @"isc"; + public const string MIT = @"mit"; + public const string OSLv3 = @"osl-3.0"; + public const string Unlicense = @"unlicense"; + public const string Unknown = @"unknown"; + public const string Other = @"other"; + public const string WTFPL = @"wtfpl"; + public const string CreativeCommons = @"cc"; + public const string CreativeCommons0v10 = @"cc0-1.0"; + public const string CreativeCommonsv20 = @"cc-by-2.0"; + public const string CreativeCommonsv25 = @"cc-by-2.5"; + public const string CreativeCommonsv30 = @"cc-by-3.0"; + public const string CreativeCommonsv40 = @"cc-by-4.0"; + public const string CreativeCommonsShareAlikev30 = @"cc-by-sa-3.0"; + public const string CreativeCommonsShareAlike40 = @"cc-by-sa-4.0"; + public const string CreativeCommonsNonCommercialv20 = @"cc-by-nc-2.0"; + public const string CreativeCommonsNonCommercialv30 = @"cc-by-nc-3.0"; + public const string CreativeCommonsNonCommercialv40 = @"cc-by-nc-4.0"; + public const string CreativeCommonsNoDerivativesv40 = @"cc-by-nd-4.0"; + public const string CreativeCommonsNonCommercialNoDerivativesv30 = @"cc-by-nc-nd-3.0"; + public const string CreativeCommonsNonCommercialNoDerivativesv40 = @"cc-by-nc-nd-4.0"; + public const string CreativeCommonsNonCommercialShareAlikev20 = @"cc-by-nc-sa-2.0"; + public const string CreativeCommonsNonCommercialShareAlikev30 = @"cc-by-nc-sa-3.0"; + public const string CreativeCommonsNonCommercialShareAlikev40 = @"cc-by-nc-sa-4.0"; + } +} \ No newline at end of file diff --git a/Runtime/API/Types/License.cs.meta b/Runtime/API/Types/License.cs.meta new file mode 100644 index 0000000..ffe47ff --- /dev/null +++ b/Runtime/API/Types/License.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 6d9437ea271d2443b8a9f78352e3606f +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API/Types/Normalization.cs b/Runtime/API/Types/Normalization.cs new file mode 100644 index 0000000..37a5cd1 --- /dev/null +++ b/Runtime/API/Types/Normalization.cs @@ -0,0 +1,27 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.API.Types { + + using System; + using API.Graph; + + /// + /// Feature normalization. + /// + [Preserve, Serializable] + public sealed class Normalization { + + /// + /// Mean. + /// + public float[] mean; + + /// + /// Standard deviation. + /// + public float[] std; + } +} \ No newline at end of file diff --git a/Runtime/API/Types/Normalization.cs.meta b/Runtime/API/Types/Normalization.cs.meta new file mode 100644 index 0000000..7e7703e --- /dev/null +++ b/Runtime/API/Types/Normalization.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 170afefd5b33c4b44a4a375edc0659ae +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API/Types/Predictor.cs b/Runtime/API/Types/Predictor.cs new file mode 100644 index 0000000..604f0d8 --- /dev/null +++ b/Runtime/API/Types/Predictor.cs @@ -0,0 +1,165 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +#nullable enable +#pragma warning disable 8618 + +namespace NatML.API.Types { + + using System; + using System.Runtime.Serialization; + using Newtonsoft.Json; + using Newtonsoft.Json.Converters; + using API.Graph; + + /// + /// ML predictor. + /// + [Preserve, Serializable] + public sealed class Predictor { + + /// + /// Predictor tag. + /// + public string tag; + + /// + /// Predictor owner. + /// + public Profile owner; + + /// + /// Predictor name. + /// + public string name; + + /// + /// Predictor description. + /// This supports Markdown. + /// + public string description; + + /// + /// Predictor status. + /// + public PredictorStatus status; + + /// + /// Predictor access. + /// + public AccessMode access; + + /// + /// Predictor graphs. + /// + [JsonProperty] + internal Graph[] graphs; + + /// + /// Predictor license. + /// + public string license; + + /// + /// Predictor topics. + /// + public string[] topics; + + /// + /// Date created. + /// + public string created; + + /// + /// Predictor media URL. + /// + public string? media; + + /// + /// Classification labels. + /// + public string[]? labels; + + /// + /// Feature normalization. + /// + public Normalization? normalization; + + /// + /// Image aspect mode. + /// + public AspectMode? aspectMode; + + /// + /// Audio format. + /// + public AudioFormat? audioFormat; + } + + /// + /// Predictor status. + /// + [JsonConverter(typeof(StringEnumConverter))] + public enum PredictorStatus : int { + /// + /// Predictor is a draft. + /// Predictor can only be viewed and used by owner. + /// + [EnumMember(Value = @"DRAFT")] + Draft = 0, + /// + /// Predictor has been published. + /// Predictor viewing and fetching permissions are dictated by the access mode. + /// + [EnumMember(Value = @"PUBLISHED")] + Published = 1, + /// + /// Predictor is archived. + /// Predictor can be viewed but cannot be used by anyone including owner. + /// + [EnumMember(Value = @"ARCHIVED")] + Archived = 2, + } + + /// + /// Access mode. + /// + [JsonConverter(typeof(StringEnumConverter))] + public enum AccessMode : int { + /// + /// Resource can only be accessed by the owner. + /// + [EnumMember(Value = @"PRIVATE")] + Private = 0, + /// + /// Resource can be accessed by anyone with NatML authentication. + /// + [EnumMember(Value = @"PUBLIC")] + Public = 1 + } + + /// + /// Image aspect mode. + /// + [JsonConverter(typeof(StringEnumConverter))] + public enum AspectMode : int { // CHECK // Must match `NatML.h` + /// + /// Scale to fit. + /// This scale mode DOES NOT preserve the aspect ratio of the image. + /// + [EnumMember(Value = @"SCALE_TO_FIT")] + ScaleToFit = 0, + /// + /// Aspect fill. + /// + [EnumMember(Value = @"ASPECT_FILL")] + AspectFill = 1, + /// + /// Aspect fit. + /// + [EnumMember(Value = @"ASPECT_FIT")] + AspectFit = 2, + } +} \ No newline at end of file diff --git a/Runtime/API/Types/Predictor.cs.meta b/Runtime/API/Types/Predictor.cs.meta new file mode 100644 index 0000000..b621be8 --- /dev/null +++ b/Runtime/API/Types/Predictor.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: b832cafa199e84a43a32f104e0aa00a0 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API/Types/PredictorSession.cs b/Runtime/API/Types/PredictorSession.cs new file mode 100644 index 0000000..8104836 --- /dev/null +++ b/Runtime/API/Types/PredictorSession.cs @@ -0,0 +1,63 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +#nullable enable +#pragma warning disable 8618 + +namespace NatML.API.Types { + + using System; + using Newtonsoft.Json; + using API.Graph; + + /// + /// Predictor graph session. + /// + [Preserve, Serializable, Obsolete(@"Deprecated in NatML 1.1.6")] + public sealed class PredictorSession { + + /// + /// Session ID. + /// + public string id; + + /// + /// Predictor for which this session was created. + /// + public Predictor predictor; + + /// + /// Session graph URL. + /// This URL is only valid for 10 minutes. + /// + public string graph; + + /// + /// Session graph format. + /// + public GraphFormat format; + + /// + /// Session flags. + /// + public int flags; + + /// + /// Session graph fingerprint. + /// This token uniquely identifies a graph across sessions. + /// + public string fingerprint; + + /// + /// Date created. + /// + public string created; + + /// + /// Session secret. + /// + public string? secret; + } +} \ No newline at end of file diff --git a/Runtime/API/Types/PredictorSession.cs.meta b/Runtime/API/Types/PredictorSession.cs.meta new file mode 100644 index 0000000..58a1eb5 --- /dev/null +++ b/Runtime/API/Types/PredictorSession.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 00be4f134952a460f965a5c19df4433b +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API/Types/Profile.cs b/Runtime/API/Types/Profile.cs new file mode 100644 index 0000000..1dd5a92 --- /dev/null +++ b/Runtime/API/Types/Profile.cs @@ -0,0 +1,60 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +#nullable enable +#pragma warning disable 8618 + +namespace NatML.API.Types { + + using System; + using API.Graph; + + /// + /// NatML user profile. + /// + [Preserve, Serializable] + public class Profile { + + /// + /// Username. + /// + public string username; + + /// + /// User email address. + /// + public string? email; // this is private to user hence nullable + + /// + /// Date created. + /// + public string created; + + /// + /// User display name. + /// + public string? name; + + /// + /// User avatar. + /// + public string? avatar; + + /// + /// User bio. + /// + public string? bio; + + /// + /// User website. + /// + public string? website; + + /// + /// User GitHub handle. + /// + public string? github; + } +} \ No newline at end of file diff --git a/Runtime/API/Types/Profile.cs.meta b/Runtime/API/Types/Profile.cs.meta new file mode 100644 index 0000000..ba5fe29 --- /dev/null +++ b/Runtime/API/Types/Profile.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 637797326d1114ec4869286113d77cf9 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API/Types/Tag.cs b/Runtime/API/Types/Tag.cs new file mode 100644 index 0000000..6d276dc --- /dev/null +++ b/Runtime/API/Types/Tag.cs @@ -0,0 +1,88 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +#nullable enable +#pragma warning disable 8618 + +namespace NatML.API.Types { + + using System; + using System.Diagnostics.CodeAnalysis; + using API.Graph; + + /// + /// NatML resource tag. + /// + [Preserve, Serializable] + public sealed class Tag { + + #region --Client API-- + /// + /// Resource owner username. + /// + public string username; + + /// + /// Resource name. + /// + public string name; + + /// + /// Resource variant. + /// + public string? variant; + + /// + /// Create a tag. + /// + /// Owner username. + /// Resource name. + /// Resource variabt. + public Tag (string username, string name, string? variant = null) { + this.username = username; + this.name = name; + this.variant = variant; + } + + /// + /// Serialize the tag. + /// + public override string ToString () { + var suffix = string.IsNullOrEmpty(variant) ? string.Empty : $"@{variant}"; + var result = $"@{username}/{name}{suffix}"; + return result; + } + + /// + /// Try to parse a predictor tag from a string. + /// + /// Input string. + /// Output tag. + /// Whether the tag was successfully parsed. + public static bool TryParse (string input, [NotNullWhen(true)] out Tag? tag) { + tag = null; + // Check username prefix + input = input.ToLowerInvariant(); + if (!input.StartsWith("@")) + return false; + // Check stem + var stem = input.Split('/'); + var extendedName = stem[1].Split('@'); + if (stem.Length != 2) + return false; + // Parse + var username = stem[0].Substring(1); + var name = extendedName[0]; + var variant = extendedName.Length > 1 ? extendedName[1] : null; + tag = new Tag(username, name, variant); + // Return + return true; + } + + public static implicit operator string (Tag tag) => tag.ToString(); + #endregion + + } +} \ No newline at end of file diff --git a/Runtime/API/Types/Tag.cs.meta b/Runtime/API/Types/Tag.cs.meta new file mode 100644 index 0000000..f097e4b --- /dev/null +++ b/Runtime/API/Types/Tag.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 0b25e529f739349e992e7f1dc641ca35 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API/Types/TypeExtensions.cs b/Runtime/API/Types/TypeExtensions.cs new file mode 100644 index 0000000..c948068 --- /dev/null +++ b/Runtime/API/Types/TypeExtensions.cs @@ -0,0 +1,55 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.API.Types { + + using System; + using System.Collections; + + public static class TypeExtensions { + + /// + /// Convert a NatML data type to a managed type. + /// + /// NatML data type. + /// Managed data type. + public static Type ToType (this Dtype dtype) => dtype switch { + Dtype.Float16 => null, // Any support for this in C#? + Dtype.Float32 => typeof(float), + Dtype.Float64 => typeof(double), + Dtype.Int8 => typeof(sbyte), + Dtype.Int16 => typeof(short), + Dtype.Int32 => typeof(int), + Dtype.Int64 => typeof(long), + Dtype.Uint8 => typeof(byte), + Dtype.Uint16 => typeof(ushort), + Dtype.Uint32 => typeof(uint), + Dtype.Uint64 => typeof(ulong), + Dtype.String => typeof(string), + Dtype.List => typeof(IList), + Dtype.Dict => typeof(IDictionary), + _ => null, + }; + + /// + /// Convert a managed type to a NatML data type. + /// + /// Managed type. + /// NatML data type. + public static Dtype ToDtype (this Type dtype) => dtype switch { + var t when t == typeof(float) => Dtype.Float32, + var t when t == typeof(double) => Dtype.Float64, + var t when t == typeof(sbyte) => Dtype.Int8, + var t when t == typeof(short) => Dtype.Int16, + var t when t == typeof(int) => Dtype.Int32, + var t when t == typeof(long) => Dtype.Int64, + var t when t == typeof(byte) => Dtype.Uint8, + var t when t == typeof(ushort) => Dtype.Uint16, + var t when t == typeof(uint) => Dtype.Uint32, + var t when t == typeof(ulong) => Dtype.Uint64, + _ => Dtype.Undefined, + }; + } +} \ No newline at end of file diff --git a/Runtime/API/Types/TypeExtensions.cs.meta b/Runtime/API/Types/TypeExtensions.cs.meta new file mode 100644 index 0000000..0fbc54a --- /dev/null +++ b/Runtime/API/Types/TypeExtensions.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 4c0ae76f3b25f49a3b704fb7c96a6f98 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API/Types/UploadType.cs b/Runtime/API/Types/UploadType.cs new file mode 100644 index 0000000..25e127b --- /dev/null +++ b/Runtime/API/Types/UploadType.cs @@ -0,0 +1,28 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.API.Types { + + using System.Runtime.Serialization; + using Newtonsoft.Json; + using Newtonsoft.Json.Converters; + + /// + /// Upload URL type. + /// + [JsonConverter(typeof(StringEnumConverter))] + public enum UploadType : int { + [EnumMember(Value = @"DEMO")] + Demo = 0, + [EnumMember(Value = @"FEATURE")] + Feature = 1, + [EnumMember(Value = @"GRAPH")] + Graph = 2, + [EnumMember(Value = @"MEDIA")] + Media = 3, + [EnumMember(Value = @"NOTEBOOK")] + Notebook = 4, + } +} \ No newline at end of file diff --git a/Runtime/API/Types/UploadType.cs.meta b/Runtime/API/Types/UploadType.cs.meta new file mode 100644 index 0000000..26b8d8a --- /dev/null +++ b/Runtime/API/Types/UploadType.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: d1410831f994442d3b796578c370d3a2 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/API/Types/User.cs b/Runtime/API/Types/User.cs new file mode 100644 index 0000000..9995c96 --- /dev/null +++ b/Runtime/API/Types/User.cs @@ -0,0 +1,18 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.API.Types { + + using System; + using API.Graph; + + /// + /// NatML user. + /// + [Preserve, Serializable] + public sealed class User : Profile { + + } +} \ No newline at end of file diff --git a/Runtime/API/Types/User.cs.meta b/Runtime/API/Types/User.cs.meta new file mode 100644 index 0000000..ce96e9c --- /dev/null +++ b/Runtime/API/Types/User.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: daf832c13bbe6466385cd9a69cada246 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Features.meta b/Runtime/Features.meta new file mode 100644 index 0000000..b6c24c7 --- /dev/null +++ b/Runtime/Features.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 71b8a0a0f85834799bdde75a9d840103 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Features/IMLCloudFeature.cs b/Runtime/Features/IMLCloudFeature.cs new file mode 100644 index 0000000..d53dfaf --- /dev/null +++ b/Runtime/Features/IMLCloudFeature.cs @@ -0,0 +1,20 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Features { + + /// + /// ML feature which can create cloud features. + /// + public interface IMLCloudFeature { + + /// + /// Create a Ccoud ML feature that is ready for prediction with cloud ML models. + /// + /// Feature type used to create the cloud ML feature. + /// Cloud ML feature. + MLCloudFeature Create (MLFeatureType featureType); + } +} \ No newline at end of file diff --git a/Runtime/Features/IMLCloudFeature.cs.meta b/Runtime/Features/IMLCloudFeature.cs.meta new file mode 100644 index 0000000..19c2517 --- /dev/null +++ b/Runtime/Features/IMLCloudFeature.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: f86ccca2be1c14e62abee3aae41c1c47 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Features/IMLEdgeFeature.cs b/Runtime/Features/IMLEdgeFeature.cs new file mode 100644 index 0000000..f0db677 --- /dev/null +++ b/Runtime/Features/IMLEdgeFeature.cs @@ -0,0 +1,20 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Features { + + /// + /// ML feature which can create edge ML features. + /// + public interface IMLEdgeFeature { + + /// + /// Create an edge ML feature that is ready for prediction with edge ML models. + /// + /// Feature type used to create the edge ML feature. + /// Edge ML feature. + MLEdgeFeature Create (MLFeatureType featureType); + } +} \ No newline at end of file diff --git a/Runtime/Features/IMLEdgeFeature.cs.meta b/Runtime/Features/IMLEdgeFeature.cs.meta new file mode 100644 index 0000000..0a5fa29 --- /dev/null +++ b/Runtime/Features/IMLEdgeFeature.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 8ee4471364a3c4adc85946bb8ec63bae +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Features/MLArrayFeature.cs b/Runtime/Features/MLArrayFeature.cs new file mode 100644 index 0000000..cff7f7e --- /dev/null +++ b/Runtime/Features/MLArrayFeature.cs @@ -0,0 +1,395 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Features { + + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using UnityEngine; + using UnityEngine.Experimental.Rendering; + using Unity.Collections; + using Unity.Collections.LowLevel.Unsafe; + using API.Types; + using Internal; + using Types; + + /// + /// ML array feature. + /// The array feature represents generic numeric tensor data. + /// The array feature never owns or allocates its own memory. Instead, it provides a "view" into existing data. + /// + public sealed unsafe class MLArrayFeature : MLFeature, IMLEdgeFeature, IMLCloudFeature where T : unmanaged { + + #region --Inspection-- + /// + /// Feature shape. + /// + public int[] shape => (type as MLArrayType).shape; + + /// + /// Feature element count. + /// + public int elementCount => (type as MLArrayType).elementCount; + #endregion + + + #region --Indexing-- + /// + /// Get or set a value at a specified index. + /// + /// Linear index. + public T this [in int idx] { + get { + fixed (T* data = this) + return data[idx]; + } + set { + fixed (T* data = this) + data[idx] = value; + } + } + + /// + /// Get or set a value at a specified index. + /// + /// Multi-index. + public T this [in int idx0, in int idx1] { + get => this[idx0 * strides[0] + idx1 * strides[1]]; + set => this[idx0 * strides[0] + idx1 * strides[1]] = value; + } + + /// + /// Get or set a value at a specified index. + /// + /// Multi-index. + public T this [in int idx0, in int idx1, in int idx2] { + get => this[idx0 * strides[0] + idx1 * strides[1] + idx2 * strides[2]]; + set => this[idx0 * strides[0] + idx1 * strides[1] + idx2 * strides[2]] = value; + } + + /// + /// Get or set a value at a specified index. + /// + /// Multi-index. + public T this [in int idx0, in int idx1, in int idx2, in int idx3] { + get => this[idx0 * strides[0] + idx1 * strides[1] + idx2 * strides[2] + idx3 * strides[3]]; + set => this[idx0 * strides[0] + idx1 * strides[1] + idx2 * strides[2] + idx3 * strides[3]] = value; + } + + /// + /// Get or set a value at a specified index. + /// + /// Multi-index. + public T this [in int idx0, in int idx1, in int idx2, in int idx3, in int idx4] { + get => this[idx0 * strides[0] + idx1 * strides[1] + idx2 * strides[2] + idx3 * strides[3] + idx4 * strides[4]]; + set => this[idx0 * strides[0] + idx1 * strides[1] + idx2 * strides[2] + idx3 * strides[3] + idx4 * strides[4]] = value; + } + + /// + /// Get or set a value at a specified index. + /// + /// Multi-index. + public T this [params int[] idx] { + get { + var linearIdx = 0; + for (var i = 0; i < strides.Length; ++i) + linearIdx += idx[i] * strides[i]; + return this[linearIdx]; + } + set { + var linearIdx = 0; + for (var i = 0; i < strides.Length; ++i) + linearIdx += idx[i] * strides[i]; + this[linearIdx] = value; + } + } + #endregion + + + #region --Constructors-- + /// + /// Create an array feature. + /// + /// Feature data. + /// Feature shape. + public MLArrayFeature (T[] data, int[] shape = null) : this(data, new MLArrayType(shape, typeof(T))) { } + + /// + /// Create an array feature. + /// + /// Feature data. + /// Feature shape. + public MLArrayFeature (NativeArray data, int[] shape = null) : this(data, new MLArrayType(shape, typeof(T))) { } + + /// + /// Create an array feature. + /// + /// Feature data. + /// Feature shape. + public unsafe MLArrayFeature (T* data, int[] shape = null) : this(data, new MLArrayType(shape, typeof(T))) { } + + /// + /// Create an array feature. + /// + /// Feature data. + /// Feature type. + public MLArrayFeature (T[] data, MLArrayType type) : base(type) { + this.array = data; + this.strides = ComputeStrides(shape); + } + + /// + /// Create an array feature. + /// + /// Feature data. + /// Feature type. + public unsafe MLArrayFeature (NativeArray data, MLArrayType type) : this((T*)data.GetUnsafePtr(), type) { } + + /// + /// Create an array feature. + /// + /// Feature data. + /// Feature type. + public unsafe MLArrayFeature (T* data, MLArrayType type) : base(type) { + this.buffer = data; + this.strides = ComputeStrides(shape); + } + + /// + /// Create an array feature from an edge feature. + /// Note that this does NOT take ownership of the edge feature. + /// As such the edge feature MUST be disposed by the client. + /// + /// Edge feature. This MUST be an array feature. + public MLArrayFeature (MLEdgeFeature feature) : this((T*)feature.data, feature.shape) { } + + /// + /// Create an array feature from a cloud feature. + /// + /// Cloud feature. This MUST be a numeric dtype. + public MLArrayFeature (MLCloudFeature feature) : this(GetFeatureData(feature), feature.shape) { } + #endregion + + + #region --Viewing-- + /// + /// Flatten this array feature into a one-dimensional feature. + /// + /// Optional starting dimension to flatten. + /// Optional ending dimension to flatten. + /// + public unsafe MLArrayFeature Flatten (int startDim = 0, int endDim = -1) { + endDim = endDim < 0 ? shape.Length - 1 : endDim; + var flatDim = shape.Skip(startDim).Take(endDim - startDim + 1).Aggregate(1, (a, b) => a * b); + var newShape = new List(); + newShape.AddRange(shape.Take(startDim)); + newShape.Add(flatDim); + newShape.AddRange(shape.Skip(endDim + 1)); + return View(newShape.ToArray()); + } + + /// + /// Permute the dimensions of this array feature. + /// This operation is a generalization of the transpose operation. + /// + /// Permuted dimensions. + /// Array feature with permuted dimensions. + public unsafe MLArrayFeature Permute (params int[] dims) { + var newShape = dims.Select(d => shape[d]).ToArray(); + var newStrides = dims.Select(d => strides[d]).ToArray(); + var result = View(newShape); + Array.Copy(newStrides, result.strides, newStrides.Length); + return result; + } + + /// + /// Remove all dimensions of size 1. + /// + /// Optional index to squeeze. + /// Array feature with singleton dimensions removed. + public unsafe MLArrayFeature Squeeze (int dim = -1) { + var newShape = new List(shape); + if (dim < 0) + newShape.RemoveAll(s => s == 1); + else if (newShape[dim] == 1) + newShape.RemoveAt(dim); + return View(newShape.ToArray()); + } + + /// + /// Create a view of this array feature with a different shape. + /// The element count of the new shape MUST match that of the feature. + /// Any single axis can be `-1`, in which case its value is dynamically calculated. + /// + /// New shape. + /// Array feature with new shape. + public unsafe MLArrayFeature View (params int[] shape) { + // Fill + var dynamicAxis = Array.IndexOf(shape, -1); + if (dynamicAxis >= 0) { + var staticAxisElementCount = shape + .Where((s, i) => i != dynamicAxis) + .Aggregate(1, (a, b) => a * b); + shape[dynamicAxis] = this.elementCount / staticAxisElementCount; + } + // Check + var elementCount = shape.Aggregate(1, (a, b) => a * b); + if (elementCount != this.elementCount) + throw new ArgumentOutOfRangeException( + nameof(shape), + "Array feature shape ("+string.Join(",", this.shape)+") cannot be viewed as ("+string.Join(",", shape)+")" + ); + // View + return buffer != null ? new MLArrayFeature(buffer, shape) : new MLArrayFeature(array, shape); + } + #endregion + + + #region --Copying-- + /// + /// Copy the array feature into another feature. + /// This copies `elementCount * sizeof(T)` bytes. + /// + /// Feature to copy data into. + public unsafe void CopyTo (MLArrayFeature destination) where U : unmanaged { + // Check that shape is specified + if (shape == null) + throw new InvalidOperationException(@"Cannot copy to destination feature because source feature does not have a shape"); + // Copy + var size = elementCount * sizeof(T); // bytes + fixed (void* src = this, dst = destination) + Buffer.MemoryCopy(src, dst, size, size); + } + + /// + /// Copy the array feature data into a texture. + /// This method MUST only be used from the Unity main thread. + /// The texture format MUST be compatible with the array feature data type. + /// + /// Texture to copy data into. + /// Whether to upload the pixel data to the GPU after copying.k + public unsafe void CopyTo ( + Texture2D destination, + bool upload = true + ) { + // Check dims + var imageType = MLImageType.FromType(type); + if (destination.width != imageType.width || destination.height != imageType.height) + throw new ArgumentException(@"Cannot copy to texture because texture size does not match feature size", nameof(destination)); + // Check format + if (!TextureFormats.TryGetValue(typeof(T), out var formats) || !formats.Contains(destination.format)) + throw new ArgumentException($"Cannot copy to texture because texture format {destination.format} is not compatible with this array feature", nameof(destination)); + // Copy data + var view = new MLArrayFeature(destination.GetRawTextureData()); + CopyTo(view); + // Upload + if (upload) + destination.Apply(); + } + + /// + /// Convert the array feature to a flattened array. + /// This method always returns a copy of the array feature data. + /// + /// Result array. + public unsafe T[] ToArray () => ToArray(); + + /// + /// Convert the array feature to a flattened array. + /// This method always returns a copy of the array feature data. + /// + /// Result array. + public unsafe U[] ToArray () where U : unmanaged { + var byteLength = elementCount * sizeof(T); + var length = byteLength / sizeof(U); + var result = new U[length]; + var feature = new MLArrayFeature(result, new [] { result.Length }); + CopyTo(feature); + return result; + } + #endregion + + + #region --Operations-- + private readonly T[] array; + private readonly T* buffer; + private readonly int[] strides; + private static readonly Dtype[] Dtypes = new [] { + Dtype.Float16, Dtype.Float32, Dtype.Float64, + Dtype.Int8, Dtype.Int16, Dtype.Int32, Dtype.Int64, + Dtype.Uint8, Dtype.Uint16, Dtype.Uint32, Dtype.Uint64, + Dtype.Bool + }; + private static readonly Dictionary> TextureFormats = new () { + [typeof(byte)] = new () { TextureFormat.RGBA32, TextureFormat.BGRA32, TextureFormat.ARGB32, TextureFormat.RGB24, TextureFormat.Alpha8, TextureFormat.R8 }, + [typeof(float)] = new () { TextureFormat.RGBAFloat, TextureFormat.RFloat }, + }; + + public ref T GetPinnableReference () => ref (buffer == null ? ref array[0] : ref *buffer); + + unsafe MLEdgeFeature IMLEdgeFeature.Create (MLFeatureType type) { + // Check array type + var arrayType = this.type as MLArrayType; + var featureType = type as MLArrayType; + if (featureType.dataType != arrayType.dataType) + throw new ArgumentException($"Cannot create {featureType.dataType} feature with {arrayType.dataType} feature", nameof(type)); + // Check that shape is fully specified + var shape = arrayType.shape ?? featureType.shape; + var dynamicAxis = Array.IndexOf(shape, -1); + if (dynamicAxis != -1) + throw new ArgumentException($"Array feature shape has unspecified size at dimension {dynamicAxis}", nameof(type)); + // Create feature + fixed (T* data = this) { + NatML.CreateFeature( + data, + shape, + shape.Length, + type.dataType.ToDtype(), + data == buffer ? 0 : 1, + out var feature + ); + return new MLEdgeFeature(feature); + } + } + + unsafe MLCloudFeature IMLCloudFeature.Create (MLFeatureType _) { + // Check shape + if (shape == null) + throw new InvalidOperationException(@"Array feature cannot be used for Cloud prediction because it has no shape"); + // Create feature + var data = ToArray(); + var stream = new MemoryStream(data, 0, data.Length, false, false); + var dtype = typeof(T).ToDtype(); + var feature = new MLCloudFeature(stream, dtype, shape); + return feature; + } + + private static int[] ComputeStrides (int[] shape) { + // Check + if (shape == null || shape.Length == 0) + return null; + // Compute + var result = new int[shape.Length]; + result[shape.Length - 1] = 1; + for (var i = shape.Length - 2; i >= 0; --i) + result[i] = shape[i + 1] * result[i + 1]; + return result; + } + + private static T[] GetFeatureData (MLCloudFeature feature) { + // Check + if (Array.IndexOf(Dtypes, feature.type) < 0) + throw new ArgumentException(@"Cloud feature is not an array feature", nameof(feature)); + // Deserialize + var rawData = feature.data.ToArray(); + var data = new T[rawData.Length / sizeof(T)]; + Buffer.BlockCopy(rawData, 0, data, 0, rawData.Length); + return data; + } + #endregion + } +} \ No newline at end of file diff --git a/Runtime/Features/MLArrayFeature.cs.meta b/Runtime/Features/MLArrayFeature.cs.meta new file mode 100644 index 0000000..cce340f --- /dev/null +++ b/Runtime/Features/MLArrayFeature.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 2d1d8a25350f04d37af87d2a7410046d +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Features/MLAudioFeature.cs b/Runtime/Features/MLAudioFeature.cs new file mode 100644 index 0000000..597b51b --- /dev/null +++ b/Runtime/Features/MLAudioFeature.cs @@ -0,0 +1,342 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Features { + + using System; + using System.Collections; + using System.Collections.Generic; + using System.IO; + using System.Text; + using System.Threading.Tasks; + using System.Linq; + using UnityEngine; + using Unity.Collections; + using Unity.Collections.LowLevel.Unsafe; + using API.Types; + using Internal; + using Types; + + /// + /// ML audio feature. + /// The audio feature will perform any necessary conversions and pre-processing to a model's desired input feature type. + /// Sample buffers used to create audio features MUST be floating-point linear PCM data interleaved by channel and in range [-1.0, 1.0]. + /// + public sealed unsafe class MLAudioFeature : MLFeature, IMLEdgeFeature, IMLCloudFeature, IEnumerable<(MLAudioFeature feature, long timestamp)> { + + #region --Preprocessing-- + /// + /// Desired sample rate for predictions. + /// + public int sampleRate; + + /// + /// Desired channel count for predictions. + /// + public int channelCount; + + /// + /// Normalization mean. + /// + public float mean = 0f; + + /// + /// Normalization standard deviation. + /// + public float std = 1f; + #endregion + + + #region --Constructors-- + /// + /// Create an audio feature from an audio clip. + /// + /// Audio clip. + /// Optional duration to extract in seconds. Negative values will use the whole clip. + public MLAudioFeature ( + AudioClip clip, + float duration = -1 + ) : this(Extract(clip, duration), clip.frequency, clip.channels) { } + + /// + /// Create an audio feature from a sample buffer. + /// + /// Linear PCM sample buffer interleaved by channel. + /// Sample rate. + /// Channel count. + public MLAudioFeature ( + float[] sampleBuffer, + int sampleRate, + int channelCount + ) : base(new MLAudioType(sampleRate, channelCount, sampleBuffer.Length)) { + this.sampleBuffer = sampleBuffer; + this.sampleRate = sampleRate; + this.channelCount = channelCount; + } + + /// + /// Create an audio feature from a sample buffer. + /// + /// Linear PCM sample buffer interleaved by channel. + /// Sample rate. + /// Channel count. + public unsafe MLAudioFeature ( + NativeArray sampleBuffer, + int sampleRate, + int channelCount + ) : this((float*)sampleBuffer.GetUnsafeReadOnlyPtr(), sampleRate, channelCount, sampleBuffer.Length) { } + + /// + /// Create an audio feature from a sample buffer. + /// + /// Linear PCM sample buffer interleaved by channel. + /// Sample rate. + /// Channel count. + /// Total sample count. + public unsafe MLAudioFeature ( + float* sampleBuffer, + int sampleRate, + int channelCount, + int sampleCount + ) : base(new MLAudioType(sampleRate, channelCount, sampleCount)) { + this.nativeBuffer = sampleBuffer; + this.sampleRate = sampleRate; + this.channelCount = channelCount; + } + + /// + /// Create an audio feature from a sample buffer list. + /// + /// List of linear PCM sample buffers interleaved by channel. + /// Sample rate. + /// Channel count. + public MLAudioFeature ( + IEnumerable bufferList, + int sampleRate, + int channelCount + ) : this(Flatten(bufferList), sampleRate, channelCount) { } + + /// + /// Create an audio feature from an audio or video file. + /// The file is not loaded into memory and as such the feature will not contain any audio samples. + /// Instead the feature must be enumerated to retrieve contained audio features that contain samples. + /// + /// Audio file path. + public MLAudioFeature (string path) : base(MLAudioType.FromFile(path)) { + this.path = path; + this.sampleRate = (type as MLAudioType).sampleRate; + this.channelCount = (type as MLAudioType).channelCount; + } + #endregion + + + #region --Copying-- + /// + /// Create an audio feature which contains all audio samples loaded into contiguous memory. + /// If the audio feature is already loaded, then `this` feature will be returned. + /// Note that this can consume large amounts of memory for large audio files. + /// For better memory consumption, manually enumerate the feature instead. + /// + /// Audio feature which contains all audio samples in contiguous memory. + public MLAudioFeature Contiguous () { + // Check + if (string.IsNullOrEmpty(path)) + return this; + // Read + var sampleBuffer = new List(); + foreach (var (feature, timestamp) in this) { + var type = feature.type as MLAudioType; + var buffer = new float[type.elementCount]; + feature.CopyTo(buffer); + sampleBuffer.AddRange(buffer); + } + return new MLAudioFeature(sampleBuffer.ToArray(), sampleRate, channelCount); + } + + /// + /// Copy the audio data in this feature into the provided sample buffer. + /// + /// Destination sample buffer. + public unsafe void CopyTo (float[] sampleBuffer) { + fixed (float* buffer = sampleBuffer) + CopyTo(buffer); + } + + /// + /// Copy the audio data in this feature into the provided sample buffer. + /// + /// Destination sample buffer. + public unsafe void CopyTo (NativeArray sampleBuffer) => CopyTo((float*)sampleBuffer.GetUnsafePtr()); + + /// + /// Copy the audio data in this feature into the provided sample buffer. + /// + /// Destination sample buffer. + public unsafe void CopyTo (float* sampleBuffer) { + // Check + if (!string.IsNullOrEmpty(path)) + throw new InvalidOperationException(@"Cannot copy to buffer because audio feature is not contiguous"); + // Copy + var sampleCount = (type as MLAudioType).elementCount; + fixed (float* srcData = this) + UnsafeUtility.MemCpy(sampleBuffer, srcData, sampleCount * sizeof(float)); + } + + /// + /// Convert the audio feature to an audio clip. + /// This method MUST only be used from the Unity main thread. + /// + /// Result audio clip. + public AudioClip ToAudioClip () { + // Check + if (!string.IsNullOrEmpty(path)) + throw new InvalidOperationException(@"Cannot convert to AudioClip because audio feature is not contiguous"); + // Convert + var type = this.type as MLAudioType; + var clip = AudioClip.Create(type.name ?? "MLAudioFeature", type.elementCount, type.channelCount, type.sampleRate, false); + var sampleBuffer = this.sampleBuffer; + if (sampleBuffer == null) { + sampleBuffer = new float[type.elementCount]; + CopyTo(sampleBuffer); + } + clip.SetData(sampleBuffer, 0); + return clip; + } + #endregion + + + #region --Operations-- + private readonly float[] sampleBuffer; + private readonly float* nativeBuffer; + private readonly string path; + + public ref float GetPinnableReference () => ref (nativeBuffer == null ? ref sampleBuffer[0] : ref *nativeBuffer); + + unsafe MLEdgeFeature IMLEdgeFeature.Create (MLFeatureType type) { + var featureType = type as MLArrayType; + var audioType = this.type as MLAudioType; + var mean = new float[channelCount]; + var std = new float[channelCount]; + Array.Fill(mean, this.mean); + Array.Fill(std, this.std); + fixed (float* data = this) { + NatML.CreateFeature( + data, + audioType.sampleRate, + audioType.shape, // (1,F,C) + sampleRate, + channelCount, + featureType.dataType.ToDtype(), + mean, + std, + 0, + out var feature + ); + return new MLEdgeFeature(feature); + } + } + + unsafe MLCloudFeature IMLCloudFeature.Create (MLFeatureType _) { + var stream = new MemoryStream(); + var type = this.type as MLAudioType; + var sampleCount = type.elementCount; + var dataLength = 44 + sampleCount * sizeof(short) - 8; + stream.Write(Encoding.UTF8.GetBytes("RIFF"), 0, 4); + stream.Write(BitConverter.GetBytes(dataLength), 0, 4); + stream.Write(Encoding.UTF8.GetBytes("WAVE"), 0, 4); + stream.Write(Encoding.UTF8.GetBytes("fmt "), 0, 4); + stream.Write(BitConverter.GetBytes(16), 0, 4); + stream.Write(BitConverter.GetBytes((ushort)1), 0, 2); + stream.Write(BitConverter.GetBytes(type.channelCount), 0, 2); + stream.Write(BitConverter.GetBytes(type.sampleRate), 0, 4); + stream.Write(BitConverter.GetBytes(type.sampleRate * type.channelCount * sizeof(short)), 0, 4); + stream.Write(BitConverter.GetBytes((ushort)(type.channelCount * 2)), 0, 2); + stream.Write(BitConverter.GetBytes((ushort)16), 0, 2); + stream.Write(Encoding.UTF8.GetBytes("data"), 0, 4); + stream.Write(BitConverter.GetBytes(sampleCount * sizeof(ushort)), 0, 4); + fixed (float* srcData = this) + fixed (short* dstData = new short[sampleCount]) + using (var dataStream = new UnmanagedMemoryStream((byte*)dstData, sampleCount * sizeof(short))) { + for (var i = 0; i < sampleCount; ++i) + dstData[i] = (short)(srcData[i] * short.MaxValue); + dataStream.CopyTo(stream); + } + stream.Seek(0, SeekOrigin.Begin); + var feature = new MLCloudFeature(stream, Dtype.Audio, type.shape); + return feature; + } + + IEnumerator<(MLAudioFeature, long)> IEnumerable<(MLAudioFeature feature, long timestamp)>.GetEnumerator () { + // Check path + if (string.IsNullOrEmpty(path)) + throw new InvalidOperationException(@"Cannot enumerate a contiguous audio feature"); + // Create reader + NatML.CreateAudioFeatureReader(path, out var reader); + if (reader == IntPtr.Zero) + throw new InvalidOperationException(@"Failed to create audio reader"); + // Read + var feature = IntPtr.Zero; + var type = this.type as MLAudioType; + try { + for (;;) { + // Read frame + feature.ReleaseFeature(); + feature = IntPtr.Zero; + reader.ReadNextFeature(out var timestamp, out feature); + // EOS + if (timestamp < 0) + break; + // Skip + if (feature == IntPtr.Zero) + continue; + // Create feature + var audioFeature = CreateAudioFeature(feature, type); + audioFeature.sampleRate = sampleRate; + audioFeature.channelCount = channelCount; + audioFeature.mean = mean; + audioFeature.std = std; + yield return (audioFeature, timestamp); + } + } + // Release + finally { + feature.ReleaseFeature(); + reader.ReleaseFeatureReader(); + } + } + + IEnumerator IEnumerable.GetEnumerator () => (this as IEnumerable<(MLAudioFeature, long)>).GetEnumerator(); + + private static float[] Extract (AudioClip clip, float duration = -1) { + var frameCount = duration < 0 ? clip.samples : Mathf.RoundToInt(clip.frequency * duration); + frameCount = Mathf.Min(frameCount, clip.samples); + var sampleBuffer = new float[frameCount * clip.channels]; + clip.GetData(sampleBuffer, 0); + return sampleBuffer; + } + + private static unsafe float[] Flatten (IEnumerable bufferList) { + var bufferSize = bufferList.Select(s => s.Length).Sum(); + var result = new float[bufferSize]; + var idx = 0; + fixed (float* dst = result) + foreach (var sampleBuffer in bufferList) + fixed (float* src = sampleBuffer) { + UnsafeUtility.MemCpy(&dst[idx], src, sampleBuffer.Length * sizeof(float)); + idx += sampleBuffer.Length; + } + return result; + } + + private static unsafe MLAudioFeature CreateAudioFeature (IntPtr feature, MLAudioType audioType) { + NatML.FeatureType(feature, out var type); + var shape = new int[3]; + type.FeatureTypeShape(shape, shape.Length); + type.ReleaseFeatureType(); + return new MLAudioFeature((float*)feature.FeatureData(), audioType.sampleRate, shape[2], shape[1] * shape[2]); + } + #endregion + } +} \ No newline at end of file diff --git a/Runtime/Features/MLAudioFeature.cs.meta b/Runtime/Features/MLAudioFeature.cs.meta new file mode 100644 index 0000000..a674777 --- /dev/null +++ b/Runtime/Features/MLAudioFeature.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: e573160b329b645eabedcf106bb4505f +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Features/MLCloudFeature.cs b/Runtime/Features/MLCloudFeature.cs new file mode 100644 index 0000000..1651f33 --- /dev/null +++ b/Runtime/Features/MLCloudFeature.cs @@ -0,0 +1,54 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +#nullable enable + +namespace NatML.Features { + + using System; + using System.IO; + using API.Types; + + /// + /// Feature used for making cloud predictions. + /// + public readonly struct MLCloudFeature : IDisposable { + + #region --Client API-- + /// + /// Feature data. + /// + public readonly MemoryStream data; + + /// + /// Feature type. + /// + public readonly Dtype type; + + /// + /// Feature shape. + /// This MUST be populated for array features. + /// + public readonly int[]? shape; + + /// + /// Dispose the feature and release resources. + /// + public readonly void Dispose () => data.Dispose(); + + /// + /// Create a cloud feature. + /// + /// Feature data stream. + /// Feature data type. + /// Feature shape. This is only used for array features. + public MLCloudFeature (MemoryStream data, Dtype type, int[]? shape = null) { + this.data = data; + this.type = type; + this.shape = shape; + } + #endregion + } +} \ No newline at end of file diff --git a/Runtime/Features/MLCloudFeature.cs.meta b/Runtime/Features/MLCloudFeature.cs.meta new file mode 100644 index 0000000..22e2d92 --- /dev/null +++ b/Runtime/Features/MLCloudFeature.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 4c743dfaaa83c463ba58cbc135bdfc78 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Features/MLDepthFeature.cs b/Runtime/Features/MLDepthFeature.cs new file mode 100644 index 0000000..375099c --- /dev/null +++ b/Runtime/Features/MLDepthFeature.cs @@ -0,0 +1,73 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Features { + + using System; + using UnityEngine; + using Types; + + /// + /// ML depth feature. + /// The depth feature is used to provide depth data to predictors that require such data. + /// Implementers can derive from this class and provide custom logic for sampling depth given a pixel location. + /// + public abstract class MLDepthFeature : MLFeature { + + #region --Inspection-- + /// + /// Depth map width. + /// + public int width => (type as MLImageType).width; + + /// + /// Depth map height. + /// + public int height => (type as MLImageType).height; + #endregion + + + #region --Sampling-- + /// + /// Sample the depth feature at a given point. + /// + /// Point to sample in normalized coordinates. + /// Depth in meters. + public abstract float Sample (Vector2 point); + + /// + /// Unproject a 2D point into 3D world space using depth data. + /// + /// Normalized point to transform in range [0., 1.]. + /// Unprojected point in 3D world space. + public abstract Vector3 Unproject (Vector2 point); + #endregion + + + #region --Operations-- + /// + /// Initialize the depth feature with the depth map dimensions. + /// + /// Depth map width. + /// Depth map height. + protected MLDepthFeature (int width, int height) : this(new MLImageType(width, height, 1, typeof(float))) { } + + /// + /// Initialize the depth feature with the depth map feature type. + /// + /// Depth map feature type. + protected MLDepthFeature (MLImageType type) : base(type) { } + #endregion + + + #region --DEPRECATED-- + [Obsolete(@"Deprecated in NatML 1.0.16. Use `ViewportToWorldPoint` method instead.", false)] + public virtual Vector3 TransformPoint (Vector2 point) => ViewportToWorldPoint(point); + + [Obsolete(@"Deprecated in NatML 1.1.4. Use `Unproject` method instead.", false)] + public virtual Vector3 ViewportToWorldPoint (Vector2 point) => Unproject(point); + #endregion + } +} \ No newline at end of file diff --git a/Runtime/Features/MLDepthFeature.cs.meta b/Runtime/Features/MLDepthFeature.cs.meta new file mode 100644 index 0000000..655005e --- /dev/null +++ b/Runtime/Features/MLDepthFeature.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 05b7c33652a3346e084ad8564a7ce560 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Features/MLEdgeFeature.cs b/Runtime/Features/MLEdgeFeature.cs new file mode 100644 index 0000000..5f3dbee --- /dev/null +++ b/Runtime/Features/MLEdgeFeature.cs @@ -0,0 +1,65 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +#nullable enable + +namespace NatML.Features { + + using System; + using API.Types; + using Internal; + + /// + /// Feature used for making edge predictions. + /// + public unsafe readonly struct MLEdgeFeature : IDisposable { + + #region --Client API-- + /// + /// Feature data. + /// + public readonly void* data => (void*)feature.FeatureData(); + + /// + /// Feature shape. + /// + public readonly int[] shape { + get { + feature.FeatureType(out var type); + var shape = new int[type.FeatureTypeDimensions()]; + type.FeatureTypeShape(shape, shape.Length); + type.ReleaseFeatureType(); + return shape; + } + } + + /// + /// Feature data type + /// + public readonly Dtype dataType { + get { + feature.FeatureType(out var type); + var result = type.FeatureTypeDataType(); + type.ReleaseFeatureType(); + return result; + } + } + + /// + /// Dispose the feature and release resources. + /// + public readonly void Dispose () => feature.ReleaseFeature(); + #endregion + + + #region --Operations-- + private readonly IntPtr feature; + + public MLEdgeFeature (IntPtr feature) => this.feature = feature; + + public static implicit operator IntPtr (MLEdgeFeature feature) => feature.feature; + #endregion + } +} \ No newline at end of file diff --git a/Runtime/Features/MLEdgeFeature.cs.meta b/Runtime/Features/MLEdgeFeature.cs.meta new file mode 100644 index 0000000..1e02e94 --- /dev/null +++ b/Runtime/Features/MLEdgeFeature.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 7ede1f505871a4ede95f0cc3c9831fd1 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Features/MLImageFeature.cs b/Runtime/Features/MLImageFeature.cs new file mode 100644 index 0000000..9aa7dc4 --- /dev/null +++ b/Runtime/Features/MLImageFeature.cs @@ -0,0 +1,383 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Features { + + using System; + using System.Collections.Generic; + using System.IO; + using UnityEngine; + using UnityEngine.Experimental.Rendering; + using Unity.Collections; + using Unity.Collections.LowLevel.Unsafe; + using API.Types; + using Internal; + using Types; + + /// + /// ML image feature. + /// The image feature will perform any necessary conversions and pre-processing to a model's desired input feature type. + /// Pixel buffers used to create image features MUST have an RGBA8888 pixel layout. + /// + public sealed unsafe class MLImageFeature : MLFeature, IMLEdgeFeature, IMLCloudFeature { + + #region --Inspection-- + /// + /// Image width. + /// + public int width => (type as MLImageType).width; + + /// + /// Image height. + /// + public int height => (type as MLImageType).height; + #endregion + + + #region --Preprocessing-- + /// + /// Normalization mean. + /// + public Vector4 mean = Vector4.zero; + + /// + /// Normalization standard deviation. + /// + public Vector4 std = Vector4.one; + + /// + /// Aspect mode. + /// + public AspectMode aspectMode = 0; + #endregion + + + #region --Constructors-- + /// + /// Create an empty image feature. + /// + /// Image feature width. + /// Image feature height. + public MLImageFeature (int width, int height) : this(new byte[width * height * 4], width, height) { } + + /// + /// Create an image feature from a texture. + /// + /// Input texture. + public MLImageFeature (Texture2D texture) : base(new MLImageType(texture.width, texture.height, 4)) { + if (!texture.isReadable) + throw new ArgumentException(@"Cannot create image feature because texture is not readable", nameof(texture)); + if (texture.format == TextureFormat.RGBA32) // zero copy :D + this.nativeBuffer = texture.GetRawTextureData().GetUnsafeReadOnlyPtr(); + else // double copy :( + this.pixelBuffer = ToPixelBuffer(texture.GetPixels32(), texture.width, texture.height); + } + + /// + /// Create an image feature from a pixel buffer. + /// + /// Color buffer. + /// Pixel buffer width. + /// Pixel buffer height. + public MLImageFeature ( + Color32[] colorBuffer, + int width, + int height + ) : this(ToPixelBuffer(colorBuffer, width, height), width, height) { } + + /// + /// Create an image feature from a pixel buffer. + /// + /// Pixel buffer. MUST have an RGBA8888 layout. + /// Pixel buffer width. + /// Pixel buffer height. + public MLImageFeature ( + byte[] pixelBuffer, + int width, + int height + ) : base(new MLImageType(width, height, 4)) => this.pixelBuffer = pixelBuffer; + + /// + /// Create an image feature from a pixel buffer. + /// Note that the native array MUST remain valid for the lifetime of the feature. + /// + /// Pixel buffer. MUST have an RGBA8888 layout. + /// Pixel buffer width. + /// Pixel buffer height. + public MLImageFeature ( + NativeArray pixelBuffer, + int width, + int height + ) : this(pixelBuffer.GetUnsafeReadOnlyPtr(), width, height) { } + + /// + /// Create an image feature from a pixel buffer. + /// Note that the pixel buffer MUST remain valid for the lifetime of the feature. + /// + /// Pixel buffer. MUST have an RGBA8888 layout. + /// Pixel buffer width. + /// Pixel buffer height. + public unsafe MLImageFeature ( + void* pixelBuffer, + int width, + int height + ) : base(new MLImageType(width, height, 4)) => this.nativeBuffer = pixelBuffer; + + /// + /// Create an image feature from a cloud feature. + /// This constructor MUST only be used on the Unity main thread. + /// + /// Cloud feature. This MUST be an `image` feature. + public MLImageFeature (MLCloudFeature feature) : base(new MLImageType(feature.shape, typeof(byte))) { + // Check + if (feature.type != Dtype.Image) + throw new ArgumentException(@"Cloud feature is not an image feature", nameof(feature)); + // Load image + var texture = new Texture2D(16, 16); + ImageConversion.LoadImage(texture, feature.data.ToArray(), false); + this.pixelBuffer = ToPixelBuffer(texture.GetPixels32(), texture.width, texture.height); + // Release image + Texture2D.Destroy(texture); + } + #endregion + + + #region --Copying-- + /// + /// Copy the image feature into another feature. + /// + /// Feature to copy data into. + public void CopyTo (MLImageFeature destination) => CopyTo( + destination, + new RectInt(0, 0, width, height) + ); + + /// + /// Copy an image feature region of interest into another feature. + /// + /// Feature to copy data into. + /// ROI rectangle in normalized coordinates. + /// Rectangle clockwise rotation in degrees. + /// Background color for unmapped pixels. + public void CopyTo ( + MLImageFeature destination, + Rect rect, + float rotation = 0f, + Color32 background = default + ) => CopyTo( + destination, + new RectInt((int)(rect.xMin * width), (int)(rect.yMin * height), (int)(rect.xMax * width), (int)(rect.yMax * height)), + rotation, + background + ); + + /// + /// Copy an image feature region of interest into another feature. + /// + /// Feature to copy data into. + /// ROI rectangle in pixel coordinates. + /// Rectangle clockwise rotation in degrees. + /// Background color for unmapped pixels. + public unsafe void CopyTo ( + MLImageFeature destination, + RectInt rect, + float rotation = 0f, + Color32 background = default + ) { + // Check + if (rect.size.x <= 0 || rect.size.y <= 0) + throw new ArgumentOutOfRangeException(nameof(rect), @"ROI rectangle must have positive width and height"); + // Check + if (rect.size.x != destination.width || rect.size.y != destination.height) + throw new ArgumentOutOfRangeException(nameof(rect), @"ROI rectangle size does not match destination feature size"); + // Copy + var center = Vector2Int.RoundToInt(rect.center); + var srcRect = stackalloc [] { center.x, center.y, rect.size.x, rect.size.y }; + var srcBuffer = this.nativeBuffer; + var dstBuffer = destination.nativeBuffer; + fixed (void* src = this, dst = destination) + NatML.CopyTo(src, width, height, srcRect, rotation, (byte*)&background, dst); + } + + /// + /// Copy the image feature data into a texture. + /// This method MUST only be used from the Unity main thread. + /// + /// Texture to copy data into. + /// Whether to upload the pixel data to the GPU after copying. + public unsafe void CopyTo ( + Texture2D destination, + bool upload = true + ) { + // Check size + if (destination.width != width || destination.height != height) + throw new ArgumentException(@"Cannot copy to texture because texture size does not match feature size", nameof(destination)); + // Check format + if (destination.format != TextureFormat.RGBA32) + throw new ArgumentException(@"Cannot copy to texture because texture format is not RGBA32", nameof(destination)); + // Copy data + var view = new MLImageFeature(destination.GetRawTextureData(), width, height); + CopyTo(view); + // Upload + if (upload) + destination.Apply(); + } + + /// + /// Create a texture from the image feature. + /// This method MUST only be used from the Unity main thread. + /// + /// Texture containing image feature data. + public Texture2D ToTexture () { + var result = new Texture2D(width, height, TextureFormat.RGBA32, false); + CopyTo(result, upload: true); + return result; + } + #endregion + + + #region --Coordinate Transformations-- + /// + /// Transform a normalized point from feature space into image space. + /// This method is used by detection models to correct for aspect ratio padding when making predictions. + /// + /// Input point. + /// Feature type that defines the input space. + /// Normalized point in image space. + public Vector2 TransformPoint (Vector2 point, MLImageType featureType) { + // Shortcut + if (aspectMode == AspectMode.ScaleToFit) + return point; + // Get normalizing factor + var scaleFactor = 1f; + if (aspectMode == AspectMode.AspectFit) + scaleFactor = Mathf.Max((float)width / featureType.width, (float)height / featureType.height); + if (aspectMode == AspectMode.AspectFill) + scaleFactor = Mathf.Min((float)width / featureType.width, (float)height / featureType.height); + // Transform + var transform = Matrix4x4.Scale(new Vector3(1f / width, 1f / height, 1f)) * + Matrix4x4.Translate(0.5f * new Vector2(width, height)) * + Matrix4x4.Scale(new Vector3(scaleFactor * featureType.width, scaleFactor * featureType.height, 1f)) * + Matrix4x4.Translate(-0.5f * Vector2.one); + var result = transform.MultiplyPoint3x4(new Vector3(point.x, point.y, 1f)); + return result; + } + + /// + /// Transform a normalized region-of-interest rectangle from feature space into image space. + /// This method is used by detection models to correct for aspect ratio padding when making predictions. + /// + /// Input rectangle. + /// Feature type that defines the input space. + /// Normalized rectangle in image space. + public Rect TransformRect (Rect rect, MLImageType featureType) { + var min = TransformPoint(rect.min, featureType); + var max = TransformPoint(rect.max, featureType); + return Rect.MinMaxRect(min.x, min.y, max.x, max.y); + } + #endregion + + + #region --Vision-- + /// + /// Perform non-max suppression on a set of candidate boxes. + /// + /// Candidate boxes. + /// Candidate scores. + /// Maximum IoU for preserving overlapping boxes. + /// Indices of boxes to keep. + public static int[] NonMaxSuppression ( + IReadOnlyList rects, + IReadOnlyList scores, + float maxIoU + ) { + var discard = new bool[rects.Count]; + for (int i = 0, ilen = discard.Length - 1; i < ilen; ++i) + if (!discard[i]) + for (var j = i + 1; j < discard.Length; ++j) + if (!discard[j]) { + var iou = IntersectionOverUnion(rects[i], rects[j]); + if (iou < maxIoU) + continue; + if (scores[i] > scores[j]) + discard[j] = true; + else { + discard[i] = true; + break; + } + } + var result = new List(); + for (var i = 0; i < discard.Length; ++i) + if (!discard[i]) + result.Add(i); + return result.ToArray(); + } + + /// + /// Calculate the intersection-over-union (IoU) of two rectangles. + /// + public static float IntersectionOverUnion (Rect a, Rect b) { + var areaA = a.width * a.height; + var areaB = b.width * b.height; + var c = Rect.MinMaxRect( + Mathf.Max(a.xMin, b.xMin), + Mathf.Max(a.yMin, b.yMin), + Mathf.Min(a.xMax, b.xMax), + Mathf.Min(a.yMax, b.yMax) + ); + var areaC = Mathf.Max(c.width, 0) * Mathf.Max(c.height, 0); + return areaC / (areaA + areaB - areaC); + } + #endregion + + + #region --Operations-- + private readonly byte[] pixelBuffer; + private readonly void* nativeBuffer; + + public ref byte GetPinnableReference () => ref (nativeBuffer == null ? ref pixelBuffer[0] : ref *(byte*)nativeBuffer); + + unsafe MLEdgeFeature IMLEdgeFeature.Create (MLFeatureType type) { + fixed (void* data = this) { + var featureType = type as MLArrayType; + var meanArr = stackalloc [] { mean.x, mean.y, mean.z, mean.w }; + var stdArr = stackalloc [] { std.x, std.y, std.z, std.w }; + NatML.CreateFeature( + data, + width, + height, + featureType.shape, + featureType.dataType.ToDtype(), + meanArr, + stdArr, + (int)aspectMode, + out var feature + ); + return new MLEdgeFeature(feature); + } + } + + unsafe MLCloudFeature IMLCloudFeature.Create (MLFeatureType _) { + fixed (void* data = this) { + var size = width * height * 4; + var pixelArray = NativeArrayUnsafeUtility.ConvertExistingDataToNativeArray(data, size, Allocator.None); + using var buffer = ImageConversion.EncodeNativeArrayToJPG(pixelArray, GraphicsFormat.R8G8B8A8_UNorm, (uint)width, (uint)height, quality: 80); + var encodedData = buffer.ToArray(); + var stream = new MemoryStream(encodedData, 0, encodedData.Length, false, false); + var shape = (type as MLImageType).shape; + var feature = new MLCloudFeature(stream, Dtype.Image, shape); + return feature; + } + } + + private static unsafe byte[] ToPixelBuffer (Color32[] colorBuffer, int width, int height) { + var pixelBuffer = new byte[width * height * 4]; + fixed (void* src = colorBuffer, dst = pixelBuffer) + Buffer.MemoryCopy(src, dst, pixelBuffer.Length, pixelBuffer.Length); + return pixelBuffer; + } + #endregion + } +} \ No newline at end of file diff --git a/Runtime/Features/MLImageFeature.cs.meta b/Runtime/Features/MLImageFeature.cs.meta new file mode 100644 index 0000000..a2f945a --- /dev/null +++ b/Runtime/Features/MLImageFeature.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 0256c19786f274c07a509bf1356b999a +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Features/MLStringFeature.cs b/Runtime/Features/MLStringFeature.cs new file mode 100644 index 0000000..2f6337f --- /dev/null +++ b/Runtime/Features/MLStringFeature.cs @@ -0,0 +1,60 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Features { + + using System; + using System.IO; + using System.Text; + using API.Types; + using Internal; + using Types; + + /// + /// ML string feature. + /// This feature will mainly be used with natural language processing models. + /// + public sealed class MLStringFeature : MLFeature, IMLCloudFeature { + + #region --Client API-- + /// + /// Feature text. + /// + public readonly string text; + + /// + /// Create the string feature from plain text. + /// + /// Feature text. + public MLStringFeature (string text) : base(new MLStringType(text.Length)) => this.text = text; + + /// + /// Create the string feature from a cloud feature. + /// + /// Cloud feature. This MUST be a `string` feature. + public MLStringFeature (MLCloudFeature feature) : base(new MLStringType(feature.shape[1])) { + // Check + if (feature.type != Dtype.String) + throw new ArgumentException(@"Cloud feature is not a string feature", nameof(feature)); + // Deserialize + using var reader = new StreamReader(feature.data); + this.text = reader.ReadToEnd(); + } + #endregion + + + #region --Operations-- + + unsafe MLCloudFeature IMLCloudFeature.Create (MLFeatureType _) { + var data = Encoding.UTF8.GetBytes(text); + var stream = new MemoryStream(data, 0, data.Length, false, false); + var feature = new MLCloudFeature(stream, Dtype.String, new [] { 1, text.Length }); + return feature; + } + + public static implicit operator string (MLStringFeature feature) => feature.text; + #endregion + } +} \ No newline at end of file diff --git a/Runtime/Features/MLStringFeature.cs.meta b/Runtime/Features/MLStringFeature.cs.meta new file mode 100644 index 0000000..bf5c7d2 --- /dev/null +++ b/Runtime/Features/MLStringFeature.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 514b5d90c65db4b3e9ace6a02af21d51 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Features/MLVideoFeature.cs b/Runtime/Features/MLVideoFeature.cs new file mode 100644 index 0000000..bd7256b --- /dev/null +++ b/Runtime/Features/MLVideoFeature.cs @@ -0,0 +1,133 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Features { + + using System; + using System.Collections; + using System.Collections.Generic; + using System.Threading.Tasks; + using UnityEngine; + using API.Types; + using Internal; + using Types; + + /// + /// ML video feature. + /// The video feature is always backed by a video file, and provides access to the contained image frames. + /// When enumerated the video feature yields image frames as `MLImageFeature` instances, along with corresponding frame timestamps. + /// The enumerated image features can then be used for predictions with ML models. + /// + public sealed class MLVideoFeature : MLFeature, IEnumerable<(MLImageFeature feature, long timestamp)> { + + #region --Inspection-- + /// + /// Video path. + /// + public readonly string path; + + /// + /// Video width. + /// + public int width => (type as MLVideoType).width; + + /// + /// Video height. + /// + public int height => (type as MLVideoType).height; + + /// + /// Video frame count. + /// Note that this is an approximate count. + /// + public int frames => (type as MLVideoType).frames; + #endregion + + + #region --Preprocessing-- + /// + /// Normalization mean. + /// + public Vector4 mean = Vector4.zero; + + /// + /// Normalization standard deviation. + /// + public Vector4 std = Vector4.one; + + /// + /// Aspect mode. + /// + public AspectMode aspectMode = 0; + #endregion + + + #region --Constructors-- + /// + /// Create an video feature from a video file. + /// + /// Video file path. + public MLVideoFeature (string path) : base(MLVideoType.FromFile(path)) => this.path = path; + + /// + /// Create a video feature from a video file in the `StreamingAssets` folder. + /// + /// Relative path to video file in `StreamingAssets` folder. + /// Video feature or `null` if no valid video can be found at the relative path. + public static async Task FromStreamingAssets (string relativePath) { + var absolutePath = await MLUnityExtensions.StreamingAssetsToAbsolutePath(relativePath); + return new MLVideoFeature(absolutePath); + } + #endregion + + + #region --Operations-- + + IEnumerator<(MLImageFeature, long)> IEnumerable<(MLImageFeature feature, long timestamp)>.GetEnumerator () { + // Create reader + NatML.CreateImageFeatureReader(path, out var reader); + if (reader == IntPtr.Zero) + throw new InvalidOperationException(@"Failed to create image reader"); + // Read + var feature = IntPtr.Zero; + try { + for (;;) { + // Read frame + feature.ReleaseFeature(); + feature = IntPtr.Zero; + reader.ReadNextFeature(out var timestamp, out feature); + // EOS + if (timestamp < 0) + break; + // Skip + if (feature == IntPtr.Zero) + continue; + // Create image + var imageFeature = CreateImageFeature(feature); + imageFeature.mean = mean; + imageFeature.std = std; + imageFeature.aspectMode = aspectMode; + yield return (imageFeature, timestamp); + } + } + // Release + finally { + feature.ReleaseFeature(); + reader.ReleaseFeatureReader(); + } + } + + IEnumerator IEnumerable.GetEnumerator () => (this as IEnumerable<(MLImageFeature, long)>).GetEnumerator(); + + private static unsafe MLImageFeature CreateImageFeature (IntPtr feature) { + NatML.FeatureType(feature, out var type); + var shape = new int[4]; + type.FeatureTypeShape(shape, shape.Length); + type.ReleaseFeatureType(); + return new MLImageFeature((void*)feature.FeatureData(), shape[2], shape[1]); + } + #endregion + } +} \ No newline at end of file diff --git a/Runtime/Features/MLVideoFeature.cs.meta b/Runtime/Features/MLVideoFeature.cs.meta new file mode 100644 index 0000000..f06096f --- /dev/null +++ b/Runtime/Features/MLVideoFeature.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 2225546880a774bc09eae262ca87e0de +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/IMLPredictor.cs b/Runtime/IMLPredictor.cs new file mode 100644 index 0000000..72931ef --- /dev/null +++ b/Runtime/IMLPredictor.cs @@ -0,0 +1,23 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML { + + using System; + + /// + /// Lightweight primitive for making predictions with a model. + /// Predictors transform raw model outputs to types that are easily usable by applications. + /// + public interface IMLPredictor : IDisposable { + + /// + /// Make a prediction on one or more input features. + /// + /// Input features. + /// Prediction output. + TOutput Predict (params MLFeature[] inputs); + } +} \ No newline at end of file diff --git a/Runtime/IMLPredictor.cs.meta b/Runtime/IMLPredictor.cs.meta new file mode 100644 index 0000000..34270e6 --- /dev/null +++ b/Runtime/IMLPredictor.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 16c120be8455743f28c19ab1dde40059 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Internal.meta b/Runtime/Internal.meta new file mode 100644 index 0000000..0ac3505 --- /dev/null +++ b/Runtime/Internal.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 5f068e71552df4495a6d3842539049aa +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Internal/NatML.cs b/Runtime/Internal/NatML.cs new file mode 100644 index 0000000..0c413f1 --- /dev/null +++ b/Runtime/Internal/NatML.cs @@ -0,0 +1,257 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Internal { + + using System; + using System.Runtime.InteropServices; + using System.Text; + using Dtype = API.Types.Dtype; + + public static class NatML { + + public const string Version = @"1.1.15"; + public const string Assembly = + #if (UNITY_IOS || UNITY_WEBGL) && !UNITY_EDITOR + @"__Internal"; + #else + @"NatML"; + #endif + + #region --Delegates-- + public delegate void ModelCreationHandler (IntPtr context, IntPtr model); + public delegate void SecretCreationHandler (IntPtr context, IntPtr secret); + #endregion + + + #region --NMLModelConfiguration-- + [DllImport(Assembly, EntryPoint = @"NMLCreateModelConfiguration")] + public static extern void CreateModelConfiguration (out IntPtr configuration); + + [DllImport(Assembly, EntryPoint = @"NMLReleaseModelConfiguration")] + public static extern void ReleaseModelConfiguration (this IntPtr configuration); + + [DllImport(Assembly, EntryPoint = @"NMLModelConfigurationSetComputeTarget")] + public static extern void SetComputeTarget ( + this IntPtr configuration, + MLEdgeModel.ComputeTarget target + ); + + [DllImport(Assembly, EntryPoint = @"NMLModelConfigurationSetComputeDevice")] + public static extern void SetComputeDevice ( + this IntPtr configuration, + IntPtr device + ); + + [DllImport(Assembly, EntryPoint = @"NMLModelConfigurationSetFingerprint")] + public static extern void SetFingerprint ( + this IntPtr configuration, + [MarshalAs(UnmanagedType.LPStr)] string fingerprint + ); + + [DllImport(Assembly, EntryPoint = @"NMLModelConfigurationSetSecret")] + public static extern void SetSecret ( + this IntPtr configuration, + [MarshalAs(UnmanagedType.LPStr)] string secret + ); + + [DllImport(Assembly, EntryPoint = @"NMLModelConfigurationCreateSecret")] + public static extern void CreateSecret ( + SecretCreationHandler handler, + IntPtr context + ); + #endregion + + + #region --NMLModel-- + [DllImport(Assembly, EntryPoint = @"NMLCreateModel")] + public static extern void CreateModel ( + byte[] buffer, + long bufferSize, + IntPtr options, + ModelCreationHandler handler, + IntPtr context + ); + + [DllImport(Assembly, EntryPoint = @"NMLReleaseModel")] + public static extern void ReleaseModel (this IntPtr model); + + [DllImport(Assembly, EntryPoint = @"NMLModelGetMetadataCount")] + public static extern int MetadataCount (this IntPtr model); + + [DllImport(Assembly, EntryPoint = @"NMLModelGetMetadataKey")] + public static extern void MetadataKey ( + this IntPtr model, + int index, + [MarshalAs(UnmanagedType.LPStr)] StringBuilder dest, + int size + ); + + [DllImport(Assembly, EntryPoint = @"NMLModelGetMetadataValue")] + public static extern void MetadataValue ( + this IntPtr model, + [MarshalAs(UnmanagedType.LPStr)] string key, + [MarshalAs(UnmanagedType.LPStr)] StringBuilder dest, + int size + ); + [DllImport(Assembly, EntryPoint = @"NMLModelGetInputFeatureCount")] + public static extern int InputFeatureCount (this IntPtr model); + + [DllImport(Assembly, EntryPoint = @"NMLModelGetInputFeatureType")] + public static extern void InputFeatureType ( + this IntPtr model, + int index, + out IntPtr type + ); + + [DllImport(Assembly, EntryPoint = @"NMLModelGetOutputFeatureCount")] + public static extern int OutputFeatureCount (this IntPtr model); + + [DllImport(Assembly, EntryPoint = @"NMLModelGetOutputFeatureType")] + public static extern void OutputFeatureType ( + this IntPtr model, + int index, + out IntPtr type + ); + + [DllImport(Assembly, EntryPoint = @"NMLModelPredict")] + public static extern void Predict ( + this IntPtr model, + [In] IntPtr[] inputs, + [Out] IntPtr[] outputs + ); + #endregion + + + #region --NMLFeature-- + [DllImport(Assembly, EntryPoint = @"NMLReleaseFeature")] + public static extern void ReleaseFeature (this IntPtr feature); + + [DllImport(Assembly, EntryPoint = @"NMLFeatureGetType")] + public static extern void FeatureType ( + this IntPtr feature, + out IntPtr type + ); + + [DllImport(Assembly, EntryPoint = @"NMLFeatureGetData")] + public static extern IntPtr FeatureData (this IntPtr feature); + + [DllImport(Assembly, EntryPoint = @"NMLCreateArrayFeature")] + public static unsafe extern void CreateFeature ( + void* data, + [In] int[] shape, + int dims, + Dtype type, + int flags, + out IntPtr feature + ); + + [DllImport(Assembly, EntryPoint = @"NMLCreateImageFeature")] + public static unsafe extern void CreateFeature ( + void* pixelBuffer, + int width, + int height, + [In] int[] shape, + Dtype type, + float* mean, + float* std, + int flags, + out IntPtr feature + ); + + [DllImport(Assembly, EntryPoint = @"NMLCreateAudioFeature")] + public static unsafe extern void CreateFeature ( + float* sampleBuffer, + int bufferSampleRate, + [In] int[] bufferShape, + int sampleRate, + int channelCount, + Dtype type, + [In] float[] mean, + [In] float[] std, + int flags, + out IntPtr feature + ); + + [DllImport(Assembly, EntryPoint = @"NMLImageFeatureCopyTo")] + public static unsafe extern void CopyTo ( + void* srcBuffer, + int width, + int height, + int* rect, + float rotation, + byte* background, + void* dstBuffer + ); + #endregion + + + #region --NMLFeatureType-- + [DllImport(Assembly, EntryPoint = @"NMLReleaseFeatureType")] + public static extern void ReleaseFeatureType (this IntPtr type); + + [DllImport(Assembly, EntryPoint = @"NMLFeatureTypeGetName")] + public static extern void FeatureTypeName ( + this IntPtr type, + [MarshalAs(UnmanagedType.LPStr)] StringBuilder dest, + int size + ); + + [DllImport(Assembly, EntryPoint = @"NMLFeatureTypeGetDataType")] + public static extern Dtype FeatureTypeDataType (this IntPtr type); + + [DllImport(Assembly, EntryPoint = @"NMLFeatureTypeGetDimensions")] + public static extern int FeatureTypeDimensions (this IntPtr type); + + [DllImport(Assembly, EntryPoint = @"NMLFeatureTypeGetShape")] + public static extern void FeatureTypeShape ( + this IntPtr type, + [Out] int[] shape, + int length + ); + #endregion + + + #region --NMLFeatureReader-- + [DllImport(Assembly, EntryPoint = @"NMLFeatureReaderGetVideoFormat")] + public static extern void GetVideoFormat ( + [MarshalAs(UnmanagedType.LPStr)] string path, + out int width, + out int height, + out int frames + ); + + [DllImport(Assembly, EntryPoint = @"NMLFeatureReaderGetAudioFormat")] + public static extern void GetAudioFormat ( + [MarshalAs(UnmanagedType.LPStr)] string path, + out int sampleRate, + out int channelCount, + out int sampleCount + ); + + [DllImport(Assembly, EntryPoint = @"NMLReleaseFeatureReader")] + public static extern void ReleaseFeatureReader (this IntPtr reader); + + [DllImport(Assembly, EntryPoint = @"NMLFeatureReaderReadNextFeature")] + public static extern void ReadNextFeature ( + this IntPtr reader, + out long timestamp, + out IntPtr feature + ); + + [DllImport(Assembly, EntryPoint = @"NMLCreateImageFeatureReader")] + public static extern void CreateImageFeatureReader ( + [MarshalAs(UnmanagedType.LPStr)] string path, + out IntPtr reader + ); + + [DllImport(Assembly, EntryPoint = @"NMLCreateAudioFeatureReader")] + public static extern void CreateAudioFeatureReader ( + [MarshalAs(UnmanagedType.LPStr)] string path, + out IntPtr reader + ); + #endregion + } +} \ No newline at end of file diff --git a/Runtime/Internal/NatML.cs.meta b/Runtime/Internal/NatML.cs.meta new file mode 100644 index 0000000..03946d0 --- /dev/null +++ b/Runtime/Internal/NatML.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 3a0fba9ad30b949caa13953a3e41d9d0 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Internal/NatMLInfo.cs b/Runtime/Internal/NatMLInfo.cs new file mode 100644 index 0000000..7d8eef1 --- /dev/null +++ b/Runtime/Internal/NatMLInfo.cs @@ -0,0 +1,23 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +using System.Reflection; +using System.Runtime.CompilerServices; +using static NatML.Internal.NatML; + +// Metadata +[assembly: AssemblyCompany(@"NatML Inc.")] +[assembly: AssemblyTitle(@"NatML.Runtime")] +[assembly: AssemblyVersionAttribute(Version)] +[assembly: AssemblyCopyright(@"Copyright © 2023 NatML Inc. All Rights Reserved.")] + +// Friends +[assembly: InternalsVisibleTo(@"NatML.Editor")] +[assembly: InternalsVisibleTo(@"NatML.Tests")] + +// Remove these ASAP +[assembly: InternalsVisibleTo(@"VideoKit")] +[assembly: InternalsVisibleTo(@"VideoKit.Runtime")] +[assembly: InternalsVisibleTo(@"VideoKit.Editor")] \ No newline at end of file diff --git a/Runtime/Internal/NatMLInfo.cs.meta b/Runtime/Internal/NatMLInfo.cs.meta new file mode 100644 index 0000000..9d49615 --- /dev/null +++ b/Runtime/Internal/NatMLInfo.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 1902a38d001a2413fa6c0beeac6455ef +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/MLAsyncPredictor.cs b/Runtime/MLAsyncPredictor.cs new file mode 100644 index 0000000..718e783 --- /dev/null +++ b/Runtime/MLAsyncPredictor.cs @@ -0,0 +1,103 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML { + + using System; + using System.Collections.Concurrent; + using System.Runtime.CompilerServices; + using System.Threading; + using System.Threading.Tasks; + + /// + /// Asynchronous preditor which runs predictions on a worker thread. + /// This predictor wraps an existing predictor and uses it to make predictions. + /// + public sealed class MLAsyncPredictor : IMLPredictor> { + + #region --Client API-- + /// + /// Backing predictor used by the async predictor. + /// + public readonly IMLPredictor predictor; + + /// + /// Whether the predictor is ready to process new requests immediately. + /// + public bool readyForPrediction { + [MethodImpl(MethodImplOptions.Synchronized)] get; + [MethodImpl(MethodImplOptions.Synchronized)] private set; + } + + /// + /// Make a prediction on one or more input features. + /// + /// Input features. + /// Prediction output. + public Task Predict (params MLFeature[] inputs) { + var tcs = new TaskCompletionSource(); + if (!fence.SafeWaitHandle.IsClosed) { + queue.Enqueue((inputs, tcs)); + fence.Set(); + } + else + tcs.SetCanceled(); + return tcs.Task; + } + + /// + /// Dispose the predictor and release resources. + /// When this is called, all outstanding prediction requests are cancelled. + /// + public void Dispose () { + // Stop worker + cts.Cancel(); + fence.Set(); + task.Wait(); + // Dispose + cts.Dispose(); + fence.Dispose(); + predictor.Dispose(); + } + #endregion + + + #region --Operations-- + private readonly ConcurrentQueue<(MLFeature[] inputs, TaskCompletionSource tcs)> queue; + private readonly AutoResetEvent fence; + private readonly CancellationTokenSource cts; + private readonly Task task; + + internal MLAsyncPredictor (IMLPredictor predictor) { + // Save + this.predictor = predictor; + this.queue = new ConcurrentQueue<(MLFeature[], TaskCompletionSource)>(); + this.fence = new AutoResetEvent(false); + this.cts = new CancellationTokenSource(); + this.task = new Task(() => { + while (!cts.Token.IsCancellationRequested) { + if (queue.TryDequeue(out var request)) + try { + readyForPrediction = false; + var result = predictor.Predict(request.inputs); + request.tcs.SetResult(result); + } catch (Exception ex) { + request.tcs.SetException(ex); + } finally { + readyForPrediction = true; + } + else + fence.WaitOne(); + } + while (queue.TryDequeue(out var request)) + request.tcs.SetCanceled(); + }, cts.Token, TaskCreationOptions.LongRunning); + // Start + task.Start(); + readyForPrediction = true; + } + #endregion + } +} \ No newline at end of file diff --git a/Runtime/MLAsyncPredictor.cs.meta b/Runtime/MLAsyncPredictor.cs.meta new file mode 100644 index 0000000..6e159f6 --- /dev/null +++ b/Runtime/MLAsyncPredictor.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 8da4deb098037493eafe2d06933eeb7f +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/MLEdgeModel.cs b/Runtime/MLEdgeModel.cs new file mode 100644 index 0000000..fcee12b --- /dev/null +++ b/Runtime/MLEdgeModel.cs @@ -0,0 +1,545 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +#nullable enable + +namespace NatML { + + using System; + using System.Collections; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Runtime.InteropServices; + using System.Text; + using System.Threading.Tasks; + using UnityEngine; + using Newtonsoft.Json; + using API; + using API.Types; + using Features; + using Internal; + using Types; + + /// + /// ML model that makes edge (on-device) predictions with a predictor graph. + /// Edge models are NOT thread safe, so predictions MUST be made from one thread at a time. + /// + public sealed class MLEdgeModel : MLModel { + + #region --Enumerations-- + /// + /// Compute target used for model predictions. + /// + [Flags] + public enum ComputeTarget : int { // CHECK // Must match `NatML.h` + /// + /// Use the default compute target for the given platform. + /// + Default = 0, + /// + /// Use the CPU. + /// + CPU = 1 << 0, + /// + /// Use the GPU. + /// + GPU = 1 << 1, + /// + /// Use the neural processing unit. + /// + NPU = 1 << 2, + /// + /// Use all available compute targets including the CPU, GPU, and neural processing units. + /// + All = CPU | GPU | NPU, + } + #endregion + + + #region --Types-- + /// + /// Edge model configuration. + /// + public sealed class Configuration { + + /// + /// Specify the compute target used for model predictions. + /// + public ComputeTarget computeTarget = ComputeTarget.Default; + + /// + /// Specify the compute device used for model predictions. + /// The native type of this pointer is platform-specific. + /// + public IntPtr computeDevice = IntPtr.Zero; + } + #endregion + + + #region --Attributes-- + /// + /// Embed an ML graph from NatML at build time so that the model is immediately available at runtime without downloading. + /// Note that the build size of the application will increase as a result of the embedded model data. + /// + [AttributeUsage(AttributeTargets.Class | AttributeTargets.Struct, AllowMultiple = true)] + public sealed class EmbedAttribute : Attribute { + + #region --Client API-- + /// + /// Embed an ML graph from NatML. + /// + /// Predictor tag. + /// NatML access key. If `null` the project access key will be used. + public EmbedAttribute (string tag, string? accessKey = null) { + this.tag = tag; + this.accessKey = accessKey; + } + #endregion + + + #region --Operations-- + internal readonly string tag; + internal readonly string? accessKey; + #endregion + } + #endregion + + + #region --Client API-- + /// + /// Predictor classification labels. + /// This is `null` if the model does not use classification labels. + /// + public string[]? labels => session.predictor?.labels; + + /// + /// Expected feature normalization for predictions with this model. + /// + public Normalization? normalization => session.predictor?.normalization; + + /// + /// Expected image aspect mode for predictions with this model. + /// + public AspectMode aspectMode => session.predictor?.aspectMode ?? 0; + + /// + /// Expected audio format for predictions with this model. + /// + public AudioFormat? audioFormat => session.predictor?.audioFormat; + + /// + /// Make a prediction on one or more edge features. + /// Input and output features MUST be disposed when no longer needed. + /// + /// Input edge features. + /// Output edge features. + public MLFeatureCollection Predict (params MLEdgeFeature[] inputs) { + Array.Clear(rawInputFeatures, 0, rawInputFeatures.Length); + Array.Clear(rawOutputFeatures, 0, rawOutputFeatures.Length); + for (var i = 0; i < rawInputFeatures.Length; ++i) + rawInputFeatures[i] = inputs[i]; + model.Predict(rawInputFeatures, rawOutputFeatures); + for (var i = 0; i < rawOutputFeatures.Length; ++i) + outputFeatures[i] = new MLEdgeFeature(rawOutputFeatures[i]); + return outputFeatures; + } + + /// + /// Dispose the model and release resources. + /// + public override void Dispose () => model.ReleaseModel(); + + /// + /// Create an edge ML model. + /// + /// Predictor tag or path to model file. + /// Optional model configuration. + /// NatML access key. + public static Task Create ( + string tagOrPath, + Configuration? configuration = null, + string? accessKey = null + ) => Create(tagOrPath, configuration, MLUnityExtensions.CreateClient(accessKey)); + + /// + /// Create an edge ML model. + /// + /// Predictor tag or path to model file. + /// Optional model configuration. + /// NatML API client. + public static async Task Create ( + string tagOrPath, + Configuration? configuration, + NatMLClient client + ) { + // Defines + PredictorSession session; + byte[] graph; + // Handle tag + if (Tag.TryParse(tagOrPath, out var tag)) { + session = SessionFromCache(tag.ToString()) ?? await SessionFromHub(tag, client); + graph = await LoadSessionGraph(session, client); + await CacheSession(session, graph); + } + // Handle file + else { + // Check format + var format = FormatForFile(tagOrPath); + if (format == null) + throw new ArgumentException(@"Model file is not a recognized ML model format", nameof(tagOrPath)); + // Read graph and session + using var stream = new FileStream(tagOrPath, FileMode.Open, FileAccess.Read); + graph = new byte[stream.Length]; + await stream.ReadAsync(graph, 0, graph.Length); + session = new PredictorSession { format = format.Value }; + } + // Return + var model = await Create(session, graph, configuration); + return model; + } + + /// + /// Create an edge ML model. + /// + /// ML model data. + /// Optional model configuration. + public static Task Create ( + MLModelData modelData, + Configuration? configuration = null + ) => Create(modelData.session, modelData.graph, configuration); + #endregion + + + #region --Operations-- + private readonly IntPtr model; + private readonly PredictorSession session; + private readonly IntPtr[] rawInputFeatures; + private readonly IntPtr[] rawOutputFeatures; + private readonly MLEdgeFeature[] outputFeatures; // prevent GC + private static string CachePath = string.Empty; + private static RuntimePlatform Platform = 0; + private static string Device = string.Empty; + private const string Extension = @".nml"; + + private unsafe MLEdgeModel (IntPtr model, PredictorSession session) { + this.model = model; + this.session = session; + // Marshal input types + this.inputs = new MLFeatureType[model.InputFeatureCount()]; + this.rawInputFeatures = new IntPtr[this.inputs.Length]; + for (var i = 0; i < inputs.Length; ++i) { + model.InputFeatureType(i, out var type); + inputs[i] = CreateFeatureType(type); + type.ReleaseFeatureType(); + } + // Marshal output types + this.outputs = new MLFeatureType[model.OutputFeatureCount()]; + this.rawOutputFeatures = new IntPtr[this.outputs.Length]; + this.outputFeatures = new MLEdgeFeature[this.outputs.Length]; + for (var i = 0; i < outputs.Length; ++i) { + model.OutputFeatureType(i, out var type); + outputs[i] = CreateFeatureType(type); + type.ReleaseFeatureType(); + } + // Marshal dictionary + var metadata = new Dictionary(); + var count = model.MetadataCount(); + var metadataBuffer = new StringBuilder(8192); + for (var i = 0; i < count; ++i) { + metadataBuffer.Clear(); + model.MetadataKey(i, metadataBuffer, metadataBuffer.Capacity); + var key = metadataBuffer.ToString(); + metadataBuffer.Clear(); + model.MetadataValue(key, metadataBuffer, metadataBuffer.Capacity); + var value = metadataBuffer.ToString(); + if (!string.IsNullOrEmpty(value)) + metadata.Add(key, value); + } + this.metadata = metadata; + } + + public override string ToString () { + var attribs = new List { GetType().Name }; + for (var i = 0; i < inputs.Length; ++i) + attribs.Add($"Input: {inputs[i]}"); + for (var i = 0; i < outputs.Length; ++i) + attribs.Add($"Output: {outputs[i]}"); + return string.Join(Environment.NewLine, attribs); + } + + private static Task Create (PredictorSession session, byte[] graph, Configuration? config) { + // Check format + if (session.format != FormatForPlatform(Platform)) + throw new InvalidOperationException($"Cannot deserialize {session.format} graph on current platform"); + // Create configuration + NatML.CreateModelConfiguration(out var configuration); + configuration.SetComputeTarget(config?.computeTarget ?? (ComputeTarget)session.flags); + configuration.SetComputeDevice(config?.computeDevice ?? IntPtr.Zero); + configuration.SetFingerprint(session.fingerprint); + configuration.SetSecret(session.secret); + // Create model + var tcs = new TaskCompletionSource(); + var request = new ModelCreationRequest (session, tcs); + var context = (IntPtr)GCHandle.Alloc(request, GCHandleType.Normal); + NatML.CreateModel(graph, graph.Length, configuration, OnCreateModel, context); + configuration.ReleaseModelConfiguration(); + return tcs.Task; + } + + /// + /// Create a predictor session secret. + /// + /// Predictor session secret. + internal static Task CreateSecret () { + var tcs = new TaskCompletionSource(); + var context = (IntPtr)GCHandle.Alloc(tcs, GCHandleType.Normal); + NatML.CreateSecret(OnCreateSecret, context); + return tcs.Task; + } + + /// + /// Clear the predictor cache. + /// This function should only be used in the Unity Editor. + /// + internal static void ClearCache () { + try { + var path = Path.Combine(Application.persistentDataPath, @"natml"); + Directory.Delete(path, true); + } catch { } + } + #endregion + + + #region --Callbacks-- + + [AOT.MonoPInvokeCallback(typeof(NatML.ModelCreationHandler))] + private static void OnCreateModel (IntPtr context, IntPtr model) { + // Get request + var handle = (GCHandle)context; + var request = handle.Target as ModelCreationRequest; + handle.Free(); + // Check + if (request == null) { + model.ReleaseModel(); + return; + } + // Create model + var session = request.session; + var tcs = request.tcs; + if (model != IntPtr.Zero) + tcs.SetResult(new MLEdgeModel(model, session)); + else + tcs.SetException(new ArgumentException(@"Failed to create MLModel from graph data")); + } + + [AOT.MonoPInvokeCallback(typeof(NatML.SecretCreationHandler))] + private static void OnCreateSecret (IntPtr context, IntPtr secret) { + // Get tcs + var handle = (GCHandle)context; + var tcs = handle.Target as TaskCompletionSource; + handle.Free(); + // Check tcs + if (tcs == null) + return; + // Create secret + if (secret != IntPtr.Zero) + tcs.SetResult(Marshal.PtrToStringAuto(secret)); + else + tcs.SetException(new ArgumentException(@"Failed to create predictor session secret")); + } + + [RuntimeInitializeOnLoadMethod(RuntimeInitializeLoadType.AfterAssembliesLoaded)] + private static void OnInitialize () { + CachePath = Path.Combine(Application.persistentDataPath, @"natml"); + Platform = Application.platform; + Device = $"{SystemInfo.deviceModel} {SystemInfo.operatingSystem}"; + } + #endregion + + + #region --Utilitiess-- + /// + /// Load a graph prediction session from the local cache. + /// + /// Predictor tag. + /// Graph prediction session or `null` if there is no cached session for the given predictor tag. + private static PredictorSession? SessionFromCache (string tag) { + // Check + var path = GetSessionCachePath(tag); + if (!File.Exists(path)) + return null; + // Load session + PredictorSession? session = null; + try { + var sessionStr = File.ReadAllText(path); + session = JsonConvert.DeserializeObject(sessionStr); + } catch { + File.Delete(path); + return null; + } + // Check session + if (session == null) + return null; + // Check graph + var graphPath = GetGraphCachePath(session.fingerprint); + if (!File.Exists(graphPath)) { + File.Delete(path); + return null; + } + // Return session + return session; + } + + /// + /// Create a graph prediction session from NatML. + /// + /// Predictor tag. + /// NatML API client. + /// Graph prediction session. + private static async Task SessionFromHub (string tag, NatMLClient client) { + var secret = await CreateSecret(); + var format = FormatForPlatform(Platform); + var session = await client.PredictorSessions.Create(tag, format, secret, Device); + return session; + } + + /// + /// + /// + /// + /// + private static async Task LoadSessionGraph (PredictorSession session, NatMLClient client) { + // Check embed + var embeddedData = NatMLSettings.Instance.embeds.FirstOrDefault(embed => embed.fingerprint == session.fingerprint); + if (embeddedData != null) + return embeddedData.data; + // Check cached graph + var path = GetGraphCachePath(session.fingerprint); + if (File.Exists(path)) { + using var fileStream = new FileStream(path, FileMode.Open, FileAccess.Read); + var graph = new byte[fileStream.Length]; + await fileStream.ReadAsync(graph, 0, graph.Length); + return graph; + } + // Download graph + using var stream = await client.Storage.Download(session.graph); + return stream.ToArray(); + } + + /// + /// Cache a graph prediction session. + /// + /// Graph prediction session. + /// Graph tag. + private static async Task CacheSession (PredictorSession session, byte[] graph) { + // Check if cached + var sessionPath = GetSessionCachePath(session.predictor.tag); + if (File.Exists(sessionPath)) + return; + // Check if cacheable + if (session.predictor.status == PredictorStatus.Draft || Platform == RuntimePlatform.WebGLPlayer) + return; + // Write graph + var graphPath = GetGraphCachePath(session.fingerprint); + Directory.CreateDirectory(CachePath); + using var graphStream = new FileStream(graphPath, FileMode.Create, FileAccess.Write, FileShare.None); + await graphStream.WriteAsync(graph, 0, graph.Length); + // Write session + session.graph = string.Empty; + var sessionStr = JsonConvert.SerializeObject(session); + using var sessionStream = new StreamWriter(sessionPath); + await sessionStream.WriteAsync(sessionStr); + } + + /// + /// Get the cache path for a given graph prediction session. + /// + /// Graph tag. + /// Session cache path. + private static string GetSessionCachePath (string tag) { + var stem = tag.Replace('/', '_'); + var path = Path.Combine(CachePath, $"{stem}{Extension}"); + return path; + } + + /// + /// Get the cache path for a given graph fingerprint. + /// + /// Graph fingerprint. + /// Graph cache path. + private static string GetGraphCachePath (string fingerprint) { + var name = $"{fingerprint}{Extension}"; + var path = Path.Combine(CachePath, name); + return path; + } + + /// + /// Get the graph format for an ML graph file based on its file extension. + /// + /// Path to ML graph. + /// Graph format. + private static GraphFormat? FormatForFile (string path) => Path.GetExtension(path) switch { + ".mlmodel" => GraphFormat.CoreML, + ".onnx" => GraphFormat.ONNX, + ".tflite" => GraphFormat.TFLite, + _ => null, + }; + + /// + /// Get the graph format for a given platform. + /// + /// Runtime platform. + /// Graph format. + private static GraphFormat FormatForPlatform (RuntimePlatform platform) => platform switch { + RuntimePlatform.Android => GraphFormat.TFLite, + RuntimePlatform.IPhonePlayer => GraphFormat.CoreML, + RuntimePlatform.OSXEditor => GraphFormat.CoreML, + RuntimePlatform.OSXPlayer => GraphFormat.CoreML, + RuntimePlatform.WebGLPlayer => GraphFormat.ONNX, + RuntimePlatform.WindowsEditor => GraphFormat.ONNX, + RuntimePlatform.WindowsPlayer => GraphFormat.ONNX, + _ => GraphFormat.ONNX, // this doesn't need to be nullable + }; + + /// + /// Convert a native `NMLFeatureType` to a managed `MLFeatureType`. + /// + /// Native `NMLFeatureType`. + /// Managed feature type. + private static MLFeatureType? CreateFeatureType (in IntPtr type) { + // Get dtype + var dtype = type.FeatureTypeDataType(); + if (dtype == Dtype.Undefined) + return null; + // Get name + var nameBuffer = new StringBuilder(2048); + type.FeatureTypeName(nameBuffer, nameBuffer.Capacity); + var name = nameBuffer.Length > 0 ? nameBuffer.ToString() : null; + // Get shape + var shape = new int[type.FeatureTypeDimensions()]; + type.FeatureTypeShape(shape, shape.Length); + // Return + switch (dtype) { + case Dtype.List: return null; + case Dtype.Dict: return null; + case var _ when shape.Length == 4: return new MLImageType(shape, dtype.ToType(), name); + default: return new MLArrayType(shape, dtype.ToType(), name); + } + } + + private sealed class ModelCreationRequest { + + public readonly PredictorSession session; + public readonly TaskCompletionSource tcs; + + public ModelCreationRequest (PredictorSession session, TaskCompletionSource tcs) { + this.session = session; + this.tcs = tcs; + } + } + #endregion + } +} \ No newline at end of file diff --git a/Runtime/MLEdgeModel.cs.meta b/Runtime/MLEdgeModel.cs.meta new file mode 100644 index 0000000..b2b049d --- /dev/null +++ b/Runtime/MLEdgeModel.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 81b7c2e877a7c4420b70fc03d39f51df +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/MLFeature.cs b/Runtime/MLFeature.cs new file mode 100644 index 0000000..5be79e8 --- /dev/null +++ b/Runtime/MLFeature.cs @@ -0,0 +1,48 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML { + + using System; + using UnityEngine; + using Features; + + /// + /// ML feature. + /// + public abstract class MLFeature { + + #region --Client API-- + /// + /// Feature type. + /// + public readonly MLFeatureType type; + #endregion + + + #region --Operations-- + + protected MLFeature (MLFeatureType type) => this.type = type; + + public static implicit operator MLFeature (float value) => new MLArrayFeature(new [] { value }, new int[0]); + + public static implicit operator MLFeature (int value) => new MLArrayFeature(new [] { value }, new int[0]); + + public static implicit operator MLFeature (bool value) => new MLArrayFeature(new [] { value }, new int[0]); + + public static implicit operator MLFeature (float[] array) => new MLArrayFeature(array, new int[array.Length]); + + public static implicit operator MLFeature (int[] array) => new MLArrayFeature(array, new [] { array.Length }); + + public static implicit operator MLFeature (Texture2D texture) => new MLImageFeature(texture); + + public static implicit operator MLFeature (WebCamTexture texture) => new MLImageFeature(texture.GetPixels32(), texture.width, texture.height); + + public static implicit operator MLFeature (AudioClip clip) => new MLAudioFeature(clip); + + public static implicit operator MLFeature (string text) => new MLStringFeature(text); + #endregion + } +} \ No newline at end of file diff --git a/Runtime/MLFeature.cs.meta b/Runtime/MLFeature.cs.meta new file mode 100644 index 0000000..1f91006 --- /dev/null +++ b/Runtime/MLFeature.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 75ecb07e801a54f51a1de3e951f6c8df +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/MLFeatureCollection.cs b/Runtime/MLFeatureCollection.cs new file mode 100644 index 0000000..f1dbab6 --- /dev/null +++ b/Runtime/MLFeatureCollection.cs @@ -0,0 +1,57 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML { + + using System; + using System.Collections; + using System.Collections.Generic; + + /// + /// Lightweight collection of ML features. + /// Disposing the collection disposes all features within the collection. + /// + public readonly struct MLFeatureCollection : IReadOnlyList, IDisposable where TFeature : IDisposable { + + #region --Client API-- + /// + /// Number of features in the feature collection. + /// + public readonly int Count => features.Length; + + /// + /// Get the feature at the specified index. + /// + public readonly TFeature this [int index] => features[index]; + + /// + /// Create a feature collection. + /// + /// Features to add to the collection. + public MLFeatureCollection (TFeature[] features) => this.features = features; + + /// + /// Dispose all features in this collection. + /// + public readonly void Dispose () { + for (var i = 0; i < features.Length; ++i) + features[i]?.Dispose(); + } + #endregion + + + #region --Operations-- + private readonly TFeature[] features; + + readonly IEnumerator IEnumerable.GetEnumerator () => (features as IEnumerable).GetEnumerator(); + + readonly IEnumerator IEnumerable.GetEnumerator () => features.GetEnumerator(); + + public static implicit operator TFeature[] (MLFeatureCollection collection) => collection.features; + + public static implicit operator MLFeatureCollection (TFeature[] features) => new MLFeatureCollection(features); + #endregion + } +} \ No newline at end of file diff --git a/Runtime/MLFeatureCollection.cs.meta b/Runtime/MLFeatureCollection.cs.meta new file mode 100644 index 0000000..d150ab9 --- /dev/null +++ b/Runtime/MLFeatureCollection.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: f8e7b4db64c7549c78a808f207624ca3 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/MLFeatureType.cs b/Runtime/MLFeatureType.cs new file mode 100644 index 0000000..53bdb7b --- /dev/null +++ b/Runtime/MLFeatureType.cs @@ -0,0 +1,36 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML { + + using System; + + /// + /// ML feature type. + /// + public abstract class MLFeatureType { + + #region --Client API-- + /// + /// Feature name. + /// + public readonly string name; + + /// + /// Feature data type. + /// This will typically be a numeric type. + /// + public readonly Type dataType; + #endregion + + + #region --Operations-- + + protected MLFeatureType (string name, Type type) => (this.name, this.dataType) = (name, type); + + public static implicit operator bool (MLFeatureType type) => type != null; + #endregion + } +} \ No newline at end of file diff --git a/Runtime/MLFeatureType.cs.meta b/Runtime/MLFeatureType.cs.meta new file mode 100644 index 0000000..a9bb492 --- /dev/null +++ b/Runtime/MLFeatureType.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 4706e8197a896405fbd1b635e2906087 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/MLModel.cs b/Runtime/MLModel.cs new file mode 100644 index 0000000..f0413e0 --- /dev/null +++ b/Runtime/MLModel.cs @@ -0,0 +1,38 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML { + + using System; + using System.Collections.Generic; + + /// + /// ML model. + /// + public abstract class MLModel : IDisposable { + + #region --Client API-- + /// + /// Input feature types. + /// + public MLFeatureType[] inputs { get; protected set; } + + /// + /// Output feature types. + /// + public MLFeatureType[] outputs { get; protected set; } + + /// + /// Metadata dictionary. + /// + public IReadOnlyDictionary metadata { get; protected set; } + + /// + /// Dispose the model and release resources. + /// + public virtual void Dispose () { } + #endregion + } +} \ No newline at end of file diff --git a/Runtime/MLModel.cs.meta b/Runtime/MLModel.cs.meta new file mode 100644 index 0000000..07e16f4 --- /dev/null +++ b/Runtime/MLModel.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 9b7669b2421304f429026adf12ef7637 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/MLPredictorExtensions.cs b/Runtime/MLPredictorExtensions.cs new file mode 100644 index 0000000..75e9732 --- /dev/null +++ b/Runtime/MLPredictorExtensions.cs @@ -0,0 +1,25 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML { + + /// + /// Common utilities for working with predictors. + /// + public static class MLPredictorExtensions { + + #region --Client API-- + /// + /// Create an async predictor from a predictor. + /// This typically results in significant performance improvements as predictions are run on a worker thread. + /// + /// Backing predictor to create an async predictor with. + /// Async predictor which runs predictions on a worker thread. + public static MLAsyncPredictor ToAsync (this IMLPredictor predictor) { + return new MLAsyncPredictor(predictor); + } + #endregion + } +} \ No newline at end of file diff --git a/Runtime/MLPredictorExtensions.cs.meta b/Runtime/MLPredictorExtensions.cs.meta new file mode 100644 index 0000000..376f471 --- /dev/null +++ b/Runtime/MLPredictorExtensions.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 64e64248be4404e6eaaf69bf405e047b +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/NatML.Runtime.asmdef b/Runtime/NatML.Runtime.asmdef new file mode 100644 index 0000000..f1d2d14 --- /dev/null +++ b/Runtime/NatML.Runtime.asmdef @@ -0,0 +1,21 @@ +{ + "name": "NatML.Runtime", + "rootNamespace": "", + "references": [], + "includePlatforms": [ + "Android", + "Editor", + "iOS", + "macOSStandalone", + "WebGL", + "WindowsStandalone64" + ], + "excludePlatforms": [], + "allowUnsafeCode": true, + "overrideReferences": false, + "precompiledReferences": [], + "autoReferenced": true, + "defineConstraints": [], + "versionDefines": [], + "noEngineReferences": false +} \ No newline at end of file diff --git a/Runtime/NatML.Runtime.asmdef.meta b/Runtime/NatML.Runtime.asmdef.meta new file mode 100644 index 0000000..d173b1e --- /dev/null +++ b/Runtime/NatML.Runtime.asmdef.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: d23f64cfd3b314bb4a18a8284c99bf5e +AssemblyDefinitionImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Types.meta b/Runtime/Types.meta new file mode 100644 index 0000000..1e18dfc --- /dev/null +++ b/Runtime/Types.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 998f230e4069f4fe88aa15e50af43cde +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Types/MLArrayType.cs b/Runtime/Types/MLArrayType.cs new file mode 100644 index 0000000..862886f --- /dev/null +++ b/Runtime/Types/MLArrayType.cs @@ -0,0 +1,53 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Types { + + using System; + using System.Linq; + + /// + /// ML array feature type. + /// + public class MLArrayType : MLFeatureType { + + #region --Client API-- + /// + /// Array shape. + /// Note that this can be `null` when array features are created without a shape. + /// + public readonly int[] shape; + + /// + /// Array dimensions. + /// + public int dims => shape?.Length ?? 0; + + /// + /// Array element count. + /// + public int elementCount => shape?.Aggregate(1, (a, b) => a * b) ?? 0; + + /// + /// Create an array feature type. + /// + /// Array feature shape. + /// Array element type. + /// Feature name. + public MLArrayType (int[] shape, Type type, string name = null) : base(name, type) => this.shape = shape; + #endregion + + + #region --Operations-- + + public override string ToString () { + var nameStr = name != null ? $"{name}: " : string.Empty; + var shape = this.shape ?? new int[0]; + var shapeStr = $"({string.Join(", ", shape)})"; + return $"{nameStr}{shapeStr} {dataType}"; + } + #endregion + } +} \ No newline at end of file diff --git a/Runtime/Types/MLArrayType.cs.meta b/Runtime/Types/MLArrayType.cs.meta new file mode 100644 index 0000000..9e5f2e1 --- /dev/null +++ b/Runtime/Types/MLArrayType.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 32815caeecade413a84884c877b6801c +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Types/MLAudioType.cs b/Runtime/Types/MLAudioType.cs new file mode 100644 index 0000000..a2af433 --- /dev/null +++ b/Runtime/Types/MLAudioType.cs @@ -0,0 +1,92 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Types { + + using System; + using System.IO; + using System.Threading.Tasks; + using Internal; + + /// + /// ML audio feature type. + /// Audio types always represent floating-point linear PCM data. + /// + public class MLAudioType : MLArrayType { + + #region --Client API-- + /// + /// Audio sample rate. + /// + public virtual int sampleRate { get; protected set; } + + /// + /// Audio channel count. + /// + public virtual int channelCount => shape[2]; + + /// + /// Audio frame count. + /// + public virtual int frames => shape[1]; + + /// + /// Create an audio feature type. + /// + /// Sample rate. + /// Channel count. + /// Total sample count. + /// Feature name. + public MLAudioType ( + int sampleRate, + int channelCount, + int sampleCount, + string name = null + ) : this(new [] { 1, sampleCount / channelCount, channelCount }, name) => this.sampleRate = sampleRate; + + /// + /// Get the audio type for an audio file at a given path. + /// + /// Audio path. + /// Corresponding audio type or `null` if the file is not a valid audio file. + public static MLAudioType FromFile (string path) { + // Check + if (!File.Exists(path)) + throw new ArgumentException(@"Cannot create audio type because file does not exist", nameof(path)); + // Populate + var name = Path.GetFileName(path); + NatML.GetAudioFormat(path, out var sampleRate, out var channelCount, out var sampleCount); + return new MLAudioType(sampleRate, channelCount, sampleCount, name); + } + + /// + /// Get the audio type for an audio file in the `StreamingAssets` folder. + /// + /// Relative path to audio file in `StreamingAssets` folder. + /// Corresponding audio type or `null` if the file is not a valid audio file. + public static async Task FromStreamingAssets (string relativePath) { + var absolutePath = await MLUnityExtensions.StreamingAssetsToAbsolutePath(relativePath); + return FromFile(absolutePath); + } + #endregion + + + #region --Operations-- + /// + /// Create an audio feature type. + /// + /// Audio tensor shape. + /// Feature name. + protected MLAudioType (int[] shape, string name = null) : base(shape, typeof(float), name) { } + + public override string ToString () { + var nameStr = name != null ? $"{name}: " : string.Empty; + var shapeStr = $"({string.Join(", ", shape)})"; + var formatStr = $"{sampleRate}Hz {channelCount}ch"; + return $"{nameStr}{shapeStr} {formatStr} {dataType}"; + } + #endregion + } +} \ No newline at end of file diff --git a/Runtime/Types/MLAudioType.cs.meta b/Runtime/Types/MLAudioType.cs.meta new file mode 100644 index 0000000..bd50f08 --- /dev/null +++ b/Runtime/Types/MLAudioType.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: dbc5a4f7e165640ec9f6c9e864f7975d +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Types/MLImageType.cs b/Runtime/Types/MLImageType.cs new file mode 100644 index 0000000..d3152fd --- /dev/null +++ b/Runtime/Types/MLImageType.cs @@ -0,0 +1,77 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Types { + + using System; + + /// + /// ML image feature type. + /// + public class MLImageType : MLArrayType { + + #region --Client API-- + /// + /// Image width. + /// + public virtual int width => shape?[interleaved ? 2 : 3] ?? 0; + + /// + /// Image height. + /// + public virtual int height => shape?[interleaved ? 1 : 2] ?? 0; + + /// + /// Image channels. + /// + public virtual int channels => shape?[interleaved ? 3 : 1] ?? 0; + + /// + /// Whether the image is interleaved or planar. + /// + public virtual bool interleaved { get; protected set; } + + /// + /// Create an image feature type. + /// This constructor assumes interleaved pixel buffers. + /// + /// Image width. + /// Image height. + /// Image channels. + public MLImageType (int width, int height, int channels) : this(width, height, channels, typeof(byte)) { } + + /// + /// Create an image feature type. + /// This constructor assumes interleaved pixel buffers. + /// + /// Image width. + /// Image height. + /// Image channels. + /// Image data type. + public MLImageType (int width, int height, int channels, Type type) : this(new [] { 1, height, width, channels }, type) { } + + /// + /// Create an image feature type. + /// + /// Image feature shape. + /// Image data type. + /// Feature name. + public MLImageType (int[] shape, Type type, string name = null) : base(shape, type, name) => this.interleaved = shape == null || shape[1] > shape[3]; + + /// + /// Get the corresponding image type for a given feature type. + /// + /// Input type. + /// Corresponding image type or `null` if input type is not an image type. + public static MLImageType FromType (MLFeatureType type) { + switch (type) { + case MLImageType imageType: return imageType; + case MLArrayType arrayType when arrayType.dims == 4: return new MLImageType(arrayType.shape, arrayType.dataType, arrayType.name); + default: return null; + } + } + #endregion + } +} \ No newline at end of file diff --git a/Runtime/Types/MLImageType.cs.meta b/Runtime/Types/MLImageType.cs.meta new file mode 100644 index 0000000..478e185 --- /dev/null +++ b/Runtime/Types/MLImageType.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 672ba00535cee404d8725012738537db +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Types/MLStringType.cs b/Runtime/Types/MLStringType.cs new file mode 100644 index 0000000..f1d4cab --- /dev/null +++ b/Runtime/Types/MLStringType.cs @@ -0,0 +1,27 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Types { + + /// + /// ML string feature type. + /// + public class MLStringType : MLFeatureType { + + #region --Client API-- + /// + /// Text length. + /// + public virtual int length { get; protected set; } + + /// + /// Create a string feature type. + /// + /// String length. + /// Feature name. + public MLStringType (int length, string name = null) : base(name, typeof(string)) => this.length = length; + #endregion + } +} \ No newline at end of file diff --git a/Runtime/Types/MLStringType.cs.meta b/Runtime/Types/MLStringType.cs.meta new file mode 100644 index 0000000..a8c8b13 --- /dev/null +++ b/Runtime/Types/MLStringType.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: e5b058c04d8f34b9daf8e54776ee74ec +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Types/MLVideoType.cs b/Runtime/Types/MLVideoType.cs new file mode 100644 index 0000000..c2314ca --- /dev/null +++ b/Runtime/Types/MLVideoType.cs @@ -0,0 +1,91 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Types { + + using System; + using System.IO; + using System.Threading.Tasks; + using Internal; + + /// + /// ML video feature type. + /// + public class MLVideoType : MLImageType { + + #region --Client API-- + /// + /// Video width. + /// + public override int width => shape?[interleaved ? 3 : 4] ?? 0; + + /// + /// Video height. + /// + public override int height => shape?[interleaved ? 2 : 3] ?? 0; + + /// + /// Video channels. + /// + public override int channels => shape?[interleaved ? 4 : 2] ?? 0; + + /// + /// Video frame count. + /// Note that this is almost always an approximate count. + /// + public virtual int frames => shape?[1] ?? 0; + + /// + /// Create an video feature type. + /// + /// Video width. + /// Video height. + /// Video frame count. + public MLVideoType (int width, int height, int frames) : this(width, height, frames, typeof(byte)) { } + + /// + /// Create an video feature type. + /// + /// Video width. + /// Video height. + /// Video frame count. + /// Video frame data type. + public MLVideoType (int width, int height, int frames, Type type) : this(new [] { 1, frames, height, width, 3 }, type) { } + + /// + /// Create an video feature type. + /// + /// Video feature shape. + /// Video frame data type. + /// Feature name. + public MLVideoType (int[] shape, Type type, string name = null) : base(shape, type, name) => this.interleaved = shape == null || shape[2] > shape[4]; + + /// + /// Get the video type for a video file at a given path. + /// + /// Video path. + /// Corresponding video type or `null` if file is not a valid video file. + public static MLVideoType FromFile (string path) { + // Check + if (!File.Exists(path)) + throw new ArgumentException(@"Cannot create video type because file does not exist", nameof(path)); + // Populate + var name = Path.GetFileName(path); + NatML.GetVideoFormat(path, out var width, out var height, out var frames); + return new MLVideoType(new [] { 1, frames, height, width, 4 }, typeof(byte), name); + } + + /// + /// Get the video type for a video file in the `StreamingAssets` folder. + /// + /// Relative path to video file in `StreamingAssets` folder. + /// Corresponding video type or `null` if the file is not a valid video file. + public static async Task FromStreamingAssets (string relativePath) { + var absolutePath = await MLUnityExtensions.StreamingAssetsToAbsolutePath(relativePath); + return FromFile(absolutePath); + } + #endregion + } +} \ No newline at end of file diff --git a/Runtime/Types/MLVideoType.cs.meta b/Runtime/Types/MLVideoType.cs.meta new file mode 100644 index 0000000..9649b29 --- /dev/null +++ b/Runtime/Types/MLVideoType.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: bdb341dd6cdf14fee90ed9e5112473e3 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: d6df589e0376343f7a3b406329630a74, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Unity.meta b/Runtime/Unity.meta new file mode 100644 index 0000000..58b215d --- /dev/null +++ b/Runtime/Unity.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: a922e8aca9b534b21b8c6e3e7adab38c +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Unity/MLModelData.cs b/Runtime/Unity/MLModelData.cs new file mode 100644 index 0000000..f4c3f97 --- /dev/null +++ b/Runtime/Unity/MLModelData.cs @@ -0,0 +1,21 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML { + + using UnityEngine; + using API.Types; + + /// + /// Self-contained archive with ML model and supplemental data needed to make predictions. + /// + public sealed class MLModelData : ScriptableObject { + + #region --Operations-- + [SerializeField, HideInInspector] internal PredictorSession session; + [SerializeField, HideInInspector] internal byte[] graph; + #endregion + } +} \ No newline at end of file diff --git a/Runtime/Unity/MLModelData.cs.meta b/Runtime/Unity/MLModelData.cs.meta new file mode 100644 index 0000000..6e4840d --- /dev/null +++ b/Runtime/Unity/MLModelData.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 6af78687be00b4f96b81eaed4da5019f +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Unity/MLUnityExtensions.cs b/Runtime/Unity/MLUnityExtensions.cs new file mode 100644 index 0000000..16fcbed --- /dev/null +++ b/Runtime/Unity/MLUnityExtensions.cs @@ -0,0 +1,88 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +#nullable enable + +namespace NatML { + + using System; + using System.IO; + using System.Threading.Tasks; + using UnityEngine; + using UnityEngine.Networking; + using API; + using API.Graph; + using API.Types; + using Internal; + + /// + /// Utilities for working with Unity. + /// + public static class MLUnityExtensions { + + /// + /// Create a NatML client that won't break on WebGL. + /// + /// NatML access key. + /// NatML client. + public static NatMLClient CreateClient (string? accessKey = null, string? url = null) { + url ??= NatMLClient.URL; + accessKey = !string.IsNullOrEmpty(accessKey) ? accessKey : NatMLSettings.Instance.accessKey; + IGraphClient graphClient = Application.platform == RuntimePlatform.WebGLPlayer ? + new UnityGraphClient(url, accessKey) : + new DotNetClient(url, accessKey); + var client = new NatMLClient(graphClient); + return client; + } + + /// + /// Convert a `StreamingAssets` path to an absolute path accessible on the file system. + /// This function will perform any necessary copying to ensure that the file is accessible. + /// + /// Relative path to target file in `StreamingAssets` folder. + /// Absolute path to file or `null` if the file cannot be found. + public static async Task StreamingAssetsToAbsolutePath (string relativePath) { + // Check persistent + var fullPath = Path.Combine(Application.streamingAssetsPath, relativePath); + // Handle other platform + if (Application.platform != RuntimePlatform.Android) + return File.Exists(fullPath) ? fullPath : null; + // Check persistent + var persistentPath = Path.Combine(Application.persistentDataPath, relativePath); + if (File.Exists(persistentPath)) + return persistentPath; + // Create directories + var directory = Path.GetDirectoryName(persistentPath); + Directory.CreateDirectory(directory); + // Download from APK/AAB + using var request = UnityWebRequest.Get(fullPath); + request.SendWebRequest(); + while (!request.isDone) + await Task.Yield(); + if (request.result != UnityWebRequest.Result.Success) + return null; + // Copy + File.WriteAllBytes(persistentPath, request.downloadHandler.data); + // Return + return persistentPath; + } + + /// + /// Deconstruct feature normalization into mean and standard deviation vectors. + /// + public static void Deconstruct (this Normalization? norm, out Vector3 mean, out Vector3 std) { + mean = norm?.mean.Length > 0 ? new Vector3(norm.mean[0], norm.mean[1], norm.mean[2]) : Vector3.zero; + std = norm?.std.Length > 0 ? new Vector3(norm.std[0], norm.std[1], norm.std[2]) : Vector3.one; + } + + /// + /// Deconstruct an audio format into the sample rate and channel count. + /// + public static void Deconstruct (this AudioFormat? format, out int sampleRate, out int channelCount) { + sampleRate = format?.sampleRate ?? 0; + channelCount = format?.channelCount ?? 0; + } + } +} \ No newline at end of file diff --git a/Runtime/Unity/MLUnityExtensions.cs.meta b/Runtime/Unity/MLUnityExtensions.cs.meta new file mode 100644 index 0000000..b539533 --- /dev/null +++ b/Runtime/Unity/MLUnityExtensions.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 253c999ae08e34281b6d516783b7a8ee +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Unity/NatMLSettings.cs b/Runtime/Unity/NatMLSettings.cs new file mode 100644 index 0000000..daa44d2 --- /dev/null +++ b/Runtime/Unity/NatMLSettings.cs @@ -0,0 +1,58 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +namespace NatML.Internal { + + using System; + using System.Collections.Generic; + using UnityEngine; + + /// + /// NatML settings. + /// + [DefaultExecutionOrder(Int32.MinValue)] + internal sealed class NatMLSettings : ScriptableObject { + + #region --Types-- + [Serializable] + internal class Embed { + public string fingerprint; + public byte[] data; + } + #endregion + + + #region --Client API-- + /// + /// Project-wide access key. + /// + [SerializeField, HideInInspector] + internal string accessKey = string.Empty; + + /// + /// Embedded model data. + /// + [SerializeField, HideInInspector] + internal Embed[] embeds = new Embed[0]; + + /// + /// Settings instance for this project. + /// + internal static NatMLSettings Instance; + #endregion + + + #region --Operations-- + + private void OnEnable () { + // Check editor + if (Application.isEditor) + return; + // Set singleton + Instance = this; + } + #endregion + } +} \ No newline at end of file diff --git a/Runtime/Unity/NatMLSettings.cs.meta b/Runtime/Unity/NatMLSettings.cs.meta new file mode 100644 index 0000000..bceb24b --- /dev/null +++ b/Runtime/Unity/NatMLSettings.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: b07b52694eaf1493b8f66120aa91087f +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Runtime/Unity/UnityGraphClient.cs b/Runtime/Unity/UnityGraphClient.cs new file mode 100644 index 0000000..53af1ea --- /dev/null +++ b/Runtime/Unity/UnityGraphClient.cs @@ -0,0 +1,118 @@ +/* +* NatML +* Copyright © 2023 NatML Inc. All rights reserved. +*/ + +#nullable enable + +namespace NatML.API.Graph { + + using System; + using System.Collections.Generic; + using System.IO; + using System.Text; + using System.Threading.Tasks; + using UnityEngine.Networking; + using Newtonsoft.Json; + + /// + /// + public sealed class UnityGraphClient : IGraphClient { + + #region --Client API-- + /// + /// Create the Unity web request client. + /// + /// NatML Graph API URL. + /// NatML access key. + public UnityGraphClient (string url, string accessKey) { + this.url = url; + this.accessKey = accessKey; + } + + /// + /// Query the NatML Graph API. + /// + /// Graph query. + /// Query result key. + /// Query inputs. + public async Task Query (string query, string key, Dictionary? variables = default) { + // Serialize payload + var payload = new GraphRequest { + query = query, + variables = variables + }; + var payloadStr = JsonConvert.SerializeObject(payload); + // Create client + using var client = new UnityWebRequest(url, UnityWebRequest.kHttpVerbPOST) { + uploadHandler = new UploadHandlerRaw(Encoding.UTF8.GetBytes(payloadStr)), + downloadHandler = new DownloadHandlerBuffer(), + disposeDownloadHandlerOnDispose = true, + disposeUploadHandlerOnDispose = true, + }; + client.SetRequestHeader(@"Content-Type", @"application/json"); + // Add auth token + if (!string.IsNullOrEmpty(accessKey)) + client.SetRequestHeader("Authorization", $"Bearer {accessKey}"); + // Post + client.SendWebRequest(); + while (!client.isDone) + await Task.Yield(); + // Parse + var responseStr = client.downloadHandler.text; + var response = JsonConvert.DeserializeObject>(responseStr); + // Check error + if (response.errors != null) + throw new InvalidOperationException(response.errors[0].message); + // Return + return response.data.TryGetValue(key, out var value) ? value : default; + } + + /// + /// Download a file. + /// + /// URL + public async Task Download (string url) { + using var request = UnityWebRequest.Get(url); + request.SendWebRequest(); + while (!request.isDone) + await Task.Yield(); + if (request.result != UnityWebRequest.Result.Success) + throw new InvalidOperationException(request.error); + var data = request.downloadHandler.data; + var stream = new MemoryStream(data, 0, data.Length, false, false); + return stream; + } + + /// + /// Upload a data stream. + /// + /// Data stream. + /// Upload URL. + /// MIME type. + public async Task Upload (MemoryStream stream, string url, string? mime = null) { + // Create client + using var client = new UnityWebRequest(url, UnityWebRequest.kHttpVerbPUT) { + uploadHandler = new UploadHandlerRaw(stream.ToArray()), + downloadHandler = new DownloadHandlerBuffer(), + disposeDownloadHandlerOnDispose = true, + disposeUploadHandlerOnDispose = true, + }; + client.SetRequestHeader(@"Content-Type", mime ?? @"application/octet-stream"); + // Put + client.SendWebRequest(); + while (!client.isDone) + await Task.Yield(); + // Check + if (client.error != null) + throw new InvalidOperationException(@"Failed to upload stream with error: {error}"); + } + #endregion + + + #region --Operations-- + private readonly string url; + private readonly string accessKey; + #endregion + } +} \ No newline at end of file diff --git a/Runtime/Unity/UnityGraphClient.cs.meta b/Runtime/Unity/UnityGraphClient.cs.meta new file mode 100644 index 0000000..4bb7dbd --- /dev/null +++ b/Runtime/Unity/UnityGraphClient.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: c73acfbd098054a2ea741872d4d91959 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/package.json b/package.json new file mode 100644 index 0000000..c24e537 --- /dev/null +++ b/package.json @@ -0,0 +1,38 @@ +{ + "name": "ai.natml.natml", + "version": "1.1.15", + "displayName": "NatML", + "description": "High performance, cross platform machine learning runtime for Unity Engine.", + "unity": "2022.3", + "dependencies": { + "com.unity.nuget.newtonsoft-json": "3.2.1" + }, + "keywords": [ + "natml", + "machine learning", + "ml", + "aritificial intelligence", + "ai", + "computer vision", + "opencv", + "android", + "ios", + "macos", + "windows", + "coreml", + "nnapi", + "directml", + "tensorflow", + "pytorch", + "torch", + "natdevice", + "natcorder" + ], + "author": { + "name": "NatML", + "email": "hi@natml.ai", + "url": "https://github.com/natmlx" + }, + "license": "Apache-2.0", + "repository": "github:natmlx/NatML" +} \ No newline at end of file diff --git a/package.json.meta b/package.json.meta new file mode 100644 index 0000000..4885381 --- /dev/null +++ b/package.json.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 67e26a1344d5649e1a25d21d86fd014d +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: