From e7676245ab7f391b9695ea6cd134efad812cce3e Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 21 May 2024 15:14:54 -0600 Subject: [PATCH 01/10] update resource changes to use either Edit or Create change where relevant. Update db model snapshot test add remote resources support. add method to upload all pending local resources, and call from SyncWithResourceUpload helper method --- .../EntityNotFoundException.cs | 3 + src/SIL.Harmony.Core/IResourceService.cs | 27 ++++ .../DbContextTests.VerifyModel.verified.txt | 34 +++++ .../ResourceTests/RemoteResourcesTests.cs | 137 ++++++++++++++++++ .../ResourceTests/RemoteServiceMock.cs | 45 ++++++ src/SIL.Harmony/Changes/EditChange.cs | 2 +- src/SIL.Harmony/CrdtConfig.cs | 22 ++- src/SIL.Harmony/DataModel.cs | 112 +++++++++++++- src/SIL.Harmony/Db/CrdtRepository.cs | 29 +++- .../Resource/CreateRemoteResourceChange.cs | 20 +++ .../CreateRemoteResourcePendingUpload.cs | 21 +++ src/SIL.Harmony/Resource/LocalResource.cs | 16 ++ src/SIL.Harmony/Resource/RemoteResource.cs | 34 +++++ .../RemoteResourceNotEnabledException.cs | 4 + .../Resource/RemoteResourceUploadedChange.cs | 21 +++ src/SIL.Harmony/SyncHelper.cs | 6 + 16 files changed, 527 insertions(+), 6 deletions(-) create mode 100644 src/SIL.Harmony.Core/EntityNotFoundException.cs create mode 100644 src/SIL.Harmony.Core/IResourceService.cs create mode 100644 src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs create mode 100644 src/SIL.Harmony.Tests/ResourceTests/RemoteServiceMock.cs create mode 100644 src/SIL.Harmony/Resource/CreateRemoteResourceChange.cs create mode 100644 src/SIL.Harmony/Resource/CreateRemoteResourcePendingUpload.cs create mode 100644 src/SIL.Harmony/Resource/LocalResource.cs create mode 100644 src/SIL.Harmony/Resource/RemoteResource.cs create mode 100644 src/SIL.Harmony/Resource/RemoteResourceNotEnabledException.cs create mode 100644 src/SIL.Harmony/Resource/RemoteResourceUploadedChange.cs diff --git a/src/SIL.Harmony.Core/EntityNotFoundException.cs b/src/SIL.Harmony.Core/EntityNotFoundException.cs new file mode 100644 index 0000000..6bcbb0a --- /dev/null +++ b/src/SIL.Harmony.Core/EntityNotFoundException.cs @@ -0,0 +1,3 @@ +namespace SIL.Harmony.Core; + +public class EntityNotFoundException(string message) : Exception(message); diff --git a/src/SIL.Harmony.Core/IResourceService.cs b/src/SIL.Harmony.Core/IResourceService.cs new file mode 100644 index 0000000..d0fad0a --- /dev/null +++ b/src/SIL.Harmony.Core/IResourceService.cs @@ -0,0 +1,27 @@ +namespace SIL.Harmony.Core; + +/// +/// interface to facilitate downloading of resources, typically implemented in application code +/// the remote Id is opaque to the CRDT lib and could be a URL or some other identifier provided by the backend +/// the local path returned for the application code to use as required, it could be a URL if needed also. +/// +public interface IResourceService +{ + /// + /// instructs application code to download a resource from the remote server + /// the service is responsible for downloading the resource and returning the local path + /// + /// ID used to identify the remote resource, could be a URL + /// path defined by the CRDT config where the resource should be stored + /// download result containing the path to the downloaded file, this is stored in the local db and not synced + Task DownloadResource(string remoteId, string localResourceCachePath); + /// + /// upload a resource to the remote server + /// + /// full path to the resource on the local machine + /// an upload result with the remote id, the id will be stored and transmitted to other clients so they can also download the resource + Task UploadResource(string localPath); +} + +public record DownloadResult(string LocalPath); +public record UploadResult(string RemoteId); diff --git a/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt b/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt index de7e1f4..fd04377 100644 --- a/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt +++ b/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt @@ -81,6 +81,40 @@ Relational:TableName: Snapshots Relational:ViewName: Relational:ViewSchema: + EntityType: LocalResource + Properties: + Id (Guid) Required PK AfterSave:Throw ValueGenerated.OnAdd + LocalPath (string) Required + Keys: + Id PK + Annotations: + DiscriminatorProperty: + Relational:FunctionName: + Relational:Schema: + Relational:SqlQuery: + Relational:TableName: LocalResource + Relational:ViewName: + Relational:ViewSchema: + EntityType: RemoteResource + Properties: + Id (Guid) Required PK AfterSave:Throw ValueGenerated.OnAdd + DeletedAt (DateTimeOffset?) + RemoteId (string) + SnapshotId (no field, Guid?) Shadow FK Index + Keys: + Id PK + Foreign keys: + RemoteResource {'SnapshotId'} -> ObjectSnapshot {'Id'} Unique SetNull + Indexes: + SnapshotId Unique + Annotations: + DiscriminatorProperty: + Relational:FunctionName: + Relational:Schema: + Relational:SqlQuery: + Relational:TableName: RemoteResource + Relational:ViewName: + Relational:ViewSchema: EntityType: Definition Properties: Id (Guid) Required PK AfterSave:Throw ValueGenerated.OnAdd diff --git a/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs b/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs new file mode 100644 index 0000000..085bd11 --- /dev/null +++ b/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs @@ -0,0 +1,137 @@ +using System.Runtime.CompilerServices; +using SIL.Harmony.Resource; + +namespace SIL.Harmony.Tests.ResourceTests; + +public class RemoteResourcesTests : DataModelTestBase +{ + private RemoteServiceMock _remoteServiceMock = new(); + + public RemoteResourcesTests() + { + } + + private string CreateFile(string contents, [CallerMemberName] string fileName = "") + { + var filePath = Path.GetFullPath(fileName + ".txt"); + File.WriteAllText(filePath, contents); + return filePath; + } + + private async Task<(Guid resourceId, string remoteId)> SetupRemoteResource(string fileContents) + { + var remoteId = _remoteServiceMock.CreateRemoteResource(fileContents); + var resourceId = Guid.NewGuid(); + await DataModel.AddChange(_localClientId, new CreateRemoteResourceChange(resourceId, remoteId)); + return (resourceId, remoteId); + } + + private async Task<(Guid resourceId, string localPath)> SetupLocalFile(string contents, [CallerMemberName] string fileName = "") + { + var file = CreateFile(contents, fileName); + //because resource service is null the file is not uploaded + var resourceId = await DataModel.AddLocalResource(file, _localClientId, resourceService: null); + return (resourceId, file); + } + + [Fact] + public async Task CreatingAResourceResultsInPendingLocalResources() + { + var (_, file) = await SetupLocalFile("contents"); + + //act + var pending = await DataModel.ListResourcesPendingUpload(); + + + pending.Should().ContainSingle().Which.LocalPath.Should().Be(file); + } + + [Fact] + public async Task ResourcesNotLocalShouldShowUpAsNotDownloaded() + { + var (resourceId, remoteId) = await SetupRemoteResource("test"); + + //act + var pending = await DataModel.ListResourcesPendingDownload(); + + + var remoteResource = pending.Should().ContainSingle().Subject; + remoteResource.RemoteId.Should().Be(remoteId); + remoteResource.Id.Should().Be(resourceId); + } + + [Fact] + public async Task CanUploadFileToRemote() + { + var fileContents = "resource"; + var localFile = CreateFile(fileContents); + + //act + var resourceId = + await DataModel.AddLocalResource(localFile, _localClientId, resourceService: _remoteServiceMock); + + + var resource = await DataModel.GetLatest(resourceId); + ArgumentNullException.ThrowIfNull(resource); + ArgumentNullException.ThrowIfNull(resource.RemoteId); + _remoteServiceMock.ReadFile(resource.RemoteId).Should().Be(fileContents); + var pendingUpload = await DataModel.ListResourcesPendingUpload(); + pendingUpload.Should().BeEmpty(); + } + + [Fact] + public async Task WillUploadMultiplePendingLocalFilesAtOnce() + { + await SetupLocalFile("file1", "file1"); + await SetupLocalFile("file2", "file2"); + + //act + await DataModel.UploadPendingResources(_localClientId, _remoteServiceMock); + + + _remoteServiceMock.ListFiles() + .Select(Path.GetFileName) + .Should() + .Contain(["file1.txt", "file2.txt"]); + } + + [Fact] + public async Task CanDownloadFileFromRemote() + { + var fileContents = "resource"; + var (resourceId, _) = await SetupRemoteResource(fileContents); + + //act + var localResource = await DataModel.DownloadResource(resourceId, _remoteServiceMock); + + + localResource.Id.Should().Be(resourceId); + var actualFileContents = await File.ReadAllTextAsync(localResource.LocalPath); + actualFileContents.Should().Be(fileContents); + var pendingDownloads = await DataModel.ListResourcesPendingDownload(); + pendingDownloads.Should().BeEmpty(); + } + + [Fact] + public async Task CanGetALocalResourceGivenAnId() + { + var file = CreateFile("resource"); + //because resource service is null the file is not uploaded + var resourceId = await DataModel.AddLocalResource(file, _localClientId, resourceService: null); + + //act + var localResource = await DataModel.GetLocalResource(resourceId); + + + localResource.Should().NotBeNull(); + localResource!.LocalPath.Should().Be(file); + } + + [Fact] + public async Task LocalResourceIsNullIfNotDownloaded() + { + var (resourceId, _) = await SetupRemoteResource("test"); + var localResource = await DataModel.GetLocalResource(resourceId); + localResource.Should().BeNull(); + } +} diff --git a/src/SIL.Harmony.Tests/ResourceTests/RemoteServiceMock.cs b/src/SIL.Harmony.Tests/ResourceTests/RemoteServiceMock.cs new file mode 100644 index 0000000..a63476d --- /dev/null +++ b/src/SIL.Harmony.Tests/ResourceTests/RemoteServiceMock.cs @@ -0,0 +1,45 @@ +using SIL.Harmony.Core; + +namespace SIL.Harmony.Tests.ResourceTests; + +public class RemoteServiceMock : IResourceService +{ + public static readonly string RemotePath = Directory.CreateTempSubdirectory("RemoteServiceMock").FullName; + + /// + /// directly creates a remote resource + /// + /// the remote id + public string CreateRemoteResource(string contents) + { + var filePath = Path.Combine(RemotePath, Guid.NewGuid().ToString("N") + ".txt"); + File.WriteAllText(filePath, contents); + return filePath; + } + + public Task DownloadResource(string remoteId, string localResourceCachePath) + { + var fileName = Path.GetFileName(remoteId); + var localPath = Path.Combine(localResourceCachePath, fileName); + Directory.CreateDirectory(localResourceCachePath); + File.Copy(remoteId, localPath); + return Task.FromResult(new DownloadResult(localPath)); + } + + public Task UploadResource(string localPath) + { + var remoteId = Path.Combine(RemotePath, Path.GetFileName(localPath)); + File.Copy(localPath, remoteId); + return Task.FromResult(new UploadResult(remoteId)); + } + + public string ReadFile(string remoteId) + { + return File.ReadAllText(remoteId); + } + + public IEnumerable ListFiles() + { + return Directory.GetFiles(RemotePath); + } +} diff --git a/src/SIL.Harmony/Changes/EditChange.cs b/src/SIL.Harmony/Changes/EditChange.cs index 5d3d70c..cdece9b 100644 --- a/src/SIL.Harmony/Changes/EditChange.cs +++ b/src/SIL.Harmony/Changes/EditChange.cs @@ -1,4 +1,4 @@ -namespace SIL.Harmony.Changes; +namespace SIL.Harmony.Changes; public abstract class EditChange(Guid entityId) : Change(entityId) where T : class diff --git a/src/SIL.Harmony/CrdtConfig.cs b/src/SIL.Harmony/CrdtConfig.cs index 1b9837a..1c24f12 100644 --- a/src/SIL.Harmony/CrdtConfig.cs +++ b/src/SIL.Harmony/CrdtConfig.cs @@ -1,4 +1,4 @@ -using System.Text.Json; +using System.Text.Json; using System.Text.Json.Serialization.Metadata; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Metadata.Builders; @@ -6,6 +6,7 @@ using SIL.Harmony.Changes; using SIL.Harmony.Db; using SIL.Harmony.Entities; +using SIL.Harmony.Resource; namespace SIL.Harmony; @@ -67,6 +68,25 @@ private void JsonTypeModifier(JsonTypeInfo typeInfo) } } } + + public bool RemoteResourcesEnabled { get; private set; } + public string LocalResourceCachePath { get; set; } = Path.GetFullPath("./localResourceCache"); + public void AddRemoteResourceEntity(string? cachePath = null) + { + RemoteResourcesEnabled = true; + LocalResourceCachePath = cachePath ?? LocalResourceCachePath; + ObjectTypeListBuilder.Add(); + ChangeTypeListBuilder.Add(); + ChangeTypeListBuilder.Add(); + ChangeTypeListBuilder.Add(); + ChangeTypeListBuilder.Add>(); + ObjectTypeListBuilder.AddDbModelConfig(builder => + { + var entity = builder.Entity(); + entity.HasKey(lr => lr.Id); + entity.Property(lr => lr.LocalPath); + }); + } } public class ChangeTypeListBuilder diff --git a/src/SIL.Harmony/DataModel.cs b/src/SIL.Harmony/DataModel.cs index a0e0292..c766332 100644 --- a/src/SIL.Harmony/DataModel.cs +++ b/src/SIL.Harmony/DataModel.cs @@ -83,8 +83,7 @@ public async Task AddChanges( private async Task Add(Commit commit, bool deferSnapshotUpdates) { if (await _crdtRepository.HasCommit(commit.Id)) return; - - await using var transaction = await _crdtRepository.BeginTransactionAsync(); + await using var transaction = _crdtRepository.IsInTransaction ? null : await _crdtRepository.BeginTransactionAsync(); await _crdtRepository.AddCommit(commit); if (!deferSnapshotUpdates) { @@ -97,7 +96,7 @@ private async Task Add(Commit commit, bool deferSnapshotUpdates) { _deferredCommits.Add(commit); } - await transaction.CommitAsync(); + if (transaction is not null) await transaction.CommitAsync(); } public async ValueTask DisposeAsync() @@ -256,4 +255,111 @@ public async Task SyncMany(ISyncable[] remotes) { await SyncHelper.SyncMany(this, remotes, _serializerOptions); } + + private void ValidateResourcesSetup() + { + if (!_crdtConfig.Value.RemoteResourcesEnabled) throw new RemoteResourceNotEnabledException(); + } + + public async Task AddLocalResource(string resourcePath, Guid clientId, Guid id = default, IResourceService? resourceService = null) + { + ValidateResourcesSetup(); + var localResource = new LocalResource + { + Id = id == default ? Guid.NewGuid() : id, + LocalPath = Path.GetFullPath(resourcePath) + }; + if (!localResource.FileExists()) throw new FileNotFoundException(localResource.LocalPath); + await using var transaction = await _crdtRepository.BeginTransactionAsync(); + await _crdtRepository.AddLocalResource(localResource); + if (resourceService is not null) + { + var uploadResult = await resourceService.UploadResource(localResource.LocalPath); + await AddChange(clientId, new CreateRemoteResourceChange(localResource.Id, uploadResult.RemoteId)); + } + else + { + await AddChange(clientId, new CreateRemoteResourcePendingUploadChange(localResource.Id)); + } + await transaction.CommitAsync(); + return localResource.Id; + } + + public async Task ListResourcesPendingUpload() + { + ValidateResourcesSetup(); + var remoteResources = await GetLatestObjects().Where(r => r.RemoteId == null).ToArrayAsync(); + var localResource = _crdtRepository.LocalResourcesByIds(remoteResources.Select(r => r.Id)); + return await localResource.ToArrayAsync(); + } + + public async Task UploadPendingResources(Guid clientId, IResourceService resourceService) + { + ValidateResourcesSetup(); + var pendingUploads = await ListResourcesPendingUpload(); + var changes = new List(pendingUploads.Length); + try + { + foreach (var localResource in pendingUploads) + { + var uploadResult = await resourceService.UploadResource(localResource.LocalPath); + changes.Add(new RemoteResourceUploadedChange(localResource.Id, uploadResult.RemoteId)); + } + } + finally + { + //if upload throws at any point we will at least save the changes that did get made. + await AddChanges(clientId, changes); + } + } + + public async Task UploadPendingResource(Guid resourceId, Guid clientId, IResourceService resourceService) + { + var localResource = await _crdtRepository.GetLocalResource(resourceId) ?? + throw new ArgumentException($"unable to find local resource with id {resourceId}"); + ValidateResourcesSetup(); + await UploadPendingResource(localResource, clientId, resourceService); + } + + public async Task UploadPendingResource(LocalResource localResource, Guid clientId, IResourceService resourceService) + { + ValidateResourcesSetup(); + var uploadResult = await resourceService.UploadResource(localResource.LocalPath); + await AddChange(clientId, new RemoteResourceUploadedChange(localResource.Id, uploadResult.RemoteId)); + } + + public async Task ListResourcesPendingDownload() + { + ValidateResourcesSetup(); + var localResourceIds = _crdtRepository.LocalResourceIds(); + var remoteResources = await GetLatestObjects() + .Where(r => r.RemoteId != null && !localResourceIds.Contains(r.Id)) + .ToArrayAsync(); + return remoteResources; + } + + public async Task DownloadResource(Guid resourceId, IResourceService resourceService) + { + ValidateResourcesSetup(); + return await DownloadResource(await GetLatest(resourceId) ?? throw new EntityNotFoundException("Unable to find remote resource"), resourceService); + } + + public async Task DownloadResource(RemoteResource remoteResource, IResourceService resourceService) + { + ValidateResourcesSetup(); + ArgumentNullException.ThrowIfNull(remoteResource.RemoteId); + var downloadResult = await resourceService.DownloadResource(remoteResource.RemoteId, _crdtConfig.Value.LocalResourceCachePath); + var localResource = new LocalResource + { + Id = remoteResource.Id, + LocalPath = downloadResult.LocalPath + }; + await _crdtRepository.AddLocalResource(localResource); + return localResource; + } + + public async Task GetLocalResource(Guid resourceId) + { + return await _crdtRepository.GetLocalResource(resourceId); + } } diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index bb94e70..4089b15 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -1,4 +1,4 @@ -using SIL.Harmony.Core; +using SIL.Harmony.Core; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.ChangeTracking; using Microsoft.EntityFrameworkCore.Storage; @@ -17,6 +17,8 @@ public Task BeginTransactionAsync() return _dbContext.Database.BeginTransactionAsync(); } + public bool IsInTransaction => _dbContext.Database.CurrentTransaction is not null; + public async Task HasCommit(Guid commitId) { @@ -281,4 +283,29 @@ public async Task AddCommits(IEnumerable commits, bool save = true) .Select(c => c.HybridDateTime) .FirstOrDefault(); } + + + public async Task AddLocalResource(LocalResource localResource) + { + _dbContext.Set().Add(localResource); + await _dbContext.SaveChangesAsync(); + } + + public IAsyncEnumerable LocalResourcesByIds(IEnumerable resourceIds) + { + return _dbContext.Set().Where(r => resourceIds.Contains(r.Id)).AsAsyncEnumerable(); + } + + /// + /// primarily for filtering other queries + /// + public IQueryable LocalResourceIds() + { + return _dbContext.Set().Select(r => r.Id); + } + + public async Task GetLocalResource(Guid resourceId) + { + return await _dbContext.Set().FindAsync(resourceId); + } } diff --git a/src/SIL.Harmony/Resource/CreateRemoteResourceChange.cs b/src/SIL.Harmony/Resource/CreateRemoteResourceChange.cs new file mode 100644 index 0000000..41ded57 --- /dev/null +++ b/src/SIL.Harmony/Resource/CreateRemoteResourceChange.cs @@ -0,0 +1,20 @@ +using SIL.Harmony.Core; +using SIL.Harmony.Changes; +using SIL.Harmony.Entities; + +namespace SIL.Harmony.Resource; + +public class CreateRemoteResourceChange(Guid resourceId, string remoteId) : CreateChange(resourceId), IPolyType +{ + public string RemoteId { get; set; } = remoteId; + public override ValueTask NewEntity(Commit commit, ChangeContext context) + { + return ValueTask.FromResult(new RemoteResource + { + Id = EntityId, + RemoteId = RemoteId + }); + } + + public static string TypeName => "create:remote-resource"; +} diff --git a/src/SIL.Harmony/Resource/CreateRemoteResourcePendingUpload.cs b/src/SIL.Harmony/Resource/CreateRemoteResourcePendingUpload.cs new file mode 100644 index 0000000..a31f292 --- /dev/null +++ b/src/SIL.Harmony/Resource/CreateRemoteResourcePendingUpload.cs @@ -0,0 +1,21 @@ +using SIL.Harmony.Changes; +using SIL.Harmony.Entities; + +namespace SIL.Harmony.Resource; + +public class CreateRemoteResourcePendingUploadChange: CreateChange, IPolyType +{ + public CreateRemoteResourcePendingUploadChange(Guid resourceId) : base(resourceId) + { + } + + public override ValueTask NewEntity(Commit commit, ChangeContext context) + { + return ValueTask.FromResult(new RemoteResource + { + Id = EntityId + }); + } + + public static string TypeName => "create:pendingUpload"; +} diff --git a/src/SIL.Harmony/Resource/LocalResource.cs b/src/SIL.Harmony/Resource/LocalResource.cs new file mode 100644 index 0000000..0beaf4c --- /dev/null +++ b/src/SIL.Harmony/Resource/LocalResource.cs @@ -0,0 +1,16 @@ +namespace SIL.Harmony.Resource; + +/// +/// a non CRDT object that tracks local resource files +/// +public class LocalResource +{ + public required Guid Id { get; set; } + //could probably be a URL if working in an electron context, not sure what would be best here. It depends on the app + public required string LocalPath { get; set; } + + public bool FileExists() + { + return File.Exists(LocalPath); + } +} diff --git a/src/SIL.Harmony/Resource/RemoteResource.cs b/src/SIL.Harmony/Resource/RemoteResource.cs new file mode 100644 index 0000000..fd13bdd --- /dev/null +++ b/src/SIL.Harmony/Resource/RemoteResource.cs @@ -0,0 +1,34 @@ +using SIL.Harmony.Entities; + +namespace SIL.Harmony.Resource; + +/// +/// represents a remote binary resource (e.g. image, video, audio, etc.) +/// +public class RemoteResource: IObjectBase +{ + public Guid Id { get; init; } + public DateTimeOffset? DeletedAt { get; set; } + /// + /// will be null when the resource has not been uploaded yet + /// + public string? RemoteId { get; set; } + public Guid[] GetReferences() + { + return []; + } + + public void RemoveReference(Guid id, Commit commit) + { + } + + public IObjectBase Copy() + { + return new RemoteResource + { + Id = Id, + RemoteId = RemoteId, + DeletedAt = DeletedAt + }; + } +} diff --git a/src/SIL.Harmony/Resource/RemoteResourceNotEnabledException.cs b/src/SIL.Harmony/Resource/RemoteResourceNotEnabledException.cs new file mode 100644 index 0000000..ee7911d --- /dev/null +++ b/src/SIL.Harmony/Resource/RemoteResourceNotEnabledException.cs @@ -0,0 +1,4 @@ +namespace SIL.Harmony.Resource; + +public class RemoteResourceNotEnabledException() + : Exception("remote recources were not enabled, to enable them call CrdtConfig.AddRemoteResourceEntity when adding the CRDT library"); diff --git a/src/SIL.Harmony/Resource/RemoteResourceUploadedChange.cs b/src/SIL.Harmony/Resource/RemoteResourceUploadedChange.cs new file mode 100644 index 0000000..bdff98b --- /dev/null +++ b/src/SIL.Harmony/Resource/RemoteResourceUploadedChange.cs @@ -0,0 +1,21 @@ +using SIL.Harmony.Changes; +using SIL.Harmony.Entities; + +namespace SIL.Harmony.Resource; + +/// +/// used when a resource is uploaded to the remote server, stores the remote url in the resource entity +/// +/// +/// +public class RemoteResourceUploadedChange(Guid entityId, string remoteId) : EditChange(entityId), IPolyType +{ + public string RemoteId { get; set; } = remoteId; + public static string TypeName => "uploaded:RemoteResource"; + + public override ValueTask ApplyChange(RemoteResource entity, ChangeContext context) + { + entity.RemoteId = RemoteId; + return ValueTask.CompletedTask; + } +} diff --git a/src/SIL.Harmony/SyncHelper.cs b/src/SIL.Harmony/SyncHelper.cs index 4e108fe..017a0aa 100644 --- a/src/SIL.Harmony/SyncHelper.cs +++ b/src/SIL.Harmony/SyncHelper.cs @@ -5,6 +5,12 @@ namespace SIL.Harmony; internal static class SyncHelper { + public static async Task SyncWithResourceUpload(this DataModel localModel, ISyncable remoteModel, IResourceService resourceService, Guid localClientId) + { + await localModel.UploadPendingResources(localClientId, resourceService); + return await localModel.SyncWith(remoteModel); + } + /// /// simple sync example, each ISyncable could be over the wire or in memory /// prefer that remote is over the wire for the best performance, however they could both be remote From 8b1237e2c49f12997fa72541900f0933a74daf7b Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 15 Oct 2024 10:05:33 +0700 Subject: [PATCH 02/10] allow multiple object adapters --- .../Adapters/CustomAdapterProvider.cs | 8 +++- .../Adapters/DefaultAdapterProvider.cs | 8 +++- .../Adapters/IObjectAdapterProvider.cs | 3 +- src/SIL.Harmony/CrdtConfig.cs | 41 +++++++++++++------ 4 files changed, 41 insertions(+), 19 deletions(-) diff --git a/src/SIL.Harmony/Adapters/CustomAdapterProvider.cs b/src/SIL.Harmony/Adapters/CustomAdapterProvider.cs index 53231cc..81a2252 100644 --- a/src/SIL.Harmony/Adapters/CustomAdapterProvider.cs +++ b/src/SIL.Harmony/Adapters/CustomAdapterProvider.cs @@ -12,8 +12,7 @@ public class CustomAdapterProvider : IObjectAd { private readonly ObjectTypeListBuilder _objectTypeListBuilder; private readonly List _objectTypes = new(); - private Dictionary> JsonTypes { get; } = []; - Dictionary> IObjectAdapterProvider.JsonTypes => JsonTypes; + private Dictionary> JsonTypes => _objectTypeListBuilder.JsonTypes; public CustomAdapterProvider(ObjectTypeListBuilder objectTypeListBuilder) { @@ -55,6 +54,11 @@ IObjectBase IObjectAdapterProvider.Adapt(object obj) { return TCustomAdapter.Create((TCommonInterface)obj); } + + public bool CanAdapt(object obj) + { + return obj is TCommonInterface; + } } // it's possible to implement this without a Common interface diff --git a/src/SIL.Harmony/Adapters/DefaultAdapterProvider.cs b/src/SIL.Harmony/Adapters/DefaultAdapterProvider.cs index 53bea0f..b764ba9 100644 --- a/src/SIL.Harmony/Adapters/DefaultAdapterProvider.cs +++ b/src/SIL.Harmony/Adapters/DefaultAdapterProvider.cs @@ -38,6 +38,10 @@ IObjectBase IObjectAdapterProvider.Adapt(object obj) $"Object is of type {obj.GetType().Name} which does not implement {nameof(IObjectBase)}"); } - private Dictionary> JsonTypes { get; } = []; - Dictionary> IObjectAdapterProvider.JsonTypes => JsonTypes; + public bool CanAdapt(object obj) + { + return obj is IObjectBase; + } + + private Dictionary> JsonTypes => objectTypeListBuilder.JsonTypes; } \ No newline at end of file diff --git a/src/SIL.Harmony/Adapters/IObjectAdapterProvider.cs b/src/SIL.Harmony/Adapters/IObjectAdapterProvider.cs index 6504c0a..98b21dd 100644 --- a/src/SIL.Harmony/Adapters/IObjectAdapterProvider.cs +++ b/src/SIL.Harmony/Adapters/IObjectAdapterProvider.cs @@ -11,6 +11,5 @@ internal interface IObjectAdapterProvider { IEnumerable GetRegistrations(); IObjectBase Adapt(object obj); - - Dictionary> JsonTypes { get; } + bool CanAdapt(object obj); } \ No newline at end of file diff --git a/src/SIL.Harmony/CrdtConfig.cs b/src/SIL.Harmony/CrdtConfig.cs index 1c24f12..da54811 100644 --- a/src/SIL.Harmony/CrdtConfig.cs +++ b/src/SIL.Harmony/CrdtConfig.cs @@ -75,12 +75,12 @@ public void AddRemoteResourceEntity(string? cachePath = null) { RemoteResourcesEnabled = true; LocalResourceCachePath = cachePath ?? LocalResourceCachePath; - ObjectTypeListBuilder.Add(); + ObjectTypeListBuilder.DefaultAdapter().Add(); ChangeTypeListBuilder.Add(); ChangeTypeListBuilder.Add(); ChangeTypeListBuilder.Add(); ChangeTypeListBuilder.Add>(); - ObjectTypeListBuilder.AddDbModelConfig(builder => + ObjectTypeListBuilder.ModelConfigurations.Add((builder, config) => { var entity = builder.Entity(); entity.HasKey(lr => lr.Id); @@ -127,8 +127,7 @@ public void Freeze() { if (_frozen) return; _frozen = true; - JsonTypes = AdapterProvider.JsonTypes; - foreach (var registration in AdapterProvider.GetRegistrations()) + foreach (var registration in AdapterProviders.SelectMany(a => a.GetRegistrations())) { ModelConfigurations.Add((builder, config) => { @@ -147,19 +146,18 @@ internal void CheckFrozen() if (_frozen) throw new InvalidOperationException($"{nameof(ObjectTypeListBuilder)} is frozen"); } - internal Dictionary>? JsonTypes { get; set; } + internal Dictionary> JsonTypes { get; } = []; internal List> ModelConfigurations { get; } = []; - internal IObjectAdapterProvider AdapterProvider => _adapterProvider ?? throw new InvalidOperationException("No adapter has been added to the builder"); - private IObjectAdapterProvider? _adapterProvider; + internal List AdapterProviders { get; } = []; public DefaultAdapterProvider DefaultAdapter() { CheckFrozen(); - if (_adapterProvider is not null) throw new InvalidOperationException("adapter has already been added"); - var adapter = new DefaultAdapterProvider(this); - _adapterProvider = adapter; + if (AdapterProviders.OfType().SingleOrDefault() is {} adapter) return adapter; + adapter = new DefaultAdapterProvider(this); + AdapterProviders.Add(adapter); return adapter; } @@ -182,9 +180,26 @@ public CustomAdapterProvider CustomAdapter, IPolyType { CheckFrozen(); - if (_adapterProvider is not null) throw new InvalidOperationException("adapter has already been added"); - var adapter = new CustomAdapterProvider(this); - _adapterProvider = adapter; + if (AdapterProviders.OfType>().SingleOrDefault() is {} adapter) return adapter; + adapter = new CustomAdapterProvider(this); + AdapterProviders.Add(adapter); return adapter; } + + internal IObjectBase Adapt(object obj) + { + if (AdapterProviders is [{ } defaultAdapter]) + { + return defaultAdapter.Adapt(obj); + } + + foreach (var objectAdapterProvider in AdapterProviders) + { + if (objectAdapterProvider.CanAdapt(obj)) + { + return objectAdapterProvider.Adapt(obj); + } + } + throw new ArgumentException($"Unable to adapt object of type {obj.GetType()}"); + } } From 034c3dce8c8274df57278c93d55d493f7656ef45 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 15 Oct 2024 10:05:51 +0700 Subject: [PATCH 03/10] fix some lingering issues after updating --- src/SIL.Harmony.Sample/CrdtSampleKernel.cs | 1 + src/SIL.Harmony/Changes/ChangeContext.cs | 2 +- src/SIL.Harmony/DataModel.cs | 1 + src/SIL.Harmony/Db/CrdtRepository.cs | 1 + src/SIL.Harmony/Resource/CreateRemoteResourceChange.cs | 4 ++-- src/SIL.Harmony/Resource/CreateRemoteResourcePendingUpload.cs | 4 ++-- 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/SIL.Harmony.Sample/CrdtSampleKernel.cs b/src/SIL.Harmony.Sample/CrdtSampleKernel.cs index 7f4c3b6..6830fae 100644 --- a/src/SIL.Harmony.Sample/CrdtSampleKernel.cs +++ b/src/SIL.Harmony.Sample/CrdtSampleKernel.cs @@ -36,6 +36,7 @@ public static IServiceCollection AddCrdtDataSample(this IServiceCollection servi services.AddCrdtData(config => { config.EnableProjectedTables = true; + config.AddRemoteResourceEntity(); config.ChangeTypeListBuilder .Add() .Add() diff --git a/src/SIL.Harmony/Changes/ChangeContext.cs b/src/SIL.Harmony/Changes/ChangeContext.cs index 15c10d7..bdcf911 100644 --- a/src/SIL.Harmony/Changes/ChangeContext.cs +++ b/src/SIL.Harmony/Changes/ChangeContext.cs @@ -25,5 +25,5 @@ internal ChangeContext(Commit commit, SnapshotWorker worker, CrdtConfig crdtConf } public async ValueTask IsObjectDeleted(Guid entityId) => (await GetSnapshot(entityId))?.EntityIsDeleted ?? true; - internal IObjectBase Adapt(object obj) => _crdtConfig.ObjectTypeListBuilder.AdapterProvider.Adapt(obj); + internal IObjectBase Adapt(object obj) => _crdtConfig.ObjectTypeListBuilder.Adapt(obj); } diff --git a/src/SIL.Harmony/DataModel.cs b/src/SIL.Harmony/DataModel.cs index c766332..2b13e0f 100644 --- a/src/SIL.Harmony/DataModel.cs +++ b/src/SIL.Harmony/DataModel.cs @@ -5,6 +5,7 @@ using SIL.Harmony.Changes; using SIL.Harmony.Db; using SIL.Harmony.Entities; +using SIL.Harmony.Resource; namespace SIL.Harmony; diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index 4089b15..20dc305 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -6,6 +6,7 @@ using SIL.Harmony.Changes; using SIL.Harmony.Entities; using SIL.Harmony.Helpers; +using SIL.Harmony.Resource; namespace SIL.Harmony.Db; diff --git a/src/SIL.Harmony/Resource/CreateRemoteResourceChange.cs b/src/SIL.Harmony/Resource/CreateRemoteResourceChange.cs index 41ded57..aa4f821 100644 --- a/src/SIL.Harmony/Resource/CreateRemoteResourceChange.cs +++ b/src/SIL.Harmony/Resource/CreateRemoteResourceChange.cs @@ -7,9 +7,9 @@ namespace SIL.Harmony.Resource; public class CreateRemoteResourceChange(Guid resourceId, string remoteId) : CreateChange(resourceId), IPolyType { public string RemoteId { get; set; } = remoteId; - public override ValueTask NewEntity(Commit commit, ChangeContext context) + public override ValueTask NewEntity(Commit commit, ChangeContext context) { - return ValueTask.FromResult(new RemoteResource + return ValueTask.FromResult(new RemoteResource { Id = EntityId, RemoteId = RemoteId diff --git a/src/SIL.Harmony/Resource/CreateRemoteResourcePendingUpload.cs b/src/SIL.Harmony/Resource/CreateRemoteResourcePendingUpload.cs index a31f292..4859f8d 100644 --- a/src/SIL.Harmony/Resource/CreateRemoteResourcePendingUpload.cs +++ b/src/SIL.Harmony/Resource/CreateRemoteResourcePendingUpload.cs @@ -9,9 +9,9 @@ public CreateRemoteResourcePendingUploadChange(Guid resourceId) : base(resourceI { } - public override ValueTask NewEntity(Commit commit, ChangeContext context) + public override ValueTask NewEntity(Commit commit, ChangeContext context) { - return ValueTask.FromResult(new RemoteResource + return ValueTask.FromResult(new RemoteResource { Id = EntityId }); From 5448bd25ee794be2e3d61ca6aa3c4dbae39bfac2 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 14 Nov 2024 15:53:44 +0700 Subject: [PATCH 04/10] refactor resource methods out of DataModel and into their own service --- ...ceService.cs => IRemoteResourceService.cs} | 2 +- .../ResourceTests/RemoteResourcesTests.cs | 24 ++-- .../ResourceTests/RemoteServiceMock.cs | 2 +- src/SIL.Harmony/CrdtKernel.cs | 6 + src/SIL.Harmony/DataModel.cs | 107 --------------- src/SIL.Harmony/ResourceService.cs | 129 ++++++++++++++++++ src/SIL.Harmony/SyncHelper.cs | 8 +- 7 files changed, 156 insertions(+), 122 deletions(-) rename src/SIL.Harmony.Core/{IResourceService.cs => IRemoteResourceService.cs} (97%) create mode 100644 src/SIL.Harmony/ResourceService.cs diff --git a/src/SIL.Harmony.Core/IResourceService.cs b/src/SIL.Harmony.Core/IRemoteResourceService.cs similarity index 97% rename from src/SIL.Harmony.Core/IResourceService.cs rename to src/SIL.Harmony.Core/IRemoteResourceService.cs index d0fad0a..22ebb06 100644 --- a/src/SIL.Harmony.Core/IResourceService.cs +++ b/src/SIL.Harmony.Core/IRemoteResourceService.cs @@ -5,7 +5,7 @@ /// the remote Id is opaque to the CRDT lib and could be a URL or some other identifier provided by the backend /// the local path returned for the application code to use as required, it could be a URL if needed also. /// -public interface IResourceService +public interface IRemoteResourceService { /// /// instructs application code to download a resource from the remote server diff --git a/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs b/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs index 085bd11..5514baf 100644 --- a/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs +++ b/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs @@ -1,4 +1,5 @@ using System.Runtime.CompilerServices; +using Microsoft.Extensions.DependencyInjection; using SIL.Harmony.Resource; namespace SIL.Harmony.Tests.ResourceTests; @@ -6,6 +7,7 @@ namespace SIL.Harmony.Tests.ResourceTests; public class RemoteResourcesTests : DataModelTestBase { private RemoteServiceMock _remoteServiceMock = new(); + private ResourceService _resourceService => _services.GetRequiredService(); public RemoteResourcesTests() { @@ -30,7 +32,7 @@ private string CreateFile(string contents, [CallerMemberName] string fileName = { var file = CreateFile(contents, fileName); //because resource service is null the file is not uploaded - var resourceId = await DataModel.AddLocalResource(file, _localClientId, resourceService: null); + var resourceId = await _resourceService.AddLocalResource(file, _localClientId, resourceService: null); return (resourceId, file); } @@ -40,7 +42,7 @@ public async Task CreatingAResourceResultsInPendingLocalResources() var (_, file) = await SetupLocalFile("contents"); //act - var pending = await DataModel.ListResourcesPendingUpload(); + var pending = await _resourceService.ListResourcesPendingUpload(); pending.Should().ContainSingle().Which.LocalPath.Should().Be(file); @@ -52,7 +54,7 @@ public async Task ResourcesNotLocalShouldShowUpAsNotDownloaded() var (resourceId, remoteId) = await SetupRemoteResource("test"); //act - var pending = await DataModel.ListResourcesPendingDownload(); + var pending = await _resourceService.ListResourcesPendingDownload(); var remoteResource = pending.Should().ContainSingle().Subject; @@ -68,14 +70,14 @@ public async Task CanUploadFileToRemote() //act var resourceId = - await DataModel.AddLocalResource(localFile, _localClientId, resourceService: _remoteServiceMock); + await _resourceService.AddLocalResource(localFile, _localClientId, resourceService: _remoteServiceMock); var resource = await DataModel.GetLatest(resourceId); ArgumentNullException.ThrowIfNull(resource); ArgumentNullException.ThrowIfNull(resource.RemoteId); _remoteServiceMock.ReadFile(resource.RemoteId).Should().Be(fileContents); - var pendingUpload = await DataModel.ListResourcesPendingUpload(); + var pendingUpload = await _resourceService.ListResourcesPendingUpload(); pendingUpload.Should().BeEmpty(); } @@ -86,7 +88,7 @@ public async Task WillUploadMultiplePendingLocalFilesAtOnce() await SetupLocalFile("file2", "file2"); //act - await DataModel.UploadPendingResources(_localClientId, _remoteServiceMock); + await _resourceService.UploadPendingResources(_localClientId, _remoteServiceMock); _remoteServiceMock.ListFiles() @@ -102,13 +104,13 @@ public async Task CanDownloadFileFromRemote() var (resourceId, _) = await SetupRemoteResource(fileContents); //act - var localResource = await DataModel.DownloadResource(resourceId, _remoteServiceMock); + var localResource = await _resourceService.DownloadResource(resourceId, _remoteServiceMock); localResource.Id.Should().Be(resourceId); var actualFileContents = await File.ReadAllTextAsync(localResource.LocalPath); actualFileContents.Should().Be(fileContents); - var pendingDownloads = await DataModel.ListResourcesPendingDownload(); + var pendingDownloads = await _resourceService.ListResourcesPendingDownload(); pendingDownloads.Should().BeEmpty(); } @@ -117,10 +119,10 @@ public async Task CanGetALocalResourceGivenAnId() { var file = CreateFile("resource"); //because resource service is null the file is not uploaded - var resourceId = await DataModel.AddLocalResource(file, _localClientId, resourceService: null); + var resourceId = await _resourceService.AddLocalResource(file, _localClientId, resourceService: null); //act - var localResource = await DataModel.GetLocalResource(resourceId); + var localResource = await _resourceService.GetLocalResource(resourceId); localResource.Should().NotBeNull(); @@ -131,7 +133,7 @@ public async Task CanGetALocalResourceGivenAnId() public async Task LocalResourceIsNullIfNotDownloaded() { var (resourceId, _) = await SetupRemoteResource("test"); - var localResource = await DataModel.GetLocalResource(resourceId); + var localResource = await _resourceService.GetLocalResource(resourceId); localResource.Should().BeNull(); } } diff --git a/src/SIL.Harmony.Tests/ResourceTests/RemoteServiceMock.cs b/src/SIL.Harmony.Tests/ResourceTests/RemoteServiceMock.cs index a63476d..92f2f03 100644 --- a/src/SIL.Harmony.Tests/ResourceTests/RemoteServiceMock.cs +++ b/src/SIL.Harmony.Tests/ResourceTests/RemoteServiceMock.cs @@ -2,7 +2,7 @@ namespace SIL.Harmony.Tests.ResourceTests; -public class RemoteServiceMock : IResourceService +public class RemoteServiceMock : IRemoteResourceService { public static readonly string RemotePath = Directory.CreateTempSubdirectory("RemoteServiceMock").FullName; diff --git a/src/SIL.Harmony/CrdtKernel.cs b/src/SIL.Harmony/CrdtKernel.cs index 67671be..6f88e23 100644 --- a/src/SIL.Harmony/CrdtKernel.cs +++ b/src/SIL.Harmony/CrdtKernel.cs @@ -26,6 +26,12 @@ public static IServiceCollection AddCrdtData(this IServiceCollection s provider.GetRequiredService(), provider.GetRequiredService>() )); + //must use factory method because ResourceService constructor is internal + services.AddScoped(provider => new ResourceService( + provider.GetRequiredService(), + provider.GetRequiredService>(), + provider.GetRequiredService() + )); return services; } diff --git a/src/SIL.Harmony/DataModel.cs b/src/SIL.Harmony/DataModel.cs index 20a3249..a8beded 100644 --- a/src/SIL.Harmony/DataModel.cs +++ b/src/SIL.Harmony/DataModel.cs @@ -298,111 +298,4 @@ public async Task SyncMany(ISyncable[] remotes) { await SyncHelper.SyncMany(this, remotes, _serializerOptions); } - - private void ValidateResourcesSetup() - { - if (!_crdtConfig.Value.RemoteResourcesEnabled) throw new RemoteResourceNotEnabledException(); - } - - public async Task AddLocalResource(string resourcePath, Guid clientId, Guid id = default, IResourceService? resourceService = null) - { - ValidateResourcesSetup(); - var localResource = new LocalResource - { - Id = id == default ? Guid.NewGuid() : id, - LocalPath = Path.GetFullPath(resourcePath) - }; - if (!localResource.FileExists()) throw new FileNotFoundException(localResource.LocalPath); - await using var transaction = await _crdtRepository.BeginTransactionAsync(); - await _crdtRepository.AddLocalResource(localResource); - if (resourceService is not null) - { - var uploadResult = await resourceService.UploadResource(localResource.LocalPath); - await AddChange(clientId, new CreateRemoteResourceChange(localResource.Id, uploadResult.RemoteId)); - } - else - { - await AddChange(clientId, new CreateRemoteResourcePendingUploadChange(localResource.Id)); - } - await transaction.CommitAsync(); - return localResource.Id; - } - - public async Task ListResourcesPendingUpload() - { - ValidateResourcesSetup(); - var remoteResources = await GetLatestObjects().Where(r => r.RemoteId == null).ToArrayAsync(); - var localResource = _crdtRepository.LocalResourcesByIds(remoteResources.Select(r => r.Id)); - return await localResource.ToArrayAsync(); - } - - public async Task UploadPendingResources(Guid clientId, IResourceService resourceService) - { - ValidateResourcesSetup(); - var pendingUploads = await ListResourcesPendingUpload(); - var changes = new List(pendingUploads.Length); - try - { - foreach (var localResource in pendingUploads) - { - var uploadResult = await resourceService.UploadResource(localResource.LocalPath); - changes.Add(new RemoteResourceUploadedChange(localResource.Id, uploadResult.RemoteId)); - } - } - finally - { - //if upload throws at any point we will at least save the changes that did get made. - await AddChanges(clientId, changes); - } - } - - public async Task UploadPendingResource(Guid resourceId, Guid clientId, IResourceService resourceService) - { - var localResource = await _crdtRepository.GetLocalResource(resourceId) ?? - throw new ArgumentException($"unable to find local resource with id {resourceId}"); - ValidateResourcesSetup(); - await UploadPendingResource(localResource, clientId, resourceService); - } - - public async Task UploadPendingResource(LocalResource localResource, Guid clientId, IResourceService resourceService) - { - ValidateResourcesSetup(); - var uploadResult = await resourceService.UploadResource(localResource.LocalPath); - await AddChange(clientId, new RemoteResourceUploadedChange(localResource.Id, uploadResult.RemoteId)); - } - - public async Task ListResourcesPendingDownload() - { - ValidateResourcesSetup(); - var localResourceIds = _crdtRepository.LocalResourceIds(); - var remoteResources = await GetLatestObjects() - .Where(r => r.RemoteId != null && !localResourceIds.Contains(r.Id)) - .ToArrayAsync(); - return remoteResources; - } - - public async Task DownloadResource(Guid resourceId, IResourceService resourceService) - { - ValidateResourcesSetup(); - return await DownloadResource(await GetLatest(resourceId) ?? throw new EntityNotFoundException("Unable to find remote resource"), resourceService); - } - - public async Task DownloadResource(RemoteResource remoteResource, IResourceService resourceService) - { - ValidateResourcesSetup(); - ArgumentNullException.ThrowIfNull(remoteResource.RemoteId); - var downloadResult = await resourceService.DownloadResource(remoteResource.RemoteId, _crdtConfig.Value.LocalResourceCachePath); - var localResource = new LocalResource - { - Id = remoteResource.Id, - LocalPath = downloadResult.LocalPath - }; - await _crdtRepository.AddLocalResource(localResource); - return localResource; - } - - public async Task GetLocalResource(Guid resourceId) - { - return await _crdtRepository.GetLocalResource(resourceId); - } } diff --git a/src/SIL.Harmony/ResourceService.cs b/src/SIL.Harmony/ResourceService.cs new file mode 100644 index 0000000..d4fea23 --- /dev/null +++ b/src/SIL.Harmony/ResourceService.cs @@ -0,0 +1,129 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Options; +using SIL.Harmony.Changes; +using SIL.Harmony.Core; +using SIL.Harmony.Db; +using SIL.Harmony.Resource; + +namespace SIL.Harmony; + +public class ResourceService +{ + private readonly CrdtRepository _crdtRepository; + private readonly IOptions _crdtConfig; + private readonly DataModel _dataModel; + + internal ResourceService(CrdtRepository crdtRepository, IOptions crdtConfig, DataModel dataModel) + { + _crdtRepository = crdtRepository; + _crdtConfig = crdtConfig; + _dataModel = dataModel; + } + + private void ValidateResourcesSetup() + { + if (!_crdtConfig.Value.RemoteResourcesEnabled) throw new RemoteResourceNotEnabledException(); + } + + public async Task AddLocalResource(string resourcePath, Guid clientId, Guid id = default, IRemoteResourceService? resourceService = null) + { + ValidateResourcesSetup(); + var localResource = new LocalResource + { + Id = id == default ? Guid.NewGuid() : id, + LocalPath = Path.GetFullPath(resourcePath) + }; + if (!localResource.FileExists()) throw new FileNotFoundException(localResource.LocalPath); + await using var transaction = await _crdtRepository.BeginTransactionAsync(); + await _crdtRepository.AddLocalResource(localResource); + if (resourceService is not null) + { + var uploadResult = await resourceService.UploadResource(localResource.LocalPath); + await _dataModel.AddChange(clientId, new CreateRemoteResourceChange(localResource.Id, uploadResult.RemoteId)); + } + else + { + await _dataModel.AddChange(clientId, new CreateRemoteResourcePendingUploadChange(localResource.Id)); + } + await transaction.CommitAsync(); + return localResource.Id; + } + + public async Task ListResourcesPendingUpload() + { + ValidateResourcesSetup(); + var remoteResources = await _dataModel.QueryLatest().Where(r => r.RemoteId == null).ToArrayAsync(); + var localResource = _crdtRepository.LocalResourcesByIds(remoteResources.Select(r => r.Id)); + return await localResource.ToArrayAsync(); + } + + public async Task UploadPendingResources(Guid clientId, IRemoteResourceService remoteResourceService) + { + ValidateResourcesSetup(); + var pendingUploads = await ListResourcesPendingUpload(); + var changes = new List(pendingUploads.Length); + try + { + foreach (var localResource in pendingUploads) + { + var uploadResult = await remoteResourceService.UploadResource(localResource.LocalPath); + changes.Add(new RemoteResourceUploadedChange(localResource.Id, uploadResult.RemoteId)); + } + } + finally + { + //if upload throws at any point we will at least save the changes that did get made. + await _dataModel.AddChanges(clientId, changes); + } + } + + public async Task UploadPendingResource(Guid resourceId, Guid clientId, IRemoteResourceService remoteResourceService) + { + var localResource = await _crdtRepository.GetLocalResource(resourceId) ?? + throw new ArgumentException($"unable to find local resource with id {resourceId}"); + ValidateResourcesSetup(); + await UploadPendingResource(localResource, clientId, remoteResourceService); + } + + public async Task UploadPendingResource(LocalResource localResource, Guid clientId, IRemoteResourceService remoteResourceService) + { + ValidateResourcesSetup(); + var uploadResult = await remoteResourceService.UploadResource(localResource.LocalPath); + await _dataModel.AddChange(clientId, new RemoteResourceUploadedChange(localResource.Id, uploadResult.RemoteId)); + } + + public async Task ListResourcesPendingDownload() + { + ValidateResourcesSetup(); + var localResourceIds = _crdtRepository.LocalResourceIds(); + var remoteResources = await _dataModel.QueryLatest() + .Where(r => r.RemoteId != null && !localResourceIds.Contains(r.Id)) + .ToArrayAsync(); + return remoteResources; + } + + public async Task DownloadResource(Guid resourceId, IRemoteResourceService remoteResourceService) + { + ValidateResourcesSetup(); + return await DownloadResource(await _dataModel.GetLatest(resourceId) ?? throw new EntityNotFoundException("Unable to find remote resource"), remoteResourceService); + } + + public async Task DownloadResource(RemoteResource remoteResource, IRemoteResourceService remoteResourceService) + { + ValidateResourcesSetup(); + ArgumentNullException.ThrowIfNull(remoteResource.RemoteId); + var downloadResult = await remoteResourceService.DownloadResource(remoteResource.RemoteId, _crdtConfig.Value.LocalResourceCachePath); + var localResource = new LocalResource + { + Id = remoteResource.Id, + LocalPath = downloadResult.LocalPath + }; + await _crdtRepository.AddLocalResource(localResource); + return localResource; + } + + public async Task GetLocalResource(Guid resourceId) + { + return await _crdtRepository.GetLocalResource(resourceId); + } +} \ No newline at end of file diff --git a/src/SIL.Harmony/SyncHelper.cs b/src/SIL.Harmony/SyncHelper.cs index 017a0aa..9ba44a6 100644 --- a/src/SIL.Harmony/SyncHelper.cs +++ b/src/SIL.Harmony/SyncHelper.cs @@ -5,9 +5,13 @@ namespace SIL.Harmony; internal static class SyncHelper { - public static async Task SyncWithResourceUpload(this DataModel localModel, ISyncable remoteModel, IResourceService resourceService, Guid localClientId) + public static async Task SyncWithResourceUpload(this DataModel localModel, + ISyncable remoteModel, + ResourceService resourceService, + IRemoteResourceService remoteResourceService, + Guid localClientId) { - await localModel.UploadPendingResources(localClientId, resourceService); + await resourceService.UploadPendingResource(localClientId, localClientId, remoteResourceService); return await localModel.SyncWith(remoteModel); } From 41abcca2e050e3ba863e387e83266d731deea67c Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 14 Nov 2024 15:54:01 +0700 Subject: [PATCH 05/10] add resource example in the sample project --- .../Changes/AddWordImageChange.cs | 15 +++++++ src/SIL.Harmony.Sample/CrdtSampleKernel.cs | 1 + src/SIL.Harmony.Sample/Models/Word.cs | 9 +++- .../ResourceTests/WordResourceTests.cs | 41 +++++++++++++++++++ 4 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 src/SIL.Harmony.Sample/Changes/AddWordImageChange.cs create mode 100644 src/SIL.Harmony.Tests/ResourceTests/WordResourceTests.cs diff --git a/src/SIL.Harmony.Sample/Changes/AddWordImageChange.cs b/src/SIL.Harmony.Sample/Changes/AddWordImageChange.cs new file mode 100644 index 0000000..2e208d1 --- /dev/null +++ b/src/SIL.Harmony.Sample/Changes/AddWordImageChange.cs @@ -0,0 +1,15 @@ +using SIL.Harmony.Changes; +using SIL.Harmony.Entities; +using SIL.Harmony.Sample.Models; + +namespace SIL.Harmony.Sample.Changes; + +public class AddWordImageChange(Guid entityId, Guid imageId) : EditChange(entityId), ISelfNamedType +{ + public Guid ImageId { get; } = imageId; + + public override async ValueTask ApplyChange(Word entity, ChangeContext context) + { + if (!await context.IsObjectDeleted(ImageId)) entity.ImageResourceId = ImageId; + } +} \ No newline at end of file diff --git a/src/SIL.Harmony.Sample/CrdtSampleKernel.cs b/src/SIL.Harmony.Sample/CrdtSampleKernel.cs index abc2bd9..410eb93 100644 --- a/src/SIL.Harmony.Sample/CrdtSampleKernel.cs +++ b/src/SIL.Harmony.Sample/CrdtSampleKernel.cs @@ -45,6 +45,7 @@ public static IServiceCollection AddCrdtDataSample(this IServiceCollection servi .Add() .Add() .Add() + .Add() .Add>() .Add() .Add>() diff --git a/src/SIL.Harmony.Sample/Models/Word.cs b/src/SIL.Harmony.Sample/Models/Word.cs index 143a425..21c2b05 100644 --- a/src/SIL.Harmony.Sample/Models/Word.cs +++ b/src/SIL.Harmony.Sample/Models/Word.cs @@ -10,10 +10,17 @@ public class Word : IObjectBase public Guid Id { get; init; } public DateTimeOffset? DeletedAt { get; set; } public Guid? AntonymId { get; set; } + public Guid? ImageResourceId { get; set; } public Guid[] GetReferences() { - return AntonymId is null ? [] : [AntonymId.Value]; + return Refs().ToArray(); + + IEnumerable Refs() + { + if (AntonymId.HasValue) yield return AntonymId.Value; + if (ImageResourceId.HasValue) yield return ImageResourceId.Value; + } } public void RemoveReference(Guid id, Commit commit) diff --git a/src/SIL.Harmony.Tests/ResourceTests/WordResourceTests.cs b/src/SIL.Harmony.Tests/ResourceTests/WordResourceTests.cs new file mode 100644 index 0000000..67a79bb --- /dev/null +++ b/src/SIL.Harmony.Tests/ResourceTests/WordResourceTests.cs @@ -0,0 +1,41 @@ +using System.Runtime.CompilerServices; +using Microsoft.Extensions.DependencyInjection; +using SIL.Harmony.Sample.Changes; +using SIL.Harmony.Sample.Models; + +namespace SIL.Harmony.Tests.ResourceTests; + +public class WordResourceTests: DataModelTestBase +{ + private RemoteServiceMock _remoteServiceMock = new(); + private ResourceService _resourceService => _services.GetRequiredService(); + private readonly Guid _entity1Id = Guid.NewGuid(); + + private string CreateFile(string contents, [CallerMemberName] string fileName = "") + { + var filePath = Path.GetFullPath(fileName + ".txt"); + File.WriteAllText(filePath, contents); + return filePath; + } + + [Fact] + public async Task CanReferenceAResourceFromAWord() + { + await WriteNextChange(SetWord(_entity1Id, "test-value")); + var imageFile = CreateFile("not image data"); + //set commit date for add local resource + MockTimeProvider.SetNextDateTime(NextDate()); + var resourceId = await _resourceService.AddLocalResource(imageFile, Guid.NewGuid(), resourceService: _remoteServiceMock); + await WriteNextChange(new AddWordImageChange(_entity1Id, resourceId)); + + var word = await DataModel.GetLatest(_entity1Id); + word.Should().NotBeNull(); + word!.ImageResourceId.Should().Be(resourceId); + + + var localResource = await _resourceService.GetLocalResource(word.ImageResourceId!.Value); + localResource.Should().NotBeNull(); + localResource!.LocalPath.Should().Be(imageFile); + (await File.ReadAllTextAsync(localResource.LocalPath)).Should().Be("not image data"); + } +} \ No newline at end of file From a3168c88642dece8ca9b1355a19f7aa685ec8db8 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Fri, 15 Nov 2024 14:14:12 +0700 Subject: [PATCH 06/10] add ImageResourceId to dbcontext model verification --- src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt b/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt index 3b3b03b..a848176 100644 --- a/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt +++ b/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt @@ -167,6 +167,7 @@ Id (Guid) Required PK AfterSave:Throw ValueGenerated.OnAdd AntonymId (Guid?) DeletedAt (DateTimeOffset?) + ImageResourceId (Guid?) Note (string) SnapshotId (no field, Guid?) Shadow FK Index Text (string) Required From 5125d33529cf80d3e33f9fe7f212320a21028291 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Fri, 29 Nov 2024 14:40:31 +0700 Subject: [PATCH 07/10] rename mock method to be more explicit --- src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs | 2 +- src/SIL.Harmony.Tests/ResourceTests/RemoteServiceMock.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs b/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs index 5514baf..c871f29 100644 --- a/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs +++ b/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs @@ -91,7 +91,7 @@ public async Task WillUploadMultiplePendingLocalFilesAtOnce() await _resourceService.UploadPendingResources(_localClientId, _remoteServiceMock); - _remoteServiceMock.ListFiles() + _remoteServiceMock.ListRemoteFiles() .Select(Path.GetFileName) .Should() .Contain(["file1.txt", "file2.txt"]); diff --git a/src/SIL.Harmony.Tests/ResourceTests/RemoteServiceMock.cs b/src/SIL.Harmony.Tests/ResourceTests/RemoteServiceMock.cs index 92f2f03..3a3f03a 100644 --- a/src/SIL.Harmony.Tests/ResourceTests/RemoteServiceMock.cs +++ b/src/SIL.Harmony.Tests/ResourceTests/RemoteServiceMock.cs @@ -38,7 +38,7 @@ public string ReadFile(string remoteId) return File.ReadAllText(remoteId); } - public IEnumerable ListFiles() + public IEnumerable ListRemoteFiles() { return Directory.GetFiles(RemotePath); } From 5ed1756357d4894185f9cee9f2d08cd1b9f66b66 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Fri, 29 Nov 2024 14:57:03 +0700 Subject: [PATCH 08/10] expose api to list all resources --- src/SIL.Harmony/Db/CrdtRepository.cs | 5 ++++ src/SIL.Harmony/Helpers/LinqHelpers.cs | 31 ++++++++++++++++++++++++ src/SIL.Harmony/Resource/CrdtResource.cs | 10 ++++++++ src/SIL.Harmony/ResourceService.cs | 17 +++++++++++++ 4 files changed, 63 insertions(+) create mode 100644 src/SIL.Harmony/Helpers/LinqHelpers.cs create mode 100644 src/SIL.Harmony/Resource/CrdtResource.cs diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index c9ae398..d6b6df3 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -310,6 +310,10 @@ public IAsyncEnumerable LocalResourcesByIds(IEnumerable res { return _dbContext.Set().Where(r => resourceIds.Contains(r.Id)).AsAsyncEnumerable(); } + public IAsyncEnumerable LocalResources() + { + return _dbContext.Set().AsAsyncEnumerable(); + } /// /// primarily for filtering other queries @@ -323,6 +327,7 @@ public IQueryable LocalResourceIds() { return await _dbContext.Set().FindAsync(resourceId); } + } internal class ScopedDbContext(ICrdtDbContext inner, Commit ignoreChangesAfter) : ICrdtDbContext diff --git a/src/SIL.Harmony/Helpers/LinqHelpers.cs b/src/SIL.Harmony/Helpers/LinqHelpers.cs new file mode 100644 index 0000000..c643928 --- /dev/null +++ b/src/SIL.Harmony/Helpers/LinqHelpers.cs @@ -0,0 +1,31 @@ +using System.Diagnostics.CodeAnalysis; + +namespace SIL.Harmony.Helpers; + +public static class LinqHelpers +{ + internal static IEnumerable FullOuterJoin( + this IEnumerable a, + IEnumerable b, + Func selectKeyA, + Func selectKeyB, + Func projection, + TA? defaultA = default(TA), + TB? defaultB = default(TB), + IEqualityComparer? cmp = null) + { + cmp ??= EqualityComparer.Default; + var alookup = a.ToLookup(selectKeyA, cmp); + var blookup = b.ToLookup(selectKeyB, cmp); + + var keys = new HashSet(alookup.Select(p => p.Key), cmp); + keys.UnionWith(blookup.Select(p => p.Key)); + + var join = from key in keys + from xa in alookup[key].DefaultIfEmpty(defaultA) + from xb in blookup[key].DefaultIfEmpty(defaultB) + select projection(xa, xb, key); + + return join; + } +} \ No newline at end of file diff --git a/src/SIL.Harmony/Resource/CrdtResource.cs b/src/SIL.Harmony/Resource/CrdtResource.cs new file mode 100644 index 0000000..9695129 --- /dev/null +++ b/src/SIL.Harmony/Resource/CrdtResource.cs @@ -0,0 +1,10 @@ +namespace SIL.Harmony.Resource; + +public class CrdtResource +{ + public required Guid Id { get; init; } + public string? RemoteId { get; init; } + public string? LocalPath { get; init; } + public bool Local => !string.IsNullOrEmpty(LocalPath); + public bool Remote => !string.IsNullOrEmpty(RemoteId); +} \ No newline at end of file diff --git a/src/SIL.Harmony/ResourceService.cs b/src/SIL.Harmony/ResourceService.cs index d4fea23..8bdc01b 100644 --- a/src/SIL.Harmony/ResourceService.cs +++ b/src/SIL.Harmony/ResourceService.cs @@ -3,6 +3,7 @@ using SIL.Harmony.Changes; using SIL.Harmony.Core; using SIL.Harmony.Db; +using SIL.Harmony.Helpers; using SIL.Harmony.Resource; namespace SIL.Harmony; @@ -126,4 +127,20 @@ public async Task DownloadResource(RemoteResource remoteResource, { return await _crdtRepository.GetLocalResource(resourceId); } + + public async Task AllResources() + { + var remoteResources = await _dataModel.QueryLatest().ToArrayAsync(); + var localResources = await _crdtRepository.LocalResources().ToArrayAsync(); + return remoteResources.FullOuterJoin(localResources, + r => r.Id, + l => l.Id, + (r, l, id) => new CrdtResource + { + Id = id, + RemoteId = r?.RemoteId, + LocalPath = l?.LocalPath + }).ToArray(); + + } } \ No newline at end of file From f8d670ab12ad7f65c32c27b3c2206edf2e7abb85 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Fri, 29 Nov 2024 16:16:54 +0700 Subject: [PATCH 09/10] test AllResources api, change `AddLocalResource` to return a CrdtResource --- .../ResourceTests/RemoteResourcesTests.cs | 39 +++++++++++++++---- .../ResourceTests/WordResourceTests.cs | 6 +-- src/SIL.Harmony/ResourceService.cs | 32 +++++++++++---- 3 files changed, 59 insertions(+), 18 deletions(-) diff --git a/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs b/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs index c871f29..544c5e9 100644 --- a/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs +++ b/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs @@ -32,8 +32,8 @@ private string CreateFile(string contents, [CallerMemberName] string fileName = { var file = CreateFile(contents, fileName); //because resource service is null the file is not uploaded - var resourceId = await _resourceService.AddLocalResource(file, _localClientId, resourceService: null); - return (resourceId, file); + var crdtResource = await _resourceService.AddLocalResource(file, _localClientId, resourceService: null); + return (crdtResource.Id, file); } [Fact] @@ -69,11 +69,10 @@ public async Task CanUploadFileToRemote() var localFile = CreateFile(fileContents); //act - var resourceId = - await _resourceService.AddLocalResource(localFile, _localClientId, resourceService: _remoteServiceMock); + var crdtResource = await _resourceService.AddLocalResource(localFile, _localClientId, resourceService: _remoteServiceMock); - var resource = await DataModel.GetLatest(resourceId); + var resource = await DataModel.GetLatest(crdtResource.Id); ArgumentNullException.ThrowIfNull(resource); ArgumentNullException.ThrowIfNull(resource.RemoteId); _remoteServiceMock.ReadFile(resource.RemoteId).Should().Be(fileContents); @@ -119,10 +118,10 @@ public async Task CanGetALocalResourceGivenAnId() { var file = CreateFile("resource"); //because resource service is null the file is not uploaded - var resourceId = await _resourceService.AddLocalResource(file, _localClientId, resourceService: null); + var crdtResource = await _resourceService.AddLocalResource(file, _localClientId, resourceService: null); //act - var localResource = await _resourceService.GetLocalResource(resourceId); + var localResource = await _resourceService.GetLocalResource(crdtResource.Id); localResource.Should().NotBeNull(); @@ -136,4 +135,30 @@ public async Task LocalResourceIsNullIfNotDownloaded() var localResource = await _resourceService.GetLocalResource(resourceId); localResource.Should().BeNull(); } + + [Fact] + public async Task CanListAllResources() + { + var (localResourceId, localResourcePath) = await SetupLocalFile("localOnly", "localOnly.txt"); + var (remoteResourceId, remoteId) = await SetupRemoteResource("remoteOnly"); + var localAndRemoteResource = await _resourceService.AddLocalResource(CreateFile("localAndRemove"), _localClientId, resourceService: _remoteServiceMock); + + var crdtResources = await _resourceService.AllResources(); + crdtResources.Should().BeEquivalentTo( + [ + new CrdtResource + { + Id = localResourceId, + LocalPath = localResourcePath, + RemoteId = null + }, + new CrdtResource + { + Id = remoteResourceId, + LocalPath = null, + RemoteId = remoteId + }, + localAndRemoteResource + ]); + } } diff --git a/src/SIL.Harmony.Tests/ResourceTests/WordResourceTests.cs b/src/SIL.Harmony.Tests/ResourceTests/WordResourceTests.cs index 67a79bb..e7fc702 100644 --- a/src/SIL.Harmony.Tests/ResourceTests/WordResourceTests.cs +++ b/src/SIL.Harmony.Tests/ResourceTests/WordResourceTests.cs @@ -25,12 +25,12 @@ public async Task CanReferenceAResourceFromAWord() var imageFile = CreateFile("not image data"); //set commit date for add local resource MockTimeProvider.SetNextDateTime(NextDate()); - var resourceId = await _resourceService.AddLocalResource(imageFile, Guid.NewGuid(), resourceService: _remoteServiceMock); - await WriteNextChange(new AddWordImageChange(_entity1Id, resourceId)); + var resource = await _resourceService.AddLocalResource(imageFile, Guid.NewGuid(), resourceService: _remoteServiceMock); + await WriteNextChange(new AddWordImageChange(_entity1Id, resource.Id)); var word = await DataModel.GetLatest(_entity1Id); word.Should().NotBeNull(); - word!.ImageResourceId.Should().Be(resourceId); + word!.ImageResourceId.Should().Be(resource.Id); var localResource = await _resourceService.GetLocalResource(word.ImageResourceId!.Value); diff --git a/src/SIL.Harmony/ResourceService.cs b/src/SIL.Harmony/ResourceService.cs index 8bdc01b..fec818b 100644 --- a/src/SIL.Harmony/ResourceService.cs +++ b/src/SIL.Harmony/ResourceService.cs @@ -26,7 +26,10 @@ private void ValidateResourcesSetup() if (!_crdtConfig.Value.RemoteResourcesEnabled) throw new RemoteResourceNotEnabledException(); } - public async Task AddLocalResource(string resourcePath, Guid clientId, Guid id = default, IRemoteResourceService? resourceService = null) + public async Task AddLocalResource(string resourcePath, + Guid clientId, + Guid id = default, + IRemoteResourceService? resourceService = null) { ValidateResourcesSetup(); var localResource = new LocalResource @@ -41,13 +44,23 @@ public async Task AddLocalResource(string resourcePath, Guid clientId, Gui { var uploadResult = await resourceService.UploadResource(localResource.LocalPath); await _dataModel.AddChange(clientId, new CreateRemoteResourceChange(localResource.Id, uploadResult.RemoteId)); + await transaction.CommitAsync(); + return new CrdtResource + { + Id = localResource.Id, + RemoteId = uploadResult.RemoteId, + LocalPath = localResource.LocalPath + }; } - else - { - await _dataModel.AddChange(clientId, new CreateRemoteResourcePendingUploadChange(localResource.Id)); - } + + await _dataModel.AddChange(clientId, new CreateRemoteResourcePendingUploadChange(localResource.Id)); await transaction.CommitAsync(); - return localResource.Id; + return new CrdtResource + { + Id = localResource.Id, + RemoteId = null, + LocalPath = localResource.LocalPath + }; } public async Task ListResourcesPendingUpload() @@ -106,7 +119,11 @@ public async Task ListResourcesPendingDownload() public async Task DownloadResource(Guid resourceId, IRemoteResourceService remoteResourceService) { ValidateResourcesSetup(); - return await DownloadResource(await _dataModel.GetLatest(resourceId) ?? throw new EntityNotFoundException("Unable to find remote resource"), remoteResourceService); + return await DownloadResource( + await _dataModel.GetLatest(resourceId) ?? + throw new EntityNotFoundException("Unable to find remote resource"), + remoteResourceService + ); } public async Task DownloadResource(RemoteResource remoteResource, IRemoteResourceService remoteResourceService) @@ -141,6 +158,5 @@ public async Task AllResources() RemoteId = r?.RemoteId, LocalPath = l?.LocalPath }).ToArray(); - } } \ No newline at end of file From 4b5369f7e96f6995bd79e6021b0eb96af8894587 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 3 Dec 2024 13:38:42 +0700 Subject: [PATCH 10/10] fix spelling mistake --- src/SIL.Harmony/Resource/RemoteResourceNotEnabledException.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SIL.Harmony/Resource/RemoteResourceNotEnabledException.cs b/src/SIL.Harmony/Resource/RemoteResourceNotEnabledException.cs index ee7911d..aaa37a0 100644 --- a/src/SIL.Harmony/Resource/RemoteResourceNotEnabledException.cs +++ b/src/SIL.Harmony/Resource/RemoteResourceNotEnabledException.cs @@ -1,4 +1,4 @@ namespace SIL.Harmony.Resource; public class RemoteResourceNotEnabledException() - : Exception("remote recources were not enabled, to enable them call CrdtConfig.AddRemoteResourceEntity when adding the CRDT library"); + : Exception("remote resources were not enabled, to enable them call CrdtConfig.AddRemoteResourceEntity when adding the CRDT library");