diff --git a/src/Umbraco.Infrastructure/Migrations/Upgrade/UmbracoPlan.cs b/src/Umbraco.Infrastructure/Migrations/Upgrade/UmbracoPlan.cs index 7d92bb93857d..c70616bfd4c3 100644 --- a/src/Umbraco.Infrastructure/Migrations/Upgrade/UmbracoPlan.cs +++ b/src/Umbraco.Infrastructure/Migrations/Upgrade/UmbracoPlan.cs @@ -105,5 +105,6 @@ protected virtual void DefinePlan() To("{37875E80-5CDD-42FF-A21A-7D4E3E23E0ED}"); To("{42E44F9E-7262-4269-922D-7310CB48E724}"); To("{7B51B4DE-5574-4484-993E-05D12D9ED703}"); + To("{F3D3EF46-1B1F-47DB-B437-7D573EEDEB98}"); } } diff --git a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/ConvertLocalLinks.cs b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/ConvertLocalLinks.cs index 20f5c6cd81e2..b0bb0ed76cb5 100644 --- a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/ConvertLocalLinks.cs +++ b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/ConvertLocalLinks.cs @@ -1,7 +1,9 @@ using System.Collections.Concurrent; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using NPoco; using Umbraco.Cms.Core; +using Umbraco.Cms.Core.DependencyInjection; using Umbraco.Cms.Core.Models; using Umbraco.Cms.Core.Models.Editors; using Umbraco.Cms.Core.Scoping; @@ -26,7 +28,9 @@ public class ConvertLocalLinks : MigrationBase private readonly LocalLinkProcessor _localLinkProcessor; private readonly IMediaTypeService _mediaTypeService; private readonly ICoreScopeProvider _coreScopeProvider; + private readonly LocalLinkMigrationTracker _linkMigrationTracker; + [Obsolete("Use non obsoleted contructor instead")] public ConvertLocalLinks( IMigrationContext context, IUmbracoContextFactory umbracoContextFactory, @@ -37,7 +41,8 @@ public ConvertLocalLinks( IJsonSerializer jsonSerializer, LocalLinkProcessor localLinkProcessor, IMediaTypeService mediaTypeService, - ICoreScopeProvider coreScopeProvider) + ICoreScopeProvider coreScopeProvider, + LocalLinkMigrationTracker linkMigrationTracker) : base(context) { _umbracoContextFactory = umbracoContextFactory; @@ -49,6 +54,33 @@ public ConvertLocalLinks( _localLinkProcessor = localLinkProcessor; _mediaTypeService = mediaTypeService; _coreScopeProvider = coreScopeProvider; + _linkMigrationTracker = linkMigrationTracker; + } + + public ConvertLocalLinks( + IMigrationContext context, + IUmbracoContextFactory umbracoContextFactory, + IContentTypeService contentTypeService, + ILogger logger, + IDataTypeService dataTypeService, + ILanguageService languageService, + IJsonSerializer jsonSerializer, + LocalLinkProcessor localLinkProcessor, + IMediaTypeService mediaTypeService, + ICoreScopeProvider coreScopeProvider) + : this( + context, + umbracoContextFactory, + contentTypeService, + logger, + dataTypeService, + languageService, + jsonSerializer, + localLinkProcessor, + mediaTypeService, + coreScopeProvider, + StaticServiceProvider.Instance.GetRequiredService()) + { } protected override void Migrate() @@ -97,6 +129,8 @@ protected override void Migrate() propertyEditorAlias); } } + + _linkMigrationTracker.MarkFixedMigrationRan(); } private bool ProcessPropertyTypes(IPropertyType[] propertyTypes, IDictionary languagesById) diff --git a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/ConvertLocalLinkComposer.cs b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/ConvertLocalLinkComposer.cs index a34def894cd7..b6a5f73e5929 100644 --- a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/ConvertLocalLinkComposer.cs +++ b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/ConvertLocalLinkComposer.cs @@ -13,5 +13,7 @@ public void Compose(IUmbracoBuilder builder) builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); } } diff --git a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/LocalLinkMigrationTracker.cs b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/LocalLinkMigrationTracker.cs new file mode 100644 index 000000000000..8b71c4e52ebb --- /dev/null +++ b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/LocalLinkMigrationTracker.cs @@ -0,0 +1,8 @@ +namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_15_0_0.LocalLinks; + +public class LocalLinkMigrationTracker +{ + public bool HasFixedMigrationRun { get; private set; } + + public void MarkFixedMigrationRan() => HasFixedMigrationRun = true; +} diff --git a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/LocalLinkProcessor.cs b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/LocalLinkProcessor.cs index a5a921fecc27..f6f8a02ed8e7 100644 --- a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/LocalLinkProcessor.cs +++ b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/LocalLinkProcessor.cs @@ -43,8 +43,8 @@ public string ProcessStringValue(string input) string newTagHref; if (tag.Udi is not null) { - newTagHref = $" type=\"{tag.Udi.EntityType}\" " - + tag.TagHref.Replace(tag.Udi.ToString(), tag.Udi.Guid.ToString()); + newTagHref = tag.TagHref.Replace(tag.Udi.ToString(), tag.Udi.Guid.ToString()) + + $"\" type=\"{tag.Udi.EntityType}"; } else if (tag.IntId is not null) { @@ -55,8 +55,8 @@ public string ProcessStringValue(string input) continue; } - newTagHref = $" type=\"{conversionResult.Value.EntityType}\" " - + tag.TagHref.Replace(tag.IntId.Value.ToString(), conversionResult.Value.Key.ToString()); + newTagHref = tag.TagHref.Replace(tag.IntId.Value.ToString(), conversionResult.Value.Key.ToString()) + + $"\" type=\"{conversionResult.Value.EntityType}"; } else { diff --git a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/LocalLinkProcessorForFaultyLinks.cs b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/LocalLinkProcessorForFaultyLinks.cs new file mode 100644 index 000000000000..fefd4ea01b63 --- /dev/null +++ b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/LocalLinkProcessorForFaultyLinks.cs @@ -0,0 +1,73 @@ +using System.Text.RegularExpressions; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Services; + +namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_15_0_0.LocalLinks; + +[Obsolete("Will be removed in V18")] +public class LocalLinkProcessorForFaultyLinks +{ + private readonly IIdKeyMap _idKeyMap; + private readonly IEnumerable _localLinkProcessors; + private const string LocalLinkLocation = "__LOCALLINKLOCATION__"; + private const string TypeAttributeLocation = "__TYPEATTRIBUTELOCATION__"; + + internal static readonly Regex FaultyHrefPattern = new( + @"href=['""] ?(? type=*?['""][^'""]*?['""] )?(?\/{localLink:[a-fA-F0-9-]+}['""])).*?>", + RegexOptions.IgnoreCase | RegexOptions.Singleline); + + public LocalLinkProcessorForFaultyLinks( + IIdKeyMap idKeyMap, + IEnumerable localLinkProcessors) + { + _idKeyMap = idKeyMap; + _localLinkProcessors = localLinkProcessors; + } + + public IEnumerable GetSupportedPropertyEditorAliases() => + _localLinkProcessors.SelectMany(p => p.PropertyEditorAliases); + + public bool ProcessToEditorValue(object? editorValue) + { + ITypedLocalLinkProcessor? processor = + _localLinkProcessors.FirstOrDefault(p => p.PropertyEditorValueType == editorValue?.GetType()); + + return processor is not null && processor.Process.Invoke(editorValue, ProcessToEditorValue, ProcessStringValue); + } + + public string ProcessStringValue(string input) + { + MatchCollection faultyTags = FaultyHrefPattern.Matches(input); + + foreach (Match fullTag in faultyTags) + { + var newValue = + fullTag.Value.Replace(fullTag.Groups["typeAttribute"].Value, LocalLinkLocation) + .Replace(fullTag.Groups["localLink"].Value, TypeAttributeLocation) + .Replace(LocalLinkLocation, fullTag.Groups["localLink"].Value) + .Replace(TypeAttributeLocation, fullTag.Groups["typeAttribute"].Value); + input = input.Replace(fullTag.Value, newValue); + } + + return input; + } + + private (Guid Key, string EntityType)? CreateIntBasedKeyType(int id) + { + // very old data, best effort replacement + Attempt documentAttempt = _idKeyMap.GetKeyForId(id, UmbracoObjectTypes.Document); + if (documentAttempt.Success) + { + return (Key: documentAttempt.Result, EntityType: UmbracoObjectTypes.Document.ToString()); + } + + Attempt mediaAttempt = _idKeyMap.GetKeyForId(id, UmbracoObjectTypes.Media); + if (mediaAttempt.Success) + { + return (Key: mediaAttempt.Result, EntityType: UmbracoObjectTypes.Media.ToString()); + } + + return null; + } +} diff --git a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_1_0/FixConvertLocalLinks.cs b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_1_0/FixConvertLocalLinks.cs new file mode 100644 index 000000000000..cd9f78763117 --- /dev/null +++ b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_1_0/FixConvertLocalLinks.cs @@ -0,0 +1,285 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; +using NPoco; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Models.Editors; +using Umbraco.Cms.Core.Scoping; +using Umbraco.Cms.Core.Serialization; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Core.Web; +using Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_15_0_0.LocalLinks; +using Umbraco.Cms.Infrastructure.Persistence; +using Umbraco.Cms.Infrastructure.Persistence.Dtos; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_15_1_0; + +public class FixConvertLocalLinks : MigrationBase +{ + private readonly IUmbracoContextFactory _umbracoContextFactory; + private readonly IContentTypeService _contentTypeService; + private readonly ILogger _logger; + private readonly IDataTypeService _dataTypeService; + private readonly ILanguageService _languageService; + private readonly IJsonSerializer _jsonSerializer; + private readonly LocalLinkProcessorForFaultyLinks _localLinkProcessor; + private readonly IMediaTypeService _mediaTypeService; + private readonly ICoreScopeProvider _coreScopeProvider; + private readonly LocalLinkMigrationTracker _localLinkMigrationTracker; + + public FixConvertLocalLinks( + IMigrationContext context, + IUmbracoContextFactory umbracoContextFactory, + IContentTypeService contentTypeService, + ILogger logger, + IDataTypeService dataTypeService, + ILanguageService languageService, + IJsonSerializer jsonSerializer, + LocalLinkProcessorForFaultyLinks localLinkProcessor, + IMediaTypeService mediaTypeService, + ICoreScopeProvider coreScopeProvider, + LocalLinkMigrationTracker localLinkMigrationTracker) + : base(context) + { + _umbracoContextFactory = umbracoContextFactory; + _contentTypeService = contentTypeService; + _logger = logger; + _dataTypeService = dataTypeService; + _languageService = languageService; + _jsonSerializer = jsonSerializer; + _localLinkProcessor = localLinkProcessor; + _mediaTypeService = mediaTypeService; + _coreScopeProvider = coreScopeProvider; + _localLinkMigrationTracker = localLinkMigrationTracker; + } + + protected override void Migrate() + { + // the original migration was fixed, we only run this if + // this migration hits + // and the fixed original migration has not run, so it must have run in the past + if (_localLinkMigrationTracker.HasFixedMigrationRun) + { + return; + } + + IEnumerable propertyEditorAliases = _localLinkProcessor.GetSupportedPropertyEditorAliases(); + + using UmbracoContextReference umbracoContextReference = _umbracoContextFactory.EnsureUmbracoContext(); + var languagesById = _languageService.GetAllAsync().GetAwaiter().GetResult() + .ToDictionary(language => language.Id); + + IEnumerable allContentTypes = _contentTypeService.GetAll(); + IEnumerable contentPropertyTypes = allContentTypes + .SelectMany(ct => ct.PropertyTypes); + + IMediaType[] allMediaTypes = _mediaTypeService.GetAll().ToArray(); + IEnumerable mediaPropertyTypes = allMediaTypes + .SelectMany(ct => ct.PropertyTypes); + + var relevantPropertyEditors = + contentPropertyTypes.Concat(mediaPropertyTypes).DistinctBy(pt => pt.Id) + .Where(pt => propertyEditorAliases.Contains(pt.PropertyEditorAlias)) + .GroupBy(pt => pt.PropertyEditorAlias) + .ToDictionary(group => group.Key, group => group.ToArray()); + + + foreach (var propertyEditorAlias in propertyEditorAliases) + { + if (relevantPropertyEditors.TryGetValue(propertyEditorAlias, out IPropertyType[]? propertyTypes) is false) + { + continue; + } + + _logger.LogInformation( + "Migration starting for all properties of type: {propertyEditorAlias}", + propertyEditorAlias); + if (ProcessPropertyTypes(propertyTypes, languagesById)) + { + _logger.LogInformation( + "Migration succeeded for all properties of type: {propertyEditorAlias}", + propertyEditorAlias); + } + else + { + _logger.LogError( + "Migration failed for one or more properties of type: {propertyEditorAlias}", + propertyEditorAlias); + } + } + } + + private bool ProcessPropertyTypes(IPropertyType[] propertyTypes, IDictionary languagesById) + { + foreach (IPropertyType propertyType in propertyTypes) + { + IDataType dataType = _dataTypeService.GetAsync(propertyType.DataTypeKey).GetAwaiter().GetResult() + ?? throw new InvalidOperationException("The data type could not be fetched."); + + IDataValueEditor valueEditor = dataType.Editor?.GetValueEditor() + ?? throw new InvalidOperationException( + "The data type value editor could not be fetched."); + + Sql sql = Sql() + .Select() + .From() + .InnerJoin() + .On((propertyData, contentVersion) => + propertyData.VersionId == contentVersion.Id) + .LeftJoin() + .On((contentVersion, documentVersion) => + contentVersion.Id == documentVersion.Id) + .Where( + (propertyData, contentVersion, documentVersion) => + (contentVersion.Current == true || documentVersion.Published == true) + && propertyData.PropertyTypeId == propertyType.Id); + + List propertyDataDtos = Database.Fetch(sql); + if (propertyDataDtos.Count < 1) + { + continue; + } + + var updateBatch = propertyDataDtos.Select(propertyDataDto => + UpdateBatch.For(propertyDataDto, Database.StartSnapshot(propertyDataDto))).ToList(); + + var updatesToSkip = new ConcurrentBag>(); + + var progress = 0; + + void HandleUpdateBatch(UpdateBatch update) + { + using UmbracoContextReference umbracoContextReference = _umbracoContextFactory.EnsureUmbracoContext(); + + progress++; + if (progress % 100 == 0) + { + _logger.LogInformation(" - finíshed {progress} of {total} properties", progress, + updateBatch.Count); + } + + PropertyDataDto propertyDataDto = update.Poco; + + if (ProcessPropertyDataDto(propertyDataDto, propertyType, languagesById, valueEditor) == false) + { + updatesToSkip.Add(update); + } + } + + if (DatabaseType == DatabaseType.SQLite) + { + // SQLite locks up if we run the migration in parallel, so... let's not. + foreach (UpdateBatch update in updateBatch) + { + HandleUpdateBatch(update); + } + } + else + { + Parallel.ForEachAsync(updateBatch, async (update, token) => + { + //Foreach here, but we need to suppress the flow before each task, but not the actuall await of the task + Task task; + using (ExecutionContext.SuppressFlow()) + { + task = Task.Run( + () => + { + using ICoreScope scope = _coreScopeProvider.CreateCoreScope(); + scope.Complete(); + HandleUpdateBatch(update); + }, + token); + } + + await task; + }).GetAwaiter().GetResult(); + } + + updateBatch.RemoveAll(updatesToSkip.Contains); + + if (updateBatch.Any() is false) + { + _logger.LogDebug(" - no properties to convert, continuing"); + continue; + } + + _logger.LogInformation(" - {totalConverted} properties converted, saving...", updateBatch.Count); + var result = Database.UpdateBatch(updateBatch, new BatchOptions { BatchSize = 100 }); + if (result != updateBatch.Count) + { + throw new InvalidOperationException( + $"The database batch update was supposed to update {updateBatch.Count} property DTO entries, but it updated {result} entries."); + } + + _logger.LogDebug( + "Migration completed for property type: {propertyTypeName} (id: {propertyTypeId}, alias: {propertyTypeAlias}, editor alias: {propertyTypeEditorAlias}) - {updateCount} property DTO entries updated.", + propertyType.Name, + propertyType.Id, + propertyType.Alias, + propertyType.PropertyEditorAlias, + result); + } + + return true; + } + + private bool ProcessPropertyDataDto(PropertyDataDto propertyDataDto, IPropertyType propertyType, + IDictionary languagesById, IDataValueEditor valueEditor) + { + // NOTE: some old property data DTOs can have variance defined, even if the property type no longer varies + var culture = propertyType.VariesByCulture() + && propertyDataDto.LanguageId.HasValue + && languagesById.TryGetValue(propertyDataDto.LanguageId.Value, out ILanguage? language) + ? language.IsoCode + : null; + + if (culture is null && propertyType.VariesByCulture()) + { + // if we end up here, the property DTO is bound to a language that no longer exists. this is an error scenario, + // and we can't really handle it in any other way than logging; in all likelihood this is an old property version, + // and it won't cause any runtime issues + _logger.LogWarning( + " - property data with id: {propertyDataId} references a language that does not exist - language id: {languageId} (property type: {propertyTypeName}, id: {propertyTypeId}, alias: {propertyTypeAlias})", + propertyDataDto.Id, + propertyDataDto.LanguageId, + propertyType.Name, + propertyType.Id, + propertyType.Alias); + return false; + } + + var segment = propertyType.VariesBySegment() ? propertyDataDto.Segment : null; + var property = new Property(propertyType); + property.SetValue(propertyDataDto.Value, culture, segment); + var toEditorValue = valueEditor.ToEditor(property, culture, segment); + + if (_localLinkProcessor.ProcessToEditorValue(toEditorValue) == false) + { + _logger.LogDebug( + " - skipping as no processor modified the data for property data with id: {propertyDataId} (property type: {propertyTypeName}, id: {propertyTypeId}, alias: {propertyTypeAlias})", + propertyDataDto.Id, + propertyType.Name, + propertyType.Id, + propertyType.Alias); + return false; + } + + var editorValue = _jsonSerializer.Serialize(toEditorValue); + var dbValue = valueEditor.FromEditor(new ContentPropertyData(editorValue, null), null); + if (dbValue is not string stringValue || stringValue.DetectIsJson() is false) + { + _logger.LogWarning( + " - value editor did not yield a valid JSON string as FromEditor value property data with id: {propertyDataId} (property type: {propertyTypeName}, id: {propertyTypeId}, alias: {propertyTypeAlias})", + propertyDataDto.Id, + propertyType.Name, + propertyType.Id, + propertyType.Alias); + return false; + } + + propertyDataDto.TextValue = stringValue; + return true; + } +}