Implemented ContentVersionCleanup scheduled task.

Note: adding ref to Microsoft.NET.Test.Sdk fixes AutoFixture AutoDataAttribute (and sub classes)
This commit is contained in:
Paul Johnson
2021-10-18 21:56:18 +01:00
parent a1ac730633
commit bba089c24c
30 changed files with 1416 additions and 25 deletions

View File

@@ -49,6 +49,7 @@ namespace Umbraco.Core.Composing.CompositionExtensions
composition.RegisterUnique<IContentTypeCommonRepository, ContentTypeCommonRepository>();
composition.RegisterUnique<IInstallationRepository, InstallationRepository>();
composition.RegisterUnique<IUpgradeCheckRepository, UpgradeCheckRepository>();
composition.RegisterUnique<IDocumentVersionRepository, DocumentVersionRepository>();
return composition;
}

View File

@@ -31,7 +31,12 @@ namespace Umbraco.Core.Composing.CompositionExtensions
composition.RegisterUnique<IDomainService, DomainService>();
composition.RegisterUnique<IAuditService, AuditService>();
composition.RegisterUnique<ITagService, TagService>();
composition.RegisterUnique<IContentService, ContentService>();
composition.RegisterUnique<ContentService>();
composition.RegisterUnique<IContentService>(factory => factory.GetInstance<ContentService>());
composition.RegisterUnique<IContentVersionCleanupService>(factory => factory.GetInstance<ContentService>());
composition.RegisterUnique<IContentVersionCleanupPolicy, DefaultContentVersionCleanupPolicy>();
composition.RegisterUnique<IUserService, UserService>();
composition.RegisterUnique<IMemberService, MemberService>();
composition.RegisterUnique<IMediaService, MediaService>();

View File

@@ -22,8 +22,8 @@ namespace Umbraco.Core.Configuration.UmbracoSettings
[ConfigurationProperty("notifications", IsRequired = true)]
internal NotificationsElement Notifications => (NotificationsElement) base["notifications"];
[ConfigurationProperty("contentVersionCleanupPolicy", IsRequired = false)]
internal ContentVersionCleanupPolicyElement ContentVersionCleanupPolicy => (ContentVersionCleanupPolicyElement) this["contentVersionCleanupPolicy"];
[ConfigurationProperty("contentVersionCleanupPolicyGlobalSettings", IsRequired = false)]
internal ContentVersionCleanupPolicyGlobalSettingsElement ContentVersionCleanupPolicyGlobalSettingsElement => (ContentVersionCleanupPolicyGlobalSettingsElement) this["contentVersionCleanupPolicyGlobalSettings"];
[ConfigurationProperty("PreviewBadge")]
internal InnerTextConfigurationElement<string> PreviewBadge => GetOptionalTextElement("PreviewBadge", DefaultPreviewBadge);
@@ -64,7 +64,7 @@ namespace Umbraco.Core.Configuration.UmbracoSettings
IEnumerable<string> IContentSection.AllowedUploadFiles => AllowedUploadFiles;
IContentVersionCleanupPolicySettings IContentSection.ContentVersionCleanupPolicySettings => ContentVersionCleanupPolicy;
IContentVersionCleanupPolicyGlobalSettings IContentSection.ContentVersionCleanupPolicyGlobalSettings => ContentVersionCleanupPolicyGlobalSettingsElement;
bool IContentSection.ShowDeprecatedPropertyEditors => ShowDeprecatedPropertyEditors;

View File

@@ -2,7 +2,7 @@
namespace Umbraco.Core.Configuration.UmbracoSettings
{
internal class ContentVersionCleanupPolicyElement : UmbracoConfigurationElement, IContentVersionCleanupPolicySettings
internal class ContentVersionCleanupPolicyGlobalSettingsElement : UmbracoConfigurationElement, IContentVersionCleanupPolicyGlobalSettings
{
[ConfigurationProperty("enable", DefaultValue = false)]
public bool EnableCleanup => (bool)this["enable"];

View File

@@ -25,7 +25,7 @@ namespace Umbraco.Core.Configuration.UmbracoSettings
IEnumerable<string> AllowedUploadFiles { get; }
IContentVersionCleanupPolicySettings ContentVersionCleanupPolicySettings { get; }
IContentVersionCleanupPolicyGlobalSettings ContentVersionCleanupPolicyGlobalSettings { get; }
/// <summary>
/// Gets a value indicating whether to show deprecated property editors in

View File

@@ -1,6 +1,6 @@
namespace Umbraco.Core.Configuration.UmbracoSettings
{
public interface IContentVersionCleanupPolicySettings
public interface IContentVersionCleanupPolicyGlobalSettings
{
bool EnableCleanup { get; }
int KeepAllVersionsNewerThanDays { get; }

View File

@@ -0,0 +1,13 @@
using System;
namespace Umbraco.Core.Models
{
public class ContentVersionCleanupPolicySettings
{
public int ContentTypeId { get; set; }
public int? KeepAllVersionsNewerThanDays { get; set; }
public int? KeepLatestVersionPerDayForDays { get; set; }
public bool PreventCleanup { get; set; }
public DateTime Updated { get; set; }
}
}

View File

@@ -0,0 +1,24 @@
using System;
namespace Umbraco.Core.Models
{
public class HistoricContentVersionMeta
{
public int ContentId { get; }
public int ContentTypeId { get; }
public int VersionId { get; }
public DateTime VersionDate { get; }
public HistoricContentVersionMeta() { }
public HistoricContentVersionMeta(int contentId, int contentTypeId, int versionId, DateTime versionDate)
{
ContentId = contentId;
ContentTypeId = contentTypeId;
VersionId = versionId;
VersionDate = versionDate;
}
public override string ToString() => $"HistoricContentVersionMeta(versionId: {VersionId}, versionDate: {VersionDate:s}";
}
}

View File

@@ -6,14 +6,13 @@ using Umbraco.Core.Persistence.DatabaseAnnotations;
namespace Umbraco.Core.Persistence.Dtos
{
[TableName(TableName)]
[PrimaryKey("contentTypeId")]
[PrimaryKey("contentTypeId", AutoIncrement = false)]
[ExplicitColumns]
internal class ContentVersionCleanupPolicyDto
{
public const string TableName = Constants.DatabaseSchema.Tables.ContentVersionCleanupPolicy;
[Column("contentTypeId")]
[PrimaryKeyColumn(AutoIncrement = false)]
[ForeignKey(typeof(ContentTypeDto), Column = "nodeId", OnDelete = Rule.Cascade)]
public int ContentTypeId { get; set; }

View File

@@ -0,0 +1,23 @@
using System.Collections.Generic;
using Umbraco.Core.Models;
namespace Umbraco.Core.Persistence.Repositories
{
public interface IDocumentVersionRepository : IRepository
{
/// <summary>
/// Gets a list of all historic content versions.
/// </summary>
public IReadOnlyCollection<HistoricContentVersionMeta> GetDocumentVersionsEligibleForCleanup();
/// <summary>
/// Gets cleanup policy override settings per content type.
/// </summary>
public IReadOnlyCollection<ContentVersionCleanupPolicySettings> GetCleanupPolicies();
/// <summary>
/// Deletes multiple content versions by ID.
/// </summary>
void DeleteVersions(IEnumerable<int> versionIds);
}
}

View File

@@ -0,0 +1,94 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Umbraco.Core.Models;
using Umbraco.Core.Persistence.Dtos;
using Umbraco.Core.Scoping;
namespace Umbraco.Core.Persistence.Repositories.Implement
{
internal class DocumentVersionRepository : IDocumentVersionRepository
{
private readonly IScopeAccessor _scopeAccessor;
public DocumentVersionRepository(IScopeAccessor scopeAccessor)
{
_scopeAccessor = scopeAccessor ?? throw new ArgumentNullException(nameof(scopeAccessor));
}
/// <inheritdoc />
/// <remarks>
/// Never includes current draft version. <br/>
/// Never includes current published version.<br/>
/// Never includes versions marked as "preventCleanup".<br/>
/// </remarks>
public IReadOnlyCollection<HistoricContentVersionMeta> GetDocumentVersionsEligibleForCleanup()
{
var query = _scopeAccessor.AmbientScope.SqlContext.Sql();
query.Select(@"umbracoDocument.nodeId as contentId,
umbracoContent.contentTypeId as contentTypeId,
umbracoContentVersion.id as versionId,
umbracoContentVersion.versionDate as versionDate")
.From<DocumentDto>()
.InnerJoin<ContentDto>()
.On<DocumentDto, ContentDto>(left => left.NodeId, right => right.NodeId)
.InnerJoin<ContentVersionDto>()
.On<ContentDto, ContentVersionDto>(left => left.NodeId, right => right.NodeId)
.InnerJoin<DocumentVersionDto>()
.On<ContentVersionDto, DocumentVersionDto>(left => left.Id, right => right.Id)
.Where<ContentVersionDto>(x => !x.Current) // Never delete current draft version
.Where<ContentVersionDto>(x => !x.PreventCleanup) // Never delete "pinned" versions
.Where<DocumentVersionDto>(x => !x.Published); // Never delete published version
return _scopeAccessor.AmbientScope.Database.Fetch<HistoricContentVersionMeta>(query);
}
/// <inheritdoc />
public IReadOnlyCollection<ContentVersionCleanupPolicySettings> GetCleanupPolicies()
{
var query = _scopeAccessor.AmbientScope.SqlContext.Sql();
query.Select<ContentVersionCleanupPolicyDto>()
.From<ContentVersionCleanupPolicyDto>();
return _scopeAccessor.AmbientScope.Database.Fetch<ContentVersionCleanupPolicySettings>(query);
}
/// <inheritdoc />
/// <remarks>
/// Deletes in batches of <see cref="Constants.Sql.MaxParameterCount"/>
/// </remarks>
public void DeleteVersions(IEnumerable<int> versionIds)
{
foreach (var group in versionIds.InGroupsOf(Constants.Sql.MaxParameterCount))
{
var groupedVersionIds = group.ToList();
// Note: We had discussed doing this in a single SQL Command.
// If you can work out how to make that work with SQL CE, let me know!
// Can use test PerformContentVersionCleanup_WithNoKeepPeriods_DeletesEverythingExceptActive to try things out.
var query = _scopeAccessor.AmbientScope.SqlContext.Sql()
.Delete<PropertyDataDto>()
.WhereIn<PropertyDataDto>(x => x.VersionId, groupedVersionIds);
_scopeAccessor.AmbientScope.Database.Execute(query);
query = _scopeAccessor.AmbientScope.SqlContext.Sql()
.Delete<ContentVersionCultureVariationDto>()
.WhereIn<ContentVersionCultureVariationDto>(x => x.VersionId, groupedVersionIds);
_scopeAccessor.AmbientScope.Database.Execute(query);
query = _scopeAccessor.AmbientScope.SqlContext.Sql()
.Delete<DocumentVersionDto>()
.WhereIn<DocumentVersionDto>(x => x.Id, groupedVersionIds);
_scopeAccessor.AmbientScope.Database.Execute(query);
query = _scopeAccessor.AmbientScope.SqlContext.Sql()
.Delete<ContentVersionDto>()
.WhereIn<ContentVersionDto>(x => x.Id, groupedVersionIds);
_scopeAccessor.AmbientScope.Database.Execute(query);
}
}
}
}

View File

@@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using Umbraco.Core.Models;
namespace Umbraco.Core.Services
{
/// <summary>
/// Used to filter historic content versions for cleanup.
/// </summary>
public interface IContentVersionCleanupPolicy
{
/// <summary>
/// Filters a set of candidates historic content versions for cleanup according to policy settings.
/// </summary>
IEnumerable<HistoricContentVersionMeta> Apply(DateTime asAtDate, IEnumerable<HistoricContentVersionMeta> items);
}
}

View File

@@ -0,0 +1,14 @@
using System;
using System.Collections.Generic;
using Umbraco.Core.Models;
namespace Umbraco.Core.Services
{
public interface IContentVersionCleanupService
{
/// <summary>
/// Removes historic content versions according to a policy.
/// </summary>
IReadOnlyCollection<HistoricContentVersionMeta> PerformContentVersionCleanup(DateTime asAtDate);
}
}

View File

@@ -19,7 +19,7 @@ namespace Umbraco.Core.Services.Implement
/// <summary>
/// Implements the content service.
/// </summary>
public class ContentService : RepositoryService, IContentService
public class ContentService : RepositoryService, IContentService, IContentVersionCleanupService
{
private readonly IDocumentRepository _documentRepository;
private readonly IEntityRepository _entityRepository;
@@ -1438,7 +1438,7 @@ namespace Umbraco.Core.Services.Implement
var result = CommitDocumentChangesInternal(scope, d, saveEventArgs, allLangs.Value, d.WriterId);
if (result.Success == false)
Logger.Error<ContentService,int,PublishResultType>(null, "Failed to publish document id={DocumentId}, reason={Reason}.", d.Id, result.Result);
Logger.Error<ContentService, int, PublishResultType>(null, "Failed to publish document id={DocumentId}, reason={Reason}.", d.Id, result.Result);
results.Add(result);
}
@@ -2452,7 +2452,7 @@ namespace Umbraco.Core.Services.Implement
if (report.FixedIssues.Count > 0)
{
//The event args needs a content item so we'll make a fake one with enough properties to not cause a null ref
var root = new Content("root", -1, new ContentType(-1)) {Id = -1, Key = Guid.Empty};
var root = new Content("root", -1, new ContentType(-1)) { Id = -1, Key = Guid.Empty };
scope.Events.Dispatch(TreeChanged, this, new TreeChange<IContent>.EventArgs(new TreeChange<IContent>(root, TreeChangeTypes.RefreshAll)));
}
@@ -3201,7 +3201,7 @@ namespace Umbraco.Core.Services.Implement
if (rollbackSaveResult.Success == false)
{
//Log the error/warning
Logger.Error<ContentService,int,int,int>("User '{UserId}' was unable to rollback content '{ContentId}' to version '{VersionId}'", userId, id, versionId);
Logger.Error<ContentService, int, int, int>("User '{UserId}' was unable to rollback content '{ContentId}' to version '{VersionId}'", userId, id, versionId);
}
else
{
@@ -3210,7 +3210,7 @@ namespace Umbraco.Core.Services.Implement
scope.Events.Dispatch(RolledBack, this, rollbackEventArgs);
//Logging & Audit message
Logger.Info<ContentService,int,int,int>("User '{UserId}' rolled back content '{ContentId}' to version '{VersionId}'", userId, id, versionId);
Logger.Info<ContentService, int, int, int>("User '{UserId}' rolled back content '{ContentId}' to version '{VersionId}'", userId, id, versionId);
Audit(AuditType.RollBack, userId, id, $"Content '{content.Name}' was rolled back to version '{versionId}'");
}
@@ -3222,7 +3222,110 @@ namespace Umbraco.Core.Services.Implement
#endregion
/// <inheritdoc />
/// <remarks>
/// In v9 this can live in another class as we publish the notifications via IEventAggregator.
/// But for v8 must be here for access to the static events.
/// </remarks>
public IReadOnlyCollection<HistoricContentVersionMeta> PerformContentVersionCleanup(DateTime asAtDate)
{
return CleanupDocumentVersions(asAtDate);
// Media - ignored
// Members - ignored
}
/// <remarks>
/// v9 - move to another class
/// </remarks>
private IReadOnlyCollection<HistoricContentVersionMeta> CleanupDocumentVersions(DateTime asAtDate)
{
// NOTE: v9 - don't service locate
var documentVersionRepository = Composing.Current.Factory.GetInstance<IDocumentVersionRepository>();
// NOTE: v9 - don't service locate
var cleanupPolicy = Composing.Current.Factory.GetInstance<IContentVersionCleanupPolicy>();
List<HistoricContentVersionMeta> versionsToDelete;
/* Why so many scopes?
*
* We could just work out the set to delete at SQL infra level which was the original plan, however we agreed that really we should fire
* ContentService.DeletingVersions so people can hook & cancel if required.
*
* On first time run of cleanup on a site with a lot of history there may be a lot of historic ContentVersions to remove e.g. 200K for our.umbraco.com.
* If we weren't supporting SQL CE we could do TVP, or use temp tables to bulk delete with joins to our list of version ids to nuke.
* (much nicer, we can kill 100k in sub second time-frames).
*
* However we are supporting SQL CE, so the easiest thing to do is use the Umbraco InGroupsOf helper to create a query with 2K args of version
* ids to delete at a time.
*
* This is already done at the repository level, however if we only had a single scope at service level we're still locking
* the ContentVersions table (and other related tables) for a couple of minutes which makes the back office unusable.
*
* As a quick fix, we can also use InGroupsOf at service level, create a scope per group to give other connections a chance
* to grab the locks and execute their queries.
*
* This makes the back office a tiny bit sluggish during first run but it is usable for loading tree and publishing content.
*
* There are optimizations we can do, we could add a bulk delete for SqlServerSyntaxProvider which differs in implementation
* and fallback to this naive approach only for SQL CE, however we agreed it is not worth the effort as this is a one time pain,
* subsequent runs shouldn't have huge numbers of versions to cleanup.
*
* tl;dr lots of scopes to enable other connections to use the DB whilst we work.
*/
using (var scope = ScopeProvider.CreateScope(autoComplete: true))
{
var allHistoricVersions = documentVersionRepository.GetDocumentVersionsEligibleForCleanup();
Logger.Debug<ContentService>("Discovered {count} candidate(s) for ContentVersion cleanup.", allHistoricVersions.Count);
versionsToDelete = new List<HistoricContentVersionMeta>(allHistoricVersions.Count);
var filteredContentVersions = cleanupPolicy.Apply(asAtDate, allHistoricVersions);
foreach (var version in filteredContentVersions)
{
var args = new DeleteRevisionsEventArgs(version.ContentId, version.VersionId);
if (scope.Events.DispatchCancelable(ContentService.DeletingVersions, this, args))
{
Logger.Debug<ContentService>("Delete cancelled for ContentVersion [{versionId}]", version.VersionId);
continue;
}
versionsToDelete.Add(version);
}
}
if (!versionsToDelete.Any())
{
Logger.Debug<ContentService>("No remaining ContentVersions for cleanup.", versionsToDelete.Count);
return Array.Empty<HistoricContentVersionMeta>();
}
Logger.Debug<ContentService>("Removing {count} ContentVersion(s).", versionsToDelete.Count);
foreach (var group in versionsToDelete.InGroupsOf(Constants.Sql.MaxParameterCount))
{
using (var scope = ScopeProvider.CreateScope(autoComplete: true))
{
scope.WriteLock(Constants.Locks.ContentTree);
var groupEnumerated = group.ToList();
documentVersionRepository.DeleteVersions(groupEnumerated.Select(x => x.VersionId));
foreach (var version in groupEnumerated)
{
var args = new DeleteRevisionsEventArgs(version.ContentId, version.VersionId);
scope.Events.Dispatch(ContentService.DeletedVersions, this, args);
}
}
}
using (var scope = ScopeProvider.CreateScope(autoComplete: true))
{
Audit(AuditType.Delete, Constants.Security.SuperUserId, -1, $"Removed {versionsToDelete.Count} ContentVersion(s) according to cleanup policy.");
}
return versionsToDelete;
}
}
}

View File

@@ -0,0 +1,90 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Umbraco.Core.Configuration.UmbracoSettings;
using Umbraco.Core.Models;
using Umbraco.Core.Persistence.Repositories;
using Umbraco.Core.Scoping;
namespace Umbraco.Core.Services.Implement
{
public class DefaultContentVersionCleanupPolicy : IContentVersionCleanupPolicy
{
private readonly IContentSection _contentSection;
private readonly IScopeProvider _scopeProvider;
private readonly IDocumentVersionRepository _documentVersionRepository;
public DefaultContentVersionCleanupPolicy(IContentSection contentSection, IScopeProvider scopeProvider, IDocumentVersionRepository documentVersionRepository)
{
_contentSection = contentSection ?? throw new ArgumentNullException(nameof(contentSection));
_scopeProvider = scopeProvider ?? throw new ArgumentNullException(nameof(scopeProvider));
_documentVersionRepository = documentVersionRepository ?? throw new ArgumentNullException(nameof(documentVersionRepository));
}
public IEnumerable<HistoricContentVersionMeta> Apply(DateTime asAtDate, IEnumerable<HistoricContentVersionMeta> items)
{
// Note: Not checking global enable flag, that's handled in the scheduled job.
// If this method is called and policy is globally disabled someone has chosen to run in code.
var globalPolicy = _contentSection.ContentVersionCleanupPolicyGlobalSettings;
var theRest = new List<HistoricContentVersionMeta>();
using(_scopeProvider.CreateScope(autoComplete: true))
{
var policyOverrides = _documentVersionRepository.GetCleanupPolicies()
.ToDictionary(x => x.ContentTypeId);
foreach (var version in items)
{
var age = asAtDate - version.VersionDate;
var overrides = GetOverridePolicy(version, policyOverrides);
var keepAll = overrides?.KeepAllVersionsNewerThanDays ?? globalPolicy.KeepAllVersionsNewerThanDays!;
var keepLatest = overrides?.KeepLatestVersionPerDayForDays ?? globalPolicy.KeepLatestVersionPerDayForDays;
var preventCleanup = overrides?.PreventCleanup ?? false;
if (preventCleanup)
{
continue;
}
if (age.TotalDays <= keepAll)
{
continue;
}
if (age.TotalDays > keepLatest)
{
yield return version;
continue;
}
theRest.Add(version);
}
var grouped = theRest.GroupBy(x => new
{
x.ContentId,
x.VersionDate.Date
});
foreach (var group in grouped)
{
yield return group.OrderByDescending(x => x.VersionId).First();
}
}
}
private ContentVersionCleanupPolicySettings GetOverridePolicy(
HistoricContentVersionMeta version,
IDictionary<int, ContentVersionCleanupPolicySettings> overrides)
{
_ = overrides.TryGetValue(version.ContentTypeId, out var value);
return value;
}
}
}

View File

@@ -130,8 +130,8 @@
<Compile Include="AssemblyExtensions.cs" />
<Compile Include="Collections\StackQueue.cs" />
<Compile Include="Configuration\ICoreDebug.cs" />
<Compile Include="Configuration\UmbracoSettings\ContentVersionCleanupPolicyElement.cs" />
<Compile Include="Configuration\UmbracoSettings\IContentVersionCleanupPolicySettings.cs" />
<Compile Include="Configuration\UmbracoSettings\ContentVersionCleanupPolicyGlobalSettingsElement.cs" />
<Compile Include="Configuration\UmbracoSettings\IContentVersionCleanupPolicyGlobalSettings.cs" />
<Compile Include="Constants-CharArrays.cs" />
<Compile Include="Collections\EventClearingObservableCollection.cs" />
<Compile Include="Constants-Sql.cs" />
@@ -163,6 +163,7 @@
<Compile Include="Models\ContentDataIntegrityReport.cs" />
<Compile Include="Models\ContentDataIntegrityReportEntry.cs" />
<Compile Include="Models\ContentDataIntegrityReportOptions.cs" />
<Compile Include="Models\ContentVersionCleanupPolicySettings.cs" />
<Compile Include="Models\Identity\ExternalLogin.cs" />
<Compile Include="Models\Identity\IExternalLogin.cs" />
<Compile Include="Models\IconModel.cs" />
@@ -179,8 +180,11 @@
<Compile Include="Persistence\Dtos\ConstraintPerTableDto.cs" />
<Compile Include="Persistence\Dtos\ContentVersionCleanupPolicyDto.cs" />
<Compile Include="Persistence\Dtos\DefaultConstraintPerColumnDto.cs" />
<Compile Include="Models\HistoricContentVersionMeta.cs" />
<Compile Include="Persistence\Dtos\UserNotificationDto.cs" />
<Compile Include="Persistence\Repositories\IDocumentVersionRepository.cs" />
<Compile Include="Persistence\Repositories\IInstallationRepository.cs" />
<Compile Include="Persistence\Repositories\Implement\DocumentVersionRepository.cs" />
<Compile Include="Persistence\Repositories\Implement\InstallationRepository.cs" />
<Compile Include="Persistence\SqlCeImageMapper.cs" />
<Compile Include="Persistence\Dtos\DefinedIndexDto.cs" />
@@ -193,7 +197,10 @@
<Compile Include="Serialization\AutoInterningStringKeyCaseInsensitiveDictionaryConverter.cs" />
<Compile Include="PropertyEditors\EyeDropperColorPickerConfiguration.cs" />
<Compile Include="PropertyEditors\ComplexPropertyEditorContentEventHandler.cs" />
<Compile Include="Services\IContentVersionCleanupPolicy.cs" />
<Compile Include="Services\IContentVersionCleanupService.cs" />
<Compile Include="Services\IIconService.cs" />
<Compile Include="Services\Implement\DefaultContentVersionCleanupPolicy.cs" />
<Compile Include="Services\Implement\InstallationService.cs" />
<Compile Include="Migrations\Upgrade\V_8_6_0\AddMainDomLock.cs" />
<Compile Include="Models\Blocks\BlockListItem.cs" />

View File

@@ -57,6 +57,8 @@
<!-- If completed, only the file extensions listed below will be allowed to be uploaded. If empty, disallowedUploadFiles will apply to prevent upload of specific file extensions. -->
<allowedUploadFiles>jpg,png,gif</allowedUploadFiles>
<contentVersionCleanupPolicyGlobalSettings enable="true" keepAllVersionsNewerThanDays="2" keepLatestVersionPerDayForDays="30" />
</content>
<security>

View File

@@ -0,0 +1,125 @@
using System.Diagnostics;
using System.Linq;
using NUnit.Framework;
using Umbraco.Core.Persistence;
using Umbraco.Core.Persistence.Dtos;
using Umbraco.Core.Persistence.Repositories.Implement;
using Umbraco.Core.Scoping;
using Umbraco.Tests.TestHelpers;
using Umbraco.Tests.TestHelpers.Entities;
using Umbraco.Tests.Testing;
namespace Umbraco.Tests.Persistence.Repositories
{
/// <remarks>
/// v9 -> Tests.Integration
/// </remarks>
[TestFixture]
[UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest)]
public class DocumentVersionRepository_Tests_Integration : TestWithDatabaseBase
{
[Test]
public void GetDocumentVersionsEligibleForCleanup_Always_ExcludesActiveVersions()
{
var contentType = MockedContentTypes.CreateSimpleContentType("umbTextpage", "Textpage");
ServiceContext.FileService.SaveTemplate(contentType.DefaultTemplate);
ServiceContext.ContentTypeService.Save(contentType);
var content = MockedContent.CreateSimpleContent(contentType);
ServiceContext.ContentService.SaveAndPublish(content);
// At this point content has 2 versions, a draft version and a published version.
ServiceContext.ContentService.SaveAndPublish(content);
// At this point content has 3 versions, a historic version, a draft version and a published version.
var scopeProvider = TestObjects.GetScopeProvider(Logger);
using (scopeProvider.CreateScope())
{
var sut = new DocumentVersionRepository((IScopeAccessor)scopeProvider);
var results = sut.GetDocumentVersionsEligibleForCleanup();
Assert.Multiple(() =>
{
Assert.AreEqual(1, results.Count);
Assert.AreEqual(1, results.First().VersionId);
});
}
}
[Test]
public void GetDocumentVersionsEligibleForCleanup_Always_ExcludesPinnedVersions()
{
var contentType = MockedContentTypes.CreateSimpleContentType("umbTextpage", "Textpage");
ServiceContext.FileService.SaveTemplate(contentType.DefaultTemplate);
ServiceContext.ContentTypeService.Save(contentType);
var content = MockedContent.CreateSimpleContent(contentType);
ServiceContext.ContentService.SaveAndPublish(content);
// At this point content has 2 versions, a draft version and a published version.
ServiceContext.ContentService.SaveAndPublish(content);
ServiceContext.ContentService.SaveAndPublish(content);
ServiceContext.ContentService.SaveAndPublish(content);
// At this point content has 5 versions, 3 historic versions, a draft version and a published version.
var allVersions = ServiceContext.ContentService.GetVersions(content.Id);
Debug.Assert(allVersions.Count() == 5); // Sanity check
var scopeProvider = TestObjects.GetScopeProvider(Logger);
using (var scope = scopeProvider.CreateScope())
{
scope.Database.Update<ContentVersionDto>("set preventCleanup = 1 where id in (1,3)");
var sut = new DocumentVersionRepository((IScopeAccessor)scopeProvider);
var results = sut.GetDocumentVersionsEligibleForCleanup();
Assert.Multiple(() =>
{
Assert.AreEqual(1, results.Count);
// We pinned 1 & 3
// 4 is current
// 5 is published
// So all that is left is 2
Assert.AreEqual(2, results.First().VersionId);
});
}
}
[Test]
public void DeleteVersions_Always_DeletesSpecifiedVersions()
{
var contentType = MockedContentTypes.CreateSimpleContentType("umbTextpage", "Textpage");
ServiceContext.FileService.SaveTemplate(contentType.DefaultTemplate);
ServiceContext.ContentTypeService.Save(contentType);
var content = MockedContent.CreateSimpleContent(contentType);
ServiceContext.ContentService.SaveAndPublish(content);
ServiceContext.ContentService.SaveAndPublish(content);
ServiceContext.ContentService.SaveAndPublish(content);
ServiceContext.ContentService.SaveAndPublish(content);
var scopeProvider = TestObjects.GetScopeProvider(Logger);
using (var scope = scopeProvider.CreateScope())
{
var query = scope.SqlContext.Sql();
query.Select<ContentVersionDto>()
.From<ContentVersionDto>();
var sut = new DocumentVersionRepository((IScopeAccessor)scopeProvider);
sut.DeleteVersions(new []{1,2,3});
var after = scope.Database.Fetch<ContentVersionDto>(query);
Assert.Multiple(() =>
{
Assert.AreEqual(2, after.Count);
Assert.True(after.All(x => x.Id > 3));
});
}
}
}
}

View File

@@ -0,0 +1,127 @@
using System;
using AutoFixture.NUnit3;
using Moq;
using NUnit.Framework;
using Umbraco.Core;
using Umbraco.Core.Configuration.UmbracoSettings;
using Umbraco.Core.Services;
using Umbraco.Core.Sync;
using Umbraco.Tests.Testing;
using Umbraco.Web.Scheduling;
namespace Umbraco.Tests.Scheduling
{
[TestFixture]
class ContentVersionCleanup_Tests_UnitTests
{
[Test, AutoMoqData]
public void ContentVersionCleanup_WhenNotEnabled_DoesNotCleanupWillRepeat(
[Frozen] Mock<IContentVersionCleanupPolicyGlobalSettings> settings,
[Frozen] Mock<IRuntimeState> state,
[Frozen] Mock<IContentVersionCleanupService> cleanupService,
ContentVersionCleanup sut)
{
settings.Setup(x => x.EnableCleanup).Returns(false);
state.Setup(x => x.Level).Returns(RuntimeLevel.Run);
state.Setup(x => x.IsMainDom).Returns(true);
state.Setup(x => x.ServerRole).Returns(ServerRole.Master);
var result = sut.PerformRun();
Assert.Multiple(() =>
{
Assert.False(result);
cleanupService.Verify(x => x.PerformContentVersionCleanup(It.IsAny<DateTime>()), Times.Never);
});
}
[Test, AutoMoqData]
public void ContentVersionCleanup_RuntimeLevelNotRun_DoesNotCleanupWillRepeat(
[Frozen] Mock<IContentVersionCleanupPolicyGlobalSettings> settings,
[Frozen] Mock<IRuntimeState> state,
[Frozen] Mock<IContentVersionCleanupService> cleanupService,
ContentVersionCleanup sut)
{
settings.Setup(x => x.EnableCleanup).Returns(true);
state.Setup(x => x.Level).Returns(RuntimeLevel.Unknown);
state.Setup(x => x.IsMainDom).Returns(true);
state.Setup(x => x.ServerRole).Returns(ServerRole.Master);
var result = sut.PerformRun();
Assert.Multiple(() =>
{
Assert.True(result);
cleanupService.Verify(x => x.PerformContentVersionCleanup(It.IsAny<DateTime>()), Times.Never);
});
}
[Test, AutoMoqData]
public void ContentVersionCleanup_ServerRoleUnknown_DoesNotCleanupWillRepeat(
[Frozen] Mock<IContentVersionCleanupPolicyGlobalSettings> settings,
[Frozen] Mock<IRuntimeState> state,
[Frozen] Mock<IContentVersionCleanupService> cleanupService,
ContentVersionCleanup sut)
{
settings.Setup(x => x.EnableCleanup).Returns(true);
state.Setup(x => x.Level).Returns(RuntimeLevel.Run);
state.Setup(x => x.IsMainDom).Returns(true);
state.Setup(x => x.ServerRole).Returns(ServerRole.Unknown);
var result = sut.PerformRun();
Assert.Multiple(() =>
{
Assert.True(result);
cleanupService.Verify(x => x.PerformContentVersionCleanup(It.IsAny<DateTime>()), Times.Never);
});
}
[Test, AutoMoqData]
public void ContentVersionCleanup_NotMainDom_DoesNotCleanupWillNotRepeat(
[Frozen] Mock<IContentVersionCleanupPolicyGlobalSettings> settings,
[Frozen] Mock<IRuntimeState> state,
[Frozen] Mock<IContentVersionCleanupService> cleanupService,
ContentVersionCleanup sut)
{
settings.Setup(x => x.EnableCleanup).Returns(true);
state.Setup(x => x.Level).Returns(RuntimeLevel.Run);
state.Setup(x => x.IsMainDom).Returns(false);
state.Setup(x => x.ServerRole).Returns(ServerRole.Master);
var result = sut.PerformRun();
Assert.Multiple(() =>
{
Assert.False(result);
cleanupService.Verify(x => x.PerformContentVersionCleanup(It.IsAny<DateTime>()), Times.Never);
});
}
[Test, AutoMoqData]
public void ContentVersionCleanup_Enabled_DelegatesToCleanupService(
[Frozen] Mock<IContentVersionCleanupPolicyGlobalSettings> settings,
[Frozen] Mock<IRuntimeState> state,
[Frozen] Mock<IContentVersionCleanupService> cleanupService,
ContentVersionCleanup sut)
{
settings.Setup(x => x.EnableCleanup).Returns(true);
state.Setup(x => x.Level).Returns(RuntimeLevel.Run);
state.Setup(x => x.IsMainDom).Returns(true);
state.Setup(x => x.ServerRole).Returns(ServerRole.Master);
var result = sut.PerformRun();
Assert.Multiple(() =>
{
Assert.True(result);
cleanupService.Verify(x => x.PerformContentVersionCleanup(It.IsAny<DateTime>()), Times.Once);
});
}
}
}

View File

@@ -0,0 +1,106 @@
using System;
using System.Data;
using System.Diagnostics;
using NUnit.Framework;
using Umbraco.Core.Models;
using Umbraco.Core.Persistence.Dtos;
using Umbraco.Core.Services;
using Umbraco.Tests.TestHelpers;
using Umbraco.Tests.TestHelpers.Entities;
using Umbraco.Tests.Testing;
namespace Umbraco.Tests.Services
{
[TestFixture]
[UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest)]
public class ContentVersionCleanupService_Tests_Integration : TestWithDatabaseBase
{
/// <remarks>
/// This is covered by the unit tests, but nice to know it deletes on infra.
/// And proves implementation is compatible with SQL CE.
/// </remarks>
[Test]
public void PerformContentVersionCleanup_WithNoKeepPeriods_DeletesEverythingExceptActive()
{
// For reference currently has
// 5000 Documents
// With 200K Versions
// With 11M Property data
var contentTypeA = MockedContentTypes.CreateSimpleContentType("contentTypeA", "contentTypeA");
ServiceContext.FileService.SaveTemplate(contentTypeA.DefaultTemplate);
ServiceContext.ContentTypeService.Save(contentTypeA);
var content = MockedContent.CreateSimpleContent(contentTypeA);
ServiceContext.ContentService.SaveAndPublish(content, raiseEvents: false);
for (var i = 0; i < 10; i++)
{
ServiceContext.ContentService.SaveAndPublish(content, raiseEvents: false);
}
var before = GetReport();
Debug.Assert(before.ContentVersions == 12); // 10 historic + current draft + current published
Debug.Assert(before.PropertyData == 12 * 3); // CreateSimpleContentType = 3 props
// Kill all historic
InsertCleanupPolicy(contentTypeA, 0, 0);
((IContentVersionCleanupService)ServiceContext.ContentService).PerformContentVersionCleanup(DateTime.Now.AddHours(1));
var after = GetReport();
Assert.Multiple(() =>
{
Assert.AreEqual(2, after.ContentVersions); // current draft, current published
Assert.AreEqual(2, after.DocumentVersions);
Assert.AreEqual(6, after.PropertyData); // CreateSimpleContentType = 3 props
});
}
private Report GetReport()
{
var scopeProvider = TestObjects.GetScopeProvider(Logger);
using (var scope = scopeProvider.CreateScope(autoComplete: true))
{
// SQL CE is fun!
var contentVersions = scope.Database.Single<int>(@"select count(1) from umbracoContentVersion");
var documentVersions = scope.Database.Single<int>(@"select count(1) from umbracoDocumentVersion");
var propertyData = scope.Database.Single<int>(@"select count(1) from umbracoPropertyData");
return new Report
{
ContentVersions = contentVersions,
DocumentVersions = documentVersions,
PropertyData = propertyData
};
}
}
private void InsertCleanupPolicy(IContentType contentType, int daysToKeepAll, int daysToRollupAll, bool preventCleanup = false)
{
var scopeProvider = TestObjects.GetScopeProvider(Logger);
using (var scope = scopeProvider.CreateScope(autoComplete: true))
{
var entity = new ContentVersionCleanupPolicyDto
{
ContentTypeId = contentType.Id,
KeepAllVersionsNewerThanDays = daysToKeepAll,
KeepLatestVersionPerDayForDays = daysToRollupAll,
PreventCleanup = preventCleanup,
Updated = DateTime.Today
};
scope.Database.Insert(entity);
}
}
class Report
{
public int ContentVersions { get; set; }
public int DocumentVersions { get; set; }
public int PropertyData { get; set; }
}
}
}

View File

@@ -0,0 +1,264 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using AutoFixture.NUnit3;
using Moq;
using NUnit.Framework;
using Umbraco.Core.Composing;
using Umbraco.Core.Events;
using Umbraco.Core.Models;
using Umbraco.Core.Persistence.Repositories;
using Umbraco.Core.Scoping;
using Umbraco.Core.Services;
using Umbraco.Core.Services.Implement;
using Umbraco.Tests.Testing;
namespace Umbraco.Tests.Services
{
/// <remarks>
/// v9 -> Tests.UnitTests
/// Sut here is ContentService, but in v9 should be a new class
/// </remarks>
[TestFixture]
public class ContentVersionCleanupService_Tests_UnitTests
{
[SetUp]
public void Setup()
{
Current.Reset();
}
/// <remarks>
/// For v9 this just needs a rewrite, no static events, no service location etc
/// </remarks>
[Test, AutoMoqData]
public void PerformContentVersionCleanup_Always_RespectsDeleteRevisionsCancellation(
[Frozen] Mock<IFactory> factory,
[Frozen] Mock<IScope> scope,
Mock<IDocumentVersionRepository> documentVersionRepository,
List<TestHistoricContentVersionMeta> someHistoricVersions,
DateTime aDateTime,
ContentService sut)
{
factory.Setup(x => x.GetInstance(typeof(IDocumentVersionRepository)))
.Returns(documentVersionRepository.Object);
factory.Setup(x => x.GetInstance(typeof(IContentVersionCleanupPolicy)))
.Returns(new EchoingCleanupPolicyStub());
documentVersionRepository.Setup(x => x.GetDocumentVersionsEligibleForCleanup())
.Returns(someHistoricVersions);
scope.Setup(x => x.Events).Returns(new PassThroughEventDispatcher());
// Wire up service locator
Current.Factory = factory.Object;
void OnDeletingVersions(IContentService sender, DeleteRevisionsEventArgs args) => args.Cancel = true;
ContentService.DeletingVersions += OnDeletingVersions;
// # Act
var report = sut.PerformContentVersionCleanup(aDateTime);
ContentService.DeletingVersions -= OnDeletingVersions;
Assert.AreEqual(0, report.Count);
}
/// <remarks>
/// For v9 this just needs a rewrite, no static events, no service location etc
/// </remarks>
[Test, AutoMoqData]
public void PerformContentVersionCleanup_Always_FiresDeletedVersionsForEachDeletedVersion(
[Frozen] Mock<IFactory> factory,
[Frozen] Mock<IScope> scope,
Mock<IDocumentVersionRepository> documentVersionRepository,
List<TestHistoricContentVersionMeta> someHistoricVersions,
DateTime aDateTime,
ContentService sut)
{
factory.Setup(x => x.GetInstance(typeof(IDocumentVersionRepository)))
.Returns(documentVersionRepository.Object);
factory.Setup(x => x.GetInstance(typeof(IContentVersionCleanupPolicy)))
.Returns(new EchoingCleanupPolicyStub());
documentVersionRepository.Setup(x => x.GetDocumentVersionsEligibleForCleanup())
.Returns(someHistoricVersions);
scope.Setup(x => x.Events).Returns(new PassThroughEventDispatcher());
// Wire up service locator
Current.Factory = factory.Object;
// v9 can Mock + Verify
var deletedAccordingToEvents = 0;
void OnDeletedVersions(IContentService sender, DeleteRevisionsEventArgs args) => deletedAccordingToEvents++;
ContentService.DeletedVersions += OnDeletedVersions;
// # Act
sut.PerformContentVersionCleanup(aDateTime);
ContentService.DeletedVersions -= OnDeletedVersions;
Assert.Multiple(() =>
{
Assert.Greater(deletedAccordingToEvents, 0);
Assert.AreEqual(someHistoricVersions.Count, deletedAccordingToEvents);
});
}
/// <remarks>
/// For v9 this just needs a rewrite, no static events, no service location etc
/// </remarks>
[Test, AutoMoqData]
public void PerformContentVersionCleanup_Always_ReturnsReportOfDeletedItems(
[Frozen] Mock<IFactory> factory,
[Frozen] Mock<IScope> scope,
Mock<IDocumentVersionRepository> documentVersionRepository,
List<TestHistoricContentVersionMeta> someHistoricVersions,
DateTime aDateTime,
ContentService sut)
{
factory.Setup(x => x.GetInstance(typeof(IDocumentVersionRepository)))
.Returns(documentVersionRepository.Object);
factory.Setup(x => x.GetInstance(typeof(IContentVersionCleanupPolicy)))
.Returns(new EchoingCleanupPolicyStub());
documentVersionRepository.Setup(x => x.GetDocumentVersionsEligibleForCleanup())
.Returns(someHistoricVersions);
scope.Setup(x => x.Events).Returns(new PassThroughEventDispatcher());
// Wire up service locator
Current.Factory = factory.Object;
// # Act
var report = sut.PerformContentVersionCleanup(aDateTime);
Assert.Multiple(() =>
{
Assert.Greater(report.Count, 0);
Assert.AreEqual(someHistoricVersions.Count, report.Count);
});
}
/// <remarks>
/// For v9 this just needs a rewrite, no static events, no service location etc
/// </remarks>
[Test, AutoMoqData]
public void PerformContentVersionCleanup_Always_AdheresToCleanupPolicy(
[Frozen] Mock<IFactory> factory,
[Frozen] Mock<IScope> scope,
Mock<IDocumentVersionRepository> documentVersionRepository,
Mock<IContentVersionCleanupPolicy> cleanupPolicy,
List<TestHistoricContentVersionMeta> someHistoricVersions,
DateTime aDateTime,
ContentService sut)
{
factory.Setup(x => x.GetInstance(typeof(IDocumentVersionRepository)))
.Returns(documentVersionRepository.Object);
factory.Setup(x => x.GetInstance(typeof(IContentVersionCleanupPolicy)))
.Returns(cleanupPolicy.Object);
documentVersionRepository.Setup(x => x.GetDocumentVersionsEligibleForCleanup())
.Returns(someHistoricVersions);
scope.Setup(x => x.Events).Returns(new PassThroughEventDispatcher());
cleanupPolicy.Setup(x => x.Apply(It.IsAny<DateTime>(), It.IsAny<IEnumerable<TestHistoricContentVersionMeta>>()))
.Returns<DateTime, IEnumerable<TestHistoricContentVersionMeta>>((_, items) => items.Take(1));
// Wire up service locator
Current.Factory = factory.Object;
// # Act
var report = sut.PerformContentVersionCleanup(aDateTime);
Debug.Assert(someHistoricVersions.Count > 1);
Assert.Multiple(() =>
{
cleanupPolicy.Verify(x => x.Apply(aDateTime, someHistoricVersions), Times.Once);
Assert.AreEqual(someHistoricVersions.First(), report.Single());
});
}
/// <remarks>
/// For v9 this just needs a rewrite, no static events, no service location etc
/// </remarks>
[Test, AutoMoqData]
public void PerformContentVersionCleanup_HasVersionsToDelete_CallsDeleteOnRepositoryWithFilteredSet(
[Frozen] Mock<IFactory> factory,
[Frozen] Mock<IScope> scope,
Mock<IDocumentVersionRepository> documentVersionRepository,
Mock<IContentVersionCleanupPolicy> cleanupPolicy,
List<TestHistoricContentVersionMeta> someHistoricVersions,
DateTime aDateTime,
ContentService sut)
{
factory.Setup(x => x.GetInstance(typeof(IDocumentVersionRepository)))
.Returns(documentVersionRepository.Object);
factory.Setup(x => x.GetInstance(typeof(IContentVersionCleanupPolicy)))
.Returns(cleanupPolicy.Object);
documentVersionRepository.Setup(x => x.GetDocumentVersionsEligibleForCleanup())
.Returns(someHistoricVersions);
scope.Setup(x => x.Events).Returns(new PassThroughEventDispatcher());
var filteredSet = someHistoricVersions.Take(1);
cleanupPolicy.Setup(x => x.Apply(It.IsAny<DateTime>(), It.IsAny<IEnumerable<TestHistoricContentVersionMeta>>()))
.Returns<DateTime, IEnumerable<TestHistoricContentVersionMeta>>((_, items) => filteredSet);
// Wire up service locator
Current.Factory = factory.Object;
// # Act
var report = sut.PerformContentVersionCleanup(aDateTime);
Debug.Assert(someHistoricVersions.Any());
var expectedId = filteredSet.First().VersionId;
documentVersionRepository.Verify(x => x.DeleteVersions(It.Is<IEnumerable<int>>(y => y.Single() == expectedId)), Times.Once);
}
class EchoingCleanupPolicyStub : IContentVersionCleanupPolicy
{
/// <summary>
/// What goes in, must come out
/// </summary>
public EchoingCleanupPolicyStub() { }
/* Note: Could just wire up a mock but its quite wordy.
*
* cleanupPolicy.Setup(x => x.Apply(It.IsAny<DateTime>(), It.IsAny<IEnumerable<TestHistoricContentVersionMeta>>()))
* .Returns<DateTime, IEnumerable<TestHistoricContentVersionMeta>>((date, items) => items);
*/
public IEnumerable<HistoricContentVersionMeta> Apply(
DateTime asAtDate,
IEnumerable<HistoricContentVersionMeta> items
) => items;
}
/// <remarks>
/// <para>NPoco &lt; 5 requires a parameter-less constructor but plays nice with get-only properties.</para>
/// <para>Moq won't play nice with get-only properties, but doesn't require a parameter-less constructor.</para>
///
/// <para>Inheritance solves this so that we get values for test data without a specimen builder</para>
/// </remarks>
public class TestHistoricContentVersionMeta : HistoricContentVersionMeta
{
public TestHistoricContentVersionMeta(int contentId, int contentTypeId, int versionId, DateTime versionDate)
: base(contentId, contentTypeId, versionId, versionDate) { }
}
}
}

View File

@@ -0,0 +1,237 @@
using System;
using System.Collections.Generic;
using System.Linq;
using AutoFixture.NUnit3;
using Moq;
using NUnit.Framework;
using Umbraco.Core.Configuration.UmbracoSettings;
using Umbraco.Core.Models;
using Umbraco.Core.Persistence.Repositories;
using Umbraco.Core.Services.Implement;
using Umbraco.Tests.Testing;
namespace Umbraco.Tests.Services
{
[TestFixture]
public class DefaultContentVersionCleanupPolicy_Tests_UnitTests
{
[Test, AutoMoqData]
public void Apply_AllOlderThanKeepSettings_AllVersionsReturned(
[Frozen] Mock<IDocumentVersionRepository> documentVersionRepository,
[Frozen] Mock<IContentSection> globalSettings,
DefaultContentVersionCleanupPolicy sut)
{
var versionId = 0;
var historicItems = new List<HistoricContentVersionMeta>
{
new HistoricContentVersionMeta(versionId: ++versionId, contentId: 1, contentTypeId: 1, versionDate: DateTime.Today.AddHours(-1)),
new HistoricContentVersionMeta(versionId: ++versionId, contentId: 1, contentTypeId: 1, versionDate: DateTime.Today.AddHours(-1)),
};
globalSettings.Setup(x => x.ContentVersionCleanupPolicyGlobalSettings)
.Returns(new TestCleanupSettings(true, 0, 0));
documentVersionRepository.Setup(x => x.GetCleanupPolicies())
.Returns(Array.Empty<ContentVersionCleanupPolicySettings>());
documentVersionRepository.Setup(x => x.GetDocumentVersionsEligibleForCleanup())
.Returns(historicItems);
var results = sut.Apply(DateTime.Today, historicItems).ToList();
Assert.AreEqual(2, results.Count);
}
[Test, AutoMoqData]
public void Apply_OverlappingKeepSettings_KeepAllVersionsNewerThanDaysTakesPriority(
[Frozen] Mock<IDocumentVersionRepository> documentVersionRepository,
[Frozen] Mock<IContentSection> globalSettings,
DefaultContentVersionCleanupPolicy sut)
{
var versionId = 0;
var historicItems = new List<HistoricContentVersionMeta>
{
new HistoricContentVersionMeta(versionId: ++versionId, contentId: 1, contentTypeId: 1, versionDate: DateTime.Today.AddHours(-1)),
new HistoricContentVersionMeta(versionId: ++versionId, contentId: 1, contentTypeId: 1, versionDate: DateTime.Today.AddHours(-1)),
};
globalSettings.Setup(x => x.ContentVersionCleanupPolicyGlobalSettings)
.Returns(new TestCleanupSettings(true, 2, 2));
documentVersionRepository.Setup(x => x.GetCleanupPolicies())
.Returns(Array.Empty<ContentVersionCleanupPolicySettings>());
documentVersionRepository.Setup(x => x.GetDocumentVersionsEligibleForCleanup())
.Returns(historicItems);
var results = sut.Apply(DateTime.Today, historicItems).ToList();
Assert.AreEqual(0, results.Count);
}
[Test, AutoMoqData]
public void Apply_WithinInKeepLatestPerDay_ReturnsSinglePerContentPerDay(
[Frozen] Mock<IDocumentVersionRepository> documentVersionRepository,
[Frozen] Mock<IContentSection> globalSettings,
DefaultContentVersionCleanupPolicy sut)
{
var historicItems = new List<HistoricContentVersionMeta>
{
new HistoricContentVersionMeta(versionId: 1, contentId: 1, contentTypeId: 1, versionDate: DateTime.Today.AddHours(-3)),
new HistoricContentVersionMeta(versionId: 2, contentId: 1, contentTypeId: 1, versionDate: DateTime.Today.AddHours(-2)),
new HistoricContentVersionMeta(versionId: 3, contentId: 1, contentTypeId: 1, versionDate: DateTime.Today.AddHours(-1)),
new HistoricContentVersionMeta(versionId: 4, contentId: 1, contentTypeId: 1, versionDate: DateTime.Today.AddDays(-1).AddHours(-3)),
new HistoricContentVersionMeta(versionId: 5, contentId: 1, contentTypeId: 1, versionDate: DateTime.Today.AddDays(-1).AddHours(-2)),
new HistoricContentVersionMeta(versionId: 6, contentId: 1, contentTypeId: 1, versionDate: DateTime.Today.AddDays(-1).AddHours(-1)),
// another content
new HistoricContentVersionMeta(versionId: 7, contentId: 2, contentTypeId: 1, versionDate: DateTime.Today.AddHours(-3)),
new HistoricContentVersionMeta(versionId: 8, contentId: 2, contentTypeId: 1, versionDate: DateTime.Today.AddHours(-2)),
new HistoricContentVersionMeta(versionId: 9, contentId: 2, contentTypeId: 1, versionDate: DateTime.Today.AddHours(-1)),
};
globalSettings.Setup(x => x.ContentVersionCleanupPolicyGlobalSettings)
.Returns(new TestCleanupSettings(true, 0, 3));
documentVersionRepository.Setup(x => x.GetCleanupPolicies())
.Returns(Array.Empty<ContentVersionCleanupPolicySettings>());
documentVersionRepository.Setup(x => x.GetDocumentVersionsEligibleForCleanup())
.Returns(historicItems);
var results = sut.Apply(DateTime.Today, historicItems).ToList();
Assert.Multiple(() =>
{
Assert.AreEqual(3, results.Count);
Assert.True(results.Exists(x => x.VersionId == 3));
Assert.True(results.Exists(x => x.VersionId == 6));
Assert.True(results.Exists(x => x.VersionId == 9));
});
}
[Test, AutoMoqData]
public void Apply_HasOverridePolicy_RespectsPreventCleanup(
[Frozen] Mock<IDocumentVersionRepository> documentVersionRepository,
[Frozen] Mock<IContentSection> globalSettings,
DefaultContentVersionCleanupPolicy sut)
{
var historicItems = new List<HistoricContentVersionMeta>
{
new HistoricContentVersionMeta(versionId: 1, contentId: 1, contentTypeId: 1, versionDate: DateTime.Today.AddHours(-3)),
new HistoricContentVersionMeta(versionId: 2, contentId: 1, contentTypeId: 1, versionDate: DateTime.Today.AddHours(-2)),
new HistoricContentVersionMeta(versionId: 3, contentId: 1, contentTypeId: 1, versionDate: DateTime.Today.AddHours(-1)),
// another content & type
new HistoricContentVersionMeta(versionId: 4, contentId: 2, contentTypeId: 2, versionDate: DateTime.Today.AddHours(-3)),
new HistoricContentVersionMeta(versionId: 5, contentId: 2, contentTypeId: 2, versionDate: DateTime.Today.AddHours(-2)),
new HistoricContentVersionMeta(versionId: 6, contentId: 2, contentTypeId: 2, versionDate: DateTime.Today.AddHours(-1)),
};
globalSettings.Setup(x => x.ContentVersionCleanupPolicyGlobalSettings)
.Returns(new TestCleanupSettings(true, 0, 0));
documentVersionRepository.Setup(x => x.GetCleanupPolicies())
.Returns(new ContentVersionCleanupPolicySettings[]
{
new ContentVersionCleanupPolicySettings{ ContentTypeId = 2, PreventCleanup = true }
});
documentVersionRepository.Setup(x => x.GetDocumentVersionsEligibleForCleanup())
.Returns(historicItems);
var results = sut.Apply(DateTime.Today, historicItems).ToList();
Assert.True(results.All(x => x.ContentTypeId == 1));
}
[Test, AutoMoqData]
public void Apply_HasOverridePolicy_RespectsKeepAll(
[Frozen] Mock<IDocumentVersionRepository> documentVersionRepository,
[Frozen] Mock<IContentSection> globalSettings,
DefaultContentVersionCleanupPolicy sut)
{
var historicItems = new List<HistoricContentVersionMeta>
{
new HistoricContentVersionMeta(versionId: 1, contentId: 1, contentTypeId: 1, versionDate: DateTime.Today.AddHours(-3)),
new HistoricContentVersionMeta(versionId: 2, contentId: 1, contentTypeId: 1, versionDate: DateTime.Today.AddHours(-2)),
new HistoricContentVersionMeta(versionId: 3, contentId: 1, contentTypeId: 1, versionDate: DateTime.Today.AddHours(-1)),
// another content & type
new HistoricContentVersionMeta(versionId: 4, contentId: 2, contentTypeId: 2, versionDate: DateTime.Today.AddHours(-3)),
new HistoricContentVersionMeta(versionId: 5, contentId: 2, contentTypeId: 2, versionDate: DateTime.Today.AddHours(-2)),
new HistoricContentVersionMeta(versionId: 6, contentId: 2, contentTypeId: 2, versionDate: DateTime.Today.AddHours(-1)),
};
globalSettings.Setup(x => x.ContentVersionCleanupPolicyGlobalSettings)
.Returns(new TestCleanupSettings(true, 0, 0));
documentVersionRepository.Setup(x => x.GetCleanupPolicies())
.Returns(new ContentVersionCleanupPolicySettings[]
{
new ContentVersionCleanupPolicySettings{ ContentTypeId = 2, PreventCleanup = false, KeepAllVersionsNewerThanDays = 3 }
});
documentVersionRepository.Setup(x => x.GetDocumentVersionsEligibleForCleanup())
.Returns(historicItems);
var results = sut.Apply(DateTime.Today, historicItems).ToList();
Assert.True(results.All(x => x.ContentTypeId == 1));
}
[Test, AutoMoqData]
public void Apply_HasOverridePolicy_RespectsKeepLatest(
[Frozen] Mock<IDocumentVersionRepository> documentVersionRepository,
[Frozen] Mock<IContentSection> globalSettings,
DefaultContentVersionCleanupPolicy sut)
{
var historicItems = new List<HistoricContentVersionMeta>
{
new HistoricContentVersionMeta(versionId: 1, contentId: 1, contentTypeId: 1, versionDate: DateTime.Today.AddHours(-3)),
new HistoricContentVersionMeta(versionId: 2, contentId: 1, contentTypeId: 1, versionDate: DateTime.Today.AddHours(-2)),
new HistoricContentVersionMeta(versionId: 3, contentId: 1, contentTypeId: 1, versionDate: DateTime.Today.AddHours(-1)),
// another content & type
new HistoricContentVersionMeta(versionId: 4, contentId: 2, contentTypeId: 2, versionDate: DateTime.Today.AddHours(-3)),
new HistoricContentVersionMeta(versionId: 5, contentId: 2, contentTypeId: 2, versionDate: DateTime.Today.AddHours(-2)),
new HistoricContentVersionMeta(versionId: 6, contentId: 2, contentTypeId: 2, versionDate: DateTime.Today.AddHours(-1)),
};
globalSettings.Setup(x => x.ContentVersionCleanupPolicyGlobalSettings)
.Returns(new TestCleanupSettings(true, 0, 0));
documentVersionRepository.Setup(x => x.GetCleanupPolicies())
.Returns(new ContentVersionCleanupPolicySettings[]
{
new ContentVersionCleanupPolicySettings{ ContentTypeId = 2, PreventCleanup = false, KeepLatestVersionPerDayForDays = 3 }
});
documentVersionRepository.Setup(x => x.GetDocumentVersionsEligibleForCleanup())
.Returns(historicItems);
var results = sut.Apply(DateTime.Today, historicItems).ToList();
Assert.Multiple(() =>
{
Assert.AreEqual(3, results.Count(x => x.ContentTypeId == 1));
Assert.AreEqual(6, results.Single(x => x.ContentTypeId == 2).VersionId);
});
}
class TestCleanupSettings : IContentVersionCleanupPolicyGlobalSettings
{
public bool EnableCleanup { get; set; }
public int KeepAllVersionsNewerThanDays { get; set; }
public int KeepLatestVersionPerDayForDays { get; set; }
public TestCleanupSettings() { }
public TestCleanupSettings(bool enable, int keepDays, int keepLatestDays)
{
EnableCleanup = enable;
KeepAllVersionsNewerThanDays = keepDays;
KeepLatestVersionPerDayForDays = keepLatestDays;
}
}
}
}

View File

@@ -19,7 +19,15 @@ namespace Umbraco.Tests.TestHelpers.Entities
public static Content CreateSimpleContent(IContentType contentType)
{
var content = new Content("Home", -1, contentType) { Level = 1, SortOrder = 1, CreatorId = 0, WriterId = 0 };
var content = new Content("Home", -1, contentType)
{
Level = 1,
SortOrder = 1,
CreatorId = 0,
WriterId = 0,
Key = Guid.NewGuid()
};
object obj =
new
{

View File

@@ -216,6 +216,7 @@ namespace Umbraco.Tests.TestHelpers.Entities
contentType.SortOrder = 1;
contentType.CreatorId = 0;
contentType.Trashed = false;
contentType.Key = Guid.NewGuid();
var contentCollection = new PropertyTypeCollection(true);
contentCollection.Add(new PropertyType(Constants.PropertyEditors.Aliases.TextBox, ValueStorageType.Ntext) { Alias = RandomAlias("title", randomizeAliases), Name = "Title", Description = "", Mandatory = false, SortOrder = 1, DataTypeId = -88, LabelOnTop = true });

View File

@@ -0,0 +1,13 @@
using AutoFixture;
using AutoFixture.AutoMoq;
using AutoFixture.NUnit3;
namespace Umbraco.Tests.Testing
{
public class AutoMoqDataAttribute : AutoDataAttribute
{
public AutoMoqDataAttribute()
: base(() => new Fixture().Customize(new AutoMoqCustomization{ ConfigureMembers = true }))
{ }
}
}

View File

@@ -78,6 +78,8 @@
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="AutoFixture.NUnit3" Version="4.17.0" />
<PackageReference Include="AutoFixture.AutoMoq" Version="4.17.0" />
<PackageReference Include="Castle.Core" Version="4.4.1" />
<PackageReference Include="Examine" Version="1.2.0" />
<PackageReference Include="HtmlAgilityPack">
@@ -100,10 +102,11 @@
<PackageReference Include="Microsoft.Owin.Testing" Version="4.0.1" />
<PackageReference Include="Microsoft.Web.Infrastructure" Version="1.0.0.0" />
<PackageReference Include="MiniProfiler" Version="4.0.138" />
<PackageReference Include="Moq" Version="4.14.5" />
<PackageReference Include="Moq" Version="4.16.1" />
<PackageReference Include="NPoco" Version="3.9.4" />
<PackageReference Include="NUnit" Version="3.11.0" />
<PackageReference Include="NUnit3TestAdapter" Version="3.12.0" />
<PackageReference Include="NUnit" Version="3.13.2" />
<PackageReference Include="NUnit3TestAdapter" Version="4.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.5.0" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.1" />
<PackageReference Include="Owin" Version="1.0" />
<PackageReference Include="Selenium.WebDriver" Version="3.141.0" />
@@ -150,6 +153,7 @@
<Compile Include="Models\VariationTests.cs" />
<Compile Include="Persistence\Mappers\MapperTestBase.cs" />
<Compile Include="Persistence\Repositories\DocumentRepositoryTest.cs" />
<Compile Include="Persistence\Repositories\DocumentVersionRepository_Tests_Integration.cs" />
<Compile Include="Persistence\Repositories\EntityRepositoryTest.cs" />
<Compile Include="PropertyEditors\BlockEditorComponentTests.cs" />
<Compile Include="PropertyEditors\BlockListPropertyValueConverterTests.cs" />
@@ -166,13 +170,17 @@
<Compile Include="Routing\RoutableDocumentFilterTests.cs" />
<Compile Include="Runtimes\StandaloneTests.cs" />
<Compile Include="Routing\GetContentUrlsTests.cs" />
<Compile Include="Scheduling\ContentVersionCleanup_Tests_UnitTests.cs" />
<Compile Include="Serialization\AutoInterningStringConverterTests.cs" />
<Compile Include="Scoping\ScopeUnitTests.cs" />
<Compile Include="Services\AmbiguousEventTests.cs" />
<Compile Include="Services\ContentServiceEventTests.cs" />
<Compile Include="Services\ContentServicePublishBranchTests.cs" />
<Compile Include="Services\ContentServiceTagsTests.cs" />
<Compile Include="Services\ContentVersionCleanupService_Tests_Integration.cs" />
<Compile Include="Services\ContentVersionCleanupService_Tests_UnitTests.cs" />
<Compile Include="Services\ContentTypeServiceVariantsTests.cs" />
<Compile Include="Services\DefaultContentVersionCleanupPolicy_Tests_UnitTests.cs" />
<Compile Include="Services\EntityXmlSerializerTests.cs" />
<Compile Include="Packaging\CreatedPackagesRepositoryTests.cs" />
<Compile Include="Services\ExternalLoginServiceTests.cs" />
@@ -183,6 +191,7 @@
<Compile Include="Templates\HtmlLocalLinkParserTests.cs" />
<Compile Include="TestHelpers\RandomIdRamDirectory.cs" />
<Compile Include="TestHelpers\TestSyncBootStateAccessor.cs" />
<Compile Include="Testing\AutoMoqDataAttribute.cs" />
<Compile Include="Testing\Objects\TestDataSource.cs" />
<Compile Include="Published\PublishedSnapshotTestObjects.cs" />
<Compile Include="Published\ModelTypeTests.cs" />

View File

@@ -191,7 +191,7 @@
<!-- You can specify your own logo for the login screen here. This path is relative to the ~/umbraco path. The default location is: /umbraco/assets/img/application/umbraco_logo_white.svg -->
<loginLogoImage>assets/img/application/umbraco_logo_white.svg</loginLogoImage>
<contentVersionCleanupPolicy enable="true" keepAllVersionsNewerThanDays="2" keepLatestVersionPerDayForDays="30" />
<contentVersionCleanupPolicyGlobalSettings enable="true" keepAllVersionsNewerThanDays="2" keepLatestVersionPerDayForDays="30" />
</content>
<security>

View File

@@ -0,0 +1,79 @@
using System;
using Umbraco.Core;
using Umbraco.Core.Configuration.UmbracoSettings;
using Umbraco.Core.Logging;
using Umbraco.Core.Services;
using Umbraco.Core.Sync;
namespace Umbraco.Web.Scheduling
{
internal class ContentVersionCleanup : RecurringTaskBase
{
private readonly IRuntimeState _runtimeState;
private readonly IProfilingLogger _logger;
private readonly IContentVersionCleanupPolicyGlobalSettings _settings;
private readonly IContentVersionCleanupService _cleanupService;
public ContentVersionCleanup(
IBackgroundTaskRunner<RecurringTaskBase> runner,
long delayMilliseconds,
long periodMilliseconds,
IRuntimeState runtimeState,
IProfilingLogger logger,
IContentVersionCleanupPolicyGlobalSettings settings,
IContentVersionCleanupService cleanupService)
: base(runner, delayMilliseconds, periodMilliseconds)
{
_runtimeState = runtimeState ?? throw new ArgumentNullException(nameof(runtimeState));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_settings = settings ?? throw new ArgumentNullException(nameof(settings));
_cleanupService = cleanupService ?? throw new ArgumentNullException(nameof(cleanupService));
}
public override bool PerformRun()
{
// Globally disabled by feature flag
if (!_settings.EnableCleanup)
{
_logger.Info<ContentVersionCleanup>("ContentVersionCleanup task will not run as it has been globally disabled via configuration.");
return false;
}
if (_runtimeState.Level != RuntimeLevel.Run)
{
return true; // repeat...
}
switch (_runtimeState.ServerRole)
{
case ServerRole.Replica:
_logger.Debug<ContentVersionCleanup>("Does not run on replica servers.");
return true; // DO repeat, server role can change
case ServerRole.Unknown:
_logger.Debug<ContentVersionCleanup>("Does not run on servers with unknown role.");
return true; // DO repeat, server role can change
case ServerRole.Single:
case ServerRole.Master:
default:
break;
}
// Ensure we do not run if not main domain, but do NOT lock it
if (!_runtimeState.IsMainDom)
{
_logger.Debug<ContentVersionCleanup>("Does not run if not MainDom.");
return false; // do NOT repeat, going down
}
_logger.Info<ContentVersionCleanup>("Starting ContentVersionCleanup task.");
var report = _cleanupService.PerformContentVersionCleanup(DateTime.Now);
_logger.Info<ContentVersionCleanup>("Finished ContentVersionCleanup task. Removed {count} item(s).", report.Count);
return true;
}
public override bool IsAsync => false;
}
}

View File

@@ -25,6 +25,7 @@ namespace Umbraco.Web.Scheduling
private readonly IRuntimeState _runtime;
private readonly IContentService _contentService;
private readonly IContentVersionCleanupService _cleanupService;
private readonly IAuditService _auditService;
private readonly IProfilingLogger _logger;
private readonly IScopeProvider _scopeProvider;
@@ -38,18 +39,26 @@ namespace Umbraco.Web.Scheduling
private BackgroundTaskRunner<IBackgroundTask> _scrubberRunner;
private BackgroundTaskRunner<IBackgroundTask> _fileCleanupRunner;
private BackgroundTaskRunner<IBackgroundTask> _healthCheckRunner;
private BackgroundTaskRunner<IBackgroundTask> _contentVersionCleanupRunner;
private bool _started;
private object _locker = new object();
private IBackgroundTask[] _tasks;
public SchedulerComponent(IRuntimeState runtime,
IContentService contentService, IAuditService auditService,
HealthCheckCollection healthChecks, HealthCheckNotificationMethodCollection notifications,
IScopeProvider scopeProvider, IUmbracoContextFactory umbracoContextFactory, IProfilingLogger logger)
public SchedulerComponent(
IRuntimeState runtime,
IContentService contentService,
IContentVersionCleanupService cleanupService,
IAuditService auditService,
HealthCheckCollection healthChecks,
HealthCheckNotificationMethodCollection notifications,
IScopeProvider scopeProvider,
IUmbracoContextFactory umbracoContextFactory,
IProfilingLogger logger)
{
_runtime = runtime;
_contentService = contentService;
_cleanupService = cleanupService;
_auditService = auditService;
_scopeProvider = scopeProvider;
_logger = logger;
@@ -68,6 +77,7 @@ namespace Umbraco.Web.Scheduling
_scrubberRunner = new BackgroundTaskRunner<IBackgroundTask>("LogScrubber", _logger);
_fileCleanupRunner = new BackgroundTaskRunner<IBackgroundTask>("TempFileCleanup", _logger);
_healthCheckRunner = new BackgroundTaskRunner<IBackgroundTask>("HealthCheckNotifier", _logger);
_contentVersionCleanupRunner = new BackgroundTaskRunner<IBackgroundTask>("ContentVersionCleanup", _logger);
// we will start the whole process when a successful request is made
UmbracoModule.RouteAttempt += RegisterBackgroundTasksOnce;
@@ -107,6 +117,7 @@ namespace Umbraco.Web.Scheduling
tasks.Add(RegisterScheduledPublishing());
tasks.Add(RegisterLogScrubber(settings));
tasks.Add(RegisterTempFileCleanup());
tasks.Add(RegisterContentVersionCleanup(settings));
var healthCheckConfig = Current.Configs.HealthChecks();
if (healthCheckConfig.NotificationSettings.Enabled)
@@ -180,5 +191,23 @@ namespace Umbraco.Web.Scheduling
_fileCleanupRunner.TryAdd(task);
return task;
}
private IBackgroundTask RegisterContentVersionCleanup(IUmbracoSettingsSection settings)
{
// content version cleanup
// install on all, will only run on non-replica servers.
var task = new ContentVersionCleanup(
_contentVersionCleanupRunner,
DefaultDelayMilliseconds,
OneHourMilliseconds,
_runtime,
_logger,
settings.Content.ContentVersionCleanupPolicyGlobalSettings,
_cleanupService);
_contentVersionCleanupRunner.TryAdd(task);
return task;
}
}
}

View File

@@ -320,6 +320,7 @@
<Compile Include="Routing\IPublishedRouter.cs" />
<Compile Include="Routing\MediaUrlProviderCollection.cs" />
<Compile Include="Routing\MediaUrlProviderCollectionBuilder.cs" />
<Compile Include="Scheduling\ContentVersionCleanup.cs" />
<Compile Include="Scheduling\SimpleTask.cs" />
<Compile Include="Scheduling\TempFileCleanup.cs" />
<Compile Include="Search\BackgroundIndexRebuilder.cs" />