From b4be203b2adf62cfa2aa26f2f322bc0b9b1d18de Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 23 Oct 2014 18:52:11 +1000 Subject: [PATCH] Added a new dashboard to rebuild xml structures for content, members, media. I haven't put it in the release dashboard config for now. Also added methods to the examine controller to check the index integrity which we can use for th 7.3 task. --- .../xmldataintegrityreport.controller.js | 62 ++++++++++++++ .../developer/xmldataintegrityreport.html | 31 +++++++ src/Umbraco.Web.UI/config/Dashboard.config | 5 ++ .../Editors/BackOfficeController.cs | 4 + .../Editors/DashboardController.cs | 4 +- src/Umbraco.Web/Editors/DashboardSecurity.cs | 7 +- .../Editors/TemplateQueryController.cs | 27 ++---- src/Umbraco.Web/Umbraco.Web.csproj | 1 + .../ExamineManagementApiController.cs | 48 +++++++++++ src/Umbraco.Web/WebServices/TagsController.cs | 5 +- .../WebServices/XmlDataIntegrityController.cs | 82 +++++++++++++++++++ .../DataServices/UmbracoMediaService.cs | 3 +- 12 files changed, 252 insertions(+), 27 deletions(-) create mode 100644 src/Umbraco.Web.UI.Client/src/views/dashboard/developer/xmldataintegrityreport.controller.js create mode 100644 src/Umbraco.Web.UI.Client/src/views/dashboard/developer/xmldataintegrityreport.html create mode 100644 src/Umbraco.Web/WebServices/XmlDataIntegrityController.cs diff --git a/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/xmldataintegrityreport.controller.js b/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/xmldataintegrityreport.controller.js new file mode 100644 index 0000000000..09b638b05f --- /dev/null +++ b/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/xmldataintegrityreport.controller.js @@ -0,0 +1,62 @@ +function xmlDataIntegrityReportController($scope, umbRequestHelper, $log, $http, $q, $timeout) { + + function check(item) { + var action = item.check; + umbRequestHelper.resourcePromise( + $http.get(umbRequestHelper.getApiUrl("xmlDataIntegrityBaseUrl", action)), + 'Failed to retrieve data integrity status') + .then(function(result) { + item.checking = false; + item.invalid = result === "false"; + }); + } + + $scope.fix = function(item) { + var action = item.fix; + if (item.fix) { + if (confirm("This will cause all xml structures for this type to be rebuilt. " + + "Depending on how much content there is in your site this could take a while. " + + "It is not recommended to rebuild xml structures if they are not out of sync, during times of high website traffic " + + "or when editors are editing content.")) { + item.fixing = true; + umbRequestHelper.resourcePromise( + $http.post(umbRequestHelper.getApiUrl("xmlDataIntegrityBaseUrl", action)), + 'Failed to retrieve data integrity status') + .then(function (result) { + item.fixing = false; + item.invalid = result === "false"; + }); + } + } + } + + $scope.items = { + "contentXml": { + label: "Content in the cmsContentXml table", + checking: true, + fixing: false, + fix: "FixContentXmlTable", + check: "CheckContentXmlTable" + }, + "mediaXml": { + label: "Media in the cmsContentXml table", + checking: true, + fixing: false, + fix: "FixMediaXmlTable", + check: "CheckMediaXmlTable" + }, + "memberXml": { + label: "Members in the cmsContentXml table", + checking: true, + fixing: false, + fix: "FixMembersXmlTable", + check: "CheckMembersXmlTable" + } + }; + + for (var i in $scope.items) { + check($scope.items[i]); + } + +} +angular.module("umbraco").controller("Umbraco.Dashboard.XmlDataIntegrityReportController", xmlDataIntegrityReportController); \ No newline at end of file diff --git a/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/xmldataintegrityreport.html b/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/xmldataintegrityreport.html new file mode 100644 index 0000000000..edfac077c0 --- /dev/null +++ b/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/xmldataintegrityreport.html @@ -0,0 +1,31 @@ +
+ +

Xml Cache Data integrity

+ +
+ Loading... +
+ +

+ This checks the data integrity for the xml structures for content, media and members that are stored in the cmsContentXml table. + This does not check the data integrity of the xml cache file, only the xml structures stored in the database used to create the xml cache file. +

+
+ {{value.label}} ... + Checking... + Ok + Error +
+ +
+
+
+
+
+ + + +
\ No newline at end of file diff --git a/src/Umbraco.Web.UI/config/Dashboard.config b/src/Umbraco.Web.UI/config/Dashboard.config index f5389cdbf9..95c59431ec 100644 --- a/src/Umbraco.Web.UI/config/Dashboard.config +++ b/src/Umbraco.Web.UI/config/Dashboard.config @@ -36,6 +36,11 @@ views/dashboard/developer/examinemanagement.html + + + views/dashboard/developer/xmldataintegrityreport.html + +
diff --git a/src/Umbraco.Web/Editors/BackOfficeController.cs b/src/Umbraco.Web/Editors/BackOfficeController.cs index 10b6989328..0e7d48d3d9 100644 --- a/src/Umbraco.Web/Editors/BackOfficeController.cs +++ b/src/Umbraco.Web/Editors/BackOfficeController.cs @@ -232,6 +232,10 @@ namespace Umbraco.Web.Editors { "examineMgmtBaseUrl", Url.GetUmbracoApiServiceBaseUrl( controller => controller.GetIndexerDetails()) + }, + { + "xmlDataIntegrityBaseUrl", Url.GetUmbracoApiServiceBaseUrl( + controller => controller.CheckContentXmlTable()) } } }, diff --git a/src/Umbraco.Web/Editors/DashboardController.cs b/src/Umbraco.Web/Editors/DashboardController.cs index 518761172a..701cca0bb2 100644 --- a/src/Umbraco.Web/Editors/DashboardController.cs +++ b/src/Umbraco.Web/Editors/DashboardController.cs @@ -9,9 +9,7 @@ using Umbraco.Core.IO; namespace Umbraco.Web.Editors { - /// - /// The API controller used for getting entity objects, basic name, icon, id representation of any umbraco object - /// + [PluginController("UmbracoApi")] public class DashboardController : UmbracoAuthorizedJsonController { diff --git a/src/Umbraco.Web/Editors/DashboardSecurity.cs b/src/Umbraco.Web/Editors/DashboardSecurity.cs index 28725d6b40..a8f5e44ef2 100644 --- a/src/Umbraco.Web/Editors/DashboardSecurity.cs +++ b/src/Umbraco.Web/Editors/DashboardSecurity.cs @@ -1,4 +1,5 @@ using System; +using System.Globalization; using System.Linq; using Umbraco.Core; using Umbraco.Core.Configuration.Dashboard; @@ -16,7 +17,7 @@ namespace Umbraco.Web.Editors public static bool AuthorizeAccess(ISection dashboardSection, IUser user, ISectionService sectionService) { - if (user.Id.ToString() == 0.ToInvariantString()) + if (user.Id.ToString(CultureInfo.InvariantCulture) == 0.ToInvariantString()) { return true; } @@ -30,7 +31,7 @@ namespace Umbraco.Web.Editors public static bool AuthorizeAccess(IDashboardTab dashboardTab, IUser user, ISectionService sectionService) { - if (user.Id.ToString() == Constants.System.Root.ToInvariantString()) + if (user.Id.ToString(CultureInfo.InvariantCulture) == Constants.System.Root.ToInvariantString()) { return true; } @@ -44,7 +45,7 @@ namespace Umbraco.Web.Editors public static bool AuthorizeAccess(IDashboardControl dashboardTab, IUser user, ISectionService sectionService) { - if (user.Id.ToString() == Constants.System.Root.ToInvariantString()) + if (user.Id.ToString(CultureInfo.InvariantCulture) == Constants.System.Root.ToInvariantString()) { return true; } diff --git a/src/Umbraco.Web/Editors/TemplateQueryController.cs b/src/Umbraco.Web/Editors/TemplateQueryController.cs index da4b90607d..2c38287d9a 100644 --- a/src/Umbraco.Web/Editors/TemplateQueryController.cs +++ b/src/Umbraco.Web/Editors/TemplateQueryController.cs @@ -5,23 +5,14 @@ using Umbraco.Core.Models; using Umbraco.Web.Mvc; using Umbraco.Web.WebApi.Filters; using Umbraco.Web.WebApi; +using System; +using System.Diagnostics; +using Umbraco.Web.Dynamics; +using Umbraco.Web.Models.TemplateQuery; namespace Umbraco.Web.Editors { - using System; - using System.Diagnostics; - using System.Diagnostics.Eventing.Reader; - using System.Drawing; - - using global::umbraco; - using global::umbraco.cms.presentation; - - using Umbraco.Core.Persistence.DatabaseModelDefinitions; - using Umbraco.Web.Dynamics; - using Umbraco.Web.Models.TemplateQuery; - using System.Linq.Expressions; - - using Umbraco.Web.Models; + /// /// The API controller used for building content queries within the template @@ -39,7 +30,7 @@ namespace Umbraco.Web.Editors { } - private static readonly IEnumerable _terms = new List() + private static readonly IEnumerable Terms = new List() { new OperathorTerm("is", Operathor.Equals, new [] {"string"}), new OperathorTerm("is not", Operathor.NotEquals, new [] {"string"}), @@ -57,7 +48,7 @@ namespace Umbraco.Web.Editors new OperathorTerm("less than or equal to", Operathor.LessThanEqualTo, new [] {"int"}) }; - private static readonly IEnumerable _properties = new List() + private static readonly IEnumerable Properties = new List() { new PropertyModel() { Name = "Id", Alias = "Id", Type = "int" }, new PropertyModel() { Name = "Name", Alias = "Name", Type = "string" }, @@ -309,7 +300,7 @@ namespace Umbraco.Web.Editors /// public IEnumerable GetAllowedProperties() { - return _properties.OrderBy(x => x.Name); + return Properties.OrderBy(x => x.Name); } /// @@ -317,7 +308,7 @@ namespace Umbraco.Web.Editors /// public IEnumerable GetFilterConditions() { - return _terms; + return Terms; } diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 5a383b4445..0329b86ded 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -1863,6 +1863,7 @@ Component + Mvc\web.config diff --git a/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs b/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs index 3361bb43a6..b56c2b1dae 100644 --- a/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs +++ b/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs @@ -18,6 +18,54 @@ namespace Umbraco.Web.WebServices { public class ExamineManagementApiController : UmbracoAuthorizedApiController { + /// + /// Checks if the member internal index is consistent with the data stored in the database + /// + /// + [HttpGet] + public bool CheckMembersInternalIndex() + { + var total = Services.MemberService.Count(); + + var criteria = ExamineManager.Instance.SearchProviderCollection["InternalMemberSearcher"] + .CreateSearchCriteria().RawQuery("__IndexType:member"); + var totalIndexed = ExamineManager.Instance.SearchProviderCollection["InternalMemberSearcher"].Search(criteria); + + return total == totalIndexed.TotalItemCount; + } + + /// + /// Checks if the media internal index is consistent with the data stored in the database + /// + /// + [HttpGet] + public bool CheckMediaInternalIndex() + { + var total = Services.MediaService.Count(); + + var criteria = ExamineManager.Instance.SearchProviderCollection["InternalSearcher"] + .CreateSearchCriteria().RawQuery("__IndexType:media"); + var totalIndexed = ExamineManager.Instance.SearchProviderCollection["InternalSearcher"].Search(criteria); + + return total == totalIndexed.TotalItemCount; + } + + /// + /// Checks if the content internal index is consistent with the data stored in the database + /// + /// + [HttpGet] + public bool CheckContentInternalIndex() + { + var total = Services.ContentService.Count(); + + var criteria = ExamineManager.Instance.SearchProviderCollection["InternalSearcher"] + .CreateSearchCriteria().RawQuery("__IndexType:content"); + var totalIndexed = ExamineManager.Instance.SearchProviderCollection["InternalSearcher"].Search(criteria); + + return total == totalIndexed.TotalItemCount; + } + /// /// Get the details for indexers /// diff --git a/src/Umbraco.Web/WebServices/TagsController.cs b/src/Umbraco.Web/WebServices/TagsController.cs index e7f1323f5d..e314e0a609 100644 --- a/src/Umbraco.Web/WebServices/TagsController.cs +++ b/src/Umbraco.Web/WebServices/TagsController.cs @@ -1,9 +1,10 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using AutoMapper; +using Examine; +using Umbraco.Core.Persistence.SqlSyntax; using Umbraco.Web.Models; using Umbraco.Web.WebApi; diff --git a/src/Umbraco.Web/WebServices/XmlDataIntegrityController.cs b/src/Umbraco.Web/WebServices/XmlDataIntegrityController.cs new file mode 100644 index 0000000000..7dcbf6519d --- /dev/null +++ b/src/Umbraco.Web/WebServices/XmlDataIntegrityController.cs @@ -0,0 +1,82 @@ +using System; +using System.Web.Http; +using Umbraco.Core; +using Umbraco.Core.Models.Rdbms; +using Umbraco.Core.Persistence; +using Umbraco.Web.WebApi; + +namespace Umbraco.Web.WebServices +{ + + public class XmlDataIntegrityController : UmbracoAuthorizedApiController + { + [HttpPost] + public bool FixContentXmlTable() + { + Services.ContentService.RebuildXmlStructures(); + return CheckContentXmlTable(); + } + + [HttpPost] + public bool FixMediaXmlTable() + { + Services.MediaService.RebuildXmlStructures(); + return CheckMediaXmlTable(); + } + + [HttpPost] + public bool FixMembersXmlTable() + { + Services.MemberService.RebuildXmlStructures(); + return CheckMembersXmlTable(); + } + + [HttpGet] + public bool CheckContentXmlTable() + { + var totalPublished = Services.ContentService.CountPublished(); + var subQuery = new Sql() + .Select("Count(DISTINCT cmsContentXml.nodeId)") + .From() + .InnerJoin() + .On(left => left.NodeId, right => right.NodeId); + var totalXml = ApplicationContext.DatabaseContext.Database.ExecuteScalar(subQuery); + + return totalXml == totalPublished; + } + + [HttpGet] + public bool CheckMediaXmlTable() + { + var total = Services.MediaService.Count(); + var mediaObjectType = Guid.Parse(Constants.ObjectTypes.Media); + var subQuery = new Sql() + .Select("Count(*)") + .From() + .InnerJoin() + .On(left => left.NodeId, right => right.NodeId) + .Where(dto => dto.NodeObjectType == mediaObjectType); + var totalXml = ApplicationContext.DatabaseContext.Database.ExecuteScalar(subQuery); + + return totalXml == total; + } + + [HttpGet] + public bool CheckMembersXmlTable() + { + var total = Services.MemberService.Count(); + var memberObjectType = Guid.Parse(Constants.ObjectTypes.Member); + var subQuery = new Sql() + .Select("Count(*)") + .From() + .InnerJoin() + .On(left => left.NodeId, right => right.NodeId) + .Where(dto => dto.NodeObjectType == memberObjectType); + var totalXml = ApplicationContext.DatabaseContext.Database.ExecuteScalar(subQuery); + + return totalXml == total; + } + + + } +} \ No newline at end of file diff --git a/src/UmbracoExamine/DataServices/UmbracoMediaService.cs b/src/UmbracoExamine/DataServices/UmbracoMediaService.cs index bf9f204e35..77946ad9de 100644 --- a/src/UmbracoExamine/DataServices/UmbracoMediaService.cs +++ b/src/UmbracoExamine/DataServices/UmbracoMediaService.cs @@ -15,6 +15,7 @@ namespace UmbracoExamine.DataServices /// /// Data service used to query for media /// + [Obsolete("This should no longer be used, latest content will be indexed by using the IMediaService directly")] public class UmbracoMediaService : IMediaService { private readonly ServiceContext _services; @@ -39,7 +40,7 @@ namespace UmbracoExamine.DataServices /// /// /// - + [Obsolete("This should no longer be used, latest content will be indexed by using the IMediaService directly")] public XDocument GetLatestMediaByXpath(string xpath) { var xmlMedia = XDocument.Parse("");