From afc236316349dca7614c8086daa68f4634fcdcbb Mon Sep 17 00:00:00 2001 From: Martin Blackwell Date: Wed, 31 Aug 2016 16:22:41 +0100 Subject: [PATCH 01/57] Resolution for U4-8927 --- src/Umbraco.Core/Persistence/Migrations/MigrationRunner.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Core/Persistence/Migrations/MigrationRunner.cs b/src/Umbraco.Core/Persistence/Migrations/MigrationRunner.cs index 51751b07ad..af894c453c 100644 --- a/src/Umbraco.Core/Persistence/Migrations/MigrationRunner.cs +++ b/src/Umbraco.Core/Persistence/Migrations/MigrationRunner.cs @@ -171,8 +171,8 @@ namespace Umbraco.Core.Persistence.Migrations from migrationAttribute in migrationAttributes where migrationAttribute != null where - migrationAttribute.TargetVersion > currentVersionToCompare && - migrationAttribute.TargetVersion <= targetVersionToCompare && + migrationAttribute.TargetVersion > targetVersionToCompare && + migrationAttribute.TargetVersion <= currentVersionToCompare && migrationAttribute.ProductName == _productName && //filter if the migration specifies a minimum current version for which to execute (migrationAttribute.MinimumCurrentVersion == null || currentVersionToCompare >= migrationAttribute.MinimumCurrentVersion) From 5f263ad851f622ff0e03923ac398fe71ba697da5 Mon Sep 17 00:00:00 2001 From: AndyButland Date: Sun, 4 Jun 2017 14:34:37 +0200 Subject: [PATCH 02/57] Added check for X-Content-Type-Options and refactored existing HTTP header check to use common base class. --- .../Checks/Security/BaseHttpHeaderCheck.cs | 220 ++++++++++++++++++ .../Checks/Security/ClickJackingCheck.cs | 210 +---------------- .../Checks/Security/NoSniffCheck.cs | 15 ++ src/Umbraco.Web/Umbraco.Web.csproj | 2 + 4 files changed, 241 insertions(+), 206 deletions(-) create mode 100644 src/Umbraco.Web/HealthCheck/Checks/Security/BaseHttpHeaderCheck.cs create mode 100644 src/Umbraco.Web/HealthCheck/Checks/Security/NoSniffCheck.cs diff --git a/src/Umbraco.Web/HealthCheck/Checks/Security/BaseHttpHeaderCheck.cs b/src/Umbraco.Web/HealthCheck/Checks/Security/BaseHttpHeaderCheck.cs new file mode 100644 index 0000000000..165d636a6d --- /dev/null +++ b/src/Umbraco.Web/HealthCheck/Checks/Security/BaseHttpHeaderCheck.cs @@ -0,0 +1,220 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Text.RegularExpressions; +using System.Xml.Linq; +using System.Xml.XPath; +using Umbraco.Core.Configuration; +using Umbraco.Core.IO; +using Umbraco.Core.Services; + +namespace Umbraco.Web.HealthCheck.Checks.Security +{ + public abstract class BaseHttpHeaderCheck : HealthCheck + { + private readonly ILocalizedTextService _textService; + + private const string SetHeaderInConfigAction = "setHeaderInConfig"; + + private readonly string _header; + private readonly string _value; + private readonly string _localizedTextPrefix; + private readonly bool _metaTagOptionAvailable; + + public BaseHttpHeaderCheck(HealthCheckContext healthCheckContext, + string header, string value, string localizedTextPrefix, bool metaTagOptionAvailable) : base(healthCheckContext) + { + _textService = healthCheckContext.ApplicationContext.Services.TextService; + + _header = header; + _value = value; + _localizedTextPrefix = localizedTextPrefix; + _metaTagOptionAvailable = metaTagOptionAvailable; + } + + /// + /// Get the status for this health check + /// + /// + public override IEnumerable GetStatus() + { + //return the statuses + return new[] { CheckForHeader() }; + } + + /// + /// Executes the action and returns it's status + /// + /// + /// + public override HealthCheckStatus ExecuteAction(HealthCheckAction action) + { + switch (action.Alias) + { + case SetHeaderInConfigAction: + return SetHeaderInConfig(); + default: + throw new InvalidOperationException("HTTP Header action requested is either not executable or does not exist"); + } + } + + protected HealthCheckStatus CheckForHeader() + { + var message = string.Empty; + var success = false; + var url = HealthCheckContext.HttpContext.Request.Url; + + // Access the site home page and check for the click-jack protection header or meta tag + var serverVariables = HealthCheckContext.HttpContext.Request.ServerVariables; + var useSsl = GlobalSettings.UseSSL || serverVariables["SERVER_PORT"] == "443"; + var address = string.Format("http{0}://{1}:{2}", useSsl ? "s" : "", url.Host.ToLower(), url.Port); + var request = WebRequest.Create(address); + request.Method = "GET"; + try + { + var response = request.GetResponse(); + + // Check first for header + success = DoHttpHeadersContainHeader(response); + + // If not found, and available, check for meta-tag + if (success == false && _metaTagOptionAvailable) + { + success = DoMetaTagsContainKeyForHeader(response); + } + + message = success + ? _textService.Localize(string.Format("healthcheck/{0}CheckHeaderFound", _localizedTextPrefix)) + : _textService.Localize(string.Format("healthcheck/{0}CheckHeaderNotFound", _localizedTextPrefix)); + } + catch (Exception ex) + { + message = _textService.Localize("healthcheck/healthCheckInvalidUrl", new[] { address, ex.Message }); + } + + var actions = new List(); + if (success == false) + { + actions.Add(new HealthCheckAction(SetHeaderInConfigAction, Id) + { + Name = _textService.Localize("healthcheck/setHeaderInConfig"), + Description = _textService.Localize(string.Format("healthcheck/{0}SetHeaderInConfigDescription", _localizedTextPrefix)) + }); + } + + return + new HealthCheckStatus(message) + { + ResultType = success ? StatusResultType.Success : StatusResultType.Error, + Actions = actions + }; + } + + private bool DoHttpHeadersContainHeader(WebResponse response) + { + return response.Headers.AllKeys.Contains(_header); + } + + private bool DoMetaTagsContainKeyForHeader(WebResponse response) + { + using (var stream = response.GetResponseStream()) + { + if (stream == null) return false; + using (var reader = new StreamReader(stream)) + { + var html = reader.ReadToEnd(); + var metaTags = ParseMetaTags(html); + return metaTags.ContainsKey(_header); + } + } + } + + private static Dictionary ParseMetaTags(string html) + { + var regex = new Regex("() + .ToDictionary(m => m.Groups[1].Value, m => m.Groups[2].Value); + } + + private HealthCheckStatus SetHeaderInConfig() + { + var errorMessage = string.Empty; + var success = SaveHeaderToConfigFile(out errorMessage); + + if (success) + { + return + new HealthCheckStatus(_textService.Localize(string.Format("healthcheck/{0}SetHeaderInConfigSuccess", _localizedTextPrefix))) + { + ResultType = StatusResultType.Success + }; + } + + return + new HealthCheckStatus(_textService.Localize("healthcheck/setHeaderInConfigError", new [] { errorMessage })) + { + ResultType = StatusResultType.Error + }; + } + + private bool SaveHeaderToConfigFile(out string errorMessage) + { + try + { + // There don't look to be any useful classes defined in https://msdn.microsoft.com/en-us/library/system.web.configuration(v=vs.110).aspx + // for working with the customHeaders section, so working with the XML directly. + var configFile = IOHelper.MapPath("~/Web.config"); + var doc = XDocument.Load(configFile); + var systemWebServerElement = doc.XPathSelectElement("/configuration/system.webServer"); + var httpProtocolElement = systemWebServerElement.Element("httpProtocol"); + if (httpProtocolElement == null) + { + httpProtocolElement = new XElement("httpProtocol"); + systemWebServerElement.Add(httpProtocolElement); + } + + var customHeadersElement = httpProtocolElement.Element("customHeaders"); + if (customHeadersElement == null) + { + customHeadersElement = new XElement("customHeaders"); + httpProtocolElement.Add(customHeadersElement); + } + + var removeHeaderElement = customHeadersElement.Elements("remove") + .SingleOrDefault(x => x.Attribute("name") != null && + x.Attribute("name").Value == _value); + if (removeHeaderElement == null) + { + removeHeaderElement = new XElement("remove"); + removeHeaderElement.Add(new XAttribute("name", _header)); + customHeadersElement.Add(removeHeaderElement); + } + + var addHeaderElement = customHeadersElement.Elements("add") + .SingleOrDefault(x => x.Attribute("name") != null && + x.Attribute("name").Value == _header); + if (addHeaderElement == null) + { + addHeaderElement = new XElement("add"); + addHeaderElement.Add(new XAttribute("name", _header)); + addHeaderElement.Add(new XAttribute("value", _value)); + customHeadersElement.Add(addHeaderElement); + } + + doc.Save(configFile); + + errorMessage = string.Empty; + return true; + } + catch (Exception ex) + { + errorMessage = ex.Message; + return false; + } + } + } +} diff --git a/src/Umbraco.Web/HealthCheck/Checks/Security/ClickJackingCheck.cs b/src/Umbraco.Web/HealthCheck/Checks/Security/ClickJackingCheck.cs index 54f72dc88a..34ba7bee19 100644 --- a/src/Umbraco.Web/HealthCheck/Checks/Security/ClickJackingCheck.cs +++ b/src/Umbraco.Web/HealthCheck/Checks/Security/ClickJackingCheck.cs @@ -1,217 +1,15 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net; -using System.Text.RegularExpressions; -using System.Xml.Linq; -using System.Xml.XPath; -using Umbraco.Core.Configuration; -using Umbraco.Core.IO; -using Umbraco.Core.Services; - -namespace Umbraco.Web.HealthCheck.Checks.Security +namespace Umbraco.Web.HealthCheck.Checks.Security { [HealthCheck( "ED0D7E40-971E-4BE8-AB6D-8CC5D0A6A5B0", "Click-Jacking Protection", Description = "Checks if your site is allowed to be IFRAMEd by another site and thus would be susceptible to click-jacking.", Group = "Security")] - public class ClickJackingCheck : HealthCheck + public class ClickJackingCheck : BaseHttpHeaderCheck { - private readonly ILocalizedTextService _textService; - - private const string SetFrameOptionsHeaderInConfigActiobn = "setFrameOptionsHeaderInConfig"; - - private const string XFrameOptionsHeader = "X-Frame-Options"; - private const string XFrameOptionsValue = "sameorigin"; // Note can't use "deny" as that would prevent Umbraco itself using IFRAMEs - - public ClickJackingCheck(HealthCheckContext healthCheckContext) : base(healthCheckContext) + public ClickJackingCheck(HealthCheckContext healthCheckContext) + : base(healthCheckContext, "X-Frame-Options", "sameorigin", "clickJacking", true) { - _textService = healthCheckContext.ApplicationContext.Services.TextService; - } - - /// - /// Get the status for this health check - /// - /// - public override IEnumerable GetStatus() - { - //return the statuses - return new[] { CheckForFrameOptionsHeader() }; - } - - /// - /// Executes the action and returns it's status - /// - /// - /// - public override HealthCheckStatus ExecuteAction(HealthCheckAction action) - { - switch (action.Alias) - { - case SetFrameOptionsHeaderInConfigActiobn: - return SetFrameOptionsHeaderInConfig(); - default: - throw new InvalidOperationException("HttpsCheck action requested is either not executable or does not exist"); - } - } - - private HealthCheckStatus CheckForFrameOptionsHeader() - { - var message = string.Empty; - var success = false; - var url = HealthCheckContext.HttpContext.Request.Url; - - // Access the site home page and check for the click-jack protection header or meta tag - var serverVariables = HealthCheckContext.HttpContext.Request.ServerVariables; - var useSsl = GlobalSettings.UseSSL || serverVariables["SERVER_PORT"] == "443"; - var address = string.Format("http{0}://{1}:{2}", useSsl ? "s" : "", url.Host.ToLower(), url.Port); - var request = WebRequest.Create(address); - request.Method = "GET"; - try - { - var response = request.GetResponse(); - - // Check first for header - success = DoHeadersContainFrameOptions(response); - - // If not found, check for meta-tag - if (success == false) - { - success = DoMetaTagsContainFrameOptions(response); - } - - message = success - ? _textService.Localize("healthcheck/clickJackingCheckHeaderFound") - : _textService.Localize("healthcheck/clickJackingCheckHeaderNotFound"); - } - catch (Exception ex) - { - message = _textService.Localize("healthcheck/httpsCheckInvalidUrl", new[] { address, ex.Message }); - } - - var actions = new List(); - if (success == false) - { - actions.Add(new HealthCheckAction(SetFrameOptionsHeaderInConfigActiobn, Id) - { - Name = _textService.Localize("healthcheck/clickJackingSetHeaderInConfig"), - Description = _textService.Localize("healthcheck/clickJackingSetHeaderInConfigDescription") - }); - } - - return - new HealthCheckStatus(message) - { - ResultType = success ? StatusResultType.Success : StatusResultType.Error, - Actions = actions - }; - } - - private static bool DoHeadersContainFrameOptions(WebResponse response) - { - return response.Headers.AllKeys.Contains(XFrameOptionsHeader); - } - - private static bool DoMetaTagsContainFrameOptions(WebResponse response) - { - using (var stream = response.GetResponseStream()) - { - if (stream == null) return false; - using (var reader = new StreamReader(stream)) - { - var html = reader.ReadToEnd(); - var metaTags = ParseMetaTags(html); - return metaTags.ContainsKey(XFrameOptionsHeader); - } - } - } - - private static Dictionary ParseMetaTags(string html) - { - var regex = new Regex("() - .ToDictionary(m => m.Groups[1].Value, m => m.Groups[2].Value); - } - - private HealthCheckStatus SetFrameOptionsHeaderInConfig() - { - var errorMessage = string.Empty; - var success = SaveHeaderToConfigFile(out errorMessage); - - if (success) - { - return - new HealthCheckStatus(_textService.Localize("healthcheck/clickJackingSetHeaderInConfigSuccess")) - { - ResultType = StatusResultType.Success - }; - } - - return - new HealthCheckStatus(_textService.Localize("healthcheck/clickJackingSetHeaderInConfigError", new [] { errorMessage })) - { - ResultType = StatusResultType.Error - }; - } - - private static bool SaveHeaderToConfigFile(out string errorMessage) - { - try - { - // There don't look to be any useful classes defined in https://msdn.microsoft.com/en-us/library/system.web.configuration(v=vs.110).aspx - // for working with the customHeaders section, so working with the XML directly. - var configFile = IOHelper.MapPath("~/Web.config"); - var doc = XDocument.Load(configFile); - var systemWebServerElement = doc.XPathSelectElement("/configuration/system.webServer"); - var httpProtocolElement = systemWebServerElement.Element("httpProtocol"); - if (httpProtocolElement == null) - { - httpProtocolElement = new XElement("httpProtocol"); - systemWebServerElement.Add(httpProtocolElement); - } - - var customHeadersElement = httpProtocolElement.Element("customHeaders"); - if (customHeadersElement == null) - { - customHeadersElement = new XElement("customHeaders"); - httpProtocolElement.Add(customHeadersElement); - } - - var removeHeaderElement = customHeadersElement.Elements("remove") - .SingleOrDefault(x => x.Attribute("name") != null && - x.Attribute("name").Value == XFrameOptionsHeader); - if (removeHeaderElement == null) - { - removeHeaderElement = new XElement("remove"); - removeHeaderElement.Add(new XAttribute("name", XFrameOptionsHeader)); - customHeadersElement.Add(removeHeaderElement); - } - - var addHeaderElement = customHeadersElement.Elements("add") - .SingleOrDefault(x => x.Attribute("name") != null && - x.Attribute("name").Value == XFrameOptionsHeader); - if (addHeaderElement == null) - { - addHeaderElement = new XElement("add"); - addHeaderElement.Add(new XAttribute("name", XFrameOptionsHeader)); - addHeaderElement.Add(new XAttribute("value", XFrameOptionsValue)); - customHeadersElement.Add(addHeaderElement); - } - - doc.Save(configFile); - - errorMessage = string.Empty; - return true; - } - catch (Exception ex) - { - errorMessage = ex.Message; - return false; - } } } } diff --git a/src/Umbraco.Web/HealthCheck/Checks/Security/NoSniffCheck.cs b/src/Umbraco.Web/HealthCheck/Checks/Security/NoSniffCheck.cs new file mode 100644 index 0000000000..4982d9f272 --- /dev/null +++ b/src/Umbraco.Web/HealthCheck/Checks/Security/NoSniffCheck.cs @@ -0,0 +1,15 @@ +namespace Umbraco.Web.HealthCheck.Checks.Security +{ + [HealthCheck( + "1CF27DB3-EFC0-41D7-A1BB-EA912064E071", + "Content/MIME Sniffing Protection", + Description = "Checks that your site contains a header used to protect against MIME sniffing vulnerabilities.", + Group = "Security")] + public class NoSniffCheck : BaseHttpHeaderCheck + { + public NoSniffCheck(HealthCheckContext healthCheckContext) + : base(healthCheckContext, "X-Content-Type-Options", "nosniff", "noSniff", false) + { + } + } +} diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 0b5652e509..d6d622315f 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -325,6 +325,8 @@ + + From 6861453166e7498527925f292ee12e3097eb9a47 Mon Sep 17 00:00:00 2001 From: AndyButland Date: Sun, 4 Jun 2017 14:35:37 +0200 Subject: [PATCH 03/57] Added localised texts and renamed some to less specific names when used across checks --- src/Umbraco.Web.UI/umbraco/config/lang/en.xml | 11 ++++++++--- src/Umbraco.Web.UI/umbraco/config/lang/en_us.xml | 6 +++--- src/Umbraco.Web.UI/umbraco/config/lang/fr.xml | 6 +++--- src/Umbraco.Web.UI/umbraco/config/lang/nl.xml | 6 +++--- src/Umbraco.Web.UI/umbraco/config/lang/ru.xml | 6 +++--- src/Umbraco.Web.UI/umbraco/config/lang/zh.xml | 6 +++--- .../Checks/Security/ExcessiveHeadersCheck.cs | 10 +++++----- .../HealthCheck/Checks/Security/HttpsCheck.cs | 4 ++-- 8 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/Umbraco.Web.UI/umbraco/config/lang/en.xml b/src/Umbraco.Web.UI/umbraco/config/lang/en.xml index d99cd8f43f..78326ab051 100644 --- a/src/Umbraco.Web.UI/umbraco/config/lang/en.xml +++ b/src/Umbraco.Web.UI/umbraco/config/lang/en.xml @@ -1476,7 +1476,7 @@ To manage your website, simply open the Umbraco back office and start adding con Your site certificate was marked as valid. Certificate validation error: '%0%' - Error pinging the URL %0% - '%1%' + Error pinging the URL %0% - '%1%' You are currently %0% viewing the site using the HTTPS scheme. The appSetting 'umbracoUseSSL' is set to 'false' in your web.config file. Once you access this site using the HTTPS scheme, that should be set to 'true'. The appSetting 'umbracoUseSSL' is set to '%0%' in your web.config file, your cookies are %1% marked as secure. @@ -1516,10 +1516,15 @@ To manage your website, simply open the Umbraco back office and start adding con X-Frame-Options used to control whether a site can be IFRAMEd by another was found.]]> X-Frame-Options used to control whether a site can be IFRAMEd by another was not found.]]> - Set Header in Config + Set Header in Config Adds a value to the httpProtocol/customHeaders section of web.config to prevent the site being IFRAMEd by other websites. A setting to create a header preventing IFRAMEing of the site by other websites has been added to your web.config file. - Could not update web.config file. Error: %0% + Could not update web.config file. Error: %0% + + X-Content-Type-Options used to protect against MIME sniffing vulnerabilities was found.]]> + X-Content-Type-Options used to protect against MIME sniffing vulnerabilities was not found.]]> + Adds a value to the httpProtocol/customHeaders section of web.config to protect against MIME sniffing vulnerabilities. + A setting to create a header protecting against MIME sniffing vulnerabilities has been added to your web.config file. From 967178cb926fa743592f806c7e939ac838ce5cb6 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Mon, 9 Apr 2018 11:43:48 +0200 Subject: [PATCH 18/57] Manually applies U4-11155 fixed error when creating member groups #2545 --- src/Umbraco.Tests/UI/LegacyDialogTests.cs | 1 - .../umbraco.presentation/umbraco/create/MemberGroupTasks.cs | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Tests/UI/LegacyDialogTests.cs b/src/Umbraco.Tests/UI/LegacyDialogTests.cs index 304f8256b8..f256d9a349 100644 --- a/src/Umbraco.Tests/UI/LegacyDialogTests.cs +++ b/src/Umbraco.Tests/UI/LegacyDialogTests.cs @@ -5,7 +5,6 @@ using Umbraco.Web.UI; using umbraco; using umbraco.BusinessLogic; using umbraco.interfaces; -using Umbraco.Web.umbraco.presentation.umbraco.create; namespace Umbraco.Tests.UI { diff --git a/src/Umbraco.Web/umbraco.presentation/umbraco/create/MemberGroupTasks.cs b/src/Umbraco.Web/umbraco.presentation/umbraco/create/MemberGroupTasks.cs index 905aa42ca3..8525170d18 100644 --- a/src/Umbraco.Web/umbraco.presentation/umbraco/create/MemberGroupTasks.cs +++ b/src/Umbraco.Web/umbraco.presentation/umbraco/create/MemberGroupTasks.cs @@ -2,9 +2,10 @@ using System.Linq; using System.Web.Security; using umbraco.BusinessLogic; using umbraco.cms.businesslogic.member; +using Umbraco.Web; using Umbraco.Web.UI; -namespace Umbraco.Web.umbraco.presentation.umbraco.create +namespace umbraco { public class MemberGroupTasks : LegacyDialogTask { From 07fc6753f5bb915e4880142030ed76becfeb09b2 Mon Sep 17 00:00:00 2001 From: Dave Woestenborghs Date: Mon, 9 Apr 2018 14:04:02 +0200 Subject: [PATCH 19/57] U4-11149 fixed error when creating new xslt file (#2540) * U4-10659 added create task back for xslt files * Add fix to UI.Release.xml as well --- src/Umbraco.Web.UI/umbraco/config/create/UI.Release.xml | 1 + src/Umbraco.Web.UI/umbraco/config/create/UI.xml | 1 + 2 files changed, 2 insertions(+) diff --git a/src/Umbraco.Web.UI/umbraco/config/create/UI.Release.xml b/src/Umbraco.Web.UI/umbraco/config/create/UI.Release.xml index ab2b89c0fb..44f2758de5 100644 --- a/src/Umbraco.Web.UI/umbraco/config/create/UI.Release.xml +++ b/src/Umbraco.Web.UI/umbraco/config/create/UI.Release.xml @@ -19,6 +19,7 @@
Macro
/create/xslt.ascx + diff --git a/src/Umbraco.Web.UI/umbraco/config/create/UI.xml b/src/Umbraco.Web.UI/umbraco/config/create/UI.xml index b0fac8393b..c1fc5c06b9 100644 --- a/src/Umbraco.Web.UI/umbraco/config/create/UI.xml +++ b/src/Umbraco.Web.UI/umbraco/config/create/UI.xml @@ -19,6 +19,7 @@
Macro
/create/xslt.ascx + From 9b33c8a0e254d4cf6eb7f344433674c48cd52aaf Mon Sep 17 00:00:00 2001 From: Dave Woestenborghs Date: Mon, 9 Apr 2018 14:04:02 +0200 Subject: [PATCH 20/57] U4-11149 fixed error when creating new xslt file (#2540) * U4-10659 added create task back for xslt files * Add fix to UI.Release.xml as well --- src/Umbraco.Web.UI/umbraco/config/create/UI.Release.xml | 1 + src/Umbraco.Web.UI/umbraco/config/create/UI.xml | 1 + 2 files changed, 2 insertions(+) diff --git a/src/Umbraco.Web.UI/umbraco/config/create/UI.Release.xml b/src/Umbraco.Web.UI/umbraco/config/create/UI.Release.xml index ab2b89c0fb..44f2758de5 100644 --- a/src/Umbraco.Web.UI/umbraco/config/create/UI.Release.xml +++ b/src/Umbraco.Web.UI/umbraco/config/create/UI.Release.xml @@ -19,6 +19,7 @@
Macro
/create/xslt.ascx + diff --git a/src/Umbraco.Web.UI/umbraco/config/create/UI.xml b/src/Umbraco.Web.UI/umbraco/config/create/UI.xml index b0fac8393b..c1fc5c06b9 100644 --- a/src/Umbraco.Web.UI/umbraco/config/create/UI.xml +++ b/src/Umbraco.Web.UI/umbraco/config/create/UI.xml @@ -19,6 +19,7 @@
Macro
/create/xslt.ascx + From 354505fde38db84e0226cd6b0f94b5868b82738b Mon Sep 17 00:00:00 2001 From: Dave Woestenborghs Date: Mon, 9 Apr 2018 14:04:02 +0200 Subject: [PATCH 21/57] U4-11149 fixed error when creating new xslt file (#2540) * U4-10659 added create task back for xslt files * Add fix to UI.Release.xml as well --- src/Umbraco.Web.UI/umbraco/config/create/UI.Release.xml | 1 + src/Umbraco.Web.UI/umbraco/config/create/UI.xml | 1 + 2 files changed, 2 insertions(+) diff --git a/src/Umbraco.Web.UI/umbraco/config/create/UI.Release.xml b/src/Umbraco.Web.UI/umbraco/config/create/UI.Release.xml index ab2b89c0fb..44f2758de5 100644 --- a/src/Umbraco.Web.UI/umbraco/config/create/UI.Release.xml +++ b/src/Umbraco.Web.UI/umbraco/config/create/UI.Release.xml @@ -19,6 +19,7 @@
Macro
/create/xslt.ascx + diff --git a/src/Umbraco.Web.UI/umbraco/config/create/UI.xml b/src/Umbraco.Web.UI/umbraco/config/create/UI.xml index b0fac8393b..c1fc5c06b9 100644 --- a/src/Umbraco.Web.UI/umbraco/config/create/UI.xml +++ b/src/Umbraco.Web.UI/umbraco/config/create/UI.xml @@ -19,6 +19,7 @@
Macro
/create/xslt.ascx + From 5380b62504b47e83d73d49ff97d6b32d2782072e Mon Sep 17 00:00:00 2001 From: Claus Date: Mon, 9 Apr 2018 15:43:17 +0200 Subject: [PATCH 22/57] fixes U4-11195 Mapping complex properties slow down some queries --- src/Umbraco.Core/Services/ContentService.cs | 2 +- src/Umbraco.Core/Services/IContentService.cs | 16 +++---- src/Umbraco.Web/Editors/ContentController.cs | 20 +++++---- .../Models/Mapping/ContentModelMapper.cs | 2 +- .../Mapping/ContentPropertyBasicConverter.cs | 44 ++++++++++++++----- .../ContentPropertyDisplayConverter.cs | 28 ++++++------ .../Mapping/ContentPropertyDtoConverter.cs | 13 +++--- .../Mapping/ContentPropertyModelMapper.cs | 4 +- 8 files changed, 78 insertions(+), 51 deletions(-) diff --git a/src/Umbraco.Core/Services/ContentService.cs b/src/Umbraco.Core/Services/ContentService.cs index b46781f76d..33131bbb52 100644 --- a/src/Umbraco.Core/Services/ContentService.cs +++ b/src/Umbraco.Core/Services/ContentService.cs @@ -1375,7 +1375,7 @@ namespace Umbraco.Core.Services { DeleteBlueprintsOfTypes(new[] {contentTypeId}, userId); } - + /// /// Saves a single object /// diff --git a/src/Umbraco.Core/Services/IContentService.cs b/src/Umbraco.Core/Services/IContentService.cs index c575255eb2..6d0ca051a1 100644 --- a/src/Umbraco.Core/Services/IContentService.cs +++ b/src/Umbraco.Core/Services/IContentService.cs @@ -16,10 +16,10 @@ namespace Umbraco.Core.Services /// explicitly. These methods will replace the normal ones in IContentService in v8 and this will be removed. /// public interface IContentServiceOperations - { - //TODO: Remove this class in v8 - - //TODO: There's probably more that needs to be added like the EmptyRecycleBin, etc... + { + //TODO: Remove this class in v8 + + //TODO: There's probably more that needs to be added like the EmptyRecycleBin, etc... /// /// Saves a single object @@ -99,7 +99,7 @@ namespace Umbraco.Core.Services IEnumerable GetBlueprintsForContentTypes(params int[] documentTypeIds); IContent GetBlueprintById(int id); IContent GetBlueprintById(Guid id); - void SaveBlueprint(IContent content, int userId = 0); + void SaveBlueprint(IContent content, int userId = 0); void DeleteBlueprint(IContent content, int userId = 0); IContent CreateContentFromBlueprint(IContent blueprint, string name, int userId = 0); void DeleteBlueprintsOfType(int contentTypeId, int userId = 0); @@ -179,7 +179,7 @@ namespace Umbraco.Core.Services /// Alias of the /// Optional id of the user creating the content /// - IContent CreateContent(string name, Guid parentId, string contentTypeAlias, int userId = 0); + IContent CreateContent(string name, Guid parentId, string contentTypeAlias, int userId = 0); /// /// Creates an object using the alias of the @@ -644,7 +644,7 @@ namespace Umbraco.Core.Services /// /// /// True if sorting succeeded, otherwise False - bool Sort(IEnumerable items, int userId = 0, bool raiseEvents = true); + bool Sort(IEnumerable items, int userId = 0, bool raiseEvents = true); /// /// Sorts a collection of objects by updating the SortOrder according @@ -704,4 +704,4 @@ namespace Umbraco.Core.Services /// IContent CreateContentWithIdentity(string name, int parentId, string contentTypeAlias, int userId = 0); } -} \ No newline at end of file +} diff --git a/src/Umbraco.Web/Editors/ContentController.cs b/src/Umbraco.Web/Editors/ContentController.cs index be78638a9e..64e3318b4c 100644 --- a/src/Umbraco.Web/Editors/ContentController.cs +++ b/src/Umbraco.Web/Editors/ContentController.cs @@ -95,8 +95,8 @@ namespace Umbraco.Web.Editors public IEnumerable PostSaveUserGroupPermissions(UserGroupPermissionsSave saveModel) { if (saveModel.ContentId <= 0) throw new HttpResponseException(Request.CreateResponse(HttpStatusCode.NotFound)); - - //TODO: Should non-admins be alowed to set granular permissions? + + //TODO: Should non-admins be alowed to set granular permissions? var content = Services.ContentService.GetById(saveModel.ContentId); if (content == null) throw new HttpResponseException(Request.CreateResponse(HttpStatusCode.NotFound)); @@ -159,8 +159,8 @@ namespace Umbraco.Web.Editors if (contentId <= 0) throw new HttpResponseException(Request.CreateResponse(HttpStatusCode.NotFound)); var content = Services.ContentService.GetById(contentId); if (content == null) throw new HttpResponseException(Request.CreateResponse(HttpStatusCode.NotFound)); - - //TODO: Should non-admins be able to see detailed permissions? + + //TODO: Should non-admins be able to see detailed permissions? var allUserGroups = Services.UserService.GetAllUserGroups(); @@ -379,7 +379,7 @@ namespace Umbraco.Web.Editors /// /// Gets the children for the content id passed in /// - /// + /// [FilterAllowedOutgoingContent(typeof(IEnumerable>), "Items")] public PagedResult> GetChildren( int id, @@ -409,9 +409,13 @@ namespace Umbraco.Web.Editors return new PagedResult>(0, 0, 0); } + // Note that we're excluding mapping of complex properties here to ensure that getting a larger amount of + // children for listviews and other similar cases, will not make everything halt when it tries to deserialize a + // complex property such as Nested Content. var pagedResult = new PagedResult>(totalChildren, pageNumber, pageSize); - pagedResult.Items = children - .Select(Mapper.Map>); + pagedResult.Items = children.Select(content => + Mapper.Map>(content, + opts => { opts.Items["ExcludeComplexProperties"] = true; })); return pagedResult; } @@ -644,7 +648,7 @@ namespace Umbraco.Web.Editors ShowMessageForPublishStatus(publishStatus.Result, display); break; } - + //If the item is new and the operation was cancelled, we need to return a different // status code so the UI can handle it since it won't be able to redirect since there // is no Id to redirect to! diff --git a/src/Umbraco.Web/Models/Mapping/ContentModelMapper.cs b/src/Umbraco.Web/Models/Mapping/ContentModelMapper.cs index c54574a08a..36b9edc3f9 100644 --- a/src/Umbraco.Web/Models/Mapping/ContentModelMapper.cs +++ b/src/Umbraco.Web/Models/Mapping/ContentModelMapper.cs @@ -195,7 +195,7 @@ namespace Umbraco.Web.Models.Mapping { var parent = _contentService.Value.GetById(source.ParentId); path = parent == null ? "-1" : parent.Path; - } + } var permissions = svc.GetPermissionsForPath( //TODO: This is certainly not ideal usage here - perhaps the best way to deal with this in the future is diff --git a/src/Umbraco.Web/Models/Mapping/ContentPropertyBasicConverter.cs b/src/Umbraco.Web/Models/Mapping/ContentPropertyBasicConverter.cs index e7104f2939..89be6859d1 100644 --- a/src/Umbraco.Web/Models/Mapping/ContentPropertyBasicConverter.cs +++ b/src/Umbraco.Web/Models/Mapping/ContentPropertyBasicConverter.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using AutoMapper; using Umbraco.Core; using Umbraco.Core.Logging; @@ -13,11 +14,13 @@ namespace Umbraco.Web.Models.Mapping /// Creates a base generic ContentPropertyBasic from a Property /// /// - internal class ContentPropertyBasicConverter : TypeConverter + internal class ContentPropertyBasicConverter : ITypeConverter where T : ContentPropertyBasic, new() { protected IDataTypeService DataTypeService { get; private set; } + private static readonly List ComplexPropertyTypeAliases = new List {"Umbraco.NestedContent"}; + public ContentPropertyBasicConverter(IDataTypeService dataTypeService) { DataTypeService = dataTypeService; @@ -26,28 +29,45 @@ namespace Umbraco.Web.Models.Mapping /// /// Assigns the PropertyEditor, Id, Alias and Value to the property /// - /// /// - protected override T ConvertCore(Property property) + public T Convert(ResolutionContext context) { + var property = context.SourceValue as Property; + if (property == null) + throw new InvalidOperationException("Source value is not a property."); + var editor = PropertyEditorResolver.Current.GetByAlias(property.PropertyType.PropertyEditorAlias); if (editor == null) { LogHelper.Error>( "No property editor found, converting to a Label", - new NullReferenceException("The property editor with alias " + property.PropertyType.PropertyEditorAlias + " does not exist")); + new NullReferenceException("The property editor with alias " + + property.PropertyType.PropertyEditorAlias + " does not exist")); editor = PropertyEditorResolver.Current.GetByAlias(Constants.PropertyEditors.NoEditAlias); } - var result = new T - { - Id = property.Id, - Value = editor.ValueEditor.ConvertDbToEditor(property, property.PropertyType, DataTypeService), - Alias = property.Alias, - PropertyEditor = editor, - Editor = editor.Alias - }; + var result = new T + { + Id = property.Id, + Alias = property.Alias, + PropertyEditor = editor, + Editor = editor.Alias + }; + + // Complex properties such as Nested Content do not need to be mapped for simpler things like list views, + // where they will not make sense to use anyways. To avoid having to do unnecessary mapping on large + // collections of items in list views - we allow excluding mapping of certain properties. + var excludeComplexProperties = false; + if (context.Options.Items.ContainsKey("ExcludeComplexProperties")) + { + excludeComplexProperties = System.Convert.ToBoolean(context.Options.Items["ExcludeComplexProperties"]); + } + if (excludeComplexProperties == false || ComplexPropertyTypeAliases.Contains(property.PropertyType.PropertyEditorAlias) == false) + { + result.Value = editor.ValueEditor.ConvertDbToEditor(property, property.PropertyType, DataTypeService); + } + return result; } } diff --git a/src/Umbraco.Web/Models/Mapping/ContentPropertyDisplayConverter.cs b/src/Umbraco.Web/Models/Mapping/ContentPropertyDisplayConverter.cs index 8267e46e25..632268ab2f 100644 --- a/src/Umbraco.Web/Models/Mapping/ContentPropertyDisplayConverter.cs +++ b/src/Umbraco.Web/Models/Mapping/ContentPropertyDisplayConverter.cs @@ -1,7 +1,5 @@ using System; -using System.Linq; -using Umbraco.Core; -using Umbraco.Core.Configuration; +using AutoMapper; using Umbraco.Core.Models; using Umbraco.Core.Services; using Umbraco.Web.Models.ContentEditing; @@ -21,27 +19,31 @@ namespace Umbraco.Web.Models.Mapping _textService = textService; } - protected override ContentPropertyDisplay ConvertCore(Property originalProp) + public new ContentPropertyDisplay Convert(ResolutionContext context) { - var display = base.ConvertCore(originalProp); + var display = base.Convert(context); + + var originalProperty = context.SourceValue as Property; + if (originalProperty == null) + throw new InvalidOperationException("Source value is not a property."); var dataTypeService = DataTypeService; - var preVals = dataTypeService.GetPreValuesCollectionByDataTypeId(originalProp.PropertyType.DataTypeDefinitionId); + var preVals = dataTypeService.GetPreValuesCollectionByDataTypeId(originalProperty.PropertyType.DataTypeDefinitionId); //configure the editor for display with the pre-values var valEditor = display.PropertyEditor.ValueEditor; valEditor.ConfigureForDisplay(preVals); //set the display properties after mapping - display.Alias = originalProp.Alias; - display.Description = originalProp.PropertyType.Description; - display.Label = originalProp.PropertyType.Name; + display.Alias = originalProperty.Alias; + display.Description = originalProperty.PropertyType.Description; + display.Label = originalProperty.PropertyType.Name; display.HideLabel = valEditor.HideLabel; - + //add the validation information - display.Validation.Mandatory = originalProp.PropertyType.Mandatory; - display.Validation.Pattern = originalProp.PropertyType.ValidationRegExp; - + display.Validation.Mandatory = originalProperty.PropertyType.Mandatory; + display.Validation.Pattern = originalProperty.PropertyType.ValidationRegExp; + if (display.PropertyEditor == null) { //display.Config = PreValueCollection.AsDictionary(preVals); diff --git a/src/Umbraco.Web/Models/Mapping/ContentPropertyDtoConverter.cs b/src/Umbraco.Web/Models/Mapping/ContentPropertyDtoConverter.cs index f3f9fbe9d5..80b8c28309 100644 --- a/src/Umbraco.Web/Models/Mapping/ContentPropertyDtoConverter.cs +++ b/src/Umbraco.Web/Models/Mapping/ContentPropertyDtoConverter.cs @@ -1,7 +1,6 @@ using System; -using Umbraco.Core; +using AutoMapper; using Umbraco.Core.Models; -using Umbraco.Core.PropertyEditors; using Umbraco.Core.Services; using Umbraco.Web.Models.ContentEditing; @@ -17,9 +16,13 @@ namespace Umbraco.Web.Models.Mapping { } - protected override ContentPropertyDto ConvertCore(Property originalProperty) + public new ContentPropertyDto Convert(ResolutionContext context) { - var propertyDto = base.ConvertCore(originalProperty); + var propertyDto = base.Convert(context); + + var originalProperty = context.SourceValue as Property; + if (originalProperty == null) + throw new InvalidOperationException("Source value is not a property."); var dataTypeService = DataTypeService; @@ -27,7 +30,7 @@ namespace Umbraco.Web.Models.Mapping propertyDto.ValidationRegExp = originalProperty.PropertyType.ValidationRegExp; propertyDto.Description = originalProperty.PropertyType.Description; propertyDto.Label = originalProperty.PropertyType.Name; - + //TODO: We should be able to look both of these up at the same time! propertyDto.DataType = dataTypeService.GetDataTypeDefinitionById(originalProperty.PropertyType.DataTypeDefinitionId); propertyDto.PreValues = dataTypeService.GetPreValuesCollectionByDataTypeId(originalProperty.PropertyType.DataTypeDefinitionId); diff --git a/src/Umbraco.Web/Models/Mapping/ContentPropertyModelMapper.cs b/src/Umbraco.Web/Models/Mapping/ContentPropertyModelMapper.cs index 5e29bcf125..f401832020 100644 --- a/src/Umbraco.Web/Models/Mapping/ContentPropertyModelMapper.cs +++ b/src/Umbraco.Web/Models/Mapping/ContentPropertyModelMapper.cs @@ -1,9 +1,7 @@ -using System; -using AutoMapper; +using AutoMapper; using Umbraco.Core; using Umbraco.Core.Models; using Umbraco.Core.Models.Mapping; -using Umbraco.Core.Services; using Umbraco.Web.Models.ContentEditing; namespace Umbraco.Web.Models.Mapping From 6d052aeeac472328de0f1a4ff374c9bd59faea82 Mon Sep 17 00:00:00 2001 From: mikkelhm Date: Tue, 10 Apr 2018 08:56:50 +0200 Subject: [PATCH 23/57] Bump version to 7.9.4 --- src/SolutionInfo.cs | 4 ++-- src/Umbraco.Core/Configuration/UmbracoVersion.cs | 2 +- src/Umbraco.Web.UI/Umbraco.Web.UI.csproj | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/SolutionInfo.cs b/src/SolutionInfo.cs index dd901ab1d1..d4f4d5f7b8 100644 --- a/src/SolutionInfo.cs +++ b/src/SolutionInfo.cs @@ -11,5 +11,5 @@ using System.Resources; [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyFileVersion("7.9.3")] -[assembly: AssemblyInformationalVersion("7.9.3")] \ No newline at end of file +[assembly: AssemblyFileVersion("7.9.4")] +[assembly: AssemblyInformationalVersion("7.9.4")] \ No newline at end of file diff --git a/src/Umbraco.Core/Configuration/UmbracoVersion.cs b/src/Umbraco.Core/Configuration/UmbracoVersion.cs index 4a81e95c26..9cfe8c7d14 100644 --- a/src/Umbraco.Core/Configuration/UmbracoVersion.cs +++ b/src/Umbraco.Core/Configuration/UmbracoVersion.cs @@ -6,7 +6,7 @@ namespace Umbraco.Core.Configuration { public class UmbracoVersion { - private static readonly Version Version = new Version("7.9.3"); + private static readonly Version Version = new Version("7.9.4"); /// /// Gets the current version of Umbraco. diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index 9a0122ad83..01273388e9 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -1023,9 +1023,9 @@ xcopy "$(ProjectDir)"..\packages\SqlServerCE.4.0.0.1\x86\*.* "$(TargetDir)x86\" True True - 7930 + 7940 / - http://localhost:7930 + http://localhost:7940 False False From 4a47c5e12953c3307d8fa7cca76da30dc984fbae Mon Sep 17 00:00:00 2001 From: Stephan Date: Thu, 12 Apr 2018 11:13:00 +0200 Subject: [PATCH 24/57] U4-11207 - fix scope leaks caused by database messenger --- src/Umbraco.Core/Persistence/PetaPoco.cs | 4 ++-- src/Umbraco.Core/Scoping/Scope.cs | 2 +- src/Umbraco.Core/Scoping/ScopeProvider.cs | 17 +++++++++----- .../BatchedDatabaseServerMessenger.cs | 23 ++++++++++++++++--- 4 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/Umbraco.Core/Persistence/PetaPoco.cs b/src/Umbraco.Core/Persistence/PetaPoco.cs index 79b3ce3871..67f967f050 100644 --- a/src/Umbraco.Core/Persistence/PetaPoco.cs +++ b/src/Umbraco.Core/Persistence/PetaPoco.cs @@ -666,8 +666,8 @@ namespace Umbraco.Core.Persistence return ExecuteScalar(sql.SQL, sql.Arguments); } - Regex rxSelect = new Regex(@"\A\s*(SELECT|EXECUTE|CALL)\s", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase | RegexOptions.Multiline); - Regex rxFrom = new Regex(@"\A\s*FROM\s", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase | RegexOptions.Multiline); + static Regex rxSelect = new Regex(@"\A\s*(SELECT|EXECUTE|CALL)\s", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase | RegexOptions.Multiline); + static Regex rxFrom = new Regex(@"\A\s*FROM\s", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase | RegexOptions.Multiline); string AddSelectClause(string sql) { if (sql.StartsWith(";")) diff --git a/src/Umbraco.Core/Scoping/Scope.cs b/src/Umbraco.Core/Scoping/Scope.cs index bfee772872..ed23913719 100644 --- a/src/Umbraco.Core/Scoping/Scope.cs +++ b/src/Umbraco.Core/Scoping/Scope.cs @@ -374,7 +374,7 @@ namespace Umbraco.Core.Scoping } var parent = ParentScope; - _scopeProvider.AmbientScope = parent; + _scopeProvider.AmbientScope = parent; // might be null = this is how scopes are removed from context objects #if DEBUG_SCOPES _scopeProvider.Disposed(this); diff --git a/src/Umbraco.Core/Scoping/ScopeProvider.cs b/src/Umbraco.Core/Scoping/ScopeProvider.cs index dfcf7985ae..a9deb3465e 100644 --- a/src/Umbraco.Core/Scoping/ScopeProvider.cs +++ b/src/Umbraco.Core/Scoping/ScopeProvider.cs @@ -66,17 +66,22 @@ namespace Umbraco.Core.Scoping // tests, any other things (see https://msdn.microsoft.com/en-us/library/dn458353(v=vs.110).aspx), // but we don't want to make all of our objects serializable since they are *not* meant to be // used in cross-AppDomain scenario anyways. + // // in addition, whatever goes into the logical call context is serialized back and forth any - // time cross-AppDomain code executes, so if we put an "object" there, we'll can *another* - // "object" instance - and so we cannot use a random object as a key. + // time cross-AppDomain code executes, so if we put an "object" there, we'll get *another* + // "object" instance back - and so we cannot use a random object as a key. + // // so what we do is: we register a guid in the call context, and we keep a table mapping those // guids to the actual objects. the guid serializes back and forth without causing any issue, // and we can retrieve the actual objects from the table. - // only issue: how are we supposed to clear the table? we can't, really. objects should take - // care of de-registering themselves from context. - // everything we use does, except the NoScope scope, which just stays there // - // during tests, NoScope can to into call context... nothing much we can do about it + // so far, the only objects that go into this table are scopes (using ScopeItemKey) and + // scope contexts (using ContextItemKey). + // + // there is no automatic way to garbage-collect the table. Normal scopes and scope contexts + // takes great care removing themselves when disposed, so it is safe. OTOH the NoScope + // *CANNOT* remove itself, so it *WILL* leak, and there is nothing we can do about it. NoScope + // exists for backward compatibility reasons, but ... relying on it is greatly discouraged. private static readonly object StaticCallContextObjectsLock = new object(); private static readonly Dictionary StaticCallContextObjects diff --git a/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs b/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs index 973ba341f2..357f36b60a 100644 --- a/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs +++ b/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs @@ -9,6 +9,7 @@ using Umbraco.Core.Models.Rdbms; using Umbraco.Core.Sync; using Umbraco.Web.Routing; using Umbraco.Core.Logging; +using Umbraco.Core.Scoping; using Umbraco.Web.Scheduling; namespace Umbraco.Web @@ -21,9 +22,12 @@ namespace Umbraco.Web /// public class BatchedDatabaseServerMessenger : DatabaseServerMessenger { + private readonly ApplicationContext _appContext; + public BatchedDatabaseServerMessenger(ApplicationContext appContext, bool enableDistCalls, DatabaseServerMessengerOptions options) : base(appContext, enableDistCalls, options) { + _appContext = appContext; Scheduler.Initializing += Scheduler_Initializing; } @@ -42,7 +46,7 @@ namespace Umbraco.Web //start the background task runner for processing instructions const int delayMilliseconds = 60000; var instructionProcessingRunner = new BackgroundTaskRunner("InstructionProcessing", ApplicationContext.ProfilingLogger.Logger); - var instructionProcessingTask = new InstructionProcessing(instructionProcessingRunner, this, delayMilliseconds, Options.ThrottleSeconds * 1000); + var instructionProcessingTask = new InstructionProcessing(instructionProcessingRunner, this, _appContext.ScopeProvider, delayMilliseconds, Options.ThrottleSeconds * 1000); instructionProcessingRunner.TryAdd(instructionProcessingTask); e.Add(instructionProcessingTask); } @@ -73,18 +77,31 @@ namespace Umbraco.Web private class InstructionProcessing : RecurringTaskBase { private readonly DatabaseServerMessenger _messenger; + private readonly IScopeProvider _scopeProvider; public InstructionProcessing(IBackgroundTaskRunner runner, DatabaseServerMessenger messenger, + IScopeProvider scopeProvider, int delayMilliseconds, int periodMilliseconds) : base(runner, delayMilliseconds, periodMilliseconds) { _messenger = messenger; + _scopeProvider = scopeProvider; } public override bool PerformRun() { - _messenger.Sync(); + // beware! + // DatabaseServerMessenger uses _appContext.DatabaseContext.Database without creating + // scopes, and since we are running in a background task, there will be no ambient + // scope (as would be the case within a web request), and so we would end up creating + // (and leaking) a NoScope instance, which is bad - better make sure we have a true + // scope here! - see U4-11207 + using (var scope = _scopeProvider.CreateScope()) + { + _messenger.Sync(); + scope.Complete(); + } //return true to repeat return true; } @@ -191,4 +208,4 @@ namespace Umbraco.Web } } -} \ No newline at end of file +} From 1076b882fb6aaec9373b578763e584fbe64e42ce Mon Sep 17 00:00:00 2001 From: Stephan Date: Thu, 12 Apr 2018 11:56:56 +0200 Subject: [PATCH 25/57] U4-11207 - collect rogue/leaked NoScope instances --- src/Umbraco.Core/Scoping/NoScope.cs | 3 ++ src/Umbraco.Core/Scoping/ScopeProvider.cs | 47 ++++++++++++++++++++--- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/src/Umbraco.Core/Scoping/NoScope.cs b/src/Umbraco.Core/Scoping/NoScope.cs index a21815173c..73353ce3da 100644 --- a/src/Umbraco.Core/Scoping/NoScope.cs +++ b/src/Umbraco.Core/Scoping/NoScope.cs @@ -20,6 +20,7 @@ namespace Umbraco.Core.Scoping public NoScope(ScopeProvider scopeProvider) { _scopeProvider = scopeProvider; + Timestamp = DateTime.Now; #if DEBUG_SCOPES _scopeProvider.RegisterScope(this); #endif @@ -28,6 +29,8 @@ namespace Umbraco.Core.Scoping private readonly Guid _instanceId = Guid.NewGuid(); public Guid InstanceId { get { return _instanceId; } } + public DateTime Timestamp { get; } + /// public bool CallContext { get { return false; } } diff --git a/src/Umbraco.Core/Scoping/ScopeProvider.cs b/src/Umbraco.Core/Scoping/ScopeProvider.cs index a9deb3465e..5ad2fe3b0a 100644 --- a/src/Umbraco.Core/Scoping/ScopeProvider.cs +++ b/src/Umbraco.Core/Scoping/ScopeProvider.cs @@ -2,10 +2,12 @@ using System.Collections; using System.Collections.Generic; using System.Data; +using System.Linq; using System.Runtime.Remoting.Messaging; using System.Text; using System.Web; using Umbraco.Core.Events; +using Umbraco.Core.Logging; using Umbraco.Core.Persistence; #if DEBUG_SCOPES using System.Linq; @@ -77,16 +79,27 @@ namespace Umbraco.Core.Scoping // // so far, the only objects that go into this table are scopes (using ScopeItemKey) and // scope contexts (using ContextItemKey). - // - // there is no automatic way to garbage-collect the table. Normal scopes and scope contexts - // takes great care removing themselves when disposed, so it is safe. OTOH the NoScope - // *CANNOT* remove itself, so it *WILL* leak, and there is nothing we can do about it. NoScope - // exists for backward compatibility reasons, but ... relying on it is greatly discouraged. private static readonly object StaticCallContextObjectsLock = new object(); private static readonly Dictionary StaticCallContextObjects = new Dictionary(); + // normal scopes and scope contexts take greate care removing themselves when disposed, so it + // is all safe. OTOH the NoScope *CANNOT* remove itself, this is by design, it *WILL* leak and + // there is little (nothing) we can do about it - NoScope exists for backward compatibility + // reasons and relying on it is greatly discouraged. + // + // however... we can *try* at protecting the app against memory leaks, by collecting NoScope + // instances that are too old. if anything actually *need* to retain a NoScope instance for + // a long time, it will break. but that's probably ok. so: the constants below define how + // long a NoScope instance can stay in the table before being removed, and how often we should + // collect the table - and collecting happens anytime SetCallContextObject is invoked + + private static readonly TimeSpan StaticCallContextNoScopeLifeSpan = TimeSpan.FromMinutes(30); + private static readonly TimeSpan StaticCallContextCollectPeriod = TimeSpan.FromMinutes(4); + private static DateTime _staticCallContextLastCollect = DateTime.MinValue; + + #if DEBUG_SCOPES public Dictionary CallContextObjects { @@ -161,6 +174,7 @@ namespace Umbraco.Core.Scoping //Logging.LogHelper.Debug("At:\r\n" + Head(Environment.StackTrace, 24)); #endif StaticCallContextObjects.Remove(objectKey); + CollectStaticCallContextObjectsLocked(); } } else @@ -176,11 +190,34 @@ namespace Umbraco.Core.Scoping //Logging.LogHelper.Debug("At:\r\n" + Head(Environment.StackTrace, 24)); #endif StaticCallContextObjects.Add(objectKey, value); + CollectStaticCallContextObjectsLocked(); } CallContext.LogicalSetData(key, objectKey); } } + private static void CollectStaticCallContextObjectsLocked() + { + // is it time to collect? + var now = DateTime.Now; + if (now - _staticCallContextLastCollect <= StaticCallContextCollectPeriod) + return; + + // disable warning: this method is invoked from within a lock + // ReSharper disable InconsistentlySynchronizedField + var threshold = now.Add(-StaticCallContextNoScopeLifeSpan); + var guids = StaticCallContextObjects + .Where(x => x.Value is NoScope noScope && noScope.Timestamp < threshold) + .Select(x => x.Key) + .ToList(); + LogHelper.Warn($"Collected {guids.Count} NoScope instances from StaticCallContextObjects."); + foreach (var guid in guids) + StaticCallContextObjects.Remove(guid); + // ReSharper restore InconsistentlySynchronizedField + + _staticCallContextLastCollect = now; + } + // this is for tests exclusively until we have a proper accessor in v8 internal static Func HttpContextItemsGetter { get; set; } From 28cbb0a51705953a3b27253377d3a550eb275e5d Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Thu, 12 Apr 2018 13:41:56 +0200 Subject: [PATCH 26/57] Bumps version to 7.11.0 --- src/SolutionInfo.cs | 4 ++-- src/Umbraco.Core/Configuration/UmbracoVersion.cs | 2 +- src/Umbraco.Web.UI/Umbraco.Web.UI.csproj | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/SolutionInfo.cs b/src/SolutionInfo.cs index cba378547a..a3a3080dd3 100644 --- a/src/SolutionInfo.cs +++ b/src/SolutionInfo.cs @@ -11,5 +11,5 @@ using System.Resources; [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyFileVersion("7.10.2")] -[assembly: AssemblyInformationalVersion("7.10.2")] \ No newline at end of file +[assembly: AssemblyFileVersion("7.11.0")] +[assembly: AssemblyInformationalVersion("7.11.0")] \ No newline at end of file diff --git a/src/Umbraco.Core/Configuration/UmbracoVersion.cs b/src/Umbraco.Core/Configuration/UmbracoVersion.cs index 434aa154ad..7eca275901 100644 --- a/src/Umbraco.Core/Configuration/UmbracoVersion.cs +++ b/src/Umbraco.Core/Configuration/UmbracoVersion.cs @@ -6,7 +6,7 @@ namespace Umbraco.Core.Configuration { public class UmbracoVersion { - private static readonly Version Version = new Version("7.10.2"); + private static readonly Version Version = new Version("7.11.0"); /// /// Gets the current version of Umbraco. diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index b2917620c3..1577442ede 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -1035,9 +1035,9 @@ xcopy "$(ProjectDir)"..\packages\SqlServerCE.4.0.0.1\x86\*.* "$(TargetDir)x86\" True True - 7102 + 7110 / - http://localhost:7102 + http://localhost:7110 False False From c630e5ef8e122473b57e2561299d4f66f6a3dea0 Mon Sep 17 00:00:00 2001 From: Claus Date: Thu, 12 Apr 2018 14:26:14 +0200 Subject: [PATCH 27/57] only warn if there's actually anything to warn about --- src/Umbraco.Core/Scoping/ScopeProvider.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Umbraco.Core/Scoping/ScopeProvider.cs b/src/Umbraco.Core/Scoping/ScopeProvider.cs index 5ad2fe3b0a..8b0f84b5d0 100644 --- a/src/Umbraco.Core/Scoping/ScopeProvider.cs +++ b/src/Umbraco.Core/Scoping/ScopeProvider.cs @@ -210,9 +210,12 @@ namespace Umbraco.Core.Scoping .Where(x => x.Value is NoScope noScope && noScope.Timestamp < threshold) .Select(x => x.Key) .ToList(); - LogHelper.Warn($"Collected {guids.Count} NoScope instances from StaticCallContextObjects."); - foreach (var guid in guids) - StaticCallContextObjects.Remove(guid); + if (guids.Count > 0) + { + LogHelper.Warn($"Collected {guids.Count} NoScope instances from StaticCallContextObjects."); + foreach (var guid in guids) + StaticCallContextObjects.Remove(guid); + } // ReSharper restore InconsistentlySynchronizedField _staticCallContextLastCollect = now; From d9b1598fa2a156d9c354732889ddc0789093845e Mon Sep 17 00:00:00 2001 From: Claus Date: Thu, 12 Apr 2018 14:26:47 +0200 Subject: [PATCH 28/57] use scope when writing instructions. --- .../BatchedDatabaseServerMessenger.cs | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs b/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs index 357f36b60a..6862986ec2 100644 --- a/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs +++ b/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs @@ -138,14 +138,17 @@ namespace Umbraco.Web batch.Clear(); //Write the instructions but only create JSON blobs with a max instruction count equal to MaxProcessingInstructionCount - foreach (var instructionsBatch in instructions.InGroupsOf(Options.MaxProcessingInstructionCount)) + using (var scope = _appContext.ScopeProvider.CreateScope()) { - WriteInstructions(instructionsBatch); + foreach (var instructionsBatch in instructions.InGroupsOf(Options.MaxProcessingInstructionCount)) + { + WriteInstructions(scope, instructionsBatch); + } + scope.Complete(); } - } - private void WriteInstructions(IEnumerable instructions) + private void WriteInstructions(IScope scope, IEnumerable instructions) { var dto = new CacheInstructionDto { @@ -154,8 +157,7 @@ namespace Umbraco.Web OriginIdentity = LocalIdentity, InstructionCount = instructions.Sum(x => x.JsonIdCount) }; - - ApplicationContext.DatabaseContext.Database.Insert(dto); + scope.Database.Insert(dto); } protected ICollection GetBatch(bool create) @@ -196,16 +198,19 @@ namespace Umbraco.Web if (batch == null) { //only write the json blob with a maximum count of the MaxProcessingInstructionCount - foreach (var maxBatch in instructions.InGroupsOf(Options.MaxProcessingInstructionCount)) + using (var scope = _appContext.ScopeProvider.CreateScope()) { - WriteInstructions(maxBatch); + foreach (var maxBatch in instructions.InGroupsOf(Options.MaxProcessingInstructionCount)) + { + WriteInstructions(scope, maxBatch); + } + scope.Complete(); } } else { batch.Add(new RefreshInstructionEnvelope(servers, refresher, instructions)); } - - } + } } } From 99ce7c07363d07ce7b943b401423add9e4124931 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Thu, 12 Apr 2018 14:35:34 +0200 Subject: [PATCH 29/57] Throw error when "downgrading" to a higher version, make sure to give the results back in the correct order (newest version first, oldest last). --- src/Umbraco.Core/Persistence/Migrations/MigrationRunner.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Umbraco.Core/Persistence/Migrations/MigrationRunner.cs b/src/Umbraco.Core/Persistence/Migrations/MigrationRunner.cs index af894c453c..7c3e131b4c 100644 --- a/src/Umbraco.Core/Persistence/Migrations/MigrationRunner.cs +++ b/src/Umbraco.Core/Persistence/Migrations/MigrationRunner.cs @@ -166,6 +166,9 @@ namespace Umbraco.Core.Persistence.Migrations var targetVersionToCompare = _targetVersion.GetVersion(3); var currentVersionToCompare = _currentVersion.GetVersion(3); + if(currentVersionToCompare <= targetVersionToCompare) + throw new ArgumentException("When downgrading, make sure that the target version is lower than the current version."); + var migrations = (from migration in foundMigrations let migrationAttributes = migration.GetType().GetCustomAttributes(false) from migrationAttribute in migrationAttributes @@ -176,7 +179,7 @@ namespace Umbraco.Core.Persistence.Migrations migrationAttribute.ProductName == _productName && //filter if the migration specifies a minimum current version for which to execute (migrationAttribute.MinimumCurrentVersion == null || currentVersionToCompare >= migrationAttribute.MinimumCurrentVersion) - orderby migrationAttribute.TargetVersion, migrationAttribute.SortOrder descending + orderby migrationAttribute.TargetVersion descending, migrationAttribute.SortOrder descending select migration).Distinct(); return migrations; } From 81c5c66e15b88669e1a03ee19703548228631149 Mon Sep 17 00:00:00 2001 From: Claus Date: Thu, 12 Apr 2018 14:41:55 +0200 Subject: [PATCH 30/57] adding readonly. --- src/Umbraco.Core/Persistence/PetaPoco.cs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/Umbraco.Core/Persistence/PetaPoco.cs b/src/Umbraco.Core/Persistence/PetaPoco.cs index 67f967f050..4692cb4b51 100644 --- a/src/Umbraco.Core/Persistence/PetaPoco.cs +++ b/src/Umbraco.Core/Persistence/PetaPoco.cs @@ -402,7 +402,7 @@ namespace Umbraco.Core.Persistence } // Helper to handle named parameters from object properties - static Regex rxParams = new Regex(@"(? args_dest) { return rxParams.Replace(_sql, m => @@ -545,7 +545,7 @@ namespace Umbraco.Core.Persistence } // Create a command - static Regex rxParamsPrefix = new Regex(@"(?(sql.SQL, sql.Arguments); } - static Regex rxSelect = new Regex(@"\A\s*(SELECT|EXECUTE|CALL)\s", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase | RegexOptions.Multiline); - static Regex rxFrom = new Regex(@"\A\s*FROM\s", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase | RegexOptions.Multiline); + static readonly Regex rxSelect = new Regex(@"\A\s*(SELECT|EXECUTE|CALL)\s", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase | RegexOptions.Multiline); + static readonly Regex rxFrom = new Regex(@"\A\s*FROM\s", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase | RegexOptions.Multiline); string AddSelectClause(string sql) { if (sql.StartsWith(";")) @@ -701,9 +701,9 @@ namespace Umbraco.Core.Persistence return Fetch(sql.SQL, sql.Arguments); } - static Regex rxColumns = new Regex(@"\A\s*SELECT\s+((?:\((?>\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|.)*?)(?\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|[\w\(\)\.])+(?:\s+(?:ASC|DESC))?(?:\s*,\s*(?:\((?>\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|[\w\(\)\.])+(?:\s+(?:ASC|DESC))?)*", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.Compiled); - static Regex rxDistinct = new Regex(@"\ADISTINCT\s", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.Compiled); + static readonly Regex rxColumns = new Regex(@"\A\s*SELECT\s+((?:\((?>\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|.)*?)(?\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|[\w\(\)\.])+(?:\s+(?:ASC|DESC))?(?:\s*,\s*(?:\((?>\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|[\w\(\)\.])+(?:\s+(?:ASC|DESC))?)*", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.Compiled); + static readonly Regex rxDistinct = new Regex(@"\ADISTINCT\s", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.Compiled); public static bool SplitSqlForPaging(string sql, out string sqlCount, out string sqlSelectRemoved, out string sqlOrderBy) { sqlSelectRemoved = null; @@ -2546,5 +2546,4 @@ namespace Umbraco.Core.Persistence } } } - } From 52701f0056a834ff120855ca417bff6cad16165e Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Thu, 12 Apr 2018 15:30:02 +0200 Subject: [PATCH 31/57] Fix unit tests that were trying to "downgrade" from version 1 to version 2 --- .../Persistence/Migrations/MigrationStartupHandlerTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Umbraco.Tests/Persistence/Migrations/MigrationStartupHandlerTests.cs b/src/Umbraco.Tests/Persistence/Migrations/MigrationStartupHandlerTests.cs index dd35f9ad3c..83fac7a786 100644 --- a/src/Umbraco.Tests/Persistence/Migrations/MigrationStartupHandlerTests.cs +++ b/src/Umbraco.Tests/Persistence/Migrations/MigrationStartupHandlerTests.cs @@ -32,7 +32,7 @@ namespace Umbraco.Tests.Persistence.Migrations conn.Setup(x => x.BeginTransaction(It.IsAny())).Returns(Mock.Of()); var db = new Mock(conn.Object); - var runner1 = new MigrationRunner(Mock.Of(), Mock.Of(), new SemVersion(1), new SemVersion(2), "Test1", + var runner1 = new MigrationRunner(Mock.Of(), Mock.Of(), new SemVersion(2), new SemVersion(1), "Test1", new IMigration[] { Mock.Of() }); var result1 = runner1.Execute(db.Object, DatabaseProviders.SqlServerCE, false); Assert.AreEqual(1, changed1.CountExecuted); @@ -52,13 +52,13 @@ namespace Umbraco.Tests.Persistence.Migrations conn.Setup(x => x.BeginTransaction(It.IsAny())).Returns(Mock.Of()); var db = new Mock(conn.Object); - var runner1 = new MigrationRunner(Mock.Of(), Mock.Of(), new SemVersion(1), new SemVersion(2), "Test1", + var runner1 = new MigrationRunner(Mock.Of(), Mock.Of(), new SemVersion(2), new SemVersion(1), "Test1", new IMigration[] { Mock.Of()}); var result1 = runner1.Execute(db.Object, DatabaseProviders.SqlServerCE, false); Assert.AreEqual(1, changed1.CountExecuted); Assert.AreEqual(0, changed2.CountExecuted); - var runner2 = new MigrationRunner(Mock.Of(), Mock.Of(), new SemVersion(1), new SemVersion(2), "Test2", + var runner2 = new MigrationRunner(Mock.Of(), Mock.Of(), new SemVersion(2), new SemVersion(1), "Test2", new IMigration[] { Mock.Of() }); var result2 = runner2.Execute(db.Object, DatabaseProviders.SqlServerCE, false); Assert.AreEqual(1, changed1.CountExecuted); From 828dbaedfb048e953fa0f647b11fade0f9890055 Mon Sep 17 00:00:00 2001 From: Bjarne Fyrstenborg Date: Thu, 12 Apr 2018 15:37:01 +0200 Subject: [PATCH 32/57] Indicate that a row is clickable and highlight with background color. (#1473) --- .../src/less/components/umb-grid.less | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/less/components/umb-grid.less b/src/Umbraco.Web.UI.Client/src/less/components/umb-grid.less index 5cc4817142..107c14ad86 100644 --- a/src/Umbraco.Web.UI.Client/src/less/components/umb-grid.less +++ b/src/Umbraco.Web.UI.Client/src/less/components/umb-grid.less @@ -8,7 +8,7 @@ -// Sortabel +// Sortable // ------------------------- // sortable-helper @@ -127,6 +127,16 @@ position: relative; margin-bottom: 40px; padding-top: 10px; + + &:hover { + background-color: @grayLighter; + } + + &[ng-click], + &[data-ng-click], + &[x-ng-click] { + cursor: pointer; + } } .umb-grid .row-tools a { @@ -431,7 +441,7 @@ background-color: @gray-10; .umb-row-title-bar { - cursor: default; + cursor: inherit; } .umb-row-title { From 19241995e867330983c7d6fe4bfa5c5445ed512f Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Thu, 12 Apr 2018 16:23:33 +0200 Subject: [PATCH 33/57] Cherry picked - Fix scope leaks caused by database messenger [U4-11207] #2580 --- src/Umbraco.Core/Persistence/PetaPoco.cs | 15 +++-- src/Umbraco.Core/Scoping/NoScope.cs | 3 + src/Umbraco.Core/Scoping/Scope.cs | 2 +- src/Umbraco.Core/Scoping/ScopeProvider.cs | 57 +++++++++++++++++-- .../BatchedDatabaseServerMessenger.cs | 48 +++++++++++----- 5 files changed, 97 insertions(+), 28 deletions(-) diff --git a/src/Umbraco.Core/Persistence/PetaPoco.cs b/src/Umbraco.Core/Persistence/PetaPoco.cs index 79b3ce3871..4692cb4b51 100644 --- a/src/Umbraco.Core/Persistence/PetaPoco.cs +++ b/src/Umbraco.Core/Persistence/PetaPoco.cs @@ -402,7 +402,7 @@ namespace Umbraco.Core.Persistence } // Helper to handle named parameters from object properties - static Regex rxParams = new Regex(@"(? args_dest) { return rxParams.Replace(_sql, m => @@ -545,7 +545,7 @@ namespace Umbraco.Core.Persistence } // Create a command - static Regex rxParamsPrefix = new Regex(@"(?(sql.SQL, sql.Arguments); } - Regex rxSelect = new Regex(@"\A\s*(SELECT|EXECUTE|CALL)\s", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase | RegexOptions.Multiline); - Regex rxFrom = new Regex(@"\A\s*FROM\s", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase | RegexOptions.Multiline); + static readonly Regex rxSelect = new Regex(@"\A\s*(SELECT|EXECUTE|CALL)\s", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase | RegexOptions.Multiline); + static readonly Regex rxFrom = new Regex(@"\A\s*FROM\s", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase | RegexOptions.Multiline); string AddSelectClause(string sql) { if (sql.StartsWith(";")) @@ -701,9 +701,9 @@ namespace Umbraco.Core.Persistence return Fetch(sql.SQL, sql.Arguments); } - static Regex rxColumns = new Regex(@"\A\s*SELECT\s+((?:\((?>\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|.)*?)(?\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|[\w\(\)\.])+(?:\s+(?:ASC|DESC))?(?:\s*,\s*(?:\((?>\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|[\w\(\)\.])+(?:\s+(?:ASC|DESC))?)*", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.Compiled); - static Regex rxDistinct = new Regex(@"\ADISTINCT\s", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.Compiled); + static readonly Regex rxColumns = new Regex(@"\A\s*SELECT\s+((?:\((?>\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|.)*?)(?\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|[\w\(\)\.])+(?:\s+(?:ASC|DESC))?(?:\s*,\s*(?:\((?>\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|[\w\(\)\.])+(?:\s+(?:ASC|DESC))?)*", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.Compiled); + static readonly Regex rxDistinct = new Regex(@"\ADISTINCT\s", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.Compiled); public static bool SplitSqlForPaging(string sql, out string sqlCount, out string sqlSelectRemoved, out string sqlOrderBy) { sqlSelectRemoved = null; @@ -2546,5 +2546,4 @@ namespace Umbraco.Core.Persistence } } } - } diff --git a/src/Umbraco.Core/Scoping/NoScope.cs b/src/Umbraco.Core/Scoping/NoScope.cs index a21815173c..73353ce3da 100644 --- a/src/Umbraco.Core/Scoping/NoScope.cs +++ b/src/Umbraco.Core/Scoping/NoScope.cs @@ -20,6 +20,7 @@ namespace Umbraco.Core.Scoping public NoScope(ScopeProvider scopeProvider) { _scopeProvider = scopeProvider; + Timestamp = DateTime.Now; #if DEBUG_SCOPES _scopeProvider.RegisterScope(this); #endif @@ -28,6 +29,8 @@ namespace Umbraco.Core.Scoping private readonly Guid _instanceId = Guid.NewGuid(); public Guid InstanceId { get { return _instanceId; } } + public DateTime Timestamp { get; } + /// public bool CallContext { get { return false; } } diff --git a/src/Umbraco.Core/Scoping/Scope.cs b/src/Umbraco.Core/Scoping/Scope.cs index bfee772872..ed23913719 100644 --- a/src/Umbraco.Core/Scoping/Scope.cs +++ b/src/Umbraco.Core/Scoping/Scope.cs @@ -374,7 +374,7 @@ namespace Umbraco.Core.Scoping } var parent = ParentScope; - _scopeProvider.AmbientScope = parent; + _scopeProvider.AmbientScope = parent; // might be null = this is how scopes are removed from context objects #if DEBUG_SCOPES _scopeProvider.Disposed(this); diff --git a/src/Umbraco.Core/Scoping/ScopeProvider.cs b/src/Umbraco.Core/Scoping/ScopeProvider.cs index dfcf7985ae..8b0f84b5d0 100644 --- a/src/Umbraco.Core/Scoping/ScopeProvider.cs +++ b/src/Umbraco.Core/Scoping/ScopeProvider.cs @@ -2,10 +2,12 @@ using System.Collections; using System.Collections.Generic; using System.Data; +using System.Linq; using System.Runtime.Remoting.Messaging; using System.Text; using System.Web; using Umbraco.Core.Events; +using Umbraco.Core.Logging; using Umbraco.Core.Persistence; #if DEBUG_SCOPES using System.Linq; @@ -66,22 +68,38 @@ namespace Umbraco.Core.Scoping // tests, any other things (see https://msdn.microsoft.com/en-us/library/dn458353(v=vs.110).aspx), // but we don't want to make all of our objects serializable since they are *not* meant to be // used in cross-AppDomain scenario anyways. + // // in addition, whatever goes into the logical call context is serialized back and forth any - // time cross-AppDomain code executes, so if we put an "object" there, we'll can *another* - // "object" instance - and so we cannot use a random object as a key. + // time cross-AppDomain code executes, so if we put an "object" there, we'll get *another* + // "object" instance back - and so we cannot use a random object as a key. + // // so what we do is: we register a guid in the call context, and we keep a table mapping those // guids to the actual objects. the guid serializes back and forth without causing any issue, // and we can retrieve the actual objects from the table. - // only issue: how are we supposed to clear the table? we can't, really. objects should take - // care of de-registering themselves from context. - // everything we use does, except the NoScope scope, which just stays there // - // during tests, NoScope can to into call context... nothing much we can do about it + // so far, the only objects that go into this table are scopes (using ScopeItemKey) and + // scope contexts (using ContextItemKey). private static readonly object StaticCallContextObjectsLock = new object(); private static readonly Dictionary StaticCallContextObjects = new Dictionary(); + // normal scopes and scope contexts take greate care removing themselves when disposed, so it + // is all safe. OTOH the NoScope *CANNOT* remove itself, this is by design, it *WILL* leak and + // there is little (nothing) we can do about it - NoScope exists for backward compatibility + // reasons and relying on it is greatly discouraged. + // + // however... we can *try* at protecting the app against memory leaks, by collecting NoScope + // instances that are too old. if anything actually *need* to retain a NoScope instance for + // a long time, it will break. but that's probably ok. so: the constants below define how + // long a NoScope instance can stay in the table before being removed, and how often we should + // collect the table - and collecting happens anytime SetCallContextObject is invoked + + private static readonly TimeSpan StaticCallContextNoScopeLifeSpan = TimeSpan.FromMinutes(30); + private static readonly TimeSpan StaticCallContextCollectPeriod = TimeSpan.FromMinutes(4); + private static DateTime _staticCallContextLastCollect = DateTime.MinValue; + + #if DEBUG_SCOPES public Dictionary CallContextObjects { @@ -156,6 +174,7 @@ namespace Umbraco.Core.Scoping //Logging.LogHelper.Debug("At:\r\n" + Head(Environment.StackTrace, 24)); #endif StaticCallContextObjects.Remove(objectKey); + CollectStaticCallContextObjectsLocked(); } } else @@ -171,11 +190,37 @@ namespace Umbraco.Core.Scoping //Logging.LogHelper.Debug("At:\r\n" + Head(Environment.StackTrace, 24)); #endif StaticCallContextObjects.Add(objectKey, value); + CollectStaticCallContextObjectsLocked(); } CallContext.LogicalSetData(key, objectKey); } } + private static void CollectStaticCallContextObjectsLocked() + { + // is it time to collect? + var now = DateTime.Now; + if (now - _staticCallContextLastCollect <= StaticCallContextCollectPeriod) + return; + + // disable warning: this method is invoked from within a lock + // ReSharper disable InconsistentlySynchronizedField + var threshold = now.Add(-StaticCallContextNoScopeLifeSpan); + var guids = StaticCallContextObjects + .Where(x => x.Value is NoScope noScope && noScope.Timestamp < threshold) + .Select(x => x.Key) + .ToList(); + if (guids.Count > 0) + { + LogHelper.Warn($"Collected {guids.Count} NoScope instances from StaticCallContextObjects."); + foreach (var guid in guids) + StaticCallContextObjects.Remove(guid); + } + // ReSharper restore InconsistentlySynchronizedField + + _staticCallContextLastCollect = now; + } + // this is for tests exclusively until we have a proper accessor in v8 internal static Func HttpContextItemsGetter { get; set; } diff --git a/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs b/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs index 973ba341f2..6862986ec2 100644 --- a/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs +++ b/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs @@ -9,6 +9,7 @@ using Umbraco.Core.Models.Rdbms; using Umbraco.Core.Sync; using Umbraco.Web.Routing; using Umbraco.Core.Logging; +using Umbraco.Core.Scoping; using Umbraco.Web.Scheduling; namespace Umbraco.Web @@ -21,9 +22,12 @@ namespace Umbraco.Web /// public class BatchedDatabaseServerMessenger : DatabaseServerMessenger { + private readonly ApplicationContext _appContext; + public BatchedDatabaseServerMessenger(ApplicationContext appContext, bool enableDistCalls, DatabaseServerMessengerOptions options) : base(appContext, enableDistCalls, options) { + _appContext = appContext; Scheduler.Initializing += Scheduler_Initializing; } @@ -42,7 +46,7 @@ namespace Umbraco.Web //start the background task runner for processing instructions const int delayMilliseconds = 60000; var instructionProcessingRunner = new BackgroundTaskRunner("InstructionProcessing", ApplicationContext.ProfilingLogger.Logger); - var instructionProcessingTask = new InstructionProcessing(instructionProcessingRunner, this, delayMilliseconds, Options.ThrottleSeconds * 1000); + var instructionProcessingTask = new InstructionProcessing(instructionProcessingRunner, this, _appContext.ScopeProvider, delayMilliseconds, Options.ThrottleSeconds * 1000); instructionProcessingRunner.TryAdd(instructionProcessingTask); e.Add(instructionProcessingTask); } @@ -73,18 +77,31 @@ namespace Umbraco.Web private class InstructionProcessing : RecurringTaskBase { private readonly DatabaseServerMessenger _messenger; + private readonly IScopeProvider _scopeProvider; public InstructionProcessing(IBackgroundTaskRunner runner, DatabaseServerMessenger messenger, + IScopeProvider scopeProvider, int delayMilliseconds, int periodMilliseconds) : base(runner, delayMilliseconds, periodMilliseconds) { _messenger = messenger; + _scopeProvider = scopeProvider; } public override bool PerformRun() { - _messenger.Sync(); + // beware! + // DatabaseServerMessenger uses _appContext.DatabaseContext.Database without creating + // scopes, and since we are running in a background task, there will be no ambient + // scope (as would be the case within a web request), and so we would end up creating + // (and leaking) a NoScope instance, which is bad - better make sure we have a true + // scope here! - see U4-11207 + using (var scope = _scopeProvider.CreateScope()) + { + _messenger.Sync(); + scope.Complete(); + } //return true to repeat return true; } @@ -121,14 +138,17 @@ namespace Umbraco.Web batch.Clear(); //Write the instructions but only create JSON blobs with a max instruction count equal to MaxProcessingInstructionCount - foreach (var instructionsBatch in instructions.InGroupsOf(Options.MaxProcessingInstructionCount)) + using (var scope = _appContext.ScopeProvider.CreateScope()) { - WriteInstructions(instructionsBatch); + foreach (var instructionsBatch in instructions.InGroupsOf(Options.MaxProcessingInstructionCount)) + { + WriteInstructions(scope, instructionsBatch); + } + scope.Complete(); } - } - private void WriteInstructions(IEnumerable instructions) + private void WriteInstructions(IScope scope, IEnumerable instructions) { var dto = new CacheInstructionDto { @@ -137,8 +157,7 @@ namespace Umbraco.Web OriginIdentity = LocalIdentity, InstructionCount = instructions.Sum(x => x.JsonIdCount) }; - - ApplicationContext.DatabaseContext.Database.Insert(dto); + scope.Database.Insert(dto); } protected ICollection GetBatch(bool create) @@ -179,16 +198,19 @@ namespace Umbraco.Web if (batch == null) { //only write the json blob with a maximum count of the MaxProcessingInstructionCount - foreach (var maxBatch in instructions.InGroupsOf(Options.MaxProcessingInstructionCount)) + using (var scope = _appContext.ScopeProvider.CreateScope()) { - WriteInstructions(maxBatch); + foreach (var maxBatch in instructions.InGroupsOf(Options.MaxProcessingInstructionCount)) + { + WriteInstructions(scope, maxBatch); + } + scope.Complete(); } } else { batch.Add(new RefreshInstructionEnvelope(servers, refresher, instructions)); } - - } + } } -} \ No newline at end of file +} From b5415844b80b9964b228d6235e98c4302c810c1f Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Thu, 12 Apr 2018 16:30:55 +0200 Subject: [PATCH 34/57] Cherry picked - Fix scope leaks caused by database messenger [U4-11207] #2580 --- src/Umbraco.Core/Persistence/PetaPoco.cs | 15 +++-- src/Umbraco.Core/Scoping/NoScope.cs | 3 + src/Umbraco.Core/Scoping/Scope.cs | 2 +- src/Umbraco.Core/Scoping/ScopeProvider.cs | 57 +++++++++++++++++-- .../BatchedDatabaseServerMessenger.cs | 48 +++++++++++----- 5 files changed, 97 insertions(+), 28 deletions(-) diff --git a/src/Umbraco.Core/Persistence/PetaPoco.cs b/src/Umbraco.Core/Persistence/PetaPoco.cs index 79b3ce3871..4692cb4b51 100644 --- a/src/Umbraco.Core/Persistence/PetaPoco.cs +++ b/src/Umbraco.Core/Persistence/PetaPoco.cs @@ -402,7 +402,7 @@ namespace Umbraco.Core.Persistence } // Helper to handle named parameters from object properties - static Regex rxParams = new Regex(@"(? args_dest) { return rxParams.Replace(_sql, m => @@ -545,7 +545,7 @@ namespace Umbraco.Core.Persistence } // Create a command - static Regex rxParamsPrefix = new Regex(@"(?(sql.SQL, sql.Arguments); } - Regex rxSelect = new Regex(@"\A\s*(SELECT|EXECUTE|CALL)\s", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase | RegexOptions.Multiline); - Regex rxFrom = new Regex(@"\A\s*FROM\s", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase | RegexOptions.Multiline); + static readonly Regex rxSelect = new Regex(@"\A\s*(SELECT|EXECUTE|CALL)\s", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase | RegexOptions.Multiline); + static readonly Regex rxFrom = new Regex(@"\A\s*FROM\s", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase | RegexOptions.Multiline); string AddSelectClause(string sql) { if (sql.StartsWith(";")) @@ -701,9 +701,9 @@ namespace Umbraco.Core.Persistence return Fetch(sql.SQL, sql.Arguments); } - static Regex rxColumns = new Regex(@"\A\s*SELECT\s+((?:\((?>\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|.)*?)(?\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|[\w\(\)\.])+(?:\s+(?:ASC|DESC))?(?:\s*,\s*(?:\((?>\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|[\w\(\)\.])+(?:\s+(?:ASC|DESC))?)*", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.Compiled); - static Regex rxDistinct = new Regex(@"\ADISTINCT\s", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.Compiled); + static readonly Regex rxColumns = new Regex(@"\A\s*SELECT\s+((?:\((?>\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|.)*?)(?\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|[\w\(\)\.])+(?:\s+(?:ASC|DESC))?(?:\s*,\s*(?:\((?>\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|[\w\(\)\.])+(?:\s+(?:ASC|DESC))?)*", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.Compiled); + static readonly Regex rxDistinct = new Regex(@"\ADISTINCT\s", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.Compiled); public static bool SplitSqlForPaging(string sql, out string sqlCount, out string sqlSelectRemoved, out string sqlOrderBy) { sqlSelectRemoved = null; @@ -2546,5 +2546,4 @@ namespace Umbraco.Core.Persistence } } } - } diff --git a/src/Umbraco.Core/Scoping/NoScope.cs b/src/Umbraco.Core/Scoping/NoScope.cs index a21815173c..73353ce3da 100644 --- a/src/Umbraco.Core/Scoping/NoScope.cs +++ b/src/Umbraco.Core/Scoping/NoScope.cs @@ -20,6 +20,7 @@ namespace Umbraco.Core.Scoping public NoScope(ScopeProvider scopeProvider) { _scopeProvider = scopeProvider; + Timestamp = DateTime.Now; #if DEBUG_SCOPES _scopeProvider.RegisterScope(this); #endif @@ -28,6 +29,8 @@ namespace Umbraco.Core.Scoping private readonly Guid _instanceId = Guid.NewGuid(); public Guid InstanceId { get { return _instanceId; } } + public DateTime Timestamp { get; } + /// public bool CallContext { get { return false; } } diff --git a/src/Umbraco.Core/Scoping/Scope.cs b/src/Umbraco.Core/Scoping/Scope.cs index bfee772872..ed23913719 100644 --- a/src/Umbraco.Core/Scoping/Scope.cs +++ b/src/Umbraco.Core/Scoping/Scope.cs @@ -374,7 +374,7 @@ namespace Umbraco.Core.Scoping } var parent = ParentScope; - _scopeProvider.AmbientScope = parent; + _scopeProvider.AmbientScope = parent; // might be null = this is how scopes are removed from context objects #if DEBUG_SCOPES _scopeProvider.Disposed(this); diff --git a/src/Umbraco.Core/Scoping/ScopeProvider.cs b/src/Umbraco.Core/Scoping/ScopeProvider.cs index dfcf7985ae..8b0f84b5d0 100644 --- a/src/Umbraco.Core/Scoping/ScopeProvider.cs +++ b/src/Umbraco.Core/Scoping/ScopeProvider.cs @@ -2,10 +2,12 @@ using System.Collections; using System.Collections.Generic; using System.Data; +using System.Linq; using System.Runtime.Remoting.Messaging; using System.Text; using System.Web; using Umbraco.Core.Events; +using Umbraco.Core.Logging; using Umbraco.Core.Persistence; #if DEBUG_SCOPES using System.Linq; @@ -66,22 +68,38 @@ namespace Umbraco.Core.Scoping // tests, any other things (see https://msdn.microsoft.com/en-us/library/dn458353(v=vs.110).aspx), // but we don't want to make all of our objects serializable since they are *not* meant to be // used in cross-AppDomain scenario anyways. + // // in addition, whatever goes into the logical call context is serialized back and forth any - // time cross-AppDomain code executes, so if we put an "object" there, we'll can *another* - // "object" instance - and so we cannot use a random object as a key. + // time cross-AppDomain code executes, so if we put an "object" there, we'll get *another* + // "object" instance back - and so we cannot use a random object as a key. + // // so what we do is: we register a guid in the call context, and we keep a table mapping those // guids to the actual objects. the guid serializes back and forth without causing any issue, // and we can retrieve the actual objects from the table. - // only issue: how are we supposed to clear the table? we can't, really. objects should take - // care of de-registering themselves from context. - // everything we use does, except the NoScope scope, which just stays there // - // during tests, NoScope can to into call context... nothing much we can do about it + // so far, the only objects that go into this table are scopes (using ScopeItemKey) and + // scope contexts (using ContextItemKey). private static readonly object StaticCallContextObjectsLock = new object(); private static readonly Dictionary StaticCallContextObjects = new Dictionary(); + // normal scopes and scope contexts take greate care removing themselves when disposed, so it + // is all safe. OTOH the NoScope *CANNOT* remove itself, this is by design, it *WILL* leak and + // there is little (nothing) we can do about it - NoScope exists for backward compatibility + // reasons and relying on it is greatly discouraged. + // + // however... we can *try* at protecting the app against memory leaks, by collecting NoScope + // instances that are too old. if anything actually *need* to retain a NoScope instance for + // a long time, it will break. but that's probably ok. so: the constants below define how + // long a NoScope instance can stay in the table before being removed, and how often we should + // collect the table - and collecting happens anytime SetCallContextObject is invoked + + private static readonly TimeSpan StaticCallContextNoScopeLifeSpan = TimeSpan.FromMinutes(30); + private static readonly TimeSpan StaticCallContextCollectPeriod = TimeSpan.FromMinutes(4); + private static DateTime _staticCallContextLastCollect = DateTime.MinValue; + + #if DEBUG_SCOPES public Dictionary CallContextObjects { @@ -156,6 +174,7 @@ namespace Umbraco.Core.Scoping //Logging.LogHelper.Debug("At:\r\n" + Head(Environment.StackTrace, 24)); #endif StaticCallContextObjects.Remove(objectKey); + CollectStaticCallContextObjectsLocked(); } } else @@ -171,11 +190,37 @@ namespace Umbraco.Core.Scoping //Logging.LogHelper.Debug("At:\r\n" + Head(Environment.StackTrace, 24)); #endif StaticCallContextObjects.Add(objectKey, value); + CollectStaticCallContextObjectsLocked(); } CallContext.LogicalSetData(key, objectKey); } } + private static void CollectStaticCallContextObjectsLocked() + { + // is it time to collect? + var now = DateTime.Now; + if (now - _staticCallContextLastCollect <= StaticCallContextCollectPeriod) + return; + + // disable warning: this method is invoked from within a lock + // ReSharper disable InconsistentlySynchronizedField + var threshold = now.Add(-StaticCallContextNoScopeLifeSpan); + var guids = StaticCallContextObjects + .Where(x => x.Value is NoScope noScope && noScope.Timestamp < threshold) + .Select(x => x.Key) + .ToList(); + if (guids.Count > 0) + { + LogHelper.Warn($"Collected {guids.Count} NoScope instances from StaticCallContextObjects."); + foreach (var guid in guids) + StaticCallContextObjects.Remove(guid); + } + // ReSharper restore InconsistentlySynchronizedField + + _staticCallContextLastCollect = now; + } + // this is for tests exclusively until we have a proper accessor in v8 internal static Func HttpContextItemsGetter { get; set; } diff --git a/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs b/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs index 973ba341f2..6862986ec2 100644 --- a/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs +++ b/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs @@ -9,6 +9,7 @@ using Umbraco.Core.Models.Rdbms; using Umbraco.Core.Sync; using Umbraco.Web.Routing; using Umbraco.Core.Logging; +using Umbraco.Core.Scoping; using Umbraco.Web.Scheduling; namespace Umbraco.Web @@ -21,9 +22,12 @@ namespace Umbraco.Web /// public class BatchedDatabaseServerMessenger : DatabaseServerMessenger { + private readonly ApplicationContext _appContext; + public BatchedDatabaseServerMessenger(ApplicationContext appContext, bool enableDistCalls, DatabaseServerMessengerOptions options) : base(appContext, enableDistCalls, options) { + _appContext = appContext; Scheduler.Initializing += Scheduler_Initializing; } @@ -42,7 +46,7 @@ namespace Umbraco.Web //start the background task runner for processing instructions const int delayMilliseconds = 60000; var instructionProcessingRunner = new BackgroundTaskRunner("InstructionProcessing", ApplicationContext.ProfilingLogger.Logger); - var instructionProcessingTask = new InstructionProcessing(instructionProcessingRunner, this, delayMilliseconds, Options.ThrottleSeconds * 1000); + var instructionProcessingTask = new InstructionProcessing(instructionProcessingRunner, this, _appContext.ScopeProvider, delayMilliseconds, Options.ThrottleSeconds * 1000); instructionProcessingRunner.TryAdd(instructionProcessingTask); e.Add(instructionProcessingTask); } @@ -73,18 +77,31 @@ namespace Umbraco.Web private class InstructionProcessing : RecurringTaskBase { private readonly DatabaseServerMessenger _messenger; + private readonly IScopeProvider _scopeProvider; public InstructionProcessing(IBackgroundTaskRunner runner, DatabaseServerMessenger messenger, + IScopeProvider scopeProvider, int delayMilliseconds, int periodMilliseconds) : base(runner, delayMilliseconds, periodMilliseconds) { _messenger = messenger; + _scopeProvider = scopeProvider; } public override bool PerformRun() { - _messenger.Sync(); + // beware! + // DatabaseServerMessenger uses _appContext.DatabaseContext.Database without creating + // scopes, and since we are running in a background task, there will be no ambient + // scope (as would be the case within a web request), and so we would end up creating + // (and leaking) a NoScope instance, which is bad - better make sure we have a true + // scope here! - see U4-11207 + using (var scope = _scopeProvider.CreateScope()) + { + _messenger.Sync(); + scope.Complete(); + } //return true to repeat return true; } @@ -121,14 +138,17 @@ namespace Umbraco.Web batch.Clear(); //Write the instructions but only create JSON blobs with a max instruction count equal to MaxProcessingInstructionCount - foreach (var instructionsBatch in instructions.InGroupsOf(Options.MaxProcessingInstructionCount)) + using (var scope = _appContext.ScopeProvider.CreateScope()) { - WriteInstructions(instructionsBatch); + foreach (var instructionsBatch in instructions.InGroupsOf(Options.MaxProcessingInstructionCount)) + { + WriteInstructions(scope, instructionsBatch); + } + scope.Complete(); } - } - private void WriteInstructions(IEnumerable instructions) + private void WriteInstructions(IScope scope, IEnumerable instructions) { var dto = new CacheInstructionDto { @@ -137,8 +157,7 @@ namespace Umbraco.Web OriginIdentity = LocalIdentity, InstructionCount = instructions.Sum(x => x.JsonIdCount) }; - - ApplicationContext.DatabaseContext.Database.Insert(dto); + scope.Database.Insert(dto); } protected ICollection GetBatch(bool create) @@ -179,16 +198,19 @@ namespace Umbraco.Web if (batch == null) { //only write the json blob with a maximum count of the MaxProcessingInstructionCount - foreach (var maxBatch in instructions.InGroupsOf(Options.MaxProcessingInstructionCount)) + using (var scope = _appContext.ScopeProvider.CreateScope()) { - WriteInstructions(maxBatch); + foreach (var maxBatch in instructions.InGroupsOf(Options.MaxProcessingInstructionCount)) + { + WriteInstructions(scope, maxBatch); + } + scope.Complete(); } } else { batch.Add(new RefreshInstructionEnvelope(servers, refresher, instructions)); } - - } + } } -} \ No newline at end of file +} From 5003a55acada5ab24b5a4ecd6d212fe2d82ca440 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Thu, 12 Apr 2018 16:32:34 +0200 Subject: [PATCH 35/57] Cherry picked - Fix scope leaks caused by database messenger [U4-11207] #2580 --- src/Umbraco.Core/Persistence/PetaPoco.cs | 15 +++-- src/Umbraco.Core/Scoping/NoScope.cs | 3 + src/Umbraco.Core/Scoping/Scope.cs | 2 +- src/Umbraco.Core/Scoping/ScopeProvider.cs | 57 +++++++++++++++++-- .../BatchedDatabaseServerMessenger.cs | 48 +++++++++++----- 5 files changed, 97 insertions(+), 28 deletions(-) diff --git a/src/Umbraco.Core/Persistence/PetaPoco.cs b/src/Umbraco.Core/Persistence/PetaPoco.cs index 79b3ce3871..4692cb4b51 100644 --- a/src/Umbraco.Core/Persistence/PetaPoco.cs +++ b/src/Umbraco.Core/Persistence/PetaPoco.cs @@ -402,7 +402,7 @@ namespace Umbraco.Core.Persistence } // Helper to handle named parameters from object properties - static Regex rxParams = new Regex(@"(? args_dest) { return rxParams.Replace(_sql, m => @@ -545,7 +545,7 @@ namespace Umbraco.Core.Persistence } // Create a command - static Regex rxParamsPrefix = new Regex(@"(?(sql.SQL, sql.Arguments); } - Regex rxSelect = new Regex(@"\A\s*(SELECT|EXECUTE|CALL)\s", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase | RegexOptions.Multiline); - Regex rxFrom = new Regex(@"\A\s*FROM\s", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase | RegexOptions.Multiline); + static readonly Regex rxSelect = new Regex(@"\A\s*(SELECT|EXECUTE|CALL)\s", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase | RegexOptions.Multiline); + static readonly Regex rxFrom = new Regex(@"\A\s*FROM\s", RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.IgnoreCase | RegexOptions.Multiline); string AddSelectClause(string sql) { if (sql.StartsWith(";")) @@ -701,9 +701,9 @@ namespace Umbraco.Core.Persistence return Fetch(sql.SQL, sql.Arguments); } - static Regex rxColumns = new Regex(@"\A\s*SELECT\s+((?:\((?>\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|.)*?)(?\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|[\w\(\)\.])+(?:\s+(?:ASC|DESC))?(?:\s*,\s*(?:\((?>\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|[\w\(\)\.])+(?:\s+(?:ASC|DESC))?)*", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.Compiled); - static Regex rxDistinct = new Regex(@"\ADISTINCT\s", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.Compiled); + static readonly Regex rxColumns = new Regex(@"\A\s*SELECT\s+((?:\((?>\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|.)*?)(?\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|[\w\(\)\.])+(?:\s+(?:ASC|DESC))?(?:\s*,\s*(?:\((?>\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|[\w\(\)\.])+(?:\s+(?:ASC|DESC))?)*", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.Compiled); + static readonly Regex rxDistinct = new Regex(@"\ADISTINCT\s", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.Compiled); public static bool SplitSqlForPaging(string sql, out string sqlCount, out string sqlSelectRemoved, out string sqlOrderBy) { sqlSelectRemoved = null; @@ -2546,5 +2546,4 @@ namespace Umbraco.Core.Persistence } } } - } diff --git a/src/Umbraco.Core/Scoping/NoScope.cs b/src/Umbraco.Core/Scoping/NoScope.cs index a21815173c..73353ce3da 100644 --- a/src/Umbraco.Core/Scoping/NoScope.cs +++ b/src/Umbraco.Core/Scoping/NoScope.cs @@ -20,6 +20,7 @@ namespace Umbraco.Core.Scoping public NoScope(ScopeProvider scopeProvider) { _scopeProvider = scopeProvider; + Timestamp = DateTime.Now; #if DEBUG_SCOPES _scopeProvider.RegisterScope(this); #endif @@ -28,6 +29,8 @@ namespace Umbraco.Core.Scoping private readonly Guid _instanceId = Guid.NewGuid(); public Guid InstanceId { get { return _instanceId; } } + public DateTime Timestamp { get; } + /// public bool CallContext { get { return false; } } diff --git a/src/Umbraco.Core/Scoping/Scope.cs b/src/Umbraco.Core/Scoping/Scope.cs index bfee772872..ed23913719 100644 --- a/src/Umbraco.Core/Scoping/Scope.cs +++ b/src/Umbraco.Core/Scoping/Scope.cs @@ -374,7 +374,7 @@ namespace Umbraco.Core.Scoping } var parent = ParentScope; - _scopeProvider.AmbientScope = parent; + _scopeProvider.AmbientScope = parent; // might be null = this is how scopes are removed from context objects #if DEBUG_SCOPES _scopeProvider.Disposed(this); diff --git a/src/Umbraco.Core/Scoping/ScopeProvider.cs b/src/Umbraco.Core/Scoping/ScopeProvider.cs index dfcf7985ae..8b0f84b5d0 100644 --- a/src/Umbraco.Core/Scoping/ScopeProvider.cs +++ b/src/Umbraco.Core/Scoping/ScopeProvider.cs @@ -2,10 +2,12 @@ using System.Collections; using System.Collections.Generic; using System.Data; +using System.Linq; using System.Runtime.Remoting.Messaging; using System.Text; using System.Web; using Umbraco.Core.Events; +using Umbraco.Core.Logging; using Umbraco.Core.Persistence; #if DEBUG_SCOPES using System.Linq; @@ -66,22 +68,38 @@ namespace Umbraco.Core.Scoping // tests, any other things (see https://msdn.microsoft.com/en-us/library/dn458353(v=vs.110).aspx), // but we don't want to make all of our objects serializable since they are *not* meant to be // used in cross-AppDomain scenario anyways. + // // in addition, whatever goes into the logical call context is serialized back and forth any - // time cross-AppDomain code executes, so if we put an "object" there, we'll can *another* - // "object" instance - and so we cannot use a random object as a key. + // time cross-AppDomain code executes, so if we put an "object" there, we'll get *another* + // "object" instance back - and so we cannot use a random object as a key. + // // so what we do is: we register a guid in the call context, and we keep a table mapping those // guids to the actual objects. the guid serializes back and forth without causing any issue, // and we can retrieve the actual objects from the table. - // only issue: how are we supposed to clear the table? we can't, really. objects should take - // care of de-registering themselves from context. - // everything we use does, except the NoScope scope, which just stays there // - // during tests, NoScope can to into call context... nothing much we can do about it + // so far, the only objects that go into this table are scopes (using ScopeItemKey) and + // scope contexts (using ContextItemKey). private static readonly object StaticCallContextObjectsLock = new object(); private static readonly Dictionary StaticCallContextObjects = new Dictionary(); + // normal scopes and scope contexts take greate care removing themselves when disposed, so it + // is all safe. OTOH the NoScope *CANNOT* remove itself, this is by design, it *WILL* leak and + // there is little (nothing) we can do about it - NoScope exists for backward compatibility + // reasons and relying on it is greatly discouraged. + // + // however... we can *try* at protecting the app against memory leaks, by collecting NoScope + // instances that are too old. if anything actually *need* to retain a NoScope instance for + // a long time, it will break. but that's probably ok. so: the constants below define how + // long a NoScope instance can stay in the table before being removed, and how often we should + // collect the table - and collecting happens anytime SetCallContextObject is invoked + + private static readonly TimeSpan StaticCallContextNoScopeLifeSpan = TimeSpan.FromMinutes(30); + private static readonly TimeSpan StaticCallContextCollectPeriod = TimeSpan.FromMinutes(4); + private static DateTime _staticCallContextLastCollect = DateTime.MinValue; + + #if DEBUG_SCOPES public Dictionary CallContextObjects { @@ -156,6 +174,7 @@ namespace Umbraco.Core.Scoping //Logging.LogHelper.Debug("At:\r\n" + Head(Environment.StackTrace, 24)); #endif StaticCallContextObjects.Remove(objectKey); + CollectStaticCallContextObjectsLocked(); } } else @@ -171,11 +190,37 @@ namespace Umbraco.Core.Scoping //Logging.LogHelper.Debug("At:\r\n" + Head(Environment.StackTrace, 24)); #endif StaticCallContextObjects.Add(objectKey, value); + CollectStaticCallContextObjectsLocked(); } CallContext.LogicalSetData(key, objectKey); } } + private static void CollectStaticCallContextObjectsLocked() + { + // is it time to collect? + var now = DateTime.Now; + if (now - _staticCallContextLastCollect <= StaticCallContextCollectPeriod) + return; + + // disable warning: this method is invoked from within a lock + // ReSharper disable InconsistentlySynchronizedField + var threshold = now.Add(-StaticCallContextNoScopeLifeSpan); + var guids = StaticCallContextObjects + .Where(x => x.Value is NoScope noScope && noScope.Timestamp < threshold) + .Select(x => x.Key) + .ToList(); + if (guids.Count > 0) + { + LogHelper.Warn($"Collected {guids.Count} NoScope instances from StaticCallContextObjects."); + foreach (var guid in guids) + StaticCallContextObjects.Remove(guid); + } + // ReSharper restore InconsistentlySynchronizedField + + _staticCallContextLastCollect = now; + } + // this is for tests exclusively until we have a proper accessor in v8 internal static Func HttpContextItemsGetter { get; set; } diff --git a/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs b/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs index 973ba341f2..6862986ec2 100644 --- a/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs +++ b/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs @@ -9,6 +9,7 @@ using Umbraco.Core.Models.Rdbms; using Umbraco.Core.Sync; using Umbraco.Web.Routing; using Umbraco.Core.Logging; +using Umbraco.Core.Scoping; using Umbraco.Web.Scheduling; namespace Umbraco.Web @@ -21,9 +22,12 @@ namespace Umbraco.Web /// public class BatchedDatabaseServerMessenger : DatabaseServerMessenger { + private readonly ApplicationContext _appContext; + public BatchedDatabaseServerMessenger(ApplicationContext appContext, bool enableDistCalls, DatabaseServerMessengerOptions options) : base(appContext, enableDistCalls, options) { + _appContext = appContext; Scheduler.Initializing += Scheduler_Initializing; } @@ -42,7 +46,7 @@ namespace Umbraco.Web //start the background task runner for processing instructions const int delayMilliseconds = 60000; var instructionProcessingRunner = new BackgroundTaskRunner("InstructionProcessing", ApplicationContext.ProfilingLogger.Logger); - var instructionProcessingTask = new InstructionProcessing(instructionProcessingRunner, this, delayMilliseconds, Options.ThrottleSeconds * 1000); + var instructionProcessingTask = new InstructionProcessing(instructionProcessingRunner, this, _appContext.ScopeProvider, delayMilliseconds, Options.ThrottleSeconds * 1000); instructionProcessingRunner.TryAdd(instructionProcessingTask); e.Add(instructionProcessingTask); } @@ -73,18 +77,31 @@ namespace Umbraco.Web private class InstructionProcessing : RecurringTaskBase { private readonly DatabaseServerMessenger _messenger; + private readonly IScopeProvider _scopeProvider; public InstructionProcessing(IBackgroundTaskRunner runner, DatabaseServerMessenger messenger, + IScopeProvider scopeProvider, int delayMilliseconds, int periodMilliseconds) : base(runner, delayMilliseconds, periodMilliseconds) { _messenger = messenger; + _scopeProvider = scopeProvider; } public override bool PerformRun() { - _messenger.Sync(); + // beware! + // DatabaseServerMessenger uses _appContext.DatabaseContext.Database without creating + // scopes, and since we are running in a background task, there will be no ambient + // scope (as would be the case within a web request), and so we would end up creating + // (and leaking) a NoScope instance, which is bad - better make sure we have a true + // scope here! - see U4-11207 + using (var scope = _scopeProvider.CreateScope()) + { + _messenger.Sync(); + scope.Complete(); + } //return true to repeat return true; } @@ -121,14 +138,17 @@ namespace Umbraco.Web batch.Clear(); //Write the instructions but only create JSON blobs with a max instruction count equal to MaxProcessingInstructionCount - foreach (var instructionsBatch in instructions.InGroupsOf(Options.MaxProcessingInstructionCount)) + using (var scope = _appContext.ScopeProvider.CreateScope()) { - WriteInstructions(instructionsBatch); + foreach (var instructionsBatch in instructions.InGroupsOf(Options.MaxProcessingInstructionCount)) + { + WriteInstructions(scope, instructionsBatch); + } + scope.Complete(); } - } - private void WriteInstructions(IEnumerable instructions) + private void WriteInstructions(IScope scope, IEnumerable instructions) { var dto = new CacheInstructionDto { @@ -137,8 +157,7 @@ namespace Umbraco.Web OriginIdentity = LocalIdentity, InstructionCount = instructions.Sum(x => x.JsonIdCount) }; - - ApplicationContext.DatabaseContext.Database.Insert(dto); + scope.Database.Insert(dto); } protected ICollection GetBatch(bool create) @@ -179,16 +198,19 @@ namespace Umbraco.Web if (batch == null) { //only write the json blob with a maximum count of the MaxProcessingInstructionCount - foreach (var maxBatch in instructions.InGroupsOf(Options.MaxProcessingInstructionCount)) + using (var scope = _appContext.ScopeProvider.CreateScope()) { - WriteInstructions(maxBatch); + foreach (var maxBatch in instructions.InGroupsOf(Options.MaxProcessingInstructionCount)) + { + WriteInstructions(scope, maxBatch); + } + scope.Complete(); } } else { batch.Add(new RefreshInstructionEnvelope(servers, refresher, instructions)); } - - } + } } -} \ No newline at end of file +} From 1e2598a3a374d35dbb62c6ab0c4a4fdb5a2662b3 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Thu, 12 Apr 2018 16:36:41 +0200 Subject: [PATCH 36/57] Bumps version to 7.8.2 --- src/SolutionInfo.cs | 4 ++-- src/Umbraco.Core/Configuration/UmbracoVersion.cs | 2 +- src/Umbraco.Web.UI/Umbraco.Web.UI.csproj | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/SolutionInfo.cs b/src/SolutionInfo.cs index 4296625d27..e39fd33eb2 100644 --- a/src/SolutionInfo.cs +++ b/src/SolutionInfo.cs @@ -11,5 +11,5 @@ using System.Resources; [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyFileVersion("7.8.1")] -[assembly: AssemblyInformationalVersion("7.8.1")] \ No newline at end of file +[assembly: AssemblyFileVersion("7.8.2")] +[assembly: AssemblyInformationalVersion("7.8.2")] \ No newline at end of file diff --git a/src/Umbraco.Core/Configuration/UmbracoVersion.cs b/src/Umbraco.Core/Configuration/UmbracoVersion.cs index 8ee2095de6..1025d97c3d 100644 --- a/src/Umbraco.Core/Configuration/UmbracoVersion.cs +++ b/src/Umbraco.Core/Configuration/UmbracoVersion.cs @@ -6,7 +6,7 @@ namespace Umbraco.Core.Configuration { public class UmbracoVersion { - private static readonly Version Version = new Version("7.8.1"); + private static readonly Version Version = new Version("7.8.2"); /// /// Gets the current version of Umbraco. diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index 03ab1f50f6..6b39c6ca05 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -1024,9 +1024,9 @@ xcopy "$(ProjectDir)"..\packages\SqlServerCE.4.0.0.1\x86\*.* "$(TargetDir)x86\" True True - 7810 + 7820 / - http://localhost:7810 + http://localhost:7820 7800 / http://localhost:7800 From c61fd65463203505e4f07fa034f69da3cdcef33b Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Thu, 12 Apr 2018 16:38:22 +0200 Subject: [PATCH 37/57] Bumps version to 7.9.5 --- src/SolutionInfo.cs | 4 ++-- src/Umbraco.Core/Configuration/UmbracoVersion.cs | 2 +- src/Umbraco.Web.UI/Umbraco.Web.UI.csproj | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/SolutionInfo.cs b/src/SolutionInfo.cs index dd901ab1d1..71cecfdde2 100644 --- a/src/SolutionInfo.cs +++ b/src/SolutionInfo.cs @@ -11,5 +11,5 @@ using System.Resources; [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyFileVersion("7.9.3")] -[assembly: AssemblyInformationalVersion("7.9.3")] \ No newline at end of file +[assembly: AssemblyFileVersion("7.9.5")] +[assembly: AssemblyInformationalVersion("7.9.5")] \ No newline at end of file diff --git a/src/Umbraco.Core/Configuration/UmbracoVersion.cs b/src/Umbraco.Core/Configuration/UmbracoVersion.cs index 4a81e95c26..b9b797aebb 100644 --- a/src/Umbraco.Core/Configuration/UmbracoVersion.cs +++ b/src/Umbraco.Core/Configuration/UmbracoVersion.cs @@ -6,7 +6,7 @@ namespace Umbraco.Core.Configuration { public class UmbracoVersion { - private static readonly Version Version = new Version("7.9.3"); + private static readonly Version Version = new Version("7.9.5"); /// /// Gets the current version of Umbraco. diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index 9a0122ad83..b680b09c71 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -1023,9 +1023,9 @@ xcopy "$(ProjectDir)"..\packages\SqlServerCE.4.0.0.1\x86\*.* "$(TargetDir)x86\" True True - 7930 + 7950 / - http://localhost:7930 + http://localhost:7950 False False From 4664266b8de967c65c95b9b7af78082c35ff5621 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Thu, 12 Apr 2018 16:40:10 +0200 Subject: [PATCH 38/57] Bumps version to 7.10.3 --- src/SolutionInfo.cs | 4 ++-- src/Umbraco.Core/Configuration/UmbracoVersion.cs | 2 +- src/Umbraco.Web.UI/Umbraco.Web.UI.csproj | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/SolutionInfo.cs b/src/SolutionInfo.cs index cba378547a..807131b63e 100644 --- a/src/SolutionInfo.cs +++ b/src/SolutionInfo.cs @@ -11,5 +11,5 @@ using System.Resources; [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyFileVersion("7.10.2")] -[assembly: AssemblyInformationalVersion("7.10.2")] \ No newline at end of file +[assembly: AssemblyFileVersion("7.10.3")] +[assembly: AssemblyInformationalVersion("7.10.3")] \ No newline at end of file diff --git a/src/Umbraco.Core/Configuration/UmbracoVersion.cs b/src/Umbraco.Core/Configuration/UmbracoVersion.cs index 434aa154ad..ef11d9ff08 100644 --- a/src/Umbraco.Core/Configuration/UmbracoVersion.cs +++ b/src/Umbraco.Core/Configuration/UmbracoVersion.cs @@ -6,7 +6,7 @@ namespace Umbraco.Core.Configuration { public class UmbracoVersion { - private static readonly Version Version = new Version("7.10.2"); + private static readonly Version Version = new Version("7.10.3"); /// /// Gets the current version of Umbraco. diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index b2917620c3..a19c92bcbb 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -1035,9 +1035,9 @@ xcopy "$(ProjectDir)"..\packages\SqlServerCE.4.0.0.1\x86\*.* "$(TargetDir)x86\" True True - 7102 + 7103 / - http://localhost:7102 + http://localhost:7103 False False From b6d5ac6a01a0b77a62e563101d5c211016162ca5 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Thu, 12 Apr 2018 17:20:12 +0200 Subject: [PATCH 39/57] Update CONTRIBUTING.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0e87f9824d..f8d8aac342 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,7 +41,7 @@ This project and everyone participating in it is governed by the [our Code of Co ### Reporting Bugs This section guides you through submitting a bug report for Umbraco CMS. Following these guidelines helps maintainers and the community understand your report 📝, reproduce the behavior 💻 💻, and find related reports 🔎. -Before creating bug reports, please check [this list](#before-submitting-a-bug-report) as you might find out that you don't need to create one. When you are creating a bug report, please [include as many details as possible](#how-do-i-submit-a-good-bug-report). Fill out [the required template](ISSUE_TEMPLATE.md), the information it asks for helps us resolve issues faster. +Before creating bug reports, please check [this list](#before-submitting-a-bug-report) as you might find out that you don't need to create one. When you are creating a bug report, please [include as many details as possible](#how-do-i-submit-a-good-bug-report). Fill out [the required template](http://issues.umbraco.org/issues#newissue=61-30118), the information it asks for helps us resolve issues faster. > **Note:** If you find a **Closed** issue that seems like it is the same thing that you're experiencing, open a new issue and include a link to the original issue in the body of your new one. From 94ed741dd51b3960e274e926daaca85183a8015e Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Thu, 12 Apr 2018 17:50:44 +0200 Subject: [PATCH 40/57] Weird revert of a PR that was merged in before --- .../src/views/propertyeditors/rte/rte.controller.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/rte/rte.controller.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/rte/rte.controller.js index f886c60bc1..17f0f3b698 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/rte/rte.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/rte/rte.controller.js @@ -266,7 +266,7 @@ angular.module("umbraco") editor.on('ObjectResized', function (e) { - var qs = "?width=" + e.width + "&height=" + e.height; + var qs = "?width=" + e.width + "&height=" + e.height + "&mode=max"; var srcAttr = $(e.target).attr("src"); var path = srcAttr.split("?")[0]; $(e.target).attr("data-mce-src", path + qs); @@ -384,7 +384,7 @@ angular.module("umbraco") // element might still be there even after the modal has been hidden. $scope.$on('$destroy', function () { unsubscribe(); - if (tinyMceEditor !== undefined && tinyMceEditor != null) { + if (tinyMceEditor !== undefined && tinyMceEditor != null) { tinyMceEditor.destroy() } }); From 29d9dcffb8306e74244bd69ed8602facb1e4378a Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Thu, 12 Apr 2018 18:38:00 +0200 Subject: [PATCH 41/57] Manually applying PR: Three new healthchecks #1522 --- src/Umbraco.Web.UI/umbraco/config/lang/en.xml | 12 ++++++++++- .../umbraco/config/lang/en_us.xml | 12 ++++++++++- .../HealthCheck/Checks/Security/HstsCheck.cs | 20 +++++++++++++++++++ .../Checks/Security/XssProtectionCheck.cs | 20 +++++++++++++++++++ src/Umbraco.Web/Umbraco.Web.csproj | 2 ++ 5 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 src/Umbraco.Web/HealthCheck/Checks/Security/HstsCheck.cs create mode 100644 src/Umbraco.Web/HealthCheck/Checks/Security/XssProtectionCheck.cs diff --git a/src/Umbraco.Web.UI/umbraco/config/lang/en.xml b/src/Umbraco.Web.UI/umbraco/config/lang/en.xml index e16e3108f1..e0f9495298 100644 --- a/src/Umbraco.Web.UI/umbraco/config/lang/en.xml +++ b/src/Umbraco.Web.UI/umbraco/config/lang/en.xml @@ -2109,7 +2109,17 @@ To manage your website, simply open the Umbraco back office and start adding con X-Content-Type-Options used to protect against MIME sniffing vulnerabilities was not found.]]> Adds a value to the httpProtocol/customHeaders section of web.config to protect against MIME sniffing vulnerabilities. A setting to create a header protecting against MIME sniffing vulnerabilities has been added to your web.config file. - + + Strict-Transport-Security, also known as the HSTS-header, was found.]]> + Strict-Transport-Security was not found.]]> + Adds the header 'Strict-Transport-Security' with the value 'max-age=10886400; preload' to the httpProtocol/customHeaders section of web.config. Use this fix only if you will have your domains running with https for the next 18 weeks (minimum). + The HSTS header has been added to your web.config file. + + X-XSS-Protection was found.]]> + X-XSS-Protection was not found.]]> + Adds the header 'X-XSS-Protection' with the value '1; mode=block' to the httpProtocol/customHeaders section of web.config. + The X-XSS-Protection header has been added to your web.config file. + diff --git a/src/Umbraco.Web.UI/umbraco/config/lang/en_us.xml b/src/Umbraco.Web.UI/umbraco/config/lang/en_us.xml index 3632b8162a..a6c0774582 100644 --- a/src/Umbraco.Web.UI/umbraco/config/lang/en_us.xml +++ b/src/Umbraco.Web.UI/umbraco/config/lang/en_us.xml @@ -2100,7 +2100,17 @@ To manage your website, simply open the Umbraco back office and start adding con X-Content-Type-Options used to protect against MIME sniffing vulnerabilities was not found.]]> Adds a value to the httpProtocol/customHeaders section of web.config to protect against MIME sniffing vulnerabilities. A setting to create a header protecting against MIME sniffing vulnerabilities has been added to your web.config file. - + + Strict-Transport-Security, also known as the HSTS-header, was found.]]> + Strict-Transport-Security was not found.]]> + Adds the header 'Strict-Transport-Security' with the value 'max-age=10886400; preload' to the httpProtocol/customHeaders section of web.config. Use this fix only if you will have your domains running with https for the next 18 weeks (minimum). + The HSTS header has been added to your web.config file. + + X-XSS-Protection was found.]]> + X-XSS-Protection was not found.]]> + Adds the header 'X-XSS-Protection' with the value '1; mode=block' to the httpProtocol/customHeaders section of web.config. + The X-XSS-Protection header has been added to your web.config file. + diff --git a/src/Umbraco.Web/HealthCheck/Checks/Security/HstsCheck.cs b/src/Umbraco.Web/HealthCheck/Checks/Security/HstsCheck.cs new file mode 100644 index 0000000000..fede5c7907 --- /dev/null +++ b/src/Umbraco.Web/HealthCheck/Checks/Security/HstsCheck.cs @@ -0,0 +1,20 @@ +namespace Umbraco.Web.HealthCheck.Checks.Security +{ + [HealthCheck( + "E2048C48-21C5-4BE1-A80B-8062162DF124", + "Cookie hijacking and protocol downgrade attacks Protection (Strict-Transport-Security Header (HSTS))", + Description = "Checks if your site, when running with HTTPS, contains the Strict-Transport-Security Header (HSTS). If not, it adds with a default of 100 days.", + Group = "Security")] + public class HstsCheck : BaseHttpHeaderCheck + { + // The check is mostly based on the instructions in the OWASP CheatSheet + // (https://www.owasp.org/index.php/HTTP_Strict_Transport_Security_Cheat_Sheet) + // and the blogpost of Troy Hunt (https://www.troyhunt.com/understanding-http-strict-transport/) + // If you want do to it perfectly, you have to submit it https://hstspreload.appspot.com/, + // but then you should include subdomains and I wouldn't suggest to do that for Umbraco-sites. + public HstsCheck(HealthCheckContext healthCheckContext) + : base(healthCheckContext, "Strict-Transport-Security", "max-age=10886400; preload", "hSTS", true) + { + } + } +} diff --git a/src/Umbraco.Web/HealthCheck/Checks/Security/XssProtectionCheck.cs b/src/Umbraco.Web/HealthCheck/Checks/Security/XssProtectionCheck.cs new file mode 100644 index 0000000000..897c92efa9 --- /dev/null +++ b/src/Umbraco.Web/HealthCheck/Checks/Security/XssProtectionCheck.cs @@ -0,0 +1,20 @@ +namespace Umbraco.Web.HealthCheck.Checks.Security +{ + [HealthCheck( + "F4D2B02E-28C5-4999-8463-05759FA15C3A", + "Cross-site scripting Protection (X-XSS-Protection header)", + Description = "This header enables the Cross-site scripting (XSS) filter in your browser. It checks for the presence of the X-XSS-Protection-header.", + Group = "Security")] + public class XssProtectionCheck : BaseHttpHeaderCheck + { + // The check is mostly based on the instructions in the OWASP CheatSheet + // (https://www.owasp.org/index.php/HTTP_Strict_Transport_Security_Cheat_Sheet) + // and the blogpost of Troy Hunt (https://www.troyhunt.com/understanding-http-strict-transport/) + // If you want do to it perfectly, you have to submit it https://hstspreload.appspot.com/, + // but then you should include subdomains and I wouldn't suggest to do that for Umbraco-sites. + public XssProtectionCheck(HealthCheckContext healthCheckContext) + : base(healthCheckContext, "X-XSS-Protection", "1; mode=block", "xssProtection", true) + { + } + } +} diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 8a0106e0b1..b578e20401 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -337,6 +337,8 @@ + + From b535a458987bec791313c17395d971285c75d0f5 Mon Sep 17 00:00:00 2001 From: Bjarne Fyrstenborg Date: Sun, 15 Apr 2018 21:39:46 +0200 Subject: [PATCH 42/57] Respect icon set on member type - by default the icon is "icon-user" --- src/Umbraco.Web.UI.Client/src/views/member/create.html | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/views/member/create.html b/src/Umbraco.Web.UI.Client/src/views/member/create.html index 188c6214eb..57a11ac068 100644 --- a/src/Umbraco.Web.UI.Client/src/views/member/create.html +++ b/src/Umbraco.Web.UI.Client/src/views/member/create.html @@ -6,10 +6,9 @@
  • - - - - {{docType.name}} + + + {{docType.name}} {{docType.description}} @@ -24,4 +23,3 @@ Do something else - From 588f5a3789fd2ba086eb4a6885b77ec01d26eed9 Mon Sep 17 00:00:00 2001 From: Bjarne Fyrstenborg Date: Sun, 15 Apr 2018 21:41:23 +0200 Subject: [PATCH 43/57] Fix some wrong descriptions --- .../src/common/resources/membertype.resource.js | 4 ++-- src/Umbraco.Web/Editors/MediaTypeController.cs | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/common/resources/membertype.resource.js b/src/Umbraco.Web.UI.Client/src/common/resources/membertype.resource.js index 6c83f69f84..eba0b41fe0 100644 --- a/src/Umbraco.Web.UI.Client/src/common/resources/membertype.resource.js +++ b/src/Umbraco.Web.UI.Client/src/common/resources/membertype.resource.js @@ -86,8 +86,8 @@ function memberTypeResource($q, $http, umbRequestHelper, umbDataFormatter) { /** * @ngdoc method - * @name umbraco.resources.contentTypeResource#save - * @methodOf umbraco.resources.contentTypeResource + * @name umbraco.resources.memberTypeResource#save + * @methodOf umbraco.resources.memberTypeResource * * @description * Saves or update a member type diff --git a/src/Umbraco.Web/Editors/MediaTypeController.cs b/src/Umbraco.Web/Editors/MediaTypeController.cs index 43312fb7c3..fbd2641e08 100644 --- a/src/Umbraco.Web/Editors/MediaTypeController.cs +++ b/src/Umbraco.Web/Editors/MediaTypeController.cs @@ -81,7 +81,7 @@ namespace Umbraco.Web.Editors } /// - /// Deletes a document type wth a given ID + /// Deletes a media type with a given ID /// /// /// @@ -138,17 +138,16 @@ namespace Umbraco.Web.Editors /// - /// Returns all member types + /// Returns all media types /// public IEnumerable GetAll() { - return Services.ContentTypeService.GetAllMediaTypes() .Select(Mapper.Map); } /// - /// Deletes a document type container wth a given ID + /// Deletes a media type container wth a given ID /// /// /// @@ -324,4 +323,4 @@ namespace Umbraco.Web.Editors doCopy: (type, i) => Services.ContentTypeService.CopyMediaType(type, i)); } } -} \ No newline at end of file +} From b7c04575faf37c767b68961e8847ab00d981bb04 Mon Sep 17 00:00:00 2001 From: Bjarne Fyrstenborg Date: Sun, 15 Apr 2018 21:42:04 +0200 Subject: [PATCH 44/57] Document media type resource save as in member type resource --- .../src/common/resources/mediatype.resource.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/Umbraco.Web.UI.Client/src/common/resources/mediatype.resource.js b/src/Umbraco.Web.UI.Client/src/common/resources/mediatype.resource.js index 06df2e0695..090d6a1ee9 100644 --- a/src/Umbraco.Web.UI.Client/src/common/resources/mediatype.resource.js +++ b/src/Umbraco.Web.UI.Client/src/common/resources/mediatype.resource.js @@ -122,6 +122,18 @@ function mediaTypeResource($q, $http, umbRequestHelper, umbDataFormatter) { 'Failed to delete content type contaier'); }, + /** + * @ngdoc method + * @name umbraco.resources.mediaTypeResource#save + * @methodOf umbraco.resources.mediaTypeResource + * + * @description + * Saves or update a media type + * + * @param {Object} content data type object to create/update + * @returns {Promise} resourcePromise object. + * + */ save: function (contentType) { var saveModel = umbDataFormatter.formatContentTypePostData(contentType); From d45c3520d468ced9f93225ce0c1f59a4c28918aa Mon Sep 17 00:00:00 2001 From: Bjarne Fyrstenborg Date: Sun, 15 Apr 2018 21:42:35 +0200 Subject: [PATCH 45/57] Format document in media create dialog --- .../src/views/media/create.html | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/views/media/create.html b/src/Umbraco.Web.UI.Client/src/views/media/create.html index cfcbf1012f..56a1c9e90c 100644 --- a/src/Umbraco.Web.UI.Client/src/views/media/create.html +++ b/src/Umbraco.Web.UI.Client/src/views/media/create.html @@ -16,24 +16,23 @@ {{docType.name}} - - {{docType.description}} - + {{docType.description}}
    • - + From 1594adf3ed6651001e5ca12dad96eff316aa82d9 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Sun, 15 Apr 2018 22:38:09 +0200 Subject: [PATCH 46/57] Makes code a bit more readable and uses ToList instead of ToArray --- src/Umbraco.Web/Editors/TourController.cs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Umbraco.Web/Editors/TourController.cs b/src/Umbraco.Web/Editors/TourController.cs index a493ea5b99..5fb54ddd32 100644 --- a/src/Umbraco.Web/Editors/TourController.cs +++ b/src/Umbraco.Web/Editors/TourController.cs @@ -118,14 +118,18 @@ namespace Umbraco.Web.Editors var contents = File.ReadAllText(tourFile); var tours = JsonConvert.DeserializeObject(contents); + var backOfficeTours = tours.Where(x => + aliasFilters.Count == 0 || aliasFilters.All(filter => filter.IsMatch(x.Alias)) == false); + + var localizedTours = backOfficeTours.Where(x => + string.IsNullOrWhiteSpace(x.Culture) || x.Culture.Equals(Security.CurrentUser.Language, + StringComparison.InvariantCultureIgnoreCase)).ToList(); + var tour = new BackOfficeTourFile { FileName = Path.GetFileNameWithoutExtension(tourFile), PluginName = pluginName, - Tours = tours - .Where(x => (aliasFilters.Count == 0 || aliasFilters.All(filter => filter.IsMatch(x.Alias)) == false) - && (x.Culture == null || string.IsNullOrEmpty(x.Culture) || x.Culture.Equals(Security.CurrentUser.Language, StringComparison.InvariantCultureIgnoreCase))) - .ToArray() + Tours = localizedTours }; //don't add if all of the tours are filtered From eca5b6e4e6b4e028ad2489c7b4e8ee15aabadd18 Mon Sep 17 00:00:00 2001 From: Dave Woestenborghs Date: Mon, 16 Apr 2018 08:35:05 +0200 Subject: [PATCH 47/57] U4-11150 added culture property of tour to the docs --- .../directives/components/application/umbtour.directive.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Umbraco.Web.UI.Client/src/common/directives/components/application/umbtour.directive.js b/src/Umbraco.Web.UI.Client/src/common/directives/components/application/umbtour.directive.js index 6dc066b282..55641b6ec7 100644 --- a/src/Umbraco.Web.UI.Client/src/common/directives/components/application/umbtour.directive.js +++ b/src/Umbraco.Web.UI.Client/src/common/directives/components/application/umbtour.directive.js @@ -25,6 +25,7 @@ The tour object consist of two parts - The overall tour configuration and a list "group": "My Custom Group" // Used to group tours in the help drawer "groupOrder": 200 // Control the order of tour groups "allowDisable": // Adds a "Don't" show this tour again"-button to the intro step + "culture" : // From v7.11+. Specifies the culture of the tour (eg. en-US), if set the tour will only be shown to users with this culture set on their profile. If omitted or left empty the tour will be visible to all users "requiredSections":["content", "media", "mySection"] // Sections that the tour will access while running, if the user does not have access to the required tour sections, the tour will not load. "steps": [] // tour steps - see next example } From 0c191a3ef784ebeba401f60df8b134b0b0c743f5 Mon Sep 17 00:00:00 2001 From: Claus Date: Tue, 17 Apr 2018 08:36:36 +0200 Subject: [PATCH 48/57] updating to use includeProperties, fixing modifier keywords for methods, and passing includeProperties from list views. --- .../src/common/resources/content.resource.js | 14 ++++---- src/Umbraco.Web/Editors/ContentController.cs | 32 ++++++++++++++++--- .../Mapping/ContentPropertyBasicConverter.cs | 25 ++++++++------- .../ContentPropertyDisplayConverter.cs | 3 +- 4 files changed, 49 insertions(+), 25 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/common/resources/content.resource.js b/src/Umbraco.Web.UI.Client/src/common/resources/content.resource.js index 642e4fb13b..1ba454ac65 100644 --- a/src/Umbraco.Web.UI.Client/src/common/resources/content.resource.js +++ b/src/Umbraco.Web.UI.Client/src/common/resources/content.resource.js @@ -57,7 +57,7 @@ function contentResource($q, $http, umbDataFormatter, umbRequestHelper) { saveModel), 'Failed to save permissions'); }, - + getRecycleBin: function () { return umbRequestHelper.resourcePromise( @@ -424,7 +424,7 @@ function contentResource($q, $http, umbDataFormatter, umbRequestHelper) { $http.get( umbRequestHelper.getApiUrl( "contentApiBaseUrl", - "GetEmpty", + "GetEmpty", [{ blueprintId: blueprintId }, { parentId: parentId}])), 'Failed to retrieve blueprint for id ' + blueprintId); }, @@ -488,6 +488,7 @@ function contentResource($q, $http, umbDataFormatter, umbRequestHelper) { getChildren: function (parentId, options) { var defaults = { + includeProperties: [], pageSize: 0, pageNumber: 0, filter: '', @@ -531,6 +532,7 @@ function contentResource($q, $http, umbDataFormatter, umbRequestHelper) { "GetChildren", { id: parentId, + includeProperties: _.pluck(options.includeProperties, 'alias').join(","), pageNumber: options.pageNumber, pageSize: options.pageSize, orderBy: options.orderBy, @@ -580,7 +582,7 @@ function contentResource($q, $http, umbDataFormatter, umbRequestHelper) { "contentApiBaseUrl", "GetDetailedPermissions", { contentId: contentId })), 'Failed to retrieve permissions for content item ' + contentId); - }, + }, getPermissions: function (nodeIds) { return umbRequestHelper.resourcePromise( @@ -741,11 +743,11 @@ function contentResource($q, $http, umbDataFormatter, umbRequestHelper) { createBlueprintFromContent: function (contentId, name) { return umbRequestHelper.resourcePromise( - $http.post( + $http.post( umbRequestHelper.getApiUrl("contentApiBaseUrl", "CreateBlueprintFromContent", { contentId: contentId, name: name - }) - ), + }) + ), "Failed to create blueprint from content with id " + contentId ); } diff --git a/src/Umbraco.Web/Editors/ContentController.cs b/src/Umbraco.Web/Editors/ContentController.cs index 64e3318b4c..26f938638c 100644 --- a/src/Umbraco.Web/Editors/ContentController.cs +++ b/src/Umbraco.Web/Editors/ContentController.cs @@ -389,7 +389,25 @@ namespace Umbraco.Web.Editors Direction orderDirection = Direction.Ascending, bool orderBySystemField = true, string filter = "") - { + { + return GetChildren(id, null, pageNumber, pageSize, orderBy, orderDirection, orderBySystemField, filter); + } + + /// + /// Gets the children for the content id passed in + /// + /// + [FilterAllowedOutgoingContent(typeof(IEnumerable>), "Items")] + public PagedResult> GetChildren( + int id, + string includeProperties, + int pageNumber = 0, //TODO: This should be '1' as it's not the index + int pageSize = 0, + string orderBy = "SortOrder", + Direction orderDirection = Direction.Ascending, + bool orderBySystemField = true, + string filter = "") + { long totalChildren; IContent[] children; if (pageNumber > 0 && pageSize > 0) @@ -409,13 +427,17 @@ namespace Umbraco.Web.Editors return new PagedResult>(0, 0, 0); } - // Note that we're excluding mapping of complex properties here to ensure that getting a larger amount of - // children for listviews and other similar cases, will not make everything halt when it tries to deserialize a - // complex property such as Nested Content. var pagedResult = new PagedResult>(totalChildren, pageNumber, pageSize); pagedResult.Items = children.Select(content => Mapper.Map>(content, - opts => { opts.Items["ExcludeComplexProperties"] = true; })); + opts => + { + // if there's a list of property aliases to map - we will make sure to store this in the mapping context. + if (String.IsNullOrWhiteSpace(includeProperties) == false) + { + opts.Items["IncludeProperties"] = includeProperties.Split(new[] { ", ", "," }, StringSplitOptions.RemoveEmptyEntries); + } + })); return pagedResult; } diff --git a/src/Umbraco.Web/Models/Mapping/ContentPropertyBasicConverter.cs b/src/Umbraco.Web/Models/Mapping/ContentPropertyBasicConverter.cs index 89be6859d1..ccf22a8c34 100644 --- a/src/Umbraco.Web/Models/Mapping/ContentPropertyBasicConverter.cs +++ b/src/Umbraco.Web/Models/Mapping/ContentPropertyBasicConverter.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using AutoMapper; using Umbraco.Core; using Umbraco.Core.Logging; @@ -30,7 +31,7 @@ namespace Umbraco.Web.Models.Mapping /// Assigns the PropertyEditor, Id, Alias and Value to the property ///     /// - public T Convert(ResolutionContext context) + public virtual T Convert(ResolutionContext context) { var property = context.SourceValue as Property; if (property == null) @@ -55,19 +56,19 @@ namespace Umbraco.Web.Models.Mapping Editor = editor.Alias }; - // Complex properties such as Nested Content do not need to be mapped for simpler things like list views, - // where they will not make sense to use anyways. To avoid having to do unnecessary mapping on large - // collections of items in list views - we allow excluding mapping of certain properties. - var excludeComplexProperties = false; - if (context.Options.Items.ContainsKey("ExcludeComplexProperties")) + // if there's a set of property aliases specified, we will check if the current property's value should be mapped. + // if it isn't one of the ones specified in 'includeProperties', we will just return the result without mapping the Value. + if (context.Options.Items.ContainsKey("IncludeProperties")) { - excludeComplexProperties = System.Convert.ToBoolean(context.Options.Items["ExcludeComplexProperties"]); + var includeProperties = context.Options.Items["IncludeProperties"] as IEnumerable; + if (includeProperties != null && includeProperties.Contains(property.Alias) == false) + { + return result; + } } - if (excludeComplexProperties == false || ComplexPropertyTypeAliases.Contains(property.PropertyType.PropertyEditorAlias) == false) - { - result.Value = editor.ValueEditor.ConvertDbToEditor(property, property.PropertyType, DataTypeService); - } - + + // if no 'IncludeProperties' were specified or this property is set to be included - we will map the value and return. + result.Value = editor.ValueEditor.ConvertDbToEditor(property, property.PropertyType, DataTypeService); return result; } } diff --git a/src/Umbraco.Web/Models/Mapping/ContentPropertyDisplayConverter.cs b/src/Umbraco.Web/Models/Mapping/ContentPropertyDisplayConverter.cs index 632268ab2f..86c79d9c59 100644 --- a/src/Umbraco.Web/Models/Mapping/ContentPropertyDisplayConverter.cs +++ b/src/Umbraco.Web/Models/Mapping/ContentPropertyDisplayConverter.cs @@ -18,8 +18,7 @@ namespace Umbraco.Web.Models.Mapping { _textService = textService; } - - public new ContentPropertyDisplay Convert(ResolutionContext context) + public override ContentPropertyDisplay Convert(ResolutionContext context) { var display = base.Convert(context); From ef47e85741b3c2e1daf4f4a65a0229a05642fa86 Mon Sep 17 00:00:00 2001 From: Robert Date: Tue, 17 Apr 2018 15:07:59 +0200 Subject: [PATCH 49/57] Removed code that populates content tree with duplicate start nodes. --- src/Umbraco.Web/Trees/ContentTreeControllerBase.cs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/Umbraco.Web/Trees/ContentTreeControllerBase.cs b/src/Umbraco.Web/Trees/ContentTreeControllerBase.cs index 99afc199c9..e8ba964f98 100644 --- a/src/Umbraco.Web/Trees/ContentTreeControllerBase.cs +++ b/src/Umbraco.Web/Trees/ContentTreeControllerBase.cs @@ -189,8 +189,7 @@ namespace Umbraco.Web.Trees { // try to parse id as an integer else use GetEntityFromId // which will grok Guids, Udis, etc and let use obtain the id - int entityId; - if (int.TryParse(id, out entityId) == false) + if (int.TryParse(id, out var entityId) == false) { var entity = GetEntityFromId(id); if (entity == null) @@ -198,15 +197,6 @@ namespace Umbraco.Web.Trees entityId = entity.Id; } - // if a request is made for the root node but user has no access to - // root node, return start nodes instead - if (entityId == Constants.System.Root && UserStartNodes.Contains(Constants.System.Root) == false) - { - return UserStartNodes.Length > 0 - ? Services.EntityService.GetAll(UmbracoObjectType, UserStartNodes) - : Enumerable.Empty(); - } - return Services.EntityService.GetChildren(entityId, UmbracoObjectType).ToArray(); } From 7d171b1e95a43b185ba035b0f5beba11e15416f9 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Tue, 17 Apr 2018 15:46:39 +0200 Subject: [PATCH 50/57] Bumps version to 7.10.4 --- src/SolutionInfo.cs | 4 ++-- src/Umbraco.Core/Configuration/UmbracoVersion.cs | 2 +- src/Umbraco.Web.UI/Umbraco.Web.UI.csproj | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/SolutionInfo.cs b/src/SolutionInfo.cs index 807131b63e..056274e8be 100644 --- a/src/SolutionInfo.cs +++ b/src/SolutionInfo.cs @@ -11,5 +11,5 @@ using System.Resources; [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyFileVersion("7.10.3")] -[assembly: AssemblyInformationalVersion("7.10.3")] \ No newline at end of file +[assembly: AssemblyFileVersion("7.10.4")] +[assembly: AssemblyInformationalVersion("7.10.4")] \ No newline at end of file diff --git a/src/Umbraco.Core/Configuration/UmbracoVersion.cs b/src/Umbraco.Core/Configuration/UmbracoVersion.cs index ef11d9ff08..2b4920f992 100644 --- a/src/Umbraco.Core/Configuration/UmbracoVersion.cs +++ b/src/Umbraco.Core/Configuration/UmbracoVersion.cs @@ -6,7 +6,7 @@ namespace Umbraco.Core.Configuration { public class UmbracoVersion { - private static readonly Version Version = new Version("7.10.3"); + private static readonly Version Version = new Version("7.10.4"); /// /// Gets the current version of Umbraco. diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index a19c92bcbb..9210613c04 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -1035,9 +1035,9 @@ xcopy "$(ProjectDir)"..\packages\SqlServerCE.4.0.0.1\x86\*.* "$(TargetDir)x86\" True True - 7103 + 7104 / - http://localhost:7103 + http://localhost:7104 False False From f25233ac2b97603d633d88a4fd01dafb57f22f62 Mon Sep 17 00:00:00 2001 From: Dave Woestenborghs Date: Wed, 18 Apr 2018 10:21:12 +0200 Subject: [PATCH 51/57] U4-11216 Multiple Content Picker allows only one item when used as macro parameter (#2593) U4-11216 set min and max number when used as parameter editor --- .../propertyeditors/contentpicker/contentpicker.controller.js | 4 ++-- .../ParameterEditors/MultipleContentPickerParameterEditor.cs | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/contentpicker/contentpicker.controller.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/contentpicker/contentpicker.controller.js index 161f989d10..5827c046fd 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/contentpicker/contentpicker.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/contentpicker/contentpicker.controller.js @@ -58,7 +58,7 @@ function contentPickerController($scope, entityResource, editorState, iconHelper $scope.renderModel = []; - $scope.dialogEditor = editorState && editorState.current && editorState.current.isDialogEditor === true; + $scope.dialogEditor = editorState && editorState.current && editorState.current.isDialogEditor === true; //the default pre-values var defaultConfig = { @@ -93,7 +93,7 @@ function contentPickerController($scope, entityResource, editorState, iconHelper $scope.model.config.showOpenButton = ($scope.model.config.showOpenButton === "1" ? true : false); $scope.model.config.showEditButton = ($scope.model.config.showEditButton === "1" ? true : false); $scope.model.config.showPathOnHover = ($scope.model.config.showPathOnHover === "1" ? true : false); - + var entityType = $scope.model.config.startNode.type === "member" ? "Member" : $scope.model.config.startNode.type === "media" diff --git a/src/Umbraco.Web/PropertyEditors/ParameterEditors/MultipleContentPickerParameterEditor.cs b/src/Umbraco.Web/PropertyEditors/ParameterEditors/MultipleContentPickerParameterEditor.cs index 1fb9b267ab..27d1076352 100644 --- a/src/Umbraco.Web/PropertyEditors/ParameterEditors/MultipleContentPickerParameterEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/ParameterEditors/MultipleContentPickerParameterEditor.cs @@ -9,6 +9,8 @@ namespace Umbraco.Web.PropertyEditors.ParameterEditors public MultipleContentPickerParameterEditor() { Configuration.Add("multiPicker", "1"); + Configuration.Add("minNumber",0 ); + Configuration.Add("maxNumber", 0); } } -} \ No newline at end of file +} From 21a8395142446e8b0f0ca7c90e99fc16177e869a Mon Sep 17 00:00:00 2001 From: Dave Woestenborghs Date: Wed, 18 Apr 2018 10:21:12 +0200 Subject: [PATCH 52/57] U4-11216 Multiple Content Picker allows only one item when used as macro parameter (#2593) U4-11216 set min and max number when used as parameter editor (cherry picked from commit f25233ac2b97603d633d88a4fd01dafb57f22f62) --- .../propertyeditors/contentpicker/contentpicker.controller.js | 4 ++-- .../ParameterEditors/MultipleContentPickerParameterEditor.cs | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/contentpicker/contentpicker.controller.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/contentpicker/contentpicker.controller.js index 161f989d10..5827c046fd 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/contentpicker/contentpicker.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/contentpicker/contentpicker.controller.js @@ -58,7 +58,7 @@ function contentPickerController($scope, entityResource, editorState, iconHelper $scope.renderModel = []; - $scope.dialogEditor = editorState && editorState.current && editorState.current.isDialogEditor === true; + $scope.dialogEditor = editorState && editorState.current && editorState.current.isDialogEditor === true; //the default pre-values var defaultConfig = { @@ -93,7 +93,7 @@ function contentPickerController($scope, entityResource, editorState, iconHelper $scope.model.config.showOpenButton = ($scope.model.config.showOpenButton === "1" ? true : false); $scope.model.config.showEditButton = ($scope.model.config.showEditButton === "1" ? true : false); $scope.model.config.showPathOnHover = ($scope.model.config.showPathOnHover === "1" ? true : false); - + var entityType = $scope.model.config.startNode.type === "member" ? "Member" : $scope.model.config.startNode.type === "media" diff --git a/src/Umbraco.Web/PropertyEditors/ParameterEditors/MultipleContentPickerParameterEditor.cs b/src/Umbraco.Web/PropertyEditors/ParameterEditors/MultipleContentPickerParameterEditor.cs index 1fb9b267ab..27d1076352 100644 --- a/src/Umbraco.Web/PropertyEditors/ParameterEditors/MultipleContentPickerParameterEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/ParameterEditors/MultipleContentPickerParameterEditor.cs @@ -9,6 +9,8 @@ namespace Umbraco.Web.PropertyEditors.ParameterEditors public MultipleContentPickerParameterEditor() { Configuration.Add("multiPicker", "1"); + Configuration.Add("minNumber",0 ); + Configuration.Add("maxNumber", 0); } } -} \ No newline at end of file +} From eeaf17c857dbb0da2fb4db66f96930c8db018f33 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Thu, 12 Apr 2018 17:50:44 +0200 Subject: [PATCH 53/57] Weird revert of a PR that was merged in before (cherry picked from commit 94ed741dd51b3960e274e926daaca85183a8015e) --- .../src/views/propertyeditors/rte/rte.controller.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/rte/rte.controller.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/rte/rte.controller.js index f886c60bc1..17f0f3b698 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/rte/rte.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/rte/rte.controller.js @@ -266,7 +266,7 @@ angular.module("umbraco") editor.on('ObjectResized', function (e) { - var qs = "?width=" + e.width + "&height=" + e.height; + var qs = "?width=" + e.width + "&height=" + e.height + "&mode=max"; var srcAttr = $(e.target).attr("src"); var path = srcAttr.split("?")[0]; $(e.target).attr("data-mce-src", path + qs); @@ -384,7 +384,7 @@ angular.module("umbraco") // element might still be there even after the modal has been hidden. $scope.$on('$destroy', function () { unsubscribe(); - if (tinyMceEditor !== undefined && tinyMceEditor != null) { + if (tinyMceEditor !== undefined && tinyMceEditor != null) { tinyMceEditor.destroy() } }); From cc3e75dc6379bf41cb022f8d7b9ae5af92a83ad6 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Wed, 18 Apr 2018 15:14:16 +0200 Subject: [PATCH 54/57] U4-11208 Error : "An item with the same key has already been added." into "Umbraco.Core.Services.IdkMap.Populate" method --- src/Umbraco.Core/Services/IdkMap.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Core/Services/IdkMap.cs b/src/Umbraco.Core/Services/IdkMap.cs index 7cf69f2e64..c1981b7b80 100644 --- a/src/Umbraco.Core/Services/IdkMap.cs +++ b/src/Umbraco.Core/Services/IdkMap.cs @@ -76,8 +76,9 @@ namespace Umbraco.Core.Services _locker.EnterWriteLock(); foreach (var pair in pairs) { - _id2Key.Add(pair.id, new TypedId(pair.key, umbracoObjectType)); - _key2Id.Add(pair.key, new TypedId(pair.id, umbracoObjectType)); + + _id2Key[pair.id] = new TypedId(pair.key, umbracoObjectType); + _key2Id[pair.key] = new TypedId(pair.id, umbracoObjectType); } } finally From 15eb40aedbf8d0bc07c19726433d2e80cc14e615 Mon Sep 17 00:00:00 2001 From: Robert Date: Tue, 17 Apr 2018 15:07:59 +0200 Subject: [PATCH 55/57] Removed code that populates content tree with duplicate start nodes. --- src/Umbraco.Web/Trees/ContentTreeControllerBase.cs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/Umbraco.Web/Trees/ContentTreeControllerBase.cs b/src/Umbraco.Web/Trees/ContentTreeControllerBase.cs index 99afc199c9..e8ba964f98 100644 --- a/src/Umbraco.Web/Trees/ContentTreeControllerBase.cs +++ b/src/Umbraco.Web/Trees/ContentTreeControllerBase.cs @@ -189,8 +189,7 @@ namespace Umbraco.Web.Trees { // try to parse id as an integer else use GetEntityFromId // which will grok Guids, Udis, etc and let use obtain the id - int entityId; - if (int.TryParse(id, out entityId) == false) + if (int.TryParse(id, out var entityId) == false) { var entity = GetEntityFromId(id); if (entity == null) @@ -198,15 +197,6 @@ namespace Umbraco.Web.Trees entityId = entity.Id; } - // if a request is made for the root node but user has no access to - // root node, return start nodes instead - if (entityId == Constants.System.Root && UserStartNodes.Contains(Constants.System.Root) == false) - { - return UserStartNodes.Length > 0 - ? Services.EntityService.GetAll(UmbracoObjectType, UserStartNodes) - : Enumerable.Empty(); - } - return Services.EntityService.GetChildren(entityId, UmbracoObjectType).ToArray(); } From 75fb0708256b1882ca3d433a581032d80f6173b3 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 19 Apr 2018 16:20:18 +1000 Subject: [PATCH 56/57] U4-11246 When user belongs to groups and none of them have browse access and one or more have a custom start node applied a YSOD occurs --- src/Umbraco.Web/Trees/ContentTreeControllerBase.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Umbraco.Web/Trees/ContentTreeControllerBase.cs b/src/Umbraco.Web/Trees/ContentTreeControllerBase.cs index e8ba964f98..4705e16c7e 100644 --- a/src/Umbraco.Web/Trees/ContentTreeControllerBase.cs +++ b/src/Umbraco.Web/Trees/ContentTreeControllerBase.cs @@ -96,6 +96,12 @@ namespace Umbraco.Web.Trees return null; var treeNode = GetSingleTreeNode(e, parentId, queryStrings); + if (treeNode == null) + { + //this means that the user has NO access to this node via permissions! They at least need to have browse permissions to see + //the node so we need to return null; + return null; + } if (hasPathAccess == false) { treeNode.AdditionalData["noAccess"] = true; From 0262b4aa76b3f6c2248c1a7e78abd26d788ea170 Mon Sep 17 00:00:00 2001 From: Robert Date: Thu, 19 Apr 2018 12:10:39 +0200 Subject: [PATCH 57/57] Changed ContentPropertyDto modifier back to override which rendered content with tags unable to be saved --- src/Umbraco.Web/Models/Mapping/ContentPropertyDtoConverter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Web/Models/Mapping/ContentPropertyDtoConverter.cs b/src/Umbraco.Web/Models/Mapping/ContentPropertyDtoConverter.cs index 80b8c28309..61fd4fbe00 100644 --- a/src/Umbraco.Web/Models/Mapping/ContentPropertyDtoConverter.cs +++ b/src/Umbraco.Web/Models/Mapping/ContentPropertyDtoConverter.cs @@ -16,7 +16,7 @@ namespace Umbraco.Web.Models.Mapping { } - public new ContentPropertyDto Convert(ResolutionContext context) + public override ContentPropertyDto Convert(ResolutionContext context) { var propertyDto = base.Convert(context);