From 38778cc7c70a877f1b4053e3c55b776323727396 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Wed, 24 Jul 2013 14:52:25 +0200 Subject: [PATCH 01/20] Fixes U4-2488 Edit datatype: Media Picker appears incorrectly --- .../umbraco/developer/DataTypes/editDatatype.aspx.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Umbraco.Web/umbraco.presentation/umbraco/developer/DataTypes/editDatatype.aspx.cs b/src/Umbraco.Web/umbraco.presentation/umbraco/developer/DataTypes/editDatatype.aspx.cs index 7e22296ed9..8149050bb8 100644 --- a/src/Umbraco.Web/umbraco.presentation/umbraco/developer/DataTypes/editDatatype.aspx.cs +++ b/src/Umbraco.Web/umbraco.presentation/umbraco/developer/DataTypes/editDatatype.aspx.cs @@ -52,6 +52,12 @@ namespace umbraco.cms.presentation.developer li.Text = ide.Key.ToString().Substring(0, ide.Key.ToString().IndexOf("|")); li.Value = ide.Value.ToString(); + //SJ Fixes U4-2488 Edit datatype: Media Picker appears incorrectly + //Apparently in some installs the media picker rendercontrol is installed twice with + //the exact same ID so we need to check for duplicates + if (ddlRenderControl.Items.Contains(li)) + continue; + if (!String.IsNullOrEmpty(datatTypeId) && li.Value.ToString() == datatTypeId) li.Selected = true; ddlRenderControl.Items.Add(li); } From 118f9b02e25a5bd880daf7091238baaa52f7a9da Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 25 Jul 2013 12:29:41 +1000 Subject: [PATCH 02/20] Fixes: U4-1027 Media Sort Usability -> changed the sorting to use Web API instead of asmx web service. --- .../FolderBrowser/Js/folderbrowser.js | 31 +++++--- src/Umbraco.Web/Editors/MediaController.cs | 79 +++++++++++++++++++ .../Models/ContentEditing/ContentSortOrder.cs | 35 ++++++++ src/Umbraco.Web/Mvc/UmbracoController.cs | 9 +++ src/Umbraco.Web/Umbraco.Web.csproj | 3 + .../WebApi/UmbracoApiController.cs | 9 +++ 6 files changed, 157 insertions(+), 9 deletions(-) create mode 100644 src/Umbraco.Web/Editors/MediaController.cs create mode 100644 src/Umbraco.Web/Models/ContentEditing/ContentSortOrder.cs diff --git a/src/Umbraco.Web.UI/umbraco_client/FolderBrowser/Js/folderbrowser.js b/src/Umbraco.Web.UI/umbraco_client/FolderBrowser/Js/folderbrowser.js index 438a04344a..497c1843b0 100644 --- a/src/Umbraco.Web.UI/umbraco_client/FolderBrowser/Js/folderbrowser.js +++ b/src/Umbraco.Web.UI/umbraco_client/FolderBrowser/Js/folderbrowser.js @@ -374,17 +374,30 @@ Umbraco.Sys.registerNamespace("Umbraco.Controls"); if (self._viewModel.filterTerm().length > 0) { $(this).sortable("cancel"); alert("Can't sort items which have been filtered"); - } else { - $.post(self._opts.umbracoPath + "/webservices/nodeSorter.asmx/UpdateSortOrder", { - ParentId: self._parentId, - SortOrder: self._viewModel.itemIds().join(","), - app: "media" - }, function (data, textStatus) { - if (textStatus == "error") { - alert("Oops. Could not update sort order"); + } + else { + + $.ajax({ + url: self._opts.umbracoPath + "/umbracoapi/media/postsort", + type: 'POST', + contentType: "application/json; charset=utf-8", + dataType: 'json', + data: JSON.stringify({ + parentId: self._parentId, + idSortOrder: self._viewModel.itemIds() + }), + processData: false, + success: function (data, textStatus) { + if (textStatus == "error") { + alert("Oops. Could not update sort order"); + self._getChildNodes(); + } + }, + error: function(data) { + alert("Oops. Could not update sort order. Err: " + data.statusText); self._getChildNodes(); } - }, "json"); + }); } } }); diff --git a/src/Umbraco.Web/Editors/MediaController.cs b/src/Umbraco.Web/Editors/MediaController.cs new file mode 100644 index 0000000000..c3d1fe3918 --- /dev/null +++ b/src/Umbraco.Web/Editors/MediaController.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Runtime.Serialization; +using System.Text; +using Umbraco.Core.Logging; +using Umbraco.Core.Models; +using Umbraco.Web.Models.ContentEditing; +using Umbraco.Web.Mvc; +using Umbraco.Web.WebApi; +using Umbraco.Web.WebServices; + +namespace Umbraco.Web.Editors +{ + //NOTE: The reason why this controller exists here with this name is because in v7, all of the editor controllers exist here, + // therefore, these methods will still be exactly the same in v7 and no URLs will need to change! + + [PluginController("UmbracoApi")] + public class MediaController : UmbracoAuthorizedApiController + { + /// + /// Remove the xml formatter... only support JSON! + /// + /// + protected override void Initialize(global::System.Web.Http.Controllers.HttpControllerContext controllerContext) + { + base.Initialize(controllerContext); + controllerContext.Configuration.Formatters.Remove(controllerContext.Configuration.Formatters.XmlFormatter); + } + + /// + /// Change the sort order for media + /// + /// + /// + public HttpResponseMessage PostSort(ContentSortOrder sorted) + { + if (sorted == null) + { + return Request.CreateResponse(HttpStatusCode.NotFound); + } + + //if there's nothing to sort just return ok + if (sorted.IdSortOrder.Length == 0) + { + return Request.CreateResponse(HttpStatusCode.OK); + } + + if (!Security.UserHasAppAccess(global::Umbraco.Core.Constants.Applications.Media, UmbracoUser)) + { + return Request.CreateErrorResponse(HttpStatusCode.Unauthorized, "User has no access to this application"); + } + + var mediaService = base.ApplicationContext.Services.MediaService; + var sortedMedia = new List(); + try + { + sortedMedia.AddRange(sorted.IdSortOrder.Select(mediaService.GetById)); + + // Save Media with new sort order and update content xml in db accordingly + if (!mediaService.Sort(sortedMedia)) + { + LogHelper.Warn("Media sorting failed, this was probably caused by an event being cancelled"); + return Request.CreateErrorResponse(HttpStatusCode.Forbidden, "Media sorting failed, this was probably caused by an event being cancelled"); + } + return Request.CreateResponse(HttpStatusCode.OK); + } + catch (Exception ex) + { + LogHelper.Error("Could not update media sort order", ex); + throw; + } + } + + + } +} diff --git a/src/Umbraco.Web/Models/ContentEditing/ContentSortOrder.cs b/src/Umbraco.Web/Models/ContentEditing/ContentSortOrder.cs new file mode 100644 index 0000000000..d6cd7b3a3e --- /dev/null +++ b/src/Umbraco.Web/Models/ContentEditing/ContentSortOrder.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.Serialization; +using System.Text; + +namespace Umbraco.Web.Models.ContentEditing +{ + /// + /// A model representing a new sort order for a content/media item + /// + [DataContract(Name = "content", Namespace = "")] + public class ContentSortOrder + { + /// + /// The parent Id of the nodes being sorted + /// + /// + /// This is nullable because currently we don't require this for media sorting + /// + [DataMember(Name = "parentId")] + public int? ParentId { get; set; } + + /// + /// An array of integer Ids representing the sort order + /// + /// + /// Of course all of these Ids should be at the same level in the heirarchy!! + /// + [DataMember(Name = "idSortOrder")] + public int[] IdSortOrder { get; set; } + + } + +} diff --git a/src/Umbraco.Web/Mvc/UmbracoController.cs b/src/Umbraco.Web/Mvc/UmbracoController.cs index 78dc6502fc..99efb77f82 100644 --- a/src/Umbraco.Web/Mvc/UmbracoController.cs +++ b/src/Umbraco.Web/Mvc/UmbracoController.cs @@ -2,6 +2,7 @@ using System.Web.Mvc; using Umbraco.Core; using Umbraco.Core.Services; +using Umbraco.Web.Security; namespace Umbraco.Web.Mvc { @@ -59,5 +60,13 @@ namespace Umbraco.Web.Mvc { get { return ApplicationContext.DatabaseContext; } } + + /// + /// Returns the WebSecurity instance + /// + public WebSecurity Security + { + get { return UmbracoContext.Security; } + } } } \ No newline at end of file diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 38370e7832..67d9b6335d 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -168,6 +168,7 @@ ..\packages\Microsoft.Net.Http.2.0.20710.0\lib\net40\System.Net.Http.WebRequest.dll + 3.5 @@ -289,6 +290,8 @@ + + diff --git a/src/Umbraco.Web/WebApi/UmbracoApiController.cs b/src/Umbraco.Web/WebApi/UmbracoApiController.cs index 55e138b1bf..acbf82f161 100644 --- a/src/Umbraco.Web/WebApi/UmbracoApiController.cs +++ b/src/Umbraco.Web/WebApi/UmbracoApiController.cs @@ -2,6 +2,7 @@ using System.Web.Http; using Umbraco.Core; using Umbraco.Core.Services; +using Umbraco.Web.Security; namespace Umbraco.Web.WebApi { @@ -59,5 +60,13 @@ namespace Umbraco.Web.WebApi /// Useful for debugging /// internal Guid InstanceId { get; private set; } + + /// + /// Returns the WebSecurity instance + /// + public WebSecurity Security + { + get { return UmbracoContext.Security; } + } } } \ No newline at end of file From a099af39c2323b09badbe898d0cea8372ab750cc Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Thu, 25 Jul 2013 11:50:56 +0200 Subject: [PATCH 03/20] xmlrpcnet was missing from nuspecs --- build/NuSpecs/UmbracoCms.Core.nuspec | 1 + 1 file changed, 1 insertion(+) diff --git a/build/NuSpecs/UmbracoCms.Core.nuspec b/build/NuSpecs/UmbracoCms.Core.nuspec index 996f05936a..009b47b89b 100644 --- a/build/NuSpecs/UmbracoCms.Core.nuspec +++ b/build/NuSpecs/UmbracoCms.Core.nuspec @@ -23,6 +23,7 @@ + From e41d3c7e844b63780a4842ccd105e8bb593e4d54 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Thu, 25 Jul 2013 11:59:37 +0200 Subject: [PATCH 04/20] Increase version to 6.1.4 --- build/Build.bat | 2 +- src/Umbraco.Core/Configuration/UmbracoVersion.cs | 2 +- src/Umbraco.Web.UI/Umbraco.Web.UI.csproj | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build/Build.bat b/build/Build.bat index bcacc995e1..d1335ea93e 100644 --- a/build/Build.bat +++ b/build/Build.bat @@ -1,5 +1,5 @@ @ECHO OFF -SET release=6.1.3 +SET release=6.1.4 SET comment= SET version=%release% diff --git a/src/Umbraco.Core/Configuration/UmbracoVersion.cs b/src/Umbraco.Core/Configuration/UmbracoVersion.cs index dc50952f8d..9863edfee3 100644 --- a/src/Umbraco.Core/Configuration/UmbracoVersion.cs +++ b/src/Umbraco.Core/Configuration/UmbracoVersion.cs @@ -5,7 +5,7 @@ namespace Umbraco.Core.Configuration { public class UmbracoVersion { - private static readonly Version Version = new Version("6.1.3"); + private static readonly Version Version = new Version("6.1.4"); /// /// Gets the current version of Umbraco. diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index 8c3e9a95cb..709c257b86 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -2574,9 +2574,9 @@ xcopy "$(ProjectDir)"..\packages\SqlServerCE.4.0.0.0\x86\*.* "$(TargetDir)x86\" True True - 6130 + 6140 / - http://localhost:6130 + http://localhost:6140 False False From ed88bbf75f94e7bd04aaaee7519392439100ec87 Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 26 Jul 2013 18:13:56 +1000 Subject: [PATCH 05/20] Fixed up BaseDatabaseFactoryTests so that it only loads in one db factory with the correct conn strings, now we can easily override to use a custom db. Added benchmark tests for bulk inserting records for the cmsContentXml table which shows how we can improve performance quite a bit especially for larger node sets. Added more unit tests for bulk inserting and fixed it up a bit more. --- src/Umbraco.Core/DatabaseContext.cs | 15 +- .../Persistence/DefaultDatabaseFactory.cs | 18 +- .../Persistence/PetaPocoExtensions.cs | 111 ++++++--- .../Querying/ModelToSqlExpressionHelper.cs | 33 +-- .../Querying/PocoToSqlExpressionHelper.cs | 39 +--- .../Persistence/Querying/QueryHelper.cs | 70 ++++++ src/Umbraco.Core/Services/ContentService.cs | 2 + src/Umbraco.Core/Umbraco.Core.csproj | 1 + .../Persistence/PetaPocoExtensionsTest.cs | 39 +++- .../Services/PerformanceTests.cs | 211 ++++++++++++++++++ .../TestHelpers/BaseDatabaseFactoryTest.cs | 30 ++- src/Umbraco.Tests/Umbraco.Tests.csproj | 1 + 12 files changed, 449 insertions(+), 121 deletions(-) create mode 100644 src/Umbraco.Core/Persistence/Querying/QueryHelper.cs create mode 100644 src/Umbraco.Tests/Services/PerformanceTests.cs diff --git a/src/Umbraco.Core/DatabaseContext.cs b/src/Umbraco.Core/DatabaseContext.cs index 4f52f61683..1311593e90 100644 --- a/src/Umbraco.Core/DatabaseContext.cs +++ b/src/Umbraco.Core/DatabaseContext.cs @@ -296,16 +296,13 @@ namespace Umbraco.Core if (databaseSettings != null && string.IsNullOrWhiteSpace(databaseSettings.ConnectionString) == false && string.IsNullOrWhiteSpace(databaseSettings.ProviderName) == false) { var providerName = "System.Data.SqlClient"; + string connString = null; if (!string.IsNullOrEmpty(ConfigurationManager.ConnectionStrings[GlobalSettings.UmbracoConnectionName].ProviderName)) { providerName = ConfigurationManager.ConnectionStrings[GlobalSettings.UmbracoConnectionName].ProviderName; - - _connectionString = - ConfigurationManager.ConnectionStrings[GlobalSettings.UmbracoConnectionName].ConnectionString; - + connString = ConfigurationManager.ConnectionStrings[GlobalSettings.UmbracoConnectionName].ConnectionString; } - - Initialize(providerName); + Initialize(providerName, connString); } else if (ConfigurationManager.AppSettings.ContainsKey(GlobalSettings.UmbracoConnectionName) && string.IsNullOrEmpty(ConfigurationManager.AppSettings[GlobalSettings.UmbracoConnectionName]) == false) { @@ -369,6 +366,12 @@ namespace Umbraco.Core } } + internal void Initialize(string providerName, string connectionString) + { + _connectionString = connectionString; + Initialize(providerName); + } + internal DatabaseSchemaResult ValidateDatabaseSchema() { if (_configured == false || (string.IsNullOrEmpty(_connectionString) || string.IsNullOrEmpty(ProviderName))) diff --git a/src/Umbraco.Core/Persistence/DefaultDatabaseFactory.cs b/src/Umbraco.Core/Persistence/DefaultDatabaseFactory.cs index 1a723166e6..2014ceb1cf 100644 --- a/src/Umbraco.Core/Persistence/DefaultDatabaseFactory.cs +++ b/src/Umbraco.Core/Persistence/DefaultDatabaseFactory.cs @@ -14,9 +14,9 @@ namespace Umbraco.Core.Persistence /// internal class DefaultDatabaseFactory : DisposableObject, IDatabaseFactory { - private readonly string _connectionStringName; - private readonly string _connectionString; - private readonly string _providerName; + private readonly string _connectionStringName; + public string ConnectionString { get; private set; } + public string ProviderName { get; private set; } //very important to have ThreadStatic: // see: http://issues.umbraco.org/issue/U4-2172 @@ -52,8 +52,8 @@ namespace Umbraco.Core.Persistence { Mandate.ParameterNotNullOrEmpty(connectionString, "connectionString"); Mandate.ParameterNotNullOrEmpty(providerName, "providerName"); - _connectionString = connectionString; - _providerName = providerName; + ConnectionString = connectionString; + ProviderName = providerName; } public UmbracoDatabase CreateDatabase() @@ -68,8 +68,8 @@ namespace Umbraco.Core.Persistence //double check if (_nonHttpInstance == null) { - _nonHttpInstance = string.IsNullOrEmpty(_connectionString) == false && string.IsNullOrEmpty(_providerName) == false - ? new UmbracoDatabase(_connectionString, _providerName) + _nonHttpInstance = string.IsNullOrEmpty(ConnectionString) == false && string.IsNullOrEmpty(ProviderName) == false + ? new UmbracoDatabase(ConnectionString, ProviderName) : new UmbracoDatabase(_connectionStringName); } } @@ -81,8 +81,8 @@ namespace Umbraco.Core.Persistence if (HttpContext.Current.Items.Contains(typeof(DefaultDatabaseFactory)) == false) { HttpContext.Current.Items.Add(typeof (DefaultDatabaseFactory), - string.IsNullOrEmpty(_connectionString) == false && string.IsNullOrEmpty(_providerName) == false - ? new UmbracoDatabase(_connectionString, _providerName) + string.IsNullOrEmpty(ConnectionString) == false && string.IsNullOrEmpty(ProviderName) == false + ? new UmbracoDatabase(ConnectionString, ProviderName) : new UmbracoDatabase(_connectionStringName)); } return (UmbracoDatabase)HttpContext.Current.Items[typeof(DefaultDatabaseFactory)]; diff --git a/src/Umbraco.Core/Persistence/PetaPocoExtensions.cs b/src/Umbraco.Core/Persistence/PetaPocoExtensions.cs index 51ae2638b0..c2db49980e 100644 --- a/src/Umbraco.Core/Persistence/PetaPocoExtensions.cs +++ b/src/Umbraco.Core/Persistence/PetaPocoExtensions.cs @@ -6,6 +6,7 @@ using Umbraco.Core.Logging; using Umbraco.Core.Models.Rdbms; using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Persistence.Migrations.Initial; +using Umbraco.Core.Persistence.Querying; using Umbraco.Core.Persistence.SqlSyntax; namespace Umbraco.Core.Persistence @@ -48,11 +49,15 @@ namespace Umbraco.Core.Persistence } else { - string sql; - using (var cmd = db.GenerateBulkInsertCommand(collection, db.Connection, out sql)) + string[] sqlStatements; + var cmds = db.GenerateBulkInsertCommand(collection, db.Connection, out sqlStatements); + for (var i = 0; i < sqlStatements.Length; i++) { - cmd.CommandText = sql; - cmd.ExecuteNonQuery(); + using (var cmd = cmds[i]) + { + cmd.CommandText = sqlStatements[i]; + cmd.ExecuteNonQuery(); + } } } @@ -66,42 +71,92 @@ namespace Umbraco.Core.Persistence } } - internal static IDbCommand GenerateBulkInsertCommand(this Database db, IEnumerable collection, IDbConnection connection, out string sql) + /// + /// Creates a bulk insert command + /// + /// + /// + /// + /// + /// + /// Sql commands with populated command parameters required to execute the sql statement + /// + /// The limits for number of parameters are 2100 (in sql server, I think there's many more allowed in mysql). So + /// we need to detect that many params and split somehow. + /// For some reason the 2100 limit is not actually allowed even though the exception from sql server mentions 2100 as a max, perhaps it is 2099 + /// that is max. I've reduced it to 2000 anyways. + /// + internal static IDbCommand[] GenerateBulkInsertCommand( + this Database db, + IEnumerable collection, + IDbConnection connection, + out string[] sql) { + //A filter used below a few times to get all columns except result cols and not the primary key if it is auto-incremental + Func, bool> includeColumn = (data, column) => + { + if (column.Value.ResultColumn) return false; + if (data.TableInfo.AutoIncrement && column.Key == data.TableInfo.PrimaryKey) return false; + return true; + }; + var pd = Database.PocoData.ForType(typeof(T)); var tableName = db.EscapeTableName(pd.TableInfo.TableName); - //get all columns but not the primary key if it is auto-incremental - var cols = string.Join(", ", ( - from c in pd.Columns - where - //don't return ResultColumns - !c.Value.ResultColumn - //if the table is auto-incremental, don't return the primary key - && (pd.TableInfo.AutoIncrement && c.Key != pd.TableInfo.PrimaryKey) - select tableName + "." + db.EscapeSqlIdentifier(c.Key)) - .ToArray()); + //get all columns to include and format for sql + var cols = string.Join(", ", + pd.Columns + .Where(c => includeColumn(pd, c)) + .Select(c => tableName + "." + db.EscapeSqlIdentifier(c.Key)).ToArray()); - var cmd = db.CreateCommand(connection, ""); + var itemArray = collection.ToArray(); - var pocoValues = new List(); - var index = 0; - foreach (var poco in collection) + //calculate number of parameters per item + var paramsPerItem = pd.Columns.Count(i => includeColumn(pd, i)); + + //Example calc: + // Given: we have 4168 items in the itemArray, each item contains 8 command parameters (values to be inserterted) + // 2100 / 8 = 262.5 + // Math.Floor(2100 / 8) = 262 items per trans + // 4168 / 262 = 15.908... = there will be 16 trans in total + + //all items will be included if we have disabled db parameters + var itemsPerTrans = Math.Floor(2000.00 / paramsPerItem); + //there will only be one transaction if we have disabled db parameters + var numTrans = Math.Ceiling(itemArray.Length / itemsPerTrans); + + var sqlQueries = new List(); + var commands = new List(); + + for (var tIndex = 0; tIndex < numTrans; tIndex++) { - var values = new List(); - foreach (var i in pd.Columns) + var itemsForTrans = itemArray + .Skip(tIndex * (int)itemsPerTrans) + .Take((int)itemsPerTrans); + + var cmd = db.CreateCommand(connection, ""); + var pocoValues = new List(); + var index = 0; + foreach (var poco in itemsForTrans) { - if (pd.TableInfo.AutoIncrement && i.Key == pd.TableInfo.PrimaryKey) + var values = new List(); + //get all columns except result cols and not the primary key if it is auto-incremental + foreach (var i in pd.Columns.Where(x => includeColumn(pd, x))) { - continue; + db.AddParam(cmd, i.Value.GetValue(poco), "@"); + values.Add(string.Format("{0}{1}", "@", index++)); } - values.Add(string.Format("{0}{1}", "@", index++)); - db.AddParam(cmd, i.Value.GetValue(poco), "@"); + pocoValues.Add("(" + string.Join(",", values.ToArray()) + ")"); } - pocoValues.Add("(" + string.Join(",", values.ToArray()) + ")"); + + var sqlResult = string.Format("INSERT INTO {0} ({1}) VALUES {2}", tableName, cols, string.Join(", ", pocoValues)); + sqlQueries.Add(sqlResult); + commands.Add(cmd); } - sql = string.Format("INSERT INTO {0} ({1}) VALUES {2}", tableName, cols, string.Join(", ", pocoValues)); - return cmd; + + sql = sqlQueries.ToArray(); + + return commands.ToArray(); } public static void CreateTable(this Database db, bool overwrite, Type modelType) diff --git a/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionHelper.cs b/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionHelper.cs index 470a098fd8..559b978721 100644 --- a/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionHelper.cs +++ b/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionHelper.cs @@ -435,38 +435,7 @@ namespace Umbraco.Core.Persistence.Querying public virtual string GetQuotedValue(object value, Type fieldType) { - if (value == null) return "NULL"; - - if (!fieldType.UnderlyingSystemType.IsValueType && fieldType != typeof(string)) - { - throw new NotSupportedException( - string.Format("Property of type: {0} is not supported", fieldType.FullName)); - } - - if (fieldType == typeof(int)) - return ((int)value).ToString(CultureInfo.InvariantCulture); - - if (fieldType == typeof(float)) - return ((float)value).ToString(CultureInfo.InvariantCulture); - - if (fieldType == typeof(double)) - return ((double)value).ToString(CultureInfo.InvariantCulture); - - if (fieldType == typeof(decimal)) - return ((decimal)value).ToString(CultureInfo.InvariantCulture); - - if (fieldType == typeof (DateTime)) - { - return "'" + EscapeParam(((DateTime)value).ToIsoString()) + "'"; - } - - - if (fieldType == typeof(bool)) - return ((bool)value) ? Convert.ToString(1, CultureInfo.InvariantCulture) : Convert.ToString(0, CultureInfo.InvariantCulture); - - return ShouldQuoteValue(fieldType) - ? "'" + EscapeParam(value) + "'" - : value.ToString(); + return QueryHelper.GetQuotedValue(value, fieldType, EscapeParam, ShouldQuoteValue); } public virtual string EscapeParam(object paramValue) diff --git a/src/Umbraco.Core/Persistence/Querying/PocoToSqlExpressionHelper.cs b/src/Umbraco.Core/Persistence/Querying/PocoToSqlExpressionHelper.cs index d065849102..f84933b3e7 100644 --- a/src/Umbraco.Core/Persistence/Querying/PocoToSqlExpressionHelper.cs +++ b/src/Umbraco.Core/Persistence/Querying/PocoToSqlExpressionHelper.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Collections.ObjectModel; -using System.Globalization; using System.Linq; using System.Linq.Expressions; using System.Text; @@ -440,43 +439,7 @@ namespace Umbraco.Core.Persistence.Querying public virtual string GetQuotedValue(object value, Type fieldType) { - if (value == null) return "NULL"; - - if (!fieldType.UnderlyingSystemType.IsValueType && fieldType != typeof(string)) - { - //if (TypeSerializer.CanCreateFromString(fieldType)) - //{ - // return "'" + EscapeParam(TypeSerializer.SerializeToString(value)) + "'"; - //} - - throw new NotSupportedException( - string.Format("Property of type: {0} is not supported", fieldType.FullName)); - } - - if (fieldType == typeof(int)) - return ((int)value).ToString(CultureInfo.InvariantCulture); - - if (fieldType == typeof(float)) - return ((float)value).ToString(CultureInfo.InvariantCulture); - - if (fieldType == typeof(double)) - return ((double)value).ToString(CultureInfo.InvariantCulture); - - if (fieldType == typeof(decimal)) - return ((decimal)value).ToString(CultureInfo.InvariantCulture); - - if (fieldType == typeof (DateTime)) - { - return "'" + EscapeParam(((DateTime)value).ToIsoString()) + "'"; - } - - - if (fieldType == typeof(bool)) - return ((bool)value) ? Convert.ToString(1, CultureInfo.InvariantCulture) : Convert.ToString(0, CultureInfo.InvariantCulture); - - return ShouldQuoteValue(fieldType) - ? "'" + EscapeParam(value) + "'" - : value.ToString(); + return QueryHelper.GetQuotedValue(value, fieldType, EscapeParam, ShouldQuoteValue); } public virtual string EscapeParam(object paramValue) diff --git a/src/Umbraco.Core/Persistence/Querying/QueryHelper.cs b/src/Umbraco.Core/Persistence/Querying/QueryHelper.cs new file mode 100644 index 0000000000..f69e106f57 --- /dev/null +++ b/src/Umbraco.Core/Persistence/Querying/QueryHelper.cs @@ -0,0 +1,70 @@ +using System; +using System.Globalization; + +namespace Umbraco.Core.Persistence.Querying +{ + /// + /// Logic that is shared with the expression helpers + /// + internal class QueryHelper + { + public static string GetQuotedValue(object value, Type fieldType, Func escapeCallback = null, Func shouldQuoteCallback = null) + { + if (value == null) return "NULL"; + + if (escapeCallback == null) + { + escapeCallback = EscapeParam; + } + if (shouldQuoteCallback == null) + { + shouldQuoteCallback = ShouldQuoteValue; + } + + if (!fieldType.UnderlyingSystemType.IsValueType && fieldType != typeof(string)) + { + //if (TypeSerializer.CanCreateFromString(fieldType)) + //{ + // return "'" + EscapeParam(TypeSerializer.SerializeToString(value)) + "'"; + //} + + throw new NotSupportedException( + string.Format("Property of type: {0} is not supported", fieldType.FullName)); + } + + if (fieldType == typeof(int)) + return ((int)value).ToString(CultureInfo.InvariantCulture); + + if (fieldType == typeof(float)) + return ((float)value).ToString(CultureInfo.InvariantCulture); + + if (fieldType == typeof(double)) + return ((double)value).ToString(CultureInfo.InvariantCulture); + + if (fieldType == typeof(decimal)) + return ((decimal)value).ToString(CultureInfo.InvariantCulture); + + if (fieldType == typeof(DateTime)) + { + return "'" + EscapeParam(((DateTime)value).ToIsoString()) + "'"; + } + + if (fieldType == typeof(bool)) + return ((bool)value) ? Convert.ToString(1, CultureInfo.InvariantCulture) : Convert.ToString(0, CultureInfo.InvariantCulture); + + return ShouldQuoteValue(fieldType) + ? "'" + EscapeParam(value) + "'" + : value.ToString(); + } + + public static string EscapeParam(object paramValue) + { + return paramValue.ToString().Replace("'", "''"); + } + + public static bool ShouldQuoteValue(Type fieldType) + { + return true; + } + } +} \ No newline at end of file diff --git a/src/Umbraco.Core/Services/ContentService.cs b/src/Umbraco.Core/Services/ContentService.cs index 94aa0f9140..3c4eedcc0d 100644 --- a/src/Umbraco.Core/Services/ContentService.cs +++ b/src/Umbraco.Core/Services/ContentService.cs @@ -1374,6 +1374,8 @@ namespace Umbraco.Core.Services { foreach (var id in contentTypeIds) { + + //first we'll clear out the data from the cmsContentXml table for this type uow.Database.Execute(@"delete from cmsContentXml where nodeId in (select cmsDocument.nodeId from cmsDocument diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index d2a5e33a28..b1b83ace68 100644 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -462,6 +462,7 @@ + diff --git a/src/Umbraco.Tests/Persistence/PetaPocoExtensionsTest.cs b/src/Umbraco.Tests/Persistence/PetaPocoExtensionsTest.cs index 1cf6aff7b0..3a76e29139 100644 --- a/src/Umbraco.Tests/Persistence/PetaPocoExtensionsTest.cs +++ b/src/Umbraco.Tests/Persistence/PetaPocoExtensionsTest.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Text.RegularExpressions; using NUnit.Framework; using Umbraco.Core; using Umbraco.Core.Models.Rdbms; @@ -73,13 +74,47 @@ namespace Umbraco.Tests.Persistence db.OpenSharedConnection(); // Act - string sql; + string[] sql; db.GenerateBulkInsertCommand(servers, db.Connection, out sql); db.CloseSharedConnection(); // Assert - Assert.That(sql, + Assert.That(sql[0], Is.EqualTo("INSERT INTO [umbracoServer] ([umbracoServer].[address], [umbracoServer].[computerName], [umbracoServer].[registeredDate], [umbracoServer].[lastNotifiedDate], [umbracoServer].[isActive]) VALUES (@0,@1,@2,@3,@4), (@5,@6,@7,@8,@9)")); } + + + [Test] + public void Generate_Bulk_Import_Sql_Exceeding_Max_Params() + { + // Arrange + var db = DatabaseContext.Database; + + var servers = new List(); + for (var i = 0; i < 1500; i++) + { + servers.Add(new ServerRegistrationDto + { + Address = "address" + i, + ComputerName = "computer" + i, + DateRegistered = DateTime.Now, + IsActive = true, + LastNotified = DateTime.Now + }); + } + db.OpenSharedConnection(); + + // Act + string[] sql; + db.GenerateBulkInsertCommand(servers, db.Connection, out sql); + db.CloseSharedConnection(); + + // Assert + Assert.That(sql.Length, Is.EqualTo(4)); + foreach (var s in sql) + { + Assert.LessOrEqual(Regex.Matches(s, "@\\d+").Count, 2000); + } + } } } \ No newline at end of file diff --git a/src/Umbraco.Tests/Services/PerformanceTests.cs b/src/Umbraco.Tests/Services/PerformanceTests.cs new file mode 100644 index 0000000000..f764a4cd04 --- /dev/null +++ b/src/Umbraco.Tests/Services/PerformanceTests.cs @@ -0,0 +1,211 @@ +using System; +using System.Collections.Generic; +using System.Configuration; +using System.Data.SqlServerCe; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Xml.Linq; +using NUnit.Framework; +using Umbraco.Core; +using Umbraco.Core.Models; +using Umbraco.Core.Models.Rdbms; +using Umbraco.Core.Persistence; +using Umbraco.Core.Persistence.Repositories; +using Umbraco.Core.Persistence.SqlSyntax; +using Umbraco.Core.Persistence.UnitOfWork; +using Umbraco.Core.Services; +using Umbraco.Tests.TestHelpers; +using Umbraco.Tests.TestHelpers.Entities; + +namespace Umbraco.Tests.Services +{ + /// + /// Tests covering all methods in the ContentService class. + /// This is more of an integration test as it involves multiple layers + /// as well as configuration. + /// + [TestFixture, RequiresSTA] + [NUnit.Framework.Ignore("These should not be run by the server, only directly as they are only benchmark tests")] + public class PerformanceTests : BaseDatabaseFactoryTest + { + [SetUp] + public override void Initialize() + { + base.Initialize(); + } + + protected override string GetDbConnectionString() + { + return @"server=.\SQLEXPRESS;database=UmbTest;user id=sa;password=test"; + } + + protected override string GetDbProviderName() + { + return "System.Data.SqlClient"; + } + + /// + /// new schema per test + /// + protected override DatabaseBehavior DatabaseTestBehavior + { + get { return DatabaseBehavior.NewSchemaPerTest; } + } + + /// + /// don't create anything, we're testing against our own server + /// + protected override void CreateSqlCeDatabase() + { + } + + [TearDown] + public override void TearDown() + { + base.TearDown(); + } + + [Test] + public void Truncate_Insert_Vs_Update_Insert() + { + var customObjectType = Guid.NewGuid(); + //chuck lots of data in the db + var nodes = PrimeDbWithLotsOfContentXmlRecords(customObjectType); + + //now we need to test the difference between truncating all records and re-inserting them as we do now, + //vs updating them (which might result in checking if they exist for or listening on an exception). + using (DisposableTimer.DebugDuration("Starting truncate + normal insert test")) + { + //do this 10x! + for (var i = 0; i < 10; i++) + { + //clear all the xml entries + DatabaseContext.Database.Execute(@"DELETE FROM cmsContentXml WHERE nodeId IN + (SELECT DISTINCT cmsContentXml.nodeId FROM cmsContentXml + INNER JOIN cmsContent ON cmsContentXml.nodeId = cmsContent.nodeId)"); + + //now we insert each record for the ones we've deleted like we do in the content service. + var xmlItems = nodes.Select(node => new ContentXmlDto { NodeId = node.NodeId, Xml = UpdatedXmlStructure }).ToList(); + foreach (var xml in xmlItems) + { + var result = DatabaseContext.Database.Insert(xml); + } + } + } + + //now, isntead of truncating, we'll attempt to update and if it doesn't work then we insert + using (DisposableTimer.DebugDuration("Starting update test")) + { + //do this 10x! + for (var i = 0; i < 10; i++) + { + //now we insert each record for the ones we've deleted like we do in the content service. + var xmlItems = nodes.Select(node => new ContentXmlDto { NodeId = node.NodeId, Xml = UpdatedXmlStructure }).ToList(); + foreach (var xml in xmlItems) + { + var result = DatabaseContext.Database.Update(xml); + } + } + } + + //now, isntead of truncating, we'll attempt to update and if it doesn't work then we insert + using (DisposableTimer.DebugDuration("Starting truncate + bulk insert test")) + { + //do this 10x! + for (var i = 0; i < 10; i++) + { + //clear all the xml entries + DatabaseContext.Database.Execute(@"DELETE FROM cmsContentXml WHERE nodeId IN + (SELECT DISTINCT cmsContentXml.nodeId FROM cmsContentXml + INNER JOIN cmsContent ON cmsContentXml.nodeId = cmsContent.nodeId)"); + + //now we insert each record for the ones we've deleted like we do in the content service. + var xmlItems = nodes.Select(node => new ContentXmlDto { NodeId = node.NodeId, Xml = UpdatedXmlStructure }).ToList(); + DatabaseContext.Database.BulkInsertRecords(xmlItems); + } + } + + + + } + + private IEnumerable PrimeDbWithLotsOfContentXmlRecords(Guid customObjectType) + { + var nodes = new List(); + for (var i = 1; i < 10000; i++) + { + nodes.Add(new NodeDto + { + Level = 1, + ParentId = -1, + NodeObjectType = customObjectType, + Text = i.ToString(CultureInfo.InvariantCulture), + UserId = 0, + CreateDate = DateTime.Now, + Trashed = false, + SortOrder = 0, + Path = "" + }); + } + DatabaseContext.Database.BulkInsertRecords(nodes); + + //re-get the nodes with ids + var sql = new Sql(); + sql.Select("*").From().Where(x => x.NodeObjectType == customObjectType); + nodes = DatabaseContext.Database.Fetch(sql); + + //create the cmsContent data, each with a new content type id (so we can query on it later if needed) + var contentTypeId = 0; + var cmsContentItems = nodes.Select(node => new ContentDto { NodeId = node.NodeId, ContentTypeId = contentTypeId++ }).ToList(); + DatabaseContext.Database.BulkInsertRecords(cmsContentItems); + + //create the xml data + var xmlItems = nodes.Select(node => new ContentXmlDto { NodeId = node.NodeId, Xml = TestXmlStructure }).ToList(); + DatabaseContext.Database.BulkInsertRecords(xmlItems); + + return nodes; + } + + private const string TestXmlStructure = @" + 0 + Standard Site for Umbraco by Koiak + + + + Built by Creative Founds +

Web ApplicationsCreative Founds design and build first class software solutions that deliver big results. We provide ASP.NET web and mobile applications, Umbraco development service & technical consultancy.

+

www.creativefounds.co.uk

]]> + Umbraco Development +

UmbracoUmbraco the the leading ASP.NET open source CMS, under pinning over 150,000 websites. Our Certified Developers are experts in developing high performance and feature rich websites.

]]> + Contact Us +

Contact Us on TwitterWe'd love to hear how this package has helped you and how it can be improved. Get in touch on the project website or via twitter

]]> + +
Standard Website MVC, Company Address, Glasgow, Postcode
+ Copyright &copy; 2012 Your Company + http://www.umbraco.org + /media/1477/umbraco_logo.png + + + + + + + 2013-07-01T00:00:00 +"; + + private const string UpdatedXmlStructure = @" + 0 + + + + Clients +

This is a standard content page.

+

Vestibulum malesuada aliquet ante, vitae ullamcorper felis faucibus vel. Vestibulum condimentum faucibus tellus porta ultrices. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas.

+

Cras at auctor orci. Praesent facilisis erat nec odio consequat at posuere ligula pretium. Nulla eget felis id nisl volutpat pellentesque. Ut id augue id ligula placerat rutrum a nec purus. Maecenas sed lectus ac mi pellentesque luctus quis sit amet turpis. Vestibulum adipiscing convallis vestibulum.

+

Duis condimentum lectus at orci placerat vitae imperdiet lorem cursus. Duis hendrerit porta lorem, non suscipit quam consectetur vitae. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean elit augue, tincidunt nec tincidunt id, elementum vel est.

]]> + +"; + + } +} \ No newline at end of file diff --git a/src/Umbraco.Tests/TestHelpers/BaseDatabaseFactoryTest.cs b/src/Umbraco.Tests/TestHelpers/BaseDatabaseFactoryTest.cs index d6ef094169..5a98e303a4 100644 --- a/src/Umbraco.Tests/TestHelpers/BaseDatabaseFactoryTest.cs +++ b/src/Umbraco.Tests/TestHelpers/BaseDatabaseFactoryTest.cs @@ -56,20 +56,23 @@ namespace Umbraco.Tests.TestHelpers var path = TestHelper.CurrentAssemblyDirectory; AppDomain.CurrentDomain.SetData("DataDirectory", path); + var dbFactory = new DefaultDatabaseFactory( + GetDbConnectionString(), + GetDbProviderName()); ApplicationContext.Current = new ApplicationContext( //assign the db context - new DatabaseContext(new DefaultDatabaseFactory()), + new DatabaseContext(dbFactory), //assign the service context - new ServiceContext(new PetaPocoUnitOfWorkProvider(), new FileUnitOfWorkProvider(), new PublishingStrategy()), + new ServiceContext(new PetaPocoUnitOfWorkProvider(dbFactory), new FileUnitOfWorkProvider(), new PublishingStrategy()), //disable cache false) { IsReady = true }; - DatabaseContext.Initialize(); + DatabaseContext.Initialize(dbFactory.ProviderName, dbFactory.ConnectionString); - CreateDatabase(); + CreateSqlCeDatabase(); InitializeDatabase(); @@ -85,10 +88,23 @@ namespace Umbraco.Tests.TestHelpers get { return DatabaseBehavior.NewSchemaPerTest; } } + protected virtual string GetDbProviderName() + { + return "System.Data.SqlServerCe.4.0"; + } + + /// + /// Get the db conn string + /// + protected virtual string GetDbConnectionString() + { + return @"Datasource=|DataDirectory|UmbracoPetaPocoTests.sdf"; + } + /// /// Creates the SqlCe database if required /// - protected virtual void CreateDatabase() + protected virtual void CreateSqlCeDatabase() { if (DatabaseTestBehavior == DatabaseBehavior.NoDatabasePerFixture) return; @@ -97,7 +113,9 @@ namespace Umbraco.Tests.TestHelpers //Get the connectionstring settings from config var settings = ConfigurationManager.ConnectionStrings[Core.Configuration.GlobalSettings.UmbracoConnectionName]; - ConfigurationManager.AppSettings.Set(Core.Configuration.GlobalSettings.UmbracoConnectionName, @"datalayer=SQLCE4Umbraco.SqlCEHelper,SQLCE4Umbraco;data source=|DataDirectory|\UmbracoPetaPocoTests.sdf"); + ConfigurationManager.AppSettings.Set( + Core.Configuration.GlobalSettings.UmbracoConnectionName, + GetDbConnectionString()); string dbFilePath = string.Concat(path, "\\UmbracoPetaPocoTests.sdf"); diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj index d9b9b857b1..66764feae5 100644 --- a/src/Umbraco.Tests/Umbraco.Tests.csproj +++ b/src/Umbraco.Tests/Umbraco.Tests.csproj @@ -188,6 +188,7 @@ + From c49becea53079e696df9a0e16a3c953142d84659 Mon Sep 17 00:00:00 2001 From: Morten Christensen Date: Fri, 26 Jul 2013 12:35:40 +0200 Subject: [PATCH 06/20] Cleanup of unused using statements in a few files --- .../umbraco/LiveEditing/Modules/SkinModule/SkinModule.cs | 6 ------ .../RazorDynamicNode/RazorLibraryCore.cs | 9 +-------- .../skinning/tasks/AddStyleSheetToTemplate.cs | 3 --- .../businesslogic/skinning/tasks/ModifyTemplate.cs | 8 -------- 4 files changed, 1 insertion(+), 25 deletions(-) diff --git a/src/Umbraco.Web/umbraco.presentation/umbraco/LiveEditing/Modules/SkinModule/SkinModule.cs b/src/Umbraco.Web/umbraco.presentation/umbraco/LiveEditing/Modules/SkinModule/SkinModule.cs index ed20f5796b..2405b37695 100644 --- a/src/Umbraco.Web/umbraco.presentation/umbraco/LiveEditing/Modules/SkinModule/SkinModule.cs +++ b/src/Umbraco.Web/umbraco.presentation/umbraco/LiveEditing/Modules/SkinModule/SkinModule.cs @@ -1,7 +1,4 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Web; using umbraco.presentation.LiveEditing.Modules; using ClientDependency.Core; using System.Web.UI.WebControls; @@ -13,9 +10,6 @@ using ClientDependency.Core.Controls; using umbraco.presentation.umbraco.controls; using HtmlAgilityPack; using umbraco.cms.businesslogic.template; -using System.Text; -using System.IO; -using System.Collections; namespace umbraco.presentation.umbraco.LiveEditing.Modules.SkinModule { diff --git a/src/umbraco.MacroEngines/RazorDynamicNode/RazorLibraryCore.cs b/src/umbraco.MacroEngines/RazorDynamicNode/RazorLibraryCore.cs index 5921ff220a..6ab1bad3d8 100644 --- a/src/umbraco.MacroEngines/RazorDynamicNode/RazorLibraryCore.cs +++ b/src/umbraco.MacroEngines/RazorDynamicNode/RazorLibraryCore.cs @@ -1,18 +1,11 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; -using System.Text; using System.Web.Mvc; -using Umbraco.Core.Dynamics; -using Umbraco.Core.Models; using Umbraco.Web; -using Umbraco.Web.Models; using umbraco.interfaces; using System.Xml.Linq; using System.Xml.XPath; using System.Web; -using System.IO; -using HtmlAgilityPack; namespace umbraco.MacroEngines.Library { diff --git a/src/umbraco.cms/businesslogic/skinning/tasks/AddStyleSheetToTemplate.cs b/src/umbraco.cms/businesslogic/skinning/tasks/AddStyleSheetToTemplate.cs index 7e2fce58ff..44f67d4506 100644 --- a/src/umbraco.cms/businesslogic/skinning/tasks/AddStyleSheetToTemplate.cs +++ b/src/umbraco.cms/businesslogic/skinning/tasks/AddStyleSheetToTemplate.cs @@ -1,7 +1,4 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; using umbraco.interfaces.skinning; using HtmlAgilityPack; using umbraco.IO; diff --git a/src/umbraco.cms/businesslogic/skinning/tasks/ModifyTemplate.cs b/src/umbraco.cms/businesslogic/skinning/tasks/ModifyTemplate.cs index e098f00af2..7d3d7a7c63 100644 --- a/src/umbraco.cms/businesslogic/skinning/tasks/ModifyTemplate.cs +++ b/src/umbraco.cms/businesslogic/skinning/tasks/ModifyTemplate.cs @@ -1,14 +1,6 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; using umbraco.interfaces.skinning; -using System.Text.RegularExpressions; -using System.IO; -using System.Web; using umbraco.IO; -using umbraco.cms.businesslogic.packager.standardPackageActions; -using System.Xml; using HtmlAgilityPack; namespace umbraco.cms.businesslogic.skinning.tasks From d734ba6136491761f2de0d4c0320d64fd6779df0 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Fri, 26 Jul 2013 12:43:19 +0200 Subject: [PATCH 07/20] U4-2543 In the back office its possible to get Response Validation errors --- src/Umbraco.Web.UI/umbraco/settings/EditNodeTypeNew.aspx | 2 +- src/Umbraco.Web/Umbraco.Web.csproj | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Web.UI/umbraco/settings/EditNodeTypeNew.aspx b/src/Umbraco.Web.UI/umbraco/settings/EditNodeTypeNew.aspx index cdda810a57..4ae6523dc6 100644 --- a/src/Umbraco.Web.UI/umbraco/settings/EditNodeTypeNew.aspx +++ b/src/Umbraco.Web.UI/umbraco/settings/EditNodeTypeNew.aspx @@ -1,4 +1,4 @@ -<%@ Page Language="c#" CodeBehind="EditNodeTypeNew.aspx.cs" AutoEventWireup="True" +<%@ Page Language="c#" CodeBehind="EditNodeTypeNew.aspx.cs" AutoEventWireup="True" ValidateRequest="false" Async="true" AsyncTimeOut="300" Trace="false" Inherits="Umbraco.Web.UI.Umbraco.Settings.EditNodeTypeNew" MasterPageFile="../masterpages/umbracoPage.Master" %> <%@ Register TagPrefix="cc1" Namespace="umbraco.uicontrols" Assembly="controls" %> diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 67d9b6335d..4fc9b92b8d 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -1935,7 +1935,9 @@ - + + ASPXCodeBehind + From f50de5b04914074c0d54cae8e54ae2a609826def Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Fri, 26 Jul 2013 15:16:55 +0200 Subject: [PATCH 08/20] U4-2550 Could not load file or assembly 'HtmlAgilityPack, Version=1.4.5.0 --- src/Umbraco.Web.UI/web.Template.Debug.config | 9 +++++++++ src/Umbraco.Web.UI/web.Template.config | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/src/Umbraco.Web.UI/web.Template.Debug.config b/src/Umbraco.Web.UI/web.Template.Debug.config index 937cee3f3a..195be82cf1 100644 --- a/src/Umbraco.Web.UI/web.Template.Debug.config +++ b/src/Umbraco.Web.UI/web.Template.Debug.config @@ -64,7 +64,16 @@ + + + + + + + + \ No newline at end of file diff --git a/src/Umbraco.Web.UI/web.Template.config b/src/Umbraco.Web.UI/web.Template.config index f439b4c0a7..fae6714db7 100644 --- a/src/Umbraco.Web.UI/web.Template.config +++ b/src/Umbraco.Web.UI/web.Template.config @@ -269,6 +269,12 @@ + + + + + + From 696306e7c9444db9c99f38eff21a9d21700fbc11 Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 29 Jul 2013 13:25:02 +1000 Subject: [PATCH 09/20] Implements the bulk insert procedure for re-populating the cmsContentXml tables, removes the need to rebuild all xml structures when sorting if in LB environments --- src/Umbraco.Core/Persistence/PetaPocoExtensions.cs | 4 ++++ src/Umbraco.Core/Services/ContentService.cs | 8 +++++--- src/Umbraco.Core/Services/MediaService.cs | 10 ++++++---- src/Umbraco.Tests/Services/PerformanceTests.cs | 2 +- .../umbraco/webservices/nodeSorter.asmx.cs | 7 ------- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/Umbraco.Core/Persistence/PetaPocoExtensions.cs b/src/Umbraco.Core/Persistence/PetaPocoExtensions.cs index c2db49980e..8917368e21 100644 --- a/src/Umbraco.Core/Persistence/PetaPocoExtensions.cs +++ b/src/Umbraco.Core/Persistence/PetaPocoExtensions.cs @@ -33,6 +33,10 @@ namespace Umbraco.Core.Persistence public static void BulkInsertRecords(this Database db, IEnumerable collection) { + //don't do anything if there are no records. + if (collection.Any() == false) + return; + using (var tr = db.GetTransaction()) { try diff --git a/src/Umbraco.Core/Services/ContentService.cs b/src/Umbraco.Core/Services/ContentService.cs index 3c4eedcc0d..83ff46d958 100644 --- a/src/Umbraco.Core/Services/ContentService.cs +++ b/src/Umbraco.Core/Services/ContentService.cs @@ -1387,15 +1387,17 @@ namespace Umbraco.Core.Services } } + var xmlItems = new List(); foreach (var c in list) { //generate the xml var xml = c.ToXml(); //create the dto to insert - var poco = new ContentXmlDto { NodeId = c.Id, Xml = xml.ToString(SaveOptions.None) }; - //insert it into the database - uow.Database.Insert(poco); + xmlItems.Add(new ContentXmlDto { NodeId = c.Id, Xml = xml.ToString(SaveOptions.None) }); } + //bulk insert it into the database + uow.Database.BulkInsertRecords(xmlItems); + } Audit.Add(AuditTypes.Publish, "RebuildXmlStructures completed, the xml has been regenerated in the database", 0, -1); } diff --git a/src/Umbraco.Core/Services/MediaService.cs b/src/Umbraco.Core/Services/MediaService.cs index 0e35802961..34f4000d4b 100644 --- a/src/Umbraco.Core/Services/MediaService.cs +++ b/src/Umbraco.Core/Services/MediaService.cs @@ -11,10 +11,11 @@ using Umbraco.Core.Models.Rdbms; using Umbraco.Core.Persistence; using Umbraco.Core.Persistence.Querying; using Umbraco.Core.Persistence.UnitOfWork; +using Umbraco.Core.Publishing; namespace Umbraco.Core.Services { - /// + /// /// Represents the Media Service, which is an easy access to operations involving /// public class MediaService : IMediaService @@ -875,15 +876,16 @@ namespace Umbraco.Core.Services } } + var xmlItems = new List(); foreach (var c in list) { //generate the xml var xml = c.ToXml(); //create the dto to insert - var poco = new ContentXmlDto { NodeId = c.Id, Xml = xml.ToString(SaveOptions.None) }; - //insert it into the database - uow.Database.Insert(poco); + xmlItems.Add(new ContentXmlDto { NodeId = c.Id, Xml = xml.ToString(SaveOptions.None) }); } + //bulk insert it into the database + uow.Database.BulkInsertRecords(xmlItems); } Audit.Add(AuditTypes.Publish, "RebuildXmlStructures completed, the xml has been regenerated in the database", 0, -1); } diff --git a/src/Umbraco.Tests/Services/PerformanceTests.cs b/src/Umbraco.Tests/Services/PerformanceTests.cs index f764a4cd04..892cc6f84d 100644 --- a/src/Umbraco.Tests/Services/PerformanceTests.cs +++ b/src/Umbraco.Tests/Services/PerformanceTests.cs @@ -109,7 +109,7 @@ namespace Umbraco.Tests.Services } } - //now, isntead of truncating, we'll attempt to update and if it doesn't work then we insert + //now, test truncating but then do bulk insertion of records using (DisposableTimer.DebugDuration("Starting truncate + bulk insert test")) { //do this 10x! diff --git a/src/Umbraco.Web/umbraco.presentation/umbraco/webservices/nodeSorter.asmx.cs b/src/Umbraco.Web/umbraco.presentation/umbraco/webservices/nodeSorter.asmx.cs index a5b2102640..b18e34292f 100644 --- a/src/Umbraco.Web/umbraco.presentation/umbraco/webservices/nodeSorter.asmx.cs +++ b/src/Umbraco.Web/umbraco.presentation/umbraco/webservices/nodeSorter.asmx.cs @@ -160,13 +160,6 @@ namespace umbraco.presentation.webservices if (parentNode != null) content.SortNodes(ref parentNode); - // Load balancing - then refresh entire cache - // NOTE: SD: This seems a bit excessive to do simply for sorting! I'm going to leave this here for now but - // the sort order should be updated in distributed calls when an item is Published (and it most likely is) - // but I guess this was put here for a reason at some point. - if (UmbracoSettings.UseDistributedCalls) - library.RefreshContent(); - // fire actionhandler, check for content BusinessLogic.Actions.Action.RunActionHandlers(new Document(parentId), ActionSort.Instance); } From b9ba350a2f209e9b2041ae87a29b66a1fa79cefe Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 29 Jul 2013 15:49:56 +1000 Subject: [PATCH 10/20] Updated the RebuildXmlStructures method with performance improvements. Added cache checking to the GetByPublishedVersion method since published content should always be 'latest' this will speed things up tremendously if items are found there. Added 2 more performance tests which show very large perf improvements, namely the Get_All_Published_Content_Of_Type shows a 77% improvement. --- .../Repositories/ContentRepository.cs | 10 +- .../Repositories/RepositoryBase.cs | 26 ++-- src/Umbraco.Core/Services/ContentService.cs | 41 ++++-- .../Services/PerformanceTests.cs | 130 ++++++++++++++++++ 4 files changed, 184 insertions(+), 23 deletions(-) diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs index 755fc5f04a..001705ebb9 100644 --- a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs @@ -449,7 +449,15 @@ namespace Umbraco.Core.Persistence.Repositories foreach (var dto in dtos) { - yield return CreateContentFromDto(dto, dto.VersionId); + var fromCache = TryGetFromCache(dto.NodeId); + if (fromCache.Success) + { + yield return fromCache.Result; + } + else + { + yield return CreateContentFromDto(dto, dto.VersionId); + } } } diff --git a/src/Umbraco.Core/Persistence/Repositories/RepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/RepositoryBase.cs index cbd8b9d252..3967aef5e6 100644 --- a/src/Umbraco.Core/Persistence/Repositories/RepositoryBase.cs +++ b/src/Umbraco.Core/Persistence/Repositories/RepositoryBase.cs @@ -85,11 +85,10 @@ namespace Umbraco.Core.Persistence.Repositories /// public TEntity Get(TId id) { - Guid key = id is int ? ConvertIdToGuid(id) : ConvertStringIdToGuid(id.ToString()); - var rEntity = _cache.GetById(typeof(TEntity), key); - if (rEntity != null) + var fromCache = TryGetFromCache(id); + if (fromCache.Success) { - return (TEntity)rEntity; + return fromCache.Result; } var entity = PerformGet(id); @@ -112,6 +111,17 @@ namespace Umbraco.Core.Persistence.Repositories return entity; } + protected Attempt TryGetFromCache(TId id) + { + Guid key = id is int ? ConvertIdToGuid(id) : ConvertStringIdToGuid(id.ToString()); + var rEntity = _cache.GetById(typeof(TEntity), key); + if (rEntity != null) + { + return new Attempt(true, (TEntity) rEntity); + } + return Attempt.False; + } + protected abstract IEnumerable PerformGetAll(params TId[] ids); /// /// Gets all entities of type TEntity or a list according to the passed in Ids @@ -173,14 +183,12 @@ namespace Umbraco.Core.Persistence.Repositories /// public bool Exists(TId id) { - Guid key = id is int ? ConvertIdToGuid(id) : ConvertStringIdToGuid(id.ToString()); - var rEntity = _cache.GetById(typeof(TEntity), key); - if (rEntity != null) + var fromCache = TryGetFromCache(id); + if (fromCache.Success) { return true; } - - return PerformExists(id); + return PerformExists(id); } protected abstract int PerformCount(IQuery query); diff --git a/src/Umbraco.Core/Services/ContentService.cs b/src/Umbraco.Core/Services/ContentService.cs index 83ff46d958..b47879bda1 100644 --- a/src/Umbraco.Core/Services/ContentService.cs +++ b/src/Umbraco.Core/Services/ContentService.cs @@ -13,6 +13,7 @@ using Umbraco.Core.Models.Rdbms; using Umbraco.Core.Persistence; using Umbraco.Core.Persistence.Caching; using Umbraco.Core.Persistence.Querying; +using Umbraco.Core.Persistence.Repositories; using Umbraco.Core.Persistence.UnitOfWork; using Umbraco.Core.Publishing; @@ -250,6 +251,17 @@ namespace Umbraco.Core.Services } } + internal IEnumerable GetPublishedContentOfContentType(int id) + { + using (var repository = _repositoryFactory.CreateContentRepository(_uowProvider.GetUnitOfWork())) + { + var query = Query.Builder.Where(x => x.ContentTypeId == id); + var contents = repository.GetByPublishedVersion(query); + + return contents; + } + } + /// /// Gets a collection of objects by Level /// @@ -431,6 +443,19 @@ namespace Umbraco.Core.Services } } + /// + /// Gets all published content items + /// + /// + internal IEnumerable GetAllPublished() + { + using (var repository = _repositoryFactory.CreateContentRepository(_uowProvider.GetUnitOfWork())) + { + var query = Query.Builder.Where(x => x.Trashed == false); + return repository.GetByPublishedVersion(query); + } + } + /// /// Gets a collection of objects, which has an expiration date less than or equal to today. /// @@ -1359,23 +1384,13 @@ namespace Umbraco.Core.Services uow.Database.Execute(@"DELETE FROM cmsContentXml WHERE nodeId IN (SELECT DISTINCT cmsContentXml.nodeId FROM cmsContentXml INNER JOIN cmsDocument ON cmsContentXml.nodeId = cmsDocument.nodeId)"); - - //get all content items that are published - // Consider creating a Path query instead of recursive method: - // var query = Query.Builder.Where(x => x.Path.StartsWith("-1")); - var rootContent = GetRootContent(); - foreach (var content in rootContent.Where(content => content.Published)) - { - list.Add(content); - list.AddRange(GetPublishedDescendants(content)); - } + + list.AddRange(GetAllPublished()); } else { foreach (var id in contentTypeIds) { - - //first we'll clear out the data from the cmsContentXml table for this type uow.Database.Execute(@"delete from cmsContentXml where nodeId in (select cmsDocument.nodeId from cmsDocument @@ -1383,7 +1398,7 @@ namespace Umbraco.Core.Services where published = 1 and contentType = @contentTypeId)", new {contentTypeId = id}); //now get all published content objects of this type and add to the list - list.AddRange(GetContentOfContentType(id).Where(content => content.Published)); + list.AddRange(GetPublishedContentOfContentType(id)); } } diff --git a/src/Umbraco.Tests/Services/PerformanceTests.cs b/src/Umbraco.Tests/Services/PerformanceTests.cs index 892cc6f84d..e49d2a1da3 100644 --- a/src/Umbraco.Tests/Services/PerformanceTests.cs +++ b/src/Umbraco.Tests/Services/PerformanceTests.cs @@ -11,6 +11,7 @@ using Umbraco.Core; using Umbraco.Core.Models; using Umbraco.Core.Models.Rdbms; using Umbraco.Core.Persistence; +using Umbraco.Core.Persistence.Caching; using Umbraco.Core.Persistence.Repositories; using Umbraco.Core.Persistence.SqlSyntax; using Umbraco.Core.Persistence.UnitOfWork; @@ -66,6 +67,90 @@ namespace Umbraco.Tests.Services base.TearDown(); } + [Test] + public void Get_All_Published_Content() + { + var result = PrimeDbWithLotsOfContent(); + var contentSvc = (ContentService) ServiceContext.ContentService; + + var countOfPublished = result.Count(x => x.Published); + var contentTypeId = result.First().ContentTypeId; + + using (DisposableTimer.DebugDuration("Getting published content normally")) + { + //do this 10x! + for (var i = 0; i < 10; i++) + { + //clear the cache to make this test valid + RuntimeCacheProvider.Current.Clear(); + + var published = new List(); + //get all content items that are published + var rootContent = contentSvc.GetRootContent(); + foreach (var content in rootContent.Where(content => content.Published)) + { + published.Add(content); + published.AddRange(contentSvc.GetPublishedDescendants(content)); + } + Assert.AreEqual(countOfPublished, published.Count(x => x.ContentTypeId == contentTypeId)); + } + + } + + using (DisposableTimer.DebugDuration("Getting published content optimized")) + { + + //do this 10x! + for (var i = 0; i < 10; i++) + { + //clear the cache to make this test valid + RuntimeCacheProvider.Current.Clear(); + + //get all content items that are published + var published = contentSvc.GetAllPublished(); + + Assert.AreEqual(countOfPublished, published.Count(x => x.ContentTypeId == contentTypeId)); + } + } + } + + [Test] + public void Get_All_Published_Content_Of_Type() + { + var result = PrimeDbWithLotsOfContent(); + var contentSvc = (ContentService)ServiceContext.ContentService; + + var countOfPublished = result.Count(x => x.Published); + var contentTypeId = result.First().ContentTypeId; + + using (DisposableTimer.DebugDuration("Getting published content of type normally")) + { + //do this 10x! + for (var i = 0; i < 10; i++) + { + //clear the cache to make this test valid + RuntimeCacheProvider.Current.Clear(); + //get all content items that are published of this type + var published = contentSvc.GetContentOfContentType(contentTypeId).Where(content => content.Published); + Assert.AreEqual(countOfPublished, published.Count(x => x.ContentTypeId == contentTypeId)); + } + } + + using (DisposableTimer.DebugDuration("Getting published content of type optimized")) + { + + //do this 10x! + for (var i = 0; i < 10; i++) + { + //clear the cache to make this test valid + RuntimeCacheProvider.Current.Clear(); + //get all content items that are published of this type + var published = contentSvc.GetPublishedContentOfContentType(contentTypeId); + Assert.AreEqual(countOfPublished, published.Count(x => x.ContentTypeId == contentTypeId)); + } + } + } + [Test] public void Truncate_Insert_Vs_Update_Insert() { @@ -130,6 +215,51 @@ namespace Umbraco.Tests.Services } + private IEnumerable PrimeDbWithLotsOfContent() + { + var contentType1 = MockedContentTypes.CreateSimpleContentType(); + contentType1.AllowedAsRoot = true; + ServiceContext.ContentTypeService.Save(contentType1); + contentType1.AllowedContentTypes = new List + { + new ContentTypeSort + { + Alias = contentType1.Alias, + Id = new Lazy(() => contentType1.Id), + SortOrder = 0 + } + }; + var result = new List(); + ServiceContext.ContentTypeService.Save(contentType1); + IContent lastParent = MockedContent.CreateSimpleContent(contentType1); + ServiceContext.ContentService.SaveAndPublish(lastParent); + result.Add(lastParent); + //create 20 deep + for (var i = 0; i < 20; i++) + { + //for each level, create 20 + IContent content = null; + for (var j = 1; j <= 10; j++) + { + content = MockedContent.CreateSimpleContent(contentType1, "Name" + j, lastParent); + //only publish evens + if (j % 2 == 0) + { + ServiceContext.ContentService.SaveAndPublish(content); + } + else + { + ServiceContext.ContentService.Save(content); + } + result.Add(content); + } + + //assign the last one as the next parent + lastParent = content; + } + return result; + } + private IEnumerable PrimeDbWithLotsOfContentXmlRecords(Guid customObjectType) { var nodes = new List(); From 55c68485e94bfe663c4a9618e338d186b1ee303d Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 29 Jul 2013 16:03:37 +1000 Subject: [PATCH 11/20] Changed the logic of the GetByPublishedVersion to only return from cache if the cache item is published. --- .../Persistence/Repositories/ContentRepository.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs index 001705ebb9..e2c981f62d 100644 --- a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs @@ -449,8 +449,12 @@ namespace Umbraco.Core.Persistence.Repositories foreach (var dto in dtos) { + //Check in the cache first. If it exists there AND it is published + // then we can use that entity. Otherwise if it is not published (which can be the case + // because we only store the 'latest' entries in the cache which might not be the published + // version) var fromCache = TryGetFromCache(dto.NodeId); - if (fromCache.Success) + if (fromCache.Success && fromCache.Result.Published) { yield return fromCache.Result; } From eed61f905302f24c3a677c7e3e2cca86c6931066 Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 29 Jul 2013 16:42:53 +1000 Subject: [PATCH 12/20] Adds unit test for U4-2556 to ensure that it is solved. --- .../Interfaces/IContentRepository.cs | 2 +- .../Services/ContentServiceTests.cs | 41 ++++++++++++++++++- src/Umbraco.Tests/unit-test-log4net.config | 8 ++++ 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IContentRepository.cs index 8453bcd6da..667395c3b4 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IContentRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IContentRepository.cs @@ -15,7 +15,7 @@ namespace Umbraco.Core.Persistence.Repositories IContent GetByLanguage(int id, string language); /// - /// Gets all published Content byh the specified query + /// Gets all published Content by the specified query /// /// Query to execute against published versions /// An enumerable list of diff --git a/src/Umbraco.Tests/Services/ContentServiceTests.cs b/src/Umbraco.Tests/Services/ContentServiceTests.cs index b85c4c05b1..b087d42827 100644 --- a/src/Umbraco.Tests/Services/ContentServiceTests.cs +++ b/src/Umbraco.Tests/Services/ContentServiceTests.cs @@ -306,12 +306,13 @@ namespace Umbraco.Tests.Services public void Can_RePublish_All_Content() { // Arrange - var contentService = ServiceContext.ContentService; + var contentService = (ContentService)ServiceContext.ContentService; var rootContent = contentService.GetRootContent(); foreach (var c in rootContent) { contentService.PublishWithChildren(c); } + var allContent = rootContent.Concat(rootContent.SelectMany(x => x.Descendants())); //for testing we need to clear out the contentXml table so we can see if it worked var provider = new PetaPocoUnitOfWorkProvider(); var uow = provider.GetUnitOfWork(); @@ -319,13 +320,49 @@ namespace Umbraco.Tests.Services { uow.Database.TruncateTable("cmsContentXml"); } - + //for this test we are also going to save a revision for a content item that is not published, this is to ensure + //that it's published version still makes it into the cmsContentXml table! + contentService.Save(allContent.Last()); + // Act var published = contentService.RePublishAll(0); // Assert Assert.IsTrue(published); + uow = provider.GetUnitOfWork(); + using (var repo = RepositoryResolver.Current.ResolveByType(uow)) + { + Assert.AreEqual(allContent.Count(), uow.Database.ExecuteScalar("select count(*) from cmsContentXml")); + } + } + + [Test] + public void Can_RePublish_All_Content_Of_Type() + { + // Arrange + var contentService = (ContentService)ServiceContext.ContentService; + var rootContent = contentService.GetRootContent(); + foreach (var c in rootContent) + { + contentService.PublishWithChildren(c); + } var allContent = rootContent.Concat(rootContent.SelectMany(x => x.Descendants())); + //for testing we need to clear out the contentXml table so we can see if it worked + var provider = new PetaPocoUnitOfWorkProvider(); + var uow = provider.GetUnitOfWork(); + using (RepositoryResolver.Current.ResolveByType(uow)) + { + uow.Database.TruncateTable("cmsContentXml"); + } + //for this test we are also going to save a revision for a content item that is not published, this is to ensure + //that it's published version still makes it into the cmsContentXml table! + contentService.Save(allContent.Last()); + + + // Act + contentService.RePublishAll(new int[]{allContent.Last().ContentTypeId}); + + // Assert uow = provider.GetUnitOfWork(); using (var repo = RepositoryResolver.Current.ResolveByType(uow)) { diff --git a/src/Umbraco.Tests/unit-test-log4net.config b/src/Umbraco.Tests/unit-test-log4net.config index ce3f527e3c..0922e4d932 100644 --- a/src/Umbraco.Tests/unit-test-log4net.config +++ b/src/Umbraco.Tests/unit-test-log4net.config @@ -5,6 +5,10 @@ + + + + @@ -13,6 +17,10 @@ + + + + From 5b26436862a16ffd020f46adec874a85ddbb424c Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 29 Jul 2013 16:45:54 +1000 Subject: [PATCH 13/20] Removes all references to the legacy Document.RepublishAll and replaces it's logic with the correct logic. --- .../umbraco/dialogs/republish.aspx.cs | 1 + src/umbraco.cms/businesslogic/web/Document.cs | 43 +------------------ 2 files changed, 2 insertions(+), 42 deletions(-) diff --git a/src/Umbraco.Web/umbraco.presentation/umbraco/dialogs/republish.aspx.cs b/src/Umbraco.Web/umbraco.presentation/umbraco/dialogs/republish.aspx.cs index 23ce0cd650..791ea13d40 100644 --- a/src/Umbraco.Web/umbraco.presentation/umbraco/dialogs/republish.aspx.cs +++ b/src/Umbraco.Web/umbraco.presentation/umbraco/dialogs/republish.aspx.cs @@ -27,6 +27,7 @@ namespace umbraco.cms.presentation if (Request.GetItemAsString("xml") != "") { Server.ScriptTimeout = 100000; + Services.ContentService.RePublishAll(); umbraco.cms.businesslogic.web.Document.RePublishAll(); } else if (Request.GetItemAsString("previews") != "") diff --git a/src/umbraco.cms/businesslogic/web/Document.cs b/src/umbraco.cms/businesslogic/web/Document.cs index 724ca09b34..66e6db5080 100644 --- a/src/umbraco.cms/businesslogic/web/Document.cs +++ b/src/umbraco.cms/businesslogic/web/Document.cs @@ -428,48 +428,7 @@ namespace umbraco.cms.businesslogic.web [Obsolete("Obsolete, Use Umbraco.Core.Services.ContentService.RePublishAll()", false)] public static void RePublishAll() { - var xd = new XmlDocument(); - - //Remove all Documents (not media or members), only Documents are stored in the cmsDocument table - SqlHelper.ExecuteNonQuery(@"DELETE FROM cmsContentXml WHERE nodeId IN - (SELECT DISTINCT cmsContentXml.nodeId FROM cmsContentXml - INNER JOIN cmsDocument ON cmsContentXml.nodeId = cmsDocument.nodeId)"); - - var dr = SqlHelper.ExecuteReader("select nodeId from cmsDocument where published = 1"); - - while (dr.Read()) - { - try - { - //create the document in optimized mode! - // (not sure why we wouldn't always do that ?!) - - new Document(true, dr.GetInt("nodeId")) - .XmlGenerate(xd); - - //The benchmark results that I found based contructing the Document object with 'true' for optimized - //mode, vs using the normal ctor. Clearly optimized mode is better! - /* - * The average page rendering time (after 10 iterations) for submitting /umbraco/dialogs/republish?xml=true when using - * optimized mode is - * - * 0.060400555555556 - * - * The average page rendering time (after 10 iterations) for submitting /umbraco/dialogs/republish?xml=true when not - * using optimized mode is - * - * 0.107037777777778 - * - * This means that by simply changing this to use optimized mode, it is a 45% improvement! - * - */ - } - catch (Exception ee) - { - LogHelper.Error("Error generating xml", ee); - } - } - dr.Close(); + ApplicationContext.Current.Services.ContentService.RePublishAll(); } public static void RegeneratePreviews() From 82a87cbd923c88150b94891c428937e8059180c9 Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 29 Jul 2013 16:46:30 +1000 Subject: [PATCH 14/20] Doh, missed file --- .../umbraco.presentation/umbraco/dialogs/republish.aspx.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Umbraco.Web/umbraco.presentation/umbraco/dialogs/republish.aspx.cs b/src/Umbraco.Web/umbraco.presentation/umbraco/dialogs/republish.aspx.cs index 791ea13d40..9142e38c1d 100644 --- a/src/Umbraco.Web/umbraco.presentation/umbraco/dialogs/republish.aspx.cs +++ b/src/Umbraco.Web/umbraco.presentation/umbraco/dialogs/republish.aspx.cs @@ -27,8 +27,7 @@ namespace umbraco.cms.presentation if (Request.GetItemAsString("xml") != "") { Server.ScriptTimeout = 100000; - Services.ContentService.RePublishAll(); - umbraco.cms.businesslogic.web.Document.RePublishAll(); + Services.ContentService.RePublishAll(); } else if (Request.GetItemAsString("previews") != "") { From a195d33332f8dfd8bd7d0d11620a1626ede5e7e2 Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 29 Jul 2013 17:45:05 +1000 Subject: [PATCH 15/20] Added another benchmark for removing/re-inserting records for cmsContentXml table using one transaction which again improves performance. Now have changed the rebuild xml method in the content service to : Lookup all data that it needs to insert first, then we begin a transaction and inside of the one transaction we clear the data and re-insert it so if anything fails in this process it should be rolled back. --- .../Persistence/PetaPocoExtensions.cs | 58 ++++++++++-------- src/Umbraco.Core/Services/ContentService.cs | 59 ++++++++++--------- .../Services/PerformanceTests.cs | 21 +++++++ 3 files changed, 86 insertions(+), 52 deletions(-) diff --git a/src/Umbraco.Core/Persistence/PetaPocoExtensions.cs b/src/Umbraco.Core/Persistence/PetaPocoExtensions.cs index 8917368e21..c962224689 100644 --- a/src/Umbraco.Core/Persistence/PetaPocoExtensions.cs +++ b/src/Umbraco.Core/Persistence/PetaPocoExtensions.cs @@ -39,39 +39,47 @@ namespace Umbraco.Core.Persistence using (var tr = db.GetTransaction()) { - try + db.BulkInsertRecords(collection, tr); + } + } + + public static void BulkInsertRecords(this Database db, IEnumerable collection, Transaction tr) + { + //don't do anything if there are no records. + if (collection.Any() == false) + return; + + try + { + if (SqlSyntaxContext.SqlSyntaxProvider is SqlCeSyntaxProvider) { - if (SqlSyntaxContext.SqlSyntaxProvider is SqlCeSyntaxProvider) + //SqlCe doesn't support bulk insert statements! + + foreach (var poco in collection) { - //SqlCe doesn't support bulk insert statements! - - foreach (var poco in collection) - { - db.Insert(poco); - } - + db.Insert(poco); } - else + } + else + { + string[] sqlStatements; + var cmds = db.GenerateBulkInsertCommand(collection, db.Connection, out sqlStatements); + for (var i = 0; i < sqlStatements.Length; i++) { - string[] sqlStatements; - var cmds = db.GenerateBulkInsertCommand(collection, db.Connection, out sqlStatements); - for (var i = 0; i < sqlStatements.Length; i++) + using (var cmd = cmds[i]) { - using (var cmd = cmds[i]) - { - cmd.CommandText = sqlStatements[i]; - cmd.ExecuteNonQuery(); - } + cmd.CommandText = sqlStatements[i]; + cmd.ExecuteNonQuery(); } } + } - tr.Complete(); - } - catch - { - tr.Dispose(); - throw; - } + tr.Complete(); + } + catch + { + tr.Dispose(); + throw; } } diff --git a/src/Umbraco.Core/Services/ContentService.cs b/src/Umbraco.Core/Services/ContentService.cs index b47879bda1..3dbc03fab2 100644 --- a/src/Umbraco.Core/Services/ContentService.cs +++ b/src/Umbraco.Core/Services/ContentService.cs @@ -1378,41 +1378,46 @@ namespace Umbraco.Core.Services var uow = _uowProvider.GetUnitOfWork(); using (var repository = _repositoryFactory.CreateContentRepository(uow)) { - if (contentTypeIds.Any() == false) - { - //Remove all Document records from the cmsContentXml table (DO NOT REMOVE Media/Members!) - uow.Database.Execute(@"DELETE FROM cmsContentXml WHERE nodeId IN - (SELECT DISTINCT cmsContentXml.nodeId FROM cmsContentXml - INNER JOIN cmsDocument ON cmsContentXml.nodeId = cmsDocument.nodeId)"); + //First we're going to get the data that needs to be inserted before clearing anything, this + //ensures that we don't accidentally leave the content xml table empty if something happens + //during the lookup process. - list.AddRange(GetAllPublished()); - } - else - { - foreach (var id in contentTypeIds) - { - //first we'll clear out the data from the cmsContentXml table for this type - uow.Database.Execute(@"delete from cmsContentXml where nodeId in -(select cmsDocument.nodeId from cmsDocument - inner join cmsContent on cmsDocument.nodeId = cmsContent.nodeId - where published = 1 and contentType = @contentTypeId)", new {contentTypeId = id}); - - //now get all published content objects of this type and add to the list - list.AddRange(GetPublishedContentOfContentType(id)); - } - } + list.AddRange(contentTypeIds.Any() == false + ? GetAllPublished() + : contentTypeIds.SelectMany(GetPublishedContentOfContentType)); var xmlItems = new List(); foreach (var c in list) { - //generate the xml var xml = c.ToXml(); - //create the dto to insert - xmlItems.Add(new ContentXmlDto { NodeId = c.Id, Xml = xml.ToString(SaveOptions.None) }); + xmlItems.Add(new ContentXmlDto {NodeId = c.Id, Xml = xml.ToString(SaveOptions.None)}); } - //bulk insert it into the database - uow.Database.BulkInsertRecords(xmlItems); + //Ok, now we need to remove the data and re-insert it, we'll do this all in one transaction too. + using (var tr = uow.Database.GetTransaction()) + { + if (contentTypeIds.Any() == false) + { + //Remove all Document records from the cmsContentXml table (DO NOT REMOVE Media/Members!) + uow.Database.Execute(@"DELETE FROM cmsContentXml WHERE nodeId IN + (SELECT DISTINCT cmsContentXml.nodeId FROM cmsContentXml + INNER JOIN cmsDocument ON cmsContentXml.nodeId = cmsDocument.nodeId)"); + } + else + { + foreach (var id in contentTypeIds) + { + //first we'll clear out the data from the cmsContentXml table for this type + uow.Database.Execute(@"delete from cmsContentXml where nodeId in +(select cmsDocument.nodeId from cmsDocument + inner join cmsContent on cmsDocument.nodeId = cmsContent.nodeId + where published = 1 and contentType = @contentTypeId)", new { contentTypeId = id }); + } + } + + //bulk insert it into the database + uow.Database.BulkInsertRecords(xmlItems, tr); + } } Audit.Add(AuditTypes.Publish, "RebuildXmlStructures completed, the xml has been regenerated in the database", 0, -1); } diff --git a/src/Umbraco.Tests/Services/PerformanceTests.cs b/src/Umbraco.Tests/Services/PerformanceTests.cs index e49d2a1da3..4a7ad2026a 100644 --- a/src/Umbraco.Tests/Services/PerformanceTests.cs +++ b/src/Umbraco.Tests/Services/PerformanceTests.cs @@ -211,6 +211,27 @@ namespace Umbraco.Tests.Services } } + //now, test truncating but then do bulk insertion of records + using (DisposableTimer.DebugDuration("Starting truncate + bulk insert test in one transaction")) + { + //do this 10x! + for (var i = 0; i < 10; i++) + { + //now we insert each record for the ones we've deleted like we do in the content service. + var xmlItems = nodes.Select(node => new ContentXmlDto { NodeId = node.NodeId, Xml = UpdatedXmlStructure }).ToList(); + + using (var tr = DatabaseContext.Database.GetTransaction()) + { + //clear all the xml entries + DatabaseContext.Database.Execute(@"DELETE FROM cmsContentXml WHERE nodeId IN + (SELECT DISTINCT cmsContentXml.nodeId FROM cmsContentXml + INNER JOIN cmsContent ON cmsContentXml.nodeId = cmsContent.nodeId)"); + + + DatabaseContext.Database.BulkInsertRecords(xmlItems, tr); + } + } + } } From ad697db42bd458ca6fd4cae278ba1a7da688586a Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 31 Jul 2013 11:10:33 +1000 Subject: [PATCH 16/20] Fixes an issue with WebSecurity logout --- src/Umbraco.Web/Security/WebSecurity.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Web/Security/WebSecurity.cs b/src/Umbraco.Web/Security/WebSecurity.cs index 972644ff49..08f5b0b67a 100644 --- a/src/Umbraco.Web/Security/WebSecurity.cs +++ b/src/Umbraco.Web/Security/WebSecurity.cs @@ -128,7 +128,7 @@ namespace Umbraco.Web.Security SqlHelper.CreateParameter("@contextId", retVal)); UmbracoUserContextId = retVal.ToString(); - LogHelper.Info(typeof(WebSecurity), "User Id: {0} logged in", () => userId); + LogHelper.Info("User Id: {0} logged in", () => userId); } @@ -147,8 +147,11 @@ namespace Umbraco.Web.Security } catch (Exception ex) { - LogHelper.Error(typeof(WebSecurity), string.Format("Login with contextId {0} didn't exist in the database", UmbracoUserContextId), ex); + LogHelper.Error(string.Format("Login with contextId {0} didn't exist in the database", UmbracoUserContextId), ex); } + + //this clears the cookie + UmbracoUserContextId = ""; } public void RenewLoginTimeout() From e2eeafcbfcb12934f825b24d7bf5aba0d6f7cfa3 Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 31 Jul 2013 16:45:11 +1000 Subject: [PATCH 17/20] Fixes: U4-2232 UmbracoAuthorizeAttribute breaks Relfection methods - ensures that no singleton instances are passed into attributes since they only ever get created once, meaning we're left with a stale version in the attribute. --- .../UmbracoInstallAuthorizeAttribute.cs | 30 +++++++++++++------ .../Mvc/MemberAuthorizeAttribute.cs | 14 ++++++--- .../Mvc/UmbracoAuthorizeAttribute.cs | 27 ++++++++++++----- .../WebApi/MemberAuthorizeAttribute.cs | 18 +++++++---- .../WebApi/UmbracoAuthorizeAttribute.cs | 21 ++++++++++--- 5 files changed, 80 insertions(+), 30 deletions(-) diff --git a/src/Umbraco.Web/Install/UmbracoInstallAuthorizeAttribute.cs b/src/Umbraco.Web/Install/UmbracoInstallAuthorizeAttribute.cs index 994703bfd3..3aafa86de3 100644 --- a/src/Umbraco.Web/Install/UmbracoInstallAuthorizeAttribute.cs +++ b/src/Umbraco.Web/Install/UmbracoInstallAuthorizeAttribute.cs @@ -13,9 +13,23 @@ namespace Umbraco.Web.Install /// internal class UmbracoInstallAuthorizeAttribute : AuthorizeAttribute { - private readonly ApplicationContext _applicationContext; + private readonly ApplicationContext _applicationContext; private readonly UmbracoContext _umbracoContext; + private ApplicationContext GetApplicationContext() + { + return _applicationContext ?? ApplicationContext.Current; + } + + private UmbracoContext GetUmbracoContext() + { + return _umbracoContext ?? UmbracoContext.Current; + } + + /// + /// THIS SHOULD BE ONLY USED FOR UNIT TESTS + /// + /// public UmbracoInstallAuthorizeAttribute(UmbracoContext umbracoContext) { if (umbracoContext == null) throw new ArgumentNullException("umbracoContext"); @@ -23,11 +37,9 @@ namespace Umbraco.Web.Install _applicationContext = _umbracoContext.Application; } - public UmbracoInstallAuthorizeAttribute() - : this(UmbracoContext.Current) - { - - } + public UmbracoInstallAuthorizeAttribute() + { + } /// /// Ensures that the user must be logged in or that the application is not configured just yet. @@ -41,13 +53,13 @@ namespace Umbraco.Web.Install try { //if its not configured then we can continue - if (!_applicationContext.IsConfigured) + if (!GetApplicationContext().IsConfigured) { return true; } - + var umbCtx = GetUmbracoContext(); //otherwise we need to ensure that a user is logged in - var isLoggedIn = _umbracoContext.Security.ValidateUserContextId(_umbracoContext.Security.UmbracoUserContextId); + var isLoggedIn = umbCtx.Security.ValidateUserContextId(umbCtx.Security.UmbracoUserContextId); if (isLoggedIn) { return true; diff --git a/src/Umbraco.Web/Mvc/MemberAuthorizeAttribute.cs b/src/Umbraco.Web/Mvc/MemberAuthorizeAttribute.cs index bdcbd275ba..dd9720cdcf 100644 --- a/src/Umbraco.Web/Mvc/MemberAuthorizeAttribute.cs +++ b/src/Umbraco.Web/Mvc/MemberAuthorizeAttribute.cs @@ -18,18 +18,24 @@ namespace Umbraco.Web.Mvc public sealed class MemberAuthorizeAttribute : AuthorizeAttribute { - private readonly ApplicationContext _applicationContext; private readonly UmbracoContext _umbracoContext; + private UmbracoContext GetUmbracoContext() + { + return _umbracoContext ?? UmbracoContext.Current; + } + + /// + /// THIS SHOULD BE ONLY USED FOR UNIT TESTS + /// + /// public MemberAuthorizeAttribute(UmbracoContext umbracoContext) { if (umbracoContext == null) throw new ArgumentNullException("umbracoContext"); _umbracoContext = umbracoContext; - _applicationContext = _umbracoContext.Application; } public MemberAuthorizeAttribute() - : this(UmbracoContext.Current) { } @@ -76,7 +82,7 @@ namespace Umbraco.Web.Mvc } } - return _umbracoContext.Security.IsMemberAuthorized(AllowAll, + return GetUmbracoContext().Security.IsMemberAuthorized(AllowAll, AllowType.Split(','), AllowGroup.Split(','), members); diff --git a/src/Umbraco.Web/Mvc/UmbracoAuthorizeAttribute.cs b/src/Umbraco.Web/Mvc/UmbracoAuthorizeAttribute.cs index 19615706d6..ff7056838b 100644 --- a/src/Umbraco.Web/Mvc/UmbracoAuthorizeAttribute.cs +++ b/src/Umbraco.Web/Mvc/UmbracoAuthorizeAttribute.cs @@ -15,6 +15,20 @@ namespace Umbraco.Web.Mvc private readonly ApplicationContext _applicationContext; private readonly UmbracoContext _umbracoContext; + private ApplicationContext GetApplicationContext() + { + return _applicationContext ?? ApplicationContext.Current; + } + + private UmbracoContext GetUmbracoContext() + { + return _umbracoContext ?? UmbracoContext.Current; + } + + /// + /// THIS SHOULD BE ONLY USED FOR UNIT TESTS + /// + /// public UmbracoAuthorizeAttribute(UmbracoContext umbracoContext) { if (umbracoContext == null) throw new ArgumentNullException("umbracoContext"); @@ -22,11 +36,9 @@ namespace Umbraco.Web.Mvc _applicationContext = _umbracoContext.Application; } - public UmbracoAuthorizeAttribute() - : this(UmbracoContext.Current) - { - - } + public UmbracoAuthorizeAttribute() + { + } /// /// Ensures that the user must be in the Administrator or the Install role @@ -40,9 +52,10 @@ namespace Umbraco.Web.Mvc try { //we need to that the app is configured and that a user is logged in - if (!_applicationContext.IsConfigured) + if (!GetApplicationContext().IsConfigured) return false; - var isLoggedIn = _umbracoContext.Security.ValidateUserContextId(_umbracoContext.Security.UmbracoUserContextId); + var umbCtx = GetUmbracoContext(); + var isLoggedIn = umbCtx.Security.ValidateUserContextId(umbCtx.Security.UmbracoUserContextId); return isLoggedIn; } catch (Exception) diff --git a/src/Umbraco.Web/WebApi/MemberAuthorizeAttribute.cs b/src/Umbraco.Web/WebApi/MemberAuthorizeAttribute.cs index bb15fe7c47..0fc04a6206 100644 --- a/src/Umbraco.Web/WebApi/MemberAuthorizeAttribute.cs +++ b/src/Umbraco.Web/WebApi/MemberAuthorizeAttribute.cs @@ -16,21 +16,27 @@ namespace Umbraco.Web.WebApi public sealed class MemberAuthorizeAttribute : AuthorizeAttribute { - private readonly ApplicationContext _applicationContext; private readonly UmbracoContext _umbracoContext; + private UmbracoContext GetUmbracoContext() + { + return _umbracoContext ?? UmbracoContext.Current; + } + + /// + /// THIS SHOULD BE ONLY USED FOR UNIT TESTS + /// + /// public MemberAuthorizeAttribute(UmbracoContext umbracoContext) { if (umbracoContext == null) throw new ArgumentNullException("umbracoContext"); _umbracoContext = umbracoContext; - _applicationContext = _umbracoContext.Application; } public MemberAuthorizeAttribute() - : this(UmbracoContext.Current) - { + { - } + } /// /// Flag for whether to allow all site visitors or just authenticated members @@ -74,7 +80,7 @@ namespace Umbraco.Web.WebApi } } - return _umbracoContext.Security.IsMemberAuthorized(AllowAll, + return GetUmbracoContext().Security.IsMemberAuthorized(AllowAll, AllowType.Split(','), AllowGroup.Split(','), members); diff --git a/src/Umbraco.Web/WebApi/UmbracoAuthorizeAttribute.cs b/src/Umbraco.Web/WebApi/UmbracoAuthorizeAttribute.cs index dc83b043df..7fee8c34e4 100644 --- a/src/Umbraco.Web/WebApi/UmbracoAuthorizeAttribute.cs +++ b/src/Umbraco.Web/WebApi/UmbracoAuthorizeAttribute.cs @@ -13,6 +13,20 @@ namespace Umbraco.Web.WebApi private readonly ApplicationContext _applicationContext; private readonly UmbracoContext _umbracoContext; + private ApplicationContext GetApplicationContext() + { + return _applicationContext ?? ApplicationContext.Current; + } + + private UmbracoContext GetUmbracoContext() + { + return _umbracoContext ?? UmbracoContext.Current; + } + + /// + /// THIS SHOULD BE ONLY USED FOR UNIT TESTS + /// + /// public UmbracoAuthorizeAttribute(UmbracoContext umbracoContext) { if (umbracoContext == null) throw new ArgumentNullException("umbracoContext"); @@ -21,9 +35,7 @@ namespace Umbraco.Web.WebApi } public UmbracoAuthorizeAttribute() - : this(UmbracoContext.Current) { - } protected override bool IsAuthorized(System.Web.Http.Controllers.HttpActionContext actionContext) @@ -31,9 +43,10 @@ namespace Umbraco.Web.WebApi try { //we need to that the app is configured and that a user is logged in - if (!_applicationContext.IsConfigured) + if (!GetApplicationContext().IsConfigured) return false; - var isLoggedIn = _umbracoContext.Security.ValidateUserContextId(_umbracoContext.Security.UmbracoUserContextId); + var umbCtx = GetUmbracoContext(); + var isLoggedIn = umbCtx.Security.ValidateUserContextId(umbCtx.Security.UmbracoUserContextId); return isLoggedIn; } catch (Exception) From b48f0f52e0a866b50c9e614376ecf9b104570c4f Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 31 Jul 2013 17:08:10 +1000 Subject: [PATCH 18/20] updates uri extensions + tests Conflicts: src/Umbraco.Core/UriExtensions.cs --- src/Umbraco.Core/UriExtensions.cs | 159 +++++++++++------- src/Umbraco.Tests/UriExtensionsTests.cs | 32 ++++ .../ServerRegistrationEventHandler.cs | 7 +- 3 files changed, 129 insertions(+), 69 deletions(-) diff --git a/src/Umbraco.Core/UriExtensions.cs b/src/Umbraco.Core/UriExtensions.cs index 45088aa1bf..53ca85e0a8 100644 --- a/src/Umbraco.Core/UriExtensions.cs +++ b/src/Umbraco.Core/UriExtensions.cs @@ -2,6 +2,7 @@ using System; using System.IO; using System.Linq; using System.Text; +using Umbraco.Core.IO; namespace Umbraco.Core { @@ -10,6 +11,38 @@ namespace Umbraco.Core /// public static class UriExtensions { + /// + /// Checks if the current uri is a back office request + /// + /// + /// + internal static bool IsBackOfficeRequest(this Uri url) + { + var authority = url.GetLeftPart(UriPartial.Authority); + var afterAuthority = url.GetLeftPart(UriPartial.Query) + .TrimStart(authority) + .TrimStart("/"); + + //check if this is in the umbraco back office + return afterAuthority.InvariantStartsWith(GlobalSettings.Path.TrimStart("/")); + } + + /// + /// Checks if the current uri is an install request + /// + /// + /// + internal static bool IsInstallerRequest(this Uri url) + { + var authority = url.GetLeftPart(UriPartial.Authority); + var afterAuthority = url.GetLeftPart(UriPartial.Query) + .TrimStart(authority) + .TrimStart("/"); + + //check if this is in the umbraco back office + return afterAuthority.InvariantStartsWith(IOHelper.ResolveUrl("~/install").TrimStart("/")); + } + /// /// This is a performance tweak to check if this is a .css, .js or .ico, .jpg, .jpeg, .png, .gif file request since /// .Net will pass these requests through to the module when in integrated mode. @@ -19,7 +52,7 @@ namespace Umbraco.Core /// internal static bool IsClientSideRequest(this Uri url) { - var toIgnore = new[] { ".js", ".css", ".ico", ".png", ".jpg", ".jpeg", ".gif" }; + var toIgnore = new[] { ".js", ".css", ".ico", ".png", ".jpg", ".jpeg", ".gif", ".html", ".svg" }; return toIgnore.Any(x => Path.GetExtension(url.LocalPath).InvariantEquals(x)); } @@ -31,14 +64,14 @@ namespace Umbraco.Core /// The rewritten uri. /// Everything else remains unchanged, except for the fragment which is removed. public static Uri Rewrite(this Uri uri, string path) - { + { if (!path.StartsWith("/")) throw new ArgumentException("Path must start with a slash.", "path"); - return uri.IsAbsoluteUri - ? new Uri(uri.GetLeftPart(UriPartial.Authority) + path + uri.Query) + return uri.IsAbsoluteUri + ? new Uri(uri.GetLeftPart(UriPartial.Authority) + path + uri.Query) : new Uri(path + uri.GetSafeQuery(), UriKind.Relative); - } + } /// /// Rewrites the path and query of a uri. @@ -49,18 +82,18 @@ namespace Umbraco.Core /// The rewritten uri. /// Everything else remains unchanged, except for the fragment which is removed. public static Uri Rewrite(this Uri uri, string path, string query) - { + { if (!path.StartsWith("/")) throw new ArgumentException("Path must start with a slash.", "path"); if (query.Length > 0 && !query.StartsWith("?")) throw new ArgumentException("Query must start with a question mark.", "query"); if (query == "?") query = ""; - - return uri.IsAbsoluteUri - ? new Uri(uri.GetLeftPart(UriPartial.Authority) + path + query) + + return uri.IsAbsoluteUri + ? new Uri(uri.GetLeftPart(UriPartial.Authority) + path + query) : new Uri(path + query, UriKind.Relative); - } + } /// /// Gets the absolute path of the uri, even if the uri is relative. @@ -68,10 +101,10 @@ namespace Umbraco.Core /// The uri. /// The absolute path of the uri. /// Default uri.AbsolutePath does not support relative uris. - public static string GetSafeAbsolutePath(this Uri uri) - { - if (uri.IsAbsoluteUri) - return uri.AbsolutePath; + public static string GetSafeAbsolutePath(this Uri uri) + { + if (uri.IsAbsoluteUri) + return uri.AbsolutePath; // cannot get .AbsolutePath on relative uri (InvalidOperation) var s = uri.OriginalString; @@ -80,7 +113,7 @@ namespace Umbraco.Core var pos = posq > 0 ? posq : (posf > 0 ? posf : 0); var path = pos > 0 ? s.Substring(0, pos) : s; return path; - } + } /// /// Gets the decoded, absolute path of the uri. @@ -89,9 +122,9 @@ namespace Umbraco.Core /// The absolute path of the uri. /// Only for absolute uris. public static string GetAbsolutePathDecoded(this Uri uri) - { - return System.Web.HttpUtility.UrlDecode(uri.AbsolutePath); - } + { + return System.Web.HttpUtility.UrlDecode(uri.AbsolutePath); + } /// /// Gets the decoded, absolute path of the uri, even if the uri is relative. @@ -100,32 +133,32 @@ namespace Umbraco.Core /// The absolute path of the uri. /// Default uri.AbsolutePath does not support relative uris. public static string GetSafeAbsolutePathDecoded(this Uri uri) - { - return System.Web.HttpUtility.UrlDecode(uri.GetSafeAbsolutePath()); - } + { + return System.Web.HttpUtility.UrlDecode(uri.GetSafeAbsolutePath()); + } - /// - /// Rewrites the path of the uri so it ends with a slash. - /// - /// The uri. - /// The rewritten uri. - /// Everything else remains unchanged. + /// + /// Rewrites the path of the uri so it ends with a slash. + /// + /// The uri. + /// The rewritten uri. + /// Everything else remains unchanged. public static Uri EndPathWithSlash(this Uri uri) - { - var path = uri.GetSafeAbsolutePath(); - if (uri.IsAbsoluteUri) - { - if (path != "/" && !path.EndsWith("/")) - uri = new Uri(uri.GetLeftPart(UriPartial.Authority) + path + "/" + uri.Query); - return uri; - } - else - { - if (path != "/" && !path.EndsWith("/")) - uri = new Uri(path + "/" + uri.Query, UriKind.Relative); - } - return uri; - } + { + var path = uri.GetSafeAbsolutePath(); + if (uri.IsAbsoluteUri) + { + if (path != "/" && !path.EndsWith("/")) + uri = new Uri(uri.GetLeftPart(UriPartial.Authority) + path + "/" + uri.Query); + return uri; + } + else + { + if (path != "/" && !path.EndsWith("/")) + uri = new Uri(path + "/" + uri.Query, UriKind.Relative); + } + return uri; + } /// /// Rewrites the path of the uri so it does not end with a slash. @@ -134,20 +167,20 @@ namespace Umbraco.Core /// The rewritten uri. /// Everything else remains unchanged. public static Uri TrimPathEndSlash(this Uri uri) - { - var path = uri.GetSafeAbsolutePath(); - if (uri.IsAbsoluteUri) - { - if (path != "/") - uri = new Uri(uri.GetLeftPart(UriPartial.Authority) + path.TrimEnd('/') + uri.Query); - } - else - { - if (path != "/") - uri = new Uri(path.TrimEnd('/') + uri.Query, UriKind.Relative); - } - return uri; - } + { + var path = uri.GetSafeAbsolutePath(); + if (uri.IsAbsoluteUri) + { + if (path != "/") + uri = new Uri(uri.GetLeftPart(UriPartial.Authority) + path.TrimEnd('/') + uri.Query); + } + else + { + if (path != "/") + uri = new Uri(path.TrimEnd('/') + uri.Query, UriKind.Relative); + } + return uri; + } /// /// Transforms a relative uri into an absolute uri. @@ -155,13 +188,13 @@ namespace Umbraco.Core /// The relative uri. /// The base absolute uri. /// The absolute uri. - public static Uri MakeAbsolute(this Uri uri, Uri baseUri) - { - if (uri.IsAbsoluteUri) - throw new ArgumentException("Uri is already absolute.", "uri"); + public static Uri MakeAbsolute(this Uri uri, Uri baseUri) + { + if (uri.IsAbsoluteUri) + throw new ArgumentException("Uri is already absolute.", "uri"); - return new Uri(baseUri.GetLeftPart(UriPartial.Authority) + uri.GetSafeAbsolutePath() + uri.GetSafeQuery()); - } + return new Uri(baseUri.GetLeftPart(UriPartial.Authority) + uri.GetSafeAbsolutePath() + uri.GetSafeQuery()); + } static string GetSafeQuery(this Uri uri) { @@ -176,5 +209,5 @@ namespace Umbraco.Core return query; } - } + } } \ No newline at end of file diff --git a/src/Umbraco.Tests/UriExtensionsTests.cs b/src/Umbraco.Tests/UriExtensionsTests.cs index 8389e5d727..61ed49d65d 100644 --- a/src/Umbraco.Tests/UriExtensionsTests.cs +++ b/src/Umbraco.Tests/UriExtensionsTests.cs @@ -10,6 +10,38 @@ namespace Umbraco.Tests [TestFixture] public class UriExtensionsTests { + [TestCase("http://www.domain.com/umbraco", true)] + [TestCase("http://www.domain.com/Umbraco/", true)] + [TestCase("http://www.domain.com/umbraco/default.aspx", true)] + [TestCase("http://www.domain.com/umbraco/test/test", true)] + [TestCase("http://www.domain.com/Umbraco/test/test.aspx", true)] + [TestCase("http://www.domain.com/umbraco/test/test.js", true)] + [TestCase("http://www.domain.com/umbrac", false)] + [TestCase("http://www.domain.com/test", false)] + [TestCase("http://www.domain.com/test/umbraco", false)] + [TestCase("http://www.domain.com/test/umbraco.aspx", false)] + public void Is_Back_Office_Request(string input, bool expected) + { + var source = new Uri(input); + Assert.AreEqual(expected, source.IsBackOfficeRequest()); + } + + [TestCase("http://www.domain.com/install", true)] + [TestCase("http://www.domain.com/Install/", true)] + [TestCase("http://www.domain.com/install/default.aspx", true)] + [TestCase("http://www.domain.com/install/test/test", true)] + [TestCase("http://www.domain.com/Install/test/test.aspx", true)] + [TestCase("http://www.domain.com/install/test/test.js", true)] + [TestCase("http://www.domain.com/instal", false)] + [TestCase("http://www.domain.com/umbraco", false)] + [TestCase("http://www.domain.com/umbraco/umbraco", false)] + [TestCase("http://www.domain.com/test/umbraco.aspx", false)] + public void Is_Installer_Request(string input, bool expected) + { + var source = new Uri(input); + Assert.AreEqual(expected, source.IsInstallerRequest()); + } + [TestCase("http://www.domain.com/foo/bar", "/", "http://www.domain.com/")] [TestCase("http://www.domain.com/foo/bar#hop", "/", "http://www.domain.com/")] [TestCase("http://www.domain.com/foo/bar?q=2#hop", "/", "http://www.domain.com/?q=2")] diff --git a/src/Umbraco.Web/Strategies/ServerRegistrationEventHandler.cs b/src/Umbraco.Web/Strategies/ServerRegistrationEventHandler.cs index bef5f2c4cd..79edd8192e 100644 --- a/src/Umbraco.Web/Strategies/ServerRegistrationEventHandler.cs +++ b/src/Umbraco.Web/Strategies/ServerRegistrationEventHandler.cs @@ -77,13 +77,8 @@ namespace Umbraco.Web.Strategies //if it is not a document request, we'll check if it is a back end request if (e.Outcome == EnsureRoutableOutcome.NotDocumentRequest) { - var authority = e.HttpContext.Request.Url.GetLeftPart(UriPartial.Authority); - var afterAuthority = e.HttpContext.Request.Url.GetLeftPart(UriPartial.Query) - .TrimStart(authority) - .TrimStart("/"); - //check if this is in the umbraco back office - if (afterAuthority.InvariantStartsWith(GlobalSettings.Path.TrimStart("/"))) + if (e.HttpContext.Request.Url.IsBackOfficeRequest()) { //yup it's a back office request! using (var lck = new UpgradeableReadLock(Locker)) From cbda86fe921f6a32d41a313e71b6a9b8f4b0ef0d Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 31 Jul 2013 17:24:36 +1000 Subject: [PATCH 19/20] Fixes build error, streamlines all calls to validate a user for base controllers. --- src/Umbraco.Core/UriExtensions.cs | 1 + .../Mvc/UmbracoAuthorizedController.cs | 47 ++-------------- .../WebApi/UmbracoApiController.cs | 24 +++++++++ .../WebApi/UmbracoAuthorizedApiController.cs | 54 ++++--------------- 4 files changed, 41 insertions(+), 85 deletions(-) diff --git a/src/Umbraco.Core/UriExtensions.cs b/src/Umbraco.Core/UriExtensions.cs index 53ca85e0a8..e5de22456b 100644 --- a/src/Umbraco.Core/UriExtensions.cs +++ b/src/Umbraco.Core/UriExtensions.cs @@ -2,6 +2,7 @@ using System; using System.IO; using System.Linq; using System.Text; +using Umbraco.Core.Configuration; using Umbraco.Core.IO; namespace Umbraco.Core diff --git a/src/Umbraco.Web/Mvc/UmbracoAuthorizedController.cs b/src/Umbraco.Web/Mvc/UmbracoAuthorizedController.cs index ae62f7acee..0f99a6c6c7 100644 --- a/src/Umbraco.Web/Mvc/UmbracoAuthorizedController.cs +++ b/src/Umbraco.Web/Mvc/UmbracoAuthorizedController.cs @@ -20,20 +20,8 @@ namespace Umbraco.Web.Mvc [UmbracoAuthorize] public abstract class UmbracoAuthorizedController : UmbracoController { - - private User _user; private bool _userisValidated = false; - /// - /// The current user ID - /// - private int _uid = 0; - - /// - /// The page timeout in seconds. - /// - private long _timeout = 0; - /// /// Returns the currently logged in Umbraco User /// @@ -41,40 +29,15 @@ namespace Umbraco.Web.Mvc { get { - if (!_userisValidated) ValidateUser(); - return _user; - } - } - - private void ValidateUser() - { - if ((UmbracoContext.Security.UmbracoUserContextId != "")) - { - _uid = UmbracoContext.Security.GetUserId(UmbracoContext.Security.UmbracoUserContextId); - _timeout = UmbracoContext.Security.GetTimeout(UmbracoContext.Security.UmbracoUserContextId); - - if (_timeout > DateTime.Now.Ticks) + //throw exceptions if not valid (true) + if (!_userisValidated) { - _user = global::umbraco.BusinessLogic.User.GetUser(_uid); - - // Check for console access - if (_user.Disabled || (_user.NoConsole && GlobalSettings.RequestIsInUmbracoApplication(HttpContext) && !GlobalSettings.RequestIsLiveEditRedirector(HttpContext))) - { - throw new ArgumentException("You have no priviledges to the umbraco console. Please contact your administrator"); - } + Security.ValidateCurrentUser(HttpContext, true); _userisValidated = true; - UmbracoContext.Security.UpdateLogin(_timeout); } - else - { - throw new ArgumentException("User has timed out!!"); - } - } - else - { - throw new InvalidOperationException("The user has no umbraco contextid - try logging in"); - } + return Security.CurrentUser; + } } } diff --git a/src/Umbraco.Web/WebApi/UmbracoApiController.cs b/src/Umbraco.Web/WebApi/UmbracoApiController.cs index acbf82f161..b8ba3f9a7b 100644 --- a/src/Umbraco.Web/WebApi/UmbracoApiController.cs +++ b/src/Umbraco.Web/WebApi/UmbracoApiController.cs @@ -1,4 +1,5 @@ using System; +using System.Web; using System.Web.Http; using Umbraco.Core; using Umbraco.Core.Services; @@ -22,6 +23,29 @@ namespace Umbraco.Web.WebApi Umbraco = new UmbracoHelper(umbracoContext); } + /// + /// Tries to retreive the current HttpContext if one exists. + /// + /// + protected Attempt TryGetHttpContext() + { + object context; + if (Request.Properties.TryGetValue("MS_HttpContext", out context)) + { + var httpContext = context as HttpContextBase; + if (httpContext != null) + { + return new Attempt(true, httpContext); + } + } + if (HttpContext.Current != null) + { + return new Attempt(true, new HttpContextWrapper(HttpContext.Current)); + } + + return Attempt.False; + } + /// /// Returns the current ApplicationContext /// diff --git a/src/Umbraco.Web/WebApi/UmbracoAuthorizedApiController.cs b/src/Umbraco.Web/WebApi/UmbracoAuthorizedApiController.cs index 9101e3ac66..d0ede6fcda 100644 --- a/src/Umbraco.Web/WebApi/UmbracoAuthorizedApiController.cs +++ b/src/Umbraco.Web/WebApi/UmbracoAuthorizedApiController.cs @@ -19,20 +19,9 @@ namespace Umbraco.Web.WebApi : base(umbracoContext) { } - - private User _user; + private bool _userisValidated = false; - - /// - /// The current user ID - /// - private int _uid = 0; - - /// - /// The page timeout in seconds. - /// - private long _timeout = 0; - + /// /// Returns the currently logged in Umbraco User /// @@ -40,40 +29,19 @@ namespace Umbraco.Web.WebApi { get { - if (!_userisValidated) ValidateUser(); - return _user; - } - } - - private void ValidateUser() - { - if ((UmbracoContext.Security.UmbracoUserContextId != "")) - { - _uid = UmbracoContext.Security.GetUserId(UmbracoContext.Security.UmbracoUserContextId); - _timeout = UmbracoContext.Security.GetTimeout(UmbracoContext.Security.UmbracoUserContextId); - - if (_timeout > DateTime.Now.Ticks) + //throw exceptions if not valid (true) + if (!_userisValidated) { - _user = global::umbraco.BusinessLogic.User.GetUser(_uid); - - // Check for console access - if (_user.Disabled || (_user.NoConsole && GlobalSettings.RequestIsInUmbracoApplication(HttpContext.Current) && !GlobalSettings.RequestIsLiveEditRedirector(HttpContext.Current))) - { - throw new ArgumentException("You have no priviledges to the umbraco console. Please contact your administrator"); - } + var ctx = TryGetHttpContext(); + if (ctx.Success == false) + throw new InvalidOperationException("To get a current user, this method must occur in a web request"); + Security.ValidateCurrentUser(ctx.Result, true); _userisValidated = true; - UmbracoContext.Security.UpdateLogin(_timeout); } - else - { - throw new ArgumentException("User has timed out!!"); - } - } - else - { - throw new InvalidOperationException("The user has no umbraco contextid - try logging in"); - } + return Security.CurrentUser; + } } + } } \ No newline at end of file From 9bc3fdd5559dccdebf68d6a155b9e48e1be1875f Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 31 Jul 2013 17:34:42 +1000 Subject: [PATCH 20/20] Makes UmbracoContext disposable and fixes the module's DisposeHttpContextItems to actual perform the disposal of each item. --- src/Umbraco.Web/UmbracoContext.cs | 16 ++++++-- src/Umbraco.Web/UmbracoModule.cs | 62 ++++++++++++++++--------------- 2 files changed, 45 insertions(+), 33 deletions(-) diff --git a/src/Umbraco.Web/UmbracoContext.cs b/src/Umbraco.Web/UmbracoContext.cs index 471e7403ad..abec9e4076 100644 --- a/src/Umbraco.Web/UmbracoContext.cs +++ b/src/Umbraco.Web/UmbracoContext.cs @@ -24,7 +24,7 @@ namespace Umbraco.Web /// /// Class that encapsulates Umbraco information of a specific HTTP request /// - public class UmbracoContext + public class UmbracoContext : DisposableObject { private const string HttpContextItemName = "Umbraco.Web.UmbracoContext"; private static readonly object Locker = new object(); @@ -361,7 +361,17 @@ namespace Umbraco.Web return null; } } - - + + protected override void DisposeResources() + { + Security.DisposeIfDisposable(); + Security = null; + _previewContent = null; + _umbracoContext = null; + //ensure not to dispose this! + Application = null; + ContentCache = null; + MediaCache = null; + } } } \ No newline at end of file diff --git a/src/Umbraco.Web/UmbracoModule.cs b/src/Umbraco.Web/UmbracoModule.cs index c93b99d180..604e61e55f 100644 --- a/src/Umbraco.Web/UmbracoModule.cs +++ b/src/Umbraco.Web/UmbracoModule.cs @@ -1,4 +1,5 @@ using System; +using System.Collections; using System.IO; using System.Linq; using System.Web; @@ -166,23 +167,6 @@ namespace Umbraco.Web return end; } - /// - /// Checks if the xml cache file needs to be updated/persisted - /// - /// - /// - /// TODO: This needs an overhaul, see the error report created here: - /// https://docs.google.com/document/d/1neGE3q3grB4lVJfgID1keWY2v9JYqf-pw75sxUUJiyo/edit - /// - void PersistXmlCache(HttpContextBase httpContext) - { - if (content.Instance.IsXmlQueuedForPersistenceToFile) - { - content.Instance.RemoveXmlFilePersistenceQueue(); - content.Instance.PersistXmlToFile(); - } - } - #endregion #region Route helper methods @@ -350,7 +334,7 @@ namespace Umbraco.Web /// /// /// - private void RewriteToUmbracoHandler(HttpContextBase context, PublishedContentRequest pcr) + private static void RewriteToUmbracoHandler(HttpContextBase context, PublishedContentRequest pcr) { // NOTE: we do not want to use TransferRequest even though many docs say it is better with IIS7, turns out this is // not what we need. The purpose of TransferRequest is to ensure that .net processes all of the rules for the newly @@ -402,6 +386,36 @@ namespace Umbraco.Web } } + /// + /// Checks if the xml cache file needs to be updated/persisted + /// + /// + /// + /// TODO: This needs an overhaul, see the error report created here: + /// https://docs.google.com/document/d/1neGE3q3grB4lVJfgID1keWY2v9JYqf-pw75sxUUJiyo/edit + /// + static void PersistXmlCache(HttpContextBase httpContext) + { + if (content.Instance.IsXmlQueuedForPersistenceToFile) + { + content.Instance.RemoveXmlFilePersistenceQueue(); + content.Instance.PersistXmlToFile(); + } + } + + /// + /// Any object that is in the HttpContext.Items collection that is IDisposable will get disposed on the end of the request + /// + /// + private static void DisposeHttpContextItems(HttpContext http) + { + foreach (DictionaryEntry i in http.Items) + { + i.Value.DisposeIfDisposable(); + i.Key.DisposeIfDisposable(); + } + } + #region IHttpModule /// @@ -469,18 +483,6 @@ namespace Umbraco.Web #endregion - /// - /// Any object that is in the HttpContext.Items collection that is IDisposable will get disposed on the end of the request - /// - /// - private static void DisposeHttpContextItems(HttpContext http) - { - foreach(var i in http.Items) - { - i.DisposeIfDisposable(); - } - } - #region Events internal static event EventHandler RouteAttempt; private void OnRouteAttempt(RoutableAttemptEventArgs args)