From 012c3289cd76b08a5c343483baa7e9cf6f8a8ea2 Mon Sep 17 00:00:00 2001 From: Stephan Date: Thu, 18 Jun 2015 15:36:04 +0200 Subject: [PATCH] U4-6675 - in-memory temp media cache --- .../PublishedMediaCacheTests.cs | 10 +- .../PublishedContent/PublishedMediaTests.cs | 6 +- src/Umbraco.Web/Cache/MediaCacheRefresher.cs | 7 +- .../XmlPublishedCache/PublishedMediaCache.cs | 246 +++++++++++++----- 4 files changed, 201 insertions(+), 68 deletions(-) diff --git a/src/Umbraco.Tests/Cache/PublishedCache/PublishedMediaCacheTests.cs b/src/Umbraco.Tests/Cache/PublishedCache/PublishedMediaCacheTests.cs index f07f757f1b..d93c4d9e3c 100644 --- a/src/Umbraco.Tests/Cache/PublishedCache/PublishedMediaCacheTests.cs +++ b/src/Umbraco.Tests/Cache/PublishedCache/PublishedMediaCacheTests.cs @@ -146,7 +146,7 @@ namespace Umbraco.Tests.Cache.PublishedCache result.Fields.Add("writerName", "Shannon"); var store = new PublishedMediaCache(ctx.Application); - var doc = store.ConvertFromSearchResult(result); + var doc = store.CreateFromCacheValues(store.ConvertFromSearchResult(result)); DoAssert(doc, 1234, 0, 0, "", "Image", 0, "Shannon", "", 0, 0, "-1,1234", default(DateTime), DateTime.Parse("2012-07-16T10:34:09"), 2); Assert.AreEqual(null, doc.Parent); @@ -160,7 +160,7 @@ namespace Umbraco.Tests.Cache.PublishedCache var xmlDoc = GetMediaXml(); var navigator = xmlDoc.SelectSingleNode("/root/Image").CreateNavigator(); var cache = new PublishedMediaCache(ctx.Application); - var doc = cache.ConvertFromXPathNavigator(navigator); + var doc = cache.CreateFromCacheValues(cache.ConvertFromXPathNavigator(navigator, true)); DoAssert(doc, 2000, 0, 2, "image1", "Image", 2044, "Shannon", "Shannon2", 22, 33, "-1,2000", DateTime.Parse("2012-06-12T14:13:17"), DateTime.Parse("2012-07-20T18:50:43"), 1); Assert.AreEqual(null, doc.Parent); @@ -246,12 +246,14 @@ namespace Umbraco.Tests.Cache.PublishedCache //there is no parent a => null, //we're not going to test this so ignore - a => new List(), + (dd, n) => new List(), (dd, a) => dd.Properties.FirstOrDefault(x => x.PropertyTypeAlias.InvariantEquals(a)), + null, false), //callback to get the children - d => children, + (dd, n) => children, (dd, a) => dd.Properties.FirstOrDefault(x => x.PropertyTypeAlias.InvariantEquals(a)), + null, false); return dicDoc; } diff --git a/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs b/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs index 959ea9d615..6ec347cc0f 100644 --- a/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs +++ b/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs @@ -445,7 +445,8 @@ namespace Umbraco.Tests.PublishedContent var nav = node.CreateNavigator(); - var converted = publishedMedia.ConvertFromXPathNodeIterator(nav.Select("/node"), nodeId); + var converted = publishedMedia.CreateFromCacheValues( + publishedMedia.ConvertFromXPathNodeIterator(nav.Select("/node"), nodeId)); Assert.AreEqual(nodeId, converted.Id); Assert.AreEqual(3, converted.Level); @@ -486,7 +487,8 @@ namespace Umbraco.Tests.PublishedContent var nav = node.CreateNavigator(); - var converted = publishedMedia.ConvertFromXPathNodeIterator(nav.Select("/Image"), nodeId); + var converted = publishedMedia.CreateFromCacheValues( + publishedMedia.ConvertFromXPathNodeIterator(nav.Select("/Image"), nodeId)); Assert.AreEqual(nodeId, converted.Id); Assert.AreEqual(3, converted.Level); diff --git a/src/Umbraco.Web/Cache/MediaCacheRefresher.cs b/src/Umbraco.Web/Cache/MediaCacheRefresher.cs index 429bf313f6..89826d668e 100644 --- a/src/Umbraco.Web/Cache/MediaCacheRefresher.cs +++ b/src/Umbraco.Web/Cache/MediaCacheRefresher.cs @@ -11,6 +11,7 @@ using Umbraco.Core.Models; using Umbraco.Core.Persistence.Repositories; using umbraco.interfaces; using System.Linq; +using Umbraco.Web.PublishedCache.XmlPublishedCache; namespace Umbraco.Web.Cache { @@ -184,9 +185,11 @@ namespace Umbraco.Web.Cache if (idPart == payload.Id.ToString(CultureInfo.InvariantCulture)) ApplicationContext.Current.ApplicationCache.RuntimeCache.ClearCacheByKeySearch( string.Format("{0}_{1}", CacheKeys.MediaCacheKey, payload.Id)); - } - } + } + + // published cache... + PublishedMediaCache.ClearCache(payload.Id); }); diff --git a/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs b/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs index f47c280d6d..c7834e0a1a 100644 --- a/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs +++ b/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs @@ -18,6 +18,9 @@ using Umbraco.Core.Xml; using Umbraco.Web.Models; using UmbracoExamine; using umbraco; +using Umbraco.Core.Cache; +using Umbraco.Core.Sync; +using Umbraco.Web.Cache; namespace Umbraco.Web.PublishedCache.XmlPublishedCache { @@ -167,9 +170,20 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache return null; } - private IPublishedContent GetUmbracoMedia(int id) - { - var searchProvider = GetSearchProviderSafe(); + private IPublishedContent GetUmbracoMedia(int id) + { + // this recreates an IPublishedContent and model each time + // it is called, but at least it should NOT hit the database + // nor Lucene each time, relying on the memory cache instead + + var cacheValues = GetCacheValues(id, GetUmbracoMediaCacheValues); + + return cacheValues == null ? null : CreateFromCacheValues(cacheValues); + } + + private CacheValues GetUmbracoMediaCacheValues(int id) + { + var searchProvider = GetSearchProviderSafe(); if (searchProvider != null) { @@ -202,12 +216,12 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache "Could not retrieve media {0} from Examine index, reverting to looking up media via legacy library.GetMedia method", () => id); - var media = global::umbraco.library.GetMedia(id, true); + var media = global::umbraco.library.GetMedia(id, false); return ConvertFromXPathNodeIterator(media, id); } - internal IPublishedContent ConvertFromXPathNodeIterator(XPathNodeIterator media, int id) + internal CacheValues ConvertFromXPathNodeIterator(XPathNodeIterator media, int id) { if (media != null && media.Current != null) { @@ -223,7 +237,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache return null; } - internal IPublishedContent ConvertFromSearchResult(SearchResult searchResult) + internal CacheValues ConvertFromSearchResult(SearchResult searchResult) { //NOTE: Some fields will not be included if the config section for the internal index has been //mucked around with. It should index everything and so the index definition should simply be: @@ -253,19 +267,24 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache values.Add("level", values["__Path"].Split(',').Length.ToString()); } + return new CacheValues + { + Values = values, + FromExamine = true + }; - var content = new DictionaryPublishedContent(values, - d => d.ParentId != -1 //parent should be null if -1 - ? GetUmbracoMedia(d.ParentId) - : null, - //callback to return the children of the current node - d => GetChildrenMedia(d.Id), - GetProperty, - true); - return content.CreateModel(); + //var content = new DictionaryPublishedContent(values, + // d => d.ParentId != -1 //parent should be null if -1 + // ? GetUmbracoMedia(d.ParentId) + // : null, + // //callback to return the children of the current node + // d => GetChildrenMedia(d.Id), + // GetProperty, + // true); + //return content.CreateModel(); } - internal IPublishedContent ConvertFromXPathNavigator(XPathNavigator xpath) + internal CacheValues ConvertFromXPathNavigator(XPathNavigator xpath, bool forceNav = false) { if (xpath == null) throw new ArgumentNullException("xpath"); @@ -313,15 +332,21 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache } } - var content = new DictionaryPublishedContent(values, - d => d.ParentId != -1 //parent should be null if -1 - ? GetUmbracoMedia(d.ParentId) - : null, - //callback to return the children of the current node based on the xml structure already found - d => GetChildrenMedia(d.Id, xpath), - GetProperty, - false); - return content.CreateModel(); + return new CacheValues + { + Values = values, + XPath = forceNav ? xpath : null // outside of tests we do NOT want to cache the navigator! + }; + + //var content = new DictionaryPublishedContent(values, + // d => d.ParentId != -1 //parent should be null if -1 + // ? GetUmbracoMedia(d.ParentId) + // : null, + // //callback to return the children of the current node based on the xml structure already found + // d => GetChildrenMedia(d.Id, xpath), + // GetProperty, + // false); + //return content.CreateModel(); } /// @@ -398,9 +423,17 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache if (results.Any()) { - return useLuceneSort - ? results.Select(ConvertFromSearchResult) //will already be sorted by lucene - : results.Select(ConvertFromSearchResult).OrderBy(x => x.SortOrder); + // var medias = results.Select(ConvertFromSearchResult); + var medias = results.Select(x => + { + int nid; + if (int.TryParse(x["__NodeId"], out nid) == false && int.TryParse(x["NodeId"], out nid) == false) + throw new Exception("Failed to extract NodeId from search result."); + var cacheValues = GetCacheValues(nid, id => ConvertFromSearchResult(x)); + return CreateFromCacheValues(cacheValues); + }); + + return useLuceneSort ? medias : medias.OrderBy(x => x.SortOrder); } else { @@ -432,29 +465,51 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache } } - //The xpath might be the whole xpath including the current ones ancestors so we need to select the current node - var item = xpath.Select("//*[@id='" + parentId + "']"); - if (item.Current == null) - { - return Enumerable.Empty(); - } - var children = item.Current.SelectChildren(XPathNodeType.Element); + var mediaList = new List(); + + // this is so bad, really + var item = xpath.Select("//*[@id='" + parentId + "']"); + if (item.Current == null) + return Enumerable.Empty(); + var items = item.Current.SelectChildren(XPathNodeType.Element); + + // and this does not work, because... meh + //var q = "//* [@id='" + parentId + "']/* [@id]"; + //var items = xpath.Select(q); + + foreach (XPathNavigator itemm in items) + { + int id; + if (int.TryParse(itemm.GetAttribute("id", ""), out id) == false) + continue; // wtf? + var captured = itemm; + var cacheValues = GetCacheValues(id, idd => ConvertFromXPathNavigator(captured)); + mediaList.Add(CreateFromCacheValues(cacheValues)); + } + + ////The xpath might be the whole xpath including the current ones ancestors so we need to select the current node + //var item = xpath.Select("//*[@id='" + parentId + "']"); + //if (item.Current == null) + //{ + // return Enumerable.Empty(); + //} + //var children = item.Current.SelectChildren(XPathNodeType.Element); + + //foreach(XPathNavigator x in children) + //{ + // //NOTE: I'm not sure why this is here, it is from legacy code of ExamineBackedMedia, but + // // will leave it here as it must have done something! + // if (x.Name != "contents") + // { + // //make sure it's actually a node, not a property + // if (!string.IsNullOrEmpty(x.GetAttribute("path", "")) && + // !string.IsNullOrEmpty(x.GetAttribute("id", ""))) + // { + // mediaList.Add(ConvertFromXPathNavigator(x)); + // } + // } + //} - var mediaList = new List(); - foreach(XPathNavigator x in children) - { - //NOTE: I'm not sure why this is here, it is from legacy code of ExamineBackedMedia, but - // will leave it here as it must have done something! - if (x.Name != "contents") - { - //make sure it's actually a node, not a property - if (!string.IsNullOrEmpty(x.GetAttribute("path", "")) && - !string.IsNullOrEmpty(x.GetAttribute("id", ""))) - { - mediaList.Add(ConvertFromXPathNavigator(x)); - } - } - } return mediaList; } @@ -477,23 +532,25 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache public DictionaryPublishedContent( IDictionary valueDictionary, - Func getParent, - Func> getChildren, + Func getParent, + Func> getChildren, Func getProperty, + XPathNavigator nav, bool fromExamine) { if (valueDictionary == null) throw new ArgumentNullException("valueDictionary"); if (getParent == null) throw new ArgumentNullException("getParent"); if (getProperty == null) throw new ArgumentNullException("getProperty"); - _getParent = getParent; - _getChildren = getChildren; + _getParent = new Lazy(() => getParent(ParentId)); + _getChildren = new Lazy>(() => getChildren(Id, nav)); _getProperty = getProperty; LoadedFromExamine = fromExamine; ValidateAndSetProperty(valueDictionary, val => _id = int.Parse(val), "id", "nodeId", "__NodeId"); //should validate the int! - ValidateAndSetProperty(valueDictionary, val => _templateId = int.Parse(val), "template", "templateId"); + // wtf are we dealing with templates for medias?! + ValidateAndSetProperty(valueDictionary, val => _templateId = int.Parse(val), "template", "templateId"); ValidateAndSetProperty(valueDictionary, val => _sortOrder = int.Parse(val), "sortOrder"); ValidateAndSetProperty(valueDictionary, val => _name = val, "nodeName", "__nodeName"); ValidateAndSetProperty(valueDictionary, val => _urlName = val, "urlName"); @@ -576,8 +633,10 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache /// internal bool LoadedFromExamine { get; private set; } - private readonly Func _getParent; - private readonly Func> _getChildren; + //private readonly Func _getParent; + private readonly Lazy _getParent; + //private readonly Func> _getChildren; + private readonly Lazy> _getChildren; private readonly Func _getProperty; /// @@ -590,7 +649,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache public override IPublishedContent Parent { - get { return _getParent(this); } + get { return _getParent.Value; } } public int ParentId { get; private set; } @@ -690,7 +749,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache public override IEnumerable Children { - get { return _getChildren(this); } + get { return _getChildren.Value; } } public override IPublishedProperty GetProperty(string alias) @@ -768,5 +827,72 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache _keysAdded.Add(key); } } - } + + // REFACTORING + + // caching the basic atomic values - and the parent id + // but NOT caching actual parent nor children and NOT even + // the list of children ids - BUT caching the path + + internal class CacheValues + { + public IDictionary Values { get; set; } + public XPathNavigator XPath { get; set; } + public bool FromExamine { get; set; } + } + + public const string PublishedMediaCacheKey = "MediaCacheMeh."; + private const int PublishedMediaCacheTimespanSeconds = 60; + private static readonly TimeSpan PublishedMediaCacheTimespan = TimeSpan.FromSeconds(PublishedMediaCacheTimespanSeconds); + + internal IPublishedContent CreateFromCacheValues(CacheValues cacheValues) + { + var content = new DictionaryPublishedContent( + cacheValues.Values, + parentId => parentId < 0 ? null : GetUmbracoMedia(parentId), + GetChildrenMedia, + GetProperty, + cacheValues.XPath, // though, outside of tests, that should be null + cacheValues.FromExamine + ); + return content.CreateModel(); + } + + private static CacheValues GetCacheValues(int id, Func func) + { + var cache = ApplicationContext.Current.ApplicationCache.RuntimeCache; + var key = PublishedMediaCacheKey + id; + return (CacheValues) cache.GetCacheItem(key, () => func(id), PublishedMediaCacheTimespan); + } + + internal static void ClearCache(int id) + { + var cache = ApplicationContext.Current.ApplicationCache.RuntimeCache; + var sid = id.ToString(); + var key = PublishedMediaCacheKey + sid; + + // we do clear a lot of things... but the cache refresher is somewhat + // convoluted and it's hard to tell what to clear exactly ;-( + + // clear the parent - NOT (why?) + //var exist = (CacheValues) cache.GetCacheItem(key); + //if (exist != null) + // cache.ClearCacheItem(PublishedMediaCacheKey + GetValuesValue(exist.Values, "parentID")); + + // clear the item + cache.ClearCacheItem(key); + + // clear all children - in case we moved and their path has changed + var fid = "/" + sid + "/"; + cache.ClearCacheObjectTypes((k, v) => + GetValuesValue(v.Values, "path", "__Path").Contains(fid)); + } + + private static string GetValuesValue(IDictionary d, params string[] keys) + { + string value = null; + var ignored = keys.Any(x => d.TryGetValue(x, out value)); + return value ?? ""; + } + } }