From 8d1cdf020b08f6ad18d2dc35a4c669f8c910fdc5 Mon Sep 17 00:00:00 2001 From: Stephan Date: Fri, 5 Jun 2015 15:45:06 +0200 Subject: [PATCH] temp - temp media memory cache --- .../PublishedMediaCacheTests.cs | 10 +- .../PublishedContent/PublishedMediaTests.cs | 6 +- .../XmlPublishedCache/PublishedMediaCache.cs | 267 ++++++++++++++---- 3 files changed, 217 insertions(+), 66 deletions(-) diff --git a/src/Umbraco.Tests/Cache/PublishedCache/PublishedMediaCacheTests.cs b/src/Umbraco.Tests/Cache/PublishedCache/PublishedMediaCacheTests.cs index f07f757f1b..d93c4d9e3c 100644 --- a/src/Umbraco.Tests/Cache/PublishedCache/PublishedMediaCacheTests.cs +++ b/src/Umbraco.Tests/Cache/PublishedCache/PublishedMediaCacheTests.cs @@ -146,7 +146,7 @@ namespace Umbraco.Tests.Cache.PublishedCache result.Fields.Add("writerName", "Shannon"); var store = new PublishedMediaCache(ctx.Application); - var doc = store.ConvertFromSearchResult(result); + var doc = store.CreateFromCacheValues(store.ConvertFromSearchResult(result)); DoAssert(doc, 1234, 0, 0, "", "Image", 0, "Shannon", "", 0, 0, "-1,1234", default(DateTime), DateTime.Parse("2012-07-16T10:34:09"), 2); Assert.AreEqual(null, doc.Parent); @@ -160,7 +160,7 @@ namespace Umbraco.Tests.Cache.PublishedCache var xmlDoc = GetMediaXml(); var navigator = xmlDoc.SelectSingleNode("/root/Image").CreateNavigator(); var cache = new PublishedMediaCache(ctx.Application); - var doc = cache.ConvertFromXPathNavigator(navigator); + var doc = cache.CreateFromCacheValues(cache.ConvertFromXPathNavigator(navigator, true)); DoAssert(doc, 2000, 0, 2, "image1", "Image", 2044, "Shannon", "Shannon2", 22, 33, "-1,2000", DateTime.Parse("2012-06-12T14:13:17"), DateTime.Parse("2012-07-20T18:50:43"), 1); Assert.AreEqual(null, doc.Parent); @@ -246,12 +246,14 @@ namespace Umbraco.Tests.Cache.PublishedCache //there is no parent a => null, //we're not going to test this so ignore - a => new List(), + (dd, n) => new List(), (dd, a) => dd.Properties.FirstOrDefault(x => x.PropertyTypeAlias.InvariantEquals(a)), + null, false), //callback to get the children - d => children, + (dd, n) => children, (dd, a) => dd.Properties.FirstOrDefault(x => x.PropertyTypeAlias.InvariantEquals(a)), + null, false); return dicDoc; } diff --git a/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs b/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs index 959ea9d615..6ec347cc0f 100644 --- a/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs +++ b/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs @@ -445,7 +445,8 @@ namespace Umbraco.Tests.PublishedContent var nav = node.CreateNavigator(); - var converted = publishedMedia.ConvertFromXPathNodeIterator(nav.Select("/node"), nodeId); + var converted = publishedMedia.CreateFromCacheValues( + publishedMedia.ConvertFromXPathNodeIterator(nav.Select("/node"), nodeId)); Assert.AreEqual(nodeId, converted.Id); Assert.AreEqual(3, converted.Level); @@ -486,7 +487,8 @@ namespace Umbraco.Tests.PublishedContent var nav = node.CreateNavigator(); - var converted = publishedMedia.ConvertFromXPathNodeIterator(nav.Select("/Image"), nodeId); + var converted = publishedMedia.CreateFromCacheValues( + publishedMedia.ConvertFromXPathNodeIterator(nav.Select("/Image"), nodeId)); Assert.AreEqual(nodeId, converted.Id); Assert.AreEqual(3, converted.Level); diff --git a/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs b/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs index f47c280d6d..6f6e07a473 100644 --- a/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs +++ b/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs @@ -18,6 +18,9 @@ using Umbraco.Core.Xml; using Umbraco.Web.Models; using UmbracoExamine; using umbraco; +using Umbraco.Core.Cache; +using Umbraco.Core.Sync; +using Umbraco.Web.Cache; namespace Umbraco.Web.PublishedCache.XmlPublishedCache { @@ -33,6 +36,8 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache { if (applicationContext == null) throw new ArgumentNullException("applicationContext"); _applicationContext = applicationContext; + + MediaCacheRefresher.CacheUpdated += MediaCacheUpdated; } /// @@ -167,9 +172,20 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache return null; } - private IPublishedContent GetUmbracoMedia(int id) - { - var searchProvider = GetSearchProviderSafe(); + private IPublishedContent GetUmbracoMedia(int id) + { + // this recreates an IPublishedContent and model each time + // it is called, but at least it should NOT hit the database + // nor Lucene each time, relying on the memory cache instead + + var cacheValues = GetCacheValues(id, GetUmbracoMediaCacheValues); + + return cacheValues == null ? null : CreateFromCacheValues(cacheValues); + } + + private CacheValues GetUmbracoMediaCacheValues(int id) + { + var searchProvider = GetSearchProviderSafe(); if (searchProvider != null) { @@ -202,12 +218,12 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache "Could not retrieve media {0} from Examine index, reverting to looking up media via legacy library.GetMedia method", () => id); - var media = global::umbraco.library.GetMedia(id, true); + var media = global::umbraco.library.GetMedia(id, false); return ConvertFromXPathNodeIterator(media, id); } - internal IPublishedContent ConvertFromXPathNodeIterator(XPathNodeIterator media, int id) + internal CacheValues ConvertFromXPathNodeIterator(XPathNodeIterator media, int id) { if (media != null && media.Current != null) { @@ -223,7 +239,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache return null; } - internal IPublishedContent ConvertFromSearchResult(SearchResult searchResult) + internal CacheValues ConvertFromSearchResult(SearchResult searchResult) { //NOTE: Some fields will not be included if the config section for the internal index has been //mucked around with. It should index everything and so the index definition should simply be: @@ -253,19 +269,24 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache values.Add("level", values["__Path"].Split(',').Length.ToString()); } + return new CacheValues + { + Values = values, + FromExamine = true + }; - var content = new DictionaryPublishedContent(values, - d => d.ParentId != -1 //parent should be null if -1 - ? GetUmbracoMedia(d.ParentId) - : null, - //callback to return the children of the current node - d => GetChildrenMedia(d.Id), - GetProperty, - true); - return content.CreateModel(); + //var content = new DictionaryPublishedContent(values, + // d => d.ParentId != -1 //parent should be null if -1 + // ? GetUmbracoMedia(d.ParentId) + // : null, + // //callback to return the children of the current node + // d => GetChildrenMedia(d.Id), + // GetProperty, + // true); + //return content.CreateModel(); } - internal IPublishedContent ConvertFromXPathNavigator(XPathNavigator xpath) + internal CacheValues ConvertFromXPathNavigator(XPathNavigator xpath, bool forceNav = false) { if (xpath == null) throw new ArgumentNullException("xpath"); @@ -313,15 +334,21 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache } } - var content = new DictionaryPublishedContent(values, - d => d.ParentId != -1 //parent should be null if -1 - ? GetUmbracoMedia(d.ParentId) - : null, - //callback to return the children of the current node based on the xml structure already found - d => GetChildrenMedia(d.Id, xpath), - GetProperty, - false); - return content.CreateModel(); + return new CacheValues + { + Values = values, + XPath = forceNav ? xpath : null // outside of tests we do NOT want to cache the navigator! + }; + + //var content = new DictionaryPublishedContent(values, + // d => d.ParentId != -1 //parent should be null if -1 + // ? GetUmbracoMedia(d.ParentId) + // : null, + // //callback to return the children of the current node based on the xml structure already found + // d => GetChildrenMedia(d.Id, xpath), + // GetProperty, + // false); + //return content.CreateModel(); } /// @@ -398,9 +425,17 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache if (results.Any()) { - return useLuceneSort - ? results.Select(ConvertFromSearchResult) //will already be sorted by lucene - : results.Select(ConvertFromSearchResult).OrderBy(x => x.SortOrder); + // var medias = results.Select(ConvertFromSearchResult); + var medias = results.Select(x => + { + int nid; + if (int.TryParse(x["__NodeId"], out nid) == false && int.TryParse(x["NodeId"], out nid) == false) + throw new Exception("Failed to extract NodeId from search result."); + var cacheValues = GetCacheValues(nid, id => ConvertFromSearchResult(x)); + return CreateFromCacheValues(cacheValues); + }); + + return useLuceneSort ? medias : medias.OrderBy(x => x.SortOrder); } else { @@ -432,29 +467,51 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache } } - //The xpath might be the whole xpath including the current ones ancestors so we need to select the current node - var item = xpath.Select("//*[@id='" + parentId + "']"); - if (item.Current == null) - { - return Enumerable.Empty(); - } - var children = item.Current.SelectChildren(XPathNodeType.Element); + var mediaList = new List(); + + // this is so bad, really + var item = xpath.Select("//*[@id='" + parentId + "']"); + if (item.Current == null) + return Enumerable.Empty(); + var items = item.Current.SelectChildren(XPathNodeType.Element); + + // and this does not work, because... meh + //var q = "//* [@id='" + parentId + "']/* [@id]"; + //var items = xpath.Select(q); + + foreach (XPathNavigator itemm in items) + { + int id; + if (int.TryParse(itemm.GetAttribute("id", ""), out id) == false) + continue; // wtf? + var captured = itemm; + var cacheValues = GetCacheValues(id, idd => ConvertFromXPathNavigator(captured)); + mediaList.Add(CreateFromCacheValues(cacheValues)); + } + + ////The xpath might be the whole xpath including the current ones ancestors so we need to select the current node + //var item = xpath.Select("//*[@id='" + parentId + "']"); + //if (item.Current == null) + //{ + // return Enumerable.Empty(); + //} + //var children = item.Current.SelectChildren(XPathNodeType.Element); + + //foreach(XPathNavigator x in children) + //{ + // //NOTE: I'm not sure why this is here, it is from legacy code of ExamineBackedMedia, but + // // will leave it here as it must have done something! + // if (x.Name != "contents") + // { + // //make sure it's actually a node, not a property + // if (!string.IsNullOrEmpty(x.GetAttribute("path", "")) && + // !string.IsNullOrEmpty(x.GetAttribute("id", ""))) + // { + // mediaList.Add(ConvertFromXPathNavigator(x)); + // } + // } + //} - var mediaList = new List(); - foreach(XPathNavigator x in children) - { - //NOTE: I'm not sure why this is here, it is from legacy code of ExamineBackedMedia, but - // will leave it here as it must have done something! - if (x.Name != "contents") - { - //make sure it's actually a node, not a property - if (!string.IsNullOrEmpty(x.GetAttribute("path", "")) && - !string.IsNullOrEmpty(x.GetAttribute("id", ""))) - { - mediaList.Add(ConvertFromXPathNavigator(x)); - } - } - } return mediaList; } @@ -477,23 +534,25 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache public DictionaryPublishedContent( IDictionary valueDictionary, - Func getParent, - Func> getChildren, + Func getParent, + Func> getChildren, Func getProperty, + XPathNavigator nav, bool fromExamine) { if (valueDictionary == null) throw new ArgumentNullException("valueDictionary"); if (getParent == null) throw new ArgumentNullException("getParent"); if (getProperty == null) throw new ArgumentNullException("getProperty"); - _getParent = getParent; - _getChildren = getChildren; + _getParent = new Lazy(() => getParent(ParentId)); + _getChildren = new Lazy>(() => getChildren(Id, nav)); _getProperty = getProperty; LoadedFromExamine = fromExamine; ValidateAndSetProperty(valueDictionary, val => _id = int.Parse(val), "id", "nodeId", "__NodeId"); //should validate the int! - ValidateAndSetProperty(valueDictionary, val => _templateId = int.Parse(val), "template", "templateId"); + // wtf are we dealing with templates for medias?! + ValidateAndSetProperty(valueDictionary, val => _templateId = int.Parse(val), "template", "templateId"); ValidateAndSetProperty(valueDictionary, val => _sortOrder = int.Parse(val), "sortOrder"); ValidateAndSetProperty(valueDictionary, val => _name = val, "nodeName", "__nodeName"); ValidateAndSetProperty(valueDictionary, val => _urlName = val, "urlName"); @@ -576,8 +635,10 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache /// internal bool LoadedFromExamine { get; private set; } - private readonly Func _getParent; - private readonly Func> _getChildren; + //private readonly Func _getParent; + private readonly Lazy _getParent; + //private readonly Func> _getChildren; + private readonly Lazy> _getChildren; private readonly Func _getProperty; /// @@ -590,7 +651,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache public override IPublishedContent Parent { - get { return _getParent(this); } + get { return _getParent.Value; } } public int ParentId { get; private set; } @@ -690,7 +751,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache public override IEnumerable Children { - get { return _getChildren(this); } + get { return _getChildren.Value; } } public override IPublishedProperty GetProperty(string alias) @@ -768,5 +829,91 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache _keysAdded.Add(key); } } - } + + // REFACTORING + + internal class CacheValues + { + public IDictionary Values { get; set; } + public XPathNavigator XPath { get; set; } + public bool FromExamine { get; set; } + } + + internal IPublishedContent CreateFromCacheValues(CacheValues cacheValues) + { + var content = new DictionaryPublishedContent( + cacheValues.Values, + parentId => parentId < 0 ? null : GetUmbracoMedia(parentId), + (id, nav) => GetChildrenMedia(id, nav), + GetProperty, + cacheValues.XPath, // though, outside of tests, that should be null + cacheValues.FromExamine + ); + + return content.CreateModel(); + } + + private const string CachePrefix = "MediaCacheMeh."; + + private CacheValues GetCacheValues(int id, Func func) + { + var cache = ApplicationContext.Current.ApplicationCache.RuntimeCache; + var key = CachePrefix + id; + + // cache for 30s - should we be more aggressive? + return (CacheValues) cache.GetCacheItem(key, () => func(id), TimeSpan.FromSeconds(30)); + } + + private void MediaCacheUpdated(MediaCacheRefresher sender, CacheRefresherEventArgs args) + { + var cache = ApplicationContext.Current.ApplicationCache.RuntimeCache; + int id; + + switch (args.MessageType) + { + case MessageType.RefreshAll: + cache.ClearCacheByKeySearch(CachePrefix); + break; + case MessageType.RefreshById: + case MessageType.RemoveById: + id = (int) args.MessageObject; + ClearCache(id); + break; + case MessageType.RefreshByInstance: + case MessageType.RemoveByInstance: + id = ((IMedia) args.MessageObject).Id; + ClearCache(id); + break; + case MessageType.RefreshByJson: + var payloads = MediaCacheRefresher.DeserializeFromJsonPayload((string) args.MessageObject); + foreach (var pid in payloads.Select(x => x.Id)) + ClearCache(pid); + break; + } + } + + private void ClearCache(int id) + { + var cache = ApplicationContext.Current.ApplicationCache.RuntimeCache; + var sid = id.ToString(); + var key = CachePrefix + sid; + // clear the parent + var exist = (CacheValues) cache.GetCacheItem(key); + if (exist != null) + cache.ClearCacheItem(CachePrefix + GetValuesValue(exist.Values, "parentID")); + // clear the item + cache.ClearCacheItem(key); + // clear all children + var fid = "/" + sid + "/"; + cache.ClearCacheObjectTypes((k, v) => + GetValuesValue(v.Values, "path", "__Path").Contains(fid)); + } + + private string GetValuesValue(IDictionary d, params string[] keys) + { + string value = null; + var found = keys.Any(x => d.TryGetValue(x, out value)); + return value ?? ""; + } + } }