From 908c8837e9fad793e354795cd9676fe29d9249a7 Mon Sep 17 00:00:00 2001 From: Stephan Date: Tue, 16 Aug 2016 15:41:52 +0200 Subject: [PATCH] U4-8856 - don't flood log with media errors --- .../XmlPublishedCache/PublishedMediaCache.cs | 1089 +++++++++-------- 1 file changed, 551 insertions(+), 538 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs b/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs index db2ca62794..da1ff94fe1 100644 --- a/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs +++ b/src/Umbraco.Web/PublishedCache/XmlPublishedCache/PublishedMediaCache.cs @@ -4,6 +4,7 @@ using System.Collections.ObjectModel; using System.Configuration; using System.IO; using System.Linq; +using System.Threading; using System.Xml.XPath; using Examine; using Examine.LuceneEngine.SearchCriteria; @@ -25,58 +26,58 @@ using Umbraco.Web.Cache; namespace Umbraco.Web.PublishedCache.XmlPublishedCache { - /// - /// An IPublishedMediaStore that first checks for the media in Examine, and then reverts to the database - /// - /// - /// NOTE: In the future if we want to properly cache all media this class can be extended or replaced when these classes/interfaces are exposed publicly. - /// + /// + /// An IPublishedMediaStore that first checks for the media in Examine, and then reverts to the database + /// + /// + /// NOTE: In the future if we want to properly cache all media this class can be extended or replaced when these classes/interfaces are exposed publicly. + /// internal class PublishedMediaCache : IPublishedMediaCache - { + { public PublishedMediaCache(ApplicationContext applicationContext) { if (applicationContext == null) throw new ArgumentNullException("applicationContext"); _applicationContext = applicationContext; } - /// - /// Generally used for unit testing to use an explicit examine searcher - /// + /// + /// Generally used for unit testing to use an explicit examine searcher + /// /// - /// - /// + /// + /// internal PublishedMediaCache(ApplicationContext applicationContext, BaseSearchProvider searchProvider, BaseIndexProvider indexProvider) - { + { if (applicationContext == null) throw new ArgumentNullException("applicationContext"); - if (searchProvider == null) throw new ArgumentNullException("searchProvider"); - if (indexProvider == null) throw new ArgumentNullException("indexProvider"); + if (searchProvider == null) throw new ArgumentNullException("searchProvider"); + if (indexProvider == null) throw new ArgumentNullException("indexProvider"); _applicationContext = applicationContext; - _searchProvider = searchProvider; - _indexProvider = indexProvider; - } + _searchProvider = searchProvider; + _indexProvider = indexProvider; + } - static PublishedMediaCache() - { - InitializeCacheConfig(); - } + static PublishedMediaCache() + { + InitializeCacheConfig(); + } private readonly ApplicationContext _applicationContext; - private readonly BaseSearchProvider _searchProvider; + private readonly BaseSearchProvider _searchProvider; private readonly BaseIndexProvider _indexProvider; public virtual IPublishedContent GetById(UmbracoContext umbracoContext, bool preview, int nodeId) - { - return GetUmbracoMedia(nodeId); - } + { + return GetUmbracoMedia(nodeId); + } - public virtual IEnumerable GetAtRoot(UmbracoContext umbracoContext, bool preview) - { + public virtual IEnumerable GetAtRoot(UmbracoContext umbracoContext, bool preview) + { //TODO: We should be able to look these ids first in Examine! - var rootMedia = _applicationContext.Services.MediaService.GetRootMedia(); - return rootMedia.Select(m => GetUmbracoMedia(m.Id)); - } + var rootMedia = _applicationContext.Services.MediaService.GetRootMedia(); + return rootMedia.Select(m => GetUmbracoMedia(m.Id)); + } public virtual IPublishedContent GetSingleByXPath(UmbracoContext umbracoContext, bool preview, string xpath, XPathVariable[] vars) { @@ -108,19 +109,19 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache public virtual bool HasContent(UmbracoContext context, bool preview) { throw new NotImplementedException(); } private ExamineManager GetExamineManagerSafe() - { - try - { - return ExamineManager.Instance; - } - catch (TypeInitializationException) - { - return null; - } - } + { + try + { + return ExamineManager.Instance; + } + catch (TypeInitializationException) + { + return null; + } + } - private BaseIndexProvider GetIndexProviderSafe() - { + private BaseIndexProvider GetIndexProviderSafe() + { if (_indexProvider != null) return _indexProvider; @@ -144,95 +145,107 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache } } return null; - } + } - private BaseSearchProvider GetSearchProviderSafe() - { - if (_searchProvider != null) - return _searchProvider; + private BaseSearchProvider GetSearchProviderSafe() + { + if (_searchProvider != null) + return _searchProvider; - var eMgr = GetExamineManagerSafe(); - if (eMgr != null) - { - try - { - //by default use the InternalSearcher - return eMgr.SearchProviderCollection[Constants.Examine.InternalSearcher]; - } - catch (FileNotFoundException) - { - //Currently examine is throwing FileNotFound exceptions when we have a loadbalanced filestore and a node is published in umbraco - //See this thread: http://examine.cdodeplex.com/discussions/264341 - //Catch the exception here for the time being, and just fallback to GetMedia - //TODO: Need to fix examine in LB scenarios! - } - catch (NullReferenceException) - { - //This will occur when the search provider cannot be initialized. In newer examine versions the initialization is lazy and therefore + var eMgr = GetExamineManagerSafe(); + if (eMgr != null) + { + try + { + //by default use the InternalSearcher + return eMgr.SearchProviderCollection[Constants.Examine.InternalSearcher]; + } + catch (FileNotFoundException) + { + //Currently examine is throwing FileNotFound exceptions when we have a loadbalanced filestore and a node is published in umbraco + //See this thread: http://examine.cdodeplex.com/discussions/264341 + //Catch the exception here for the time being, and just fallback to GetMedia + //TODO: Need to fix examine in LB scenarios! + } + catch (NullReferenceException) + { + //This will occur when the search provider cannot be initialized. In newer examine versions the initialization is lazy and therefore // the manager will return the singleton without throwing initialization errors, however if examine isn't configured correctly a null // reference error will occur because the examine settings are null. - } - } - return null; - } + } + } + return null; + } - private IPublishedContent GetUmbracoMedia(int id) - { + private IPublishedContent GetUmbracoMedia(int id) + { // this recreates an IPublishedContent and model each time // it is called, but at least it should NOT hit the database // nor Lucene each time, relying on the memory cache instead - if (id <= 0) return null; // fail fast + if (id <= 0) return null; // fail fast - var cacheValues = GetCacheValues(id, GetUmbracoMediaCacheValues); + var cacheValues = GetCacheValues(id, GetUmbracoMediaCacheValues); - return cacheValues == null ? null : CreateFromCacheValues(cacheValues); - } + return cacheValues == null ? null : CreateFromCacheValues(cacheValues); + } private CacheValues GetUmbracoMediaCacheValues(int id) { - var searchProvider = GetSearchProviderSafe(); + var searchProvider = GetSearchProviderSafe(); - if (searchProvider != null) - { - try - { - //first check in Examine as this is WAY faster - var criteria = searchProvider.CreateSearchCriteria("media"); + if (searchProvider != null) + { + try + { + // first check in Examine as this is WAY faster + // + // the filter will create a query like this: + // +(+__NodeId:3113 -__Path:-1,-21,*) +__IndexType:media + // + // note that since the use of the wildcard, it automatically escapes it in Lucene. + var criteria = searchProvider.CreateSearchCriteria("media"); var filter = criteria.Id(id).Not().Field(UmbracoContentIndexer.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard()); - //the above filter will create a query like this, NOTE: That since the use of the wildcard, it automatically escapes it in Lucene. - //+(+__NodeId:3113 -__Path:-1,-21,*) +__IndexType:media - var results = searchProvider.Search(filter.Compile()); - if (results.Any()) - { - return ConvertFromSearchResult(results.First()); - } - } - catch (FileNotFoundException ex) - { - //Currently examine is throwing FileNotFound exceptions when we have a loadbalanced filestore and a node is published in umbraco - //See this thread: http://examine.cdodeplex.com/discussions/264341 - //Catch the exception here for the time being, and just fallback to GetMedia - //TODO: Need to fix examine in LB scenarios! + var result = searchProvider.Search(filter.Compile()).FirstOrDefault(); + if (result != null) return ConvertFromSearchResult(result); + } + catch (FileNotFoundException ex) + { + //Currently examine is throwing FileNotFound exceptions when we have a loadbalanced filestore and a node is published in umbraco + //See this thread: http://examine.cdodeplex.com/discussions/264341 + //Catch the exception here for the time being, and just fallback to GetMedia + //TODO: Need to fix examine in LB scenarios! LogHelper.Error("Could not load data from Examine index for media", ex); - } - } + } + } - LogHelper.Warn( - "Could not retrieve media {0} from Examine index, reverting to looking up media via legacy library.GetMedia method", - () => id); - - //var media = global::umbraco.library.GetMedia(id, false); - //return ConvertFromXPathNodeIterator(media, id); + // don't log a warning here, as it can flood the log in case of eg a media picker referencing a media + // that has been deleted, hence is not in the Examine index anymore (for a good reason). try to get + // the media from the service, first var media = ApplicationContext.Current.Services.MediaService.GetById(id); - return media == null ? null : ConvertFromIMedia(media); + if (media == null) return null; // not found, ok + + // so, the media was not found in Examine's index *yet* it exists, which probably indicates that + // the index is corrupted. Or not up-to-date. Log a warning, but only once, and only if seeing the + // error more that a number of times. + + var miss = Interlocked.CompareExchange(ref _examineIndexMiss, 0, 0); // volatile read + if (miss < ExamineIndexMissMax && Interlocked.Increment(ref _examineIndexMiss) == ExamineIndexMissMax) + LogHelper.Warn("Failed ({0} times) to retrieve medias from Examine index and had to load" + + " them from DB. This may indicate that the Examine index is corrupted.", + () => ExamineIndexMissMax); + + return ConvertFromIMedia(media); } + private const int ExamineIndexMissMax = 10; + private int _examineIndexMiss; + internal CacheValues ConvertFromXPathNodeIterator(XPathNodeIterator media, int id) - { + { if (media != null && media.Current != null) { return media.Current.Name.InvariantEquals("error") @@ -245,47 +258,47 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache () => id); return null; - } + } - internal CacheValues ConvertFromSearchResult(SearchResult searchResult) - { - //NOTE: Some fields will not be included if the config section for the internal index has been + internal CacheValues ConvertFromSearchResult(SearchResult searchResult) + { + //NOTE: Some fields will not be included if the config section for the internal index has been //mucked around with. It should index everything and so the index definition should simply be: // - var values = new Dictionary(searchResult.Fields); - //we need to ensure some fields exist, because of the above issue - if (!new []{"template", "templateId"}.Any(values.ContainsKey)) - values.Add("template", 0.ToString()); - if (!new[] { "sortOrder" }.Any(values.ContainsKey)) - values.Add("sortOrder", 0.ToString()); - if (!new[] { "urlName" }.Any(values.ContainsKey)) - values.Add("urlName", ""); - if (!new[] { "nodeType" }.Any(values.ContainsKey)) - values.Add("nodeType", 0.ToString()); - if (!new[] { "creatorName" }.Any(values.ContainsKey)) - values.Add("creatorName", ""); - if (!new[] { "writerID" }.Any(values.ContainsKey)) - values.Add("writerID", 0.ToString()); - if (!new[] { "creatorID" }.Any(values.ContainsKey)) - values.Add("creatorID", 0.ToString()); - if (!new[] { "createDate" }.Any(values.ContainsKey)) - values.Add("createDate", default(DateTime).ToString("yyyy-MM-dd HH:mm:ss")); - if (!new[] { "level" }.Any(values.ContainsKey)) - { - values.Add("level", values["__Path"].Split(',').Length.ToString()); - } + var values = new Dictionary(searchResult.Fields); + //we need to ensure some fields exist, because of the above issue + if (!new[] { "template", "templateId" }.Any(values.ContainsKey)) + values.Add("template", 0.ToString()); + if (!new[] { "sortOrder" }.Any(values.ContainsKey)) + values.Add("sortOrder", 0.ToString()); + if (!new[] { "urlName" }.Any(values.ContainsKey)) + values.Add("urlName", ""); + if (!new[] { "nodeType" }.Any(values.ContainsKey)) + values.Add("nodeType", 0.ToString()); + if (!new[] { "creatorName" }.Any(values.ContainsKey)) + values.Add("creatorName", ""); + if (!new[] { "writerID" }.Any(values.ContainsKey)) + values.Add("writerID", 0.ToString()); + if (!new[] { "creatorID" }.Any(values.ContainsKey)) + values.Add("creatorID", 0.ToString()); + if (!new[] { "createDate" }.Any(values.ContainsKey)) + values.Add("createDate", default(DateTime).ToString("yyyy-MM-dd HH:mm:ss")); + if (!new[] { "level" }.Any(values.ContainsKey)) + { + values.Add("level", values["__Path"].Split(',').Length.ToString()); + } // because, migration if (values.ContainsKey("key") == false) values["key"] = Guid.Empty.ToString(); - return new CacheValues - { - Values = values, + return new CacheValues + { + Values = values, FromExamine = true - }; + }; //var content = new DictionaryPublishedContent(values, // d => d.ParentId != -1 //parent should be null if -1 @@ -296,104 +309,104 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache // GetProperty, // true); //return content.CreateModel(); - } + } - internal CacheValues ConvertFromXPathNavigator(XPathNavigator xpath, bool forceNav = false) - { - if (xpath == null) throw new ArgumentNullException("xpath"); + internal CacheValues ConvertFromXPathNavigator(XPathNavigator xpath, bool forceNav = false) + { + if (xpath == null) throw new ArgumentNullException("xpath"); - var values = new Dictionary {{"nodeName", xpath.GetAttribute("nodeName", "")}}; - if (!UmbracoConfig.For.UmbracoSettings().Content.UseLegacyXmlSchema) - { - values["nodeTypeAlias"] = xpath.Name; - } + var values = new Dictionary { { "nodeName", xpath.GetAttribute("nodeName", "") } }; + if (!UmbracoConfig.For.UmbracoSettings().Content.UseLegacyXmlSchema) + { + values["nodeTypeAlias"] = xpath.Name; + } - var result = xpath.SelectChildren(XPathNodeType.Element); - //add the attributes e.g. id, parentId etc - if (result.Current != null && result.Current.HasAttributes) - { - if (result.Current.MoveToFirstAttribute()) - { - //checking for duplicate keys because of the 'nodeTypeAlias' might already be added above. - if (!values.ContainsKey(result.Current.Name)) - { - values[result.Current.Name] = result.Current.Value; - } - while (result.Current.MoveToNextAttribute()) - { - if (!values.ContainsKey(result.Current.Name)) - { - values[result.Current.Name] = result.Current.Value; - } - } - result.Current.MoveToParent(); - } - } + var result = xpath.SelectChildren(XPathNodeType.Element); + //add the attributes e.g. id, parentId etc + if (result.Current != null && result.Current.HasAttributes) + { + if (result.Current.MoveToFirstAttribute()) + { + //checking for duplicate keys because of the 'nodeTypeAlias' might already be added above. + if (!values.ContainsKey(result.Current.Name)) + { + values[result.Current.Name] = result.Current.Value; + } + while (result.Current.MoveToNextAttribute()) + { + if (!values.ContainsKey(result.Current.Name)) + { + values[result.Current.Name] = result.Current.Value; + } + } + result.Current.MoveToParent(); + } + } // because, migration - if (values.ContainsKey("key") == false) - values["key"] = Guid.Empty.ToString(); - //add the user props - while (result.MoveNext()) - { - if (result.Current != null && !result.Current.HasAttributes) - { - string value = result.Current.Value; - if (string.IsNullOrEmpty(value)) - { - if (result.Current.HasAttributes || result.Current.SelectChildren(XPathNodeType.Element).Count > 0) - { - value = result.Current.OuterXml; - } - } - values[result.Current.Name] = value; - } - } + if (values.ContainsKey("key") == false) + values["key"] = Guid.Empty.ToString(); + //add the user props + while (result.MoveNext()) + { + if (result.Current != null && !result.Current.HasAttributes) + { + string value = result.Current.Value; + if (string.IsNullOrEmpty(value)) + { + if (result.Current.HasAttributes || result.Current.SelectChildren(XPathNodeType.Element).Count > 0) + { + value = result.Current.OuterXml; + } + } + values[result.Current.Name] = value; + } + } - return new CacheValues - { + return new CacheValues + { Values = values, XPath = forceNav ? xpath : null // outside of tests we do NOT want to cache the navigator! - }; + }; - //var content = new DictionaryPublishedContent(values, - // d => d.ParentId != -1 //parent should be null if -1 - // ? GetUmbracoMedia(d.ParentId) - // : null, - // //callback to return the children of the current node based on the xml structure already found - // d => GetChildrenMedia(d.Id, xpath), - // GetProperty, - // false); - //return content.CreateModel(); - } + //var content = new DictionaryPublishedContent(values, + // d => d.ParentId != -1 //parent should be null if -1 + // ? GetUmbracoMedia(d.ParentId) + // : null, + // //callback to return the children of the current node based on the xml structure already found + // d => GetChildrenMedia(d.Id, xpath), + // GetProperty, + // false); + //return content.CreateModel(); + } - internal CacheValues ConvertFromIMedia(IMedia media) - { - var values = new Dictionary(); + internal CacheValues ConvertFromIMedia(IMedia media) + { + var values = new Dictionary(); - var creator = _applicationContext.Services.UserService.GetProfileById(media.CreatorId); + var creator = _applicationContext.Services.UserService.GetProfileById(media.CreatorId); var creatorName = creator == null ? "" : creator.Name; - values["id"] = media.Id.ToString(); - values["key"] = media.Key.ToString(); - values["parentID"] = media.ParentId.ToString(); - values["level"] = media.Level.ToString(); - values["creatorID"] = media.CreatorId.ToString(); - values["creatorName"] = creatorName; + values["id"] = media.Id.ToString(); + values["key"] = media.Key.ToString(); + values["parentID"] = media.ParentId.ToString(); + values["level"] = media.Level.ToString(); + values["creatorID"] = media.CreatorId.ToString(); + values["creatorName"] = creatorName; values["writerID"] = media.CreatorId.ToString(); - values["writerName"] = creatorName; + values["writerName"] = creatorName; values["template"] = "0"; values["urlName"] = ""; values["sortOrder"] = media.SortOrder.ToString(); - values["createDate"] = media.CreateDate.ToString("yyyy-MM-dd HH:mm:ss"); - values["updateDate"] = media.UpdateDate.ToString("yyyy-MM-dd HH:mm:ss"); - values["nodeName"] = media.Name; - values["path"] = media.Path; - values["nodeType"] = media.ContentType.Id.ToString(); - values["nodeTypeAlias"] = media.ContentType.Alias; + values["createDate"] = media.CreateDate.ToString("yyyy-MM-dd HH:mm:ss"); + values["updateDate"] = media.UpdateDate.ToString("yyyy-MM-dd HH:mm:ss"); + values["nodeName"] = media.Name; + values["path"] = media.Path; + values["nodeType"] = media.ContentType.Id.ToString(); + values["nodeTypeAlias"] = media.ContentType.Alias; // add the user props - foreach (var prop in media.Properties) - values[prop.Alias] = prop.Value == null ? null : prop.Value.ToString(); + foreach (var prop in media.Properties) + values[prop.Alias] = prop.Value == null ? null : prop.Value.ToString(); return new CacheValues { @@ -409,7 +422,7 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache /// /// private IPublishedProperty GetProperty(DictionaryPublishedContent dd, string alias) - { + { //lets check if the alias does not exist on the document. //NOTE: Examine will not index empty values and we do not output empty XML Elements to the cache - either of these situations // would mean that the property is missing from the collection whether we are getting the value from Examine or from the library media cache. @@ -418,50 +431,50 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache return null; } - if (dd.LoadedFromExamine) - { - //We are going to check for a special field however, that is because in some cases we store a 'Raw' - //value in the index such as for xml/html. - var rawValue = dd.Properties.FirstOrDefault(x => x.PropertyTypeAlias.InvariantEquals(UmbracoContentIndexer.RawFieldPrefix + alias)); - return rawValue - ?? dd.Properties.FirstOrDefault(x => x.PropertyTypeAlias.InvariantEquals(alias)); - } + if (dd.LoadedFromExamine) + { + //We are going to check for a special field however, that is because in some cases we store a 'Raw' + //value in the index such as for xml/html. + var rawValue = dd.Properties.FirstOrDefault(x => x.PropertyTypeAlias.InvariantEquals(UmbracoContentIndexer.RawFieldPrefix + alias)); + return rawValue + ?? dd.Properties.FirstOrDefault(x => x.PropertyTypeAlias.InvariantEquals(alias)); + } //if its not loaded from examine, then just return the property - return dd.Properties.FirstOrDefault(x => x.PropertyTypeAlias.InvariantEquals(alias)); - } + return dd.Properties.FirstOrDefault(x => x.PropertyTypeAlias.InvariantEquals(alias)); + } - /// - /// A Helper methods to return the children for media whther it is based on examine or xml - /// - /// - /// - /// - private IEnumerable GetChildrenMedia(int parentId, XPathNavigator xpath = null) - { + /// + /// A Helper methods to return the children for media whther it is based on examine or xml + /// + /// + /// + /// + private IEnumerable GetChildrenMedia(int parentId, XPathNavigator xpath = null) + { - //if there is no navigator, try examine first, then re-look it up - if (xpath == null) - { - var searchProvider = GetSearchProviderSafe(); + //if there is no navigator, try examine first, then re-look it up + if (xpath == null) + { + var searchProvider = GetSearchProviderSafe(); - if (searchProvider != null) - { - try - { - //first check in Examine as this is WAY faster - var criteria = searchProvider.CreateSearchCriteria("media"); + if (searchProvider != null) + { + try + { + //first check in Examine as this is WAY faster + var criteria = searchProvider.CreateSearchCriteria("media"); var filter = criteria.ParentId(parentId).Not().Field(UmbracoContentIndexer.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard()); //the above filter will create a query like this, NOTE: That since the use of the wildcard, it automatically escapes it in Lucene. //+(+parentId:3113 -__Path:-1,-21,*) +__IndexType:media - ISearchResults results; + ISearchResults results; //we want to check if the indexer for this searcher has "sortOrder" flagged as sortable. //if so, we'll use Lucene to do the sorting, if not we'll have to manually sort it (slower). var indexer = GetIndexProviderSafe(); - var useLuceneSort = indexer != null && indexer.IndexerData.StandardFields.Any(x => x.Name.InvariantEquals("sortOrder") && x.EnableSorting); + var useLuceneSort = indexer != null && indexer.IndexerData.StandardFields.Any(x => x.Name.InvariantEquals("sortOrder") && x.EnableSorting); if (useLuceneSort) { //we have a sortOrder field declared to be sorted, so we'll use Examine @@ -473,49 +486,49 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache results = searchProvider.Search(filter.Compile()); } - if (results.Any()) - { + if (results.Any()) + { // var medias = results.Select(ConvertFromSearchResult); - var medias = results.Select(x => - { - int nid; - if (int.TryParse(x["__NodeId"], out nid) == false && int.TryParse(x["NodeId"], out nid) == false) - throw new Exception("Failed to extract NodeId from search result."); - var cacheValues = GetCacheValues(nid, id => ConvertFromSearchResult(x)); - return CreateFromCacheValues(cacheValues); - }); + var medias = results.Select(x => + { + int nid; + if (int.TryParse(x["__NodeId"], out nid) == false && int.TryParse(x["NodeId"], out nid) == false) + throw new Exception("Failed to extract NodeId from search result."); + var cacheValues = GetCacheValues(nid, id => ConvertFromSearchResult(x)); + return CreateFromCacheValues(cacheValues); + }); - return useLuceneSort ? medias : medias.OrderBy(x => x.SortOrder); - } - else - { - //if there's no result then return null. Previously we defaulted back to library.GetMedia below + return useLuceneSort ? medias : medias.OrderBy(x => x.SortOrder); + } + else + { + //if there's no result then return null. Previously we defaulted back to library.GetMedia below //but this will always get called for when we are getting descendents since many items won't have //children and then we are hitting the database again! //So instead we're going to rely on Examine to have the correct results like it should. - return Enumerable.Empty(); - } - } - catch (FileNotFoundException) - { - //Currently examine is throwing FileNotFound exceptions when we have a loadbalanced filestore and a node is published in umbraco - //See this thread: http://examine.cdodeplex.com/discussions/264341 - //Catch the exception here for the time being, and just fallback to GetMedia - } - } + return Enumerable.Empty(); + } + } + catch (FileNotFoundException) + { + //Currently examine is throwing FileNotFound exceptions when we have a loadbalanced filestore and a node is published in umbraco + //See this thread: http://examine.cdodeplex.com/discussions/264341 + //Catch the exception here for the time being, and just fallback to GetMedia + } + } //falling back to get media - var media = library.GetMedia(parentId, true); - if (media != null && media.Current != null) - { - xpath = media.Current; - } - else - { + var media = library.GetMedia(parentId, true); + if (media != null && media.Current != null) + { + xpath = media.Current; + } + else + { return Enumerable.Empty(); - } - } + } + } var mediaList = new List(); @@ -526,18 +539,18 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache var items = item.Current.SelectChildren(XPathNodeType.Element); // and this does not work, because... meh - //var q = "//* [@id='" + parentId + "']/* [@id]"; + //var q = "//* [@id='" + parentId + "']/* [@id]"; //var items = xpath.Select(q); - foreach (XPathNavigator itemm in items) - { - int id; - if (int.TryParse(itemm.GetAttribute("id", ""), out id) == false) - continue; // wtf? - var captured = itemm; - var cacheValues = GetCacheValues(id, idd => ConvertFromXPathNavigator(captured)); - mediaList.Add(CreateFromCacheValues(cacheValues)); - } + foreach (XPathNavigator itemm in items) + { + int id; + if (int.TryParse(itemm.GetAttribute("id", ""), out id) == false) + continue; // wtf? + var captured = itemm; + var cacheValues = GetCacheValues(id, idd => ConvertFromXPathNavigator(captured)); + mediaList.Add(CreateFromCacheValues(cacheValues)); + } ////The xpath might be the whole xpath including the current ones ancestors so we need to select the current node //var item = xpath.Select("//*[@id='" + parentId + "']"); @@ -562,18 +575,18 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache // } //} - return mediaList; - } + return mediaList; + } - /// - /// An IPublishedContent that is represented all by a dictionary. - /// - /// - /// This is a helper class and definitely not intended for public use, it expects that all of the values required - /// to create an IPublishedContent exist in the dictionary by specific aliases. - /// - internal class DictionaryPublishedContent : PublishedContentWithKeyBase - { + /// + /// An IPublishedContent that is represented all by a dictionary. + /// + /// + /// This is a helper class and definitely not intended for public use, it expects that all of the values required + /// to create an IPublishedContent exist in the dictionary by specific aliases. + /// + internal class DictionaryPublishedContent : PublishedContentWithKeyBase + { // note: I'm not sure this class fully complies with IPublishedContent rules especially // I'm not sure that _properties contains all properties including those without a value, // neither that GetProperty will return a property without a value vs. null... @zpqrtbnk @@ -582,59 +595,59 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache // anything in the ContentType, so they must be ignored. private static readonly string[] IgnoredKeys = { "version", "isDoc" }; - public DictionaryPublishedContent( - IDictionary valueDictionary, - Func getParent, - Func> getChildren, - Func getProperty, + public DictionaryPublishedContent( + IDictionary valueDictionary, + Func getParent, + Func> getChildren, + Func getProperty, XPathNavigator nav, - bool fromExamine) - { - if (valueDictionary == null) throw new ArgumentNullException("valueDictionary"); - if (getParent == null) throw new ArgumentNullException("getParent"); - if (getProperty == null) throw new ArgumentNullException("getProperty"); + bool fromExamine) + { + if (valueDictionary == null) throw new ArgumentNullException("valueDictionary"); + if (getParent == null) throw new ArgumentNullException("getParent"); + if (getProperty == null) throw new ArgumentNullException("getProperty"); - _getParent = new Lazy(() => getParent(ParentId)); - _getChildren = new Lazy>(() => getChildren(Id, nav)); - _getProperty = getProperty; + _getParent = new Lazy(() => getParent(ParentId)); + _getChildren = new Lazy>(() => getChildren(Id, nav)); + _getProperty = getProperty; - LoadedFromExamine = fromExamine; + LoadedFromExamine = fromExamine; - ValidateAndSetProperty(valueDictionary, val => _id = int.Parse(val), "id", "nodeId", "__NodeId"); //should validate the int! + ValidateAndSetProperty(valueDictionary, val => _id = int.Parse(val), "id", "nodeId", "__NodeId"); //should validate the int! ValidateAndSetProperty(valueDictionary, val => _key = Guid.Parse(val), "key"); - // wtf are we dealing with templates for medias?! + // wtf are we dealing with templates for medias?! ValidateAndSetProperty(valueDictionary, val => _templateId = int.Parse(val), "template", "templateId"); - ValidateAndSetProperty(valueDictionary, val => _sortOrder = int.Parse(val), "sortOrder"); - ValidateAndSetProperty(valueDictionary, val => _name = val, "nodeName", "__nodeName"); - ValidateAndSetProperty(valueDictionary, val => _urlName = val, "urlName"); - ValidateAndSetProperty(valueDictionary, val => _documentTypeAlias = val, "nodeTypeAlias", UmbracoContentIndexer.NodeTypeAliasFieldName); - ValidateAndSetProperty(valueDictionary, val => _documentTypeId = int.Parse(val), "nodeType"); - ValidateAndSetProperty(valueDictionary, val => _writerName = val, "writerName"); - ValidateAndSetProperty(valueDictionary, val => _creatorName = val, "creatorName", "writerName"); //this is a bit of a hack fix for: U4-1132 - ValidateAndSetProperty(valueDictionary, val => _writerId = int.Parse(val), "writerID"); - ValidateAndSetProperty(valueDictionary, val => _creatorId = int.Parse(val), "creatorID", "writerID"); //this is a bit of a hack fix for: U4-1132 - ValidateAndSetProperty(valueDictionary, val => _path = val, "path", "__Path"); - ValidateAndSetProperty(valueDictionary, val => _createDate = ParseDateTimeValue(val), "createDate"); - ValidateAndSetProperty(valueDictionary, val => _updateDate = ParseDateTimeValue(val), "updateDate"); - ValidateAndSetProperty(valueDictionary, val => _level = int.Parse(val), "level"); - ValidateAndSetProperty(valueDictionary, val => - { - int pId; - ParentId = -1; - if (int.TryParse(val, out pId)) - { - ParentId = pId; - } - }, "parentID"); + ValidateAndSetProperty(valueDictionary, val => _sortOrder = int.Parse(val), "sortOrder"); + ValidateAndSetProperty(valueDictionary, val => _name = val, "nodeName", "__nodeName"); + ValidateAndSetProperty(valueDictionary, val => _urlName = val, "urlName"); + ValidateAndSetProperty(valueDictionary, val => _documentTypeAlias = val, "nodeTypeAlias", UmbracoContentIndexer.NodeTypeAliasFieldName); + ValidateAndSetProperty(valueDictionary, val => _documentTypeId = int.Parse(val), "nodeType"); + ValidateAndSetProperty(valueDictionary, val => _writerName = val, "writerName"); + ValidateAndSetProperty(valueDictionary, val => _creatorName = val, "creatorName", "writerName"); //this is a bit of a hack fix for: U4-1132 + ValidateAndSetProperty(valueDictionary, val => _writerId = int.Parse(val), "writerID"); + ValidateAndSetProperty(valueDictionary, val => _creatorId = int.Parse(val), "creatorID", "writerID"); //this is a bit of a hack fix for: U4-1132 + ValidateAndSetProperty(valueDictionary, val => _path = val, "path", "__Path"); + ValidateAndSetProperty(valueDictionary, val => _createDate = ParseDateTimeValue(val), "createDate"); + ValidateAndSetProperty(valueDictionary, val => _updateDate = ParseDateTimeValue(val), "updateDate"); + ValidateAndSetProperty(valueDictionary, val => _level = int.Parse(val), "level"); + ValidateAndSetProperty(valueDictionary, val => + { + int pId; + ParentId = -1; + if (int.TryParse(val, out pId)) + { + ParentId = pId; + } + }, "parentID"); - _contentType = PublishedContentType.Get(PublishedItemType.Media, _documentTypeAlias); - _properties = new Collection(); + _contentType = PublishedContentType.Get(PublishedItemType.Media, _documentTypeAlias); + _properties = new Collection(); //handle content type properties //make sure we create them even if there's no value - foreach (var propertyType in _contentType.PropertyTypes) - { - var alias = propertyType.PropertyTypeAlias; + foreach (var propertyType in _contentType.PropertyTypes) + { + var alias = propertyType.PropertyTypeAlias; _keysAdded.Add(alias); string value; const bool isPreviewing = false; // false :: never preview a media @@ -642,13 +655,13 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache ? new XmlPublishedProperty(propertyType, isPreviewing) : new XmlPublishedProperty(propertyType, isPreviewing, value); _properties.Add(property); - } + } - //loop through remaining values that haven't been applied - foreach (var i in valueDictionary.Where(x => + //loop through remaining values that haven't been applied + foreach (var i in valueDictionary.Where(x => _keysAdded.Contains(x.Key) == false // not already processed && IgnoredKeys.Contains(x.Key) == false)) // not ignorable - { + { if (i.Key.InvariantStartsWith("__")) { // no type for that one, dunno how to convert @@ -660,157 +673,157 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache // this is a property that does not correspond to anything, ignore and log LogHelper.Warn("Dropping property \"" + i.Key + "\" because it does not belong to the content type."); } - } - } + } + } - private DateTime ParseDateTimeValue(string val) - { - if (LoadedFromExamine) - { - try - { - //we might need to parse the date time using Lucene converters - return DateTools.StringToDate(val); - } - catch (FormatException) - { - //swallow exception, its not formatted correctly so revert to just trying to parse - } - } + private DateTime ParseDateTimeValue(string val) + { + if (LoadedFromExamine) + { + try + { + //we might need to parse the date time using Lucene converters + return DateTools.StringToDate(val); + } + catch (FormatException) + { + //swallow exception, its not formatted correctly so revert to just trying to parse + } + } - return DateTime.Parse(val); - } + return DateTime.Parse(val); + } - /// - /// Flag to get/set if this was laoded from examine cache - /// - internal bool LoadedFromExamine { get; private set; } + /// + /// Flag to get/set if this was laoded from examine cache + /// + internal bool LoadedFromExamine { get; private set; } - //private readonly Func _getParent; - private readonly Lazy _getParent; - //private readonly Func> _getChildren; - private readonly Lazy> _getChildren; - private readonly Func _getProperty; + //private readonly Func _getParent; + private readonly Lazy _getParent; + //private readonly Func> _getChildren; + private readonly Lazy> _getChildren; + private readonly Func _getProperty; - /// - /// Returns 'Media' as the item type - /// - public override PublishedItemType ItemType - { - get { return PublishedItemType.Media; } - } + /// + /// Returns 'Media' as the item type + /// + public override PublishedItemType ItemType + { + get { return PublishedItemType.Media; } + } - public override IPublishedContent Parent - { - get { return _getParent.Value; } - } + public override IPublishedContent Parent + { + get { return _getParent.Value; } + } - public int ParentId { get; private set; } - public override int Id - { - get { return _id; } - } + public int ParentId { get; private set; } + public override int Id + { + get { return _id; } + } public override Guid Key { get { return _key; } } - public override int TemplateId - { - get - { - //TODO: should probably throw a not supported exception since media doesn't actually support this. - return _templateId; - } - } + public override int TemplateId + { + get + { + //TODO: should probably throw a not supported exception since media doesn't actually support this. + return _templateId; + } + } - public override int SortOrder - { - get { return _sortOrder; } - } + public override int SortOrder + { + get { return _sortOrder; } + } - public override string Name - { - get { return _name; } - } + public override string Name + { + get { return _name; } + } - public override string UrlName - { - get { return _urlName; } - } + public override string UrlName + { + get { return _urlName; } + } - public override string DocumentTypeAlias - { - get { return _documentTypeAlias; } - } + public override string DocumentTypeAlias + { + get { return _documentTypeAlias; } + } - public override int DocumentTypeId - { - get { return _documentTypeId; } - } + public override int DocumentTypeId + { + get { return _documentTypeId; } + } - public override string WriterName - { - get { return _writerName; } - } + public override string WriterName + { + get { return _writerName; } + } - public override string CreatorName - { - get { return _creatorName; } - } + public override string CreatorName + { + get { return _creatorName; } + } - public override int WriterId - { - get { return _writerId; } - } + public override int WriterId + { + get { return _writerId; } + } - public override int CreatorId - { - get { return _creatorId; } - } + public override int CreatorId + { + get { return _creatorId; } + } - public override string Path - { - get { return _path; } - } + public override string Path + { + get { return _path; } + } - public override DateTime CreateDate - { - get { return _createDate; } - } + public override DateTime CreateDate + { + get { return _createDate; } + } - public override DateTime UpdateDate - { - get { return _updateDate; } - } + public override DateTime UpdateDate + { + get { return _updateDate; } + } - public override Guid Version - { - get { return _version; } - } + public override Guid Version + { + get { return _version; } + } - public override int Level - { - get { return _level; } - } + public override int Level + { + get { return _level; } + } public override bool IsDraft { get { return false; } } - public override ICollection Properties - { - get { return _properties; } - } + public override ICollection Properties + { + get { return _properties; } + } - public override IEnumerable Children - { - get { return _getChildren.Value; } - } + public override IEnumerable Children + { + get { return _getChildren.Value; } + } - public override IPublishedProperty GetProperty(string alias) - { - return _getProperty(this, alias); - } + public override IPublishedProperty GetProperty(string alias) + { + return _getProperty(this, alias); + } public override PublishedContentType ContentType { @@ -850,38 +863,38 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache return property; } - private readonly List _keysAdded = new List(); - private int _id; - private Guid _key; - private int _templateId; - private int _sortOrder; - private string _name; - private string _urlName; - private string _documentTypeAlias; - private int _documentTypeId; - private string _writerName; - private string _creatorName; - private int _writerId; - private int _creatorId; - private string _path; - private DateTime _createDate; - private DateTime _updateDate; - private Guid _version; - private int _level; - private readonly ICollection _properties; - private readonly PublishedContentType _contentType; + private readonly List _keysAdded = new List(); + private int _id; + private Guid _key; + private int _templateId; + private int _sortOrder; + private string _name; + private string _urlName; + private string _documentTypeAlias; + private int _documentTypeId; + private string _writerName; + private string _creatorName; + private int _writerId; + private int _creatorId; + private string _path; + private DateTime _createDate; + private DateTime _updateDate; + private Guid _version; + private int _level; + private readonly ICollection _properties; + private readonly PublishedContentType _contentType; - private void ValidateAndSetProperty(IDictionary valueDictionary, Action setProperty, params string[] potentialKeys) - { - var key = potentialKeys.FirstOrDefault(x => valueDictionary.ContainsKey(x) && valueDictionary[x] != null); - if (key == null) - { - throw new FormatException("The valueDictionary is not formatted correctly and is missing any of the '" + string.Join(",", potentialKeys) + "' elements"); - } + private void ValidateAndSetProperty(IDictionary valueDictionary, Action setProperty, params string[] potentialKeys) + { + var key = potentialKeys.FirstOrDefault(x => valueDictionary.ContainsKey(x) && valueDictionary[x] != null); + if (key == null) + { + throw new FormatException("The valueDictionary is not formatted correctly and is missing any of the '" + string.Join(",", potentialKeys) + "' elements"); + } - setProperty(valueDictionary[key]); - _keysAdded.Add(key); - } + setProperty(valueDictionary[key]); + _keysAdded.Add(key); + } } // REFACTORING @@ -890,34 +903,34 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache // but NOT caching actual parent nor children and NOT even // the list of children ids - BUT caching the path - internal class CacheValues - { + internal class CacheValues + { public IDictionary Values { get; set; } public XPathNavigator XPath { get; set; } public bool FromExamine { get; set; } - } + } public const string PublishedMediaCacheKey = "MediaCacheMeh."; - private const int PublishedMediaCacheTimespanSeconds = 4 * 60; // 4 mins + private const int PublishedMediaCacheTimespanSeconds = 4 * 60; // 4 mins private static TimeSpan _publishedMediaCacheTimespan; - private static bool _publishedMediaCacheEnabled; + private static bool _publishedMediaCacheEnabled; - private static void InitializeCacheConfig() - { - var value = ConfigurationManager.AppSettings["Umbraco.PublishedMediaCache.Seconds"]; - int seconds; - if (int.TryParse(value, out seconds) == false) - seconds = PublishedMediaCacheTimespanSeconds; - if (seconds > 0) - { - _publishedMediaCacheEnabled = true; - _publishedMediaCacheTimespan = TimeSpan.FromSeconds(seconds); - } - else - { - _publishedMediaCacheEnabled = false; - } - } + private static void InitializeCacheConfig() + { + var value = ConfigurationManager.AppSettings["Umbraco.PublishedMediaCache.Seconds"]; + int seconds; + if (int.TryParse(value, out seconds) == false) + seconds = PublishedMediaCacheTimespanSeconds; + if (seconds > 0) + { + _publishedMediaCacheEnabled = true; + _publishedMediaCacheTimespan = TimeSpan.FromSeconds(seconds); + } + else + { + _publishedMediaCacheEnabled = false; + } + } internal IPublishedContent CreateFromCacheValues(CacheValues cacheValues) { @@ -932,20 +945,20 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache return content.CreateModel(); } - private static CacheValues GetCacheValues(int id, Func func) - { - if (_publishedMediaCacheEnabled == false) - return func(id); + private static CacheValues GetCacheValues(int id, Func func) + { + if (_publishedMediaCacheEnabled == false) + return func(id); - var cache = ApplicationContext.Current.ApplicationCache.RuntimeCache; - var key = PublishedMediaCacheKey + id; - return (CacheValues) cache.GetCacheItem(key, () => func(id), _publishedMediaCacheTimespan); - } - - internal static void ClearCache(int id) - { var cache = ApplicationContext.Current.ApplicationCache.RuntimeCache; - var sid = id.ToString(); + var key = PublishedMediaCacheKey + id; + return (CacheValues)cache.GetCacheItem(key, () => func(id), _publishedMediaCacheTimespan); + } + + internal static void ClearCache(int id) + { + var cache = ApplicationContext.Current.ApplicationCache.RuntimeCache; + var sid = id.ToString(); var key = PublishedMediaCacheKey + sid; // we do clear a lot of things... but the cache refresher is somewhat @@ -960,16 +973,16 @@ namespace Umbraco.Web.PublishedCache.XmlPublishedCache cache.ClearCacheItem(key); // clear all children - in case we moved and their path has changed - var fid = "/" + sid + "/"; + var fid = "/" + sid + "/"; cache.ClearCacheObjectTypes((k, v) => GetValuesValue(v.Values, "path", "__Path").Contains(fid)); } - private static string GetValuesValue(IDictionary d, params string[] keys) - { - string value = null; - var ignored = keys.Any(x => d.TryGetValue(x, out value)); + private static string GetValuesValue(IDictionary d, params string[] keys) + { + string value = null; + var ignored = keys.Any(x => d.TryGetValue(x, out value)); return value ?? ""; - } + } } }