Files
Umbraco-CMS/tests/Umbraco.Tests/LegacyXmlPublishedCache/PublishedMediaCache.cs

699 lines
31 KiB
C#
Raw Normal View History

using System;
using System.Collections.Generic;
using System.Configuration;
using System.IO;
using System.Linq;
using System.Threading;
using System.Xml.XPath;
using Examine;
using Examine.Search;
using Lucene.Net.Store;
using Microsoft.Extensions.Logging;
using Umbraco.Cms.Core;
using Umbraco.Cms.Core.Cache;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.Models.PublishedContent;
using Umbraco.Cms.Core.PublishedCache;
using Umbraco.Cms.Core.Services;
using Umbraco.Cms.Core.Web;
using Umbraco.Cms.Core.Xml;
2021-02-12 10:57:50 +01:00
using Umbraco.Cms.Infrastructure.Examine;
using Umbraco.Extensions;
using Umbraco.Web.Composing;
using Constants = Umbraco.Cms.Core.Constants;
namespace Umbraco.Tests.LegacyXmlPublishedCache
{
/// <summary>
/// An IPublishedMediaStore that first checks for the media in Examine, and then reverts to the database
/// </summary>
/// <remarks>
/// NOTE: In the future if we want to properly cache all media this class can be extended or replaced when these classes/interfaces are exposed publicly.
/// </remarks>
internal class PublishedMediaCache : PublishedCacheBase, IPublishedMediaCache
{
private readonly IMediaService _mediaService;
private readonly IUserService _userService;
// by default these are null unless specified by the ctor dedicated to tests
// when they are null the cache derives them from the ExamineManager, see
// method GetExamineManagerSafe().
//
private readonly ISearcher _searchProvider;
private readonly XmlStore _xmlStore;
private readonly PublishedContentTypeCache _contentTypeCache;
private readonly IEntityXmlSerializer _entitySerializer;
private readonly IUmbracoContextAccessor _umbracoContextAccessor;
private readonly IVariationContextAccessor _variationContextAccessor;
private readonly IExamineManager _examineManager = new ExamineManager();
// must be specified by the ctor
2019-01-18 08:14:08 +01:00
private readonly IAppCache _appCache;
public PublishedMediaCache(XmlStore xmlStore, IMediaService mediaService, IUserService userService,
IAppCache appCache, PublishedContentTypeCache contentTypeCache, IEntityXmlSerializer entitySerializer,
IUmbracoContextAccessor umbracoContextAccessor, IVariationContextAccessor variationContextAccessor)
: base(false)
{
_mediaService = mediaService ?? throw new ArgumentNullException(nameof(mediaService));
_userService = userService ?? throw new ArgumentNullException(nameof(userService));
2019-01-18 08:14:08 +01:00
_appCache = appCache;
_xmlStore = xmlStore;
_contentTypeCache = contentTypeCache;
_entitySerializer = entitySerializer;
_umbracoContextAccessor = umbracoContextAccessor;
_variationContextAccessor = variationContextAccessor;
}
/// <summary>
/// Generally used for unit testing to use an explicit examine searcher
/// </summary>
/// <param name="mediaService"></param>
/// <param name="userService"></param>
/// <param name="searchProvider"></param>
2019-01-18 08:14:08 +01:00
/// <param name="appCache"></param>
/// <param name="contentTypeCache"></param>
/// <param name="entitySerializer"></param>
2019-02-15 22:45:30 +01:00
internal PublishedMediaCache(IMediaService mediaService, IUserService userService, ISearcher searchProvider, IAppCache appCache, PublishedContentTypeCache contentTypeCache, IEntityXmlSerializer entitySerializer, IUmbracoContextAccessor umbracoContextAccessor)
: base(false)
{
_mediaService = mediaService ?? throw new ArgumentNullException(nameof(mediaService));
_userService = userService ?? throw new ArgumentNullException(nameof(userService));
_searchProvider = searchProvider ?? throw new ArgumentNullException(nameof(searchProvider));
2019-01-18 08:14:08 +01:00
_appCache = appCache;
_contentTypeCache = contentTypeCache;
_entitySerializer = entitySerializer;
2019-02-15 22:45:30 +01:00
_umbracoContextAccessor = umbracoContextAccessor;
}
static PublishedMediaCache()
{
InitializeCacheConfig();
}
public override IPublishedContent GetById(bool preview, int nodeId)
{
return GetUmbracoMedia(nodeId);
}
public override IPublishedContent GetById(bool preview, Guid nodeId)
{
throw new NotImplementedException();
}
2019-04-12 16:05:43 +02:00
public override IPublishedContent GetById(bool preview, Udi nodeId)
=> throw new NotSupportedException();
public override bool HasById(bool preview, int contentId)
{
return GetUmbracoMedia(contentId) != null;
}
2019-06-07 11:15:58 +02:00
public override IEnumerable<IPublishedContent> GetAtRoot(bool preview, string culture = null)
{
var searchProvider = GetSearchProviderSafe();
if (searchProvider != null)
{
try
{
// first check in Examine for the cache values
// +(+parentID:-1) +__IndexType:media
var criteria = searchProvider.CreateQuery("media");
var filter = criteria.ParentId(-1).Not().Field(UmbracoExamineFieldNames.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard());
var result = filter.Execute();
if (result != null)
return result.Select(x => CreateFromCacheValues(ConvertFromSearchResult(x)));
}
catch (Exception ex)
{
if (ex is FileNotFoundException)
{
//Currently examine is throwing FileNotFound exceptions when we have a load balanced filestore and a node is published in umbraco
//See this thread: http://examine.cdodeplex.com/discussions/264341
//Catch the exception here for the time being, and just fallback to GetMedia
// TODO: Need to fix examine in LB scenarios!
Current.Logger.LogError(ex, "Could not load data from Examine index for media");
}
Examine 2.0 integration (#10241) * Init commit for examine 2.0 work, most old umb examine tests working, probably a lot that doesn't * Gets Umbraco Examine tests passing and makes some sense out of them, fixes some underlying issues. * Large refactor, remove TaskHelper, rename Notifications to be consistent, Gets all examine/lucene indexes building and startup ordered in the correct way, removes old files, creates new IUmbracoIndexingHandler for abstracting out all index operations for umbraco data, abstracts out IIndexRebuilder, Fixes Stack overflow with LiveModelsProvider and loading assemblies, ports some changes from v8 for startup handling with cold boots, refactors out LastSyncedFileManager * fix up issues with rebuilding and management dashboard. * removes old files, removes NetworkHelper, fixes LastSyncedFileManager implementation to ensure the machine name is used, fix up logging with cold boot state. * Makes MainDom safer to use and makes PublishedSnapshotService lazily register with MainDom * lazily acquire application id (fix unit tests) * Fixes resource casing and missing test file * Ensures caches when requiring internal services for PublishedSnapshotService, UseNuCache is a separate call, shouldn't be buried in AddWebComponents, was also causing issues in integration tests since nucache was being used for the Id2Key service. * For UmbracoTestServerTestBase enable nucache services * Fixing tests * Fix another test * Fixes tests, use TestHostingEnvironment, make Tests.Common use net5, remove old Lucene.Net.Contrib ref. * Fixes up some review notes * Fixes issue with doubly registering PublishedSnapshotService meanig there could be 2x instances of it * Checks for parseexception when executing the query * Use application root instead of duplicating functionality. * Added Examine project to netcore only solution file * Fixed casing issue with LazyLoad, that is not lowercase. * uses cancellationToken instead of bool flag, fixes always reading lastId from the LastSyncedFileManager, fixes RecurringHostedServiceBase so that there isn't an overlapping thread for the same task type * Fix tests * remove legacy test project from solution file * Fix test Co-authored-by: Bjarke Berg <mail@bergmania.dk>
2021-05-18 18:31:38 +10:00
else if (ex is ObjectDisposedException)
{
//If the app domain is shutting down and the site is under heavy load the index reader will be closed and it really cannot
//be re-opened since the app domain is shutting down. In this case we have no option but to try to load the data from the db.
Current.Logger.LogError(ex, "Could not load data from Examine index for media, the app domain is most likely in a shutdown state");
}
else throw;
}
}
//something went wrong, fetch from the db
var rootMedia = _mediaService.GetRootMedia();
return rootMedia.Select(m => GetUmbracoMedia(m.Id));
}
public override IPublishedContent GetSingleByXPath(bool preview, string xpath, XPathVariable[] vars)
{
throw new NotImplementedException("PublishedMediaCache does not support XPath.");
//var navigator = CreateNavigator(preview);
//var iterator = navigator.Select(xpath, vars);
//return GetSingleByXPath(iterator);
}
public override IPublishedContent GetSingleByXPath(bool preview, XPathExpression xpath, XPathVariable[] vars)
{
throw new NotImplementedException("PublishedMediaCache does not support XPath.");
//var navigator = CreateNavigator(preview);
//var iterator = navigator.Select(xpath, vars);
//return GetSingleByXPath(iterator);
}
private IPublishedContent GetSingleByXPath(XPathNodeIterator iterator)
{
throw new NotImplementedException("PublishedMediaCache does not support XPath.");
//if (iterator.MoveNext() == false) return null;
//var idAttr = iterator.Current.GetAttribute("id", "");
//int id;
//return int.TryParse(idAttr, out id) ? GetUmbracoMedia(id) : null;
}
public override IEnumerable<IPublishedContent> GetByXPath(bool preview, string xpath, XPathVariable[] vars)
{
throw new NotImplementedException("PublishedMediaCache does not support XPath.");
//var navigator = CreateNavigator(preview);
//var iterator = navigator.Select(xpath, vars);
//return GetByXPath(iterator);
}
public override IEnumerable<IPublishedContent> GetByXPath(bool preview, XPathExpression xpath, XPathVariable[] vars)
{
throw new NotImplementedException("PublishedMediaCache does not support XPath.");
//var navigator = CreateNavigator(preview);
//var iterator = navigator.Select(xpath, vars);
//return GetByXPath(iterator);
}
private IEnumerable<IPublishedContent> GetByXPath(XPathNodeIterator iterator)
{
while (iterator.MoveNext())
{
var idAttr = iterator.Current.GetAttribute("id", "");
int id;
if (int.TryParse(idAttr, out id))
yield return GetUmbracoMedia(id);
}
}
public override XPathNavigator CreateNavigator(bool preview)
{
throw new NotImplementedException("PublishedMediaCache does not support XPath.");
//var doc = _xmlStore.GetMediaXml();
//return doc.CreateNavigator();
}
public override XPathNavigator CreateNodeNavigator(int id, bool preview)
{
// preview is ignored for media cache
// this code is mostly used when replacing old media.ToXml() code, and that code
// stored the XML attached to the media itself - so for some time in memory - so
// unless we implement some sort of cache here, we're probably degrading perfs.
XPathNavigator navigator = null;
var node = _xmlStore.GetMediaXmlNode(id);
if (node != null)
{
navigator = node.CreateNavigator();
}
return navigator;
}
public override bool HasContent(bool preview) { throw new NotImplementedException(); }
private ISearcher GetSearchProviderSafe()
{
if (_searchProvider != null)
return _searchProvider;
try
{
return _examineManager.TryGetIndex(Constants.UmbracoIndexes.InternalIndexName, out var index) ? index.GetSearcher() : null;
}
catch (FileNotFoundException)
{
//Currently examine is throwing FileNotFound exceptions when we have a load balanced filestore and a node is published in umbraco
//See this thread: http://examine.cdodeplex.com/discussions/264341
//Catch the exception here for the time being, and just fallback to GetMedia
// TODO: Need to fix examine in LB scenarios!
}
catch (NullReferenceException)
{
//This will occur when the search provider cannot be initialized. In newer examine versions the initialization is lazy and therefore
// the manager will return the singleton without throwing initialization errors, however if examine isn't configured correctly a null
// reference error will occur because the examine settings are null.
}
Examine 2.0 integration (#10241) * Init commit for examine 2.0 work, most old umb examine tests working, probably a lot that doesn't * Gets Umbraco Examine tests passing and makes some sense out of them, fixes some underlying issues. * Large refactor, remove TaskHelper, rename Notifications to be consistent, Gets all examine/lucene indexes building and startup ordered in the correct way, removes old files, creates new IUmbracoIndexingHandler for abstracting out all index operations for umbraco data, abstracts out IIndexRebuilder, Fixes Stack overflow with LiveModelsProvider and loading assemblies, ports some changes from v8 for startup handling with cold boots, refactors out LastSyncedFileManager * fix up issues with rebuilding and management dashboard. * removes old files, removes NetworkHelper, fixes LastSyncedFileManager implementation to ensure the machine name is used, fix up logging with cold boot state. * Makes MainDom safer to use and makes PublishedSnapshotService lazily register with MainDom * lazily acquire application id (fix unit tests) * Fixes resource casing and missing test file * Ensures caches when requiring internal services for PublishedSnapshotService, UseNuCache is a separate call, shouldn't be buried in AddWebComponents, was also causing issues in integration tests since nucache was being used for the Id2Key service. * For UmbracoTestServerTestBase enable nucache services * Fixing tests * Fix another test * Fixes tests, use TestHostingEnvironment, make Tests.Common use net5, remove old Lucene.Net.Contrib ref. * Fixes up some review notes * Fixes issue with doubly registering PublishedSnapshotService meanig there could be 2x instances of it * Checks for parseexception when executing the query * Use application root instead of duplicating functionality. * Added Examine project to netcore only solution file * Fixed casing issue with LazyLoad, that is not lowercase. * uses cancellationToken instead of bool flag, fixes always reading lastId from the LastSyncedFileManager, fixes RecurringHostedServiceBase so that there isn't an overlapping thread for the same task type * Fix tests * remove legacy test project from solution file * Fix test Co-authored-by: Bjarke Berg <mail@bergmania.dk>
2021-05-18 18:31:38 +10:00
catch (ObjectDisposedException)
{
//If the app domain is shutting down and the site is under heavy load the index reader will be closed and it really cannot
//be re-opened since the app domain is shutting down. In this case we have no option but to try to load the data from the db.
}
return null;
}
private IPublishedContent GetUmbracoMedia(int id)
{
// this recreates an IPublishedContent and model each time
// it is called, but at least it should NOT hit the database
// nor Lucene each time, relying on the memory cache instead
if (id <= 0) return null; // fail fast
var cacheValues = GetCacheValues(id, GetUmbracoMediaCacheValues);
return cacheValues == null ? null : CreateFromCacheValues(cacheValues);
}
private CacheValues GetUmbracoMediaCacheValues(int id)
{
var searchProvider = GetSearchProviderSafe();
if (searchProvider != null)
{
try
{
// first check in Examine as this is WAY faster
//
// the filter will create a query like this:
// +(+__NodeId:3113 -__Path:-1,-21,*) +__IndexType:media
//
// note that since the use of the wildcard, it automatically escapes it in Lucene.
var criteria = searchProvider.CreateQuery("media");
var filter = criteria.Id(id.ToInvariantString()).Not().Field(UmbracoExamineFieldNames.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard());
var result = filter.Execute().FirstOrDefault();
if (result != null) return ConvertFromSearchResult(result);
}
catch (Exception ex)
{
if (ex is FileNotFoundException)
{
//Currently examine is throwing FileNotFound exceptions when we have a load balanced filestore and a node is published in umbraco
//See this thread: http://examine.cdodeplex.com/discussions/264341
//Catch the exception here for the time being, and just fallback to GetMedia
// TODO: Need to fix examine in LB scenarios!
Current.Logger.LogError(ex, "Could not load data from Examine index for media");
}
Examine 2.0 integration (#10241) * Init commit for examine 2.0 work, most old umb examine tests working, probably a lot that doesn't * Gets Umbraco Examine tests passing and makes some sense out of them, fixes some underlying issues. * Large refactor, remove TaskHelper, rename Notifications to be consistent, Gets all examine/lucene indexes building and startup ordered in the correct way, removes old files, creates new IUmbracoIndexingHandler for abstracting out all index operations for umbraco data, abstracts out IIndexRebuilder, Fixes Stack overflow with LiveModelsProvider and loading assemblies, ports some changes from v8 for startup handling with cold boots, refactors out LastSyncedFileManager * fix up issues with rebuilding and management dashboard. * removes old files, removes NetworkHelper, fixes LastSyncedFileManager implementation to ensure the machine name is used, fix up logging with cold boot state. * Makes MainDom safer to use and makes PublishedSnapshotService lazily register with MainDom * lazily acquire application id (fix unit tests) * Fixes resource casing and missing test file * Ensures caches when requiring internal services for PublishedSnapshotService, UseNuCache is a separate call, shouldn't be buried in AddWebComponents, was also causing issues in integration tests since nucache was being used for the Id2Key service. * For UmbracoTestServerTestBase enable nucache services * Fixing tests * Fix another test * Fixes tests, use TestHostingEnvironment, make Tests.Common use net5, remove old Lucene.Net.Contrib ref. * Fixes up some review notes * Fixes issue with doubly registering PublishedSnapshotService meanig there could be 2x instances of it * Checks for parseexception when executing the query * Use application root instead of duplicating functionality. * Added Examine project to netcore only solution file * Fixed casing issue with LazyLoad, that is not lowercase. * uses cancellationToken instead of bool flag, fixes always reading lastId from the LastSyncedFileManager, fixes RecurringHostedServiceBase so that there isn't an overlapping thread for the same task type * Fix tests * remove legacy test project from solution file * Fix test Co-authored-by: Bjarke Berg <mail@bergmania.dk>
2021-05-18 18:31:38 +10:00
else if (ex is ObjectDisposedException)
{
//If the app domain is shutting down and the site is under heavy load the index reader will be closed and it really cannot
//be re-opened since the app domain is shutting down. In this case we have no option but to try to load the data from the db.
Current.Logger.LogError(ex, "Could not load data from Examine index for media, the app domain is most likely in a shutdown state");
}
else throw;
}
}
// don't log a warning here, as it can flood the log in case of eg a media picker referencing a media
// that has been deleted, hence is not in the Examine index anymore (for a good reason). try to get
// the media from the service, first
var media = _mediaService.GetById(id);
if (media == null || media.Trashed) return null; // not found, ok
// so, the media was not found in Examine's index *yet* it exists, which probably indicates that
// the index is corrupted. Or not up-to-date. Log a warning, but only once, and only if seeing the
// error more that a number of times.
var miss = Interlocked.CompareExchange(ref _examineIndexMiss, 0, 0); // volatile read
if (miss < ExamineIndexMissMax && Interlocked.Increment(ref _examineIndexMiss) == ExamineIndexMissMax)
Current.Logger.LogWarning("Failed ({ExamineIndexMissMax} times) to retrieve medias from Examine index and had to load"
+ " them from DB. This may indicate that the Examine index is corrupted.", ExamineIndexMissMax);
return ConvertFromIMedia(media);
}
private const int ExamineIndexMissMax = 10;
private int _examineIndexMiss;
internal CacheValues ConvertFromXPathNodeIterator(XPathNodeIterator media, int id)
{
if (media?.Current != null)
{
return media.Current.Name.InvariantEquals("error")
? null
: ConvertFromXPathNavigator(media.Current);
}
Current.Logger.LogWarning("Could not retrieve media {MediaId} from Examine index or from legacy library.GetMedia method", id);
return null;
}
internal CacheValues ConvertFromSearchResult(ISearchResult searchResult)
{
// note: fixing fields in 7.x, removed by Shan for 8.0
return new CacheValues
{
2018-11-26 17:20:15 +11:00
Values = searchResult.Values,
FromExamine = true
};
}
internal CacheValues ConvertFromXPathNavigator(XPathNavigator xpath, bool forceNav = false)
{
if (xpath == null) throw new ArgumentNullException(nameof(xpath));
var values = new Dictionary<string, string> { { "nodeName", xpath.GetAttribute("nodeName", "") } };
values["nodeTypeAlias"] = xpath.Name;
var result = xpath.SelectChildren(XPathNodeType.Element);
//add the attributes e.g. id, parentId etc
if (result.Current != null && result.Current.HasAttributes)
{
if (result.Current.MoveToFirstAttribute())
{
//checking for duplicate keys because of the 'nodeTypeAlias' might already be added above.
if (values.ContainsKey(result.Current.Name) == false)
{
values[result.Current.Name] = result.Current.Value;
}
while (result.Current.MoveToNextAttribute())
{
if (values.ContainsKey(result.Current.Name) == false)
{
values[result.Current.Name] = result.Current.Value;
}
}
result.Current.MoveToParent();
}
}
// because, migration
if (values.ContainsKey("key") == false)
values["key"] = Guid.Empty.ToString();
//add the user props
while (result.MoveNext())
{
if (result.Current != null && result.Current.HasAttributes == false)
{
var value = result.Current.Value;
if (string.IsNullOrEmpty(value))
{
if (result.Current.HasAttributes || result.Current.SelectChildren(XPathNodeType.Element).Count > 0)
{
value = result.Current.OuterXml;
}
}
values[result.Current.Name] = value;
}
}
return new CacheValues
{
Values = values,
XPath = forceNav ? xpath : null // outside of tests we do NOT want to cache the navigator!
};
}
internal CacheValues ConvertFromIMedia(IMedia media)
{
var values = new Dictionary<string, string>();
var creator = _userService.GetProfileById(media.CreatorId);
var creatorName = creator == null ? "" : creator.Name;
values["id"] = media.Id.ToString();
values["key"] = media.Key.ToString();
values["parentID"] = media.ParentId.ToString();
values["level"] = media.Level.ToString();
values["creatorID"] = media.CreatorId.ToString();
values["creatorName"] = creatorName;
values["writerID"] = media.CreatorId.ToString();
values["writerName"] = creatorName;
values["template"] = "0";
values["urlName"] = "";
values["sortOrder"] = media.SortOrder.ToString();
values["createDate"] = media.CreateDate.ToString("yyyy-MM-dd HH:mm:ss");
values["updateDate"] = media.UpdateDate.ToString("yyyy-MM-dd HH:mm:ss");
values["nodeName"] = media.Name;
values["path"] = media.Path;
values["nodeType"] = media.ContentType.Id.ToString();
values["nodeTypeAlias"] = media.ContentType.Alias;
// add the user props
foreach (var prop in media.Properties)
values[prop.Alias] = prop.GetValue()?.ToString();
return new CacheValues
{
Values = values
};
}
/// <summary>
/// We will need to first check if the document was loaded by Examine, if so we'll need to check if this property exists
/// in the results, if it does not, then we'll have to revert to looking up in the db.
/// </summary>
/// <param name="dd"> </param>
/// <param name="alias"></param>
/// <returns></returns>
private IPublishedProperty GetProperty(DictionaryPublishedContent dd, string alias)
{
//lets check if the alias does not exist on the document.
//NOTE: Examine will not index empty values and we do not output empty XML Elements to the cache - either of these situations
// would mean that the property is missing from the collection whether we are getting the value from Examine or from the library media cache.
if (dd.Properties.All(x => x.Alias.InvariantEquals(alias) == false))
{
return null;
}
if (dd.LoadedFromExamine)
{
//We are going to check for a special field however, that is because in some cases we store a 'Raw'
//value in the index such as for xml/html.
var rawValue = dd.Properties.FirstOrDefault(x => x.Alias.InvariantEquals(UmbracoExamineFieldNames.RawFieldPrefix + alias));
return rawValue
?? dd.Properties.FirstOrDefault(x => x.Alias.InvariantEquals(alias));
}
//if its not loaded from examine, then just return the property
return dd.Properties.FirstOrDefault(x => x.Alias.InvariantEquals(alias));
}
/// <summary>
/// A Helper methods to return the children for media whether it is based on examine or xml
/// </summary>
/// <param name="parentId"></param>
/// <param name="xpath"></param>
/// <returns></returns>
private IEnumerable<IPublishedContent> GetChildrenMedia(int parentId, XPathNavigator xpath = null)
{
2018-07-09 17:29:50 +02:00
// if there *is* a navigator, directly look it up
if (xpath != null)
{
return ToIPublishedContent(parentId, xpath);
}
2018-07-09 17:29:50 +02:00
// otherwise, try examine first, then re-look it up
var searchProvider = GetSearchProviderSafe();
if (searchProvider != null)
{
try
{
//first check in Examine as this is WAY faster
var criteria = searchProvider.CreateQuery("media");
var filter = criteria.ParentId(parentId).Not().Field(UmbracoExamineFieldNames.IndexPathFieldName, "-1,-21,".MultipleCharacterWildcard())
.OrderBy(new SortableField("sortOrder", SortType.Int));
//the above filter will create a query like this, NOTE: That since the use of the wildcard, it automatically escapes it in Lucene.
//+(+parentId:3113 -__Path:-1,-21,*) +__IndexType:media
// sort with the Sort field (updated for 8.0)
var results = filter.Execute();
if (results.Any())
{
// var medias = results.Select(ConvertFromSearchResult);
var medias = results.Select(x =>
{
int nid;
if (int.TryParse(x["__NodeId"], out nid) == false && int.TryParse(x["NodeId"], out nid) == false)
throw new Exception("Failed to extract NodeId from search result.");
var cacheValues = GetCacheValues(nid, id => ConvertFromSearchResult(x));
return CreateFromCacheValues(cacheValues);
});
return medias;
}
//if there's no result then return null. Previously we defaulted back to library.GetMedia below
//but this will always get called for when we are getting descendants since many items won't have
//children and then we are hitting the database again!
//So instead we're going to rely on Examine to have the correct results like it should.
return Enumerable.Empty<IPublishedContent>();
}
catch (FileNotFoundException)
{
//Currently examine is throwing FileNotFound exceptions when we have a load balanced filestore and a node is published in umbraco
//See this thread: http://examine.cdodeplex.com/discussions/264341
//Catch the exception here for the time being, and just fallback to GetMedia
}
}
2018-07-09 17:29:50 +02:00
// falling back to get media
// was library.GetMedia which had its own cache, but MediaService *also* caches
// so, library.GetMedia is gone and now we directly work with MediaService
// (code below copied from what library was doing)
var media = _mediaService.GetById(parentId);
if (media == null)
{
return Enumerable.Empty<IPublishedContent>();
}
var serialized = _entitySerializer.Serialize(media, true);
var mediaIterator = serialized.CreateNavigator().Select("/");
2018-07-09 17:29:50 +02:00
return mediaIterator.Current == null
? Enumerable.Empty<IPublishedContent>()
: ToIPublishedContent(parentId, mediaIterator.Current);
}
internal IEnumerable<IPublishedContent> ToIPublishedContent(int parentId, XPathNavigator xpath)
{
var mediaList = new List<IPublishedContent>();
// this is so bad, really
var item = xpath.Select("//*[@id='" + parentId + "']");
if (item.Current == null)
return Enumerable.Empty<IPublishedContent>();
var items = item.Current.SelectChildren(XPathNodeType.Element);
// and this does not work, because... meh
//var q = "//* [@id='" + parentId + "']/* [@id]";
//var items = xpath.Select(q);
foreach (XPathNavigator itemm in items)
{
int id;
if (int.TryParse(itemm.GetAttribute("id", ""), out id) == false)
2019-02-06 15:05:56 +01:00
continue; // uh?
var captured = itemm;
var cacheValues = GetCacheValues(id, idd => ConvertFromXPathNavigator(captured));
mediaList.Add(CreateFromCacheValues(cacheValues));
}
return mediaList;
}
internal void Resync()
{
// clear recursive properties cached by XmlPublishedContent.GetProperty
// assume that nothing else is going to cache IPublishedProperty items (else would need to do ByKeySearch)
// NOTE all properties cleared when clearing the content cache (see content cache)
2019-01-18 08:14:08 +01:00
//_appCache.ClearCacheObjectTypes<IPublishedProperty>();
//_appCache.ClearCacheByKeySearch("XmlPublishedCache.PublishedMediaCache:RecursiveProperty-");
}
#region Content types
Block Editor List (#8273) * add style to create-option in itempicker + removing overflow hidden * style adjustment * clean up of html * correct sentence to use the number 7 * correct overlays, so they can use size * numberrange prevalue editor * add confirmRemove overlay * correcting primary btn colors * move confirmMessage below content of overlay. * min max validation for numberrange * remove comment * improved actions for block list * use file tree for view picker * style adjustment to border of creator item in item-picker * vertical align * clean up + validation limit range * rename ElementTypes to Blocks * implement block list editor * renaming * use Chrome Headless for unit tests * test for blockEditorService * safer code * block list editor * rename view to overlayView * block editor work * Revert "rename view to overlayView" This reverts commit 5b910c178a4f193d190367c4f1da3402aa8c4d0e. * block editor implementation * sync models * block list editor copy paste feature * order var declarations * remove unused paste function * block list editor better naming * simpler label generation * clean up * compile config for test mode * Chrome Debug for VS code * promise test working * space change * another two tests * more tests for block list editor * provide key on blockModel for angularJS performance optimization * syncronization from infinite editing layers * use an isolated scope * more tests * fix C# test * remove unused block watcher component * clean css * only show on hover or focus for block-actions * clean up and prepare for implementing settings * remove console * Add ability to render block list editor using Grid style rendering extension * Enable Block List Editor settings editing * Add Stacked Content to Block List migration * Block Editor: Clean-up, refactoring, one step closer being ready for Content-Apps * changes naming to Submit, to avoid misunderstanding. * use a common variable on the block model for isOpen, to be able to make Actions show when open. * NotSupported property editor, to be used when an editor is not supported in the given context. * remove unused controller * Hide group header if only one group is presented * rename notsupport property editor css class * smaller header for property group * hide description if no description is presented * css adjustments * Inline create button styling: Better spacing, darker blue color for Focus Outline, moving the plus icon to mouse position for better visual appearance. * css correction * Add references for picked items * Revert commit 45e892f3505059674779c6e1a43084a367c2862f - Changes api to GetData * Use the .Data propertry as opposed to GetData in this PartialView * Fix block list test failures * Just parsing layout as model for partial views. * minor adjustments * Remove DB migrations so that they can be reviewed as a block * Add migrations for new block editor * Update default rendering partial view * Add error handling to default template * Handle color picker data in stacked content * BlockList PreValue Editor opens configurations as overlay * translation for prevalue editor property group headlines * blockcard corrections * block list prevalue corrections * revert agressive clean up * Block Picker * MaxPropertyWidth PreValue + Implementation * Incorporate latest block list editor changes, update migration for changed configuration * Change declared converter type * Handle invalid data type references * Remove code duplicated from PR #7957 * use ElementModel for the ContentModel of an ElementType. So we can use ElementTypeModel for the ModelDefinition aka. the Type. * do still show itempicker for BlockConfiguration even though there is no ElementTypes to pick. This enables the option to create a new ElementType to be used. * use the right wrapper, for correct spacing * parse item * correct fallback for label * removed unused callback * paste feature for block-picker * localize block-picker tabs * Slightly change for shadow on block-picker item hover * Localization of BlockEditor Settings Tab * localizationService * only filter when more than 8 items available * Add multiple blocks if hold down CTRL or SuperKey * adds notes * ability to add a scoped stylesheet for block view * make scoped block draggable + style adjustments * provide index for custom view * rename contentModel to data + rename layoutModel to layout * clean up * more localization * openSettings option for block editor * minor changes for a better developer experience * paste mistake corrected * only manipulate content-content-app if its present * make small overlays slightly bigger * moved block list entry editor into block list editor folder * limit labelinterpretator to only runs ones pr. edit. and lets make sure to have a label thought we dont have any properties. * fixed inline views gulp watcher * changed vm to a better controller instance name * make watch for views work again. * able to re run watch * make js up to date * fix white background of image-picker * media-picker container class * loading indication * adjust unit tests to latest interface * getting started on JS Docs * converting code to use contentTypeKey instead of contentTypeAlias, still missing a few TODOs. * revert change * add todo * use Guid for Key * use key * Updates the caching layer to handle GUID keys for content types while preserving backwards compat, fixes unit tests, removes the strongly typed lists for the block editor value since it's unecessary * Reverts the nested content changes, fixes up the GetEmptyByKey * Returns ContentTypeKey from the server, updates js to use this everywhere and fix all js tests. * Allows key in SimpleContentType * correct for the new spelling * appended this. since the method is a non-static class method. * only add background-image if value isnt null * simplifyed limits validation * clean up * no need to execute a digest. * define the full model for new configurations * removed setDataFromBlockModel and general clean up and added documentation * default size should be medium * use retriveValuesFrom method to transfer data. * ability to disable an navigation item * createElementTypeAndCallback working again for settings. * still have the ability to retrive a scaffold model by alias, since we are still using alias in clipboard service. * disable clipboard tab if no available paste options * ups, should stay as alias * disable clipboard when empty * use property alias for content add button * use a grey that can be seen on top of grey and on top of white. * some accessibility improvements * rename entry to block * appended ' and added space in Element Type * use background color for hover to corospond with active state * make nested content unsupported * Moving BlockEditorModelObject into its own file for Documentation purpose. Same goes for renaming BlockModel to BlockObject. and a lot of documentation written. * fix links in js docs * added a blocklistentryeditor for unsupported blocks * ability to retrive the $scope from UmbVariantContentEditors, needed for Block Editor to get a scope existing across variants and splitview. * Appending the block objects to layout, to share it across variants and in split-view. * removed trailing comma * Unsupported block * Dont copy unsupported blocks * use grey color for unsupported block * text correction * we dont have this fallback anymore * sort properties * Text change * css correction * use active to determine if an inline editor is expanded. To enable the inline editor to be expanded on creation. * using udi instead of key. * bringing the runtime block key back * live editing ability * live editing for settings data * call formSubmit before property editor are removed from DOM. Both for overlay-editing and inline-editing. Fire an postFormSubmitting event after formSubmitting event to let editors know that data could have been updated. * make sure settings object exists * only set active to false if it was false before opening the editor. * update test with new scope parameter * move destroy responsibility to blockObject * rename onDestroy to destroy * added some JS-Docs * correction jsDocs * Update ElementType Resource to not use hardcoded URL but to use the Umbraco.Sys.ServerVariables.umbracoUrls instead * Remove partially completed ConvertToElement migration, fixed in issue 7939 instead. * Remove external property editor migration * corrected naming of umbBlockListScopedBlock and umbBlockListBlock * correct ngdoc type * removed vscode specific configuration of karma * Finished Todo, gets name of documentType if copying all entities in an infinite editor * changed comment from TODO to something that explains the state. * stop tracking build output files. * rename files to match file name conventions * this should not happen. * remove dublicated code * rename requestCopyBlock to copyBlock * make sure images does not repeat. * scale thumbnail for block showcase * renamed blockcard.component to umb-block-card and moved it. * removed inline style * correct style location * corrected filepath * corrected file path * keep elementTypes up to date through the EventService. * mark Umbraco.BlockList as unsupported inside Nested Content * correct js docs name * remove comment * remove comment * remove unused controller * rename inline method name * corrected spelling mistake * remove not very used vars * make binding one-way * split in multiple lines * corrected default rendering * removing documentation that is relevant for developers of new block editors. this documentation will be transfered to Our documentation. * added danish translation * corrected blog to blok * Remove invalid using statement * use native forEach Co-authored-by: Niels Lyngsø <nsl@umbraco.dk> Co-authored-by: Benjamin Carleski <benjamin@proworks.com> Co-authored-by: Warren Buckley <warren@umbraco.com> Co-authored-by: Niels Lyngsø <nsl@umbraco.com> Co-authored-by: Niels Lyngsø <niels.lyngso@gmail.com> Co-authored-by: Claus <claus@claus.nu>
2020-06-30 19:52:42 +10:00
public override IPublishedContentType GetContentType(int id) => _contentTypeCache.Get(PublishedItemType.Media, id);
Block Editor List (#8273) * add style to create-option in itempicker + removing overflow hidden * style adjustment * clean up of html * correct sentence to use the number 7 * correct overlays, so they can use size * numberrange prevalue editor * add confirmRemove overlay * correcting primary btn colors * move confirmMessage below content of overlay. * min max validation for numberrange * remove comment * improved actions for block list * use file tree for view picker * style adjustment to border of creator item in item-picker * vertical align * clean up + validation limit range * rename ElementTypes to Blocks * implement block list editor * renaming * use Chrome Headless for unit tests * test for blockEditorService * safer code * block list editor * rename view to overlayView * block editor work * Revert "rename view to overlayView" This reverts commit 5b910c178a4f193d190367c4f1da3402aa8c4d0e. * block editor implementation * sync models * block list editor copy paste feature * order var declarations * remove unused paste function * block list editor better naming * simpler label generation * clean up * compile config for test mode * Chrome Debug for VS code * promise test working * space change * another two tests * more tests for block list editor * provide key on blockModel for angularJS performance optimization * syncronization from infinite editing layers * use an isolated scope * more tests * fix C# test * remove unused block watcher component * clean css * only show on hover or focus for block-actions * clean up and prepare for implementing settings * remove console * Add ability to render block list editor using Grid style rendering extension * Enable Block List Editor settings editing * Add Stacked Content to Block List migration * Block Editor: Clean-up, refactoring, one step closer being ready for Content-Apps * changes naming to Submit, to avoid misunderstanding. * use a common variable on the block model for isOpen, to be able to make Actions show when open. * NotSupported property editor, to be used when an editor is not supported in the given context. * remove unused controller * Hide group header if only one group is presented * rename notsupport property editor css class * smaller header for property group * hide description if no description is presented * css adjustments * Inline create button styling: Better spacing, darker blue color for Focus Outline, moving the plus icon to mouse position for better visual appearance. * css correction * Add references for picked items * Revert commit 45e892f3505059674779c6e1a43084a367c2862f - Changes api to GetData * Use the .Data propertry as opposed to GetData in this PartialView * Fix block list test failures * Just parsing layout as model for partial views. * minor adjustments * Remove DB migrations so that they can be reviewed as a block * Add migrations for new block editor * Update default rendering partial view * Add error handling to default template * Handle color picker data in stacked content * BlockList PreValue Editor opens configurations as overlay * translation for prevalue editor property group headlines * blockcard corrections * block list prevalue corrections * revert agressive clean up * Block Picker * MaxPropertyWidth PreValue + Implementation * Incorporate latest block list editor changes, update migration for changed configuration * Change declared converter type * Handle invalid data type references * Remove code duplicated from PR #7957 * use ElementModel for the ContentModel of an ElementType. So we can use ElementTypeModel for the ModelDefinition aka. the Type. * do still show itempicker for BlockConfiguration even though there is no ElementTypes to pick. This enables the option to create a new ElementType to be used. * use the right wrapper, for correct spacing * parse item * correct fallback for label * removed unused callback * paste feature for block-picker * localize block-picker tabs * Slightly change for shadow on block-picker item hover * Localization of BlockEditor Settings Tab * localizationService * only filter when more than 8 items available * Add multiple blocks if hold down CTRL or SuperKey * adds notes * ability to add a scoped stylesheet for block view * make scoped block draggable + style adjustments * provide index for custom view * rename contentModel to data + rename layoutModel to layout * clean up * more localization * openSettings option for block editor * minor changes for a better developer experience * paste mistake corrected * only manipulate content-content-app if its present * make small overlays slightly bigger * moved block list entry editor into block list editor folder * limit labelinterpretator to only runs ones pr. edit. and lets make sure to have a label thought we dont have any properties. * fixed inline views gulp watcher * changed vm to a better controller instance name * make watch for views work again. * able to re run watch * make js up to date * fix white background of image-picker * media-picker container class * loading indication * adjust unit tests to latest interface * getting started on JS Docs * converting code to use contentTypeKey instead of contentTypeAlias, still missing a few TODOs. * revert change * add todo * use Guid for Key * use key * Updates the caching layer to handle GUID keys for content types while preserving backwards compat, fixes unit tests, removes the strongly typed lists for the block editor value since it's unecessary * Reverts the nested content changes, fixes up the GetEmptyByKey * Returns ContentTypeKey from the server, updates js to use this everywhere and fix all js tests. * Allows key in SimpleContentType * correct for the new spelling * appended this. since the method is a non-static class method. * only add background-image if value isnt null * simplifyed limits validation * clean up * no need to execute a digest. * define the full model for new configurations * removed setDataFromBlockModel and general clean up and added documentation * default size should be medium * use retriveValuesFrom method to transfer data. * ability to disable an navigation item * createElementTypeAndCallback working again for settings. * still have the ability to retrive a scaffold model by alias, since we are still using alias in clipboard service. * disable clipboard tab if no available paste options * ups, should stay as alias * disable clipboard when empty * use property alias for content add button * use a grey that can be seen on top of grey and on top of white. * some accessibility improvements * rename entry to block * appended ' and added space in Element Type * use background color for hover to corospond with active state * make nested content unsupported * Moving BlockEditorModelObject into its own file for Documentation purpose. Same goes for renaming BlockModel to BlockObject. and a lot of documentation written. * fix links in js docs * added a blocklistentryeditor for unsupported blocks * ability to retrive the $scope from UmbVariantContentEditors, needed for Block Editor to get a scope existing across variants and splitview. * Appending the block objects to layout, to share it across variants and in split-view. * removed trailing comma * Unsupported block * Dont copy unsupported blocks * use grey color for unsupported block * text correction * we dont have this fallback anymore * sort properties * Text change * css correction * use active to determine if an inline editor is expanded. To enable the inline editor to be expanded on creation. * using udi instead of key. * bringing the runtime block key back * live editing ability * live editing for settings data * call formSubmit before property editor are removed from DOM. Both for overlay-editing and inline-editing. Fire an postFormSubmitting event after formSubmitting event to let editors know that data could have been updated. * make sure settings object exists * only set active to false if it was false before opening the editor. * update test with new scope parameter * move destroy responsibility to blockObject * rename onDestroy to destroy * added some JS-Docs * correction jsDocs * Update ElementType Resource to not use hardcoded URL but to use the Umbraco.Sys.ServerVariables.umbracoUrls instead * Remove partially completed ConvertToElement migration, fixed in issue 7939 instead. * Remove external property editor migration * corrected naming of umbBlockListScopedBlock and umbBlockListBlock * correct ngdoc type * removed vscode specific configuration of karma * Finished Todo, gets name of documentType if copying all entities in an infinite editor * changed comment from TODO to something that explains the state. * stop tracking build output files. * rename files to match file name conventions * this should not happen. * remove dublicated code * rename requestCopyBlock to copyBlock * make sure images does not repeat. * scale thumbnail for block showcase * renamed blockcard.component to umb-block-card and moved it. * removed inline style * correct style location * corrected filepath * corrected file path * keep elementTypes up to date through the EventService. * mark Umbraco.BlockList as unsupported inside Nested Content * correct js docs name * remove comment * remove comment * remove unused controller * rename inline method name * corrected spelling mistake * remove not very used vars * make binding one-way * split in multiple lines * corrected default rendering * removing documentation that is relevant for developers of new block editors. this documentation will be transfered to Our documentation. * added danish translation * corrected blog to blok * Remove invalid using statement * use native forEach Co-authored-by: Niels Lyngsø <nsl@umbraco.dk> Co-authored-by: Benjamin Carleski <benjamin@proworks.com> Co-authored-by: Warren Buckley <warren@umbraco.com> Co-authored-by: Niels Lyngsø <nsl@umbraco.com> Co-authored-by: Niels Lyngsø <niels.lyngso@gmail.com> Co-authored-by: Claus <claus@claus.nu>
2020-06-30 19:52:42 +10:00
public override IPublishedContentType GetContentType(string alias) => _contentTypeCache.Get(PublishedItemType.Media, alias);
public override IPublishedContentType GetContentType(Guid key) => _contentTypeCache.Get(PublishedItemType.Media, key);
2019-04-15 13:04:14 +02:00
public override IEnumerable<IPublishedContent> GetByContentType(IPublishedContentType contentType)
{
throw new NotSupportedException();
}
#endregion
// REFACTORING
// caching the basic atomic values - and the parent id
// but NOT caching actual parent nor children and NOT even
// the list of children ids - BUT caching the path
internal class CacheValues
{
public IReadOnlyDictionary<string, string> Values { get; set; }
public XPathNavigator XPath { get; set; }
public bool FromExamine { get; set; }
}
public const string PublishedMediaCacheKey = "MediaCacheMeh.";
private const int PublishedMediaCacheTimespanSeconds = 4 * 60; // 4 mins
private static TimeSpan _publishedMediaCacheTimespan;
private static bool _publishedMediaCacheEnabled;
private static void InitializeCacheConfig()
{
_publishedMediaCacheEnabled = true;
_publishedMediaCacheTimespan = TimeSpan.FromSeconds(PublishedMediaCacheTimespanSeconds);
}
internal IPublishedContent CreateFromCacheValues(CacheValues cacheValues)
{
var content = new DictionaryPublishedContent(
cacheValues.Values,
parentId => parentId < 0 ? null : GetUmbracoMedia(parentId),
GetChildrenMedia,
GetProperty,
2019-01-18 08:14:08 +01:00
_appCache,
_variationContextAccessor,
_contentTypeCache,
cacheValues.XPath, // though, outside of tests, that should be null
cacheValues.FromExamine
);
return content.CreateModel(Current.PublishedModelFactory);
}
private static CacheValues GetCacheValues(int id, Func<int, CacheValues> func)
{
if (_publishedMediaCacheEnabled == false)
return func(id);
2019-01-18 08:14:08 +01:00
var cache = Current.AppCaches.RuntimeCache;
var key = PublishedMediaCacheKey + id;
2019-01-17 11:01:23 +01:00
return (CacheValues)cache.Get(key, () => func(id), _publishedMediaCacheTimespan);
}
internal static void ClearCache(int id)
{
2019-01-18 08:14:08 +01:00
var cache = Current.AppCaches.RuntimeCache;
var sid = id.ToString();
var key = PublishedMediaCacheKey + sid;
// we do clear a lot of things... but the cache refresher is somewhat
// convoluted and it's hard to tell what to clear exactly ;-(
// clear the parent - NOT (why?)
//var exist = (CacheValues) cache.GetCacheItem(key);
//if (exist != null)
// cache.ClearCacheItem(PublishedMediaCacheKey + GetValuesValue(exist.Values, "parentID"));
// clear the item
2019-01-17 11:01:23 +01:00
cache.Clear(key);
// clear all children - in case we moved and their path has changed
var fid = "/" + sid + "/";
2019-01-17 11:01:23 +01:00
cache.ClearOfType<CacheValues>((k, v) =>
GetValuesValue(v.Values, "path", "__Path").Contains(fid));
}
private static string GetValuesValue(IReadOnlyDictionary<string, string> d, params string[] keys)
{
string value = null;
var ignored = keys.Any(x => d.TryGetValue(x, out value));
return value ?? "";
}
}
}