Renamed the project to Umbraco.PublishedCache.NuCache - and move non NuCache related stuff to abstractions and infrastructure.

src/Umbraco.PublishedCache.NuCache/DataSource/BTree.ContentDataSerializer.cs

@@ -0,0 +1,43 @@
+using System.IO;
+using CSharpTest.Net.Serialization;
+
+namespace Umbraco.Web.PublishedCache.NuCache.DataSource
+{
+    class ContentDataSerializer : ISerializer<ContentData>
+    {
+        private static readonly DictionaryOfPropertyDataSerializer PropertiesSerializer = new DictionaryOfPropertyDataSerializer();
+        private static readonly DictionaryOfCultureVariationSerializer CultureVariationsSerializer = new DictionaryOfCultureVariationSerializer();
+
+        public ContentData ReadFrom(Stream stream)
+        {
+            return new ContentData
+            {
+                Published = PrimitiveSerializer.Boolean.ReadFrom(stream),
+                Name = PrimitiveSerializer.String.ReadFrom(stream),
+                UrlSegment = PrimitiveSerializer.String.ReadFrom(stream),
+                VersionId = PrimitiveSerializer.Int32.ReadFrom(stream),
+                VersionDate = PrimitiveSerializer.DateTime.ReadFrom(stream),
+                WriterId = PrimitiveSerializer.Int32.ReadFrom(stream),
+                TemplateId = PrimitiveSerializer.Int32.ReadFrom(stream),
+                Properties = PropertiesSerializer.ReadFrom(stream),
+                CultureInfos = CultureVariationsSerializer.ReadFrom(stream)
+            };
+        }
+
+        public void WriteTo(ContentData value, Stream stream)
+        {
+            PrimitiveSerializer.Boolean.WriteTo(value.Published, stream);
+            PrimitiveSerializer.String.WriteTo(value.Name, stream);
+            PrimitiveSerializer.String.WriteTo(value.UrlSegment, stream);
+            PrimitiveSerializer.Int32.WriteTo(value.VersionId, stream);
+            PrimitiveSerializer.DateTime.WriteTo(value.VersionDate, stream);
+            PrimitiveSerializer.Int32.WriteTo(value.WriterId, stream);
+            if (value.TemplateId.HasValue)
+            {
+                PrimitiveSerializer.Int32.WriteTo(value.TemplateId.Value, stream);
+            }
+            PropertiesSerializer.WriteTo(value.Properties, stream);
+            CultureVariationsSerializer.WriteTo(value.CultureInfos, stream);
+        }
+    }
+}

src/Umbraco.PublishedCache.NuCache/DataSource/BTree.ContentNodeKitSerializer.cs

@@ -0,0 +1,57 @@
+using System.IO;
+using CSharpTest.Net.Serialization;
+
+namespace Umbraco.Web.PublishedCache.NuCache.DataSource
+{
+    internal class ContentNodeKitSerializer : ISerializer<ContentNodeKit>
+    {
+        static readonly ContentDataSerializer DataSerializer = new ContentDataSerializer();
+        //static readonly ListOfIntSerializer ChildContentIdsSerializer = new ListOfIntSerializer();
+
+        public ContentNodeKit ReadFrom(Stream stream)
+        {
+            var kit = new ContentNodeKit
+            {
+                Node = new ContentNode(
+                    PrimitiveSerializer.Int32.ReadFrom(stream), // id
+                    PrimitiveSerializer.Guid.ReadFrom(stream), // uid
+                    PrimitiveSerializer.Int32.ReadFrom(stream), // level
+                    PrimitiveSerializer.String.ReadFrom(stream), // path
+                    PrimitiveSerializer.Int32.ReadFrom(stream), // sort order
+                    PrimitiveSerializer.Int32.ReadFrom(stream), // parent id
+                    PrimitiveSerializer.DateTime.ReadFrom(stream), // date created
+                    PrimitiveSerializer.Int32.ReadFrom(stream) // creator id
+                ),
+                ContentTypeId = PrimitiveSerializer.Int32.ReadFrom(stream)
+            };
+            var hasDraft = PrimitiveSerializer.Boolean.ReadFrom(stream);
+            if (hasDraft)
+                kit.DraftData = DataSerializer.ReadFrom(stream);
+            var hasPublished = PrimitiveSerializer.Boolean.ReadFrom(stream);
+            if (hasPublished)
+                kit.PublishedData = DataSerializer.ReadFrom(stream);
+            return kit;
+        }
+
+        public void WriteTo(ContentNodeKit value, Stream stream)
+        {
+            PrimitiveSerializer.Int32.WriteTo(value.Node.Id, stream);
+            PrimitiveSerializer.Guid.WriteTo(value.Node.Uid, stream);
+            PrimitiveSerializer.Int32.WriteTo(value.Node.Level, stream);
+            PrimitiveSerializer.String.WriteTo(value.Node.Path, stream);
+            PrimitiveSerializer.Int32.WriteTo(value.Node.SortOrder, stream);
+            PrimitiveSerializer.Int32.WriteTo(value.Node.ParentContentId, stream);
+            PrimitiveSerializer.DateTime.WriteTo(value.Node.CreateDate, stream);
+            PrimitiveSerializer.Int32.WriteTo(value.Node.CreatorId, stream);
+            PrimitiveSerializer.Int32.WriteTo(value.ContentTypeId, stream);
+
+            PrimitiveSerializer.Boolean.WriteTo(value.DraftData != null, stream);
+            if (value.DraftData != null)
+                DataSerializer.WriteTo(value.DraftData, stream);
+
+            PrimitiveSerializer.Boolean.WriteTo(value.PublishedData != null, stream);
+            if (value.PublishedData != null)
+                DataSerializer.WriteTo(value.PublishedData, stream);
+        }
+    }
+}

src/Umbraco.PublishedCache.NuCache/DataSource/BTree.DictionaryOfCultureVariationSerializer.cs

@@ -0,0 +1,49 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using CSharpTest.Net.Serialization;
+using Umbraco.Core;
+
+namespace Umbraco.Web.PublishedCache.NuCache.DataSource
+{
+    internal class DictionaryOfCultureVariationSerializer : SerializerBase, ISerializer<IReadOnlyDictionary<string, CultureVariation>>
+    {
+        public IReadOnlyDictionary<string, CultureVariation> ReadFrom(Stream stream)
+        {
+            // read variations count
+            var pcount = PrimitiveSerializer.Int32.ReadFrom(stream);
+            if (pcount == 0) return Empty;
+
+            // read each variation
+            var dict = new Dictionary<string, CultureVariation>(StringComparer.InvariantCultureIgnoreCase);
+            for (var i = 0; i < pcount; i++)
+            {
+                var languageId = PrimitiveSerializer.String.ReadFrom(stream);
+                var cultureVariation = new CultureVariation { Name = ReadStringObject(stream), UrlSegment = ReadStringObject(stream), Date = ReadDateTime(stream) };
+                dict[languageId] = cultureVariation;
+            }
+            return dict;
+        }
+
+        private static readonly IReadOnlyDictionary<string, CultureVariation> Empty = new Dictionary<string, CultureVariation>();
+
+        public void WriteTo(IReadOnlyDictionary<string, CultureVariation> value, Stream stream)
+        {
+            var variations = value ?? Empty;
+
+            // write variations count
+            PrimitiveSerializer.Int32.WriteTo(variations.Count, stream);
+
+            // write each variation
+            foreach (var (culture, variation) in variations)
+            {
+                // TODO: it's weird we're dealing with cultures here, and languageId in properties
+
+                PrimitiveSerializer.String.WriteTo(culture, stream); // should never be null
+                WriteObject(variation.Name, stream); // write an object in case it's null (though... should not happen)
+                WriteObject(variation.UrlSegment, stream); // write an object in case it's null (though... should not happen)
+                PrimitiveSerializer.DateTime.WriteTo(variation.Date, stream);
+            }
+        }
+    }
+}

src/Umbraco.PublishedCache.NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs

@@ -0,0 +1,78 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using CSharpTest.Net.Serialization;
+using Umbraco.Core;
+
+namespace Umbraco.Web.PublishedCache.NuCache.DataSource
+{
+    internal class DictionaryOfPropertyDataSerializer : SerializerBase, ISerializer<IDictionary<string, PropertyData[]>>
+    {
+        public IDictionary<string, PropertyData[]> ReadFrom(Stream stream)
+        {
+            var dict = new Dictionary<string, PropertyData[]>(StringComparer.InvariantCultureIgnoreCase);
+
+            // read properties count
+            var pcount = PrimitiveSerializer.Int32.ReadFrom(stream);
+
+            // read each property
+            for (var i = 0; i < pcount; i++)
+            {
+                // read property alias
+                var key = PrimitiveSerializer.String.ReadFrom(stream);
+
+                // read values count
+                var vcount = PrimitiveSerializer.Int32.ReadFrom(stream);
+
+                // create pdata and add to the dictionary
+                var pdatas = new List<PropertyData>();
+
+                // for each value, read and add to pdata
+                for (var j = 0; j < vcount; j++)
+                {
+                    var pdata = new PropertyData();
+                    pdatas.Add(pdata);
+
+                    // everything that can be null is read/written as object
+                    //  even though - culture and segment should never be null here, as 'null' represents
+                    //  the 'current' value, and string.Empty should be used to represent the invariant or
+                    //  neutral values - PropertyData throws when getting nulls, so falling back to
+                    //  string.Empty here - what else?
+                    pdata.Culture = ReadStringObject(stream) ?? string.Empty;
+                    pdata.Segment = ReadStringObject(stream) ?? string.Empty;
+                    pdata.Value = ReadObject(stream);
+                }
+
+                dict[key] = pdatas.ToArray();
+            }
+            return dict;
+        }
+
+        public void WriteTo(IDictionary<string, PropertyData[]> value, Stream stream)
+        {
+            // write properties count
+            PrimitiveSerializer.Int32.WriteTo(value.Count, stream);
+
+            // write each property
+            foreach (var (alias, values) in value)
+            {
+                // write alias
+                PrimitiveSerializer.String.WriteTo(alias, stream);
+
+                // write values count
+                PrimitiveSerializer.Int32.WriteTo(values.Length, stream);
+
+                // write each value
+                foreach (var pdata in values)
+                {
+                    // everything that can be null is read/written as object
+                    //  even though - culture and segment should never be null here,
+                    //  see note in ReadFrom() method above
+                    WriteObject(pdata.Culture ?? string.Empty, stream);
+                    WriteObject(pdata.Segment ?? string.Empty, stream);
+                    WriteObject(pdata.Value, stream);
+                }
+            }
+        }
+    }
+}

src/Umbraco.PublishedCache.NuCache/DataSource/BTree.cs

@@ -0,0 +1,78 @@
+using System.Configuration;
+using CSharpTest.Net.Collections;
+using CSharpTest.Net.Serialization;
+
+namespace Umbraco.Web.PublishedCache.NuCache.DataSource
+{
+    internal class BTree
+    {
+        public static BPlusTree<int, ContentNodeKit> GetTree(string filepath, bool exists)
+        {
+            var keySerializer = new PrimitiveSerializer();
+            var valueSerializer = new ContentNodeKitSerializer();
+            var options = new BPlusTree<int, ContentNodeKit>.OptionsV2(keySerializer, valueSerializer)
+            {
+                CreateFile = exists ? CreatePolicy.IfNeeded : CreatePolicy.Always,
+                FileName = filepath,
+
+                // read or write but do *not* keep in memory
+                CachePolicy = CachePolicy.None,
+
+                // default is 4096, min 2^9 = 512, max 2^16 = 64K
+                FileBlockSize = GetBlockSize(),
+
+                // other options?
+            };
+
+            var tree = new BPlusTree<int, ContentNodeKit>(options);
+
+            // anything?
+            //btree.
+
+            return tree;
+        }
+
+        private static int GetBlockSize()
+        {
+            var blockSize = 4096;
+
+            var appSetting = ConfigurationManager.AppSettings["Umbraco.Web.PublishedCache.NuCache.BTree.BlockSize"];
+            if (appSetting == null)
+                return blockSize;
+
+            if (!int.TryParse(appSetting, out blockSize))
+                throw new ConfigurationErrorsException($"Invalid block size value \"{appSetting}\": not a number.");
+
+            var bit = 0;
+            for (var i = blockSize; i != 1; i >>= 1)
+                bit++;
+            if (1 << bit != blockSize)
+                throw new ConfigurationErrorsException($"Invalid block size value \"{blockSize}\": must be a power of two.");
+            if (blockSize < 512 || blockSize > 65536)
+                throw new ConfigurationErrorsException($"Invalid block size value \"{blockSize}\": must be >= 512 and <= 65536.");
+
+            return blockSize;
+        }
+
+        /*
+        class ListOfIntSerializer : ISerializer<List<int>>
+        {
+            public List<int> ReadFrom(Stream stream)
+            {
+                var list = new List<int>();
+                var count = PrimitiveSerializer.Int32.ReadFrom(stream);
+                for (var i = 0; i < count; i++)
+                    list.Add(PrimitiveSerializer.Int32.ReadFrom(stream));
+                return list;
+            }
+
+            public void WriteTo(List<int> value, Stream stream)
+            {
+                PrimitiveSerializer.Int32.WriteTo(value.Count, stream);
+                foreach (var item in value)
+                    PrimitiveSerializer.Int32.WriteTo(item, stream);
+            }
+        }
+        */
+    }
+}

src/Umbraco.PublishedCache.NuCache/DataSource/ContentData.cs

@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+
+namespace Umbraco.Web.PublishedCache.NuCache.DataSource
+{
+    // represents everything that is specific to edited or published version
+    public class ContentData
+    {
+        public string Name { get; set; }
+        public string UrlSegment { get; set; }
+        public int VersionId { get; set; }
+        public DateTime VersionDate { get; set; }
+        public int WriterId { get; set; }
+        public int? TemplateId { get; set; }
+        public bool Published { get; set; }
+
+        public IDictionary<string, PropertyData[]> Properties { get; set; }
+
+        /// <summary>
+        /// The collection of language Id to name for the content item
+        /// </summary>
+        public IReadOnlyDictionary<string, CultureVariation> CultureInfos { get; set; }
+    }
+}

src/Umbraco.PublishedCache.NuCache/DataSource/ContentNestedData.cs

@@ -0,0 +1,23 @@
+using Newtonsoft.Json;
+using System.Collections.Generic;
+using Umbraco.Core.Serialization;
+
+namespace Umbraco.Web.PublishedCache.NuCache.DataSource
+{
+    /// <summary>
+    /// The content item 1:M data that is serialized to JSON
+    /// </summary>
+    internal class ContentNestedData
+    {
+        [JsonProperty("properties")]
+        [JsonConverter(typeof(CaseInsensitiveDictionaryConverter<PropertyData[]>))]
+        public Dictionary<string, PropertyData[]> PropertyData { get; set; }
+
+        [JsonProperty("cultureData")]
+        [JsonConverter(typeof(CaseInsensitiveDictionaryConverter<CultureVariation>))]
+        public Dictionary<string, CultureVariation> CultureData { get; set; }
+
+        [JsonProperty("urlSegment")]
+        public string UrlSegment { get; set; }
+    }
+}

src/Umbraco.PublishedCache.NuCache/DataSource/ContentSourceDto.cs

@@ -0,0 +1,39 @@
+using System;
+
+namespace Umbraco.Web.PublishedCache.NuCache.DataSource
+{
+    // read-only dto
+    internal class ContentSourceDto
+    {
+        public int Id { get; set; }
+        public Guid Uid { get; set; }
+        public int ContentTypeId { get; set; }
+
+        public int Level { get; set; }
+        public string Path { get; set; }
+        public int SortOrder { get; set; }
+        public int ParentId { get; set; }
+
+        public bool Published { get; set; }
+        public bool Edited { get; set; }
+
+        public DateTime CreateDate { get; set; }
+        public int CreatorId { get; set; }
+
+        // edited data
+        public int VersionId { get; set; }
+        public string EditName { get; set; }
+        public DateTime EditVersionDate { get; set; }
+        public int EditWriterId { get; set; }
+        public int EditTemplateId { get; set; }
+        public string EditData { get; set; }
+
+        // published data
+        public int PublishedVersionId { get; set; }
+        public string PubName { get; set; }
+        public DateTime PubVersionDate { get; set; }
+        public int PubWriterId { get; set; }
+        public int PubTemplateId { get; set; }
+        public string PubData { get; set; }
+    }
+}

src/Umbraco.PublishedCache.NuCache/DataSource/CultureVariation.cs

@@ -0,0 +1,23 @@
+using System;
+using Newtonsoft.Json;
+
+namespace Umbraco.Web.PublishedCache.NuCache.DataSource
+{
+    /// <summary>
+    /// Represents the culture variation information on a content item
+    /// </summary>
+    public class CultureVariation
+    {
+        [JsonProperty("name")]
+        public string Name { get; set; }
+
+        [JsonProperty("urlSegment")]
+        public string UrlSegment { get; set; }
+
+        [JsonProperty("date")]
+        public DateTime Date { get; set; }
+
+        [JsonProperty("isDraft")]
+        public bool IsDraft { get; set; }
+    }
+}

src/Umbraco.PublishedCache.NuCache/DataSource/DatabaseDataSource.cs

@@ -0,0 +1,308 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using Newtonsoft.Json;
+using NPoco;
+using Umbraco.Core;
+using Umbraco.Core.Logging;
+using Umbraco.Core.Persistence;
+using Umbraco.Core.Persistence.Dtos;
+using Umbraco.Core.Scoping;
+using Umbraco.Core.Serialization;
+using static Umbraco.Core.Persistence.SqlExtensionsStatics;
+
+namespace Umbraco.Web.PublishedCache.NuCache.DataSource
+{
+    // TODO: use SqlTemplate for these queries else it's going to be horribly slow!
+
+    // provides efficient database access for NuCache
+    internal class DatabaseDataSource : IDataSource
+    {
+        private readonly ILogger _logger;
+
+        public DatabaseDataSource(ILogger logger)
+        {
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        }
+
+        // we want arrays, we want them all loaded, not an enumerable
+
+        private Sql<ISqlContext> ContentSourcesSelect(IScope scope, Func<Sql<ISqlContext>, Sql<ISqlContext>> joins = null)
+        {
+            var sql = scope.SqlContext.Sql()
+
+                .Select<NodeDto>(x => Alias(x.NodeId, "Id"), x => Alias(x.UniqueId, "Uid"),
+                    x => Alias(x.Level, "Level"), x => Alias(x.Path, "Path"), x => Alias(x.SortOrder, "SortOrder"), x => Alias(x.ParentId, "ParentId"),
+                    x => Alias(x.CreateDate, "CreateDate"), x => Alias(x.UserId, "CreatorId"))
+                .AndSelect<ContentDto>(x => Alias(x.ContentTypeId, "ContentTypeId"))
+                .AndSelect<DocumentDto>(x => Alias(x.Published, "Published"), x => Alias(x.Edited, "Edited"))
+
+                .AndSelect<ContentVersionDto>(x => Alias(x.Id, "VersionId"), x => Alias(x.Text, "EditName"), x => Alias(x.VersionDate, "EditVersionDate"), x => Alias(x.UserId, "EditWriterId"))
+                .AndSelect<DocumentVersionDto>(x => Alias(x.TemplateId, "EditTemplateId"))
+
+                .AndSelect<ContentVersionDto>("pcver", x => Alias(x.Id, "PublishedVersionId"), x => Alias(x.Text, "PubName"), x => Alias(x.VersionDate, "PubVersionDate"), x => Alias(x.UserId, "PubWriterId"))
+                .AndSelect<DocumentVersionDto>("pdver", x => Alias(x.TemplateId, "PubTemplateId"))
+
+                .AndSelect<ContentNuDto>("nuEdit", x => Alias(x.Data, "EditData"))
+                .AndSelect<ContentNuDto>("nuPub", x => Alias(x.Data, "PubData"))
+
+                .From<NodeDto>();
+
+            if (joins != null)
+                sql = joins(sql);
+
+            sql = sql
+                .InnerJoin<ContentDto>().On<NodeDto, ContentDto>((left, right) => left.NodeId == right.NodeId)
+                .InnerJoin<DocumentDto>().On<NodeDto, DocumentDto>((left, right) => left.NodeId == right.NodeId)
+
+                .InnerJoin<ContentVersionDto>().On<NodeDto, ContentVersionDto>((left, right) => left.NodeId == right.NodeId && right.Current)
+                .InnerJoin<DocumentVersionDto>().On<ContentVersionDto, DocumentVersionDto>((left, right) => left.Id == right.Id)
+
+                .LeftJoin<ContentVersionDto>(j =>
+                    j.InnerJoin<DocumentVersionDto>("pdver").On<ContentVersionDto, DocumentVersionDto>((left, right) => left.Id == right.Id && right.Published, "pcver", "pdver"), "pcver")
+                .On<NodeDto, ContentVersionDto>((left, right) => left.NodeId == right.NodeId, aliasRight: "pcver")
+
+                .LeftJoin<ContentNuDto>("nuEdit").On<NodeDto, ContentNuDto>((left, right) => left.NodeId == right.NodeId && !right.Published, aliasRight: "nuEdit")
+                .LeftJoin<ContentNuDto>("nuPub").On<NodeDto, ContentNuDto>((left, right) => left.NodeId == right.NodeId && right.Published, aliasRight: "nuPub");
+
+            return sql;
+        }
+
+        public ContentNodeKit GetContentSource(IScope scope, int id)
+        {
+            var sql = ContentSourcesSelect(scope)
+                .Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Document && x.NodeId == id && !x.Trashed)
+                .OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
+
+            var dto = scope.Database.Fetch<ContentSourceDto>(sql).FirstOrDefault();
+            return dto == null ? new ContentNodeKit() : CreateContentNodeKit(dto);
+        }
+
+        public IEnumerable<ContentNodeKit> GetAllContentSources(IScope scope)
+        {
+            var sql = ContentSourcesSelect(scope)
+                .Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed)
+                .OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
+
+            return scope.Database.Query<ContentSourceDto>(sql).Select(CreateContentNodeKit);
+        }
+
+        public IEnumerable<ContentNodeKit> GetBranchContentSources(IScope scope, int id)
+        {
+            var syntax = scope.SqlContext.SqlSyntax;
+            var sql = ContentSourcesSelect(scope, s => s
+
+                    .InnerJoin<NodeDto>("x").On<NodeDto, NodeDto>((left, right) => left.NodeId == right.NodeId || SqlText<bool>(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x"))
+
+                .Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed)
+                .Where<NodeDto>(x => x.NodeId == id, "x")
+                .OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
+
+            return scope.Database.Query<ContentSourceDto>(sql).Select(CreateContentNodeKit);
+        }
+
+        public IEnumerable<ContentNodeKit> GetTypeContentSources(IScope scope, IEnumerable<int> ids)
+        {
+            if (!ids.Any()) return Enumerable.Empty<ContentNodeKit>();
+
+            var sql = ContentSourcesSelect(scope)
+                .Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed)
+                .WhereIn<ContentDto>(x => x.ContentTypeId, ids)
+                .OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
+
+            return scope.Database.Query<ContentSourceDto>(sql).Select(CreateContentNodeKit);
+        }
+
+        private Sql<ISqlContext> MediaSourcesSelect(IScope scope, Func<Sql<ISqlContext>, Sql<ISqlContext>> joins = null)
+        {
+            var sql = scope.SqlContext.Sql()
+
+                .Select<NodeDto>(x => Alias(x.NodeId, "Id"), x => Alias(x.UniqueId, "Uid"),
+                    x => Alias(x.Level, "Level"), x => Alias(x.Path, "Path"), x => Alias(x.SortOrder, "SortOrder"), x => Alias(x.ParentId, "ParentId"),
+                    x => Alias(x.CreateDate, "CreateDate"), x => Alias(x.UserId, "CreatorId"))
+                .AndSelect<ContentDto>(x => Alias(x.ContentTypeId, "ContentTypeId"))
+
+                .AndSelect<ContentVersionDto>(x => Alias(x.Id, "VersionId"), x => Alias(x.Text, "EditName"), x => Alias(x.VersionDate, "EditVersionDate"), x => Alias(x.UserId, "EditWriterId"))
+
+                .AndSelect<ContentNuDto>("nuEdit", x => Alias(x.Data, "EditData"))
+
+                .From<NodeDto>();
+
+            if (joins != null)
+                sql = joins(sql);
+
+            sql = sql
+                .InnerJoin<ContentDto>().On<NodeDto, ContentDto>((left, right) => left.NodeId == right.NodeId)
+
+                .InnerJoin<ContentVersionDto>().On<NodeDto, ContentVersionDto>((left, right) => left.NodeId == right.NodeId && right.Current)
+
+                .LeftJoin<ContentNuDto>("nuEdit").On<NodeDto, ContentNuDto>((left, right) => left.NodeId == right.NodeId && !right.Published, aliasRight: "nuEdit");
+
+            return sql;
+        }
+
+        public ContentNodeKit GetMediaSource(IScope scope, int id)
+        {
+            var sql = MediaSourcesSelect(scope)
+                .Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Media && x.NodeId == id && !x.Trashed)
+                .OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
+
+            var dto = scope.Database.Fetch<ContentSourceDto>(sql).FirstOrDefault();
+            return dto == null ? new ContentNodeKit() : CreateMediaNodeKit(dto);
+        }
+
+        public IEnumerable<ContentNodeKit> GetAllMediaSources(IScope scope)
+        {
+            var sql = MediaSourcesSelect(scope)
+                .Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed)
+                .OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
+
+            return scope.Database.Query<ContentSourceDto>(sql).Select(CreateMediaNodeKit);
+        }
+
+        public IEnumerable<ContentNodeKit> GetBranchMediaSources(IScope scope, int id)
+        {
+            var syntax = scope.SqlContext.SqlSyntax;
+            var sql = MediaSourcesSelect(scope, s => s
+
+                    .InnerJoin<NodeDto>("x").On<NodeDto, NodeDto>((left, right) => left.NodeId == right.NodeId || SqlText<bool>(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x"))
+
+                .Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed)
+                .Where<NodeDto>(x => x.NodeId == id, "x")
+                .OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
+
+            return scope.Database.Query<ContentSourceDto>(sql).Select(CreateMediaNodeKit);
+        }
+
+        public IEnumerable<ContentNodeKit> GetTypeMediaSources(IScope scope, IEnumerable<int> ids)
+        {
+            if (!ids.Any()) return Enumerable.Empty<ContentNodeKit>();
+
+            var sql = MediaSourcesSelect(scope)
+                    .Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed)
+                    .WhereIn<ContentDto>(x => x.ContentTypeId, ids)
+                    .OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
+
+            return scope.Database.Query<ContentSourceDto>(sql).Select(CreateMediaNodeKit);
+        }
+
+        private ContentNodeKit CreateContentNodeKit(ContentSourceDto dto)
+        {
+            ContentData d = null;
+            ContentData p = null;
+
+            if (dto.Edited)
+            {
+                if (dto.EditData == null)
+                {
+                    if (Debugger.IsAttached)
+                        throw new Exception("Missing cmsContentNu edited content for node " + dto.Id + ", consider rebuilding.");
+                    _logger.Warn<DatabaseDataSource>("Missing cmsContentNu edited content for node {NodeId}, consider rebuilding.", dto.Id);
+                }
+                else
+                {
+                    var nested = DeserializeNestedData(dto.EditData);
+
+                    d = new ContentData
+                    {
+                        Name = dto.EditName,
+                        Published = false,
+                        TemplateId = dto.EditTemplateId,
+                        VersionId = dto.VersionId,
+                        VersionDate = dto.EditVersionDate,
+                        WriterId = dto.EditWriterId,
+                        Properties = nested.PropertyData,
+                        CultureInfos = nested.CultureData,
+                        UrlSegment = nested.UrlSegment
+                    };
+                }
+            }
+
+            if (dto.Published)
+            {
+                if (dto.PubData == null)
+                {
+                    if (Debugger.IsAttached)
+                        throw new Exception("Missing cmsContentNu published content for node " + dto.Id + ", consider rebuilding.");
+                    _logger.Warn<DatabaseDataSource>("Missing cmsContentNu published content for node {NodeId}, consider rebuilding.", dto.Id);
+                }
+                else
+                {
+                    var nested = DeserializeNestedData(dto.PubData);
+
+                    p = new ContentData
+                    {
+                        Name = dto.PubName,
+                        UrlSegment = nested.UrlSegment,
+                        Published = true,
+                        TemplateId = dto.PubTemplateId,
+                        VersionId = dto.VersionId,
+                        VersionDate = dto.PubVersionDate,
+                        WriterId = dto.PubWriterId,
+                        Properties = nested.PropertyData,
+                        CultureInfos = nested.CultureData
+                    };
+                }
+            }
+
+            var n = new ContentNode(dto.Id, dto.Uid,
+                dto.Level, dto.Path, dto.SortOrder, dto.ParentId, dto.CreateDate, dto.CreatorId);
+
+            var s = new ContentNodeKit
+            {
+                Node = n,
+                ContentTypeId = dto.ContentTypeId,
+                DraftData = d,
+                PublishedData = p
+            };
+
+            return s;
+        }
+
+        private static ContentNodeKit CreateMediaNodeKit(ContentSourceDto dto)
+        {
+            if (dto.EditData == null)
+                throw new Exception("No data for media " + dto.Id);
+
+            var nested = DeserializeNestedData(dto.EditData);
+
+            var p = new ContentData
+            {
+                Name = dto.EditName,
+                Published = true,
+                TemplateId = -1,
+                VersionId = dto.VersionId,
+                VersionDate = dto.EditVersionDate,
+                WriterId = dto.CreatorId, // what-else?
+                Properties = nested.PropertyData,
+                CultureInfos = nested.CultureData
+            };
+
+            var n = new ContentNode(dto.Id, dto.Uid,
+                dto.Level, dto.Path, dto.SortOrder, dto.ParentId, dto.CreateDate, dto.CreatorId);
+
+            var s = new ContentNodeKit
+            {
+                Node = n,
+                ContentTypeId = dto.ContentTypeId,
+                PublishedData = p
+            };
+
+            return s;
+        }
+
+        private static ContentNestedData DeserializeNestedData(string data)
+        {
+            // by default JsonConvert will deserialize our numeric values as Int64
+            // which is bad, because they were Int32 in the database - take care
+
+            var settings = new JsonSerializerSettings
+            {
+                Converters = new List<JsonConverter> { new ForceInt32Converter() }
+            };
+
+            return JsonConvert.DeserializeObject<ContentNestedData>(data, settings);
+        }
+    }
+}

src/Umbraco.PublishedCache.NuCache/DataSource/IDataSource.cs

@@ -0,0 +1,77 @@
+using System.Collections.Generic;
+using Umbraco.Core.Scoping;
+
+namespace Umbraco.Web.PublishedCache.NuCache.DataSource
+{
+    /// <summary>
+    /// Defines a data source for NuCache.
+    /// </summary>
+    internal interface IDataSource
+    {
+        //TODO: For these required sort orders, would sorting on Path 'just work'?
+
+        ContentNodeKit GetContentSource(IScope scope, int id);
+
+        /// <summary>
+        /// Returns all content ordered by level + sortOrder
+        /// </summary>
+        /// <param name="scope"></param>
+        /// <returns></returns>
+        /// <remarks>
+        /// MUST be ordered by level + parentId + sortOrder!
+        /// </remarks>
+        IEnumerable<ContentNodeKit> GetAllContentSources(IScope scope);
+
+        /// <summary>
+        /// Returns branch for content ordered by level + sortOrder
+        /// </summary>
+        /// <param name="scope"></param>
+        /// <returns></returns>
+        /// <remarks>
+        /// MUST be ordered by level + parentId + sortOrder!
+        /// </remarks>
+        IEnumerable<ContentNodeKit> GetBranchContentSources(IScope scope, int id);
+
+        /// <summary>
+        /// Returns content by Ids ordered by level + sortOrder
+        /// </summary>
+        /// <param name="scope"></param>
+        /// <returns></returns>
+        /// <remarks>
+        /// MUST be ordered by level + parentId + sortOrder!
+        /// </remarks>
+        IEnumerable<ContentNodeKit> GetTypeContentSources(IScope scope, IEnumerable<int> ids);
+
+        ContentNodeKit GetMediaSource(IScope scope, int id);
+
+        /// <summary>
+        /// Returns all media ordered by level + sortOrder
+        /// </summary>
+        /// <param name="scope"></param>
+        /// <returns></returns>
+        /// <remarks>
+        /// MUST be ordered by level + parentId + sortOrder!
+        /// </remarks>
+        IEnumerable<ContentNodeKit> GetAllMediaSources(IScope scope);
+
+        /// <summary>
+        /// Returns branch for media ordered by level + sortOrder
+        /// </summary>
+        /// <param name="scope"></param>
+        /// <returns></returns>
+        /// <remarks>
+        /// MUST be ordered by level + parentId + sortOrder!
+        /// </remarks>
+        IEnumerable<ContentNodeKit> GetBranchMediaSources(IScope scope, int id); // must order by level, sortOrder
+
+        /// <summary>
+        /// Returns media by Ids ordered by level + sortOrder
+        /// </summary>
+        /// <param name="scope"></param>
+        /// <returns></returns>
+        /// <remarks>
+        /// MUST be ordered by level + parentId + sortOrder!
+        /// </remarks>
+        IEnumerable<ContentNodeKit> GetTypeMediaSources(IScope scope, IEnumerable<int> ids);
+    }
+}

src/Umbraco.PublishedCache.NuCache/DataSource/PropertyData.cs

@@ -0,0 +1,28 @@
+using System;
+using Newtonsoft.Json;
+
+namespace Umbraco.Web.PublishedCache.NuCache.DataSource
+{
+    public class PropertyData
+    {
+        private string _culture;
+        private string _segment;
+
+        [JsonProperty("culture")]
+        public string Culture
+        {
+            get => _culture;
+            set => _culture = value ?? throw new ArgumentNullException(nameof(value)); // TODO: or fallback to string.Empty? CANNOT be null
+        }
+
+        [JsonProperty("seg")]
+        public string Segment
+        {
+            get => _segment;
+            set => _segment = value ?? throw new ArgumentNullException(nameof(value)); // TODO: or fallback to string.Empty? CANNOT be null
+        }
+
+        [JsonProperty("val")]
+        public object Value { get; set; }
+    }
+}

src/Umbraco.PublishedCache.NuCache/DataSource/SerializerBase.cs

@@ -0,0 +1,107 @@
+using System;
+using System.IO;
+using CSharpTest.Net.Serialization;
+
+namespace Umbraco.Web.PublishedCache.NuCache.DataSource
+{
+    internal abstract class SerializerBase
+    {
+        protected string ReadString(Stream stream) => PrimitiveSerializer.String.ReadFrom(stream);
+        protected int ReadInt(Stream stream) => PrimitiveSerializer.Int32.ReadFrom(stream);
+        protected long ReadLong(Stream stream) => PrimitiveSerializer.Int64.ReadFrom(stream);
+        protected float ReadFloat(Stream stream) => PrimitiveSerializer.Float.ReadFrom(stream);
+        protected double ReadDouble(Stream stream) => PrimitiveSerializer.Double.ReadFrom(stream);
+        protected DateTime ReadDateTime(Stream stream) => PrimitiveSerializer.DateTime.ReadFrom(stream);
+
+        private T? ReadObject<T>(Stream stream, char t, Func<Stream, T> read)
+            where T : struct
+        {
+            var type = PrimitiveSerializer.Char.ReadFrom(stream);
+            if (type == 'N') return null;
+            if (type != t)
+                throw new NotSupportedException($"Cannot deserialize type '{type}', expected '{t}'.");
+            return read(stream);
+        }
+
+        protected string ReadStringObject(Stream stream) // required 'cos string is not a struct
+        {
+            var type = PrimitiveSerializer.Char.ReadFrom(stream);
+            if (type == 'N') return null;
+            if (type != 'S')
+                throw new NotSupportedException($"Cannot deserialize type '{type}', expected 'S'.");
+            return PrimitiveSerializer.String.ReadFrom(stream);
+        }
+
+        protected int? ReadIntObject(Stream stream) => ReadObject(stream, 'I', ReadInt);
+        protected long? ReadLongObject(Stream stream) => ReadObject(stream, 'L', ReadLong);
+        protected float? ReadFloatObject(Stream stream) => ReadObject(stream, 'F', ReadFloat);
+        protected double? ReadDoubleObject(Stream stream) => ReadObject(stream, 'B', ReadDouble);
+        protected DateTime? ReadDateTimeObject(Stream stream) => ReadObject(stream, 'D', ReadDateTime);
+
+        protected object ReadObject(Stream stream)
+            => ReadObject(PrimitiveSerializer.Char.ReadFrom(stream), stream);
+
+        protected object ReadObject(char type, Stream stream)
+        {
+            switch (type)
+            {
+                case 'N':
+                    return null;
+                case 'S':
+                    return PrimitiveSerializer.String.ReadFrom(stream);
+                case 'I':
+                    return PrimitiveSerializer.Int32.ReadFrom(stream);
+                case 'L':
+                    return PrimitiveSerializer.Int64.ReadFrom(stream);
+                case 'F':
+                    return PrimitiveSerializer.Float.ReadFrom(stream);
+                case 'B':
+                    return PrimitiveSerializer.Double.ReadFrom(stream);
+                case 'D':
+                    return PrimitiveSerializer.DateTime.ReadFrom(stream);
+                default:
+                    throw new NotSupportedException($"Cannot deserialize unknown type '{type}'.");
+            }
+        }
+
+        protected void WriteObject(object value, Stream stream)
+        {
+            if (value == null)
+            {
+                PrimitiveSerializer.Char.WriteTo('N', stream);
+            }
+            else if (value is string stringValue)
+            {
+                PrimitiveSerializer.Char.WriteTo('S', stream);
+                PrimitiveSerializer.String.WriteTo(stringValue, stream);
+            }
+            else if (value is int intValue)
+            {
+                PrimitiveSerializer.Char.WriteTo('I', stream);
+                PrimitiveSerializer.Int32.WriteTo(intValue, stream);
+            }
+            else if (value is long longValue)
+            {
+                PrimitiveSerializer.Char.WriteTo('L', stream);
+                PrimitiveSerializer.Int64.WriteTo(longValue, stream);
+            }
+            else if (value is float floatValue)
+            {
+                PrimitiveSerializer.Char.WriteTo('F', stream);
+                PrimitiveSerializer.Float.WriteTo(floatValue, stream);
+            }
+            else if (value is double doubleValue)
+            {
+                PrimitiveSerializer.Char.WriteTo('B', stream);
+                PrimitiveSerializer.Double.WriteTo(doubleValue, stream);
+            }
+            else if (value is DateTime dateValue)
+            {
+                PrimitiveSerializer.Char.WriteTo('D', stream);
+                PrimitiveSerializer.DateTime.WriteTo(dateValue, stream);
+            }
+            else
+                throw new NotSupportedException("Value type " + value.GetType().FullName + " cannot be serialized.");
+        }
+    }
+}