Return null from UmbracoRouteValueTransformer when there is no matches, use a custom IEndpointSelectorPolicy to deal with 404s.

src/Umbraco.Web.Website/DependencyInjection/UmbracoBuilderExtensions.cs

@@ -1,5 +1,7 @@
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Routing;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection.Extensions;
 using Microsoft.Extensions.Options;
 using Umbraco.Cms.Core.DependencyInjection;
 using Umbraco.Cms.Infrastructure.DependencyInjection;
@@ -11,6 +13,7 @@ using Umbraco.Cms.Web.Website.Middleware;
 using Umbraco.Cms.Web.Website.Models;
 using Umbraco.Cms.Web.Website.Routing;
 using Umbraco.Cms.Web.Website.ViewEngines;
+using static Microsoft.Extensions.DependencyInjection.ServiceDescriptor;
 
 namespace Umbraco.Extensions
 {
@@ -40,6 +43,7 @@ namespace Umbraco.Extensions
 
             builder.Services.AddSingleton<UmbracoRouteValueTransformer>();
             builder.Services.AddSingleton<IControllerActionSearcher, ControllerActionSearcher>();
+            builder.Services.TryAddEnumerable(Singleton<MatcherPolicy, NotFoundSelectorPolicy>());
             builder.Services.AddSingleton<IUmbracoRouteValuesFactory, UmbracoRouteValuesFactory>();
             builder.Services.AddSingleton<IRoutableDocumentFilter, RoutableDocumentFilter>();
 

src/Umbraco.Web.Website/Routing/NotFoundSelectorPolicy.cs

@@ -0,0 +1,79 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Mvc.Controllers;
+using Microsoft.AspNetCore.Routing;
+using Microsoft.AspNetCore.Routing.Matching;
+using Umbraco.Cms.Core.Routing;
+using Umbraco.Cms.Web.Common.Controllers;
+using Umbraco.Cms.Web.Common.Routing;
+
+namespace Umbraco.Cms.Web.Website.Routing
+{
+    /// <summary>
+    /// Used to handle 404 routes that haven't been handled by the end user
+    /// </summary>
+    internal class NotFoundSelectorPolicy : MatcherPolicy, IEndpointSelectorPolicy
+    {
+        private readonly Lazy<Endpoint> _notFound;
+        private readonly EndpointDataSource _endpointDataSource;
+
+        public NotFoundSelectorPolicy(EndpointDataSource endpointDataSource)
+        {
+            _notFound = new Lazy<Endpoint>(GetNotFoundEndpoint);
+            _endpointDataSource = endpointDataSource;            
+        }
+
+        // return the endpoint for the RenderController.Index action.
+        private Endpoint GetNotFoundEndpoint()
+        {
+            Endpoint e = _endpointDataSource.Endpoints.First(x =>
+            {
+                // return the endpoint for the RenderController.Index action.
+                ControllerActionDescriptor descriptor = x.Metadata?.GetMetadata<ControllerActionDescriptor>();
+                return descriptor.ControllerTypeInfo == typeof(RenderController)
+                    && descriptor.ActionName == nameof(RenderController.Index);
+            });
+            return e;
+        }
+
+        public override int Order => 0;
+
+        public bool AppliesToEndpoints(IReadOnlyList<Endpoint> endpoints)
+        {
+            // Don't apply this filter to any endpoint group that is a controller route
+            // i.e. only dynamic routes.            
+            foreach(Endpoint endpoint in endpoints)
+            {
+                ControllerAttribute controller = endpoint.Metadata?.GetMetadata<ControllerAttribute>();
+                if (controller != null)
+                {
+                    return false;
+                }
+            }
+
+            // then ensure this is only applied if all endpoints are IDynamicEndpointMetadata
+            return endpoints.All(x => x.Metadata?.GetMetadata<IDynamicEndpointMetadata>() != null);
+        }
+
+        public Task ApplyAsync(HttpContext httpContext, CandidateSet candidates)
+        {
+            if (candidates.Count == 1 && candidates[0].Values == null)
+            {
+                UmbracoRouteValues umbracoRouteValues = httpContext.Features.Get<UmbracoRouteValues>();
+                if (umbracoRouteValues?.PublishedRequest != null
+                    && !umbracoRouteValues.PublishedRequest.HasPublishedContent()
+                    && umbracoRouteValues.PublishedRequest.ResponseStatusCode == StatusCodes.Status404NotFound)
+                {
+                    // not found/404
+                    httpContext.SetEndpoint(_notFound.Value);
+                }
+            }
+            
+            return Task.CompletedTask;            
+        }
+    }
+}

src/Umbraco.Web.Website/Routing/UmbracoRouteValueTransformer.cs

@@ -28,6 +28,7 @@ using RouteDirection = Umbraco.Cms.Core.Routing.RouteDirection;
 
 namespace Umbraco.Cms.Web.Website.Routing
 {
+
     /// <summary>
     /// The route value transformer for Umbraco front-end routes
     /// </summary>
@@ -138,6 +139,16 @@ namespace Umbraco.Cms.Web.Website.Routing
                 return HandlePostedValues(postedInfo, httpContext);
             }
 
+            if (!umbracoRouteValues?.PublishedRequest?.HasPublishedContent() ?? false)
+            {
+                // No content was found, not by any registered 404 handlers and
+                // not by the IContentLastChanceFinder. In this case we want to return
+                // our default 404 page but we cannot return route values now because
+                // it's possible that a developer is handling dynamic routes too.
+                // Our 404 page will be handled with the NotFoundSelectorPolicy
+                return null;
+            }
+
             // See https://docs.microsoft.com/en-us/dotnet/api/microsoft.aspnetcore.mvc.routing.dynamicroutevaluetransformer.transformasync?view=aspnetcore-5.0#Microsoft_AspNetCore_Mvc_Routing_DynamicRouteValueTransformer_TransformAsync_Microsoft_AspNetCore_Http_HttpContext_Microsoft_AspNetCore_Routing_RouteValueDictionary_
             // We should apparenlty not be modified these values.
             // So we create new ones.
@@ -150,11 +161,6 @@ namespace Umbraco.Cms.Web.Website.Routing
                 newValues[ActionToken] = umbracoRouteValues.ActionName;
             }
 
-            // NOTE: If we are never returning null it means that it is not possible for another
-            // DynamicRouteValueTransformer to execute to set the route values. This one will
-            // always win even if it is a 404 because we manage all 404s via Umbraco and 404
-            // handlers.
-
             return newValues;
         }