From 88cad2ad1acc158c48de950e0b0adb03d2d84b89 Mon Sep 17 00:00:00 2001 From: Shadowghost Date: Mon, 4 May 2026 01:56:21 +0200 Subject: [PATCH 1/4] Speed-up LatestItems for Music --- Emby.Server.Implementations/Dto/DtoService.cs | 50 +++++++++-- .../Item/BaseItemRepository.Querying.cs | 89 +++++++++++-------- 2 files changed, 96 insertions(+), 43 deletions(-) diff --git a/Emby.Server.Implementations/Dto/DtoService.cs b/Emby.Server.Implementations/Dto/DtoService.cs index cc57d183b6..9f36577410 100644 --- a/Emby.Server.Implementations/Dto/DtoService.cs +++ b/Emby.Server.Implementations/Dto/DtoService.cs @@ -203,6 +203,39 @@ namespace Emby.Server.Implementations.Dto } } + // Batch-fetch MusicArtist lookups across all items to avoid N+1 queries. + IReadOnlyDictionary? artistsBatch = null; + HashSet? artistNames = null; + foreach (var item in accessibleItems) + { + if (item is IHasArtist hasArtist) + { + foreach (var name in hasArtist.Artists) + { + if (!string.IsNullOrWhiteSpace(name)) + { + (artistNames ??= new HashSet(StringComparer.Ordinal)).Add(name); + } + } + } + + if (item is IHasAlbumArtist hasAlbumArtist) + { + foreach (var name in hasAlbumArtist.AlbumArtists) + { + if (!string.IsNullOrWhiteSpace(name)) + { + (artistNames ??= new HashSet(StringComparer.Ordinal)).Add(name); + } + } + } + } + + if (artistNames is { Count: > 0 }) + { + artistsBatch = _libraryManager.GetArtists(artistNames.ToArray()); + } + for (int index = 0; index < accessibleItems.Count; index++) { var item = accessibleItems[index]; @@ -214,7 +247,8 @@ namespace Emby.Server.Implementations.Dto userDataBatch?.GetValueOrDefault(item.Id), allCollectionFolders, childCountBatch, - playedCountBatch); + playedCountBatch, + artistsBatch); if (item is LiveTvChannel tvChannel) { @@ -274,7 +308,8 @@ namespace Emby.Server.Implementations.Dto UserItemData? userData = null, List? allCollectionFolders = null, Dictionary? childCountBatch = null, - Dictionary? playedCountBatch = null) + Dictionary? playedCountBatch = null, + IReadOnlyDictionary? artistsBatch = null) { var dto = new BaseItemDto { @@ -334,7 +369,7 @@ namespace Emby.Server.Implementations.Dto AttachStudios(dto, item); } - AttachBasicFields(dto, item, owner, options); + AttachBasicFields(dto, item, owner, options, artistsBatch); if (options.ContainsField(ItemFields.CanDelete)) { @@ -907,7 +942,8 @@ namespace Emby.Server.Implementations.Dto /// The item. /// The owner. /// The options. - private void AttachBasicFields(BaseItemDto dto, BaseItem item, BaseItem? owner, DtoOptions options) + /// Optional pre-fetched artist lookup shared across a batch of items. + private void AttachBasicFields(BaseItemDto dto, BaseItem item, BaseItem? owner, DtoOptions options, IReadOnlyDictionary? artistsBatch = null) { if (options.ContainsField(ItemFields.DateCreated)) { @@ -1152,7 +1188,8 @@ namespace Emby.Server.Implementations.Dto // Include artists that are not in the database yet, e.g., just added via metadata editor // var foundArtists = artistItems.Items.Select(i => i.Item1.Name).ToList(); - var artistsLookup = _libraryManager.GetArtists([.. hasArtist.Artists.Where(e => !string.IsNullOrWhiteSpace(e))]); + var artistsLookup = artistsBatch + ?? _libraryManager.GetArtists([.. hasArtist.Artists.Where(e => !string.IsNullOrWhiteSpace(e))]); dto.ArtistItems = hasArtist.Artists .Where(name => !string.IsNullOrWhiteSpace(name)) @@ -1186,7 +1223,8 @@ namespace Emby.Server.Implementations.Dto // }) // .ToList(); - var albumArtistsLookup = _libraryManager.GetArtists([.. hasAlbumArtist.AlbumArtists.Where(e => !string.IsNullOrWhiteSpace(e))]); + var albumArtistsLookup = artistsBatch + ?? _libraryManager.GetArtists([.. hasAlbumArtist.AlbumArtists.Where(e => !string.IsNullOrWhiteSpace(e))]); dto.AlbumArtists = hasAlbumArtist.AlbumArtists .Where(name => !string.IsNullOrWhiteSpace(name)) diff --git a/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs b/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs index d516bc0d13..5a96205b04 100644 --- a/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs +++ b/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Linq.Expressions; using System.Threading.Tasks; using Jellyfin.Data.Enums; using Jellyfin.Database.Implementations; @@ -125,53 +124,69 @@ public sealed partial class BaseItemRepository return GetLatestTvShowItems(context, baseQuery, filter, limit); } - // Find the top N group keys ordered by most recent DateCreated. - // Movies group by PresentationUniqueKey (alternate versions like 4K/1080p share a key). - // Music groups by Album. - Expression> groupKeyFilter; - Expression> groupKeySelector; - + // Resolve the top N result item ids in a single SQL statement, ordered by the + // group's most recent DateCreated. Movies and music differ in what an "item" + // is, so the grouping shape is per-branch. + List firstIds; if (collectionType is CollectionType.movies) { - groupKeyFilter = e => e.PresentationUniqueKey != null; - groupKeySelector = e => e.PresentationUniqueKey; + // Movies group by PresentationUniqueKey. Alternate versions (4K/1080p of the + // same movie) share that key, but they're already filtered out upstream by + // PrimaryVersionId IS NULL. + var topGroupItems = baseQuery + .Where(e => e.PresentationUniqueKey != null) + .GroupBy(e => e.PresentationUniqueKey) + .Select(g => new + { + MaxDate = g.Max(e => e.DateCreated), + FirstId = g.OrderByDescending(e => e.DateCreated).ThenByDescending(e => e.Id).Select(e => e.Id).First() + }) + .OrderByDescending(g => g.MaxDate); + + var idsQuery = filter.Limit.HasValue + ? topGroupItems.Take(filter.Limit.Value).Select(g => g.FirstId) + : topGroupItems.Select(g => g.FirstId); + + firstIds = idsQuery.ToList(); } else { - groupKeyFilter = e => e.Album != null; - groupKeySelector = e => e.Album; + // Music returns MusicAlbum entities, ordered by their latest track's + // DateCreated. Group by the MusicAlbum ancestor of each track via + // AncestorIds. + var musicAlbumType = _itemTypeLookup.BaseItemKindNames[BaseItemKind.MusicAlbum]!; + + var topGroupItems = + from ancestor in context.AncestorIds + join track in baseQuery on ancestor.ItemId equals track.Id + join album in context.BaseItems on ancestor.ParentItemId equals album.Id + where album.Type == musicAlbumType + group track.DateCreated by album.Id into g + orderby g.Max() descending + select new { AlbumId = g.Key, MaxDate = g.Max() }; + + var idsQuery = filter.Limit.HasValue + ? topGroupItems.Take(filter.Limit.Value).Select(g => g.AlbumId) + : topGroupItems.Select(g => g.AlbumId); + + firstIds = idsQuery.ToList(); } - // Group by GroupKey, pick the latest item per group (correlated subquery: ORDER BY DateCreated DESC, Id DESC LIMIT 1), - // order groups by group max date, take the top N — all in a single SQL statement. - // ThenByDescending(Id) is the tiebreaker for deterministic ordering when items share a DateCreated. - var topGroupItems = baseQuery - .Where(groupKeyFilter) - .GroupBy(groupKeySelector) - .Select(g => new - { - MaxDate = g.Max(e => e.DateCreated), - FirstId = g.OrderByDescending(e => e.DateCreated).ThenByDescending(e => e.Id).Select(e => e.Id).First() - }) - .OrderByDescending(g => g.MaxDate); + // Load the result items by id. The order from firstIds is the group ordering + // we want; we re-apply it via dictionary lookup because for music the loaded + // album's own DateCreated may not match the album's latest-track date, so a + // SQL ORDER BY DateCreated wouldn't preserve it. + var itemsQuery = ApplyNavigations( + context.BaseItems.AsNoTracking().WhereOneOrMany(firstIds, e => e.Id), + filter); - var firstIdsQuery = filter.Limit.HasValue - ? topGroupItems.Take(filter.Limit.Value).Select(g => g.FirstId) - : topGroupItems.Select(g => g.FirstId); - - var firstIds = firstIdsQuery.ToList(); - - // Single bound JSON / array parameter via WhereOneOrMany — keeps SQL small regardless of N. - var itemsQuery = context.BaseItems.AsNoTracking().WhereOneOrMany(firstIds, e => e.Id); - itemsQuery = ApplyNavigations(itemsQuery, filter); - - return itemsQuery - .OrderByDescending(e => e.DateCreated) - .ThenByDescending(e => e.Id) + var itemsById = itemsQuery .AsEnumerable() .Select(w => DeserializeBaseItem(w, filter.SkipDeserialization)) .Where(dto => dto != null) - .ToArray()!; + .ToDictionary(i => i!.Id); + + return firstIds.Where(itemsById.ContainsKey).Select(id => itemsById[id]).ToArray()!; } /// From 2365cea6260d9b25ef80a2350caf631020a254cb Mon Sep 17 00:00:00 2001 From: Shadowghost Date: Mon, 4 May 2026 15:13:00 +0200 Subject: [PATCH 2/4] Only consider Album creation date --- .../Item/BaseItemRepository.Querying.cs | 73 +++++++++---------- 1 file changed, 33 insertions(+), 40 deletions(-) diff --git a/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs b/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs index 5a96205b04..b7b40e76d8 100644 --- a/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs +++ b/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs @@ -124,15 +124,9 @@ public sealed partial class BaseItemRepository return GetLatestTvShowItems(context, baseQuery, filter, limit); } - // Resolve the top N result item ids in a single SQL statement, ordered by the - // group's most recent DateCreated. Movies and music differ in what an "item" - // is, so the grouping shape is per-branch. - List firstIds; if (collectionType is CollectionType.movies) { - // Movies group by PresentationUniqueKey. Alternate versions (4K/1080p of the - // same movie) share that key, but they're already filtered out upstream by - // PrimaryVersionId IS NULL. + // Group by PresentationUniqueKey, pick the newest item per group. var topGroupItems = baseQuery .Where(e => e.PresentationUniqueKey != null) .GroupBy(e => e.PresentationUniqueKey) @@ -143,50 +137,49 @@ public sealed partial class BaseItemRepository }) .OrderByDescending(g => g.MaxDate); - var idsQuery = filter.Limit.HasValue + var firstIdsQuery = filter.Limit.HasValue ? topGroupItems.Take(filter.Limit.Value).Select(g => g.FirstId) : topGroupItems.Select(g => g.FirstId); - firstIds = idsQuery.ToList(); - } - else - { - // Music returns MusicAlbum entities, ordered by their latest track's - // DateCreated. Group by the MusicAlbum ancestor of each track via - // AncestorIds. - var musicAlbumType = _itemTypeLookup.BaseItemKindNames[BaseItemKind.MusicAlbum]!; - - var topGroupItems = - from ancestor in context.AncestorIds - join track in baseQuery on ancestor.ItemId equals track.Id - join album in context.BaseItems on ancestor.ParentItemId equals album.Id - where album.Type == musicAlbumType - group track.DateCreated by album.Id into g - orderby g.Max() descending - select new { AlbumId = g.Key, MaxDate = g.Max() }; - - var idsQuery = filter.Limit.HasValue - ? topGroupItems.Take(filter.Limit.Value).Select(g => g.AlbumId) - : topGroupItems.Select(g => g.AlbumId); - - firstIds = idsQuery.ToList(); + return LoadLatestByIds(context, firstIdsQuery, filter); } - // Load the result items by id. The order from firstIds is the group ordering - // we want; we re-apply it via dictionary lookup because for music the loaded - // album's own DateCreated may not match the album's latest-track date, so a - // SQL ORDER BY DateCreated wouldn't preserve it. + // Albums whose Id is the parent of any track matching the user's filter. + var musicAlbumType = _itemTypeLookup.BaseItemKindNames[BaseItemKind.MusicAlbum]!; + + var albumIdsWithMatchingTrack = context.AncestorIds + .Join(baseQuery, ai => ai.ItemId, t => t.Id, (ai, _) => ai.ParentItemId); + + var topAlbumsQuery = context.BaseItems.AsNoTracking() + .Where(album => album.Type == musicAlbumType) + .Where(album => albumIdsWithMatchingTrack.Contains(album.Id)) + .OrderByDescending(album => album.DateCreated) + .ThenByDescending(album => album.Id); + + var albumIdsQuery = filter.Limit.HasValue + ? topAlbumsQuery.Take(filter.Limit.Value).Select(a => a.Id) + : topAlbumsQuery.Select(a => a.Id); + + return LoadLatestByIds(context, albumIdsQuery, filter); + } + + // Keeping idsQuery deferred lets EF emit `WHERE Id IN ()`. + private IReadOnlyList LoadLatestByIds( + JellyfinDbContext context, + IQueryable idsQuery, + InternalItemsQuery filter) + { var itemsQuery = ApplyNavigations( - context.BaseItems.AsNoTracking().WhereOneOrMany(firstIds, e => e.Id), + context.BaseItems.AsNoTracking().Where(e => idsQuery.Contains(e.Id)), filter); - var itemsById = itemsQuery + return itemsQuery + .OrderByDescending(e => e.DateCreated) + .ThenByDescending(e => e.Id) .AsEnumerable() .Select(w => DeserializeBaseItem(w, filter.SkipDeserialization)) .Where(dto => dto != null) - .ToDictionary(i => i!.Id); - - return firstIds.Where(itemsById.ContainsKey).Select(id => itemsById[id]).ToArray()!; + .ToArray()!; } /// From 0d58c773f9ffa33794bd272d1b0783603e2e46d6 Mon Sep 17 00:00:00 2001 From: Shadowghost Date: Mon, 4 May 2026 21:42:31 +0200 Subject: [PATCH 3/4] Fix review comments --- Emby.Server.Implementations/Dto/DtoService.cs | 8 ++++---- .../Item/BaseItemRepository.Querying.cs | 4 +--- .../Item/BaseItemRepository.cs | 2 ++ 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Emby.Server.Implementations/Dto/DtoService.cs b/Emby.Server.Implementations/Dto/DtoService.cs index 9f36577410..94e2468719 100644 --- a/Emby.Server.Implementations/Dto/DtoService.cs +++ b/Emby.Server.Implementations/Dto/DtoService.cs @@ -205,7 +205,7 @@ namespace Emby.Server.Implementations.Dto // Batch-fetch MusicArtist lookups across all items to avoid N+1 queries. IReadOnlyDictionary? artistsBatch = null; - HashSet? artistNames = null; + var artistNames = new HashSet(StringComparer.Ordinal); foreach (var item in accessibleItems) { if (item is IHasArtist hasArtist) @@ -214,7 +214,7 @@ namespace Emby.Server.Implementations.Dto { if (!string.IsNullOrWhiteSpace(name)) { - (artistNames ??= new HashSet(StringComparer.Ordinal)).Add(name); + artistNames.Add(name); } } } @@ -225,13 +225,13 @@ namespace Emby.Server.Implementations.Dto { if (!string.IsNullOrWhiteSpace(name)) { - (artistNames ??= new HashSet(StringComparer.Ordinal)).Add(name); + artistNames.Add(name); } } } } - if (artistNames is { Count: > 0 }) + if (artistNames.Count > 0) { artistsBatch = _libraryManager.GetArtists(artistNames.ToArray()); } diff --git a/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs b/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs index b7b40e76d8..85f49eeadd 100644 --- a/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs +++ b/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs @@ -145,13 +145,11 @@ public sealed partial class BaseItemRepository } // Albums whose Id is the parent of any track matching the user's filter. - var musicAlbumType = _itemTypeLookup.BaseItemKindNames[BaseItemKind.MusicAlbum]!; - var albumIdsWithMatchingTrack = context.AncestorIds .Join(baseQuery, ai => ai.ItemId, t => t.Id, (ai, _) => ai.ParentItemId); var topAlbumsQuery = context.BaseItems.AsNoTracking() - .Where(album => album.Type == musicAlbumType) + .Where(album => album.Type == _musicAlbumTypeName) .Where(album => albumIdsWithMatchingTrack.Contains(album.Id)) .OrderByDescending(album => album.DateCreated) .ThenByDescending(album => album.Id); diff --git a/Jellyfin.Server.Implementations/Item/BaseItemRepository.cs b/Jellyfin.Server.Implementations/Item/BaseItemRepository.cs index 94dedaeba8..e2c77437a8 100644 --- a/Jellyfin.Server.Implementations/Item/BaseItemRepository.cs +++ b/Jellyfin.Server.Implementations/Item/BaseItemRepository.cs @@ -39,6 +39,7 @@ public sealed partial class BaseItemRepository private readonly IItemTypeLookup _itemTypeLookup; private readonly IServerConfigurationManager _serverConfigurationManager; private readonly ILogger _logger; + private readonly string _musicAlbumTypeName; private static readonly IReadOnlyList _getAllArtistsValueTypes = [ItemValueType.Artist, ItemValueType.AlbumArtist]; private static readonly IReadOnlyList _getArtistValueTypes = [ItemValueType.Artist]; @@ -66,6 +67,7 @@ public sealed partial class BaseItemRepository _itemTypeLookup = itemTypeLookup; _serverConfigurationManager = serverConfigurationManager; _logger = logger; + _musicAlbumTypeName = itemTypeLookup.BaseItemKindNames[BaseItemKind.MusicAlbum]!; } /// From d4f91ab5cac87e087657b825def6bb30841d2963 Mon Sep 17 00:00:00 2001 From: Shadowghost Date: Mon, 4 May 2026 23:48:09 +0200 Subject: [PATCH 4/4] Fixup --- .../Item/BaseItemRepository.Querying.cs | 3 ++- Jellyfin.Server.Implementations/Item/BaseItemRepository.cs | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs b/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs index 85f49eeadd..dc16c3b1b3 100644 --- a/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs +++ b/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs @@ -148,8 +148,9 @@ public sealed partial class BaseItemRepository var albumIdsWithMatchingTrack = context.AncestorIds .Join(baseQuery, ai => ai.ItemId, t => t.Id, (ai, _) => ai.ParentItemId); + var musicAlbumTypeName = _itemTypeLookup.BaseItemKindNames[BaseItemKind.MusicAlbum]!; var topAlbumsQuery = context.BaseItems.AsNoTracking() - .Where(album => album.Type == _musicAlbumTypeName) + .Where(album => album.Type == musicAlbumTypeName) .Where(album => albumIdsWithMatchingTrack.Contains(album.Id)) .OrderByDescending(album => album.DateCreated) .ThenByDescending(album => album.Id); diff --git a/Jellyfin.Server.Implementations/Item/BaseItemRepository.cs b/Jellyfin.Server.Implementations/Item/BaseItemRepository.cs index e2c77437a8..94dedaeba8 100644 --- a/Jellyfin.Server.Implementations/Item/BaseItemRepository.cs +++ b/Jellyfin.Server.Implementations/Item/BaseItemRepository.cs @@ -39,7 +39,6 @@ public sealed partial class BaseItemRepository private readonly IItemTypeLookup _itemTypeLookup; private readonly IServerConfigurationManager _serverConfigurationManager; private readonly ILogger _logger; - private readonly string _musicAlbumTypeName; private static readonly IReadOnlyList _getAllArtistsValueTypes = [ItemValueType.Artist, ItemValueType.AlbumArtist]; private static readonly IReadOnlyList _getArtistValueTypes = [ItemValueType.Artist]; @@ -67,7 +66,6 @@ public sealed partial class BaseItemRepository _itemTypeLookup = itemTypeLookup; _serverConfigurationManager = serverConfigurationManager; _logger = logger; - _musicAlbumTypeName = itemTypeLookup.BaseItemKindNames[BaseItemKind.MusicAlbum]!; } ///