Optimize Validator and Filter Performance

This commit is contained in:
Shadowghost
2026-01-31 19:19:26 +01:00
parent 694db80d4c
commit 2789532aa8
17 changed files with 2077 additions and 92 deletions

View File

@@ -3495,5 +3495,11 @@ namespace Emby.Server.Implementations.Library
{
return _itemRepository.RerouteLinkedChildren(fromChildId, toChildId);
}
/// <inheritdoc />
public QueryFiltersLegacy GetQueryFiltersLegacy(InternalItemsQuery query)
{
return _itemRepository.GetQueryFiltersLegacy(query);
}
}
}

View File

@@ -55,6 +55,8 @@ public class ArtistsValidator
IncludeItemTypes = [BaseItemKind.MusicArtist]
}).ToHashSet();
var existingArtists = _libraryManager.GetArtists(names);
var numComplete = 0;
var count = names.Count;
var refreshed = 0;
@@ -63,7 +65,15 @@ public class ArtistsValidator
{
try
{
var item = _libraryManager.GetArtist(name);
MusicArtist? item = null;
if (existingArtists.TryGetValue(name, out var artists) && artists.Length > 0)
{
item = artists.OrderBy(i => i.IsAccessedByName ? 1 : 0).First();
}
// Fall back to GetArtist if not found (creates new item if needed)
item ??= _libraryManager.GetArtist(name);
if (!existingArtistIds.Contains(item.Id))
{
await item.RefreshMetadata(cancellationToken).ConfigureAwait(false);

View File

@@ -54,6 +54,13 @@ public class GenresValidator
IncludeItemTypes = [BaseItemKind.Genre]
}).ToHashSet();
var existingGenres = _libraryManager.GetItemList(new InternalItemsQuery
{
IncludeItemTypes = [BaseItemKind.Genre]
}).Cast<Genre>()
.GroupBy(g => g.Name, StringComparer.OrdinalIgnoreCase)
.ToDictionary(g => g.Key, g => g.First(), StringComparer.OrdinalIgnoreCase);
var numComplete = 0;
var count = names.Count;
var refreshed = 0;
@@ -62,7 +69,15 @@ public class GenresValidator
{
try
{
var item = _libraryManager.GetGenre(name);
Genre? item = null;
if (existingGenres.TryGetValue(name, out var existingGenre))
{
item = existingGenre;
}
// Fall back to GetGenre if not found (creates new item if needed)
item ??= _libraryManager.GetGenre(name);
if (!existingGenreIds.Contains(item.Id))
{
await item.RefreshMetadata(cancellationToken).ConfigureAwait(false);

View File

@@ -55,6 +55,13 @@ public class StudiosValidator
IncludeItemTypes = [BaseItemKind.Studio]
}).ToHashSet();
var existingStudios = _libraryManager.GetItemList(new InternalItemsQuery
{
IncludeItemTypes = [BaseItemKind.Studio]
}).Cast<Studio>()
.GroupBy(s => s.Name, StringComparer.OrdinalIgnoreCase)
.ToDictionary(g => g.Key, g => g.First(), StringComparer.OrdinalIgnoreCase);
var numComplete = 0;
var count = names.Count;
var refreshed = 0;
@@ -63,7 +70,15 @@ public class StudiosValidator
{
try
{
var item = _libraryManager.GetStudio(name);
Studio? item = null;
if (existingStudios.TryGetValue(name, out var existingStudio))
{
item = existingStudio;
}
// Fall back to GetStudio if not found (creates new item if needed)
item ??= _libraryManager.GetStudio(name);
if (!existingStudioIds.Contains(item.Id))
{
await item.RefreshMetadata(cancellationToken).ConfigureAwait(false);

View File

@@ -68,53 +68,27 @@ public class FilterController : BaseJellyfinApiController
item = _libraryManager.GetParentItem(parentId, user?.Id);
}
var query = new InternalItemsQuery
{
User = user,
MediaTypes = mediaTypes,
IncludeItemTypes = includeItemTypes,
Recursive = true,
EnableTotalRecordCount = false,
DtoOptions = new DtoOptions
{
Fields = new[] { ItemFields.Genres, ItemFields.Tags },
EnableImages = false,
EnableUserData = false
}
};
if (item is not Folder folder)
{
return new QueryFiltersLegacy();
}
var itemList = folder.GetItemList(query);
return new QueryFiltersLegacy
var query = new InternalItemsQuery(user)
{
Years = itemList.Select(i => i.ProductionYear ?? -1)
.Where(i => i > 0)
.Distinct()
.Order()
.ToArray(),
Genres = itemList.SelectMany(i => i.Genres)
.DistinctNames()
.Order()
.ToArray(),
Tags = itemList
.SelectMany(i => i.Tags)
.Distinct(StringComparer.OrdinalIgnoreCase)
.Order()
.ToArray(),
OfficialRatings = itemList
.Select(i => i.OfficialRating)
.Where(i => !string.IsNullOrWhiteSpace(i))
.Distinct(StringComparer.OrdinalIgnoreCase)
.Order()
.ToArray()
MediaTypes = mediaTypes,
IncludeItemTypes = includeItemTypes,
Recursive = true,
EnableTotalRecordCount = false,
AncestorIds = [folder.Id],
DtoOptions = new DtoOptions
{
Fields = [],
EnableImages = false,
EnableUserData = false
}
};
return _libraryManager.GetQueryFiltersLegacy(query);
}
/// <summary>

View File

@@ -993,7 +993,7 @@ public sealed class BaseItemRepository
dbQuery = dbQuery.Include(e => e.Extras);
}
return dbQuery;
return dbQuery.AsSingleQuery();
}
private IQueryable<BaseItemEntity> ApplyQueryPaging(IQueryable<BaseItemEntity> dbQuery, InternalItemsQuery filter)
@@ -1046,6 +1046,62 @@ public sealed class BaseItemRepository
return dbQuery.Count();
}
/// <inheritdoc />
public QueryFiltersLegacy GetQueryFiltersLegacy(InternalItemsQuery filter)
{
ArgumentNullException.ThrowIfNull(filter);
PrepareFilterQuery(filter);
using var context = _dbProvider.CreateDbContext();
var baseQuery = PrepareItemQuery(context, filter);
baseQuery = TranslateQuery(baseQuery, context, filter);
// Get matching item IDs as a subquery (not materialized)
var matchingItemIds = baseQuery.Select(e => e.Id);
// Query distinct years directly from the database
var years = baseQuery
.Where(e => e.ProductionYear != null && e.ProductionYear > 0)
.Select(e => e.ProductionYear!.Value)
.Distinct()
.OrderBy(y => y)
.ToArray();
// Query distinct official ratings directly from the database
var officialRatings = baseQuery
.Where(e => e.OfficialRating != null && e.OfficialRating != string.Empty)
.Select(e => e.OfficialRating!)
.Distinct()
.OrderBy(r => r)
.ToArray();
// Tags via ItemValuesMap JOIN - uses subquery for matching items
var tags = context.ItemValuesMap
.Where(ivm => ivm.ItemValue.Type == ItemValueType.Tags)
.Where(ivm => matchingItemIds.Contains(ivm.ItemId))
.Select(ivm => ivm.ItemValue.CleanValue)
.Distinct()
.OrderBy(t => t)
.ToArray();
// Genres via ItemValuesMap JOIN - uses subquery for matching items
var genres = context.ItemValuesMap
.Where(ivm => ivm.ItemValue.Type == ItemValueType.Genre)
.Where(ivm => matchingItemIds.Contains(ivm.ItemId))
.Select(ivm => ivm.ItemValue.CleanValue)
.Distinct()
.OrderBy(g => g)
.ToArray();
return new QueryFiltersLegacy
{
Years = years,
OfficialRatings = officialRatings,
Tags = tags,
Genres = genres
};
}
/// <inheritdoc />
public ItemCounts GetItemCounts(InternalItemsQuery filter)
{
@@ -1229,8 +1285,6 @@ public sealed class BaseItemRepository
}
}
context.SaveChanges();
var itemValueMaps = tuples
.Select(e => (e.Item, Values: GetItemValuesToSave(e.Item, e.InheritedTags)))
.ToArray();
@@ -1255,7 +1309,6 @@ public sealed class BaseItemRepository
Value = f.Value
}).ToArray();
context.ItemValues.AddRange(missingItemValues);
context.SaveChanges();
var itemValuesStore = existingValues.Concat(missingItemValues).ToArray();
var valueMap = itemValueMaps
@@ -1291,8 +1344,6 @@ public sealed class BaseItemRepository
context.ItemValuesMap.RemoveRange(itemMappedValues);
}
context.SaveChanges();
var itemsWithAncestors = tuples
.Where(t => t.Item.SupportsAncestors && t.AncestorIds != null)
.Select(t => t.Item.Id)
@@ -1619,7 +1670,8 @@ public sealed class BaseItemRepository
.Include(e => e.LockedFields)
.Include(e => e.UserData)
.Include(e => e.Images)
.Include(e => e.LinkedChildEntities);
.Include(e => e.LinkedChildEntities)
.AsSingleQuery();
var item = dbQuery.FirstOrDefault(e => e.Id == id);
if (item is null)
@@ -2780,7 +2832,7 @@ public sealed class BaseItemRepository
if (filter.TrailerTypes.Length > 0)
{
var trailerTypes = filter.TrailerTypes.Select(e => (int)e).ToArray();
baseQuery = baseQuery.Where(e => trailerTypes.Any(f => e.TrailerTypes!.Any(w => w.Id == f)));
baseQuery = baseQuery.Where(e => e.TrailerTypes!.Any(w => trailerTypes.Contains(w.Id)));
}
if (filter.IsAiring.HasValue)
@@ -2896,29 +2948,31 @@ public sealed class BaseItemRepository
if (filter.IsFavoriteOrLiked.HasValue)
{
var favoriteItemIds = context.UserData
.Where(ud => ud.UserId == filter.User!.Id && ud.IsFavorite)
.Select(ud => ud.ItemId);
if (filter.IsFavoriteOrLiked.Value)
{
baseQuery = baseQuery
.Where(e => e.UserData!.Any(f => f.UserId == filter.User!.Id && f.IsFavorite));
baseQuery = baseQuery.Where(e => favoriteItemIds.Contains(e.Id));
}
else
{
baseQuery = baseQuery
.Where(e => !e.UserData!.Any(f => f.UserId == filter.User!.Id && f.IsFavorite));
baseQuery = baseQuery.Where(e => !favoriteItemIds.Contains(e.Id));
}
}
if (filter.IsFavorite.HasValue)
{
var favoriteItemIds = context.UserData
.Where(ud => ud.UserId == filter.User!.Id && ud.IsFavorite)
.Select(ud => ud.ItemId);
if (filter.IsFavorite.Value)
{
baseQuery = baseQuery
.Where(e => e.UserData!.Any(f => f.UserId == filter.User!.Id && f.IsFavorite));
baseQuery = baseQuery.Where(e => favoriteItemIds.Contains(e.Id));
}
else
{
baseQuery = baseQuery
.Where(e => !e.UserData!.Any(f => f.UserId == filter.User!.Id && f.IsFavorite));
baseQuery = baseQuery.Where(e => !favoriteItemIds.Contains(e.Id));
}
}
@@ -2927,44 +2981,56 @@ public sealed class BaseItemRepository
// We should probably figure this out for all folders, but for right now, this is the only place where we need it
if (filter.IncludeItemTypes.Length == 1 && filter.IncludeItemTypes[0] == BaseItemKind.Series)
{
// Use subquery to find series with played episodes - stays in SQL instead of materializing to HashSet
var playedSeriesKeysSubquery = context.BaseItems
.Where(e => !e.IsFolder && !e.IsVirtualItem && e.SeriesPresentationUniqueKey != null)
.Where(e => e.UserData!.Any(ud => ud.UserId == filter.User!.Id && ud.Played))
.Select(e => e.SeriesPresentationUniqueKey!);
// Get played series IDs by joining episodes to UserData via SeriesId (Guid foreign key).
// Don't filter episodes by TopParentIds here - the series will be filtered by baseQuery anyway.
// This allows the materialized list to be reused across library-scoped queries.
var playedSeriesIdList = context.BaseItems
.Where(e => !e.IsFolder && !e.IsVirtualItem && e.SeriesId.HasValue)
.Join(
context.UserData.Where(ud => ud.UserId == filter.User!.Id && ud.Played),
episode => episode.Id,
ud => ud.ItemId,
(episode, ud) => episode.SeriesId!.Value)
.Distinct()
.ToList();
if (filter.IsPlayed.Value)
{
baseQuery = baseQuery.Where(e => playedSeriesKeysSubquery.Contains(e.PresentationUniqueKey!));
baseQuery = baseQuery.Where(s => playedSeriesIdList.Contains(s.Id));
}
else
{
baseQuery = baseQuery.Where(e => !playedSeriesKeysSubquery.Contains(e.PresentationUniqueKey!));
baseQuery = baseQuery.Where(s => !playedSeriesIdList.Contains(s.Id));
}
}
else if (filter.IsPlayed.Value)
{
baseQuery = baseQuery
.Where(e => e.UserData!.Any(f => f.UserId == filter.User!.Id && f.Played));
}
else
{
baseQuery = baseQuery
.Where(e => !e.UserData!.Any(f => f.UserId == filter.User!.Id && f.Played));
var playedItemIds = context.UserData
.Where(ud => ud.UserId == filter.User!.Id && ud.Played)
.Select(ud => ud.ItemId);
if (filter.IsPlayed.Value)
{
baseQuery = baseQuery.Where(e => playedItemIds.Contains(e.Id));
}
else
{
baseQuery = baseQuery.Where(e => !playedItemIds.Contains(e.Id));
}
}
}
if (filter.IsResumable.HasValue)
{
var resumableItemIds = context.UserData
.Where(ud => ud.UserId == filter.User!.Id && ud.PlaybackPositionTicks > 0)
.Select(ud => ud.ItemId);
if (filter.IsResumable.Value)
{
baseQuery = baseQuery
.Where(e => e.UserData!.Any(f => f.UserId == filter.User!.Id && f.PlaybackPositionTicks > 0));
baseQuery = baseQuery.Where(e => resumableItemIds.Contains(e.Id));
}
else
{
baseQuery = baseQuery
.Where(e => !e.UserData!.Any(f => f.UserId == filter.User!.Id && f.PlaybackPositionTicks > 0));
baseQuery = baseQuery.Where(e => !resumableItemIds.Contains(e.Id));
}
}
@@ -3681,9 +3747,12 @@ public sealed class BaseItemRepository
private static (int Played, int Total) GetPlayedAndTotalCountFromQuery(IQueryable<BaseItemEntity> query, Guid userId)
{
// GroupBy with a constant key aggregates all rows into a single group for server-side counting.
// OrderBy is required before FirstOrDefault to avoid EF Core warnings about unpredictable results.
var result = query
.Select(b => b.UserData!.Any(u => u.UserId == userId && u.Played))
.GroupBy(_ => 1) // Hack to aggregate over entire set
.GroupBy(_ => 1)
.OrderBy(g => g.Key)
.Select(g => new
{
Total = g.Count(),

View File

@@ -26,13 +26,25 @@ internal static class FolderAwareFilterExtensions
JellyfinDbContext context,
Expression<Func<BaseItemEntity, bool>> condition)
{
// Use correlated Any() subqueries instead of UNION + Contains for better index utilization
var matchingIds = context.BaseItems.Where(condition).Select(b => b.Id);
// Get IDs of items that directly match the condition
var directMatchIds = context.BaseItems.Where(condition).Select(b => b.Id);
return query.Where(e =>
matchingIds.Contains(e.Id)
|| context.AncestorIds.Any(a => a.ParentItemId == e.Id && matchingIds.Contains(a.ItemId))
|| context.LinkedChildren.Any(lc => lc.ParentId == e.Id && matchingIds.Contains(lc.ChildId)));
// Get parent IDs where a descendant (via AncestorIds) matches
var ancestorMatchIds = context.AncestorIds
.Where(a => directMatchIds.Contains(a.ItemId))
.Select(a => a.ParentItemId);
// Get parent IDs where a linked child matches
var linkedMatchIds = context.LinkedChildren
.Where(lc => directMatchIds.Contains(lc.ChildId))
.Select(lc => lc.ParentId);
var allMatchingIds = directMatchIds
.Concat(ancestorMatchIds)
.Concat(linkedMatchIds)
.Distinct();
return query.Where(e => allMatchingIds.Contains(e.Id));
}
/// <summary>
@@ -48,11 +60,24 @@ internal static class FolderAwareFilterExtensions
JellyfinDbContext context,
Expression<Func<BaseItemEntity, bool>> condition)
{
var matchingIds = context.BaseItems.Where(condition).Select(b => b.Id);
// Get IDs of items that directly match the condition
var directMatchIds = context.BaseItems.Where(condition).Select(b => b.Id);
return query.Where(e =>
!matchingIds.Contains(e.Id)
&& !context.AncestorIds.Any(a => a.ParentItemId == e.Id && matchingIds.Contains(a.ItemId))
&& !context.LinkedChildren.Any(lc => lc.ParentId == e.Id && matchingIds.Contains(lc.ChildId)));
// Get parent IDs where a descendant (via AncestorIds) matches
var ancestorMatchIds = context.AncestorIds
.Where(a => directMatchIds.Contains(a.ItemId))
.Select(a => a.ParentItemId);
// Get parent IDs where a linked child matches
var linkedMatchIds = context.LinkedChildren
.Where(lc => directMatchIds.Contains(lc.ChildId))
.Select(lc => lc.ParentId);
var allMatchingIds = directMatchIds
.Concat(ancestorMatchIds)
.Concat(linkedMatchIds)
.Distinct();
return query.Where(e => !allMatchingIds.Contains(e.Id));
}
}

View File

@@ -62,10 +62,9 @@ public class PeopleRepository(IDbContextFactory<JellyfinDbContext> dbProvider, I
using var context = _dbProvider.CreateDbContext();
var dbQuery = TranslateQuery(context.Peoples.AsNoTracking(), context, filter).Select(e => e.Name).Distinct();
// dbQuery = dbQuery.OrderBy(e => e.ListOrder);
if (filter.Limit > 0)
{
dbQuery = dbQuery.Take(filter.Limit);
dbQuery = dbQuery.OrderBy(e => e).Take(filter.Limit);
}
return dbQuery.ToArray();

View File

@@ -93,7 +93,7 @@ namespace Jellyfin.Server.Implementations.Users
_users = new ConcurrentDictionary<Guid, User>();
using var dbContext = _dbProvider.CreateDbContext();
foreach (var user in dbContext.Users
.AsSplitQuery()
.AsSingleQuery()
.Include(user => user.Permissions)
.Include(user => user.Preferences)
.Include(user => user.AccessSchedules)
@@ -607,6 +607,7 @@ namespace Jellyfin.Server.Implementations.Users
.Include(u => u.Preferences)
.Include(u => u.AccessSchedules)
.Include(u => u.ProfileImage)
.AsSingleQuery()
.FirstOrDefault(u => u.Id.Equals(userId))
?? throw new ArgumentException("No user exists with given Id!");
@@ -651,6 +652,7 @@ namespace Jellyfin.Server.Implementations.Users
.Include(u => u.Preferences)
.Include(u => u.AccessSchedules)
.Include(u => u.ProfileImage)
.AsSingleQuery()
.FirstOrDefault(u => u.Id.Equals(userId))
?? throw new ArgumentException("No user exists with given Id!");

View File

@@ -723,5 +723,12 @@ namespace MediaBrowser.Controller.Library
/// <param name="toChildId">The child ID to re-route to.</param>
/// <returns>Number of references updated.</returns>
int RerouteLinkedChildReferences(Guid fromChildId, Guid toChildId);
/// <summary>
/// Gets legacy query filters for filtering UI.
/// </summary>
/// <param name="query">The query filter.</param>
/// <returns>Aggregated filter values.</returns>
QueryFiltersLegacy GetQueryFiltersLegacy(InternalItemsQuery query);
}
}

View File

@@ -218,6 +218,13 @@ public interface IItemRepository
/// <returns>List of parent IDs that reference the child.</returns>
IReadOnlyList<Guid> GetManualLinkedParentIds(Guid childId);
/// <summary>
/// Gets legacy query filters (Years, Genres, Tags, OfficialRatings) aggregated directly from the database.
/// </summary>
/// <param name="filter">The query filter.</param>
/// <returns>Aggregated filter values.</returns>
QueryFiltersLegacy GetQueryFiltersLegacy(InternalItemsQuery filter);
/// <summary>
/// Updates LinkedChildren references from one child to another, preserving SortOrder.
/// Handles duplicates: if parent already references toChildId, removes the old reference instead.

View File

@@ -36,6 +36,7 @@ public class BaseItemConfiguration : IEntityTypeConfiguration<BaseItemEntity>
builder.HasIndex(e => e.Path);
builder.HasIndex(e => e.ParentId);
builder.HasIndex(e => e.OwnerId);
builder.HasIndex(e => e.Name);
builder.HasIndex(e => e.ExtraType);
builder.HasIndex(e => new { e.ExtraType, e.OwnerId });
builder.HasIndex(e => e.PresentationUniqueKey);

View File

@@ -15,6 +15,7 @@ public class PeopleBaseItemMapConfiguration : IEntityTypeConfiguration<PeopleBas
builder.HasKey(e => new { e.ItemId, e.PeopleId, e.Role });
builder.HasIndex(e => new { e.ItemId, e.SortOrder });
builder.HasIndex(e => new { e.ItemId, e.ListOrder });
builder.HasIndex(e => e.PeopleId);
builder.HasOne(e => e.Item);
builder.HasOne(e => e.People);
}

View File

@@ -18,6 +18,8 @@ public class UserDataConfiguration : IEntityTypeConfiguration<UserData>
builder.HasIndex(d => new { d.ItemId, d.UserId, d.IsFavorite });
builder.HasIndex(d => new { d.ItemId, d.UserId, d.LastPlayedDate });
builder.HasIndex(d => new { d.UserId, d.ItemId, d.LastPlayedDate });
builder.HasIndex(d => new { d.UserId, d.Played, d.ItemId });
builder.HasIndex(d => new { d.UserId, d.IsFavorite, d.ItemId });
builder.HasOne(e => e.Item).WithMany(e => e.UserData);
}
}

View File

@@ -0,0 +1,45 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Jellyfin.Database.Providers.Sqlite.Migrations
{
/// <inheritdoc />
public partial class AddBaseItemNameIndex : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateIndex(
name: "IX_UserData_UserId_IsFavorite_ItemId",
table: "UserData",
columns: new[] { "UserId", "IsFavorite", "ItemId" });
migrationBuilder.CreateIndex(
name: "IX_UserData_UserId_Played_ItemId",
table: "UserData",
columns: new[] { "UserId", "Played", "ItemId" });
migrationBuilder.CreateIndex(
name: "IX_BaseItems_Name",
table: "BaseItems",
column: "Name");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "IX_UserData_UserId_IsFavorite_ItemId",
table: "UserData");
migrationBuilder.DropIndex(
name: "IX_UserData_UserId_Played_ItemId",
table: "UserData");
migrationBuilder.DropIndex(
name: "IX_BaseItems_Name",
table: "BaseItems");
}
}
}

View File

@@ -362,6 +362,8 @@ namespace Jellyfin.Server.Implementations.Migrations
b.HasIndex("ExtraType");
b.HasIndex("Name");
b.HasIndex("OwnerId");
b.HasIndex("ParentId");
@@ -1446,8 +1448,12 @@ namespace Jellyfin.Server.Implementations.Migrations
b.HasIndex("ItemId", "UserId", "Played");
b.HasIndex("UserId", "IsFavorite", "ItemId");
b.HasIndex("UserId", "ItemId", "LastPlayedDate");
b.HasIndex("UserId", "Played", "ItemId");
b.ToTable("UserData");
b.HasAnnotation("Sqlite:UseSqlReturningClause", false);