Implement Similarity providers

This commit is contained in:
Shadowghost
2026-05-03 23:43:01 +02:00
parent 622947e374
commit 4ebce39070
36 changed files with 1830 additions and 61 deletions

View File

@@ -25,6 +25,7 @@ using Emby.Server.Implementations.Dto;
using Emby.Server.Implementations.HttpServer.Security;
using Emby.Server.Implementations.IO;
using Emby.Server.Implementations.Library;
using Emby.Server.Implementations.Library.SimilarItems;
using Emby.Server.Implementations.Localization;
using Emby.Server.Implementations.Playlists;
using Emby.Server.Implementations.Plugins;
@@ -92,7 +93,11 @@ using MediaBrowser.Model.System;
using MediaBrowser.Model.Tasks;
using MediaBrowser.Providers.Lyric;
using MediaBrowser.Providers.Manager;
using MediaBrowser.Providers.Plugins.ListenBrainz;
using MediaBrowser.Providers.Plugins.ListenBrainz.Api;
using MediaBrowser.Providers.Plugins.Tmdb;
using MediaBrowser.Providers.Plugins.Tmdb.Movies;
using MediaBrowser.Providers.Plugins.Tmdb.TV;
using MediaBrowser.Providers.Subtitles;
using MediaBrowser.XbmcMetadata.Providers;
using Microsoft.AspNetCore.Http;
@@ -485,6 +490,11 @@ namespace Emby.Server.Implementations
serviceCollection.AddScoped<ISystemManager, SystemManager>();
serviceCollection.AddSingleton<TmdbClientManager>();
serviceCollection.AddSingleton<TmdbMovieSimilarProvider>();
serviceCollection.AddSingleton<TmdbSeriesSimilarProvider>();
serviceCollection.AddSingleton<ListenBrainzLabsClient>();
serviceCollection.AddSingleton<ListenBrainzSimilarArtistProvider>();
serviceCollection.AddSingleton(NetManager);
@@ -537,6 +547,8 @@ namespace Emby.Server.Implementations
serviceCollection.AddSingleton<ILibraryMonitor, LibraryMonitor>();
serviceCollection.AddSingleton<ISimilarItemsManager, SimilarItemsManager>();
serviceCollection.AddSingleton<ISearchEngine, SearchEngine>();
serviceCollection.AddSingleton<IWebSocketManager, WebSocketManager>();
@@ -694,6 +706,8 @@ namespace Emby.Server.Implementations
GetExports<IExternalUrlProvider>());
Resolve<IMediaSourceManager>().AddParts(GetExports<IMediaSourceProvider>());
Resolve<ISimilarItemsManager>().AddParts(GetExports<ISimilarItemsProvider>());
}
/// <summary>

View File

@@ -0,0 +1,55 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Jellyfin.Data.Enums;
using Jellyfin.Database.Implementations.Enums;
using MediaBrowser.Controller.Dto;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.Entities.Audio;
using MediaBrowser.Controller.Library;
using MediaBrowser.Model.Configuration;
namespace Emby.Server.Implementations.Library.SimilarItems;
/// <summary>
/// Provides similar items for audio tracks.
/// </summary>
public class AudioSimilarItemsProvider : ILocalSimilarItemsProvider<Audio>
{
private readonly ILibraryManager _libraryManager;
/// <summary>
/// Initializes a new instance of the <see cref="AudioSimilarItemsProvider"/> class.
/// </summary>
/// <param name="libraryManager">The library manager.</param>
public AudioSimilarItemsProvider(ILibraryManager libraryManager)
{
_libraryManager = libraryManager;
}
/// <inheritdoc/>
public string Name => "Local Genre/Tag";
/// <inheritdoc/>
public MetadataPluginType Type => MetadataPluginType.LocalSimilarityProvider;
/// <inheritdoc/>
public Task<IReadOnlyList<BaseItem>> GetSimilarItemsAsync(Audio item, SimilarItemsQuery query, CancellationToken cancellationToken)
{
var internalQuery = new InternalItemsQuery(query.User)
{
Genres = item.Genres,
Tags = item.Tags,
Limit = query.Limit,
DtoOptions = query.DtoOptions ?? new DtoOptions(),
ExcludeItemIds = [.. query.ExcludeItemIds],
ExcludeArtistIds = [.. query.ExcludeArtistIds],
IncludeItemTypes = [BaseItemKind.Audio],
EnableGroupByMetadataKey = false,
EnableTotalRecordCount = true,
OrderBy = [(ItemSortBy.Random, SortOrder.Ascending)]
};
return Task.FromResult(_libraryManager.GetItemList(internalQuery));
}
}

View File

@@ -0,0 +1,94 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Jellyfin.Data.Enums;
using Jellyfin.Database.Implementations.Enums;
using MediaBrowser.Controller.Configuration;
using MediaBrowser.Controller.Dto;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.Library;
using MediaBrowser.Controller.LiveTv;
using MediaBrowser.Model.Configuration;
namespace Emby.Server.Implementations.Library.SimilarItems;
/// <summary>
/// Provides similar items for Live TV programs.
/// </summary>
public class LiveTvProgramSimilarItemsProvider : ILocalSimilarItemsProvider<LiveTvProgram>
{
private readonly ILibraryManager _libraryManager;
private readonly IServerConfigurationManager _serverConfigurationManager;
/// <summary>
/// Initializes a new instance of the <see cref="LiveTvProgramSimilarItemsProvider"/> class.
/// </summary>
/// <param name="libraryManager">The library manager.</param>
/// <param name="serverConfigurationManager">The server configuration manager.</param>
public LiveTvProgramSimilarItemsProvider(
ILibraryManager libraryManager,
IServerConfigurationManager serverConfigurationManager)
{
_libraryManager = libraryManager;
_serverConfigurationManager = serverConfigurationManager;
}
/// <inheritdoc/>
public string Name => "Local Genre/Tag";
/// <inheritdoc/>
public MetadataPluginType Type => MetadataPluginType.LocalSimilarityProvider;
/// <inheritdoc/>
public Task<IReadOnlyList<BaseItem>> GetSimilarItemsAsync(LiveTvProgram item, SimilarItemsQuery query, CancellationToken cancellationToken)
{
BaseItemKind[] includeItemTypes;
bool enableGroupByMetadataKey;
bool enableTotalRecordCount;
if (item.IsMovie)
{
// Movie-like program
var itemTypes = new List<BaseItemKind> { BaseItemKind.Movie };
if (_serverConfigurationManager.Configuration.EnableExternalContentInSuggestions)
{
itemTypes.Add(BaseItemKind.Trailer);
itemTypes.Add(BaseItemKind.LiveTvProgram);
}
includeItemTypes = [.. itemTypes];
enableGroupByMetadataKey = true;
enableTotalRecordCount = false;
}
else if (item.IsSeries)
{
// Series-like program
includeItemTypes = [BaseItemKind.Series];
enableGroupByMetadataKey = false;
enableTotalRecordCount = true;
}
else
{
// Default - match same type
includeItemTypes = [item.GetBaseItemKind()];
enableGroupByMetadataKey = false;
enableTotalRecordCount = true;
}
var internalQuery = new InternalItemsQuery(query.User)
{
Genres = item.Genres,
Tags = item.Tags,
Limit = query.Limit,
DtoOptions = query.DtoOptions ?? new DtoOptions(),
ExcludeItemIds = [.. query.ExcludeItemIds],
IncludeItemTypes = includeItemTypes,
EnableGroupByMetadataKey = enableGroupByMetadataKey,
EnableTotalRecordCount = enableTotalRecordCount,
OrderBy = [(ItemSortBy.Random, SortOrder.Ascending)]
};
return Task.FromResult(_libraryManager.GetItemList(internalQuery));
}
}

View File

@@ -0,0 +1,79 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Jellyfin.Data.Enums;
using Jellyfin.Database.Implementations.Enums;
using MediaBrowser.Controller.Configuration;
using MediaBrowser.Controller.Dto;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.Entities.Movies;
using MediaBrowser.Controller.Library;
using MediaBrowser.Model.Configuration;
namespace Emby.Server.Implementations.Library.SimilarItems;
/// <summary>
/// Provides similar items for movies and trailers.
/// </summary>
public class MovieSimilarItemsProvider : ILocalSimilarItemsProvider<Movie>, ILocalSimilarItemsProvider<Trailer>
{
private readonly ILibraryManager _libraryManager;
private readonly IServerConfigurationManager _serverConfigurationManager;
/// <summary>
/// Initializes a new instance of the <see cref="MovieSimilarItemsProvider"/> class.
/// </summary>
/// <param name="libraryManager">The library manager.</param>
/// <param name="serverConfigurationManager">The server configuration manager.</param>
public MovieSimilarItemsProvider(
ILibraryManager libraryManager,
IServerConfigurationManager serverConfigurationManager)
{
_libraryManager = libraryManager;
_serverConfigurationManager = serverConfigurationManager;
}
/// <inheritdoc/>
public string Name => "Local Genre/Tag";
/// <inheritdoc/>
public MetadataPluginType Type => MetadataPluginType.LocalSimilarityProvider;
/// <inheritdoc/>
public Task<IReadOnlyList<BaseItem>> GetSimilarItemsAsync(Movie item, SimilarItemsQuery query, CancellationToken cancellationToken)
{
return Task.FromResult(GetSimilarMovieItems(item, query));
}
/// <inheritdoc/>
public Task<IReadOnlyList<BaseItem>> GetSimilarItemsAsync(Trailer item, SimilarItemsQuery query, CancellationToken cancellationToken)
{
return Task.FromResult(GetSimilarMovieItems(item, query));
}
private IReadOnlyList<BaseItem> GetSimilarMovieItems(BaseItem item, SimilarItemsQuery query)
{
var includeItemTypes = new List<BaseItemKind> { BaseItemKind.Movie };
if (_serverConfigurationManager.Configuration.EnableExternalContentInSuggestions)
{
includeItemTypes.Add(BaseItemKind.Trailer);
includeItemTypes.Add(BaseItemKind.LiveTvProgram);
}
var internalQuery = new InternalItemsQuery(query.User)
{
Genres = item.Genres,
Tags = item.Tags,
Limit = query.Limit,
DtoOptions = query.DtoOptions ?? new DtoOptions(),
ExcludeItemIds = [.. query.ExcludeItemIds],
IncludeItemTypes = [.. includeItemTypes],
EnableGroupByMetadataKey = true,
EnableTotalRecordCount = false,
OrderBy = [(ItemSortBy.Random, SortOrder.Ascending)]
};
return _libraryManager.GetItemList(internalQuery);
}
}

View File

@@ -0,0 +1,55 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Jellyfin.Data.Enums;
using Jellyfin.Database.Implementations.Enums;
using MediaBrowser.Controller.Dto;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.Entities.Audio;
using MediaBrowser.Controller.Library;
using MediaBrowser.Model.Configuration;
namespace Emby.Server.Implementations.Library.SimilarItems;
/// <summary>
/// Provides similar items for music albums.
/// </summary>
public class MusicAlbumSimilarItemsProvider : ILocalSimilarItemsProvider<MusicAlbum>
{
private readonly ILibraryManager _libraryManager;
/// <summary>
/// Initializes a new instance of the <see cref="MusicAlbumSimilarItemsProvider"/> class.
/// </summary>
/// <param name="libraryManager">The library manager.</param>
public MusicAlbumSimilarItemsProvider(ILibraryManager libraryManager)
{
_libraryManager = libraryManager;
}
/// <inheritdoc/>
public string Name => "Local Genre/Tag";
/// <inheritdoc/>
public MetadataPluginType Type => MetadataPluginType.LocalSimilarityProvider;
/// <inheritdoc/>
public Task<IReadOnlyList<BaseItem>> GetSimilarItemsAsync(MusicAlbum item, SimilarItemsQuery query, CancellationToken cancellationToken)
{
var internalQuery = new InternalItemsQuery(query.User)
{
Genres = item.Genres,
Tags = item.Tags,
Limit = query.Limit,
DtoOptions = query.DtoOptions ?? new DtoOptions(),
ExcludeItemIds = [.. query.ExcludeItemIds],
ExcludeArtistIds = [.. query.ExcludeArtistIds],
IncludeItemTypes = [BaseItemKind.MusicAlbum],
EnableGroupByMetadataKey = false,
EnableTotalRecordCount = true,
OrderBy = [(ItemSortBy.Random, SortOrder.Ascending)]
};
return Task.FromResult(_libraryManager.GetItemList(internalQuery));
}
}

View File

@@ -0,0 +1,55 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Jellyfin.Data.Enums;
using Jellyfin.Database.Implementations.Enums;
using MediaBrowser.Controller.Dto;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.Entities.Audio;
using MediaBrowser.Controller.Library;
using MediaBrowser.Model.Configuration;
namespace Emby.Server.Implementations.Library.SimilarItems;
/// <summary>
/// Provides similar items for music artists.
/// </summary>
public class MusicArtistSimilarItemsProvider : ILocalSimilarItemsProvider<MusicArtist>
{
private readonly ILibraryManager _libraryManager;
/// <summary>
/// Initializes a new instance of the <see cref="MusicArtistSimilarItemsProvider"/> class.
/// </summary>
/// <param name="libraryManager">The library manager.</param>
public MusicArtistSimilarItemsProvider(ILibraryManager libraryManager)
{
_libraryManager = libraryManager;
}
/// <inheritdoc/>
public string Name => "Local Genre/Tag";
/// <inheritdoc/>
public MetadataPluginType Type => MetadataPluginType.LocalSimilarityProvider;
/// <inheritdoc/>
public Task<IReadOnlyList<BaseItem>> GetSimilarItemsAsync(MusicArtist item, SimilarItemsQuery query, CancellationToken cancellationToken)
{
var internalQuery = new InternalItemsQuery(query.User)
{
Genres = item.Genres,
Tags = item.Tags,
Limit = query.Limit,
DtoOptions = query.DtoOptions ?? new DtoOptions(),
ExcludeItemIds = [.. query.ExcludeItemIds],
ExcludeArtistIds = [.. query.ExcludeArtistIds],
IncludeItemTypes = [BaseItemKind.MusicArtist],
EnableGroupByMetadataKey = false,
EnableTotalRecordCount = true,
OrderBy = [(ItemSortBy.Random, SortOrder.Ascending)]
};
return Task.FromResult(_libraryManager.GetItemList(internalQuery));
}
}

View File

@@ -0,0 +1,54 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Jellyfin.Data.Enums;
using Jellyfin.Database.Implementations.Enums;
using MediaBrowser.Controller.Dto;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.Entities.TV;
using MediaBrowser.Controller.Library;
using MediaBrowser.Model.Configuration;
namespace Emby.Server.Implementations.Library.SimilarItems;
/// <summary>
/// Provides similar items for TV series.
/// </summary>
public class SeriesSimilarItemsProvider : ILocalSimilarItemsProvider<Series>
{
private readonly ILibraryManager _libraryManager;
/// <summary>
/// Initializes a new instance of the <see cref="SeriesSimilarItemsProvider"/> class.
/// </summary>
/// <param name="libraryManager">The library manager.</param>
public SeriesSimilarItemsProvider(ILibraryManager libraryManager)
{
_libraryManager = libraryManager;
}
/// <inheritdoc/>
public string Name => "Local Genre/Tag";
/// <inheritdoc/>
public MetadataPluginType Type => MetadataPluginType.LocalSimilarityProvider;
/// <inheritdoc/>
public Task<IReadOnlyList<BaseItem>> GetSimilarItemsAsync(Series item, SimilarItemsQuery query, CancellationToken cancellationToken)
{
var internalQuery = new InternalItemsQuery(query.User)
{
Genres = item.Genres,
Tags = item.Tags,
Limit = query.Limit,
DtoOptions = query.DtoOptions ?? new DtoOptions(),
ExcludeItemIds = [.. query.ExcludeItemIds],
IncludeItemTypes = [BaseItemKind.Series],
EnableGroupByMetadataKey = false,
EnableTotalRecordCount = true,
OrderBy = [(ItemSortBy.Random, SortOrder.Ascending)]
};
return Task.FromResult(_libraryManager.GetItemList(internalQuery));
}
}

View File

@@ -0,0 +1,423 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Jellyfin.Data.Enums;
using Jellyfin.Database.Implementations.Entities;
using Jellyfin.Extensions.Json;
using MediaBrowser.Controller;
using MediaBrowser.Controller.Dto;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.Library;
using MediaBrowser.Model.Configuration;
using MediaBrowser.Model.Entities;
using MediaBrowser.Model.IO;
using MediaBrowser.Model.Querying;
using Microsoft.Extensions.Logging;
namespace Emby.Server.Implementations.Library.SimilarItems;
/// <summary>
/// Manages similar items providers and orchestrates similar items operations.
/// </summary>
public class SimilarItemsManager : ISimilarItemsManager
{
private static readonly ConcurrentDictionary<Type, MethodInfo> _genericMethodCache = new();
private static readonly MethodInfo _getSimilarItemsInternalMethod = typeof(SimilarItemsManager)
.GetMethod(nameof(GetSimilarItemsInternalAsync), BindingFlags.NonPublic | BindingFlags.Instance)!;
private readonly ILogger<SimilarItemsManager> _logger;
private readonly IServerApplicationPaths _appPaths;
private readonly ILibraryManager _libraryManager;
private readonly IFileSystem _fileSystem;
private ISimilarItemsProvider[] _similarItemsProviders = [];
/// <summary>
/// Initializes a new instance of the <see cref="SimilarItemsManager"/> class.
/// </summary>
/// <param name="logger">The logger.</param>
/// <param name="appPaths">The server application paths.</param>
/// <param name="libraryManager">The library manager.</param>
/// <param name="fileSystem">The file system.</param>
public SimilarItemsManager(
ILogger<SimilarItemsManager> logger,
IServerApplicationPaths appPaths,
ILibraryManager libraryManager,
IFileSystem fileSystem)
{
_logger = logger;
_appPaths = appPaths;
_libraryManager = libraryManager;
_fileSystem = fileSystem;
}
/// <inheritdoc/>
public void AddParts(IEnumerable<ISimilarItemsProvider> providers)
{
_similarItemsProviders = providers.ToArray();
}
/// <inheritdoc/>
public IReadOnlyList<ISimilarItemsProvider> GetSimilarItemsProviders<T>()
where T : BaseItem
{
return _similarItemsProviders
.OfType<ILocalSimilarItemsProvider<T>>()
.Cast<ISimilarItemsProvider>()
.Concat(_similarItemsProviders.OfType<IRemoteSimilarItemsProvider<T>>())
.ToList();
}
/// <inheritdoc/>
public async Task<IReadOnlyList<BaseItem>> GetSimilarItemsAsync(
BaseItem item,
IReadOnlyList<Guid> excludeArtistIds,
User? user,
DtoOptions dtoOptions,
int? limit,
LibraryOptions? libraryOptions,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(item);
ArgumentNullException.ThrowIfNull(excludeArtistIds);
var itemType = item.GetType();
var method = _genericMethodCache.GetOrAdd(itemType, static type => _getSimilarItemsInternalMethod.MakeGenericMethod(type));
var task = (Task<IReadOnlyList<BaseItem>>)method.Invoke(this, [item, excludeArtistIds, user, dtoOptions, limit, libraryOptions, cancellationToken])!;
return await task.ConfigureAwait(false);
}
private async Task<IReadOnlyList<BaseItem>> GetSimilarItemsInternalAsync<T>(
T item,
IReadOnlyList<Guid> excludeArtistIds,
User? user,
DtoOptions dtoOptions,
int? limit,
LibraryOptions? libraryOptions,
CancellationToken cancellationToken)
where T : BaseItem
{
var requestedLimit = limit ?? 50;
var itemKind = item.GetBaseItemKind();
// Ensure ProviderIds is included in DtoOptions for matching remote provider responses
if (!dtoOptions.Fields.Contains(ItemFields.ProviderIds))
{
dtoOptions.Fields = dtoOptions.Fields.Concat([ItemFields.ProviderIds]).ToArray();
}
// Local providers are always enabled. Remote providers must be explicitly enabled.
var localProviders = _similarItemsProviders.OfType<ILocalSimilarItemsProvider<T>>().Cast<ISimilarItemsProvider>().ToList();
var remoteProviders = _similarItemsProviders.OfType<IRemoteSimilarItemsProvider<T>>().Cast<ISimilarItemsProvider>();
var matchingProviders = new List<ISimilarItemsProvider>(localProviders);
var typeOptions = libraryOptions?.GetTypeOptions(typeof(T).Name);
if (typeOptions?.SimilarItemProviders?.Length > 0)
{
matchingProviders.AddRange(remoteProviders
.Where(p => typeOptions.SimilarItemProviders.Contains(p.Name, StringComparer.OrdinalIgnoreCase)));
}
var orderConfig = typeOptions?.SimilarItemProviderOrder is { Length: > 0 } order
? order
: typeOptions?.SimilarItemProviders;
var orderedProviders = matchingProviders
.OrderBy(p => GetConfiguredSimilarProviderOrder(orderConfig, p.Name))
.ToList();
var allResults = new List<(BaseItem Item, float Score)>();
var excludeIds = new HashSet<Guid> { item.Id };
foreach (var (providerOrder, provider) in orderedProviders.Index())
{
if (allResults.Count >= requestedLimit || cancellationToken.IsCancellationRequested)
{
break;
}
try
{
if (provider is ILocalSimilarItemsProvider<T> localProvider)
{
var query = new SimilarItemsQuery
{
User = user,
Limit = requestedLimit - allResults.Count,
DtoOptions = dtoOptions,
ExcludeItemIds = [.. excludeIds],
ExcludeArtistIds = excludeArtistIds
};
var items = await localProvider.GetSimilarItemsAsync(item, query, cancellationToken).ConfigureAwait(false);
foreach (var (position, resultItem) in items.Index())
{
if (excludeIds.Add(resultItem.Id))
{
var score = CalculateScore(null, providerOrder, position);
allResults.Add((resultItem, score));
}
}
}
else if (provider is IRemoteSimilarItemsProvider<T> remoteProvider)
{
var cachePath = GetSimilarItemsCachePath(provider.Name, typeof(T).Name, item.Id);
var cachedReferences = await TryReadSimilarItemsCacheAsync(cachePath, cancellationToken).ConfigureAwait(false);
if (cachedReferences is not null)
{
var resolvedItems = ResolveRemoteReferences(cachedReferences, providerOrder, user, dtoOptions, itemKind, excludeIds);
allResults.AddRange(resolvedItems);
continue;
}
var query = new SimilarItemsQuery
{
User = user,
Limit = requestedLimit - allResults.Count,
DtoOptions = dtoOptions,
ExcludeItemIds = [.. excludeIds],
ExcludeArtistIds = excludeArtistIds
};
// Collect references in batches and resolve against local library.
// Stop fetching once we have enough resolved local items.
const int BatchSize = 20;
var remaining = requestedLimit - allResults.Count;
var collectedReferences = new List<SimilarItemReference>();
var pendingBatch = new List<SimilarItemReference>();
await foreach (var reference in remoteProvider.GetSimilarItemsAsync(item, query, cancellationToken).ConfigureAwait(false))
{
collectedReferences.Add(reference);
pendingBatch.Add(reference);
if (pendingBatch.Count >= BatchSize)
{
var resolvedItems = ResolveRemoteReferences(pendingBatch, providerOrder, user, dtoOptions, itemKind, excludeIds);
allResults.AddRange(resolvedItems);
remaining -= resolvedItems.Count;
pendingBatch.Clear();
if (remaining <= 0)
{
break;
}
}
}
// Resolve any remaining references in the last partial batch
if (pendingBatch.Count > 0)
{
var resolvedItems = ResolveRemoteReferences(pendingBatch, providerOrder, user, dtoOptions, itemKind, excludeIds);
allResults.AddRange(resolvedItems);
}
if (collectedReferences.Count > 0 && provider.CacheDuration is not null)
{
await SaveSimilarItemsCacheAsync(cachePath, collectedReferences, provider.CacheDuration.Value, cancellationToken).ConfigureAwait(false);
}
}
}
catch (OperationCanceledException)
{
break;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Similar items provider {ProviderName} failed for item {ItemId}", provider.Name, item.Id);
}
}
return allResults
.OrderByDescending(x => x.Score)
.Select(x => x.Item)
.Take(requestedLimit)
.ToList();
}
private List<(BaseItem Item, float Score)> ResolveRemoteReferences(
IReadOnlyList<SimilarItemReference> references,
int providerOrder,
User? user,
DtoOptions dtoOptions,
BaseItemKind itemKind,
HashSet<Guid> excludeIds)
{
if (references.Count == 0)
{
return [];
}
var resolvedById = new Dictionary<Guid, (BaseItem Item, float Score)>();
var providerLookup = new Dictionary<(string ProviderName, string ProviderId), (float? Score, int Position)>(StringTupleComparer.Instance);
foreach (var (position, match) in references.Index())
{
var lookupKey = (match.ProviderName, match.ProviderId);
if (!providerLookup.TryGetValue(lookupKey, out var existing))
{
providerLookup[lookupKey] = (match.Score, position);
}
else if (match.Score > existing.Score || (match.Score == existing.Score && position < existing.Position))
{
providerLookup[lookupKey] = (match.Score, position);
}
}
var allProviderIds = providerLookup
.GroupBy(kvp => kvp.Key.ProviderName)
.ToDictionary(g => g.Key, g => g.Select(x => x.Key.ProviderId).ToArray());
var query = new InternalItemsQuery(user)
{
HasAnyProviderIds = allProviderIds,
IncludeItemTypes = [itemKind],
DtoOptions = dtoOptions
};
var items = _libraryManager.GetItemList(query);
foreach (var item in items)
{
if (excludeIds.Contains(item.Id) || resolvedById.ContainsKey(item.Id))
{
continue;
}
foreach (var providerName in allProviderIds.Keys)
{
if (item.TryGetProviderId(providerName, out var itemProviderId) && providerLookup.TryGetValue((providerName, itemProviderId), out var matchInfo))
{
var score = CalculateScore(matchInfo.Score, providerOrder, matchInfo.Position);
if (!resolvedById.TryGetValue(item.Id, out var existing) || existing.Score < score)
{
excludeIds.Add(item.Id);
resolvedById[item.Id] = (item, score);
}
break;
}
}
}
return [.. resolvedById.Values];
}
private static float CalculateScore(float? matchScore, int providerOrder, int position)
{
// Use provider-supplied score if available, otherwise derive from position
var baseScore = matchScore ?? (1.0f - (position * 0.02f));
// Apply small boost based on provider order (higher priority providers get small bonus)
var priorityBoost = Math.Max(0, 10 - providerOrder) * 0.005f;
return Math.Clamp(baseScore + priorityBoost, 0f, 1f);
}
private static int GetConfiguredSimilarProviderOrder(string[]? orderConfig, string providerName)
{
if (orderConfig is null || orderConfig.Length == 0)
{
return int.MaxValue;
}
var index = Array.FindIndex(orderConfig, name => string.Equals(name, providerName, StringComparison.OrdinalIgnoreCase));
return index >= 0 ? index : int.MaxValue;
}
private string GetSimilarItemsCachePath(string providerName, string baseItemType, Guid itemId)
{
var dataPath = Path.Combine(
_appPaths.CachePath,
$"{providerName.ToLowerInvariant()}-similar-{baseItemType.ToLowerInvariant()}");
return Path.Combine(dataPath, $"{itemId.ToString("N", CultureInfo.InvariantCulture)}.json");
}
private async Task<List<SimilarItemReference>?> TryReadSimilarItemsCacheAsync(string cachePath, CancellationToken cancellationToken)
{
var fileInfo = _fileSystem.GetFileSystemInfo(cachePath);
if (!fileInfo.Exists || fileInfo.Length == 0)
{
return null;
}
try
{
var stream = File.OpenRead(cachePath);
await using (stream.ConfigureAwait(false))
{
var cache = await JsonSerializer.DeserializeAsync<SimilarItemsCache>(stream, JsonDefaults.Options, cancellationToken).ConfigureAwait(false);
if (cache?.References is not null && DateTime.UtcNow < cache.ExpiresAt)
{
return cache.References;
}
}
}
catch (IOException ex)
{
_logger.LogWarning(ex, "Failed to read similar items cache from {CachePath}", cachePath);
}
catch (JsonException ex)
{
_logger.LogWarning(ex, "Failed to parse similar items cache from {CachePath}", cachePath);
}
return null;
}
private async Task SaveSimilarItemsCacheAsync(string cachePath, List<SimilarItemReference> references, TimeSpan cacheDuration, CancellationToken cancellationToken)
{
try
{
var directory = Path.GetDirectoryName(cachePath);
if (!string.IsNullOrEmpty(directory))
{
Directory.CreateDirectory(directory);
}
var cache = new SimilarItemsCache
{
References = references,
ExpiresAt = DateTime.UtcNow.Add(cacheDuration)
};
var stream = File.Create(cachePath);
await using (stream.ConfigureAwait(false))
{
await JsonSerializer.SerializeAsync(stream, cache, JsonDefaults.Options, cancellationToken).ConfigureAwait(false);
}
}
catch (IOException ex)
{
_logger.LogWarning(ex, "Failed to save similar items cache to {CachePath}", cachePath);
}
}
private sealed class SimilarItemsCache
{
public List<SimilarItemReference>? References { get; set; }
public DateTime ExpiresAt { get; set; }
}
private sealed class StringTupleComparer : IEqualityComparer<(string Key, string Value)>
{
public static readonly StringTupleComparer Instance = new();
public bool Equals((string Key, string Value) x, (string Key, string Value) y)
=> string.Equals(x.Key, y.Key, StringComparison.OrdinalIgnoreCase) &&
string.Equals(x.Value, y.Value, StringComparison.OrdinalIgnoreCase);
public int GetHashCode((string Key, string Value) obj)
=> HashCode.Combine(
StringComparer.OrdinalIgnoreCase.GetHashCode(obj.Key),
StringComparer.OrdinalIgnoreCase.GetHashCode(obj.Value));
}
}