using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Mime;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using AsyncKeyedLock;
using Jellyfin.Data.Enums;
using Jellyfin.Data.Events;
using Jellyfin.Extensions;
using Jellyfin.Extensions.Json;
using MediaBrowser.Common.Net;
using MediaBrowser.Controller;
using MediaBrowser.Controller.BaseItemManager;
using MediaBrowser.Controller.Configuration;
using MediaBrowser.Controller.Dto;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.Entities.Audio;
using MediaBrowser.Controller.Entities.Movies;
using MediaBrowser.Controller.Library;
using MediaBrowser.Controller.Lyrics;
using MediaBrowser.Controller.MediaSegments;
using MediaBrowser.Controller.Providers;
using MediaBrowser.Controller.Subtitles;
using MediaBrowser.Model.Configuration;
using MediaBrowser.Model.Entities;
using MediaBrowser.Model.Extensions;
using MediaBrowser.Model.IO;
using MediaBrowser.Model.Net;
using MediaBrowser.Model.Providers;
using MediaBrowser.Model.Querying;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Book = MediaBrowser.Controller.Entities.Book;
using Episode = MediaBrowser.Controller.Entities.TV.Episode;
using Movie = MediaBrowser.Controller.Entities.Movies.Movie;
using MusicAlbum = MediaBrowser.Controller.Entities.Audio.MusicAlbum;
using Season = MediaBrowser.Controller.Entities.TV.Season;
using Series = MediaBrowser.Controller.Entities.TV.Series;
namespace MediaBrowser.Providers.Manager
{
///
/// Class ProviderManager.
///
public class ProviderManager : IProviderManager, IDisposable
{
private readonly Lock _refreshQueueLock = new();
private readonly ILogger _logger;
private readonly IHttpClientFactory _httpClientFactory;
private readonly ILibraryMonitor _libraryMonitor;
private readonly IFileSystem _fileSystem;
private readonly IServerApplicationPaths _appPaths;
private readonly ILibraryManager _libraryManager;
private readonly ISubtitleManager _subtitleManager;
private readonly ILyricManager _lyricManager;
private readonly IServerConfigurationManager _configurationManager;
private readonly IBaseItemManager _baseItemManager;
private readonly ConcurrentDictionary _activeRefreshes = new();
private readonly CancellationTokenSource _disposeCancellationTokenSource = new();
private readonly PriorityQueue<(Guid ItemId, MetadataRefreshOptions RefreshOptions), RefreshPriority> _refreshQueue = new();
private readonly IMemoryCache _memoryCache;
private readonly IMediaSegmentManager _mediaSegmentManager;
private readonly ISimilarItemsManager _similarItemsManager;
private readonly AsyncKeyedLocker _imageSaveLock = new(o =>
{
o.PoolSize = 20;
o.PoolInitialFill = 1;
});
///
/// Cache for ordered metadata providers per library/item type combination.
/// Key: (LibraryPath, ItemTypeName, IncludeDisabled, ForceEnableInternetMetadata).
/// Value: Array of ordered metadata providers (before per-item filtering).
///
private readonly ConcurrentDictionary _metadataProviderCache = new();
private IImageProvider[] _imageProviders = [];
private IMetadataService[] _metadataServices = [];
private IMetadataProvider[] _metadataProviders = [];
private IMetadataSaver[] _savers = [];
private IExternalId[] _externalIds = [];
private IExternalUrlProvider[] _externalUrlProviders = [];
private bool _isProcessingRefreshQueue;
private bool _disposed;
///
/// Initializes a new instance of the class.
///
/// The Http client factory.
/// The subtitle manager.
/// The configuration manager.
/// The library monitor.
/// The logger.
/// The filesystem.
/// The server application paths.
/// The library manager.
/// The BaseItem manager.
/// The lyric manager.
/// The memory cache.
/// The media segment manager.
/// The similar items manager.
public ProviderManager(
IHttpClientFactory httpClientFactory,
ISubtitleManager subtitleManager,
IServerConfigurationManager configurationManager,
ILibraryMonitor libraryMonitor,
ILogger logger,
IFileSystem fileSystem,
IServerApplicationPaths appPaths,
ILibraryManager libraryManager,
IBaseItemManager baseItemManager,
ILyricManager lyricManager,
IMemoryCache memoryCache,
IMediaSegmentManager mediaSegmentManager,
ISimilarItemsManager similarItemsManager)
{
_logger = logger;
_httpClientFactory = httpClientFactory;
_configurationManager = configurationManager;
_libraryMonitor = libraryMonitor;
_fileSystem = fileSystem;
_appPaths = appPaths;
_libraryManager = libraryManager;
_subtitleManager = subtitleManager;
_baseItemManager = baseItemManager;
_lyricManager = lyricManager;
_memoryCache = memoryCache;
_mediaSegmentManager = mediaSegmentManager;
_similarItemsManager = similarItemsManager;
CollectionFolder.LibraryOptionsUpdated += OnLibraryOptionsUpdated;
}
///
public event EventHandler>? RefreshStarted;
///
public event EventHandler>? RefreshCompleted;
///
public event EventHandler>>? RefreshProgress;
///
public void AddParts(
IEnumerable imageProviders,
IEnumerable metadataServices,
IEnumerable metadataProviders,
IEnumerable metadataSavers,
IEnumerable externalIds,
IEnumerable externalUrlProviders)
{
_imageProviders = imageProviders.ToArray();
_metadataServices = metadataServices.OrderBy(i => i.Order).ToArray();
_metadataProviders = metadataProviders.ToArray();
_externalIds = externalIds.OrderBy(i => i.ProviderName).ToArray();
_externalUrlProviders = externalUrlProviders.OrderBy(i => i.Name).ToArray();
_savers = metadataSavers.ToArray();
}
///
public Task RefreshSingleItem(BaseItem item, MetadataRefreshOptions options, CancellationToken cancellationToken)
{
var type = item.GetType();
var service = _metadataServices.FirstOrDefault(current => current.CanRefreshPrimary(type))
?? _metadataServices.FirstOrDefault(current => current.CanRefresh(item));
if (service is null)
{
_logger.LogError("Unable to find a metadata service for item of type {TypeName}", type.Name);
return Task.FromResult(ItemUpdateType.None);
}
return service.RefreshMetadata(item, options, cancellationToken);
}
///
public async Task SaveImage(BaseItem item, string url, ImageType type, int? imageIndex, CancellationToken cancellationToken)
{
using (await _imageSaveLock.LockAsync(url, cancellationToken).ConfigureAwait(false))
{
if (_memoryCache.TryGetValue(url, out (string ContentType, byte[] ImageContents)? cachedValue)
&& cachedValue is not null)
{
var imageContents = cachedValue.Value.ImageContents;
var cacheStream = new MemoryStream(imageContents, 0, imageContents.Length, false);
await using (cacheStream.ConfigureAwait(false))
{
await SaveImage(
item,
cacheStream,
cachedValue.Value.ContentType,
type,
imageIndex,
cancellationToken).ConfigureAwait(false);
return;
}
}
var httpClient = _httpClientFactory.CreateClient(NamedClient.Default);
using var response = await httpClient.GetAsync(url, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
var contentType = response.Content.Headers.ContentType?.MediaType;
// Workaround for tvheadend channel icons
// TODO: Isolate this hack into the tvh plugin
if (string.IsNullOrEmpty(contentType))
{
// Special case for imagecache
if (url.Contains("/imagecache/", StringComparison.OrdinalIgnoreCase))
{
contentType = MediaTypeNames.Image.Png;
}
}
// some providers don't correctly report media type, extract from url if no extension found
if (contentType is null || contentType.Equals(MediaTypeNames.Application.Octet, StringComparison.OrdinalIgnoreCase))
{
// Strip query parameters from url to get actual path.
contentType = MimeTypes.GetMimeType(new Uri(url).GetLeftPart(UriPartial.Path));
}
if (!contentType.StartsWith("image/", StringComparison.OrdinalIgnoreCase))
{
throw new HttpRequestException($"Request returned '{contentType}' instead of an image type", null, HttpStatusCode.NotFound);
}
var responseBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
var stream = new MemoryStream(responseBytes, 0, responseBytes.Length, false);
await using (stream.ConfigureAwait(false))
{
_memoryCache.Set(url, (contentType, responseBytes), TimeSpan.FromSeconds(10));
await SaveImage(
item,
stream,
contentType,
type,
imageIndex,
cancellationToken).ConfigureAwait(false);
}
}
}
///
public Task SaveImage(BaseItem item, Stream source, string mimeType, ImageType type, int? imageIndex, CancellationToken cancellationToken)
{
return new ImageSaver(_configurationManager, _libraryMonitor, _fileSystem, _logger).SaveImage(item, source, mimeType, type, imageIndex, cancellationToken);
}
///
public async Task SaveImage(BaseItem item, string source, string mimeType, ImageType type, int? imageIndex, bool? saveLocallyWithMedia, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(source))
{
throw new ArgumentNullException(nameof(source));
}
try
{
var fileStream = AsyncFile.OpenRead(source);
await new ImageSaver(_configurationManager, _libraryMonitor, _fileSystem, _logger)
.SaveImage(item, fileStream, mimeType, type, imageIndex, saveLocallyWithMedia, cancellationToken)
.ConfigureAwait(false);
}
finally
{
try
{
File.Delete(source);
}
catch (Exception ex)
{
_logger.LogError(ex, "Source file {Source} not found or in use, skip removing", source);
}
}
}
///
public Task SaveImage(Stream source, string mimeType, string path)
{
return new ImageSaver(_configurationManager, _libraryMonitor, _fileSystem, _logger)
.SaveImage(source, path);
}
///
public async Task> GetAvailableRemoteImages(BaseItem item, RemoteImageQuery query, CancellationToken cancellationToken)
{
var providers = GetRemoteImageProviders(item, query.IncludeDisabledProviders);
if (!string.IsNullOrEmpty(query.ProviderName))
{
var providerName = query.ProviderName;
providers = providers.Where(i => string.Equals(i.Name, providerName, StringComparison.OrdinalIgnoreCase));
}
if (query.ImageType is not null)
{
providers = providers.Where(i => i.GetSupportedImages(item).Contains(query.ImageType.Value));
}
var preferredLanguage = item.GetPreferredMetadataLanguage();
var tasks = providers.Select(i => GetImages(item, i, preferredLanguage, query.IncludeAllLanguages, cancellationToken, query.ImageType));
var results = await Task.WhenAll(tasks).ConfigureAwait(false);
return results.SelectMany(i => i);
}
///
/// Gets the images.
///
/// The item.
/// The provider.
/// The preferred language.
/// Whether to include all languages in results.
/// The cancellation token.
/// The type.
/// Task{IEnumerable{RemoteImageInfo}}.
private async Task> GetImages(
BaseItem item,
IRemoteImageProvider provider,
string preferredLanguage,
bool includeAllLanguages,
CancellationToken cancellationToken,
ImageType? type = null)
{
bool hasPreferredLanguage = !string.IsNullOrWhiteSpace(preferredLanguage);
try
{
var result = await provider.GetImages(item, cancellationToken).ConfigureAwait(false);
if (type.HasValue)
{
result = result.Where(i => i.Type == type.Value);
}
if (!includeAllLanguages && hasPreferredLanguage)
{
// Filter out languages that do not match the preferred languages.
//
// TODO: should exception case of "en" (English) eventually be removed?
result = result.Where(i => string.IsNullOrWhiteSpace(i.Language) ||
string.Equals(preferredLanguage, i.Language, StringComparison.OrdinalIgnoreCase) ||
string.Equals(i.Language, "en", StringComparison.OrdinalIgnoreCase));
}
return result.OrderByLanguageDescending(preferredLanguage);
}
catch (OperationCanceledException)
{
return Enumerable.Empty();
}
catch (Exception ex)
{
_logger.LogError(ex, "{ProviderName} failed in GetImageInfos for type {ItemType} at {ItemPath}", provider.GetType().Name, item.GetType().Name, item.Path);
return Enumerable.Empty();
}
}
///
public IEnumerable GetRemoteImageProviderInfo(BaseItem item)
{
return GetRemoteImageProviders(item, true).Select(i => new ImageProviderInfo(i.Name, i.GetSupportedImages(item).ToArray()));
}
private IEnumerable GetRemoteImageProviders(BaseItem item, bool includeDisabled)
{
var options = GetMetadataOptions(item);
var libraryOptions = _libraryManager.GetLibraryOptions(item);
return GetImageProvidersInternal(
item,
libraryOptions,
options,
new ImageRefreshOptions(new DirectoryService(_fileSystem)),
includeDisabled).OfType();
}
///
public IEnumerable GetImageProviders(BaseItem item, ImageRefreshOptions refreshOptions)
{
return GetImageProvidersInternal(item, _libraryManager.GetLibraryOptions(item), GetMetadataOptions(item), refreshOptions, false);
}
private IEnumerable GetImageProvidersInternal(BaseItem item, LibraryOptions libraryOptions, MetadataOptions options, ImageRefreshOptions refreshOptions, bool includeDisabled)
{
var typeOptions = libraryOptions.GetTypeOptions(item.GetType().Name);
var fetcherOrder = typeOptions?.ImageFetcherOrder ?? options.ImageFetcherOrder;
return _imageProviders.Where(i => CanRefreshImages(i, item, typeOptions, refreshOptions, includeDisabled))
.OrderBy(i => GetConfiguredOrder(fetcherOrder, i.Name))
.ThenBy(GetDefaultOrder);
}
private bool CanRefreshImages(
IImageProvider provider,
BaseItem item,
TypeOptions? libraryTypeOptions,
ImageRefreshOptions refreshOptions,
bool includeDisabled)
{
try
{
if (!provider.Supports(item))
{
return false;
}
}
catch (Exception ex)
{
_logger.LogError(ex, "{ProviderName} failed in Supports for type {ItemType} at {ItemPath}", provider.GetType().Name, item.GetType().Name, item.Path);
return false;
}
if (includeDisabled || provider is ILocalImageProvider)
{
return true;
}
if (item.IsLocked && refreshOptions.ImageRefreshMode != MetadataRefreshMode.FullRefresh)
{
return false;
}
return _baseItemManager.IsImageFetcherEnabled(item, libraryTypeOptions, provider.Name);
}
///
public IEnumerable> GetMetadataProviders(BaseItem item, LibraryOptions libraryOptions)
where T : BaseItem
{
var globalMetadataOptions = GetMetadataOptions(item);
var libraryPath = GetLibraryPathForItem(item);
return GetMetadataProvidersInternal(item, libraryOptions, globalMetadataOptions, false, false, libraryPath);
}
///
/// Gets metadata providers for the specified item.
///
/// The item type.
/// The item.
/// The library options.
/// Whether to include disabled providers.
/// The metadata providers.
public IEnumerable> GetMetadataProviders(BaseItem item, LibraryOptions libraryOptions, bool includeDisabled)
where T : BaseItem
{
var globalMetadataOptions = GetMetadataOptions(item);
var libraryPath = GetLibraryPathForItem(item);
return GetMetadataProvidersInternal(item, libraryOptions, globalMetadataOptions, includeDisabled, false, libraryPath);
}
private static string GetLibraryPathForItem(BaseItem item)
{
if (item is CollectionFolder collectionFolder)
{
return collectionFolder.Path ?? string.Empty;
}
var topParent = item.GetTopParent();
return topParent?.Path ?? string.Empty;
}
///
public IEnumerable GetMetadataSavers(BaseItem item, LibraryOptions libraryOptions)
{
return _savers.Where(i => IsSaverEnabledForItem(i, item, libraryOptions, ItemUpdateType.MetadataEdit, false));
}
private IEnumerable> GetMetadataProvidersInternal(BaseItem item, LibraryOptions libraryOptions, MetadataOptions globalMetadataOptions, bool includeDisabled, bool forceEnableInternetMetadata, string libraryPath)
where T : BaseItem
{
var typeOptions = libraryOptions.GetTypeOptions(item.GetType().Name);
var orderedProviders = GetOrCreateOrderedProviders(item.GetType().Name, libraryOptions, globalMetadataOptions, includeDisabled, forceEnableInternetMetadata, libraryPath);
return orderedProviders.Where(i => CanRefreshMetadata(i, item, typeOptions, includeDisabled, forceEnableInternetMetadata));
}
private IMetadataProvider[] GetOrCreateOrderedProviders(
string itemTypeName,
LibraryOptions libraryOptions,
MetadataOptions globalMetadataOptions,
bool includeDisabled,
bool forceEnableInternetMetadata,
string libraryPath)
where T : BaseItem
{
var cacheKey = new MetadataProviderCacheKey(libraryPath, itemTypeName, includeDisabled, forceEnableInternetMetadata);
if (_metadataProviderCache.TryGetValue(cacheKey, out var cachedProviders))
{
return cachedProviders.OfType>().ToArray();
}
var localMetadataReaderOrder = libraryOptions.LocalMetadataReaderOrder ?? globalMetadataOptions.LocalMetadataReaderOrder;
var typeOptions = libraryOptions.GetTypeOptions(itemTypeName);
var metadataFetcherOrder = typeOptions?.MetadataFetcherOrder ?? globalMetadataOptions.MetadataFetcherOrder;
var orderedProviders = _metadataProviders.OfType>()
.Where(i => CanRefreshMetadataForCache(i, typeOptions, includeDisabled, forceEnableInternetMetadata))
.OrderBy(i =>
// local and remote providers will be interleaved in the final order
// only relative order within a type matters: consumers of the list filter to one or the other
i switch
{
ILocalMetadataProvider => GetConfiguredOrder(localMetadataReaderOrder, i.Name),
IRemoteMetadataProvider => GetConfiguredOrder(metadataFetcherOrder, i.Name),
// Default to end
_ => int.MaxValue
})
.ThenBy(GetDefaultOrder)
.ToArray();
_metadataProviderCache.TryAdd(cacheKey, orderedProviders.Cast().ToArray());
return orderedProviders;
}
private static bool CanRefreshMetadataForCache(
IMetadataProvider provider,
TypeOptions? libraryTypeOptions,
bool includeDisabled,
bool forceEnableInternetMetadata)
{
if (includeDisabled)
{
return true;
}
if (forceEnableInternetMetadata || provider is not IRemoteMetadataProvider)
{
return true;
}
if (libraryTypeOptions?.MetadataFetchers is { Length: > 0 } metadataFetchers)
{
return metadataFetchers.Contains(provider.Name, StringComparer.OrdinalIgnoreCase);
}
return true;
}
private bool CanRefreshMetadata(
IMetadataProvider provider,
BaseItem item,
TypeOptions? libraryTypeOptions,
bool includeDisabled,
bool forceEnableInternetMetadata)
{
if (!item.SupportsLocalMetadata && provider is ILocalMetadataProvider)
{
return false;
}
if (includeDisabled)
{
return true;
}
// If locked only allow local providers
if (item.IsLocked && provider is not ILocalMetadataProvider && provider is not IForcedProvider)
{
return false;
}
if (forceEnableInternetMetadata || provider is not IRemoteMetadataProvider)
{
return true;
}
// Artists without a folder structure that are derived from metadata have no real path in the library,
// so GetLibraryOptions returns null. Allow all providers through rather than blocking them.
if (item is MusicArtist && libraryTypeOptions is null)
{
return true;
}
return _baseItemManager.IsMetadataFetcherEnabled(item, libraryTypeOptions, provider.Name);
}
private static int GetConfiguredOrder(string[] order, string providerName)
{
var index = Array.IndexOf(order, providerName);
if (index != -1)
{
return index;
}
// default to end
return int.MaxValue;
}
private static int GetDefaultOrder(object provider)
{
if (provider is IHasOrder hasOrder)
{
return hasOrder.Order;
}
// after items that want to be first (~0) but before items that want to be last (~100)
return 50;
}
///
public MetadataPluginSummary[] GetAllMetadataPlugins()
{
return new[]
{
GetPluginSummary(),
GetPluginSummary(),
GetPluginSummary(),
GetPluginSummary(),
GetPluginSummary(),
GetPluginSummary(),
GetPluginSummary(),
GetPluginSummary(),
GetPluginSummary