diff --git a/.github/workflows/ci-codeql-analysis.yml b/.github/workflows/ci-codeql-analysis.yml index 442114dd80..65752af977 100644 --- a/.github/workflows/ci-codeql-analysis.yml +++ b/.github/workflows/ci-codeql-analysis.yml @@ -32,13 +32,13 @@ jobs: dotnet-version: '10.0.x' - name: Initialize CodeQL - uses: github/codeql-action/init@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2 + uses: github/codeql-action/init@e46ed2cbd01164d986452f91f178727624ae40d7 # v4.35.3 with: languages: ${{ matrix.language }} queries: +security-extended - name: Autobuild - uses: github/codeql-action/autobuild@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2 + uses: github/codeql-action/autobuild@e46ed2cbd01164d986452f91f178727624ae40d7 # v4.35.3 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2 + uses: github/codeql-action/analyze@e46ed2cbd01164d986452f91f178727624ae40d7 # v4.35.3 diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index f0ecb166b4..2a26bf15a4 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -35,7 +35,7 @@ jobs: --verbosity minimal - name: Merge code coverage results - uses: danielpalme/ReportGenerator-GitHub-Action@a003c8fb9ac008fd0fffd5faa4f7d3ecb52e0675 # v5.5.7 + uses: danielpalme/ReportGenerator-GitHub-Action@c31aa4ed4f12f147061186cf2a029f307b5c3636 # v5.5.9 with: reports: "**/coverage.cobertura.xml" targetdir: "merged/" diff --git a/Directory.Packages.props b/Directory.Packages.props index 9768d39a27..e8901b4a1d 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -49,7 +49,7 @@ - + diff --git a/Emby.Naming/Common/NamingOptions.cs b/Emby.Naming/Common/NamingOptions.cs index 3792fbdd3f..21638ba9e7 100644 --- a/Emby.Naming/Common/NamingOptions.cs +++ b/Emby.Naming/Common/NamingOptions.cs @@ -379,6 +379,14 @@ namespace Emby.Naming.Common IsNamed = true }, + // "Name - 101.mkv", "Name - 101 [720p].mkv", "Name - 101 (2020).mkv" + // Handles absolute episode numbers with hyphen delimiter (common in anime) + // Without brackets (bracketed version handled above) + new EpisodeExpression(@".*[\\\/](?[^\\\/]+?)[\s_]+-[\s_]+(?[0-9]+)[\s_]*(?:\[.*?\]|\(.*?\))*[\s_]*(?:\.\w+)?$") + { + IsNamed = true + }, + // /server/anything_102.mp4 // /server/james.corden.2017.04.20.anne.hathaway.720p.hdtv.x264-crooks.mkv // /server/anything_1996.11.14.mp4 diff --git a/Emby.Server.Implementations/AppBase/BaseApplicationPaths.cs b/Emby.Server.Implementations/AppBase/BaseApplicationPaths.cs index de722332a4..56d977bbcb 100644 --- a/Emby.Server.Implementations/AppBase/BaseApplicationPaths.cs +++ b/Emby.Server.Implementations/AppBase/BaseApplicationPaths.cs @@ -90,6 +90,7 @@ namespace Emby.Server.Implementations.AppBase CreateAndCheckMarker(ProgramDataPath, "data"); CreateAndCheckMarker(CachePath, "cache"); CreateAndCheckMarker(DataPath, "data"); + CreateCacheDirTag(CachePath); } /// @@ -100,6 +101,26 @@ namespace Emby.Server.Implementations.AppBase CheckOrCreateMarker(path, $".jellyfin-{markerName}", recursive); } + /// + /// Creates a CACHEDIR.TAG file in the specified directory per the Cache Directory Tagging specification. + /// This signals to backup tools (e.g. Restic, Borg) that the directory contains cached data + /// and can be excluded from backups. + /// + /// The cache directory path. + internal static void CreateCacheDirTag(string path) + { + var tagPath = Path.Combine(path, "CACHEDIR.TAG"); + if (!File.Exists(tagPath)) + { + File.WriteAllText( + tagPath, + "Signature: 8a477f597d28d172789f06886806bc55\n" + + "# This file is a cache directory tag created by Jellyfin.\n" + + "# For information about cache directory tags, see:\n" + + "#\thttps://bford.info/cachedir/\n"); + } + } + private IEnumerable GetMarkers(string path, bool recursive = false) { return Directory.EnumerateFiles(path, ".jellyfin-*", recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly); diff --git a/Emby.Server.Implementations/AppBase/BaseConfigurationManager.cs b/Emby.Server.Implementations/AppBase/BaseConfigurationManager.cs index 81ef0e5f9a..ef5fa8bef9 100644 --- a/Emby.Server.Implementations/AppBase/BaseConfigurationManager.cs +++ b/Emby.Server.Implementations/AppBase/BaseConfigurationManager.cs @@ -228,6 +228,7 @@ namespace Emby.Server.Implementations.AppBase Logger.LogInformation("Setting cache path: {Path}", cachePath); ((BaseApplicationPaths)CommonApplicationPaths).CachePath = cachePath; CommonApplicationPaths.CreateAndCheckMarker(((BaseApplicationPaths)CommonApplicationPaths).CachePath, "cache"); + BaseApplicationPaths.CreateCacheDirTag(cachePath); } /// diff --git a/Emby.Server.Implementations/ApplicationHost.cs b/Emby.Server.Implementations/ApplicationHost.cs index cbb0f6c565..b7aa2f3d06 100644 --- a/Emby.Server.Implementations/ApplicationHost.cs +++ b/Emby.Server.Implementations/ApplicationHost.cs @@ -507,7 +507,13 @@ namespace Emby.Server.Implementations serviceCollection.AddSingleton(); - serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(sp => sp.GetRequiredService()); + serviceCollection.AddSingleton(sp => sp.GetRequiredService()); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); @@ -530,6 +536,7 @@ namespace Emby.Server.Implementations serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); @@ -641,6 +648,7 @@ namespace Emby.Server.Implementations BaseItem.ConfigurationManager = ConfigurationManager; BaseItem.FileSystem = Resolve(); BaseItem.ItemRepository = Resolve(); + BaseItem.ItemCountService = Resolve(); BaseItem.LibraryManager = Resolve(); BaseItem.LocalizationManager = Resolve(); BaseItem.Logger = Resolve>(); diff --git a/Emby.Server.Implementations/Chapters/ChapterManager.cs b/Emby.Server.Implementations/Chapters/ChapterManager.cs index d09ed30ae3..8a4721ce62 100644 --- a/Emby.Server.Implementations/Chapters/ChapterManager.cs +++ b/Emby.Server.Implementations/Chapters/ChapterManager.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using Jellyfin.Extensions; using MediaBrowser.Controller.Chapters; using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Entities.Audio; using MediaBrowser.Controller.IO; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.MediaEncoding; @@ -128,7 +129,7 @@ public class ChapterManager : IChapterManager var averageChapterDuration = GetAverageDurationBetweenChapters(chapters); var threshold = TimeSpan.FromSeconds(1).Ticks; - if (averageChapterDuration < threshold) + if (chapters.Count >= 2 && averageChapterDuration < threshold) { _logger.LogInformation("Skipping chapter image extraction for {Video} as the average chapter duration {AverageDuration} was lower than the minimum threshold {Threshold}", video.Name, averageChapterDuration, threshold); extractImages = false; @@ -232,12 +233,22 @@ public class ChapterManager : IChapterManager } /// - public void SaveChapters(Video video, IReadOnlyList chapters) + public bool Supports(BaseItem item) + => item is Video or Audio; + + /// + public void SaveChapters(BaseItem item, IReadOnlyList chapters) { - // Remove any chapters that are outside of the runtime of the video - var validChapters = chapters.Where(c => c.StartPositionTicks < video.RunTimeTicks).ToList(); - _chapterRepository.SaveChapters(video.Id, validChapters); - } + if (!Supports(item)) + { + _logger.LogWarning("Attempted to save chapters for unsupported item type {Type}: {Name} ({Id})", item.GetType().Name, item.Name, item.Id); + return; + } + + // Remove any chapters that are outside of the runtime of the item + var validChapters = chapters.Where(c => c.StartPositionTicks < item.RunTimeTicks).ToList(); + _chapterRepository.SaveChapters(item.Id, validChapters); +} /// public ChapterInfo? GetChapter(Guid baseItemId, int index) diff --git a/Emby.Server.Implementations/Collections/CollectionManager.cs b/Emby.Server.Implementations/Collections/CollectionManager.cs index a320a774c6..0ede5665f9 100644 --- a/Emby.Server.Implementations/Collections/CollectionManager.cs +++ b/Emby.Server.Implementations/Collections/CollectionManager.cs @@ -272,7 +272,7 @@ namespace Emby.Server.Implementations.Collections { var childItem = _libraryManager.GetItemById(guidId); - var child = collection.LinkedChildren.FirstOrDefault(i => (i.ItemId.HasValue && i.ItemId.Value.Equals(guidId)) || (childItem is not null && string.Equals(childItem.Path, i.Path, StringComparison.OrdinalIgnoreCase))); + var child = collection.LinkedChildren.FirstOrDefault(i => i.ItemId.HasValue && i.ItemId.Value.Equals(guidId)); if (child is null) { @@ -342,7 +342,7 @@ namespace Emby.Server.Implementations.Collections // this is kind of a performance hack because only Video has alternate versions that should be in a box set? if (item is Video video) { - foreach (var childId in video.GetLocalAlternateVersionIds()) + foreach (var childId in _libraryManager.GetLocalAlternateVersionIds(video)) { if (!results.ContainsKey(childId)) { diff --git a/Emby.Server.Implementations/Data/CleanDatabaseScheduledTask.cs b/Emby.Server.Implementations/Data/CleanDatabaseScheduledTask.cs index 676bb7f816..17355960c3 100644 --- a/Emby.Server.Implementations/Data/CleanDatabaseScheduledTask.cs +++ b/Emby.Server.Implementations/Data/CleanDatabaseScheduledTask.cs @@ -5,10 +5,12 @@ using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Jellyfin.Data.Enums; using Jellyfin.Database.Implementations; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.IO; using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Playlists; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; @@ -35,7 +37,11 @@ public class CleanDatabaseScheduledTask : ILibraryPostScanTask public async Task Run(IProgress progress, CancellationToken cancellationToken) { - await CleanDeadItems(cancellationToken, progress).ConfigureAwait(false); + var deadItemsProgress = new Progress(val => progress.Report(val * 0.8)); + await CleanDeadItems(cancellationToken, deadItemsProgress).ConfigureAwait(false); + + var playlistProgress = new Progress(val => progress.Report(80 + (val * 0.2))); + await CleanOrphanedFilePlaylistsAsync(cancellationToken, playlistProgress).ConfigureAwait(false); } private async Task CleanDeadItems(CancellationToken cancellationToken, IProgress progress) @@ -116,4 +122,32 @@ public class CleanDatabaseScheduledTask : ILibraryPostScanTask progress.Report(100); } + + private async Task CleanOrphanedFilePlaylistsAsync(CancellationToken cancellationToken, IProgress progress) + { + var playlists = _libraryManager.GetItemList(new InternalItemsQuery + { + IncludeItemTypes = [BaseItemKind.Playlist], + Recursive = true + }).OfType().ToList(); + + var numComplete = 0; + var numItems = Math.Max(playlists.Count, 1); + + foreach (var playlist in playlists) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (playlist.IsFile && !File.Exists(playlist.Path)) + { + _logger.LogInformation("Removing file-based playlist {Name} because source file {Path} no longer exists", playlist.Name, playlist.Path); + _libraryManager.DeleteItem(playlist, new DeleteOptions { DeleteFileLocation = false }); + } + + numComplete++; + progress.Report((double)numComplete / numItems * 100); + } + + progress.Report(100); + } } diff --git a/Emby.Server.Implementations/Dto/DtoService.cs b/Emby.Server.Implementations/Dto/DtoService.cs index 08ced387b8..cc57d183b6 100644 --- a/Emby.Server.Implementations/Dto/DtoService.cs +++ b/Emby.Server.Implementations/Dto/DtoService.cs @@ -153,17 +153,68 @@ namespace Emby.Server.Implementations.Dto private ILiveTvManager LivetvManager => _livetvManagerFactory.Value; /// - public IReadOnlyList GetBaseItemDtos(IReadOnlyList items, DtoOptions options, User? user = null, BaseItem? owner = null) + public IReadOnlyList GetBaseItemDtos( + IReadOnlyList items, + DtoOptions options, + User? user = null, + BaseItem? owner = null, + bool skipVisibilityCheck = false) { - var accessibleItems = user is null ? items : items.Where(x => x.IsVisible(user)).ToList(); + var accessibleItems = skipVisibilityCheck || user is null ? items : items.Where(x => x.IsVisible(user)).ToList(); var returnItems = new BaseItemDto[accessibleItems.Count]; List<(BaseItem, BaseItemDto)>? programTuples = null; List<(BaseItemDto, LiveTvChannel)>? channelTuples = null; + // Batch-fetch user data for all items + Dictionary? userDataBatch = null; + if (user is not null && options.EnableUserData) + { + userDataBatch = _userDataRepository.GetUserDataBatch(accessibleItems, user); + } + + // Pre-compute collection folders once to avoid N+1 queries in CanDelete + List? allCollectionFolders = null; + if (user is not null && options.ContainsField(ItemFields.CanDelete)) + { + allCollectionFolders = _libraryManager.GetUserRootFolder().Children.OfType().ToList(); + } + + // Batch-fetch child counts for all folders to avoid N+1 queries + Dictionary? childCountBatch = null; + if (options.ContainsField(ItemFields.ChildCount)) + { + var folderIds = accessibleItems.OfType().Select(f => f.Id).ToList(); + if (folderIds.Count > 0) + { + childCountBatch = _libraryManager.GetChildCountBatch(folderIds, user?.Id); + } + } + + // Batch-fetch played/total counts for all folders to avoid N+1 queries + Dictionary? playedCountBatch = null; + if (user is not null && options.EnableUserData) + { + var folderIds = accessibleItems.OfType() + .Where(f => f.SupportsUserDataFromChildren && (f.SupportsPlayedStatus || options.ContainsField(ItemFields.RecursiveItemCount))) + .Select(f => f.Id).ToList(); + if (folderIds.Count > 0) + { + playedCountBatch = _libraryManager.GetPlayedAndTotalCountBatch(folderIds, user); + } + } + for (int index = 0; index < accessibleItems.Count; index++) { var item = accessibleItems[index]; - var dto = GetBaseItemDtoInternal(item, options, user, owner); + var dto = GetBaseItemDtoInternal( + item, + options, + user, + owner, + userDataBatch?.GetValueOrDefault(item.Id), + allCollectionFolders, + childCountBatch, + playedCountBatch); if (item is LiveTvChannel tvChannel) { @@ -197,7 +248,7 @@ namespace Emby.Server.Implementations.Dto public BaseItemDto GetBaseItemDto(BaseItem item, DtoOptions options, User? user = null, BaseItem? owner = null) { - var dto = GetBaseItemDtoInternal(item, options, user, owner); + var dto = GetBaseItemDtoInternal(item, options, user, owner, null); if (item is LiveTvChannel tvChannel) { LivetvManager.AddChannelInfo(new[] { (dto, tvChannel) }, options, user); @@ -215,7 +266,15 @@ namespace Emby.Server.Implementations.Dto return dto; } - private BaseItemDto GetBaseItemDtoInternal(BaseItem item, DtoOptions options, User? user = null, BaseItem? owner = null) + private BaseItemDto GetBaseItemDtoInternal( + BaseItem item, + DtoOptions options, + User? user = null, + BaseItem? owner = null, + UserItemData? userData = null, + List? allCollectionFolders = null, + Dictionary? childCountBatch = null, + Dictionary? playedCountBatch = null) { var dto = new BaseItemDto { @@ -252,7 +311,14 @@ namespace Emby.Server.Implementations.Dto if (user is not null) { - AttachUserSpecificInfo(dto, item, user, options); + AttachUserSpecificInfo( + dto, + item, + user, + options, + userData, + childCountBatch, + playedCountBatch); } if (item is IHasMediaSources @@ -274,7 +340,9 @@ namespace Emby.Server.Implementations.Dto { dto.CanDelete = user is null ? item.CanDelete() - : item.CanDelete(user); + : allCollectionFolders is not null + ? item.CanDelete(user, allCollectionFolders) + : item.CanDelete(user); } if (options.ContainsField(ItemFields.CanDownload)) @@ -378,37 +446,7 @@ namespace Emby.Server.Implementations.Dto return; } - var query = new InternalItemsQuery(user) - { - Recursive = true, - DtoOptions = new DtoOptions(false) { EnableImages = false }, - IncludeItemTypes = relatedItemKinds - }; - - switch (dto.Type) - { - case BaseItemKind.Genre: - case BaseItemKind.MusicGenre: - query.GenreIds = [dto.Id]; - break; - case BaseItemKind.MusicArtist: - query.ArtistIds = [dto.Id]; - break; - case BaseItemKind.Person: - query.PersonIds = [dto.Id]; - break; - case BaseItemKind.Studio: - query.StudioIds = [dto.Id]; - break; - case BaseItemKind.Year - when int.TryParse(dto.Name, NumberStyles.Integer, CultureInfo.InvariantCulture, out var year): - query.Years = [year]; - break; - default: - return; - } - - var counts = _libraryManager.GetItemCounts(query); + var counts = _libraryManager.GetItemCountsForNameItem(dto.Type, dto.Id, relatedItemKinds, user); dto.AlbumCount = counts.AlbumCount; dto.ArtistCount = counts.ArtistCount; @@ -458,7 +496,14 @@ namespace Emby.Server.Implementations.Dto /// /// Attaches the user specific info. /// - private void AttachUserSpecificInfo(BaseItemDto dto, BaseItem item, User user, DtoOptions options) + private void AttachUserSpecificInfo( + BaseItemDto dto, + BaseItem item, + User user, + DtoOptions options, + UserItemData? userData = null, + Dictionary? childCountBatch = null, + Dictionary? playedCountBatch = null) { if (item.IsFolder) { @@ -466,7 +511,19 @@ namespace Emby.Server.Implementations.Dto if (options.EnableUserData) { - dto.UserData = _userDataRepository.GetUserDataDto(item, dto, user, options); + if (userData is not null) + { + // Use pre-fetched user data + dto.UserData = GetUserItemDataDto(userData, item.Id); + (int Played, int Total)? precomputed = playedCountBatch is not null + && playedCountBatch.TryGetValue(item.Id, out var counts) ? counts : null; + item.FillUserDataDtoValues(dto.UserData, userData, dto, user, options, precomputed); + } + else + { + // Fall back to individual fetch + dto.UserData = _userDataRepository.GetUserDataDto(item, dto, user, options); + } } if (!dto.ChildCount.HasValue && item.SourceType == SourceType.Library) @@ -485,7 +542,7 @@ namespace Emby.Server.Implementations.Dto if (options.ContainsField(ItemFields.ChildCount)) { - dto.ChildCount ??= GetChildCount(folder, user); + dto.ChildCount ??= GetChildCount(folder, user, childCountBatch); } } @@ -503,7 +560,17 @@ namespace Emby.Server.Implementations.Dto { if (options.EnableUserData) { - dto.UserData = _userDataRepository.GetUserDataDto(item, user); + if (userData is not null) + { + // Use pre-fetched user data + dto.UserData = GetUserItemDataDto(userData, item.Id); + item.FillUserDataDtoValues(dto.UserData, userData, dto, user, options); + } + else + { + // Fall back to individual fetch + dto.UserData = _userDataRepository.GetUserDataDto(item, user); + } } } @@ -513,7 +580,25 @@ namespace Emby.Server.Implementations.Dto } } - private static int GetChildCount(Folder folder, User user) + private static UserItemDataDto GetUserItemDataDto(UserItemData data, Guid itemId) + { + ArgumentNullException.ThrowIfNull(data); + + return new UserItemDataDto + { + IsFavorite = data.IsFavorite, + Likes = data.Likes, + PlaybackPositionTicks = data.PlaybackPositionTicks, + PlayCount = data.PlayCount, + Rating = data.Rating, + Played = data.Played, + LastPlayedDate = data.LastPlayedDate, + ItemId = itemId, + Key = data.Key + }; + } + + private static int GetChildCount(Folder folder, User user, Dictionary? childCountBatch) { // Right now this is too slow to calculate for top level folders on a per-user basis // Just return something so that apps that are expecting a value won't think the folders are empty @@ -522,6 +607,13 @@ namespace Emby.Server.Implementations.Dto return Random.Shared.Next(1, 10); } + // Use pre-fetched batch data if available + if (childCountBatch is not null && childCountBatch.TryGetValue(folder.Id, out var count)) + { + return count; + } + + // Fall back to individual query for special cases (Series, Season, etc.) return folder.GetChildCount(user); } @@ -1132,11 +1224,6 @@ namespace Emby.Server.Implementations.Dto } } - if (options.ContainsField(ItemFields.Chapters)) - { - dto.Chapters = _chapterManager.GetChapters(item.Id).ToList(); - } - if (options.ContainsField(ItemFields.Trickplay)) { var trickplay = _trickplayManager.GetTrickplayManifest(item).GetAwaiter().GetResult(); @@ -1150,6 +1237,11 @@ namespace Emby.Server.Implementations.Dto dto.ExtraType = video.ExtraType; } + if (options.ContainsField(ItemFields.Chapters)) + { + dto.Chapters = _chapterManager.GetChapters(item.Id).ToList(); + } + if (options.ContainsField(ItemFields.MediaStreams)) { // Add VideoInfo diff --git a/Emby.Server.Implementations/IO/LibraryMonitor.cs b/Emby.Server.Implementations/IO/LibraryMonitor.cs index 23bd5cf200..1bf0f8c76c 100644 --- a/Emby.Server.Implementations/IO/LibraryMonitor.cs +++ b/Emby.Server.Implementations/IO/LibraryMonitor.cs @@ -21,6 +21,7 @@ namespace Emby.Server.Implementations.IO private readonly ILibraryManager _libraryManager; private readonly IServerConfigurationManager _configurationManager; private readonly IFileSystem _fileSystem; + private readonly DotIgnoreIgnoreRule _dotIgnoreIgnoreRule; /// /// The file system watchers. @@ -47,17 +48,20 @@ namespace Emby.Server.Implementations.IO /// The configuration manager. /// The filesystem. /// The . + /// The .ignore rule handler. public LibraryMonitor( ILogger logger, ILibraryManager libraryManager, IServerConfigurationManager configurationManager, IFileSystem fileSystem, - IHostApplicationLifetime appLifetime) + IHostApplicationLifetime appLifetime, + DotIgnoreIgnoreRule dotIgnoreIgnoreRule) { _libraryManager = libraryManager; _logger = logger; _configurationManager = configurationManager; _fileSystem = fileSystem; + _dotIgnoreIgnoreRule = dotIgnoreIgnoreRule; appLifetime.ApplicationStarted.Register(Start); appLifetime.ApplicationStopping.Register(Stop); @@ -354,7 +358,7 @@ namespace Emby.Server.Implementations.IO } var fileInfo = _fileSystem.GetFileSystemInfo(path); - if (DotIgnoreIgnoreRule.IsIgnored(fileInfo, null)) + if (_dotIgnoreIgnoreRule.ShouldIgnore(fileInfo, null)) { return; } diff --git a/Emby.Server.Implementations/IO/ManagedFileSystem.cs b/Emby.Server.Implementations/IO/ManagedFileSystem.cs index 4d68cb4444..199407044b 100644 --- a/Emby.Server.Implementations/IO/ManagedFileSystem.cs +++ b/Emby.Server.Implementations/IO/ManagedFileSystem.cs @@ -586,6 +586,12 @@ namespace Emby.Server.Implementations.IO /// public virtual IEnumerable GetFiles(string path, string searchPattern, IReadOnlyList? extensions, bool enableCaseSensitiveExtensions, bool recursive = false) { + if (!Directory.Exists(path)) + { + _logger.LogWarning("Directory does not exist: {Path}", path); + return []; + } + var enumerationOptions = GetEnumerationOptions(recursive); // On linux and macOS the search pattern is case-sensitive diff --git a/Emby.Server.Implementations/Library/DotIgnoreIgnoreRule.cs b/Emby.Server.Implementations/Library/DotIgnoreIgnoreRule.cs index ef5d24c70f..023c1e8915 100644 --- a/Emby.Server.Implementations/Library/DotIgnoreIgnoreRule.cs +++ b/Emby.Server.Implementations/Library/DotIgnoreIgnoreRule.cs @@ -1,6 +1,8 @@ using System; +using System.Collections.Generic; using System.IO; using System.Text.RegularExpressions; +using BitFaster.Caching.Lru; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.IO; using MediaBrowser.Controller.Resolvers; @@ -15,22 +17,36 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule { private static readonly bool IsWindows = OperatingSystem.IsWindows(); - private static FileInfo? FindIgnoreFile(DirectoryInfo directory) - { - for (var current = directory; current is not null; current = current.Parent) - { - var ignorePath = Path.Join(current.FullName, ".ignore"); - if (File.Exists(ignorePath)) - { - return new FileInfo(ignorePath); - } - } + private readonly FastConcurrentLru _directoryCache; + private readonly FastConcurrentLru _rulesCache; - return null; + /// + /// Initializes a new instance of the class. + /// + public DotIgnoreIgnoreRule() + { + var cacheSize = Math.Max(100, Environment.ProcessorCount * 100); + _directoryCache = new FastConcurrentLru( + Environment.ProcessorCount, + cacheSize, + StringComparer.Ordinal); + _rulesCache = new FastConcurrentLru( + Environment.ProcessorCount, + Math.Max(32, cacheSize / 4), + StringComparer.Ordinal); } /// - public bool ShouldIgnore(FileSystemMetadata fileInfo, BaseItem? parent) => IsIgnored(fileInfo, parent); + public bool ShouldIgnore(FileSystemMetadata fileInfo, BaseItem? parent) => IsIgnoredInternal(fileInfo, parent); + + /// + /// Clears the directory lookup cache. The parsed rules cache is not cleared + /// as it validates file modification time on each access. + /// + public void ClearDirectoryCache() + { + _directoryCache.Clear(); + } /// /// Checks whether or not the file is ignored. @@ -38,40 +54,38 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule /// The file information. /// The parent BaseItem. /// True if the file should be ignored. - public static bool IsIgnored(FileSystemMetadata fileInfo, BaseItem? parent) + public bool IsIgnoredInternal(FileSystemMetadata fileInfo, BaseItem? parent) { var searchDirectory = fileInfo.IsDirectory - ? new DirectoryInfo(fileInfo.FullName) - : new DirectoryInfo(Path.GetDirectoryName(fileInfo.FullName) ?? string.Empty); + ? fileInfo.FullName + : Path.GetDirectoryName(fileInfo.FullName); - if (string.IsNullOrEmpty(searchDirectory.FullName)) + if (string.IsNullOrEmpty(searchDirectory)) { return false; } - var ignoreFile = FindIgnoreFile(searchDirectory); + var ignoreFile = FindIgnoreFileCached(searchDirectory); if (ignoreFile is null) { return false; } - // Fast path in case the ignore files isn't a symlink and is empty - if (ignoreFile.LinkTarget is null && ignoreFile.Length == 0) + var parsedEntry = GetParsedRules(ignoreFile); + if (parsedEntry is null) + { + // File was deleted after we cached the path - clear the directory cache entry and return false + _directoryCache.TryRemove(searchDirectory, out _); + return false; + } + + // Empty file means ignore everything + if (parsedEntry.IsEmpty) { - // Ignore directory if we just have the file return true; } - var content = GetFileContent(ignoreFile); - return string.IsNullOrWhiteSpace(content) - || CheckIgnoreRules(fileInfo.FullName, content, fileInfo.IsDirectory); - } - - private static bool CheckIgnoreRules(string path, string ignoreFileContent, bool isDirectory) - { - // If file has content, base ignoring off the content .gitignore-style rules - var rules = ignoreFileContent.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - return CheckIgnoreRules(path, rules, isDirectory); + return parsedEntry.Rules.IsIgnored(GetPathToCheck(fileInfo.FullName, fileInfo.IsDirectory)); } /// @@ -117,8 +131,8 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule return true; } - // Mitigate the problem of the Ignore library not handling Windows paths correctly. - // See https://github.com/jellyfin/jellyfin/issues/15484 + // Mitigate the problem of the Ignore library not handling Windows paths correctly. + // See https://github.com/jellyfin/jellyfin/issues/15484 var pathToCheck = normalizePath ? path.NormalizePath('/') : path; // Add trailing slash for directories to match "folder/" @@ -130,11 +144,196 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule return ignore.IsIgnored(pathToCheck); } - private static string GetFileContent(FileInfo ignoreFile) + private FileInfo? FindIgnoreFileCached(string directory) { - ignoreFile = FileSystemHelper.ResolveLinkTarget(ignoreFile, returnFinalTarget: true) ?? ignoreFile; - return ignoreFile.Exists - ? File.ReadAllText(ignoreFile.FullName) - : string.Empty; + // Check if we have a cached result for this directory + if (_directoryCache.TryGet(directory, out var cached)) + { + return cached.IgnoreFileDirectory is null + ? null + : new FileInfo(Path.Join(cached.IgnoreFileDirectory, ".ignore")); + } + + DirectoryInfo startDir; + try + { + startDir = new DirectoryInfo(directory); + } + catch (ArgumentException) + { + return null; + } + + // Walk up the directory tree to find .ignore file using DirectoryInfo.Parent + var checkedDirs = new List { directory }; + + for (var current = startDir; current is not null; current = current.Parent) + { + var currentPath = current.FullName; + + // Check if this intermediate directory is cached + if (current != startDir && _directoryCache.TryGet(currentPath, out var parentCached)) + { + // Cache the result for all directories we checked + var entry = new IgnoreFileCacheEntry(parentCached.IgnoreFileDirectory); + foreach (var dir in checkedDirs) + { + _directoryCache.AddOrUpdate(dir, entry); + } + + return parentCached.IgnoreFileDirectory is null + ? null + : new FileInfo(Path.Join(parentCached.IgnoreFileDirectory, ".ignore")); + } + + var ignoreFile = new FileInfo(Path.Join(currentPath, ".ignore")); + if (ignoreFile.Exists) + { + // Cache for all directories we checked + var entry = new IgnoreFileCacheEntry(currentPath); + foreach (var dir in checkedDirs) + { + _directoryCache.AddOrUpdate(dir, entry); + } + + return ignoreFile; + } + + if (current != startDir) + { + checkedDirs.Add(currentPath); + } + } + + // No .ignore file found - cache null result for all directories + var nullEntry = new IgnoreFileCacheEntry((string?)null); + foreach (var dir in checkedDirs) + { + _directoryCache.AddOrUpdate(dir, nullEntry); + } + + return null; + } + + private ParsedIgnoreCacheEntry? GetParsedRules(FileInfo ignoreFile) + { + if (!ignoreFile.Exists) + { + _rulesCache.TryRemove(ignoreFile.FullName, out _); + return null; + } + + var lastModified = ignoreFile.LastWriteTimeUtc; + var fileLength = ignoreFile.Length; + var key = ignoreFile.FullName; + + // Check cache + if (_rulesCache.TryGet(key, out var cached)) + { + if (cached.FileLastModified == lastModified && cached.FileLength == fileLength) + { + return cached; + } + + // Stale - need to reparse + _rulesCache.TryRemove(key, out _); + } + + // Parse the file + var parsedEntry = ParseIgnoreFile(ignoreFile, lastModified, fileLength); + _rulesCache.AddOrUpdate(key, parsedEntry); + return parsedEntry; + } + + private static ParsedIgnoreCacheEntry ParseIgnoreFile(FileInfo ignoreFile, DateTime lastModified, long fileLength) + { + if (ignoreFile.LinkTarget is null && fileLength == 0) + { + return new ParsedIgnoreCacheEntry + { + Rules = new Ignore.Ignore(), + FileLastModified = lastModified, + FileLength = fileLength, + IsEmpty = true + }; + } + + // Resolve symlinks + var resolvedFile = FileSystemHelper.ResolveLinkTarget(ignoreFile, returnFinalTarget: true) ?? ignoreFile; + if (!resolvedFile.Exists) + { + return new ParsedIgnoreCacheEntry + { + Rules = new Ignore.Ignore(), + FileLastModified = lastModified, + FileLength = fileLength, + IsEmpty = true + }; + } + + var content = File.ReadAllText(resolvedFile.FullName); + if (string.IsNullOrWhiteSpace(content)) + { + return new ParsedIgnoreCacheEntry + { + Rules = new Ignore.Ignore(), + FileLastModified = lastModified, + FileLength = fileLength, + IsEmpty = true + }; + } + + var rules = content.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + var ignore = new Ignore.Ignore(); + var validRulesAdded = 0; + + foreach (var rule in rules) + { + try + { + ignore.Add(rule); + validRulesAdded++; + } + catch (RegexParseException) + { + // Ignore invalid patterns + } + } + + // No valid rules means treat as empty (ignore all) + return new ParsedIgnoreCacheEntry + { + Rules = ignore, + FileLastModified = lastModified, + FileLength = fileLength, + IsEmpty = validRulesAdded == 0 + }; + } + + private static string GetPathToCheck(string path, bool isDirectory) + { + // Normalize Windows paths + var pathToCheck = IsWindows ? path.NormalizePath('/') : path; + + // Add trailing slash for directories to match "folder/" + if (isDirectory) + { + pathToCheck = string.Concat(pathToCheck.AsSpan().TrimEnd('/'), "/"); + } + + return pathToCheck; + } + + private readonly record struct IgnoreFileCacheEntry(string? IgnoreFileDirectory); + + private sealed class ParsedIgnoreCacheEntry + { + public required Ignore.Ignore Rules { get; init; } + + public required DateTime FileLastModified { get; init; } + + public required long FileLength { get; init; } + + public required bool IsEmpty { get; init; } } } diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index eee87c4d8b..11f1496086 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -30,18 +30,17 @@ using MediaBrowser.Controller.Drawing; using MediaBrowser.Controller.Dto; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Entities.Audio; +using MediaBrowser.Controller.Entities.Movies; using MediaBrowser.Controller.IO; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.LiveTv; using MediaBrowser.Controller.MediaEncoding; -using MediaBrowser.Controller.MediaSegments; using MediaBrowser.Controller.Persistence; +using MediaBrowser.Controller.Playlists; using MediaBrowser.Controller.Providers; using MediaBrowser.Controller.Resolvers; using MediaBrowser.Controller.Sorting; -using MediaBrowser.Controller.Trickplay; using MediaBrowser.Model.Configuration; -using MediaBrowser.Model.Dlna; using MediaBrowser.Model.Drawing; using MediaBrowser.Model.Dto; using MediaBrowser.Model.Entities; @@ -77,12 +76,17 @@ namespace Emby.Server.Implementations.Library private readonly IMediaEncoder _mediaEncoder; private readonly IFileSystem _fileSystem; private readonly IItemRepository _itemRepository; + private readonly IItemPersistenceService _persistenceService; + private readonly INextUpService _nextUpService; + private readonly IItemCountService _countService; + private readonly ILinkedChildrenService _linkedChildrenService; private readonly IImageProcessor _imageProcessor; private readonly NamingOptions _namingOptions; private readonly IPeopleRepository _peopleRepository; private readonly ExtraResolver _extraResolver; private readonly IPathManager _pathManager; private readonly FastConcurrentLru _cache; + private readonly DotIgnoreIgnoreRule _dotIgnoreIgnoreRule; /// /// The _root folder sync lock. @@ -115,11 +119,16 @@ namespace Emby.Server.Implementations.Library /// The user view manager. /// The media encoder. /// The item repository. + /// The item persistence service. + /// The next up service. + /// The item count service. + /// The linked children service. /// The image processor. /// The naming options. /// The directory service. /// The people repository. /// The path manager. + /// The .ignore rule handler. public LibraryManager( IServerApplicationHost appHost, ILoggerFactory loggerFactory, @@ -133,11 +142,16 @@ namespace Emby.Server.Implementations.Library Lazy userViewManagerFactory, IMediaEncoder mediaEncoder, IItemRepository itemRepository, + IItemPersistenceService persistenceService, + INextUpService nextUpService, + IItemCountService countService, + ILinkedChildrenService linkedChildrenService, IImageProcessor imageProcessor, NamingOptions namingOptions, IDirectoryService directoryService, IPeopleRepository peopleRepository, - IPathManager pathManager) + IPathManager pathManager, + DotIgnoreIgnoreRule dotIgnoreIgnoreRule) { _appHost = appHost; _logger = loggerFactory.CreateLogger(); @@ -151,6 +165,10 @@ namespace Emby.Server.Implementations.Library _userViewManagerFactory = userViewManagerFactory; _mediaEncoder = mediaEncoder; _itemRepository = itemRepository; + _persistenceService = persistenceService; + _nextUpService = nextUpService; + _countService = countService; + _linkedChildrenService = linkedChildrenService; _imageProcessor = imageProcessor; _cache = new FastConcurrentLru(_configurationManager.Configuration.CacheSize); @@ -158,6 +176,7 @@ namespace Emby.Server.Implementations.Library _namingOptions = namingOptions; _peopleRepository = peopleRepository; _pathManager = pathManager; + _dotIgnoreIgnoreRule = dotIgnoreIgnoreRule; _extraResolver = new ExtraResolver(loggerFactory.CreateLogger(), namingOptions, directoryService); _configurationManager.ConfigurationUpdated += ConfigurationUpdated; @@ -327,9 +346,17 @@ namespace Emby.Server.Implementations.Library DeleteItem(item, options, parent, notifyParentItem); } - public void DeleteItemsUnsafeFast(IEnumerable items) + public void DeleteItemsUnsafeFast(IReadOnlyCollection items, bool deleteSourceFiles = false) { - var pathMaps = items.Select(e => (Item: e, InternalPath: GetInternalMetadataPaths(e), DeletePaths: e.GetDeletePaths())).ToArray(); + if (items.Count == 0) + { + return; + } + + var pathMaps = items.Select(e => + (Item: e, + InternalPath: GetInternalMetadataPaths(e), + DeletePaths: deleteSourceFiles ? e.GetDeletePaths() : [])).ToArray(); foreach (var (item, internalPaths, pathsToDelete) in pathMaps) { @@ -363,7 +390,7 @@ namespace Emby.Server.Implementations.Library } } - _itemRepository.DeleteItem([.. pathMaps.Select(f => f.Item.Id)]); + _persistenceService.DeleteItem([.. pathMaps.Select(f => f.Item.Id)]); } public void DeleteItem(BaseItem item, DeleteOptions options, BaseItem parent, bool notifyParentItem) @@ -406,6 +433,99 @@ namespace Emby.Server.Implementations.Library item.Id); } + // If deleting a primary version video, clear PrimaryVersionId from alternate versions + // OwnerId check: items with OwnerId set are alternate versions or extras, not primaries + if (item is Video video && !video.PrimaryVersionId.HasValue && video.OwnerId.IsEmpty()) + { + var localAlternateIds = GetLocalAlternateVersionIds(video).ToHashSet(); + var allAlternateVersions = localAlternateIds + .Concat(GetLinkedAlternateVersions(video).Select(v => v.Id)) + .Distinct() + .Select(id => GetItemById(id)) + .OfType