alteropen/jellyfin
1
0
Fork 0
mirror of https://github.com/jellyfin/jellyfin.git synced 2026-05-13 04:06:31 +01:00
Code Issues Actions 7 Packages Projects Releases Wiki Activity

Merge pull request #16166 from Shadowghost/ignore-caching

Browse Source 
Implement ignore rule caching
This commit is contained in:
Niels van Velzen
2026-05-04 17:59:48 +02:00
committed by GitHub
parent ba268cc3fb 0f6bab03eb
commit d359d2f7a8
8 changed files with 665 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Emby.Server.Implementations/ApplicationHost.cs
View File

@@ -536,6 +536,7 @@ namespace Emby.Server.Implementations
serviceCollection.AddSingleton<IMusicManager, MusicManager>();
serviceCollection.AddSingleton<ILibraryMonitor, LibraryMonitor>();
serviceCollection.AddSingleton<DotIgnoreIgnoreRule>();
serviceCollection.AddSingleton<ISearchEngine, SearchEngine>();

8
Emby.Server.Implementations/IO/LibraryMonitor.cs
View File

@@ -21,6 +21,7 @@ namespace Emby.Server.Implementations.IO
private readonly ILibraryManager _libraryManager;
private readonly IServerConfigurationManager _configurationManager;
private readonly IFileSystem _fileSystem;
private readonly DotIgnoreIgnoreRule _dotIgnoreIgnoreRule;
/// <summary>
/// The file system watchers.
@@ -47,17 +48,20 @@ namespace Emby.Server.Implementations.IO
/// <param name="configurationManager">The configuration manager.</param>
/// <param name="fileSystem">The filesystem.</param>
/// <param name="appLifetime">The <see cref="IHostApplicationLifetime"/>.</param>
/// <param name="dotIgnoreIgnoreRule">The .ignore rule handler.</param>
public LibraryMonitor(
ILogger<LibraryMonitor> logger,
ILibraryManager libraryManager,
IServerConfigurationManager configurationManager,
IFileSystem fileSystem,
IHostApplicationLifetime appLifetime)
IHostApplicationLifetime appLifetime,
DotIgnoreIgnoreRule dotIgnoreIgnoreRule)
{
_libraryManager = libraryManager;
_logger = logger;
_configurationManager = configurationManager;
_fileSystem = fileSystem;
_dotIgnoreIgnoreRule = dotIgnoreIgnoreRule;
appLifetime.ApplicationStarted.Register(Start);
appLifetime.ApplicationStopping.Register(Stop);
@@ -354,7 +358,7 @@ namespace Emby.Server.Implementations.IO
}
var fileInfo = _fileSystem.GetFileSystemInfo(path);
if (DotIgnoreIgnoreRule.IsIgnored(fileInfo, null))
if (_dotIgnoreIgnoreRule.ShouldIgnore(fileInfo, null))
{
return;
}

273
Emby.Server.Implementations/Library/DotIgnoreIgnoreRule.cs
View File

@@ -1,6 +1,8 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text.RegularExpressions;
using BitFaster.Caching.Lru;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.IO;
using MediaBrowser.Controller.Resolvers;
@@ -15,22 +17,36 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule
{
private static readonly bool IsWindows = OperatingSystem.IsWindows();
private static FileInfo? FindIgnoreFile(DirectoryInfo directory)
{
for (var current = directory; current is not null; current = current.Parent)
{
var ignorePath = Path.Join(current.FullName, ".ignore");
if (File.Exists(ignorePath))
{
return new FileInfo(ignorePath);
}
}
private readonly FastConcurrentLru<string, IgnoreFileCacheEntry> _directoryCache;
private readonly FastConcurrentLru<string, ParsedIgnoreCacheEntry> _rulesCache;
return null;
/// <summary>
/// Initializes a new instance of the <see cref="DotIgnoreIgnoreRule"/> class.
/// </summary>
public DotIgnoreIgnoreRule()
{
var cacheSize = Math.Max(100, Environment.ProcessorCount * 100);
_directoryCache = new FastConcurrentLru<string, IgnoreFileCacheEntry>(
Environment.ProcessorCount,
cacheSize,
StringComparer.Ordinal);
_rulesCache = new FastConcurrentLru<string, ParsedIgnoreCacheEntry>(
Environment.ProcessorCount,
Math.Max(32, cacheSize / 4),
StringComparer.Ordinal);
}
/// <inheritdoc />
public bool ShouldIgnore(FileSystemMetadata fileInfo, BaseItem? parent) => IsIgnored(fileInfo, parent);
public bool ShouldIgnore(FileSystemMetadata fileInfo, BaseItem? parent) => IsIgnoredInternal(fileInfo, parent);
/// <summary>
/// Clears the directory lookup cache. The parsed rules cache is not cleared
/// as it validates file modification time on each access.
/// </summary>
public void ClearDirectoryCache()
{
_directoryCache.Clear();
}
/// <summary>
/// Checks whether or not the file is ignored.
@@ -38,40 +54,38 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule
/// <param name="fileInfo">The file information.</param>
/// <param name="parent">The parent BaseItem.</param>
/// <returns>True if the file should be ignored.</returns>
public static bool IsIgnored(FileSystemMetadata fileInfo, BaseItem? parent)
public bool IsIgnoredInternal(FileSystemMetadata fileInfo, BaseItem? parent)
{
var searchDirectory = fileInfo.IsDirectory
? new DirectoryInfo(fileInfo.FullName)
: new DirectoryInfo(Path.GetDirectoryName(fileInfo.FullName) ?? string.Empty);
? fileInfo.FullName
: Path.GetDirectoryName(fileInfo.FullName);
if (string.IsNullOrEmpty(searchDirectory.FullName))
if (string.IsNullOrEmpty(searchDirectory))
{
return false;
}
var ignoreFile = FindIgnoreFile(searchDirectory);
var ignoreFile = FindIgnoreFileCached(searchDirectory);
if (ignoreFile is null)
{
return false;
}
// Fast path in case the ignore files isn't a symlink and is empty
if (ignoreFile.LinkTarget is null && ignoreFile.Length == 0)
var parsedEntry = GetParsedRules(ignoreFile);
if (parsedEntry is null)
{
// File was deleted after we cached the path - clear the directory cache entry and return false
_directoryCache.TryRemove(searchDirectory, out _);
return false;
}
// Empty file means ignore everything
if (parsedEntry.IsEmpty)
{
// Ignore directory if we just have the file
return true;
}
var content = GetFileContent(ignoreFile);
return string.IsNullOrWhiteSpace(content)
|| CheckIgnoreRules(fileInfo.FullName, content, fileInfo.IsDirectory);
}
private static bool CheckIgnoreRules(string path, string ignoreFileContent, bool isDirectory)
{
// If file has content, base ignoring off the content .gitignore-style rules
var rules = ignoreFileContent.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
return CheckIgnoreRules(path, rules, isDirectory);
return parsedEntry.Rules.IsIgnored(GetPathToCheck(fileInfo.FullName, fileInfo.IsDirectory));
}
/// <summary>
@@ -117,8 +131,8 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule
return true;
}
// Mitigate the problem of the Ignore library not handling Windows paths correctly.
// See https://github.com/jellyfin/jellyfin/issues/15484
// Mitigate the problem of the Ignore library not handling Windows paths correctly.
// See https://github.com/jellyfin/jellyfin/issues/15484
var pathToCheck = normalizePath ? path.NormalizePath('/') : path;
// Add trailing slash for directories to match "folder/"
@@ -130,11 +144,196 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule
return ignore.IsIgnored(pathToCheck);
}
private static string GetFileContent(FileInfo ignoreFile)
private FileInfo? FindIgnoreFileCached(string directory)
{
ignoreFile = FileSystemHelper.ResolveLinkTarget(ignoreFile, returnFinalTarget: true) ?? ignoreFile;
return ignoreFile.Exists
? File.ReadAllText(ignoreFile.FullName)
: string.Empty;
// Check if we have a cached result for this directory
if (_directoryCache.TryGet(directory, out var cached))
{
return cached.IgnoreFileDirectory is null
? null
: new FileInfo(Path.Join(cached.IgnoreFileDirectory, ".ignore"));
}
DirectoryInfo startDir;
try
{
startDir = new DirectoryInfo(directory);
}
catch (ArgumentException)
{
return null;
}
// Walk up the directory tree to find .ignore file using DirectoryInfo.Parent
var checkedDirs = new List<string> { directory };
for (var current = startDir; current is not null; current = current.Parent)
{
var currentPath = current.FullName;
// Check if this intermediate directory is cached
if (current != startDir && _directoryCache.TryGet(currentPath, out var parentCached))
{
// Cache the result for all directories we checked
var entry = new IgnoreFileCacheEntry(parentCached.IgnoreFileDirectory);
foreach (var dir in checkedDirs)
{
_directoryCache.AddOrUpdate(dir, entry);
}
return parentCached.IgnoreFileDirectory is null
? null
: new FileInfo(Path.Join(parentCached.IgnoreFileDirectory, ".ignore"));
}
var ignoreFile = new FileInfo(Path.Join(currentPath, ".ignore"));
if (ignoreFile.Exists)
{
// Cache for all directories we checked
var entry = new IgnoreFileCacheEntry(currentPath);
foreach (var dir in checkedDirs)
{
_directoryCache.AddOrUpdate(dir, entry);
}
return ignoreFile;
}
if (current != startDir)
{
checkedDirs.Add(currentPath);
}
}
// No .ignore file found - cache null result for all directories
var nullEntry = new IgnoreFileCacheEntry((string?)null);
foreach (var dir in checkedDirs)
{
_directoryCache.AddOrUpdate(dir, nullEntry);
}
return null;
}
private ParsedIgnoreCacheEntry? GetParsedRules(FileInfo ignoreFile)
{
if (!ignoreFile.Exists)
{
_rulesCache.TryRemove(ignoreFile.FullName, out _);
return null;
}
var lastModified = ignoreFile.LastWriteTimeUtc;
var fileLength = ignoreFile.Length;
var key = ignoreFile.FullName;
// Check cache
if (_rulesCache.TryGet(key, out var cached))
{
if (cached.FileLastModified == lastModified && cached.FileLength == fileLength)
{
return cached;
}
// Stale - need to reparse
_rulesCache.TryRemove(key, out _);
}
// Parse the file
var parsedEntry = ParseIgnoreFile(ignoreFile, lastModified, fileLength);
_rulesCache.AddOrUpdate(key, parsedEntry);
return parsedEntry;
}
private static ParsedIgnoreCacheEntry ParseIgnoreFile(FileInfo ignoreFile, DateTime lastModified, long fileLength)
{
if (ignoreFile.LinkTarget is null && fileLength == 0)
{
return new ParsedIgnoreCacheEntry
{
Rules = new Ignore.Ignore(),
FileLastModified = lastModified,
FileLength = fileLength,
IsEmpty = true
};
}
// Resolve symlinks
var resolvedFile = FileSystemHelper.ResolveLinkTarget(ignoreFile, returnFinalTarget: true) ?? ignoreFile;
if (!resolvedFile.Exists)
{
return new ParsedIgnoreCacheEntry
{
Rules = new Ignore.Ignore(),
FileLastModified = lastModified,
FileLength = fileLength,
IsEmpty = true
};
}
var content = File.ReadAllText(resolvedFile.FullName);
if (string.IsNullOrWhiteSpace(content))
{
return new ParsedIgnoreCacheEntry
{
Rules = new Ignore.Ignore(),
FileLastModified = lastModified,
FileLength = fileLength,
IsEmpty = true
};
}
var rules = content.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
var ignore = new Ignore.Ignore();
var validRulesAdded = 0;
foreach (var rule in rules)
{
try
{
ignore.Add(rule);
validRulesAdded++;
}
catch (RegexParseException)
{
// Ignore invalid patterns
}
}
// No valid rules means treat as empty (ignore all)
return new ParsedIgnoreCacheEntry
{
Rules = ignore,
FileLastModified = lastModified,
FileLength = fileLength,
IsEmpty = validRulesAdded == 0
};
}
private static string GetPathToCheck(string path, bool isDirectory)
{
// Normalize Windows paths
var pathToCheck = IsWindows ? path.NormalizePath('/') : path;
// Add trailing slash for directories to match "folder/"
if (isDirectory)
{
pathToCheck = string.Concat(pathToCheck.AsSpan().TrimEnd('/'), "/");
}
return pathToCheck;
}
private readonly record struct IgnoreFileCacheEntry(string? IgnoreFileDirectory);
private sealed class ParsedIgnoreCacheEntry
{
public required Ignore.Ignore Rules { get; init; }
public required DateTime FileLastModified { get; init; }
public required long FileLength { get; init; }
public required bool IsEmpty { get; init; }
}
}

19
Emby.Server.Implementations/Library/LibraryManager.cs
View File

@@ -86,6 +86,7 @@ namespace Emby.Server.Implementations.Library
private readonly ExtraResolver _extraResolver;
private readonly IPathManager _pathManager;
private readonly FastConcurrentLru<Guid, BaseItem> _cache;
private readonly DotIgnoreIgnoreRule _dotIgnoreIgnoreRule;
/// <summary>
/// The _root folder sync lock.
@@ -127,6 +128,7 @@ namespace Emby.Server.Implementations.Library
/// <param name="directoryService">The directory service.</param>
/// <param name="peopleRepository">The people repository.</param>
/// <param name="pathManager">The path manager.</param>
/// <param name="dotIgnoreIgnoreRule">The .ignore rule handler.</param>
public LibraryManager(
IServerApplicationHost appHost,
ILoggerFactory loggerFactory,
@@ -148,7 +150,8 @@ namespace Emby.Server.Implementations.Library
NamingOptions namingOptions,
IDirectoryService directoryService,
IPeopleRepository peopleRepository,
IPathManager pathManager)
IPathManager pathManager,
DotIgnoreIgnoreRule dotIgnoreIgnoreRule)
{
_appHost = appHost;
_logger = loggerFactory.CreateLogger<LibraryManager>();
@@ -173,6 +176,7 @@ namespace Emby.Server.Implementations.Library
_namingOptions = namingOptions;
_peopleRepository = peopleRepository;
_pathManager = pathManager;
_dotIgnoreIgnoreRule = dotIgnoreIgnoreRule;
_extraResolver = new ExtraResolver(loggerFactory.CreateLogger<ExtraResolver>(), namingOptions, directoryService);
_configurationManager.ConfigurationUpdated += ConfigurationUpdated;
@@ -1305,6 +1309,7 @@ namespace Emby.Server.Implementations.Library
public async Task ValidateMediaLibraryInternal(IProgress<double> progress, CancellationToken cancellationToken)
{
IsScanRunning = true;
ClearIgnoreRuleCache();
LibraryMonitor.Stop();
try
@@ -1313,6 +1318,7 @@ namespace Emby.Server.Implementations.Library
}
finally
{
ClearIgnoreRuleCache();
LibraryMonitor.Start();
IsScanRunning = false;
}
@@ -1320,6 +1326,7 @@ namespace Emby.Server.Implementations.Library
public async Task ValidateTopLibraryFolders(CancellationToken cancellationToken, bool removeRoot = false)
{
ClearIgnoreRuleCache();
RootFolder.Children = null;
await RootFolder.RefreshMetadata(cancellationToken).ConfigureAwait(false);
@@ -1362,6 +1369,14 @@ namespace Emby.Server.Implementations.Library
{
_persistenceService.DeleteItem(toDelete.ToArray());
}
ClearIgnoreRuleCache();
}
/// <inheritdoc />
public void ClearIgnoreRuleCache()
{
_dotIgnoreIgnoreRule.ClearDirectoryCache();
}
private async Task PerformLibraryValidation(IProgress<double> progress, CancellationToken cancellationToken)
@@ -3182,7 +3197,7 @@ namespace Emby.Server.Implementations.Library
public IEnumerable<BaseItem> FindExtras(BaseItem owner, IReadOnlyList<FileSystemMetadata> fileSystemChildren, IDirectoryService directoryService)
{
// Apply .ignore rules
var filtered = fileSystemChildren.Where(c => !DotIgnoreIgnoreRule.IsIgnored(c, owner)).ToList();
var filtered = fileSystemChildren.Where(c => !_dotIgnoreIgnoreRule.ShouldIgnore(c, owner)).ToList();
var isFolder = owner.IsFolder || (owner is Video video && (video.VideoType == VideoType.BluRay || video.VideoType == VideoType.Dvd));
var ownerVideoInfo = VideoResolver.Resolve(owner.Path, isFolder, _namingOptions, libraryRoot: owner.ContainingFolderPath);
if (ownerVideoInfo is null)