alteropen/jellyfin
1
0
Fork 0
mirror of https://github.com/jellyfin/jellyfin.git synced 2026-05-13 04:06:31 +01:00
Code Issues Actions 7 Packages Projects Releases Wiki Activity

Implement ignore rule caching

Browse Source
This commit is contained in:
Shadowghost
2026-02-01 23:07:01 +01:00
parent 622947e374
commit d20c775daf
8 changed files with 663 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Emby.Server.Implementations/ApplicationHost.cs
View File

@@ -536,6 +536,7 @@ namespace Emby.Server.Implementations
serviceCollection.AddSingleton<IMusicManager, MusicManager>();
serviceCollection.AddSingleton<ILibraryMonitor, LibraryMonitor>();
serviceCollection.AddSingleton<DotIgnoreIgnoreRule>();
serviceCollection.AddSingleton<ISearchEngine, SearchEngine>();

8
Emby.Server.Implementations/IO/LibraryMonitor.cs
View File

@@ -21,6 +21,7 @@ namespace Emby.Server.Implementations.IO
private readonly ILibraryManager _libraryManager;
private readonly IServerConfigurationManager _configurationManager;
private readonly IFileSystem _fileSystem;
private readonly DotIgnoreIgnoreRule _dotIgnoreIgnoreRule;
/// <summary>
/// The file system watchers.
@@ -47,17 +48,20 @@ namespace Emby.Server.Implementations.IO
/// <param name="configurationManager">The configuration manager.</param>
/// <param name="fileSystem">The filesystem.</param>
/// <param name="appLifetime">The <see cref="IHostApplicationLifetime"/>.</param>
/// <param name="dotIgnoreIgnoreRule">The .ignore rule handler.</param>
public LibraryMonitor(
ILogger<LibraryMonitor> logger,
ILibraryManager libraryManager,
IServerConfigurationManager configurationManager,
IFileSystem fileSystem,
IHostApplicationLifetime appLifetime)
IHostApplicationLifetime appLifetime,
DotIgnoreIgnoreRule dotIgnoreIgnoreRule)
{
_libraryManager = libraryManager;
_logger = logger;
_configurationManager = configurationManager;
_fileSystem = fileSystem;
_dotIgnoreIgnoreRule = dotIgnoreIgnoreRule;
appLifetime.ApplicationStarted.Register(Start);
appLifetime.ApplicationStopping.Register(Stop);
@@ -354,7 +358,7 @@ namespace Emby.Server.Implementations.IO
}
var fileInfo = _fileSystem.GetFileSystemInfo(path);
if (DotIgnoreIgnoreRule.IsIgnored(fileInfo, null))
if (_dotIgnoreIgnoreRule.ShouldIgnore(fileInfo, null))
{
return;
}

272
Emby.Server.Implementations/Library/DotIgnoreIgnoreRule.cs
View File

@@ -1,6 +1,7 @@
using System;
using System.IO;
using System.Text.RegularExpressions;
using BitFaster.Caching.Lru;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.IO;
using MediaBrowser.Controller.Resolvers;
@@ -15,22 +16,36 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule
{
private static readonly bool IsWindows = OperatingSystem.IsWindows();
private static FileInfo? FindIgnoreFile(DirectoryInfo directory)
{
for (var current = directory; current is not null; current = current.Parent)
{
var ignorePath = Path.Join(current.FullName, ".ignore");
if (File.Exists(ignorePath))
{
return new FileInfo(ignorePath);
}
}
private readonly FastConcurrentLru<string, IgnoreFileCacheEntry> _directoryCache;
private readonly FastConcurrentLru<string, ParsedIgnoreCacheEntry> _rulesCache;
return null;
/// <summary>
/// Initializes a new instance of the <see cref="DotIgnoreIgnoreRule"/> class.
/// </summary>
public DotIgnoreIgnoreRule()
{
var cacheSize = Math.Max(100, Environment.ProcessorCount * 100);
_directoryCache = new FastConcurrentLru<string, IgnoreFileCacheEntry>(
Environment.ProcessorCount,
cacheSize,
StringComparer.Ordinal);
_rulesCache = new FastConcurrentLru<string, ParsedIgnoreCacheEntry>(
Environment.ProcessorCount,
Math.Max(32, cacheSize / 4),
StringComparer.Ordinal);
}
/// <inheritdoc />
public bool ShouldIgnore(FileSystemMetadata fileInfo, BaseItem? parent) => IsIgnored(fileInfo, parent);
public bool ShouldIgnore(FileSystemMetadata fileInfo, BaseItem? parent) => IsIgnoredInternal(fileInfo, parent);
/// <summary>
/// Clears the directory lookup cache. The parsed rules cache is not cleared
/// as it validates file modification time on each access.
/// </summary>
public void ClearDirectoryCache()
{
_directoryCache.Clear();
}
/// <summary>
/// Checks whether or not the file is ignored.
@@ -38,40 +53,38 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule
/// <param name="fileInfo">The file information.</param>
/// <param name="parent">The parent BaseItem.</param>
/// <returns>True if the file should be ignored.</returns>
public static bool IsIgnored(FileSystemMetadata fileInfo, BaseItem? parent)
public bool IsIgnoredInternal(FileSystemMetadata fileInfo, BaseItem? parent)
{
var searchDirectory = fileInfo.IsDirectory
? new DirectoryInfo(fileInfo.FullName)
: new DirectoryInfo(Path.GetDirectoryName(fileInfo.FullName) ?? string.Empty);
? fileInfo.FullName
: Path.GetDirectoryName(fileInfo.FullName);
if (string.IsNullOrEmpty(searchDirectory.FullName))
if (string.IsNullOrEmpty(searchDirectory))
{
return false;
}
var ignoreFile = FindIgnoreFile(searchDirectory);
if (ignoreFile is null)
var ignoreFilePath = FindIgnoreFileCached(searchDirectory);
if (ignoreFilePath is null)
{
return false;
}
// Fast path in case the ignore files isn't a symlink and is empty
if (ignoreFile.LinkTarget is null && ignoreFile.Length == 0)
var parsedEntry = GetParsedRules(ignoreFilePath);
if (parsedEntry is null)
{
// File was deleted after we cached the path - clear the directory cache entry and return false
_directoryCache.TryRemove(searchDirectory, out _);
return false;
}
// Empty file means ignore everything
if (parsedEntry.IsEmpty)
{
// Ignore directory if we just have the file
return true;
}
var content = GetFileContent(ignoreFile);
return string.IsNullOrWhiteSpace(content)
|| CheckIgnoreRules(fileInfo.FullName, content, fileInfo.IsDirectory);
}
private static bool CheckIgnoreRules(string path, string ignoreFileContent, bool isDirectory)
{
// If file has content, base ignoring off the content .gitignore-style rules
var rules = ignoreFileContent.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
return CheckIgnoreRules(path, rules, isDirectory);
return parsedEntry.Rules.IsIgnored(GetPathToCheck(fileInfo.FullName, fileInfo.IsDirectory));
}
/// <summary>
@@ -117,8 +130,8 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule
return true;
}
// Mitigate the problem of the Ignore library not handling Windows paths correctly.
// See https://github.com/jellyfin/jellyfin/issues/15484
// Mitigate the problem of the Ignore library not handling Windows paths correctly.
// See https://github.com/jellyfin/jellyfin/issues/15484
var pathToCheck = normalizePath ? path.NormalizePath('/') : path;
// Add trailing slash for directories to match "folder/"
@@ -130,11 +143,194 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule
return ignore.IsIgnored(pathToCheck);
}
private static string GetFileContent(FileInfo ignoreFile)
private string? FindIgnoreFileCached(string directory)
{
ignoreFile = FileSystemHelper.ResolveLinkTarget(ignoreFile, returnFinalTarget: true) ?? ignoreFile;
return ignoreFile.Exists
? File.ReadAllText(ignoreFile.FullName)
: string.Empty;
// Check if we have a cached result for this directory
if (_directoryCache.TryGet(directory, out var cached))
{
return cached.IgnoreFilePath;
}
// Walk up the directory tree to find .ignore file
var current = directory;
var checkedDirs = new System.Collections.Generic.List<string> { directory };
while (!string.IsNullOrEmpty(current))
{
// Check if this intermediate directory is cached
if (current != directory && _directoryCache.TryGet(current, out var parentCached))
{
// Cache the result for all directories we checked
var entry = new IgnoreFileCacheEntry(parentCached.IgnoreFilePath);
foreach (var dir in checkedDirs)
{
_directoryCache.AddOrUpdate(dir, entry);
}
return parentCached.IgnoreFilePath;
}
var ignorePath = Path.Join(current, ".ignore");
if (File.Exists(ignorePath))
{
// Cache for all directories we checked
var entry = new IgnoreFileCacheEntry(ignorePath);
foreach (var dir in checkedDirs)
{
_directoryCache.AddOrUpdate(dir, entry);
}
return ignorePath;
}
var parent = Path.GetDirectoryName(current);
if (parent == current || string.IsNullOrEmpty(parent))
{
break;
}
current = parent;
checkedDirs.Add(current);
}
// No .ignore file found - cache null result for all directories
var nullEntry = new IgnoreFileCacheEntry(null);
foreach (var dir in checkedDirs)
{
_directoryCache.AddOrUpdate(dir, nullEntry);
}
return null;
}
private ParsedIgnoreCacheEntry? GetParsedRules(string ignoreFilePath)
{
FileInfo fileInfo;
try
{
fileInfo = new FileInfo(ignoreFilePath);
if (!fileInfo.Exists)
{
_rulesCache.TryRemove(ignoreFilePath, out _);
return null;
}
}
catch
{
_rulesCache.TryRemove(ignoreFilePath, out _);
return null;
}
var lastModified = fileInfo.LastWriteTimeUtc;
var fileLength = fileInfo.Length;
// Check cache
if (_rulesCache.TryGet(ignoreFilePath, out var cached))
{
if (cached.FileLastModified == lastModified && cached.FileLength == fileLength)
{
return cached;
}
// Stale - need to reparse
_rulesCache.TryRemove(ignoreFilePath, out _);
}
// Parse the file
var parsedEntry = ParseIgnoreFile(fileInfo, lastModified, fileLength);
_rulesCache.AddOrUpdate(ignoreFilePath, parsedEntry);
return parsedEntry;
}
private static ParsedIgnoreCacheEntry ParseIgnoreFile(FileInfo ignoreFile, DateTime lastModified, long fileLength)
{
if (ignoreFile.LinkTarget is null && fileLength == 0)
{
return new ParsedIgnoreCacheEntry
{
Rules = new Ignore.Ignore(),
FileLastModified = lastModified,
FileLength = fileLength,
IsEmpty = true
};
}
// Resolve symlinks
var resolvedFile = FileSystemHelper.ResolveLinkTarget(ignoreFile, returnFinalTarget: true) ?? ignoreFile;
if (!resolvedFile.Exists)
{
return new ParsedIgnoreCacheEntry
{
Rules = new Ignore.Ignore(),
FileLastModified = lastModified,
FileLength = fileLength,
IsEmpty = true
};
}
var content = File.ReadAllText(resolvedFile.FullName);
if (string.IsNullOrWhiteSpace(content))
{
return new ParsedIgnoreCacheEntry
{
Rules = new Ignore.Ignore(),
FileLastModified = lastModified,
FileLength = fileLength,
IsEmpty = true
};
}
var rules = content.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
var ignore = new Ignore.Ignore();
var validRulesAdded = 0;
foreach (var rule in rules)
{
try
{
ignore.Add(rule);
validRulesAdded++;
}
catch (RegexParseException)
{
// Ignore invalid patterns
}
}
// No valid rules means treat as empty (ignore all)
return new ParsedIgnoreCacheEntry
{
Rules = ignore,
FileLastModified = lastModified,
FileLength = fileLength,
IsEmpty = validRulesAdded == 0
};
}
private static string GetPathToCheck(string path, bool isDirectory)
{
// Normalize Windows paths
var pathToCheck = IsWindows ? path.NormalizePath('/') : path;
// Add trailing slash for directories to match "folder/"
if (isDirectory)
{
pathToCheck = string.Concat(pathToCheck.AsSpan().TrimEnd('/'), "/");
}
return pathToCheck;
}
private readonly record struct IgnoreFileCacheEntry(string? IgnoreFilePath);
private sealed class ParsedIgnoreCacheEntry
{
public required Ignore.Ignore Rules { get; init; }
public required DateTime FileLastModified { get; init; }
public required long FileLength { get; init; }
public required bool IsEmpty { get; init; }
}
}

19
Emby.Server.Implementations/Library/LibraryManager.cs
View File

@@ -84,6 +84,7 @@ namespace Emby.Server.Implementations.Library
private readonly ExtraResolver _extraResolver;
private readonly IPathManager _pathManager;
private readonly FastConcurrentLru<Guid, BaseItem> _cache;
private readonly DotIgnoreIgnoreRule _dotIgnoreIgnoreRule;
/// <summary>
/// The _root folder sync lock.
@@ -125,6 +126,7 @@ namespace Emby.Server.Implementations.Library
/// <param name="directoryService">The directory service.</param>
/// <param name="peopleRepository">The people repository.</param>
/// <param name="pathManager">The path manager.</param>
/// <param name="dotIgnoreIgnoreRule">The .ignore rule handler.</param>
public LibraryManager(
IServerApplicationHost appHost,
ILoggerFactory loggerFactory,
@@ -146,7 +148,8 @@ namespace Emby.Server.Implementations.Library
NamingOptions namingOptions,
IDirectoryService directoryService,
IPeopleRepository peopleRepository,
IPathManager pathManager)
IPathManager pathManager,
DotIgnoreIgnoreRule dotIgnoreIgnoreRule)
{
_appHost = appHost;
_logger = loggerFactory.CreateLogger<LibraryManager>();
@@ -171,6 +174,7 @@ namespace Emby.Server.Implementations.Library
_namingOptions = namingOptions;
_peopleRepository = peopleRepository;
_pathManager = pathManager;
_dotIgnoreIgnoreRule = dotIgnoreIgnoreRule;
_extraResolver = new ExtraResolver(loggerFactory.CreateLogger<ExtraResolver>(), namingOptions, directoryService);
_configurationManager.ConfigurationUpdated += ConfigurationUpdated;
@@ -1303,6 +1307,7 @@ namespace Emby.Server.Implementations.Library
public async Task ValidateMediaLibraryInternal(IProgress<double> progress, CancellationToken cancellationToken)
{
IsScanRunning = true;
ClearIgnoreRuleCache();
LibraryMonitor.Stop();
try
@@ -1311,6 +1316,7 @@ namespace Emby.Server.Implementations.Library
}
finally
{
ClearIgnoreRuleCache();
LibraryMonitor.Start();
IsScanRunning = false;
}
@@ -1318,6 +1324,7 @@ namespace Emby.Server.Implementations.Library
public async Task ValidateTopLibraryFolders(CancellationToken cancellationToken, bool removeRoot = false)
{
ClearIgnoreRuleCache();
RootFolder.Children = null;
await RootFolder.RefreshMetadata(cancellationToken).ConfigureAwait(false);
@@ -1360,6 +1367,14 @@ namespace Emby.Server.Implementations.Library
{
_persistenceService.DeleteItem(toDelete.ToArray());
}
ClearIgnoreRuleCache();
}
/// <inheritdoc />
public void ClearIgnoreRuleCache()
{
_dotIgnoreIgnoreRule.ClearDirectoryCache();
}
private async Task PerformLibraryValidation(IProgress<double> progress, CancellationToken cancellationToken)
@@ -3161,7 +3176,7 @@ namespace Emby.Server.Implementations.Library
public IEnumerable<BaseItem> FindExtras(BaseItem owner, IReadOnlyList<FileSystemMetadata> fileSystemChildren, IDirectoryService directoryService)
{
// Apply .ignore rules
var filtered = fileSystemChildren.Where(c => !DotIgnoreIgnoreRule.IsIgnored(c, owner)).ToList();
var filtered = fileSystemChildren.Where(c => !_dotIgnoreIgnoreRule.ShouldIgnore(c, owner)).ToList();
var isFolder = owner.IsFolder || (owner is Video video && (video.VideoType == VideoType.BluRay || video.VideoType == VideoType.Dvd));
var ownerVideoInfo = VideoResolver.Resolve(owner.Path, isFolder, _namingOptions, libraryRoot: owner.ContainingFolderPath);
if (ownerVideoInfo is null)