alteropen/jellyfin
1
0
Fork 0
mirror of https://github.com/jellyfin/jellyfin.git synced 2026-05-03 23:36:38 +01:00
Code Issues Actions 7 Packages Projects Releases Wiki Activity

Optimize migrations

Browse Source
This commit is contained in:
Shadowghost
2026-03-07 21:23:16 +01:00
parent ea1c1d0468
commit 5882006ee7
5 changed files with 109 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
Emby.Server.Implementations/IO/ManagedFileSystem.cs
View File

@@ -586,6 +586,12 @@ namespace Emby.Server.Implementations.IO
/// <inheritdoc />
public virtual IEnumerable<FileSystemMetadata> GetFiles(string path, string searchPattern, IReadOnlyList<string>? extensions, bool enableCaseSensitiveExtensions, bool recursive = false)
{
if (!Directory.Exists(path))
{
_logger.LogWarning("Directory does not exist: {Path}", path);
return [];
}
var enumerationOptions = GetEnumerationOptions(recursive);
// On linux and macOS the search pattern is case-sensitive

36
Jellyfin.Server/Migrations/Routines/CleanupOrphanedExtras.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
@@ -12,6 +13,7 @@ using MediaBrowser.Controller.Library;
using MediaBrowser.Controller.LiveTv;
using MediaBrowser.Controller.MediaSegments;
using MediaBrowser.Controller.Persistence;
using MediaBrowser.Model.IO;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
@@ -42,6 +44,7 @@ public class CleanupOrphanedExtras : IAsyncMigrationRoutine
/// <param name="mediaSourceManager">The media source manager.</param>
/// <param name="mediaSegmentManager">The media segments manager.</param>
/// <param name="configurationManager">The configuration manager.</param>
/// <param name="fileSystem">The file system.</param>
public CleanupOrphanedExtras(
IStartupLogger<CleanupOrphanedExtras> logger,
IDbContextFactory<JellyfinDbContext> dbContextFactory,
@@ -52,7 +55,8 @@ public class CleanupOrphanedExtras : IAsyncMigrationRoutine
IRecordingsManager recordingsManager,
IMediaSourceManager mediaSourceManager,
IMediaSegmentManager mediaSegmentManager,
IServerConfigurationManager configurationManager)
IServerConfigurationManager configurationManager,
IFileSystem fileSystem)
{
_logger = logger;
_dbContextFactory = dbContextFactory;
@@ -64,6 +68,7 @@ public class CleanupOrphanedExtras : IAsyncMigrationRoutine
BaseItem.MediaSourceManager ??= mediaSourceManager;
BaseItem.MediaSegmentManager ??= mediaSegmentManager;
BaseItem.ConfigurationManager ??= configurationManager;
BaseItem.FileSystem ??= fileSystem;
Video.RecordingsManager ??= recordingsManager;
}
@@ -88,35 +93,20 @@ public class CleanupOrphanedExtras : IAsyncMigrationRoutine
_logger.LogInformation("Found {Count} orphaned extras to remove", orphanedItemIds.Count);
var deleteOptions = new DeleteOptions
{
DeleteFileLocation = false // Extras don't have their own media files
};
var deletedCount = 0;
// Batch-resolve items for metadata path cleanup, then delete all at once
var itemsToDelete = new List<BaseItem>();
foreach (var itemId in orphanedItemIds)
{
cancellationToken.ThrowIfCancellationRequested();
var item = _libraryManager.GetItemById(itemId);
if (item is null)
if (item is not null)
{
_logger.LogDebug("Item {ItemId} not found in library, may have been already deleted", itemId);
continue;
}
try
{
_libraryManager.DeleteItem(item, deleteOptions, notifyParentItem: false);
deletedCount++;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to delete orphaned item {ItemId} ({ItemName})", item.Id, item.Name);
itemsToDelete.Add(item);
}
}
_logger.LogInformation("Successfully removed {Count} orphaned extras", deletedCount);
_libraryManager.DeleteItemsUnsafeFast(itemsToDelete);
_logger.LogInformation("Successfully removed {Count} orphaned extras", itemsToDelete.Count);
}
}
}

139
Jellyfin.Server/Migrations/Routines/FixIncorrectOwnerIdRelationships.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
@@ -84,12 +85,8 @@ public class FixIncorrectOwnerIdRelationships : IAsyncMigrationRoutine
_logger.LogInformation("Found {Count} paths with duplicate database entries", duplicatePaths.Count);
var deleteOptions = new DeleteOptions
{
DeleteFileLocation = false // Don't delete the actual file, just the database entry
};
var deletedCount = 0;
// Collect all duplicate IDs to delete in one batch
var allIdsToDelete = new List<Guid>();
foreach (var path in duplicatePaths)
{
cancellationToken.ThrowIfCancellationRequested();
@@ -125,52 +122,28 @@ public class FixIncorrectOwnerIdRelationships : IAsyncMigrationRoutine
continue;
}
// Delete all other items with this path
var itemsToDelete = itemsWithPath.Where(i => !i.Id.Equals(itemToKeep.Id)).Select(i => i.Id).ToList();
foreach (var itemId in itemsToDelete)
{
var item = _libraryManager.GetItemById(itemId);
if (item is not null)
{
var deleted = false;
try
{
_libraryManager.DeleteItem(item, deleteOptions, notifyParentItem: false);
deletedCount++;
deleted = true;
}
catch (InvalidOperationException ex)
{
_logger.LogWarning(ex, "Failed to delete duplicate item {ItemId} at path {Path}", itemId, path);
}
catch (UnauthorizedAccessException ex)
{
_logger.LogWarning(ex, "Failed to delete duplicate item {ItemId} at path {Path}", itemId, path);
}
catch (NullReferenceException ex)
{
_logger.LogWarning(ex, "Failed to delete duplicate item {ItemId} at path {Path} via LibraryManager - falling back to direct database deletion", itemId, path);
}
allIdsToDelete.AddRange(itemsWithPath.Where(i => !i.Id.Equals(itemToKeep.Id)).Select(i => i.Id));
}
// If LibraryManager.DeleteItem failed, delete directly from database
if (!deleted)
{
try
{
_persistenceService.DeleteItem([itemId]);
deletedCount++;
_logger.LogInformation("Successfully deleted duplicate item {ItemId} at path {Path} via direct database deletion", itemId, path);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to delete duplicate item {ItemId} at path {Path} via direct database deletion", itemId, path);
}
}
}
if (allIdsToDelete.Count > 0)
{
// Batch-resolve items for metadata path cleanup, then delete all at once
var itemsToDelete = allIdsToDelete
.Select(id => _libraryManager.GetItemById(id))
.Where(item => item is not null)
.ToList();
_libraryManager.DeleteItemsUnsafeFast(itemsToDelete!);
// Fall back to direct DB deletion for any items that couldn't be resolved via LibraryManager
var deletedIds = itemsToDelete.Select(i => i!.Id).ToHashSet();
var unresolvedIds = allIdsToDelete.Where(id => !deletedIds.Contains(id)).ToList();
if (unresolvedIds.Count > 0)
{
_persistenceService.DeleteItem(unresolvedIds);
}
}
_logger.LogInformation("Successfully removed {Count} duplicate database entries", deletedCount);
_logger.LogInformation("Successfully removed {Count} duplicate database entries", allIdsToDelete.Count);
}
private async Task ClearIncorrectOwnerIdsAsync(JellyfinDbContext context, CancellationToken cancellationToken)
@@ -236,39 +209,56 @@ public class FixIncorrectOwnerIdRelationships : IAsyncMigrationRoutine
_logger.LogInformation("Found {Count} orphaned extras to reassign", orphanedExtras.Count);
// Build a lookup of directory -> first video/movie item for parent resolution
var extraDirectories = orphanedExtras
.Where(e => !string.IsNullOrEmpty(e.Path))
.Select(e => System.IO.Path.GetDirectoryName(e.Path))
.Where(d => !string.IsNullOrEmpty(d))
.Distinct()
.ToList();
// Load all potential parent video/movies with paths in one query
var videoTypes = new[]
{
"MediaBrowser.Controller.Entities.Video",
"MediaBrowser.Controller.Entities.Movies.Movie"
};
var potentialParents = await context.BaseItems
.Where(b => b.Path != null && videoTypes.Contains(b.Type))
.Select(b => new { b.Id, b.Path })
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
// Build directory -> parent ID mapping
var dirToParent = new Dictionary<string, Guid>();
foreach (var dir in extraDirectories)
{
var parent = potentialParents
.Where(p => p.Path!.StartsWith(dir!, StringComparison.OrdinalIgnoreCase))
.OrderBy(p => p.Id)
.FirstOrDefault();
if (parent is not null)
{
dirToParent[dir!] = parent.Id;
}
}
var reassignedCount = 0;
foreach (var extra in orphanedExtras)
{
cancellationToken.ThrowIfCancellationRequested();
// Find the parent path from the extra's path (extras are usually in same directory as parent)
if (string.IsNullOrEmpty(extra.Path))
{
continue;
}
var extraDirectory = System.IO.Path.GetDirectoryName(extra.Path);
if (string.IsNullOrEmpty(extraDirectory))
if (!string.IsNullOrEmpty(extraDirectory) && dirToParent.TryGetValue(extraDirectory, out var parentId))
{
continue;
}
// Find potential parent in same directory
var potentialParent = await context.BaseItems
.Where(b => b.Path != null && b.Path.StartsWith(extraDirectory))
.Where(b => b.Type == "MediaBrowser.Controller.Entities.Video" || b.Type == "MediaBrowser.Controller.Entities.Movies.Movie")
.OrderBy(b => b.Id)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
if (potentialParent is not null)
{
extra.OwnerId = potentialParent.Id;
extra.OwnerId = parentId;
reassignedCount++;
}
else
{
// Can't find a parent, clear the OwnerId
extra.OwnerId = null;
}
}
@@ -301,16 +291,17 @@ public class FixIncorrectOwnerIdRelationships : IAsyncMigrationRoutine
_logger.LogInformation("Found {Count} alternate version items that need PrimaryVersionId populated", alternateVersionLinks.Count);
// Batch-load all child items in a single query
var childIds = alternateVersionLinks.Select(l => l.ChildId).Distinct().ToList();
var childItems = await context.BaseItems
.Where(b => childIds.Contains(b.Id))
.ToDictionaryAsync(b => b.Id, cancellationToken)
.ConfigureAwait(false);
var updatedCount = 0;
foreach (var link in alternateVersionLinks)
{
cancellationToken.ThrowIfCancellationRequested();
var childItem = await context.BaseItems
.FirstOrDefaultAsync(b => b.Id.Equals(link.ChildId), cancellationToken)
.ConfigureAwait(false);
if (childItem is not null)
if (childItems.TryGetValue(link.ChildId, out var childItem))
{
childItem.PrimaryVersionId = link.ParentId;
updatedCount++;

60
Jellyfin.Server/Migrations/Routines/MigrateLinkedChildren.cs
View File

@@ -272,16 +272,13 @@ internal class MigrateLinkedChildren : IDatabaseMigrationRoutine
_logger.LogInformation("Found {Count} wrong-type alternate version items to remove.", wrongTypeChildIds.Count);
foreach (var childId in wrongTypeChildIds)
{
var item = _libraryManager.GetItemById(childId);
if (item is not null)
{
_libraryManager.DeleteItem(item, new DeleteOptions { DeleteFileLocation = false });
}
}
var itemsToDelete = wrongTypeChildIds
.Select(id => _libraryManager.GetItemById(id))
.Where(item => item is not null)
.ToList();
_libraryManager.DeleteItemsUnsafeFast(itemsToDelete!);
_logger.LogInformation("Removed {Count} wrong-type alternate version items. They will be recreated with the correct type on next library scan.", wrongTypeChildIds.Count);
_logger.LogInformation("Removed {Count} wrong-type alternate version items. They will be recreated with the correct type on next library scan.", itemsToDelete.Count);
}
private void CleanupOrphanedAlternateVersionBaseItems(JellyfinDbContext context)
@@ -306,16 +303,13 @@ internal class MigrateLinkedChildren : IDatabaseMigrationRoutine
_logger.LogInformation("Found {Count} orphaned alternate version BaseItems to remove.", orphanedVersionIds.Count);
foreach (var id in orphanedVersionIds)
{
var item = _libraryManager.GetItemById(id);
if (item is not null)
{
_libraryManager.DeleteItem(item, new DeleteOptions { DeleteFileLocation = false });
}
}
var itemsToDelete = orphanedVersionIds
.Select(id => _libraryManager.GetItemById(id))
.Where(item => item is not null)
.ToList();
_libraryManager.DeleteItemsUnsafeFast(itemsToDelete!);
_logger.LogInformation("Removed {Count} orphaned alternate version BaseItems.", orphanedVersionIds.Count);
_logger.LogInformation("Removed {Count} orphaned alternate version BaseItems.", itemsToDelete.Count);
}
private void CleanupItemsFromDeletedLibraries(JellyfinDbContext context)
@@ -338,16 +332,13 @@ internal class MigrateLinkedChildren : IDatabaseMigrationRoutine
_logger.LogInformation("Found {Count} items from deleted libraries to remove.", orphanedIds.Count);
foreach (var id in orphanedIds)
{
var item = _libraryManager.GetItemById(id);
if (item is not null)
{
_libraryManager.DeleteItem(item, new DeleteOptions { DeleteFileLocation = false });
}
}
var itemsToDelete = orphanedIds
.Select(id => _libraryManager.GetItemById(id))
.Where(item => item is not null)
.ToList();
_libraryManager.DeleteItemsUnsafeFast(itemsToDelete!);
_logger.LogInformation("Removed {Count} items from deleted libraries.", orphanedIds.Count);
_logger.LogInformation("Removed {Count} items from deleted libraries.", itemsToDelete.Count);
}
private void CleanupStaleFileEntries(JellyfinDbContext context)
@@ -429,16 +420,13 @@ internal class MigrateLinkedChildren : IDatabaseMigrationRoutine
_logger.LogInformation("Found {Count} stale items to remove.", staleIds.Count);
foreach (var id in staleIds)
{
var item = _libraryManager.GetItemById(id);
if (item is not null)
{
_libraryManager.DeleteItem(item, new DeleteOptions { DeleteFileLocation = false });
}
}
var itemsToDelete = staleIds
.Select(id => _libraryManager.GetItemById(id))
.Where(item => item is not null)
.ToList();
_libraryManager.DeleteItemsUnsafeFast(itemsToDelete!);
_logger.LogInformation("Removed {Count} stale items.", staleIds.Count);
_logger.LogInformation("Removed {Count} stale items.", itemsToDelete.Count);
}
private void CleanupOrphanedLinkedChildren(JellyfinDbContext context)

5
Jellyfin.Server/Migrations/Routines/RefreshCleanNames.cs
View File

@@ -1,13 +1,10 @@
using System;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Jellyfin.Database.Implementations;
using Jellyfin.Database.Implementations.Entities;
using Jellyfin.Extensions;
using Jellyfin.Server.Implementations.Item;
using Jellyfin.Server.ServerSetupApp;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
@@ -40,7 +37,7 @@ public class RefreshCleanNames : IAsyncMigrationRoutine
/// <inheritdoc />
public async Task PerformAsync(CancellationToken cancellationToken)
{
const int Limit = 1000;
const int Limit = 10000;
int itemCount = 0;
var sw = Stopwatch.StartNew();