alteropen/jellyfin
1
0
Fork 0
mirror of https://github.com/jellyfin/jellyfin.git synced 2026-06-28 10:30:57 +01:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

Batch duplicate-cleanup deletes in merge migrations

Browse Source
This commit is contained in:
Shadowghost
2026-06-22 23:04:12 +02:00
parent 3741d71965
commit af82aceadb
3 changed files with 79 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
Jellyfin.Server/Migrations/Routines/20260113230000_CleanupOrphanedExtras.cs
View File

@@ -76,25 +76,36 @@ public class CleanupOrphanedExtras : IAsyncMigrationRoutine
_logger.LogInformation("Found {Count} orphaned extras to remove", orphanedItemIds.Count);
// Batch-resolve items for metadata path cleanup, then delete all at once
var itemsToDelete = new List<BaseItem>();
foreach (var itemId in orphanedItemIds)
// Resolve items for metadata path cleanup, then delete in batches so we never issue one
// massive delete transaction and progress stays visible on large libraries.
_logger.LogInformation("Deleting {Count} orphaned extras...", orphanedItemIds.Count);
const int deleteBatchSize = 500;
var deletedSoFar = 0;
for (var offset = 0; offset < orphanedItemIds.Count; offset += deleteBatchSize)
{
itemsToDelete.Add(BaseItemMapper.DeserializeBaseItem(
new Database.Implementations.Entities.BaseItemEntity()
{
Id = itemId.Id,
Path = itemId.Path,
Type = itemId.Type
},
_logger,
null,
true)!);
cancellationToken.ThrowIfCancellationRequested();
var batch = orphanedItemIds.GetRange(offset, Math.Min(deleteBatchSize, orphanedItemIds.Count - offset));
var itemsToDelete = batch
.Select(itemId => BaseItemMapper.DeserializeBaseItem(
new Database.Implementations.Entities.BaseItemEntity()
{
Id = itemId.Id,
Path = itemId.Path,
Type = itemId.Type
},
_logger,
null,
true)!)
.ToList();
_libraryManager.DeleteItemsUnsafeFast(itemsToDelete);
deletedSoFar += batch.Count;
_logger.LogInformation("Deleting orphaned extras: {Deleted}/{Total}", deletedSoFar, orphanedItemIds.Count);
}
_libraryManager.DeleteItemsUnsafeFast(itemsToDelete);
_logger.LogInformation("Successfully removed {Count} orphaned extras", itemsToDelete.Count);
_logger.LogInformation("Successfully removed {Count} orphaned extras", orphanedItemIds.Count);
}
}
}

40
Jellyfin.Server/Migrations/Routines/20260508120000_MergeDuplicateMusicArtists.cs
View File

@@ -182,23 +182,35 @@ public class MergeDuplicateMusicArtists : IAsyncMigrationRoutine
// Resolve via LibraryManager so DeleteItemsUnsafeFast can also remove the
// %MetadataPath%/artists/<Name> directories that the duplicate stubs left behind.
// Fall back to the persistence service for any items the LibraryManager can't resolve.
var itemsToDelete = idsToDelete
.Select(id => _libraryManager.GetItemById(id))
.Where(item => item is not null)
.ToList();
if (itemsToDelete.Count > 0)
// Delete in batches so we never issue one massive delete transaction and progress stays visible.
_logger.LogInformation("Deleting {Count} duplicate MusicArtist records...", idsToDelete.Count);
const int deleteBatchSize = 500;
var deletedSoFar = 0;
for (var offset = 0; offset < idsToDelete.Count; offset += deleteBatchSize)
{
_libraryManager.DeleteItemsUnsafeFast(itemsToDelete!);
}
cancellationToken.ThrowIfCancellationRequested();
var deletedIds = itemsToDelete.Select(i => i!.Id).ToHashSet();
var unresolvedIds = idsToDelete.Where(id => !deletedIds.Contains(id)).ToList();
if (unresolvedIds.Count > 0)
{
_persistenceService.DeleteItem(unresolvedIds);
}
var batchIds = idsToDelete.GetRange(offset, Math.Min(deleteBatchSize, idsToDelete.Count - offset));
_logger.LogInformation("Removed {Count} duplicate MusicArtist records.", idsToDelete.Count);
var itemsToDelete = batchIds
.Select(id => _libraryManager.GetItemById(id))
.Where(item => item is not null)
.ToList();
if (itemsToDelete.Count > 0)
{
_libraryManager.DeleteItemsUnsafeFast(itemsToDelete!);
}
var deletedIds = itemsToDelete.Select(i => i!.Id).ToHashSet();
var unresolvedIds = batchIds.Where(id => !deletedIds.Contains(id)).ToList();
if (unresolvedIds.Count > 0)
{
_persistenceService.DeleteItem(unresolvedIds);
}
deletedSoFar += batchIds.Count;
_logger.LogInformation("Deleting duplicate MusicArtist records: {Deleted}/{Total}", deletedSoFar, idsToDelete.Count);
}
}
}
}

40
Jellyfin.Server/Migrations/Routines/20260508130000_MergeDuplicatePeople.cs
View File

@@ -184,23 +184,35 @@ public class MergeDuplicatePeople : IAsyncMigrationRoutine
// Resolve via LibraryManager so DeleteItemsUnsafeFast can also remove the
// %MetadataPath%/People/<Letter>/<Name> directories the duplicate stubs left behind.
var itemsToDelete = idsToDelete
.Select(id => _libraryManager.GetItemById(id))
.Where(item => item is not null)
.ToList();
if (itemsToDelete.Count > 0)
// Delete in batches so we never issue one massive delete transaction and progress stays visible.
_logger.LogInformation("Deleting {Count} duplicate Person BaseItems...", idsToDelete.Count);
const int deleteBatchSize = 500;
var deletedSoFar = 0;
for (var offset = 0; offset < idsToDelete.Count; offset += deleteBatchSize)
{
_libraryManager.DeleteItemsUnsafeFast(itemsToDelete!);
}
cancellationToken.ThrowIfCancellationRequested();
var deletedIds = itemsToDelete.Select(i => i!.Id).ToHashSet();
var unresolvedIds = idsToDelete.Where(id => !deletedIds.Contains(id)).ToList();
if (unresolvedIds.Count > 0)
{
_persistenceService.DeleteItem(unresolvedIds);
}
var batchIds = idsToDelete.GetRange(offset, Math.Min(deleteBatchSize, idsToDelete.Count - offset));
_logger.LogInformation("Removed {Count} duplicate Person BaseItems.", idsToDelete.Count);
var itemsToDelete = batchIds
.Select(id => _libraryManager.GetItemById(id))
.Where(item => item is not null)
.ToList();
if (itemsToDelete.Count > 0)
{
_libraryManager.DeleteItemsUnsafeFast(itemsToDelete!);
}
var deletedIds = itemsToDelete.Select(i => i!.Id).ToHashSet();
var unresolvedIds = batchIds.Where(id => !deletedIds.Contains(id)).ToList();
if (unresolvedIds.Count > 0)
{
_persistenceService.DeleteItem(unresolvedIds);
}
deletedSoFar += batchIds.Count;
_logger.LogInformation("Deleting duplicate Person BaseItems: {Deleted}/{Total}", deletedSoFar, idsToDelete.Count);
}
}
private async Task MergePeoplesRowsAsync(JellyfinDbContext context, CancellationToken cancellationToken)