mirror of
https://github.com/jellyfin/jellyfin.git
synced 2026-02-11 15:12:24 +00:00
begin to rework repositories
This commit is contained in:
Luke Pulverenti
123
Emby.Server.Implementations/Data/BaseSqliteRepository.cs
Normal file
123
Emby.Server.Implementations/Data/BaseSqliteRepository.cs
Normal file
@@ -0,0 +1,123 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MediaBrowser.Model.Logging;
|
||||
using SQLitePCL.pretty;
|
||||
|
||||
namespace Emby.Server.Implementations.Data
|
||||
{
|
||||
public abstract class BaseSqliteRepository : IDisposable
|
||||
{
|
||||
protected string DbFilePath { get; set; }
|
||||
protected SemaphoreSlim WriteLock = new SemaphoreSlim(1, 1);
|
||||
protected ILogger Logger { get; private set; }
|
||||
|
||||
protected BaseSqliteRepository(ILogger logger)
|
||||
{
|
||||
Logger = logger;
|
||||
}
|
||||
|
||||
protected virtual bool EnableConnectionPooling
|
||||
{
|
||||
get { return true; }
|
||||
}
|
||||
|
||||
protected virtual SQLiteDatabaseConnection CreateConnection(bool isReadOnly = false)
|
||||
{
|
||||
SQLite3.EnableSharedCache = false;
|
||||
|
||||
ConnectionFlags connectionFlags;
|
||||
|
||||
if (isReadOnly)
|
||||
{
|
||||
connectionFlags = ConnectionFlags.ReadOnly;
|
||||
//connectionFlags = ConnectionFlags.Create;
|
||||
//connectionFlags |= ConnectionFlags.ReadWrite;
|
||||
}
|
||||
else
|
||||
{
|
||||
connectionFlags = ConnectionFlags.Create;
|
||||
connectionFlags |= ConnectionFlags.ReadWrite;
|
||||
}
|
||||
|
||||
if (EnableConnectionPooling)
|
||||
{
|
||||
connectionFlags |= ConnectionFlags.SharedCached;
|
||||
}
|
||||
else
|
||||
{
|
||||
connectionFlags |= ConnectionFlags.PrivateCache;
|
||||
}
|
||||
|
||||
connectionFlags |= ConnectionFlags.NoMutex;
|
||||
|
||||
var db = SQLite3.Open(DbFilePath, connectionFlags, null);
|
||||
|
||||
var queries = new[]
|
||||
{
|
||||
"PRAGMA page_size=4096",
|
||||
"PRAGMA journal_mode=WAL",
|
||||
"PRAGMA temp_store=memory",
|
||||
"PRAGMA synchronous=Normal",
|
||||
//"PRAGMA cache size=-10000"
|
||||
};
|
||||
|
||||
//foreach (var query in queries)
|
||||
//{
|
||||
// db.Execute(query);
|
||||
//}
|
||||
|
||||
db.ExecuteAll(string.Join(";", queries));
|
||||
|
||||
return db;
|
||||
}
|
||||
|
||||
private bool _disposed;
|
||||
protected void CheckDisposed()
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
throw new ObjectDisposedException(GetType().Name + " has been disposed and cannot be accessed.");
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_disposed = true;
|
||||
Dispose(true);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
private readonly object _disposeLock = new object();
|
||||
|
||||
/// <summary>
|
||||
/// Releases unmanaged and - optionally - managed resources.
|
||||
/// </summary>
|
||||
/// <param name="dispose"><c>true</c> to release both managed and unmanaged resources; <c>false</c> to release only unmanaged resources.</param>
|
||||
protected virtual void Dispose(bool dispose)
|
||||
{
|
||||
if (dispose)
|
||||
{
|
||||
try
|
||||
{
|
||||
lock (_disposeLock)
|
||||
{
|
||||
WriteLock.Wait();
|
||||
|
||||
CloseConnection();
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.ErrorException("Error disposing database", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected virtual void CloseConnection()
|
||||
{
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
361
Emby.Server.Implementations/Data/CleanDatabaseScheduledTask.cs
Normal file
361
Emby.Server.Implementations/Data/CleanDatabaseScheduledTask.cs
Normal file
@@ -0,0 +1,361 @@
|
||||
using MediaBrowser.Common.Progress;
|
||||
using MediaBrowser.Controller.Configuration;
|
||||
using MediaBrowser.Controller.Entities;
|
||||
using MediaBrowser.Controller.Library;
|
||||
using MediaBrowser.Controller.Persistence;
|
||||
using MediaBrowser.Model.Entities;
|
||||
using MediaBrowser.Model.Logging;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MediaBrowser.Common.IO;
|
||||
using MediaBrowser.Model.IO;
|
||||
using MediaBrowser.Controller.Channels;
|
||||
using MediaBrowser.Controller.Entities.Audio;
|
||||
using MediaBrowser.Controller.IO;
|
||||
using MediaBrowser.Controller.LiveTv;
|
||||
using MediaBrowser.Controller.Net;
|
||||
using MediaBrowser.Model.Globalization;
|
||||
using MediaBrowser.Model.Tasks;
|
||||
using Emby.Server.Implementations.ScheduledTasks;
|
||||
|
||||
namespace Emby.Server.Implementations.Data
|
||||
{
|
||||
public class CleanDatabaseScheduledTask : IScheduledTask
|
||||
{
|
||||
private readonly ILibraryManager _libraryManager;
|
||||
private readonly IItemRepository _itemRepo;
|
||||
private readonly ILogger _logger;
|
||||
private readonly IServerConfigurationManager _config;
|
||||
private readonly IFileSystem _fileSystem;
|
||||
private readonly IHttpServer _httpServer;
|
||||
private readonly ILocalizationManager _localization;
|
||||
private readonly ITaskManager _taskManager;
|
||||
|
||||
public const int MigrationVersion = 23;
|
||||
public static bool EnableUnavailableMessage = false;
|
||||
const int LatestSchemaVersion = 109;
|
||||
|
||||
public CleanDatabaseScheduledTask(ILibraryManager libraryManager, IItemRepository itemRepo, ILogger logger, IServerConfigurationManager config, IFileSystem fileSystem, IHttpServer httpServer, ILocalizationManager localization, ITaskManager taskManager)
|
||||
{
|
||||
_libraryManager = libraryManager;
|
||||
_itemRepo = itemRepo;
|
||||
_logger = logger;
|
||||
_config = config;
|
||||
_fileSystem = fileSystem;
|
||||
_httpServer = httpServer;
|
||||
_localization = localization;
|
||||
_taskManager = taskManager;
|
||||
}
|
||||
|
||||
public string Name
|
||||
{
|
||||
get { return "Clean Database"; }
|
||||
}
|
||||
|
||||
public string Description
|
||||
{
|
||||
get { return "Deletes obsolete content from the database."; }
|
||||
}
|
||||
|
||||
public string Category
|
||||
{
|
||||
get { return "Library"; }
|
||||
}
|
||||
|
||||
public async Task Execute(CancellationToken cancellationToken, IProgress<double> progress)
|
||||
{
|
||||
OnProgress(0);
|
||||
|
||||
// Ensure these objects are lazy loaded.
|
||||
// Without this there is a deadlock that will need to be investigated
|
||||
var rootChildren = _libraryManager.RootFolder.Children.ToList();
|
||||
rootChildren = _libraryManager.GetUserRootFolder().Children.ToList();
|
||||
|
||||
var innerProgress = new ActionableProgress<double>();
|
||||
innerProgress.RegisterAction(p =>
|
||||
{
|
||||
double newPercentCommplete = .4 * p;
|
||||
OnProgress(newPercentCommplete);
|
||||
|
||||
progress.Report(newPercentCommplete);
|
||||
});
|
||||
|
||||
await UpdateToLatestSchema(cancellationToken, innerProgress).ConfigureAwait(false);
|
||||
|
||||
innerProgress = new ActionableProgress<double>();
|
||||
innerProgress.RegisterAction(p =>
|
||||
{
|
||||
double newPercentCommplete = 40 + .05 * p;
|
||||
OnProgress(newPercentCommplete);
|
||||
progress.Report(newPercentCommplete);
|
||||
});
|
||||
await CleanDeadItems(cancellationToken, innerProgress).ConfigureAwait(false);
|
||||
progress.Report(45);
|
||||
|
||||
innerProgress = new ActionableProgress<double>();
|
||||
innerProgress.RegisterAction(p =>
|
||||
{
|
||||
double newPercentCommplete = 45 + .55 * p;
|
||||
OnProgress(newPercentCommplete);
|
||||
progress.Report(newPercentCommplete);
|
||||
});
|
||||
await CleanDeletedItems(cancellationToken, innerProgress).ConfigureAwait(false);
|
||||
progress.Report(100);
|
||||
|
||||
await _itemRepo.UpdateInheritedValues(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (_config.Configuration.MigrationVersion < MigrationVersion)
|
||||
{
|
||||
_config.Configuration.MigrationVersion = MigrationVersion;
|
||||
_config.SaveConfiguration();
|
||||
}
|
||||
|
||||
if (_config.Configuration.SchemaVersion < LatestSchemaVersion)
|
||||
{
|
||||
_config.Configuration.SchemaVersion = LatestSchemaVersion;
|
||||
_config.SaveConfiguration();
|
||||
}
|
||||
|
||||
if (EnableUnavailableMessage)
|
||||
{
|
||||
EnableUnavailableMessage = false;
|
||||
_httpServer.GlobalResponse = null;
|
||||
_taskManager.QueueScheduledTask<RefreshMediaLibraryTask>();
|
||||
}
|
||||
|
||||
_taskManager.SuspendTriggers = false;
|
||||
}
|
||||
|
||||
private void OnProgress(double newPercentCommplete)
|
||||
{
|
||||
if (EnableUnavailableMessage)
|
||||
{
|
||||
var html = "<!doctype html><html><head><title>Emby</title></head><body>";
|
||||
var text = _localization.GetLocalizedString("DbUpgradeMessage");
|
||||
html += string.Format(text, newPercentCommplete.ToString("N2", CultureInfo.InvariantCulture));
|
||||
|
||||
html += "<script>setTimeout(function(){window.location.reload(true);}, 5000);</script>";
|
||||
html += "</body></html>";
|
||||
|
||||
_httpServer.GlobalResponse = html;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task UpdateToLatestSchema(CancellationToken cancellationToken, IProgress<double> progress)
|
||||
{
|
||||
var itemIds = _libraryManager.GetItemIds(new InternalItemsQuery
|
||||
{
|
||||
IsCurrentSchema = false,
|
||||
ExcludeItemTypes = new[] { typeof(LiveTvProgram).Name }
|
||||
});
|
||||
|
||||
var numComplete = 0;
|
||||
var numItems = itemIds.Count;
|
||||
|
||||
_logger.Debug("Upgrading schema for {0} items", numItems);
|
||||
|
||||
var list = new List<BaseItem>();
|
||||
|
||||
foreach (var itemId in itemIds)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (itemId != Guid.Empty)
|
||||
{
|
||||
// Somehow some invalid data got into the db. It probably predates the boundary checking
|
||||
var item = _libraryManager.GetItemById(itemId);
|
||||
|
||||
if (item != null)
|
||||
{
|
||||
list.Add(item);
|
||||
}
|
||||
}
|
||||
|
||||
if (list.Count >= 1000)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _itemRepo.SaveItems(list, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.ErrorException("Error saving item", ex);
|
||||
}
|
||||
|
||||
list.Clear();
|
||||
}
|
||||
|
||||
numComplete++;
|
||||
double percent = numComplete;
|
||||
percent /= numItems;
|
||||
progress.Report(percent * 100);
|
||||
}
|
||||
|
||||
if (list.Count > 0)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _itemRepo.SaveItems(list, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.ErrorException("Error saving item", ex);
|
||||
}
|
||||
}
|
||||
|
||||
progress.Report(100);
|
||||
}
|
||||
|
||||
private async Task CleanDeadItems(CancellationToken cancellationToken, IProgress<double> progress)
|
||||
{
|
||||
var itemIds = _libraryManager.GetItemIds(new InternalItemsQuery
|
||||
{
|
||||
HasDeadParentId = true
|
||||
});
|
||||
|
||||
var numComplete = 0;
|
||||
var numItems = itemIds.Count;
|
||||
|
||||
_logger.Debug("Cleaning {0} items with dead parent links", numItems);
|
||||
|
||||
foreach (var itemId in itemIds)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var item = _libraryManager.GetItemById(itemId);
|
||||
|
||||
if (item != null)
|
||||
{
|
||||
_logger.Info("Cleaning item {0} type: {1} path: {2}", item.Name, item.GetType().Name, item.Path ?? string.Empty);
|
||||
|
||||
await item.Delete(new DeleteOptions
|
||||
{
|
||||
DeleteFileLocation = false
|
||||
|
||||
}).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
numComplete++;
|
||||
double percent = numComplete;
|
||||
percent /= numItems;
|
||||
progress.Report(percent * 100);
|
||||
}
|
||||
|
||||
progress.Report(100);
|
||||
}
|
||||
|
||||
private async Task CleanDeletedItems(CancellationToken cancellationToken, IProgress<double> progress)
|
||||
{
|
||||
var result = _itemRepo.GetItemIdsWithPath(new InternalItemsQuery
|
||||
{
|
||||
LocationTypes = new[] { LocationType.FileSystem },
|
||||
//Limit = limit,
|
||||
|
||||
// These have their own cleanup routines
|
||||
ExcludeItemTypes = new[]
|
||||
{
|
||||
typeof(Person).Name,
|
||||
typeof(Genre).Name,
|
||||
typeof(MusicGenre).Name,
|
||||
typeof(GameGenre).Name,
|
||||
typeof(Studio).Name,
|
||||
typeof(Year).Name,
|
||||
typeof(Channel).Name,
|
||||
typeof(AggregateFolder).Name,
|
||||
typeof(CollectionFolder).Name
|
||||
}
|
||||
});
|
||||
|
||||
var numComplete = 0;
|
||||
var numItems = result.Items.Length;
|
||||
|
||||
foreach (var item in result.Items)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var path = item.Item2;
|
||||
|
||||
try
|
||||
{
|
||||
if (_fileSystem.FileExists(path) || _fileSystem.DirectoryExists(path))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var libraryItem = _libraryManager.GetItemById(item.Item1);
|
||||
|
||||
if (libraryItem.IsTopParent)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var hasDualAccess = libraryItem as IHasDualAccess;
|
||||
if (hasDualAccess != null && hasDualAccess.IsAccessedByName)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var libraryItemPath = libraryItem.Path;
|
||||
if (!string.Equals(libraryItemPath, path, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
_logger.Error("CleanDeletedItems aborting delete for item {0}-{1} because paths don't match. {2}---{3}", libraryItem.Id, libraryItem.Name, libraryItem.Path ?? string.Empty, path ?? string.Empty);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (Folder.IsPathOffline(path))
|
||||
{
|
||||
await libraryItem.UpdateIsOffline(true).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
_logger.Info("Deleting item from database {0} because path no longer exists. type: {1} path: {2}", libraryItem.Name, libraryItem.GetType().Name, libraryItemPath ?? string.Empty);
|
||||
|
||||
await libraryItem.OnFileDeleted().ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.ErrorException("Error in CleanDeletedItems. File {0}", ex, path);
|
||||
}
|
||||
|
||||
numComplete++;
|
||||
double percent = numComplete;
|
||||
percent /= numItems;
|
||||
progress.Report(percent * 100);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates the triggers that define when the task will run
|
||||
/// </summary>
|
||||
/// <returns>IEnumerable{BaseTaskTrigger}.</returns>
|
||||
public IEnumerable<TaskTriggerInfo> GetDefaultTriggers()
|
||||
{
|
||||
return new[] {
|
||||
|
||||
// Every so often
|
||||
new TaskTriggerInfo { Type = TaskTriggerInfo.TriggerInterval, IntervalTicks = TimeSpan.FromHours(24).Ticks}
|
||||
};
|
||||
}
|
||||
|
||||
public string Key
|
||||
{
|
||||
get { return "CleanDatabase"; }
|
||||
}
|
||||
}
|
||||
}
|
||||
105
Emby.Server.Implementations/Data/SqliteExtensions.cs
Normal file
105
Emby.Server.Implementations/Data/SqliteExtensions.cs
Normal file
@@ -0,0 +1,105 @@
|
||||
using System;
|
||||
using System.Globalization;
|
||||
using SQLitePCL.pretty;
|
||||
|
||||
namespace Emby.Server.Implementations.Data
|
||||
{
|
||||
public static class SqliteExtensions
|
||||
{
|
||||
public static void RunQueries(this SQLiteDatabaseConnection connection, string[] queries)
|
||||
{
|
||||
if (queries == null)
|
||||
{
|
||||
throw new ArgumentNullException("queries");
|
||||
}
|
||||
|
||||
connection.RunInTransaction(conn =>
|
||||
{
|
||||
//foreach (var query in queries)
|
||||
//{
|
||||
// conn.Execute(query);
|
||||
//}
|
||||
conn.ExecuteAll(string.Join(";", queries));
|
||||
});
|
||||
}
|
||||
|
||||
public static byte[] ToGuidParamValue(this string str)
|
||||
{
|
||||
return new Guid(str).ToByteArray();
|
||||
}
|
||||
|
||||
public static Guid ReadGuid(this IResultSetValue result)
|
||||
{
|
||||
return new Guid(result.ToBlob());
|
||||
}
|
||||
|
||||
public static string ToDateTimeParamValue(this DateTime dateValue)
|
||||
{
|
||||
var kind = DateTimeKind.Utc;
|
||||
|
||||
return (dateValue.Kind == DateTimeKind.Unspecified)
|
||||
? DateTime.SpecifyKind(dateValue, kind).ToString(
|
||||
GetDateTimeKindFormat(kind),
|
||||
CultureInfo.InvariantCulture)
|
||||
: dateValue.ToString(
|
||||
GetDateTimeKindFormat(dateValue.Kind),
|
||||
CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
private static string GetDateTimeKindFormat(
|
||||
DateTimeKind kind)
|
||||
{
|
||||
return (kind == DateTimeKind.Utc) ? _datetimeFormatUtc : _datetimeFormatLocal;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// An array of ISO-8601 DateTime formats that we support parsing.
|
||||
/// </summary>
|
||||
private static string[] _datetimeFormats = new string[] {
|
||||
"THHmmssK",
|
||||
"THHmmK",
|
||||
"HH:mm:ss.FFFFFFFK",
|
||||
"HH:mm:ssK",
|
||||
"HH:mmK",
|
||||
"yyyy-MM-dd HH:mm:ss.FFFFFFFK", /* NOTE: UTC default (5). */
|
||||
"yyyy-MM-dd HH:mm:ssK",
|
||||
"yyyy-MM-dd HH:mmK",
|
||||
"yyyy-MM-ddTHH:mm:ss.FFFFFFFK",
|
||||
"yyyy-MM-ddTHH:mmK",
|
||||
"yyyy-MM-ddTHH:mm:ssK",
|
||||
"yyyyMMddHHmmssK",
|
||||
"yyyyMMddHHmmK",
|
||||
"yyyyMMddTHHmmssFFFFFFFK",
|
||||
"THHmmss",
|
||||
"THHmm",
|
||||
"HH:mm:ss.FFFFFFF",
|
||||
"HH:mm:ss",
|
||||
"HH:mm",
|
||||
"yyyy-MM-dd HH:mm:ss.FFFFFFF", /* NOTE: Non-UTC default (19). */
|
||||
"yyyy-MM-dd HH:mm:ss",
|
||||
"yyyy-MM-dd HH:mm",
|
||||
"yyyy-MM-ddTHH:mm:ss.FFFFFFF",
|
||||
"yyyy-MM-ddTHH:mm",
|
||||
"yyyy-MM-ddTHH:mm:ss",
|
||||
"yyyyMMddHHmmss",
|
||||
"yyyyMMddHHmm",
|
||||
"yyyyMMddTHHmmssFFFFFFF",
|
||||
"yyyy-MM-dd",
|
||||
"yyyyMMdd",
|
||||
"yy-MM-dd"
|
||||
};
|
||||
|
||||
private static string _datetimeFormatUtc = _datetimeFormats[5];
|
||||
private static string _datetimeFormatLocal = _datetimeFormats[19];
|
||||
|
||||
public static DateTime ReadDateTime(this IResultSetValue result)
|
||||
{
|
||||
var dateText = result.ToString();
|
||||
|
||||
return DateTime.ParseExact(
|
||||
dateText, _datetimeFormats,
|
||||
DateTimeFormatInfo.InvariantInfo,
|
||||
DateTimeStyles.None).ToUniversalTime();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user