switch to flat file storage

This commit is contained in:
Luke Pulverenti
2013-06-17 16:35:43 -04:00
parent 95f471e8c3
commit e677a57bf1
44 changed files with 958 additions and 1594 deletions

View File

@@ -0,0 +1,164 @@
using MediaBrowser.Common.Configuration;
using MediaBrowser.Controller.Persistence;
using MediaBrowser.Model.Entities;
using MediaBrowser.Model.Logging;
using MediaBrowser.Model.Serialization;
using System;
using System.Collections.Concurrent;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace MediaBrowser.Server.Implementations.Persistence
{
public class JsonDisplayPreferencesRepository : IDisplayPreferencesRepository
{
private readonly ConcurrentDictionary<string, SemaphoreSlim> _fileLocks = new ConcurrentDictionary<string, SemaphoreSlim>();
private SemaphoreSlim GetLock(string filename)
{
return _fileLocks.GetOrAdd(filename, key => new SemaphoreSlim(1, 1));
}
/// <summary>
/// Gets the name of the repository
/// </summary>
/// <value>The name.</value>
public string Name
{
get
{
return "Json";
}
}
/// <summary>
/// The _json serializer
/// </summary>
private readonly IJsonSerializer _jsonSerializer;
private readonly string _dataPath;
/// <summary>
/// Initializes a new instance of the <see cref="JsonUserDataRepository" /> class.
/// </summary>
/// <param name="appPaths">The app paths.</param>
/// <param name="jsonSerializer">The json serializer.</param>
/// <param name="logManager">The log manager.</param>
/// <exception cref="System.ArgumentNullException">
/// jsonSerializer
/// or
/// appPaths
/// </exception>
public JsonDisplayPreferencesRepository(IApplicationPaths appPaths, IJsonSerializer jsonSerializer, ILogManager logManager)
{
if (jsonSerializer == null)
{
throw new ArgumentNullException("jsonSerializer");
}
if (appPaths == null)
{
throw new ArgumentNullException("appPaths");
}
_jsonSerializer = jsonSerializer;
_dataPath = Path.Combine(appPaths.DataPath, "display-preferences");
}
/// <summary>
/// Opens the connection to the database
/// </summary>
/// <returns>Task.</returns>
public Task Initialize()
{
return Task.FromResult(true);
}
/// <summary>
/// Save the display preferences associated with an item in the repo
/// </summary>
/// <param name="displayPreferences">The display preferences.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>Task.</returns>
/// <exception cref="System.ArgumentNullException">item</exception>
public async Task SaveDisplayPreferences(DisplayPreferences displayPreferences, CancellationToken cancellationToken)
{
if (displayPreferences == null)
{
throw new ArgumentNullException("displayPreferences");
}
if (displayPreferences.Id == Guid.Empty)
{
throw new ArgumentNullException("displayPreferences.Id");
}
if (cancellationToken == null)
{
throw new ArgumentNullException("cancellationToken");
}
cancellationToken.ThrowIfCancellationRequested();
if (!Directory.Exists(_dataPath))
{
Directory.CreateDirectory(_dataPath);
}
var path = Path.Combine(_dataPath, displayPreferences.Id + ".json");
var semaphore = GetLock(path);
await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
_jsonSerializer.SerializeToFile(displayPreferences, path);
}
finally
{
semaphore.Release();
}
}
/// <summary>
/// Gets the display preferences.
/// </summary>
/// <param name="displayPreferencesId">The display preferences id.</param>
/// <returns>Task{DisplayPreferences}.</returns>
/// <exception cref="System.ArgumentNullException">item</exception>
public Task<DisplayPreferences> GetDisplayPreferences(Guid displayPreferencesId)
{
if (displayPreferencesId == Guid.Empty)
{
throw new ArgumentNullException("displayPreferencesId");
}
return Task.Run(() =>
{
var path = Path.Combine(_dataPath, displayPreferencesId + ".json");
try
{
return _jsonSerializer.DeserializeFromFile<DisplayPreferences>(path);
}
catch (IOException)
{
// File doesn't exist or is currently bring written to
return null;
}
});
}
public void Dispose()
{
// Wait up to two seconds for any existing writes to finish
var locks = _fileLocks.Values.ToList()
.Where(i => i.CurrentCount == 1)
.Select(i => i.WaitAsync(2000));
var task = Task.WhenAll(locks);
Task.WaitAll(task);
}
}
}

View File

@@ -0,0 +1,235 @@
using MediaBrowser.Common.Configuration;
using MediaBrowser.Common.IO;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.Persistence;
using MediaBrowser.Model.Entities;
using MediaBrowser.Model.Logging;
using MediaBrowser.Model.Serialization;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace MediaBrowser.Server.Implementations.Persistence
{
public class JsonItemRepository : IItemRepository
{
private readonly ConcurrentDictionary<string, SemaphoreSlim> _fileLocks = new ConcurrentDictionary<string, SemaphoreSlim>();
private SemaphoreSlim GetLock(string filename)
{
return _fileLocks.GetOrAdd(filename, key => new SemaphoreSlim(1, 1));
}
/// <summary>
/// Gets the name of the repository
/// </summary>
/// <value>The name.</value>
public string Name
{
get
{
return "Json";
}
}
/// <summary>
/// Gets the json serializer.
/// </summary>
/// <value>The json serializer.</value>
private readonly IJsonSerializer _jsonSerializer;
private readonly string _criticReviewsPath;
private readonly FileSystemRepository _itemRepo;
/// <summary>
/// Initializes a new instance of the <see cref="JsonUserDataRepository" /> class.
/// </summary>
/// <param name="appPaths">The app paths.</param>
/// <param name="jsonSerializer">The json serializer.</param>
/// <param name="logManager">The log manager.</param>
/// <exception cref="System.ArgumentNullException">appPaths</exception>
public JsonItemRepository(IApplicationPaths appPaths, IJsonSerializer jsonSerializer, ILogManager logManager)
{
if (appPaths == null)
{
throw new ArgumentNullException("appPaths");
}
if (jsonSerializer == null)
{
throw new ArgumentNullException("jsonSerializer");
}
_jsonSerializer = jsonSerializer;
_criticReviewsPath = Path.Combine(appPaths.DataPath, "critic-reviews");
_itemRepo = new FileSystemRepository(Path.Combine(appPaths.DataPath, "library"));
}
/// <summary>
/// Opens the connection to the database
/// </summary>
/// <returns>Task.</returns>
public Task Initialize()
{
return Task.FromResult(true);
}
/// <summary>
/// Save a standard item in the repo
/// </summary>
/// <param name="item">The item.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>Task.</returns>
/// <exception cref="System.ArgumentNullException">item</exception>
public async Task SaveItem(BaseItem item, CancellationToken cancellationToken)
{
if (item == null)
{
throw new ArgumentNullException("item");
}
if (!Directory.Exists(_criticReviewsPath))
{
Directory.CreateDirectory(_criticReviewsPath);
}
var path = _itemRepo.GetResourcePath(item.Id + ".json");
var parentPath = Path.GetDirectoryName(path);
if (!Directory.Exists(parentPath))
{
Directory.CreateDirectory(parentPath);
}
var semaphore = GetLock(path);
await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
_jsonSerializer.SerializeToFile(item, path);
}
finally
{
semaphore.Release();
}
}
/// <summary>
/// Saves the items.
/// </summary>
/// <param name="items">The items.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>Task.</returns>
/// <exception cref="System.ArgumentNullException">
/// items
/// or
/// cancellationToken
/// </exception>
public Task SaveItems(IEnumerable<BaseItem> items, CancellationToken cancellationToken)
{
if (items == null)
{
throw new ArgumentNullException("items");
}
if (cancellationToken == null)
{
throw new ArgumentNullException("cancellationToken");
}
var tasks = items.Select(i => SaveItem(i, cancellationToken));
return Task.WhenAll(tasks);
}
/// <summary>
/// Retrieves the item.
/// </summary>
/// <param name="id">The id.</param>
/// <param name="type">The type.</param>
/// <returns>BaseItem.</returns>
/// <exception cref="System.ArgumentNullException">id</exception>
public BaseItem RetrieveItem(Guid id, Type type)
{
if (id == Guid.Empty)
{
throw new ArgumentNullException("id");
}
var path = _itemRepo.GetResourcePath(id + ".json");
try
{
return (BaseItem)_jsonSerializer.DeserializeFromFile(type, path);
}
catch (IOException)
{
// File doesn't exist or is currently bring written to
return null;
}
}
/// <summary>
/// Gets the critic reviews.
/// </summary>
/// <param name="itemId">The item id.</param>
/// <returns>Task{IEnumerable{ItemReview}}.</returns>
public Task<IEnumerable<ItemReview>> GetCriticReviews(Guid itemId)
{
return Task.Run<IEnumerable<ItemReview>>(() =>
{
var path = Path.Combine(_criticReviewsPath, itemId + ".json");
try
{
return _jsonSerializer.DeserializeFromFile<List<ItemReview>>(path);
}
catch (IOException)
{
// File doesn't exist or is currently bring written to
return new List<ItemReview>();
}
});
}
/// <summary>
/// Saves the critic reviews.
/// </summary>
/// <param name="itemId">The item id.</param>
/// <param name="criticReviews">The critic reviews.</param>
/// <returns>Task.</returns>
public Task SaveCriticReviews(Guid itemId, IEnumerable<ItemReview> criticReviews)
{
return Task.Run(() =>
{
if (!Directory.Exists(_criticReviewsPath))
{
Directory.CreateDirectory(_criticReviewsPath);
}
var path = Path.Combine(_criticReviewsPath, itemId + ".json");
_jsonSerializer.SerializeToFile(criticReviews.ToList(), path);
});
}
public void Dispose()
{
// Wait up to two seconds for any existing writes to finish
var locks = _fileLocks.Values.ToList()
.Where(i => i.CurrentCount == 1)
.Select(i => i.WaitAsync(2000));
var task = Task.WhenAll(locks);
Task.WaitAll(task);
}
}
}

View File

@@ -0,0 +1,246 @@
using MediaBrowser.Common.Configuration;
using MediaBrowser.Common.Extensions;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.Persistence;
using MediaBrowser.Model.Logging;
using MediaBrowser.Model.Serialization;
using System;
using System.Collections.Concurrent;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace MediaBrowser.Server.Implementations.Persistence
{
public class JsonUserDataRepository : IUserDataRepository
{
private readonly ConcurrentDictionary<string, SemaphoreSlim> _fileLocks = new ConcurrentDictionary<string, SemaphoreSlim>();
private SemaphoreSlim GetLock(string filename)
{
return _fileLocks.GetOrAdd(filename, key => new SemaphoreSlim(1, 1));
}
private readonly ConcurrentDictionary<string, UserItemData> _userData = new ConcurrentDictionary<string, UserItemData>();
/// <summary>
/// Gets the name of the repository
/// </summary>
/// <value>The name.</value>
public string Name
{
get
{
return "Json";
}
}
private readonly IJsonSerializer _jsonSerializer;
private readonly string _dataPath;
private readonly ILogger _logger;
/// <summary>
/// Initializes a new instance of the <see cref="JsonUserDataRepository" /> class.
/// </summary>
/// <param name="appPaths">The app paths.</param>
/// <param name="jsonSerializer">The json serializer.</param>
/// <param name="logManager">The log manager.</param>
/// <exception cref="System.ArgumentNullException">
/// jsonSerializer
/// or
/// appPaths
/// </exception>
public JsonUserDataRepository(IApplicationPaths appPaths, IJsonSerializer jsonSerializer, ILogManager logManager)
{
if (jsonSerializer == null)
{
throw new ArgumentNullException("jsonSerializer");
}
if (appPaths == null)
{
throw new ArgumentNullException("appPaths");
}
_logger = logManager.GetLogger(GetType().Name);
_jsonSerializer = jsonSerializer;
_dataPath = Path.Combine(appPaths.DataPath, "userdata");
}
/// <summary>
/// Opens the connection to the database
/// </summary>
/// <returns>Task.</returns>
public Task Initialize()
{
return Task.FromResult(true);
}
/// <summary>
/// Saves the user data.
/// </summary>
/// <param name="userId">The user id.</param>
/// <param name="key">The key.</param>
/// <param name="userData">The user data.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>Task.</returns>
/// <exception cref="System.ArgumentNullException">userData
/// or
/// cancellationToken
/// or
/// userId
/// or
/// userDataId</exception>
public async Task SaveUserData(Guid userId, string key, UserItemData userData, CancellationToken cancellationToken)
{
if (userData == null)
{
throw new ArgumentNullException("userData");
}
if (cancellationToken == null)
{
throw new ArgumentNullException("cancellationToken");
}
if (userId == Guid.Empty)
{
throw new ArgumentNullException("userId");
}
if (string.IsNullOrEmpty(key))
{
throw new ArgumentNullException("key");
}
cancellationToken.ThrowIfCancellationRequested();
try
{
await PersistUserData(userId, key, userData, cancellationToken).ConfigureAwait(false);
// Once it succeeds, put it into the dictionary to make it available to everyone else
_userData.AddOrUpdate(GetInternalKey(userId, key), userData, delegate { return userData; });
}
catch (Exception ex)
{
_logger.ErrorException("Error saving user data", ex);
throw;
}
}
/// <summary>
/// Gets the internal key.
/// </summary>
/// <param name="userId">The user id.</param>
/// <param name="key">The key.</param>
/// <returns>System.String.</returns>
private string GetInternalKey(Guid userId, string key)
{
return userId + key;
}
/// <summary>
/// Persists the user data.
/// </summary>
/// <param name="userId">The user id.</param>
/// <param name="key">The key.</param>
/// <param name="userData">The user data.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>Task.</returns>
public async Task PersistUserData(Guid userId, string key, UserItemData userData, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var path = GetUserDataPath(userId, key);
var parentPath = Path.GetDirectoryName(path);
if (!Directory.Exists(parentPath))
{
Directory.CreateDirectory(parentPath);
}
var semaphore = GetLock(path);
await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
_jsonSerializer.SerializeToFile(userData, path);
}
finally
{
semaphore.Release();
}
}
/// <summary>
/// Gets the user data.
/// </summary>
/// <param name="userId">The user id.</param>
/// <param name="key">The key.</param>
/// <returns>Task{UserItemData}.</returns>
/// <exception cref="System.ArgumentNullException">
/// userId
/// or
/// key
/// </exception>
public UserItemData GetUserData(Guid userId, string key)
{
if (userId == Guid.Empty)
{
throw new ArgumentNullException("userId");
}
if (string.IsNullOrEmpty(key))
{
throw new ArgumentNullException("key");
}
return _userData.GetOrAdd(GetInternalKey(userId, key), keyName => RetrieveUserData(userId, key));
}
/// <summary>
/// Retrieves the user data.
/// </summary>
/// <param name="userId">The user id.</param>
/// <param name="key">The key.</param>
/// <returns>Task{UserItemData}.</returns>
private UserItemData RetrieveUserData(Guid userId, string key)
{
var path = GetUserDataPath(userId, key);
try
{
return _jsonSerializer.DeserializeFromFile<UserItemData>(path);
}
catch (IOException)
{
// File doesn't exist or is currently bring written to
return new UserItemData { UserId = userId };
}
}
private string GetUserDataPath(Guid userId, string key)
{
var userFolder = Path.Combine(_dataPath, userId.ToString());
var keyHash = key.GetMD5().ToString();
var prefix = keyHash.Substring(0, 1);
return Path.Combine(userFolder, prefix, keyHash + ".json");
}
public void Dispose()
{
// Wait up to two seconds for any existing writes to finish
var locks = _fileLocks.Values.ToList()
.Where(i => i.CurrentCount == 1)
.Select(i => i.WaitAsync(2000));
var task = Task.WhenAll(locks);
Task.WaitAll(task);
}
}
}

View File

@@ -0,0 +1,189 @@
using MediaBrowser.Common.Configuration;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.Persistence;
using MediaBrowser.Model.Logging;
using MediaBrowser.Model.Serialization;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace MediaBrowser.Server.Implementations.Persistence
{
public class JsonUserRepository : IUserRepository
{
private readonly ConcurrentDictionary<string, SemaphoreSlim> _fileLocks = new ConcurrentDictionary<string, SemaphoreSlim>();
private SemaphoreSlim GetLock(string filename)
{
return _fileLocks.GetOrAdd(filename, key => new SemaphoreSlim(1, 1));
}
/// <summary>
/// Gets the name of the repository
/// </summary>
/// <value>The name.</value>
public string Name
{
get
{
return "Json";
}
}
/// <summary>
/// Gets the json serializer.
/// </summary>
/// <value>The json serializer.</value>
private readonly IJsonSerializer _jsonSerializer;
private readonly string _dataPath;
/// <summary>
/// Initializes a new instance of the <see cref="JsonUserRepository"/> class.
/// </summary>
/// <param name="appPaths">The app paths.</param>
/// <param name="jsonSerializer">The json serializer.</param>
/// <param name="logManager">The log manager.</param>
/// <exception cref="System.ArgumentNullException">
/// appPaths
/// or
/// jsonSerializer
/// </exception>
public JsonUserRepository(IApplicationPaths appPaths, IJsonSerializer jsonSerializer, ILogManager logManager)
{
if (appPaths == null)
{
throw new ArgumentNullException("appPaths");
}
if (jsonSerializer == null)
{
throw new ArgumentNullException("jsonSerializer");
}
_jsonSerializer = jsonSerializer;
_dataPath = Path.Combine(appPaths.DataPath, "users");
}
/// <summary>
/// Opens the connection to the database
/// </summary>
/// <returns>Task.</returns>
public Task Initialize()
{
return Task.FromResult(true);
}
/// <summary>
/// Save a user in the repo
/// </summary>
/// <param name="user">The user.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>Task.</returns>
/// <exception cref="System.ArgumentNullException">user</exception>
public async Task SaveUser(User user, CancellationToken cancellationToken)
{
if (user == null)
{
throw new ArgumentNullException("user");
}
if (cancellationToken == null)
{
throw new ArgumentNullException("cancellationToken");
}
cancellationToken.ThrowIfCancellationRequested();
if (!Directory.Exists(_dataPath))
{
Directory.CreateDirectory(_dataPath);
}
var path = Path.Combine(_dataPath, user.Id + ".json");
var semaphore = GetLock(path);
await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
_jsonSerializer.SerializeToFile(user, path);
}
finally
{
semaphore.Release();
}
}
/// <summary>
/// Retrieve all users from the database
/// </summary>
/// <returns>IEnumerable{User}.</returns>
public IEnumerable<User> RetrieveAllUsers()
{
try
{
return Directory.EnumerateFiles(_dataPath, "*.json", SearchOption.TopDirectoryOnly)
.Select(i => _jsonSerializer.DeserializeFromFile<User>(i));
}
catch (IOException)
{
return new List<User>();
}
}
/// <summary>
/// Deletes the user.
/// </summary>
/// <param name="user">The user.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>Task.</returns>
/// <exception cref="System.ArgumentNullException">user</exception>
public async Task DeleteUser(User user, CancellationToken cancellationToken)
{
if (user == null)
{
throw new ArgumentNullException("user");
}
if (cancellationToken == null)
{
throw new ArgumentNullException("cancellationToken");
}
cancellationToken.ThrowIfCancellationRequested();
var path = Path.Combine(_dataPath, user.Id + ".json");
var semaphore = GetLock(path);
await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
File.Delete(path);
}
finally
{
semaphore.Release();
}
}
public void Dispose()
{
// Wait up to two seconds for any existing writes to finish
var locks = _fileLocks.Values.ToList()
.Where(i => i.CurrentCount == 1)
.Select(i => i.WaitAsync(2000));
var task = Task.WhenAll(locks);
Task.WaitAll(task);
}
}
}