alteropen/jellyfin
1
0
Fork 0
mirror of https://github.com/jellyfin/jellyfin.git synced 2026-03-14 22:26:34 +00:00
Code Issues Actions 7 Packages Projects Releases Wiki Activity

rework server sync

Browse Source
This commit is contained in:
Luke Pulverenti
2015-04-13 01:12:02 -04:00
parent 933fca78e6
commit 65fb01bbe2
23 changed files with 162 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
MediaBrowser.Server.Implementations/Sync/MediaSync.cs
View File

@@ -1,5 +1,4 @@
using System.Globalization;
using MediaBrowser.Common.Configuration;
using MediaBrowser.Common.Configuration;
using MediaBrowser.Common.IO;
using MediaBrowser.Common.Progress;
using MediaBrowser.Controller;
@@ -12,12 +11,14 @@ using MediaBrowser.Model.MediaInfo;
using MediaBrowser.Model.Sync;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Patterns.IO;
namespace MediaBrowser.Server.Implementations.Sync
{
@@ -29,6 +30,9 @@ namespace MediaBrowser.Server.Implementations.Sync
private readonly IFileSystem _fileSystem;
private readonly IConfigurationManager _config;
public const string PathSeparatorString = "/";
public const char PathSeparatorChar = '/';
public MediaSync(ILogger logger, ISyncManager syncManager, IServerApplicationHost appHost, IFileSystem fileSystem, IConfigurationManager config)
{
_logger = logger;
@@ -71,7 +75,24 @@ namespace MediaBrowser.Server.Implementations.Sync
SyncTarget target,
CancellationToken cancellationToken)
{
var jobItemIds = await dataProvider.GetSyncJobItemIds(target, serverId).ConfigureAwait(false);
var localItems = await dataProvider.GetLocalItems(target, serverId).ConfigureAwait(false);
var remoteFiles = await provider.GetFiles(new FileQuery(), target, cancellationToken).ConfigureAwait(false);
var remoteIds = remoteFiles.Items.Select(i => i.Id).ToList();
var jobItemIds = new List<string>();
foreach (var localItem in localItems)
{
// TODO: Remove this after a while
if (string.IsNullOrWhiteSpace(localItem.FileId))
{
jobItemIds.Add(localItem.SyncJobItemId);
}
else if (remoteIds.Contains(localItem.FileId, StringComparer.OrdinalIgnoreCase))
{
jobItemIds.Add(localItem.SyncJobItemId);
}
}
var result = await _syncManager.SyncData(new SyncDataRequest
{
@@ -163,7 +184,7 @@ namespace MediaBrowser.Server.Implementations.Sync
var fileTransferProgress = new ActionableProgress<double>();
fileTransferProgress.RegisterAction(pct => progress.Report(pct * .92));
var sendFileResult = await SendFile(provider, internalSyncJobItem.OutputPath, localItem.LocalPath, target, options, fileTransferProgress, cancellationToken).ConfigureAwait(false);
var sendFileResult = await SendFile(provider, internalSyncJobItem.OutputPath, localItem.LocalPath.Split(PathSeparatorChar), target, options, fileTransferProgress, cancellationToken).ConfigureAwait(false);
if (localItem.Item.MediaSources != null)
{
@@ -177,6 +198,8 @@ namespace MediaBrowser.Server.Implementations.Sync
}
}
localItem.FileId = sendFileResult.Id;
// Create db record
await dataProvider.AddOrUpdate(target, localItem).ConfigureAwait(false);
@@ -228,10 +251,10 @@ namespace MediaBrowser.Server.Implementations.Sync
var sendFileResult = await SendFile(provider, mediaStream.Path, remotePath, target, options, new Progress<double>(), cancellationToken).ConfigureAwait(false);
// This is the path that will be used when talking to the provider
mediaStream.ExternalId = remotePath;
mediaStream.ExternalId = sendFileResult.Id;
// Keep track of all additional files for cleanup later.
localItem.AdditionalFiles.Add(remotePath);
localItem.AdditionalFiles.Add(sendFileResult.Id);
// This is the public path clients will use
mediaStream.Path = sendFileResult.Path;
@@ -256,17 +279,15 @@ namespace MediaBrowser.Server.Implementations.Sync
}
}
private string GetRemoteSubtitlePath(LocalItem item, MediaStream stream, IServerSyncProvider provider, SyncTarget target)
private string[] GetRemoteSubtitlePath(LocalItem item, MediaStream stream, IServerSyncProvider provider, SyncTarget target)
{
var path = item.LocalPath;
var filename = GetSubtitleSaveFileName(item, stream.Language, stream.IsForced) + "." + stream.Codec.ToLower();
var parentPath = provider.GetParentDirectoryPath(path, target);
var pathParts = item.LocalPath.Split(PathSeparatorChar);
var list = pathParts.Take(pathParts.Length - 1).ToList();
list.Add(filename);
path = Path.Combine(parentPath, filename);
return path;
return list.ToArray();
}
private string GetSubtitleSaveFileName(LocalItem item, string language, bool isForced)
@@ -300,12 +321,16 @@ namespace MediaBrowser.Server.Implementations.Sync
foreach (var localItem in localItems)
{
var files = localItem.AdditionalFiles.ToList();
files.Insert(0, localItem.LocalPath);
// TODO: Remove this. Have to check it for now since this is a new property
if (!string.IsNullOrWhiteSpace(localItem.FileId))
{
files.Insert(0, localItem.FileId);
}
foreach (var file in files)
{
_logger.Debug("Removing {0} from {1}.", file, target.Name);
await provider.DeleteFile(file, target, cancellationToken).ConfigureAwait(false);
}
@@ -313,9 +338,9 @@ namespace MediaBrowser.Server.Implementations.Sync
}
}
private async Task<SyncedFileInfo> SendFile(IServerSyncProvider provider, string inputPath, string remotePath, SyncTarget target, SyncOptions options, IProgress<double> progress, CancellationToken cancellationToken)
private async Task<SyncedFileInfo> SendFile(IServerSyncProvider provider, string inputPath, string[] pathParts, SyncTarget target, SyncOptions options, IProgress<double> progress, CancellationToken cancellationToken)
{
_logger.Debug("Sending {0} to {1}. Remote path: {2}", inputPath, provider.Name, remotePath);
_logger.Debug("Sending {0} to {1}. Remote path: {2}", inputPath, provider.Name, string.Join("/", pathParts));
using (var fileStream = _fileSystem.GetFileStream(inputPath, FileMode.Open, FileAccess.Read, FileShare.Read, true))
{
Stream stream = fileStream;
@@ -325,7 +350,7 @@ namespace MediaBrowser.Server.Implementations.Sync
stream = new ThrottledStream(stream, options.UploadSpeedLimitBytes);
}
return await provider.SendFile(stream, remotePath, target, progress, cancellationToken).ConfigureAwait(false);
return await provider.SendFile(stream, pathParts, target, progress, cancellationToken).ConfigureAwait(false);
}
}
@@ -349,7 +374,7 @@ namespace MediaBrowser.Server.Implementations.Sync
var path = GetDirectoryPath(provider, job, syncedItem, libraryItem, serverName);
path.Add(GetLocalFileName(provider, libraryItem, originalFileName));
var localPath = provider.GetFullPath(path, target);
var localPath = string.Join(PathSeparatorString, path.ToArray());
foreach (var mediaSource in libraryItem.MediaSources)
{

6
MediaBrowser.Server.Implementations/Sync/SyncManager.cs
View File

@@ -793,8 +793,6 @@ namespace MediaBrowser.Server.Implementations.Sync
}
else
{
_logger.Debug("Setting status to Queued for {0} because it is no longer on the device.", jobItem.ItemId);
// Content is no longer on the device
if (jobItem.IsMarkedForRemoval)
{
@@ -802,6 +800,7 @@ namespace MediaBrowser.Server.Implementations.Sync
}
else
{
_logger.Debug("Setting status to Queued for {0} because it is no longer on the device.", jobItem.ItemId);
jobItem.Status = SyncJobItemStatus.Queued;
}
requiresSaving = true;
@@ -902,8 +901,6 @@ namespace MediaBrowser.Server.Implementations.Sync
}
else
{
_logger.Debug("Setting status to Queued for {0} because it is no longer on the device.", jobItem.Id);
// Content is no longer on the device
if (jobItem.IsMarkedForRemoval)
{
@@ -911,6 +908,7 @@ namespace MediaBrowser.Server.Implementations.Sync
}
else
{
_logger.Debug("Setting status to Queued for {0} because it is no longer on the device.", jobItem.Id);
jobItem.Status = SyncJobItemStatus.Queued;
}
requiresSaving = true;

7
MediaBrowser.Server.Implementations/Sync/SyncedMediaSourceProvider.cs
View File

@@ -109,8 +109,13 @@ namespace MediaBrowser.Server.Implementations.Sync
var dataProvider = _syncManager.GetDataProvider(provider, target);
var localItem = await dataProvider.Get(target, openKeys[2]).ConfigureAwait(false);
var fileId = localItem.FileId;
if (string.IsNullOrWhiteSpace(fileId))
{
}
var requiresDynamicAccess = (IHasDynamicAccess)provider;
var dynamicInfo = await requiresDynamicAccess.GetSyncedFileInfo(localItem.LocalPath, target, cancellationToken).ConfigureAwait(false);
var dynamicInfo = await requiresDynamicAccess.GetSyncedFileInfo(fileId, target, cancellationToken).ConfigureAwait(false);
var mediaSource = localItem.Item.MediaSources.First();
mediaSource.LiveStreamId = Guid.NewGuid().ToString();

35
MediaBrowser.Server.Implementations/Sync/TargetDataProvider.cs
View File

@@ -1,5 +1,4 @@
using MediaBrowser.Common.Configuration;
using MediaBrowser.Common.Extensions;
using MediaBrowser.Common.IO;
using MediaBrowser.Controller;
using MediaBrowser.Controller.Sync;
@@ -12,6 +11,7 @@ using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Patterns.IO;
namespace MediaBrowser.Server.Implementations.Sync
{
@@ -29,8 +29,6 @@ namespace MediaBrowser.Server.Implementations.Sync
private readonly IApplicationPaths _appPaths;
private readonly IServerApplicationHost _appHost;
private readonly SemaphoreSlim _cacheFileLock = new SemaphoreSlim(1, 1);
public TargetDataProvider(IServerSyncProvider provider, SyncTarget target, IServerApplicationHost appHost, ILogger logger, IJsonSerializer json, IFileSystem fileSystem, IApplicationPaths appPaths)
{
_logger = logger;
@@ -42,7 +40,7 @@ namespace MediaBrowser.Server.Implementations.Sync
_appHost = appHost;
}
private string GetRemotePath()
private string[] GetRemotePath()
{
var parts = new List<string>
{
@@ -52,7 +50,7 @@ namespace MediaBrowser.Server.Implementations.Sync
parts = parts.Select(i => GetValidFilename(_provider, i)).ToList();
return _provider.GetFullPath(parts, _target);
return parts.ToArray();
}
private string GetValidFilename(IServerSyncProvider provider, string filename)
@@ -65,22 +63,22 @@ namespace MediaBrowser.Server.Implementations.Sync
{
if (_items == null)
{
try
_logger.Debug("Getting {0} from {1}", string.Join(MediaSync.PathSeparatorString, GetRemotePath().ToArray()), _provider.Name);
var fileResult = await _provider.GetFiles(new FileQuery
{
var path = GetRemotePath();
FullPath = GetRemotePath().ToArray()
_logger.Debug("Getting {0} from {1}", path, _provider.Name);
}, _target, cancellationToken).ConfigureAwait(false);
using (var stream = await _provider.GetFile(path, _target, new Progress<double>(), cancellationToken))
if (fileResult.Items.Length > 0)
{
using (var stream = await _provider.GetFile(fileResult.Items[0].Id, _target, new Progress<double>(), cancellationToken))
{
_items = _json.DeserializeFromStream<List<LocalItem>>(stream);
}
}
catch (FileNotFoundException)
{
_items = new List<LocalItem>();
}
catch (DirectoryNotFoundException)
else
{
_items = new List<LocalItem>();
}
@@ -133,14 +131,9 @@ namespace MediaBrowser.Server.Implementations.Sync
}
}
public Task<List<string>> GetServerItemIds(SyncTarget target, string serverId)
public Task<List<LocalItem>> GetLocalItems(SyncTarget target, string serverId)
{
return GetData(items => items.Where(i => string.Equals(i.ServerId, serverId, StringComparison.OrdinalIgnoreCase)).Select(i => i.ItemId).ToList());
}
public Task<List<string>> GetSyncJobItemIds(SyncTarget target, string serverId)
{
return GetData(items => items.Where(i => string.Equals(i.ServerId, serverId, StringComparison.OrdinalIgnoreCase)).Select(i => i.SyncJobItemId).Where(i => !string.IsNullOrWhiteSpace(i)).ToList());
return GetData(items => items.Where(i => string.Equals(i.ServerId, serverId, StringComparison.OrdinalIgnoreCase)).ToList());
}
public Task AddOrUpdate(SyncTarget target, LocalItem item)