add channel downloading settings

This commit is contained in:
Luke Pulverenti
2014-06-02 15:32:41 -04:00
parent 36648d2708
commit 858c37b860
45 changed files with 980 additions and 303 deletions

View File

@@ -0,0 +1,277 @@
using MediaBrowser.Common.Extensions;
using MediaBrowser.Common.IO;
using MediaBrowser.Common.Net;
using MediaBrowser.Common.ScheduledTasks;
using MediaBrowser.Controller.Channels;
using MediaBrowser.Controller.Configuration;
using MediaBrowser.Controller.Library;
using MediaBrowser.Model.Channels;
using MediaBrowser.Model.Dto;
using MediaBrowser.Model.Entities;
using MediaBrowser.Model.Logging;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace MediaBrowser.Server.Implementations.Channels
{
public class ChannelDownloadScheduledTask : IScheduledTask, IConfigurableScheduledTask
{
private readonly IChannelManager _manager;
private readonly IServerConfigurationManager _config;
private readonly ILogger _logger;
private readonly IHttpClient _httpClient;
private readonly IFileSystem _fileSystem;
private readonly ILibraryManager _libraryManager;
public ChannelDownloadScheduledTask(IChannelManager manager, IServerConfigurationManager config, ILogger logger, IHttpClient httpClient, IFileSystem fileSystem, ILibraryManager libraryManager)
{
_manager = manager;
_config = config;
_logger = logger;
_httpClient = httpClient;
_fileSystem = fileSystem;
_libraryManager = libraryManager;
}
public string Name
{
get { return "Download channel content"; }
}
public string Description
{
get { return "Downloads channel content based on configuration."; }
}
public string Category
{
get { return "Channels"; }
}
public async Task Execute(CancellationToken cancellationToken, IProgress<double> progress)
{
CleanChannelContent(cancellationToken);
progress.Report(5);
await DownloadChannelContent(cancellationToken, progress).ConfigureAwait(false);
progress.Report(100);
}
private void CleanChannelContent(CancellationToken cancellationToken)
{
if (!_config.Configuration.ChannelOptions.MaxDownloadAge.HasValue)
{
return;
}
var minDateModified = DateTime.UtcNow.AddDays(0 - _config.Configuration.ChannelOptions.MaxDownloadAge.Value);
var path = _manager.ChannelDownloadPath;
try
{
DeleteCacheFilesFromDirectory(cancellationToken, path, minDateModified, new Progress<double>());
}
catch (DirectoryNotFoundException)
{
// No biggie here. Nothing to delete
}
}
private async Task DownloadChannelContent(CancellationToken cancellationToken, IProgress<double> progress)
{
if (_config.Configuration.ChannelOptions.DownloadingChannels.Length == 0)
{
return;
}
var result = await _manager.GetAllMedia(new AllChannelMediaQuery
{
ChannelIds = _config.Configuration.ChannelOptions.DownloadingChannels
}, cancellationToken).ConfigureAwait(false);
var path = _manager.ChannelDownloadPath;
var numComplete = 0;
foreach (var item in result.Items)
{
try
{
await DownloadChannelItem(item, cancellationToken, path);
}
catch (OperationCanceledException)
{
break;
}
catch (Exception ex)
{
_logger.ErrorException("Error downloading channel content for {0}", ex, item.Name);
}
numComplete++;
double percent = numComplete;
percent /= result.Items.Length;
progress.Report(percent * 95 + 5);
}
}
private async Task DownloadChannelItem(BaseItemDto item,
CancellationToken cancellationToken,
string path)
{
var sources = await _manager.GetChannelItemMediaSources(item.Id, cancellationToken)
.ConfigureAwait(false);
var list = sources.ToList();
var cachedVersions = list.Where(i => i.LocationType == LocationType.FileSystem).ToList();
if (cachedVersions.Count > 0)
{
await RefreshMediaSourceItems(cachedVersions, item.IsVideo, cancellationToken).ConfigureAwait(false);
return;
}
var source = list.First();
var options = new HttpRequestOptions
{
CancellationToken = cancellationToken,
Url = source.Path,
Progress = new Progress<double>()
};
foreach (var header in source.RequiredHttpHeaders)
{
options.RequestHeaders[header.Key] = header.Value;
}
var destination = Path.Combine(path, item.ChannelId, source.Path.GetMD5().ToString("N"));
Directory.CreateDirectory(Path.GetDirectoryName(destination));
// Determine output extension
var response = await _httpClient.GetTempFileResponse(options).ConfigureAwait(false);
if (item.IsVideo && response.ContentType.StartsWith("video/", StringComparison.OrdinalIgnoreCase))
{
var extension = response.ContentType.Split('/')
.Last();
destination += "." + extension;
}
else if (item.IsAudio && response.ContentType.StartsWith("audio/", StringComparison.OrdinalIgnoreCase))
{
var extension = response.ContentType.Replace("audio/mpeg", "audio/mp3", StringComparison.OrdinalIgnoreCase)
.Split('/')
.Last();
destination += "." + extension;
}
else
{
throw new ApplicationException("Unexpected response type encountered: " + response.ContentType);
}
File.Move(response.TempFilePath, destination);
await RefreshMediaSourceItem(destination, item.IsVideo, cancellationToken).ConfigureAwait(false);
}
private async Task RefreshMediaSourceItems(IEnumerable<MediaSourceInfo> items, bool isVideo, CancellationToken cancellationToken)
{
foreach (var item in items)
{
await RefreshMediaSourceItem(item.Path, isVideo, cancellationToken).ConfigureAwait(false);
}
}
private async Task RefreshMediaSourceItem(string path, bool isVideo, CancellationToken cancellationToken)
{
var item = _libraryManager.ResolvePath(new FileInfo(path));
if (item != null)
{
await item.RefreshMetadata(cancellationToken).ConfigureAwait(false);
}
}
public IEnumerable<ITaskTrigger> GetDefaultTriggers()
{
return new ITaskTrigger[]
{
new DailyTrigger { TimeOfDay = TimeSpan.FromHours(3) },
};
}
/// <summary>
/// Deletes the cache files from directory with a last write time less than a given date
/// </summary>
/// <param name="cancellationToken">The task cancellation token.</param>
/// <param name="directory">The directory.</param>
/// <param name="minDateModified">The min date modified.</param>
/// <param name="progress">The progress.</param>
private void DeleteCacheFilesFromDirectory(CancellationToken cancellationToken, string directory, DateTime minDateModified, IProgress<double> progress)
{
var filesToDelete = new DirectoryInfo(directory).EnumerateFiles("*", SearchOption.AllDirectories)
.Where(f => _fileSystem.GetLastWriteTimeUtc(f) < minDateModified)
.ToList();
var index = 0;
foreach (var file in filesToDelete)
{
double percent = index;
percent /= filesToDelete.Count;
progress.Report(100 * percent);
cancellationToken.ThrowIfCancellationRequested();
DeleteFile(file.FullName);
index++;
}
progress.Report(100);
}
/// <summary>
/// Deletes the file.
/// </summary>
/// <param name="path">The path.</param>
private void DeleteFile(string path)
{
try
{
File.Delete(path);
}
catch (IOException ex)
{
_logger.ErrorException("Error deleting file {0}", ex, path);
}
}
public bool IsHidden
{
get
{
return !_manager.GetAllChannelFeatures()
.Any(i => i.CanDownloadAllMedia && _config.Configuration.ChannelOptions.DownloadingChannels.Contains(i.Id));
}
}
public bool IsEnabled
{
get
{
return true;
}
}
}
}

View File

@@ -54,6 +54,19 @@ namespace MediaBrowser.Server.Implementations.Channels
_factories = factories.ToArray();
}
public string ChannelDownloadPath
{
get
{
if (!string.IsNullOrWhiteSpace(_config.Configuration.ChannelOptions.DownloadPath))
{
return _config.Configuration.ChannelOptions.DownloadPath;
}
return Path.Combine(_config.ApplicationPaths.ProgramDataPath, "channels");
}
}
private IEnumerable<IChannel> GetAllChannels()
{
return _factories
@@ -156,7 +169,7 @@ namespace MediaBrowser.Server.Implementations.Channels
progress.Report(100);
}
public Task<IEnumerable<ChannelMediaInfo>> GetChannelItemMediaSources(string id, CancellationToken cancellationToken)
public async Task<IEnumerable<MediaSourceInfo>> GetChannelItemMediaSources(string id, CancellationToken cancellationToken)
{
var item = (IChannelMediaItem)_libraryManager.GetItemById(id);
@@ -166,12 +179,149 @@ namespace MediaBrowser.Server.Implementations.Channels
var requiresCallback = channelPlugin as IRequiresMediaInfoCallback;
IEnumerable<ChannelMediaInfo> results;
if (requiresCallback != null)
{
return requiresCallback.GetChannelItemMediaInfo(item.ExternalId, cancellationToken);
results = await requiresCallback.GetChannelItemMediaInfo(item.ExternalId, cancellationToken)
.ConfigureAwait(false);
}
else
{
results = item.ChannelMediaSources;
}
return Task.FromResult<IEnumerable<ChannelMediaInfo>>(item.ChannelMediaSources);
var sources = SortMediaInfoResults(results).Select(i => GetMediaSource(item, i))
.ToList();
var channelIdString = channel.Id.ToString("N");
var isVideo = string.Equals(item.MediaType, MediaType.Video, StringComparison.OrdinalIgnoreCase);
var cachedVersionTasks = sources
.Select(i => GetCachedVersion(channelIdString, i, isVideo, cancellationToken));
var cachedVersions = await Task.WhenAll(cachedVersionTasks).ConfigureAwait(false);
sources.InsertRange(0, cachedVersions.Where(i => i != null));
return sources;
}
private MediaSourceInfo GetMediaSource(IChannelMediaItem item, ChannelMediaInfo info)
{
var id = info.Path.GetMD5().ToString("N");
var source = new MediaSourceInfo
{
MediaStreams = GetMediaStreams(info).ToList(),
Container = info.Container,
LocationType = info.IsRemote ? LocationType.Remote : LocationType.FileSystem,
Path = info.Path,
RequiredHttpHeaders = info.RequiredHttpHeaders,
RunTimeTicks = item.RunTimeTicks,
Name = id,
Id = id
};
return source;
}
private async Task<MediaSourceInfo> GetCachedVersion(string channelId,
MediaSourceInfo info,
bool isVideo,
CancellationToken cancellationToken)
{
var filename = info.Path.GetMD5().ToString("N");
var path = Path.Combine(ChannelDownloadPath, channelId, filename);
try
{
var file = Directory.EnumerateFiles(Path.GetDirectoryName(path), "*", SearchOption.TopDirectoryOnly)
.FirstOrDefault(i => (Path.GetFileName(i) ?? string.Empty).StartsWith(filename, StringComparison.OrdinalIgnoreCase));
if (!string.IsNullOrWhiteSpace(file))
{
var source = new MediaSourceInfo
{
Path = file,
LocationType = LocationType.FileSystem,
Name = "Cached " + info.Name,
Id = file.GetMD5().ToString("N")
};
if (isVideo)
{
source.VideoType = VideoType.VideoFile;
}
return source;
}
}
catch (DirectoryNotFoundException)
{
return null;
}
return null;
}
private IEnumerable<MediaStream> GetMediaStreams(ChannelMediaInfo info)
{
var list = new List<MediaStream>();
if (!string.IsNullOrWhiteSpace(info.VideoCodec) &&
!string.IsNullOrWhiteSpace(info.AudioCodec))
{
list.Add(new MediaStream
{
Type = MediaStreamType.Video,
Width = info.Width,
RealFrameRate = info.Framerate,
Profile = info.VideoProfile,
Level = info.VideoLevel,
Index = -1,
Height = info.Height,
Codec = info.VideoCodec,
BitRate = info.VideoBitrate,
AverageFrameRate = info.Framerate
});
list.Add(new MediaStream
{
Type = MediaStreamType.Audio,
Index = -1,
Codec = info.AudioCodec,
BitRate = info.AudioBitrate,
Channels = info.AudioChannels,
SampleRate = info.AudioSampleRate
});
}
return list;
}
private IEnumerable<ChannelMediaInfo> SortMediaInfoResults(IEnumerable<ChannelMediaInfo> channelMediaSources)
{
var list = channelMediaSources.ToList();
var width = _config.Configuration.ChannelOptions.PreferredStreamingWidth;
if (width.HasValue)
{
var val = width.Value;
return list
.OrderBy(i => i.Width.HasValue && i.Width.Value <= val)
.ThenBy(i => Math.Abs(i.Width ?? 0 - val))
.ThenByDescending(i => i.Width ?? 0)
.ThenBy(list.IndexOf);
}
return list
.OrderByDescending(i => i.Width ?? 0)
.ThenBy(list.IndexOf);
}
private async Task<Channel> GetChannel(IChannel channelInfo, CancellationToken cancellationToken)
@@ -237,26 +387,37 @@ namespace MediaBrowser.Server.Implementations.Channels
return (Channel)_libraryManager.GetItemById(new Guid(id));
}
public IEnumerable<ChannelFeatures> GetAllChannelFeatures()
{
return _channelEntities
.OrderBy(i => i.SortName)
.Select(i => GetChannelFeatures(i.Id.ToString("N")));
}
public ChannelFeatures GetChannelFeatures(string id)
{
var channel = GetChannel(id);
var channelProvider = GetChannelProvider(channel);
return GetChannelFeaturesDto(channelProvider.GetChannelFeatures());
return GetChannelFeaturesDto(channel, channelProvider.GetChannelFeatures());
}
public ChannelFeatures GetChannelFeaturesDto(InternalChannelFeatures features)
public ChannelFeatures GetChannelFeaturesDto(Channel channel, InternalChannelFeatures features)
{
return new ChannelFeatures
{
CanFilter = !features.MaxPageSize.HasValue,
CanGetAllMedia = features.CanGetAllMedia,
CanSearch = features.CanSearch,
ContentTypes = features.ContentTypes,
DefaultSortFields = features.DefaultSortFields,
MaxPageSize = features.MaxPageSize,
MediaTypes = features.MediaTypes,
SupportsSortOrderToggle = features.SupportsSortOrderToggle
SupportsSortOrderToggle = features.SupportsSortOrderToggle,
Name = channel.Name,
Id = channel.Id.ToString("N"),
CanDownloadAllMedia = features.CanGetAllMedia
};
}
@@ -270,6 +431,85 @@ namespace MediaBrowser.Server.Implementations.Channels
return ("Channel " + name).GetMBId(typeof(Channel));
}
public async Task<QueryResult<BaseItemDto>> GetAllMedia(AllChannelMediaQuery query, CancellationToken cancellationToken)
{
var user = string.IsNullOrWhiteSpace(query.UserId)
? null
: _userManager.GetUserById(new Guid(query.UserId));
var channels = _channels;
if (query.ChannelIds.Length > 0)
{
channels = channels
.Where(i => query.ChannelIds.Contains(GetInternalChannelId(i.Name).ToString("N")))
.ToArray();
}
var tasks = channels
.Where(i => i.GetChannelFeatures().CanGetAllMedia)
.Select(async i =>
{
try
{
var result = await i.GetAllMedia(new InternalAllChannelMediaQuery
{
User = user
}, cancellationToken).ConfigureAwait(false);
return new Tuple<IChannel, ChannelItemResult>(i, result);
}
catch (Exception ex)
{
_logger.ErrorException("Error getting all media from {0}", ex, i.Name);
return new Tuple<IChannel, ChannelItemResult>(i, new ChannelItemResult { });
}
});
var results = await Task.WhenAll(tasks).ConfigureAwait(false);
var totalCount = results.Length;
IEnumerable<Tuple<IChannel, ChannelItemInfo>> items = results
.SelectMany(i => i.Item2.Items.Select(m => new Tuple<IChannel, ChannelItemInfo>(i.Item1, m)))
.OrderBy(i => i.Item2.Name);
if (query.StartIndex.HasValue)
{
items = items.Skip(query.StartIndex.Value);
}
if (query.Limit.HasValue)
{
items = items.Take(query.Limit.Value);
}
// Avoid implicitly captured closure
var token = cancellationToken;
var itemTasks = items.Select(i =>
{
var channelProvider = i.Item1;
var channel = GetChannel(GetInternalChannelId(channelProvider.Name).ToString("N"));
return GetChannelItemEntity(i.Item2, channelProvider, channel, token);
});
var internalItems = await Task.WhenAll(itemTasks).ConfigureAwait(false);
// Get everything
var fields = Enum.GetNames(typeof(ItemFields))
.Select(i => (ItemFields)Enum.Parse(typeof(ItemFields), i, true))
.ToList();
var returnItemArray = internalItems.Select(i => _dtoService.GetBaseItemDto(i, fields, user))
.ToArray();
return new QueryResult<BaseItemDto>
{
TotalRecordCount = totalCount,
Items = returnItemArray
};
}
public async Task<QueryResult<BaseItemDto>> GetChannelItems(ChannelItemQuery query, CancellationToken cancellationToken)
{
var queryChannelId = query.ChannelId;
@@ -301,7 +541,7 @@ namespace MediaBrowser.Server.Implementations.Channels
ChannelItemSortField? sortField = null;
ChannelItemSortField parsedField;
if (query.SortBy.Length == 1 &&
if (query.SortBy.Length == 1 &&
Enum.TryParse(query.SortBy[0], true, out parsedField))
{
sortField = parsedField;
@@ -309,11 +549,11 @@ namespace MediaBrowser.Server.Implementations.Channels
var sortDescending = query.SortOrder.HasValue && query.SortOrder.Value == SortOrder.Descending;
var itemsResult = await GetChannelItems(channelProvider,
user,
query.FolderId,
providerStartIndex,
providerLimit,
var itemsResult = await GetChannelItems(channelProvider,
user,
query.FolderId,
providerStartIndex,
providerLimit,
sortField,
sortDescending,
cancellationToken)