extracted provider manager. took more off the kernel

MediaBrowser.Server.Implementations/IO/DirectoryWatchers.cs

@@ -0,0 +1,543 @@
+using MediaBrowser.Common.ScheduledTasks;
+using MediaBrowser.Controller.Configuration;
+using MediaBrowser.Controller.Entities;
+using MediaBrowser.Controller.IO;
+using MediaBrowser.Controller.Library;
+using MediaBrowser.Controller.ScheduledTasks;
+using MediaBrowser.Model.Logging;
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace MediaBrowser.Server.Implementations.IO
+{
+    /// <summary>
+    /// Class DirectoryWatchers
+    /// </summary>
+    public class DirectoryWatchers : IDirectoryWatchers
+    {
+        /// <summary>
+        /// The file system watchers
+        /// </summary>
+        private ConcurrentBag<FileSystemWatcher> _fileSystemWatchers = new ConcurrentBag<FileSystemWatcher>();
+        /// <summary>
+        /// The update timer
+        /// </summary>
+        private Timer _updateTimer;
+        /// <summary>
+        /// The affected paths
+        /// </summary>
+        private readonly ConcurrentDictionary<string, string> _affectedPaths = new ConcurrentDictionary<string, string>();
+
+        /// <summary>
+        /// A dynamic list of paths that should be ignored.  Added to during our own file sytem modifications.
+        /// </summary>
+        private readonly ConcurrentDictionary<string,string> _tempIgnoredPaths = new ConcurrentDictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+
+        /// <summary>
+        /// The timer lock
+        /// </summary>
+        private readonly object _timerLock = new object();
+
+        /// <summary>
+        /// Add the path to our temporary ignore list.  Use when writing to a path within our listening scope.
+        /// </summary>
+        /// <param name="path">The path.</param>
+        public void TemporarilyIgnore(string path)
+        {
+            _tempIgnoredPaths[path] = path;
+        }
+
+        /// <summary>
+        /// Removes the temp ignore.
+        /// </summary>
+        /// <param name="path">The path.</param>
+        public void RemoveTempIgnore(string path)
+        {
+            string val;
+            _tempIgnoredPaths.TryRemove(path, out val);
+        }
+
+        /// <summary>
+        /// Gets or sets the logger.
+        /// </summary>
+        /// <value>The logger.</value>
+        private ILogger Logger { get; set; }
+
+        /// <summary>
+        /// Gets or sets the task manager.
+        /// </summary>
+        /// <value>The task manager.</value>
+        private ITaskManager TaskManager { get; set; }
+
+        private ILibraryManager LibraryManager { get; set; }
+        private IServerConfigurationManager ConfigurationManager { get; set; }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="DirectoryWatchers" /> class.
+        /// </summary>
+        public DirectoryWatchers(ILogManager logManager, ITaskManager taskManager, ILibraryManager libraryManager, IServerConfigurationManager configurationManager)
+        {
+            if (taskManager == null)
+            {
+                throw new ArgumentNullException("taskManager");
+            }
+
+            LibraryManager = libraryManager;
+            TaskManager = taskManager;
+            Logger = logManager.GetLogger("DirectoryWatchers");
+            ConfigurationManager = configurationManager;
+        }
+        
+        /// <summary>
+        /// Starts this instance.
+        /// </summary>
+        public void Start()
+        {
+            LibraryManager.LibraryChanged += Instance_LibraryChanged;
+
+            var pathsToWatch = new List<string> { LibraryManager.RootFolder.Path };
+
+            var paths = LibraryManager.RootFolder.Children.OfType<Folder>()
+                .SelectMany(f =>
+                    {
+                        try
+                        {
+                            // Accessing ResolveArgs could involve file system access
+                            return f.ResolveArgs.PhysicalLocations;
+                        }
+                        catch (IOException)
+                        {
+                            return new string[] {};
+                        }
+
+                    })
+                .Where(Path.IsPathRooted);
+
+            foreach (var path in paths)
+            {
+                if (!ContainsParentFolder(pathsToWatch, path))
+                {
+                    pathsToWatch.Add(path);
+                }
+            }
+
+            foreach (var path in pathsToWatch)
+            {
+                StartWatchingPath(path);
+            }
+        }
+
+        /// <summary>
+        /// Examine a list of strings assumed to be file paths to see if it contains a parent of
+        /// the provided path.
+        /// </summary>
+        /// <param name="lst">The LST.</param>
+        /// <param name="path">The path.</param>
+        /// <returns><c>true</c> if [contains parent folder] [the specified LST]; otherwise, <c>false</c>.</returns>
+        /// <exception cref="System.ArgumentNullException">path</exception>
+        private static bool ContainsParentFolder(IEnumerable<string> lst, string path)
+        {
+            if (string.IsNullOrEmpty(path))
+            {
+                throw new ArgumentNullException("path");
+            }
+
+            path = path.TrimEnd(Path.DirectorySeparatorChar);
+
+            return lst.Any(str =>
+            {
+                //this should be a little quicker than examining each actual parent folder...
+                var compare = str.TrimEnd(Path.DirectorySeparatorChar);
+
+                return (path.Equals(compare, StringComparison.OrdinalIgnoreCase) || (path.StartsWith(compare, StringComparison.OrdinalIgnoreCase) && path[compare.Length] == Path.DirectorySeparatorChar));
+            });
+        }
+
+        /// <summary>
+        /// Starts the watching path.
+        /// </summary>
+        /// <param name="path">The path.</param>
+        private void StartWatchingPath(string path)
+        {
+            // Creating a FileSystemWatcher over the LAN can take hundreds of milliseconds, so wrap it in a Task to do them all in parallel
+            Task.Run(() =>
+            {
+                var newWatcher = new FileSystemWatcher(path, "*") { IncludeSubdirectories = true, InternalBufferSize = 32767 };
+
+                newWatcher.Created += watcher_Changed;
+                newWatcher.Deleted += watcher_Changed;
+                newWatcher.Renamed += watcher_Changed;
+                newWatcher.Changed += watcher_Changed;
+
+                newWatcher.Error += watcher_Error;
+
+                try
+                {
+                    newWatcher.EnableRaisingEvents = true;
+                    _fileSystemWatchers.Add(newWatcher);
+
+                    Logger.Info("Watching directory " + path);
+                }
+                catch (IOException ex)
+                {
+                    Logger.ErrorException("Error watching path: {0}", ex, path);
+                }
+                catch (PlatformNotSupportedException ex)
+                {
+                    Logger.ErrorException("Error watching path: {0}", ex, path);
+                }
+            });
+        }
+
+        /// <summary>
+        /// Stops the watching path.
+        /// </summary>
+        /// <param name="path">The path.</param>
+        private void StopWatchingPath(string path)
+        {
+            var watcher = _fileSystemWatchers.FirstOrDefault(f => f.Path.Equals(path, StringComparison.OrdinalIgnoreCase));
+
+            if (watcher != null)
+            {
+                DisposeWatcher(watcher);
+            }
+        }
+
+        /// <summary>
+        /// Disposes the watcher.
+        /// </summary>
+        /// <param name="watcher">The watcher.</param>
+        private void DisposeWatcher(FileSystemWatcher watcher)
+        {
+            Logger.Info("Stopping directory watching for path {0}", watcher.Path);
+
+            watcher.EnableRaisingEvents = false;
+            watcher.Dispose();
+
+            var watchers = _fileSystemWatchers.ToList();
+
+            watchers.Remove(watcher);
+
+            _fileSystemWatchers = new ConcurrentBag<FileSystemWatcher>(watchers);
+        }
+
+        /// <summary>
+        /// Handles the LibraryChanged event of the Kernel
+        /// </summary>
+        /// <param name="sender">The source of the event.</param>
+        /// <param name="e">The <see cref="MediaBrowser.Controller.Library.ChildrenChangedEventArgs" /> instance containing the event data.</param>
+        void Instance_LibraryChanged(object sender, ChildrenChangedEventArgs e)
+        {
+            if (e.Folder is AggregateFolder && e.HasAddOrRemoveChange)
+            {
+                if (e.ItemsRemoved != null)
+                {
+                    foreach (var item in e.ItemsRemoved.OfType<Folder>())
+                    {
+                        StopWatchingPath(item.Path);
+                    }
+                }
+                if (e.ItemsAdded != null)
+                {
+                    foreach (var item in e.ItemsAdded.OfType<Folder>())
+                    {
+                        StartWatchingPath(item.Path);
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Handles the Error event of the watcher control.
+        /// </summary>
+        /// <param name="sender">The source of the event.</param>
+        /// <param name="e">The <see cref="ErrorEventArgs" /> instance containing the event data.</param>
+        async void watcher_Error(object sender, ErrorEventArgs e)
+        {
+            var ex = e.GetException();
+            var dw = (FileSystemWatcher) sender;
+
+            Logger.ErrorException("Error in Directory watcher for: "+dw.Path, ex);
+
+            if (ex.Message.Contains("network name is no longer available"))
+            {
+                //Network either dropped or, we are coming out of sleep and it hasn't reconnected yet - wait and retry
+                Logger.Warn("Network connection lost - will retry...");
+                var retries = 0;
+                var success = false;
+                while (!success && retries < 10)
+                {
+                    await Task.Delay(500).ConfigureAwait(false);
+
+                    try
+                    {
+                        dw.EnableRaisingEvents = false;
+                        dw.EnableRaisingEvents = true;
+                        success = true;
+                    }
+                    catch (IOException)
+                    {
+                        Logger.Warn("Network still unavailable...");
+                        retries++;
+                    }
+                }
+                if (!success)
+                {
+                    Logger.Warn("Unable to access network. Giving up.");
+                    DisposeWatcher(dw);
+                }
+
+            }
+            else
+            {
+                if (!ex.Message.Contains("BIOS command limit"))
+                {
+                    Logger.Info("Attempting to re-start watcher.");
+
+                    dw.EnableRaisingEvents = false;
+                    dw.EnableRaisingEvents = true;
+                }
+                
+            }
+        }
+
+        /// <summary>
+        /// Handles the Changed event of the watcher control.
+        /// </summary>
+        /// <param name="sender">The source of the event.</param>
+        /// <param name="e">The <see cref="FileSystemEventArgs" /> instance containing the event data.</param>
+        void watcher_Changed(object sender, FileSystemEventArgs e)
+        {
+            if (e.ChangeType == WatcherChangeTypes.Created && e.Name == "New folder")
+            {
+                return;
+            }
+            if (_tempIgnoredPaths.ContainsKey(e.FullPath))
+            {
+                Logger.Info("Watcher requested to ignore change to " + e.FullPath);
+                return;
+            }
+
+            Logger.Info("Watcher sees change of type " + e.ChangeType.ToString() + " to " + e.FullPath);
+
+            //Since we're watching created, deleted and renamed we always want the parent of the item to be the affected path
+            var affectedPath = e.FullPath;
+
+            _affectedPaths.AddOrUpdate(affectedPath, affectedPath, (key, oldValue) => affectedPath);
+
+            lock (_timerLock)
+            {
+                if (_updateTimer == null)
+                {
+                    _updateTimer = new Timer(TimerStopped, null, TimeSpan.FromSeconds(ConfigurationManager.Configuration.FileWatcherDelay), TimeSpan.FromMilliseconds(-1));
+                }
+                else
+                {
+                    _updateTimer.Change(TimeSpan.FromSeconds(ConfigurationManager.Configuration.FileWatcherDelay), TimeSpan.FromMilliseconds(-1));
+                }
+            }
+        }
+
+        /// <summary>
+        /// Timers the stopped.
+        /// </summary>
+        /// <param name="stateInfo">The state info.</param>
+        private async void TimerStopped(object stateInfo)
+        {
+            lock (_timerLock)
+            {
+                // Extend the timer as long as any of the paths are still being written to.
+                if (_affectedPaths.Any(p => IsFileLocked(p.Key)))
+                {
+                    Logger.Info("Timer extended.");
+                    _updateTimer.Change(TimeSpan.FromSeconds(ConfigurationManager.Configuration.FileWatcherDelay), TimeSpan.FromMilliseconds(-1));
+                    return;
+                }
+
+                Logger.Info("Timer stopped.");
+
+                _updateTimer.Dispose();
+                _updateTimer = null;
+            }
+
+            var paths = _affectedPaths.Keys.ToList();
+            _affectedPaths.Clear();
+
+            await ProcessPathChanges(paths).ConfigureAwait(false);
+        }
+
+        /// <summary>
+        /// Try and determine if a file is locked
+        /// This is not perfect, and is subject to race conditions, so I'd rather not make this a re-usable library method.
+        /// </summary>
+        /// <param name="path">The path.</param>
+        /// <returns><c>true</c> if [is file locked] [the specified path]; otherwise, <c>false</c>.</returns>
+        private bool IsFileLocked(string path)
+        {
+            try
+            {
+                var data = FileSystem.GetFileData(path);
+
+                if (!data.HasValue || data.Value.IsDirectory)
+                {
+                    return false;
+                }
+            }
+            catch (IOException)
+            {
+                return false;
+            }
+
+            FileStream stream = null;
+
+            try
+            {
+                stream = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite);
+            }
+            catch
+            {
+                //the file is unavailable because it is:
+                //still being written to
+                //or being processed by another thread
+                //or does not exist (has already been processed)
+                return true;
+            }
+            finally
+            {
+                if (stream != null)
+                    stream.Close();
+            }
+
+            //file is not locked
+            return false;
+        }
+
+        /// <summary>
+        /// Processes the path changes.
+        /// </summary>
+        /// <param name="paths">The paths.</param>
+        /// <returns>Task.</returns>
+        private async Task ProcessPathChanges(List<string> paths)
+        {
+            var itemsToRefresh = paths.Select(Path.GetDirectoryName)
+                .Select(GetAffectedBaseItem)
+                .Where(item => item != null)
+                .Distinct()
+                .ToList();
+
+            foreach (var p in paths) Logger.Info(p + " reports change.");
+
+            // If the root folder changed, run the library task so the user can see it
+            if (itemsToRefresh.Any(i => i is AggregateFolder))
+            {
+                TaskManager.CancelIfRunningAndQueue<RefreshMediaLibraryTask>();
+                return;
+            }
+
+            await Task.WhenAll(itemsToRefresh.Select(i => Task.Run(async () =>
+            {
+                Logger.Info(i.Name + " (" + i.Path + ") will be refreshed.");
+                
+                try
+                {
+                    await i.ChangedExternally().ConfigureAwait(false);
+                }
+                catch (IOException ex)
+                {
+                    // For now swallow and log. 
+                    // Research item: If an IOException occurs, the item may be in a disconnected state (media unavailable)
+                    // Should we remove it from it's parent?
+                    Logger.ErrorException("Error refreshing {0}", ex, i.Name);
+                }
+
+            }))).ConfigureAwait(false);
+        }
+
+        /// <summary>
+        /// Gets the affected base item.
+        /// </summary>
+        /// <param name="path">The path.</param>
+        /// <returns>BaseItem.</returns>
+        private BaseItem GetAffectedBaseItem(string path)
+        {
+            BaseItem item = null;
+
+            while (item == null && !string.IsNullOrEmpty(path))
+            {
+                item = LibraryManager.RootFolder.FindByPath(path);
+
+                path = Path.GetDirectoryName(path);
+            }
+
+            if (item != null)
+            {
+                // If the item has been deleted find the first valid parent that still exists
+                while (!Directory.Exists(item.Path) && !File.Exists(item.Path))
+                {
+                    item = item.Parent;
+
+                    if (item == null)
+                    {
+                        break;
+                    }
+                }
+            }
+
+            return item;
+        }
+
+        /// <summary>
+        /// Stops this instance.
+        /// </summary>
+        public void Stop()
+        {
+            LibraryManager.LibraryChanged -= Instance_LibraryChanged;
+
+            FileSystemWatcher watcher;
+
+            while (_fileSystemWatchers.TryTake(out watcher))
+            {
+                watcher.Changed -= watcher_Changed;
+                watcher.EnableRaisingEvents = false;
+                watcher.Dispose();
+            }
+
+            lock (_timerLock)
+            {
+                if (_updateTimer != null)
+                {
+                    _updateTimer.Dispose();
+                    _updateTimer = null;
+                }
+            } 
+
+            _affectedPaths.Clear();
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        /// <summary>
+        /// Releases unmanaged and - optionally - managed resources.
+        /// </summary>
+        /// <param name="dispose"><c>true</c> to release both managed and unmanaged resources; <c>false</c> to release only unmanaged resources.</param>
+        protected virtual void Dispose(bool dispose)
+        {
+            if (dispose)
+            {
+                Stop();
+            }
+        }
+    }
+}

MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj

@@ -111,6 +111,7 @@
     <Compile Include="HttpServer\ServerFactory.cs" />
     <Compile Include="HttpServer\StreamWriter.cs" />
     <Compile Include="HttpServer\SwaggerService.cs" />
+    <Compile Include="IO\DirectoryWatchers.cs" />
     <Compile Include="Library\CoreResolutionIgnoreRule.cs" />
     <Compile Include="Library\LibraryManager.cs" />
     <Compile Include="Library\ResolverHelper.cs" />
@@ -128,6 +129,7 @@
     <Compile Include="Library\Resolvers\VideoResolver.cs" />
     <Compile Include="Library\UserManager.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Providers\ProviderManager.cs" />
     <Compile Include="Reflection\TypeMapper.cs" />
     <Compile Include="ScheduledTasks\PeopleValidationTask.cs" />
     <Compile Include="ScheduledTasks\ChapterImagesTask.cs" />

MediaBrowser.Server.Implementations/Providers/ProviderManager.cs

@@ -0,0 +1,487 @@
+using MediaBrowser.Common.Extensions;
+using MediaBrowser.Common.IO;
+using MediaBrowser.Common.Net;
+using MediaBrowser.Controller;
+using MediaBrowser.Controller.Configuration;
+using MediaBrowser.Controller.Entities;
+using MediaBrowser.Controller.IO;
+using MediaBrowser.Controller.Providers;
+using MediaBrowser.Model.Logging;
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace MediaBrowser.Server.Implementations.Providers
+{
+    /// <summary>
+    /// Class ProviderManager
+    /// </summary>
+    public class ProviderManager : IProviderManager
+    {
+        /// <summary>
+        /// The remote image cache
+        /// </summary>
+        private readonly FileSystemRepository _remoteImageCache;
+
+        /// <summary>
+        /// The currently running metadata providers
+        /// </summary>
+        private readonly ConcurrentDictionary<string, Tuple<BaseMetadataProvider, BaseItem, CancellationTokenSource>> _currentlyRunningProviders =
+            new ConcurrentDictionary<string, Tuple<BaseMetadataProvider, BaseItem, CancellationTokenSource>>();
+
+        /// <summary>
+        /// The _logger
+        /// </summary>
+        private readonly ILogger _logger;
+
+        /// <summary>
+        /// The _HTTP client
+        /// </summary>
+        private readonly IHttpClient _httpClient;
+
+        /// <summary>
+        /// The _directory watchers
+        /// </summary>
+        private readonly IDirectoryWatchers _directoryWatchers;
+
+        /// <summary>
+        /// Gets or sets the configuration manager.
+        /// </summary>
+        /// <value>The configuration manager.</value>
+        private IServerConfigurationManager ConfigurationManager { get; set; }
+
+        /// <summary>
+        /// Gets the list of currently registered metadata prvoiders
+        /// </summary>
+        /// <value>The metadata providers enumerable.</value>
+        private BaseMetadataProvider[] MetadataProviders { get; set; }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ProviderManager" /> class.
+        /// </summary>
+        /// <param name="httpClient">The HTTP client.</param>
+        /// <param name="configurationManager">The configuration manager.</param>
+        /// <param name="directoryWatchers">The directory watchers.</param>
+        /// <param name="logManager">The log manager.</param>
+        public ProviderManager(IHttpClient httpClient, IServerConfigurationManager configurationManager, IDirectoryWatchers directoryWatchers, ILogManager logManager)
+        {
+            _logger = logManager.GetLogger("ProviderManager");
+            _httpClient = httpClient;
+            ConfigurationManager = configurationManager;
+            _directoryWatchers = directoryWatchers;
+            _remoteImageCache = new FileSystemRepository(configurationManager.ApplicationPaths.DownloadedImagesDataPath);
+
+            configurationManager.ConfigurationUpdated += configurationManager_ConfigurationUpdated;
+        }
+
+        /// <summary>
+        /// Handles the ConfigurationUpdated event of the configurationManager control.
+        /// </summary>
+        /// <param name="sender">The source of the event.</param>
+        /// <param name="e">The <see cref="EventArgs" /> instance containing the event data.</param>
+        void configurationManager_ConfigurationUpdated(object sender, EventArgs e)
+        {
+            // Validate currently executing providers, in the background
+            Task.Run(() =>
+            {
+                ValidateCurrentlyRunningProviders();
+            });
+        }
+
+        /// <summary>
+        /// Gets or sets the supported providers key.
+        /// </summary>
+        /// <value>The supported providers key.</value>
+        private Guid SupportedProvidersKey { get; set; }
+
+        /// <summary>
+        /// Adds the metadata providers.
+        /// </summary>
+        /// <param name="providers">The providers.</param>
+        public void AddMetadataProviders(IEnumerable<BaseMetadataProvider> providers)
+        {
+            MetadataProviders = providers.ToArray();
+        }
+
+        /// <summary>
+        /// Runs all metadata providers for an entity, and returns true or false indicating if at least one was refreshed and requires persistence
+        /// </summary>
+        /// <param name="item">The item.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <param name="force">if set to <c>true</c> [force].</param>
+        /// <param name="allowSlowProviders">if set to <c>true</c> [allow slow providers].</param>
+        /// <returns>Task{System.Boolean}.</returns>
+        public async Task<bool> ExecuteMetadataProviders(BaseItem item, CancellationToken cancellationToken, bool force = false, bool allowSlowProviders = true)
+        {
+            // Allow providers of the same priority to execute in parallel
+            MetadataProviderPriority? currentPriority = null;
+            var currentTasks = new List<Task<bool>>();
+
+            var result = false;
+
+            cancellationToken.ThrowIfCancellationRequested();
+
+            // Determine if supported providers have changed
+            var supportedProviders = MetadataProviders.Where(p => p.Supports(item)).ToList();
+
+            BaseProviderInfo supportedProvidersInfo;
+
+            if (SupportedProvidersKey == Guid.Empty)
+            {
+                SupportedProvidersKey = "SupportedProviders".GetMD5();
+            }
+
+            var supportedProvidersHash = string.Join("+", supportedProviders.Select(i => i.GetType().Name)).GetMD5();
+            bool providersChanged;
+
+            item.ProviderData.TryGetValue(SupportedProvidersKey, out supportedProvidersInfo);
+            if (supportedProvidersInfo == null)
+            {
+                // First time
+                supportedProvidersInfo = new BaseProviderInfo { ProviderId = SupportedProvidersKey, FileSystemStamp = supportedProvidersHash };
+                providersChanged = force = true;
+            }
+            else
+            {
+                // Force refresh if the supported providers have changed
+                providersChanged = force = force || supportedProvidersInfo.FileSystemStamp != supportedProvidersHash;
+            }
+
+            // If providers have changed, clear provider info and update the supported providers hash
+            if (providersChanged)
+            {
+                _logger.Debug("Providers changed for {0}. Clearing and forcing refresh.", item.Name);
+                item.ProviderData.Clear();
+                supportedProvidersInfo.FileSystemStamp = supportedProvidersHash;
+            }
+
+            if (force) item.ClearMetaValues();
+
+            // Run the normal providers sequentially in order of priority
+            foreach (var provider in supportedProviders)
+            {
+                cancellationToken.ThrowIfCancellationRequested();
+
+                // Skip if internet providers are currently disabled
+                if (provider.RequiresInternet && !ConfigurationManager.Configuration.EnableInternetProviders)
+                {
+                    continue;
+                }
+
+                // Skip if is slow and we aren't allowing slow ones
+                if (provider.IsSlow && !allowSlowProviders)
+                {
+                    continue;
+                }
+
+                // Skip if internet provider and this type is not allowed
+                if (provider.RequiresInternet && ConfigurationManager.Configuration.EnableInternetProviders && ConfigurationManager.Configuration.InternetProviderExcludeTypes.Contains(item.GetType().Name, StringComparer.OrdinalIgnoreCase))
+                {
+                    continue;
+                }
+
+                // When a new priority is reached, await the ones that are currently running and clear the list
+                if (currentPriority.HasValue && currentPriority.Value != provider.Priority && currentTasks.Count > 0)
+                {
+                    var results = await Task.WhenAll(currentTasks).ConfigureAwait(false);
+                    result |= results.Contains(true);
+
+                    currentTasks.Clear();
+                }
+
+                // Put this check below the await because the needs refresh of the next tier of providers may depend on the previous ones running
+                //  This is the case for the fan art provider which depends on the movie and tv providers having run before them
+                if (!force && !provider.NeedsRefresh(item))
+                {
+                    continue;
+                }
+
+                currentTasks.Add(FetchAsync(provider, item, force, cancellationToken));
+                currentPriority = provider.Priority;
+            }
+
+            if (currentTasks.Count > 0)
+            {
+                var results = await Task.WhenAll(currentTasks).ConfigureAwait(false);
+                result |= results.Contains(true);
+            }
+
+            if (providersChanged)
+            {
+                item.ProviderData[SupportedProvidersKey] = supportedProvidersInfo;
+            }
+            
+            return result || providersChanged;
+        }
+
+        /// <summary>
+        /// Fetches metadata and returns true or false indicating if any work that requires persistence was done
+        /// </summary>
+        /// <param name="provider">The provider.</param>
+        /// <param name="item">The item.</param>
+        /// <param name="force">if set to <c>true</c> [force].</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns>Task{System.Boolean}.</returns>
+        /// <exception cref="System.ArgumentNullException"></exception>
+        private async Task<bool> FetchAsync(BaseMetadataProvider provider, BaseItem item, bool force, CancellationToken cancellationToken)
+        {
+            if (item == null)
+            {
+                throw new ArgumentNullException();
+            }
+
+            cancellationToken.ThrowIfCancellationRequested();
+
+            _logger.Info("Running {0} for {1}", provider.GetType().Name, item.Path ?? item.Name ?? "--Unknown--");
+
+            // This provides the ability to cancel just this one provider
+            var innerCancellationTokenSource = new CancellationTokenSource();
+
+            OnProviderRefreshBeginning(provider, item, innerCancellationTokenSource);
+
+            try
+            {
+                return await provider.FetchAsync(item, force, CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, innerCancellationTokenSource.Token).Token).ConfigureAwait(false);
+            }
+            catch (OperationCanceledException ex)
+            {
+                _logger.Info("{0} cancelled for {1}", provider.GetType().Name, item.Name);
+
+                // If the outer cancellation token is the one that caused the cancellation, throw it
+                if (cancellationToken.IsCancellationRequested && ex.CancellationToken == cancellationToken)
+                {
+                    throw;
+                }
+
+                return false;
+            }
+            catch (Exception ex)
+            {
+                _logger.ErrorException("{0} failed refreshing {1}", ex, provider.GetType().Name, item.Name);
+
+                provider.SetLastRefreshed(item, DateTime.UtcNow, ProviderRefreshStatus.Failure);
+                return true;
+            }
+            finally
+            {
+                innerCancellationTokenSource.Dispose();
+
+                OnProviderRefreshCompleted(provider, item);
+            }
+        }
+
+        /// <summary>
+        /// Notifies the kernal that a provider has begun refreshing
+        /// </summary>
+        /// <param name="provider">The provider.</param>
+        /// <param name="item">The item.</param>
+        /// <param name="cancellationTokenSource">The cancellation token source.</param>
+        public void OnProviderRefreshBeginning(BaseMetadataProvider provider, BaseItem item, CancellationTokenSource cancellationTokenSource)
+        {
+            var key = item.Id + provider.GetType().Name;
+
+            Tuple<BaseMetadataProvider, BaseItem, CancellationTokenSource> current;
+
+            if (_currentlyRunningProviders.TryGetValue(key, out current))
+            {
+                try
+                {
+                    current.Item3.Cancel();
+                }
+                catch (ObjectDisposedException)
+                {
+                    
+                }
+            }
+
+            var tuple = new Tuple<BaseMetadataProvider, BaseItem, CancellationTokenSource>(provider, item, cancellationTokenSource);
+
+            _currentlyRunningProviders.AddOrUpdate(key, tuple, (k, v) => tuple);
+        }
+
+        /// <summary>
+        /// Notifies the kernal that a provider has completed refreshing
+        /// </summary>
+        /// <param name="provider">The provider.</param>
+        /// <param name="item">The item.</param>
+        public void OnProviderRefreshCompleted(BaseMetadataProvider provider, BaseItem item)
+        {
+            var key = item.Id + provider.GetType().Name;
+
+            Tuple<BaseMetadataProvider, BaseItem, CancellationTokenSource> current;
+
+            if (_currentlyRunningProviders.TryRemove(key, out current))
+            {
+                current.Item3.Dispose();
+            }
+        }
+
+        /// <summary>
+        /// Validates the currently running providers and cancels any that should not be run due to configuration changes
+        /// </summary>
+        private void ValidateCurrentlyRunningProviders()
+        {
+            _logger.Info("Validing currently running providers");
+
+            var enableInternetProviders = ConfigurationManager.Configuration.EnableInternetProviders;
+            var internetProviderExcludeTypes = ConfigurationManager.Configuration.InternetProviderExcludeTypes;
+
+            foreach (var tuple in _currentlyRunningProviders.Values
+                .Where(p => p.Item1.RequiresInternet && (!enableInternetProviders || internetProviderExcludeTypes.Contains(p.Item2.GetType().Name, StringComparer.OrdinalIgnoreCase)))
+                .ToList())
+            {
+                tuple.Item3.Cancel();
+            }
+        }
+
+        /// <summary>
+        /// Downloads the and save image.
+        /// </summary>
+        /// <param name="item">The item.</param>
+        /// <param name="source">The source.</param>
+        /// <param name="targetName">Name of the target.</param>
+        /// <param name="resourcePool">The resource pool.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns>Task{System.String}.</returns>
+        /// <exception cref="System.ArgumentNullException">item</exception>
+        public async Task<string> DownloadAndSaveImage(BaseItem item, string source, string targetName, SemaphoreSlim resourcePool, CancellationToken cancellationToken)
+        {
+            if (item == null)
+            {
+                throw new ArgumentNullException("item");
+            }
+            if (string.IsNullOrEmpty(source))
+            {
+                throw new ArgumentNullException("source");
+            }
+            if (string.IsNullOrEmpty(targetName))
+            {
+                throw new ArgumentNullException("targetName");
+            }
+            if (resourcePool == null)
+            {
+                throw new ArgumentNullException("resourcePool");
+            }
+
+            //download and save locally
+            var localPath = ConfigurationManager.Configuration.SaveLocalMeta ?
+                Path.Combine(item.MetaLocation, targetName) :
+                _remoteImageCache.GetResourcePath(item.GetType().FullName + item.Path.ToLower(), targetName);
+
+            var img = await _httpClient.GetMemoryStream(source, resourcePool, cancellationToken).ConfigureAwait(false);
+
+            if (ConfigurationManager.Configuration.SaveLocalMeta) // queue to media directories
+            {
+                await SaveToLibraryFilesystem(item, localPath, img, cancellationToken).ConfigureAwait(false);
+            }
+            else
+            {
+                // we can write directly here because it won't affect the watchers
+
+                try
+                {
+                    using (var fs = new FileStream(localPath, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous))
+                    {
+                        await img.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false);
+                    }
+                }
+                catch (OperationCanceledException)
+                {
+                    throw;
+                }
+                catch (Exception e)
+                {
+                    _logger.ErrorException("Error downloading and saving image " + localPath, e);
+                    throw;
+                }
+                finally
+                {
+                    img.Dispose();
+                }
+
+            }
+            return localPath;
+        }
+
+
+        /// <summary>
+        /// Saves to library filesystem.
+        /// </summary>
+        /// <param name="item">The item.</param>
+        /// <param name="path">The path.</param>
+        /// <param name="dataToSave">The data to save.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns>Task.</returns>
+        /// <exception cref="System.ArgumentNullException"></exception>
+        public async Task SaveToLibraryFilesystem(BaseItem item, string path, Stream dataToSave, CancellationToken cancellationToken)
+        {
+            if (item == null)
+            {
+                throw new ArgumentNullException();
+            }
+            if (string.IsNullOrEmpty(path))
+            {
+                throw new ArgumentNullException();
+            }
+            if (dataToSave == null)
+            {
+                throw new ArgumentNullException();
+            }
+            if (cancellationToken == null)
+            {
+                throw new ArgumentNullException();
+            }
+
+            cancellationToken.ThrowIfCancellationRequested();
+
+            //Tell the watchers to ignore
+            _directoryWatchers.TemporarilyIgnore(path);
+
+            //Make the mod
+
+            dataToSave.Position = 0;
+
+            try
+            {
+                using (var fs = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous))
+                {
+                    await dataToSave.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false);
+
+                    dataToSave.Dispose();
+
+                    // If this is ever used for something other than metadata we can add a file type param
+                    item.ResolveArgs.AddMetadataFile(path);
+                }
+            }
+            finally
+            {
+                //Remove the ignore
+                _directoryWatchers.RemoveTempIgnore(path);
+            }
+        }
+
+        /// <summary>
+        /// Releases unmanaged and - optionally - managed resources.
+        /// </summary>
+        /// <param name="dispose"><c>true</c> to release both managed and unmanaged resources; <c>false</c> to release only unmanaged resources.</param>
+        protected virtual void Dispose(bool dispose)
+        {
+            if (dispose)
+            {
+                _remoteImageCache.Dispose();
+            }
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        public void Dispose()
+        {
+            Dispose(true);
+        }
+    }
+}

MediaBrowser.Server.Implementations/ScheduledTasks/ImageCleanupTask.cs

@@ -27,17 +27,21 @@ namespace MediaBrowser.Server.Implementations.ScheduledTasks
         /// </summary>
         private readonly ILogger _logger;
         private readonly ILibraryManager _libraryManager;
+        private readonly IServerApplicationPaths _appPaths;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="ImageCleanupTask" /> class.
         /// </summary>
         /// <param name="kernel">The kernel.</param>
         /// <param name="logger">The logger.</param>
-        public ImageCleanupTask(Kernel kernel, ILogger logger, ILibraryManager libraryManager)
+        /// <param name="libraryManager">The library manager.</param>
+        /// <param name="appPaths">The app paths.</param>
+        public ImageCleanupTask(Kernel kernel, ILogger logger, ILibraryManager libraryManager, IServerApplicationPaths appPaths)
         {
             _kernel = kernel;
             _logger = logger;
             _libraryManager = libraryManager;
+            _appPaths = appPaths;
         }
 
         /// <summary>
@@ -65,7 +69,7 @@ namespace MediaBrowser.Server.Implementations.ScheduledTasks
             // First gather all image files
             var files = GetFiles(_kernel.FFMpegManager.AudioImagesDataPath)
                 .Concat(GetFiles(_kernel.FFMpegManager.VideoImagesDataPath))
-                .Concat(GetFiles(_kernel.ProviderManager.ImagesDataPath))
+                .Concat(GetFiles(_appPaths.DownloadedImagesDataPath))
                 .ToList();
 
             // Now gather all items

MediaBrowser.Server.Implementations/ServerApplicationPaths.cs

@@ -302,7 +302,7 @@ namespace MediaBrowser.Server.Implementations
         /// Gets the FF MPEG stream cache path.
         /// </summary>
         /// <value>The FF MPEG stream cache path.</value>
-        public string FFMpegStreamCachePath
+        public string EncodedMediaCachePath
         {
             get
             {
@@ -345,5 +345,31 @@ namespace MediaBrowser.Server.Implementations
                 return _mediaToolsPath;
             }
         }
+
+        /// <summary>
+        /// The _images data path
+        /// </summary>
+        private string _downloadedImagesDataPath;
+        /// <summary>
+        /// Gets the images data path.
+        /// </summary>
+        /// <value>The images data path.</value>
+        public string DownloadedImagesDataPath
+        {
+            get
+            {
+                if (_downloadedImagesDataPath == null)
+                {
+                    _downloadedImagesDataPath = Path.Combine(DataPath, "remote-images");
+
+                    if (!Directory.Exists(_downloadedImagesDataPath))
+                    {
+                        Directory.CreateDirectory(_downloadedImagesDataPath);
+                    }
+                }
+
+                return _downloadedImagesDataPath;
+            }
+        }
     }
 }

MediaBrowser.Server.Implementations/ServerManager/ServerManager.cs

@@ -104,8 +104,8 @@ namespace MediaBrowser.Server.Implementations.ServerManager
         /// <value>The web socket listeners.</value>
         private readonly List<IWebSocketListener> _webSocketListeners = new List<IWebSocketListener>();
 
-        private Kernel _kernel;
-
+        private readonly Kernel _kernel;
+        
         /// <summary>
         /// Initializes a new instance of the <see cref="ServerManager" /> class.
         /// </summary>