Auto-Organize: Async operation and instant feedback UI (reworked)

This commit includes changes to enable and stabilize asyncronous
operation in the auto-organize area. Here are the key points:

- The auto-organize correction dialog is now closed (almost) instantly.
This means that the user does not have to wait until the file copy/move
operation is completed in order to continue. (even with local HDs the
copy/move process can take several minutes or even much longer with
network destination).
- This commit also implements locking of files to be organized in order
to prevent parallel processing of the same item. In effect, there can be
2 or more manual organization operations active even while the normal
auto-organization task is running without causing any problems
- The items that are currently being processed are indicated as such in
the log with an orange color and a spinner graphic
- The client display is refreshed through websocket messages
- A side effect of this is that other clients showing the auto-organize
log at the same time are always up-to-date as well

MediaBrowser.Server.Implementations/FileOrganization/EpisodeFileOrganizer.cs

@@ -272,6 +272,18 @@ namespace MediaBrowser.Server.Implementations.FileOrganization
 
             var originalExtractedSeriesString = result.ExtractedName;
 
+            bool isNew = string.IsNullOrWhiteSpace(result.Id);
+
+            if (isNew)
+            {
+                await _organizationService.SaveResult(result, cancellationToken);
+            }
+
+            if (!_organizationService.AddToInProgressList(result, isNew))
+            {
+                throw new Exception("File is currently processed otherwise. Please try again later.");
+            }
+            
             try
             {
             // Proceed to sort the file
@@ -363,6 +375,10 @@ namespace MediaBrowser.Server.Implementations.FileOrganization
                 _logger.Warn(ex.Message);
                 return;
             }
+            finally
+            {
+                _organizationService.RemoveFromInprogressList(result);
+            }
 
             if (rememberCorrection)
             {

MediaBrowser.Server.Implementations/FileOrganization/FileOrganizationNotifier.cs

@@ -0,0 +1,68 @@
+using MediaBrowser.Controller.FileOrganization;
+using MediaBrowser.Controller.Plugins;
+using MediaBrowser.Model.Logging;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using MediaBrowser.Model.Events;
+using MediaBrowser.Model.FileOrganization;
+using MediaBrowser.Controller.Session;
+using System.Threading;
+
+namespace MediaBrowser.Server.Implementations.FileOrganization
+{
+    /// <summary>
+    /// Class SessionInfoWebSocketListener
+    /// </summary>
+    class FileOrganizationNotifier : IServerEntryPoint
+    {
+        private readonly IFileOrganizationService _organizationService;
+        private readonly ISessionManager _sessionManager;
+
+        public FileOrganizationNotifier(ILogger logger, IFileOrganizationService organizationService, ISessionManager sessionManager)
+        {
+            _organizationService = organizationService;
+            _sessionManager = sessionManager;
+        }
+
+        public void Run()
+        {
+            _organizationService.ItemAdded += _organizationService_ItemAdded;
+            _organizationService.ItemRemoved += _organizationService_ItemRemoved;
+            _organizationService.ItemUpdated += _organizationService_ItemUpdated;
+            _organizationService.LogReset += _organizationService_LogReset;
+        }
+
+        private void _organizationService_LogReset(object sender, EventArgs e)
+        {
+            _sessionManager.SendMessageToAdminSessions("AutoOrganizeUpdate", (FileOrganizationResult)null, CancellationToken.None);
+        }
+
+        private void _organizationService_ItemUpdated(object sender, GenericEventArgs<FileOrganizationResult> e)
+        {
+            _sessionManager.SendMessageToAdminSessions("AutoOrganizeUpdate", e.Argument, CancellationToken.None);
+        }
+
+        private void _organizationService_ItemRemoved(object sender, GenericEventArgs<FileOrganizationResult> e)
+        {
+            _sessionManager.SendMessageToAdminSessions("AutoOrganizeUpdate", (FileOrganizationResult)null, CancellationToken.None);
+        }
+
+        private void _organizationService_ItemAdded(object sender, GenericEventArgs<FileOrganizationResult> e)
+        {
+            _sessionManager.SendMessageToAdminSessions("AutoOrganizeUpdate", (FileOrganizationResult)null, CancellationToken.None);
+        }
+
+        public void Dispose()
+        {
+            _organizationService.ItemAdded -= _organizationService_ItemAdded;
+            _organizationService.ItemRemoved -= _organizationService_ItemRemoved;
+            _organizationService.ItemUpdated -= _organizationService_ItemUpdated;
+            _organizationService.LogReset -= _organizationService_LogReset;
+        }
+
+
+    }
+}

MediaBrowser.Server.Implementations/FileOrganization/FileOrganizationService.cs

@@ -3,16 +3,21 @@ using MediaBrowser.Common.ScheduledTasks;
 using MediaBrowser.Controller.Configuration;
 using MediaBrowser.Controller.FileOrganization;
 using MediaBrowser.Controller.Library;
+using MediaBrowser.Controller.Net;
 using MediaBrowser.Controller.Persistence;
 using MediaBrowser.Controller.Providers;
 using MediaBrowser.Model.FileOrganization;
 using MediaBrowser.Model.Logging;
 using MediaBrowser.Model.Querying;
 using System;
+using System.Collections.Concurrent;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using CommonIO;
+using MediaBrowser.Controller.Session;
+using MediaBrowser.Model.Events;
+using MediaBrowser.Common.Events;
 
 namespace MediaBrowser.Server.Implementations.FileOrganization
 {
@@ -26,6 +31,12 @@ namespace MediaBrowser.Server.Implementations.FileOrganization
         private readonly IServerConfigurationManager _config;
         private readonly IFileSystem _fileSystem;
         private readonly IProviderManager _providerManager;
+        private readonly ConcurrentDictionary<string, bool> _inProgressItemIds = new ConcurrentDictionary<string, bool>();
+
+        public event EventHandler<GenericEventArgs<FileOrganizationResult>> ItemAdded;
+        public event EventHandler<GenericEventArgs<FileOrganizationResult>> ItemUpdated;
+        public event EventHandler<GenericEventArgs<FileOrganizationResult>> ItemRemoved;
+        public event EventHandler LogReset;
 
         public FileOrganizationService(ITaskManager taskManager, IFileOrganizationRepository repo, ILogger logger, ILibraryMonitor libraryMonitor, ILibraryManager libraryManager, IServerConfigurationManager config, IFileSystem fileSystem, IProviderManager providerManager)
         {
@@ -58,12 +69,26 @@ namespace MediaBrowser.Server.Implementations.FileOrganization
 
         public QueryResult<FileOrganizationResult> GetResults(FileOrganizationResultQuery query)
         {
-            return _repo.GetResults(query);
+            var results = _repo.GetResults(query);
+
+            foreach (var result in results.Items)
+            {
+                result.IsInProgress = _inProgressItemIds.ContainsKey(result.Id);
+            }
+
+            return results;
         }
 
         public FileOrganizationResult GetResult(string id)
         {
-            return _repo.GetResult(id);
+            var result = _repo.GetResult(id);
+
+            if (result != null)
+            {
+                result.IsInProgress = _inProgressItemIds.ContainsKey(result.Id);
+            }
+
+            return result;
         }
 
         public FileOrganizationResult GetResultBySourcePath(string path)
@@ -78,11 +103,17 @@ namespace MediaBrowser.Server.Implementations.FileOrganization
             return GetResult(id);
         }
 
-        public Task DeleteOriginalFile(string resultId)
+        public async Task DeleteOriginalFile(string resultId)
         {
             var result = _repo.GetResult(resultId);
 
             _logger.Info("Requested to delete {0}", result.OriginalPath);
+
+            if (!AddToInProgressList(result, false))
+            {
+                throw new Exception("Path is currently processed otherwise. Please try again later.");
+            }
+
             try
             {
                 _fileSystem.DeleteFile(result.OriginalPath);
@@ -91,8 +122,14 @@ namespace MediaBrowser.Server.Implementations.FileOrganization
             {
                 _logger.ErrorException("Error deleting {0}", ex, result.OriginalPath);
             }
+            finally
+            {
+                RemoveFromInprogressList(result);
+            }
 
-            return _repo.Delete(resultId);
+            await _repo.Delete(resultId);
+
+            EventHelper.FireEventIfNotNull(ItemRemoved, this, new GenericEventArgs<FileOrganizationResult>(result), _logger);
         }
 
         private AutoOrganizeOptions GetAutoOrganizeOptions()
@@ -121,9 +158,10 @@ namespace MediaBrowser.Server.Implementations.FileOrganization
             }
         }
 
-        public Task ClearLog()
+        public async Task ClearLog()
         {
-            return _repo.DeleteAll();
+            await _repo.DeleteAll();
+            EventHelper.FireEventIfNotNull(LogReset, this, EventArgs.Empty, _logger);
         }
 
         public async Task PerformEpisodeOrganization(EpisodeFileOrganizationRequest request)
@@ -189,5 +227,55 @@ namespace MediaBrowser.Server.Implementations.FileOrganization
                 _config.SaveAutoOrganizeOptions(options);
             }
         }
+
+        /// <summary>
+        /// Attempts to add a an item to the list of currently processed items.
+        /// </summary>
+        /// <param name="result">The result item.</param>
+        /// <param name="isNewItem">Passing true will notify the client to reload all items, otherwise only a single item will be refreshed.</param>
+        /// <returns>True if the item was added, False if the item is already contained in the list.</returns>
+        public bool AddToInProgressList(FileOrganizationResult result, bool isNewItem)
+        {
+            if (string.IsNullOrWhiteSpace(result.Id))
+            {
+                result.Id = result.OriginalPath.GetMD5().ToString("N");
+            }
+
+            if (!_inProgressItemIds.TryAdd(result.Id, false))
+            {
+                return false;
+            }
+
+            result.IsInProgress = true;
+
+            if (isNewItem)
+            {
+                EventHelper.FireEventIfNotNull(ItemAdded, this, new GenericEventArgs<FileOrganizationResult>(result), _logger);
+            }
+            else
+            {
+                EventHelper.FireEventIfNotNull(ItemUpdated, this, new GenericEventArgs<FileOrganizationResult>(result), _logger);
+            }
+
+            return true;
+        }
+
+        /// <summary>
+        /// Removes an item from the list of currently processed items.
+        /// </summary>
+        /// <param name="result">The result item.</param>
+        /// <returns>True if the item was removed, False if the item was not contained in the list.</returns>
+        public bool RemoveFromInprogressList(FileOrganizationResult result)
+        {
+            bool itemValue;
+            var retval = _inProgressItemIds.TryRemove(result.Id, out itemValue);
+
+            result.IsInProgress = false;
+
+            EventHelper.FireEventIfNotNull(ItemUpdated, this, new GenericEventArgs<FileOrganizationResult>(result), _logger);
+
+            return retval;
+        }
+
     }
 }

MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj

@@ -149,6 +149,7 @@
     <Compile Include="EntryPoints\UsageReporter.cs" />
     <Compile Include="FileOrganization\EpisodeFileOrganizer.cs" />
     <Compile Include="FileOrganization\Extensions.cs" />
+    <Compile Include="FileOrganization\FileOrganizationNotifier.cs" />
     <Compile Include="FileOrganization\FileOrganizationService.cs" />
     <Compile Include="FileOrganization\NameUtils.cs" />
     <Compile Include="FileOrganization\TvFolderOrganizer.cs" />

MediaBrowser.Server.Implementations/Session/SessionManager.cs

@@ -1869,6 +1869,27 @@ namespace MediaBrowser.Server.Implementations.Session
             return GetSessionByAuthenticationToken(info, deviceId, remoteEndpoint, null);
         }
 
+        public Task SendMessageToAdminSessions<T>(string name, T data, CancellationToken cancellationToken)
+        {
+            // TODO: How to identify admin sessions?
+            var sessions = Sessions.Where(i => i.IsActive && i.SessionController != null).ToList();
+
+            var tasks = sessions.Select(session => Task.Run(async () =>
+            {
+                try
+                {
+                    await session.SessionController.SendMessage(name, data, cancellationToken).ConfigureAwait(false);
+                }
+                catch (Exception ex)
+                {
+                    _logger.ErrorException("Error sending message", ex);
+                }
+
+            }, cancellationToken));
+
+            return Task.WhenAll(tasks);
+        }
+
         public Task SendMessageToUserSessions<T>(string userId, string name, T data,
             CancellationToken cancellationToken)
         {