Pull ProviderData out of memory

MediaBrowser.Server.Implementations/Library/UserManager.cs

@@ -196,6 +196,8 @@ namespace MediaBrowser.Server.Implementations.Library
 
                 var user = InstantiateNewUser(name);
 
+                user.DateLastSaved = DateTime.UtcNow;
+
                 var task = UserRepository.SaveUser(user, CancellationToken.None);
 
                 // Hate having to block threads
@@ -274,6 +276,7 @@ namespace MediaBrowser.Server.Implementations.Library
             }
 
             user.DateModified = DateTime.UtcNow;
+            user.DateLastSaved = DateTime.UtcNow;
 
             await UserRepository.SaveUser(user, CancellationToken.None).ConfigureAwait(false);
 
@@ -307,6 +310,8 @@ namespace MediaBrowser.Server.Implementations.Library
             list.Add(user);
             Users = list;
 
+            user.DateLastSaved = DateTime.UtcNow;
+            
             await UserRepository.SaveUser(user, CancellationToken.None).ConfigureAwait(false);
 
             EventHelper.QueueEventIfNotNull(UserCreated, this, new GenericEventArgs<User> { Argument = user }, _logger);

MediaBrowser.Server.Implementations/LiveTv/ChannelImageProvider.cs

@@ -36,11 +36,11 @@ namespace MediaBrowser.Server.Implementations.LiveTv
             return !item.HasImage(ImageType.Primary);
         }
 
-        public override async Task<bool> FetchAsync(BaseItem item, bool force, CancellationToken cancellationToken)
+        public override async Task<bool> FetchAsync(BaseItem item, bool force, BaseProviderInfo providerInfo, CancellationToken cancellationToken)
         {
             if (item.HasImage(ImageType.Primary))
             {
-                SetLastRefreshed(item, DateTime.UtcNow);
+                SetLastRefreshed(item, DateTime.UtcNow, providerInfo);
                 return true;
             }
 
@@ -58,7 +58,7 @@ namespace MediaBrowser.Server.Implementations.LiveTv
             }
 
 
-            SetLastRefreshed(item, DateTime.UtcNow);
+            SetLastRefreshed(item, DateTime.UtcNow, providerInfo);
             return true;
         }
 

MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj

@@ -170,6 +170,7 @@
     <Compile Include="Persistence\SqliteExtensions.cs" />
     <Compile Include="Persistence\SqliteMediaStreamsRepository.cs" />
     <Compile Include="Persistence\SqliteNotificationsRepository.cs" />
+    <Compile Include="Persistence\SqliteProviderInfoRepository.cs" />
     <Compile Include="Persistence\TypeMapper.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="Providers\ImageSaver.cs" />

MediaBrowser.Server.Implementations/Persistence/SqliteItemRepository.cs

@@ -1,6 +1,7 @@
 using MediaBrowser.Common.Configuration;
 using MediaBrowser.Controller.Entities;
 using MediaBrowser.Controller.Persistence;
+using MediaBrowser.Controller.Providers;
 using MediaBrowser.Model.Entities;
 using MediaBrowser.Model.Logging;
 using MediaBrowser.Model.Serialization;
@@ -57,6 +58,7 @@ namespace MediaBrowser.Server.Implementations.Persistence
 
         private SqliteChapterRepository _chapterRepository;
         private SqliteMediaStreamsRepository _mediaStreamsRepository;
+        private SqliteProviderInfoRepository _providerInfoRepository;
 
         private IDbCommand _deleteChildrenCommand;
         private IDbCommand _saveChildrenCommand;
@@ -91,16 +93,16 @@ namespace MediaBrowser.Server.Implementations.Persistence
             _logger = logManager.GetLogger(GetType().Name);
 
             var chapterDbFile = Path.Combine(_appPaths.DataPath, "chapters.db");
-
             var chapterConnection = SqliteExtensions.ConnectToDb(chapterDbFile, _logger).Result;
-
             _chapterRepository = new SqliteChapterRepository(chapterConnection, logManager);
 
             var mediaStreamsDbFile = Path.Combine(_appPaths.DataPath, "mediainfo.db");
-
             var mediaStreamsConnection = SqliteExtensions.ConnectToDb(mediaStreamsDbFile, _logger).Result;
-
             _mediaStreamsRepository = new SqliteMediaStreamsRepository(mediaStreamsConnection, logManager);
+
+            var providerInfosDbFile = Path.Combine(_appPaths.DataPath, "providerinfo.db");
+            var providerInfoConnection = SqliteExtensions.ConnectToDb(providerInfosDbFile, _logger).Result;
+            _providerInfoRepository = new SqliteProviderInfoRepository(providerInfoConnection, logManager);
         }
 
         /// <summary>
@@ -128,8 +130,9 @@ namespace MediaBrowser.Server.Implementations.Persistence
             _connection.RunQueries(queries, _logger);
 
             PrepareStatements();
-
+            
             _mediaStreamsRepository.Initialize();
+            _providerInfoRepository.Initialize();
             _chapterRepository.Initialize();
         }
 
@@ -427,6 +430,12 @@ namespace MediaBrowser.Server.Implementations.Persistence
                     _mediaStreamsRepository.Dispose();
                     _mediaStreamsRepository = null;
                 }
+
+                if (_providerInfoRepository != null)
+                {
+                    _providerInfoRepository.Dispose();
+                    _providerInfoRepository = null;
+                }
             }
         }
 
@@ -535,5 +544,15 @@ namespace MediaBrowser.Server.Implementations.Persistence
         {
             return _mediaStreamsRepository.SaveMediaStreams(id, streams, cancellationToken);
         }
+
+        public IEnumerable<BaseProviderInfo> GetProviderHistory(Guid itemId)
+        {
+            return _providerInfoRepository.GetBaseProviderInfos(itemId);
+        }
+
+        public Task SaveProviderHistory(Guid id, IEnumerable<BaseProviderInfo> history, CancellationToken cancellationToken)
+        {
+            return _providerInfoRepository.SaveProviderInfos(id, history, cancellationToken);
+        }
     }
 }

MediaBrowser.Server.Implementations/Persistence/SqliteProviderInfoRepository.cs

@@ -0,0 +1,265 @@
+using MediaBrowser.Controller.Providers;
+using MediaBrowser.Model.Logging;
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace MediaBrowser.Server.Implementations.Persistence
+{
+    class SqliteProviderInfoRepository
+    {
+        private IDbConnection _connection;
+
+        private readonly ILogger _logger;
+
+        private IDbCommand _deleteInfosCommand;
+        private IDbCommand _saveInfoCommand;
+
+        public SqliteProviderInfoRepository(IDbConnection connection, ILogManager logManager)
+        {
+            _connection = connection;
+
+            _logger = logManager.GetLogger(GetType().Name);
+        }
+
+        /// <summary>
+        /// Opens the connection to the database
+        /// </summary>
+        /// <returns>Task.</returns>
+        public void Initialize()
+        {
+            var createTableCommand
+                = "create table if not exists providerinfos ";
+
+            createTableCommand += "(ItemId GUID, ProviderId GUID, ProviderVersion TEXT, FileStamp GUID, LastRefreshStatus TEXT, LastRefreshed datetime, PRIMARY KEY (ItemId, ProviderId))";
+
+            string[] queries = {
+
+                                createTableCommand,
+                                "create index if not exists idx_providerinfos on providerinfos(ItemId, ProviderId)",
+
+                                //pragmas
+                                "pragma temp_store = memory"
+                               };
+
+            _connection.RunQueries(queries, _logger);
+
+            PrepareStatements();
+        }
+
+        private static readonly string[] SaveColumns =
+        {
+            "ItemId",
+            "ProviderId",
+            "ProviderVersion",
+            "FileStamp",
+            "LastRefreshStatus",
+            "LastRefreshed"
+        };
+
+        private readonly string[] _selectColumns = SaveColumns.Skip(1).ToArray();
+
+        /// <summary>
+        /// The _write lock
+        /// </summary>
+        private readonly SemaphoreSlim _writeLock = new SemaphoreSlim(1, 1);
+
+        /// <summary>
+        /// Prepares the statements.
+        /// </summary>
+        private void PrepareStatements()
+        {
+            _deleteInfosCommand = _connection.CreateCommand();
+            _deleteInfosCommand.CommandText = "delete from providerinfos where ItemId=@ItemId";
+            _deleteInfosCommand.Parameters.Add(_deleteInfosCommand, "@ItemId");
+
+            _saveInfoCommand = _connection.CreateCommand();
+
+            _saveInfoCommand.CommandText = string.Format("replace into providerinfos ({0}) values ({1})",
+                string.Join(",", SaveColumns),
+                string.Join(",", SaveColumns.Select(i => "@" + i).ToArray()));
+
+            foreach (var col in SaveColumns)
+            {
+                _saveInfoCommand.Parameters.Add(_saveInfoCommand, "@" + col);
+            }
+        }
+
+        public IEnumerable<BaseProviderInfo> GetBaseProviderInfos(Guid itemId)
+        {
+            if (itemId == Guid.Empty)
+            {
+                throw new ArgumentNullException("itemId");
+            }
+
+            using (var cmd = _connection.CreateCommand())
+            {
+                var cmdText = "select " + string.Join(",", _selectColumns) + " from providerinfos where";
+
+                cmdText += " ItemId=@ItemId";
+                cmd.Parameters.Add(cmd, "@ItemId", DbType.Guid).Value = itemId;
+
+                cmd.CommandText = cmdText;
+
+                using (var reader = cmd.ExecuteReader(CommandBehavior.SequentialAccess | CommandBehavior.SingleResult))
+                {
+                    while (reader.Read())
+                    {
+                        yield return GetBaseProviderInfo(reader);
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Gets the chapter.
+        /// </summary>
+        /// <param name="reader">The reader.</param>
+        /// <returns>ChapterInfo.</returns>
+        private BaseProviderInfo GetBaseProviderInfo(IDataReader reader)
+        {
+            var item = new BaseProviderInfo
+            {
+                ProviderId = reader.GetGuid(0)
+            };
+
+            if (!reader.IsDBNull(1))
+            {
+                item.ProviderVersion = reader.GetString(1);
+            }
+
+            item.FileStamp = reader.GetGuid(2);
+            item.LastRefreshStatus = (ProviderRefreshStatus)Enum.Parse(typeof(ProviderRefreshStatus), reader.GetString(3), true);
+            item.LastRefreshed = reader.GetDateTime(4).ToUniversalTime();
+
+            return item;
+        }
+
+        public async Task SaveProviderInfos(Guid id, IEnumerable<BaseProviderInfo> infos, CancellationToken cancellationToken)
+        {
+            if (id == Guid.Empty)
+            {
+                throw new ArgumentNullException("id");
+            }
+
+            if (infos == null)
+            {
+                throw new ArgumentNullException("infos");
+            }
+
+            cancellationToken.ThrowIfCancellationRequested();
+
+            await _writeLock.WaitAsync(cancellationToken).ConfigureAwait(false);
+
+            IDbTransaction transaction = null;
+
+            try
+            {
+                transaction = _connection.BeginTransaction();
+
+                // First delete chapters
+                _deleteInfosCommand.GetParameter(0).Value = id;
+
+                _deleteInfosCommand.Transaction = transaction;
+
+                _deleteInfosCommand.ExecuteNonQuery();
+
+                foreach (var stream in infos)
+                {
+                    if (stream.LastRefreshed == DateTime.MinValue)
+                    {
+                        throw new Exception("LastRefreshed still has DateTime.MinValue");
+
+                    }
+                    cancellationToken.ThrowIfCancellationRequested();
+
+                    _saveInfoCommand.GetParameter(0).Value = id;
+                    _saveInfoCommand.GetParameter(1).Value = stream.ProviderId;
+                    _saveInfoCommand.GetParameter(2).Value = stream.ProviderVersion;
+                    _saveInfoCommand.GetParameter(3).Value = stream.FileStamp;
+                    _saveInfoCommand.GetParameter(4).Value = stream.LastRefreshStatus.ToString();
+                    _saveInfoCommand.GetParameter(5).Value = stream.LastRefreshed;
+
+                    _saveInfoCommand.Transaction = transaction;
+                    _saveInfoCommand.ExecuteNonQuery();
+                }
+
+                transaction.Commit();
+            }
+            catch (OperationCanceledException)
+            {
+                if (transaction != null)
+                {
+                    transaction.Rollback();
+                }
+
+                throw;
+            }
+            catch (Exception e)
+            {
+                _logger.ErrorException("Failed to save provider info:", e);
+
+                if (transaction != null)
+                {
+                    transaction.Rollback();
+                }
+
+                throw;
+            }
+            finally
+            {
+                if (transaction != null)
+                {
+                    transaction.Dispose();
+                }
+
+                _writeLock.Release();
+            }
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        private readonly object _disposeLock = new object();
+
+        /// <summary>
+        /// Releases unmanaged and - optionally - managed resources.
+        /// </summary>
+        /// <param name="dispose"><c>true</c> to release both managed and unmanaged resources; <c>false</c> to release only unmanaged resources.</param>
+        protected virtual void Dispose(bool dispose)
+        {
+            if (dispose)
+            {
+                try
+                {
+                    lock (_disposeLock)
+                    {
+                        if (_connection != null)
+                        {
+                            if (_connection.IsOpen())
+                            {
+                                _connection.Close();
+                            }
+
+                            _connection.Dispose();
+                            _connection = null;
+                        }
+                    }
+                }
+                catch (Exception ex)
+                {
+                    _logger.ErrorException("Error disposing database", ex);
+                }
+            }
+        }
+    }
+}

MediaBrowser.Server.Implementations/Persistence/SqliteUserDataRepository.cs

@@ -233,7 +233,7 @@ namespace MediaBrowser.Server.Implementations.Persistence
 
                         if (!reader.IsDBNull(5))
                         {
-                            userData.LastPlayedDate = reader.GetDateTime(5);
+                            userData.LastPlayedDate = reader.GetDateTime(5).ToUniversalTime();
                         }
                     }
                 }

MediaBrowser.Server.Implementations/Providers/ProviderManager.cs

@@ -4,6 +4,7 @@ using MediaBrowser.Controller.Configuration;
 using MediaBrowser.Controller.Entities;
 using MediaBrowser.Controller.IO;
 using MediaBrowser.Controller.Library;
+using MediaBrowser.Controller.Persistence;
 using MediaBrowser.Controller.Providers;
 using MediaBrowser.Model.Entities;
 using MediaBrowser.Model.Logging;
@@ -52,6 +53,8 @@ namespace MediaBrowser.Server.Implementations.Providers
         private IImageProvider[] ImageProviders { get; set; }
         private readonly IFileSystem _fileSystem;
 
+        private readonly IItemRepository _itemRepo;
+
         /// <summary>
         /// Initializes a new instance of the <see cref="ProviderManager" /> class.
         /// </summary>
@@ -59,13 +62,14 @@ namespace MediaBrowser.Server.Implementations.Providers
         /// <param name="configurationManager">The configuration manager.</param>
         /// <param name="directoryWatchers">The directory watchers.</param>
         /// <param name="logManager">The log manager.</param>
-        public ProviderManager(IHttpClient httpClient, IServerConfigurationManager configurationManager, IDirectoryWatchers directoryWatchers, ILogManager logManager, IFileSystem fileSystem)
+        public ProviderManager(IHttpClient httpClient, IServerConfigurationManager configurationManager, IDirectoryWatchers directoryWatchers, ILogManager logManager, IFileSystem fileSystem, IItemRepository itemRepo)
         {
             _logger = logManager.GetLogger("ProviderManager");
             _httpClient = httpClient;
             ConfigurationManager = configurationManager;
             _directoryWatchers = directoryWatchers;
             _fileSystem = fileSystem;
+            _itemRepo = itemRepo;
         }
 
         /// <summary>
@@ -102,6 +106,10 @@ namespace MediaBrowser.Server.Implementations.Providers
             var enableInternetProviders = ConfigurationManager.Configuration.EnableInternetProviders;
             var excludeTypes = ConfigurationManager.Configuration.InternetProviderExcludeTypes;
 
+            var providerHistories = item.DateLastSaved == DateTime.MinValue ?
+                new List<BaseProviderInfo>() :
+                _itemRepo.GetProviderHistory(item.Id).ToList();
+
             // Run the normal providers sequentially in order of priority
             foreach (var provider in MetadataProviders)
             {
@@ -140,9 +148,20 @@ namespace MediaBrowser.Server.Implementations.Providers
                     continue;
                 }
 
+                var providerInfo = providerHistories.FirstOrDefault(i => i.ProviderId == provider.Id);
+
+                if (providerInfo == null)
+                {
+                    providerInfo = new BaseProviderInfo
+                    {
+                        ProviderId = provider.Id
+                    };
+                    providerHistories.Add(providerInfo);
+                }
+
                 try
                 {
-                    if (!force && !provider.NeedsRefresh(item))
+                    if (!force && !provider.NeedsRefresh(item, providerInfo))
                     {
                         continue;
                     }
@@ -152,7 +171,7 @@ namespace MediaBrowser.Server.Implementations.Providers
                     _logger.Error("Error determining NeedsRefresh for {0}", ex, item.Path);
                 }
 
-                var updateType = await FetchAsync(provider, item, force, cancellationToken).ConfigureAwait(false);
+                var updateType = await FetchAsync(provider, item, providerInfo, force, cancellationToken).ConfigureAwait(false);
 
                 if (updateType.HasValue)
                 {
@@ -167,6 +186,11 @@ namespace MediaBrowser.Server.Implementations.Providers
                 }
             }
 
+            if (result.HasValue || force)
+            {
+                await _itemRepo.SaveProviderHistory(item.Id, providerHistories, cancellationToken);
+            }
+
             return result;
         }
 
@@ -194,11 +218,12 @@ namespace MediaBrowser.Server.Implementations.Providers
         /// </summary>
         /// <param name="provider">The provider.</param>
         /// <param name="item">The item.</param>
+        /// <param name="providerInfo">The provider information.</param>
         /// <param name="force">if set to <c>true</c> [force].</param>
         /// <param name="cancellationToken">The cancellation token.</param>
         /// <returns>Task{System.Boolean}.</returns>
-        /// <exception cref="System.ArgumentNullException"></exception>
-        private async Task<ItemUpdateType?> FetchAsync(BaseMetadataProvider provider, BaseItem item, bool force, CancellationToken cancellationToken)
+        /// <exception cref="System.ArgumentNullException">item</exception>
+        private async Task<ItemUpdateType?> FetchAsync(BaseMetadataProvider provider, BaseItem item, BaseProviderInfo providerInfo, bool force, CancellationToken cancellationToken)
         {
             if (item == null)
             {
@@ -215,7 +240,7 @@ namespace MediaBrowser.Server.Implementations.Providers
 
             try
             {
-                var changed = await provider.FetchAsync(item, force, cancellationToken).ConfigureAwait(false);
+                var changed = await provider.FetchAsync(item, force, providerInfo, cancellationToken).ConfigureAwait(false);
 
                 if (changed)
                 {
@@ -240,7 +265,7 @@ namespace MediaBrowser.Server.Implementations.Providers
             {
                 _logger.ErrorException("{0} failed refreshing {1} {2}", ex, provider.GetType().Name, item.Name, item.Path ?? string.Empty);
 
-                provider.SetLastRefreshed(item, DateTime.UtcNow, ProviderRefreshStatus.Failure);
+                provider.SetLastRefreshed(item, DateTime.UtcNow, providerInfo, ProviderRefreshStatus.Failure);
 
                 return ItemUpdateType.Unspecified;
             }