Merge pull request 'Add EnableRetryOnFailure to handle transient connection failures (57P01 terminating connection)' (#4) from development into main

Reviewed-on: #4

Jellyfin.Api/Middleware/ExceptionMiddleware.cs

@@ -83,12 +83,23 @@ public class ExceptionMiddleware
             {
                 if (isAuthenticationError)
                 {
-                    // Log authentication errors as warnings with user-friendly message
+                    // WebSocket authentication failures are expected when clients connect without tokens
+                    // Log at Debug level to reduce noise, unless it's not a WebSocket endpoint
+                    if (context.Request.Path.StartsWithSegments("/socket", StringComparison.OrdinalIgnoreCase))
+                    {
+                        _logger.LogDebug(
+                            "WebSocket connection rejected: Authentication token missing or invalid. IP: {IP}",
+                            context.Connection.RemoteIpAddress);
+                    }
+                    else
+                    {
+                        // Log authentication errors for non-WebSocket endpoints as warnings
                         _logger.LogWarning(
                             "Access denied: Unable to process request. Authentication token missing or invalid. URL {Method} {Url}.",
                             context.Request.Method,
                             context.Request.Path);
                     }
+                }
                 else
                 {
                     _logger.LogError(

MediaBrowser.Controller/Providers/DirectoryService.cs

@@ -7,6 +7,7 @@
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using MediaBrowser.Model.IO;
 
@@ -30,7 +31,22 @@ namespace MediaBrowser.Controller.Providers
 
         public FileSystemMetadata[] GetFileSystemEntries(string path)
         {
-            return _cache.GetOrAdd(path, static (p, fileSystem) => fileSystem.GetFileSystemEntries(p).ToArray(), _fileSystem);
+            if (_cache.TryGetValue(path, out var result))
+            {
+                return result;
+            }
+
+            try
+            {
+                result = _fileSystem.GetFileSystemEntries(path).ToArray();
+                _cache.TryAdd(path, result);
+                return result;
+            }
+            catch (DirectoryNotFoundException)
+            {
+                // A folder may be deleted between scans; treat it as empty so refresh logic can continue.
+                return Array.Empty<FileSystemMetadata>();
+            }
         }
 
         public List<FileSystemMetadata> GetDirectories(string path)

README-POSTGRESQL.md

@@ -398,6 +398,108 @@ psql -h your-server -U jellyfin -d jellyfin -f jellyfin_backup.sql
 
 ---
 
+## Troubleshooting Connection Issues
+
+### Error: `57P01: terminating connection due to administrator command`
+
+**Cause**: PostgreSQL server forcibly terminated the connection during an operation
+
+**Common reasons**:
+- PostgreSQL server restart or reload
+- Connection timeout (statement_timeout, idle_in_transaction_session_timeout)
+- Manual connection termination via `pg_terminate_backend()`
+- Network instability or firewall issues
+- Resource limits exceeded (max_connections, memory pressure)
+
+**Solutions**:
+
+#### 1. Configure Automatic Retry (Already Enabled)
+
+Jellyfin now includes automatic retry logic for transient failures:
+- **Max retries**: 3 attempts
+- **Max delay**: 5 seconds between retries
+- **Retryable errors**: Includes `57P01` (connection termination)
+
+#### 2. Increase PostgreSQL Timeouts
+
+Edit PostgreSQL configuration (`/etc/postgresql/*/main/postgresql.conf`):
+
+```
+# Increase statement timeout
+statement_timeout = 300000              # 5 minutes (in milliseconds)
+
+# Increase idle transaction timeout
+idle_in_transaction_session_timeout = 600000  # 10 minutes
+
+# Enable TCP keepalives (prevent network timeouts)
+tcp_keepalives_idle = 60                # seconds
+tcp_keepalives_interval = 10            # seconds
+tcp_keepalives_count = 6
+```
+
+Reload PostgreSQL:
+```bash
+sudo systemctl reload postgresql
+# OR
+psql -U postgres -c "SELECT pg_reload_conf();"
+```
+
+#### 3. Optimize Connection String
+
+Add keepalive and timeout parameters to your connection string:
+
+```xml
+<ConnectionString>Host=192.168.129.253;Port=5432;Database=jellyfin;Username=jellyfin;Password=yourpass;Pooling=True;Minimum Pool Size=0;Maximum Pool Size=50;Connection Idle Lifetime=300;Connection Pruning Interval=10;Timeout=30;Command Timeout=300;Keepalive=60</ConnectionString>
+```
+
+**Key parameters**:
+- `Timeout=30` - Connection establishment timeout (30 seconds)
+- `Command Timeout=300` - Query execution timeout (5 minutes)
+- `Keepalive=60` - TCP keepalive interval (60 seconds)
+- `Connection Idle Lifetime=300` - Close idle connections after 5 minutes
+- `Connection Pruning Interval=10` - Check for stale connections every 10 seconds
+
+#### 4. Check PostgreSQL Logs
+
+On your PostgreSQL server:
+
+```bash
+# View recent logs
+sudo tail -f /var/log/postgresql/postgresql-*.log
+
+# Or check systemd journal
+sudo journalctl -u postgresql -n 100 --no-pager
+```
+
+Look for:
+- `received fast shutdown request`
+- `terminating connection`
+- `too many connections`
+- `out of memory`
+
+#### 5. Monitor Active Connections
+
+```sql
+-- Check current connections
+SELECT 
+    datname, 
+    usename, 
+    application_name,
+    state,
+    state_change
+FROM pg_stat_activity
+WHERE datname = 'jellyfin'
+ORDER BY state_change DESC;
+
+-- Check connection limits
+SELECT 
+    (SELECT setting::int FROM pg_settings WHERE name = 'max_connections') AS max_connections,
+    (SELECT COUNT(*) FROM pg_stat_activity) AS current_connections,
+    (SELECT COUNT(*) FROM pg_stat_activity WHERE datname = 'jellyfin') AS jellyfin_connections;
+```
+
+---
+
 ## Building from Source
 
 ```bash

src/Jellyfin.Database/Jellyfin.Database.Providers.Postgres/PostgresDatabaseProvider.cs

@@ -277,6 +277,12 @@ public sealed class PostgresDatabaseProvider : IJellyfinDatabaseProvider
                 {
                     npgsqlOptions.MigrationsAssembly(GetType().Assembly.FullName);
                     npgsqlOptions.UseQuerySplittingBehavior(QuerySplittingBehavior.SplitQuery);
+
+                    // Enable automatic retry on transient failures
+                    npgsqlOptions.EnableRetryOnFailure(
+                        maxRetryCount: 3,
+                        maxRetryDelay: TimeSpan.FromSeconds(5),
+                        errorCodesToAdd: new[] { "57P01" }); // Add "terminating connection" to retryable errors
                 })
             .ConfigureWarnings(warnings =>
                 warnings.Ignore(Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning));
@@ -301,7 +307,6 @@ public sealed class PostgresDatabaseProvider : IJellyfinDatabaseProvider
 
         // Start with connection string if provided, then override with individual options
         NpgsqlConnectionStringBuilder connectionBuilder;
-        
         if (!string.IsNullOrWhiteSpace(databaseConfiguration.CustomProviderOptions?.ConnectionString))
         {
             // Parse the existing connection string to get all parameters
@@ -682,69 +687,6 @@ public sealed class PostgresDatabaseProvider : IJellyfinDatabaseProvider
         }
     }
 
-    /// <summary>
-    /// Manually records migrations as applied in the migrations history table.
-    /// This is used to recover from situations where tables were created but the history wasn't updated.
-    /// </summary>
-    /// <param name="context">The database context.</param>
-    /// <param name="migrationIds">List of migration IDs to record.</param>
-    /// <param name="cancellationToken">Cancellation token.</param>
-    private async Task RecordMigrationAsAppliedAsync(JellyfinDbContext context, IList<string> migrationIds, CancellationToken cancellationToken)
-    {
-        var connection = context.Database.GetDbConnection();
-        var wasOpened = false;
-
-        try
-        {
-            if (connection.State != System.Data.ConnectionState.Open)
-            {
-                await connection.OpenAsync(cancellationToken).ConfigureAwait(false);
-                wasOpened = true;
-            }
-
-            // Get the current product version from the assembly
-            var productVersion = typeof(JellyfinDbContext).Assembly
-                .GetCustomAttribute<System.Reflection.AssemblyInformationalVersionAttribute>()?
-                .InformationalVersion ?? "Unknown";
-
-            foreach (var migrationId in migrationIds)
-            {
-                logger.LogInformation("Recording migration '{MigrationId}' as applied", migrationId);
-
-                var insertQuery = @"
-                    INSERT INTO ""__EFMigrationsHistory"" (""MigrationId"", ""ProductVersion"")
-                    VALUES (@migrationId, @productVersion)
-                    ON CONFLICT (""MigrationId"") DO NOTHING";
-
-                await using (var command = connection.CreateCommand())
-                {
-                    command.CommandText = insertQuery;
-
-                    var migrationParam = command.CreateParameter();
-                    migrationParam.ParameterName = "@migrationId";
-                    migrationParam.Value = migrationId;
-                    command.Parameters.Add(migrationParam);
-
-                    var versionParam = command.CreateParameter();
-                    versionParam.ParameterName = "@productVersion";
-                    versionParam.Value = productVersion;
-                    command.Parameters.Add(versionParam);
-
-                    await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
-                }
-            }
-
-            logger.LogInformation("Successfully recorded {Count} migrations in history table", migrationIds.Count);
-        }
-        finally
-        {
-            if (wasOpened)
-            {
-                await connection.CloseAsync().ConfigureAwait(false);
-            }
-        }
-    }
-
     /// <inheritdoc/>
     public async Task RunScheduledOptimisation(CancellationToken cancellationToken)
     {

src/Jellyfin.MediaEncoding.Hls/Cache/CacheDecorator.cs

@@ -16,7 +16,7 @@ using Microsoft.Extensions.Logging;
 namespace Jellyfin.MediaEncoding.Hls.Cache;
 
 /// <inheritdoc />
-public class CacheDecorator : IKeyframeExtractor
+public partial class CacheDecorator : IKeyframeExtractor
 {
     private readonly IKeyframeRepository _keyframeRepository;
     private readonly IKeyframeExtractor _keyframeExtractor;
@@ -58,11 +58,11 @@ public class CacheDecorator : IKeyframeExtractor
         {
             if (!_keyframeExtractor.TryExtractKeyframes(itemId, filePath, out var result))
             {
-                _logger.LogDebug("Failed to extract keyframes using {ExtractorName}", _keyframeExtractorName);
+                LogKeyframeExtractionFailed(_logger, _keyframeExtractorName);
                 return false;
             }
 
-            _logger.LogDebug("Successfully extracted keyframes using {ExtractorName}", _keyframeExtractorName);
+            LogKeyframeExtractionSucceeded(_logger, _keyframeExtractorName);
             keyframeData = result;
             _keyframeRepository.SaveKeyframeDataAsync(itemId, keyframeData, CancellationToken.None)
                 .GetAwaiter()
@@ -71,4 +71,10 @@ public class CacheDecorator : IKeyframeExtractor
 
         return true;
     }
+
+    [LoggerMessage(Level = LogLevel.Debug, Message = "Failed to extract keyframes using {ExtractorName}")]
+    private static partial void LogKeyframeExtractionFailed(ILogger logger, string extractorName);
+
+    [LoggerMessage(Level = LogLevel.Debug, Message = "Successfully extracted keyframes using {ExtractorName}")]
+    private static partial void LogKeyframeExtractionSucceeded(ILogger logger, string extractorName);
 }

src/Jellyfin.MediaEncoding.Hls/Extractors/FfProbeKeyframeExtractor.cs

@@ -15,7 +15,7 @@ using Microsoft.Extensions.Logging;
 using Extractor = Jellyfin.MediaEncoding.Keyframes.FfProbe.FfProbeKeyframeExtractor;
 
 /// <inheritdoc />
-public class FfProbeKeyframeExtractor : IKeyframeExtractor
+public partial class FfProbeKeyframeExtractor : IKeyframeExtractor
 {
     private readonly IMediaEncoder _mediaEncoder;
     private readonly NamingOptions _namingOptions;
@@ -51,12 +51,27 @@ public class FfProbeKeyframeExtractor : IKeyframeExtractor
             keyframeData = Extractor.GetKeyframeData(_mediaEncoder.ProbePath, filePath);
             return keyframeData.KeyframeTicks.Count > 0;
         }
-        catch (Exception ex)
+        catch (FileNotFoundException ex)
         {
-            _logger.LogError(ex, "Extracting keyframes from {FilePath} using ffprobe failed", filePath);
+            LogKeyframeExtractionError(_logger, filePath, ex);
+        }
+        catch (UnauthorizedAccessException ex)
+        {
+            LogKeyframeExtractionError(_logger, filePath, ex);
+        }
+        catch (IOException ex)
+        {
+            LogKeyframeExtractionError(_logger, filePath, ex);
+        }
+        catch (InvalidDataException ex)
+        {
+            LogKeyframeExtractionError(_logger, filePath, ex);
         }
 
         keyframeData = null;
         return false;
     }
+
+    [LoggerMessage(Level = LogLevel.Error, Message = "Extracting keyframes from {FilePath} using ffprobe failed")]
+    private static partial void LogKeyframeExtractionError(ILogger logger, string filePath, Exception exception);
 }

src/Jellyfin.MediaEncoding.Hls/Extractors/MatroskaKeyframeExtractor.cs

@@ -6,12 +6,13 @@ namespace Jellyfin.MediaEncoding.Hls.Extractors;
 
 using System;
 using System.Diagnostics.CodeAnalysis;
+using System.IO;
 using Jellyfin.MediaEncoding.Keyframes;
 using Microsoft.Extensions.Logging;
 using Extractor = Jellyfin.MediaEncoding.Keyframes.Matroska.MatroskaKeyframeExtractor;
 
 /// <inheritdoc />
-public class MatroskaKeyframeExtractor : IKeyframeExtractor
+public partial class MatroskaKeyframeExtractor : IKeyframeExtractor
 {
     private readonly ILogger<MatroskaKeyframeExtractor> _logger;
 
@@ -41,12 +42,27 @@ public class MatroskaKeyframeExtractor : IKeyframeExtractor
             keyframeData = Extractor.GetKeyframeData(filePath);
             return keyframeData.KeyframeTicks.Count > 0;
         }
-        catch (Exception ex)
+        catch (FileNotFoundException ex)
         {
-            _logger.LogError(ex, "Extracting keyframes from {FilePath} using matroska metadata failed", filePath);
+            LogKeyframeExtractionError(_logger, filePath, ex);
+        }
+        catch (UnauthorizedAccessException ex)
+        {
+            LogKeyframeExtractionError(_logger, filePath, ex);
+        }
+        catch (IOException ex)
+        {
+            LogKeyframeExtractionError(_logger, filePath, ex);
+        }
+        catch (InvalidDataException ex)
+        {
+            LogKeyframeExtractionError(_logger, filePath, ex);
         }
 
         keyframeData = null;
         return false;
     }
+
+    [LoggerMessage(Level = LogLevel.Error, Message = "Extracting keyframes from {FilePath} using matroska metadata failed")]
+    private static partial void LogKeyframeExtractionError(ILogger logger, string filePath, Exception exception);
 }

src/Jellyfin.MediaEncoding.Hls/Playlist/DynamicHlsPlaylistGenerator.cs

@@ -37,6 +37,8 @@ public class DynamicHlsPlaylistGenerator : IDynamicHlsPlaylistGenerator
     /// <inheritdoc />
     public string CreateMainPlaylist(CreateMainPlaylistRequest request)
     {
+        ArgumentNullException.ThrowIfNull(request);
+
         IReadOnlyList<double> segments;
         // For video transcodes it is sufficient with equal length segments as ffmpeg will create new keyframes
         if (request.IsRemuxingVideo

src/Jellyfin.MediaEncoding.Hls/ScheduledTasks/KeyframeExtractionScheduledTask.cs

@@ -56,6 +56,8 @@ public class KeyframeExtractionScheduledTask : IScheduledTask
     /// <inheritdoc />
     public Task ExecuteAsync(IProgress<double> progress, CancellationToken cancellationToken)
     {
+        ArgumentNullException.ThrowIfNull(progress);
+
         var query = new InternalItemsQuery
         {
             MediaTypes = [MediaType.Video],

src/Jellyfin.MediaEncoding.Keyframes/FfProbe/FfProbeKeyframeExtractor.cs

@@ -50,10 +50,13 @@ public static class FfProbeKeyframeExtractor
             {
                 process.PriorityClass = ProcessPriorityClass.BelowNormal;
             }
-            catch
+            catch (InvalidOperationException)
             {
-                // We do not care if process priority setting fails
-                // Ideally log a warning but this does not have a logger available
+                // Process may have already exited - ignore
+            }
+            catch (System.ComponentModel.Win32Exception)
+            {
+                // Priority setting may not be supported on this platform - ignore
             }
 
             return ParseStream(process.StandardOutput);
@@ -67,9 +70,13 @@ public static class FfProbeKeyframeExtractor
                     process.Kill();
                 }
             }
-            catch
+            catch (InvalidOperationException)
             {
-                // We do not care if this fails
+                // Process may have already exited - ignore
+            }
+            catch (System.ComponentModel.Win32Exception)
+            {
+                // Process termination may fail - ignore
             }
 
             throw;