From dd7c38fb5d1e63b6c71d27482803abcd049814da Mon Sep 17 00:00:00 2001 From: Wendell Jones Date: Wed, 29 Apr 2026 08:12:44 -0400 Subject: [PATCH 1/4] Add EnableRetryOnFailure to handle transient connection failures (57P01 terminating connection) --- .../Providers/DirectoryService.cs | 18 ++++- .../PostgresDatabaseProvider.cs | 70 ++----------------- 2 files changed, 23 insertions(+), 65 deletions(-) diff --git a/MediaBrowser.Controller/Providers/DirectoryService.cs b/MediaBrowser.Controller/Providers/DirectoryService.cs index a29bb9eb..a4f21351 100644 --- a/MediaBrowser.Controller/Providers/DirectoryService.cs +++ b/MediaBrowser.Controller/Providers/DirectoryService.cs @@ -7,6 +7,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.IO; using System.Linq; using MediaBrowser.Model.IO; @@ -30,7 +31,22 @@ namespace MediaBrowser.Controller.Providers public FileSystemMetadata[] GetFileSystemEntries(string path) { - return _cache.GetOrAdd(path, static (p, fileSystem) => fileSystem.GetFileSystemEntries(p).ToArray(), _fileSystem); + if (_cache.TryGetValue(path, out var result)) + { + return result; + } + + try + { + result = _fileSystem.GetFileSystemEntries(path).ToArray(); + _cache.TryAdd(path, result); + return result; + } + catch (DirectoryNotFoundException) + { + // A folder may be deleted between scans; treat it as empty so refresh logic can continue. + return Array.Empty(); + } } public List GetDirectories(string path) diff --git a/src/Jellyfin.Database/Jellyfin.Database.Providers.Postgres/PostgresDatabaseProvider.cs b/src/Jellyfin.Database/Jellyfin.Database.Providers.Postgres/PostgresDatabaseProvider.cs index f33acbb1..f98bbcd8 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Providers.Postgres/PostgresDatabaseProvider.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Providers.Postgres/PostgresDatabaseProvider.cs @@ -277,6 +277,12 @@ public sealed class PostgresDatabaseProvider : IJellyfinDatabaseProvider { npgsqlOptions.MigrationsAssembly(GetType().Assembly.FullName); npgsqlOptions.UseQuerySplittingBehavior(QuerySplittingBehavior.SplitQuery); + + // Enable automatic retry on transient failures + npgsqlOptions.EnableRetryOnFailure( + maxRetryCount: 3, + maxRetryDelay: TimeSpan.FromSeconds(5), + errorCodesToAdd: new[] { "57P01" }); // Add "terminating connection" to retryable errors }) .ConfigureWarnings(warnings => warnings.Ignore(Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning)); @@ -301,7 +307,6 @@ public sealed class PostgresDatabaseProvider : IJellyfinDatabaseProvider // Start with connection string if provided, then override with individual options NpgsqlConnectionStringBuilder connectionBuilder; - if (!string.IsNullOrWhiteSpace(databaseConfiguration.CustomProviderOptions?.ConnectionString)) { // Parse the existing connection string to get all parameters @@ -682,69 +687,6 @@ public sealed class PostgresDatabaseProvider : IJellyfinDatabaseProvider } } - /// - /// Manually records migrations as applied in the migrations history table. - /// This is used to recover from situations where tables were created but the history wasn't updated. - /// - /// The database context. - /// List of migration IDs to record. - /// Cancellation token. - private async Task RecordMigrationAsAppliedAsync(JellyfinDbContext context, IList migrationIds, CancellationToken cancellationToken) - { - var connection = context.Database.GetDbConnection(); - var wasOpened = false; - - try - { - if (connection.State != System.Data.ConnectionState.Open) - { - await connection.OpenAsync(cancellationToken).ConfigureAwait(false); - wasOpened = true; - } - - // Get the current product version from the assembly - var productVersion = typeof(JellyfinDbContext).Assembly - .GetCustomAttribute()? - .InformationalVersion ?? "Unknown"; - - foreach (var migrationId in migrationIds) - { - logger.LogInformation("Recording migration '{MigrationId}' as applied", migrationId); - - var insertQuery = @" - INSERT INTO ""__EFMigrationsHistory"" (""MigrationId"", ""ProductVersion"") - VALUES (@migrationId, @productVersion) - ON CONFLICT (""MigrationId"") DO NOTHING"; - - await using (var command = connection.CreateCommand()) - { - command.CommandText = insertQuery; - - var migrationParam = command.CreateParameter(); - migrationParam.ParameterName = "@migrationId"; - migrationParam.Value = migrationId; - command.Parameters.Add(migrationParam); - - var versionParam = command.CreateParameter(); - versionParam.ParameterName = "@productVersion"; - versionParam.Value = productVersion; - command.Parameters.Add(versionParam); - - await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); - } - } - - logger.LogInformation("Successfully recorded {Count} migrations in history table", migrationIds.Count); - } - finally - { - if (wasOpened) - { - await connection.CloseAsync().ConfigureAwait(false); - } - } - } - /// public async Task RunScheduledOptimisation(CancellationToken cancellationToken) { From aead802df68bd8331a8e6f5e5a9e1844581752ee Mon Sep 17 00:00:00 2001 From: Wendell Jones Date: Wed, 29 Apr 2026 08:13:31 -0400 Subject: [PATCH 2/4] Add troubleshooting section for 57P01 connection termination errors --- README-POSTGRESQL.md | 102 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/README-POSTGRESQL.md b/README-POSTGRESQL.md index f112d4d9..9bb72182 100644 --- a/README-POSTGRESQL.md +++ b/README-POSTGRESQL.md @@ -398,6 +398,108 @@ psql -h your-server -U jellyfin -d jellyfin -f jellyfin_backup.sql --- +## Troubleshooting Connection Issues + +### Error: `57P01: terminating connection due to administrator command` + +**Cause**: PostgreSQL server forcibly terminated the connection during an operation + +**Common reasons**: +- PostgreSQL server restart or reload +- Connection timeout (statement_timeout, idle_in_transaction_session_timeout) +- Manual connection termination via `pg_terminate_backend()` +- Network instability or firewall issues +- Resource limits exceeded (max_connections, memory pressure) + +**Solutions**: + +#### 1. Configure Automatic Retry (Already Enabled) + +Jellyfin now includes automatic retry logic for transient failures: +- **Max retries**: 3 attempts +- **Max delay**: 5 seconds between retries +- **Retryable errors**: Includes `57P01` (connection termination) + +#### 2. Increase PostgreSQL Timeouts + +Edit PostgreSQL configuration (`/etc/postgresql/*/main/postgresql.conf`): + +``` +# Increase statement timeout +statement_timeout = 300000 # 5 minutes (in milliseconds) + +# Increase idle transaction timeout +idle_in_transaction_session_timeout = 600000 # 10 minutes + +# Enable TCP keepalives (prevent network timeouts) +tcp_keepalives_idle = 60 # seconds +tcp_keepalives_interval = 10 # seconds +tcp_keepalives_count = 6 +``` + +Reload PostgreSQL: +```bash +sudo systemctl reload postgresql +# OR +psql -U postgres -c "SELECT pg_reload_conf();" +``` + +#### 3. Optimize Connection String + +Add keepalive and timeout parameters to your connection string: + +```xml +Host=192.168.129.253;Port=5432;Database=jellyfin;Username=jellyfin;Password=yourpass;Pooling=True;Minimum Pool Size=0;Maximum Pool Size=50;Connection Idle Lifetime=300;Connection Pruning Interval=10;Timeout=30;Command Timeout=300;Keepalive=60 +``` + +**Key parameters**: +- `Timeout=30` - Connection establishment timeout (30 seconds) +- `Command Timeout=300` - Query execution timeout (5 minutes) +- `Keepalive=60` - TCP keepalive interval (60 seconds) +- `Connection Idle Lifetime=300` - Close idle connections after 5 minutes +- `Connection Pruning Interval=10` - Check for stale connections every 10 seconds + +#### 4. Check PostgreSQL Logs + +On your PostgreSQL server: + +```bash +# View recent logs +sudo tail -f /var/log/postgresql/postgresql-*.log + +# Or check systemd journal +sudo journalctl -u postgresql -n 100 --no-pager +``` + +Look for: +- `received fast shutdown request` +- `terminating connection` +- `too many connections` +- `out of memory` + +#### 5. Monitor Active Connections + +```sql +-- Check current connections +SELECT + datname, + usename, + application_name, + state, + state_change +FROM pg_stat_activity +WHERE datname = 'jellyfin' +ORDER BY state_change DESC; + +-- Check connection limits +SELECT + (SELECT setting::int FROM pg_settings WHERE name = 'max_connections') AS max_connections, + (SELECT COUNT(*) FROM pg_stat_activity) AS current_connections, + (SELECT COUNT(*) FROM pg_stat_activity WHERE datname = 'jellyfin') AS jellyfin_connections; +``` + +--- + ## Building from Source ```bash From 786c61ba09de59f474be6baf561f49e82582777c Mon Sep 17 00:00:00 2001 From: Wendell Jones Date: Wed, 29 Apr 2026 08:24:34 -0400 Subject: [PATCH 3/4] Fix 12 code analysis errors in MediaEncoding projects - use LoggerMessage source generators and specific exception types --- .../Cache/CacheDecorator.cs | 12 +++++++--- .../Extractors/FfProbeKeyframeExtractor.cs | 21 +++++++++++++++--- .../Extractors/MatroskaKeyframeExtractor.cs | 22 ++++++++++++++++--- .../Playlist/DynamicHlsPlaylistGenerator.cs | 2 ++ .../KeyframeExtractionScheduledTask.cs | 2 ++ .../FfProbe/FfProbeKeyframeExtractor.cs | 17 +++++++++----- 6 files changed, 62 insertions(+), 14 deletions(-) diff --git a/src/Jellyfin.MediaEncoding.Hls/Cache/CacheDecorator.cs b/src/Jellyfin.MediaEncoding.Hls/Cache/CacheDecorator.cs index 5348360f..42bdedd2 100644 --- a/src/Jellyfin.MediaEncoding.Hls/Cache/CacheDecorator.cs +++ b/src/Jellyfin.MediaEncoding.Hls/Cache/CacheDecorator.cs @@ -16,7 +16,7 @@ using Microsoft.Extensions.Logging; namespace Jellyfin.MediaEncoding.Hls.Cache; /// -public class CacheDecorator : IKeyframeExtractor +public partial class CacheDecorator : IKeyframeExtractor { private readonly IKeyframeRepository _keyframeRepository; private readonly IKeyframeExtractor _keyframeExtractor; @@ -58,11 +58,11 @@ public class CacheDecorator : IKeyframeExtractor { if (!_keyframeExtractor.TryExtractKeyframes(itemId, filePath, out var result)) { - _logger.LogDebug("Failed to extract keyframes using {ExtractorName}", _keyframeExtractorName); + LogKeyframeExtractionFailed(_logger, _keyframeExtractorName); return false; } - _logger.LogDebug("Successfully extracted keyframes using {ExtractorName}", _keyframeExtractorName); + LogKeyframeExtractionSucceeded(_logger, _keyframeExtractorName); keyframeData = result; _keyframeRepository.SaveKeyframeDataAsync(itemId, keyframeData, CancellationToken.None) .GetAwaiter() @@ -71,4 +71,10 @@ public class CacheDecorator : IKeyframeExtractor return true; } + + [LoggerMessage(Level = LogLevel.Debug, Message = "Failed to extract keyframes using {ExtractorName}")] + private static partial void LogKeyframeExtractionFailed(ILogger logger, string extractorName); + + [LoggerMessage(Level = LogLevel.Debug, Message = "Successfully extracted keyframes using {ExtractorName}")] + private static partial void LogKeyframeExtractionSucceeded(ILogger logger, string extractorName); } diff --git a/src/Jellyfin.MediaEncoding.Hls/Extractors/FfProbeKeyframeExtractor.cs b/src/Jellyfin.MediaEncoding.Hls/Extractors/FfProbeKeyframeExtractor.cs index 47cef8d2..cb0b3316 100644 --- a/src/Jellyfin.MediaEncoding.Hls/Extractors/FfProbeKeyframeExtractor.cs +++ b/src/Jellyfin.MediaEncoding.Hls/Extractors/FfProbeKeyframeExtractor.cs @@ -15,7 +15,7 @@ using Microsoft.Extensions.Logging; using Extractor = Jellyfin.MediaEncoding.Keyframes.FfProbe.FfProbeKeyframeExtractor; /// -public class FfProbeKeyframeExtractor : IKeyframeExtractor +public partial class FfProbeKeyframeExtractor : IKeyframeExtractor { private readonly IMediaEncoder _mediaEncoder; private readonly NamingOptions _namingOptions; @@ -51,12 +51,27 @@ public class FfProbeKeyframeExtractor : IKeyframeExtractor keyframeData = Extractor.GetKeyframeData(_mediaEncoder.ProbePath, filePath); return keyframeData.KeyframeTicks.Count > 0; } - catch (Exception ex) + catch (FileNotFoundException ex) { - _logger.LogError(ex, "Extracting keyframes from {FilePath} using ffprobe failed", filePath); + LogKeyframeExtractionError(_logger, filePath, ex); + } + catch (UnauthorizedAccessException ex) + { + LogKeyframeExtractionError(_logger, filePath, ex); + } + catch (IOException ex) + { + LogKeyframeExtractionError(_logger, filePath, ex); + } + catch (InvalidDataException ex) + { + LogKeyframeExtractionError(_logger, filePath, ex); } keyframeData = null; return false; } + + [LoggerMessage(Level = LogLevel.Error, Message = "Extracting keyframes from {FilePath} using ffprobe failed")] + private static partial void LogKeyframeExtractionError(ILogger logger, string filePath, Exception exception); } diff --git a/src/Jellyfin.MediaEncoding.Hls/Extractors/MatroskaKeyframeExtractor.cs b/src/Jellyfin.MediaEncoding.Hls/Extractors/MatroskaKeyframeExtractor.cs index 0726aa99..515ee344 100644 --- a/src/Jellyfin.MediaEncoding.Hls/Extractors/MatroskaKeyframeExtractor.cs +++ b/src/Jellyfin.MediaEncoding.Hls/Extractors/MatroskaKeyframeExtractor.cs @@ -6,12 +6,13 @@ namespace Jellyfin.MediaEncoding.Hls.Extractors; using System; using System.Diagnostics.CodeAnalysis; +using System.IO; using Jellyfin.MediaEncoding.Keyframes; using Microsoft.Extensions.Logging; using Extractor = Jellyfin.MediaEncoding.Keyframes.Matroska.MatroskaKeyframeExtractor; /// -public class MatroskaKeyframeExtractor : IKeyframeExtractor +public partial class MatroskaKeyframeExtractor : IKeyframeExtractor { private readonly ILogger _logger; @@ -41,12 +42,27 @@ public class MatroskaKeyframeExtractor : IKeyframeExtractor keyframeData = Extractor.GetKeyframeData(filePath); return keyframeData.KeyframeTicks.Count > 0; } - catch (Exception ex) + catch (FileNotFoundException ex) { - _logger.LogError(ex, "Extracting keyframes from {FilePath} using matroska metadata failed", filePath); + LogKeyframeExtractionError(_logger, filePath, ex); + } + catch (UnauthorizedAccessException ex) + { + LogKeyframeExtractionError(_logger, filePath, ex); + } + catch (IOException ex) + { + LogKeyframeExtractionError(_logger, filePath, ex); + } + catch (InvalidDataException ex) + { + LogKeyframeExtractionError(_logger, filePath, ex); } keyframeData = null; return false; } + + [LoggerMessage(Level = LogLevel.Error, Message = "Extracting keyframes from {FilePath} using matroska metadata failed")] + private static partial void LogKeyframeExtractionError(ILogger logger, string filePath, Exception exception); } diff --git a/src/Jellyfin.MediaEncoding.Hls/Playlist/DynamicHlsPlaylistGenerator.cs b/src/Jellyfin.MediaEncoding.Hls/Playlist/DynamicHlsPlaylistGenerator.cs index 67ab3cd6..00d691a4 100644 --- a/src/Jellyfin.MediaEncoding.Hls/Playlist/DynamicHlsPlaylistGenerator.cs +++ b/src/Jellyfin.MediaEncoding.Hls/Playlist/DynamicHlsPlaylistGenerator.cs @@ -37,6 +37,8 @@ public class DynamicHlsPlaylistGenerator : IDynamicHlsPlaylistGenerator /// public string CreateMainPlaylist(CreateMainPlaylistRequest request) { + ArgumentNullException.ThrowIfNull(request); + IReadOnlyList segments; // For video transcodes it is sufficient with equal length segments as ffmpeg will create new keyframes if (request.IsRemuxingVideo diff --git a/src/Jellyfin.MediaEncoding.Hls/ScheduledTasks/KeyframeExtractionScheduledTask.cs b/src/Jellyfin.MediaEncoding.Hls/ScheduledTasks/KeyframeExtractionScheduledTask.cs index 68fab1da..027a818f 100644 --- a/src/Jellyfin.MediaEncoding.Hls/ScheduledTasks/KeyframeExtractionScheduledTask.cs +++ b/src/Jellyfin.MediaEncoding.Hls/ScheduledTasks/KeyframeExtractionScheduledTask.cs @@ -56,6 +56,8 @@ public class KeyframeExtractionScheduledTask : IScheduledTask /// public Task ExecuteAsync(IProgress progress, CancellationToken cancellationToken) { + ArgumentNullException.ThrowIfNull(progress); + var query = new InternalItemsQuery { MediaTypes = [MediaType.Video], diff --git a/src/Jellyfin.MediaEncoding.Keyframes/FfProbe/FfProbeKeyframeExtractor.cs b/src/Jellyfin.MediaEncoding.Keyframes/FfProbe/FfProbeKeyframeExtractor.cs index 5975aebd..6c2cffbf 100644 --- a/src/Jellyfin.MediaEncoding.Keyframes/FfProbe/FfProbeKeyframeExtractor.cs +++ b/src/Jellyfin.MediaEncoding.Keyframes/FfProbe/FfProbeKeyframeExtractor.cs @@ -50,10 +50,13 @@ public static class FfProbeKeyframeExtractor { process.PriorityClass = ProcessPriorityClass.BelowNormal; } - catch + catch (InvalidOperationException) { - // We do not care if process priority setting fails - // Ideally log a warning but this does not have a logger available + // Process may have already exited - ignore + } + catch (System.ComponentModel.Win32Exception) + { + // Priority setting may not be supported on this platform - ignore } return ParseStream(process.StandardOutput); @@ -67,9 +70,13 @@ public static class FfProbeKeyframeExtractor process.Kill(); } } - catch + catch (InvalidOperationException) { - // We do not care if this fails + // Process may have already exited - ignore + } + catch (System.ComponentModel.Win32Exception) + { + // Process termination may fail - ignore } throw; From bad167656ac0e747f643f171440aa7ec024a816e Mon Sep 17 00:00:00 2001 From: Wendell Jones Date: Wed, 29 Apr 2026 08:30:54 -0400 Subject: [PATCH 4/4] Reduce log noise for WebSocket authentication failures - downgrade to Debug level --- .../Middleware/ExceptionMiddleware.cs | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/Jellyfin.Api/Middleware/ExceptionMiddleware.cs b/Jellyfin.Api/Middleware/ExceptionMiddleware.cs index 30c2b5f2..41906582 100644 --- a/Jellyfin.Api/Middleware/ExceptionMiddleware.cs +++ b/Jellyfin.Api/Middleware/ExceptionMiddleware.cs @@ -83,11 +83,22 @@ public class ExceptionMiddleware { if (isAuthenticationError) { - // Log authentication errors as warnings with user-friendly message - _logger.LogWarning( - "Access denied: Unable to process request. Authentication token missing or invalid. URL {Method} {Url}.", - context.Request.Method, - context.Request.Path); + // WebSocket authentication failures are expected when clients connect without tokens + // Log at Debug level to reduce noise, unless it's not a WebSocket endpoint + if (context.Request.Path.StartsWithSegments("/socket", StringComparison.OrdinalIgnoreCase)) + { + _logger.LogDebug( + "WebSocket connection rejected: Authentication token missing or invalid. IP: {IP}", + context.Connection.RemoteIpAddress); + } + else + { + // Log authentication errors for non-WebSocket endpoints as warnings + _logger.LogWarning( + "Access denied: Unable to process request. Authentication token missing or invalid. URL {Method} {Url}.", + context.Request.Method, + context.Request.Path); + } } else {