Add EnableRetryOnFailure to handle transient connection failures (57P01 terminating connection) #4

Merged
wjones merged 4 commits from development into main 2026-04-30 08:49:05 -04:00
10 changed files with 203 additions and 84 deletions
+16 -5
View File
@@ -83,11 +83,22 @@ public class ExceptionMiddleware
{ {
if (isAuthenticationError) if (isAuthenticationError)
{ {
// Log authentication errors as warnings with user-friendly message // WebSocket authentication failures are expected when clients connect without tokens
_logger.LogWarning( // Log at Debug level to reduce noise, unless it's not a WebSocket endpoint
"Access denied: Unable to process request. Authentication token missing or invalid. URL {Method} {Url}.", if (context.Request.Path.StartsWithSegments("/socket", StringComparison.OrdinalIgnoreCase))
context.Request.Method, {
context.Request.Path); _logger.LogDebug(
"WebSocket connection rejected: Authentication token missing or invalid. IP: {IP}",
context.Connection.RemoteIpAddress);
}
else
{
// Log authentication errors for non-WebSocket endpoints as warnings
_logger.LogWarning(
"Access denied: Unable to process request. Authentication token missing or invalid. URL {Method} {Url}.",
context.Request.Method,
context.Request.Path);
}
} }
else else
{ {
@@ -7,6 +7,7 @@
using System; using System;
using System.Collections.Concurrent; using System.Collections.Concurrent;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO;
using System.Linq; using System.Linq;
using MediaBrowser.Model.IO; using MediaBrowser.Model.IO;
@@ -30,7 +31,22 @@ namespace MediaBrowser.Controller.Providers
public FileSystemMetadata[] GetFileSystemEntries(string path) public FileSystemMetadata[] GetFileSystemEntries(string path)
{ {
return _cache.GetOrAdd(path, static (p, fileSystem) => fileSystem.GetFileSystemEntries(p).ToArray(), _fileSystem); if (_cache.TryGetValue(path, out var result))
{
return result;
}
try
{
result = _fileSystem.GetFileSystemEntries(path).ToArray();
_cache.TryAdd(path, result);
return result;
}
catch (DirectoryNotFoundException)
{
// A folder may be deleted between scans; treat it as empty so refresh logic can continue.
return Array.Empty<FileSystemMetadata>();
}
} }
public List<FileSystemMetadata> GetDirectories(string path) public List<FileSystemMetadata> GetDirectories(string path)
+102
View File
@@ -398,6 +398,108 @@ psql -h your-server -U jellyfin -d jellyfin -f jellyfin_backup.sql
--- ---
## Troubleshooting Connection Issues
### Error: `57P01: terminating connection due to administrator command`
**Cause**: PostgreSQL server forcibly terminated the connection during an operation
**Common reasons**:
- PostgreSQL server restart or reload
- Connection timeout (statement_timeout, idle_in_transaction_session_timeout)
- Manual connection termination via `pg_terminate_backend()`
- Network instability or firewall issues
- Resource limits exceeded (max_connections, memory pressure)
**Solutions**:
#### 1. Configure Automatic Retry (Already Enabled)
Jellyfin now includes automatic retry logic for transient failures:
- **Max retries**: 3 attempts
- **Max delay**: 5 seconds between retries
- **Retryable errors**: Includes `57P01` (connection termination)
#### 2. Increase PostgreSQL Timeouts
Edit PostgreSQL configuration (`/etc/postgresql/*/main/postgresql.conf`):
```
# Increase statement timeout
statement_timeout = 300000 # 5 minutes (in milliseconds)
# Increase idle transaction timeout
idle_in_transaction_session_timeout = 600000 # 10 minutes
# Enable TCP keepalives (prevent network timeouts)
tcp_keepalives_idle = 60 # seconds
tcp_keepalives_interval = 10 # seconds
tcp_keepalives_count = 6
```
Reload PostgreSQL:
```bash
sudo systemctl reload postgresql
# OR
psql -U postgres -c "SELECT pg_reload_conf();"
```
#### 3. Optimize Connection String
Add keepalive and timeout parameters to your connection string:
```xml
<ConnectionString>Host=192.168.129.253;Port=5432;Database=jellyfin;Username=jellyfin;Password=yourpass;Pooling=True;Minimum Pool Size=0;Maximum Pool Size=50;Connection Idle Lifetime=300;Connection Pruning Interval=10;Timeout=30;Command Timeout=300;Keepalive=60</ConnectionString>
```
**Key parameters**:
- `Timeout=30` - Connection establishment timeout (30 seconds)
- `Command Timeout=300` - Query execution timeout (5 minutes)
- `Keepalive=60` - TCP keepalive interval (60 seconds)
- `Connection Idle Lifetime=300` - Close idle connections after 5 minutes
- `Connection Pruning Interval=10` - Check for stale connections every 10 seconds
#### 4. Check PostgreSQL Logs
On your PostgreSQL server:
```bash
# View recent logs
sudo tail -f /var/log/postgresql/postgresql-*.log
# Or check systemd journal
sudo journalctl -u postgresql -n 100 --no-pager
```
Look for:
- `received fast shutdown request`
- `terminating connection`
- `too many connections`
- `out of memory`
#### 5. Monitor Active Connections
```sql
-- Check current connections
SELECT
datname,
usename,
application_name,
state,
state_change
FROM pg_stat_activity
WHERE datname = 'jellyfin'
ORDER BY state_change DESC;
-- Check connection limits
SELECT
(SELECT setting::int FROM pg_settings WHERE name = 'max_connections') AS max_connections,
(SELECT COUNT(*) FROM pg_stat_activity) AS current_connections,
(SELECT COUNT(*) FROM pg_stat_activity WHERE datname = 'jellyfin') AS jellyfin_connections;
```
---
## Building from Source ## Building from Source
```bash ```bash
@@ -277,6 +277,12 @@ public sealed class PostgresDatabaseProvider : IJellyfinDatabaseProvider
{ {
npgsqlOptions.MigrationsAssembly(GetType().Assembly.FullName); npgsqlOptions.MigrationsAssembly(GetType().Assembly.FullName);
npgsqlOptions.UseQuerySplittingBehavior(QuerySplittingBehavior.SplitQuery); npgsqlOptions.UseQuerySplittingBehavior(QuerySplittingBehavior.SplitQuery);
// Enable automatic retry on transient failures
npgsqlOptions.EnableRetryOnFailure(
maxRetryCount: 3,
maxRetryDelay: TimeSpan.FromSeconds(5),
errorCodesToAdd: new[] { "57P01" }); // Add "terminating connection" to retryable errors
}) })
.ConfigureWarnings(warnings => .ConfigureWarnings(warnings =>
warnings.Ignore(Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning)); warnings.Ignore(Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning));
@@ -301,7 +307,6 @@ public sealed class PostgresDatabaseProvider : IJellyfinDatabaseProvider
// Start with connection string if provided, then override with individual options // Start with connection string if provided, then override with individual options
NpgsqlConnectionStringBuilder connectionBuilder; NpgsqlConnectionStringBuilder connectionBuilder;
if (!string.IsNullOrWhiteSpace(databaseConfiguration.CustomProviderOptions?.ConnectionString)) if (!string.IsNullOrWhiteSpace(databaseConfiguration.CustomProviderOptions?.ConnectionString))
{ {
// Parse the existing connection string to get all parameters // Parse the existing connection string to get all parameters
@@ -682,69 +687,6 @@ public sealed class PostgresDatabaseProvider : IJellyfinDatabaseProvider
} }
} }
/// <summary>
/// Manually records migrations as applied in the migrations history table.
/// This is used to recover from situations where tables were created but the history wasn't updated.
/// </summary>
/// <param name="context">The database context.</param>
/// <param name="migrationIds">List of migration IDs to record.</param>
/// <param name="cancellationToken">Cancellation token.</param>
private async Task RecordMigrationAsAppliedAsync(JellyfinDbContext context, IList<string> migrationIds, CancellationToken cancellationToken)
{
var connection = context.Database.GetDbConnection();
var wasOpened = false;
try
{
if (connection.State != System.Data.ConnectionState.Open)
{
await connection.OpenAsync(cancellationToken).ConfigureAwait(false);
wasOpened = true;
}
// Get the current product version from the assembly
var productVersion = typeof(JellyfinDbContext).Assembly
.GetCustomAttribute<System.Reflection.AssemblyInformationalVersionAttribute>()?
.InformationalVersion ?? "Unknown";
foreach (var migrationId in migrationIds)
{
logger.LogInformation("Recording migration '{MigrationId}' as applied", migrationId);
var insertQuery = @"
INSERT INTO ""__EFMigrationsHistory"" (""MigrationId"", ""ProductVersion"")
VALUES (@migrationId, @productVersion)
ON CONFLICT (""MigrationId"") DO NOTHING";
await using (var command = connection.CreateCommand())
{
command.CommandText = insertQuery;
var migrationParam = command.CreateParameter();
migrationParam.ParameterName = "@migrationId";
migrationParam.Value = migrationId;
command.Parameters.Add(migrationParam);
var versionParam = command.CreateParameter();
versionParam.ParameterName = "@productVersion";
versionParam.Value = productVersion;
command.Parameters.Add(versionParam);
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
}
logger.LogInformation("Successfully recorded {Count} migrations in history table", migrationIds.Count);
}
finally
{
if (wasOpened)
{
await connection.CloseAsync().ConfigureAwait(false);
}
}
}
/// <inheritdoc/> /// <inheritdoc/>
public async Task RunScheduledOptimisation(CancellationToken cancellationToken) public async Task RunScheduledOptimisation(CancellationToken cancellationToken)
{ {
@@ -16,7 +16,7 @@ using Microsoft.Extensions.Logging;
namespace Jellyfin.MediaEncoding.Hls.Cache; namespace Jellyfin.MediaEncoding.Hls.Cache;
/// <inheritdoc /> /// <inheritdoc />
public class CacheDecorator : IKeyframeExtractor public partial class CacheDecorator : IKeyframeExtractor
{ {
private readonly IKeyframeRepository _keyframeRepository; private readonly IKeyframeRepository _keyframeRepository;
private readonly IKeyframeExtractor _keyframeExtractor; private readonly IKeyframeExtractor _keyframeExtractor;
@@ -58,11 +58,11 @@ public class CacheDecorator : IKeyframeExtractor
{ {
if (!_keyframeExtractor.TryExtractKeyframes(itemId, filePath, out var result)) if (!_keyframeExtractor.TryExtractKeyframes(itemId, filePath, out var result))
{ {
_logger.LogDebug("Failed to extract keyframes using {ExtractorName}", _keyframeExtractorName); LogKeyframeExtractionFailed(_logger, _keyframeExtractorName);
return false; return false;
} }
_logger.LogDebug("Successfully extracted keyframes using {ExtractorName}", _keyframeExtractorName); LogKeyframeExtractionSucceeded(_logger, _keyframeExtractorName);
keyframeData = result; keyframeData = result;
_keyframeRepository.SaveKeyframeDataAsync(itemId, keyframeData, CancellationToken.None) _keyframeRepository.SaveKeyframeDataAsync(itemId, keyframeData, CancellationToken.None)
.GetAwaiter() .GetAwaiter()
@@ -71,4 +71,10 @@ public class CacheDecorator : IKeyframeExtractor
return true; return true;
} }
[LoggerMessage(Level = LogLevel.Debug, Message = "Failed to extract keyframes using {ExtractorName}")]
private static partial void LogKeyframeExtractionFailed(ILogger logger, string extractorName);
[LoggerMessage(Level = LogLevel.Debug, Message = "Successfully extracted keyframes using {ExtractorName}")]
private static partial void LogKeyframeExtractionSucceeded(ILogger logger, string extractorName);
} }
@@ -15,7 +15,7 @@ using Microsoft.Extensions.Logging;
using Extractor = Jellyfin.MediaEncoding.Keyframes.FfProbe.FfProbeKeyframeExtractor; using Extractor = Jellyfin.MediaEncoding.Keyframes.FfProbe.FfProbeKeyframeExtractor;
/// <inheritdoc /> /// <inheritdoc />
public class FfProbeKeyframeExtractor : IKeyframeExtractor public partial class FfProbeKeyframeExtractor : IKeyframeExtractor
{ {
private readonly IMediaEncoder _mediaEncoder; private readonly IMediaEncoder _mediaEncoder;
private readonly NamingOptions _namingOptions; private readonly NamingOptions _namingOptions;
@@ -51,12 +51,27 @@ public class FfProbeKeyframeExtractor : IKeyframeExtractor
keyframeData = Extractor.GetKeyframeData(_mediaEncoder.ProbePath, filePath); keyframeData = Extractor.GetKeyframeData(_mediaEncoder.ProbePath, filePath);
return keyframeData.KeyframeTicks.Count > 0; return keyframeData.KeyframeTicks.Count > 0;
} }
catch (Exception ex) catch (FileNotFoundException ex)
{ {
_logger.LogError(ex, "Extracting keyframes from {FilePath} using ffprobe failed", filePath); LogKeyframeExtractionError(_logger, filePath, ex);
}
catch (UnauthorizedAccessException ex)
{
LogKeyframeExtractionError(_logger, filePath, ex);
}
catch (IOException ex)
{
LogKeyframeExtractionError(_logger, filePath, ex);
}
catch (InvalidDataException ex)
{
LogKeyframeExtractionError(_logger, filePath, ex);
} }
keyframeData = null; keyframeData = null;
return false; return false;
} }
[LoggerMessage(Level = LogLevel.Error, Message = "Extracting keyframes from {FilePath} using ffprobe failed")]
private static partial void LogKeyframeExtractionError(ILogger logger, string filePath, Exception exception);
} }
@@ -6,12 +6,13 @@ namespace Jellyfin.MediaEncoding.Hls.Extractors;
using System; using System;
using System.Diagnostics.CodeAnalysis; using System.Diagnostics.CodeAnalysis;
using System.IO;
using Jellyfin.MediaEncoding.Keyframes; using Jellyfin.MediaEncoding.Keyframes;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Extractor = Jellyfin.MediaEncoding.Keyframes.Matroska.MatroskaKeyframeExtractor; using Extractor = Jellyfin.MediaEncoding.Keyframes.Matroska.MatroskaKeyframeExtractor;
/// <inheritdoc /> /// <inheritdoc />
public class MatroskaKeyframeExtractor : IKeyframeExtractor public partial class MatroskaKeyframeExtractor : IKeyframeExtractor
{ {
private readonly ILogger<MatroskaKeyframeExtractor> _logger; private readonly ILogger<MatroskaKeyframeExtractor> _logger;
@@ -41,12 +42,27 @@ public class MatroskaKeyframeExtractor : IKeyframeExtractor
keyframeData = Extractor.GetKeyframeData(filePath); keyframeData = Extractor.GetKeyframeData(filePath);
return keyframeData.KeyframeTicks.Count > 0; return keyframeData.KeyframeTicks.Count > 0;
} }
catch (Exception ex) catch (FileNotFoundException ex)
{ {
_logger.LogError(ex, "Extracting keyframes from {FilePath} using matroska metadata failed", filePath); LogKeyframeExtractionError(_logger, filePath, ex);
}
catch (UnauthorizedAccessException ex)
{
LogKeyframeExtractionError(_logger, filePath, ex);
}
catch (IOException ex)
{
LogKeyframeExtractionError(_logger, filePath, ex);
}
catch (InvalidDataException ex)
{
LogKeyframeExtractionError(_logger, filePath, ex);
} }
keyframeData = null; keyframeData = null;
return false; return false;
} }
[LoggerMessage(Level = LogLevel.Error, Message = "Extracting keyframes from {FilePath} using matroska metadata failed")]
private static partial void LogKeyframeExtractionError(ILogger logger, string filePath, Exception exception);
} }
@@ -37,6 +37,8 @@ public class DynamicHlsPlaylistGenerator : IDynamicHlsPlaylistGenerator
/// <inheritdoc /> /// <inheritdoc />
public string CreateMainPlaylist(CreateMainPlaylistRequest request) public string CreateMainPlaylist(CreateMainPlaylistRequest request)
{ {
ArgumentNullException.ThrowIfNull(request);
IReadOnlyList<double> segments; IReadOnlyList<double> segments;
// For video transcodes it is sufficient with equal length segments as ffmpeg will create new keyframes // For video transcodes it is sufficient with equal length segments as ffmpeg will create new keyframes
if (request.IsRemuxingVideo if (request.IsRemuxingVideo
@@ -56,6 +56,8 @@ public class KeyframeExtractionScheduledTask : IScheduledTask
/// <inheritdoc /> /// <inheritdoc />
public Task ExecuteAsync(IProgress<double> progress, CancellationToken cancellationToken) public Task ExecuteAsync(IProgress<double> progress, CancellationToken cancellationToken)
{ {
ArgumentNullException.ThrowIfNull(progress);
var query = new InternalItemsQuery var query = new InternalItemsQuery
{ {
MediaTypes = [MediaType.Video], MediaTypes = [MediaType.Video],
@@ -50,10 +50,13 @@ public static class FfProbeKeyframeExtractor
{ {
process.PriorityClass = ProcessPriorityClass.BelowNormal; process.PriorityClass = ProcessPriorityClass.BelowNormal;
} }
catch catch (InvalidOperationException)
{ {
// We do not care if process priority setting fails // Process may have already exited - ignore
// Ideally log a warning but this does not have a logger available }
catch (System.ComponentModel.Win32Exception)
{
// Priority setting may not be supported on this platform - ignore
} }
return ParseStream(process.StandardOutput); return ParseStream(process.StandardOutput);
@@ -67,9 +70,13 @@ public static class FfProbeKeyframeExtractor
process.Kill(); process.Kill();
} }
} }
catch catch (InvalidOperationException)
{ {
// We do not care if this fails // Process may have already exited - ignore
}
catch (System.ComponentModel.Win32Exception)
{
// Process termination may fail - ignore
} }
throw; throw;