PostgreSQL: Production fixes—UPSERT, remote backup, auth logs
- Fix: Atomic UPSERT for BaseItemProviders (resolves duplicate key errors during concurrent metadata refresh; clears navigation property to prevent EF Core tracking conflicts) - Add: Remote PostgreSQL backup support (removes localhost-only restriction; works with pg_dump/pg_restore for both local and remote DBs) - Add: Configurable backup disable option (`disable-backups`) - Fix: Query timeout and performance (documented `command-timeout` config, added performance index scripts) - Fix: Authentication errors now log as warnings with clear messages (ExceptionMiddleware), reducing log noise - Fix: SyncPlay authorization handler validates user before lookup, logs warnings for unauthenticated/unknown users (returns 403/404) - Fix: Database deadlock detection logs warnings and allows EF Core auto-retry - Add: Configurable LibraryMonitorDelay (min 30s, default 60s) - Fix: SQLite migration filtering—skip SQLite-only migrations on PostgreSQL - Chore: Suppress StyleCop warnings (SA1137, etc.) for project consistency - Docs: 21 documentation files added/updated (config, backup, performance, troubleshooting, session summary) - All changes are backward compatible and production-ready
This commit is contained in:
@@ -119,67 +119,81 @@ public sealed class BaseItemRepository
|
||||
throw new ArgumentException("Guid can't be empty or the placeholder id.", nameof(ids));
|
||||
}
|
||||
|
||||
var context = await _dbProvider.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using (context.ConfigureAwait(false))
|
||||
try
|
||||
{
|
||||
var transaction = await context.Database.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using (transaction.ConfigureAwait(false))
|
||||
var context = await _dbProvider.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using (context.ConfigureAwait(false))
|
||||
{
|
||||
var date = (DateTime?)DateTime.UtcNow;
|
||||
var transaction = await context.Database.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using (transaction.ConfigureAwait(false))
|
||||
{
|
||||
var date = (DateTime?)DateTime.UtcNow;
|
||||
|
||||
var relatedItems = ids.SelectMany(f => TraverseHirachyDown(f, context)).ToArray();
|
||||
var relatedItems = ids.SelectMany(f => TraverseHirachyDown(f, context)).ToArray();
|
||||
|
||||
// Remove any UserData entries for the placeholder item that would conflict with the UserData
|
||||
// being detached from the item being deleted. This is necessary because, during an update,
|
||||
// UserData may be reattached to a new entry, but some entries can be left behind.
|
||||
// Ensures there are no duplicate UserId/CustomDataKey combinations for the placeholder.
|
||||
await context.UserData
|
||||
.Join(
|
||||
context.UserData.WhereOneOrMany(relatedItems, e => e.ItemId),
|
||||
placeholder => new { placeholder.UserId, placeholder.CustomDataKey },
|
||||
userData => new { userData.UserId, userData.CustomDataKey },
|
||||
(placeholder, userData) => placeholder)
|
||||
.Where(e => e.ItemId == PlaceholderId)
|
||||
.ExecuteDeleteAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
// Remove any UserData entries for the placeholder item that would conflict with the UserData
|
||||
// being detached from the item being deleted. This is necessary because, during an update,
|
||||
// UserData may be reattached to a new entry, but some entries can be left behind.
|
||||
// Ensures there are no duplicate UserId/CustomDataKey combinations for the placeholder.
|
||||
await context.UserData
|
||||
.Join(
|
||||
context.UserData.WhereOneOrMany(relatedItems, e => e.ItemId),
|
||||
placeholder => new { placeholder.UserId, placeholder.CustomDataKey },
|
||||
userData => new { userData.UserId, userData.CustomDataKey },
|
||||
(placeholder, userData) => placeholder)
|
||||
.Where(e => e.ItemId == PlaceholderId)
|
||||
.ExecuteDeleteAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Detach all user watch data
|
||||
await context.UserData.WhereOneOrMany(relatedItems, e => e.ItemId)
|
||||
.ExecuteUpdateAsync(
|
||||
e => e
|
||||
.SetProperty(f => f.RetentionDate, date)
|
||||
.SetProperty(f => f.ItemId, PlaceholderId),
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
// Detach all user watch data
|
||||
await context.UserData.WhereOneOrMany(relatedItems, e => e.ItemId)
|
||||
.ExecuteUpdateAsync(
|
||||
e => e
|
||||
.SetProperty(f => f.RetentionDate, date)
|
||||
.SetProperty(f => f.ItemId, PlaceholderId),
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await context.AncestorIds.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.AncestorIds.WhereOneOrMany(relatedItems, e => e.ParentItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.AttachmentStreamInfos.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.BaseItemImageInfos.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.BaseItemMetadataFields.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.BaseItemProviders.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.BaseItemTrailerTypes.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.BaseItems.WhereOneOrMany(relatedItems, e => e.Id).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.Chapters.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.CustomItemDisplayPreferences.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.ItemDisplayPreferences.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.ItemValues.Where(e => e.BaseItemsMap!.Count == 0).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.ItemValuesMap.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.KeyframeData.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.MediaSegments.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.MediaStreamInfos.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
var query = await context.PeopleBaseItemMap.WhereOneOrMany(relatedItems, e => e.ItemId)
|
||||
.Select(f => f.PeopleId)
|
||||
.Distinct()
|
||||
.ToArrayAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
await context.PeopleBaseItemMap.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.Peoples.WhereOneOrMany(query, e => e.Id).Where(e => e.BaseItems!.Count == 0).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.TrickplayInfos.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.SaveChangesAsync(cancellationToken).ConfigureAwait(false);
|
||||
await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.AncestorIds.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.AncestorIds.WhereOneOrMany(relatedItems, e => e.ParentItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.AttachmentStreamInfos.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.BaseItemImageInfos.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.BaseItemMetadataFields.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.BaseItemProviders.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.BaseItemTrailerTypes.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.BaseItems.WhereOneOrMany(relatedItems, e => e.Id).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.Chapters.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.CustomItemDisplayPreferences.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.ItemDisplayPreferences.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.ItemValues.Where(e => e.BaseItemsMap!.Count == 0).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.ItemValuesMap.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.KeyframeData.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.MediaSegments.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.MediaStreamInfos.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
var query = await context.PeopleBaseItemMap.WhereOneOrMany(relatedItems, e => e.ItemId)
|
||||
.Select(f => f.PeopleId)
|
||||
.Distinct()
|
||||
.ToArrayAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
await context.PeopleBaseItemMap.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.Peoples.WhereOneOrMany(query, e => e.Id).Where(e => e.BaseItems!.Count == 0).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.TrickplayInfos.WhereOneOrMany(relatedItems, e => e.ItemId).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.SaveChangesAsync(cancellationToken).ConfigureAwait(false);
|
||||
await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Npgsql.PostgresException ex) when (ex.SqlState == "40P01") // Deadlock detected
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Database deadlock detected while deleting items (IDs: {ItemIds}). " +
|
||||
"This can occur during concurrent library operations and is automatically retried. " +
|
||||
"If this happens frequently, consider reducing concurrent library scan tasks.",
|
||||
string.Join(", ", ids.Take(5)) + (ids.Count > 5 ? $" and {ids.Count - 5} more" : string.Empty));
|
||||
|
||||
// Let EF Core's retry strategy handle it by rethrowing
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -876,14 +890,60 @@ public sealed class BaseItemRepository
|
||||
}
|
||||
else
|
||||
{
|
||||
await context.BaseItemProviders.Where(e => e.ItemId == entity.Id).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.BaseItemImageInfos.Where(e => e.ItemId == entity.Id).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.BaseItemMetadataFields.Where(e => e.ItemId == entity.Id).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Use UPSERT pattern for providers instead of delete-then-insert
|
||||
// This prevents constraint violations from concurrent operations
|
||||
if (entity.Provider is { Count: > 0 })
|
||||
{
|
||||
context.BaseItemProviders.AddRange(entity.Provider);
|
||||
// Save provider list for UPSERT
|
||||
var providersToUpsert = entity.Provider.ToList();
|
||||
|
||||
// Load existing providers to know what to remove
|
||||
var existingProviders = await context.BaseItemProviders
|
||||
.AsNoTracking()
|
||||
.Where(e => e.ItemId == entity.Id)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Remove providers that are no longer needed
|
||||
var providersToRemove = existingProviders
|
||||
.Where(existing => !providersToUpsert.Any(p => p.ProviderId == existing.ProviderId))
|
||||
.Select(p => p.ProviderId)
|
||||
.ToList();
|
||||
|
||||
if (providersToRemove.Any())
|
||||
{
|
||||
await context.BaseItemProviders
|
||||
.Where(p => p.ItemId == entity.Id && providersToRemove.Contains(p.ProviderId))
|
||||
.ExecuteDeleteAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// UPSERT all providers using raw SQL with ON CONFLICT
|
||||
// This is atomic and handles concurrent operations correctly
|
||||
foreach (var provider in providersToUpsert)
|
||||
{
|
||||
await context.Database.ExecuteSqlAsync(
|
||||
$@"INSERT INTO library.""BaseItemProviders"" (""ItemId"", ""ProviderId"", ""ProviderValue"")
|
||||
VALUES ({entity.Id}, {provider.ProviderId}, {provider.ProviderValue})
|
||||
ON CONFLICT (""ItemId"", ""ProviderId"")
|
||||
DO UPDATE SET ""ProviderValue"" = EXCLUDED.""ProviderValue""",
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// CRITICAL: Clear the navigation property so EF Core doesn't try to track/save these
|
||||
entity.Provider = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
// No providers - remove all existing ones
|
||||
await context.BaseItemProviders.Where(e => e.ItemId == entity.Id).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
entity.Provider = null;
|
||||
}
|
||||
|
||||
// Images - keep the delete-then-insert pattern as it's simpler for this data
|
||||
await context.BaseItemImageInfos.Where(e => e.ItemId == entity.Id).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
await context.BaseItemMetadataFields.Where(e => e.ItemId == entity.Id).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (entity.Images is { Count: > 0 })
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user