wjones/pgsql-jellyfin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement migration tracking and history management for PostgreSQL

Browse Source
This commit is contained in:
Wendell Jones
2026-05-04 12:47:12 -04:00
parent e67c191843
commit 4ccd84342b
12 changed files with 191 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs
+10 -14
View File
@@ -15,13 +15,13 @@ using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Jellyfin.Database.Implementations;
using Jellyfin.Server.Implementations.Migrations;
using Jellyfin.Server.Implementations.StorageHelpers;
using Jellyfin.Server.Implementations.SystemBackupService;
using MediaBrowser.Controller;
using MediaBrowser.Controller.SystemBackupService;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
@@ -156,34 +156,31 @@ public class BackupService : IBackupService
await using (dbContext.ConfigureAwait(false))
{
// restore migration history manually
var historyEntry = zipArchive.GetEntry(NormalizePathSeparator(Path.Combine("Database", $"{nameof(HistoryRow)}.json")));
var historyEntry = zipArchive.GetEntry(NormalizePathSeparator(Path.Combine("Database", $"{MigrationTracker.BackupEntryName}.json")));
if (historyEntry is null)
{
_logger.LogInformation("No backup of the history table in archive. This is required for Jellyfin operation");
throw new InvalidOperationException("Cannot restore backup that has no History data.");
}
HistoryRow[] historyEntries;
MigrationRecord[] historyEntries;
var historyArchive = await historyEntry.OpenAsync().ConfigureAwait(false);
await using (historyArchive.ConfigureAwait(false))
{
historyEntries = await JsonSerializer.DeserializeAsync<HistoryRow[]>(historyArchive).ConfigureAwait(false) ??
historyEntries = await JsonSerializer.DeserializeAsync<MigrationRecord[]>(historyArchive).ConfigureAwait(false) ??
throw new InvalidOperationException("Cannot restore backup that has no History data.");
}
var historyRepository = dbContext.GetService<IHistoryRepository>();
await historyRepository.CreateIfNotExistsAsync().ConfigureAwait(false);
await MigrationTracker.EnsureTableExistsAsync(dbContext).ConfigureAwait(false);
foreach (var item in await historyRepository.GetAppliedMigrationsAsync(CancellationToken.None).ConfigureAwait(false))
foreach (var item in await MigrationTracker.GetAppliedMigrationIdsAsync(dbContext, CancellationToken.None).ConfigureAwait(false))
{
var insertScript = historyRepository.GetDeleteScript(item.MigrationId);
await dbContext.Database.ExecuteSqlRawAsync(insertScript).ConfigureAwait(false);
await MigrationTracker.DeleteAsync(dbContext, item).ConfigureAwait(false);
}
foreach (var item in historyEntries)
{
var insertScript = historyRepository.GetInsertScript(item);
await dbContext.Database.ExecuteSqlRawAsync(insertScript).ConfigureAwait(false);
await MigrationTracker.InsertAsync(dbContext, item.MigrationId, item.ProductVersion).ConfigureAwait(false);
}
dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking;
@@ -310,8 +307,7 @@ public class BackupService : IBackupService
}
// include the migration history as well
var historyRepository = dbContext.GetService<IHistoryRepository>();
var migrations = await historyRepository.GetAppliedMigrationsAsync().ConfigureAwait(false);
var migrations = await MigrationTracker.GetAppliedMigrationsAsync(dbContext).ConfigureAwait(false);
ICollection<(Type Type, string SourceName, Func<IAsyncEnumerable<object>> ValueFactory)> entityTypes =
[
@@ -319,7 +315,7 @@ public class BackupService : IBackupService
.GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance)
.Where(e => e.PropertyType.IsAssignableTo(typeof(IQueryable)))
.Select(e => (Type: e.PropertyType, dbContext.Model.FindEntityType(e.PropertyType.GetGenericArguments()[0])!.GetSchemaQualifiedTableName()!, ValueFactory: new Func<IAsyncEnumerable<object>>(() => GetValues((IQueryable)e.GetValue(dbContext)!)))),
(Type: typeof(HistoryRow), SourceName: nameof(HistoryRow), ValueFactory: () => migrations.ToAsyncEnumerable())
(Type: typeof(MigrationRecord), SourceName: MigrationTracker.BackupEntryName, ValueFactory: () => migrations.ToAsyncEnumerable())
];
manifest.DatabaseTables = entityTypes.Select(e => e.Type.Name).ToArray();
Jellyfin.Server.Implementations/Item/BaseItemRepository.cs
+37 -22
View File
@@ -1157,18 +1157,17 @@ public sealed class BaseItemRepository
.ConfigureAwait(false);
}
// UPSERT all providers using raw SQL with ON CONFLICT
// This is atomic and handles concurrent operations correctly
foreach (var provider in providersToUpsert)
{
await context.Database.ExecuteSqlAsync(
$@"INSERT INTO library.base_item_providers (item_id, provider_id, provider_value)
VALUES ({entity.Id}, {provider.ProviderId}, {provider.ProviderValue})
ON CONFLICT (item_id, provider_id)
DO UPDATE SET provider_value = EXCLUDED.provider_value",
cancellationToken)
.ConfigureAwait(false);
}
// UPSERT all providers in a single UNNEST-based INSERT instead of one round-trip per provider
var providerItemIds = providersToUpsert.Select(_ => entity.Id).ToArray();
var providerIds = providersToUpsert.Select(p => p.ProviderId).ToArray();
var providerValues = providersToUpsert.Select(p => p.ProviderValue).ToArray();
await context.Database.ExecuteSqlAsync(
$@"INSERT INTO library.base_item_providers (item_id, provider_id, provider_value)
SELECT * FROM UNNEST({providerItemIds}, {providerIds}, {providerValues})
ON CONFLICT (item_id, provider_id)
DO UPDATE SET provider_value = EXCLUDED.provider_value",
cancellationToken)
.ConfigureAwait(false);
// CRITICAL: Clear the navigation property so EF Core doesn't try to track/save these
entity.Provider = null;
@@ -1291,7 +1290,6 @@ public sealed class BaseItemRepository
Value = f.Value
}).ToArray();
context.ItemValues.AddRange(missingItemValues);
await context.SaveChangesAsync(cancellationToken).ConfigureAwait(false);
var itemValuesStore = existingValues.Concat(missingItemValues).ToArray();
var valueMap = itemValueMaps
@@ -1332,19 +1330,36 @@ public sealed class BaseItemRepository
await context.SaveChangesAsync(cancellationToken).ConfigureAwait(false);
// Hoist ancestor queries outside the per-item loop to avoid N×2 DB round-trips
var ancestorItemIds = tuples
.Where(t => t.Item.SupportsAncestors && t.AncestorIds != null)
.Select(t => t.Item.Id)
.ToArray();
var allNeededAncestorIds = tuples
.Where(t => t.Item.SupportsAncestors && t.AncestorIds != null)
.SelectMany(t => t.AncestorIds!)
.Distinct()
.ToArray();
var allExistingAncestorIds = ancestorItemIds.Length > 0
? await context.AncestorIds
.Where(e => ancestorItemIds.Contains(e.ItemId))
.ToListAsync(cancellationToken)
.ConfigureAwait(false)
: [];
var validAncestorIdSet = allNeededAncestorIds.Length > 0
? (await context.BaseItems
.Where(e => allNeededAncestorIds.Contains(e.Id))
.Select(f => f.Id)
.ToArrayAsync(cancellationToken)
.ConfigureAwait(false)).ToHashSet()
: [];
foreach (var item in tuples)
{
if (item.Item.SupportsAncestors && item.AncestorIds != null)
{
var existingAncestorIds = await context.AncestorIds
.Where(e => e.ItemId == item.Item.Id)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
var validAncestorIds = await context.BaseItems
.Where(e => item.AncestorIds.Contains(e.Id))
.Select(f => f.Id)
.ToArrayAsync(cancellationToken)
.ConfigureAwait(false);
var existingAncestorIds = allExistingAncestorIds.Where(e => e.ItemId == item.Item.Id).ToList();
var validAncestorIds = item.AncestorIds.Where(validAncestorIdSet.Contains).ToArray();
foreach (var ancestorId in validAncestorIds)
{
var existingAncestorId = existingAncestorIds.FirstOrDefault(e => e.ParentItemId == ancestorId);
Jellyfin.Server.Implementations/Migrations/MigrationRecord.cs
+15
View File
@@ -0,0 +1,15 @@
// <copyright file="MigrationRecord.cs" company="PlaceholderCompany">
// Copyright (c) PlaceholderCompany. All rights reserved.
// </copyright>
namespace Jellyfin.Server.Implementations.Migrations;
/// <summary>Represents one row of the EF migrations history table.</summary>
public sealed class MigrationRecord
{
/// <summary>Gets or sets the migration identifier.</summary>
public string MigrationId { get; set; } = string.Empty;
/// <summary>Gets or sets the product version that applied this migration.</summary>
public string ProductVersion { get; set; } = string.Empty;
}
Jellyfin.Server.Implementations/Migrations/MigrationTracker.cs
+80
View File
@@ -0,0 +1,80 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Jellyfin.Database.Implementations;
using Microsoft.EntityFrameworkCore;
namespace Jellyfin.Server.Implementations.Migrations;
/// <summary>
/// Plain-SQL tracker for the EF migrations history table.
/// Replaces the EF-internal IHistoryRepository / SnakeCaseHistoryRepository.
/// </summary>
public static class MigrationTracker
{
/// <summary>
/// Entry name used when writing/reading migration history from backup archives.
/// Kept as "HistoryRow" for backward compatibility with existing backup files.
/// </summary>
public const string BackupEntryName = "HistoryRow";
private const string CreateTableSql = """
CREATE TABLE IF NOT EXISTS public."__EFMigrationsHistory" (
migration_id character varying(150) NOT NULL,
product_version character varying(32) NOT NULL,
CONSTRAINT "PK___EFMigrationsHistory" PRIMARY KEY (migration_id)
)
""";
/// <summary>Ensures the migrations history table exists.</summary>
/// <param name="dbContext">The database context.</param>
/// <param name="ct">Cancellation token.</param>
/// <returns>A <see cref="Task"/> representing the operation.</returns>
public static async Task EnsureTableExistsAsync(JellyfinDbContext dbContext, CancellationToken ct = default)
=> await dbContext.Database.ExecuteSqlRawAsync(CreateTableSql, ct).ConfigureAwait(false);
/// <summary>Returns the migration_id of every applied migration.</summary>
/// <param name="dbContext">The database context.</param>
/// <param name="ct">Cancellation token.</param>
/// <returns>A read-only list of applied migration identifiers.</returns>
public static async Task<IReadOnlyList<string>> GetAppliedMigrationIdsAsync(JellyfinDbContext dbContext, CancellationToken ct = default)
=> await dbContext.Database
.SqlQueryRaw<string>("""SELECT migration_id AS "Value" FROM public."__EFMigrationsHistory" """)
.ToListAsync(ct).ConfigureAwait(false);
/// <summary>Returns all applied migration rows.</summary>
/// <param name="dbContext">The database context.</param>
/// <param name="ct">Cancellation token.</param>
/// <returns>A read-only list of <see cref="MigrationRecord"/> instances.</returns>
public static async Task<IReadOnlyList<MigrationRecord>> GetAppliedMigrationsAsync(JellyfinDbContext dbContext, CancellationToken ct = default)
=> await dbContext.Database
.SqlQueryRaw<MigrationRecord>("""
SELECT migration_id AS "MigrationId", product_version AS "ProductVersion"
FROM public."__EFMigrationsHistory"
""")
.ToListAsync(ct).ConfigureAwait(false);
/// <summary>Inserts a migration row, ignoring conflicts.</summary>
/// <param name="dbContext">The database context.</param>
/// <param name="migrationId">The migration identifier.</param>
/// <param name="productVersion">The product version.</param>
/// <param name="ct">Cancellation token.</param>
/// <returns>A <see cref="Task"/> representing the operation.</returns>
public static async Task InsertAsync(JellyfinDbContext dbContext, string migrationId, string productVersion, CancellationToken ct = default)
=> await dbContext.Database.ExecuteSqlAsync(
$"""
INSERT INTO public."__EFMigrationsHistory" (migration_id, product_version)
VALUES ({migrationId}, {productVersion}) ON CONFLICT DO NOTHING
""",
ct).ConfigureAwait(false);
/// <summary>Deletes a migration row by id.</summary>
/// <param name="dbContext">The database context.</param>
/// <param name="migrationId">The migration identifier to delete.</param>
/// <param name="ct">Cancellation token.</param>
/// <returns>A <see cref="Task"/> representing the operation.</returns>
public static async Task DeleteAsync(JellyfinDbContext dbContext, string migrationId, CancellationToken ct = default)
=> await dbContext.Database.ExecuteSqlAsync(
$"""DELETE FROM public."__EFMigrationsHistory" WHERE migration_id = {migrationId}""",
ct).ConfigureAwait(false);
}