Add automated PostgreSQL performance index management
- Add scripts and SQL for base and supplementary indexes (total 23) - Integrate index checks and auto-creation into Jellyfin startup - Update csproj to include SQL files in build/publish output - Add diagnostics, publish/copy scripts, and troubleshooting docs - Provide detailed guides for migration, verification, and merging - Ensures robust, automated, and well-documented index setup
This commit is contained in:
+140
@@ -0,0 +1,140 @@
|
||||
// <copyright file="20260226170000_AddBasePerformanceIndexes.cs" company="PlaceholderCompany">
|
||||
// Copyright (c) PlaceholderCompany. All rights reserved.
|
||||
// </copyright>
|
||||
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace Jellyfin.Database.Providers.Postgres.Migrations
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public partial class AddBasePerformanceIndexes : Migration
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
// These are the base performance indexes that should have been in InitialCreate
|
||||
// but were missing. They're from the original schema dump.
|
||||
|
||||
// BaseItems performance indexes
|
||||
migrationBuilder.Sql(@"
|
||||
CREATE INDEX IF NOT EXISTS baseitems_communityrating_idx
|
||||
ON library.""BaseItems"" (""CommunityRating"" DESC);
|
||||
");
|
||||
|
||||
migrationBuilder.Sql(@"
|
||||
CREATE INDEX IF NOT EXISTS baseitems_datecreated_idx
|
||||
ON library.""BaseItems"" (""DateCreated"" DESC);
|
||||
");
|
||||
|
||||
migrationBuilder.Sql(@"
|
||||
CREATE INDEX IF NOT EXISTS baseitems_datemodified_idx
|
||||
ON library.""BaseItems"" (""DateModified"" DESC);
|
||||
");
|
||||
|
||||
migrationBuilder.Sql(@"
|
||||
CREATE INDEX IF NOT EXISTS baseitems_parentid_idx
|
||||
ON library.""BaseItems"" (""ParentId"", ""Type"");
|
||||
");
|
||||
|
||||
migrationBuilder.Sql(@"
|
||||
CREATE INDEX IF NOT EXISTS baseitems_premieredate_idx
|
||||
ON library.""BaseItems"" (""PremiereDate"" DESC);
|
||||
");
|
||||
|
||||
migrationBuilder.Sql(@"
|
||||
CREATE INDEX IF NOT EXISTS baseitems_productionyear_idx
|
||||
ON library.""BaseItems"" (""ProductionYear"" DESC);
|
||||
");
|
||||
|
||||
migrationBuilder.Sql(@"
|
||||
CREATE INDEX IF NOT EXISTS baseitems_seriespresentationuniquekey_idx
|
||||
ON library.""BaseItems"" (""SeriesPresentationUniqueKey"", ""IndexNumber"", ""ParentIndexNumber"");
|
||||
");
|
||||
|
||||
migrationBuilder.Sql(@"
|
||||
CREATE INDEX IF NOT EXISTS baseitems_sortname_idx
|
||||
ON library.""BaseItems"" (""SortName"");
|
||||
");
|
||||
|
||||
migrationBuilder.Sql(@"
|
||||
CREATE INDEX IF NOT EXISTS baseitems_topparentid_idx
|
||||
ON library.""BaseItems"" (""TopParentId"", ""Type"");
|
||||
");
|
||||
|
||||
// BaseItemProviders performance indexes
|
||||
migrationBuilder.Sql(@"
|
||||
CREATE INDEX IF NOT EXISTS baseitemproviders_providerid_idx
|
||||
ON library.""BaseItemProviders"" (""ProviderId"", ""ItemId"");
|
||||
");
|
||||
|
||||
migrationBuilder.Sql(@"
|
||||
CREATE INDEX IF NOT EXISTS baseitemproviders_providervalue_idx
|
||||
ON library.""BaseItemProviders"" (""ProviderValue"", ""ProviderId"");
|
||||
");
|
||||
|
||||
// MediaStreamInfos performance indexes
|
||||
migrationBuilder.Sql(@"
|
||||
CREATE INDEX IF NOT EXISTS mediastreaminfos_codec_idx
|
||||
ON library.""MediaStreamInfos"" (""Codec"");
|
||||
");
|
||||
|
||||
migrationBuilder.Sql(@"
|
||||
CREATE INDEX IF NOT EXISTS mediastreaminfos_itemid_idx
|
||||
ON library.""MediaStreamInfos"" (""ItemId"", ""StreamType"");
|
||||
");
|
||||
|
||||
// PeopleBaseItemMap performance indexes
|
||||
migrationBuilder.Sql(@"
|
||||
CREATE INDEX IF NOT EXISTS peoplebaseitemmap_itemid_idx
|
||||
ON library.""PeopleBaseItemMap"" (""ItemId"", ""PeopleId"");
|
||||
");
|
||||
|
||||
migrationBuilder.Sql(@"
|
||||
CREATE INDEX IF NOT EXISTS peoplebaseitemmap_peopleid_idx
|
||||
ON library.""PeopleBaseItemMap"" (""PeopleId"", ""ItemId"");
|
||||
");
|
||||
|
||||
// UserData performance indexes
|
||||
migrationBuilder.Sql(@"
|
||||
CREATE INDEX IF NOT EXISTS userdata_lastplayeddate_idx
|
||||
ON library.""UserData"" (""LastPlayedDate"" DESC);
|
||||
");
|
||||
|
||||
migrationBuilder.Sql(@"
|
||||
CREATE INDEX IF NOT EXISTS userdata_userid_isfavorite_idx
|
||||
ON library.""UserData"" (""UserId"", ""IsFavorite"");
|
||||
");
|
||||
|
||||
migrationBuilder.Sql(@"
|
||||
CREATE INDEX IF NOT EXISTS userdata_userid_played_idx
|
||||
ON library.""UserData"" (""UserId"", ""Played"");
|
||||
");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
// Drop all the base performance indexes
|
||||
migrationBuilder.Sql(@"DROP INDEX IF EXISTS library.baseitems_communityrating_idx;");
|
||||
migrationBuilder.Sql(@"DROP INDEX IF EXISTS library.baseitems_datecreated_idx;");
|
||||
migrationBuilder.Sql(@"DROP INDEX IF EXISTS library.baseitems_datemodified_idx;");
|
||||
migrationBuilder.Sql(@"DROP INDEX IF EXISTS library.baseitems_parentid_idx;");
|
||||
migrationBuilder.Sql(@"DROP INDEX IF EXISTS library.baseitems_premieredate_idx;");
|
||||
migrationBuilder.Sql(@"DROP INDEX IF EXISTS library.baseitems_productionyear_idx;");
|
||||
migrationBuilder.Sql(@"DROP INDEX IF EXISTS library.baseitems_seriespresentationuniquekey_idx;");
|
||||
migrationBuilder.Sql(@"DROP INDEX IF EXISTS library.baseitems_sortname_idx;");
|
||||
migrationBuilder.Sql(@"DROP INDEX IF EXISTS library.baseitems_topparentid_idx;");
|
||||
migrationBuilder.Sql(@"DROP INDEX IF EXISTS library.baseitemproviders_providerid_idx;");
|
||||
migrationBuilder.Sql(@"DROP INDEX IF EXISTS library.baseitemproviders_providervalue_idx;");
|
||||
migrationBuilder.Sql(@"DROP INDEX IF EXISTS library.mediastreaminfos_codec_idx;");
|
||||
migrationBuilder.Sql(@"DROP INDEX IF EXISTS library.mediastreaminfos_itemid_idx;");
|
||||
migrationBuilder.Sql(@"DROP INDEX IF EXISTS library.peoplebaseitemmap_itemid_idx;");
|
||||
migrationBuilder.Sql(@"DROP INDEX IF EXISTS library.peoplebaseitemmap_peopleid_idx;");
|
||||
migrationBuilder.Sql(@"DROP INDEX IF EXISTS library.userdata_lastplayeddate_idx;");
|
||||
migrationBuilder.Sql(@"DROP INDEX IF EXISTS library.userdata_userid_isfavorite_idx;");
|
||||
migrationBuilder.Sql(@"DROP INDEX IF EXISTS library.userdata_userid_played_idx;");
|
||||
}
|
||||
}
|
||||
}
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
// <copyright file="20260227000000_AddSupplementaryIndexes.cs" company="PlaceholderCompany">
|
||||
// Copyright (c) PlaceholderCompany. All rights reserved.
|
||||
// </copyright>
|
||||
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace Jellyfin.Database.Providers.Postgres.Migrations
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public partial class AddSupplementaryIndexes : Migration
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
// 1. Composite index for filtered library browsing (type + virtual + parent)
|
||||
// Use case: Browse items by type in a specific library, excluding virtual items
|
||||
migrationBuilder.Sql(@"
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_baseitems_type_isvirtualitem_topparentid
|
||||
ON library.""BaseItems"" (""Type"", ""IsVirtualItem"", ""TopParentId"")
|
||||
WHERE ""IsVirtualItem"" = false;
|
||||
");
|
||||
|
||||
// 2. Index for folder hierarchy queries
|
||||
// Use case: Navigate folder structures efficiently
|
||||
migrationBuilder.Sql(@"
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_baseitems_topparentid_isfolder
|
||||
ON library.""BaseItems"" (""TopParentId"", ""IsFolder"", ""IsVirtualItem"")
|
||||
WHERE ""TopParentId"" IS NOT NULL;
|
||||
");
|
||||
|
||||
// 3. Index for recently added content with filters
|
||||
// Use case: ""Recently Added"" view across all libraries
|
||||
migrationBuilder.Sql(@"
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_baseitems_datecreated_filtered
|
||||
ON library.""BaseItems"" (""DateCreated"" DESC, ""Type"", ""IsVirtualItem"")
|
||||
WHERE ""DateCreated"" IS NOT NULL AND ""IsVirtualItem"" = false;
|
||||
");
|
||||
|
||||
// 4. Index for reverse lookup (find items by genre/tag)
|
||||
// Use case: ""Show all items with genre 'Action'""
|
||||
migrationBuilder.Sql(@"
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_itemvaluesmap_itemvalueid_itemid
|
||||
ON library.""ItemValuesMap"" (""ItemValueId"", ""ItemId"");
|
||||
");
|
||||
|
||||
// 5. Index for user-specific activity queries
|
||||
// Use case: ""Show activity for user X""
|
||||
migrationBuilder.Sql(@"
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema = 'activitylog'
|
||||
AND table_name = 'ActivityLogs'
|
||||
) THEN
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_activitylogs_userid_datecreated
|
||||
ON activitylog.""ActivityLogs"" (""UserId"", ""DateCreated"" DESC)
|
||||
WHERE ""UserId"" IS NOT NULL;
|
||||
END IF;
|
||||
END $$;
|
||||
");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
// Drop the supplementary indexes
|
||||
migrationBuilder.Sql(@"DROP INDEX CONCURRENTLY IF EXISTS library.idx_baseitems_type_isvirtualitem_topparentid;");
|
||||
migrationBuilder.Sql(@"DROP INDEX CONCURRENTLY IF EXISTS library.idx_baseitems_topparentid_isfolder;");
|
||||
migrationBuilder.Sql(@"DROP INDEX CONCURRENTLY IF EXISTS library.idx_baseitems_datecreated_filtered;");
|
||||
migrationBuilder.Sql(@"DROP INDEX CONCURRENTLY IF EXISTS library.idx_itemvaluesmap_itemvalueid_itemid;");
|
||||
migrationBuilder.Sql(@"DROP INDEX CONCURRENTLY IF EXISTS activitylog.idx_activitylogs_userid_datecreated;");
|
||||
}
|
||||
}
|
||||
}
|
||||
+153
@@ -452,6 +452,12 @@ public sealed class PostgresDatabaseProvider : IJellyfinDatabaseProvider
|
||||
}
|
||||
|
||||
logger.LogInformation("Database table verification complete");
|
||||
|
||||
// Check for supplementary performance indexes
|
||||
await CheckSupplementaryIndexesAsync(connection, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Apply performance indexes from SQL script if missing
|
||||
await ApplyPerformanceIndexesFromScriptAsync(connection, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -833,4 +839,151 @@ public sealed class PostgresDatabaseProvider : IJellyfinDatabaseProvider
|
||||
|
||||
await dbContext.Database.ExecuteSqlRawAsync(deleteAllQuery).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if supplementary performance indexes exist and logs warnings if missing.
|
||||
/// These indexes complement the base schema and improve specific query patterns.
|
||||
/// </summary>
|
||||
/// <param name="connection">The database connection.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
private async Task CheckSupplementaryIndexesAsync(System.Data.Common.DbConnection connection, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var checkQuery = @"
|
||||
SELECT indexname
|
||||
FROM pg_indexes
|
||||
WHERE indexname IN (
|
||||
'idx_baseitems_type_isvirtualitem_topparentid',
|
||||
'idx_baseitems_topparentid_isfolder',
|
||||
'idx_baseitems_datecreated_filtered',
|
||||
'idx_itemvaluesmap_itemvalueid_itemid',
|
||||
'idx_activitylogs_userid_datecreated'
|
||||
)
|
||||
ORDER BY indexname";
|
||||
|
||||
var foundIndexes = new List<string>();
|
||||
|
||||
await using var command = connection.CreateCommand();
|
||||
command.CommandText = checkQuery;
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
foundIndexes.Add(reader.GetString(0));
|
||||
}
|
||||
|
||||
var expectedIndexes = new[]
|
||||
{
|
||||
"idx_baseitems_type_isvirtualitem_topparentid",
|
||||
"idx_baseitems_topparentid_isfolder",
|
||||
"idx_baseitems_datecreated_filtered",
|
||||
"idx_itemvaluesmap_itemvalueid_itemid",
|
||||
"idx_activitylogs_userid_datecreated"
|
||||
};
|
||||
|
||||
var missingIndexes = expectedIndexes.Except(foundIndexes).ToList();
|
||||
|
||||
if (missingIndexes.Count > 0)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Found {FoundCount}/5 supplementary performance indexes. Missing: {MissingIndexes}",
|
||||
foundIndexes.Count,
|
||||
string.Join(", ", missingIndexes));
|
||||
logger.LogWarning(
|
||||
"Performance indexes will be created automatically from sql/all_performance_indexes.sql");
|
||||
}
|
||||
else
|
||||
{
|
||||
logger.LogInformation("All 5 supplementary performance indexes are present.");
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogWarning(ex, "Failed to check supplementary indexes. This is non-critical and startup will continue.");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Applies performance indexes from the SQL script if they are missing.
|
||||
/// This ensures optimal query performance after database initialization.
|
||||
/// </summary>
|
||||
/// <param name="connection">The database connection.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
private async Task ApplyPerformanceIndexesFromScriptAsync(System.Data.Common.DbConnection connection, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Check if base performance indexes exist
|
||||
var checkQuery = @"
|
||||
SELECT COUNT(*)
|
||||
FROM pg_indexes
|
||||
WHERE schemaname = 'library'
|
||||
AND indexname IN (
|
||||
'baseitems_communityrating_idx',
|
||||
'baseitems_datecreated_idx',
|
||||
'baseitems_datemodified_idx',
|
||||
'baseitems_parentid_idx',
|
||||
'baseitems_premieredate_idx',
|
||||
'baseitems_productionyear_idx',
|
||||
'baseitems_seriespresentationuniquekey_idx',
|
||||
'baseitems_sortname_idx',
|
||||
'baseitems_topparentid_idx'
|
||||
)";
|
||||
|
||||
await using (var command = connection.CreateCommand())
|
||||
{
|
||||
command.CommandText = checkQuery;
|
||||
var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
|
||||
var existingIndexCount = result != null ? Convert.ToInt32(result) : 0;
|
||||
|
||||
// If we have all 9 base indexes, assume performance indexes are already applied
|
||||
if (existingIndexCount >= 9)
|
||||
{
|
||||
logger.LogDebug("Base performance indexes already exist ({Count}/9). Skipping SQL script execution.", existingIndexCount);
|
||||
return;
|
||||
}
|
||||
|
||||
logger.LogInformation("Found {Count}/9 base performance indexes. Applying SQL script to create missing indexes...", existingIndexCount);
|
||||
}
|
||||
|
||||
// Find the SQL script path
|
||||
var sqlScriptPath = Path.Combine(AppContext.BaseDirectory, "sql", "all_performance_indexes.sql");
|
||||
|
||||
if (!File.Exists(sqlScriptPath))
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Performance indexes SQL script not found at: {ScriptPath}. " +
|
||||
"Database will function but may have suboptimal performance. " +
|
||||
"Run 'Add-All-Indexes.bat' or apply sql/all_performance_indexes.sql manually.",
|
||||
sqlScriptPath);
|
||||
return;
|
||||
}
|
||||
|
||||
logger.LogInformation("Executing performance indexes script: {ScriptPath}", sqlScriptPath);
|
||||
|
||||
// Read and execute the SQL script
|
||||
var sqlScript = await File.ReadAllTextAsync(sqlScriptPath, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Remove psql-specific commands (\echo, etc.) since we're executing via code
|
||||
sqlScript = System.Text.RegularExpressions.Regex.Replace(sqlScript, @"\\echo.*$", string.Empty, System.Text.RegularExpressions.RegexOptions.Multiline);
|
||||
|
||||
await using (var command = connection.CreateCommand())
|
||||
{
|
||||
command.CommandText = sqlScript;
|
||||
command.CommandTimeout = 600; // 10 minutes for index creation
|
||||
|
||||
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
logger.LogInformation("Performance indexes created successfully. Database is now fully optimized.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogError(
|
||||
ex,
|
||||
"Failed to apply performance indexes from SQL script. " +
|
||||
"Database will function but performance may be suboptimal. " +
|
||||
"Consider running 'Add-All-Indexes.bat' or applying sql/all_performance_indexes.sql manually.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user