Add automated PostgreSQL performance index management
- Add scripts and SQL for base and supplementary indexes (total 23) - Integrate index checks and auto-creation into Jellyfin startup - Update csproj to include SQL files in build/publish output - Add diagnostics, publish/copy scripts, and troubleshooting docs - Provide detailed guides for migration, verification, and merging - Ensures robust, automated, and well-documented index setup
This commit is contained in:
@@ -0,0 +1,175 @@
|
||||
# Apply Supplementary Indexes Migration to PostgreSQL Database
|
||||
|
||||
Write-Host "========================================" -ForegroundColor Cyan
|
||||
Write-Host "Apply Supplementary Indexes Migration" -ForegroundColor Cyan
|
||||
Write-Host "========================================" -ForegroundColor Cyan
|
||||
Write-Host ""
|
||||
|
||||
# Configuration
|
||||
$connectionString = "Host=localhost;Port=5432;Database=jellyfin_testsdata;Username=jellyfin;Password=YOUR_PASSWORD"
|
||||
$migrationName = "AddSupplementaryIndexes"
|
||||
|
||||
Write-Host "Step 1: Build the solution" -ForegroundColor Yellow
|
||||
Write-Host "This ensures the migration is compiled..." -ForegroundColor Gray
|
||||
dotnet build src\Jellyfin.Database\Jellyfin.Database.Providers.Postgres\Jellyfin.Database.Providers.Postgres.csproj
|
||||
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Host "❌ Build failed. Please fix compilation errors." -ForegroundColor Red
|
||||
exit 1
|
||||
}
|
||||
|
||||
Write-Host "✓ Build successful" -ForegroundColor Green
|
||||
Write-Host ""
|
||||
|
||||
Write-Host "Step 2: Check current database state" -ForegroundColor Yellow
|
||||
Write-Host "Connecting to jellyfin_testsdata..." -ForegroundColor Gray
|
||||
|
||||
# Test database connection
|
||||
try {
|
||||
$testQuery = @"
|
||||
SELECT EXISTS (
|
||||
SELECT 1
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'library'
|
||||
AND table_name = 'BaseItems'
|
||||
) as table_exists;
|
||||
"@
|
||||
|
||||
$result = psql -U jellyfin -d jellyfin_testsdata -t -c $testQuery 2>&1
|
||||
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
Write-Host "✓ Database connection successful" -ForegroundColor Green
|
||||
} else {
|
||||
Write-Host "❌ Database connection failed" -ForegroundColor Red
|
||||
Write-Host "Error: $result" -ForegroundColor Red
|
||||
Write-Host ""
|
||||
Write-Host "Please ensure:" -ForegroundColor Yellow
|
||||
Write-Host " 1. PostgreSQL is running" -ForegroundColor Gray
|
||||
Write-Host " 2. Database 'jellyfin_testsdata' exists" -ForegroundColor Gray
|
||||
Write-Host " 3. Credentials are correct" -ForegroundColor Gray
|
||||
exit 1
|
||||
}
|
||||
} catch {
|
||||
Write-Host "❌ Failed to test database connection: $_" -ForegroundColor Red
|
||||
exit 1
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
|
||||
Write-Host "Step 3: Check for existing indexes" -ForegroundColor Yellow
|
||||
Write-Host "Checking which supplementary indexes already exist..." -ForegroundColor Gray
|
||||
|
||||
$checkIndexQuery = @"
|
||||
SELECT
|
||||
indexname,
|
||||
CASE WHEN indexname IS NOT NULL THEN 'EXISTS' ELSE 'MISSING' END as status
|
||||
FROM (
|
||||
VALUES
|
||||
('idx_baseitems_type_isvirtualitem_topparentid'),
|
||||
('idx_baseitems_topparentid_isfolder'),
|
||||
('idx_baseitems_datecreated_filtered'),
|
||||
('idx_itemvaluesmap_itemvalueid_itemid'),
|
||||
('idx_activitylogs_userid_datecreated')
|
||||
) AS wanted(indexname)
|
||||
LEFT JOIN pg_indexes ON pg_indexes.indexname = wanted.indexname
|
||||
ORDER BY wanted.indexname;
|
||||
"@
|
||||
|
||||
$indexStatus = psql -U jellyfin -d jellyfin_testsdata -c $checkIndexQuery
|
||||
|
||||
Write-Host $indexStatus
|
||||
Write-Host ""
|
||||
|
||||
Write-Host "Step 4: Apply migration using EF Core" -ForegroundColor Yellow
|
||||
Write-Host "This will create any missing indexes..." -ForegroundColor Gray
|
||||
|
||||
# Option 1: Use dotnet ef (if you have it installed)
|
||||
Write-Host ""
|
||||
Write-Host "Choose migration method:" -ForegroundColor Cyan
|
||||
Write-Host "1. Use dotnet ef database update (requires dotnet-ef tool)" -ForegroundColor White
|
||||
Write-Host "2. Use SQL script generation" -ForegroundColor White
|
||||
Write-Host "3. Apply SQL directly (recommended)" -ForegroundColor White
|
||||
Write-Host ""
|
||||
|
||||
$choice = Read-Host "Enter your choice (1, 2, or 3)"
|
||||
|
||||
switch ($choice) {
|
||||
"1" {
|
||||
Write-Host "Applying migration using dotnet ef..." -ForegroundColor Yellow
|
||||
$env:ConnectionStrings__DefaultConnection = $connectionString
|
||||
|
||||
dotnet ef database update --project src\Jellyfin.Database\Jellyfin.Database.Providers.Postgres\Jellyfin.Database.Providers.Postgres.csproj
|
||||
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
Write-Host "✓ Migration applied successfully" -ForegroundColor Green
|
||||
} else {
|
||||
Write-Host "❌ Migration failed" -ForegroundColor Red
|
||||
}
|
||||
}
|
||||
"2" {
|
||||
Write-Host "Generating SQL script..." -ForegroundColor Yellow
|
||||
|
||||
dotnet ef migrations script --project src\Jellyfin.Database\Jellyfin.Database.Providers.Postgres\Jellyfin.Database.Providers.Postgres.csproj --output sql\migration_supplementary_indexes.sql
|
||||
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
Write-Host "✓ SQL script generated at: sql\migration_supplementary_indexes.sql" -ForegroundColor Green
|
||||
Write-Host ""
|
||||
Write-Host "To apply, run:" -ForegroundColor Yellow
|
||||
Write-Host "psql -U jellyfin -d jellyfin_testsdata -f sql\migration_supplementary_indexes.sql" -ForegroundColor White
|
||||
} else {
|
||||
Write-Host "❌ Script generation failed" -ForegroundColor Red
|
||||
}
|
||||
}
|
||||
"3" {
|
||||
Write-Host "Applying SQL directly..." -ForegroundColor Yellow
|
||||
|
||||
# Apply the SQL from the schema_init script
|
||||
psql -U jellyfin -d jellyfin_testsdata -f sql\schema_init\10_create_supplementary_indexes.sql
|
||||
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
Write-Host "✓ Indexes created successfully" -ForegroundColor Green
|
||||
} else {
|
||||
Write-Host "❌ Index creation failed" -ForegroundColor Red
|
||||
}
|
||||
}
|
||||
default {
|
||||
Write-Host "Invalid choice. Exiting." -ForegroundColor Red
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Step 5: Verify indexes were created" -ForegroundColor Yellow
|
||||
|
||||
$verifyQuery = @"
|
||||
SELECT
|
||||
schemaname,
|
||||
tablename,
|
||||
indexname,
|
||||
pg_size_pretty(pg_relation_size(indexrelid)) as size
|
||||
FROM pg_indexes
|
||||
JOIN pg_stat_user_indexes USING (schemaname, tablename, indexname)
|
||||
WHERE indexname IN (
|
||||
'idx_baseitems_type_isvirtualitem_topparentid',
|
||||
'idx_baseitems_topparentid_isfolder',
|
||||
'idx_baseitems_datecreated_filtered',
|
||||
'idx_itemvaluesmap_itemvalueid_itemid',
|
||||
'idx_activitylogs_userid_datecreated'
|
||||
)
|
||||
ORDER BY schemaname, tablename, indexname;
|
||||
"@
|
||||
|
||||
$verification = psql -U jellyfin -d jellyfin_testsdata -c $verifyQuery
|
||||
|
||||
Write-Host $verification
|
||||
Write-Host ""
|
||||
|
||||
Write-Host "========================================" -ForegroundColor Cyan
|
||||
Write-Host "Migration Complete!" -ForegroundColor Green
|
||||
Write-Host "========================================" -ForegroundColor Cyan
|
||||
Write-Host ""
|
||||
Write-Host "Next steps:" -ForegroundColor Yellow
|
||||
Write-Host "1. Restart Jellyfin to clear connection pools" -ForegroundColor White
|
||||
Write-Host "2. Monitor index usage with: psql -U jellyfin -d jellyfin_testsdata -f sql\diagnostics.sql" -ForegroundColor White
|
||||
Write-Host "3. Check performance improvements" -ForegroundColor White
|
||||
Write-Host ""
|
||||
@@ -0,0 +1,126 @@
|
||||
# Quick Connect and Add Supplementary Indexes
|
||||
|
||||
This script connects to your `jellyfin_testsdata` database and adds the 5 supplementary indexes.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. PostgreSQL is running
|
||||
2. Database `jellyfin_testsdata` exists
|
||||
3. `psql` is in your PATH
|
||||
4. You have the jellyfin user credentials
|
||||
|
||||
## Quick Run
|
||||
|
||||
### Option 1: Direct SQL (Fastest)
|
||||
```powershell
|
||||
# Just add the indexes directly
|
||||
psql -U jellyfin -d jellyfin_testsdata -f sql\schema_init\10_create_supplementary_indexes.sql
|
||||
```
|
||||
|
||||
### Option 2: With verification and checks
|
||||
```powershell
|
||||
# Run the full script with checks and verification
|
||||
.\scripts\Apply-SupplementaryIndexes.ps1
|
||||
```
|
||||
|
||||
### Option 3: Manual Step-by-Step
|
||||
|
||||
1. **Connect to database**:
|
||||
```powershell
|
||||
psql -U jellyfin -d jellyfin_testsdata
|
||||
```
|
||||
|
||||
2. **Check current indexes**:
|
||||
```sql
|
||||
SELECT indexname
|
||||
FROM pg_indexes
|
||||
WHERE schemaname = 'library'
|
||||
AND indexname LIKE 'idx_baseitems_%'
|
||||
ORDER BY indexname;
|
||||
```
|
||||
|
||||
3. **Apply supplementary indexes**:
|
||||
```sql
|
||||
\i sql/schema_init/10_create_supplementary_indexes.sql
|
||||
```
|
||||
|
||||
4. **Verify creation**:
|
||||
```sql
|
||||
SELECT
|
||||
indexname,
|
||||
pg_size_pretty(pg_relation_size(indexrelid)) as size,
|
||||
idx_scan as times_used
|
||||
FROM pg_stat_user_indexes
|
||||
WHERE indexname IN (
|
||||
'idx_baseitems_type_isvirtualitem_topparentid',
|
||||
'idx_baseitems_topparentid_isfolder',
|
||||
'idx_baseitems_datecreated_filtered',
|
||||
'idx_itemvaluesmap_itemvalueid_itemid',
|
||||
'idx_activitylogs_userid_datecreated'
|
||||
);
|
||||
```
|
||||
|
||||
## What Gets Added
|
||||
|
||||
5 supplementary indexes:
|
||||
1. `idx_baseitems_type_isvirtualitem_topparentid` - Filtered library browsing
|
||||
2. `idx_baseitems_topparentid_isfolder` - Folder hierarchy
|
||||
3. `idx_baseitems_datecreated_filtered` - Recently added view
|
||||
4. `idx_itemvaluesmap_itemvalueid_itemid` - Genre/tag reverse lookup
|
||||
5. `idx_activitylogs_userid_datecreated` - User activity queries
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### "CONCURRENTLY cannot be used" error
|
||||
Remove `CONCURRENTLY` from the SQL if you get this error (less common in newer PostgreSQL versions).
|
||||
|
||||
### "Index already exists"
|
||||
This is safe - the script checks before creating. The index already exists and doesn't need to be created again.
|
||||
|
||||
### "Connection refused"
|
||||
Ensure PostgreSQL is running:
|
||||
```powershell
|
||||
# Check if PostgreSQL service is running
|
||||
Get-Service postgresql*
|
||||
|
||||
# Or try to ping the database
|
||||
psql -U jellyfin -d jellyfin_testsdata -c "SELECT version();"
|
||||
```
|
||||
|
||||
### Wrong password
|
||||
Update the password in the script or use `.pgpass` file:
|
||||
```powershell
|
||||
# Windows: %APPDATA%\postgresql\pgpass.conf
|
||||
# Linux/Mac: ~/.pgpass
|
||||
# Format: hostname:port:database:username:password
|
||||
localhost:5432:jellyfin_testsdata:jellyfin:YOUR_PASSWORD
|
||||
```
|
||||
|
||||
## Monitor Performance
|
||||
|
||||
After applying, monitor index usage:
|
||||
```powershell
|
||||
# Wait a few days, then check usage
|
||||
psql -U jellyfin -d jellyfin_testsdata -c "
|
||||
SELECT
|
||||
indexname,
|
||||
idx_scan as times_used,
|
||||
idx_tup_read as rows_read,
|
||||
pg_size_pretty(pg_relation_size(indexrelid)) as size
|
||||
FROM pg_stat_user_indexes
|
||||
WHERE indexname LIKE 'idx_%'
|
||||
AND schemaname = 'library'
|
||||
ORDER BY idx_scan DESC;
|
||||
"
|
||||
```
|
||||
|
||||
## Rollback
|
||||
|
||||
If you need to remove the indexes:
|
||||
```sql
|
||||
DROP INDEX CONCURRENTLY IF EXISTS library.idx_baseitems_type_isvirtualitem_topparentid;
|
||||
DROP INDEX CONCURRENTLY IF EXISTS library.idx_baseitems_topparentid_isfolder;
|
||||
DROP INDEX CONCURRENTLY IF EXISTS library.idx_baseitems_datecreated_filtered;
|
||||
DROP INDEX CONCURRENTLY IF EXISTS library.idx_itemvaluesmap_itemvalueid_itemid;
|
||||
DROP INDEX CONCURRENTLY IF EXISTS activitylog.idx_activitylogs_userid_datecreated;
|
||||
```
|
||||
Reference in New Issue
Block a user