wjones
1abf61a05f
- Introduced PostgresSubprocessException to handle errors during PostgreSQL subprocess execution. - Refactored PostgresDatabaseProvider to improve schema initialization script discovery. - Enhanced error handling for psql command execution, including logging of output and errors. - Implemented methods to find the schema initialization script and the psql executable path.
407 lines
15 KiB
Bash
407 lines
15 KiB
Bash
#!/bin/bash
|
|
#
|
|
# SYNOPSIS
|
|
# Automated weekly performance monitoring for Jellyfin PostgreSQL database
|
|
#
|
|
# DESCRIPTION
|
|
# This script:
|
|
# 1. Runs comprehensive diagnostics
|
|
# 2. Analyzes slow queries
|
|
# 3. Tracks index usage over time
|
|
# 4. Generates weekly performance reports
|
|
# 5. Optionally runs VACUUM ANALYZE on tables with high dead tuples
|
|
# 6. Can be scheduled via cron
|
|
#
|
|
# USAGE
|
|
# ./Weekly-Performance-Monitor.sh [OPTIONS]
|
|
#
|
|
# OPTIONS
|
|
# -o, --output-directory DIR Directory to save reports (default: reports/weekly)
|
|
# -e, --email-report If specified, emails the report (requires email configuration)
|
|
# -c, --compare-with-previous Compare current stats with previous week
|
|
# -a, --auto-vacuum-analyze Enable auto vacuum analyze
|
|
# --dead-tuple-threshold NUM Minimum dead tuples to trigger vacuum (default: 100)
|
|
# --dead-tuple-percent-threshold NUM Minimum dead tuple percentage to trigger vacuum (default: 0.02)
|
|
#
|
|
# EXAMPLE
|
|
# ./Weekly-Performance-Monitor.sh
|
|
# ./Weekly-Performance-Monitor.sh -o "/reports/jellyfin" -c
|
|
# ./Weekly-Performance-Monitor.sh -a --dead-tuple-threshold 10000 --dead-tuple-percent-threshold 20
|
|
#
|
|
# CRON JOB EXAMPLE
|
|
# # Run every Monday at 6am
|
|
# 0 6 * * 1 /path/to/Weekly-Performance-Monitor.sh -a
|
|
#
|
|
|
|
# ============================================================================
|
|
# Default Configuration
|
|
# ============================================================================
|
|
OUTPUT_DIRECTORY="reports/weekly"
|
|
EMAIL_REPORT=false
|
|
COMPARE_WITH_PREVIOUS=false
|
|
AUTO_VACUUM_ANALYZE=false
|
|
DEAD_TUPLE_THRESHOLD=100
|
|
DEAD_TUPLE_PERCENT_THRESHOLD=0.02
|
|
|
|
# ============================================================================
|
|
# Helper Functions
|
|
# ============================================================================
|
|
|
|
# Color definitions
|
|
CYAN='\033[0;36m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
RED='\033[0;31m'
|
|
GRAY='\033[0;90m'
|
|
NC='\033[0m' # No Color
|
|
|
|
write_color_output() {
|
|
local message="$1"
|
|
local color="$2"
|
|
echo -e "${color}${message}${NC}"
|
|
}
|
|
|
|
get_report_timestamp() {
|
|
date +"%Y-%m-%d_%H%M%S"
|
|
}
|
|
|
|
get_week_number() {
|
|
date +%V
|
|
}
|
|
|
|
# ============================================================================
|
|
# Script Argument Parsing
|
|
# ============================================================================
|
|
while [[ "$#" -gt 0 ]]; do
|
|
case $1 in
|
|
-o|--output-directory) OUTPUT_DIRECTORY="$2"; shift ;;
|
|
-e|--email-report) EMAIL_REPORT=true ;;
|
|
-c|--compare-with-previous) COMPARE_WITH_PREVIOUS=true ;;
|
|
-a|--auto-vacuum-analyze) AUTO_VACUUM_ANALYZE=true ;;
|
|
--dead-tuple-threshold) DEAD_TUPLE_THRESHOLD="$2"; shift ;;
|
|
--dead-tuple-percent-threshold) DEAD_TUPLE_PERCENT_THRESHOLD="$2"; shift ;;
|
|
*) write_color_output "Unknown parameter passed: $1" "$RED"; exit 1 ;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
# ============================================================================
|
|
# Configuration
|
|
# ============================================================================
|
|
|
|
set -e # Exit on error
|
|
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
|
SQL_DIR="$PROJECT_ROOT/scripts/sql"
|
|
|
|
# Load database configuration
|
|
source "./db-config.sh"
|
|
|
|
# ============================================================================
|
|
# Functions
|
|
# ============================================================================
|
|
|
|
test_database_connection() {
|
|
write_color_output "\nTesting database connection..." "$CYAN"
|
|
|
|
if PGPASSWORD=$DB_PASS $PSQL_PATH -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME -c "SELECT version();" > /dev/null 2>&1; then
|
|
write_color_output "[OK] Database connection successful" "$GREEN"
|
|
return 0
|
|
else
|
|
write_color_output "[ERROR] Database connection failed" "$RED"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
invoke_diagnostic_report() {
|
|
local output_file="$1"
|
|
write_color_output "\nRunning comprehensive diagnostics..." "$CYAN"
|
|
|
|
run_sql_files_from_dir "$SQL_DIR/diagnostics" "$output_file"
|
|
|
|
if [ $? -eq 0 ]; then
|
|
write_color_output "[OK] Diagnostics completed: $output_file" "$GREEN"
|
|
else
|
|
write_color_output "[WARN] Diagnostics completed with warnings" "$YELLOW"
|
|
fi
|
|
}
|
|
|
|
get_index_usage_stats() {
|
|
local output_file="$1"
|
|
write_color_output "\nGathering index usage statistics..." "$CYAN"
|
|
|
|
local query="
|
|
SELECT
|
|
schemaname,
|
|
relname AS tablename,
|
|
indexrelname,
|
|
idx_scan,
|
|
idx_tup_read,
|
|
idx_tup_fetch,
|
|
pg_size_pretty(pg_relation_size(indexrelid)) AS index_size
|
|
FROM pg_stat_user_indexes
|
|
WHERE schemaname IN ('library', 'users', 'authentication', 'displaypreferences', 'activitylog')
|
|
ORDER BY schemaname, relname, idx_scan DESC;
|
|
"
|
|
|
|
if PGPASSWORD=$DB_PASS $PSQL_PATH -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME -c "$query" > "$output_file" 2>&1; then
|
|
write_color_output "[OK] Index stats gathered: $output_file" "$GREEN"
|
|
else
|
|
write_color_output "[WARN] Index stats completed with warnings" "$YELLOW"
|
|
fi
|
|
}
|
|
|
|
get_table_size_stats() {
|
|
local output_file="$1"
|
|
write_color_output "\nGathering table size statistics..." "$CYAN"
|
|
|
|
local query="
|
|
SELECT
|
|
schemaname,
|
|
relname,
|
|
pg_size_pretty(pg_total_relation_size(schemaname||'.'||relname)) AS total_size,
|
|
pg_size_pretty(pg_relation_size(schemaname||'.'||relname)) AS table_size,
|
|
pg_size_pretty(pg_total_relation_size(schemaname||'.'||relname) - pg_relation_size(schemaname||'.'||relname)) AS index_size,
|
|
n_live_tup AS row_count,
|
|
n_dead_tup AS dead_rows,
|
|
ROUND(100.0 * n_dead_tup / NULLIF(n_live_tup, 0), 2) AS dead_pct
|
|
FROM pg_stat_user_tables
|
|
WHERE schemaname IN ('library', 'users', 'authentication', 'displaypreferences', 'activitylog')
|
|
ORDER BY pg_total_relation_size(schemaname||'.'||relname) DESC;
|
|
"
|
|
|
|
if PGPASSWORD=$DB_PASS $PSQL_PATH -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME -c "$query" > "$output_file" 2>&1; then
|
|
write_color_output "[OK] Table size stats gathered: $output_file" "$GREEN"
|
|
else
|
|
write_color_output "[WARN] Table size stats completed with warnings" "$YELLOW"
|
|
fi
|
|
}
|
|
|
|
invoke_conditional_vacuum_analyze() {
|
|
local min_dead_tuples="$1"
|
|
local min_dead_percent="$2"
|
|
|
|
write_color_output "\nChecking for tables that need VACUUM ANALYZE..." "$CYAN"
|
|
write_color_output "Thresholds: dead tuples >= $min_dead_tuples and dead percent >= $min_dead_percent" "$GRAY"
|
|
|
|
local table_query="
|
|
SELECT
|
|
format('%I.%I', schemaname, relname) AS qualified_table,
|
|
n_dead_tup,
|
|
ROUND(100.0 * n_dead_tup / NULLIF(n_live_tup, 0), 2) AS dead_pct
|
|
FROM pg_stat_user_tables
|
|
WHERE schemaname IN ('library', 'users', 'authentication', 'displaypreferences', 'activitylog')
|
|
AND n_live_tup > 0
|
|
AND n_dead_tup >= $min_dead_tuples
|
|
AND (100.0 * n_dead_tup / NULLIF(n_live_tup, 0)) >= $min_dead_percent
|
|
ORDER BY n_dead_tup DESC;
|
|
"
|
|
|
|
tables=$(PGPASSWORD=$DB_PASS $PSQL_PATH -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME -t -A -F "|" -c "$table_query")
|
|
|
|
if [ -z "$tables" ]; then
|
|
write_color_output "[OK] No tables exceeded dead tuple thresholds" "$GREEN"
|
|
return
|
|
fi
|
|
|
|
write_color_output "Found $(echo "$tables" | wc -l) table(s) requiring VACUUM ANALYZE" "$YELLOW"
|
|
|
|
while IFS= read -r line; do
|
|
qualified_table=$(echo "$line" | cut -d'|' -f1)
|
|
dead_tuples=$(echo "$line" | cut -d'|' -f2)
|
|
dead_percent=$(echo "$line" | cut -d'|' -f3)
|
|
|
|
write_color_output "Running VACUUM ANALYZE on $qualified_table (dead tuples: $dead_tuples, dead percent: $dead_percent)" "$CYAN"
|
|
|
|
if PGPASSWORD=$DB_PASS $PSQL_PATH -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME -c "VACUUM (ANALYZE, VERBOSE) $qualified_table;"; then
|
|
write_color_output "[OK] VACUUM ANALYZE completed for $qualified_table" "$GREEN"
|
|
else
|
|
write_color_output "[ERROR] VACUUM ANALYZE failed for $qualified_table" "$RED"
|
|
fi
|
|
done <<< "$tables"
|
|
}
|
|
|
|
compare_with_previous() {
|
|
local current_report="$1"
|
|
local output_dir="$2"
|
|
|
|
write_color_output "\nComparing with previous week's report..." "$CYAN"
|
|
|
|
previous_report=$(find "$output_dir" -name "diagnostics_*.txt" ! -name "$(basename "$current_report")" -printf '%T@ %p\n' | sort -n | tail -1 | cut -d' ' -f2-)
|
|
|
|
if [ -z "$previous_report" ]; then
|
|
write_color_output "[WARN] No previous reports found for comparison" "$YELLOW"
|
|
return
|
|
fi
|
|
|
|
write_color_output "Comparing with: $(basename "$previous_report")" "$CYAN"
|
|
|
|
# TODO: Add detailed comparison logic
|
|
write_color_output "[OK] Comparison data available" "$GREEN"
|
|
write_color_output " Current: $current_report" "$GRAY"
|
|
write_color_output " Previous: $previous_report" "$GRAY"
|
|
}
|
|
|
|
new_summary_report() {
|
|
local output_dir="$1"
|
|
local timestamp="$2"
|
|
|
|
write_color_output "\nGenerating summary report..." "$CYAN"
|
|
|
|
local summary_file="$output_dir/summary_$timestamp.txt"
|
|
|
|
cat > "$summary_file" <<EOF
|
|
================================================================================
|
|
JELLYFIN DATABASE WEEKLY PERFORMANCE SUMMARY
|
|
================================================================================
|
|
Report Date: $(date +"%Y-%m-%d %H:%M:%S")
|
|
Database: $DB_NAME at $DB_HOST
|
|
Week Number: $(get_week_number)
|
|
================================================================================
|
|
|
|
REPORTS GENERATED:
|
|
* diagnostics_$timestamp.txt : Full diagnostic report
|
|
* index_usage_$timestamp.txt : Index usage statistics
|
|
* table_sizes_$timestamp.txt : Table size and bloat info
|
|
* summary_$timestamp.txt : This summary
|
|
|
|
================================================================================
|
|
QUICK HEALTH CHECK:
|
|
================================================================================
|
|
|
|
To view critical issues check diagnostics file for:
|
|
* Section 5: TABLES WITH HIGH SEQUENTIAL SCANS
|
|
* Section 8: UNUSED OR RARELY USED INDEXES
|
|
* Section 10: SLOWEST QUERIES
|
|
|
|
To identify slow queries check query_analysis file for:
|
|
* Section 2: TOP 20 SLOWEST QUERIES (by max execution time)
|
|
* Section 5: BASEITEMS TABLE QUERIES ANALYSIS
|
|
|
|
================================================================================
|
|
RECOMMENDED ACTIONS:
|
|
================================================================================
|
|
|
|
* Review slow queries taking longer than 10 seconds
|
|
* Check if new indexes are being utilized
|
|
* Look for tables with many sequential scans
|
|
* Monitor cache hit ratio (should be above 95 percent)
|
|
* Check for tables with high dead tuples percentage (need VACUUM)
|
|
|
|
================================================================================
|
|
AUTOMATION SETUP:
|
|
================================================================================
|
|
|
|
To schedule this script weekly, add the following to your crontab:
|
|
0 6 * * 1 /path/to/$(basename "$0")
|
|
|
|
================================================================================
|
|
NEXT REPORT: $(date -d "+7 days" +"%Y-%m-%d")
|
|
================================================================================
|
|
EOF
|
|
|
|
write_color_output "[OK] Summary generated: $summary_file" "$GREEN"
|
|
}
|
|
|
|
# ============================================================================
|
|
# Helper Functions
|
|
# ============================================================================
|
|
|
|
run_sql_files_from_dir() {
|
|
local sql_dir="$1"
|
|
local output_file="$2"
|
|
|
|
if [ ! -d "$sql_dir" ]; then
|
|
write_color_output "[ERROR] Directory not found: $sql_dir" "$RED"
|
|
return 1
|
|
fi
|
|
|
|
# Clear the output file before running scripts
|
|
> "$output_file"
|
|
|
|
for sql_file in "$sql_dir"/*.sql; do
|
|
echo -e "\n---\n-- Executing: $(basename "$sql_file")\n---\n" >> "$output_file"
|
|
write_color_output "Running SQL file: $(basename "$sql_file")" "$CYAN"
|
|
if PGPASSWORD=$DB_PASS $PSQL_PATH -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME -f "$sql_file" >> "$output_file" 2>&1; then
|
|
write_color_output "[OK] Successfully executed $(basename "$sql_file")" "$GREEN"
|
|
else
|
|
write_color_output "[WARN] Completed with warnings: $(basename "$sql_file")" "$YELLOW"
|
|
fi
|
|
done
|
|
}
|
|
|
|
# ============================================================================
|
|
# Main Execution
|
|
# ============================================================================
|
|
|
|
write_color_output "\n${CYAN}========================================${NC}"
|
|
write_color_output "Jellyfin Database Weekly Monitor" "$CYAN"
|
|
write_color_output "========================================${NC}"
|
|
write_color_output "Database: $DB_NAME" "$YELLOW"
|
|
write_color_output "Host: $DB_HOST" "$YELLOW"
|
|
write_color_output "Week: $(get_week_number)" "$YELLOW"
|
|
write_color_output "Date: $(date +'%Y-%m-%d %H:%M:%S')" "$YELLOW"
|
|
write_color_output "========================================\n${NC}"
|
|
|
|
# Create output directory
|
|
report_dir="./$OUTPUT_DIRECTORY"
|
|
mkdir -p "$report_dir"
|
|
write_color_output "Created report directory: $report_dir" "$GREEN"
|
|
|
|
# Generate timestamp for this run
|
|
timestamp=$(get_report_timestamp)
|
|
|
|
# Test connection
|
|
if ! test_database_connection; then
|
|
write_color_output "\n[ERROR] Cannot connect to database. Exiting." "$RED"
|
|
exit 1
|
|
fi
|
|
|
|
# Run diagnostics
|
|
diagnostics_file="$report_dir/diagnostics_$timestamp.txt"
|
|
invoke_diagnostic_report "$diagnostics_file"
|
|
|
|
# Gather index usage stats
|
|
index_usage_file="$report_dir/index_usage_$timestamp.txt"
|
|
get_index_usage_stats "$index_usage_file"
|
|
|
|
# Gather table size stats
|
|
table_sizes_file="$report_dir/table_sizes_$timestamp.txt"
|
|
get_table_size_stats "$table_sizes_file"
|
|
|
|
# Optionally vacuum/analyze tables
|
|
if [ "$AUTO_VACUUM_ANALYZE" = true ]; then
|
|
invoke_conditional_vacuum_analyze "$DEAD_TUPLE_THRESHOLD" "$DEAD_TUPLE_PERCENT_THRESHOLD"
|
|
fi
|
|
|
|
# Compare with previous week
|
|
if [ "$COMPARE_WITH_PREVIOUS" = true ]; then
|
|
compare_with_previous "$diagnostics_file" "$report_dir"
|
|
fi
|
|
|
|
# Generate summary
|
|
new_summary_report "$report_dir" "$timestamp"
|
|
|
|
# Cleanup old reports (keep last 12 weeks)
|
|
write_color_output "\nCleaning up old reports (keeping last 12 weeks)..." "$CYAN"
|
|
find "$report_dir" -type f -mtime +84 -delete
|
|
write_color_output "[OK] Cleanup complete" "$GREEN"
|
|
|
|
write_color_output "\n${GREEN}========================================${NC}"
|
|
write_color_output "[OK] WEEKLY MONITORING COMPLETE" "$GREEN"
|
|
write_color_output "========================================${NC}"
|
|
write_color_output "Reports saved to: $report_dir" "$YELLOW"
|
|
write_color_output "Summary: summary_$timestamp.txt" "$YELLOW"
|
|
write_color_output "\nNext steps:" "$CYAN"
|
|
write_color_output "1. Review summary_$timestamp.txt" "$NC"
|
|
write_color_output "2. Check for critical issues in diagnostics report" "$NC"
|
|
write_color_output "3. Analyze slow queries in query_analysis report" "$NC"
|
|
write_color_output "========================================\n${NC}"
|
|
|
|
# Email report if requested
|
|
if [ "$EMAIL_REPORT" = true ]; then
|
|
write_color_output "[WARN] Email functionality not yet implemented" "$YELLOW"
|
|
write_color_output " Configure a mail client like 'mail' or 'sendmail'" "$GRAY"
|
|
fi
|
|
|
|
exit 0 |