diff --git a/personal/cron_scripts/clean_walfiles.sh b/personal/cron_scripts/clean_walfiles.sh deleted file mode 100644 index 9967925..0000000 --- a/personal/cron_scripts/clean_walfiles.sh +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -ARCHIVE_DIR="/var/lib/postgresql_archive/18/main/pg_wal" -MIN_AGE_DAYS=7 -DRY_RUN=1 - -usage() { - cat <<'EOF' -Usage: clean_walfiles.sh [--apply] [--archive-dir PATH] [--min-age-days N] - -Cleans PostgreSQL archive WAL files using both rules: -1) Candidate file mtime is older than the oldest *.backup file mtime -2) Candidate file mtime is older than N days (default: 7) - -Defaults to dry-run. Use --apply to actually delete files. -EOF -} - -while [[ $# -gt 0 ]]; do - case "$1" in - --apply) - DRY_RUN=0 - shift - ;; - --archive-dir) - ARCHIVE_DIR="$2" - shift 2 - ;; - --min-age-days) - MIN_AGE_DAYS="$2" - shift 2 - ;; - -h|--help) - usage - exit 0 - ;; - *) - echo "Unknown argument: $1" >&2 - usage - exit 2 - ;; - esac -done - -if [[ ! -d "$ARCHIVE_DIR" ]]; then - echo "Archive directory does not exist: $ARCHIVE_DIR" >&2 - exit 1 -fi - -if ! [[ "$MIN_AGE_DAYS" =~ ^[0-9]+$ ]]; then - echo "--min-age-days must be a non-negative integer" >&2 - exit 2 -fi - -mapfile -d '' backup_files < <(find "$ARCHIVE_DIR" -maxdepth 1 -type f -name '*.backup' -print0) - -if [[ ${#backup_files[@]} -eq 0 ]]; then - echo "No .backup files found in $ARCHIVE_DIR; nothing to do." - exit 0 -fi - -oldest_backup="" -oldest_backup_ts="" - -for backup_file in "${backup_files[@]}"; do - backup_ts=$(stat -c %Y "$backup_file") - if [[ -z "$oldest_backup_ts" || "$backup_ts" -lt "$oldest_backup_ts" ]]; then - oldest_backup_ts="$backup_ts" - oldest_backup="$backup_file" - fi -done - -age_cutoff_ts=$(date -d "$MIN_AGE_DAYS days ago" +%s) - -echo "Archive dir : $ARCHIVE_DIR" -echo "Oldest .backup file : $oldest_backup" -echo "Oldest .backup mtime: $oldest_backup_ts" -echo "Age cutoff (epoch) : $age_cutoff_ts" - -declare -a files_to_delete=() - -while IFS= read -r -d '' file; do - file_ts=$(stat -c %Y "$file") - - # Delete only when both constraints are true. - if [[ "$file_ts" -lt "$oldest_backup_ts" && "$file_ts" -lt "$age_cutoff_ts" ]]; then - files_to_delete+=("$file") - fi -done < <(find "$ARCHIVE_DIR" -maxdepth 1 -type f ! -name '*.backup' -print0) - -if [[ ${#files_to_delete[@]} -eq 0 ]]; then - echo "No files matched cleanup criteria." - exit 0 -fi - -echo "" -echo "Files matching cleanup criteria (${#files_to_delete[@]}):" -for file in "${files_to_delete[@]}"; do - echo " $file" -done - -if [[ "$DRY_RUN" -eq 1 ]]; then - echo "" - echo "Dry-run mode: no files were deleted." - echo "Re-run with --apply to delete the files above." - exit 0 -fi - -deleted=0 -for file in "${files_to_delete[@]}"; do - rm -f -- "$file" - ((deleted += 1)) -done - -echo "" -echo "Deleted $deleted file(s)." diff --git a/personal/cron_scripts/pg_backup.sh b/personal/cron_scripts/pg_backup.sh index d1a1e8e..ac4485e 100644 --- a/personal/cron_scripts/pg_backup.sh +++ b/personal/cron_scripts/pg_backup.sh @@ -1,5 +1,117 @@ #!/bin/bash # Configuration + +# If started by root (e.g., root crontab), re-run as postgres. +if [ "$(id -u)" -eq 0 ] && [ "$(id -un)" != "postgres" ]; then + if command -v runuser >/dev/null 2>&1; then + exec runuser -u postgres -- "$0" "$@" + fi + + # Fallback for systems without runuser. + exec su - postgres -s /bin/bash -c "$(printf '%q ' "$0" "$@")" +fi + +# Avoid permission warnings when root launches this script and postgres inherits /root as cwd. +cd / || exit 1 + +log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; } + +: "${WAL_CLEANUP_ENABLED:=1}" +WAL_ARCHIVE_DIR="/var/lib/postgresql_archive/18/main/pg_wal" +WAL_MIN_AGE_DAYS=7 + +clean_walfiles() { + local oldest_backup="" + local oldest_backup_ts="" + local oldest_backup_dt="" + local age_cutoff_ts="" + local age_cutoff_dt="" + local backup_ts="" + local file_ts="" + local deleted=0 + local backup_file="" + local file="" + local -a backup_files=() + local -a files_to_delete=() + + if [[ ! -d "$WAL_ARCHIVE_DIR" ]]; then + log "Archive directory does not exist: $WAL_ARCHIVE_DIR" >&2 + return 1 + fi + + if ! [[ "$WAL_MIN_AGE_DAYS" =~ ^[0-9]+$ ]]; then + log "WAL_MIN_AGE_DAYS must be a non-negative integer" >&2 + return 1 + fi + + mapfile -d '' backup_files < <(find "$WAL_ARCHIVE_DIR" -maxdepth 1 -type f -name '*.backup' -print0) + + if [[ ${#backup_files[@]} -eq 0 ]]; then + log "No .backup files found in $WAL_ARCHIVE_DIR; skipping WAL cleanup." + return 0 + fi + + for backup_file in "${backup_files[@]}"; do + backup_ts=$(stat -c %Y "$backup_file") + if [[ -z "$oldest_backup_ts" || "$backup_ts" -lt "$oldest_backup_ts" ]]; then + oldest_backup_ts="$backup_ts" + oldest_backup="$backup_file" + fi + done + + age_cutoff_ts=$(date -d "$WAL_MIN_AGE_DAYS days ago" +%s) + oldest_backup_dt=$(date -d "@$oldest_backup_ts" '+%Y-%m-%d %H:%M:%S') + age_cutoff_dt=$(date -d "@$age_cutoff_ts" '+%Y-%m-%d %H:%M:%S') + + log "WAL archive dir : $WAL_ARCHIVE_DIR" + log "Oldest .backup file : $oldest_backup" + log "Oldest .backup mtime : $oldest_backup_dt" + log "WAL age cutoff : $age_cutoff_dt" + + while IFS= read -r -d '' file; do + file_ts=$(stat -c %Y "$file") + + # Delete only when both constraints are true. + if [[ "$file_ts" -lt "$oldest_backup_ts" && "$file_ts" -lt "$age_cutoff_ts" ]]; then + files_to_delete+=("$file") + fi + done < <(find "$WAL_ARCHIVE_DIR" -maxdepth 1 -type f ! -name '*.backup' -print0) + + if [[ ${#files_to_delete[@]} -eq 0 ]]; then + log "No WAL files matched cleanup criteria." + return 0 + fi + + echo "" + log "WAL files matching cleanup criteria (${#files_to_delete[@]}):" + for file in "${files_to_delete[@]}"; do + log " $file" + done + + for file in "${files_to_delete[@]}"; do + rm -f -- "$file" + ((deleted += 1)) + done + + echo "" + log "Deleted $deleted WAL file(s)." +} + +log "Beginning base backup." +BASEBACKUP_ROOT="/var/lib/postgresql_archive/pgbackups" +BACKUP_DIR="/var/lib/postgresql_archive/pgbackups/basebackup" +mkdir -p "$BACKUP_DIR" +# Perform base backup with WAL streaming +pg_basebackup -h localhost -U wjones -D "$BACKUP_DIR" -Fp -Xs -P +# Optional: Compress +BASEBACKUP_ARCHIVE="${BACKUP_DIR}_$(date +%Y%m%d_%H%M%S).tar.gz" +tar -czf "$BASEBACKUP_ARCHIVE" -C "$(dirname "$BACKUP_DIR")" "$(basename "$BACKUP_DIR")" +rm -rf "$BACKUP_DIR" +# Retention: Delete backups older than 7 days +find "$BASEBACKUP_ROOT" -type f -name "*.tar.gz" -mtime +7 -delete + +log "Base backup completed.Now backing up individual databases." + BACKUP_DIR="/var/lib/postgresql_archive/pg_dumpfiles" TIMESTAMP=$(date +%Y%m%d_%H%M%S) RETENTION_DAYS=7 @@ -10,8 +122,17 @@ mkdir -p "$BACKUP_DIR" # Get all non-system databases and back each one up. psql -U postgres -d postgres -Atc "SELECT datname FROM pg_database WHERE datistemplate = false AND datname <> 'postgres' ORDER BY datname" | while IFS= read -r DB_NAME; do pg_dump -U postgres -d "$DB_NAME" | gzip > "$BACKUP_DIR/${DB_NAME}_$TIMESTAMP.sql.gz" - echo "Backup completed: ${DB_NAME}_$TIMESTAMP.sql.gz" + log "Backup completed: ${DB_NAME}_$TIMESTAMP.sql.gz" done # Delete backups older than retention period -find "$BACKUP_DIR" -type f -mtime +$RETENTION_DAYS -delete \ No newline at end of file +find "$BACKUP_DIR" -type f -mtime +"$RETENTION_DAYS" -delete + +log "Individual database backups completed and old backups cleaned up." +if [[ "$WAL_CLEANUP_ENABLED" -eq 1 ]]; then + log "Starting WAL archive cleanup." + clean_walfiles +else + log "WAL archive cleanup disabled; skipping final cleanup." +fi +log "Backup workflow completed." \ No newline at end of file diff --git a/personal/cron_scripts/pg_basebackup.sh b/personal/cron_scripts/pg_basebackup.sh deleted file mode 100644 index 2180120..0000000 --- a/personal/cron_scripts/pg_basebackup.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -BACKUP_DIR="/var/lib/postgresql_archive/pgbackups/$(date +%Y%m%d)" -mkdir -p $BACKUP_DIR -# Perform base backup with WAL streaming -pg_basebackup -h localhost -U wjones -D $BACKUP_DIR -Fp -Xs -P -# Optional: Compress -tar -czf ${BACKUP_DIR}_$(date +%Y%m%d_%H%M%S).tar.gz $BACKUP_DIR -rm -rf $BACKUP_DIR -# Retention: Delete backups older than 7 days -find /var/lib/postgresql_archive/pgbackups/ -type f -mtime +7 -name "*.tar.gz" -delete