combined all there postgresql scripts into one to backup base, databases and clean up old archived WAL files.
Co-authored-by: Copilot <copilot@github.com>
This commit is contained in:
@@ -1,118 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ARCHIVE_DIR="/var/lib/postgresql_archive/18/main/pg_wal"
|
||||
MIN_AGE_DAYS=7
|
||||
DRY_RUN=1
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage: clean_walfiles.sh [--apply] [--archive-dir PATH] [--min-age-days N]
|
||||
|
||||
Cleans PostgreSQL archive WAL files using both rules:
|
||||
1) Candidate file mtime is older than the oldest *.backup file mtime
|
||||
2) Candidate file mtime is older than N days (default: 7)
|
||||
|
||||
Defaults to dry-run. Use --apply to actually delete files.
|
||||
EOF
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--apply)
|
||||
DRY_RUN=0
|
||||
shift
|
||||
;;
|
||||
--archive-dir)
|
||||
ARCHIVE_DIR="$2"
|
||||
shift 2
|
||||
;;
|
||||
--min-age-days)
|
||||
MIN_AGE_DAYS="$2"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown argument: $1" >&2
|
||||
usage
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ ! -d "$ARCHIVE_DIR" ]]; then
|
||||
echo "Archive directory does not exist: $ARCHIVE_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! [[ "$MIN_AGE_DAYS" =~ ^[0-9]+$ ]]; then
|
||||
echo "--min-age-days must be a non-negative integer" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
mapfile -d '' backup_files < <(find "$ARCHIVE_DIR" -maxdepth 1 -type f -name '*.backup' -print0)
|
||||
|
||||
if [[ ${#backup_files[@]} -eq 0 ]]; then
|
||||
echo "No .backup files found in $ARCHIVE_DIR; nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
oldest_backup=""
|
||||
oldest_backup_ts=""
|
||||
|
||||
for backup_file in "${backup_files[@]}"; do
|
||||
backup_ts=$(stat -c %Y "$backup_file")
|
||||
if [[ -z "$oldest_backup_ts" || "$backup_ts" -lt "$oldest_backup_ts" ]]; then
|
||||
oldest_backup_ts="$backup_ts"
|
||||
oldest_backup="$backup_file"
|
||||
fi
|
||||
done
|
||||
|
||||
age_cutoff_ts=$(date -d "$MIN_AGE_DAYS days ago" +%s)
|
||||
|
||||
echo "Archive dir : $ARCHIVE_DIR"
|
||||
echo "Oldest .backup file : $oldest_backup"
|
||||
echo "Oldest .backup mtime: $oldest_backup_ts"
|
||||
echo "Age cutoff (epoch) : $age_cutoff_ts"
|
||||
|
||||
declare -a files_to_delete=()
|
||||
|
||||
while IFS= read -r -d '' file; do
|
||||
file_ts=$(stat -c %Y "$file")
|
||||
|
||||
# Delete only when both constraints are true.
|
||||
if [[ "$file_ts" -lt "$oldest_backup_ts" && "$file_ts" -lt "$age_cutoff_ts" ]]; then
|
||||
files_to_delete+=("$file")
|
||||
fi
|
||||
done < <(find "$ARCHIVE_DIR" -maxdepth 1 -type f ! -name '*.backup' -print0)
|
||||
|
||||
if [[ ${#files_to_delete[@]} -eq 0 ]]; then
|
||||
echo "No files matched cleanup criteria."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Files matching cleanup criteria (${#files_to_delete[@]}):"
|
||||
for file in "${files_to_delete[@]}"; do
|
||||
echo " $file"
|
||||
done
|
||||
|
||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||
echo ""
|
||||
echo "Dry-run mode: no files were deleted."
|
||||
echo "Re-run with --apply to delete the files above."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
deleted=0
|
||||
for file in "${files_to_delete[@]}"; do
|
||||
rm -f -- "$file"
|
||||
((deleted += 1))
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Deleted $deleted file(s)."
|
||||
@@ -1,5 +1,117 @@
|
||||
#!/bin/bash
|
||||
# Configuration
|
||||
|
||||
# If started by root (e.g., root crontab), re-run as postgres.
|
||||
if [ "$(id -u)" -eq 0 ] && [ "$(id -un)" != "postgres" ]; then
|
||||
if command -v runuser >/dev/null 2>&1; then
|
||||
exec runuser -u postgres -- "$0" "$@"
|
||||
fi
|
||||
|
||||
# Fallback for systems without runuser.
|
||||
exec su - postgres -s /bin/bash -c "$(printf '%q ' "$0" "$@")"
|
||||
fi
|
||||
|
||||
# Avoid permission warnings when root launches this script and postgres inherits /root as cwd.
|
||||
cd / || exit 1
|
||||
|
||||
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
|
||||
|
||||
: "${WAL_CLEANUP_ENABLED:=1}"
|
||||
WAL_ARCHIVE_DIR="/var/lib/postgresql_archive/18/main/pg_wal"
|
||||
WAL_MIN_AGE_DAYS=7
|
||||
|
||||
clean_walfiles() {
|
||||
local oldest_backup=""
|
||||
local oldest_backup_ts=""
|
||||
local oldest_backup_dt=""
|
||||
local age_cutoff_ts=""
|
||||
local age_cutoff_dt=""
|
||||
local backup_ts=""
|
||||
local file_ts=""
|
||||
local deleted=0
|
||||
local backup_file=""
|
||||
local file=""
|
||||
local -a backup_files=()
|
||||
local -a files_to_delete=()
|
||||
|
||||
if [[ ! -d "$WAL_ARCHIVE_DIR" ]]; then
|
||||
log "Archive directory does not exist: $WAL_ARCHIVE_DIR" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! [[ "$WAL_MIN_AGE_DAYS" =~ ^[0-9]+$ ]]; then
|
||||
log "WAL_MIN_AGE_DAYS must be a non-negative integer" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
mapfile -d '' backup_files < <(find "$WAL_ARCHIVE_DIR" -maxdepth 1 -type f -name '*.backup' -print0)
|
||||
|
||||
if [[ ${#backup_files[@]} -eq 0 ]]; then
|
||||
log "No .backup files found in $WAL_ARCHIVE_DIR; skipping WAL cleanup."
|
||||
return 0
|
||||
fi
|
||||
|
||||
for backup_file in "${backup_files[@]}"; do
|
||||
backup_ts=$(stat -c %Y "$backup_file")
|
||||
if [[ -z "$oldest_backup_ts" || "$backup_ts" -lt "$oldest_backup_ts" ]]; then
|
||||
oldest_backup_ts="$backup_ts"
|
||||
oldest_backup="$backup_file"
|
||||
fi
|
||||
done
|
||||
|
||||
age_cutoff_ts=$(date -d "$WAL_MIN_AGE_DAYS days ago" +%s)
|
||||
oldest_backup_dt=$(date -d "@$oldest_backup_ts" '+%Y-%m-%d %H:%M:%S')
|
||||
age_cutoff_dt=$(date -d "@$age_cutoff_ts" '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
log "WAL archive dir : $WAL_ARCHIVE_DIR"
|
||||
log "Oldest .backup file : $oldest_backup"
|
||||
log "Oldest .backup mtime : $oldest_backup_dt"
|
||||
log "WAL age cutoff : $age_cutoff_dt"
|
||||
|
||||
while IFS= read -r -d '' file; do
|
||||
file_ts=$(stat -c %Y "$file")
|
||||
|
||||
# Delete only when both constraints are true.
|
||||
if [[ "$file_ts" -lt "$oldest_backup_ts" && "$file_ts" -lt "$age_cutoff_ts" ]]; then
|
||||
files_to_delete+=("$file")
|
||||
fi
|
||||
done < <(find "$WAL_ARCHIVE_DIR" -maxdepth 1 -type f ! -name '*.backup' -print0)
|
||||
|
||||
if [[ ${#files_to_delete[@]} -eq 0 ]]; then
|
||||
log "No WAL files matched cleanup criteria."
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
log "WAL files matching cleanup criteria (${#files_to_delete[@]}):"
|
||||
for file in "${files_to_delete[@]}"; do
|
||||
log " $file"
|
||||
done
|
||||
|
||||
for file in "${files_to_delete[@]}"; do
|
||||
rm -f -- "$file"
|
||||
((deleted += 1))
|
||||
done
|
||||
|
||||
echo ""
|
||||
log "Deleted $deleted WAL file(s)."
|
||||
}
|
||||
|
||||
log "Beginning base backup."
|
||||
BASEBACKUP_ROOT="/var/lib/postgresql_archive/pgbackups"
|
||||
BACKUP_DIR="/var/lib/postgresql_archive/pgbackups/basebackup"
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
# Perform base backup with WAL streaming
|
||||
pg_basebackup -h localhost -U wjones -D "$BACKUP_DIR" -Fp -Xs -P
|
||||
# Optional: Compress
|
||||
BASEBACKUP_ARCHIVE="${BACKUP_DIR}_$(date +%Y%m%d_%H%M%S).tar.gz"
|
||||
tar -czf "$BASEBACKUP_ARCHIVE" -C "$(dirname "$BACKUP_DIR")" "$(basename "$BACKUP_DIR")"
|
||||
rm -rf "$BACKUP_DIR"
|
||||
# Retention: Delete backups older than 7 days
|
||||
find "$BASEBACKUP_ROOT" -type f -name "*.tar.gz" -mtime +7 -delete
|
||||
|
||||
log "Base backup completed.Now backing up individual databases."
|
||||
|
||||
BACKUP_DIR="/var/lib/postgresql_archive/pg_dumpfiles"
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
RETENTION_DAYS=7
|
||||
@@ -10,8 +122,17 @@ mkdir -p "$BACKUP_DIR"
|
||||
# Get all non-system databases and back each one up.
|
||||
psql -U postgres -d postgres -Atc "SELECT datname FROM pg_database WHERE datistemplate = false AND datname <> 'postgres' ORDER BY datname" | while IFS= read -r DB_NAME; do
|
||||
pg_dump -U postgres -d "$DB_NAME" | gzip > "$BACKUP_DIR/${DB_NAME}_$TIMESTAMP.sql.gz"
|
||||
echo "Backup completed: ${DB_NAME}_$TIMESTAMP.sql.gz"
|
||||
log "Backup completed: ${DB_NAME}_$TIMESTAMP.sql.gz"
|
||||
done
|
||||
|
||||
# Delete backups older than retention period
|
||||
find "$BACKUP_DIR" -type f -mtime +$RETENTION_DAYS -delete
|
||||
find "$BACKUP_DIR" -type f -mtime +"$RETENTION_DAYS" -delete
|
||||
|
||||
log "Individual database backups completed and old backups cleaned up."
|
||||
if [[ "$WAL_CLEANUP_ENABLED" -eq 1 ]]; then
|
||||
log "Starting WAL archive cleanup."
|
||||
clean_walfiles
|
||||
else
|
||||
log "WAL archive cleanup disabled; skipping final cleanup."
|
||||
fi
|
||||
log "Backup workflow completed."
|
||||
@@ -1,10 +0,0 @@
|
||||
#!/bin/bash
|
||||
BACKUP_DIR="/var/lib/postgresql_archive/pgbackups/$(date +%Y%m%d)"
|
||||
mkdir -p $BACKUP_DIR
|
||||
# Perform base backup with WAL streaming
|
||||
pg_basebackup -h localhost -U wjones -D $BACKUP_DIR -Fp -Xs -P
|
||||
# Optional: Compress
|
||||
tar -czf ${BACKUP_DIR}_$(date +%Y%m%d_%H%M%S).tar.gz $BACKUP_DIR
|
||||
rm -rf $BACKUP_DIR
|
||||
# Retention: Delete backups older than 7 days
|
||||
find /var/lib/postgresql_archive/pgbackups/ -type f -mtime +7 -name "*.tar.gz" -delete
|
||||
Reference in New Issue
Block a user