Refactor pg_backup.sh to track newest backup files and update deletion logic
- Changed variable names from oldest to newest for clarity. - Updated logic to find the newest backup file instead of the oldest. - Adjusted logging to reflect the change in backup file tracking. - Modified file deletion criteria to use the newest backup timestamp. Add batch_rsync.ps1 for parallel rsync operations in PowerShell - Implemented a PowerShell script to launch multiple rsync processes concurrently. - Added features for auto-tuning worker count based on file size and destination type. - Included verbose output options and support for dry runs. - Ensured compatibility with Windows paths and rsync arguments. Enhance pyrsync.py with recursive file handling and auto-tuning - Added support for balanced recursive file chunking in rsync operations. - Implemented auto-tuning of worker count based on average file size. - Improved output handling and error reporting for rsync tasks. - Updated command-line argument parsing for enhanced usability. Create sync_data.ps1 for FreeFileSync job management - Developed a PowerShell script to manage synchronization jobs using FreeFileSync. - Implemented logging for synchronization processes and error handling. - Added functionality to copy HTML reports generated by FreeFileSync. - Included an option to force shutdown regardless of synchronization success.
This commit is contained in:
@@ -21,9 +21,9 @@ WAL_ARCHIVE_DIR="/var/lib/postgresql_archive/18/main/pg_wal"
|
||||
WAL_MIN_AGE_DAYS=7
|
||||
|
||||
clean_walfiles() {
|
||||
local oldest_backup=""
|
||||
local oldest_backup_ts=""
|
||||
local oldest_backup_dt=""
|
||||
local newest_backup=""
|
||||
local newest_backup_ts=""
|
||||
local newest_backup_dt=""
|
||||
local age_cutoff_ts=""
|
||||
local age_cutoff_dt=""
|
||||
local backup_ts=""
|
||||
@@ -53,26 +53,26 @@ clean_walfiles() {
|
||||
|
||||
for backup_file in "${backup_files[@]}"; do
|
||||
backup_ts=$(stat -c %Y "$backup_file")
|
||||
if [[ -z "$oldest_backup_ts" || "$backup_ts" -lt "$oldest_backup_ts" ]]; then
|
||||
oldest_backup_ts="$backup_ts"
|
||||
oldest_backup="$backup_file"
|
||||
if [[ -z "$newest_backup_ts" || "$backup_ts" -gt "$newest_backup_ts" ]]; then
|
||||
newest_backup_ts="$backup_ts"
|
||||
newest_backup="$backup_file"
|
||||
fi
|
||||
done
|
||||
|
||||
age_cutoff_ts=$(date -d "$WAL_MIN_AGE_DAYS days ago" +%s)
|
||||
oldest_backup_dt=$(date -d "@$oldest_backup_ts" '+%Y-%m-%d %H:%M:%S')
|
||||
newest_backup_dt=$(date -d "@$newest_backup_ts" '+%Y-%m-%d %H:%M:%S')
|
||||
age_cutoff_dt=$(date -d "@$age_cutoff_ts" '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
log "WAL archive dir : $WAL_ARCHIVE_DIR"
|
||||
log "Oldest .backup file : $oldest_backup"
|
||||
log "Oldest .backup mtime : $oldest_backup_dt"
|
||||
log "Newest .backup file : $newest_backup"
|
||||
log "Newest .backup mtime : $newest_backup_dt"
|
||||
log "WAL age cutoff : $age_cutoff_dt"
|
||||
|
||||
while IFS= read -r -d '' file; do
|
||||
file_ts=$(stat -c %Y "$file")
|
||||
|
||||
# Delete only when both constraints are true.
|
||||
if [[ "$file_ts" -lt "$oldest_backup_ts" && "$file_ts" -lt "$age_cutoff_ts" ]]; then
|
||||
if [[ "$file_ts" -lt "$newest_backup_ts" && "$file_ts" -lt "$age_cutoff_ts" ]]; then
|
||||
files_to_delete+=("$file")
|
||||
fi
|
||||
done < <(find "$WAL_ARCHIVE_DIR" -maxdepth 1 -type f ! -name '*.backup' -print0)
|
||||
|
||||
@@ -1,105 +0,0 @@
|
||||
#Requires -Version 5.1
|
||||
|
||||
|
||||
# Use -ShutdownRegardless to force shutdown even if one or more sync jobs fail.
|
||||
param(
|
||||
[switch]$ShutdownRegardless
|
||||
)
|
||||
|
||||
$FfsExe = 'C:\Program Files\FreeFileSync\FreeFileSync.exe'
|
||||
$ScriptDir = $PSScriptRoot
|
||||
$LogDir = Join-Path $ScriptDir 'logs'
|
||||
$ReportDir = Join-Path $ScriptDir 'reports'
|
||||
|
||||
$SyncJobs = @(
|
||||
'D:\jobs\shuttlebay_SyncSettings.ffs_batch'
|
||||
'D:\jobs\main_bridge_SyncSettings.ffs_batch'
|
||||
'D:\jobs\engineering_SyncSettings.ffs_batch'
|
||||
)
|
||||
|
||||
# ── Setup ──────────────────────────────────────────────────────────────────────
|
||||
foreach ($dir in $LogDir, $ReportDir) {
|
||||
if (-not (Test-Path $dir)) { New-Item -ItemType Directory -Path $dir | Out-Null }
|
||||
}
|
||||
|
||||
$RunTs = Get-Date -Format 'yyyyMMdd_HHmmss'
|
||||
$LogFile = Join-Path $LogDir "sync_history_$RunTs.log"
|
||||
|
||||
function Write-Log {
|
||||
param([string]$Message)
|
||||
$line = "[$(Get-Date -Format 'ddd MM/dd/yyyy HH:mm:ss.ff')] $Message"
|
||||
Write-Host $line
|
||||
Add-Content -Path $LogFile -Value $line
|
||||
}
|
||||
|
||||
Write-Log 'Sync run started'
|
||||
Write-Host "Log file: $LogFile"
|
||||
|
||||
# ── Run sync jobs ──────────────────────────────────────────────────────────────
|
||||
$HadFatalError = $false
|
||||
$FatalExitCodes = @()
|
||||
|
||||
foreach ($job in $SyncJobs) {
|
||||
$jobName = Split-Path $job -Leaf
|
||||
Write-Host ''
|
||||
Write-Log "Running synchronization: $jobName"
|
||||
|
||||
$tmpOut = [IO.Path]::GetTempFileName()
|
||||
$proc = Start-Process -FilePath $FfsExe -ArgumentList $job `
|
||||
-RedirectStandardOutput $tmpOut -NoNewWindow -PassThru -Wait
|
||||
$rc = $proc.ExitCode
|
||||
$output = Get-Content -Path $tmpOut -Raw
|
||||
Remove-Item $tmpOut -Force
|
||||
if ($output) { Add-Content -Path $LogFile -Value $output }
|
||||
|
||||
switch ($rc) {
|
||||
0 { Write-Log 'Synchronization completed successfully.' }
|
||||
1 { Write-Log 'Synchronization completed with warnings.' }
|
||||
default {
|
||||
Write-Log "Synchronization failed or was aborted. Exit code: $rc"
|
||||
$HadFatalError = $true
|
||||
$FatalExitCodes += $rc
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host ''
|
||||
Write-Log 'All synchronization jobs completed.'
|
||||
|
||||
# ── Copy HTML reports ──────────────────────────────────────────────────────────
|
||||
$logPaths = Select-String -Path $LogFile -Pattern '"logFile"\s*:\s*"([^"]+)"' |
|
||||
ForEach-Object { $_.Matches[0].Groups[1].Value -replace '\\\\', '\' } |
|
||||
Sort-Object -Unique
|
||||
|
||||
$copied = 0
|
||||
foreach ($path in $logPaths) {
|
||||
if ([IO.Path]::GetExtension($path) -ieq '.html') {
|
||||
if (Test-Path -LiteralPath $path) {
|
||||
Copy-Item -LiteralPath $path -Destination $ReportDir -Force
|
||||
Write-Log "Copied report: $(Split-Path $path -Leaf)"
|
||||
$copied++
|
||||
} else {
|
||||
Write-Log "Report not found: $path"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Write-Log "Total reports copied: $copied"
|
||||
|
||||
# ── Shutdown ───────────────────────────────────────────────────────────────────
|
||||
if ($HadFatalError) {
|
||||
$codes = ($FatalExitCodes | Sort-Object -Unique) -join ', '
|
||||
Write-Log "One or more synchronization jobs failed. Fatal exit code(s): $codes"
|
||||
|
||||
if (-not $ShutdownRegardless) {
|
||||
Write-Log 'Skipping shutdown because not all synchronization jobs completed successfully.'
|
||||
Write-Log 'Use -ShutdownRegardless to force shutdown even when jobs fail.'
|
||||
exit ($FatalExitCodes | Select-Object -Last 1)
|
||||
}
|
||||
|
||||
Write-Log 'Forced shutdown enabled. Continuing to shutdown despite synchronization failures.'
|
||||
}
|
||||
|
||||
Write-Log 'Shutting down server now.'
|
||||
Stop-Computer -Force
|
||||
Reference in New Issue
Block a user