wjones/utilities
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor pg_backup.sh to track newest backup files and update deletion logic

Browse Source 
- Changed variable names from oldest to newest for clarity.
- Updated logic to find the newest backup file instead of the oldest.
- Adjusted logging to reflect the change in backup file tracking.
- Modified file deletion criteria to use the newest backup timestamp.

Add batch_rsync.ps1 for parallel rsync operations in PowerShell

- Implemented a PowerShell script to launch multiple rsync processes concurrently.
- Added features for auto-tuning worker count based on file size and destination type.
- Included verbose output options and support for dry runs.
- Ensured compatibility with Windows paths and rsync arguments.

Enhance pyrsync.py with recursive file handling and auto-tuning

- Added support for balanced recursive file chunking in rsync operations.
- Implemented auto-tuning of worker count based on average file size.
- Improved output handling and error reporting for rsync tasks.
- Updated command-line argument parsing for enhanced usability.

Create sync_data.ps1 for FreeFileSync job management

- Developed a PowerShell script to manage synchronization jobs using FreeFileSync.
- Implemented logging for synchronization processes and error handling.
- Added functionality to copy HTML reports generated by FreeFileSync.
- Included an option to force shutdown regardless of synchronization success.
This commit is contained in:
Wendell Jones
2026-06-03 09:12:20 -04:00
parent f637c53930
commit 3f4e1425a2
6 changed files with 911 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
personal/tools/pyrsync.py
+316 -22
View File
@@ -3,18 +3,56 @@
Parallel rsync launcher.
Requires rsync on PATH (e.g. via MSYS/Cygwin on Windows).
Runs multiple rsync invocations in parallel (one per file/entry).
Runs multiple rsync invocations in parallel using either top-level entries
or balanced recursive file chunks.
"""
# Usage examples:
#
# Basic sync (recursive-files mode, auto worker count):
# python pyrsync.py /src/dir user@host:/dest/
#
# Auto-tune workers based on file size and local vs remote destination:
# python pyrsync.py /src/dir user@host:/dest/ --auto-tune
#
# Limit to a specific number of parallel workers:
# python pyrsync.py /src/dir /local/dest/ -w 8
#
# Use top-level mode (one rsync process per immediate source entry):
# python pyrsync.py /src/dir /local/dest/ --mode top-level
#
# Dry run (show what would be transferred without doing it):
# python pyrsync.py /src/dir user@host:/dest/ --dry-run
#
# Verbose output (shows command templates, task list preview, and elapsed time):
# python pyrsync.py /src/dir /local/dest/ -v
#
# Show rsync output for all tasks:
# python pyrsync.py /src/dir /local/dest/ --show-rsync-output
#
# Show rsync output only when a task fails:
# python pyrsync.py /src/dir user@host:/dest/ --show-rsync-output-on-failure
#
# Full-featured example (auto-tune, verbose, show failures):
# python pyrsync.py /src/dir user@host:/dest/ --auto-tune -v --show-rsync-output-on-failure
#
# Override rsync flags entirely:
# python pyrsync.py /src/dir /local/dest/ -r "-az --checksum"
import argparse
import os
import re
import shlex
import subprocess
import sys
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import List
from typing import List, Sequence, Tuple
#DEFAULT_RSYNC_ARGS = ["-av", "--partial", "--compress", "--info=progress2"]
DEFAULT_RSYNC_ARGS = ["-avP", "--partial"]
# DEFAULT_RSYNC_ARGS = ["-av", "--partial", "--compress", "--info=progress2"]
DEFAULT_RSYNC_ARGS = ["-a", "--partial", "--info=stats2"]
def build_file_list(src: str) -> List[Path]:
@@ -29,7 +67,105 @@ def build_file_list(src: str) -> List[Path]:
return [p]
def run_rsync(item: Path, dest: str, rsync_args: List[str], dry_run: bool) -> (Path, int, str, str):
def build_recursive_file_list(src_dir: Path) -> List[Path]:
"""Return all files under src_dir as relative paths for --files-from."""
return [p.relative_to(src_dir) for p in src_dir.rglob("*") if p.is_file()]
def get_sized_files(src_dir: Path, rel_files: Sequence[Path]) -> List[Tuple[int, Path]]:
"""Return (size_bytes, rel_path) tuples with best-effort size lookup."""
sized_files: List[Tuple[int, Path]] = []
for rel_path in rel_files:
abs_path = src_dir / rel_path
try:
size = abs_path.stat().st_size
except OSError:
size = 0
sized_files.append((size, rel_path))
return sized_files
def partition_by_size(sized_files: Sequence[Tuple[int, Path]], workers: int) -> List[List[Path]]:
"""Greedy size-balanced partitioning to reduce worker skew."""
buckets: List[List[Path]] = [[] for _ in range(workers)]
bucket_sizes = [0] * workers
# Largest files first gives better load balancing across workers.
sorted_files = sorted(sized_files, key=lambda x: x[0], reverse=True)
for size, rel_path in sorted_files:
idx = min(range(workers), key=lambda i: bucket_sizes[i])
buckets[idx].append(rel_path)
bucket_sizes[idx] += size
return [b for b in buckets if b]
def detect_remote_destination(dest: str) -> bool:
"""Heuristic detection of rsync remote destinations."""
if dest.startswith("rsync://"):
return True
if "::" in dest:
return True
if re.match(r"^[a-zA-Z]:[\\/]", dest):
return False
if re.match(r"^\\\\", dest):
return False
return bool(re.match(r"^[^:/\\]+@?[^:/\\]*:[^/].*", dest))
def format_bytes(num_bytes: float) -> str:
units = ["B", "KB", "MB", "GB", "TB"]
value = float(num_bytes)
for unit in units:
if value < 1024.0 or unit == units[-1]:
return f"{value:.1f}{unit}"
value /= 1024.0
return f"{num_bytes:.1f}B"
def auto_tune_workers(
total_units: int,
avg_size_bytes: float,
is_remote: bool,
cpu_count: int,
configured_workers: int,
) -> int:
"""Pick worker count from workload size profile and destination type."""
cpu = max(1, cpu_count)
if avg_size_bytes < 1 * 1024 * 1024:
base = min(32, cpu * 4)
elif avg_size_bytes < 32 * 1024 * 1024:
base = min(16, cpu * 2)
elif avg_size_bytes < 256 * 1024 * 1024:
base = min(8, cpu)
else:
base = min(4, max(2, cpu // 2))
if is_remote:
base = max(2, min(base, 8))
else:
base = min(base + 2, 24)
tuned = max(1, min(base, total_units))
return min(tuned, configured_workers)
def maybe_print_task_output(label: str, out: str, err: str, show_output: bool, show_on_fail_only: bool, code: int) -> None:
if not show_output:
return
if show_on_fail_only and code == 0:
return
if out.strip():
print(f"--- stdout: {label} ---")
print(out.rstrip())
if err.strip():
print(f"--- stderr: {label} ---", file=sys.stderr)
print(err.rstrip(), file=sys.stderr)
def run_rsync(item: Path, dest: str, rsync_args: List[str], dry_run: bool) -> Tuple[Path, int, str, str]:
cmd = ["rsync", *rsync_args]
if dry_run:
cmd.append("--dry-run")
@@ -42,40 +178,198 @@ def run_rsync(item: Path, dest: str, rsync_args: List[str], dry_run: bool) -> (P
return item, 127, "", "rsync not found on PATH"
def run_rsync_files_from(
src_dir: Path,
rel_files: Sequence[Path],
dest: str,
rsync_args: List[str],
dry_run: bool,
) -> Tuple[str, int, str, str]:
cmd = ["rsync", *rsync_args]
if dry_run:
cmd.append("--dry-run")
cmd += ["--from0", "--files-from=-", f"{src_dir}{os.sep}", dest]
payload = "\0".join(str(p) for p in rel_files) + "\0"
try:
proc = subprocess.run(cmd, input=payload, capture_output=True, text=True)
label = f"chunk({len(rel_files)} files)"
return label, proc.returncode, proc.stdout, proc.stderr
except FileNotFoundError:
return "chunk(0 files)", 127, "", "rsync not found on PATH"
def main():
ap = argparse.ArgumentParser(description="Transfer files using rsync in parallel.")
ap.add_argument("src", help="Source file, directory or glob pattern")
ap.add_argument("dest", help="Rsync destination (local path or user@host:/path)")
ap.add_argument("-w", "--workers", type=int, default=4, help="Parallel rsync workers")
ap.add_argument(
"-w",
"--workers",
type=int,
default=min(8, (os.cpu_count() or 4)),
help="Parallel rsync workers (default: min(8, CPU count))",
)
ap.add_argument(
"--mode",
choices=["top-level", "recursive-files"],
default="recursive-files",
help=(
"top-level: one rsync per immediate src entry; "
"recursive-files: split full file tree into balanced rsync chunks"
),
)
ap.add_argument("-r", "--rsync-args", default=" ".join(DEFAULT_RSYNC_ARGS),
help="Additional rsync args (quoted). Default: -a --partial --compress --progress")
help="Additional rsync args (quoted). Default: -a --partial --info=stats2")
ap.add_argument(
"--auto-tune",
action="store_true",
help=(
"Auto-tune worker count from average file size and destination type. "
"Uses --workers as an upper bound."
),
)
ap.add_argument(
"-v",
"--verbose",
action="store_true",
help="Show detailed progress, including task start/completion and rsync command lines.",
)
ap.add_argument(
"--show-rsync-output",
action="store_true",
help="Print rsync stdout/stderr for each task (can be noisy).",
)
ap.add_argument(
"--show-rsync-output-on-failure",
action="store_true",
help="Print rsync stdout/stderr only for failed tasks.",
)
ap.add_argument("--dry-run", action="store_true", help="Don't transfer, just show what would happen")
args = ap.parse_args()
items = build_file_list(args.src)
if not items:
print("No files found for:", args.src, file=sys.stderr)
if args.workers < 1:
print("--workers must be >= 1", file=sys.stderr)
sys.exit(2)
rsync_args = args.rsync_args.split()
rsync_args = shlex.split(args.rsync_args)
results = []
print(f"Starting {len(items)} rsync tasks with {args.workers} workers...")
started_at = time.perf_counter()
with ThreadPoolExecutor(max_workers=args.workers) as ex:
futures = {ex.submit(run_rsync, item, args.dest, rsync_args, args.dry_run): item for item in items}
for fut in as_completed(futures):
item, code, out, err = fut.result()
results.append((item, code, out, err))
status = "OK" if code == 0 else f"ERR({code})"
print(f"[{status}] {item}")
if args.show_rsync_output and args.show_rsync_output_on_failure:
print(
"Both --show-rsync-output and --show-rsync-output-on-failure set; "
"showing output for all tasks.",
file=sys.stderr,
)
show_output = args.show_rsync_output or args.show_rsync_output_on_failure
show_on_fail_only = args.show_rsync_output_on_failure and not args.show_rsync_output
src_path = Path(args.src)
is_remote_dest = detect_remote_destination(args.dest)
cpu_count = os.cpu_count() or 4
if args.mode == "recursive-files" and src_path.is_dir():
rel_files = build_recursive_file_list(src_path)
if not rel_files:
print("No files found for:", args.src, file=sys.stderr)
sys.exit(2)
sized_files = get_sized_files(src_path, rel_files)
avg_size = sum(size for size, _ in sized_files) / max(1, len(sized_files))
if args.auto_tune:
workers = auto_tune_workers(len(rel_files), avg_size, is_remote_dest, cpu_count, args.workers)
dest_kind = "remote" if is_remote_dest else "local"
print(
f"Auto-tuned workers={workers} ({dest_kind} dest, avg file {format_bytes(avg_size)})"
)
else:
workers = min(args.workers, len(rel_files))
chunks = partition_by_size(sized_files, workers)
print(f"Starting {len(chunks)} rsync chunk task(s) with {workers} worker(s)...")
if args.verbose:
cmd_preview = ["rsync", *rsync_args]
if args.dry_run:
cmd_preview.append("--dry-run")
cmd_preview += ["--from0", "--files-from=-", f"{src_path}{os.sep}", args.dest]
print("Command template:", " ".join(shlex.quote(part) for part in cmd_preview))
with ThreadPoolExecutor(max_workers=workers) as ex:
futures = {
ex.submit(run_rsync_files_from, src_path, chunk, args.dest, rsync_args, args.dry_run): chunk
for chunk in chunks
}
for fut in as_completed(futures):
label, code, out, err = fut.result()
results.append((label, code, out, err))
status = "OK" if code == 0 else f"ERR({code})"
print(f"[{status}] {label}")
maybe_print_task_output(label, out, err, show_output, show_on_fail_only, code)
else:
items = build_file_list(args.src)
if not items:
print("No files found for:", args.src, file=sys.stderr)
sys.exit(2)
size_samples = []
for item in items:
try:
if item.is_file():
size_samples.append(item.stat().st_size)
else:
size_samples.append(64 * 1024 * 1024)
except OSError:
size_samples.append(0)
avg_size = sum(size_samples) / max(1, len(size_samples))
if args.auto_tune:
workers = auto_tune_workers(len(items), avg_size, is_remote_dest, cpu_count, args.workers)
dest_kind = "remote" if is_remote_dest else "local"
print(
f"Auto-tuned workers={workers} ({dest_kind} dest, avg unit {format_bytes(avg_size)})"
)
else:
workers = min(args.workers, len(items))
print(f"Starting {len(items)} rsync task(s) with {workers} worker(s)...")
if args.verbose:
print("Task list preview:")
preview_limit = 10
for idx, item in enumerate(items[:preview_limit], start=1):
print(f" {idx}. {item}")
if len(items) > preview_limit:
print(f" ... and {len(items) - preview_limit} more")
with ThreadPoolExecutor(max_workers=workers) as ex:
futures = {}
for item in items:
if args.verbose:
cmd_preview = ["rsync", *rsync_args]
if args.dry_run:
cmd_preview.append("--dry-run")
cmd_preview += [str(item), args.dest]
print("Scheduling:", " ".join(shlex.quote(part) for part in cmd_preview))
futures[ex.submit(run_rsync, item, args.dest, rsync_args, args.dry_run)] = item
for fut in as_completed(futures):
item, code, out, err = fut.result()
results.append((str(item), code, out, err))
status = "OK" if code == 0 else f"ERR({code})"
print(f"[{status}] {item}")
maybe_print_task_output(str(item), out, err, show_output, show_on_fail_only, code)
# summary
failed = [r for r in results if r[1] != 0]
print(f"\nCompleted: {len(results)} tasks, {len(failed)} failed.")
elapsed = time.perf_counter() - started_at
print(f"\nCompleted: {len(results)} tasks, {len(failed)} failed in {elapsed:.2f}s.")
if failed:
print("Failures detail:")
for item, code, out, err in failed:
print(f"--- {item} (exit {code}) ---")
for label, code, out, err in failed:
print(f"--- {label} (exit {code}) ---")
if out:
print(out.strip())
if err: