89 lines
3.1 KiB
Python
89 lines
3.1 KiB
Python
#!/bin/python3
|
|
"""
|
|
Parallel rsync launcher.
|
|
|
|
Requires rsync on PATH (e.g. via MSYS/Cygwin on Windows).
|
|
Runs multiple rsync invocations in parallel (one per file/entry).
|
|
"""
|
|
|
|
import argparse
|
|
import subprocess
|
|
import sys
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from pathlib import Path
|
|
from typing import List
|
|
|
|
#DEFAULT_RSYNC_ARGS = ["-av", "--partial", "--compress", "--info=progress2"]
|
|
DEFAULT_RSYNC_ARGS = ["-avP", "--partial"]
|
|
|
|
|
|
def build_file_list(src: str) -> List[Path]:
|
|
p = Path(src)
|
|
if p.is_dir():
|
|
# sync each top-level entry separately (files and immediate subdirs)
|
|
return [x for x in p.iterdir()]
|
|
# allow glob patterns
|
|
if any(ch in src for ch in ["*", "?"]):
|
|
return list(Path(".").glob(src))
|
|
# single file
|
|
return [p]
|
|
|
|
|
|
def run_rsync(item: Path, dest: str, rsync_args: List[str], dry_run: bool) -> (Path, int, str, str):
|
|
cmd = ["rsync", *rsync_args]
|
|
if dry_run:
|
|
cmd.append("--dry-run")
|
|
# preserve relative path by sending the item name, not its parent
|
|
cmd += [str(item), dest]
|
|
try:
|
|
proc = subprocess.run(cmd, capture_output=True, text=True)
|
|
return item, proc.returncode, proc.stdout, proc.stderr
|
|
except FileNotFoundError:
|
|
return item, 127, "", "rsync not found on PATH"
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description="Transfer files using rsync in parallel.")
|
|
ap.add_argument("src", help="Source file, directory or glob pattern")
|
|
ap.add_argument("dest", help="Rsync destination (local path or user@host:/path)")
|
|
ap.add_argument("-w", "--workers", type=int, default=4, help="Parallel rsync workers")
|
|
ap.add_argument("-r", "--rsync-args", default=" ".join(DEFAULT_RSYNC_ARGS),
|
|
help="Additional rsync args (quoted). Default: -a --partial --compress --progress")
|
|
ap.add_argument("--dry-run", action="store_true", help="Don't transfer, just show what would happen")
|
|
args = ap.parse_args()
|
|
|
|
items = build_file_list(args.src)
|
|
if not items:
|
|
print("No files found for:", args.src, file=sys.stderr)
|
|
sys.exit(2)
|
|
|
|
rsync_args = args.rsync_args.split()
|
|
results = []
|
|
print(f"Starting {len(items)} rsync tasks with {args.workers} workers...")
|
|
|
|
with ThreadPoolExecutor(max_workers=args.workers) as ex:
|
|
futures = {ex.submit(run_rsync, item, args.dest, rsync_args, args.dry_run): item for item in items}
|
|
for fut in as_completed(futures):
|
|
item, code, out, err = fut.result()
|
|
results.append((item, code, out, err))
|
|
status = "OK" if code == 0 else f"ERR({code})"
|
|
print(f"[{status}] {item}")
|
|
|
|
# summary
|
|
failed = [r for r in results if r[1] != 0]
|
|
print(f"\nCompleted: {len(results)} tasks, {len(failed)} failed.")
|
|
if failed:
|
|
print("Failures detail:")
|
|
for item, code, out, err in failed:
|
|
print(f"--- {item} (exit {code}) ---")
|
|
if out:
|
|
print(out.strip())
|
|
if err:
|
|
print(err.strip())
|
|
|
|
sys.exit(1 if failed else 0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|