wjones/utilities
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

merge into: wjones/utilities:main
Branches Tags
wjones/utilities:main wjones/utilities:development
..
pull from: wjones/utilities:development
Branches Tags
wjones/utilities:main wjones/utilities:development

2 Commits
main .. development

Author SHA1 Message Date
wjones 0b843113b6 Add bash script to start containers on boot as python script is not executing 2025-07-07 09:36:14 -04:00
wjones d2965d4439 update execution for vm shutdown 2025-07-03 16:01:30 -04:00
56 changed files with 63 additions and 10042 deletions
Expand all files Collapse all files
.gitignore
-26
View File
@@ -1,26 +0,0 @@
# PyInstaller
build/
dist/
*.spec
# Python
__pycache__/
*.py[cod]
*.pyo
*.pyd
.Python
*.egg-info/
.eggs/
# Virtual environments
.venv/
venv/
env/
# Credentials (never commit these)
credentials
*.credentials
# OS
.DS_Store
Thumbs.db
build_all.py
-41
View File
@@ -1,41 +0,0 @@
#!/usr/bin/env python3
import subprocess
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent
CRON_SCRIPTS = REPO_ROOT / "personal" / "cron_scripts"
BUILD_SCRIPTS = [
CRON_SCRIPTS / "build_multi_powercycle.py",
CRON_SCRIPTS / "build_set_fanspeed.py",
CRON_SCRIPTS / "build_ilo_status.py",
CRON_SCRIPTS / "build_ilo_powercycle.py",
CRON_SCRIPTS / "build_ipmi_poweron.py",
CRON_SCRIPTS / "build_ipmi_poweroff.py",
]
def main():
failures = []
for script in BUILD_SCRIPTS:
print(f"\n{'='*60}")
print(f"Building: {script.name}")
print(f"{'='*60}")
result = subprocess.run([sys.executable, str(script)], cwd=REPO_ROOT)
if result.returncode != 0:
failures.append(script.name)
print(f"\n{'='*60}")
if failures:
print(f"FAILED builds ({len(failures)}/{len(BUILD_SCRIPTS)}):")
for name in failures:
print(f" - {name}")
raise SystemExit(1)
else:
print(f"All {len(BUILD_SCRIPTS)} builds completed successfully.")
print(f"Executables are in: {REPO_ROOT / 'dist'}")
if __name__ == "__main__":
main()
personal/configs/cron/credentials.ini
-13
View File
@@ -1,13 +0,0 @@
[credentials]
username=<username>
password=<password>
domain=<domain>
[ipmi_credentials]
username=<username>
password=<password>
IPMIEK=0000000000000000000000000000000000000000
[ilo_credentials]
username=<username>
password=<password>
personal/configs/cron/serverlist.ini
-6
View File
@@ -1,6 +0,0 @@
[ipmi]
#list of IPMI hosts to power cycle
[ilo]
#list of iLO hosts to power cycle
personal/configs/cron/set_password.py
-149
View File
@@ -1,149 +0,0 @@
#!/usr/bin/env python3
"""
set_password.py — Encrypt and store a password in any section of credentials.ini
Usage:
python set_password.py # interactive prompts
python set_password.py -s ipmi_credentials # pre-select section
python set_password.py --credentials /path/to/credentials.ini
python set_password.py --keyfile /path/to/secret.key
The first run generates a secret.key file next to credentials.ini.
Keep that key file secure (chmod 600); without it, passwords cannot be decrypted.
Decrypting at runtime (in other scripts):
from cryptography.fernet import Fernet
key = open("secret.key", "rb").read()
fernet = Fernet(key)
plaintext = fernet.decrypt(encrypted_value.encode()).decode()
"""
import argparse
import configparser
import getpass
import os
import sys
try:
from cryptography.fernet import Fernet
except ImportError:
sys.exit("Error: 'cryptography' package is required. Run: pip install cryptography")
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
DEFAULT_CREDENTIALS = os.path.join(SCRIPT_DIR, "credentials.ini")
DEFAULT_KEYFILE = os.path.join(SCRIPT_DIR, "secret.key")
def load_or_create_key(keyfile: str) -> bytes:
"""Load an existing Fernet key or generate and save a new one."""
if os.path.exists(keyfile):
with open(keyfile, "rb") as fh:
return fh.read()
key = Fernet.generate_key()
with open(keyfile, "wb") as fh:
fh.write(key)
# Restrict permissions on Unix-like systems
try:
os.chmod(keyfile, 0o600)
except OSError:
pass
print(f"New encryption key created: {keyfile}")
print("Keep this file secure — it is required to decrypt passwords.")
return key
def encrypt_password(plaintext: str, fernet: Fernet) -> str:
return fernet.encrypt(plaintext.encode()).decode()
def main():
parser = argparse.ArgumentParser(
description="Encrypt and store a password in credentials.ini"
)
parser.add_argument(
"--credentials",
default=DEFAULT_CREDENTIALS,
help=f"Path to credentials file (default: {DEFAULT_CREDENTIALS})",
)
parser.add_argument(
"--keyfile",
default=DEFAULT_KEYFILE,
help=f"Path to encryption key file (default: {DEFAULT_KEYFILE})",
)
parser.add_argument(
"-s",
"--section",
help="Section to update (skips interactive section prompt)",
)
args = parser.parse_args()
config = configparser.ConfigParser()
if not os.path.exists(args.credentials):
print(f"Credentials file not found. Creating: {args.credentials}")
os.makedirs(os.path.dirname(os.path.abspath(args.credentials)), exist_ok=True)
# Seed with the standard sections and placeholder values
config["credentials"] = {"username": "", "password": "", "domain": ""}
config["ipmi_credentials"] = {
"username": "",
"password": "",
"IPMIEK": "0000000000000000000000000000000000000000",
}
config["ilo_credentials"] = {"username": "", "password": ""}
with open(args.credentials, "w") as fh:
config.write(fh)
try:
os.chmod(args.credentials, 0o600)
except OSError:
pass
print(f"Created {args.credentials} — fill in usernames and re-run to set passwords.")
else:
config.read(args.credentials)
sections = config.sections()
if not sections:
sys.exit(f"Error: no sections found in {args.credentials}")
# Resolve section
if args.section:
section = args.section
if section not in sections:
sys.exit(
f"Error: section [{section}] not found in {args.credentials}\n"
f"Available sections: {', '.join(sections)}"
)
else:
print("Available sections:")
for i, s in enumerate(sections, 1):
print(f" {i}. {s}")
choice = input("Select section number: ").strip()
try:
section = sections[int(choice) - 1]
except (ValueError, IndexError):
sys.exit("Error: invalid selection")
# Prompt for new password (hidden input, confirmed)
while True:
password = getpass.getpass(f"New password for [{section}]: ")
confirm = getpass.getpass("Confirm password: ")
if password == confirm:
break
print("Passwords do not match. Try again.")
# Load/create key and encrypt
key = load_or_create_key(args.keyfile)
fernet = Fernet(key)
encrypted = encrypt_password(password, fernet)
config.set(section, "password", encrypted)
with open(args.credentials, "w") as fh:
config.write(fh)
print(f"Password updated and encrypted in [{section}] of {args.credentials}")
if __name__ == "__main__":
main()
personal/configs/frigate/config.yml
-642
View File
@@ -1,642 +0,0 @@
mqtt:
enabled: false
detectors:
coral:
type: edgetpu
device: pci
ffmpeg:
hwaccel_args: preset-nvidia
record:
enabled: true
alerts:
retain:
days: 30
mode: motion
detections:
retain:
days: 30
mode: motion
continuous:
days: 0
motion:
days: 3
go2rtc:
ffmpeg:
bin: ffmpeg
# Nvidia NVDEC hardware decode templates (used when #hardware is in stream URL)
h264: "-hwaccel cuda -hwaccel_output_format cuda -c:v h264_cuvid {input} -c:v copy -f rtsp {output}"
h265: "-hwaccel cuda -hwaccel_output_format cuda -c:v hevc_cuvid {input} -c:v copy -f rtsp {output}"
streams:
# Backbasement:
# - ffmpeg:rtsp://tpadmin:Optimus0329@192.168.128.222:554/live/ch0#video=copy#audio=copy#audio=aac#hardware
# SubBackBasement:
# - ffmpeg:rtsp://tpadmin:Optimus0329@192.168.128.222:554/live/ch1#video=copy#audio=copy#audio=aac#hardware
# FrontBasement:
# - ffmpeg:rtsp://tpadmin:Optimus0329@192.168.128.223:554/live/ch0?token=e6cf2225e8bf5fb028543898fab9a545#video=copy#audio=copy#audio=aac#hardware
# SubFrontBasement:
# - ffmpeg:rtsp://tpadmin:Optimus0329@192.168.128.223:554/live/ch1?token=e6cf2225e8bf5fb028543898fab9a545#video=copy#audio=copy#audio=aac#hardware
# Eningeering:
# - ffmpeg:rtsp://tpadmin:Optimus0329@192.168.128.221:554/live/ch0?token=e6cf2225e8bf5fb028543898fab9a545#video=copy#audio=copy#audio=aac#hardware
# SubEngineering:
# - ffmpeg:rtsp://tpadmin:Optimus0329@192.168.128.221:554/live/ch1?token=e6cf2225e8bf5fb028543898fab9a545#video=copy#audio=copy#audio=aac#hardware
Driveway:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch12_0.264#video=copy#audio=copy#audio=aac#hardware
SubDriveway:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch12_1.264#video=copy#audio=copy#audio=aac#hardware
Porch:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch8_0.264#video=copy#audio=copy#audio=aac#hardware
SubPorch:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch8_1.264#video=copy#audio=copy#audio=aac#hardware
FenceGate:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch1_0.264#video=copy#audio=copy#audio=aac#hardware
SubFenceGate:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch1_1.264#video=copy#audio=copy#audio=aac#hardware
GardenArea:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch2_0.264#video=copy#audio=copy#audio=aac#hardware
SubGardenArea:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch2_1.264#video=copy#audio=copy#audio=aac#hardware
Backyard1:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch3_0.264#video=copy#audio=copy#audio=aac#hardware
SubBackyard1:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch3_1.264#video=copy#audio=copy#audio=aac#hardware
Backyard2:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch4_0.264#video=copy#audio=copy#audio=aac#hardware
SubBackyard2:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch4_1.264#video=copy#audio=copy#audio=aac#hardware
Backyard3:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch0_0.264#video=copy#audio=copy#audio=aac#hardware
SubBackyard3:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch0_1.264#video=copy#audio=copy#audio=aac#hardware
Backyard4:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch10_0.264#video=copy#audio=copy#audio=aac#hardware
SubBackyard4:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch10_1.264#video=copy#audio=copy#audio=aac#hardware
Backyard5:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch7_0.264#video=copy#audio=copy#audio=aac#hardware
SubBackyard5:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch7_1.264#video=copy#audio=copy#audio=aac#hardware
PatioArea:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch6_0.264#video=copy#audio=copy#audio=aac#hardware
SubPatioArea:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch6_1.264#video=copy#audio=copy#audio=aac#hardware
PoolArea:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch9_0.264#video=copy#audio=copy#audio=aac#hardware
SubPoolArea:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch9_1.264#video=copy#audio=copy#audio=aac#hardware
SideHouse:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch11_0.264#video=copy#audio=copy#audio=aac#hardware
SubSideHouse:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch11_1.264#video=copy#audio=copy#audio=aac#hardware
BackSlider:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch5_0.264#video=copy#audio=copy#audio=aac#hardware
SubBackSlider:
- ffmpeg:rtsp://admin:Optimus0329@192.168.128.11:80/ch5_1.264#video=copy#audio=copy#audio=aac#hardware
cameras:
# name_of_your_camera: # <------ Name the camera
# enabled: True
# ffmpeg:
# inputs:
# - path: rtsp://10.0.10.10:554/rtsp # <----- The stream you want to use for detection
# roles:
# - detect
# detect:
# enabled: False # <---- disable detection until you have a working camera feed
# width: 1280
# height: 720
# back_basement:
# enabled: true
# ffmpeg:
# inputs:
# # - path: rtsp://tpadmin:Optimus0329@192.168.128.225/stream1
# - path: rtsp://127.0.0.1:8554/BackBasement
# #input_args: preset-rtsp-restream
# roles:
# - record
# - path: rtsp://127.0.0.1:8554/SubBackBasement
# #input_args: preset-rtsp-restream
# roles:
# - detect
# detect:
# enabled: true # <---- disable detection until you have a working camera feed
# width: 1280
# height: 720
# fps: 5
# motion:
# enabled: true
# threshold: 30
# contour_area: 30
# mask: 0.683,0.006,0.683,0.056,0.766,0.055,0.996,0.054,0.996,0.006
# improve_contrast: true
# snapshots:
# enabled: true
# timestamp: true
# bounding_box: true
# retain:
# objects:
# person: 7
# front_basement:
# enabled: true
# ffmpeg:
# inputs:
# # - path: rtsp://tpadmin:Optimus0329@192.168.128.225/stream1
# - path: rtsp://127.0.0.1:8554/FrontBasement
# #input_args: preset-rtsp-restream
# roles:
# - record
# - path: rtsp://127.0.0.1:8554/SubFrontBasement
# #input_args: preset-rtsp-restream
# roles:
# - detect
# detect:
# enabled: true # <---- disable detection until you have a working camera feed
# width: 1280
# height: 720
# fps: 5
# motion:
# enabled: true
# threshold: 30
# contour_area: 30
# mask: 0.723,0,0.724,0.05,1,0.048,0.999,0.001
# improve_contrast: true
# snapshots:
# enabled: true
# timestamp: true
# bounding_box: true
# retain:
# objects:
# person: 7
# record:
# enabled: false
# engineering:
# enabled: true
# ffmpeg:
# inputs:
# # - path: rtsp://tpadmin:Optimus0329@192.168.128.225/stream1
# - path: rtsp://127.0.0.1:8554/Engineering
# #input_args: preset-rtsp-restream
# roles:
# - record
# - path: rtsp://127.0.0.1:8554/SubEngineering
# #input_args: preset-rtsp-restream
# roles:
# - detect
# detect:
# enabled: true # <---- disable detection until you have a working camera feed
# width: 1280
# height: 720
# fps: 5
# motion:
# enabled: true
# threshold: 14
# contour_area: 15
# improve_contrast: true
# snapshots:
# enabled: true
# timestamp: true
# bounding_box: true
# retain:
# objects:
# person: 7
# record:
# enabled: false
driveway:
enabled: true
ffmpeg:
inputs:
# - path: rtsp://tpadmin:Optimus0329@192.168.128.225/stream1
# - path: rtsp://127.0.0.1:8554/Driveway
# input_args: preset-rtsp-restream
# roles:
# - record
- path: rtsp://127.0.0.1:8554/SubDriveway
input_args: preset-rtsp-restream
roles:
- record
- detect
detect:
enabled: true # <---- disable detection until you have a working camera feed
width: 1280
height: 720
fps: 5
snapshots:
enabled: true
timestamp: true
bounding_box: true
retain:
objects:
person: 7
record:
enabled: true
zones:
drivewayview:
coordinates: 0,0.373,0,1,1,1,1,0.381,0.47,0.343,0.134,0.352
loitering_time: 0
friendly_name: DrivewayView
porch:
enabled: true
ffmpeg:
inputs:
# - path: rtsp://tpadmin:Optimus0329@192.168.128.225/stream1
# - path: rtsp://127.0.0.1:8554/Porch
# input_args: preset-rtsp-restream
# roles:
# - record
- path: rtsp://127.0.0.1:8554/SubPorch
input_args: preset-rtsp-restream
roles:
- record
- detect
detect:
enabled: true # <---- disable detection until you have a working camera feed
width: 1280
height: 720
fps: 5
snapshots:
enabled: true
timestamp: true
bounding_box: true
retain:
objects:
person: 7
record:
enabled: true
fence_gate:
enabled: true
ffmpeg:
inputs:
# - path: rtsp://tpadmin:Optimus0329@192.168.128.225/stream1
# - path: rtsp://127.0.0.1:8554/FenceGate
# input_args: preset-rtsp-restream
# roles:
# - record
- path: rtsp://127.0.0.1:8554/SubFenceGate
input_args: preset-rtsp-restream
roles:
- record
- detect
detect:
enabled: true # <---- disable detection until you have a working camera feed
width: 1280
height: 720
fps: 5
snapshots:
enabled: true
timestamp: true
bounding_box: true
retain:
objects:
person: 7
record:
enabled: true
zones:
fencegate:
coordinates:
0,0.395,0.318,0.259,0.598,0.175,0.916,0.237,1,0.296,1,1,0.003,0.996
loitering_time: 0
friendly_name: FenceGate
garden_area:
enabled: true
ffmpeg:
inputs:
# - path: rtsp://tpadmin:Optimus0329@192.168.128.225/stream1
# - path: rtsp://127.0.0.1:8554/GardenArea
# input_args: preset-rtsp-restream
# roles:
# - record
- path: rtsp://127.0.0.1:8554/SubGardenArea
input_args: preset-rtsp-restream
roles:
- record
- detect
detect:
enabled: true # <---- disable detection until you have a working camera feed
width: 1280
height: 720
fps: 5
snapshots:
enabled: true
timestamp: true
bounding_box: true
retain:
objects:
person: 7
record:
enabled: true
backyard_1:
enabled: true
ffmpeg:
inputs:
# - path: rtsp://tpadmin:Optimus0329@192.168.128.225/stream1
# - path: rtsp://127.0.0.1:8554/Backyard1
# input_args: preset-rtsp-restream
# roles:
# - record
- path: rtsp://127.0.0.1:8554/SubBackyard1
input_args: preset-rtsp-restream
roles:
- record
- detect
detect:
enabled: true # <---- disable detection until you have a working camera feed
width: 1280
height: 720
fps: 5
snapshots:
enabled: true
timestamp: true
bounding_box: true
retain:
objects:
person: 7
record:
enabled: true
zones:
backyard1:
coordinates:
0.001,0.274,0.124,0.162,0.228,0.143,0.382,0.131,0.555,0.133,0.714,0.16,0.854,0.196,0.998,0.253,0.998,0.996,0,0.992
loitering_time: 0
friendly_name: Backyard1
backyard_2:
enabled: true
ffmpeg:
inputs:
# - path: rtsp://tpadmin:Optimus0329@192.168.128.225/stream1
# - path: rtsp://127.0.0.1:8554/Backyard2
# input_args: preset-rtsp-restream
# roles:
# - record
- path: rtsp://127.0.0.1:8554/SubBackyard2
input_args: preset-rtsp-restream
roles:
- record
- detect
detect:
enabled: true # <---- disable detection until you have a working camera feed
width: 1280
height: 720
fps: 5
snapshots:
enabled: true
timestamp: true
bounding_box: true
retain:
objects:
person: 7
record:
enabled: true
zones:
backyard2:
coordinates:
0.002,0.147,0.086,0.106,0.172,0.091,0.271,0.078,0.386,0.077,0.499,0.077,0.601,0.086,0.698,0.099,0.787,0.117,0.859,0.087,0.995,0.155,0.998,0.992,0.001,0.995
loitering_time: 0
friendly_name: Backyard2
backyard_3:
enabled: true
ffmpeg:
inputs:
# - path: rtsp://tpadmin:Optimus0329@192.168.128.225/stream1
# - path: rtsp://127.0.0.1:8554/Backyard3
# input_args: preset-rtsp-restream
# roles:
# - record
- path: rtsp://127.0.0.1:8554/SubBackyard3
input_args: preset-rtsp-restream
roles:
- record
- detect
detect:
enabled: true # <---- disable detection until you have a working camera feed
width: 1280
height: 720
fps: 5
snapshots:
enabled: true
timestamp: true
bounding_box: true
retain:
objects:
person: 7
record:
enabled: true
zones:
backyard3:
coordinates:
0.001,0.337,0.166,0.332,0.327,0.342,0.419,0.344,0.51,0.355,0.603,0.363,0.692,0.374,0.771,0.381,0.848,0.389,0.951,0.405,0.997,0.423,0.998,0.996,0.003,0.994
loitering_time: 0
friendly_name: Backyard3
motion:
mask: 0.79,0.539,0.845,0.541,0.82,0.858,0.772,0.844
backyard_4:
enabled: true
ffmpeg:
inputs:
# - path: rtsp://tpadmin:Optimus0329@192.168.128.225/stream1
# - path: rtsp://127.0.0.1:8554/Backyard4
# input_args: preset-rtsp-restream
# roles:
# - record
- path: rtsp://127.0.0.1:8554/SubBackyard4
input_args: preset-rtsp-restream
roles:
- record
- detect
detect:
enabled: true # <---- disable detection until you have a working camera feed
width: 1280
height: 720
fps: 5
snapshots:
enabled: true
timestamp: true
bounding_box: true
retain:
objects:
person: 7
record:
enabled: true
motion:
mask:
- 0.866,0.001,0.866,0.209,1,0.202,1,0
- 0.001,0,0.249,0.001,0.249,0.045,0.002,0.044
threshold: 60
contour_area: 10
improve_contrast: true
backyard_5:
enabled: true
ffmpeg:
inputs:
# - path: rtsp://tpadmin:Optimus0329@192.168.128.225/stream1
# - path: rtsp://127.0.0.1:8554/Backyard5
# input_args: preset-rtsp-restream
# roles:
# - record
- path: rtsp://127.0.0.1:8554/SubBackyard5
input_args: preset-rtsp-restream
roles:
- record
- detect
detect:
enabled: true # <---- disable detection until you have a working camera feed
width: 1280
height: 720
fps: 5
snapshots:
enabled: true
timestamp: true
bounding_box: true
retain:
objects:
person: 7
record:
enabled: true
motion:
mask:
- 0.029,0.31,0.13,0.305,0.134,0.54,0.032,0.544
- 0.004,0.002,0.004,0.044,0.245,0.044,0.244,0
threshold: 60
contour_area: 10
improve_contrast: true
zones:
backyard5:
coordinates:
0.001,0.305,0.043,0.264,0.113,0.194,0.19,0.128,0.271,0.109,0.371,0.089,0.525,0.057,0.709,0.057,0.89,0.076,0.997,0.122,0.997,0.995,0.002,0.998
loitering_time: 0
friendly_name: Backyard5
patio_area:
enabled: true
ffmpeg:
inputs:
# - path: rtsp://tpadmin:Optimus0329@192.168.128.225/stream1
# - path: rtsp://127.0.0.1:8554/PatioArea
# input_args: preset-rtsp-restream
# roles:
# - record
- path: rtsp://127.0.0.1:8554/SubPatioArea
input_args: preset-rtsp-restream
roles:
- record
- detect
detect:
enabled: true # <---- disable detection until you have a working camera feed
width: 1280
height: 720
fps: 5
snapshots:
enabled: true
timestamp: true
bounding_box: true
retain:
objects:
person: 7
record:
enabled: true
pool_area:
enabled: true
ffmpeg:
inputs:
# - path: rtsp://tpadmin:Optimus0329@192.168.128.225/stream1
# - path: rtsp://127.0.0.1:8554/PoolArea
# input_args: preset-rtsp-restream
# roles:
# - record
- path: rtsp://127.0.0.1:8554/SubPoolArea
input_args: preset-rtsp-restream
roles:
- record
- detect
detect:
enabled: true # <---- disable detection until you have a working camera feed
width: 1280
height: 720
fps: 5
snapshots:
enabled: true
timestamp: true
bounding_box: true
retain:
objects:
person: 7
record:
enabled: true
motion:
mask:
0.43,0.146,0.704,0.194,0.532,0.808,0.199,0.704,0.002,0.559,0.252,0.282
back_slider:
enabled: true
ffmpeg:
inputs:
# - path: rtsp://tpadmin:Optimus0329@192.168.128.225/stream1
# - path: rtsp://127.0.0.1:8554/BackSlider
# input_args: preset-rtsp-restream
# roles:
# - record
- path: rtsp://127.0.0.1:8554/SubBackSlider
input_args: preset-rtsp-restream
roles:
- record
- detect
detect:
enabled: true # <---- disable detection until you have a working camera feed
width: 1280
height: 720
fps: 5
snapshots:
enabled: true
timestamp: true
bounding_box: true
retain:
objects:
person: 7
record:
enabled: true
side_house:
enabled: true
ffmpeg:
inputs:
# - path: rtsp://tpadmin:Optimus0329@192.168.128.225/stream1
# - path: rtsp://127.0.0.1:8554/SideHouse
# input_args: preset-rtsp-restream
# roles:
# - record
- path: rtsp://127.0.0.1:8554/SubSideHouse
input_args: preset-rtsp-restream
roles:
- record
- detect
detect:
enabled: true # <---- disable detection until you have a working camera feed
width: 1280
height: 720
fps: 5
snapshots:
enabled: true
timestamp: true
bounding_box: true
retain:
objects:
person: 7
record:
enabled: true
motion:
threshold: 60
contour_area: 40
improve_contrast: true
mask:
0.001,0.859,0.085,0.717,0.203,0.637,0.383,0.736,0.65,0.421,0.642,0.997,0.003,0.999
zones:
sidehouse:
coordinates: 0.661,0.001,0.662,0.996,1,1,1,0
loitering_time: 0
friendly_name: SideHouse
objects:
filters:
person: {}
version: 0.17-0
personal/configs/go2rtc/go2rtc.yaml
-43
View File
@@ -1,43 +0,0 @@
api:
listen: ":1984"
log:
level: "info"
format: "color"
output: "file:go2rtc.log"
rtsp:
listen: ":8554" # RTSP Server TCP port, default - 8554
#username: "admin" # optional, default - disabled
#password: "pass" # optional, default - disabled
#default_query: "video=h264" # optional, default codecs filters
ffmpeg:
bin: ffmpeg
# Nvidia NVDEC hardware decode templates (used when #hardware is in stream URL)
h264: "-hwaccel cuda -hwaccel_output_format cuda -c:v h264_cuvid {input} -c:v copy -f rtsp {output}"
h265: "-hwaccel cuda -hwaccel_output_format cuda -c:v hevc_cuvid {input} -c:v copy -f rtsp {output}"
streams:
porch_steps: rtsp://admin:Optimus0329@192.168.128.11:80/ch8_0.264#video=h264#hardware=cuda
driveway: rtsp://admin:Optimus0329@192.168.128.11:80/ch12_0.264#video=h264#hardware=cuda
fence_gate: rtsp://admin:Optimus0329@192.168.128.11:80/ch1_0.264#video=h264#hardware=cuda
shed_area: rtsp://admin:Optimus0329@192.168.128.11:80/ch13_0.264#video=h264#hardware=cuda
garden_area: rtsp://admin:Optimus0329@192.168.128.11:80/ch2_0.264#video=h264#hardware=cuda
backyard_1: rtsp://admin:Optimus0329@192.168.128.11:80/ch3_0.264#video=h264#hardware=cuda
backyard_2: rtsp://admin:Optimus0329@192.168.128.11:80/ch4_0.264#video=h264#hardware=cuda
backyard_3: rtsp://admin:Optimus0329@192.168.128.11:80/ch0_0.264#video=h264#hardware=cuda
backyard_4: rtsp://admin:Optimus0329@192.168.128.11:80/ch10_0.264#video=h264#hardware=cuda
backyard_5: rtsp://admin:Optimus0329@192.168.128.11:80/ch7_0.264#video=h264#hardware=cuda
inside_garage: rtsp://admin:Optimus0329@192.168.128.11:80/ch6_0.264#video=h264#hardware=cuda
pool_area: rtsp://admin:Optimus0329@192.168.128.11:80/ch9_0.264#video=h264#hardware=cuda
back_slider: rtsp://admin:Optimus0329@192.168.128.11:80/ch5_0.264#video=h264#hardware=cuda
side_house: rtsp://admin:Optimus0329@192.168.128.11:80/ch11_0.264#video=h264#hardware=cuda
patio_area: rtsp://admin:Optimus0329@192.168.128.11:80/ch14_0.264
engineering: rtsp://admin:Optimus0329@192.168.128.65:1554/live/ch0
front_basement: rtsp://admin:Optimus0329@192.168.128.223:554/live/ch0
back_basement: rtsp://admin:Optimus0329@192.168.128.95:554/live/ch0
back_basement2: rtsp://admin:Optimus0329@192.168.128.183:554/live/ch0
webrtc:
listen: ":8555"
personal/cron_scripts/archived_versions/ilo_powercycle.py.04262026
-66
View File
@@ -1,66 +0,0 @@
#!/usr/bin/python3
import argparse
import configparser
import os
import sys
import urllib3
import redfish
from datetime import datetime
def log(message, is_error=False):
"""Print message with timestamp"""
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if is_error:
print(f"[{timestamp}] {message}", file=sys.stderr)
else:
print(f"[{timestamp}] {message}")
# Suppress SSL warnings for self-signed iLO certificates
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
parser = argparse.ArgumentParser(description="Power off an HPE server via iLO Redfish")
parser.add_argument("hostname", help="iLO hostname or IP address")
parser.add_argument("-c", "--credentials", default="/etc/ilo_credentials",
help="Path to credentials file (default: /etc/ilo_credentials)")
args = parser.parse_args()
# Read credentials from file (chmod 600 recommended: sudo chmod 600 /etc/ilo_credentials)
# File format:
# [ilo]
# username = root
# password = yourpassword
cred_path = args.credentials
if not os.path.exists(cred_path):
log(f"Error: credentials file not found: {cred_path}", is_error=True)
sys.exit(1)
config = configparser.ConfigParser()
config.read(cred_path)
try:
LOGIN_ACCOUNT = config.get("ilo_credentials", "username", fallback="root")
LOGIN_PASSWORD = config.get("ilo_credentials", "password")
except (configparser.NoSectionError, configparser.NoOptionError) as e:
log(f"Error reading credentials file: {e}", is_error=True)
sys.exit(1)
BASE_URL = f"https://{args.hostname}"
REDFISHOBJ = redfish.RedfishClient(
base_url=BASE_URL,
username=LOGIN_ACCOUNT,
password=LOGIN_PASSWORD,
)
REDFISHOBJ.login(auth="session")
# Discover the correct reset action URI from the system resource
sys_response = REDFISHOBJ.get("/redfish/v1/Systems/1/")
reset_uri = sys_response.dict["Actions"]["#ComputerSystem.Reset"]["target"]
body = {"ResetType": "PushPowerButton"} # iLO 4: PushPowerButton = graceful shutdown; use "ForceOff" for hard power cut
response = REDFISHOBJ.post(reset_uri, body=body)
log(f"Status: {response.status}")
if response.status >= 400:
log(f"Error: {response.read}", is_error=True)
REDFISHOBJ.logout()
personal/cron_scripts/archived_versions/ilo_poweron.py.04262026
-66
View File
@@ -1,66 +0,0 @@
#!/usr/bin/python3
import argparse
import configparser
import os
import sys
import urllib3
import redfish
from datetime import datetime
def log(message, is_error=False):
"""Print message with timestamp"""
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if is_error:
print(f"[{timestamp}] {message}", file=sys.stderr)
else:
print(f"[{timestamp}] {message}")
# Suppress SSL warnings for self-signed iLO certificates
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
parser = argparse.ArgumentParser(description="Power off an HPE server via iLO Redfish")
parser.add_argument("hostname", help="iLO hostname or IP address")
parser.add_argument("-c", "--credentials", default="/etc/ilo_credentials",
help="Path to credentials file (default: /etc/ilo_credentials)")
args = parser.parse_args()
# Read credentials from file (chmod 600 recommended: sudo chmod 600 /etc/ilo_credentials)
# File format:
# [ilo]
# username = root
# password = yourpassword
cred_path = args.credentials
if not os.path.exists(cred_path):
log(f"Error: credentials file not found: {cred_path}", is_error=True)
sys.exit(1)
config = configparser.ConfigParser()
config.read(cred_path)
try:
LOGIN_ACCOUNT = config.get("ilo", "username", fallback="root")
LOGIN_PASSWORD = config.get("ilo", "password")
except (configparser.NoSectionError, configparser.NoOptionError) as e:
log(f"Error reading credentials file: {e}", is_error=True)
sys.exit(1)
BASE_URL = f"https://{args.hostname}"
REDFISHOBJ = redfish.RedfishClient(
base_url=BASE_URL,
username=LOGIN_ACCOUNT,
password=LOGIN_PASSWORD,
)
REDFISHOBJ.login(auth="session")
# Discover the correct reset action URI from the system resource
sys_response = REDFISHOBJ.get("/redfish/v1/Systems/1/")
reset_uri = sys_response.dict["Actions"]["#ComputerSystem.Reset"]["target"]
body = {"ResetType": "PushPowerButton"} # iLO 4: PushPowerButton = graceful shutdown; use "ForceOff" for hard power cut
response = REDFISHOBJ.post(reset_uri, body=body)
log(f"Status: {response.status}")
if response.status >= 400:
log(f"Error: {response.read}", is_error=True)
REDFISHOBJ.logout()
personal/cron_scripts/archived_versions/ipmi_multipoweron.py.04262026
-159
View File
@@ -1,159 +0,0 @@
#!/usr/bin/env python3
import argparse
import configparser
import os
import subprocess
import sys
import datetime
# Examples:
# ./ipmi_multipoweron.py host1 host2 host3
# ./ipmi_multipoweron.py host1 host2 host3 -c /path/to/credentials
# ./ipmi_multipoweron.py -f serverlist.txt
# ./ipmi_multipoweron.py -f serverlist.txt -c /path/to/credentials
def ts_print(msg):
"""Print message prefixed with a timestamp (YYYY-MM-DD HH:MM:SS)."""
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(f"[{now}] {msg}")
DEFAULT_EK = "0000000000000000000000000000000000000000"
def find_credentials_file(provided_path):
"""Find credentials file in standard locations if provided path doesn't exist."""
# If provided path exists, use it
if os.path.exists(provided_path):
return provided_path
# Check standard locations
standard_paths = [
os.path.expanduser("~/credentials"),
"/etc/credentials",
]
for path in standard_paths:
if os.path.exists(path):
return path
# Return the provided path (will fail with proper error message later)
return provided_path
def load_credentials(path):
cred_path = find_credentials_file(path)
if not os.path.exists(cred_path):
ts_print(f"Error: credentials file not found: {cred_path}")
sys.exit(1)
config = configparser.ConfigParser()
config.read(cred_path)
try:
username = config.get("credentials", "username", fallback="root")
password = config.get("credentials", "password")
encryption_key = config.get("credentials", "encryption_key", fallback=DEFAULT_EK)
except (configparser.NoSectionError, configparser.NoOptionError) as e:
ts_print(f"Error reading credentials file: {e}")
sys.exit(1)
return username, password, encryption_key
def power_on_host(host, ipmi_user, ipmi_pw, ipmi_ek):
"""Power on a single host via IPMI. Returns True if successful."""
cmd = [
"ipmitool",
"-I", "lanplus",
"-H", host,
"-U", ipmi_user,
"-P", ipmi_pw,
"-y", ipmi_ek,
"power", "on",
]
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if proc.returncode != 0:
ts_print(f"ERROR: Failed to power on {host}: {proc.stderr.strip()}")
return False
else:
ts_print(f"SUCCESS: {host} - {proc.stdout.strip()}")
return True
def load_servers_from_file(filepath):
"""Load server list from a file (one server per line, lines starting with # are ignored)."""
if not os.path.exists(filepath):
ts_print(f"Error: server list file not found: {filepath}")
sys.exit(1)
servers = []
try:
with open(filepath, 'r') as f:
for line in f:
line = line.strip()
# Skip empty lines and comments
if line and not line.startswith('#'):
servers.append(line)
except Exception as e:
ts_print(f"Error reading server list file: {e}")
sys.exit(1)
return servers
def main():
p = argparse.ArgumentParser(description="Send IPMI power on to multiple servers")
# Either provide servers as arguments or a file
group = p.add_mutually_exclusive_group(required=True)
group.add_argument(
"servers",
nargs="*",
help="List of IPMI host addresses (space-separated)"
)
group.add_argument(
"-f", "--file",
help="Path to file containing list of servers (one per line)"
)
p.add_argument(
"-c",
"--credentials",
default="credentials",
help="Path to credentials file (default: credentials; also checks ~/ and /etc/ if not found)",
)
args = p.parse_args()
# Determine server list
if args.file:
servers = load_servers_from_file(args.file)
else:
servers = args.servers
if not servers:
ts_print("Error: No servers specified")
sys.exit(1)
ts_print(f"Starting power on sequence for {len(servers)} server(s)")
ipmi_user, ipmi_pw, ipmi_ek = load_credentials(args.credentials)
success_count = 0
failed_count = 0
for host in servers:
if power_on_host(host, ipmi_user, ipmi_pw, ipmi_ek):
success_count += 1
else:
failed_count += 1
ts_print(f"Completed: {success_count} successful, {failed_count} failed")
if failed_count > 0:
sys.exit(1)
if __name__ == "__main__":
main()
personal/cron_scripts/archived_versions/ipmi_poweroff.py.04262026
-70
View File
@@ -1,70 +0,0 @@
#!/usr/bin/env python3
import argparse
import configparser
import os
import subprocess
import sys
from datetime import datetime
DEFAULT_EK = "0000000000000000000000000000000000000000"
def log(message, is_error=False):
"""Print message with timestamp"""
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if is_error:
print(f"[{timestamp}] {message}", file=sys.stderr)
else:
print(f"[{timestamp}] {message}")
def load_credentials(path):
if not os.path.exists(path):
log(f"Error: credentials file not found: {path}", is_error=True)
sys.exit(1)
config = configparser.ConfigParser()
config.read(path)
try:
username = config.get("credentials", "username", fallback="root")
password = config.get("credentials", "password")
encryption_key = config.get("credentials", "encryption_key", fallback=DEFAULT_EK)
except (configparser.NoSectionError, configparser.NoOptionError) as e:
log(f"Error reading credentials file: {e}", is_error=True)
sys.exit(1)
return username, password, encryption_key
def main():
p = argparse.ArgumentParser(description="Send IPMI power off")
p.add_argument("-H", "--host", required=True, help="IPMI host address")
p.add_argument(
"-c",
"--credentials",
default="credentials",
help="Path to credentials file (default: credentials)",
)
args = p.parse_args()
ipmi_user, ipmi_pw, ipmi_ek = load_credentials(args.credentials)
cmd = [
"ipmitool",
"-I", "lanplus",
"-H", args.host,
"-U", ipmi_user,
"-P", ipmi_pw,
"-y", ipmi_ek,
"power", "off",
]
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if proc.returncode != 0:
log("ipmitool failed: " + proc.stderr.strip(), is_error=True)
sys.exit(proc.returncode)
log(proc.stdout.strip())
if __name__ == "__main__":
main()
personal/cron_scripts/archived_versions/ipmi_poweron.py.04262026
-65
View File
@@ -1,65 +0,0 @@
#!/usr/bin/env python3
import argparse
import configparser
import os
import subprocess
import sys
import datetime
def ts_print(msg):
"""ts_print message prefixed with a timestamp (YYYY-MM-DD HH:MM:SS)."""
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(f"{now} {msg}")
DEFAULT_EK = "0000000000000000000000000000000000000000"
def load_credentials(path):
if not os.path.exists(path):
print(f"Error: credentials file not found: {path}", file=sys.stderr)
sys.exit(1)
config = configparser.ConfigParser()
config.read(path)
try:
username = config.get("ilo", "username", fallback="root")
password = config.get("ilo", "password")
encryption_key = config.get("ilo", "encryption_key", fallback=DEFAULT_EK)
except (configparser.NoSectionError, configparser.NoOptionError) as e:
print(f"Error reading credentials file: {e}", file=sys.stderr)
sys.exit(1)
return username, password, encryption_key
def main():
p = argparse.ArgumentParser(description="Send IPMI power on")
p.add_argument("-H", "--host", required=True, help="IPMI host address")
p.add_argument(
"-c",
"--credentials",
default="credentials",
help="Path to credentials file (default: credentials)",
)
args = p.parse_args()
ipmi_user, ipmi_pw, ipmi_ek = load_credentials(args.credentials)
cmd = [
"ipmitool",
"-I", "lanplus",
"-H", args.host,
"-U", ipmi_user,
"-P", ipmi_pw,
"-y", ipmi_ek,
"power", "on",
]
print(f"Sending power on to {args.host}")
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if proc.returncode != 0:
print("ipmitool failed:", proc.stderr.strip(), file=sys.stderr)
sys.exit(proc.returncode)
print(proc.stdout.strip())
if __name__ == "__main__":
main()
personal/cron_scripts/build_ilo_powercycle.py
-40
View File
@@ -1,40 +0,0 @@
#!/usr/bin/env python3
import subprocess
import sys
from pathlib import Path
SCRIPT_DIR = Path(__file__).resolve().parent
REPO_ROOT = SCRIPT_DIR.parents[1]
ENTRYPOINT = REPO_ROOT / "personal" / "tools" / "ilo_powercycle.py"
DIST_DIR = REPO_ROOT / "dist"
def main():
command = [
sys.executable,
"-m",
"PyInstaller",
"--noconfirm",
"--clean",
"--onefile",
"--name",
"ilo_powercycle",
"--collect-submodules",
"redfish",
str(ENTRYPOINT),
]
try:
subprocess.run(command, cwd=REPO_ROOT, check=True)
except subprocess.CalledProcessError as exc:
raise SystemExit(exc.returncode) from exc
output_name = "ilo_powercycle.exe" if sys.platform.startswith("win") else "ilo_powercycle"
output_path = DIST_DIR / output_name
print(f"Built executable: {output_path}")
print("Build once on each target OS to get a native executable for that platform.")
if __name__ == "__main__":
main()
personal/cron_scripts/build_ilo_status.py
-40
View File
@@ -1,40 +0,0 @@
#!/usr/bin/env python3
import subprocess
import sys
from pathlib import Path
SCRIPT_DIR = Path(__file__).resolve().parent
REPO_ROOT = SCRIPT_DIR.parents[1]
ENTRYPOINT = SCRIPT_DIR / "ilo_status.py"
DIST_DIR = REPO_ROOT / "dist"
def main():
command = [
sys.executable,
"-m",
"PyInstaller",
"--noconfirm",
"--clean",
"--onefile",
"--name",
"ilo_status",
"--collect-submodules",
"redfish",
str(ENTRYPOINT),
]
try:
subprocess.run(command, cwd=REPO_ROOT, check=True)
except subprocess.CalledProcessError as exc:
raise SystemExit(exc.returncode) from exc
output_name = "ilo_status.exe" if sys.platform.startswith("win") else "ilo_status"
output_path = DIST_DIR / output_name
print(f"Built executable: {output_path}")
print("Build once on each target OS to get a native executable for that platform.")
if __name__ == "__main__":
main()
personal/cron_scripts/build_ipmi_poweroff.py
-38
View File
@@ -1,38 +0,0 @@
#!/usr/bin/env python3
import subprocess
import sys
from pathlib import Path
SCRIPT_DIR = Path(__file__).resolve().parent
REPO_ROOT = SCRIPT_DIR.parents[1]
ENTRYPOINT = SCRIPT_DIR / "power" / "ipmi_poweroff.py"
DIST_DIR = REPO_ROOT / "dist"
def main():
command = [
sys.executable,
"-m",
"PyInstaller",
"--noconfirm",
"--clean",
"--onefile",
"--name",
"ipmi_poweroff",
str(ENTRYPOINT),
]
try:
subprocess.run(command, cwd=REPO_ROOT, check=True)
except subprocess.CalledProcessError as exc:
raise SystemExit(exc.returncode) from exc
output_name = "ipmi_poweroff.exe" if sys.platform.startswith("win") else "ipmi_poweroff"
output_path = DIST_DIR / output_name
print(f"Built executable: {output_path}")
print("Build once on each target OS to get a native executable for that platform.")
if __name__ == "__main__":
main()
personal/cron_scripts/build_ipmi_poweron.py
-38
View File
@@ -1,38 +0,0 @@
#!/usr/bin/env python3
import subprocess
import sys
from pathlib import Path
SCRIPT_DIR = Path(__file__).resolve().parent
REPO_ROOT = SCRIPT_DIR.parents[1]
ENTRYPOINT = SCRIPT_DIR / "power" / "ipmi_poweron.py"
DIST_DIR = REPO_ROOT / "dist"
def main():
command = [
sys.executable,
"-m",
"PyInstaller",
"--noconfirm",
"--clean",
"--onefile",
"--name",
"ipmi_poweron",
str(ENTRYPOINT),
]
try:
subprocess.run(command, cwd=REPO_ROOT, check=True)
except subprocess.CalledProcessError as exc:
raise SystemExit(exc.returncode) from exc
output_name = "ipmi_poweron.exe" if sys.platform.startswith("win") else "ipmi_poweron"
output_path = DIST_DIR / output_name
print(f"Built executable: {output_path}")
print("Build once on each target OS to get a native executable for that platform.")
if __name__ == "__main__":
main()
personal/cron_scripts/build_multi_powercycle.py
-40
View File
@@ -1,40 +0,0 @@
#!/usr/bin/env python3
import subprocess
import sys
from pathlib import Path
SCRIPT_DIR = Path(__file__).resolve().parent
REPO_ROOT = SCRIPT_DIR.parents[1]
ENTRYPOINT = SCRIPT_DIR / "multi_powercycle.py"
DIST_DIR = REPO_ROOT / "dist"
def main():
command = [
sys.executable,
"-m",
"PyInstaller",
"--noconfirm",
"--clean",
"--onefile",
"--name",
"multi_powercycle",
"--collect-submodules",
"redfish",
str(ENTRYPOINT),
]
try:
subprocess.run(command, cwd=REPO_ROOT, check=True)
except subprocess.CalledProcessError as exc:
raise SystemExit(exc.returncode) from exc
output_name = "multi_powercycle.exe" if sys.platform.startswith("win") else "multi_powercycle"
output_path = DIST_DIR / output_name
print(f"Built executable: {output_path}")
print("Build once on each target OS to get a native executable for that platform.")
if __name__ == "__main__":
main()
personal/cron_scripts/build_set_fanspeed.py
-38
View File
@@ -1,38 +0,0 @@
#!/usr/bin/env python3
import subprocess
import sys
from pathlib import Path
SCRIPT_DIR = Path(__file__).resolve().parent
REPO_ROOT = SCRIPT_DIR.parents[1]
ENTRYPOINT = SCRIPT_DIR / "fancontrol" / "set_fanspeed.py"
DIST_DIR = REPO_ROOT / "dist"
def main():
command = [
sys.executable,
"-m",
"PyInstaller",
"--noconfirm",
"--clean",
"--onefile",
"--name",
"set_fanspeed",
str(ENTRYPOINT),
]
try:
subprocess.run(command, cwd=REPO_ROOT, check=True)
except subprocess.CalledProcessError as exc:
raise SystemExit(exc.returncode) from exc
output_name = "set_fanspeed.exe" if sys.platform.startswith("win") else "set_fanspeed"
output_path = DIST_DIR / output_name
print(f"Built executable: {output_path}")
print("Build once on each target OS to get a native executable for that platform.")
if __name__ == "__main__":
main()
personal/cron_scripts/crontab
-32
View File
@@ -1,32 +0,0 @@
# Edit this file to introduce tasks to be run by cron.
#
# Each task to run has to be defined through a single line
# indicating with different fields when the task will be run
# and what command to run for the task
#
# To define the time you can provide concrete values for
# minute (m), hour (h), day of month (dom), month (mon),
# and day of week (dow) or use '*' in these fields (for 'any').
#
# Notice that tasks will be started based on the cron's system
# daemon's notion of time and timezones.
#
# Output of the crontab jobs (including errors) is sent through
# email to the user the crontab file belongs to (unless redirected).
#
# For example, you can run a backup of all your user accounts
# at 5 a.m every week with:
# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/
#
# For more information see the manual pages of crontab(5) and cron(8)
#
# m h dom mon dow command
30 0 * * * /cron_jobs/power/ipmi_poweroff.py -H 192.168.129.240 >> /var/log/cron_scripts.log 2>&1
30 0 * * * /cron_jobs/power/ipmi_poweroff.py -H 192.168.129.241 >> /var/log/cron_scripts.log 2>&1
0 1 * * * /cron_jobs/power/ilo_poweroff.py -H 192.168.129.242 -a ForceOff --no-serverinfo >> /var/log/cron_scripts.log 2>&1
0 5 * * * /cron_jobs/system/ilo_poweron.py -H 192.168.129.242 -a On --no-serverinfo >> /var/log/cron_scripts.log 2>&1
10 5 * * * /cron_jobs/power/ipmi_poweron.py -H 192.168.129.240 >> /var/log/cron_scripts.log 2>&1
10 5 * * * /cron_jobs/power/ipmi_poweron.py -H 192.168.129.241 >> /var/log/cron_scripts.log 2>&1
35 0 * * * /cron_jobs/system/update_system.py >> /var/log/cron_scripts.log 2>&1
*/3 * * * * /cron_jobs/fancontrol/set_fanspeed.py -H 192.168.129.240 >> /var/log/cron_scripts.log 2>&1
*/3 * * * * /cron_jobs/fancontrol/set_fanspeed.py -H 192.168.129.241 >> /var/log/cron_scripts.log 2>&1
personal/cron_scripts/fancontrol/set_fanspeed.py
-282
View File
@@ -1,282 +0,0 @@
#!/usr/bin/env python3
import subprocess
import re
import sys
import datetime
import argparse
import configparser
import os
try:
from cryptography.fernet import Fernet, InvalidToken
except ImportError:
sys.exit("Error: 'cryptography' package is required. Run: pip install cryptography")
DEFAULT_EK = "0000000000000000000000000000000000000000"
DEFAULT_CREDENTIALS_FILE = "/root/cron_serverlists/credentials.ini"
DEFAULT_KEYFILE_NAME = "secret.key"
# IPMI settings (loaded at runtime from credentials file)
IPMIUSER = None
IPMIPW = None
IPMIEK = DEFAULT_EK
# Fan / temperature settings
FANSPEED = 15 # percent
MAXTEMP = 37 # Celsius
def build_ipmi_base():
base = ["ipmitool", "-I", IPMIINTERFACE, "-H", IPMIHOST, "-U", IPMIUSER, "-P", IPMIPW, "-y", IPMIEK]
if IPMICIPHER is not None:
base += ["-C", str(IPMICIPHER)]
return base
def has_ipmi_session_error(stderr_text):
s = (stderr_text or "").lower()
markers = [
"unable to establish ipmi v2 / rmcp+ session",
"rakp",
"session timeout",
"authentication type",
"invalid user name",
]
return any(marker in s for marker in markers)
def run_ipmi(args):
base = build_ipmi_base()
cmd = base + args
p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if p.returncode != 0 or has_ipmi_session_error(p.stderr):
raise RuntimeError(
f"ipmitool failed: {' '.join(cmd)}\n"
f"stdout: {(p.stdout or '').strip() or '<empty>'}\n"
f"stderr: {(p.stderr or '').strip() or '<empty>'}"
)
return p.stdout
def run_ipmi_raw(args):
base = build_ipmi_base()
cmd = base + args
p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
return cmd, p.returncode, p.stdout, p.stderr
def log_systemd(tag, msg):
try:
subprocess.run(["systemd-cat", "-t", tag], input=msg + "\n", text=True, check=True)
except Exception:
# fallback to stdout
ts_print(msg)
def ts_print(msg):
"""ts_print message prefixed with a timestamp (YYYY-MM-DD HH:MM:SS)."""
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(f"{now} {msg}")
def is_fernet_token(value):
return isinstance(value, str) and value.startswith("gAAAA")
def load_fernet(keyfile):
if not os.path.exists(keyfile):
raise RuntimeError(f"key file not found: {keyfile}")
with open(keyfile, "rb") as fh:
key = fh.read()
try:
return Fernet(key)
except ValueError as e:
raise RuntimeError(f"invalid key format in key file: {keyfile}") from e
def decrypt_password(value, fernet):
# Support both encrypted and plaintext credentials for smoother migrations.
if not is_fernet_token(value):
return value
if fernet is None:
raise RuntimeError(
"encrypted password detected but key file was not found. "
"Use --keyfile /path/to/secret.key"
)
try:
return fernet.decrypt(value.encode()).decode()
except InvalidToken as e:
raise RuntimeError("failed to decrypt password; key file may not match credentials file") from e
def resolve_credentials_path(cli_path):
if cli_path and os.path.exists(cli_path):
return cli_path
candidates = []
if cli_path:
candidates.append(cli_path)
candidates.append(os.path.join(os.getcwd(), "credentials.ini"))
if DEFAULT_CREDENTIALS_FILE not in candidates:
candidates.append(DEFAULT_CREDENTIALS_FILE)
for candidate in candidates:
if os.path.exists(candidate):
return candidate
raise RuntimeError("credentials file not found. Looked in: " + ", ".join(candidates))
def resolve_keyfile(cli_keyfile, credentials_path):
if cli_keyfile:
return cli_keyfile if os.path.exists(cli_keyfile) else None
candidates = [
os.path.join(os.path.dirname(credentials_path), DEFAULT_KEYFILE_NAME),
os.path.join(os.path.dirname(DEFAULT_CREDENTIALS_FILE), DEFAULT_KEYFILE_NAME),
]
for candidate in candidates:
if os.path.exists(candidate):
return candidate
return None
def load_credentials(path, fernet):
if not os.path.exists(path):
raise RuntimeError(f"credentials file not found: {path}")
config = configparser.ConfigParser()
config.read(path)
try:
section = "ipmi_credentials"
username = config.get(section, "username", fallback="root")
encrypted_password = config.get(section, "password")
password = decrypt_password(encrypted_password, fernet)
encryption_key = config.get(
section,
"ipmiek",
fallback=config.get(section, "encryption_key", fallback=DEFAULT_EK),
)
except (configparser.NoSectionError, configparser.NoOptionError) as e:
raise RuntimeError(f"error reading credentials file: {e}")
return username, password, encryption_key
def get_exhaust_temp():
commands_to_try = [
["sensor", "get", "Exhaust Temp"],
["sdr", "type", "temperature"],
["sensor", "list"],
["sdr", "elist", "full"],
]
debug_sections = []
session_error_seen = False
for args in commands_to_try:
cmd, rc, out, err = run_ipmi_raw(args)
if has_ipmi_session_error(err):
session_error_seen = True
cmd_text = " ".join(cmd)
section = [
f"Command: {cmd_text}",
f"Return code: {rc}",
"STDOUT:",
out.strip() or "<empty>",
"STDERR:",
err.strip() or "<empty>",
]
debug_sections.append("\n".join(section))
# Parse explicit sensor-get output first.
if args[:2] == ["sensor", "get"] and out:
m = re.search(r"Sensor\s+Reading\s*:\s*(-?\d+(?:\.\d+)?)\b.*degrees\s*C", out, re.IGNORECASE)
if m:
return int(float(m.group(1)))
# Parse table outputs that have pipe-delimited sensor rows.
for line in out.splitlines():
parts = [p.strip() for p in line.split("|")]
if len(parts) < 2:
continue
sensor_name = parts[0]
normalized_name = " ".join(sensor_name.lower().split())
if not (normalized_name == "exhaust temp" or normalized_name.startswith("exhaust temp ")):
continue
for part in parts[1:]:
m = re.search(r"(-?\d+(?:\.\d+)?)\s*degrees\s*C", part, re.IGNORECASE)
if m:
return int(float(m.group(1)))
if session_error_seen:
raise RuntimeError(
"Failed to establish an IPMI session. "
"Check host/credentials, then try different options like "
"--interface lan --cipher 3 or --interface lanplus --cipher 17.\n\n"
+ "Debug output from attempted commands:\n\n"
+ "\n\n---\n\n".join(debug_sections)
)
raise RuntimeError(
"Failed to read Exhaust Temp from ipmitool output.\n"
+ "Debug output from attempted commands:\n\n"
+ "\n\n---\n\n".join(debug_sections)
)
def main():
# parse CLI args (IPMI host can be overridden with -H/--host)
parser = argparse.ArgumentParser(description="Set fan speed via IPMI")
parser.add_argument("-H", "--host", required=True, help="IPMI host/IP")
parser.add_argument(
"-c",
"--credentials",
default=DEFAULT_CREDENTIALS_FILE,
help="Path to credentials file (default: %(default)s)",
)
parser.add_argument(
"--keyfile",
default=None,
help="Path to Fernet key file for encrypted passwords (default: auto-detect next to credentials)",
)
parser.add_argument("--interface", default="lanplus", choices=["lanplus", "lan"], help="IPMI interface (default: %(default)s)")
parser.add_argument("--cipher", type=int, default=None, help="Optional IPMI cipher suite ID (e.g. 3 or 17)")
args = parser.parse_args()
global IPMIHOST, IPMIINTERFACE, IPMICIPHER, IPMIUSER, IPMIPW, IPMIEK
# set globals so run_ipmi() uses provided connection settings and credentials
IPMIHOST = args.host
IPMIINTERFACE = args.interface
IPMICIPHER = args.cipher
credentials_path = resolve_credentials_path(args.credentials)
keyfile_path = resolve_keyfile(args.keyfile, credentials_path)
fernet = load_fernet(keyfile_path) if keyfile_path else None
IPMIUSER, IPMIPW, IPMIEK = load_credentials(credentials_path, fernet)
ts_print(f"{IPMIHOST}: Fan speed check for host {IPMIHOST}")
try:
temp = get_exhaust_temp()
ts_print(f"{IPMIHOST}: Exhaust temperature: {temp} C")
except Exception as e:
ts_print(f"{IPMIHOST}: Error reading temperature: {e}")
sys.exit(1)
speed_hex = format(FANSPEED, "x")
if temp > MAXTEMP:
msg = f"{IPMIHOST}:Warning: Temperature is too high! Activating dynamic fan control! ({temp} C)"
log_systemd("R710-IPMI-TEMP", msg)
ts_print(msg)
run_ipmi(["raw", "0x30", "0x30", "0x01", "0x01"])
else:
okmsg = f"{IPMIHOST}: Temperature is OK ({temp} C)"
log_systemd("R710-IPMI-TEMP", okmsg)
ts_print(okmsg)
ts_print(f"{IPMIHOST}: Activating manual fan speeds! (requested %d%%)" % FANSPEED)
run_ipmi(["raw", "0x30", "0x30", "0x01", "0x00"])
run_ipmi(["raw", "0x30", "0x30", "0x02", "0xff", "0x" + speed_hex])
if __name__ == "__main__":
main()
personal/cron_scripts/ilo_status.py
-146
View File
@@ -1,146 +0,0 @@
#!/usr/bin/env python3
import argparse
import configparser
import os
import sys
import urllib3
import redfish
from datetime import datetime
try:
from cryptography.fernet import Fernet, InvalidToken
except ImportError:
sys.exit("Error: 'cryptography' package is required. Run: pip install cryptography")
DEFAULT_CREDENTIALS_FILE = "/root/cron_serverlists/credentials.ini"
DEFAULT_KEYFILE_NAME = "secret.key"
def log(message, is_error=False):
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
stream = sys.stderr if is_error else sys.stdout
print(f"[{timestamp}] {message}", file=stream)
def resolve_credentials_path(cli_path):
"""Resolve credentials path. Prefer explicit CLI path, then search common locations."""
if cli_path and os.path.exists(cli_path):
return cli_path
candidates = []
if cli_path:
candidates.append(cli_path)
candidates.append(os.path.join(os.getcwd(), "credentials.ini"))
if DEFAULT_CREDENTIALS_FILE not in candidates:
candidates.append(DEFAULT_CREDENTIALS_FILE)
for candidate in candidates:
if os.path.exists(candidate):
return candidate
log(
"Error: credentials file not found. Looked in: " + ", ".join(candidates),
is_error=True,
)
sys.exit(1)
def resolve_keyfile(cli_keyfile, credentials_path):
"""Resolve keyfile path. Prefer explicit --keyfile, else search next to credentials file."""
if cli_keyfile:
return cli_keyfile if os.path.exists(cli_keyfile) else None
candidate = os.path.join(os.path.dirname(credentials_path), DEFAULT_KEYFILE_NAME)
if os.path.exists(candidate):
return candidate
return None
def load_fernet(keyfile):
"""Load Fernet instance from the key file created by set_password.py."""
if not os.path.exists(keyfile):
log(f"Error: key file not found: {keyfile}", is_error=True)
sys.exit(1)
with open(keyfile, "rb") as fh:
key = fh.read()
try:
return Fernet(key)
except ValueError:
log(f"Error: invalid key format in key file: {keyfile}", is_error=True)
sys.exit(1)
def is_fernet_token(value):
return isinstance(value, str) and value.startswith("gAAAA")
def decrypt_password(encrypted_password, fernet):
"""Decrypt password encrypted by set_password.py, or pass through plaintext."""
if not is_fernet_token(encrypted_password):
return encrypted_password
if fernet is None:
log(
"Error: encrypted password detected but key file was not found. "
"Use --keyfile /path/to/secret.key.",
is_error=True,
)
sys.exit(1)
try:
return fernet.decrypt(encrypted_password.encode()).decode()
except InvalidToken:
log("Error: failed to decrypt password. Check that --keyfile matches the credentials file.", is_error=True)
sys.exit(1)
def load_ilo_credentials(path, fernet):
config = configparser.ConfigParser()
config.read(path)
section = "ilo_credentials"
if not config.has_section(section):
log(f"Error: missing [{section}] section in {path}", is_error=True)
sys.exit(1)
username = config.get(section, "username", fallback="root")
encrypted_password = config.get(section, "password", fallback=None)
if not encrypted_password:
log(f"Error: missing password in [{section}] section of {path}", is_error=True)
sys.exit(1)
password = decrypt_password(encrypted_password, fernet)
return username, password
# Suppress SSL warnings for self-signed iLO certificates
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
parser = argparse.ArgumentParser(description="Get the power status of an HPE server via iLO Redfish")
parser.add_argument("hostname", help="iLO hostname or IP address")
parser.add_argument("-c", "--credentials", default=None,
help="Path to credentials file (default: auto-detected credentials.ini)")
parser.add_argument("--keyfile", default=None,
help="Path to Fernet key file for decrypting passwords (default: auto-detected secret.key)")
args = parser.parse_args()
cred_path = resolve_credentials_path(args.credentials)
keyfile = resolve_keyfile(args.keyfile, cred_path)
fernet = load_fernet(keyfile) if keyfile else None
LOGIN_ACCOUNT, LOGIN_PASSWORD = load_ilo_credentials(cred_path, fernet)
BASE_URL = f"https://{args.hostname}"
REDFISHOBJ = redfish.RedfishClient(
base_url=BASE_URL,
username=LOGIN_ACCOUNT,
password=LOGIN_PASSWORD,
)
REDFISHOBJ.login(auth="session")
sys_response = REDFISHOBJ.get("/redfish/v1/Systems/1/")
power_state = sys_response.dict.get("PowerState", "Unknown")
log(f"Power State: {power_state}")
REDFISHOBJ.logout()
personal/cron_scripts/multi_powercycle.py
-358
View File
@@ -1,358 +0,0 @@
#!/usr/bin/env python3
import argparse
import configparser
import os
import subprocess
import sys
from datetime import datetime
import redfish
import urllib3
try:
from cryptography.fernet import Fernet, InvalidToken
except ImportError:
sys.exit("Error: 'cryptography' package is required. Run: pip install cryptography")
DEFAULT_CREDENTIALS_FILE = "/root/cron_serverlists/credentials.ini"
DEFAULT_KEYFILE_NAME = "/root/cron_serverlists/secret.key"
DEFAULT_SERVER_LIST_FILE = "/root/cron_serverlists/serverlist.ini"
# Examples:
# ./multi_powercycle.py on
# ./multi_powercycle.py on --keyfile /root/cron_serverlists/secret.key
# ./multi_powercycle.py off -f /root/cron_serverlists/serverlist.ini --keyfile /root/cron_serverlists/secret.key
# ./multi_powercycle.py on
# ./multi_powercycle.py off
# ./multi_powercycle.py on -f server_groups.txt
#
# File format:
# [ipmi]
# 192.168.129.240
# 192.168.129.241
#
# [ilo]
# 192.168.129.242
DEFAULT_EK = "0000000000000000000000000000000000000000"
def load_fernet(keyfile: str):
"""Load Fernet instance from the key file created by set_password.py."""
if not os.path.exists(keyfile):
log(f"Error: key file not found: {keyfile}", is_error=True)
sys.exit(1)
with open(keyfile, "rb") as fh:
key = fh.read()
try:
return Fernet(key)
except ValueError:
log(f"Error: invalid key format in key file: {keyfile}", is_error=True)
sys.exit(1)
def resolve_keyfile(cli_keyfile, ipmi_credentials_path, ilo_credentials_path):
"""Resolve keyfile path. Prefer explicit --keyfile, else search next to credentials files."""
if cli_keyfile:
return cli_keyfile if os.path.exists(cli_keyfile) else None
candidates = []
for cred_path in (ipmi_credentials_path, ilo_credentials_path):
key_candidate = os.path.join(os.path.dirname(cred_path), DEFAULT_KEYFILE_NAME)
if key_candidate not in candidates:
candidates.append(key_candidate)
# Backward-compatible fallback
legacy_default = os.path.join(os.path.dirname(DEFAULT_CREDENTIALS_FILE), DEFAULT_KEYFILE_NAME)
if legacy_default not in candidates:
candidates.append(legacy_default)
for candidate in candidates:
if os.path.exists(candidate):
return candidate
return None
def resolve_credentials_path(cli_path, server_file_path):
"""Resolve credentials path. Prefer explicit CLI path, then search common locations."""
if cli_path and os.path.exists(cli_path):
return cli_path
candidates = []
if cli_path:
candidates.append(cli_path)
# Credentials next to server list file is a common deployment pattern
candidates.append(os.path.join(os.path.dirname(server_file_path), "credentials.ini"))
# Current working directory fallback
candidates.append(os.path.join(os.getcwd(), "credentials.ini"))
# Legacy default fallback
if DEFAULT_CREDENTIALS_FILE not in candidates:
candidates.append(DEFAULT_CREDENTIALS_FILE)
for candidate in candidates:
if os.path.exists(candidate):
return candidate
log(
"Error: credentials file not found. Looked in: " + ", ".join(candidates),
is_error=True,
)
sys.exit(1)
def is_fernet_token(value: str) -> bool:
"""Best-effort check for Fernet token format from set_password.py."""
return isinstance(value, str) and value.startswith("gAAAA")
def decrypt_password(encrypted_password: str, fernet):
"""Decrypt password encrypted by set_password.py, or pass through plaintext."""
if not is_fernet_token(encrypted_password):
return encrypted_password
if fernet is None:
log(
"Error: encrypted password detected but key file was not found. "
"Use --keyfile /path/to/secret.key.",
is_error=True,
)
sys.exit(1)
try:
return fernet.decrypt(encrypted_password.encode()).decode()
except InvalidToken:
log("Error: failed to decrypt password. Check that --keyfile matches the credentials file.", is_error=True)
sys.exit(1)
def log(message, is_error=False):
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
stream = sys.stderr if is_error else sys.stdout
print(f"[{timestamp}] {message}", file=stream)
def parse_server_groups(path):
if not os.path.exists(path):
log(f"Error: server list file not found: {path}", is_error=True)
sys.exit(1)
groups = {"ipmi": [], "ilo": []}
current_group = None
aliases = {
"ipmi": "ipmi",
"dell": "ipmi",
"ilo": "ilo",
}
try:
with open(path, "r", encoding="utf-8") as handle:
for raw_line in handle:
line = raw_line.strip()
if not line or line.startswith("#"):
continue
if line.startswith("[") and line.endswith("]"):
section_name = line[1:-1].strip().lower()
current_group = aliases.get(section_name)
if current_group is None:
log(f"Error: unsupported server group [{section_name}] in {path}", is_error=True)
sys.exit(1)
continue
if current_group is None:
log(f"Error: host entry found before any section header in {path}: {line}", is_error=True)
sys.exit(1)
groups[current_group].append(line)
except OSError as exc:
log(f"Error reading server list file: {exc}", is_error=True)
sys.exit(1)
return groups
def load_ipmi_credentials(path, fernet):
if not os.path.exists(path):
log(f"Error: IPMI credentials file not found: {path}", is_error=True)
sys.exit(1)
config = configparser.ConfigParser()
config.read(path)
section = "ipmi_credentials"
if not config.has_section(section):
log(f"Error reading IPMI credentials file: missing [{section}] section in {path}", is_error=True)
sys.exit(1)
username = config.get(section, "username", fallback="root")
encrypted_password = config.get(section, "password", fallback=None)
encryption_key = config.get(
section,
"ipmiek",
fallback=config.get(section, "encryption_key", fallback=DEFAULT_EK),
)
if not encrypted_password:
log(f"Error reading IPMI credentials file: missing password in [{section}] section of {path}", is_error=True)
sys.exit(1)
password = decrypt_password(encrypted_password, fernet)
return username, password, encryption_key
def load_ilo_credentials(path, fernet):
if not os.path.exists(path):
log(f"Error: iLO credentials file not found: {path}", is_error=True)
sys.exit(1)
config = configparser.ConfigParser()
config.read(path)
section = "ilo_credentials"
if not config.has_section(section):
log(f"Error reading iLO credentials file: missing [{section}] section in {path}", is_error=True)
sys.exit(1)
username = config.get(section, "username", fallback="root")
encrypted_password = config.get(section, "password", fallback=None)
if not encrypted_password:
log(f"Error reading iLO credentials file: missing password in [{section}] section of {path}", is_error=True)
sys.exit(1)
password = decrypt_password(encrypted_password, fernet)
return username, password
def ipmi_power(host, action, username, password, encryption_key):
cmd = [
"ipmitool",
"-I",
"lanplus",
"-H",
host,
"-U",
username,
"-P",
password,
"-y",
encryption_key,
"power",
action,
]
log(f"Sending IPMI power {action} to {host}")
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if proc.returncode != 0:
detail = proc.stderr.strip() or proc.stdout.strip() or "unknown error"
log(f"IPMI {action} failed for {host}: {detail}", is_error=True)
return False
detail = proc.stdout.strip() or f"power {action} request accepted"
log(f"IPMI {action} succeeded for {host}: {detail}")
return True
def ilo_powercycle(host, username, password):
base_url = f"https://{host}"
client = redfish.RedfishClient(base_url=base_url, username=username, password=password)
try:
client.login(auth="session")
sys_response = client.get("/redfish/v1/Systems/1/")
reset_uri = sys_response.dict["Actions"]["#ComputerSystem.Reset"]["target"]
response = client.post(reset_uri, body={"ResetType": "PushPowerButton"})
log(f"iLO powercycle status for {host}: {response.status}")
if response.status >= 400:
log(f"iLO powercycle failed for {host}: {response.read}", is_error=True)
return False
return True
except Exception as exc:
log(f"iLO powercycle failed for {host}: {exc}", is_error=True)
return False
finally:
try:
client.logout()
except Exception:
pass
def main():
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
parser = argparse.ArgumentParser(
description="Dispatch power operations to IPMI hosts and iLO hosts from a grouped server list"
)
parser.add_argument("action", choices=("on", "off"), help="Power action for IPMI hosts")
parser.add_argument(
"-f",
"--file",
dest="server_file",
default=DEFAULT_SERVER_LIST_FILE,
help=f"Path to server list file (default: {DEFAULT_SERVER_LIST_FILE})",
)
parser.add_argument(
"--ipmi-credentials",
default=DEFAULT_CREDENTIALS_FILE,
help=f"Path to IPMI credentials file (default: {DEFAULT_CREDENTIALS_FILE})",
)
parser.add_argument(
"--ilo-credentials",
default=DEFAULT_CREDENTIALS_FILE,
help=f"Path to iLO credentials file (default: {DEFAULT_CREDENTIALS_FILE})",
)
parser.add_argument(
"--keyfile",
default=None,
help="Path to Fernet key file created by set_password.py (default: auto-detect next to credentials file)",
)
args = parser.parse_args()
args.ipmi_credentials = resolve_credentials_path(args.ipmi_credentials, args.server_file)
args.ilo_credentials = resolve_credentials_path(args.ilo_credentials, args.server_file)
keyfile = resolve_keyfile(args.keyfile, args.ipmi_credentials, args.ilo_credentials)
fernet = load_fernet(keyfile) if keyfile else None
groups = parse_server_groups(args.server_file)
total_hosts = len(groups["ipmi"]) + len(groups["ilo"])
if total_hosts == 0:
log(f"Error: no servers found in {args.server_file}", is_error=True)
sys.exit(1)
log(
f"Starting run for {total_hosts} host(s): {len(groups['ipmi'])} IPMI host(s), {len(groups['ilo'])} iLO host(s)"
)
ipmi_creds = None
ilo_creds = None
failures = 0
if groups["ipmi"]:
ipmi_creds = load_ipmi_credentials(args.ipmi_credentials, fernet)
for host in groups["ipmi"]:
if not ipmi_power(host, args.action, *ipmi_creds):
failures += 1
if groups["ilo"]:
ilo_creds = load_ilo_credentials(args.ilo_credentials, fernet)
for host in groups["ilo"]:
if not ilo_powercycle(host, *ilo_creds):
failures += 1
successes = total_hosts - failures
log(f"Completed run: {successes} successful, {failures} failed")
if failures:
sys.exit(1)
if __name__ == "__main__":
main()
personal/cron_scripts/pg_backup.sh
-148
View File
@@ -1,148 +0,0 @@
#!/bin/bash
# Configuration
# If started by root (e.g., root crontab), re-run as postgres.
if [ "$(id -u)" -eq 0 ] && [ "$(id -un)" != "postgres" ]; then
if command -v runuser >/dev/null 2>&1; then
exec runuser -u postgres -- "$0" "$@"
fi
# Fallback for systems without runuser.
exec su - postgres -s /bin/bash -c "$(printf '%q ' "$0" "$@")"
fi
# Avoid permission warnings when root launches this script and postgres inherits /root as cwd.
cd / || exit 1
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
: "${WAL_CLEANUP_ENABLED:=1}"
WAL_ARCHIVE_DIR="/var/lib/postgresql_archive/18/main/pg_wal"
WAL_MIN_AGE_DAYS=7
clean_walfiles() {
local newest_backup=""
local newest_backup_ts=""
local newest_backup_dt=""
local age_cutoff_ts=""
local age_cutoff_dt=""
local backup_ts=""
local file_ts=""
local deleted=0
local backup_file=""
local file=""
local -a backup_files=()
local -a files_to_delete=()
if [[ ! -d "$WAL_ARCHIVE_DIR" ]]; then
log "Archive directory does not exist: $WAL_ARCHIVE_DIR" >&2
return 1
fi
if ! [[ "$WAL_MIN_AGE_DAYS" =~ ^[0-9]+$ ]]; then
log "WAL_MIN_AGE_DAYS must be a non-negative integer" >&2
return 1
fi
mapfile -d '' backup_files < <(find "$WAL_ARCHIVE_DIR" -maxdepth 1 -type f -name '*.backup' -print0)
if [[ ${#backup_files[@]} -eq 0 ]]; then
log "No .backup files found in $WAL_ARCHIVE_DIR; skipping WAL cleanup."
return 0
fi
for backup_file in "${backup_files[@]}"; do
backup_ts=$(stat -c %Y "$backup_file")
if [[ -z "$newest_backup_ts" || "$backup_ts" -gt "$newest_backup_ts" ]]; then
newest_backup_ts="$backup_ts"
newest_backup="$backup_file"
fi
done
age_cutoff_ts=$(date -d "$WAL_MIN_AGE_DAYS days ago" +%s)
newest_backup_dt=$(date -d "@$newest_backup_ts" '+%Y-%m-%d %H:%M:%S')
age_cutoff_dt=$(date -d "@$age_cutoff_ts" '+%Y-%m-%d %H:%M:%S')
log "WAL archive dir : $WAL_ARCHIVE_DIR"
log "Newest .backup file : $newest_backup"
log "Newest .backup mtime : $newest_backup_dt"
log "WAL age cutoff : $age_cutoff_dt"
while IFS= read -r -d '' file; do
file_ts=$(stat -c %Y "$file")
# Delete only when both constraints are true.
if [[ "$file_ts" -lt "$newest_backup_ts" && "$file_ts" -lt "$age_cutoff_ts" ]]; then
files_to_delete+=("$file")
fi
done < <(find "$WAL_ARCHIVE_DIR" -maxdepth 1 -type f ! -name '*.backup' -print0)
# Also delete old .backup files (except the newest one) that exceed retention period
for backup_file in "${backup_files[@]}"; do
if [[ "$backup_file" != "$newest_backup" ]]; then
backup_ts=$(stat -c %Y "$backup_file")
if [[ "$backup_ts" -lt "$age_cutoff_ts" ]]; then
files_to_delete+=("$backup_file")
fi
fi
done
if [[ ${#files_to_delete[@]} -eq 0 ]]; then
log "No WAL files matched cleanup criteria."
return 0
fi
echo ""
log "WAL files matching cleanup criteria (${#files_to_delete[@]}):"
for file in "${files_to_delete[@]}"; do
log " $file"
done
for file in "${files_to_delete[@]}"; do
rm -f -- "$file"
((deleted += 1))
done
echo ""
log "Deleted $deleted WAL file(s)."
}
log "Beginning base backup."
BASEBACKUP_ROOT="/var/lib/postgresql_archive/pgbackups"
BACKUP_DIR="/var/lib/postgresql_archive/pgbackups/basebackup"
mkdir -p "$BACKUP_DIR"
# Perform base backup with WAL streaming
pg_basebackup -h localhost -U wjones -D "$BACKUP_DIR" -Fp -Xs -P
# Optional: Compress
BASEBACKUP_ARCHIVE="${BACKUP_DIR}_$(date +%Y%m%d_%H%M%S).tar.gz"
tar -czf "$BASEBACKUP_ARCHIVE" -C "$(dirname "$BACKUP_DIR")" "$(basename "$BACKUP_DIR")"
rm -rf "$BACKUP_DIR"
# Retention: Delete backups older than 7 days
find "$BASEBACKUP_ROOT" -type f -name "*.tar.gz" -mtime +7 -delete
log "Base backup completed.Now backing up individual databases."
BACKUP_DIR="/var/lib/postgresql_archive/pg_dumpfiles"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
RETENTION_DAYS=7
# Ensure directory exists
mkdir -p "$BACKUP_DIR"
# Get all non-system databases and back each one up.
psql -U postgres -d postgres -Atc "SELECT datname FROM pg_database WHERE datistemplate = false AND datname <> 'postgres' ORDER BY datname" | while IFS= read -r DB_NAME; do
pg_dump -U postgres -d "$DB_NAME" | gzip > "$BACKUP_DIR/${DB_NAME}_$TIMESTAMP.sql.gz"
log "Backup completed: ${DB_NAME}_$TIMESTAMP.sql.gz"
done
# Delete backups older than retention period
find "$BACKUP_DIR" -type f -mtime +"$RETENTION_DAYS" -delete
log "Individual database backups completed and old backups cleaned up."
if [[ "$WAL_CLEANUP_ENABLED" -eq 1 ]]; then
log "Starting WAL archive cleanup."
clean_walfiles
else
log "WAL archive cleanup disabled; skipping final cleanup."
fi
log "Backup workflow completed."
personal/cron_scripts/power/ilo_poweroff.py
-38
View File
@@ -1,38 +0,0 @@
#!/usr/bin/env python3
import argparse
import subprocess
import getpass
import sys
import datetime
def ts_print(msg):
"""ts_print message prefixed with a timestamp (YYYY-MM-DD HH:MM:SS)."""
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(f"{now} {msg}")
def run(cmd):
try:
subprocess.run(cmd, check=True)
except subprocess.CalledProcessError as e:
print(f"Command failed: {' '.join(cmd)}", file=sys.stderr)
sys.exit(e.returncode)
def main():
p = argparse.ArgumentParser(description="Run iLO REST commands (login, serverinfo, reboot)")
p.add_argument("-H", "--host", required=True, help="iLO host/IP")
p.add_argument("-a", "--action", default="ForceOff", choices=["ForceOff", "GracefulRestart", "Reset"],
help="reboot action to run (default: ForceOff)")
p.add_argument("--no-serverinfo", action="store_true", help="skip running serverinfo")
args = p.parse_args()
# login
ts_print(f"Sending {args.action} to {args.host}")
run(['ilorest', 'login', args.host, '-u', '<username>', '-p', '<password>'])
# optional serverinfo
if not args.no_serverinfo:
run(['ilorest', 'serverinfo'])
# power off / reboot action
run(['ilorest', 'reboot', args.action])
if __name__ == "__main__":
main()
personal/cron_scripts/power/ilo_poweron.py
-35
View File
@@ -1,35 +0,0 @@
#!/usr/bin/env python3
import argparse
import subprocess
import getpass
import sys
import datetime
def ts_print(msg):
"""ts_print message prefixed with a timestamp (YYYY-MM-DD HH:MM:SS)."""
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(f"{now} {msg}")
def run(cmd):
try:
subprocess.run(cmd, check=True)
except subprocess.CalledProcessError as e:
print(f"Command failed: {' '.join(cmd)}", file=sys.stderr)
sys.exit(e.returncode)
def main():
p = argparse.ArgumentParser(description="Login to iLO, show serverinfo and power on")
p.add_argument("-H", "--host", required=True, help="iLO host/IP")
p.add_argument("-a", "--action", default="On", choices=["On","Off","GracefulRestart","ForceOff","Reset"],
help="power/reboot action to run (default: On)")
p.add_argument("--no-serverinfo", action="store_true", help="skip running serverinfo")
args = p.parse_args()
ts_print(f"Sending {args.action} to {args.host}")
run(['ilorest', 'login', args.host, '-u', '<username>', '-p', '<password>'])
if not args.no_serverinfo:
run(['ilorest', 'serverinfo'])
run(['ilorest', 'reboot', args.action])
if __name__ == "__main__":
main()
personal/cron_scripts/power/ipmi_poweroff.py
-31
View File
@@ -1,31 +0,0 @@
#!/usr/bin/env python3
import argparse
import subprocess
import sys
IPMIUSER = "<username>"
IPMIPW = "<password>"
IPMIEK = "0000000000000000000000000000000000000000"
def main():
p = argparse.ArgumentParser(description="Send IPMI power off")
p.add_argument("-H", "--host", required=True, help="IPMI host address")
args = p.parse_args()
cmd = [
"ipmitool",
"-I", "lanplus",
"-H", args.host,
"-U", IPMIUSER,
"-P", IPMIPW,
"-y", IPMIEK,
"power", "off",
]
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if proc.returncode != 0:
print("ipmitool failed:", proc.stderr.strip(), file=sys.stderr)
sys.exit(proc.returncode)
print(proc.stdout.strip())
if __name__ == "__main__":
main()
personal/cron_scripts/power/ipmi_poweron.py
-38
View File
@@ -1,38 +0,0 @@
#!/usr/bin/env python3
import argparse
import subprocess
import sys
import datetime
def ts_print(msg):
"""ts_print message prefixed with a timestamp (YYYY-MM-DD HH:MM:SS)."""
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(f"{now} {msg}")
IPMIUSER = "<username>"
IPMIPW = "<password>"
IPMIEK = "0000000000000000000000000000000000000000"
def main():
p = argparse.ArgumentParser(description="Send IPMI power off")
p.add_argument("-H", "--host", required=True, help="IPMI host address")
args = p.parse_args()
cmd = [
"ipmitool",
"-I", "lanplus",
"-H", args.host,
"-U", IPMIUSER,
"-P", IPMIPW,
"-y", IPMIEK,
"power", "on",
]
print(f"Sending power on to {args.host}")
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if proc.returncode != 0:
print("ipmitool failed:", proc.stderr.strip(), file=sys.stderr)
sys.exit(proc.returncode)
print(proc.stdout.strip())
if __name__ == "__main__":
main()
personal/cron_scripts/system/update_system.py
-45
View File
@@ -1,45 +0,0 @@
#!/usr/bin/python3
import subprocess
import os
from datetime import datetime, timezone
from typing import Optional
def generate_timestamp(fmt: Optional[str] = None, utc: bool = False) -> str:
"""Return a timestamp string.
- If `fmt` is provided, return `datetime.strftime(fmt)`.
- If `fmt` is None, return an ISO 8601 timestamp. When `utc` is True
the timestamp ends with 'Z' (UTC). When `utc` is False it returns
the local time ISO format.
Examples:
- `generate_timestamp()` -> '2025-11-20T12:34:56Z'
- `generate_timestamp("%Y-%m-%d %H:%M:%S")` -> '2025-11-20 12:34:56'
"""
if fmt:
dt = datetime.now(timezone.utc) if utc else datetime.now()
return dt.strftime(fmt)
dt = datetime.now(timezone.utc) if utc else datetime.now()
if utc:
# force trailing Z for UTC instead of +00:00
return dt.replace(tzinfo=timezone.utc).isoformat().replace('+00:00', 'Z')
return dt.isoformat()
print(f"Starting system update at {generate_timestamp()}")
# Update package lists
print(f"{generate_timestamp()}: Updating package lists...")
subprocess.run(['sudo', 'apt', 'update'], check=True)
# Upgrade packages
print(f"{generate_timestamp()}: Upgrading packages...")
subprocess.run(['sudo', 'apt', 'upgrade', '-y'], check=True)
# Check if reboot is required
if os.path.exists('/var/run/reboot-required'):
print(f"{generate_timestamp()}: Reboot required. Rebooting now...")
subprocess.run(['sudo', 'reboot'])
else:
print(f"{generate_timestamp()}: No reboot required.")
personal/fancontrol/setspeed_fanspeed.sh
+41
View File
@@ -0,0 +1,41 @@
#!/bin/bash
# IPMI IP address
IPMIHOST=<hostname>
# IPMI Username
IPMIUSER=<username
# IPMI Password
IPMIPW=<password>
# Your IPMI Encryption Key
IPMIEK=0<IPMI Encryption Key>
# Fan Speed / utilisation in percentage, for example 9 for 9% utilisation
# Please note that each fan can have a different rpm and will not all be the same speed
FANSPEED=10
# TEMPERATURE
# Change this to the temperature in Celsius you are comfortable with.
# If the temperature goes above the set degrees it will send raw IPMI command to enable dynamic fan control
MAXTEMP=30
# This variable sends a IPMI command to get the temperature, and outputs it as two digits.
# Do not edit unless you know what you do.
# Side note, if you are running ipmitool on the system you are controlling, you don't need to specify -H,-U,-P - from the OS installed on the host, ipmitool is assumed permitted. You only need host/user/pass for remote access.
TEMP=$(ipmitool -I lanplus -H $IPMIHOST -U $IPMIUSER -P $IPMIPW -y $IPMIEK sdr type temperature |grep Ambient |grep degrees |grep -Po '\d{2}' | tail -1)
# Dont edit this, this converts decimal to hex
SPEEDHEX=$( printf "%x" $FANSPEED )
if [[ $TEMP > $MAXTEMP ]];
then
printf "Warning: Temperature is too high! Activating dynamic fan control! ($TEMP C)" | systemd-cat -t R710-IPMI-TEMP
echo "Warning: Temperature is too high! Activating dynamic fan control! ($TEMP C)"
# This sets the fans to auto mode, so the motherboard will set it to a speed that it will need do cool the server down
ipmitool -I lanplus -H $IPMIHOST -U $IPMIUSER -P $IPMIPW -y $IPMIEK raw 0x30 0x30 0x01 0x01
else
printf "Temperature is OK ($TEMP C)" | systemd-cat -t R710-IPMI-TEMP
printf "Activating manual fan speeds! (1560 RPM)"
# This sets the fans to manual mode
ipmitool -I lanplus -H $IPMIHOST -U $IPMIUSER -P $IPMIPW -y $IPMIEK raw 0x30 0x30 0x01 0x00
# This is where we set the slower, quite speed
ipmitool -I lanplus -H $IPMIHOST -U $IPMIUSER -P $IPMIPW -y $IPMIEK raw 0x30 0x30 0x02 0xff 0x$SPEEDHEX
fi
personal/power/poweroff.sh
+2
View File
@@ -0,0 +1,2 @@
#!/bin/bash
ipmitool -I lanplus -H 192.168.128.223 -U root -P Optimus0329 -y 0000000000000000000000000000000000000000 power off
personal/power/poweron.sh
+2
View File
@@ -0,0 +1,2 @@
#!/bin/bash
ipmitool -I lanplus -H 192.168.128.223 -U root -P Optimus0329 -y 0000000000000000000000000000000000000000 power on
personal/cron_scripts/system/vm_powerup.py → personal/power/vm_powerup.py
View File
personal/system/start_clients.sh
+14
View File
@@ -0,0 +1,14 @@
#!/bin/bash
echo "Client Type: $1"
echo "Client IDs: $2"
client_string=$2
client_list=()
IFS=', ' read -a array <<< "${client_string}"; echo "${array[@]}"
for index in "${!array[@]}"
do
echo "Starting container: ${array[index]}"
lxc-start ${array[index]}
done
personal/cron_scripts/system/start_wrapper.sh → personal/system/start_wrapper.sh
View File
personal/cron_scripts/system/stop_wrapper.sh → personal/system/stop_wrapper.sh
View File
personal/cron_scripts/system/vm_restart.py → personal/system/vm_restart.py
View File
personal/cron_scripts/system/vm_shutdown.py.v1 → personal/system/vm_shutdown.py.v1
View File
personal/cron_scripts/system/vm_start.py → personal/system/vm_start.py
View File
personal/cron_scripts/system/vm_stop.py → personal/system/vm_stop.py
+3 -2
View File
@@ -12,11 +12,11 @@ def get_time():
def client_shutdown(client_type, clientid): def client_shutdown(client_type, clientid):
lxc_filepath = "/usr/bin/lxc-stop" lxc_filepath = "/usr/bin/lxc-stop"
qemu_filepath = "/usr/sbin/qm" qemu_filepath = f"/usr/sbin/qm shutdown {clientid}"
try: try:
print(f'Stopping {clientid}') print(f'Stopping {clientid}')
if client_type == 'vm': if client_type == 'vm':
process_id = os.spawnv(os.P_NOWAIT, qemu_filepath, ['shutdown', clientid]) process_id = os.spawnv(os.P_NOWAIT, qemu_filepath, ['--skip-lock', 'true'])
elif client_type == 'lxc': elif client_type == 'lxc':
process_id = os.spawnv(os.P_NOWAIT, lxc_filepath, [client_type, clientid]) process_id = os.spawnv(os.P_NOWAIT, lxc_filepath, [client_type, clientid])
print(process_id) print(process_id)
@@ -37,6 +37,7 @@ def main():
print(f'Start time: {timestamp}') print(f'Start time: {timestamp}')
for clientid in client_list: for clientid in client_list:
client_shutdown(client_type, clientid) client_shutdown(client_type, clientid)
sleep(1)
else: else:
print(f'Successfully stopped client list.') print(f'Successfully stopped client list.')
personal/cron_scripts/system/vm_stop.py.v1 → personal/system/vm_stop.py.v1
View File
personal/tools/airsonic_pg_diagnostics.sh
-276
View File
@@ -1,276 +0,0 @@
#!/bin/bash
set -euo pipefail
# Airsonic Advanced + PostgreSQL diagnostics
# Focus: slow media scans, index efficiency, write pressure, and cache tuning signals.
PGHOST="${PGHOST:-localhost}"
PGPORT="${PGPORT:-5432}"
PGUSER="${PGUSER:-postgres}"
PGDATABASE="${PGDATABASE:-airsonic}"
OUT_FILE="${1:-airsonic_pg_diagnostics_$(date +%Y%m%d_%H%M%S).txt}"
PSQL=(psql -X -v ON_ERROR_STOP=1 -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE")
run_section() {
local title="$1"
local sql="$2"
{
echo
echo "================================================================"
echo "$title"
echo "================================================================"
} >> "$OUT_FILE"
if ! "${PSQL[@]}" -c "$sql" >> "$OUT_FILE" 2>&1; then
echo "Query failed for section: $title" >> "$OUT_FILE"
fi
}
query_single_value() {
local sql="$1"
"${PSQL[@]}" -Atc "$sql" 2>/dev/null || true
}
bytes_to_human() {
local bytes="$1"
if [[ -z "$bytes" || ! "$bytes" =~ ^[0-9]+$ ]]; then
echo "unknown"
return
fi
local kib=$((1024))
local mib=$((1024 * 1024))
local gib=$((1024 * 1024 * 1024))
if (( bytes >= gib )); then
awk -v b="$bytes" -v g="$gib" 'BEGIN { printf "%.2f GiB", b/g }'
elif (( bytes >= mib )); then
awk -v b="$bytes" -v m="$mib" 'BEGIN { printf "%.2f MiB", b/m }'
elif (( bytes >= kib )); then
awk -v b="$bytes" -v k="$kib" 'BEGIN { printf "%.2f KiB", b/k }'
else
echo "${bytes} B"
fi
}
{
echo "Airsonic PostgreSQL Diagnostics"
echo "Generated at: $(date -Iseconds)"
echo "Target: host=$PGHOST port=$PGPORT db=$PGDATABASE user=$PGUSER"
} > "$OUT_FILE"
run_section "1) Server identity and uptime" "
SELECT version();
SELECT now() AS current_time, pg_postmaster_start_time() AS postmaster_start_time;
"
run_section "2) Key PostgreSQL tuning parameters" "
SELECT name, setting, unit, short_desc
FROM pg_settings
WHERE name IN (
'shared_buffers',
'work_mem',
'maintenance_work_mem',
'effective_cache_size',
'max_connections',
'effective_io_concurrency',
'random_page_cost',
'seq_page_cost',
'checkpoint_timeout',
'checkpoint_completion_target',
'autovacuum',
'autovacuum_max_workers',
'autovacuum_naptime'
)
ORDER BY name;
"
run_section "3) Database cache hit ratio and temp usage" "
SELECT
datname,
blks_read,
blks_hit,
ROUND(100.0 * blks_hit / NULLIF(blks_hit + blks_read, 0), 2) AS cache_hit_pct,
temp_files,
pg_size_pretty(temp_bytes) AS temp_bytes,
deadlocks,
stats_reset
FROM pg_stat_database
WHERE datname = current_database();
"
run_section "4) Most read tables and their index-vs-seq scan profile" "
SELECT
relname,
seq_scan,
idx_scan,
n_live_tup,
n_dead_tup,
ROUND((100.0 * idx_scan / NULLIF(idx_scan + seq_scan, 0))::numeric, 2) AS index_scan_pct,
ROUND((100.0 * heap_blks_hit / NULLIF(heap_blks_hit + heap_blks_read, 0))::numeric, 2) AS table_cache_hit_pct
FROM pg_stat_user_tables t
JOIN pg_statio_user_tables io USING (relid)
ORDER BY (seq_scan + idx_scan) DESC NULLS LAST
LIMIT 25;
"
run_section "5) Potentially unused indexes (can slow writes during scans/indexing)" "
SELECT
s.schemaname,
s.relname AS table_name,
s.indexrelname AS index_name,
s.idx_scan,
pg_size_pretty(pg_relation_size(s.indexrelid)) AS index_size,
t.n_tup_ins + t.n_tup_upd + t.n_tup_del AS table_writes_since_reset
FROM pg_stat_user_indexes s
JOIN pg_index i ON i.indexrelid = s.indexrelid
JOIN pg_stat_user_tables t ON t.relid = s.relid
WHERE s.idx_scan = 0
AND NOT i.indisunique
AND NOT i.indisprimary
ORDER BY pg_relation_size(s.indexrelid) DESC
LIMIT 50;
"
run_section "6) Write-heavy tables (hot spot during media metadata indexing)" "
SELECT
relname,
n_tup_ins,
n_tup_upd,
n_tup_del,
n_tup_hot_upd,
n_dead_tup,
vacuum_count,
autovacuum_count,
analyze_count,
autoanalyze_count,
COALESCE(last_autovacuum::text, 'never') AS last_autovacuum,
COALESCE(last_autoanalyze::text, 'never') AS last_autoanalyze
FROM pg_stat_user_tables
ORDER BY (n_tup_ins + n_tup_upd + n_tup_del) DESC
LIMIT 25;
"
run_section "7) Largest relations (tables and indexes)" "
SELECT
n.nspname AS schema_name,
c.relname,
c.relkind,
pg_size_pretty(pg_total_relation_size(c.oid)) AS total_size
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname NOT IN ('pg_catalog', 'information_schema')
AND c.relkind IN ('r', 'i')
ORDER BY pg_total_relation_size(c.oid) DESC
LIMIT 30;
"
if [[ "$(query_single_value "SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements';")" == "1" ]]; then
run_section "8) Slowest normalized SQL from pg_stat_statements" "
SELECT
calls,
ROUND(total_exec_time::numeric, 2) AS total_exec_ms,
ROUND((total_exec_time / NULLIF(calls, 0))::numeric, 2) AS mean_exec_ms,
ROUND(rows::numeric / NULLIF(calls, 0), 2) AS avg_rows,
shared_blks_read,
shared_blks_hit,
temp_blks_read,
temp_blks_written,
LEFT(query, 300) AS query_sample
FROM pg_stat_statements
WHERE dbid = (SELECT oid FROM pg_database WHERE datname = current_database())
ORDER BY mean_exec_ms DESC
LIMIT 20;
"
else
{
echo
echo "================================================================"
echo "8) Slowest normalized SQL from pg_stat_statements"
echo "================================================================"
echo "Extension pg_stat_statements is not installed."
echo "Install with: CREATE EXTENSION pg_stat_statements;"
} >> "$OUT_FILE"
fi
# Heuristic recommendation summary
CACHE_HIT_PCT="$(query_single_value "
SELECT ROUND(100.0 * blks_hit / NULLIF(blks_hit + blks_read, 0), 2)
FROM pg_stat_database
WHERE datname = current_database();")"
UNUSED_INDEX_COUNT="$(query_single_value "
SELECT COUNT(*)
FROM pg_stat_user_indexes s
JOIN pg_index i ON i.indexrelid = s.indexrelid
WHERE s.idx_scan = 0
AND NOT i.indisunique
AND NOT i.indisprimary;")"
SHARED_BUFFERS_BYTES="$(query_single_value "
SELECT pg_size_bytes(setting || unit)
FROM pg_settings
WHERE name = 'shared_buffers';")"
TOTAL_RAM_KB=""
if [[ -r /proc/meminfo ]]; then
TOTAL_RAM_KB="$(awk '/MemTotal:/ {print $2}' /proc/meminfo)"
fi
{
echo
echo "================================================================"
echo "9) Recommendations summary"
echo "================================================================"
if [[ -n "$CACHE_HIT_PCT" ]]; then
echo "Cache hit ratio: ${CACHE_HIT_PCT}%"
awk -v hit="$CACHE_HIT_PCT" 'BEGIN {
if (hit < 98.0) {
print "- Action: cache hit ratio is low for media workloads. Increase shared_buffers/effective_cache_size and review slow seq scans."
} else if (hit < 99.0) {
print "- Action: cache hit ratio is acceptable but can likely improve for large libraries."
} else {
print "- Action: cache hit ratio looks healthy."
}
}'
else
echo "Cache hit ratio: unavailable"
fi
if [[ -n "$UNUSED_INDEX_COUNT" ]]; then
echo "Potentially unused non-unique indexes: $UNUSED_INDEX_COUNT"
if [[ "$UNUSED_INDEX_COUNT" =~ ^[0-9]+$ ]] && (( UNUSED_INDEX_COUNT > 0 )); then
echo "- Action: validate and drop truly unused indexes to reduce write overhead during scans/indexing."
else
echo "- Action: no obvious unused non-unique indexes from current stats."
fi
fi
if [[ -n "$SHARED_BUFFERS_BYTES" ]]; then
echo "shared_buffers: $(bytes_to_human "$SHARED_BUFFERS_BYTES")"
if [[ -n "$TOTAL_RAM_KB" && "$TOTAL_RAM_KB" =~ ^[0-9]+$ ]]; then
TOTAL_RAM_BYTES=$((TOTAL_RAM_KB * 1024))
SHARED_BUFFER_PCT=$(awk -v s="$SHARED_BUFFERS_BYTES" -v t="$TOTAL_RAM_BYTES" 'BEGIN { printf "%.2f", (s/t)*100 }')
echo "Estimated shared_buffers/system RAM: ${SHARED_BUFFER_PCT}%"
awk -v p="$SHARED_BUFFER_PCT" 'BEGIN {
if (p < 10.0) {
print "- Action: shared_buffers appears low; typical starting point is ~15-25% of system RAM."
} else if (p > 40.0) {
print "- Action: shared_buffers appears high; ensure OS cache remains adequate."
} else {
print "- Action: shared_buffers proportion looks reasonable."
}
}'
else
echo "- Action: compare shared_buffers to system RAM manually (target often 15-25% for dedicated DB hosts)."
fi
fi
echo "- Action: for media scans, verify high-read tables use selective indexes and that autovacuum keeps dead tuples controlled."
echo "- Action: rerun this script before and during a scan to compare write pressure and cache behavior over time."
} >> "$OUT_FILE"
echo "Diagnostics complete. Report written to: $OUT_FILE"
personal/tools/batch_rsync.ps1
-536
View File
@@ -1,536 +0,0 @@
#Requires -Version 7.0
# =============================================================================
# batch_rsync.ps1 — Parallel rsync launcher for Windows PowerShell
# =============================================================================
#
# KEY IMPLEMENTATION NOTES
# ------------------------
#
# Parallelism
# Uses RunspacePool + [PowerShell]::Create() so multiple rsync processes
# run concurrently without blocking the main thread. All runspace script
# blocks are self-contained (no closures over outer variables).
#
# As-completed result collection
# WaitHandle.WaitAny() polls the async handles so results are printed as
# soon as each task finishes, matching the behaviour of Python's as_completed().
#
# --files-from (recursive-files mode)
# rsync is fed a NUL-delimited temp file via --from0 --files-from=<tmpfile>
# rather than stdin, because stdin is not reliably available inside a
# runspace. The temp file is always cleaned up in a finally block.
#
# Path handling
# All source paths are converted from Windows backslashes to forward slashes
# before being passed to rsync, which is required for MSYS2/Cygwin rsync
# to interpret them correctly (e.g. C:/data/... not C:\data\...).
#
# Argument passing to rsync
# Uses ProcessStartInfo.ArgumentList (System.Collections.Generic.List<string>)
# rather than a single argument string. This avoids quoting/escaping issues
# with paths that contain spaces or special characters.
# Requires .NET 5+ — hence the #Requires -Version 7.0 constraint.
#
# Auto-tune worker selection
# Workers are derived from average file size and destination type (local vs
# remote). Smaller files favour higher concurrency; large files and remote
# destinations are throttled to avoid saturating the link or dest I/O.
# The -Workers value is always treated as an upper bound.
#
# Size-balanced chunk partitioning (recursive-files mode)
# Files are sorted largest-first and distributed across worker buckets with
# a greedy least-loaded algorithm to minimise the slowest-worker stall.
#
# Remote destination detection
# Heuristics check for rsync://, :: (daemon), Windows drive letters, UNC
# paths, and user@host:/path patterns to decide whether the destination is
# local or remote without requiring an actual connection.
#
# Verbose output
# Uses PowerShell's built-in -Verbose switch (CmdletBinding) so there is no
# custom flag. Pass -Verbose on the command line to enable detailed output.
#
# Requirements
# - PowerShell 7.0+ (for ProcessStartInfo.ArgumentList via .NET 5+)
# - rsync on PATH (MSYS2, Cygwin, or WSL interop on Windows)
#
# =============================================================================
<#
.SYNOPSIS
Parallel rsync launcher for Windows PowerShell.
.DESCRIPTION
Runs multiple rsync invocations in parallel using either top-level entries
or balanced recursive file chunks. Requires rsync on PATH (e.g. via MSYS2
or Cygwin on Windows).
Paths are converted to forward-slash notation for rsync compatibility.
MSYS2 rsync accepts Windows absolute paths using forward slashes (C:/path/...).
.EXAMPLE
# Basic sync (recursive-files mode, auto worker count):
.\batch_rsync.ps1 C:\data user@host:/dest/
.EXAMPLE
# Auto-tune workers based on file size and local vs remote destination:
.\batch_rsync.ps1 C:\data user@host:/dest/ -AutoTune
.EXAMPLE
# Limit to a specific number of parallel workers:
.\batch_rsync.ps1 C:\data D:\backup -Workers 8
.EXAMPLE
# Use top-level mode (one rsync process per immediate source entry):
.\batch_rsync.ps1 C:\data D:\backup -Mode top-level
.EXAMPLE
# Dry run (show what would be transferred without doing it):
.\batch_rsync.ps1 C:\data user@host:/dest/ -DryRun
.EXAMPLE
# Verbose output (command templates, task previews, elapsed time):
.\batch_rsync.ps1 C:\data D:\backup -Verbose
.EXAMPLE
# Show rsync output for all tasks:
.\batch_rsync.ps1 C:\data D:\backup -ShowRsyncOutput
.EXAMPLE
# Show rsync output only when a task fails:
.\batch_rsync.ps1 C:\data user@host:/dest/ -ShowRsyncOutputOnFailure
.EXAMPLE
# Full-featured (auto-tune, verbose, show on failure):
.\batch_rsync.ps1 C:\data user@host:/dest/ -AutoTune -Verbose -ShowRsyncOutputOnFailure
.EXAMPLE
# Override rsync flags entirely:
.\batch_rsync.ps1 C:\data D:\backup -RsyncArgs '-az --checksum'
#>
[CmdletBinding()]
param (
[Parameter(Mandatory, Position = 0)]
[string]$Src,
[Parameter(Mandatory, Position = 1)]
[string]$Dest,
[ValidateRange(1, [int]::MaxValue)]
[int]$Workers = [Math]::Min(8, [Environment]::ProcessorCount),
[ValidateSet('recursive-files', 'top-level')]
[string]$Mode = 'recursive-files',
[string]$RsyncArgs = '-a --partial --info=stats2',
[switch]$AutoTune,
[switch]$ShowRsyncOutput,
[switch]$ShowRsyncOutputOnFailure,
[switch]$DryRun
)
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
#region Helper functions
function Format-Bytes {
param([double]$Bytes)
$units = @('B', 'KB', 'MB', 'GB', 'TB')
$value = $Bytes
foreach ($unit in $units) {
if ($value -lt 1024 -or $unit -eq 'TB') {
return '{0:F1}{1}' -f $value, $unit
}
$value /= 1024
}
return '{0:F1}B' -f $Bytes
}
function Test-RemoteDestination {
param([string]$Destination)
if ($Destination -match '^rsync://') { return $true }
if ($Destination -match '::') { return $true }
# Windows drive letters and UNC paths are local
if ($Destination -match '^[a-zA-Z]:[/\\]') { return $false }
if ($Destination -match '^\\\\') { return $false }
# user@host:/path or host:/path patterns are remote
return $Destination -match '^[^:/\\]+@?[^:/\\]*:[^/]'
}
function Get-AutoTunedWorkers {
param(
[int] $TotalUnits,
[double]$AvgSizeBytes,
[bool] $IsRemote,
[int] $CpuCount,
[int] $MaxWorkers
)
$cpu = [Math]::Max(1, $CpuCount)
$MB1 = 1 * 1024 * 1024
$MB32 = 32 * 1024 * 1024
$MB256 = 256 * 1024 * 1024
if ($AvgSizeBytes -lt $MB1) {
$base = [Math]::Min(32, $cpu * 4)
} elseif ($AvgSizeBytes -lt $MB32) {
$base = [Math]::Min(16, $cpu * 2)
} elseif ($AvgSizeBytes -lt $MB256) {
$base = [Math]::Min(8, $cpu)
} else {
$base = [Math]::Min(4, [Math]::Max(2, [int]($cpu / 2)))
}
if ($IsRemote) {
$base = [Math]::Max(2, [Math]::Min($base, 8))
} else {
$base = [Math]::Min($base + 2, 24)
}
$tuned = [Math]::Max(1, [Math]::Min($base, $TotalUnits))
return [Math]::Min($tuned, $MaxWorkers)
}
function Get-RecursiveFileList {
param([string]$SrcDir)
$root = $SrcDir.TrimEnd('\', '/')
Get-ChildItem -LiteralPath $root -Recurse -File |
ForEach-Object { $_.FullName.Substring($root.Length).TrimStart('\', '/') }
}
function Get-SizedFiles {
param([string]$SrcDir, [string[]]$RelFiles)
foreach ($rel in $RelFiles) {
$abs = Join-Path $SrcDir $rel
try { $size = (Get-Item -LiteralPath $abs -ErrorAction Stop).Length }
catch { $size = 0 }
[PSCustomObject]@{ Size = [long]$size; RelPath = $rel }
}
}
function Get-BalancedChunks {
param($SizedFiles, [int]$Workers)
$buckets = @(1..$Workers | ForEach-Object { [System.Collections.Generic.List[string]]::new() })
$bucketSizes = [long[]]::new($Workers)
foreach ($entry in ($SizedFiles | Sort-Object -Property Size -Descending)) {
$minIdx = 0
for ($i = 1; $i -lt $Workers; $i++) {
if ($bucketSizes[$i] -lt $bucketSizes[$minIdx]) { $minIdx = $i }
}
$buckets[$minIdx].Add($entry.RelPath)
$bucketSizes[$minIdx] += $entry.Size
}
$buckets | Where-Object { $_.Count -gt 0 }
}
#endregion
#region Runspace script blocks
# These blocks run inside isolated runspaces — all needed logic must be self-contained.
$rsyncChunkScriptBlock = {
param(
[string] $SrcDir,
[string[]]$RelFiles,
[string] $Dest,
[string[]]$RsyncArgsList,
[bool] $IsDryRun
)
$filesFromTmp = [System.IO.Path]::GetTempFileName()
try {
# Build a NUL-delimited file list using forward slashes for rsync
$lines = $RelFiles | ForEach-Object { $_ -replace '\\', '/' }
$allBytes = [System.Collections.Generic.List[byte]]::new()
foreach ($line in $lines) {
$allBytes.AddRange([System.Text.Encoding]::UTF8.GetBytes($line))
$allBytes.Add([byte]0)
}
[System.IO.File]::WriteAllBytes($filesFromTmp, $allBytes.ToArray())
$srcRoot = ($SrcDir.TrimEnd('\', '/') -replace '\\', '/') + '/'
$cmdArgs = [System.Collections.Generic.List[string]]::new($RsyncArgsList)
if ($IsDryRun) { $cmdArgs.Insert(0, '--dry-run') }
$cmdArgs.AddRange([string[]]@('--from0', "--files-from=$filesFromTmp", $srcRoot, $Dest))
$psi = [System.Diagnostics.ProcessStartInfo]::new('rsync')
$psi.RedirectStandardOutput = $true
$psi.RedirectStandardError = $true
$psi.UseShellExecute = $false
$psi.CreateNoWindow = $true
foreach ($arg in $cmdArgs) { $psi.ArgumentList.Add($arg) }
try {
$proc = [System.Diagnostics.Process]::Start($psi)
$stdout = $proc.StandardOutput.ReadToEnd()
$stderr = $proc.StandardError.ReadToEnd()
$proc.WaitForExit()
$exitCode = $proc.ExitCode
} catch {
$stdout = ''
$stderr = "rsync not found on PATH: $($_.Exception.Message)"
$exitCode = 127
}
return [PSCustomObject]@{
Label = "chunk($($RelFiles.Count) files)"
ExitCode = $exitCode
Stdout = $stdout
Stderr = $stderr
}
} finally {
Remove-Item -LiteralPath $filesFromTmp -Force -ErrorAction SilentlyContinue
}
}
$rsyncItemScriptBlock = {
param(
[string] $Item,
[string] $Dest,
[string[]]$RsyncArgsList,
[bool] $IsDryRun
)
$srcPath = $Item -replace '\\', '/'
$cmdArgs = [System.Collections.Generic.List[string]]::new($RsyncArgsList)
if ($IsDryRun) { $cmdArgs.Insert(0, '--dry-run') }
$cmdArgs.Add($srcPath)
$cmdArgs.Add($Dest)
$psi = [System.Diagnostics.ProcessStartInfo]::new('rsync')
$psi.RedirectStandardOutput = $true
$psi.RedirectStandardError = $true
$psi.UseShellExecute = $false
$psi.CreateNoWindow = $true
foreach ($arg in $cmdArgs) { $psi.ArgumentList.Add($arg) }
try {
$proc = [System.Diagnostics.Process]::Start($psi)
$stdout = $proc.StandardOutput.ReadToEnd()
$stderr = $proc.StandardError.ReadToEnd()
$proc.WaitForExit()
$exitCode = $proc.ExitCode
} catch {
$stdout = ''
$stderr = "rsync not found on PATH: $($_.Exception.Message)"
$exitCode = 127
}
return [PSCustomObject]@{
Label = $Item
ExitCode = $exitCode
Stdout = $stdout
Stderr = $stderr
}
}
#endregion
#region Parallel job runner (as-completed using WaitHandle.WaitAny)
function Invoke-ParallelJobs {
param(
[object[]] $Jobs, # PSCustomObjects with .PS and .Handle
[bool] $ShowOutput,
[bool] $ShowOnFailOnly
)
$results = [System.Collections.Generic.List[object]]::new()
$pending = [System.Collections.Generic.List[object]]::new()
foreach ($j in $Jobs) { $pending.Add($j) }
while ($pending.Count -gt 0) {
$handles = @($pending | ForEach-Object { $_.Handle.AsyncWaitHandle })
$idx = [System.Threading.WaitHandle]::WaitAny($handles)
$job = $pending[$idx]
$pending.RemoveAt($idx)
$r = $job.PS.EndInvoke($job.Handle)[0]
$job.PS.Dispose()
$results.Add($r)
$status = if ($r.ExitCode -eq 0) { 'OK' } else { "ERR($($r.ExitCode))" }
Write-Host "[$status] $($r.Label)"
$printOutput = $ShowOutput -and (-not $ShowOnFailOnly -or $r.ExitCode -ne 0)
if ($printOutput) {
if ($r.Stdout.Trim()) {
Write-Host "--- stdout: $($r.Label) ---"
Write-Host $r.Stdout.TrimEnd()
}
if ($r.Stderr.Trim()) {
Write-Warning "--- stderr: $($r.Label) ---`n$($r.Stderr.TrimEnd())"
}
}
}
return $results
}
#endregion
#region Main
$timer = [System.Diagnostics.Stopwatch]::StartNew()
$rsyncArgsList = $RsyncArgs -split '\s+' | Where-Object { $_ -ne '' }
$srcPath = $Src.TrimEnd('\', '/')
$isRemote = Test-RemoteDestination -Destination $Dest
$cpuCount = [Environment]::ProcessorCount
$showOutput = $ShowRsyncOutput -or $ShowRsyncOutputOnFailure
$showOnFail = $ShowRsyncOutputOnFailure -and -not $ShowRsyncOutput
if ($ShowRsyncOutput -and $ShowRsyncOutputOnFailure) {
Write-Warning 'Both -ShowRsyncOutput and -ShowRsyncOutputOnFailure set; showing output for all tasks.'
}
$allResults = $null
if ($Mode -eq 'recursive-files' -and (Test-Path -LiteralPath $srcPath -PathType Container)) {
$relFiles = @(Get-RecursiveFileList -SrcDir $srcPath)
if (-not $relFiles -or $relFiles.Count -eq 0) {
Write-Error "No files found for: $Src"
exit 2
}
$sizedFiles = @(Get-SizedFiles -SrcDir $srcPath -RelFiles $relFiles)
$totalBytes = ($sizedFiles | Measure-Object -Property Size -Sum).Sum
$avgSize = if ($sizedFiles.Count -gt 0) { $totalBytes / $sizedFiles.Count } else { 0 }
if ($AutoTune) {
$effectiveWorkers = Get-AutoTunedWorkers -TotalUnits $sizedFiles.Count -AvgSizeBytes $avgSize `
-IsRemote $isRemote -CpuCount $cpuCount -MaxWorkers $Workers
$destKind = if ($isRemote) { 'remote' } else { 'local' }
Write-Host "Auto-tuned workers=$effectiveWorkers ($destKind dest, avg file $(Format-Bytes $avgSize))"
} else {
$effectiveWorkers = [Math]::Min($Workers, $sizedFiles.Count)
}
$chunks = @(Get-BalancedChunks -SizedFiles $sizedFiles -Workers $effectiveWorkers)
Write-Host "Starting $($chunks.Count) rsync chunk task(s) with $effectiveWorkers worker(s)..."
if ($VerbosePreference -ne 'SilentlyContinue') {
$srcRoot = ($srcPath -replace '\\', '/') + '/'
$previewArgs = (@('rsync') + $rsyncArgsList + @('--from0', '--files-from=<tmpfile>', $srcRoot, $Dest))
if ($DryRun) { $previewArgs = @('rsync', '--dry-run') + $previewArgs[1..($previewArgs.Count-1)] }
Write-Verbose "Command template: $($previewArgs -join ' ')"
}
$pool = [RunspaceFactory]::CreateRunspacePool(1, $effectiveWorkers)
$pool.Open()
$jobs = foreach ($chunk in $chunks) {
$ps = [PowerShell]::Create()
$ps.RunspacePool = $pool
[void]$ps.AddScript($rsyncChunkScriptBlock)
[void]$ps.AddParameter('SrcDir', $srcPath)
[void]$ps.AddParameter('RelFiles', [string[]]$chunk)
[void]$ps.AddParameter('Dest', $Dest)
[void]$ps.AddParameter('RsyncArgsList', [string[]]$rsyncArgsList)
[void]$ps.AddParameter('IsDryRun', $DryRun.IsPresent)
[PSCustomObject]@{ PS = $ps; Handle = $ps.BeginInvoke() }
}
$allResults = Invoke-ParallelJobs -Jobs $jobs -ShowOutput $showOutput -ShowOnFailOnly $showOnFail
$pool.Close()
$pool.Dispose()
} else {
if (Test-Path -LiteralPath $srcPath -PathType Container) {
$items = @(Get-ChildItem -LiteralPath $srcPath | ForEach-Object { $_.FullName })
} elseif ($srcPath -match '[*?]') {
$items = @(Resolve-Path -Path $srcPath | ForEach-Object { $_.ProviderPath })
} else {
$items = @($srcPath)
}
if (-not $items -or $items.Count -eq 0) {
Write-Error "No files found for: $Src"
exit 2
}
$sizeSamples = foreach ($item in $items) {
try {
$fi = Get-Item -LiteralPath $item -ErrorAction Stop
if ($fi.PSIsContainer) { [long](64 * 1024 * 1024) } else { $fi.Length }
} catch { [long]0 }
}
$totalBytes = ($sizeSamples | Measure-Object -Sum).Sum
$avgSize = if ($items.Count -gt 0) { $totalBytes / $items.Count } else { 0 }
if ($AutoTune) {
$effectiveWorkers = Get-AutoTunedWorkers -TotalUnits $items.Count -AvgSizeBytes $avgSize `
-IsRemote $isRemote -CpuCount $cpuCount -MaxWorkers $Workers
$destKind = if ($isRemote) { 'remote' } else { 'local' }
Write-Host "Auto-tuned workers=$effectiveWorkers ($destKind dest, avg unit $(Format-Bytes $avgSize))"
} else {
$effectiveWorkers = [Math]::Min($Workers, $items.Count)
}
Write-Host "Starting $($items.Count) rsync task(s) with $effectiveWorkers worker(s)..."
if ($VerbosePreference -ne 'SilentlyContinue') {
Write-Verbose 'Task list preview:'
$previewItems = $items | Select-Object -First 10
for ($i = 0; $i -lt $previewItems.Count; $i++) {
Write-Verbose (' {0}. {1}' -f ($i + 1), $previewItems[$i])
}
if ($items.Count -gt 10) { Write-Verbose " ... and $($items.Count - 10) more" }
}
$pool = [RunspaceFactory]::CreateRunspacePool(1, $effectiveWorkers)
$pool.Open()
$jobs = foreach ($item in $items) {
if ($VerbosePreference -ne 'SilentlyContinue') {
$previewArgs = (@('rsync') + $rsyncArgsList + @(($item -replace '\\', '/'), $Dest))
if ($DryRun) { $previewArgs = @('rsync', '--dry-run') + $previewArgs[1..($previewArgs.Count-1)] }
Write-Verbose "Scheduling: $($previewArgs -join ' ')"
}
$ps = [PowerShell]::Create()
$ps.RunspacePool = $pool
[void]$ps.AddScript($rsyncItemScriptBlock)
[void]$ps.AddParameter('Item', $item)
[void]$ps.AddParameter('Dest', $Dest)
[void]$ps.AddParameter('RsyncArgsList', [string[]]$rsyncArgsList)
[void]$ps.AddParameter('IsDryRun', $DryRun.IsPresent)
[PSCustomObject]@{ PS = $ps; Handle = $ps.BeginInvoke() }
}
$allResults = Invoke-ParallelJobs -Jobs $jobs -ShowOutput $showOutput -ShowOnFailOnly $showOnFail
$pool.Close()
$pool.Dispose()
}
#endregion
#region Summary
$failed = @($allResults | Where-Object { $_.ExitCode -ne 0 })
$elapsed = [Math]::Round($timer.Elapsed.TotalSeconds, 2)
Write-Host "`nCompleted: $($allResults.Count) tasks, $($failed.Count) failed in ${elapsed}s."
if ($failed.Count -gt 0) {
Write-Host 'Failures detail:'
foreach ($r in $failed) {
Write-Host "--- $($r.Label) (exit $($r.ExitCode)) ---"
if ($r.Stdout.Trim()) { Write-Host $r.Stdout.TrimEnd() }
if ($r.Stderr.Trim()) { Write-Host $r.Stderr.TrimEnd() }
}
exit 1
}
exit 0
#endregion
personal/tools/fix_csv.py
-47
View File
@@ -1,47 +0,0 @@
#!/usr/bin/env python3
"""Fix the powerball CSV file format."""
import shutil
# Read the original file
with open('powerball_numbers.csv', 'r') as f:
lines = f.readlines()
# Create backup
shutil.copy('powerball_numbers.csv', 'powerball_numbers.csv.backup')
# Open output file
with open('powerball_numbers.csv', 'w') as f:
# Write header
f.write('Draw Date,Winning Numbers,Multiplier\n')
# Process each line
for line in lines:
parts = line.strip().split(',')
# Extract components
month = parts[0]
day_year = parts[1].split('/')
day = day_year[0]
year = day_year[1]
# Format date as MM/DD/YYYY
date = f"{int(month):02d}/{int(day):02d}/{year}"
# Get the 6 winning numbers (5 white balls + 1 powerball)
numbers = ' '.join(parts[2:8])
# Get the multiplier
multiplier = parts[8]
# Write the formatted line
f.write(f'{date},{numbers},{multiplier}\n')
print("✓ File fixed! Created backup at powerball_numbers.csv.backup")
print("\nFirst few lines of fixed file:")
with open('powerball_numbers.csv', 'r') as f:
for i, line in enumerate(f):
if i < 5:
print(line.strip())
else:
break
personal/tools/ilo_powercycle.py
-57
View File
@@ -1,57 +0,0 @@
#!/usr/bin/python3
import argparse
import configparser
import os
import sys
import urllib3
import redfish
# Suppress SSL warnings for self-signed iLO certificates
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
parser = argparse.ArgumentParser(description="Power off an HPE server via iLO Redfish")
parser.add_argument("hostname", help="iLO hostname or IP address")
parser.add_argument("-c", "--credentials", default="/etc/ilo_credentials",
help="Path to credentials file (default: /etc/ilo_credentials)")
args = parser.parse_args()
# Read credentials from file (chmod 600 recommended: sudo chmod 600 /etc/ilo_credentials)
# File format:
# [ilo]
# username = root
# password = yourpassword
cred_path = args.credentials
if not os.path.exists(cred_path):
print(f"Error: credentials file not found: {cred_path}", file=sys.stderr)
sys.exit(1)
config = configparser.ConfigParser()
config.read(cred_path)
try:
LOGIN_ACCOUNT = config.get("ilo", "username", fallback="root")
LOGIN_PASSWORD = config.get("ilo", "password")
except (configparser.NoSectionError, configparser.NoOptionError) as e:
print(f"Error reading credentials file: {e}", file=sys.stderr)
sys.exit(1)
BASE_URL = f"https://{args.hostname}"
REDFISHOBJ = redfish.RedfishClient(
base_url=BASE_URL,
username=LOGIN_ACCOUNT,
password=LOGIN_PASSWORD,
)
REDFISHOBJ.login(auth="session")
# Discover the correct reset action URI from the system resource
sys_response = REDFISHOBJ.get("/redfish/v1/Systems/1/")
reset_uri = sys_response.dict["Actions"]["#ComputerSystem.Reset"]["target"]
body = {"ResetType": "PushPowerButton"} # iLO 4: PushPowerButton = graceful shutdown; use "ForceOff" for hard power cut
response = REDFISHOBJ.post(reset_uri, body=body)
print(f"Status: {response.status}")
if response.status >= 400:
print(f"Error: {response.read}")
REDFISHOBJ.logout()
personal/tools/ipmi_poweroff.py
-61
View File
@@ -1,61 +0,0 @@
#!/usr/bin/env python3
import argparse
import configparser
import os
import subprocess
import sys
DEFAULT_EK = "0000000000000000000000000000000000000000"
def load_credentials(path):
if not os.path.exists(path):
print(f"Error: credentials file not found: {path}", file=sys.stderr)
sys.exit(1)
config = configparser.ConfigParser()
config.read(path)
try:
username = config.get("credentials", "username", fallback="root")
password = config.get("credentials", "password")
encryption_key = config.get("credentials", "encryption_key", fallback=DEFAULT_EK)
except (configparser.NoSectionError, configparser.NoOptionError) as e:
print(f"Error reading credentials file: {e}", file=sys.stderr)
sys.exit(1)
return username, password, encryption_key
def main():
p = argparse.ArgumentParser(description="Send IPMI power off")
p.add_argument("-H", "--host", required=True, help="IPMI host address")
p.add_argument(
"-c",
"--credentials",
default="credentials",
help="Path to credentials file (default: credentials)",
)
args = p.parse_args()
ipmi_user, ipmi_pw, ipmi_ek = load_credentials(args.credentials)
cmd = [
"ipmitool",
"-I", "lanplus",
"-H", args.host,
"-U", ipmi_user,
"-P", ipmi_pw,
"-y", ipmi_ek,
"power", "off",
]
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if proc.returncode != 0:
print("ipmitool failed:", proc.stderr.strip(), file=sys.stderr)
sys.exit(proc.returncode)
print(proc.stdout.strip())
if __name__ == "__main__":
main()
personal/tools/ipmi_poweron.py
-65
View File
@@ -1,65 +0,0 @@
#!/usr/bin/env python3
import argparse
import configparser
import os
import subprocess
import sys
import datetime
def ts_print(msg):
"""ts_print message prefixed with a timestamp (YYYY-MM-DD HH:MM:SS)."""
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(f"{now} {msg}")
DEFAULT_EK = "0000000000000000000000000000000000000000"
def load_credentials(path):
if not os.path.exists(path):
print(f"Error: credentials file not found: {path}", file=sys.stderr)
sys.exit(1)
config = configparser.ConfigParser()
config.read(path)
try:
username = config.get("ilo", "username", fallback="root")
password = config.get("ilo", "password")
encryption_key = config.get("ilo", "encryption_key", fallback=DEFAULT_EK)
except (configparser.NoSectionError, configparser.NoOptionError) as e:
print(f"Error reading credentials file: {e}", file=sys.stderr)
sys.exit(1)
return username, password, encryption_key
def main():
p = argparse.ArgumentParser(description="Send IPMI power on")
p.add_argument("-H", "--host", required=True, help="IPMI host address")
p.add_argument(
"-c",
"--credentials",
default="credentials",
help="Path to credentials file (default: credentials)",
)
args = p.parse_args()
ipmi_user, ipmi_pw, ipmi_ek = load_credentials(args.credentials)
cmd = [
"ipmitool",
"-I", "lanplus",
"-H", args.host,
"-U", ipmi_user,
"-P", ipmi_pw,
"-y", ipmi_ek,
"power", "on",
]
print(f"Sending power on to {args.host}")
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if proc.returncode != 0:
print("ipmitool failed:", proc.stderr.strip(), file=sys.stderr)
sys.exit(proc.returncode)
print(proc.stdout.strip())
if __name__ == "__main__":
main()
personal/tools/pgbench_benchmark.ps1
-268
View File
@@ -1,268 +0,0 @@
param(
[string]$PgHost = "localhost",
[int]$PgPort = 5432,
[string]$PgUser = "postgres",
[string]$BenchDb = "pgbench",
[int]$Scale = 50,
[int]$Duration = 60,
[string]$Clients = "1 4 8 16 32",
[int]$Jobs = 4,
[int]$WarmupSeconds = 15,
[bool]$RunReadOnly = $true,
[bool]$RunCustom = $false,
[string]$CustomSqlFile = "pgbench_custom_workload.sql",
[string]$CustomModeName = "custom_sql",
[ValidateRange(1, 99)][int]$CustomReadPct = 80,
[ValidateRange(1, 100)][int]$CustomHotPct = 90,
[ValidateRange(1, 4)][int]$CustomTxnSize = 2,
[ValidateRange(100, 100000000)][int]$CustomHotAccounts = 10000,
[bool]$Reinit = $false
)
$ErrorActionPreference = "Stop"
function Require-Command {
param([Parameter(Mandatory = $true)][string]$Name)
if (-not (Get-Command $Name -ErrorAction SilentlyContinue)) {
throw "Missing required command: $Name"
}
}
function Invoke-PsqlScalar {
param(
[Parameter(Mandatory = $true)][string]$Database,
[Parameter(Mandatory = $true)][string]$Sql
)
$output = & psql -X -v ON_ERROR_STOP=1 -h $PgHost -p $PgPort -U $PgUser -d $Database -Atc $Sql
if ($LASTEXITCODE -ne 0) {
throw "psql failed running SQL: $Sql"
}
return ($output | Select-Object -First 1)
}
function Invoke-PsqlNonQuery {
param(
[Parameter(Mandatory = $true)][string]$Database,
[Parameter(Mandatory = $true)][string]$Sql
)
& psql -X -v ON_ERROR_STOP=1 -h $PgHost -p $PgPort -U $PgUser -d $Database -c $Sql | Out-Null
if ($LASTEXITCODE -ne 0) {
throw "psql failed running SQL: $Sql"
}
}
function Ensure-BenchmarkDatabase {
$exists = Invoke-PsqlScalar -Database "postgres" -Sql "SELECT 1 FROM pg_database WHERE datname = '$BenchDb';"
if ($exists -ne "1") {
Write-Host "Creating benchmark database: $BenchDb"
Invoke-PsqlNonQuery -Database "postgres" -Sql "CREATE DATABASE $BenchDb;"
}
}
function Needs-Init {
$hasTable = Invoke-PsqlScalar -Database $BenchDb -Sql "SELECT to_regclass('public.pgbench_accounts') IS NOT NULL;"
return $hasTable -ne "t"
}
function Init-OrReinitData {
if ($Reinit -or (Needs-Init)) {
Write-Host "Initializing pgbench schema/data (scale=$Scale)"
& pgbench -h $PgHost -p $PgPort -U $PgUser -i -s $Scale $BenchDb | Out-Host
if ($LASTEXITCODE -ne 0) {
throw "pgbench initialization failed"
}
}
else {
Write-Host "Using existing pgbench data in $BenchDb"
}
}
function Parse-Tps {
param([Parameter(Mandatory = $true)][string]$Text)
$matches = [regex]::Matches($Text, "(?m)^tps\s*=\s*([0-9]+(?:\.[0-9]+)?)")
if ($matches.Count -gt 0) {
return $matches[$matches.Count - 1].Groups[1].Value
}
return ""
}
function Parse-AvgLatencyMs {
param([Parameter(Mandatory = $true)][string]$Text)
$matches = [regex]::Matches($Text, "(?m)^latency average\s*=\s*([0-9]+(?:\.[0-9]+)?)\s*ms")
if ($matches.Count -gt 0) {
return $matches[$matches.Count - 1].Groups[1].Value
}
return ""
}
function Run-Case {
param(
[Parameter(Mandatory = $true)][string]$Mode,
[Parameter(Mandatory = $true)][int]$ClientCount,
[AllowEmptyCollection()][string[]]$ExtraArgs = @(),
[Parameter(Mandatory = $true)][string]$ResultDir,
[Parameter(Mandatory = $true)][string]$SummaryCsv
)
$logFile = Join-Path $ResultDir ("{0}_c{1}.txt" -f $Mode, $ClientCount)
$txLogPrefix = Join-Path $ResultDir ("{0}_c{1}_txlog" -f $Mode, $ClientCount)
Write-Host "Running mode=$Mode, clients=$ClientCount, duration=${Duration}s"
# Warm cache and avoid measuring first-run effects.
& pgbench -h $PgHost -p $PgPort -U $PgUser -n -M prepared -j $Jobs -c $ClientCount -T $WarmupSeconds @ExtraArgs $BenchDb | Out-Null
if ($LASTEXITCODE -ne 0) {
throw "Warm-up run failed for mode=$Mode clients=$ClientCount"
}
$runOutput = & pgbench -h $PgHost -p $PgPort -U $PgUser -n -M prepared -j $Jobs -c $ClientCount -T $Duration -r -l --log-prefix $txLogPrefix @ExtraArgs $BenchDb 2>&1
if ($LASTEXITCODE -ne 0) {
throw "Measured run failed for mode=$Mode clients=$ClientCount"
}
$runOutput | Set-Content -Path $logFile -Encoding UTF8
$text = ($runOutput -join [Environment]::NewLine)
$tps = Parse-Tps -Text $text
$latencyMs = Parse-AvgLatencyMs -Text $text
$percentiles = Get-PercentileLatency -LogPrefix $txLogPrefix
"$Mode,$ClientCount,$Duration,$Jobs,$tps,$latencyMs,$($percentiles.P95Ms),$($percentiles.P99Ms),$logFile" | Add-Content -Path $SummaryCsv
}
function Get-PercentileLatency {
param([Parameter(Mandatory = $true)][string]$LogPrefix)
$logFiles = Get-ChildItem -Path ($LogPrefix + "*") -File -ErrorAction SilentlyContinue
if (-not $logFiles -or $logFiles.Count -eq 0) {
return @{ P95Ms = ""; P99Ms = "" }
}
# Keep memory bounded even for very large transaction logs.
$maxSampleSize = 200000
$reservoir = New-Object System.Collections.Generic.List[double]
$reservoir.Capacity = $maxSampleSize
$random = [System.Random]::new()
$seen = 0
foreach ($file in $logFiles) {
$reader = [System.IO.File]::OpenText($file.FullName)
try {
while (($line = $reader.ReadLine()) -ne $null) {
if ([string]::IsNullOrWhiteSpace($line) -or $line.StartsWith("#")) {
continue
}
$tokens = $line -split "[\s,]+" | Where-Object { $_ -ne "" }
if ($tokens.Count -ge 3 -and $tokens[2] -match "^[0-9]+(?:\.[0-9]+)?$") {
$latUs = [double]$tokens[2]
$seen++
if ($reservoir.Count -lt $maxSampleSize) {
$reservoir.Add($latUs)
}
else {
$idx = $random.Next(0, $seen)
if ($idx -lt $maxSampleSize) {
$reservoir[$idx] = $latUs
}
}
}
}
}
finally {
$reader.Dispose()
}
}
if ($reservoir.Count -eq 0) {
return @{ P95Ms = ""; P99Ms = "" }
}
$sorted = $reservoir | Sort-Object
$p95Us = Select-PercentileValue -SortedValues $sorted -Percentile 95
$p99Us = Select-PercentileValue -SortedValues $sorted -Percentile 99
return @{
P95Ms = [math]::Round(($p95Us / 1000.0), 3)
P99Ms = [math]::Round(($p99Us / 1000.0), 3)
}
}
function Select-PercentileValue {
param(
[Parameter(Mandatory = $true)][double[]]$SortedValues,
[Parameter(Mandatory = $true)][ValidateRange(1, 99)][int]$Percentile
)
$n = $SortedValues.Count
$rank = [math]::Ceiling(($Percentile / 100.0) * $n)
if ($rank -lt 1) { $rank = 1 }
if ($rank -gt $n) { $rank = $n }
return $SortedValues[$rank - 1]
}
function Resolve-CustomSqlPath {
if ([System.IO.Path]::IsPathRooted($CustomSqlFile)) {
return $CustomSqlFile
}
return (Join-Path (Get-Location) $CustomSqlFile)
}
Require-Command -Name "psql"
Require-Command -Name "pgbench"
$timestamp = Get-Date -Format "yyyyMMdd_HHmmss"
$resultDir = Join-Path (Get-Location) ("pgbench_results_{0}" -f $timestamp)
$summaryCsv = Join-Path $resultDir "summary.csv"
New-Item -Path $resultDir -ItemType Directory -Force | Out-Null
"mode,clients,duration_seconds,jobs,tps,latency_ms,p95_ms,p99_ms,details_file" | Set-Content -Path $summaryCsv -Encoding UTF8
Ensure-BenchmarkDatabase
Init-OrReinitData
$customSqlPath = ""
if ($RunCustom) {
$customSqlPath = Resolve-CustomSqlPath
if (-not (Test-Path -Path $customSqlPath -PathType Leaf)) {
throw "Custom SQL file not found: $customSqlPath"
}
}
$clientValues = $Clients -split "\s+" | Where-Object { $_ -match "^[0-9]+$" }
if ($clientValues.Count -eq 0) {
throw "No valid client values found. Example: -Clients '1 4 8 16 32'"
}
foreach ($c in $clientValues) {
$clientInt = [int]$c
Run-Case -Mode "read_write" -ClientCount $clientInt -ExtraArgs @() -ResultDir $resultDir -SummaryCsv $summaryCsv
if ($RunReadOnly) {
Run-Case -Mode "read_only" -ClientCount $clientInt -ExtraArgs @("-S") -ResultDir $resultDir -SummaryCsv $summaryCsv
}
if ($RunCustom) {
Run-Case -Mode $CustomModeName -ClientCount $clientInt -ExtraArgs @(
"-D", "read_pct=$CustomReadPct",
"-D", "hot_pct=$CustomHotPct",
"-D", "txn_size=$CustomTxnSize",
"-D", "hot_accounts=$CustomHotAccounts",
"-f", $customSqlPath
) -ResultDir $resultDir -SummaryCsv $summaryCsv
}
}
Write-Host ""
Write-Host "Benchmark completed."
Write-Host "Summary: $summaryCsv"
Write-Host "Raw outputs: $resultDir"
personal/tools/pgbench_benchmark.sh
-188
View File
@@ -1,188 +0,0 @@
#!/bin/bash
set -euo pipefail
# Repeatable pgbench benchmark harness.
# Usage example:
# PGHOST=localhost PGPORT=5432 PGUSER=postgres PGPASSWORD=secret \
# ./pgbench_benchmark.sh
#
# Optional environment variables:
# BENCH_DB=pgbench
# SCALE=50
# DURATION=60
# CLIENTS="1 4 8 16 32"
# JOBS=4
# WARMUP_SECONDS=15
# RUN_READ_ONLY=1
# RUN_CUSTOM=0
# CUSTOM_SQL_FILE=pgbench_custom_workload.sql
# CUSTOM_MODE_NAME=custom_sql
# CUSTOM_READ_PCT=80
# CUSTOM_HOT_PCT=90
# CUSTOM_TXN_SIZE=2
# CUSTOM_HOT_ACCOUNTS=10000
# REINIT=0
PGHOST="${PGHOST:-localhost}"
PGPORT="${PGPORT:-5432}"
PGUSER="${PGUSER:-postgres}"
BENCH_DB="${BENCH_DB:-pgbench}"
SCALE="${SCALE:-50}"
DURATION="${DURATION:-60}"
CLIENTS="${CLIENTS:-1 4 8 16 32}"
JOBS="${JOBS:-4}"
WARMUP_SECONDS="${WARMUP_SECONDS:-15}"
RUN_READ_ONLY="${RUN_READ_ONLY:-1}"
RUN_CUSTOM="${RUN_CUSTOM:-0}"
CUSTOM_SQL_FILE="${CUSTOM_SQL_FILE:-pgbench_custom_workload.sql}"
CUSTOM_MODE_NAME="${CUSTOM_MODE_NAME:-custom_sql}"
CUSTOM_READ_PCT="${CUSTOM_READ_PCT:-80}"
CUSTOM_HOT_PCT="${CUSTOM_HOT_PCT:-90}"
CUSTOM_TXN_SIZE="${CUSTOM_TXN_SIZE:-2}"
CUSTOM_HOT_ACCOUNTS="${CUSTOM_HOT_ACCOUNTS:-10000}"
REINIT="${REINIT:-0}"
TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
RESULT_DIR="pgbench_results_${TIMESTAMP}"
SUMMARY_CSV="${RESULT_DIR}/summary.csv"
PSQL=(psql -X -v ON_ERROR_STOP=1 -h "$PGHOST" -p "$PGPORT" -U "$PGUSER")
PGBENCH=(pgbench -h "$PGHOST" -p "$PGPORT" -U "$PGUSER")
require_command() {
local cmd="$1"
if ! command -v "$cmd" >/dev/null 2>&1; then
echo "Missing required command: $cmd" >&2
exit 1
fi
}
create_database_if_missing() {
local exists
exists="$("${PSQL[@]}" -d postgres -Atc "SELECT 1 FROM pg_database WHERE datname = '${BENCH_DB}';" || true)"
if [[ "$exists" != "1" ]]; then
echo "Creating benchmark database: ${BENCH_DB}"
"${PSQL[@]}" -d postgres -c "CREATE DATABASE ${BENCH_DB};"
fi
}
needs_init() {
local rel
rel="$("${PSQL[@]}" -d "$BENCH_DB" -Atc "SELECT to_regclass('public.pgbench_accounts') IS NOT NULL;")"
[[ "$rel" != "t" ]]
}
init_or_reinit_data() {
if [[ "$REINIT" == "1" ]] || needs_init; then
echo "Initializing pgbench schema/data (scale=${SCALE})"
"${PGBENCH[@]}" -i -s "$SCALE" "$BENCH_DB"
else
echo "Using existing pgbench data in ${BENCH_DB}"
fi
}
extract_tps() {
local output_file="$1"
awk -F'= ' '/tps =/{print $2}' "$output_file" | awk '{print $1}' | tail -n 1
}
extract_avg_latency() {
local output_file="$1"
awk -F'= ' '/latency average =/{print $2}' "$output_file" | awk '{print $1}' | tail -n 1
}
select_percentile_ms_from_prefix() {
local log_prefix="$1"
local percentile="$2"
local -a log_files
local -a values_us
local count rank value_us
shopt -s nullglob
log_files=("${log_prefix}"*)
shopt -u nullglob
if (( ${#log_files[@]} == 0 )); then
echo ""
return
fi
mapfile -t values_us < <(awk 'NF >= 3 && $3 ~ /^[0-9]+(\.[0-9]+)?$/ {print $3}' "${log_files[@]}" | sort -n)
count="${#values_us[@]}"
if (( count == 0 )); then
echo ""
return
fi
rank=$(( (percentile * count + 99) / 100 ))
if (( rank < 1 )); then rank=1; fi
if (( rank > count )); then rank=count; fi
value_us="${values_us[$((rank - 1))]}"
awk -v us="$value_us" 'BEGIN { printf "%.3f", us/1000.0 }'
}
run_case() {
local mode="$1"
local clients="$2"
shift 2
local -a extra_args=("$@")
local log_file="${RESULT_DIR}/${mode}_c${clients}.txt"
local tx_log_prefix="${RESULT_DIR}/${mode}_c${clients}_txlog"
echo "Running mode=${mode}, clients=${clients}, duration=${DURATION}s"
# Warm cache and avoid measuring first-run effects.
"${PGBENCH[@]}" -n -M prepared -j "$JOBS" -c "$clients" -T "$WARMUP_SECONDS" "${extra_args[@]}" "$BENCH_DB" > /dev/null
"${PGBENCH[@]}" -n -M prepared -j "$JOBS" -c "$clients" -T "$DURATION" -r -l --log-prefix="$tx_log_prefix" "${extra_args[@]}" "$BENCH_DB" > "$log_file"
local tps latency p95 p99
tps="$(extract_tps "$log_file")"
latency="$(extract_avg_latency "$log_file")"
p95="$(select_percentile_ms_from_prefix "$tx_log_prefix" 95)"
p99="$(select_percentile_ms_from_prefix "$tx_log_prefix" 99)"
echo "${mode},${clients},${DURATION},${JOBS},${tps},${latency},${p95},${p99},${log_file}" >> "$SUMMARY_CSV"
}
main() {
require_command psql
require_command pgbench
mkdir -p "$RESULT_DIR"
echo "mode,clients,duration_seconds,jobs,tps,latency_ms,p95_ms,p99_ms,details_file" > "$SUMMARY_CSV"
create_database_if_missing
init_or_reinit_data
if [[ "$RUN_CUSTOM" == "1" ]]; then
if [[ ! -f "$CUSTOM_SQL_FILE" ]]; then
echo "Custom SQL file not found: $CUSTOM_SQL_FILE" >&2
exit 1
fi
fi
for c in $CLIENTS; do
run_case "read_write" "$c"
if [[ "$RUN_READ_ONLY" == "1" ]]; then
run_case "read_only" "$c" "-S"
fi
if [[ "$RUN_CUSTOM" == "1" ]]; then
run_case "$CUSTOM_MODE_NAME" "$c" \
"-D" "read_pct=${CUSTOM_READ_PCT}" \
"-D" "hot_pct=${CUSTOM_HOT_PCT}" \
"-D" "txn_size=${CUSTOM_TXN_SIZE}" \
"-D" "hot_accounts=${CUSTOM_HOT_ACCOUNTS}" \
"-f" "$CUSTOM_SQL_FILE"
fi
done
echo
echo "Benchmark completed."
echo "Summary: $SUMMARY_CSV"
echo "Raw outputs: $RESULT_DIR"
}
main "$@"
personal/tools/pgbench_export_report.py
-306
View File
@@ -1,306 +0,0 @@
#!/usr/bin/env python3
"""Export pgbench summary.csv into an HTML report with TPS and latency charts."""
import argparse
import csv
import json
from pathlib import Path
from typing import Dict, List, Optional
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Create an HTML chart report from pgbench summary.csv")
parser.add_argument("summary_csv", help="Path to summary.csv generated by pgbench benchmark script")
parser.add_argument(
"-o",
"--output",
default=None,
help="Output HTML path (default: <summary_dir>/pgbench_report.html)",
)
return parser.parse_args()
def load_rows(path: Path) -> List[Dict[str, str]]:
with path.open("r", encoding="utf-8-sig", newline="") as f:
reader = csv.DictReader(f)
expected = {"mode", "clients", "tps", "latency_ms"}
if not expected.issubset(set(reader.fieldnames or [])):
missing = sorted(expected - set(reader.fieldnames or []))
raise ValueError(f"summary.csv missing required columns: {', '.join(missing)}")
return list(reader)
def parse_optional_float(raw: str) -> Optional[float]:
value = (raw or "").strip()
if not value:
return None
try:
return float(value)
except ValueError:
return None
def build_series(rows: List[Dict[str, str]]) -> Dict[str, List[Dict[str, float]]]:
series: Dict[str, List[Dict[str, float]]] = {}
for row in rows:
mode = row["mode"].strip()
try:
clients = int(row["clients"])
tps = float(row["tps"])
latency = float(row["latency_ms"])
except ValueError:
continue
point: Dict[str, float] = {
"clients": clients,
"tps": tps,
"latency_ms": latency,
}
p95 = parse_optional_float(row.get("p95_ms", ""))
p99 = parse_optional_float(row.get("p99_ms", ""))
if p95 is not None:
point["p95_ms"] = p95
if p99 is not None:
point["p99_ms"] = p99
series.setdefault(mode, []).append(point)
for mode in series:
series[mode].sort(key=lambda item: item["clients"])
return series
def html_report(title: str, data: Dict[str, List[Dict[str, float]]]) -> str:
payload = json.dumps(data)
return f"""<!doctype html>
<html lang=\"en\">
<head>
<meta charset=\"utf-8\" />
<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" />
<title>{title}</title>
<style>
:root {{
--bg: #f7f7f2;
--fg: #182020;
--panel: #ffffff;
--muted: #4f5b5b;
--grid: #dde4e4;
}}
* {{ box-sizing: border-box; }}
body {{
margin: 0;
font-family: "Segoe UI", Tahoma, Geneva, Verdana, sans-serif;
background: radial-gradient(circle at 8% 15%, #e7f6f2, var(--bg));
color: var(--fg);
}}
.container {{
max-width: 1100px;
margin: 0 auto;
padding: 24px;
}}
.card {{
background: var(--panel);
border: 1px solid #e2ebeb;
border-radius: 14px;
padding: 16px;
margin-bottom: 18px;
box-shadow: 0 10px 26px rgba(20, 30, 30, 0.06);
}}
h1 {{ margin: 0 0 8px 0; font-size: 1.6rem; }}
p {{ margin: 0; color: var(--muted); }}
canvas {{ width: 100%; height: 360px; display: block; }}
.legend {{ display: flex; gap: 12px; flex-wrap: wrap; margin-top: 10px; }}
.legend-item {{ display: inline-flex; align-items: center; gap: 8px; font-size: 0.9rem; color: var(--muted); }}
.swatch {{ width: 14px; height: 14px; border-radius: 3px; display: inline-block; }}
</style>
</head>
<body>
<div class=\"container\">
<div class=\"card\">
<h1>{title}</h1>
<p>Generated from summary.csv. X-axis = clients. Lines are grouped by mode.</p>
</div>
<div class=\"card\">
<h2>TPS by Clients</h2>
<canvas id=\"tpsChart\" width=\"1040\" height=\"360\"></canvas>
<div id=\"legend\" class=\"legend\"></div>
</div>
<div class=\"card\">
<h2>Latency (ms) by Clients</h2>
<canvas id=\"latencyChart\" width=\"1040\" height=\"360\"></canvas>
</div>
<div class=\"card\">
<h2>p95 Latency (ms) by Clients</h2>
<canvas id=\"p95Chart\" width=\"1040\" height=\"360\"></canvas>
</div>
<div class=\"card\">
<h2>p99 Latency (ms) by Clients</h2>
<canvas id=\"p99Chart\" width=\"1040\" height=\"360\"></canvas>
</div>
</div>
<script>
const series = {payload};
const palette = ["#0b6e4f", "#f26419", "#33658a", "#2f4858", "#5f0f40", "#9a031e", "#386641", "#6a4c93"];
function flattenValues(metric) {{
const out = [];
Object.values(series).forEach(items => items.forEach(i => {{
const v = i[metric];
if (typeof v === 'number' && Number.isFinite(v)) out.push(v);
}}));
return out;
}}
function flattenClients() {{
const out = [];
Object.values(series).forEach(items => items.forEach(i => out.push(i.clients)));
return out;
}}
function drawChart(canvasId, metric, yLabel) {{
const c = document.getElementById(canvasId);
const ctx = c.getContext('2d');
const W = c.width;
const H = c.height;
const m = {{ top: 22, right: 20, bottom: 44, left: 60 }};
const x0 = m.left, y0 = H - m.bottom, x1 = W - m.right, y1 = m.top;
ctx.clearRect(0, 0, W, H);
ctx.fillStyle = '#182020';
ctx.font = '13px Segoe UI, sans-serif';
ctx.fillText(yLabel, 8, 16);
const clientsAll = flattenClients();
const valuesAll = flattenValues(metric);
if (!clientsAll.length || !valuesAll.length) {{
ctx.fillText('No data in summary.csv for this metric', x0 + 10, y0 - 10);
return;
}}
const minX = Math.min(...clientsAll);
const maxX = Math.max(...clientsAll);
const minY = 0;
const maxY = Math.max(...valuesAll) * 1.1;
const x = v => x0 + ((v - minX) / Math.max(1, maxX - minX)) * (x1 - x0);
const y = v => y0 - ((v - minY) / Math.max(1e-9, maxY - minY)) * (y0 - y1);
ctx.strokeStyle = '#dde4e4';
ctx.lineWidth = 1;
for (let i = 0; i <= 5; i++) {{
const yy = y0 - ((y0 - y1) * i / 5);
ctx.beginPath();
ctx.moveTo(x0, yy);
ctx.lineTo(x1, yy);
ctx.stroke();
const val = (maxY * i / 5).toFixed(1);
ctx.fillStyle = '#4f5b5b';
ctx.fillText(val, 16, yy + 4);
}}
const xTicks = [...new Set(clientsAll)].sort((a, b) => a - b);
xTicks.forEach(v => {{
const xx = x(v);
ctx.beginPath();
ctx.moveTo(xx, y0);
ctx.lineTo(xx, y0 + 5);
ctx.strokeStyle = '#9da9a9';
ctx.stroke();
ctx.fillStyle = '#4f5b5b';
ctx.fillText(String(v), xx - 6, y0 + 18);
}});
ctx.strokeStyle = '#182020';
ctx.beginPath();
ctx.moveTo(x0, y0);
ctx.lineTo(x1, y0);
ctx.stroke();
ctx.beginPath();
ctx.moveTo(x0, y0);
ctx.lineTo(x0, y1);
ctx.stroke();
const legend = document.getElementById('legend');
if (metric === 'tps') legend.innerHTML = '';
Object.entries(series).forEach(([mode, items], idx) => {{
const color = palette[idx % palette.length];
const sorted = [...items]
.filter(p => typeof p[metric] === 'number' && Number.isFinite(p[metric]))
.sort((a, b) => a.clients - b.clients);
if (!sorted.length) return;
ctx.strokeStyle = color;
ctx.lineWidth = 2;
ctx.beginPath();
sorted.forEach((p, i) => {{
const px = x(p.clients);
const py = y(p[metric]);
if (i === 0) ctx.moveTo(px, py); else ctx.lineTo(px, py);
}});
ctx.stroke();
sorted.forEach(p => {{
const px = x(p.clients);
const py = y(p[metric]);
ctx.fillStyle = color;
ctx.beginPath();
ctx.arc(px, py, 3, 0, Math.PI * 2);
ctx.fill();
}});
if (metric === 'tps') {{
const item = document.createElement('div');
item.className = 'legend-item';
item.innerHTML = `<span class=\"swatch\" style=\"background:${{color}}\"></span>${{mode}}`;
legend.appendChild(item);
}}
}});
}}
drawChart('tpsChart', 'tps', 'TPS');
drawChart('latencyChart', 'latency_ms', 'Latency (ms)');
drawChart('p95Chart', 'p95_ms', 'p95 (ms)');
drawChart('p99Chart', 'p99_ms', 'p99 (ms)');
</script>
</body>
</html>
"""
def main() -> int:
args = parse_args()
summary_path = Path(args.summary_csv).resolve()
if not summary_path.is_file():
raise FileNotFoundError(f"summary.csv not found: {summary_path}")
rows = load_rows(summary_path)
if not rows:
raise ValueError("summary.csv has no data rows")
series = build_series(rows)
if not series:
raise ValueError("No valid numeric rows found in summary.csv")
output_path = (
Path(args.output).resolve()
if args.output
else summary_path.parent / "pgbench_report.html"
)
title = f"pgbench report: {summary_path.parent.name}"
output_path.write_text(html_report(title=title, data=series), encoding="utf-8")
print(f"Report written: {output_path}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
personal/tools/pgbench_quickstart.md
-102
View File
@@ -1,102 +0,0 @@
# pgbench benchmark quickstart
This workspace now includes repeatable benchmark harnesses:
- `pgbench_benchmark.ps1` (Windows PowerShell)
- `pgbench_benchmark.sh` (Bash)
- `pgbench_custom_workload.sql` (sample custom SQL transaction mix)
- `pgbench_export_report.py` (exports TPS/latency chart report from `summary.csv`)
## 1) Prerequisites
- PostgreSQL client tools installed (`psql`, `pgbench`)
- Network access to your PostgreSQL server
- Credentials available via environment variables (`PGHOST`, `PGPORT`, `PGUSER`, `PGPASSWORD`)
## 2) Run the benchmark (Windows PowerShell)
From this folder:
```powershell
$env:PGPASSWORD = "your_password"
.\pgbench_benchmark.ps1 -PgHost localhost -PgPort 5432 -PgUser postgres
```
You can override defaults:
```powershell
$env:PGPASSWORD = "your_password"
.\pgbench_benchmark.ps1 -BenchDb pgbench -Scale 100 -Duration 120 -Clients "1 8 16 32 64" -Jobs 8 -Reinit $true
```
Run with custom SQL workload mode enabled:
```powershell
$env:PGPASSWORD = "your_password"
.\pgbench_benchmark.ps1 -RunCustom $true -CustomSqlFile .\pgbench_custom_workload.sql -CustomModeName app_like -CustomReadPct 85 -CustomHotPct 92 -CustomTxnSize 3 -CustomHotAccounts 12000
```
## 3) Optional Bash usage (WSL/Git Bash)
```bash
chmod +x pgbench_benchmark.sh
PGHOST=localhost PGPORT=5432 PGUSER=postgres PGPASSWORD=your_password ./pgbench_benchmark.sh
```
Run with custom SQL workload mode enabled:
```bash
PGHOST=localhost PGPORT=5432 PGUSER=postgres PGPASSWORD=your_password RUN_CUSTOM=1 CUSTOM_SQL_FILE=./pgbench_custom_workload.sql CUSTOM_MODE_NAME=app_like CUSTOM_READ_PCT=85 CUSTOM_HOT_PCT=92 CUSTOM_TXN_SIZE=3 CUSTOM_HOT_ACCOUNTS=12000 ./pgbench_benchmark.sh
```
## 4) Optional tuning inputs (Bash script)
```bash
BENCH_DB=pgbench SCALE=100 DURATION=120 CLIENTS="1 8 16 32 64" JOBS=8 REINIT=1 ./pgbench_benchmark.sh
```
Environment variables:
- `BENCH_DB`: database name for benchmark data (default: `pgbench`)
- `SCALE`: data size scale factor (default: `50`)
- `DURATION`: measured run time per test case in seconds (default: `60`)
- `CLIENTS`: space-separated client counts (default: `"1 4 8 16 32"`)
- `JOBS`: worker threads for pgbench (default: `4`)
- `WARMUP_SECONDS`: unmeasured warm-up before each test (default: `15`)
- `RUN_READ_ONLY`: set to `0` to skip `-S` read-only tests (default: `1`)
- `REINIT`: set to `1` to reinitialize data each run (default: `0`)
- `RUN_CUSTOM`: set to `1` to run custom SQL workload mode (default: `0`)
- `CUSTOM_SQL_FILE`: path to custom SQL file for `pgbench -f` (default: `pgbench_custom_workload.sql`)
- `CUSTOM_MODE_NAME`: mode label written to `summary.csv` for custom runs (default: `custom_sql`)
- `CUSTOM_READ_PCT` / `-CustomReadPct`: read share percentage for custom mode; writes happen in `100-read_pct` (default: `80`)
- `CUSTOM_HOT_PCT` / `-CustomHotPct`: chance to pick from hot account set in custom mode (default: `90`)
- `CUSTOM_TXN_SIZE` / `-CustomTxnSize`: custom transaction size tier from `1` to `4` (default: `2`)
- `CUSTOM_HOT_ACCOUNTS` / `-CustomHotAccounts`: number of hot accounts near start of keyspace (default: `10000`)
## 5) Output
Each run creates:
- `pgbench_results_YYYYMMDD_HHMMSS/summary.csv`
- Per-test raw output files in the same results directory
Use `summary.csv` to compare TPS and latency across runs.
`summary.csv` now includes percentile latency columns:
- `p95_ms`
- `p99_ms`
## 6) Export chart report from summary.csv
Generate an HTML report with TPS and latency charts:
```powershell
python .\pgbench_export_report.py .\pgbench_results_YYYYMMDD_HHMMSS\summary.csv
```
Optional output path:
```powershell
python .\pgbench_export_report.py .\pgbench_results_YYYYMMDD_HHMMSS\summary.csv -o .\pgbench_results_YYYYMMDD_HHMMSS\my_report.html
```
personal/tools/powerball_numbers.csv
-1964
View File
File diff suppressed because it is too large Load Diff
personal/tools/powerball_numbers.csv.backup
-1963
View File
File diff suppressed because it is too large Load Diff
personal/tools/powerball_reader.py
-946
View File
@@ -1,946 +0,0 @@
#!/usr/bin/env python3
"""
Script to read and parse Powerball numbers from a CSV file with advanced pattern detection.
"""
import csv
from datetime import datetime, timedelta
from pathlib import Path
from typing import List, Dict, Tuple, Optional, Set
from collections import Counter, defaultdict
from itertools import combinations
def read_powerball_csv(filepath: str) -> List[Dict]:
"""
Read a CSV file containing Powerball numbers.
Expected CSV format:
Draw Date,Winning Numbers,Multiplier
09/26/2020,11 21 27 36 62 24,3
Args:
filepath: Path to the CSV file
Returns:
List of dictionaries containing parsed powerball data
"""
powerball_data = []
try:
with open(filepath, 'r', encoding='utf-8') as csvfile:
reader = csv.DictReader(csvfile)
for row_num, row in enumerate(reader, start=2): # start=2 because row 1 is header
try:
# Skip empty rows
if not any(row.values()):
continue
# Parse the row
draw_date = datetime.strptime(row['Draw Date'].strip(), '%m/%d/%Y')
# Parse winning numbers - filter out empty strings
numbers_str = row['Winning Numbers'].strip()
winning_numbers = [int(n) for n in numbers_str.split() if n.strip()]
multiplier = int(row['Multiplier'].strip())
powerball_data.append({
'draw_date': draw_date,
'date_str': draw_date.strftime('%Y-%m-%d'),
'winning_numbers': winning_numbers,
'multiplier': multiplier
})
except ValueError as e:
print(f"Warning: Skipping row {row_num} due to error: {e}")
print(f" Row data: {row}")
continue
except FileNotFoundError:
print(f"Error: File '{filepath}' not found.")
return []
except Exception as e:
print(f"Error: Failed to read CSV file - {e}")
return []
return powerball_data
def display_powerball_data(data: List[Dict]) -> None:
"""
Display powerball data in a formatted table.
Args:
data: List of powerball data dictionaries
"""
if not data:
print("No data to display.")
return
print(f"{'Draw Date':<12} {'Winning Numbers':<30} {'Multiplier':<10}")
print("-" * 52)
for entry in data:
numbers_str = ' '.join(map(str, entry['winning_numbers']))
print(f"{entry['date_str']:<12} {numbers_str:<30} {entry['multiplier']:<10}")
def _get_unweighted_top_numbers(draws: List[Dict]) -> Tuple[List[int], Optional[int]]:
"""Return top 5 white balls and top Powerball from a draw list."""
if not draws:
return [], None
white_counter = Counter()
powerball_counter = Counter()
for entry in draws:
numbers = entry['winning_numbers']
for number in numbers[:5]:
white_counter[number] += 1
if len(numbers) >= 6:
powerball_counter[numbers[5]] += 1
top_white = [number for number, _ in white_counter.most_common(5)]
top_powerball = powerball_counter.most_common(1)[0][0] if powerball_counter else None
return top_white, top_powerball
def _get_recency_weighted_top_numbers(draws: List[Dict]) -> Tuple[List[int], Optional[int]]:
"""Return top numbers weighted so newer draws contribute more."""
if not draws:
return [], None
sorted_draws = sorted(draws, key=lambda x: x['draw_date'])
total_draws = len(sorted_draws)
white_scores: Dict[int, float] = defaultdict(float)
powerball_scores: Dict[int, float] = defaultdict(float)
for index, entry in enumerate(sorted_draws, start=1):
weight = index / total_draws
numbers = entry['winning_numbers']
for number in numbers[:5]:
white_scores[number] += weight
if len(numbers) >= 6:
powerball_scores[numbers[5]] += weight
top_white = [number for number, _ in sorted(white_scores.items(), key=lambda x: x[1], reverse=True)[:5]]
top_powerball = max(powerball_scores.items(), key=lambda x: x[1])[0] if powerball_scores else None
return top_white, top_powerball
def _get_aggressive_recency_weighted_top_numbers(draws: List[Dict]) -> Tuple[List[int], Optional[int]]:
"""Return top numbers with an aggressive recency curve (cubic weighting)."""
if not draws:
return [], None
sorted_draws = sorted(draws, key=lambda x: x['draw_date'])
total_draws = len(sorted_draws)
white_scores: Dict[int, float] = defaultdict(float)
powerball_scores: Dict[int, float] = defaultdict(float)
for index, entry in enumerate(sorted_draws, start=1):
normalized_position = index / total_draws
weight = normalized_position ** 3
numbers = entry['winning_numbers']
for number in numbers[:5]:
white_scores[number] += weight
if len(numbers) >= 6:
powerball_scores[numbers[5]] += weight
top_white = [number for number, _ in sorted(white_scores.items(), key=lambda x: x[1], reverse=True)[:5]]
top_powerball = max(powerball_scores.items(), key=lambda x: x[1])[0] if powerball_scores else None
return top_white, top_powerball
def analyze_most_frequent_numbers(data: List[Dict]) -> Dict:
"""
Analyze and display the most frequent number in each slot position.
Args:
data: List of powerball data dictionaries
"""
if not data:
print("No data to analyze.")
return {}
# Initialize lists to hold numbers for each position
num_positions = len(data[0]['winning_numbers'])
position_numbers = [[] for _ in range(num_positions)]
# Collect all numbers for each position
for entry in data:
for position, number in enumerate(entry['winning_numbers']):
position_numbers[position].append(number)
# Find most common number in each position
print("\n" + "=" * 60)
print("Most Frequent Numbers by Slot Position")
print("=" * 60)
most_frequent = []
for position, numbers in enumerate(position_numbers, start=1):
counter = Counter(numbers)
most_common_num, frequency = counter.most_common(1)[0]
most_frequent.append(most_common_num)
percentage = (frequency / len(numbers)) * 100
print(f"Position {position}: {most_common_num:<3} (appears {frequency} times, {percentage:.1f}%)")
print("-" * 60)
print(f"Most frequent numbers combined: {' '.join(map(str, most_frequent))}")
# Analyze most frequent number across first 5 positions combined
print("\n" + "=" * 60)
print("Additional Analysis")
print("=" * 60)
# Combine all numbers from positions 1-5
first_five_combined = []
for i in range(min(5, num_positions)):
first_five_combined.extend(position_numbers[i])
top_five_numbers = []
top_ten_numbers = []
if first_five_combined:
counter_first_five = Counter(first_five_combined)
top_five_numbers = counter_first_five.most_common(5)
top_ten_numbers = counter_first_five.most_common(10)
print("Top 5 most drawn numbers in first 5 positions combined:")
for rank, (number, frequency) in enumerate(top_five_numbers, start=1):
percentage = (frequency / len(first_five_combined)) * 100
print(f" {rank}. {number:<3} (appears {frequency} times, {percentage:.1f}%)")
# Analyze the sixth position (Powerball)
powerball_ranked = []
most_common_sixth = None
if num_positions >= 6:
sixth_position_numbers = position_numbers[5]
counter_sixth = Counter(sixth_position_numbers)
powerball_ranked = counter_sixth.most_common(5)
most_common_sixth, freq_sixth = counter_sixth.most_common(1)[0]
percentage_sixth = (freq_sixth / len(sixth_position_numbers)) * 100
print(f"\nMost drawn number in position 6 (Powerball): {most_common_sixth} (appears {freq_sixth} times, {percentage_sixth:.1f}%)")
# Third strategy: most frequent numbers from the most recent 104 draws
sorted_by_date = sorted(data, key=lambda x: x['draw_date'])
recent_104_draws = sorted_by_date[-104:]
recent_104_top_five, recent_104_powerball = _get_unweighted_top_numbers(recent_104_draws)
# Fourth strategy: weighted-by-recency across all draws
weighted_top_five, weighted_powerball = _get_recency_weighted_top_numbers(data)
# Fifth strategy: aggressive weighted-by-recency across all draws
aggressive_weighted_top_five, aggressive_weighted_powerball = _get_aggressive_recency_weighted_top_numbers(data)
# Return statistics for prediction
return {
'top_five_combined': [num for num, freq in top_five_numbers] if first_five_combined else [],
'top_ten_combined': [num for num, freq in top_ten_numbers],
'most_common_powerball': most_common_sixth,
'powerball_ranked': [num for num, freq in powerball_ranked],
'recent_104_top_five': recent_104_top_five,
'recent_104_powerball': recent_104_powerball,
'weighted_top_five': weighted_top_five,
'weighted_powerball': weighted_powerball,
'aggressive_weighted_top_five': aggressive_weighted_top_five,
'aggressive_weighted_powerball': aggressive_weighted_powerball
}
def generate_suggested_combination(stats: dict) -> None:
"""
Generate a suggested combination based on statistical analysis.
Args:
stats: Dictionary containing statistical analysis results
"""
print("\n" + "=" * 60)
print("Suggested Combination Based on Statistical Analysis")
print("=" * 60)
print("\n*** DISCLAIMER: This is purely statistical analysis.")
print("Past results do NOT predict future outcomes.")
print("Each draw is random and independent.\n")
top_five = stats['top_five_combined']
top_ten = stats.get('top_ten_combined', [])
powerball = stats['most_common_powerball']
ranked_powerballs = stats.get('powerball_ranked', [])
recent_104_top_five = stats.get('recent_104_top_five', [])
recent_104_powerball = stats.get('recent_104_powerball')
weighted_top_five = stats.get('weighted_top_five', [])
weighted_powerball = stats.get('weighted_powerball')
aggressive_weighted_top_five = stats.get('aggressive_weighted_top_five', [])
aggressive_weighted_powerball = stats.get('aggressive_weighted_powerball')
if len(top_five) >= 5 and powerball:
# Use the top 5 most frequent numbers from positions 1-5
primary_numbers = sorted(top_five[:5])
primary_powerball = powerball
# Build a secondary set from the next-most-frequent historical numbers.
secondary_pool = top_ten[5:10]
if len(secondary_pool) < 5:
secondary_pool = top_ten[:5]
secondary_numbers = sorted(secondary_pool[:5])
secondary_powerball = ranked_powerballs[1] if len(ranked_powerballs) > 1 else primary_powerball
third_numbers = sorted(recent_104_top_five[:5]) if len(recent_104_top_five) >= 5 else []
third_powerball = recent_104_powerball
fourth_numbers = sorted(weighted_top_five[:5]) if len(weighted_top_five) >= 5 else []
fourth_powerball = weighted_powerball
fifth_numbers = sorted(aggressive_weighted_top_five[:5]) if len(aggressive_weighted_top_five) >= 5 else []
fifth_powerball = aggressive_weighted_powerball
print("Primary set (most frequent):")
print(f" White balls: {' '.join(map(str, primary_numbers))}")
print(f" Powerball: {primary_powerball}")
print("\nSecondary set (next-most frequent):")
print(f" White balls: {' '.join(map(str, secondary_numbers))}")
print(f" Powerball: {secondary_powerball}")
if third_numbers and third_powerball:
print("\nThird set (most frequent in recent 104 draws):")
print(f" White balls: {' '.join(map(str, third_numbers))}")
print(f" Powerball: {third_powerball}")
if fourth_numbers and fourth_powerball:
print("\nFourth set (weighted by recency):")
print(f" White balls: {' '.join(map(str, fourth_numbers))}")
print(f" Powerball: {fourth_powerball}")
if fifth_numbers and fifth_powerball:
print("\nFifth set (aggressive recency weighting):")
print(f" White balls: {' '.join(map(str, fifth_numbers))}")
print(f" Powerball: {fifth_powerball}")
print()
print(f"Primary combination: {' '.join(map(str, primary_numbers))} + {primary_powerball}")
print(f"Secondary combination: {' '.join(map(str, secondary_numbers))} + {secondary_powerball}")
if third_numbers and third_powerball:
print(f"Third combination: {' '.join(map(str, third_numbers))} + {third_powerball}")
if fourth_numbers and fourth_powerball:
print(f"Fourth combination: {' '.join(map(str, fourth_numbers))} + {fourth_powerball}")
if fifth_numbers and fifth_powerball:
print(f"Fifth combination: {' '.join(map(str, fifth_numbers))} + {fifth_powerball}")
print()
print("These combinations are based on:")
print(" • The 5 most frequently drawn numbers across all white ball positions")
print(f" • The most frequently drawn Powerball number ({primary_powerball})")
print(" • A secondary set from the next-most-frequent white ball and Powerball values")
print(" • Recent-window frequency (last 104 draws)")
print(" • Recency-weighted scoring where newer draws count more")
print(" • Aggressive recency weighting using a cubic curve")
else:
print("Insufficient data to generate a suggestion.")
def detect_consecutive_patterns(data: List[Dict]) -> Dict:
"""
Analyze frequency of consecutive numbers in draws.
Args:
data: List of powerball data dictionaries
Returns:
Dictionary with consecutive number statistics
"""
consecutive_counts = Counter()
total_draws = len(data)
draws_with_consecutives = 0
consecutive_pairs = []
for entry in data:
white_balls = sorted(entry['winning_numbers'][:5])
has_consecutive = False
for i in range(len(white_balls) - 1):
if white_balls[i+1] - white_balls[i] == 1:
consecutive_counts[f"{white_balls[i]}-{white_balls[i+1]}"] += 1
consecutive_pairs.append((white_balls[i], white_balls[i+1]))
has_consecutive = True
if has_consecutive:
draws_with_consecutives += 1
print("\n" + "=" * 60)
print("PATTERN: Consecutive Numbers Analysis")
print("=" * 60)
percentage = (draws_with_consecutives / total_draws) * 100 if total_draws > 0 else 0
print(f"Draws with consecutive numbers: {draws_with_consecutives}/{total_draws} ({percentage:.1f}%)")
if consecutive_counts:
print("\nMost common consecutive pairs:")
for pair, count in consecutive_counts.most_common(10):
pair_pct = (count / total_draws) * 100
print(f" {pair}: {count} times ({pair_pct:.1f}%)")
return {
'draws_with_consecutives': draws_with_consecutives,
'percentage': percentage,
'common_pairs': dict(consecutive_counts.most_common(10))
}
def detect_odd_even_patterns(data: List[Dict]) -> Dict:
"""
Analyze odd/even distribution patterns.
Args:
data: List of powerball data dictionaries
Returns:
Dictionary with odd/even distribution statistics
"""
distribution = Counter()
for entry in data:
white_balls = entry['winning_numbers'][:5]
odd_count = sum(1 for num in white_balls if num % 2 == 1)
even_count = 5 - odd_count
distribution[f"{odd_count}odd-{even_count}even"] += 1
print("\n" + "=" * 60)
print("PATTERN: Odd/Even Distribution")
print("=" * 60)
print("Distribution of odd vs even numbers:")
total = sum(distribution.values())
for pattern, count in sorted(distribution.items(), key=lambda x: x[1], reverse=True):
percentage = (count / total) * 100
print(f" {pattern}: {count} times ({percentage:.1f}%)")
return {'distribution': dict(distribution)}
def detect_high_low_patterns(data: List[Dict]) -> Dict:
"""
Analyze high/low number distribution (1-35 = low, 36-69 = high).
Args:
data: List of powerball data dictionaries
Returns:
Dictionary with high/low distribution statistics
"""
distribution = Counter()
SPLIT_POINT = 35
for entry in data:
white_balls = entry['winning_numbers'][:5]
low_count = sum(1 for num in white_balls if num <= SPLIT_POINT)
high_count = 5 - low_count
distribution[f"{low_count}low-{high_count}high"] += 1
print("\n" + "=" * 60)
print(f"PATTERN: High/Low Distribution (Low: 1-{SPLIT_POINT}, High: {SPLIT_POINT+1}-69)")
print("=" * 60)
print("Distribution of low vs high numbers:")
total = sum(distribution.values())
for pattern, count in sorted(distribution.items(), key=lambda x: x[1], reverse=True):
percentage = (count / total) * 100
print(f" {pattern}: {count} times ({percentage:.1f}%)")
return {'distribution': dict(distribution)}
def detect_sum_patterns(data: List[Dict]) -> Dict:
"""
Analyze the sum of winning numbers.
Args:
data: List of powerball data dictionaries
Returns:
Dictionary with sum statistics
"""
sums = []
for entry in data:
white_balls = entry['winning_numbers'][:5]
total = sum(white_balls)
sums.append(total)
print("\n" + "=" * 60)
print("PATTERN: Sum of Winning Numbers")
print("=" * 60)
avg_sum = sum(sums) / len(sums) if sums else 0
min_sum = min(sums) if sums else 0
max_sum = max(sums) if sums else 0
print(f"Average sum: {avg_sum:.1f}")
print(f"Range: {min_sum} - {max_sum}")
# Create buckets for sum ranges
buckets = Counter()
for s in sums:
bucket = (s // 25) * 25 # Group into 25-number ranges
buckets[f"{bucket}-{bucket+24}"] += 1
print("\nSum distribution by range:")
for range_str, count in sorted(buckets.items(), key=lambda x: int(x[0].split('-')[0])):
percentage = (count / len(sums)) * 100
print(f" {range_str}: {count} times ({percentage:.1f}%)")
return {
'average': avg_sum,
'min': min_sum,
'max': max_sum,
'buckets': dict(buckets)
}
def detect_gap_patterns(data: List[Dict]) -> Dict:
"""
Analyze gaps (spacing) between consecutive numbers when sorted.
Args:
data: List of powerball data dictionaries
Returns:
Dictionary with gap statistics
"""
all_gaps = []
for entry in data:
white_balls = sorted(entry['winning_numbers'][:5])
gaps = [white_balls[i+1] - white_balls[i] for i in range(len(white_balls) - 1)]
all_gaps.extend(gaps)
print("\n" + "=" * 60)
print("PATTERN: Gap Analysis (spacing between sorted numbers)")
print("=" * 60)
avg_gap = sum(all_gaps) / len(all_gaps) if all_gaps else 0
gap_counter = Counter(all_gaps)
print(f"Average gap: {avg_gap:.1f}")
print("\nMost common gap sizes:")
for gap, count in gap_counter.most_common(10):
percentage = (count / len(all_gaps)) * 100
print(f" Gap of {gap}: {count} times ({percentage:.1f}%)")
return {
'average_gap': avg_gap,
'gap_distribution': dict(gap_counter.most_common(15))
}
def detect_hot_cold_numbers(data: List[Dict], recent_draws: int = 20) -> Dict:
"""
Identify hot (frequently appearing) and cold (rarely appearing) numbers in recent draws.
Args:
data: List of powerball data dictionaries
recent_draws: Number of recent draws to analyze
Returns:
Dictionary with hot/cold number analysis
"""
sorted_data = sorted(data, key=lambda x: x['draw_date'])
recent = sorted_data[-recent_draws:] if len(sorted_data) >= recent_draws else sorted_data
white_counter = Counter()
powerball_counter = Counter()
for entry in recent:
for num in entry['winning_numbers'][:5]:
white_counter[num] += 1
if len(entry['winning_numbers']) >= 6:
powerball_counter[entry['winning_numbers'][5]] += 1
print("\n" + "=" * 60)
print(f"PATTERN: Hot & Cold Numbers (Last {len(recent)} draws)")
print("=" * 60)
print(f"\nHottest white balls (appeared most in last {len(recent)} draws):")
for num, count in white_counter.most_common(10):
print(f" {num}: {count} times")
print(f"\nHottest Powerballs:")
for num, count in powerball_counter.most_common(5):
print(f" {num}: {count} times")
# Cold numbers - all possible white ball numbers (1-69) that appeared least
all_white_numbers = set(range(1, 70))
appeared = set(white_counter.keys())
not_appeared = all_white_numbers - appeared
print(f"\nColdest white balls (appeared least or not at all):")
cold_numbers = sorted(
[(num, white_counter.get(num, 0)) for num in all_white_numbers],
key=lambda x: (x[1], x[0])
)[:10]
for num, count in cold_numbers:
if count == 0:
print(f" {num}: 0 times (not appeared)")
else:
print(f" {num}: {count} times")
return {
'hot_white': dict(white_counter.most_common(10)),
'hot_powerball': dict(powerball_counter.most_common(5)),
'cold_white': dict(cold_numbers[:10])
}
def detect_overdue_numbers(data: List[Dict]) -> Dict:
"""
Identify numbers that haven't appeared in the longest time.
Args:
data: List of powerball data dictionaries
Returns:
Dictionary with overdue number analysis
"""
sorted_data = sorted(data, key=lambda x: x['draw_date'])
# Track last appearance of each number
last_seen_white = {}
last_seen_powerball = {}
for entry in sorted_data:
draw_date = entry['draw_date']
for num in entry['winning_numbers'][:5]:
last_seen_white[num] = draw_date
if len(entry['winning_numbers']) >= 6:
last_seen_powerball[entry['winning_numbers'][5]] = draw_date
most_recent_draw = sorted_data[-1]['draw_date']
# Calculate days since last appearance
overdue_white = {}
for num in range(1, 70): # White balls are 1-69
if num in last_seen_white:
days_overdue = (most_recent_draw - last_seen_white[num]).days
overdue_white[num] = days_overdue
else:
overdue_white[num] = float('inf') # Never appeared
overdue_powerball = {}
for num in range(1, 27): # Powerballs are 1-26
if num in last_seen_powerball:
days_overdue = (most_recent_draw - last_seen_powerball[num]).days
overdue_powerball[num] = days_overdue
else:
overdue_powerball[num] = float('inf')
print("\n" + "=" * 60)
print("PATTERN: Overdue Numbers (longest time since last appearance)")
print("=" * 60)
print("\nMost overdue white balls:")
sorted_overdue_white = sorted(overdue_white.items(), key=lambda x: x[1], reverse=True)[:10]
for num, days in sorted_overdue_white:
if days == float('inf'):
print(f" {num}: Never appeared")
else:
print(f" {num}: {days} days ago")
print("\nMost overdue Powerballs:")
sorted_overdue_powerball = sorted(overdue_powerball.items(), key=lambda x: x[1], reverse=True)[:5]
for num, days in sorted_overdue_powerball:
if days == float('inf'):
print(f" {num}: Never appeared")
else:
print(f" {num}: {days} days ago")
return {
'overdue_white': dict(sorted_overdue_white),
'overdue_powerball': dict(sorted_overdue_powerball)
}
def detect_number_pairs(data: List[Dict], top_n: int = 10) -> Dict:
"""
Identify which number pairs appear together most frequently.
Args:
data: List of powerball data dictionaries
top_n: Number of top pairs to display
Returns:
Dictionary with pair frequency analysis
"""
pair_counter = Counter()
for entry in data:
white_balls = entry['winning_numbers'][:5]
# Get all pairs from this draw
for pair in combinations(sorted(white_balls), 2):
pair_counter[pair] += 1
print("\n" + "=" * 60)
print("PATTERN: Number Pair Analysis")
print("=" * 60)
print(f"\nTop {top_n} most common number pairs:")
for (num1, num2), count in pair_counter.most_common(top_n):
percentage = (count / len(data)) * 100
print(f" ({num1}, {num2}): appeared together {count} times ({percentage:.1f}%)")
return {
'top_pairs': dict(pair_counter.most_common(top_n))
}
def detect_decade_distribution(data: List[Dict]) -> Dict:
"""
Analyze how numbers are distributed across decades (1-9, 10-19, 20-29, etc.).
Args:
data: List of powerball data dictionaries
Returns:
Dictionary with decade distribution statistics
"""
decade_counts = Counter()
for entry in data:
white_balls = entry['winning_numbers'][:5]
for num in white_balls:
decade = (num // 10) * 10
decade_label = f"{decade:02d}-{decade+9:02d}" if decade < 60 else "60-69"
decade_counts[decade_label] += 1
print("\n" + "=" * 60)
print("PATTERN: Decade Distribution")
print("=" * 60)
print("How often numbers from each range appear:")
total = sum(decade_counts.values())
for decade in ["00-09", "10-19", "20-29", "30-39", "40-49", "50-59", "60-69"]:
count = decade_counts.get(decade, 0)
percentage = (count / total) * 100 if total > 0 else 0
print(f" {decade}: {count} times ({percentage:.1f}%)")
return {'decade_distribution': dict(decade_counts)}
def run_all_pattern_analyses(data: List[Dict]) -> Dict:
"""
Run all pattern detection analyses.
Args:
data: List of powerball data dictionaries
Returns:
Dictionary containing all pattern analysis results
"""
print("\n" + "#" * 60)
print("# ADVANCED PATTERN DETECTION ANALYSIS")
print("#" * 60)
results = {}
results['consecutive'] = detect_consecutive_patterns(data)
results['odd_even'] = detect_odd_even_patterns(data)
results['high_low'] = detect_high_low_patterns(data)
results['sum'] = detect_sum_patterns(data)
results['gaps'] = detect_gap_patterns(data)
results['hot_cold'] = detect_hot_cold_numbers(data, recent_draws=20)
results['overdue'] = detect_overdue_numbers(data)
results['pairs'] = detect_number_pairs(data, top_n=15)
results['decades'] = detect_decade_distribution(data)
return results
def generate_pattern_based_combinations(data: List[Dict], pattern_results: Dict, num_combinations: int = 5) -> List[Dict]:
"""
Generate number combinations based on pattern analysis.
Args:
data: List of powerball data dictionaries
pattern_results: Results from run_all_pattern_analyses
num_combinations: Number of combinations to generate
Returns:
List of dictionaries containing combination details
"""
import random
print("\n" + "=" * 60)
print("PATTERN-BASED NUMBER COMBINATIONS")
print("=" * 60)
print("\n*** DISCLAIMER: These combinations are based on pattern analysis.")
print("Lottery draws are random. Past patterns do NOT predict future results.")
print("Play responsibly.\n")
combinations = []
# Get pattern insights
hot_white = list(pattern_results['hot_cold']['hot_white'].keys())[:15]
cold_white = [num for num, _ in pattern_results['hot_cold']['cold_white'].items() if _ < 3][:10]
overdue_white = [num for num, days in pattern_results['overdue']['overdue_white'].items() if days != float('inf')][:15]
hot_powerball = list(pattern_results['hot_cold']['hot_powerball'].keys())[:3]
overdue_powerball = [num for num, days in pattern_results['overdue']['overdue_powerball'].items() if days != float('inf')][:5]
# Get most common distributions
odd_even_dist = pattern_results['odd_even']['distribution']
most_common_oe = max(odd_even_dist.items(), key=lambda x: x[1])[0]
target_odds = int(most_common_oe.split('odd')[0])
high_low_dist = pattern_results['high_low']['distribution']
most_common_hl = max(high_low_dist.items(), key=lambda x: x[1])[0]
target_lows = int(most_common_hl.split('low')[0])
# Get average sum
avg_sum = pattern_results['sum']['average']
sum_tolerance = 30
# Get top pairs
top_pairs = list(pattern_results['pairs']['top_pairs'].keys())[:20]
# Strategy descriptions
strategies = [
("Hot Numbers Focus", hot_white, "Uses frequently drawn recent numbers"),
("Balanced Hot + Overdue", hot_white[:8] + overdue_white[:12], "Mixes hot numbers with overdue picks"),
("Overdue Numbers Focus", overdue_white, "Focuses on numbers that haven't appeared recently"),
("Cold Numbers Gamble", cold_white + hot_white[:10], "Includes rarely drawn numbers"),
("Pair-Based Selection", [n for pair in top_pairs for n in pair], "Uses numbers from common pairs")
]
for strategy_idx in range(min(num_combinations, len(strategies))):
strategy_name, number_pool, strategy_desc = strategies[strategy_idx]
# Ensure we have enough numbers in pool
if len(number_pool) < 20:
number_pool = list(set(number_pool + hot_white + list(range(1, 70))))
# Try to generate a combination that fits the patterns
attempts = 0
max_attempts = 1000
selected = [] # Initialize to avoid unbound variable
while attempts < max_attempts:
# Select 5 random numbers from pool
selected = random.sample(number_pool[:30], min(5, len(number_pool[:30])))
selected = sorted(selected)
# Check odd/even distribution
odds_count = sum(1 for n in selected if n % 2 == 1)
if abs(odds_count - target_odds) > 1:
attempts += 1
continue
# Check high/low distribution
lows_count = sum(1 for n in selected if n <= 35)
if abs(lows_count - target_lows) > 1:
attempts += 1
continue
# Check sum is reasonable
total = sum(selected)
if abs(total - avg_sum) > sum_tolerance:
attempts += 1
continue
# Check gaps are reasonable (not too clustered or too spread)
gaps = [selected[i+1] - selected[i] for i in range(4)]
if min(gaps) < 2 or max(gaps) > 25:
attempts += 1
continue
# Good combination found
break
# If we didn't find a perfect match, selected still has the last attempt
if not selected:
selected = sorted(random.sample(range(1, 70), 5))
# Select powerball
if strategy_idx % 2 == 0 and hot_powerball:
powerball = random.choice(hot_powerball)
elif overdue_powerball:
powerball = random.choice(overdue_powerball[:5])
else:
powerball = random.randint(1, 26)
# Calculate combination stats
odds = sum(1 for n in selected if n % 2 == 1)
evens = 5 - odds
lows = sum(1 for n in selected if n <= 35)
highs = 5 - lows
total = sum(selected)
combinations.append({
'strategy': strategy_name,
'description': strategy_desc,
'numbers': selected,
'powerball': powerball,
'odds': odds,
'evens': evens,
'lows': lows,
'highs': highs,
'sum': total,
'attempts': attempts
})
# Display combinations
for i, combo in enumerate(combinations, 1):
print(f"\nCombination #{i}: {combo['strategy']}")
print(f" Strategy: {combo['description']}")
print(f" Numbers: {' '.join(map(str, combo['numbers']))} + Powerball: {combo['powerball']}")
print(f" Stats: {combo['odds']}odd-{combo['evens']}even, {combo['lows']}low-{combo['highs']}high, Sum: {combo['sum']}")
print("\n" + "=" * 60)
print("Quick Pick Lines:")
print("=" * 60)
for i, combo in enumerate(combinations, 1):
numbers_str = ' - '.join(f"{n:2d}" for n in combo['numbers'])
print(f" Line {i}: {numbers_str} | PB: {combo['powerball']:2d}")
return combinations
def main():
"""Main function."""
# Default to 'powerball_numbers.csv' in the same directory as the script
script_dir = Path(__file__).parent
csv_file = script_dir / 'powerball_numbers.csv'
# You can also specify a different file path here
# csv_file = Path('path/to/your/file.csv')
print(f"Reading Powerball numbers from: {csv_file}\n")
powerball_data = read_powerball_csv(str(csv_file))
if powerball_data:
print(f"Successfully loaded {len(powerball_data)} records.\n")
display_powerball_data(powerball_data)
stats = analyze_most_frequent_numbers(powerball_data)
generate_suggested_combination(stats)
# Run advanced pattern detection
pattern_results = run_all_pattern_analyses(powerball_data)
# Generate pattern-based combinations
pattern_combos = generate_pattern_based_combinations(powerball_data, pattern_results, num_combinations=5)
else:
print("Failed to load powerball data.")
if __name__ == '__main__':
main()
personal/tools/pyrsync.py
-382
View File
@@ -1,382 +0,0 @@
#!/bin/python3
"""
Parallel rsync launcher.
Requires rsync on PATH (e.g. via MSYS/Cygwin on Windows).
Runs multiple rsync invocations in parallel using either top-level entries
or balanced recursive file chunks.
"""
# Usage examples:
#
# Basic sync (recursive-files mode, auto worker count):
# python pyrsync.py /src/dir user@host:/dest/
#
# Auto-tune workers based on file size and local vs remote destination:
# python pyrsync.py /src/dir user@host:/dest/ --auto-tune
#
# Limit to a specific number of parallel workers:
# python pyrsync.py /src/dir /local/dest/ -w 8
#
# Use top-level mode (one rsync process per immediate source entry):
# python pyrsync.py /src/dir /local/dest/ --mode top-level
#
# Dry run (show what would be transferred without doing it):
# python pyrsync.py /src/dir user@host:/dest/ --dry-run
#
# Verbose output (shows command templates, task list preview, and elapsed time):
# python pyrsync.py /src/dir /local/dest/ -v
#
# Show rsync output for all tasks:
# python pyrsync.py /src/dir /local/dest/ --show-rsync-output
#
# Show rsync output only when a task fails:
# python pyrsync.py /src/dir user@host:/dest/ --show-rsync-output-on-failure
#
# Full-featured example (auto-tune, verbose, show failures):
# python pyrsync.py /src/dir user@host:/dest/ --auto-tune -v --show-rsync-output-on-failure
#
# Override rsync flags entirely:
# python pyrsync.py /src/dir /local/dest/ -r "-az --checksum"
import argparse
import os
import re
import shlex
import subprocess
import sys
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import List, Sequence, Tuple
# DEFAULT_RSYNC_ARGS = ["-av", "--partial", "--compress", "--info=progress2"]
DEFAULT_RSYNC_ARGS = ["-a", "--partial", "--info=stats2"]
def build_file_list(src: str) -> List[Path]:
p = Path(src)
if p.is_dir():
# sync each top-level entry separately (files and immediate subdirs)
return [x for x in p.iterdir()]
# allow glob patterns
if any(ch in src for ch in ["*", "?"]):
return list(Path(".").glob(src))
# single file
return [p]
def build_recursive_file_list(src_dir: Path) -> List[Path]:
"""Return all files under src_dir as relative paths for --files-from."""
return [p.relative_to(src_dir) for p in src_dir.rglob("*") if p.is_file()]
def get_sized_files(src_dir: Path, rel_files: Sequence[Path]) -> List[Tuple[int, Path]]:
"""Return (size_bytes, rel_path) tuples with best-effort size lookup."""
sized_files: List[Tuple[int, Path]] = []
for rel_path in rel_files:
abs_path = src_dir / rel_path
try:
size = abs_path.stat().st_size
except OSError:
size = 0
sized_files.append((size, rel_path))
return sized_files
def partition_by_size(sized_files: Sequence[Tuple[int, Path]], workers: int) -> List[List[Path]]:
"""Greedy size-balanced partitioning to reduce worker skew."""
buckets: List[List[Path]] = [[] for _ in range(workers)]
bucket_sizes = [0] * workers
# Largest files first gives better load balancing across workers.
sorted_files = sorted(sized_files, key=lambda x: x[0], reverse=True)
for size, rel_path in sorted_files:
idx = min(range(workers), key=lambda i: bucket_sizes[i])
buckets[idx].append(rel_path)
bucket_sizes[idx] += size
return [b for b in buckets if b]
def detect_remote_destination(dest: str) -> bool:
"""Heuristic detection of rsync remote destinations."""
if dest.startswith("rsync://"):
return True
if "::" in dest:
return True
if re.match(r"^[a-zA-Z]:[\\/]", dest):
return False
if re.match(r"^\\\\", dest):
return False
return bool(re.match(r"^[^:/\\]+@?[^:/\\]*:[^/].*", dest))
def format_bytes(num_bytes: float) -> str:
units = ["B", "KB", "MB", "GB", "TB"]
value = float(num_bytes)
for unit in units:
if value < 1024.0 or unit == units[-1]:
return f"{value:.1f}{unit}"
value /= 1024.0
return f"{num_bytes:.1f}B"
def auto_tune_workers(
total_units: int,
avg_size_bytes: float,
is_remote: bool,
cpu_count: int,
configured_workers: int,
) -> int:
"""Pick worker count from workload size profile and destination type."""
cpu = max(1, cpu_count)
if avg_size_bytes < 1 * 1024 * 1024:
base = min(32, cpu * 4)
elif avg_size_bytes < 32 * 1024 * 1024:
base = min(16, cpu * 2)
elif avg_size_bytes < 256 * 1024 * 1024:
base = min(8, cpu)
else:
base = min(4, max(2, cpu // 2))
if is_remote:
base = max(2, min(base, 8))
else:
base = min(base + 2, 24)
tuned = max(1, min(base, total_units))
return min(tuned, configured_workers)
def maybe_print_task_output(label: str, out: str, err: str, show_output: bool, show_on_fail_only: bool, code: int) -> None:
if not show_output:
return
if show_on_fail_only and code == 0:
return
if out.strip():
print(f"--- stdout: {label} ---")
print(out.rstrip())
if err.strip():
print(f"--- stderr: {label} ---", file=sys.stderr)
print(err.rstrip(), file=sys.stderr)
def run_rsync(item: Path, dest: str, rsync_args: List[str], dry_run: bool) -> Tuple[Path, int, str, str]:
cmd = ["rsync", *rsync_args]
if dry_run:
cmd.append("--dry-run")
# preserve relative path by sending the item name, not its parent
cmd += [str(item), dest]
try:
proc = subprocess.run(cmd, capture_output=True, text=True)
return item, proc.returncode, proc.stdout, proc.stderr
except FileNotFoundError:
return item, 127, "", "rsync not found on PATH"
def run_rsync_files_from(
src_dir: Path,
rel_files: Sequence[Path],
dest: str,
rsync_args: List[str],
dry_run: bool,
) -> Tuple[str, int, str, str]:
cmd = ["rsync", *rsync_args]
if dry_run:
cmd.append("--dry-run")
cmd += ["--from0", "--files-from=-", f"{src_dir}{os.sep}", dest]
payload = "\0".join(str(p) for p in rel_files) + "\0"
try:
proc = subprocess.run(cmd, input=payload, capture_output=True, text=True)
label = f"chunk({len(rel_files)} files)"
return label, proc.returncode, proc.stdout, proc.stderr
except FileNotFoundError:
return "chunk(0 files)", 127, "", "rsync not found on PATH"
def main():
ap = argparse.ArgumentParser(description="Transfer files using rsync in parallel.")
ap.add_argument("src", help="Source file, directory or glob pattern")
ap.add_argument("dest", help="Rsync destination (local path or user@host:/path)")
ap.add_argument(
"-w",
"--workers",
type=int,
default=min(8, (os.cpu_count() or 4)),
help="Parallel rsync workers (default: min(8, CPU count))",
)
ap.add_argument(
"--mode",
choices=["top-level", "recursive-files"],
default="recursive-files",
help=(
"top-level: one rsync per immediate src entry; "
"recursive-files: split full file tree into balanced rsync chunks"
),
)
ap.add_argument("-r", "--rsync-args", default=" ".join(DEFAULT_RSYNC_ARGS),
help="Additional rsync args (quoted). Default: -a --partial --info=stats2")
ap.add_argument(
"--auto-tune",
action="store_true",
help=(
"Auto-tune worker count from average file size and destination type. "
"Uses --workers as an upper bound."
),
)
ap.add_argument(
"-v",
"--verbose",
action="store_true",
help="Show detailed progress, including task start/completion and rsync command lines.",
)
ap.add_argument(
"--show-rsync-output",
action="store_true",
help="Print rsync stdout/stderr for each task (can be noisy).",
)
ap.add_argument(
"--show-rsync-output-on-failure",
action="store_true",
help="Print rsync stdout/stderr only for failed tasks.",
)
ap.add_argument("--dry-run", action="store_true", help="Don't transfer, just show what would happen")
args = ap.parse_args()
if args.workers < 1:
print("--workers must be >= 1", file=sys.stderr)
sys.exit(2)
rsync_args = shlex.split(args.rsync_args)
results = []
started_at = time.perf_counter()
if args.show_rsync_output and args.show_rsync_output_on_failure:
print(
"Both --show-rsync-output and --show-rsync-output-on-failure set; "
"showing output for all tasks.",
file=sys.stderr,
)
show_output = args.show_rsync_output or args.show_rsync_output_on_failure
show_on_fail_only = args.show_rsync_output_on_failure and not args.show_rsync_output
src_path = Path(args.src)
is_remote_dest = detect_remote_destination(args.dest)
cpu_count = os.cpu_count() or 4
if args.mode == "recursive-files" and src_path.is_dir():
rel_files = build_recursive_file_list(src_path)
if not rel_files:
print("No files found for:", args.src, file=sys.stderr)
sys.exit(2)
sized_files = get_sized_files(src_path, rel_files)
avg_size = sum(size for size, _ in sized_files) / max(1, len(sized_files))
if args.auto_tune:
workers = auto_tune_workers(len(rel_files), avg_size, is_remote_dest, cpu_count, args.workers)
dest_kind = "remote" if is_remote_dest else "local"
print(
f"Auto-tuned workers={workers} ({dest_kind} dest, avg file {format_bytes(avg_size)})"
)
else:
workers = min(args.workers, len(rel_files))
chunks = partition_by_size(sized_files, workers)
print(f"Starting {len(chunks)} rsync chunk task(s) with {workers} worker(s)...")
if args.verbose:
cmd_preview = ["rsync", *rsync_args]
if args.dry_run:
cmd_preview.append("--dry-run")
cmd_preview += ["--from0", "--files-from=-", f"{src_path}{os.sep}", args.dest]
print("Command template:", " ".join(shlex.quote(part) for part in cmd_preview))
with ThreadPoolExecutor(max_workers=workers) as ex:
futures = {
ex.submit(run_rsync_files_from, src_path, chunk, args.dest, rsync_args, args.dry_run): chunk
for chunk in chunks
}
for fut in as_completed(futures):
label, code, out, err = fut.result()
results.append((label, code, out, err))
status = "OK" if code == 0 else f"ERR({code})"
print(f"[{status}] {label}")
maybe_print_task_output(label, out, err, show_output, show_on_fail_only, code)
else:
items = build_file_list(args.src)
if not items:
print("No files found for:", args.src, file=sys.stderr)
sys.exit(2)
size_samples = []
for item in items:
try:
if item.is_file():
size_samples.append(item.stat().st_size)
else:
size_samples.append(64 * 1024 * 1024)
except OSError:
size_samples.append(0)
avg_size = sum(size_samples) / max(1, len(size_samples))
if args.auto_tune:
workers = auto_tune_workers(len(items), avg_size, is_remote_dest, cpu_count, args.workers)
dest_kind = "remote" if is_remote_dest else "local"
print(
f"Auto-tuned workers={workers} ({dest_kind} dest, avg unit {format_bytes(avg_size)})"
)
else:
workers = min(args.workers, len(items))
print(f"Starting {len(items)} rsync task(s) with {workers} worker(s)...")
if args.verbose:
print("Task list preview:")
preview_limit = 10
for idx, item in enumerate(items[:preview_limit], start=1):
print(f" {idx}. {item}")
if len(items) > preview_limit:
print(f" ... and {len(items) - preview_limit} more")
with ThreadPoolExecutor(max_workers=workers) as ex:
futures = {}
for item in items:
if args.verbose:
cmd_preview = ["rsync", *rsync_args]
if args.dry_run:
cmd_preview.append("--dry-run")
cmd_preview += [str(item), args.dest]
print("Scheduling:", " ".join(shlex.quote(part) for part in cmd_preview))
futures[ex.submit(run_rsync, item, args.dest, rsync_args, args.dry_run)] = item
for fut in as_completed(futures):
item, code, out, err = fut.result()
results.append((str(item), code, out, err))
status = "OK" if code == 0 else f"ERR({code})"
print(f"[{status}] {item}")
maybe_print_task_output(str(item), out, err, show_output, show_on_fail_only, code)
# summary
failed = [r for r in results if r[1] != 0]
elapsed = time.perf_counter() - started_at
print(f"\nCompleted: {len(results)} tasks, {len(failed)} failed in {elapsed:.2f}s.")
if failed:
print("Failures detail:")
for label, code, out, err in failed:
print(f"--- {label} (exit {code}) ---")
if out:
print(out.strip())
if err:
print(err.strip())
sys.exit(1 if failed else 0)
if __name__ == "__main__":
main()
personal/tools/read_full_table_list.py
-40
View File
@@ -1,40 +0,0 @@
import csv
from pathlib import Path
def collect_first_tokens_from_full_table_files(directory: Path) -> list[str]:
first_tokens: list[str] = []
for file_path in directory.iterdir():
# Match file names like "Full_Table_List_..." regardless of case.
if file_path.is_file() and file_path.name.lower().startswith("full_table_list"):
with file_path.open("r", encoding="utf-8", errors="ignore") as f:
for line in f:
parts = line.strip().split()
if parts:
first_tokens.append(parts[0])
return first_tokens
def main() -> None:
current_directory = Path.cwd()
tokens = collect_first_tokens_from_full_table_files(current_directory)
tokens.sort()
for token in tokens:
print(token)
output_file = current_directory / "full_table_list_tokens.csv"
with output_file.open("w", newline="", encoding="utf-8") as csv_file:
writer = csv.writer(csv_file)
writer.writerow(["value"])
for token in tokens:
writer.writerow([token])
print(f"\nTotal extracted values: {len(tokens)}")
print(f"Saved CSV file: {output_file}")
if __name__ == "__main__":
main()
personal/tools/sync_data.ps1
-105
View File
@@ -1,105 +0,0 @@
#Requires -Version 5.1
# Use -ShutdownRegardless to force shutdown even if one or more sync jobs fail.
param(
[switch]$ShutdownRegardless
)
$FfsExe = 'C:\Program Files\FreeFileSync\FreeFileSync.exe'
$ScriptDir = $PSScriptRoot
$LogDir = Join-Path $ScriptDir 'logs'
$ReportDir = Join-Path $ScriptDir 'reports'
$SyncJobs = @(
'C:\sync_jobs\shuttlebay_SyncSettings.ffs_batch'
'C:\sync_jobs\main_bridge_SyncSettings.ffs_batch'
'C:\sync_jobs\engineering_SyncSettings.ffs_batch'
)
# ── Setup ──────────────────────────────────────────────────────────────────────
foreach ($dir in $LogDir, $ReportDir) {
if (-not (Test-Path $dir)) { New-Item -ItemType Directory -Path $dir | Out-Null }
}
$RunTs = Get-Date -Format 'yyyyMMdd_HHmmss'
$LogFile = Join-Path $LogDir "sync_history_$RunTs.log"
function Write-Log {
param([string]$Message)
$line = "[$(Get-Date -Format 'ddd MM/dd/yyyy HH:mm:ss.ff')] $Message"
Write-Host $line
Add-Content -Path $LogFile -Value $line
}
Write-Log 'Sync run started'
Write-Host "Log file: $LogFile"
# ── Run sync jobs ──────────────────────────────────────────────────────────────
$HadFatalError = $false
$FatalExitCodes = @()
foreach ($job in $SyncJobs) {
$jobName = Split-Path $job -Leaf
Write-Host ''
Write-Log "Running synchronization: $jobName"
$tmpOut = [IO.Path]::GetTempFileName()
$proc = Start-Process -FilePath $FfsExe -ArgumentList $job `
-RedirectStandardOutput $tmpOut -NoNewWindow -PassThru -Wait
$rc = $proc.ExitCode
$output = Get-Content -Path $tmpOut -Raw
Remove-Item $tmpOut -Force
if ($output) { Add-Content -Path $LogFile -Value $output }
switch ($rc) {
0 { Write-Log 'Synchronization completed successfully.' }
1 { Write-Log 'Synchronization completed with warnings.' }
default {
Write-Log "Synchronization failed or was aborted. Exit code: $rc"
$HadFatalError = $true
$FatalExitCodes += $rc
continue
}
}
}
Write-Host ''
Write-Log 'All synchronization jobs completed.'
# ── Copy HTML reports ──────────────────────────────────────────────────────────
$logPaths = Select-String -Path $LogFile -Pattern '"logFile"\s*:\s*"([^"]+)"' |
ForEach-Object { $_.Matches[0].Groups[1].Value -replace '\\\\', '\' } |
Sort-Object -Unique
$copied = 0
foreach ($path in $logPaths) {
if ([IO.Path]::GetExtension($path) -ieq '.html') {
if (Test-Path -LiteralPath $path) {
Copy-Item -LiteralPath $path -Destination $ReportDir -Force
Write-Log "Copied report: $(Split-Path $path -Leaf)"
$copied++
} else {
Write-Log "Report not found: $path"
}
}
}
Write-Log "Total reports copied: $copied"
# ── Shutdown ───────────────────────────────────────────────────────────────────
if ($HadFatalError) {
$codes = ($FatalExitCodes | Sort-Object -Unique) -join ', '
Write-Log "One or more synchronization jobs failed. Fatal exit code(s): $codes"
if (-not $ShutdownRegardless) {
Write-Log 'Skipping shutdown because not all synchronization jobs completed successfully.'
Write-Log 'Use -ShutdownRegardless to force shutdown even when jobs fail.'
exit ($FatalExitCodes | Select-Object -Last 1)
}
Write-Log 'Forced shutdown enabled. Continuing to shutdown despite synchronization failures.'
}
Write-Log 'Shutting down server now.'
Invoke-CimMethod -ClassName Win32_OperatingSystem -MethodName Win32Shutdown -Arguments @{ Flags = 5 }