diff --git a/personal/cron_scripts/ilo_powercycle.py b/personal/cron_scripts/ilo_powercycle.py new file mode 100644 index 0000000..4e07fc7 --- /dev/null +++ b/personal/cron_scripts/ilo_powercycle.py @@ -0,0 +1,57 @@ +#!/usr/bin/python3 +import argparse +import configparser +import os +import sys +import urllib3 +import redfish + +# Suppress SSL warnings for self-signed iLO certificates +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +parser = argparse.ArgumentParser(description="Power off an HPE server via iLO Redfish") +parser.add_argument("hostname", help="iLO hostname or IP address") +parser.add_argument("-c", "--credentials", default="/etc/ilo_credentials", + help="Path to credentials file (default: /etc/ilo_credentials)") +args = parser.parse_args() + +# Read credentials from file (chmod 600 recommended: sudo chmod 600 /etc/ilo_credentials) +# File format: +# [ilo] +# username = root +# password = yourpassword +cred_path = args.credentials +if not os.path.exists(cred_path): + print(f"Error: credentials file not found: {cred_path}", file=sys.stderr) + sys.exit(1) + +config = configparser.ConfigParser() +config.read(cred_path) +try: + LOGIN_ACCOUNT = config.get("ilo", "username", fallback="root") + LOGIN_PASSWORD = config.get("ilo", "password") +except (configparser.NoSectionError, configparser.NoOptionError) as e: + print(f"Error reading credentials file: {e}", file=sys.stderr) + sys.exit(1) + +BASE_URL = f"https://{args.hostname}" + +REDFISHOBJ = redfish.RedfishClient( + base_url=BASE_URL, + username=LOGIN_ACCOUNT, + password=LOGIN_PASSWORD, +) +REDFISHOBJ.login(auth="session") + +# Discover the correct reset action URI from the system resource +sys_response = REDFISHOBJ.get("/redfish/v1/Systems/1/") +reset_uri = sys_response.dict["Actions"]["#ComputerSystem.Reset"]["target"] + +body = {"ResetType": "PushPowerButton"} # iLO 4: PushPowerButton = graceful shutdown; use "ForceOff" for hard power cut +response = REDFISHOBJ.post(reset_uri, body=body) + +print(f"Status: {response.status}") +if response.status >= 400: + print(f"Error: {response.read}") + +REDFISHOBJ.logout() \ No newline at end of file diff --git a/personal/cron_scripts/ilo_poweron.py b/personal/cron_scripts/ilo_poweron.py new file mode 100644 index 0000000..4e07fc7 --- /dev/null +++ b/personal/cron_scripts/ilo_poweron.py @@ -0,0 +1,57 @@ +#!/usr/bin/python3 +import argparse +import configparser +import os +import sys +import urllib3 +import redfish + +# Suppress SSL warnings for self-signed iLO certificates +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +parser = argparse.ArgumentParser(description="Power off an HPE server via iLO Redfish") +parser.add_argument("hostname", help="iLO hostname or IP address") +parser.add_argument("-c", "--credentials", default="/etc/ilo_credentials", + help="Path to credentials file (default: /etc/ilo_credentials)") +args = parser.parse_args() + +# Read credentials from file (chmod 600 recommended: sudo chmod 600 /etc/ilo_credentials) +# File format: +# [ilo] +# username = root +# password = yourpassword +cred_path = args.credentials +if not os.path.exists(cred_path): + print(f"Error: credentials file not found: {cred_path}", file=sys.stderr) + sys.exit(1) + +config = configparser.ConfigParser() +config.read(cred_path) +try: + LOGIN_ACCOUNT = config.get("ilo", "username", fallback="root") + LOGIN_PASSWORD = config.get("ilo", "password") +except (configparser.NoSectionError, configparser.NoOptionError) as e: + print(f"Error reading credentials file: {e}", file=sys.stderr) + sys.exit(1) + +BASE_URL = f"https://{args.hostname}" + +REDFISHOBJ = redfish.RedfishClient( + base_url=BASE_URL, + username=LOGIN_ACCOUNT, + password=LOGIN_PASSWORD, +) +REDFISHOBJ.login(auth="session") + +# Discover the correct reset action URI from the system resource +sys_response = REDFISHOBJ.get("/redfish/v1/Systems/1/") +reset_uri = sys_response.dict["Actions"]["#ComputerSystem.Reset"]["target"] + +body = {"ResetType": "PushPowerButton"} # iLO 4: PushPowerButton = graceful shutdown; use "ForceOff" for hard power cut +response = REDFISHOBJ.post(reset_uri, body=body) + +print(f"Status: {response.status}") +if response.status >= 400: + print(f"Error: {response.read}") + +REDFISHOBJ.logout() \ No newline at end of file diff --git a/personal/cron_scripts/ipmi_poweroff.py b/personal/cron_scripts/ipmi_poweroff.py new file mode 100644 index 0000000..9847003 --- /dev/null +++ b/personal/cron_scripts/ipmi_poweroff.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +import argparse +import configparser +import os +import subprocess +import sys + +DEFAULT_EK = "0000000000000000000000000000000000000000" + + +def load_credentials(path): + if not os.path.exists(path): + print(f"Error: credentials file not found: {path}", file=sys.stderr) + sys.exit(1) + + config = configparser.ConfigParser() + config.read(path) + + try: + username = config.get("credentials", "username", fallback="root") + password = config.get("credentials", "password") + encryption_key = config.get("credentials", "encryption_key", fallback=DEFAULT_EK) + except (configparser.NoSectionError, configparser.NoOptionError) as e: + print(f"Error reading credentials file: {e}", file=sys.stderr) + sys.exit(1) + + return username, password, encryption_key + + +def main(): + p = argparse.ArgumentParser(description="Send IPMI power off") + p.add_argument("-H", "--host", required=True, help="IPMI host address") + p.add_argument( + "-c", + "--credentials", + default="credentials", + help="Path to credentials file (default: credentials)", + ) + args = p.parse_args() + + ipmi_user, ipmi_pw, ipmi_ek = load_credentials(args.credentials) + + cmd = [ + "ipmitool", + "-I", "lanplus", + "-H", args.host, + "-U", ipmi_user, + "-P", ipmi_pw, + "-y", ipmi_ek, + "power", "off", + ] + + proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if proc.returncode != 0: + print("ipmitool failed:", proc.stderr.strip(), file=sys.stderr) + sys.exit(proc.returncode) + + print(proc.stdout.strip()) + +if __name__ == "__main__": + main() diff --git a/personal/cron_scripts/ipmi_poweron.py b/personal/cron_scripts/ipmi_poweron.py new file mode 100644 index 0000000..1ed2228 --- /dev/null +++ b/personal/cron_scripts/ipmi_poweron.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +import argparse +import configparser +import os +import subprocess +import sys +import datetime + +def ts_print(msg): + """ts_print message prefixed with a timestamp (YYYY-MM-DD HH:MM:SS).""" + now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + print(f"{now} {msg}") + +DEFAULT_EK = "0000000000000000000000000000000000000000" + + +def load_credentials(path): + if not os.path.exists(path): + print(f"Error: credentials file not found: {path}", file=sys.stderr) + sys.exit(1) + + config = configparser.ConfigParser() + config.read(path) + + try: + username = config.get("ilo", "username", fallback="root") + password = config.get("ilo", "password") + encryption_key = config.get("ilo", "encryption_key", fallback=DEFAULT_EK) + except (configparser.NoSectionError, configparser.NoOptionError) as e: + print(f"Error reading credentials file: {e}", file=sys.stderr) + sys.exit(1) + + return username, password, encryption_key + +def main(): + p = argparse.ArgumentParser(description="Send IPMI power on") + p.add_argument("-H", "--host", required=True, help="IPMI host address") + p.add_argument( + "-c", + "--credentials", + default="credentials", + help="Path to credentials file (default: credentials)", + ) + args = p.parse_args() + + ipmi_user, ipmi_pw, ipmi_ek = load_credentials(args.credentials) + + cmd = [ + "ipmitool", + "-I", "lanplus", + "-H", args.host, + "-U", ipmi_user, + "-P", ipmi_pw, + "-y", ipmi_ek, + "power", "on", + ] + print(f"Sending power on to {args.host}") + proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if proc.returncode != 0: + print("ipmitool failed:", proc.stderr.strip(), file=sys.stderr) + sys.exit(proc.returncode) + print(proc.stdout.strip()) + +if __name__ == "__main__": + main() diff --git a/personal/cron_scripts/pg_backup.sh b/personal/cron_scripts/pg_backup.sh new file mode 100644 index 0000000..d1a1e8e --- /dev/null +++ b/personal/cron_scripts/pg_backup.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Configuration +BACKUP_DIR="/var/lib/postgresql_archive/pg_dumpfiles" +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +RETENTION_DAYS=7 + +# Ensure directory exists +mkdir -p "$BACKUP_DIR" + +# Get all non-system databases and back each one up. +psql -U postgres -d postgres -Atc "SELECT datname FROM pg_database WHERE datistemplate = false AND datname <> 'postgres' ORDER BY datname" | while IFS= read -r DB_NAME; do + pg_dump -U postgres -d "$DB_NAME" | gzip > "$BACKUP_DIR/${DB_NAME}_$TIMESTAMP.sql.gz" + echo "Backup completed: ${DB_NAME}_$TIMESTAMP.sql.gz" +done + +# Delete backups older than retention period +find "$BACKUP_DIR" -type f -mtime +$RETENTION_DAYS -delete \ No newline at end of file diff --git a/personal/cron_scripts/pg_basebackup.sh b/personal/cron_scripts/pg_basebackup.sh new file mode 100644 index 0000000..2180120 --- /dev/null +++ b/personal/cron_scripts/pg_basebackup.sh @@ -0,0 +1,10 @@ +#!/bin/bash +BACKUP_DIR="/var/lib/postgresql_archive/pgbackups/$(date +%Y%m%d)" +mkdir -p $BACKUP_DIR +# Perform base backup with WAL streaming +pg_basebackup -h localhost -U wjones -D $BACKUP_DIR -Fp -Xs -P +# Optional: Compress +tar -czf ${BACKUP_DIR}_$(date +%Y%m%d_%H%M%S).tar.gz $BACKUP_DIR +rm -rf $BACKUP_DIR +# Retention: Delete backups older than 7 days +find /var/lib/postgresql_archive/pgbackups/ -type f -mtime +7 -name "*.tar.gz" -delete diff --git a/personal/tools/airsonic_pg_diagnostics.sh b/personal/tools/airsonic_pg_diagnostics.sh new file mode 100644 index 0000000..410a561 --- /dev/null +++ b/personal/tools/airsonic_pg_diagnostics.sh @@ -0,0 +1,276 @@ +#!/bin/bash +set -euo pipefail + +# Airsonic Advanced + PostgreSQL diagnostics +# Focus: slow media scans, index efficiency, write pressure, and cache tuning signals. + +PGHOST="${PGHOST:-localhost}" +PGPORT="${PGPORT:-5432}" +PGUSER="${PGUSER:-postgres}" +PGDATABASE="${PGDATABASE:-airsonic}" +OUT_FILE="${1:-airsonic_pg_diagnostics_$(date +%Y%m%d_%H%M%S).txt}" + +PSQL=(psql -X -v ON_ERROR_STOP=1 -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE") + +run_section() { + local title="$1" + local sql="$2" + + { + echo + echo "================================================================" + echo "$title" + echo "================================================================" + } >> "$OUT_FILE" + + if ! "${PSQL[@]}" -c "$sql" >> "$OUT_FILE" 2>&1; then + echo "Query failed for section: $title" >> "$OUT_FILE" + fi +} + +query_single_value() { + local sql="$1" + "${PSQL[@]}" -Atc "$sql" 2>/dev/null || true +} + +bytes_to_human() { + local bytes="$1" + if [[ -z "$bytes" || ! "$bytes" =~ ^[0-9]+$ ]]; then + echo "unknown" + return + fi + + local kib=$((1024)) + local mib=$((1024 * 1024)) + local gib=$((1024 * 1024 * 1024)) + + if (( bytes >= gib )); then + awk -v b="$bytes" -v g="$gib" 'BEGIN { printf "%.2f GiB", b/g }' + elif (( bytes >= mib )); then + awk -v b="$bytes" -v m="$mib" 'BEGIN { printf "%.2f MiB", b/m }' + elif (( bytes >= kib )); then + awk -v b="$bytes" -v k="$kib" 'BEGIN { printf "%.2f KiB", b/k }' + else + echo "${bytes} B" + fi +} + +{ + echo "Airsonic PostgreSQL Diagnostics" + echo "Generated at: $(date -Iseconds)" + echo "Target: host=$PGHOST port=$PGPORT db=$PGDATABASE user=$PGUSER" +} > "$OUT_FILE" + +run_section "1) Server identity and uptime" " +SELECT version(); +SELECT now() AS current_time, pg_postmaster_start_time() AS postmaster_start_time; +" + +run_section "2) Key PostgreSQL tuning parameters" " +SELECT name, setting, unit, short_desc +FROM pg_settings +WHERE name IN ( + 'shared_buffers', + 'work_mem', + 'maintenance_work_mem', + 'effective_cache_size', + 'max_connections', + 'effective_io_concurrency', + 'random_page_cost', + 'seq_page_cost', + 'checkpoint_timeout', + 'checkpoint_completion_target', + 'autovacuum', + 'autovacuum_max_workers', + 'autovacuum_naptime' +) +ORDER BY name; +" + +run_section "3) Database cache hit ratio and temp usage" " +SELECT + datname, + blks_read, + blks_hit, + ROUND(100.0 * blks_hit / NULLIF(blks_hit + blks_read, 0), 2) AS cache_hit_pct, + temp_files, + pg_size_pretty(temp_bytes) AS temp_bytes, + deadlocks, + stats_reset +FROM pg_stat_database +WHERE datname = current_database(); +" + +run_section "4) Most read tables and their index-vs-seq scan profile" " +SELECT + relname, + seq_scan, + idx_scan, + n_live_tup, + n_dead_tup, + ROUND((100.0 * idx_scan / NULLIF(idx_scan + seq_scan, 0))::numeric, 2) AS index_scan_pct, + ROUND((100.0 * heap_blks_hit / NULLIF(heap_blks_hit + heap_blks_read, 0))::numeric, 2) AS table_cache_hit_pct +FROM pg_stat_user_tables t +JOIN pg_statio_user_tables io USING (relid) +ORDER BY (seq_scan + idx_scan) DESC NULLS LAST +LIMIT 25; +" + +run_section "5) Potentially unused indexes (can slow writes during scans/indexing)" " +SELECT + s.schemaname, + s.relname AS table_name, + s.indexrelname AS index_name, + s.idx_scan, + pg_size_pretty(pg_relation_size(s.indexrelid)) AS index_size, + t.n_tup_ins + t.n_tup_upd + t.n_tup_del AS table_writes_since_reset +FROM pg_stat_user_indexes s +JOIN pg_index i ON i.indexrelid = s.indexrelid +JOIN pg_stat_user_tables t ON t.relid = s.relid +WHERE s.idx_scan = 0 + AND NOT i.indisunique + AND NOT i.indisprimary +ORDER BY pg_relation_size(s.indexrelid) DESC +LIMIT 50; +" + +run_section "6) Write-heavy tables (hot spot during media metadata indexing)" " +SELECT + relname, + n_tup_ins, + n_tup_upd, + n_tup_del, + n_tup_hot_upd, + n_dead_tup, + vacuum_count, + autovacuum_count, + analyze_count, + autoanalyze_count, + COALESCE(last_autovacuum::text, 'never') AS last_autovacuum, + COALESCE(last_autoanalyze::text, 'never') AS last_autoanalyze +FROM pg_stat_user_tables +ORDER BY (n_tup_ins + n_tup_upd + n_tup_del) DESC +LIMIT 25; +" + +run_section "7) Largest relations (tables and indexes)" " +SELECT + n.nspname AS schema_name, + c.relname, + c.relkind, + pg_size_pretty(pg_total_relation_size(c.oid)) AS total_size +FROM pg_class c +JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') + AND c.relkind IN ('r', 'i') +ORDER BY pg_total_relation_size(c.oid) DESC +LIMIT 30; +" + +if [[ "$(query_single_value "SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements';")" == "1" ]]; then + run_section "8) Slowest normalized SQL from pg_stat_statements" " + SELECT + calls, + ROUND(total_exec_time::numeric, 2) AS total_exec_ms, + ROUND((total_exec_time / NULLIF(calls, 0))::numeric, 2) AS mean_exec_ms, + ROUND(rows::numeric / NULLIF(calls, 0), 2) AS avg_rows, + shared_blks_read, + shared_blks_hit, + temp_blks_read, + temp_blks_written, + LEFT(query, 300) AS query_sample + FROM pg_stat_statements + WHERE dbid = (SELECT oid FROM pg_database WHERE datname = current_database()) + ORDER BY mean_exec_ms DESC + LIMIT 20; + " +else + { + echo + echo "================================================================" + echo "8) Slowest normalized SQL from pg_stat_statements" + echo "================================================================" + echo "Extension pg_stat_statements is not installed." + echo "Install with: CREATE EXTENSION pg_stat_statements;" + } >> "$OUT_FILE" +fi + +# Heuristic recommendation summary +CACHE_HIT_PCT="$(query_single_value " +SELECT ROUND(100.0 * blks_hit / NULLIF(blks_hit + blks_read, 0), 2) +FROM pg_stat_database +WHERE datname = current_database();")" + +UNUSED_INDEX_COUNT="$(query_single_value " +SELECT COUNT(*) +FROM pg_stat_user_indexes s +JOIN pg_index i ON i.indexrelid = s.indexrelid +WHERE s.idx_scan = 0 + AND NOT i.indisunique + AND NOT i.indisprimary;")" + +SHARED_BUFFERS_BYTES="$(query_single_value " +SELECT pg_size_bytes(setting || unit) +FROM pg_settings +WHERE name = 'shared_buffers';")" + +TOTAL_RAM_KB="" +if [[ -r /proc/meminfo ]]; then + TOTAL_RAM_KB="$(awk '/MemTotal:/ {print $2}' /proc/meminfo)" +fi + +{ + echo + echo "================================================================" + echo "9) Recommendations summary" + echo "================================================================" + + if [[ -n "$CACHE_HIT_PCT" ]]; then + echo "Cache hit ratio: ${CACHE_HIT_PCT}%" + awk -v hit="$CACHE_HIT_PCT" 'BEGIN { + if (hit < 98.0) { + print "- Action: cache hit ratio is low for media workloads. Increase shared_buffers/effective_cache_size and review slow seq scans." + } else if (hit < 99.0) { + print "- Action: cache hit ratio is acceptable but can likely improve for large libraries." + } else { + print "- Action: cache hit ratio looks healthy." + } + }' + else + echo "Cache hit ratio: unavailable" + fi + + if [[ -n "$UNUSED_INDEX_COUNT" ]]; then + echo "Potentially unused non-unique indexes: $UNUSED_INDEX_COUNT" + if [[ "$UNUSED_INDEX_COUNT" =~ ^[0-9]+$ ]] && (( UNUSED_INDEX_COUNT > 0 )); then + echo "- Action: validate and drop truly unused indexes to reduce write overhead during scans/indexing." + else + echo "- Action: no obvious unused non-unique indexes from current stats." + fi + fi + + if [[ -n "$SHARED_BUFFERS_BYTES" ]]; then + echo "shared_buffers: $(bytes_to_human "$SHARED_BUFFERS_BYTES")" + if [[ -n "$TOTAL_RAM_KB" && "$TOTAL_RAM_KB" =~ ^[0-9]+$ ]]; then + TOTAL_RAM_BYTES=$((TOTAL_RAM_KB * 1024)) + SHARED_BUFFER_PCT=$(awk -v s="$SHARED_BUFFERS_BYTES" -v t="$TOTAL_RAM_BYTES" 'BEGIN { printf "%.2f", (s/t)*100 }') + echo "Estimated shared_buffers/system RAM: ${SHARED_BUFFER_PCT}%" + awk -v p="$SHARED_BUFFER_PCT" 'BEGIN { + if (p < 10.0) { + print "- Action: shared_buffers appears low; typical starting point is ~15-25% of system RAM." + } else if (p > 40.0) { + print "- Action: shared_buffers appears high; ensure OS cache remains adequate." + } else { + print "- Action: shared_buffers proportion looks reasonable." + } + }' + else + echo "- Action: compare shared_buffers to system RAM manually (target often 15-25% for dedicated DB hosts)." + fi + fi + + echo "- Action: for media scans, verify high-read tables use selective indexes and that autovacuum keeps dead tuples controlled." + echo "- Action: rerun this script before and during a scan to compare write pressure and cache behavior over time." +} >> "$OUT_FILE" + +echo "Diagnostics complete. Report written to: $OUT_FILE" diff --git a/personal/tools/pgbench_benchmark.ps1 b/personal/tools/pgbench_benchmark.ps1 new file mode 100644 index 0000000..9880f31 --- /dev/null +++ b/personal/tools/pgbench_benchmark.ps1 @@ -0,0 +1,268 @@ +param( + [string]$PgHost = "localhost", + [int]$PgPort = 5432, + [string]$PgUser = "postgres", + [string]$BenchDb = "pgbench", + [int]$Scale = 50, + [int]$Duration = 60, + [string]$Clients = "1 4 8 16 32", + [int]$Jobs = 4, + [int]$WarmupSeconds = 15, + [bool]$RunReadOnly = $true, + [bool]$RunCustom = $false, + [string]$CustomSqlFile = "pgbench_custom_workload.sql", + [string]$CustomModeName = "custom_sql", + [ValidateRange(1, 99)][int]$CustomReadPct = 80, + [ValidateRange(1, 100)][int]$CustomHotPct = 90, + [ValidateRange(1, 4)][int]$CustomTxnSize = 2, + [ValidateRange(100, 100000000)][int]$CustomHotAccounts = 10000, + [bool]$Reinit = $false +) + +$ErrorActionPreference = "Stop" + +function Require-Command { + param([Parameter(Mandatory = $true)][string]$Name) + + if (-not (Get-Command $Name -ErrorAction SilentlyContinue)) { + throw "Missing required command: $Name" + } +} + +function Invoke-PsqlScalar { + param( + [Parameter(Mandatory = $true)][string]$Database, + [Parameter(Mandatory = $true)][string]$Sql + ) + + $output = & psql -X -v ON_ERROR_STOP=1 -h $PgHost -p $PgPort -U $PgUser -d $Database -Atc $Sql + if ($LASTEXITCODE -ne 0) { + throw "psql failed running SQL: $Sql" + } + + return ($output | Select-Object -First 1) +} + +function Invoke-PsqlNonQuery { + param( + [Parameter(Mandatory = $true)][string]$Database, + [Parameter(Mandatory = $true)][string]$Sql + ) + + & psql -X -v ON_ERROR_STOP=1 -h $PgHost -p $PgPort -U $PgUser -d $Database -c $Sql | Out-Null + if ($LASTEXITCODE -ne 0) { + throw "psql failed running SQL: $Sql" + } +} + +function Ensure-BenchmarkDatabase { + $exists = Invoke-PsqlScalar -Database "postgres" -Sql "SELECT 1 FROM pg_database WHERE datname = '$BenchDb';" + if ($exists -ne "1") { + Write-Host "Creating benchmark database: $BenchDb" + Invoke-PsqlNonQuery -Database "postgres" -Sql "CREATE DATABASE $BenchDb;" + } +} + +function Needs-Init { + $hasTable = Invoke-PsqlScalar -Database $BenchDb -Sql "SELECT to_regclass('public.pgbench_accounts') IS NOT NULL;" + return $hasTable -ne "t" +} + +function Init-OrReinitData { + if ($Reinit -or (Needs-Init)) { + Write-Host "Initializing pgbench schema/data (scale=$Scale)" + & pgbench -h $PgHost -p $PgPort -U $PgUser -i -s $Scale $BenchDb | Out-Host + if ($LASTEXITCODE -ne 0) { + throw "pgbench initialization failed" + } + } + else { + Write-Host "Using existing pgbench data in $BenchDb" + } +} + +function Parse-Tps { + param([Parameter(Mandatory = $true)][string]$Text) + + $matches = [regex]::Matches($Text, "(?m)^tps\s*=\s*([0-9]+(?:\.[0-9]+)?)") + if ($matches.Count -gt 0) { + return $matches[$matches.Count - 1].Groups[1].Value + } + + return "" +} + +function Parse-AvgLatencyMs { + param([Parameter(Mandatory = $true)][string]$Text) + + $matches = [regex]::Matches($Text, "(?m)^latency average\s*=\s*([0-9]+(?:\.[0-9]+)?)\s*ms") + if ($matches.Count -gt 0) { + return $matches[$matches.Count - 1].Groups[1].Value + } + + return "" +} + +function Run-Case { + param( + [Parameter(Mandatory = $true)][string]$Mode, + [Parameter(Mandatory = $true)][int]$ClientCount, + [AllowEmptyCollection()][string[]]$ExtraArgs = @(), + [Parameter(Mandatory = $true)][string]$ResultDir, + [Parameter(Mandatory = $true)][string]$SummaryCsv + ) + + $logFile = Join-Path $ResultDir ("{0}_c{1}.txt" -f $Mode, $ClientCount) + $txLogPrefix = Join-Path $ResultDir ("{0}_c{1}_txlog" -f $Mode, $ClientCount) + Write-Host "Running mode=$Mode, clients=$ClientCount, duration=${Duration}s" + + # Warm cache and avoid measuring first-run effects. + & pgbench -h $PgHost -p $PgPort -U $PgUser -n -M prepared -j $Jobs -c $ClientCount -T $WarmupSeconds @ExtraArgs $BenchDb | Out-Null + if ($LASTEXITCODE -ne 0) { + throw "Warm-up run failed for mode=$Mode clients=$ClientCount" + } + + $runOutput = & pgbench -h $PgHost -p $PgPort -U $PgUser -n -M prepared -j $Jobs -c $ClientCount -T $Duration -r -l --log-prefix $txLogPrefix @ExtraArgs $BenchDb 2>&1 + if ($LASTEXITCODE -ne 0) { + throw "Measured run failed for mode=$Mode clients=$ClientCount" + } + + $runOutput | Set-Content -Path $logFile -Encoding UTF8 + + $text = ($runOutput -join [Environment]::NewLine) + $tps = Parse-Tps -Text $text + $latencyMs = Parse-AvgLatencyMs -Text $text + $percentiles = Get-PercentileLatency -LogPrefix $txLogPrefix + + "$Mode,$ClientCount,$Duration,$Jobs,$tps,$latencyMs,$($percentiles.P95Ms),$($percentiles.P99Ms),$logFile" | Add-Content -Path $SummaryCsv +} + +function Get-PercentileLatency { + param([Parameter(Mandatory = $true)][string]$LogPrefix) + + $logFiles = Get-ChildItem -Path ($LogPrefix + "*") -File -ErrorAction SilentlyContinue + if (-not $logFiles -or $logFiles.Count -eq 0) { + return @{ P95Ms = ""; P99Ms = "" } + } + + # Keep memory bounded even for very large transaction logs. + $maxSampleSize = 200000 + $reservoir = New-Object System.Collections.Generic.List[double] + $reservoir.Capacity = $maxSampleSize + $random = [System.Random]::new() + $seen = 0 + + foreach ($file in $logFiles) { + $reader = [System.IO.File]::OpenText($file.FullName) + try { + while (($line = $reader.ReadLine()) -ne $null) { + if ([string]::IsNullOrWhiteSpace($line) -or $line.StartsWith("#")) { + continue + } + + $tokens = $line -split "[\s,]+" | Where-Object { $_ -ne "" } + if ($tokens.Count -ge 3 -and $tokens[2] -match "^[0-9]+(?:\.[0-9]+)?$") { + $latUs = [double]$tokens[2] + $seen++ + + if ($reservoir.Count -lt $maxSampleSize) { + $reservoir.Add($latUs) + } + else { + $idx = $random.Next(0, $seen) + if ($idx -lt $maxSampleSize) { + $reservoir[$idx] = $latUs + } + } + } + } + } + finally { + $reader.Dispose() + } + } + + if ($reservoir.Count -eq 0) { + return @{ P95Ms = ""; P99Ms = "" } + } + + $sorted = $reservoir | Sort-Object + $p95Us = Select-PercentileValue -SortedValues $sorted -Percentile 95 + $p99Us = Select-PercentileValue -SortedValues $sorted -Percentile 99 + + return @{ + P95Ms = [math]::Round(($p95Us / 1000.0), 3) + P99Ms = [math]::Round(($p99Us / 1000.0), 3) + } +} + +function Select-PercentileValue { + param( + [Parameter(Mandatory = $true)][double[]]$SortedValues, + [Parameter(Mandatory = $true)][ValidateRange(1, 99)][int]$Percentile + ) + + $n = $SortedValues.Count + $rank = [math]::Ceiling(($Percentile / 100.0) * $n) + if ($rank -lt 1) { $rank = 1 } + if ($rank -gt $n) { $rank = $n } + return $SortedValues[$rank - 1] +} + +function Resolve-CustomSqlPath { + if ([System.IO.Path]::IsPathRooted($CustomSqlFile)) { + return $CustomSqlFile + } + + return (Join-Path (Get-Location) $CustomSqlFile) +} + +Require-Command -Name "psql" +Require-Command -Name "pgbench" + +$timestamp = Get-Date -Format "yyyyMMdd_HHmmss" +$resultDir = Join-Path (Get-Location) ("pgbench_results_{0}" -f $timestamp) +$summaryCsv = Join-Path $resultDir "summary.csv" + +New-Item -Path $resultDir -ItemType Directory -Force | Out-Null +"mode,clients,duration_seconds,jobs,tps,latency_ms,p95_ms,p99_ms,details_file" | Set-Content -Path $summaryCsv -Encoding UTF8 + +Ensure-BenchmarkDatabase +Init-OrReinitData + +$customSqlPath = "" +if ($RunCustom) { + $customSqlPath = Resolve-CustomSqlPath + if (-not (Test-Path -Path $customSqlPath -PathType Leaf)) { + throw "Custom SQL file not found: $customSqlPath" + } +} + +$clientValues = $Clients -split "\s+" | Where-Object { $_ -match "^[0-9]+$" } +if ($clientValues.Count -eq 0) { + throw "No valid client values found. Example: -Clients '1 4 8 16 32'" +} + +foreach ($c in $clientValues) { + $clientInt = [int]$c + Run-Case -Mode "read_write" -ClientCount $clientInt -ExtraArgs @() -ResultDir $resultDir -SummaryCsv $summaryCsv + + if ($RunReadOnly) { + Run-Case -Mode "read_only" -ClientCount $clientInt -ExtraArgs @("-S") -ResultDir $resultDir -SummaryCsv $summaryCsv + } + + if ($RunCustom) { + Run-Case -Mode $CustomModeName -ClientCount $clientInt -ExtraArgs @( + "-D", "read_pct=$CustomReadPct", + "-D", "hot_pct=$CustomHotPct", + "-D", "txn_size=$CustomTxnSize", + "-D", "hot_accounts=$CustomHotAccounts", + "-f", $customSqlPath + ) -ResultDir $resultDir -SummaryCsv $summaryCsv + } +} + +Write-Host "" +Write-Host "Benchmark completed." +Write-Host "Summary: $summaryCsv" +Write-Host "Raw outputs: $resultDir" diff --git a/personal/tools/pgbench_benchmark.sh b/personal/tools/pgbench_benchmark.sh new file mode 100644 index 0000000..486c316 --- /dev/null +++ b/personal/tools/pgbench_benchmark.sh @@ -0,0 +1,188 @@ +#!/bin/bash +set -euo pipefail + +# Repeatable pgbench benchmark harness. +# Usage example: +# PGHOST=localhost PGPORT=5432 PGUSER=postgres PGPASSWORD=secret \ +# ./pgbench_benchmark.sh +# +# Optional environment variables: +# BENCH_DB=pgbench +# SCALE=50 +# DURATION=60 +# CLIENTS="1 4 8 16 32" +# JOBS=4 +# WARMUP_SECONDS=15 +# RUN_READ_ONLY=1 +# RUN_CUSTOM=0 +# CUSTOM_SQL_FILE=pgbench_custom_workload.sql +# CUSTOM_MODE_NAME=custom_sql +# CUSTOM_READ_PCT=80 +# CUSTOM_HOT_PCT=90 +# CUSTOM_TXN_SIZE=2 +# CUSTOM_HOT_ACCOUNTS=10000 +# REINIT=0 + +PGHOST="${PGHOST:-localhost}" +PGPORT="${PGPORT:-5432}" +PGUSER="${PGUSER:-postgres}" +BENCH_DB="${BENCH_DB:-pgbench}" +SCALE="${SCALE:-50}" +DURATION="${DURATION:-60}" +CLIENTS="${CLIENTS:-1 4 8 16 32}" +JOBS="${JOBS:-4}" +WARMUP_SECONDS="${WARMUP_SECONDS:-15}" +RUN_READ_ONLY="${RUN_READ_ONLY:-1}" +RUN_CUSTOM="${RUN_CUSTOM:-0}" +CUSTOM_SQL_FILE="${CUSTOM_SQL_FILE:-pgbench_custom_workload.sql}" +CUSTOM_MODE_NAME="${CUSTOM_MODE_NAME:-custom_sql}" +CUSTOM_READ_PCT="${CUSTOM_READ_PCT:-80}" +CUSTOM_HOT_PCT="${CUSTOM_HOT_PCT:-90}" +CUSTOM_TXN_SIZE="${CUSTOM_TXN_SIZE:-2}" +CUSTOM_HOT_ACCOUNTS="${CUSTOM_HOT_ACCOUNTS:-10000}" +REINIT="${REINIT:-0}" + +TIMESTAMP="$(date +%Y%m%d_%H%M%S)" +RESULT_DIR="pgbench_results_${TIMESTAMP}" +SUMMARY_CSV="${RESULT_DIR}/summary.csv" + +PSQL=(psql -X -v ON_ERROR_STOP=1 -h "$PGHOST" -p "$PGPORT" -U "$PGUSER") +PGBENCH=(pgbench -h "$PGHOST" -p "$PGPORT" -U "$PGUSER") + +require_command() { + local cmd="$1" + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "Missing required command: $cmd" >&2 + exit 1 + fi +} + +create_database_if_missing() { + local exists + exists="$("${PSQL[@]}" -d postgres -Atc "SELECT 1 FROM pg_database WHERE datname = '${BENCH_DB}';" || true)" + if [[ "$exists" != "1" ]]; then + echo "Creating benchmark database: ${BENCH_DB}" + "${PSQL[@]}" -d postgres -c "CREATE DATABASE ${BENCH_DB};" + fi +} + +needs_init() { + local rel + rel="$("${PSQL[@]}" -d "$BENCH_DB" -Atc "SELECT to_regclass('public.pgbench_accounts') IS NOT NULL;")" + [[ "$rel" != "t" ]] +} + +init_or_reinit_data() { + if [[ "$REINIT" == "1" ]] || needs_init; then + echo "Initializing pgbench schema/data (scale=${SCALE})" + "${PGBENCH[@]}" -i -s "$SCALE" "$BENCH_DB" + else + echo "Using existing pgbench data in ${BENCH_DB}" + fi +} + +extract_tps() { + local output_file="$1" + awk -F'= ' '/tps =/{print $2}' "$output_file" | awk '{print $1}' | tail -n 1 +} + +extract_avg_latency() { + local output_file="$1" + awk -F'= ' '/latency average =/{print $2}' "$output_file" | awk '{print $1}' | tail -n 1 +} + +select_percentile_ms_from_prefix() { + local log_prefix="$1" + local percentile="$2" + local -a log_files + local -a values_us + local count rank value_us + + shopt -s nullglob + log_files=("${log_prefix}"*) + shopt -u nullglob + + if (( ${#log_files[@]} == 0 )); then + echo "" + return + fi + + mapfile -t values_us < <(awk 'NF >= 3 && $3 ~ /^[0-9]+(\.[0-9]+)?$/ {print $3}' "${log_files[@]}" | sort -n) + count="${#values_us[@]}" + if (( count == 0 )); then + echo "" + return + fi + + rank=$(( (percentile * count + 99) / 100 )) + if (( rank < 1 )); then rank=1; fi + if (( rank > count )); then rank=count; fi + + value_us="${values_us[$((rank - 1))]}" + awk -v us="$value_us" 'BEGIN { printf "%.3f", us/1000.0 }' +} + +run_case() { + local mode="$1" + local clients="$2" + shift 2 + local -a extra_args=("$@") + local log_file="${RESULT_DIR}/${mode}_c${clients}.txt" + local tx_log_prefix="${RESULT_DIR}/${mode}_c${clients}_txlog" + + echo "Running mode=${mode}, clients=${clients}, duration=${DURATION}s" + # Warm cache and avoid measuring first-run effects. + "${PGBENCH[@]}" -n -M prepared -j "$JOBS" -c "$clients" -T "$WARMUP_SECONDS" "${extra_args[@]}" "$BENCH_DB" > /dev/null + + "${PGBENCH[@]}" -n -M prepared -j "$JOBS" -c "$clients" -T "$DURATION" -r -l --log-prefix="$tx_log_prefix" "${extra_args[@]}" "$BENCH_DB" > "$log_file" + + local tps latency p95 p99 + tps="$(extract_tps "$log_file")" + latency="$(extract_avg_latency "$log_file")" + p95="$(select_percentile_ms_from_prefix "$tx_log_prefix" 95)" + p99="$(select_percentile_ms_from_prefix "$tx_log_prefix" 99)" + + echo "${mode},${clients},${DURATION},${JOBS},${tps},${latency},${p95},${p99},${log_file}" >> "$SUMMARY_CSV" +} + +main() { + require_command psql + require_command pgbench + + mkdir -p "$RESULT_DIR" + echo "mode,clients,duration_seconds,jobs,tps,latency_ms,p95_ms,p99_ms,details_file" > "$SUMMARY_CSV" + + create_database_if_missing + init_or_reinit_data + + if [[ "$RUN_CUSTOM" == "1" ]]; then + if [[ ! -f "$CUSTOM_SQL_FILE" ]]; then + echo "Custom SQL file not found: $CUSTOM_SQL_FILE" >&2 + exit 1 + fi + fi + + for c in $CLIENTS; do + run_case "read_write" "$c" + + if [[ "$RUN_READ_ONLY" == "1" ]]; then + run_case "read_only" "$c" "-S" + fi + + if [[ "$RUN_CUSTOM" == "1" ]]; then + run_case "$CUSTOM_MODE_NAME" "$c" \ + "-D" "read_pct=${CUSTOM_READ_PCT}" \ + "-D" "hot_pct=${CUSTOM_HOT_PCT}" \ + "-D" "txn_size=${CUSTOM_TXN_SIZE}" \ + "-D" "hot_accounts=${CUSTOM_HOT_ACCOUNTS}" \ + "-f" "$CUSTOM_SQL_FILE" + fi + done + + echo + echo "Benchmark completed." + echo "Summary: $SUMMARY_CSV" + echo "Raw outputs: $RESULT_DIR" +} + +main "$@" diff --git a/personal/tools/pgbench_export_report.py b/personal/tools/pgbench_export_report.py new file mode 100644 index 0000000..7dbfd47 --- /dev/null +++ b/personal/tools/pgbench_export_report.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python3 +"""Export pgbench summary.csv into an HTML report with TPS and latency charts.""" + +import argparse +import csv +import json +from pathlib import Path +from typing import Dict, List, Optional + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Create an HTML chart report from pgbench summary.csv") + parser.add_argument("summary_csv", help="Path to summary.csv generated by pgbench benchmark script") + parser.add_argument( + "-o", + "--output", + default=None, + help="Output HTML path (default: /pgbench_report.html)", + ) + return parser.parse_args() + + +def load_rows(path: Path) -> List[Dict[str, str]]: + with path.open("r", encoding="utf-8-sig", newline="") as f: + reader = csv.DictReader(f) + expected = {"mode", "clients", "tps", "latency_ms"} + if not expected.issubset(set(reader.fieldnames or [])): + missing = sorted(expected - set(reader.fieldnames or [])) + raise ValueError(f"summary.csv missing required columns: {', '.join(missing)}") + return list(reader) + + +def parse_optional_float(raw: str) -> Optional[float]: + value = (raw or "").strip() + if not value: + return None + try: + return float(value) + except ValueError: + return None + + +def build_series(rows: List[Dict[str, str]]) -> Dict[str, List[Dict[str, float]]]: + series: Dict[str, List[Dict[str, float]]] = {} + for row in rows: + mode = row["mode"].strip() + try: + clients = int(row["clients"]) + tps = float(row["tps"]) + latency = float(row["latency_ms"]) + except ValueError: + continue + + point: Dict[str, float] = { + "clients": clients, + "tps": tps, + "latency_ms": latency, + } + + p95 = parse_optional_float(row.get("p95_ms", "")) + p99 = parse_optional_float(row.get("p99_ms", "")) + if p95 is not None: + point["p95_ms"] = p95 + if p99 is not None: + point["p99_ms"] = p99 + + series.setdefault(mode, []).append(point) + + for mode in series: + series[mode].sort(key=lambda item: item["clients"]) + + return series + + +def html_report(title: str, data: Dict[str, List[Dict[str, float]]]) -> str: + payload = json.dumps(data) + return f""" + + + + +{title} + + + +
+
+

{title}

+

Generated from summary.csv. X-axis = clients. Lines are grouped by mode.

+
+ +
+

TPS by Clients

+ +
+
+ +
+

Latency (ms) by Clients

+ +
+ +
+

p95 Latency (ms) by Clients

+ +
+ +
+

p99 Latency (ms) by Clients

+ +
+
+ + + + +""" + + +def main() -> int: + args = parse_args() + summary_path = Path(args.summary_csv).resolve() + if not summary_path.is_file(): + raise FileNotFoundError(f"summary.csv not found: {summary_path}") + + rows = load_rows(summary_path) + if not rows: + raise ValueError("summary.csv has no data rows") + + series = build_series(rows) + if not series: + raise ValueError("No valid numeric rows found in summary.csv") + + output_path = ( + Path(args.output).resolve() + if args.output + else summary_path.parent / "pgbench_report.html" + ) + + title = f"pgbench report: {summary_path.parent.name}" + output_path.write_text(html_report(title=title, data=series), encoding="utf-8") + print(f"Report written: {output_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/personal/tools/pgbench_quickstart.md b/personal/tools/pgbench_quickstart.md new file mode 100644 index 0000000..41e5c5c --- /dev/null +++ b/personal/tools/pgbench_quickstart.md @@ -0,0 +1,102 @@ +# pgbench benchmark quickstart + +This workspace now includes repeatable benchmark harnesses: + +- `pgbench_benchmark.ps1` (Windows PowerShell) +- `pgbench_benchmark.sh` (Bash) +- `pgbench_custom_workload.sql` (sample custom SQL transaction mix) +- `pgbench_export_report.py` (exports TPS/latency chart report from `summary.csv`) + +## 1) Prerequisites + +- PostgreSQL client tools installed (`psql`, `pgbench`) +- Network access to your PostgreSQL server +- Credentials available via environment variables (`PGHOST`, `PGPORT`, `PGUSER`, `PGPASSWORD`) + +## 2) Run the benchmark (Windows PowerShell) + +From this folder: + +```powershell +$env:PGPASSWORD = "your_password" +.\pgbench_benchmark.ps1 -PgHost localhost -PgPort 5432 -PgUser postgres +``` + +You can override defaults: + +```powershell +$env:PGPASSWORD = "your_password" +.\pgbench_benchmark.ps1 -BenchDb pgbench -Scale 100 -Duration 120 -Clients "1 8 16 32 64" -Jobs 8 -Reinit $true +``` + +Run with custom SQL workload mode enabled: + +```powershell +$env:PGPASSWORD = "your_password" +.\pgbench_benchmark.ps1 -RunCustom $true -CustomSqlFile .\pgbench_custom_workload.sql -CustomModeName app_like -CustomReadPct 85 -CustomHotPct 92 -CustomTxnSize 3 -CustomHotAccounts 12000 +``` + +## 3) Optional Bash usage (WSL/Git Bash) + +```bash +chmod +x pgbench_benchmark.sh +PGHOST=localhost PGPORT=5432 PGUSER=postgres PGPASSWORD=your_password ./pgbench_benchmark.sh +``` + +Run with custom SQL workload mode enabled: + +```bash +PGHOST=localhost PGPORT=5432 PGUSER=postgres PGPASSWORD=your_password RUN_CUSTOM=1 CUSTOM_SQL_FILE=./pgbench_custom_workload.sql CUSTOM_MODE_NAME=app_like CUSTOM_READ_PCT=85 CUSTOM_HOT_PCT=92 CUSTOM_TXN_SIZE=3 CUSTOM_HOT_ACCOUNTS=12000 ./pgbench_benchmark.sh +``` + +## 4) Optional tuning inputs (Bash script) + +```bash +BENCH_DB=pgbench SCALE=100 DURATION=120 CLIENTS="1 8 16 32 64" JOBS=8 REINIT=1 ./pgbench_benchmark.sh +``` + +Environment variables: + +- `BENCH_DB`: database name for benchmark data (default: `pgbench`) +- `SCALE`: data size scale factor (default: `50`) +- `DURATION`: measured run time per test case in seconds (default: `60`) +- `CLIENTS`: space-separated client counts (default: `"1 4 8 16 32"`) +- `JOBS`: worker threads for pgbench (default: `4`) +- `WARMUP_SECONDS`: unmeasured warm-up before each test (default: `15`) +- `RUN_READ_ONLY`: set to `0` to skip `-S` read-only tests (default: `1`) +- `REINIT`: set to `1` to reinitialize data each run (default: `0`) +- `RUN_CUSTOM`: set to `1` to run custom SQL workload mode (default: `0`) +- `CUSTOM_SQL_FILE`: path to custom SQL file for `pgbench -f` (default: `pgbench_custom_workload.sql`) +- `CUSTOM_MODE_NAME`: mode label written to `summary.csv` for custom runs (default: `custom_sql`) +- `CUSTOM_READ_PCT` / `-CustomReadPct`: read share percentage for custom mode; writes happen in `100-read_pct` (default: `80`) +- `CUSTOM_HOT_PCT` / `-CustomHotPct`: chance to pick from hot account set in custom mode (default: `90`) +- `CUSTOM_TXN_SIZE` / `-CustomTxnSize`: custom transaction size tier from `1` to `4` (default: `2`) +- `CUSTOM_HOT_ACCOUNTS` / `-CustomHotAccounts`: number of hot accounts near start of keyspace (default: `10000`) + +## 5) Output + +Each run creates: + +- `pgbench_results_YYYYMMDD_HHMMSS/summary.csv` +- Per-test raw output files in the same results directory + +Use `summary.csv` to compare TPS and latency across runs. + +`summary.csv` now includes percentile latency columns: + +- `p95_ms` +- `p99_ms` + +## 6) Export chart report from summary.csv + +Generate an HTML report with TPS and latency charts: + +```powershell +python .\pgbench_export_report.py .\pgbench_results_YYYYMMDD_HHMMSS\summary.csv +``` + +Optional output path: + +```powershell +python .\pgbench_export_report.py .\pgbench_results_YYYYMMDD_HHMMSS\summary.csv -o .\pgbench_results_YYYYMMDD_HHMMSS\my_report.html +``` diff --git a/personal/tools/powerball_reader.py b/personal/tools/powerball_reader.py index 8bc3cfd..410a624 100644 --- a/personal/tools/powerball_reader.py +++ b/personal/tools/powerball_reader.py @@ -6,7 +6,7 @@ Script to read and parse Powerball numbers from a CSV file. import csv from datetime import datetime from pathlib import Path -from typing import List, Dict +from typing import List, Dict, Tuple, Optional from collections import Counter @@ -85,7 +85,84 @@ def display_powerball_data(data: List[Dict]) -> None: print(f"{entry['date_str']:<12} {numbers_str:<30} {entry['multiplier']:<10}") -def analyze_most_frequent_numbers(data: List[Dict]) -> None: +def _get_unweighted_top_numbers(draws: List[Dict]) -> Tuple[List[int], Optional[int]]: + """Return top 5 white balls and top Powerball from a draw list.""" + if not draws: + return [], None + + white_counter = Counter() + powerball_counter = Counter() + + for entry in draws: + numbers = entry['winning_numbers'] + for number in numbers[:5]: + white_counter[number] += 1 + + if len(numbers) >= 6: + powerball_counter[numbers[5]] += 1 + + top_white = [number for number, _ in white_counter.most_common(5)] + top_powerball = powerball_counter.most_common(1)[0][0] if powerball_counter else None + + return top_white, top_powerball + + +def _get_recency_weighted_top_numbers(draws: List[Dict]) -> Tuple[List[int], Optional[int]]: + """Return top numbers weighted so newer draws contribute more.""" + if not draws: + return [], None + + sorted_draws = sorted(draws, key=lambda x: x['draw_date']) + total_draws = len(sorted_draws) + + white_scores = Counter() + powerball_scores = Counter() + + for index, entry in enumerate(sorted_draws, start=1): + weight = index / total_draws + numbers = entry['winning_numbers'] + + for number in numbers[:5]: + white_scores[number] += weight + + if len(numbers) >= 6: + powerball_scores[numbers[5]] += weight + + top_white = [number for number, _ in white_scores.most_common(5)] + top_powerball = powerball_scores.most_common(1)[0][0] if powerball_scores else None + + return top_white, top_powerball + + +def _get_aggressive_recency_weighted_top_numbers(draws: List[Dict]) -> Tuple[List[int], Optional[int]]: + """Return top numbers with an aggressive recency curve (cubic weighting).""" + if not draws: + return [], None + + sorted_draws = sorted(draws, key=lambda x: x['draw_date']) + total_draws = len(sorted_draws) + + white_scores = Counter() + powerball_scores = Counter() + + for index, entry in enumerate(sorted_draws, start=1): + normalized_position = index / total_draws + weight = normalized_position ** 3 + numbers = entry['winning_numbers'] + + for number in numbers[:5]: + white_scores[number] += weight + + if len(numbers) >= 6: + powerball_scores[numbers[5]] += weight + + top_white = [number for number, _ in white_scores.most_common(5)] + top_powerball = powerball_scores.most_common(1)[0][0] if powerball_scores else None + + return top_white, top_powerball + + +def analyze_most_frequent_numbers(data: List[Dict]) -> Dict: """ Analyze and display the most frequent number in each slot position. @@ -131,27 +208,51 @@ def analyze_most_frequent_numbers(data: List[Dict]) -> None: for i in range(min(5, num_positions)): first_five_combined.extend(position_numbers[i]) + top_five_numbers = [] + top_ten_numbers = [] if first_five_combined: counter_first_five = Counter(first_five_combined) top_five_numbers = counter_first_five.most_common(5) + top_ten_numbers = counter_first_five.most_common(10) print("Top 5 most drawn numbers in first 5 positions combined:") for rank, (number, frequency) in enumerate(top_five_numbers, start=1): percentage = (frequency / len(first_five_combined)) * 100 print(f" {rank}. {number:<3} (appears {frequency} times, {percentage:.1f}%)") # Analyze the sixth position (Powerball) + powerball_ranked = [] most_common_sixth = None if num_positions >= 6: sixth_position_numbers = position_numbers[5] counter_sixth = Counter(sixth_position_numbers) + powerball_ranked = counter_sixth.most_common(5) most_common_sixth, freq_sixth = counter_sixth.most_common(1)[0] percentage_sixth = (freq_sixth / len(sixth_position_numbers)) * 100 print(f"\nMost drawn number in position 6 (Powerball): {most_common_sixth} (appears {freq_sixth} times, {percentage_sixth:.1f}%)") + + # Third strategy: most frequent numbers from the most recent 104 draws + sorted_by_date = sorted(data, key=lambda x: x['draw_date']) + recent_104_draws = sorted_by_date[-104:] + recent_104_top_five, recent_104_powerball = _get_unweighted_top_numbers(recent_104_draws) + + # Fourth strategy: weighted-by-recency across all draws + weighted_top_five, weighted_powerball = _get_recency_weighted_top_numbers(data) + + # Fifth strategy: aggressive weighted-by-recency across all draws + aggressive_weighted_top_five, aggressive_weighted_powerball = _get_aggressive_recency_weighted_top_numbers(data) # Return statistics for prediction return { 'top_five_combined': [num for num, freq in top_five_numbers] if first_five_combined else [], - 'most_common_powerball': most_common_sixth + 'top_ten_combined': [num for num, freq in top_ten_numbers], + 'most_common_powerball': most_common_sixth, + 'powerball_ranked': [num for num, freq in powerball_ranked], + 'recent_104_top_five': recent_104_top_five, + 'recent_104_powerball': recent_104_powerball, + 'weighted_top_five': weighted_top_five, + 'weighted_powerball': weighted_powerball, + 'aggressive_weighted_top_five': aggressive_weighted_top_five, + 'aggressive_weighted_powerball': aggressive_weighted_powerball } @@ -171,20 +272,80 @@ def generate_suggested_combination(stats: dict) -> None: print("Each draw is random and independent.\n") top_five = stats['top_five_combined'] + top_ten = stats.get('top_ten_combined', []) powerball = stats['most_common_powerball'] + ranked_powerballs = stats.get('powerball_ranked', []) + recent_104_top_five = stats.get('recent_104_top_five', []) + recent_104_powerball = stats.get('recent_104_powerball') + weighted_top_five = stats.get('weighted_top_five', []) + weighted_powerball = stats.get('weighted_powerball') + aggressive_weighted_top_five = stats.get('aggressive_weighted_top_five', []) + aggressive_weighted_powerball = stats.get('aggressive_weighted_powerball') if len(top_five) >= 5 and powerball: # Use the top 5 most frequent numbers from positions 1-5 - suggested_numbers = sorted(top_five[:5]) + primary_numbers = sorted(top_five[:5]) + primary_powerball = powerball + + # Build a secondary set from the next-most-frequent historical numbers. + secondary_pool = top_ten[5:10] + if len(secondary_pool) < 5: + secondary_pool = top_ten[:5] + + secondary_numbers = sorted(secondary_pool[:5]) + secondary_powerball = ranked_powerballs[1] if len(ranked_powerballs) > 1 else primary_powerball + + third_numbers = sorted(recent_104_top_five[:5]) if len(recent_104_top_five) >= 5 else [] + third_powerball = recent_104_powerball + + fourth_numbers = sorted(weighted_top_five[:5]) if len(weighted_top_five) >= 5 else [] + fourth_powerball = weighted_powerball + + fifth_numbers = sorted(aggressive_weighted_top_five[:5]) if len(aggressive_weighted_top_five) >= 5 else [] + fifth_powerball = aggressive_weighted_powerball - print(f"Suggested white ball numbers: {' '.join(map(str, suggested_numbers))}") - print(f"Suggested Powerball number: {powerball}") + print("Primary set (most frequent):") + print(f" White balls: {' '.join(map(str, primary_numbers))}") + print(f" Powerball: {primary_powerball}") + + print("\nSecondary set (next-most frequent):") + print(f" White balls: {' '.join(map(str, secondary_numbers))}") + print(f" Powerball: {secondary_powerball}") + + if third_numbers and third_powerball: + print("\nThird set (most frequent in recent 104 draws):") + print(f" White balls: {' '.join(map(str, third_numbers))}") + print(f" Powerball: {third_powerball}") + + if fourth_numbers and fourth_powerball: + print("\nFourth set (weighted by recency):") + print(f" White balls: {' '.join(map(str, fourth_numbers))}") + print(f" Powerball: {fourth_powerball}") + + if fifth_numbers and fifth_powerball: + print("\nFifth set (aggressive recency weighting):") + print(f" White balls: {' '.join(map(str, fifth_numbers))}") + print(f" Powerball: {fifth_powerball}") print() - print(f"Complete combination: {' '.join(map(str, suggested_numbers))} + {powerball}") + print(f"Primary combination: {' '.join(map(str, primary_numbers))} + {primary_powerball}") + print(f"Secondary combination: {' '.join(map(str, secondary_numbers))} + {secondary_powerball}") + + if third_numbers and third_powerball: + print(f"Third combination: {' '.join(map(str, third_numbers))} + {third_powerball}") + + if fourth_numbers and fourth_powerball: + print(f"Fourth combination: {' '.join(map(str, fourth_numbers))} + {fourth_powerball}") + + if fifth_numbers and fifth_powerball: + print(f"Fifth combination: {' '.join(map(str, fifth_numbers))} + {fifth_powerball}") print() - print("This combination is based on:") + print("These combinations are based on:") print(" • The 5 most frequently drawn numbers across all white ball positions") - print(f" • The most frequently drawn Powerball number ({powerball})") + print(f" • The most frequently drawn Powerball number ({primary_powerball})") + print(" • A secondary set from the next-most-frequent white ball and Powerball values") + print(" • Recent-window frequency (last 104 draws)") + print(" • Recency-weighted scoring where newer draws count more") + print(" • Aggressive recency weighting using a cubic curve") else: print("Insufficient data to generate a suggestion.") diff --git a/personal/tools/read_full_table_list.py b/personal/tools/read_full_table_list.py new file mode 100644 index 0000000..b6f12f2 --- /dev/null +++ b/personal/tools/read_full_table_list.py @@ -0,0 +1,40 @@ +import csv +from pathlib import Path + + +def collect_first_tokens_from_full_table_files(directory: Path) -> list[str]: + first_tokens: list[str] = [] + + for file_path in directory.iterdir(): + # Match file names like "Full_Table_List_..." regardless of case. + if file_path.is_file() and file_path.name.lower().startswith("full_table_list"): + with file_path.open("r", encoding="utf-8", errors="ignore") as f: + for line in f: + parts = line.strip().split() + if parts: + first_tokens.append(parts[0]) + + return first_tokens + + +def main() -> None: + current_directory = Path.cwd() + tokens = collect_first_tokens_from_full_table_files(current_directory) + tokens.sort() + + for token in tokens: + print(token) + + output_file = current_directory / "full_table_list_tokens.csv" + with output_file.open("w", newline="", encoding="utf-8") as csv_file: + writer = csv.writer(csv_file) + writer.writerow(["value"]) + for token in tokens: + writer.writerow([token]) + + print(f"\nTotal extracted values: {len(tokens)}") + print(f"Saved CSV file: {output_file}") + + +if __name__ == "__main__": + main()