wjones/new_dbsync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor update_mongo.py for improved readability and maintainability

Browse Source
This commit is contained in:
Wendell Jones
2025-12-09 13:00:48 -05:00
parent b390fc9f9a
commit c5502711e7
2 changed files with 93 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
settings/tvsync_settings.cfg
+1 -1
View File
@@ -4,7 +4,7 @@ loglevel=debug
[dbsettings]
sqlserver_driver=ODBC Driver 13 for SQL Server
hostname=192.168.128.7
mghostname=192.168.128.8
mghostname=192.168.128.24
dbname=media_dbsync
mgdbname=test2
mysqlusername=root
update_mongo.py
+89 -43
View File
@@ -2,6 +2,7 @@
# -*- coding: utf-8 -*-
import os
import time
import sys
import url.urls
import url.process
import api.tvmaze.cast as mazecast
@@ -17,11 +18,25 @@ from datetime import datetime
response = None
def set_apienv(urls, dbExec, updatesBase):
def set_apienv(urls, uprocess, jget, dbengine, dbExec, updatesBase, lprint):
"""Populate updates table from the API and return available updates.
Args:
urls: URL configuration with an "updatesurl" key.
uprocess: URL loader instance with `get_data`.
jget: JSON processor with `jconvert`.
dbengine: database engine dict (expects key "engine").
dbExec: database execution helper with `update_tvupdates` and `rawsql_select`.
updatesBase: list-like containing tables (uses index 8 for updates table).
lprint: logger instance with `logprint`.
Returns:
List of tuples (seriesid, timestamp) representing available updates.
"""
updatetable = updatesBase[8]
inputdata = uprocess.get_data(urls["updatesurl"])
availableupdates = jget.jconvert(inputdata)
print(f"Retrieved {len(availableupdates)} rows for processing......")
lprint.logprint("info", f"Retrieved {len(availableupdates)} rows for processing......")
dbExec.update_tvupdates(dbengine["engine"], availableupdates, updatetable)
newupdates = dbExec.rawsql_select(
dbengine["engine"],
@@ -35,31 +50,35 @@ def print_starttime(lprint, ts1):
lprint.logprint("info", f"Sync start time: {str(ts1)}")
def get_tvdb():
tvdb_urls = TVDBURLS.get_data()
return tvdb_urls
def get_tvdb(tvdburls):
return tvdburls.get_data()
def get_tvmaze():
tvmaze_urls = TVMAZEURLS.get_data()
return tvmaze_urls
def get_tvmaze(tvmazeurls):
return tvmazeurls.get_data()
if __name__ == "__main__":
def main() -> int:
ROOTDIR = os.getcwd()
directories = {}
if os.name == "nt":
LOGDIR = f"{ROOTDIR}\\log"
directories["SERIESDIR"] = f"{ROOTDIR}\\cache\\series\\"
directories["EPISODEDIR"] = f"{ROOTDIR}\\cache\\episodes\\"
directories["CASTDIR"] = f"{ROOTDIR}\\cache\\cast\\"
directories["CREWDIR"] = f"{ROOTDIR}\\cache\\crew\\"
else:
LOGDIR = f"{ROOTDIR}/log"
directories["SERIESDIR"] = f"{ROOTDIR}/cache/series/"
directories["EPISODEDIR"] = f"{ROOTDIR}/cache/episodes/"
directories["CASTDIR"] = f"{ROOTDIR}/cache/cast/"
directories["CREWDIR"] = f"{ROOTDIR}/cache/crew/"
# Use cross-platform path builders and ensure directories exist
LOGDIR = os.path.join(ROOTDIR, "log")
directories["SERIESDIR"] = os.path.join(ROOTDIR, "cache", "series") + os.sep
directories["EPISODEDIR"] = os.path.join(ROOTDIR, "cache", "episodes") + os.sep
directories["CASTDIR"] = os.path.join(ROOTDIR, "cache", "cast") + os.sep
directories["CREWDIR"] = os.path.join(ROOTDIR, "cache", "crew") + os.sep
# Create directories if missing so later code doesn't fail
try:
os.makedirs(LOGDIR, exist_ok=True)
os.makedirs(os.path.join(ROOTDIR, "cache", "series"), exist_ok=True)
os.makedirs(os.path.join(ROOTDIR, "cache", "episodes"), exist_ok=True)
os.makedirs(os.path.join(ROOTDIR, "cache", "cast"), exist_ok=True)
os.makedirs(os.path.join(ROOTDIR, "cache", "crew"), exist_ok=True)
except Exception:
# If directory creation fails, let the logging initialization surface the error
pass
lprint = logs.Logger.LPrint(LOGDIR, False)
lprint.start_log()
import settings.config
@@ -71,20 +90,20 @@ if __name__ == "__main__":
dbclass, dbengine, apiengine = dbs.dbclass, dbs.dbengine, dbs.apiengine
uprocess = url.process.Loadurl()
TVMAZEURLS = url.urls.Tvmazeurls()
TVMAZE_URLS = get_tvmaze()
TVMAZE_URLS = get_tvmaze(TVMAZEURLS)
TVDBURLS = url.urls.TVDBurls()
TVDB_URLS = get_tvdb()
mseries = mazeseries.tvshow()
mcast = mazecast.cast()
mepisode = mazeepisodes.episodes()
mcrew = mazecrew.crew()
TVDB_URLS = get_tvdb(TVDBURLS)
#mseries = mazeseries.tvshow()
#mcast = mazecast.cast()
#mepisode = mazeepisodes.episodes()
#mcrew = mazecrew.crew()
countrydictionary = {}
updateRange = None
lprint.logprint("info", "Creating JSON instance for data retrieval.")
jget = jprocessor.process.Jsonload()
lprint.logprint("info", "JSON instance created.")
print(f"Update type is {options['updatetype']}")
lprint.logprint("info", f"Update type is {options['updatetype']}")
timedifference = options["updatetype"]
currenttimestamp = time.time()
if timedifference.lower() == "full":
@@ -92,22 +111,22 @@ if __name__ == "__main__":
basetime = str(currenttimestamp).split(".")
currenttimestamp = int(basetime[0])
ts1 = str(datetime.now()).split(".")[0]
print("Sync start time: " + str(ts1))
lprint.logprint("info", "Sync start time: " + str(ts1))
print_starttime(lprint, ts1)
updatesBase = dbengine["updatesBase"]
#updatesBase = dbengine["updatesBase"]
dboBase = dbengine["dboBase"]
session = dbengine["session"]
#session = dbengine["session"]
dbExec = db.sql.Dbexec()
createTables = dbclass["Createtables"]
Tables = dbclass["Tables"]
#Tables = dbclass["Tables"]
createTempTables = dbclass["CreateTemptables"]
new_updates = {}
sqlexec = db.sql.SQLGenerate()
mt = createTables(
dbengine["session"],
dbengine["metadata"],
dbengine["engine"]
)
#sqlexec = db.sql.SQLGenerate()
#mt = createTables(
# dbengine["session"],
# dbengine["metadata"],
# dbengine["engine"]
#)
ct = createTempTables(
dbengine["session"],
dbengine["metadata"],
@@ -116,14 +135,32 @@ if __name__ == "__main__":
tempTableList = ct.tablelist
if options["apitype"].lower() == "tvmaze":
new_updates = set_apienv(TVMAZE_URLS, dbExec, tempTableList)
try:
new_updates = set_apienv(
TVMAZE_URLS,
uprocess,
jget,
dbengine,
dbExec,
tempTableList,
lprint,
)
except Exception as exc: # noqa: PLC0301 - broad handler to log failures
lprint.logprint("error", f"Failed to set API env: {exc}")
return 1
from cache.process import Cacher
cacher = Cacher()
try:
cacher.process(options, jget, new_updates, ROOTDIR, TVMAZE_URLS)
except Exception as exc:
lprint.logprint("error", f"Cache processing failed: {exc}")
return 1
timeDifference = options["updatetype"]
currentTimestamp = time.time()
baseTime = str(currentTimestamp).split(".")
#baseTime = str(currentTimestamp).split(".")
if timeDifference.lower() == "full" or options["initload"] == "1":
tdifference = 0
else:
@@ -140,12 +177,17 @@ if __name__ == "__main__":
needed_updates.append(updateid)
updateList.append(updateid)
print("Update Type: " + str(options["updatetype"]))
lprint.logprint("info", "Update Type: " + str(options["updatetype"]))
updateRange = len(updateList)
print("Number of updates: " + str(updateRange))
lprint.logprint("info", "Number of updates: " + str(updateRange))
try:
cacher.update_cache(
jget, updateList, ROOTDIR, TVMAZE_URLS, options, lprint
)
except Exception as exc:
lprint.logprint("error", f"update_cache failed: {exc}")
return 1
countrylisting = dbExec.rawsql_select(
dbengine["engine"],
"select row_id, country_name from dbo.countrydata",
@@ -167,4 +209,8 @@ if __name__ == "__main__":
dboBase,
lprint,
)
exit(0)
return 0
if __name__ == "__main__":
sys.exit(main())