Refactor update_mongo.py to enhance JSON downloading and file handling; update MongoDB host in test_tableddl_from_yaml.py
This commit is contained in:
Regular → Executable
+187
-33
@@ -5,15 +5,18 @@ import time
|
||||
import sys
|
||||
import url.urls
|
||||
import url.process
|
||||
import api.tvmaze.cast as mazecast
|
||||
import api.tvmaze.crew as mazecrew
|
||||
import api.tvmaze.episodes as mazeepisodes
|
||||
import api.tvmaze.series as mazeseries
|
||||
# import api.tvmaze.cast as mazecast
|
||||
# import api.tvmaze.crew as mazecrew
|
||||
# import api.tvmaze.episodes as mazeepisodes
|
||||
# import api.tvmaze.series as mazeseries
|
||||
from pathlib import Path
|
||||
import db.sql
|
||||
import jprocessor.process
|
||||
import logs.Logger
|
||||
from db.functions import settype, dbmongo
|
||||
from datetime import datetime
|
||||
import json_downloader
|
||||
import json
|
||||
|
||||
response = None
|
||||
|
||||
@@ -57,6 +60,23 @@ def get_tvdb(tvdburls):
|
||||
def get_tvmaze(tvmazeurls):
|
||||
return tvmazeurls.get_data()
|
||||
|
||||
def find_file_in_multiple_dirs(filename, directories):
|
||||
"""
|
||||
Checks if a file exists in any of the provided directories.
|
||||
|
||||
Args:
|
||||
filename (str): The name of the file to search for.
|
||||
directories (list): A list of directory paths to check.
|
||||
|
||||
Returns:
|
||||
Path or None: The full path to the file if found, otherwise None.
|
||||
"""
|
||||
directories_found = []
|
||||
for directory in directories:
|
||||
if os.path.isfile(f'{directory}{filename}'):
|
||||
directories_found.append(directory)
|
||||
|
||||
return directories_found
|
||||
|
||||
def main() -> int:
|
||||
ROOTDIR = os.getcwd()
|
||||
@@ -67,6 +87,9 @@ def main() -> int:
|
||||
directories["EPISODEDIR"] = os.path.join(ROOTDIR, "cache", "episodes") + os.sep
|
||||
directories["CASTDIR"] = os.path.join(ROOTDIR, "cache", "cast") + os.sep
|
||||
directories["CREWDIR"] = os.path.join(ROOTDIR, "cache", "crew") + os.sep
|
||||
directories["CREDITSDIR"] = os.path.join(ROOTDIR, "cache", "credits") + os.sep
|
||||
directories["ALIASDIR"] = os.path.join(ROOTDIR, "cache", "aliases") + os.sep
|
||||
tempdir = os.path.join(ROOTDIR, 'temp') + os.sep
|
||||
|
||||
# Create directories if missing so later code doesn't fail
|
||||
try:
|
||||
@@ -93,10 +116,6 @@ def main() -> int:
|
||||
TVMAZE_URLS = get_tvmaze(TVMAZEURLS)
|
||||
TVDBURLS = url.urls.TVDBurls()
|
||||
TVDB_URLS = get_tvdb(TVDBURLS)
|
||||
#mseries = mazeseries.tvshow()
|
||||
#mcast = mazecast.cast()
|
||||
#mepisode = mazeepisodes.episodes()
|
||||
#mcrew = mazecrew.crew()
|
||||
countrydictionary = {}
|
||||
updateRange = None
|
||||
|
||||
@@ -171,37 +190,172 @@ def main() -> int:
|
||||
lprint.logprint("info", "Update Type: " + str(options["updatetype"]))
|
||||
updateRange = len(updateList)
|
||||
lprint.logprint("info", "Number of updates: " + str(updateRange))
|
||||
|
||||
|
||||
# Save a downloaded copy of the updates JSON for debugging/inspection
|
||||
try:
|
||||
cacher.update_cache(
|
||||
jget, updateList, ROOTDIR, TVMAZE_URLS, options, lprint
|
||||
)
|
||||
os.makedirs(tempdir, exist_ok=True)
|
||||
out_file = 'tvmaze_updates.json'
|
||||
# TVMAZE_URLS is a dict of urls; use the updatesurl if available
|
||||
updates_url = TVMAZE_URLS.get('updatesurl') if isinstance(TVMAZE_URLS, dict) else None
|
||||
if updates_url:
|
||||
json_downloader.download_json_to_file(updates_url, tempdir, out_file)
|
||||
# load and convert saved JSON into a dict (id -> timestamp)
|
||||
try:
|
||||
downloaded_updates_dict = {}
|
||||
downloaded_updates_list = []
|
||||
if os.path.exists(f"{tempdir}{out_file}"):
|
||||
with open(f"{tempdir}{out_file}", 'r') as jf:
|
||||
loaded = json.load(jf)
|
||||
|
||||
if isinstance(loaded, dict):
|
||||
for k, v in loaded.items():
|
||||
try:
|
||||
kid = int(k)
|
||||
except Exception:
|
||||
kid = k
|
||||
try:
|
||||
vt = int(v)
|
||||
except Exception:
|
||||
vt = v
|
||||
downloaded_updates_dict[kid] = vt
|
||||
downloaded_updates_list = list(downloaded_updates_dict.items())
|
||||
|
||||
elif isinstance(loaded, list):
|
||||
for item in loaded:
|
||||
if isinstance(item, list) and len(item) >= 2:
|
||||
k = item[0]; v = item[1]
|
||||
elif isinstance(item, dict):
|
||||
if 'id' in item and 'timestamp' in item:
|
||||
k = item['id']; v = item['timestamp']
|
||||
elif 'seriesid' in item and 'timestamp' in item:
|
||||
k = item['seriesid']; v = item['timestamp']
|
||||
else:
|
||||
downloaded_updates_list.append(item)
|
||||
continue
|
||||
else:
|
||||
k = item; v = None
|
||||
try:
|
||||
kid = int(k)
|
||||
except Exception:
|
||||
kid = k
|
||||
try:
|
||||
vt = int(v) if v is not None and str(v).isdigit() else v
|
||||
except Exception:
|
||||
vt = v
|
||||
downloaded_updates_dict[kid] = vt
|
||||
downloaded_updates_list.append((kid, vt))
|
||||
|
||||
else:
|
||||
downloaded_updates_list = [loaded]
|
||||
lprint.logprint("info", f"Loaded {len(downloaded_updates_list)} entries from tvmaze_updates.json")
|
||||
else:
|
||||
downloaded_updates_dict = {}
|
||||
downloaded_updates_list = []
|
||||
except Exception:
|
||||
downloaded_updates_dict = {}
|
||||
downloaded_updates_list = []
|
||||
except Exception:
|
||||
# non-fatal; continue
|
||||
pass
|
||||
del downloaded_updates_list
|
||||
previouslisting = dbExec.rawsql_select(
|
||||
dbengine["engine"],
|
||||
"select seriesid, timestamp from updates.tvupdates order by seriesid",
|
||||
lprint
|
||||
)
|
||||
# convert previouslisting (list of tuples) into a dict: row_id -> country_name
|
||||
try:
|
||||
_previouslisting_list = previouslisting
|
||||
previous_listing_dict = {row[0]: row[1] for row in _previouslisting_list}
|
||||
except Exception:
|
||||
# leave previouslisting as-is on any failure
|
||||
pass
|
||||
# Ensure dictionaries exist
|
||||
try:
|
||||
downloaded_updates_dict
|
||||
except NameError:
|
||||
downloaded_updates_dict = {}
|
||||
try:
|
||||
previous_listing_dict
|
||||
except NameError:
|
||||
previous_listing_dict = {}
|
||||
files_needed = []
|
||||
directory_list = []
|
||||
for value in directories.values():
|
||||
directory_list.append(value)
|
||||
for seriesid, ts in downloaded_updates_dict.items():
|
||||
if seriesid in previous_listing_dict:
|
||||
prev_ts = previous_listing_dict[seriesid]
|
||||
try:
|
||||
if int(ts) == int(prev_ts):
|
||||
found_path = find_file_in_multiple_dirs(f"{seriesid}.json", directory_list)
|
||||
#print(f"Found path for series ID {seriesid}: {found_path}")
|
||||
if len(found_path) == 6:
|
||||
continue
|
||||
else:
|
||||
for directory in directory_list:
|
||||
if directory not in found_path:
|
||||
files_needed.append([f"{directory}", f"{seriesid}.json"])
|
||||
else:
|
||||
for directory in directory_list:
|
||||
files_needed.append([f"{directory}", f"{seriesid}.json"])
|
||||
except Exception:
|
||||
for directory in directory_list:
|
||||
files_needed.append([f"{directory}", f"{seriesid}.json"])
|
||||
else:
|
||||
for directory in directory_list:
|
||||
files_needed.append([f"{directory}", f"{seriesid}.json"])
|
||||
lprint.logprint("debug", f"Series {seriesid} present in downloaded updates but missing in previous listing")
|
||||
|
||||
lprint.logprint("info", f"Downloadinng JSON for {len(files_needed)} series IDs.")
|
||||
print(f"Downloadinng JSON for {len(files_needed)} series IDs.")
|
||||
|
||||
try:
|
||||
for directory, filename in files_needed:
|
||||
if 'credits' in directory:
|
||||
url_template = TVMAZE_URLS["creditsurl"]
|
||||
elif 'crew' in directory:
|
||||
url_template = TVMAZE_URLS["crewurl"]
|
||||
elif 'cast' in directory:
|
||||
url_template = TVMAZE_URLS["casturl"]
|
||||
elif 'episodes' in directory:
|
||||
url_template = TVMAZE_URLS["episodesurl"]
|
||||
elif 'aliases' in directory:
|
||||
url_template = TVMAZE_URLS["aliasurl"]
|
||||
else:
|
||||
url_template = TVMAZE_URLS["showurl"]
|
||||
seriesid = filename.replace('.json', '')
|
||||
json_downloader.download_json_to_file(url_template.replace("<seriesid>", str(seriesid)), directory, f'{seriesid}.json')
|
||||
except Exception as exc:
|
||||
lprint.logprint("error", f"update_cache failed: {exc}")
|
||||
return 1
|
||||
|
||||
countrylisting = dbExec.rawsql_select(
|
||||
dbengine["engine"],
|
||||
"select row_id, country_name from dbo.countrydata",
|
||||
lprint
|
||||
)
|
||||
for country in countrylisting:
|
||||
country_name = country[1]
|
||||
country_id = country[0]
|
||||
countrydictionary[country_name] = country_id
|
||||
mongo_updater = dbmongo(options)
|
||||
mongo_updater.update_mongo(
|
||||
updateList,
|
||||
ROOTDIR,
|
||||
directories,
|
||||
jget,
|
||||
countrydictionary,
|
||||
dbExec,
|
||||
dbengine,
|
||||
dboBase,
|
||||
lprint,
|
||||
)
|
||||
return 0
|
||||
|
||||
# countrylisting = dbExec.rawsql_select(
|
||||
# dbengine["engine"],
|
||||
# "select row_id, country_name from dbo.countrydata",
|
||||
# lprint
|
||||
# )
|
||||
# for country in countrylisting:
|
||||
# country_name = country[1]
|
||||
# country_id = country[0]
|
||||
# countrydictionary[country_name] = country_id
|
||||
# mongo_updater = dbmongo(options)
|
||||
# mongo_updater.update_mongo(
|
||||
# updateList,
|
||||
# ROOTDIR,
|
||||
# directories,
|
||||
# jget,
|
||||
# countrydictionary,
|
||||
# dbExec,
|
||||
# dbengine,
|
||||
# dboBase,
|
||||
# lprint,
|
||||
# )
|
||||
# return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
||||
|
||||
Reference in New Issue
Block a user