diff --git a/db/functions.py b/db/functions.py index 80fe9f4..efc6677 100644 --- a/db/functions.py +++ b/db/functions.py @@ -21,6 +21,7 @@ import api.tvmaze.cast import api.tvmaze.series from pymongo import MongoClient from tqdm import tqdm +from datetime import datetime Createtables = Createtables CreateTempTables = CreateTempTables @@ -170,6 +171,11 @@ class dbmongo(object): global mgcharacters global mgcrew + def convert_updated_time(self, timestamp): + new_timestamp = datetime.fromtimestamp(timestamp) + update_time = str(new_timestamp).split(".")[0] + return update_time + def __init__(self, options): host = options["mghostname"] port = options["mgport"] @@ -233,6 +239,8 @@ class dbmongo(object): "info", f"Unable to process series file for {seriesid}" ) else: + update_time = self.convert_updated_time(seriesdata["updated"]) + seriesdata["updated"] = update_time self.update_country_data( country, countrydictionary, @@ -475,6 +483,7 @@ class dbmongo(object): persondata = [] characterdata = [] for item in data: + #update_time = convert_updated_time(item["person"]["updated"]) if "person" in item: persondata.append(item["person"]) if "character" in item: @@ -512,7 +521,11 @@ class dbmongo(object): ) else: for item in data: - item["seriesid"] = seriesid + item["person"]["seriesid"] = seriesid + item["person"]["type"] = item["type"] + update_time =self.convert_updated_time(item["person"]["updated"]) + if update_time: + item["person"]["updated"] = str(update_time).split(".")[0] if "person" in item: crewdata.append(item["person"]) try: @@ -526,7 +539,7 @@ class dbmongo(object): f"Unable to process {data_type} data.{e}" ) return failed_file_count - + return data for series in tqdm( range(len(updateList)), desc="Update MongoDB", unit="series" ): @@ -577,24 +590,35 @@ class dbmongo(object): ) if isinstance(fcount, int): episode_failed_count = episode_failed_count + fcount + else: + for episode in fcount: + self.insert_or_update_mongo( + self.mgepisodes, episode, "id" + ) fcount = process_data( castcachefile, "cast", self.process_cast_data, failed_file_count, lprint ) - if isinstance(fcount, int): - cast_failed_count = cast_failed_count + fcount - try: + # Need to split cast between actors and characters collections + # if isinstance(fcount, int): + # cast_failed_count = cast_failed_count + fcount + # else: + # for cast in fcount: + # self.insert_or_update_mongo( + # self.mg, episode, "id" + # ) + if os.path.isfile(crewcachefile): fcount = process_data( crewcachefile, "crew", self.process_crew_data, failed_file_count, lprint ) - except Exception: - crew_failed_count = crew_failed_count + 1 - else: if isinstance(fcount, int): crew_failed_count = crew_failed_count + fcount - processed_count += 1 - + else: + for crew in fcount: + self.insert_or_update_mongo( + self.mgcrew, crew, "id" + ) print(f"Processing complete. {processed_count} series processed.") print(f"Failed to load {series_failed_count} series files.") print(f"Failed to load {episode_failed_count} episode files.") diff --git a/json_downloader.py b/json_downloader.py index 52ad995..6dfdd1e 100644 --- a/json_downloader.py +++ b/json_downloader.py @@ -84305,11 +84305,6 @@ creditsdir = os.path.join(cachedir, 'credits') for seriesid in listing: cachefile = f'{seriesid}.json' - #os.chdir(seriesdir) - #download_json_to_file(showapi.replace('', seriesid), seriesdir, cachefile) - #os.chdir(episodesdir) - #download_json_to_file(episodeapi.replace('', seriesid), episodesdir, cachefile) - #time.sleep(0.5) os.chdir(castdir) download_json_to_file(castapi.replace('', seriesid), castdir, cachefile) os.chdir(crewdir)