Add timestamp conversion for updated fields in dbmongo class and clean up json_downloader.py

db/functions.py

@@ -21,6 +21,7 @@ import api.tvmaze.cast
 import api.tvmaze.series
 from pymongo import MongoClient
 from tqdm import tqdm
+from datetime import datetime
 
 Createtables = Createtables
 CreateTempTables = CreateTempTables
@@ -170,6 +171,11 @@ class dbmongo(object):
     global mgcharacters
     global mgcrew
 
+    def convert_updated_time(self, timestamp):
+        new_timestamp = datetime.fromtimestamp(timestamp)
+        update_time = str(new_timestamp).split(".")[0]
+        return update_time
+        
     def __init__(self, options):
         host = options["mghostname"]
         port = options["mgport"]
@@ -233,6 +239,8 @@ class dbmongo(object):
                 "info", f"Unable to process series file for {seriesid}"
             )
         else:
+            update_time = self.convert_updated_time(seriesdata["updated"])
+            seriesdata["updated"] = update_time
             self.update_country_data(
                 country,
                 countrydictionary,
@@ -475,6 +483,7 @@ class dbmongo(object):
                     persondata = []
                     characterdata = []
                     for item in data:
+                        #update_time = convert_updated_time(item["person"]["updated"])
                         if "person" in item:
                             persondata.append(item["person"])
                         if "character" in item:
@@ -512,7 +521,11 @@ class dbmongo(object):
                         )
                     else:
                         for item in data:
-                            item["seriesid"] = seriesid
+                            item["person"]["seriesid"] = seriesid
+                            item["person"]["type"] = item["type"]
+                            update_time =self.convert_updated_time(item["person"]["updated"])
+                            if update_time:
+                                item["person"]["updated"] = str(update_time).split(".")[0]
                             if "person" in item:
                                 crewdata.append(item["person"])
                         try:
@@ -526,7 +539,7 @@ class dbmongo(object):
                                 f"Unable to process {data_type} data.{e}"
                             )
                             return failed_file_count
-
+                return data
         for series in tqdm(
             range(len(updateList)), desc="Update MongoDB", unit="series"
         ):
@@ -577,24 +590,35 @@ class dbmongo(object):
             )
             if isinstance(fcount, int):
                 episode_failed_count = episode_failed_count + fcount
+            else:
+                for episode in fcount:
+                    self.insert_or_update_mongo(
+                        self.mgepisodes, episode, "id"
+                    )
             fcount = process_data(
                 castcachefile, "cast", self.process_cast_data,
                 failed_file_count, lprint
             )
-            if isinstance(fcount, int):
-                cast_failed_count = cast_failed_count + fcount
-            try:
+            # Need to split cast between actors and characters collections
+            # if isinstance(fcount, int):
+            #     cast_failed_count = cast_failed_count + fcount
+            # else:
+            #     for cast in fcount:
+            #         self.insert_or_update_mongo(
+            #             self.mg, episode, "id"
+            #         )    
+            if os.path.isfile(crewcachefile):
                 fcount = process_data(
                     crewcachefile, "crew", self.process_crew_data,
                     failed_file_count, lprint
                 )
-            except Exception:
-                crew_failed_count = crew_failed_count + 1
-            else:
                 if isinstance(fcount, int):
                     crew_failed_count = crew_failed_count + fcount
-                processed_count += 1
-
+                else:
+                    for crew in fcount:
+                        self.insert_or_update_mongo(
+                            self.mgcrew, crew, "id"
+                        )
         print(f"Processing complete. {processed_count} series processed.")
         print(f"Failed to load {series_failed_count} series files.")
         print(f"Failed to load {episode_failed_count} episode files.")

json_downloader.py

@@ -84305,11 +84305,6 @@ creditsdir =  os.path.join(cachedir, 'credits')
 
 for seriesid in listing:
     cachefile = f'{seriesid}.json'
-    #os.chdir(seriesdir)
-    #download_json_to_file(showapi.replace('<tvmazeid>', seriesid), seriesdir, cachefile)
-    #os.chdir(episodesdir)
-    #download_json_to_file(episodeapi.replace('<tvmazeid>', seriesid), episodesdir, cachefile)
-    #time.sleep(0.5)
     os.chdir(castdir)
     download_json_to_file(castapi.replace('<tvmazeid>', seriesid), castdir, cachefile)
     os.chdir(crewdir)