Comment out SQL update logic in Dbexec class for verification; adjust sleep duration in JSON downloader to reduce server load; enhance logging in update_mongo.py for clarity on processing updates.

db/sql.py

@@ -24,12 +24,13 @@ class Dbexec():
             )
         ]
 
-        print('Applying updates to table.')
+        # Commented out code for applying updates for now. Will re-enable later after downloads are verified.
-        with engine.connect() as conn:
+        # print('Applying updates to table.')
-            conn.execute(text(f"truncate table {tablename}"))
+        # with engine.connect() as conn:
-            conn.execute(tablename.insert(), updatelist)
+        #     conn.execute(text(f"truncate table {tablename}"))
-            conn.commit()
+        #     conn.execute(tablename.insert(), updatelist)
-        print('Update of tvupdates table is complete.')
+        #     conn.commit()
+        # print('Update of tvupdates table is complete.')
 
     def rawsql_select(self, engine, sqlquery, lprint):
         session = Session(engine)

json_downloader.py

@@ -16,7 +16,7 @@ def download_json_to_file(api_url, outputdir, output_file):
         with open(f"{outputdir}{output_file}", 'w') as f:
             json.dump(data, f, indent=4)
         #print(f"JSON data saved to {outputdir}{output_file}")
-        time.sleep(1)  # Sleep for 1 second to avoid overwhelming the server
+        time.sleep(.5)  # Sleep for 0.5 second to avoid overwhelming the server
     except Exception as e:
         print(f"Failed to download or save JSON: {e}")
 

update_mongo.py

@@ -5,11 +5,7 @@ import time
 import sys
 import url.urls
 import url.process
-# import api.tvmaze.cast as mazecast
+from tqdm import tqdm
-# import api.tvmaze.crew as mazecrew
-# import api.tvmaze.episodes as mazeepisodes
-# import api.tvmaze.series as mazeseries
-from pathlib import Path
 import db.sql
 import jprocessor.process
 import logs.Logger
@@ -288,6 +284,7 @@ def main() -> int:
     directory_list = []
     for value in directories.values():
         directory_list.append(value)
+    print(f"Processing {len(downloaded_updates_dict)} downloaded updates against {len(previous_listing_dict)} previous listings for files needed.")
     for seriesid, ts in downloaded_updates_dict.items():
         if seriesid in previous_listing_dict:
             prev_ts = previous_listing_dict[seriesid]
@@ -315,8 +312,8 @@ def main() -> int:
     lprint.logprint("info", f"Downloadinng JSON for {len(files_needed)} series IDs.")
     print(f"Downloadinng JSON for {len(files_needed)} files.")
 
-    try:
+    for i in tqdm(range(len(files_needed)), desc="Downloading JSON files", unit="filename"):
-        for directory, filename in files_needed:
+        for directory, file in files_needed[i:i+1]:
             if 'credits' in directory:
                 url_template = TVMAZE_URLS["creditsurl"]
             elif 'crew' in directory:
@@ -329,11 +326,10 @@ def main() -> int:
                 url_template = TVMAZE_URLS["aliasurl"]
             else:
                 url_template = TVMAZE_URLS["showurl"]
-            seriesid = filename.replace('.json', '')
+            seriesid = file.replace('.json', '')
             json_downloader.download_json_to_file(url_template.replace("<seriesid>", str(seriesid)), directory, f'{seriesid}.json')
-    except Exception as exc:
+
-        lprint.logprint("error", f"update_cache failed: {exc}")
+
-        return 1
 
 
     # countrylisting = dbExec.rawsql_select(