Refactor update_mongo.py to enhance JSON downloading and file handling; update MongoDB host in test_tableddl_from_yaml.py

mongo2pgsql.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python3
+"""Convert a MongoDB collection into a PostgreSQL table.
+
+Contains `convert_mongo_to_pg(mongo_uri, mongo_db, collection_name, pg_dsn, pg_table=None, ...)`
+and a small CLI.
+
+Notes:
+- Uses `pymongo` to read documents and `psycopg2` to write to Postgres.
+- Infers basic column types from a sample of documents; nested objects/arrays
+  are stored as `JSONB`.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+from typing import Any, Dict, Iterable, List, Optional
+
+import pymongo
+import psycopg2
+import psycopg2.extras as pgextras
+import psycopg2.sql as sql
+
+LOG = logging.getLogger("mongo2pgsql")
+
+
+def _detect_type(value: Any) -> str:
+    if value is None:
+        return "null"
+    if isinstance(value, bool):
+        return "boolean"
+    if isinstance(value, int) and not isinstance(value, bool):
+        return "integer"
+    if isinstance(value, float):
+        return "real"
+    if isinstance(value, (dict, list)):
+        return "jsonb"
+    return "text"
+
+
+def _merge_types(types: List[str]) -> str:
+    # priority: jsonb > text > real > integer > boolean > null
+    s = set(types)
+    if "jsonb" in s:
+        return "jsonb"
+    if "text" in s:
+        return "text"
+    if "real" in s:
+        return "real"
+    if "integer" in s:
+        return "integer"
+    if "boolean" in s:
+        return "boolean"
+    return "text"
+
+
+def _pg_type_for(kind: str) -> str:
+    return {
+        "integer": "BIGINT",
+        "real": "DOUBLE PRECISION",
+        "boolean": "BOOLEAN",
+        "jsonb": "JSONB",
+        "text": "TEXT",
+    }.get(kind, "TEXT")
+
+
+def convert_mongo_to_pg(
+    mongo_uri: str,
+    mongo_db: str,
+    collection_name: str,
+    pg_dsn: str,
+    pg_table: Optional[str] = None,
+    sample_size: int = 500,
+    batch_size: int = 1000,
+    create_table: bool = True,
+    pk_field: str = "_id",
+) -> None:
+    """Copy a MongoDB collection to PostgreSQL.
+
+    - `pg_table` defaults to `collection_name` if omitted.
+    - Nested documents/arrays are stored as JSONB.
+    - `_id` is stored as TEXT by default.
+    """
+    if pg_table is None:
+        pg_table = collection_name
+
+    client = pymongo.MongoClient(mongo_uri)
+    coll = client[mongo_db][collection_name]
+
+    sample = list(coll.find({}, projection=None, limit=sample_size))
+    if not sample:
+        raise ValueError("collection is empty or not found")
+
+    # gather all field names and observed types
+    field_types: Dict[str, List[str]] = {}
+    for doc in sample:
+        for k, v in doc.items():
+            t = _detect_type(v)
+            field_types.setdefault(k, []).append(t)
+
+    # finalize types
+    final_types: Dict[str, str] = {}
+    for k, types in field_types.items():
+        merged = _merge_types(types)
+        final_types[k] = _pg_type_for(merged)
+
+    # ensure primary key present
+    if pk_field not in final_types:
+        final_types[pk_field] = "TEXT"
+
+    cols = list(final_types.items())
+
+    # create table
+    if create_table:
+        # Build CREATE TABLE statement using psycopg2.sql for safe identifiers
+        col_defs = []
+        for name, ptype in cols:
+            col_defs.append(sql.SQL("{} {}").format(sql.Identifier(name), sql.SQL(ptype)))
+
+        create_body = sql.SQL(", ").join(col_defs)
+        if pk_field in final_types:
+            pk_clause = sql.SQL(", PRIMARY KEY ({})").format(sql.SQL(",").join(map(sql.Identifier, [pk_field])))
+            create_body = create_body + pk_clause
+
+        create_stmt = sql.SQL("CREATE TABLE IF NOT EXISTS {} ({})").format(sql.Identifier(pg_table), create_body)
+        with psycopg2.connect(pg_dsn) as pgconn:
+            with pgconn.cursor() as cur:
+                LOG.info("Creating table %s", pg_table)
+                cur.execute(create_stmt)
+                pgconn.commit()
+
+    # stream documents and insert
+    insert_cols = [n for n, _ in cols]
+
+    with psycopg2.connect(pg_dsn) as pgconn:
+        with pgconn.cursor() as cur:
+            # build INSERT statement for execute_values: it expects a single %s
+            base_insert = sql.SQL("INSERT INTO {} ({}) VALUES %s").format(
+                sql.Identifier(pg_table),
+                sql.SQL(', ').join(map(sql.Identifier, insert_cols)),
+            ).as_string(pgconn)
+            batch = []
+            # Use an explicit session when using no_cursor_timeout to avoid pymongo warning
+            with client.start_session() as session:
+                cursor = coll.find({}, no_cursor_timeout=True, session=session).batch_size(batch_size)
+                try:
+                    for doc in cursor:
+                        row = []
+                        for col_name in insert_cols:
+                            v = doc.get(col_name)
+                            if v is None:
+                                row.append(None)
+                                continue
+                            # convert ObjectId and other non-JSON types
+                            if isinstance(v, (dict, list)):
+                                row.append(pgextras.Json(v))
+                            else:
+                                row.append(str(v) if col_name == pk_field else v)
+                        batch.append(tuple(row))
+                        if len(batch) >= batch_size:
+                            pgextras.execute_values(cur, base_insert, batch, template=None, page_size=batch_size)
+                            pgconn.commit()
+                            batch = []
+                finally:
+                    try:
+                        cursor.close()
+                    except Exception:
+                        pass
+            if batch:
+                pgextras.execute_values(cur, base_insert, batch, template=None, page_size=len(batch))
+                pgconn.commit()
+
+    LOG.info("Finished copying collection %s.%s to %s", mongo_db, collection_name, pg_table)
+
+
+def _cli() -> None:
+    parser = argparse.ArgumentParser(description="Copy a MongoDB collection to PostgreSQL")
+    parser.add_argument("mongo_uri", help="MongoDB connection URI, e.g. mongodb://user:pw@host:27017")
+    parser.add_argument("mongo_db", help="MongoDB database name")
+    parser.add_argument("collection", help="Collection name to copy")
+    parser.add_argument("pg_dsn", help="Postgres DSN, e.g. \"host=.. dbname=.. user=.. password=..\"")
+    parser.add_argument("--pg-table", help="Target Postgres table name (defaults to collection name)")
+    parser.add_argument("--sample", type=int, default=500, help="Sample size to infer schema")
+    parser.add_argument("--batch", type=int, default=1000, help="Insert batch size")
+    parser.add_argument("--no-create", dest="create", action="store_false", help="Don't attempt to create table in PG")
+    parser.add_argument("--verbose", "-v", action="store_true")
+    args = parser.parse_args()
+
+    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
+
+    convert_mongo_to_pg(
+        args.mongo_uri,
+        args.mongo_db,
+        args.collection,
+        args.pg_dsn,
+        pg_table=args.pg_table,
+        sample_size=args.sample,
+        batch_size=args.batch,
+        create_table=args.create,
+    )
+
+
+if __name__ == "__main__":
+    _cli()

update_mongo.py

@@ -5,15 +5,18 @@ import time
 import sys
 import url.urls
 import url.process
-import api.tvmaze.cast as mazecast
+# import api.tvmaze.cast as mazecast
-import api.tvmaze.crew as mazecrew
+# import api.tvmaze.crew as mazecrew
-import api.tvmaze.episodes as mazeepisodes
+# import api.tvmaze.episodes as mazeepisodes
-import api.tvmaze.series as mazeseries
+# import api.tvmaze.series as mazeseries
+from pathlib import Path
 import db.sql
 import jprocessor.process
 import logs.Logger
 from db.functions import settype, dbmongo
 from datetime import datetime
+import json_downloader
+import json
 
 response = None
 
@@ -57,6 +60,23 @@ def get_tvdb(tvdburls):
 def get_tvmaze(tvmazeurls):
     return tvmazeurls.get_data()
 
+def find_file_in_multiple_dirs(filename, directories):
+    """
+    Checks if a file exists in any of the provided directories.
+
+    Args:
+        filename (str): The name of the file to search for.
+        directories (list): A list of directory paths to check.
+
+    Returns:
+        Path or None: The full path to the file if found, otherwise None.
+    """
+    directories_found = []
+    for directory in directories:
+        if os.path.isfile(f'{directory}{filename}'):
+            directories_found.append(directory)
+                
+    return directories_found
 
 def main() -> int:
     ROOTDIR = os.getcwd()
@@ -67,6 +87,9 @@ def main() -> int:
     directories["EPISODEDIR"] = os.path.join(ROOTDIR, "cache", "episodes") + os.sep
     directories["CASTDIR"] = os.path.join(ROOTDIR, "cache", "cast") + os.sep
     directories["CREWDIR"] = os.path.join(ROOTDIR, "cache", "crew") + os.sep
+    directories["CREDITSDIR"] = os.path.join(ROOTDIR, "cache", "credits") + os.sep
+    directories["ALIASDIR"] = os.path.join(ROOTDIR, "cache", "aliases") + os.sep
+    tempdir = os.path.join(ROOTDIR, 'temp') + os.sep
 
     # Create directories if missing so later code doesn't fail
     try:
@@ -93,10 +116,6 @@ def main() -> int:
     TVMAZE_URLS = get_tvmaze(TVMAZEURLS)
     TVDBURLS = url.urls.TVDBurls()
     TVDB_URLS = get_tvdb(TVDBURLS)
-    #mseries = mazeseries.tvshow()
-    #mcast = mazecast.cast()
-    #mepisode = mazeepisodes.episodes()
-    #mcrew = mazecrew.crew()
     countrydictionary = {}
     updateRange = None
 
@@ -171,37 +190,172 @@ def main() -> int:
     lprint.logprint("info", "Update Type: " + str(options["updatetype"]))
     updateRange = len(updateList)
     lprint.logprint("info", "Number of updates: " + str(updateRange))
+
+
+    # Save a downloaded copy of the updates JSON for debugging/inspection
     try:
-        cacher.update_cache(
+        os.makedirs(tempdir, exist_ok=True)
-            jget, updateList, ROOTDIR, TVMAZE_URLS, options, lprint
+        out_file = 'tvmaze_updates.json'
+        # TVMAZE_URLS is a dict of urls; use the updatesurl if available
+        updates_url = TVMAZE_URLS.get('updatesurl') if isinstance(TVMAZE_URLS, dict) else None
+        if updates_url:
+            json_downloader.download_json_to_file(updates_url, tempdir, out_file)
+            # load and convert saved JSON into a dict (id -> timestamp)
+            try:
+                downloaded_updates_dict = {}
+                downloaded_updates_list = []
+                if os.path.exists(f"{tempdir}{out_file}"):
+                    with open(f"{tempdir}{out_file}", 'r') as jf:
+                        loaded = json.load(jf)
+
+                    if isinstance(loaded, dict):
+                        for k, v in loaded.items():
+                            try:
+                                kid = int(k)
+                            except Exception:
+                                kid = k
+                            try:
+                                vt = int(v)
+                            except Exception:
+                                vt = v
+                            downloaded_updates_dict[kid] = vt
+                        downloaded_updates_list = list(downloaded_updates_dict.items())
+
+                    elif isinstance(loaded, list):
+                        for item in loaded:
+                            if isinstance(item, list) and len(item) >= 2:
+                                k = item[0]; v = item[1]
+                            elif isinstance(item, dict):
+                                if 'id' in item and 'timestamp' in item:
+                                    k = item['id']; v = item['timestamp']
+                                elif 'seriesid' in item and 'timestamp' in item:
+                                    k = item['seriesid']; v = item['timestamp']
+                                else:
+                                    downloaded_updates_list.append(item)
+                                    continue
+                            else:
+                                k = item; v = None
+                            try:
+                                kid = int(k)
+                            except Exception:
+                                kid = k
+                            try:
+                                vt = int(v) if v is not None and str(v).isdigit() else v
+                            except Exception:
+                                vt = v
+                            downloaded_updates_dict[kid] = vt
+                            downloaded_updates_list.append((kid, vt))
+
+                    else:
+                        downloaded_updates_list = [loaded]
+                    lprint.logprint("info", f"Loaded {len(downloaded_updates_list)} entries from tvmaze_updates.json")
+                else:
+                    downloaded_updates_dict = {}
+                    downloaded_updates_list = []
+            except Exception:
+                downloaded_updates_dict = {}
+                downloaded_updates_list = []
+    except Exception:
+        # non-fatal; continue
+        pass
+    del downloaded_updates_list
+    previouslisting = dbExec.rawsql_select(
+        dbengine["engine"],
+        "select seriesid, timestamp from updates.tvupdates order by seriesid",
+        lprint
     )
+    # convert previouslisting (list of tuples) into a dict: row_id -> country_name
+    try:
+        _previouslisting_list = previouslisting
+        previous_listing_dict = {row[0]: row[1] for row in _previouslisting_list}
+    except Exception:
+        # leave previouslisting as-is on any failure
+        pass
+    # Ensure dictionaries exist
+    try:
+        downloaded_updates_dict
+    except NameError:
+        downloaded_updates_dict = {}
+    try:
+        previous_listing_dict
+    except NameError:
+        previous_listing_dict = {}
+    files_needed = []
+    directory_list = []
+    for value in directories.values():
+        directory_list.append(value)
+    for seriesid, ts in downloaded_updates_dict.items():
+        if seriesid in previous_listing_dict:
+            prev_ts = previous_listing_dict[seriesid]
+            try:
+                if int(ts) == int(prev_ts):
+                    found_path = find_file_in_multiple_dirs(f"{seriesid}.json", directory_list)
+                    #print(f"Found path for series ID {seriesid}: {found_path}")
+                    if len(found_path) == 6:
+                        continue
+                    else:
+                        for directory in directory_list:
+                            if directory not in found_path:
+                                files_needed.append([f"{directory}", f"{seriesid}.json"])
+                else:
+                    for directory in directory_list:
+                        files_needed.append([f"{directory}", f"{seriesid}.json"])
+            except Exception:
+                for directory in directory_list:
+                    files_needed.append([f"{directory}", f"{seriesid}.json"])
+        else:
+            for directory in directory_list:
+                files_needed.append([f"{directory}", f"{seriesid}.json"])
+            lprint.logprint("debug", f"Series {seriesid} present in downloaded updates but missing in previous listing")
+
+    lprint.logprint("info", f"Downloadinng JSON for {len(files_needed)} series IDs.")
+    print(f"Downloadinng JSON for {len(files_needed)} series IDs.")
+
+    try:
+        for directory, filename in files_needed:
+            if 'credits' in directory:
+                url_template = TVMAZE_URLS["creditsurl"]
+            elif 'crew' in directory:
+                url_template = TVMAZE_URLS["crewurl"]
+            elif 'cast' in directory:
+                url_template = TVMAZE_URLS["casturl"]
+            elif 'episodes' in directory:
+                url_template = TVMAZE_URLS["episodesurl"]
+            elif 'aliases' in directory:
+                url_template = TVMAZE_URLS["aliasurl"]
+            else:
+                url_template = TVMAZE_URLS["showurl"]
+            seriesid = filename.replace('.json', '')
+            json_downloader.download_json_to_file(url_template.replace("<seriesid>", str(seriesid)), directory, f'{seriesid}.json')
     except Exception as exc:
         lprint.logprint("error", f"update_cache failed: {exc}")
         return 1
 
-    countrylisting = dbExec.rawsql_select(
+
-        dbengine["engine"],
+    # countrylisting = dbExec.rawsql_select(
-        "select row_id, country_name from dbo.countrydata",
+    #     dbengine["engine"],
-        lprint
+    #     "select row_id, country_name from dbo.countrydata",
-    )
+    #     lprint
-    for country in countrylisting:
+    # )
-        country_name = country[1]
+    # for country in countrylisting:
-        country_id = country[0]
+    #     country_name = country[1]
-        countrydictionary[country_name] = country_id
+    #     country_id = country[0]
-    mongo_updater = dbmongo(options)
+    #     countrydictionary[country_name] = country_id
-    mongo_updater.update_mongo(
+    # mongo_updater = dbmongo(options)
-        updateList,
+    # mongo_updater.update_mongo(
-        ROOTDIR,
+    #     updateList,
-        directories,
+    #     ROOTDIR,
-        jget,
+    #     directories,
-        countrydictionary,
+    #     jget,
-        dbExec,
+    #     countrydictionary,
-        dbengine,
+    #     dbExec,
-        dboBase,
+    #     dbengine,
-        lprint,
+    #     dboBase,
-    )
+    #     lprint,
-    return 0
+    # )
+    # return 0
 
 
 if __name__ == "__main__":
     sys.exit(main())
+

yaml_ddl/test_tableddl_from_yaml.py

@@ -55,7 +55,7 @@ except Exception as e:
     exit()
 
 
-host = '192.168.128.8'
+host = '192.168.128.24'
 port = 27017
 print(f"Connecting to mongodb at {host}:{port}")
 
@@ -83,7 +83,7 @@ else:
         f"Connection to mongodb at {host}:"
         f"{port} successful."
     )
-os.chdir('C:\\Projects\\new_dbsync\\db\\ddl')
+#os.chdir('C:\\Projects\\new_dbsync\\db\\ddl')
 # Create the database and tables in PostgreSQL
 # Check if the database exists, if not create it
 # Check if the tables exist, if not create them