Refactor MongoDB to PostgreSQL data synchronization: remove json2postgres.py, add mongodb2postgres.py for improved schema generation and data insertion

mongodb2postgres.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import os
+import time
+import sys
+from tqdm import tqdm
+import logs.Logger
+from db.functions import settype, dbmongo
+from datetime import datetime
+import json
+from db.schema_generator import MongoToPostgresSchemaGenerator, MongoDocumentInserter
+import settings.config
+    
+def set_apienv(urls, uprocess, dbengine, dbExec, updatesBase, lprint):
+    """Populate updates table from the API and return available updates.
+
+    Args:
+        urls: URL configuration with an "updatesurl" key.
+        uprocess: URL loader instance with `get_data`.
+        jget: JSON processor with `jconvert`.
+        dbengine: database engine dict (expects key "engine").
+        dbExec: database execution helper with `update_tvupdates` and `rawsql_select`.
+        updatesBase: list-like containing tables (uses index 8 for updates table).
+        lprint: logger instance with `logprint`.
+
+    Returns:
+        List of tuples (seriesid, timestamp) representing available updates.
+    """
+    #updatetable = updatesBase[8]
+    #inputdata = uprocess.get_data(urls["updatesurl"])
+    #lprint.logprint("info", f"Retrieved {len(availableupdates)} rows for processing......")
+    #
+    newupdates = dbExec.rawsql_select(
+        dbengine["engine"],
+        "select seriesid,timestamp from updates.tvupdates",
+        lprint
+    )
+    return newupdates
+
+
+ROOTDIR = os.getcwd()
+config_options = settings.config.Config(ROOTDIR)
+options = config_options.config_options
+dbtype, apitype = options["dbtype"], options["apitype"]
+dbs = settype(dbtype, apitype, options)
+dbclass, dbengine = dbs.dbclass, dbs.dbengine
+mongo_updater = dbmongo(options)
+generator = MongoToPostgresSchemaGenerator(sample_size=100)
+
+tableNames = ['seriesdata', 'episodesdata', 'actorsdata', 'charactersdata', 'crewdata']
+
+for tableName in tableNames:
+    # Generate and print the schema for each collection
+    schema = generator.analyze_collection(mongo_updater, getattr(mongo_updater, f'mg{tableName[:-4]}'))
+    sql = generator.generate_create_table_sql(
+        tableName,
+        schema_name='updates',
+        pk_field='id' if tableName == 'seriesdata' else 'id',
+    )
+    print(f"--- SQL Schema for {tableName} ---")
+    print(sql)
+    print("\n")
+
+    # Or create it directly
+    generator.create_table_in_postgres(
+        dbengine,
+        tableName,
+        schema_name='updates',
+        #pk_field='seriesid' if tableName == 'seriesdata' else 'id',
+        pk_field='id',
+        drop_existing=True,
+    )
+
+
+
+inserter = MongoDocumentInserter(batch_size=1000)
+
+# Map table names to their MongoDB collections and primary keys
+collection_map = {
+    'seriesdata': (mongo_updater.mgseries, 'id'),
+    'episodesdata': (mongo_updater.mgepisodes, 'id'),
+    'actorsdata': (mongo_updater.mgactors, 'id'),
+    'charactersdata': (mongo_updater.mgcharacters, 'id'),
+    'crewdata': (mongo_updater.mgcrew, 'id'),
+}
+
+# Insert documents from collections with UPSERT
+for tableName in tableNames:
+    if tableName in collection_map:
+        collection, pk_field = collection_map[tableName]
+        # Build ON CONFLICT ... DO UPDATE clause
+        # Re-analyze the specific collection so we have the correct fields
+        schema = generator.analyze_collection(mongo_updater, collection)
+        # Use lowercased column names to match PostgreSQL unquoted identifiers
+        update_cols = [col.lower() for col in schema.keys() if col.lower() != pk_field.lower() and col != '_id']
+        update_clause = ', '.join([f"{col}=EXCLUDED.{col}" for col in update_cols])
+        on_conflict = f"ON CONFLICT ({pk_field}) DO UPDATE SET {update_clause}"
+        
+        count = inserter.insert_from_collection(
+            engine=dbengine,
+            table_name=tableName,
+            collection=collection,
+            schema_name='updates',
+            on_conflict=on_conflict,
+        )
+        print(f"Inserted/Updated {count} documents in {tableName}")
+    else:
+        print(f"Warning: No collection mapping for {tableName}")
+
+# # Or directly from MongoDB collection
+# count = inserter.insert_from_collection(
+#     engine=dbengine,
+#     table_name='seriesdata',
+#     collection=mongo_db['series'],
+#     schema_name='dbo',
+#     on_conflict="DO NOTHING",
+# )
+# if __name__ == "__main__":
+#     sys.exit(main())
+