#!/usr/bin/env python3 # -*- coding: utf-8 -*- import os import time import sys from tqdm import tqdm import logs.Logger from db.functions import settype, dbmongo from datetime import datetime import json from db.schema_generator import MongoToPostgresSchemaGenerator, MongoDocumentInserter import settings.config def set_apienv(urls, uprocess, dbengine, dbExec, updatesBase, lprint): """Populate updates table from the API and return available updates. Args: urls: URL configuration with an "updatesurl" key. uprocess: URL loader instance with `get_data`. jget: JSON processor with `jconvert`. dbengine: database engine dict (expects key "engine"). dbExec: database execution helper with `update_tvupdates` and `rawsql_select`. updatesBase: list-like containing tables (uses index 8 for updates table). lprint: logger instance with `logprint`. Returns: List of tuples (seriesid, timestamp) representing available updates. """ #updatetable = updatesBase[8] #inputdata = uprocess.get_data(urls["updatesurl"]) #lprint.logprint("info", f"Retrieved {len(availableupdates)} rows for processing......") # newupdates = dbExec.rawsql_select( dbengine["engine"], "select seriesid,timestamp from updates.tvupdates", lprint ) return newupdates ROOTDIR = os.getcwd() config_options = settings.config.Config(ROOTDIR) options = config_options.config_options dbtype, apitype = options["dbtype"], options["apitype"] dbs = settype(dbtype, apitype, options) dbclass, dbengine = dbs.dbclass, dbs.dbengine mongo_updater = dbmongo(options) generator = MongoToPostgresSchemaGenerator(sample_size=100) tableNames = ['seriesdata', 'episodesdata', 'actorsdata', 'charactersdata', 'crewdata'] for tableName in tableNames: # Generate and print the schema for each collection schema = generator.analyze_collection(mongo_updater, getattr(mongo_updater, f'mg{tableName[:-4]}')) sql = generator.generate_create_table_sql( tableName, schema_name='updates', pk_field='id' if tableName == 'seriesdata' else 'id', ) print(f"--- SQL Schema for {tableName} ---") print(sql) print("\n") # Or create it directly generator.create_table_in_postgres( dbengine, tableName, schema_name='updates', #pk_field='seriesid' if tableName == 'seriesdata' else 'id', pk_field='id', drop_existing=True, ) inserter = MongoDocumentInserter(batch_size=10000) # Map table names to their MongoDB collections and primary keys collection_map = { 'seriesdata': (mongo_updater.mgseries, 'id'), 'episodesdata': (mongo_updater.mgepisodes, 'id'), 'actorsdata': (mongo_updater.mgactors, 'id'), 'charactersdata': (mongo_updater.mgcharacters, 'id'), 'crewdata': (mongo_updater.mgcrew, 'id'), } # Insert documents from collections with UPSERT for tableName in tableNames: if tableName in collection_map: collection, pk_field = collection_map[tableName] # Build ON CONFLICT ... DO UPDATE clause # Re-analyze the specific collection so we have the correct fields schema = generator.analyze_collection(mongo_updater, collection) # Use lowercased column names to match PostgreSQL unquoted identifiers update_cols = [col.lower() for col in schema.keys() if col.lower() != pk_field.lower() and col != '_id'] update_clause = ', '.join([f"{col}=EXCLUDED.{col}" for col in update_cols]) on_conflict = f"ON CONFLICT ({pk_field}) DO UPDATE SET {update_clause}" count = inserter.insert_from_collection( engine=dbengine, table_name=tableName, collection=collection, schema_name='updates', on_conflict=on_conflict, ) print(f"Inserted/Updated {count} documents in {tableName}") else: print(f"Warning: No collection mapping for {tableName}") # # Or directly from MongoDB collection # count = inserter.insert_from_collection( # engine=dbengine, # table_name='seriesdata', # collection=mongo_db['series'], # schema_name='dbo', # on_conflict="DO NOTHING", # ) # if __name__ == "__main__": # sys.exit(main()) # Below are the relevant methods from db/functions.py and db/query_utils.py that are used in the above code. They are included here for completeness and context. # Needs to be incorporated into logic so that only releveant documents are updated. # from db.query_utils import MongoQueryUtils # # With additional filters # recent = MongoQueryUtils.get_recent_updates( # collection=mongo_updater.mgseries, # minutes=60, # updated_field='updated', # query_filter={'status': 'Ended'}, # Only ended shows # ) # # Since a specific timestamp # docs = MongoQueryUtils.get_updates_since( # collection=mongo_updater.mgseries, # since_timestamp=None, # Epoch or ISO string # ) # for doc in docs: # print(doc)