140 lines
4.9 KiB
Python
140 lines
4.9 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
import os
|
|
import time
|
|
import sys
|
|
from tqdm import tqdm
|
|
import logs.Logger
|
|
from db.functions import settype, dbmongo
|
|
from datetime import datetime
|
|
import json
|
|
from db.schema_generator import MongoToPostgresSchemaGenerator, MongoDocumentInserter
|
|
import settings.config
|
|
|
|
def set_apienv(urls, uprocess, dbengine, dbExec, updatesBase, lprint):
|
|
"""Populate updates table from the API and return available updates.
|
|
|
|
Args:
|
|
urls: URL configuration with an "updatesurl" key.
|
|
uprocess: URL loader instance with `get_data`.
|
|
jget: JSON processor with `jconvert`.
|
|
dbengine: database engine dict (expects key "engine").
|
|
dbExec: database execution helper with `update_tvupdates` and `rawsql_select`.
|
|
updatesBase: list-like containing tables (uses index 8 for updates table).
|
|
lprint: logger instance with `logprint`.
|
|
|
|
Returns:
|
|
List of tuples (seriesid, timestamp) representing available updates.
|
|
"""
|
|
#updatetable = updatesBase[8]
|
|
#inputdata = uprocess.get_data(urls["updatesurl"])
|
|
#lprint.logprint("info", f"Retrieved {len(availableupdates)} rows for processing......")
|
|
#
|
|
newupdates = dbExec.rawsql_select(
|
|
dbengine["engine"],
|
|
"select seriesid,timestamp from updates.tvupdates",
|
|
lprint
|
|
)
|
|
return newupdates
|
|
|
|
|
|
ROOTDIR = os.getcwd()
|
|
config_options = settings.config.Config(ROOTDIR)
|
|
options = config_options.config_options
|
|
dbtype, apitype = options["dbtype"], options["apitype"]
|
|
dbs = settype(dbtype, apitype, options)
|
|
dbclass, dbengine = dbs.dbclass, dbs.dbengine
|
|
mongo_updater = dbmongo(options)
|
|
generator = MongoToPostgresSchemaGenerator(sample_size=100)
|
|
|
|
tableNames = ['seriesdata', 'episodesdata', 'actorsdata', 'charactersdata', 'crewdata']
|
|
|
|
for tableName in tableNames:
|
|
# Generate and print the schema for each collection
|
|
schema = generator.analyze_collection(mongo_updater, getattr(mongo_updater, f'mg{tableName[:-4]}'))
|
|
sql = generator.generate_create_table_sql(
|
|
tableName,
|
|
schema_name='updates',
|
|
pk_field='id' if tableName == 'seriesdata' else 'id',
|
|
)
|
|
print(f"--- SQL Schema for {tableName} ---")
|
|
print(sql)
|
|
print("\n")
|
|
|
|
# Or create it directly
|
|
generator.create_table_in_postgres(
|
|
dbengine,
|
|
tableName,
|
|
schema_name='updates',
|
|
#pk_field='seriesid' if tableName == 'seriesdata' else 'id',
|
|
pk_field='id',
|
|
drop_existing=True,
|
|
)
|
|
|
|
|
|
|
|
inserter = MongoDocumentInserter(batch_size=10000)
|
|
|
|
# Map table names to their MongoDB collections and primary keys
|
|
collection_map = {
|
|
'seriesdata': (mongo_updater.mgseries, 'id'),
|
|
'episodesdata': (mongo_updater.mgepisodes, 'id'),
|
|
'actorsdata': (mongo_updater.mgactors, 'id'),
|
|
'charactersdata': (mongo_updater.mgcharacters, 'id'),
|
|
'crewdata': (mongo_updater.mgcrew, 'id'),
|
|
}
|
|
|
|
# Insert documents from collections with UPSERT
|
|
for tableName in tableNames:
|
|
if tableName in collection_map:
|
|
collection, pk_field = collection_map[tableName]
|
|
# Build ON CONFLICT ... DO UPDATE clause
|
|
# Re-analyze the specific collection so we have the correct fields
|
|
schema = generator.analyze_collection(mongo_updater, collection)
|
|
# Use lowercased column names to match PostgreSQL unquoted identifiers
|
|
update_cols = [col.lower() for col in schema.keys() if col.lower() != pk_field.lower() and col != '_id']
|
|
update_clause = ', '.join([f"{col}=EXCLUDED.{col}" for col in update_cols])
|
|
on_conflict = f"ON CONFLICT ({pk_field}) DO UPDATE SET {update_clause}"
|
|
|
|
count = inserter.insert_from_collection(
|
|
engine=dbengine,
|
|
table_name=tableName,
|
|
collection=collection,
|
|
schema_name='updates',
|
|
on_conflict=on_conflict,
|
|
)
|
|
print(f"Inserted/Updated {count} documents in {tableName}")
|
|
else:
|
|
print(f"Warning: No collection mapping for {tableName}")
|
|
|
|
# # Or directly from MongoDB collection
|
|
# count = inserter.insert_from_collection(
|
|
# engine=dbengine,
|
|
# table_name='seriesdata',
|
|
# collection=mongo_db['series'],
|
|
# schema_name='dbo',
|
|
# on_conflict="DO NOTHING",
|
|
# )
|
|
# if __name__ == "__main__":
|
|
# sys.exit(main())
|
|
|
|
# Below are the relevant methods from db/functions.py and db/query_utils.py that are used in the above code. They are included here for completeness and context.
|
|
# Needs to be incorporated into logic so that only releveant documents are updated.
|
|
|
|
# from db.query_utils import MongoQueryUtils
|
|
|
|
# # With additional filters
|
|
# recent = MongoQueryUtils.get_recent_updates(
|
|
# collection=mongo_updater.mgseries,
|
|
# minutes=60,
|
|
# updated_field='updated',
|
|
# query_filter={'status': 'Ended'}, # Only ended shows
|
|
# )
|
|
|
|
# # Since a specific timestamp
|
|
# docs = MongoQueryUtils.get_updates_since(
|
|
# collection=mongo_updater.mgseries,
|
|
# since_timestamp=None, # Epoch or ISO string
|
|
# )
|
|
# for doc in docs:
|
|
# print(doc) |