Merge pull request 'development' (#4) from development into main

Reviewed-on: #4

.gitignore

@@ -5,11 +5,19 @@
 /.vs/new_dbsync/FileContentIndex
 /.vs
 *.pyc
-*.json
+/cache/series/*.json
+/cache/episodes/*.json
+/cache/cast/*.json
+/cache/crew/*.json
+/cache/guestcast/*.json
+/cache/guestcrew/*.json
+/cache/aliases/*.json
 *.DS_Store
 *.log
 /pyproject.toml
 dbsync.log.*
 .idea
 /log
-venv
+/venv
+/temp/post_tvmaze_updates.json.*
+/archived

Makefile

@@ -1,15 +0,0 @@
-CC = gcc
-CFLAGS = -fPIC -O2 -I/usr/include
-LDFLAGS = -shared
-LIBS = -lpq
-
-TARGET = pg_vtab.so
-SRC = pg_vtab.c
-
-all: $(TARGET)
-
-$(TARGET): $(SRC)
-	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)
-
-clean:
-	rm -f $(TARGET)

README_pg_vtab.md

@@ -1,38 +0,0 @@
-# SQLite -> PostgreSQL virtual table (minimal)
-
-This directory contains a minimal SQLite extension that implements a read-only virtual table backed directly by PostgreSQL using libpq.
-
-Build
------
-You need `libpq` (Postgres client) and a C compiler. On Debian/Ubuntu:
-
-```bash
-sudo apt-get install libpq-dev build-essential
-make
-```
-
-This produces `pg_vtab.so`.
-
-Usage
------
-Start `sqlite3`, load the extension and create a virtual table pointing at a Postgres table:
-
-```sql
-.load ./pg_vtab.so
-CREATE VIRTUAL TABLE remote_tbl USING pg_vtab('host=localhost dbname=mydb user=me password=secret', 'public.mytable');
-SELECT * FROM remote_tbl LIMIT 10;
-```
-
-Limitations
------------
-- Minimal proof-of-concept: supports only full table scans.
-- All columns are presented as `TEXT`.
-- No constraint pushdown or type mapping sophistication.
-- Error handling and SQL injection hygiene are minimal; do not use on untrusted inputs without review.
-
-Next steps (optional)
----------------------
-- Add support for WHERE constraints and indexed access (`xBestIndex`).
-- Map Postgres types more accurately to SQLite storage classes.
-- Implement server-side cursors for large tables.
-- Harden error reporting and resource cleanup.

db/Tables.py

@@ -250,7 +250,6 @@ class CreateTempTables():
             pass
         self.create(Base, engine, metadata, session, self.tablelist)
         session.commit()
-        #self.send(tablelist)
     
     def settables(self, metadata, engine):
         tablelist = []
@@ -379,12 +378,12 @@ class CreateTempTables():
                 schema = "updates",
                 extend_existing=True)
         tablelist.append(countries)
-        tvupdates = Table('tvupdates', metadata,
+        # tvupdates = Table('tvupdates', metadata,
-                Column('seriesid', BIGINT, primary_key = True),
+        #         Column('seriesid', BIGINT, primary_key = True),
-                Column('timestamp', VARCHAR(15)),
+        #         Column('timestamp', VARCHAR(15)),
-                schema = "updates",
+        #         schema = "updates",
-                extend_existing=True)
+        #         extend_existing=True)
-        tablelist.append(tvupdates)
+        # tablelist.append(tvupdates)
         return(tablelist)
     
     def create(self, Base, engine, metadata, session, tablelist):

db/sql.py

@@ -24,10 +24,14 @@ class Dbexec():
             )
         ]
 
+        # Commented out code for applying updates for now. Will re-enable later after downloads are verified.
         print('Applying updates to table.')
         with engine.connect() as conn:
             conn.execute(text(f"truncate table {tablename}"))
-            conn.execute(tablename.insert(), updatelist)
+            for key, value in available_updates.items():
+                seriesid = str(key)
+                timestamp = value
+                conn.execute(text(f"INSERT INTO {tablename} (seriesid, timestamp) VALUES (:seriesid, :timestamp)"), {'seriesid': seriesid, 'timestamp': timestamp})
             conn.commit()
         print('Update of tvupdates table is complete.')
 

jprocessor/process.py

@@ -1,105 +0,0 @@
-__author__ = 'Wendell Jones'
-import json
-import progressbar
-from url.process import Loadurl
-
-class Jsonload():
-    def __init__(self):
-        pass
-    
-    def load(self, rawdata):
-        jdata = {}
-        jdata = json.loads(rawdata)
-        newdata = self.dump(jdata)
-        return(newdata)
-    
-    def dump(self, rawdata):
-        jdata = json.dumps(rawdata, indent=4)
-        return(jdata)
-
-    def jconvert(self, inputdata):
-        outputdata = json.loads(inputdata)
-        return(outputdata)
-    
-    def populate_cache(self, jdata, jfile):
-        try:
-            with open(jfile, 'w') as datafile:
-                datafile.write(jdata)
-        except:
-            print('Unable to save cache file.')
-        else:
-            datafile.close()
-    
-    def fetch_json(self, weburl, seriesid, cachefile):
-        uprocess = Loadurl()
-        try:
-            webdata = uprocess.get_data(weburl)
-            jdata = self.load(webdata)
-        except Exception as e:
-            print('Unable to get JSON data for ' + str(seriesid) + ': ' + str(e))
-        else:
-            if jdata == '[]':
-                pass
-            else:
-                self.populate_cache(jdata, cachefile)
-
-    def processSeriesJson(self, seriesdata):
-        seriesresults = {}
-        networkchannel = {}
-        webchannel = {}
-        country = {}
-
-        seriesresults['id'] = seriesdata['id']
-        seriesresults['url'] = seriesdata['url']
-        seriesresults['name'] = seriesdata['name']
-        seriesresults['type'] = seriesdata['type']
-        seriesresults['language'] = seriesdata['language']
-        seriesresults['genres'] = ', '.join(seriesdata['genres'])
-        seriesresults['status'] = seriesdata['status']
-        seriesresults['runtime'] = seriesdata['runtime']
-        seriesresults['premiered'] = seriesdata['premiered']
-        seriesresults['officalsite'] = seriesdata['officialSite']
-        if len(seriesdata['schedule']['days']) > 0:
-            seriesresults['days'] = seriesdata['schedule']['days'][0]
-            seriesresults['time'] = seriesdata['schedule']['time']
-        seriesresults['rating'] = seriesdata['rating']['average']
-        seriesresults['weight'] = seriesdata['weight']
-        seriesresults['networkid'] = seriesdata['id']
-        seriesresults['webchannel'] = seriesdata['id']
-        seriesresults['tvrage'] = seriesdata['externals']['tvrage']
-        seriesresults['thetvdb'] = seriesdata['externals']['thetvdb']
-        seriesresults['imdb'] = seriesdata['externals']['imdb']
-        if seriesdata['image']:
-            seriesresults['medium'] = seriesdata['image']['medium']
-            seriesresults['original'] = seriesdata['image']['original']
-        seriesresults['summary'] = seriesdata['summary']
-        seriesresults['serieslink'] = seriesdata['_links']['self']['href']
-        if 'previousepisode' in seriesdata['_links'].keys():
-            seriesresults['previousepisode'] = seriesdata['_links']['previousepisode']['href']
-        if 'nextepisode' in seriesdata['_links'].keys():
-            seriesresults['nextepisode'] = seriesdata['_links']['nextepisode']['href']
-
-        
-        if seriesdata['network'] and seriesdata['network'] != None:
-            networkchannel['id'] = seriesdata['network']['id']
-            networkchannel['name'] = seriesdata['network']['name']
-            #networkchannel['countryid'] = country['id']
-            if seriesdata['network']['country']:
-            #    country['id'] = "auto incremented id"
-                country['name'] = seriesdata['network']['country']['name']
-                country['code'] = seriesdata['network']['country']['code']
-                country['timezone'] = seriesdata['network']['country']['timezone']
-        if seriesdata['webChannel'] and seriesdata['webChannel'] != None:
-            webchannel['id'] = seriesdata['webChannel']['id']
-            webchannel['name'] = seriesdata['webChannel']['name']
-            #webchannel['countryid'] = country['id']    
-            if seriesdata['webChannel']['country']:
-                #    country['id'] = "auto incremented id"
-                country['name'] = seriesdata['webchannel']['country']['name']
-                country['code'] = seriesdata['webchannel']['country']['code']
-                country['timezone'] = seriesdata['webchannel']['country']['timezone']
-        
-        return(seriesresults, networkchannel, country, webchannel)
-    
-    def __del__(self):
-        pass

logs/Logger.py

@@ -79,6 +79,23 @@ class LPrint:
         else:
             logging.error(f'Unknown log level: {loglevel}. Message: {logmessage}')
 
+    def screenprint(self, loglevel, logmessage):
+        """Log a message with a timestamp."""
+        logstamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+        log_message = f'{logstamp}: {logmessage}'
+        loglevel = loglevel.lower()
+        if loglevel == 'debug':
+            logging.debug(log_message)
+            print(f"{log_message}")
+        elif loglevel == 'warning':
+            logging.warning(log_message)
+            print(f"{log_message}")
+        elif loglevel == 'info':
+            logging.info(log_message)
+            print(f"{log_message}")
+        else:
+            logging.error(f'Unknown log level: {loglevel}. Message: {logmessage}')
+            
     def create_rotating_log(self, logpath, max_bytes=10_000_000, backup_count=5):
         """Create a rotating log handler."""
         logger = logging.getLogger("Rotating Log")

mongo2pgsql.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python3
+"""Convert a MongoDB collection into a PostgreSQL table.
+
+Contains `convert_mongo_to_pg(mongo_uri, mongo_db, collection_name, pg_dsn, pg_table=None, ...)`
+and a small CLI.
+
+Notes:
+- Uses `pymongo` to read documents and `psycopg2` to write to Postgres.
+- Infers basic column types from a sample of documents; nested objects/arrays
+  are stored as `JSONB`.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+from typing import Any, Dict, Iterable, List, Optional
+
+import pymongo
+import psycopg2
+import psycopg2.extras as pgextras
+import psycopg2.sql as sql
+
+LOG = logging.getLogger("mongo2pgsql")
+
+
+def _detect_type(value: Any) -> str:
+    if value is None:
+        return "null"
+    if isinstance(value, bool):
+        return "boolean"
+    if isinstance(value, int) and not isinstance(value, bool):
+        return "integer"
+    if isinstance(value, float):
+        return "real"
+    if isinstance(value, (dict, list)):
+        return "jsonb"
+    return "text"
+
+
+def _merge_types(types: List[str]) -> str:
+    # priority: jsonb > text > real > integer > boolean > null
+    s = set(types)
+    if "jsonb" in s:
+        return "jsonb"
+    if "text" in s:
+        return "text"
+    if "real" in s:
+        return "real"
+    if "integer" in s:
+        return "integer"
+    if "boolean" in s:
+        return "boolean"
+    return "text"
+
+
+def _pg_type_for(kind: str) -> str:
+    return {
+        "integer": "BIGINT",
+        "real": "DOUBLE PRECISION",
+        "boolean": "BOOLEAN",
+        "jsonb": "JSONB",
+        "text": "TEXT",
+    }.get(kind, "TEXT")
+
+
+def convert_mongo_to_pg(
+    mongo_uri: str,
+    mongo_db: str,
+    collection_name: str,
+    pg_dsn: str,
+    pg_table: Optional[str] = None,
+    sample_size: int = 500,
+    batch_size: int = 1000,
+    create_table: bool = True,
+    pk_field: str = "_id",
+) -> None:
+    """Copy a MongoDB collection to PostgreSQL.
+
+    - `pg_table` defaults to `collection_name` if omitted.
+    - Nested documents/arrays are stored as JSONB.
+    - `_id` is stored as TEXT by default.
+    """
+    if pg_table is None:
+        pg_table = collection_name
+
+    client = pymongo.MongoClient(mongo_uri)
+    coll = client[mongo_db][collection_name]
+
+    sample = list(coll.find({}, projection=None, limit=sample_size))
+    if not sample:
+        raise ValueError("collection is empty or not found")
+
+    # gather all field names and observed types
+    field_types: Dict[str, List[str]] = {}
+    for doc in sample:
+        for k, v in doc.items():
+            t = _detect_type(v)
+            field_types.setdefault(k, []).append(t)
+
+    # finalize types
+    final_types: Dict[str, str] = {}
+    for k, types in field_types.items():
+        merged = _merge_types(types)
+        final_types[k] = _pg_type_for(merged)
+
+    # ensure primary key present
+    if pk_field not in final_types:
+        final_types[pk_field] = "TEXT"
+
+    cols = list(final_types.items())
+
+    # create table
+    if create_table:
+        # Build CREATE TABLE statement using psycopg2.sql for safe identifiers
+        col_defs = []
+        for name, ptype in cols:
+            col_defs.append(sql.SQL("{} {}").format(sql.Identifier(name), sql.SQL(ptype)))
+
+        create_body = sql.SQL(", ").join(col_defs)
+        if pk_field in final_types:
+            pk_clause = sql.SQL(", PRIMARY KEY ({})").format(sql.SQL(",").join(map(sql.Identifier, [pk_field])))
+            create_body = create_body + pk_clause
+
+        create_stmt = sql.SQL("CREATE TABLE IF NOT EXISTS {} ({})").format(sql.Identifier(pg_table), create_body)
+        with psycopg2.connect(pg_dsn) as pgconn:
+            with pgconn.cursor() as cur:
+                LOG.info("Creating table %s", pg_table)
+                cur.execute(create_stmt)
+                pgconn.commit()
+
+    # stream documents and insert
+    insert_cols = [n for n, _ in cols]
+
+    with psycopg2.connect(pg_dsn) as pgconn:
+        with pgconn.cursor() as cur:
+            # build INSERT statement for execute_values: it expects a single %s
+            base_insert = sql.SQL("INSERT INTO {} ({}) VALUES %s").format(
+                sql.Identifier(pg_table),
+                sql.SQL(', ').join(map(sql.Identifier, insert_cols)),
+            ).as_string(pgconn)
+            batch = []
+            # Use an explicit session when using no_cursor_timeout to avoid pymongo warning
+            with client.start_session() as session:
+                cursor = coll.find({}, no_cursor_timeout=True, session=session).batch_size(batch_size)
+                try:
+                    for doc in cursor:
+                        row = []
+                        for col_name in insert_cols:
+                            v = doc.get(col_name)
+                            if v is None:
+                                row.append(None)
+                                continue
+                            # convert ObjectId and other non-JSON types
+                            if isinstance(v, (dict, list)):
+                                row.append(pgextras.Json(v))
+                            else:
+                                row.append(str(v) if col_name == pk_field else v)
+                        batch.append(tuple(row))
+                        if len(batch) >= batch_size:
+                            pgextras.execute_values(cur, base_insert, batch, template=None, page_size=batch_size)
+                            pgconn.commit()
+                            batch = []
+                finally:
+                    try:
+                        cursor.close()
+                    except Exception:
+                        pass
+            if batch:
+                pgextras.execute_values(cur, base_insert, batch, template=None, page_size=len(batch))
+                pgconn.commit()
+
+    LOG.info("Finished copying collection %s.%s to %s", mongo_db, collection_name, pg_table)
+
+
+def _cli() -> None:
+    parser = argparse.ArgumentParser(description="Copy a MongoDB collection to PostgreSQL")
+    parser.add_argument("mongo_uri", help="MongoDB connection URI, e.g. mongodb://user:pw@host:27017")
+    parser.add_argument("mongo_db", help="MongoDB database name")
+    parser.add_argument("collection", help="Collection name to copy")
+    parser.add_argument("pg_dsn", help="Postgres DSN, e.g. \"host=.. dbname=.. user=.. password=..\"")
+    parser.add_argument("--pg-table", help="Target Postgres table name (defaults to collection name)")
+    parser.add_argument("--sample", type=int, default=500, help="Sample size to infer schema")
+    parser.add_argument("--batch", type=int, default=1000, help="Insert batch size")
+    parser.add_argument("--no-create", dest="create", action="store_false", help="Don't attempt to create table in PG")
+    parser.add_argument("--verbose", "-v", action="store_true")
+    args = parser.parse_args()
+
+    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
+
+    convert_mongo_to_pg(
+        args.mongo_uri,
+        args.mongo_db,
+        args.collection,
+        args.pg_dsn,
+        pg_table=args.pg_table,
+        sample_size=args.sample,
+        batch_size=args.batch,
+        create_table=args.create,
+    )
+
+
+if __name__ == "__main__":
+    _cli()

pg_vtab.c

@@ -1,259 +0,0 @@
-/*
- * Minimal SQLite virtual table module backed by PostgreSQL via libpq.
- *
- * Usage (after building as shared object):
- *   sqlite3> .load ./pg_vtab.so
- *   sqlite> CREATE VIRTUAL TABLE t1 USING pg_vtab('host=... dbname=... user=... password=...', 'schema.table');
- *   sqlite> SELECT * FROM t1 LIMIT 10;
- *
- * Notes:
- * - This is a minimal read-only implementation.
- * - It only supports full table scans (no constraint pushdown).
- * - All values are returned as TEXT.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <assert.h>
-
-#include <libpq-fe.h>
-#include <sqlite3.h>
-
-/* vtab object */
-typedef struct PgVTab {
-    sqlite3_vtab base;    /* must be first */
-    char *conninfo;
-    char *schema_table;   /* qualified table name */
-    int ncol;
-    char **colnames;
-} PgVTab;
-
-/* cursor object */
-typedef struct PgVTabCursor {
-    sqlite3_vtab_cursor base; /* must be first */
-    PGconn *pgconn;
-    PGresult *res;
-    int row;                /* current row index in res */
-    sqlite3_int64 rowid;    /* generated rowid */
-} PgVTabCursor;
-
-/* forward declarations */
-static int pgConnect(sqlite3 *db, void *pAux, int argc, const char *const *argv, sqlite3_vtab **ppVTab, char **pzErr);
-static int pgDisconnect(sqlite3_vtab *pVTab);
-static int pgBestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pIdx);
-static int pgOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor);
-static int pgClose(sqlite3_vtab_cursor *cur);
-static int pgFilter(sqlite3_vtab_cursor *pVtabCursor, int idxNum, const char *idxStr, int argc, sqlite3_value **argv);
-static int pgNext(sqlite3_vtab_cursor *cur);
-static int pgEof(sqlite3_vtab_cursor *cur);
-static int pgColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i);
-static int pgRowid(sqlite3_vtab_cursor *cur, sqlite3_int64 *pRowid);
-
-static sqlite3_module PgModule = {
-    0,                       /* iVersion */
-    pgConnect,
-    pgConnect,               /* xCreate == xConnect for this module */
-    pgBestIndex,
-    pgDisconnect,
-    pgDisconnect,            /* xDestroy == xDisconnect */
-    pgOpen,
-    pgClose,
-    pgFilter,
-    pgNext,
-    pgEof,
-    pgColumn,
-    pgRowid,
-    NULL, NULL, NULL, NULL, NULL, NULL, NULL
-};
-
-/* helper: strdup-like */
-static char *str_dup(const char *s){
-    if(!s) return NULL;
-    size_t n = strlen(s)+1;
-    char *r = (char*)malloc(n);
-    if(r) memcpy(r,s,n);
-    return r;
-}
-
-/* read column names from POSTGRES by executing SELECT * FROM table LIMIT 0 */
-static int fetch_pg_columns(const char *conninfo, const char *schema_table, int *pncol, char ***pnames, char **pErr){
-    PGconn *conn = PQconnectdb(conninfo);
-    if (PQstatus(conn) != CONNECTION_OK) {
-        if(pErr) *pErr = str_dup(PQerrorMessage(conn));
-        PQfinish(conn);
-        return SQLITE_ERROR;
-    }
-    char q[1024];
-    snprintf(q, sizeof(q), "SELECT * FROM %s LIMIT 0", schema_table);
-    PGresult *res = PQexec(conn, q);
-    if (PQresultStatus(res) != PGRES_TUPLES_OK && PQresultStatus(res) != PGRES_COMMAND_OK) {
-        if(pErr) *pErr = str_dup(PQerrorMessage(conn));
-        PQclear(res);
-        PQfinish(conn);
-        return SQLITE_ERROR;
-    }
-    int n = PQnfields(res);
-    char **names = (char**)malloc(sizeof(char*)*n);
-    for(int i=0;i<n;i++){
-        names[i] = str_dup(PQfname(res,i));
-    }
-    PQclear(res);
-    PQfinish(conn);
-    *pncol = n;
-    *pnames = names;
-    return SQLITE_OK;
-}
-
-static int pgConnect(sqlite3 *db, void *pAux, int argc, const char *const *argv, sqlite3_vtab **ppVTab, char **pzErr){
-    /* argv: module name, db name, conninfo, schema.table */
-    if(argc < 4){
-        if(pzErr) *pzErr = str_dup("pg_vtab requires connection string and qualified table name");
-        return SQLITE_ERROR;
-    }
-    const char *conninfo = argv[2];
-    const char *schema_table = argv[3];
-
-    int ncol = 0;
-    char **names = NULL;
-    int rc = fetch_pg_columns(conninfo, schema_table, &ncol, &names, pzErr);
-    if(rc != SQLITE_OK) return rc;
-
-    /* build CREATE TABLE declaration with TEXT columns */
-    size_t needed = 64;
-    for(int i=0;i<ncol;i++) needed += strlen(names[i]) + 8;
-    char *decl = (char*)malloc(needed);
-    if(!decl){
-        if(pzErr) *pzErr = str_dup("out of memory");
-        return SQLITE_NOMEM;
-    }
-    strcpy(decl, "CREATE TABLE x(");
-    for(int i=0;i<ncol;i++){
-        if(i) strcat(decl, ",");
-        strcat(decl, names[i]);
-        strcat(decl, " TEXT");
-    }
-    strcat(decl, ")");
-
-    rc = sqlite3_declare_vtab(db, decl);
-    free(decl);
-    if(rc != SQLITE_OK){
-        if(pzErr) *pzErr = str_dup("sqlite3_declare_vtab failed");
-        for(int i=0;i<ncol;i++) free(names[i]); free(names);
-        return rc;
-    }
-
-    PgVTab *vtab = (PgVTab*)sqlite3_malloc(sizeof(PgVTab));
-    if(!vtab) return SQLITE_NOMEM;
-    memset(vtab,0,sizeof(PgVTab));
-    vtab->conninfo = str_dup(conninfo);
-    vtab->schema_table = str_dup(schema_table);
-    vtab->ncol = ncol;
-    vtab->colnames = names;
-
-    *ppVTab = (sqlite3_vtab*)vtab;
-    return SQLITE_OK;
-}
-
-static int pgDisconnect(sqlite3_vtab *pVTab){
-    PgVTab *v = (PgVTab*)pVTab;
-    if(!v) return SQLITE_OK;
-    if(v->conninfo) free(v->conninfo);
-    if(v->schema_table) free(v->schema_table);
-    for(int i=0;i<v->ncol;i++) free(v->colnames[i]);
-    if(v->colnames) free(v->colnames);
-    sqlite3_free(v);
-    return SQLITE_OK;
-}
-
-static int pgBestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pIdx){
-    /* do not support indexes or constraints; full table scan */
-    pIdx->estimatedCost = 1e6;
-    return SQLITE_OK;
-}
-
-static int pgOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
-    PgVTabCursor *cur = (PgVTabCursor*)sqlite3_malloc(sizeof(PgVTabCursor));
-    if(!cur) return SQLITE_NOMEM;
-    memset(cur,0,sizeof(PgVTabCursor));
-    *ppCursor = &cur->base;
-    cur->pgconn = NULL;
-    cur->res = NULL;
-    cur->row = 0;
-    cur->rowid = 0;
-    return SQLITE_OK;
-}
-
-static int pgClose(sqlite3_vtab_cursor *curp){
-    PgVTabCursor *cur = (PgVTabCursor*)curp;
-    if(cur->res) PQclear(cur->res);
-    if(cur->pgconn) PQfinish(cur->pgconn);
-    sqlite3_free(cur);
-    return SQLITE_OK;
-}
-
-static int pgFilter(sqlite3_vtab_cursor *pVtabCursor, int idxNum, const char *idxStr, int argc, sqlite3_value **argv){
-    PgVTabCursor *cur = (PgVTabCursor*)pVtabCursor;
-    PgVTab *vtab = (PgVTab*)(pVtabCursor->pVtab);
-    /* connect to pg */
-    if(cur->res){ PQclear(cur->res); cur->res = NULL; }
-    if(cur->pgconn){ PQfinish(cur->pgconn); cur->pgconn = NULL; }
-    cur->pgconn = PQconnectdb(vtab->conninfo);
-    if(PQstatus(cur->pgconn) != CONNECTION_OK){
-        /* can't report error via sqlite3_errmsg here, return error */
-        return SQLITE_ERROR;
-    }
-
-    /* simple select all */
-    char q[1024];
-    snprintf(q, sizeof(q), "SELECT * FROM %s", vtab->schema_table);
-    cur->res = PQexec(cur->pgconn, q);
-    if(PQresultStatus(cur->res) != PGRES_TUPLES_OK && PQresultStatus(cur->res) != PGRES_COMMAND_OK){
-        PQclear(cur->res); cur->res = NULL; return SQLITE_ERROR;
-    }
-    cur->row = 0;
-    cur->rowid = 0;
-    return SQLITE_OK;
-}
-
-static int pgNext(sqlite3_vtab_cursor *curp){
-    PgVTabCursor *cur = (PgVTabCursor*)curp;
-    cur->row++;
-    cur->rowid++;
-    return SQLITE_OK;
-}
-
-static int pgEof(sqlite3_vtab_cursor *curp){
-    PgVTabCursor *cur = (PgVTabCursor*)curp;
-    if(!cur->res) return 1;
-    int nt = PQntuples(cur->res);
-    return cur->row >= nt;
-}
-
-static int pgColumn(sqlite3_vtab_cursor *curp, sqlite3_context *ctx, int i){
-    PgVTabCursor *cur = (PgVTabCursor*)curp;
-    if(!cur->res) { sqlite3_result_null(ctx); return SQLITE_OK; }
-    if(PQgetisnull(cur->res, cur->row, i)){
-        sqlite3_result_null(ctx);
-    } else {
-        const char *val = PQgetvalue(cur->res, cur->row, i);
-        sqlite3_result_text(ctx, val, -1, SQLITE_TRANSIENT);
-    }
-    return SQLITE_OK;
-}
-
-static int pgRowid(sqlite3_vtab_cursor *curp, sqlite3_int64 *pRowid){
-    PgVTabCursor *cur = (PgVTabCursor*)curp;
-    *pRowid = cur->rowid;
-    return SQLITE_OK;
-}
-
-/* Entry point called by sqlite3 when loading extension */
-int sqlite3_pg_vtab_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi){
-    SQLITE_EXTENSION_INIT2(pApi)}
-    int rc = sqlite3_create_module(db, "pg_vtab", &PgModule, NULL);
-    if(rc != SQLITE_OK){
-        if(pzErrMsg) *pzErrMsg = str_dup("failed to register pg_vtab module");
-    }
-    return rc;
-}

requirements.txt

@@ -1,7 +1,7 @@
 logrotater==1.3
 numpy==2.2.3
 progressbar33==2.4
-psycopg2==2.9.10
+psycopg2-binary==2.9.10
 pymongo==4.11.2
 pymssql==2.3.2
 Requests==2.32.3

run_update_mongo.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+cd /srv/Projects/new_dbsync
+/srv/Projects/new_dbsync/update_json.py >> /home/wjones/update_mongo.log

search mongodb.py

@@ -0,0 +1,266 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import os, time, sys, datetime
+from db.functions import settype, dbmongo
+import json
+
+
+def set_apienv(urls, uprocess, dbengine, dbExec, updatesBase, lprint):
+    """Populate updates table from the API and return available updates.
+
+    Args:
+        urls: URL configuration with an "updatesurl" key.
+        uprocess: URL loader instance with `get_data`.
+        jget: JSON processor with `jconvert`.
+        dbengine: database engine dict (expects key "engine").
+        dbExec: database execution helper with `update_tvupdates` and `rawsql_select`.
+        updatesBase: list-like containing tables (uses index 8 for updates table).
+        lprint: logger instance with `logprint`.
+
+    Returns:
+        List of tuples (seriesid, timestamp) representing available updates.
+    """
+    #updatetable = updatesBase[8]
+    #inputdata = uprocess.get_data(urls["updatesurl"])
+    #lprint.logprint("info", f"Retrieved {len(availableupdates)} rows for processing......")
+    #
+    newupdates = dbExec.rawsql_select(
+        dbengine["engine"],
+        "select seriesid,timestamp from updates.tvupdates",
+        lprint
+    )
+    return newupdates
+
+
+def print_starttime(lprint, ts1):
+    lprint.logprint("info", f"Sync start time: {str(ts1)}")
+
+
+def find_file_in_multiple_dirs(filename, directories):
+    """
+    Checks if a file exists in any of the provided directories.
+
+    Args:
+        filename (str): The name of the file to search for.
+        directories (list): A list of directory paths to check.
+
+    Returns:
+        Path or None: The full path to the file if found, otherwise None.
+    """
+    directories_found = []
+    for directory in directories:
+        if os.path.isfile(f'{directory}{filename}'):
+            directories_found.append(directory)
+                
+    return directories_found
+        
+def load_json_to_mongodb(mongo_updater, directories, update_list, updateDict, lprint, track_changes=True):
+    """
+    Load downloaded JSON files into MongoDB collections with change detection.
+    
+    Processes series, episodes, cast, and crew data from cached JSON files
+    and inserts or updates them in the appropriate MongoDB collections.
+    Automatically detects if documents are new, updated, or unchanged by comparing
+    updateDict timestamps with existing document 'updated' fields.
+    
+    Args:
+        mongo_updater: dbmongo instance for MongoDB operations
+        directories: dict containing cache directory paths for different data types
+        update_list: list of series IDs to process
+        updateDict: dict with seriesid as key and timestamp as value
+        lprint: logger instance for logging operations
+        track_changes: bool, if True tracks inserted vs updated documents (default: True)
+    
+    Returns:
+        tuple: (successful_count, failed_count, stats_dict) for series processed
+               stats_dict contains: {'inserted': count, 'updated': count}
+    """
+    successful_count = 0
+    failed_count = 0
+    stats = {'series_inserted': 0, 'series_updated': 0, 'series_skipped': 0, 
+             'episodes_inserted': 0, 'episodes_updated': 0, 'episodes_skipped': 0, 
+             'cast_inserted': 0, 'cast_updated': 0, 'cast_skipped': 0,
+             'crew_inserted': 0, 'crew_updated': 0, 'crew_skipped': 0}
+    
+    lprint.logprint("info", f"Starting to load {len(update_list)} series into MongoDB")
+    
+    for seriesid in tqdm(update_list, desc="Loading series data into MongoDB", unit="series"):
+        try:
+            # Load series data
+            series_file = f"{directories['SERIESDIR']}{seriesid}.json"
+            if os.path.exists(series_file):
+                try:
+                    with open(series_file, 'r') as f:
+                        series_data = json.load(f)
+                    
+                    # Insert or update series in MongoDB
+                    if series_data:
+                        if track_changes:
+                            existing = mongo_updater.mgseries.find_one({"id": series_data.get("id")})
+                            seriesid_str = str(seriesid)
+                            update_timestamp = updateDict.get(seriesid_str)
+                            if existing:
+                                # Check if data has changed by comparing update timestamp
+                                if update_timestamp and int(update_timestamp) > existing.get("updated", 0):
+                                    mongo_updater.bulk_upsert_by_id(mongo_updater.mgseries, [series_data])
+                                    stats['series_updated'] += 1
+                                    lprint.logprint("debug", f"Updated series data for ID {seriesid}")
+                                else:
+                                    lprint.logprint("debug", f"Series {seriesid} unchanged, skipping")
+                                    stats['series_skipped'] += 1
+                                    pass  # Document unchanged
+                            else:
+                                mongo_updater.bulk_upsert_by_id(mongo_updater.mgseries, [series_data])
+                                stats['series_inserted'] += 1
+                                lprint.logprint("debug", f"Inserted new series data for ID {seriesid}")
+                                # Load episodes data
+                                episodes_file = f"{directories['EPISODEDIR']}{seriesid}.json"
+                                if os.path.exists(episodes_file):
+                                    try:
+                                        with open(episodes_file, 'r') as f:
+                                            episodes_data = json.load(f)
+                                        
+                                        if isinstance(episodes_data, list) and episodes_data:
+                                            # Add seriesid to each episode
+                                            for episode in episodes_data:
+                                                episode['seriesid'] = seriesid
+                                            
+                                            if track_changes:
+                                                for episode in episodes_data:
+                                                    existing = mongo_updater.mgepisodes.find_one({"id": episode.get("id")})
+                                                    seriesid_str = str(seriesid)
+                                                    update_timestamp = updateDict.get(seriesid_str)
+                                                    if existing:
+                                                        if update_timestamp and int(update_timestamp) > existing.get("updated", 0):
+                                                            mongo_updater.bulk_upsert_by_id(mongo_updater.mgepisodes, episodes_data)
+                                                            stats['episodes_updated'] += 1
+                                                        else: 
+                                                            lprint.logprint("debug", f"Episode Data for Series {seriesid} unchanged, skipping")
+                                                            pass  # Document unchanged
+                                                    else:
+                                                        mongo_updater.bulk_upsert_by_id(mongo_updater.mgepisodes, episodes_data)
+                                                        stats['episodes_inserted'] += 1
+                                            
+                                            #mongo_updater.bulk_upsert_by_id(mongo_updater.mgepisodes, episodes_data)
+                                            lprint.logprint("debug", f"Loaded {len(episodes_data)} episodes for series {seriesid}")
+                                    except Exception as e:
+                                        lprint.logprint("warning", f"Failed to load episodes file for {seriesid}: {e}")
+                                
+                                # Load cast data
+                                cast_file = f"{directories['CASTDIR']}{seriesid}.json"
+                                if os.path.exists(cast_file):
+                                    try:
+                                        with open(cast_file, 'r') as f:
+                                            cast_data = json.load(f)
+                                        
+                                        if isinstance(cast_data, list) and cast_data:
+                                            actors = []
+                                            characters = []
+                                            for item in cast_data:
+                                                if 'person' in item and item['person']:
+                                                    item['person']['seriesid'] = seriesid
+                                                    actors.append(item['person'])
+                                                if 'character' in item and item['character']:
+                                                    item['character']['seriesid'] = seriesid
+                                                    characters.append(item['character'])
+                                            
+                                            if track_changes:
+                                                for actor in actors:
+                                                    existing = mongo_updater.mgactors.find_one({"id": actor.get("id")})
+                                                    seriesid_str = str(seriesid)
+                                                    update_timestamp = updateDict.get(seriesid_str)
+                                                    if existing:
+                                                        if update_timestamp and int(update_timestamp) > existing.get("updated", 0):
+                                                            stats['cast_updated'] += 1
+                                                    else:
+                                                        stats['cast_inserted'] += 1
+                                            
+                                            if actors:
+                                                mongo_updater.bulk_upsert_by_id(mongo_updater.mgactors, actors)
+                                            if characters:
+                                                mongo_updater.bulk_upsert_by_id(mongo_updater.mgcharacters, characters)
+                                            lprint.logprint("debug", f"Loaded cast data for series {seriesid}")
+                                    except Exception as e:
+                                        lprint.logprint("warning", f"Failed to load cast file for {seriesid}: {e}")
+                                
+                                # Load crew data
+                                crew_file = f"{directories['CREWDIR']}{seriesid}.json"
+                                if os.path.exists(crew_file):
+                                    try:
+                                        with open(crew_file, 'r') as f:
+                                            crew_data = json.load(f)
+                                        
+                                        if isinstance(crew_data, list) and crew_data:
+                                            crew_list = []
+                                            for item in crew_data:
+                                                if 'person' in item and item['person']:
+                                                    item['person']['seriesid'] = seriesid
+                                                    item['person']['crew_type'] = item.get('type', 'unknown')
+                                                    crew_list.append(item['person'])
+                                            
+                                            if track_changes:
+                                                for crew in crew_list:
+                                                    existing = mongo_updater.mgcrew.find_one({"id": crew.get("id")})
+                                                    seriesid_str = str(seriesid)
+                                                    update_timestamp = updateDict.get(seriesid_str)
+                                                    if existing:
+                                                        if update_timestamp and int(update_timestamp) > existing.get("updated", 0):
+                                                            stats['crew_updated'] += 1
+                                                    else:
+                                                        stats['crew_inserted'] += 1
+                                            
+                                            if crew_list:
+                                                mongo_updater.bulk_upsert_by_id(mongo_updater.mgcrew, crew_list)
+                                                lprint.logprint("debug", f"Loaded crew data for series {seriesid}")
+                                    except Exception as e:
+                                        lprint.logprint("warning", f"Failed to load crew file for {seriesid}: {e}")
+                                
+                                successful_count += 1
+                                lprint.logprint("info", f"Successfully processed series {seriesid}")                        
+                        #mongo_updater.bulk_upsert_by_id(mongo_updater.mgseries, [series_data])
+                except Exception as e:
+                    lprint.logprint("warning", f"Failed to load series file for {seriesid}: {e}")
+                    failed_count += 1
+                    continue
+        except Exception as e:
+            failed_count += 1
+            lprint.logprint("error", f"Unexpected error processing series {seriesid}: {e}")
+    
+    lprint.logprint("info", f"MongoDB loading complete.")
+    return successful_count, failed_count, stats
+
+def main() -> int:
+    ROOTDIR = os.getcwd()
+    directories = {}
+    # Use cross-platform path builders and ensure directories exist
+    LOGDIR = os.path.join(ROOTDIR, "log")
+    directories["SERIESDIR"] = os.path.join(ROOTDIR, "cache", "series") + os.sep
+    directories["EPISODEDIR"] = os.path.join(ROOTDIR, "cache", "episodes") + os.sep
+    directories["CASTDIR"] = os.path.join(ROOTDIR, "cache", "cast") + os.sep
+    directories["GUESTCASTDIR"] = os.path.join(ROOTDIR, "cache", "guestcast") + os.sep
+    directories["CREWDIR"] = os.path.join(ROOTDIR, "cache", "crew") + os.sep
+    #directories["GUESTCREWDIR"] = os.path.join(ROOTDIR, "cache", "guestcrew") + os.sep    
+    #directories["CREDITSDIR"] = os.path.join(ROOTDIR, "cache", "credits") + os.sep
+    #directories["ALIASDIR"] = os.path.join(ROOTDIR, "cache", "aliases") + os.sep
+    tempdir = os.path.join(ROOTDIR, 'temp') + os.sep
+    
+    import settings.config
+
+    updateList = []
+    updateDict = {}
+    needed_updates = []
+    
+    config_options = settings.config.Config(ROOTDIR)
+    options = config_options.config_options
+    mongo_query = dbmongo(options)
+    
+    seriesid = 4
+    query = { "id": int(seriesid)}
+    for doc in mongo_query.mgseries.find(query):
+        print(doc)
+    query = { "seriesid": str(seriesid)}
+    for doc in mongo_query.mgepisodes.find(query):
+        print(doc)
+if __name__ == "__main__":
+    sys.exit(main())
+

settings/tvsync_settings.cfg

@@ -1,9 +1,9 @@
 [global]
-loglevel=debug
+loglevel=info
 
 [dbsettings]
 sqlserver_driver=ODBC Driver 13 for SQL Server
-hostname=192.168.128.7
+hostname=192.168.128.3
 mghostname=192.168.128.24
 dbname=media_dbsync
 mgdbname=tvdata

sqlite2pgsql

@@ -1,193 +0,0 @@
-#!/usr/bin/env python3
-"""Simple utility to copy a single table from SQLite to PostgreSQL.
-
-Usage example:
-    python sqlite2pgsql /path/to/sqlite.db "host=... dbname=... user=... password=..." mytable
-
-Implements `convert_table_sqlite_to_pg(sqlite_path, pg_dsn, table_name, create_table=True, batch_size=1000)`
-which copies schema (basic types) and data in batches.
-"""
-
-from __future__ import annotations
-
-import argparse
-import logging
-import sqlite3
-from typing import List, Tuple
-
-import psycopg2
-from psycopg2 import sql
-
-LOG = logging.getLogger("sqlite2pgsql")
-
-
-def _map_sqlite_type_to_pg(sqlite_type: str) -> str:
-    t = (sqlite_type or "").upper()
-    if "INT" in t:
-        return "INTEGER"
-    if any(x in t for x in ("CHAR", "CLOB", "TEXT")):
-        return "TEXT"
-    if "BLOB" in t:
-        return "BYTEA"
-    if any(x in t for x in ("REAL", "FLOA", "DOUB")):
-        return "REAL"
-    if "NUM" in t or "DEC" in t:
-        return "NUMERIC"
-    # default fallback
-    return "TEXT"
-
-
-def _get_table_info_sqlite(conn: sqlite3.Connection, table: str) -> List[Tuple]:
-    cur = conn.execute(f"PRAGMA table_info('{table}')")
-    rows = cur.fetchall()
-    return rows
-
-
-def list_tables_sqlite(sqlite_path: str) -> List[str]:
-    """Return a list of non-system table names from the SQLite database."""
-    conn = sqlite3.connect(sqlite_path)
-    try:
-        cur = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name;")
-        return [r[0] for r in cur.fetchall()]
-    finally:
-        conn.close()
-
-
-def convert_table_sqlite_to_pg(
-    sqlite_path: str,
-    pg_dsn: str,
-    table_name: str,
-    create_table: bool = True,
-    batch_size: int = 1000,
-) -> None:
-    """Copy a table from a SQLite file to PostgreSQL.
-
-    Arguments:
-        sqlite_path: path to the .db file
-        pg_dsn: libpq connection string for psycopg2
-        table_name: table to copy
-        create_table: if True, attempt to create the table in Postgres (basic mapping)
-        batch_size: number of rows per insert batch
-    """
-    LOG.info("Opening SQLite DB %s", sqlite_path)
-    sconn = sqlite3.connect(sqlite_path)
-    sconn.row_factory = sqlite3.Row
-
-    try:
-        info = _get_table_info_sqlite(sconn, table_name)
-        if not info:
-            raise ValueError(f"Table {table_name} not found in {sqlite_path}")
-
-        columns = []
-        pk_cols = []
-        for col in info:
-            # PRAGMA table_info: cid, name, type, notnull, dflt_value, pk
-            name = col[1]
-            ctype = col[2]
-            notnull = bool(col[3])
-            dflt = col[4]
-            pk = bool(col[5])
-            pgtype = _map_sqlite_type_to_pg(ctype)
-            columns.append((name, pgtype, notnull, dflt))
-            if pk:
-                pk_cols.append(name)
-
-        if create_table:
-            LOG.info("Connecting to Postgres to create table %s", table_name)
-            with psycopg2.connect(pg_dsn) as pgconn:
-                with pgconn.cursor() as cur:
-                    col_defs = []
-                    for name, pgtype, notnull, _dflt in columns:
-                        part = sql.SQL("{} {}{}") .format(
-                            sql.Identifier(name),
-                            sql.SQL(pgtype),
-                            sql.SQL(" NOT NULL") if notnull else sql.SQL(""),
-                        )
-                        col_defs.append(part)
-
-                    if pk_cols:
-                        pk_clause = sql.SQL(", PRIMARY KEY ({})").format(sql.SQL(",").join(map(sql.Identifier, pk_cols)))
-                        create_body = sql.SQL(", ").join(col_defs) + pk_clause
-                    else:
-                        create_body = sql.SQL(", ").join(col_defs)
-
-                    create_stmt = sql.SQL("CREATE TABLE IF NOT EXISTS {} ({})").format(
-                        sql.Identifier(table_name), create_body
-                    )
-                    cur.execute(create_stmt)
-                    pgconn.commit()
-
-        # Now copy data
-        LOG.info("Copying data for table %s", table_name)
-        s_cur = sconn.cursor()
-        s_cur.execute(f"SELECT * FROM '{table_name}'")
-
-        cols = [d[0] for d in s_cur.description]
-        placeholders = ",".join(["%s"] * len(cols))
-        insert_sql = sql.SQL("INSERT INTO {} ({}) VALUES ({})").format(
-            sql.Identifier(table_name),
-            sql.SQL(",").join(map(sql.Identifier, cols)),
-            sql.SQL(placeholders),
-        )
-
-        with psycopg2.connect(pg_dsn) as pgconn:
-            with pgconn.cursor() as pgcur:
-                batch = []
-                for row in s_cur:
-                    batch.append(tuple(row))
-                    if len(batch) >= batch_size:
-                        pgcur.executemany(insert_sql.as_string(pgconn), batch)
-                        pgconn.commit()
-                        batch = []
-                if batch:
-                    pgcur.executemany(insert_sql.as_string(pgconn), batch)
-                    pgconn.commit()
-
-    finally:
-        sconn.close()
-
-
-def _cli() -> None:
-    parser = argparse.ArgumentParser(description="Copy one table from SQLite to PostgreSQL")
-    parser.add_argument("sqlite_db", help="Path to SQLite database file")
-    parser.add_argument("pg_dsn", help="Postgres DSN, e.g. \"host=.. dbname=.. user=.. password=..\"")
-    parser.add_argument("table", nargs="?", help="Table name to copy (optional if --tables or --all used)")
-    parser.add_argument("--tables", help="Comma-separated list of tables to copy")
-    parser.add_argument("--all", action="store_true", help="Copy all non-system tables from the SQLite DB")
-    parser.add_argument("--no-create", dest="create", action="store_false", help="Don't attempt to create table in PG")
-    parser.add_argument("--list", dest="list_tables", action="store_true", help="List tables in the SQLite DB and exit")
-    parser.add_argument("--batch", type=int, default=1000, help="Batch size for inserts")
-    parser.add_argument("--verbose", "-v", action="store_true", help="Verbose logging")
-    args = parser.parse_args()
-
-    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
-    if getattr(args, "list_tables", False):
-        for t in list_tables_sqlite(args.sqlite_db):
-            print(t)
-        return
-
-    # Determine tables to process
-    table_names = []
-    if args.all:
-        table_names = list_tables_sqlite(args.sqlite_db)
-    elif args.tables:
-        table_names = [t.strip() for t in args.tables.split(",") if t.strip()]
-    elif args.table:
-        table_names = [args.table]
-    else:
-        parser.error("must specify a table, --tables, or --all unless --list is used")
-
-    if not args.pg_dsn:
-        parser.error("pg_dsn is required to copy tables to Postgres")
-
-    # Process tables sequentially
-    for tbl in table_names:
-        try:
-            LOG.info("Processing table %s", tbl)
-            convert_table_sqlite_to_pg(args.sqlite_db, args.pg_dsn, tbl, create_table=args.create, batch_size=args.batch)
-        except Exception as exc:
-            LOG.exception("Failed to process table %s: %s", tbl, exc)
-
-
-if __name__ == "__main__":
-    _cli()

sqlite_pg_plugin.py

@@ -1,154 +0,0 @@
-#!/usr/bin/env python3
-"""Utility to expose a PostgreSQL table to SQLite by copying it into a sqlite file.
-
-This is a pragmatic "plugin": it creates a local SQLite table (read-only by default)
-that mirrors a PostgreSQL table so SQLite queries can run locally against the data.
-
-Usage examples:
-  # copy single table into sqlite file
-  python sqlite_pg_plugin.py copy --pg "host=... dbname=... user=... password=..." public.mytable /tmp/my.db
-
-  # list tables in Postgres
-  python sqlite_pg_plugin.py list --pg "host=... dbname=... user=... password=..."
-
-This avoids needing to build a native virtual-table extension. For a direct virtual
-table (no copy) a C extension using SQLite virtual table API + libpq would be needed.
-"""
-
-from __future__ import annotations
-
-import argparse
-import logging
-import sqlite3
-from typing import List, Tuple
-
-import psycopg2
-from psycopg2 import sql
-
-LOG = logging.getLogger("sqlite_pg_plugin")
-
-
-def list_tables_pg(pg_dsn: str) -> List[str]:
-    q = "SELECT table_schema||'.'||table_name FROM information_schema.tables WHERE table_type='BASE TABLE' AND table_schema NOT IN ('pg_catalog','information_schema') ORDER BY 1;"
-    with psycopg2.connect(pg_dsn) as conn:
-        with conn.cursor() as cur:
-            cur.execute(q)
-            return [r[0] for r in cur.fetchall()]
-
-
-def fetch_table_schema_pg(pg_dsn: str, qualified_table: str) -> List[Tuple[str, str, bool]]:
-    """Return list of (colname, pgtype, notnull) for the given qualified table (schema.table)."""
-    if '.' in qualified_table:
-        schema, table = qualified_table.split('.', 1)
-    else:
-        schema, table = 'public', qualified_table
-
-    q = sql.SQL("SELECT column_name, data_type, is_nullable FROM information_schema.columns WHERE table_schema=%s AND table_name=%s ORDER BY ordinal_position;")
-    with psycopg2.connect(pg_dsn) as conn:
-        with conn.cursor() as cur:
-            cur.execute(q, (schema, table))
-            out = []
-            for name, dtype, is_nullable in cur.fetchall():
-                out.append((name, dtype, is_nullable == 'NO'))
-            return out
-
-
-def _map_pg_to_sqlite(pg_type: str) -> str:
-    t = (pg_type or '').lower()
-    if 'int' in t or t in ('bigserial', 'serial'):
-        return 'INTEGER'
-    if any(x in t for x in ('char', 'text', 'uuid')):
-        return 'TEXT'
-    if any(x in t for x in ('timestamp', 'date', 'time')):
-        return 'TEXT'
-    if any(x in t for x in ('numeric', 'decimal', 'real', 'double', 'float')):
-        return 'REAL'
-    if 'bool' in t:
-        return 'INTEGER'
-    return 'TEXT'
-
-
-def copy_table_pg_to_sqlite(pg_dsn: str, qualified_table: str, sqlite_path: str, sqlite_table: str | None = None, batch_size: int = 1000, overwrite: bool = True) -> None:
-    """Copy schema and data from Postgres table into a SQLite file.
-
-    - `qualified_table` can be 'schema.table' or 'table'
-    - `sqlite_table` is optional name to use in sqlite; defaults to table part
-    """
-    if '.' in qualified_table and not sqlite_table:
-        sqlite_table = qualified_table.split('.', 1)[1]
-    elif not sqlite_table:
-        sqlite_table = qualified_table
-
-    LOG.info('Fetching schema for %s', qualified_table)
-    cols = fetch_table_schema_pg(pg_dsn, qualified_table)
-    if not cols:
-        raise ValueError(f'No columns found for {qualified_table}')
-
-    # create sqlite table
-    sconn = sqlite3.connect(sqlite_path)
-    try:
-        s_cur = sconn.cursor()
-        if overwrite:
-            s_cur.execute(f"DROP TABLE IF EXISTS '{sqlite_table}'")
-
-        col_defs = []
-        for name, pgtype, notnull in cols:
-            stype = _map_pg_to_sqlite(pgtype)
-            col_defs.append(f"{name} {stype}{' NOT NULL' if notnull else ''}")
-
-        create_sql = f"CREATE TABLE IF NOT EXISTS '{sqlite_table}' ({', '.join(col_defs)})"
-        s_cur.execute(create_sql)
-        sconn.commit()
-
-        # stream rows from Postgres
-        with psycopg2.connect(pg_dsn) as pgconn:
-            with pgconn.cursor(name='stream_cursor') as pgcur:
-                pgcur.itersize = batch_size
-                pgcur.execute(sql.SQL('SELECT {} FROM {}').format(sql.SQL(',').join(sql.Identifier(c[0]) for c in cols), sql.Identifier(*qualified_table.split('.', 1)) if '.' in qualified_table else sql.Identifier(qualified_table)))
-
-                placeholders = ','.join('?' for _ in cols)
-                insert_sql = f"INSERT INTO '{sqlite_table}' ({', '.join(n for n,_,_ in cols)}) VALUES ({placeholders})"
-                batch = []
-                for row in pgcur:
-                    batch.append(tuple(row))
-                    if len(batch) >= batch_size:
-                        s_cur.executemany(insert_sql, batch)
-                        sconn.commit()
-                        batch = []
-                if batch:
-                    s_cur.executemany(insert_sql, batch)
-                    sconn.commit()
-    finally:
-        sconn.close()
-
-
-def _cli() -> None:
-    parser = argparse.ArgumentParser(description='Expose Postgres table(s) to SQLite by copying them')
-    sub = parser.add_subparsers(dest='cmd', required=True)
-
-    p_list = sub.add_parser('list', help='List tables in Postgres')
-    p_list.add_argument('--pg', required=True, help='Postgres DSN')
-
-    p_copy = sub.add_parser('copy', help='Copy a Postgres table into sqlite file')
-    p_copy.add_argument('--pg', required=True, help='Postgres DSN')
-    p_copy.add_argument('table', help='Qualified table name (schema.table or table)')
-    p_copy.add_argument('sqlite', help='Path to sqlite file to write to')
-    p_copy.add_argument('--target', help='Target sqlite table name (defaults to original table)')
-    p_copy.add_argument('--batch', type=int, default=1000)
-    p_copy.add_argument('--no-overwrite', dest='overwrite', action='store_false')
-    p_copy.add_argument('--verbose', '-v', action='store_true')
-
-    args = parser.parse_args()
-    logging.basicConfig(level=logging.DEBUG if getattr(args, 'verbose', False) else logging.INFO)
-
-    if args.cmd == 'list':
-        for t in list_tables_pg(args.pg):
-            print(t)
-        return
-
-    if args.cmd == 'copy':
-        copy_table_pg_to_sqlite(args.pg, args.table, args.sqlite, sqlite_table=args.target, batch_size=args.batch, overwrite=args.overwrite)
-
-
-if __name__ == '__main__':
-    _cli()

temp/skip_ids.txt

@@ -0,0 +1,986 @@
+137;http://api.tvmaze.com/people/137/castcredits
+145;http://api.tvmaze.com/people/145/castcredits
+137;http://api.tvmaze.com/people/137/castcredits
+145;http://api.tvmaze.com/people/145/castcredits
+211;http://api.tvmaze.com/people/211/castcredits
+214;http://api.tvmaze.com/people/214/castcredits
+315;http://api.tvmaze.com/people/315/castcredits
+368;http://api.tvmaze.com/people/368/castcredits
+416;http://api.tvmaze.com/people/416/castcredits
+473;http://api.tvmaze.com/people/473/castcredits
+557;http://api.tvmaze.com/people/557/castcredits
+577;http://api.tvmaze.com/people/577/castcredits
+583;http://api.tvmaze.com/people/583/castcredits
+635;http://api.tvmaze.com/people/635/castcredits
+688;http://api.tvmaze.com/people/688/castcredits
+743;http://api.tvmaze.com/people/743/castcredits
+771;http://api.tvmaze.com/people/771/castcredits
+811;http://api.tvmaze.com/people/811/castcredits
+817;http://api.tvmaze.com/people/817/castcredits
+823;http://api.tvmaze.com/people/823/castcredits
+864;http://api.tvmaze.com/people/864/castcredits
+1008;http://api.tvmaze.com/people/1008/castcredits
+1037;http://api.tvmaze.com/people/1037/castcredits
+1100;http://api.tvmaze.com/people/1100/castcredits
+1238;http://api.tvmaze.com/people/1238/castcredits
+1267;http://api.tvmaze.com/people/1267/castcredits
+1314;http://api.tvmaze.com/people/1314/castcredits
+1358;http://api.tvmaze.com/people/1358/castcredits
+1375;http://api.tvmaze.com/people/1375/castcredits
+1394;http://api.tvmaze.com/people/1394/castcredits
+1555;http://api.tvmaze.com/people/1555/castcredits
+1611;http://api.tvmaze.com/people/1611/castcredits
+1749;http://api.tvmaze.com/people/1749/castcredits
+1813;http://api.tvmaze.com/people/1813/castcredits
+2002;http://api.tvmaze.com/people/2002/castcredits
+2041;http://api.tvmaze.com/people/2041/castcredits
+2101;http://api.tvmaze.com/people/2101/castcredits
+2191;http://api.tvmaze.com/people/2191/castcredits
+2233;http://api.tvmaze.com/people/2233/castcredits
+2280;http://api.tvmaze.com/people/2280/castcredits
+2292;http://api.tvmaze.com/people/2292/castcredits
+2307;http://api.tvmaze.com/people/2307/castcredits
+2339;http://api.tvmaze.com/people/2339/castcredits
+2342;http://api.tvmaze.com/people/2342/castcredits
+2408;http://api.tvmaze.com/people/2408/castcredits
+2428;http://api.tvmaze.com/people/2428/castcredits
+2466;http://api.tvmaze.com/people/2466/castcredits
+2542;http://api.tvmaze.com/people/2542/castcredits
+2545;http://api.tvmaze.com/people/2545/castcredits
+2654;http://api.tvmaze.com/people/2654/castcredits
+2692;http://api.tvmaze.com/people/2692/castcredits
+2735;http://api.tvmaze.com/people/2735/castcredits
+2736;http://api.tvmaze.com/people/2736/castcredits
+2737;http://api.tvmaze.com/people/2737/castcredits
+2743;http://api.tvmaze.com/people/2743/castcredits
+2745;http://api.tvmaze.com/people/2745/castcredits
+2747;http://api.tvmaze.com/people/2747/castcredits
+2963;http://api.tvmaze.com/people/2963/castcredits
+2972;http://api.tvmaze.com/people/2972/castcredits
+3020;http://api.tvmaze.com/people/3020/castcredits
+3032;http://api.tvmaze.com/people/3032/castcredits
+3046;http://api.tvmaze.com/people/3046/castcredits
+3092;http://api.tvmaze.com/people/3092/castcredits
+3261;http://api.tvmaze.com/people/3261/castcredits
+3302;http://api.tvmaze.com/people/3302/castcredits
+3346;http://api.tvmaze.com/people/3346/castcredits
+3428;http://api.tvmaze.com/people/3428/castcredits
+3454;http://api.tvmaze.com/people/3454/castcredits
+3464;http://api.tvmaze.com/people/3464/castcredits
+3505;http://api.tvmaze.com/people/3505/castcredits
+3581;http://api.tvmaze.com/people/3581/castcredits
+3777;http://api.tvmaze.com/people/3777/castcredits
+3841;http://api.tvmaze.com/people/3841/castcredits
+3878;http://api.tvmaze.com/people/3878/castcredits
+3909;http://api.tvmaze.com/people/3909/castcredits
+3942;http://api.tvmaze.com/people/3942/castcredits
+3944;http://api.tvmaze.com/people/3944/castcredits
+3946;http://api.tvmaze.com/people/3946/castcredits
+3948;http://api.tvmaze.com/people/3948/castcredits
+3950;http://api.tvmaze.com/people/3950/castcredits
+3952;http://api.tvmaze.com/people/3952/castcredits
+3954;http://api.tvmaze.com/people/3954/castcredits
+3965;http://api.tvmaze.com/people/3965/castcredits
+4046;http://api.tvmaze.com/people/4046/castcredits
+4193;http://api.tvmaze.com/people/4193/castcredits
+4261;http://api.tvmaze.com/people/4261/castcredits
+4276;http://api.tvmaze.com/people/4276/castcredits
+4379;http://api.tvmaze.com/people/4379/castcredits
+4395;http://api.tvmaze.com/people/4395/castcredits
+4406;http://api.tvmaze.com/people/4406/castcredits
+4445;http://api.tvmaze.com/people/4445/castcredits
+4492;http://api.tvmaze.com/people/4492/castcredits
+4548;http://api.tvmaze.com/people/4548/castcredits
+4585;http://api.tvmaze.com/people/4585/castcredits
+4601;http://api.tvmaze.com/people/4601/castcredits
+4677;http://api.tvmaze.com/people/4677/castcredits
+4820;http://api.tvmaze.com/people/4820/castcredits
+4841;http://api.tvmaze.com/people/4841/castcredits
+4851;http://api.tvmaze.com/people/4851/castcredits
+4908;http://api.tvmaze.com/people/4908/castcredits
+4911;http://api.tvmaze.com/people/4911/castcredits
+4941;http://api.tvmaze.com/people/4941/castcredits
+4983;http://api.tvmaze.com/people/4983/castcredits
+5010;http://api.tvmaze.com/people/5010/castcredits
+5013;http://api.tvmaze.com/people/5013/castcredits
+5019;http://api.tvmaze.com/people/5019/castcredits
+5051;http://api.tvmaze.com/people/5051/castcredits
+5066;http://api.tvmaze.com/people/5066/castcredits
+5198;http://api.tvmaze.com/people/5198/castcredits
+5205;http://api.tvmaze.com/people/5205/castcredits
+5208;http://api.tvmaze.com/people/5208/castcredits
+5209;http://api.tvmaze.com/people/5209/castcredits
+5210;http://api.tvmaze.com/people/5210/castcredits
+5211;http://api.tvmaze.com/people/5211/castcredits
+5212;http://api.tvmaze.com/people/5212/castcredits
+5233;http://api.tvmaze.com/people/5233/castcredits
+5277;http://api.tvmaze.com/people/5277/castcredits
+5425;http://api.tvmaze.com/people/5425/castcredits
+5450;http://api.tvmaze.com/people/5450/castcredits
+5584;http://api.tvmaze.com/people/5584/castcredits
+5615;http://api.tvmaze.com/people/5615/castcredits
+5622;http://api.tvmaze.com/people/5622/castcredits
+5758;http://api.tvmaze.com/people/5758/castcredits
+5760;http://api.tvmaze.com/people/5760/castcredits
+5777;http://api.tvmaze.com/people/5777/castcredits
+5885;http://api.tvmaze.com/people/5885/castcredits
+5934;http://api.tvmaze.com/people/5934/castcredits
+5936;http://api.tvmaze.com/people/5936/castcredits
+5950;http://api.tvmaze.com/people/5950/castcredits
+5952;http://api.tvmaze.com/people/5952/castcredits
+5954;http://api.tvmaze.com/people/5954/castcredits
+5992;http://api.tvmaze.com/people/5992/castcredits
+5994;http://api.tvmaze.com/people/5994/castcredits
+5996;http://api.tvmaze.com/people/5996/castcredits
+5998;http://api.tvmaze.com/people/5998/castcredits
+6000;http://api.tvmaze.com/people/6000/castcredits
+6002;http://api.tvmaze.com/people/6002/castcredits
+6004;http://api.tvmaze.com/people/6004/castcredits
+6006;http://api.tvmaze.com/people/6006/castcredits
+6008;http://api.tvmaze.com/people/6008/castcredits
+6010;http://api.tvmaze.com/people/6010/castcredits
+6012;http://api.tvmaze.com/people/6012/castcredits
+6014;http://api.tvmaze.com/people/6014/castcredits
+6016;http://api.tvmaze.com/people/6016/castcredits
+6018;http://api.tvmaze.com/people/6018/castcredits
+6071;http://api.tvmaze.com/people/6071/castcredits
+6130;http://api.tvmaze.com/people/6130/castcredits
+6158;http://api.tvmaze.com/people/6158/castcredits
+6185;http://api.tvmaze.com/people/6185/castcredits
+6197;http://api.tvmaze.com/people/6197/castcredits
+6199;http://api.tvmaze.com/people/6199/castcredits
+6201;http://api.tvmaze.com/people/6201/castcredits
+6203;http://api.tvmaze.com/people/6203/castcredits
+6205;http://api.tvmaze.com/people/6205/castcredits
+6207;http://api.tvmaze.com/people/6207/castcredits
+6463;http://api.tvmaze.com/people/6463/castcredits
+6471;http://api.tvmaze.com/people/6471/castcredits
+6642;http://api.tvmaze.com/people/6642/castcredits
+6693;http://api.tvmaze.com/people/6693/castcredits
+6719;http://api.tvmaze.com/people/6719/castcredits
+6741;http://api.tvmaze.com/people/6741/castcredits
+6801;http://api.tvmaze.com/people/6801/castcredits
+6943;http://api.tvmaze.com/people/6943/castcredits
+7063;http://api.tvmaze.com/people/7063/castcredits
+7251;http://api.tvmaze.com/people/7251/castcredits
+7282;http://api.tvmaze.com/people/7282/castcredits
+7320;http://api.tvmaze.com/people/7320/castcredits
+7574;http://api.tvmaze.com/people/7574/castcredits
+7591;http://api.tvmaze.com/people/7591/castcredits
+7686;http://api.tvmaze.com/people/7686/castcredits
+7730;http://api.tvmaze.com/people/7730/castcredits
+7740;http://api.tvmaze.com/people/7740/castcredits
+7815;http://api.tvmaze.com/people/7815/castcredits
+7834;http://api.tvmaze.com/people/7834/castcredits
+7848;http://api.tvmaze.com/people/7848/castcredits
+7876;http://api.tvmaze.com/people/7876/castcredits
+7950;http://api.tvmaze.com/people/7950/castcredits
+8071;http://api.tvmaze.com/people/8071/castcredits
+8081;http://api.tvmaze.com/people/8081/castcredits
+8089;http://api.tvmaze.com/people/8089/castcredits
+8171;http://api.tvmaze.com/people/8171/castcredits
+8218;http://api.tvmaze.com/people/8218/castcredits
+8382;http://api.tvmaze.com/people/8382/castcredits
+8450;http://api.tvmaze.com/people/8450/castcredits
+8497;http://api.tvmaze.com/people/8497/castcredits
+8524;http://api.tvmaze.com/people/8524/castcredits
+8525;http://api.tvmaze.com/people/8525/castcredits
+8527;http://api.tvmaze.com/people/8527/castcredits
+8560;http://api.tvmaze.com/people/8560/castcredits
+8574;http://api.tvmaze.com/people/8574/castcredits
+8581;http://api.tvmaze.com/people/8581/castcredits
+8585;http://api.tvmaze.com/people/8585/castcredits
+8587;http://api.tvmaze.com/people/8587/castcredits
+8589;http://api.tvmaze.com/people/8589/castcredits
+8590;http://api.tvmaze.com/people/8590/castcredits
+8591;http://api.tvmaze.com/people/8591/castcredits
+8593;http://api.tvmaze.com/people/8593/castcredits
+8692;http://api.tvmaze.com/people/8692/castcredits
+8747;http://api.tvmaze.com/people/8747/castcredits
+8752;http://api.tvmaze.com/people/8752/castcredits
+8789;http://api.tvmaze.com/people/8789/castcredits
+8848;http://api.tvmaze.com/people/8848/castcredits
+9010;http://api.tvmaze.com/people/9010/castcredits
+9037;http://api.tvmaze.com/people/9037/castcredits
+9162;http://api.tvmaze.com/people/9162/castcredits
+9212;http://api.tvmaze.com/people/9212/castcredits
+9217;http://api.tvmaze.com/people/9217/castcredits
+9219;http://api.tvmaze.com/people/9219/castcredits
+9221;http://api.tvmaze.com/people/9221/castcredits
+9223;http://api.tvmaze.com/people/9223/castcredits
+9225;http://api.tvmaze.com/people/9225/castcredits
+9237;http://api.tvmaze.com/people/9237/castcredits
+9286;http://api.tvmaze.com/people/9286/castcredits
+9317;http://api.tvmaze.com/people/9317/castcredits
+9320;http://api.tvmaze.com/people/9320/castcredits
+9321;http://api.tvmaze.com/people/9321/castcredits
+9322;http://api.tvmaze.com/people/9322/castcredits
+9323;http://api.tvmaze.com/people/9323/castcredits
+9324;http://api.tvmaze.com/people/9324/castcredits
+9325;http://api.tvmaze.com/people/9325/castcredits
+9326;http://api.tvmaze.com/people/9326/castcredits
+9327;http://api.tvmaze.com/people/9327/castcredits
+9328;http://api.tvmaze.com/people/9328/castcredits
+9329;http://api.tvmaze.com/people/9329/castcredits
+9330;http://api.tvmaze.com/people/9330/castcredits
+9331;http://api.tvmaze.com/people/9331/castcredits
+9332;http://api.tvmaze.com/people/9332/castcredits
+9334;http://api.tvmaze.com/people/9334/castcredits
+9335;http://api.tvmaze.com/people/9335/castcredits
+9336;http://api.tvmaze.com/people/9336/castcredits
+9355;http://api.tvmaze.com/people/9355/castcredits
+9666;http://api.tvmaze.com/people/9666/castcredits
+9754;http://api.tvmaze.com/people/9754/castcredits
+9772;http://api.tvmaze.com/people/9772/castcredits
+9801;http://api.tvmaze.com/people/9801/castcredits
+9851;http://api.tvmaze.com/people/9851/castcredits
+9891;http://api.tvmaze.com/people/9891/castcredits
+9913;http://api.tvmaze.com/people/9913/castcredits
+9929;http://api.tvmaze.com/people/9929/castcredits
+9931;http://api.tvmaze.com/people/9931/castcredits
+9933;http://api.tvmaze.com/people/9933/castcredits
+9935;http://api.tvmaze.com/people/9935/castcredits
+9937;http://api.tvmaze.com/people/9937/castcredits
+9939;http://api.tvmaze.com/people/9939/castcredits
+9948;http://api.tvmaze.com/people/9948/castcredits
+9971;http://api.tvmaze.com/people/9971/castcredits
+10118;http://api.tvmaze.com/people/10118/castcredits
+10119;http://api.tvmaze.com/people/10119/castcredits
+10120;http://api.tvmaze.com/people/10120/castcredits
+10121;http://api.tvmaze.com/people/10121/castcredits
+10122;http://api.tvmaze.com/people/10122/castcredits
+10123;http://api.tvmaze.com/people/10123/castcredits
+10124;http://api.tvmaze.com/people/10124/castcredits
+10125;http://api.tvmaze.com/people/10125/castcredits
+10126;http://api.tvmaze.com/people/10126/castcredits
+10127;http://api.tvmaze.com/people/10127/castcredits
+137;http://api.tvmaze.com/people/137/castcredits
+145;http://api.tvmaze.com/people/145/castcredits
+211;http://api.tvmaze.com/people/211/castcredits
+214;http://api.tvmaze.com/people/214/castcredits
+315;http://api.tvmaze.com/people/315/castcredits
+368;http://api.tvmaze.com/people/368/castcredits
+416;http://api.tvmaze.com/people/416/castcredits
+473;http://api.tvmaze.com/people/473/castcredits
+557;http://api.tvmaze.com/people/557/castcredits
+577;http://api.tvmaze.com/people/577/castcredits
+583;http://api.tvmaze.com/people/583/castcredits
+635;http://api.tvmaze.com/people/635/castcredits
+688;http://api.tvmaze.com/people/688/castcredits
+743;http://api.tvmaze.com/people/743/castcredits
+771;http://api.tvmaze.com/people/771/castcredits
+10148;http://api.tvmaze.com/people/10148/castcredits
+10149;http://api.tvmaze.com/people/10149/castcredits
+10162;http://api.tvmaze.com/people/10162/castcredits
+10177;http://api.tvmaze.com/people/10177/castcredits
+10213;http://api.tvmaze.com/people/10213/castcredits
+10271;http://api.tvmaze.com/people/10271/castcredits
+10283;http://api.tvmaze.com/people/10283/castcredits
+10309;http://api.tvmaze.com/people/10309/castcredits
+10333;http://api.tvmaze.com/people/10333/castcredits
+10355;http://api.tvmaze.com/people/10355/castcredits
+10383;http://api.tvmaze.com/people/10383/castcredits
+10390;http://api.tvmaze.com/people/10390/castcredits
+10455;http://api.tvmaze.com/people/10455/castcredits
+10462;http://api.tvmaze.com/people/10462/castcredits
+10477;http://api.tvmaze.com/people/10477/castcredits
+10513;http://api.tvmaze.com/people/10513/castcredits
+10529;http://api.tvmaze.com/people/10529/castcredits
+10532;http://api.tvmaze.com/people/10532/castcredits
+10534;http://api.tvmaze.com/people/10534/castcredits
+10616;http://api.tvmaze.com/people/10616/castcredits
+10647;http://api.tvmaze.com/people/10647/castcredits
+10659;http://api.tvmaze.com/people/10659/castcredits
+10720;http://api.tvmaze.com/people/10720/castcredits
+10862;http://api.tvmaze.com/people/10862/castcredits
+10864;http://api.tvmaze.com/people/10864/castcredits
+10865;http://api.tvmaze.com/people/10865/castcredits
+10866;http://api.tvmaze.com/people/10866/castcredits
+10867;http://api.tvmaze.com/people/10867/castcredits
+10868;http://api.tvmaze.com/people/10868/castcredits
+10970;http://api.tvmaze.com/people/10970/castcredits
+11039;http://api.tvmaze.com/people/11039/castcredits
+11047;http://api.tvmaze.com/people/11047/castcredits
+11079;http://api.tvmaze.com/people/11079/castcredits
+11150;http://api.tvmaze.com/people/11150/castcredits
+11178;http://api.tvmaze.com/people/11178/castcredits
+11312;http://api.tvmaze.com/people/11312/castcredits
+11402;http://api.tvmaze.com/people/11402/castcredits
+11453;http://api.tvmaze.com/people/11453/castcredits
+11457;http://api.tvmaze.com/people/11457/castcredits
+11493;http://api.tvmaze.com/people/11493/castcredits
+11546;http://api.tvmaze.com/people/11546/castcredits
+11547;http://api.tvmaze.com/people/11547/castcredits
+11548;http://api.tvmaze.com/people/11548/castcredits
+11549;http://api.tvmaze.com/people/11549/castcredits
+11550;http://api.tvmaze.com/people/11550/castcredits
+11551;http://api.tvmaze.com/people/11551/castcredits
+11593;http://api.tvmaze.com/people/11593/castcredits
+11621;http://api.tvmaze.com/people/11621/castcredits
+11759;http://api.tvmaze.com/people/11759/castcredits
+11790;http://api.tvmaze.com/people/11790/castcredits
+11952;http://api.tvmaze.com/people/11952/castcredits
+11971;http://api.tvmaze.com/people/11971/castcredits
+11989;http://api.tvmaze.com/people/11989/castcredits
+12043;http://api.tvmaze.com/people/12043/castcredits
+12066;http://api.tvmaze.com/people/12066/castcredits
+12096;http://api.tvmaze.com/people/12096/castcredits
+12097;http://api.tvmaze.com/people/12097/castcredits
+12098;http://api.tvmaze.com/people/12098/castcredits
+12099;http://api.tvmaze.com/people/12099/castcredits
+12100;http://api.tvmaze.com/people/12100/castcredits
+12101;http://api.tvmaze.com/people/12101/castcredits
+12102;http://api.tvmaze.com/people/12102/castcredits
+12141;http://api.tvmaze.com/people/12141/castcredits
+12146;http://api.tvmaze.com/people/12146/castcredits
+12147;http://api.tvmaze.com/people/12147/castcredits
+12148;http://api.tvmaze.com/people/12148/castcredits
+12149;http://api.tvmaze.com/people/12149/castcredits
+12150;http://api.tvmaze.com/people/12150/castcredits
+12201;http://api.tvmaze.com/people/12201/castcredits
+12379;http://api.tvmaze.com/people/12379/castcredits
+12445;http://api.tvmaze.com/people/12445/castcredits
+12575;http://api.tvmaze.com/people/12575/castcredits
+12650;http://api.tvmaze.com/people/12650/castcredits
+12666;http://api.tvmaze.com/people/12666/castcredits
+12842;http://api.tvmaze.com/people/12842/castcredits
+12872;http://api.tvmaze.com/people/12872/castcredits
+12929;http://api.tvmaze.com/people/12929/castcredits
+13169;http://api.tvmaze.com/people/13169/castcredits
+13232;http://api.tvmaze.com/people/13232/castcredits
+13262;http://api.tvmaze.com/people/13262/castcredits
+13277;http://api.tvmaze.com/people/13277/castcredits
+13509;http://api.tvmaze.com/people/13509/castcredits
+13513;http://api.tvmaze.com/people/13513/castcredits
+13516;http://api.tvmaze.com/people/13516/castcredits
+13518;http://api.tvmaze.com/people/13518/castcredits
+13519;http://api.tvmaze.com/people/13519/castcredits
+13520;http://api.tvmaze.com/people/13520/castcredits
+13679;http://api.tvmaze.com/people/13679/castcredits
+13760;http://api.tvmaze.com/people/13760/castcredits
+13774;http://api.tvmaze.com/people/13774/castcredits
+13922;http://api.tvmaze.com/people/13922/castcredits
+13957;http://api.tvmaze.com/people/13957/castcredits
+13968;http://api.tvmaze.com/people/13968/castcredits
+13997;http://api.tvmaze.com/people/13997/castcredits
+14057;http://api.tvmaze.com/people/14057/castcredits
+14099;http://api.tvmaze.com/people/14099/castcredits
+14132;http://api.tvmaze.com/people/14132/castcredits
+14159;http://api.tvmaze.com/people/14159/castcredits
+14162;http://api.tvmaze.com/people/14162/castcredits
+14307;http://api.tvmaze.com/people/14307/castcredits
+14309;http://api.tvmaze.com/people/14309/castcredits
+14310;http://api.tvmaze.com/people/14310/castcredits
+14397;http://api.tvmaze.com/people/14397/castcredits
+14436;http://api.tvmaze.com/people/14436/castcredits
+14472;http://api.tvmaze.com/people/14472/castcredits
+14501;http://api.tvmaze.com/people/14501/castcredits
+14525;http://api.tvmaze.com/people/14525/castcredits
+14541;http://api.tvmaze.com/people/14541/castcredits
+14566;http://api.tvmaze.com/people/14566/castcredits
+14572;http://api.tvmaze.com/people/14572/castcredits
+14625;http://api.tvmaze.com/people/14625/castcredits
+14627;http://api.tvmaze.com/people/14627/castcredits
+14671;http://api.tvmaze.com/people/14671/castcredits
+14672;http://api.tvmaze.com/people/14672/castcredits
+14673;http://api.tvmaze.com/people/14673/castcredits
+14674;http://api.tvmaze.com/people/14674/castcredits
+14853;http://api.tvmaze.com/people/14853/castcredits
+14864;http://api.tvmaze.com/people/14864/castcredits
+14872;http://api.tvmaze.com/people/14872/castcredits
+14960;http://api.tvmaze.com/people/14960/castcredits
+14977;http://api.tvmaze.com/people/14977/castcredits
+14996;http://api.tvmaze.com/people/14996/castcredits
+15000;http://api.tvmaze.com/people/15000/castcredits
+15021;http://api.tvmaze.com/people/15021/castcredits
+15028;http://api.tvmaze.com/people/15028/castcredits
+15029;http://api.tvmaze.com/people/15029/castcredits
+15032;http://api.tvmaze.com/people/15032/castcredits
+15036;http://api.tvmaze.com/people/15036/castcredits
+15043;http://api.tvmaze.com/people/15043/castcredits
+15048;http://api.tvmaze.com/people/15048/castcredits
+15074;http://api.tvmaze.com/people/15074/castcredits
+15098;http://api.tvmaze.com/people/15098/castcredits
+15108;http://api.tvmaze.com/people/15108/castcredits
+15114;http://api.tvmaze.com/people/15114/castcredits
+15128;http://api.tvmaze.com/people/15128/castcredits
+15147;http://api.tvmaze.com/people/15147/castcredits
+15150;http://api.tvmaze.com/people/15150/castcredits
+15172;http://api.tvmaze.com/people/15172/castcredits
+15188;http://api.tvmaze.com/people/15188/castcredits
+15189;http://api.tvmaze.com/people/15189/castcredits
+15196;http://api.tvmaze.com/people/15196/castcredits
+15219;http://api.tvmaze.com/people/15219/castcredits
+15220;http://api.tvmaze.com/people/15220/castcredits
+15234;http://api.tvmaze.com/people/15234/castcredits
+15257;http://api.tvmaze.com/people/15257/castcredits
+15270;http://api.tvmaze.com/people/15270/castcredits
+15276;http://api.tvmaze.com/people/15276/castcredits
+15279;http://api.tvmaze.com/people/15279/castcredits
+15281;http://api.tvmaze.com/people/15281/castcredits
+15285;http://api.tvmaze.com/people/15285/castcredits
+15292;http://api.tvmaze.com/people/15292/castcredits
+15295;http://api.tvmaze.com/people/15295/castcredits
+15315;http://api.tvmaze.com/people/15315/castcredits
+15344;http://api.tvmaze.com/people/15344/castcredits
+15369;http://api.tvmaze.com/people/15369/castcredits
+15371;http://api.tvmaze.com/people/15371/castcredits
+15426;http://api.tvmaze.com/people/15426/castcredits
+15443;http://api.tvmaze.com/people/15443/castcredits
+15448;http://api.tvmaze.com/people/15448/castcredits
+15449;http://api.tvmaze.com/people/15449/castcredits
+15450;http://api.tvmaze.com/people/15450/castcredits
+15451;http://api.tvmaze.com/people/15451/castcredits
+15454;http://api.tvmaze.com/people/15454/castcredits
+15473;http://api.tvmaze.com/people/15473/castcredits
+15474;http://api.tvmaze.com/people/15474/castcredits
+15496;http://api.tvmaze.com/people/15496/castcredits
+15509;http://api.tvmaze.com/people/15509/castcredits
+15520;http://api.tvmaze.com/people/15520/castcredits
+15534;http://api.tvmaze.com/people/15534/castcredits
+15535;http://api.tvmaze.com/people/15535/castcredits
+15574;http://api.tvmaze.com/people/15574/castcredits
+15587;http://api.tvmaze.com/people/15587/castcredits
+15613;http://api.tvmaze.com/people/15613/castcredits
+15630;http://api.tvmaze.com/people/15630/castcredits
+15631;http://api.tvmaze.com/people/15631/castcredits
+15643;http://api.tvmaze.com/people/15643/castcredits
+15644;http://api.tvmaze.com/people/15644/castcredits
+15645;http://api.tvmaze.com/people/15645/castcredits
+15647;http://api.tvmaze.com/people/15647/castcredits
+15654;http://api.tvmaze.com/people/15654/castcredits
+15676;http://api.tvmaze.com/people/15676/castcredits
+15677;http://api.tvmaze.com/people/15677/castcredits
+15678;http://api.tvmaze.com/people/15678/castcredits
+15688;http://api.tvmaze.com/people/15688/castcredits
+15689;http://api.tvmaze.com/people/15689/castcredits
+15693;http://api.tvmaze.com/people/15693/castcredits
+15698;http://api.tvmaze.com/people/15698/castcredits
+15701;http://api.tvmaze.com/people/15701/castcredits
+15706;http://api.tvmaze.com/people/15706/castcredits
+15716;http://api.tvmaze.com/people/15716/castcredits
+15731;http://api.tvmaze.com/people/15731/castcredits
+15751;http://api.tvmaze.com/people/15751/castcredits
+15767;http://api.tvmaze.com/people/15767/castcredits
+15771;http://api.tvmaze.com/people/15771/castcredits
+15778;http://api.tvmaze.com/people/15778/castcredits
+15812;http://api.tvmaze.com/people/15812/castcredits
+15824;http://api.tvmaze.com/people/15824/castcredits
+15826;http://api.tvmaze.com/people/15826/castcredits
+15847;http://api.tvmaze.com/people/15847/castcredits
+15855;http://api.tvmaze.com/people/15855/castcredits
+15866;http://api.tvmaze.com/people/15866/castcredits
+15882;http://api.tvmaze.com/people/15882/castcredits
+15909;http://api.tvmaze.com/people/15909/castcredits
+15928;http://api.tvmaze.com/people/15928/castcredits
+16316;http://api.tvmaze.com/people/16316/castcredits
+16317;http://api.tvmaze.com/people/16317/castcredits
+16331;http://api.tvmaze.com/people/16331/castcredits
+16332;http://api.tvmaze.com/people/16332/castcredits
+16340;http://api.tvmaze.com/people/16340/castcredits
+16341;http://api.tvmaze.com/people/16341/castcredits
+16343;http://api.tvmaze.com/people/16343/castcredits
+16348;http://api.tvmaze.com/people/16348/castcredits
+16352;http://api.tvmaze.com/people/16352/castcredits
+16362;http://api.tvmaze.com/people/16362/castcredits
+16363;http://api.tvmaze.com/people/16363/castcredits
+16367;http://api.tvmaze.com/people/16367/castcredits
+16368;http://api.tvmaze.com/people/16368/castcredits
+16383;http://api.tvmaze.com/people/16383/castcredits
+16386;http://api.tvmaze.com/people/16386/castcredits
+16387;http://api.tvmaze.com/people/16387/castcredits
+16394;http://api.tvmaze.com/people/16394/castcredits
+16395;http://api.tvmaze.com/people/16395/castcredits
+16398;http://api.tvmaze.com/people/16398/castcredits
+16399;http://api.tvmaze.com/people/16399/castcredits
+16400;http://api.tvmaze.com/people/16400/castcredits
+16404;http://api.tvmaze.com/people/16404/castcredits
+16407;http://api.tvmaze.com/people/16407/castcredits
+16417;http://api.tvmaze.com/people/16417/castcredits
+16424;http://api.tvmaze.com/people/16424/castcredits
+16435;http://api.tvmaze.com/people/16435/castcredits
+16440;http://api.tvmaze.com/people/16440/castcredits
+16442;http://api.tvmaze.com/people/16442/castcredits
+16443;http://api.tvmaze.com/people/16443/castcredits
+16448;http://api.tvmaze.com/people/16448/castcredits
+16449;http://api.tvmaze.com/people/16449/castcredits
+16469;http://api.tvmaze.com/people/16469/castcredits
+16470;http://api.tvmaze.com/people/16470/castcredits
+16473;http://api.tvmaze.com/people/16473/castcredits
+16474;http://api.tvmaze.com/people/16474/castcredits
+16475;http://api.tvmaze.com/people/16475/castcredits
+16476;http://api.tvmaze.com/people/16476/castcredits
+16481;http://api.tvmaze.com/people/16481/castcredits
+16492;http://api.tvmaze.com/people/16492/castcredits
+16493;http://api.tvmaze.com/people/16493/castcredits
+16496;http://api.tvmaze.com/people/16496/castcredits
+16497;http://api.tvmaze.com/people/16497/castcredits
+16506;http://api.tvmaze.com/people/16506/castcredits
+16507;http://api.tvmaze.com/people/16507/castcredits
+16510;http://api.tvmaze.com/people/16510/castcredits
+16511;http://api.tvmaze.com/people/16511/castcredits
+16512;http://api.tvmaze.com/people/16512/castcredits
+16514;http://api.tvmaze.com/people/16514/castcredits
+16515;http://api.tvmaze.com/people/16515/castcredits
+16522;http://api.tvmaze.com/people/16522/castcredits
+16523;http://api.tvmaze.com/people/16523/castcredits
+16528;http://api.tvmaze.com/people/16528/castcredits
+16534;http://api.tvmaze.com/people/16534/castcredits
+16535;http://api.tvmaze.com/people/16535/castcredits
+16539;http://api.tvmaze.com/people/16539/castcredits
+16540;http://api.tvmaze.com/people/16540/castcredits
+16542;http://api.tvmaze.com/people/16542/castcredits
+16547;http://api.tvmaze.com/people/16547/castcredits
+16548;http://api.tvmaze.com/people/16548/castcredits
+16549;http://api.tvmaze.com/people/16549/castcredits
+16551;http://api.tvmaze.com/people/16551/castcredits
+16552;http://api.tvmaze.com/people/16552/castcredits
+16559;http://api.tvmaze.com/people/16559/castcredits
+16563;http://api.tvmaze.com/people/16563/castcredits
+16564;http://api.tvmaze.com/people/16564/castcredits
+16567;http://api.tvmaze.com/people/16567/castcredits
+16568;http://api.tvmaze.com/people/16568/castcredits
+16569;http://api.tvmaze.com/people/16569/castcredits
+16574;http://api.tvmaze.com/people/16574/castcredits
+16578;http://api.tvmaze.com/people/16578/castcredits
+16579;http://api.tvmaze.com/people/16579/castcredits
+16580;http://api.tvmaze.com/people/16580/castcredits
+16586;http://api.tvmaze.com/people/16586/castcredits
+16587;http://api.tvmaze.com/people/16587/castcredits
+16588;http://api.tvmaze.com/people/16588/castcredits
+16593;http://api.tvmaze.com/people/16593/castcredits
+16596;http://api.tvmaze.com/people/16596/castcredits
+16597;http://api.tvmaze.com/people/16597/castcredits
+16598;http://api.tvmaze.com/people/16598/castcredits
+16601;http://api.tvmaze.com/people/16601/castcredits
+16602;http://api.tvmaze.com/people/16602/castcredits
+16605;http://api.tvmaze.com/people/16605/castcredits
+16617;http://api.tvmaze.com/people/16617/castcredits
+16618;http://api.tvmaze.com/people/16618/castcredits
+16626;http://api.tvmaze.com/people/16626/castcredits
+16627;http://api.tvmaze.com/people/16627/castcredits
+16628;http://api.tvmaze.com/people/16628/castcredits
+16633;http://api.tvmaze.com/people/16633/castcredits
+16634;http://api.tvmaze.com/people/16634/castcredits
+16635;http://api.tvmaze.com/people/16635/castcredits
+16642;http://api.tvmaze.com/people/16642/castcredits
+16643;http://api.tvmaze.com/people/16643/castcredits
+16649;http://api.tvmaze.com/people/16649/castcredits
+16651;http://api.tvmaze.com/people/16651/castcredits
+16657;http://api.tvmaze.com/people/16657/castcredits
+16664;http://api.tvmaze.com/people/16664/castcredits
+16665;http://api.tvmaze.com/people/16665/castcredits
+16666;http://api.tvmaze.com/people/16666/castcredits
+16670;http://api.tvmaze.com/people/16670/castcredits
+16671;http://api.tvmaze.com/people/16671/castcredits
+16672;http://api.tvmaze.com/people/16672/castcredits
+16678;http://api.tvmaze.com/people/16678/castcredits
+16680;http://api.tvmaze.com/people/16680/castcredits
+16685;http://api.tvmaze.com/people/16685/castcredits
+16690;http://api.tvmaze.com/people/16690/castcredits
+16691;http://api.tvmaze.com/people/16691/castcredits
+16692;http://api.tvmaze.com/people/16692/castcredits
+16696;http://api.tvmaze.com/people/16696/castcredits
+16697;http://api.tvmaze.com/people/16697/castcredits
+16698;http://api.tvmaze.com/people/16698/castcredits
+16704;http://api.tvmaze.com/people/16704/castcredits
+16705;http://api.tvmaze.com/people/16705/castcredits
+16706;http://api.tvmaze.com/people/16706/castcredits
+16708;http://api.tvmaze.com/people/16708/castcredits
+16712;http://api.tvmaze.com/people/16712/castcredits
+16718;http://api.tvmaze.com/people/16718/castcredits
+16719;http://api.tvmaze.com/people/16719/castcredits
+16720;http://api.tvmaze.com/people/16720/castcredits
+16726;http://api.tvmaze.com/people/16726/castcredits
+16727;http://api.tvmaze.com/people/16727/castcredits
+16737;http://api.tvmaze.com/people/16737/castcredits
+16738;http://api.tvmaze.com/people/16738/castcredits
+16741;http://api.tvmaze.com/people/16741/castcredits
+16751;http://api.tvmaze.com/people/16751/castcredits
+16753;http://api.tvmaze.com/people/16753/castcredits
+16759;http://api.tvmaze.com/people/16759/castcredits
+16766;http://api.tvmaze.com/people/16766/castcredits
+16769;http://api.tvmaze.com/people/16769/castcredits
+16771;http://api.tvmaze.com/people/16771/castcredits
+16787;http://api.tvmaze.com/people/16787/castcredits
+16788;http://api.tvmaze.com/people/16788/castcredits
+16797;http://api.tvmaze.com/people/16797/castcredits
+16798;http://api.tvmaze.com/people/16798/castcredits
+16802;http://api.tvmaze.com/people/16802/castcredits
+16803;http://api.tvmaze.com/people/16803/castcredits
+16804;http://api.tvmaze.com/people/16804/castcredits
+16808;http://api.tvmaze.com/people/16808/castcredits
+16813;http://api.tvmaze.com/people/16813/castcredits
+16814;http://api.tvmaze.com/people/16814/castcredits
+16817;http://api.tvmaze.com/people/16817/castcredits
+16824;http://api.tvmaze.com/people/16824/castcredits
+16825;http://api.tvmaze.com/people/16825/castcredits
+16826;http://api.tvmaze.com/people/16826/castcredits
+16827;http://api.tvmaze.com/people/16827/castcredits
+16828;http://api.tvmaze.com/people/16828/castcredits
+16834;http://api.tvmaze.com/people/16834/castcredits
+16835;http://api.tvmaze.com/people/16835/castcredits
+16836;http://api.tvmaze.com/people/16836/castcredits
+16837;http://api.tvmaze.com/people/16837/castcredits
+16838;http://api.tvmaze.com/people/16838/castcredits
+16839;http://api.tvmaze.com/people/16839/castcredits
+16844;http://api.tvmaze.com/people/16844/castcredits
+16845;http://api.tvmaze.com/people/16845/castcredits
+16854;http://api.tvmaze.com/people/16854/castcredits
+16858;http://api.tvmaze.com/people/16858/castcredits
+16859;http://api.tvmaze.com/people/16859/castcredits
+16971;http://api.tvmaze.com/people/16971/castcredits
+17044;http://api.tvmaze.com/people/17044/castcredits
+17049;http://api.tvmaze.com/people/17049/castcredits
+17306;http://api.tvmaze.com/people/17306/castcredits
+17334;http://api.tvmaze.com/people/17334/castcredits
+17371;http://api.tvmaze.com/people/17371/castcredits
+17445;http://api.tvmaze.com/people/17445/castcredits
+17470;http://api.tvmaze.com/people/17470/castcredits
+17505;http://api.tvmaze.com/people/17505/castcredits
+17558;http://api.tvmaze.com/people/17558/castcredits
+17588;http://api.tvmaze.com/people/17588/castcredits
+17640;http://api.tvmaze.com/people/17640/castcredits
+17657;http://api.tvmaze.com/people/17657/castcredits
+17719;http://api.tvmaze.com/people/17719/castcredits
+17736;http://api.tvmaze.com/people/17736/castcredits
+17783;http://api.tvmaze.com/people/17783/castcredits
+17818;http://api.tvmaze.com/people/17818/castcredits
+17890;http://api.tvmaze.com/people/17890/castcredits
+17898;http://api.tvmaze.com/people/17898/castcredits
+17954;http://api.tvmaze.com/people/17954/castcredits
+18043;http://api.tvmaze.com/people/18043/castcredits
+18080;http://api.tvmaze.com/people/18080/castcredits
+18086;http://api.tvmaze.com/people/18086/castcredits
+18308;http://api.tvmaze.com/people/18308/castcredits
+18469;http://api.tvmaze.com/people/18469/castcredits
+18508;http://api.tvmaze.com/people/18508/castcredits
+18676;http://api.tvmaze.com/people/18676/castcredits
+18880;http://api.tvmaze.com/people/18880/castcredits
+18882;http://api.tvmaze.com/people/18882/castcredits
+19047;http://api.tvmaze.com/people/19047/castcredits
+19107;http://api.tvmaze.com/people/19107/castcredits
+19194;http://api.tvmaze.com/people/19194/castcredits
+19205;http://api.tvmaze.com/people/19205/castcredits
+19214;http://api.tvmaze.com/people/19214/castcredits
+19228;http://api.tvmaze.com/people/19228/castcredits
+19441;http://api.tvmaze.com/people/19441/castcredits
+19502;http://api.tvmaze.com/people/19502/castcredits
+19504;http://api.tvmaze.com/people/19504/castcredits
+19536;http://api.tvmaze.com/people/19536/castcredits
+19559;http://api.tvmaze.com/people/19559/castcredits
+19606;http://api.tvmaze.com/people/19606/castcredits
+19611;http://api.tvmaze.com/people/19611/castcredits
+19619;http://api.tvmaze.com/people/19619/castcredits
+19640;http://api.tvmaze.com/people/19640/castcredits
+19686;http://api.tvmaze.com/people/19686/castcredits
+19689;http://api.tvmaze.com/people/19689/castcredits
+19690;http://api.tvmaze.com/people/19690/castcredits
+19691;http://api.tvmaze.com/people/19691/castcredits
+19699;http://api.tvmaze.com/people/19699/castcredits
+19769;http://api.tvmaze.com/people/19769/castcredits
+19788;http://api.tvmaze.com/people/19788/castcredits
+19840;http://api.tvmaze.com/people/19840/castcredits
+20084;http://api.tvmaze.com/people/20084/castcredits
+20213;http://api.tvmaze.com/people/20213/castcredits
+20287;http://api.tvmaze.com/people/20287/castcredits
+20289;http://api.tvmaze.com/people/20289/castcredits
+20327;http://api.tvmaze.com/people/20327/castcredits
+20385;http://api.tvmaze.com/people/20385/castcredits
+20402;http://api.tvmaze.com/people/20402/castcredits
+20421;http://api.tvmaze.com/people/20421/castcredits
+20463;http://api.tvmaze.com/people/20463/castcredits
+20468;http://api.tvmaze.com/people/20468/castcredits
+20540;http://api.tvmaze.com/people/20540/castcredits
+20570;http://api.tvmaze.com/people/20570/castcredits
+20580;http://api.tvmaze.com/people/20580/castcredits
+20647;http://api.tvmaze.com/people/20647/castcredits
+20668;http://api.tvmaze.com/people/20668/castcredits
+20671;http://api.tvmaze.com/people/20671/castcredits
+20797;http://api.tvmaze.com/people/20797/castcredits
+20811;http://api.tvmaze.com/people/20811/castcredits
+20812;http://api.tvmaze.com/people/20812/castcredits
+20813;http://api.tvmaze.com/people/20813/castcredits
+20814;http://api.tvmaze.com/people/20814/castcredits
+20822;http://api.tvmaze.com/people/20822/castcredits
+20830;http://api.tvmaze.com/people/20830/castcredits
+20862;http://api.tvmaze.com/people/20862/castcredits
+20886;http://api.tvmaze.com/people/20886/castcredits
+20896;http://api.tvmaze.com/people/20896/castcredits
+20905;http://api.tvmaze.com/people/20905/castcredits
+20907;http://api.tvmaze.com/people/20907/castcredits
+20929;http://api.tvmaze.com/people/20929/castcredits
+21001;http://api.tvmaze.com/people/21001/castcredits
+21003;http://api.tvmaze.com/people/21003/castcredits
+21035;http://api.tvmaze.com/people/21035/castcredits
+21057;http://api.tvmaze.com/people/21057/castcredits
+21062;http://api.tvmaze.com/people/21062/castcredits
+21075;http://api.tvmaze.com/people/21075/castcredits
+21077;http://api.tvmaze.com/people/21077/castcredits
+21078;http://api.tvmaze.com/people/21078/castcredits
+21084;http://api.tvmaze.com/people/21084/castcredits
+21085;http://api.tvmaze.com/people/21085/castcredits
+21128;http://api.tvmaze.com/people/21128/castcredits
+21165;http://api.tvmaze.com/people/21165/castcredits
+21170;http://api.tvmaze.com/people/21170/castcredits
+21171;http://api.tvmaze.com/people/21171/castcredits
+21176;http://api.tvmaze.com/people/21176/castcredits
+21177;http://api.tvmaze.com/people/21177/castcredits
+21181;http://api.tvmaze.com/people/21181/castcredits
+21185;http://api.tvmaze.com/people/21185/castcredits
+21186;http://api.tvmaze.com/people/21186/castcredits
+21189;http://api.tvmaze.com/people/21189/castcredits
+21190;http://api.tvmaze.com/people/21190/castcredits
+21191;http://api.tvmaze.com/people/21191/castcredits
+21192;http://api.tvmaze.com/people/21192/castcredits
+21193;http://api.tvmaze.com/people/21193/castcredits
+21196;http://api.tvmaze.com/people/21196/castcredits
+21198;http://api.tvmaze.com/people/21198/castcredits
+21199;http://api.tvmaze.com/people/21199/castcredits
+21203;http://api.tvmaze.com/people/21203/castcredits
+21204;http://api.tvmaze.com/people/21204/castcredits
+21205;http://api.tvmaze.com/people/21205/castcredits
+21206;http://api.tvmaze.com/people/21206/castcredits
+21208;http://api.tvmaze.com/people/21208/castcredits
+21209;http://api.tvmaze.com/people/21209/castcredits
+21210;http://api.tvmaze.com/people/21210/castcredits
+21212;http://api.tvmaze.com/people/21212/castcredits
+21213;http://api.tvmaze.com/people/21213/castcredits
+21214;http://api.tvmaze.com/people/21214/castcredits
+21218;http://api.tvmaze.com/people/21218/castcredits
+21219;http://api.tvmaze.com/people/21219/castcredits
+21220;http://api.tvmaze.com/people/21220/castcredits
+21222;http://api.tvmaze.com/people/21222/castcredits
+21223;http://api.tvmaze.com/people/21223/castcredits
+21224;http://api.tvmaze.com/people/21224/castcredits
+21225;http://api.tvmaze.com/people/21225/castcredits
+21226;http://api.tvmaze.com/people/21226/castcredits
+21227;http://api.tvmaze.com/people/21227/castcredits
+21228;http://api.tvmaze.com/people/21228/castcredits
+21232;http://api.tvmaze.com/people/21232/castcredits
+21233;http://api.tvmaze.com/people/21233/castcredits
+21234;http://api.tvmaze.com/people/21234/castcredits
+21235;http://api.tvmaze.com/people/21235/castcredits
+21236;http://api.tvmaze.com/people/21236/castcredits
+21237;http://api.tvmaze.com/people/21237/castcredits
+21238;http://api.tvmaze.com/people/21238/castcredits
+21239;http://api.tvmaze.com/people/21239/castcredits
+21242;http://api.tvmaze.com/people/21242/castcredits
+21243;http://api.tvmaze.com/people/21243/castcredits
+21244;http://api.tvmaze.com/people/21244/castcredits
+21245;http://api.tvmaze.com/people/21245/castcredits
+21246;http://api.tvmaze.com/people/21246/castcredits
+21249;http://api.tvmaze.com/people/21249/castcredits
+21250;http://api.tvmaze.com/people/21250/castcredits
+21251;http://api.tvmaze.com/people/21251/castcredits
+21253;http://api.tvmaze.com/people/21253/castcredits
+21254;http://api.tvmaze.com/people/21254/castcredits
+21255;http://api.tvmaze.com/people/21255/castcredits
+21256;http://api.tvmaze.com/people/21256/castcredits
+21262;http://api.tvmaze.com/people/21262/castcredits
+21263;http://api.tvmaze.com/people/21263/castcredits
+21264;http://api.tvmaze.com/people/21264/castcredits
+21265;http://api.tvmaze.com/people/21265/castcredits
+21266;http://api.tvmaze.com/people/21266/castcredits
+21267;http://api.tvmaze.com/people/21267/castcredits
+21270;http://api.tvmaze.com/people/21270/castcredits
+21272;http://api.tvmaze.com/people/21272/castcredits
+21273;http://api.tvmaze.com/people/21273/castcredits
+21275;http://api.tvmaze.com/people/21275/castcredits
+21276;http://api.tvmaze.com/people/21276/castcredits
+21277;http://api.tvmaze.com/people/21277/castcredits
+21278;http://api.tvmaze.com/people/21278/castcredits
+21279;http://api.tvmaze.com/people/21279/castcredits
+21280;http://api.tvmaze.com/people/21280/castcredits
+21281;http://api.tvmaze.com/people/21281/castcredits
+21282;http://api.tvmaze.com/people/21282/castcredits
+21283;http://api.tvmaze.com/people/21283/castcredits
+21284;http://api.tvmaze.com/people/21284/castcredits
+21285;http://api.tvmaze.com/people/21285/castcredits
+21287;http://api.tvmaze.com/people/21287/castcredits
+21288;http://api.tvmaze.com/people/21288/castcredits
+21289;http://api.tvmaze.com/people/21289/castcredits
+21290;http://api.tvmaze.com/people/21290/castcredits
+21291;http://api.tvmaze.com/people/21291/castcredits
+21292;http://api.tvmaze.com/people/21292/castcredits
+21293;http://api.tvmaze.com/people/21293/castcredits
+21294;http://api.tvmaze.com/people/21294/castcredits
+21295;http://api.tvmaze.com/people/21295/castcredits
+21296;http://api.tvmaze.com/people/21296/castcredits
+21297;http://api.tvmaze.com/people/21297/castcredits
+21298;http://api.tvmaze.com/people/21298/castcredits
+21299;http://api.tvmaze.com/people/21299/castcredits
+21300;http://api.tvmaze.com/people/21300/castcredits
+21301;http://api.tvmaze.com/people/21301/castcredits
+21303;http://api.tvmaze.com/people/21303/castcredits
+21304;http://api.tvmaze.com/people/21304/castcredits
+21305;http://api.tvmaze.com/people/21305/castcredits
+21307;http://api.tvmaze.com/people/21307/castcredits
+21308;http://api.tvmaze.com/people/21308/castcredits
+21313;http://api.tvmaze.com/people/21313/castcredits
+21314;http://api.tvmaze.com/people/21314/castcredits
+21317;http://api.tvmaze.com/people/21317/castcredits
+21319;http://api.tvmaze.com/people/21319/castcredits
+21320;http://api.tvmaze.com/people/21320/castcredits
+21321;http://api.tvmaze.com/people/21321/castcredits
+21322;http://api.tvmaze.com/people/21322/castcredits
+21323;http://api.tvmaze.com/people/21323/castcredits
+21327;http://api.tvmaze.com/people/21327/castcredits
+21328;http://api.tvmaze.com/people/21328/castcredits
+21329;http://api.tvmaze.com/people/21329/castcredits
+21330;http://api.tvmaze.com/people/21330/castcredits
+21331;http://api.tvmaze.com/people/21331/castcredits
+21332;http://api.tvmaze.com/people/21332/castcredits
+21335;http://api.tvmaze.com/people/21335/castcredits
+21338;http://api.tvmaze.com/people/21338/castcredits
+21339;http://api.tvmaze.com/people/21339/castcredits
+21342;http://api.tvmaze.com/people/21342/castcredits
+21343;http://api.tvmaze.com/people/21343/castcredits
+21344;http://api.tvmaze.com/people/21344/castcredits
+21345;http://api.tvmaze.com/people/21345/castcredits
+21410;http://api.tvmaze.com/people/21410/castcredits
+21416;http://api.tvmaze.com/people/21416/castcredits
+21420;http://api.tvmaze.com/people/21420/castcredits
+21435;http://api.tvmaze.com/people/21435/castcredits
+21438;http://api.tvmaze.com/people/21438/castcredits
+21447;http://api.tvmaze.com/people/21447/castcredits
+21449;http://api.tvmaze.com/people/21449/castcredits
+21458;http://api.tvmaze.com/people/21458/castcredits
+21459;http://api.tvmaze.com/people/21459/castcredits
+21465;http://api.tvmaze.com/people/21465/castcredits
+21466;http://api.tvmaze.com/people/21466/castcredits
+21467;http://api.tvmaze.com/people/21467/castcredits
+21468;http://api.tvmaze.com/people/21468/castcredits
+21481;http://api.tvmaze.com/people/21481/castcredits
+21488;http://api.tvmaze.com/people/21488/castcredits
+21501;http://api.tvmaze.com/people/21501/castcredits
+21502;http://api.tvmaze.com/people/21502/castcredits
+21509;http://api.tvmaze.com/people/21509/castcredits
+21510;http://api.tvmaze.com/people/21510/castcredits
+21519;http://api.tvmaze.com/people/21519/castcredits
+21520;http://api.tvmaze.com/people/21520/castcredits
+21521;http://api.tvmaze.com/people/21521/castcredits
+21522;http://api.tvmaze.com/people/21522/castcredits
+21523;http://api.tvmaze.com/people/21523/castcredits
+21534;http://api.tvmaze.com/people/21534/castcredits
+21535;http://api.tvmaze.com/people/21535/castcredits
+21539;http://api.tvmaze.com/people/21539/castcredits
+21545;http://api.tvmaze.com/people/21545/castcredits
+21549;http://api.tvmaze.com/people/21549/castcredits
+21572;http://api.tvmaze.com/people/21572/castcredits
+21581;http://api.tvmaze.com/people/21581/castcredits
+21584;http://api.tvmaze.com/people/21584/castcredits
+21585;http://api.tvmaze.com/people/21585/castcredits
+21592;http://api.tvmaze.com/people/21592/castcredits
+21599;http://api.tvmaze.com/people/21599/castcredits
+21609;http://api.tvmaze.com/people/21609/castcredits
+21614;http://api.tvmaze.com/people/21614/castcredits
+21622;http://api.tvmaze.com/people/21622/castcredits
+21623;http://api.tvmaze.com/people/21623/castcredits
+21627;http://api.tvmaze.com/people/21627/castcredits
+21628;http://api.tvmaze.com/people/21628/castcredits
+21642;http://api.tvmaze.com/people/21642/castcredits
+21643;http://api.tvmaze.com/people/21643/castcredits
+21644;http://api.tvmaze.com/people/21644/castcredits
+21645;http://api.tvmaze.com/people/21645/castcredits
+21648;http://api.tvmaze.com/people/21648/castcredits
+21649;http://api.tvmaze.com/people/21649/castcredits
+21652;http://api.tvmaze.com/people/21652/castcredits
+21653;http://api.tvmaze.com/people/21653/castcredits
+21666;http://api.tvmaze.com/people/21666/castcredits
+21667;http://api.tvmaze.com/people/21667/castcredits
+21671;http://api.tvmaze.com/people/21671/castcredits
+21674;http://api.tvmaze.com/people/21674/castcredits
+21695;http://api.tvmaze.com/people/21695/castcredits
+21709;http://api.tvmaze.com/people/21709/castcredits
+21710;http://api.tvmaze.com/people/21710/castcredits
+21719;http://api.tvmaze.com/people/21719/castcredits
+21720;http://api.tvmaze.com/people/21720/castcredits
+21721;http://api.tvmaze.com/people/21721/castcredits
+21739;http://api.tvmaze.com/people/21739/castcredits
+21744;http://api.tvmaze.com/people/21744/castcredits
+21745;http://api.tvmaze.com/people/21745/castcredits
+21752;http://api.tvmaze.com/people/21752/castcredits
+21755;http://api.tvmaze.com/people/21755/castcredits
+21756;http://api.tvmaze.com/people/21756/castcredits
+21770;http://api.tvmaze.com/people/21770/castcredits
+21776;http://api.tvmaze.com/people/21776/castcredits
+21783;http://api.tvmaze.com/people/21783/castcredits
+21789;http://api.tvmaze.com/people/21789/castcredits
+21790;http://api.tvmaze.com/people/21790/castcredits
+21799;http://api.tvmaze.com/people/21799/castcredits
+21806;http://api.tvmaze.com/people/21806/castcredits
+21807;http://api.tvmaze.com/people/21807/castcredits
+21821;http://api.tvmaze.com/people/21821/castcredits
+21822;http://api.tvmaze.com/people/21822/castcredits
+21836;http://api.tvmaze.com/people/21836/castcredits
+21840;http://api.tvmaze.com/people/21840/castcredits
+21841;http://api.tvmaze.com/people/21841/castcredits
+21842;http://api.tvmaze.com/people/21842/castcredits
+21844;http://api.tvmaze.com/people/21844/castcredits
+21845;http://api.tvmaze.com/people/21845/castcredits
+21854;http://api.tvmaze.com/people/21854/castcredits
+21855;http://api.tvmaze.com/people/21855/castcredits
+21872;http://api.tvmaze.com/people/21872/castcredits
+21873;http://api.tvmaze.com/people/21873/castcredits
+21874;http://api.tvmaze.com/people/21874/castcredits
+21883;http://api.tvmaze.com/people/21883/castcredits
+21884;http://api.tvmaze.com/people/21884/castcredits
+21885;http://api.tvmaze.com/people/21885/castcredits
+21886;http://api.tvmaze.com/people/21886/castcredits
+21901;http://api.tvmaze.com/people/21901/castcredits
+21902;http://api.tvmaze.com/people/21902/castcredits
+21905;http://api.tvmaze.com/people/21905/castcredits
+21906;http://api.tvmaze.com/people/21906/castcredits
+21911;http://api.tvmaze.com/people/21911/castcredits
+21919;http://api.tvmaze.com/people/21919/castcredits
+21920;http://api.tvmaze.com/people/21920/castcredits
+21925;http://api.tvmaze.com/people/21925/castcredits
+21928;http://api.tvmaze.com/people/21928/castcredits
+21930;http://api.tvmaze.com/people/21930/castcredits
+21936;http://api.tvmaze.com/people/21936/castcredits
+21939;http://api.tvmaze.com/people/21939/castcredits
+21942;http://api.tvmaze.com/people/21942/castcredits
+21943;http://api.tvmaze.com/people/21943/castcredits
+22007;http://api.tvmaze.com/people/22007/castcredits
+22059;http://api.tvmaze.com/people/22059/castcredits
+22063;http://api.tvmaze.com/people/22063/castcredits
+22064;http://api.tvmaze.com/people/22064/castcredits
+22069;http://api.tvmaze.com/people/22069/castcredits
+22076;http://api.tvmaze.com/people/22076/castcredits
+22078;http://api.tvmaze.com/people/22078/castcredits
+22086;http://api.tvmaze.com/people/22086/castcredits
+22095;http://api.tvmaze.com/people/22095/castcredits
+22099;http://api.tvmaze.com/people/22099/castcredits
+22104;http://api.tvmaze.com/people/22104/castcredits
+22146;http://api.tvmaze.com/people/22146/castcredits
+22147;http://api.tvmaze.com/people/22147/castcredits
+22172;http://api.tvmaze.com/people/22172/castcredits
+22176;http://api.tvmaze.com/people/22176/castcredits
+22193;http://api.tvmaze.com/people/22193/castcredits
+22198;http://api.tvmaze.com/people/22198/castcredits
+22199;http://api.tvmaze.com/people/22199/castcredits
+22204;http://api.tvmaze.com/people/22204/castcredits
+22211;http://api.tvmaze.com/people/22211/castcredits
+22214;http://api.tvmaze.com/people/22214/castcredits
+22215;http://api.tvmaze.com/people/22215/castcredits
+22222;http://api.tvmaze.com/people/22222/castcredits
+22223;http://api.tvmaze.com/people/22223/castcredits
+22243;http://api.tvmaze.com/people/22243/castcredits
+22245;http://api.tvmaze.com/people/22245/castcredits
+22250;http://api.tvmaze.com/people/22250/castcredits
+22321;http://api.tvmaze.com/people/22321/castcredits
+22395;http://api.tvmaze.com/people/22395/castcredits
+22458;http://api.tvmaze.com/people/22458/castcredits
+22522;http://api.tvmaze.com/people/22522/castcredits
+22536;http://api.tvmaze.com/people/22536/castcredits
+22697;http://api.tvmaze.com/people/22697/castcredits
+22792;http://api.tvmaze.com/people/22792/castcredits
+22798;http://api.tvmaze.com/people/22798/castcredits
+22803;http://api.tvmaze.com/people/22803/castcredits
+22899;http://api.tvmaze.com/people/22899/castcredits
+23153;http://api.tvmaze.com/people/23153/castcredits
+23177;http://api.tvmaze.com/people/23177/castcredits
+23198;http://api.tvmaze.com/people/23198/castcredits
+23204;http://api.tvmaze.com/people/23204/castcredits
+23262;http://api.tvmaze.com/people/23262/castcredits

update_json.py

@@ -0,0 +1,370 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import os
+import time
+import sys
+import url.urls
+import url.process
+from tqdm import tqdm
+import db.sql
+import logs.Logger
+from db.functions import settype, dbmongo
+from datetime import datetime
+import json
+import json_downloader
+
+response = None
+
+
+def set_apienv(urls, uprocess, dbengine, dbExec, updatesBase, lprint):
+    """Populate updates table from the API and return available updates.
+
+    Args:
+        urls: URL configuration with an "updatesurl" key.
+        uprocess: URL loader instance with `get_data`.
+        jget: JSON processor with `jconvert`.
+        dbengine: database engine dict (expects key "engine").
+        dbExec: database execution helper with `update_tvupdates` and `rawsql_select`.
+        updatesBase: list-like containing tables (uses index 8 for updates table).
+        lprint: logger instance with `logprint`.
+
+    Returns:
+        List of tuples (seriesid, timestamp) representing available updates.
+    """
+    #updatetable = updatesBase[8]
+    #inputdata = uprocess.get_data(urls["updatesurl"])
+    #lprint.logprint("info", f"Retrieved {len(availableupdates)} rows for processing......")
+    #
+    newupdates = dbExec.rawsql_select(
+        dbengine["engine"],
+        "select seriesid,timestamp from updates.tvupdates",
+        lprint
+    )
+    return newupdates
+
+
+def print_starttime(lprint, ts1):
+    lprint.logprint("info", f"Sync start time: {str(ts1)}")
+
+
+def get_tvdb(tvdburls):
+    return tvdburls.get_data()
+
+
+def get_tvmaze(tvmazeurls):
+    return tvmazeurls.get_data()
+
+def find_file_in_multiple_dirs(filename, directories):
+    """
+    Checks if a file exists in any of the provided directories.
+
+    Args:
+        filename (str): The name of the file to search for.
+        directories (list): A list of directory paths to check.
+
+    Returns:
+        Path or None: The full path to the file if found, otherwise None.
+    """
+    directories_found = []
+    for directory in directories:
+        if os.path.isfile(f'{directory}{filename}'):
+            directories_found.append(directory)
+                
+    return directories_found
+
+def main() -> int:
+    ROOTDIR = os.getcwd()
+    directories = {}
+    # Use cross-platform path builders and ensure directories exist
+    LOGDIR = os.path.join(ROOTDIR, "log")
+    directories["SERIESDIR"] = os.path.join(ROOTDIR, "cache", "series") + os.sep
+    directories["EPISODEDIR"] = os.path.join(ROOTDIR, "cache", "episodes") + os.sep
+    directories["CASTDIR"] = os.path.join(ROOTDIR, "cache", "cast") + os.sep
+    directories["GUESTCASTDIR"] = os.path.join(ROOTDIR, "cache", "guestcast") + os.sep
+    directories["CREWDIR"] = os.path.join(ROOTDIR, "cache", "crew") + os.sep
+    #directories["GUESTCREWDIR"] = os.path.join(ROOTDIR, "cache", "guestcrew") + os.sep    
+    #directories["CREDITSDIR"] = os.path.join(ROOTDIR, "cache", "credits") + os.sep
+    #directories["ALIASDIR"] = os.path.join(ROOTDIR, "cache", "aliases") + os.sep
+    tempdir = os.path.join(ROOTDIR, 'temp') + os.sep
+    updatetable = 'updates.tvupdates'
+    skip_file = f"{tempdir}skip_ids.txt"
+    
+    # Create directories if missing so later code doesn't fail
+    try:
+        os.makedirs(LOGDIR, exist_ok=True)
+        os.makedirs(os.path.join(ROOTDIR, "cache", "series"), exist_ok=True)
+        os.makedirs(os.path.join(ROOTDIR, "cache", "episodes"), exist_ok=True)
+        os.makedirs(os.path.join(ROOTDIR, "cache", "cast"), exist_ok=True)
+        os.makedirs(os.path.join(ROOTDIR, "cache", "guestcast"), exist_ok=True)
+        os.makedirs(os.path.join(ROOTDIR, "cache", "crew"), exist_ok=True)
+        os.makedirs(os.path.join(ROOTDIR, "cache", "guestcrew"), exist_ok=True)
+    except Exception:
+        # If directory creation fails, let the logging initialization surface the error
+        pass
+
+    lprint = logs.Logger.LPrint(LOGDIR, False)
+    lprint.start_log()
+    import settings.config
+
+    config_options = settings.config.Config(ROOTDIR)
+    options = config_options.config_options
+    mongo_updater = dbmongo(options)  # Create an instance of dbmongo
+    dbtype, apitype = options["dbtype"], options["apitype"]
+    dbs = settype(dbtype, apitype, options)
+    dbclass, dbengine, apiengine = dbs.dbclass, dbs.dbengine, dbs.apiengine
+    uprocess = url.process.Loadurl()
+    TVMAZEURLS = url.urls.Tvmazeurls()
+    TVMAZE_URLS = get_tvmaze(TVMAZEURLS)
+    TVDBURLS = url.urls.TVDBurls()
+    TVDB_URLS = get_tvdb(TVDBURLS)
+    updateRange = None
+
+    lprint.logprint("info", "Creating JSON instance for data retrieval.")
+    lprint.logprint("info", "JSON instance created.")
+    lprint.logprint("info", f"Update type is {options['updatetype']}")
+    timedifference = options["updatetype"]
+    currenttimestamp = time.time()
+    if timedifference.lower() == "full":
+        timedifference = int(currenttimestamp)
+    basetime = str(currenttimestamp).split(".")
+    currenttimestamp = int(basetime[0])
+    ts1 = str(datetime.now()).split(".")[0]
+    lprint.logprint("info", "Sync start time: " + str(ts1))
+    print_starttime(lprint, ts1)
+    dbExec = db.sql.Dbexec()
+    createTempTables = dbclass["CreateTemptables"]
+    new_updates = {}
+    
+    ct = createTempTables(
+        dbengine["session"],
+        dbengine["metadata"],
+        dbengine["engine"]
+    )
+    tempTableList = ct.tablelist
+
+    new_updates = []
+    if options["apitype"].lower() == "tvmaze":
+        try:
+            new_updates = set_apienv(
+                TVMAZE_URLS,
+                uprocess,
+                dbengine,
+                dbExec,
+                tempTableList,
+                lprint,
+            )
+        except Exception as exc:  # noqa: PLC0301 - broad handler to log failures
+            lprint.logprint("error", f"Failed to set API env: {exc}")
+            return 1
+
+    timeDifference = options["updatetype"]
+    currentTimestamp = time.time()
+    if timeDifference.lower() == "full" or options["initload"] == "1":
+        tdifference = 0
+    else:
+        tdifference = int(currentTimestamp) - int(
+            options[options["updatetype"]]
+        )
+    needed_updates = []
+    updateList = []
+    for result in new_updates:
+        updateid = result[0]
+        updatestamp = int(result[1])
+        tdif = int(tdifference)
+        if updatestamp > tdif:
+            needed_updates.append(updateid)
+            updateList.append(updateid)
+
+    lprint.logprint("info", "Update Type: " + str(options["updatetype"]))
+    updateRange = len(updateList)
+    lprint.logprint("info", "Number of updates: " + str(updateRange))
+
+
+    # Save a downloaded copy of the updates JSON for debugging/inspection
+    try:
+        os.makedirs(tempdir, exist_ok=True)
+        out_file = 'tvmaze_updates.json'
+        prev_file = f'post_tvmaze_updates.json.{currentTimestamp}'
+        previous_updates_file = f"{tempdir}{prev_file}"
+        # TVMAZE_URLS is a dict of urls; use the updatesurl if available
+        updates_url = TVMAZE_URLS.get('updatesurl') if isinstance(TVMAZE_URLS, dict) else None
+        if updates_url:
+            json_downloader.download_json_to_file(updates_url, tempdir, out_file, lprint)
+            # load and convert saved JSON into a dict (id -> timestamp)
+            try:
+                downloaded_updates_dict = {}
+                downloaded_updates_list = []
+                if os.path.exists(f"{tempdir}{out_file}"):
+                    with open(f"{tempdir}{out_file}", 'r') as jf:
+                        loaded = json.load(jf)
+
+                    if isinstance(loaded, dict):
+                        for k, v in loaded.items():
+                            try:
+                                kid = int(k)
+                            except Exception:
+                                kid = k
+                            try:
+                                vt = int(v)
+                            except Exception:
+                                vt = v
+                            downloaded_updates_dict[kid] = vt
+                        downloaded_updates_list = list(downloaded_updates_dict.items())
+
+                    elif isinstance(loaded, list):
+                        for item in loaded:
+                            if isinstance(item, list) and len(item) >= 2:
+                                k = item[0]; v = item[1]
+                            elif isinstance(item, dict):
+                                if 'id' in item and 'timestamp' in item:
+                                    k = item['id']; v = item['timestamp']
+                                elif 'seriesid' in item and 'timestamp' in item:
+                                    k = item['seriesid']; v = item['timestamp']
+                                else:
+                                    downloaded_updates_list.append(item)
+                                    continue
+                            else:
+                                k = item; v = None
+                            try:
+                                kid = int(k)
+                            except Exception:
+                                kid = k
+                            try:
+                                vt = int(v) if v is not None and str(v).isdigit() else v
+                            except Exception:
+                                vt = v
+                            downloaded_updates_dict[kid] = vt
+                            downloaded_updates_list.append((kid, vt))
+
+                    else:
+                        downloaded_updates_list = [loaded]
+                    lprint.logprint("info", f"Loaded {len(downloaded_updates_list)} entries from tvmaze_updates.json")
+                else:
+                    downloaded_updates_dict = {}
+                    downloaded_updates_list = []
+            except Exception:
+                downloaded_updates_dict = {}
+                downloaded_updates_list = []
+    except Exception:
+        # non-fatal; continue
+        lprint.logprint("warning", "Failed to save or load downloaded updates JSON.")
+    previouslisting = dbExec.rawsql_select(
+        dbengine["engine"],
+        "select seriesid, timestamp from updates.tvupdates order by seriesid",
+        lprint
+    )
+    print(f"Saving previous updates to file.")
+    with open(f"{previous_updates_file}", 'w') as pjf:
+        for pl in previouslisting:
+            pjf.write(f"{pl[0]}: {pl[1]}\n")
+    pjf.close()
+    # convert previouslisting (list of tuples) into a dict: row_id -> country_name
+    try:
+        _previouslisting_list = previouslisting
+        previous_listing_dict = {row[0]: row[1] for row in _previouslisting_list}
+    except Exception:
+        # leave previouslisting as-is on any failure
+        pass
+    # Ensure dictionaries exist
+    try:
+        downloaded_updates_dict
+    except NameError:
+        downloaded_updates_dict = {}
+    try:
+        previous_listing_dict
+    except NameError:
+        previous_listing_dict = {}
+    files_needed = []
+    directory_list = []
+    for value in directories.values():
+        directory_list.append(value)
+    skip_ids = {}
+    skipfile = open(skip_file, 'r')
+    for line in skipfile:
+        field = line.split(';')
+        skip_ids[field[0]] =  field[1]
+    print(f"Updating update table.")
+    dbExec.update_tvupdates(
+        dbengine["engine"],
+        downloaded_updates_dict,
+        "updates.tvupdates"
+    )
+    print("update complete.")
+    print(f"Processing {len(downloaded_updates_dict)} downloaded updates against {len(previous_listing_dict)} previous updates for files needed.")
+    for seriesid, ts in downloaded_updates_dict.items():
+        if seriesid in previous_listing_dict:
+            prev_ts = previous_listing_dict[seriesid]
+            # Check to see if all json files are present for seriesid
+            try:
+                if int(ts) == int(prev_ts):
+                    found_path = find_file_in_multiple_dirs(f"{seriesid}.json", directory_list)
+                    if len(found_path) == 5:
+                        continue
+                    else:
+                        for directory in directory_list:
+                            if directory not in found_path:
+                                files_needed.append([f"{directory}", f"{seriesid}.json"])
+                elif int(ts) > int(prev_ts):
+                    for directory in directory_list:
+                        files_needed.append([f"{directory}", f"{seriesid}.json"])
+            except Exception:
+                lprint.logprint("debug", f"Unable to process Series {seriesid} timestamps: downloaded {ts} vs previous {prev_ts}")
+        else:
+            # Add needed json files to needed file list.
+            for directory in directory_list:
+                files_needed.append([f"{directory}", f"{seriesid}.json"])
+            lprint.logprint("debug", f"Series {seriesid} present in downloaded updates but missing in previous listing")
+
+    lprint.logprint("info", f"Downloadinng JSON for {len(files_needed)} series IDs.")
+    print(f"Downloading JSON for {len(files_needed)} files.")
+    for i in tqdm(range(len(files_needed)), desc="Downloading JSON files", unit="filename"):
+        for directory, file in files_needed[i:i+1]:
+            if 'credits' in directory:
+                if file.replace('.json', '') in skip_ids and 'castcredits' in skip_ids[file.replace('.json', '')].strip():
+                    lprint.logprint("info", f"Skipping credits download for series ID {file.replace('.json', '')} as per skip_ids.txt")
+                    continue
+                else:
+                    url_template = TVMAZE_URLS["creditsurl"]
+            elif 'crew' in directory:
+                url_template = TVMAZE_URLS["crewurl"]
+            elif 'cast' in directory:
+                url_template = TVMAZE_URLS["casturl"]
+            elif 'episodes' in directory:
+                url_template = TVMAZE_URLS["episodesurl"]
+            elif 'aliases' in directory:
+                url_template = TVMAZE_URLS["aliasurl"]
+            else:
+                url_template = TVMAZE_URLS["showurl"]
+            seriesid = file.replace('.json', '')
+            json_downloader.download_json_to_file(url_template.replace("<seriesid>", str(seriesid)), directory, f'{seriesid}.json', lprint)
+    # dbExec.update_tvupdates(dbengine["engine"], downloaded_updates_dict, updatetable)
+
+
+
+    # countrylisting = dbExec.rawsql_select(
+    #     dbengine["engine"],
+    #     "select row_id, country_name from dbo.countrydata",
+    #     lprint
+    # )
+    # for country in countrylisting:
+    #     country_name = country[1]
+    #     country_id = country[0]
+    #     countrydictionary[country_name] = country_id
+    # mongo_updater = dbmongo(options)
+    # mongo_updater.update_mongo(
+    #     updateList,
+    #     ROOTDIR,
+    #     directories,
+    #     jget,
+    #     countrydictionary,
+    #     dbExec,
+    #     dbengine,
+    #     dboBase,
+    #     lprint,
+    # )
+    # return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+

update_mongo.py

@@ -1,207 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-import os
-import time
-import sys
-import url.urls
-import url.process
-import api.tvmaze.cast as mazecast
-import api.tvmaze.crew as mazecrew
-import api.tvmaze.episodes as mazeepisodes
-import api.tvmaze.series as mazeseries
-import db.sql
-import jprocessor.process
-import logs.Logger
-from db.functions import settype, dbmongo
-from datetime import datetime
-
-response = None
-
-
-def set_apienv(urls, uprocess, jget, dbengine, dbExec, updatesBase, lprint):
-    """Populate updates table from the API and return available updates.
-
-    Args:
-        urls: URL configuration with an "updatesurl" key.
-        uprocess: URL loader instance with `get_data`.
-        jget: JSON processor with `jconvert`.
-        dbengine: database engine dict (expects key "engine").
-        dbExec: database execution helper with `update_tvupdates` and `rawsql_select`.
-        updatesBase: list-like containing tables (uses index 8 for updates table).
-        lprint: logger instance with `logprint`.
-
-    Returns:
-        List of tuples (seriesid, timestamp) representing available updates.
-    """
-    updatetable = updatesBase[8]
-    inputdata = uprocess.get_data(urls["updatesurl"])
-    availableupdates = jget.jconvert(inputdata)
-    lprint.logprint("info", f"Retrieved {len(availableupdates)} rows for processing......")
-    dbExec.update_tvupdates(dbengine["engine"], availableupdates, updatetable)
-    newupdates = dbExec.rawsql_select(
-        dbengine["engine"],
-        "select seriesid,timestamp from updates.tvupdates",
-        lprint
-    )
-    return newupdates
-
-
-def print_starttime(lprint, ts1):
-    lprint.logprint("info", f"Sync start time: {str(ts1)}")
-
-
-def get_tvdb(tvdburls):
-    return tvdburls.get_data()
-
-
-def get_tvmaze(tvmazeurls):
-    return tvmazeurls.get_data()
-
-
-def main() -> int:
-    ROOTDIR = os.getcwd()
-    directories = {}
-    # Use cross-platform path builders and ensure directories exist
-    LOGDIR = os.path.join(ROOTDIR, "log")
-    directories["SERIESDIR"] = os.path.join(ROOTDIR, "cache", "series") + os.sep
-    directories["EPISODEDIR"] = os.path.join(ROOTDIR, "cache", "episodes") + os.sep
-    directories["CASTDIR"] = os.path.join(ROOTDIR, "cache", "cast") + os.sep
-    directories["CREWDIR"] = os.path.join(ROOTDIR, "cache", "crew") + os.sep
-
-    # Create directories if missing so later code doesn't fail
-    try:
-        os.makedirs(LOGDIR, exist_ok=True)
-        os.makedirs(os.path.join(ROOTDIR, "cache", "series"), exist_ok=True)
-        os.makedirs(os.path.join(ROOTDIR, "cache", "episodes"), exist_ok=True)
-        os.makedirs(os.path.join(ROOTDIR, "cache", "cast"), exist_ok=True)
-        os.makedirs(os.path.join(ROOTDIR, "cache", "crew"), exist_ok=True)
-    except Exception:
-        # If directory creation fails, let the logging initialization surface the error
-        pass
-
-    lprint = logs.Logger.LPrint(LOGDIR, False)
-    lprint.start_log()
-    import settings.config
-
-    config_options = settings.config.Config(ROOTDIR)
-    options = config_options.config_options
-    dbtype, apitype = options["dbtype"], options["apitype"]
-    dbs = settype(dbtype, apitype, options)
-    dbclass, dbengine, apiengine = dbs.dbclass, dbs.dbengine, dbs.apiengine
-    uprocess = url.process.Loadurl()
-    TVMAZEURLS = url.urls.Tvmazeurls()
-    TVMAZE_URLS = get_tvmaze(TVMAZEURLS)
-    TVDBURLS = url.urls.TVDBurls()
-    TVDB_URLS = get_tvdb(TVDBURLS)
-    #mseries = mazeseries.tvshow()
-    #mcast = mazecast.cast()
-    #mepisode = mazeepisodes.episodes()
-    #mcrew = mazecrew.crew()
-    countrydictionary = {}
-    updateRange = None
-
-    lprint.logprint("info", "Creating JSON instance for data retrieval.")
-    jget = jprocessor.process.Jsonload()
-    lprint.logprint("info", "JSON instance created.")
-    lprint.logprint("info", f"Update type is {options['updatetype']}")
-    timedifference = options["updatetype"]
-    currenttimestamp = time.time()
-    if timedifference.lower() == "full":
-        timedifference = int(currenttimestamp)
-    basetime = str(currenttimestamp).split(".")
-    currenttimestamp = int(basetime[0])
-    ts1 = str(datetime.now()).split(".")[0]
-    lprint.logprint("info", "Sync start time: " + str(ts1))
-    print_starttime(lprint, ts1)
-    dboBase = dbengine["dboBase"]
-    dbExec = db.sql.Dbexec()
-    createTables = dbclass["Createtables"]
-    createTempTables = dbclass["CreateTemptables"]
-    new_updates = {}
-    
-    ct = createTempTables(
-        dbengine["session"],
-        dbengine["metadata"],
-        dbengine["engine"]
-    )
-    tempTableList = ct.tablelist
-
-    if options["apitype"].lower() == "tvmaze":
-        try:
-            new_updates = set_apienv(
-                TVMAZE_URLS,
-                uprocess,
-                jget,
-                dbengine,
-                dbExec,
-                tempTableList,
-                lprint,
-            )
-        except Exception as exc:  # noqa: PLC0301 - broad handler to log failures
-            lprint.logprint("error", f"Failed to set API env: {exc}")
-            return 1
-
-    from cache.process import Cacher
-
-    cacher = Cacher()
-    try:
-        cacher.process(options, jget, new_updates, ROOTDIR, TVMAZE_URLS)
-    except Exception as exc:
-        lprint.logprint("error", f"Cache processing failed: {exc}")
-        return 1
-
-    timeDifference = options["updatetype"]
-    currentTimestamp = time.time()
-    if timeDifference.lower() == "full" or options["initload"] == "1":
-        tdifference = 0
-    else:
-        tdifference = int(currentTimestamp) - int(
-            options[options["updatetype"]]
-        )
-    needed_updates = []
-    updateList = []
-    for result in new_updates:
-        updateid = result[0]
-        updatestamp = int(result[1])
-        tdif = int(tdifference)
-        if updatestamp > tdif:
-            needed_updates.append(updateid)
-            updateList.append(updateid)
-
-    lprint.logprint("info", "Update Type: " + str(options["updatetype"]))
-    updateRange = len(updateList)
-    lprint.logprint("info", "Number of updates: " + str(updateRange))
-    try:
-        cacher.update_cache(
-            jget, updateList, ROOTDIR, TVMAZE_URLS, options, lprint
-        )
-    except Exception as exc:
-        lprint.logprint("error", f"update_cache failed: {exc}")
-        return 1
-
-    countrylisting = dbExec.rawsql_select(
-        dbengine["engine"],
-        "select row_id, country_name from dbo.countrydata",
-        lprint
-    )
-    for country in countrylisting:
-        country_name = country[1]
-        country_id = country[0]
-        countrydictionary[country_name] = country_id
-    mongo_updater = dbmongo(options)
-    mongo_updater.update_mongo(
-        updateList,
-        ROOTDIR,
-        directories,
-        jget,
-        countrydictionary,
-        dbExec,
-        dbengine,
-        dboBase,
-        lprint,
-    )
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

update_mongodb.py

@@ -0,0 +1,305 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import os
+import time
+import sys
+from tqdm import tqdm
+import logs.Logger
+from db.functions import settype, dbmongo
+from datetime import datetime
+import json
+
+
+def set_apienv(urls, uprocess, dbengine, dbExec, updatesBase, lprint):
+    """Populate updates table from the API and return available updates.
+
+    Args:
+        urls: URL configuration with an "updatesurl" key.
+        uprocess: URL loader instance with `get_data`.
+        jget: JSON processor with `jconvert`.
+        dbengine: database engine dict (expects key "engine").
+        dbExec: database execution helper with `update_tvupdates` and `rawsql_select`.
+        updatesBase: list-like containing tables (uses index 8 for updates table).
+        lprint: logger instance with `logprint`.
+
+    Returns:
+        List of tuples (seriesid, timestamp) representing available updates.
+    """
+    #updatetable = updatesBase[8]
+    #inputdata = uprocess.get_data(urls["updatesurl"])
+    #lprint.logprint("info", f"Retrieved {len(availableupdates)} rows for processing......")
+    #
+    newupdates = dbExec.rawsql_select(
+        dbengine["engine"],
+        "select seriesid,timestamp from updates.tvupdates",
+        lprint
+    )
+    return newupdates
+
+
+def print_starttime(lprint, ts1):
+    lprint.logprint("info", f"Sync start time: {str(ts1)}")
+
+
+def find_file_in_multiple_dirs(filename, directories):
+    """
+    Checks if a file exists in any of the provided directories.
+
+    Args:
+        filename (str): The name of the file to search for.
+        directories (list): A list of directory paths to check.
+
+    Returns:
+        Path or None: The full path to the file if found, otherwise None.
+    """
+    directories_found = []
+    for directory in directories:
+        if os.path.isfile(f'{directory}{filename}'):
+            directories_found.append(directory)
+                
+    return directories_found
+        
+def load_json_to_mongodb(mongo_updater, directories, update_list, updateDict, lprint, track_changes=True):
+    """
+    Load downloaded JSON files into MongoDB collections with change detection.
+    
+    Processes series, episodes, cast, and crew data from cached JSON files
+    and inserts or updates them in the appropriate MongoDB collections.
+    Automatically detects if documents are new, updated, or unchanged by comparing
+    updateDict timestamps with existing document 'updated' fields.
+    
+    Args:
+        mongo_updater: dbmongo instance for MongoDB operations
+        directories: dict containing cache directory paths for different data types
+        update_list: list of series IDs to process
+        updateDict: dict with seriesid as key and timestamp as value
+        lprint: logger instance for logging operations
+        track_changes: bool, if True tracks inserted vs updated documents (default: True)
+    
+    Returns:
+        tuple: (successful_count, failed_count, stats_dict) for series processed
+               stats_dict contains: {'inserted': count, 'updated': count}
+    """
+    successful_count = 0
+    failed_count = 0
+    stats = {'series_inserted': 0, 'series_updated': 0, 'series_skipped': 0, 
+             'episodes_inserted': 0, 'episodes_updated': 0, 'episodes_skipped': 0, 
+             'cast_inserted': 0, 'cast_updated': 0, 'cast_skipped': 0,
+             'crew_inserted': 0, 'crew_updated': 0, 'crew_skipped': 0}
+    
+    lprint.logprint("info", f"Starting to load {len(update_list)} series into MongoDB")
+    
+    for seriesid in tqdm(update_list[1:-1], desc="Loading series data into MongoDB", unit="series"):
+        seriesid_str = int(seriesid)
+        try:
+            # Load series data
+            series_file = f"{directories['SERIESDIR']}{seriesid}.json"
+            if os.path.exists(series_file):
+                try:
+                    with open(series_file, 'r') as f:
+                        series_data = json.load(f)
+                    
+                    # Insert or update series in MongoDB
+                    if series_data:
+                        if track_changes:
+                            existing = mongo_updater.mgseries.find_one({"id": series_data.get("id")})
+                            #seriesid_str = str(seriesid)
+                            update_timestamp = updateDict.get(seriesid_str)
+                            if existing:
+                                # Check if data has changed by comparing update timestamp
+                                if update_timestamp and int(update_timestamp) > existing.get("updated", 0):
+                                    mongo_updater.bulk_upsert_by_id(mongo_updater.mgseries, [series_data])
+                                    stats['series_updated'] += 1
+                                    lprint.logprint("debug", f"Updated series data for ID {seriesid}")
+                                else:
+                                    lprint.logprint("debug", f"Series {seriesid} unchanged, skipping")
+                                    stats['series_skipped'] += 1
+                                    pass  # Document unchanged
+                            else:
+                                mongo_updater.bulk_upsert_by_id(mongo_updater.mgseries, [series_data])
+                                stats['series_inserted'] += 1
+                                lprint.logprint("debug", f"Inserted new series data for ID {seriesid}")
+                                # Load episodes data
+                                episodes_file = f"{directories['EPISODEDIR']}{seriesid}.json"
+                                if os.path.exists(episodes_file):
+                                    try:
+                                        with open(episodes_file, 'r') as f:
+                                            episodes_data = json.load(f)
+                                        
+                                        if isinstance(episodes_data, list) and episodes_data:
+                                            # Add seriesid to each episode
+                                            for episode in episodes_data:
+                                                episode['seriesid'] = seriesid_str
+                                            
+                                            if track_changes:
+                                                for episode in episodes_data:
+                                                    existing = mongo_updater.mgepisodes.find_one({"id": episode.get("id")})
+                                                    #seriesid_str = str(seriesid)
+                                                    update_timestamp = updateDict.get(seriesid_str)
+                                                    if existing:
+                                                        if update_timestamp and int(update_timestamp) > existing.get("updated", 0):
+                                                            mongo_updater.bulk_upsert_by_id(mongo_updater.mgepisodes, episodes_data)
+                                                            stats['episodes_updated'] += 1
+                                                        else: 
+                                                            lprint.logprint("debug", f"Episode Data for Series {seriesid} unchanged, skipping")
+                                                            pass  # Document unchanged
+                                                    else:
+                                                        mongo_updater.bulk_upsert_by_id(mongo_updater.mgepisodes, episodes_data)
+                                                        stats['episodes_inserted'] += 1
+                                            
+                                            #mongo_updater.bulk_upsert_by_id(mongo_updater.mgepisodes, episodes_data)
+                                            lprint.logprint("debug", f"Loaded {len(episodes_data)} episodes for series {seriesid}")
+                                    except Exception as e:
+                                        lprint.logprint("warning", f"Failed to load episodes file for {seriesid}: {e}")
+                                
+                                # Load cast data
+                                cast_file = f"{directories['CASTDIR']}{seriesid}.json"
+                                if os.path.exists(cast_file):
+                                    try:
+                                        with open(cast_file, 'r') as f:
+                                            cast_data = json.load(f)
+                                        
+                                        if isinstance(cast_data, list) and cast_data:
+                                            actors = []
+                                            characters = []
+                                            for item in cast_data:
+                                                if 'person' in item and item['person']:
+                                                    item['person']['seriesid'] = seriesid_str
+                                                    actors.append(item['person'])
+                                                if 'character' in item and item['character']:
+                                                    item['character']['seriesid'] = seriesid_str
+                                                    characters.append(item['character'])
+                                            
+                                            if track_changes:
+                                                for actor in actors:
+                                                    existing = mongo_updater.mgactors.find_one({"id": actor.get("id")})
+                                                    #seriesid_str = str(seriesid)
+                                                    update_timestamp = updateDict.get(seriesid_str)
+                                                    if existing:
+                                                        if update_timestamp and int(update_timestamp) > existing.get("updated", 0):
+                                                            stats['cast_updated'] += 1
+                                                    else:
+                                                        stats['cast_inserted'] += 1
+                                            
+                                            if actors:
+                                                mongo_updater.bulk_upsert_by_id(mongo_updater.mgactors, actors)
+                                            if characters:
+                                                mongo_updater.bulk_upsert_by_id(mongo_updater.mgcharacters, characters)
+                                            lprint.logprint("debug", f"Loaded cast data for series {seriesid}")
+                                    except Exception as e:
+                                        lprint.logprint("warning", f"Failed to load cast file for {seriesid}: {e}")
+                                
+                                # Load crew data
+                                crew_file = f"{directories['CREWDIR']}{seriesid}.json"
+                                if os.path.exists(crew_file):
+                                    try:
+                                        with open(crew_file, 'r') as f:
+                                            crew_data = json.load(f)
+                                        
+                                        if isinstance(crew_data, list) and crew_data:
+                                            crew_list = []
+                                            for item in crew_data:
+                                                if 'person' in item and item['person']:
+                                                    item['person']['seriesid'] = seriesid_str
+                                                    item['person']['crew_type'] = item.get('type', 'unknown')
+                                                    crew_list.append(item['person'])
+                                            
+                                            if track_changes:
+                                                for crew in crew_list:
+                                                    existing = mongo_updater.mgcrew.find_one({"id": crew.get("id")})
+                                                    #seriesid_str = str(seriesid)
+                                                    update_timestamp = updateDict.get(seriesid_str)
+                                                    if existing:
+                                                        if update_timestamp and int(update_timestamp) > existing.get("updated", 0):
+                                                            stats['crew_updated'] += 1
+                                                    else:
+                                                        stats['crew_inserted'] += 1
+                                            
+                                            if crew_list:
+                                                mongo_updater.bulk_upsert_by_id(mongo_updater.mgcrew, crew_list)
+                                                lprint.logprint("debug", f"Loaded crew data for series {seriesid}")
+                                    except Exception as e:
+                                        lprint.logprint("warning", f"Failed to load crew file for {seriesid}: {e}")
+                                
+                                successful_count += 1
+                                lprint.logprint("info", f"Successfully processed series {seriesid}")                        
+                        #mongo_updater.bulk_upsert_by_id(mongo_updater.mgseries, [series_data])
+                except Exception as e:
+                    lprint.logprint("warning", f"Failed to load series file for {seriesid}: {e}")
+                    failed_count += 1
+                    continue
+        except Exception as e:
+            failed_count += 1
+            lprint.logprint("error", f"Unexpected error processing series {seriesid}: {e}")
+    
+    lprint.logprint("info", f"MongoDB loading complete.")
+    return successful_count, failed_count, stats
+
+def main() -> int:
+    ROOTDIR = os.getcwd()
+    directories = {}
+    # Use cross-platform path builders and ensure directories exist
+    LOGDIR = os.path.join(ROOTDIR, "log")
+    directories["SERIESDIR"] = os.path.join(ROOTDIR, "cache", "series") + os.sep
+    directories["EPISODEDIR"] = os.path.join(ROOTDIR, "cache", "episodes") + os.sep
+    directories["CASTDIR"] = os.path.join(ROOTDIR, "cache", "cast") + os.sep
+    directories["GUESTCASTDIR"] = os.path.join(ROOTDIR, "cache", "guestcast") + os.sep
+    directories["CREWDIR"] = os.path.join(ROOTDIR, "cache", "crew") + os.sep
+    #directories["GUESTCREWDIR"] = os.path.join(ROOTDIR, "cache", "guestcrew") + os.sep    
+    #directories["CREDITSDIR"] = os.path.join(ROOTDIR, "cache", "credits") + os.sep
+    #directories["ALIASDIR"] = os.path.join(ROOTDIR, "cache", "aliases") + os.sep
+    tempdir = os.path.join(ROOTDIR, 'temp') + os.sep
+    
+    lprint = logs.Logger.LPrint(LOGDIR, False)
+    lprint.start_log()
+    import settings.config
+
+    updateList = []
+    updateDict = {}
+    needed_updates = []
+    
+    new_updates = open(f'{tempdir}tvmaze_updates.json', 'r').readlines()
+    for line in new_updates:
+        line = line.strip().replace('\n', '').replace('\r', '')
+        linefield = line.split(":")
+        updateList.append(linefield[0].replace('"',''))
+        try:
+            updateDict[linefield[0].replace('"','')] = linefield[1].replace(',','').strip()
+        except IndexError:
+            lprint.logprint("warning", f"Failed to parse line: {line}")
+    # for result in new_updates:
+    #     updateid = result[0]
+    #     updatestamp = int(result[1])
+    #     tdif = int(tdifference)
+    #     if updatestamp > tdif:
+    #         needed_updates.append(updateid)
+    #         updateList.append(updateid)
+            
+    config_options = settings.config.Config(ROOTDIR)
+    options = config_options.config_options
+    mongo_updater = dbmongo(options)
+    dbtype, apitype = options["dbtype"], options["apitype"]
+    
+    lprint.logprint("info", "Creating JSON instance for data retrieval.")
+    lprint.logprint("info", "JSON instance created.")
+    lprint.logprint("info", f"Update type is {options['updatetype']}")
+    timedifference = options["updatetype"]
+    currenttimestamp = time.time()
+    if timedifference.lower() == "full":
+        timedifference = int(currenttimestamp)
+    basetime = str(currenttimestamp).split(".")
+    currenttimestamp = int(basetime[0])
+    ts1 = str(datetime.now()).split(".")[0]
+    lprint.logprint("info", "Sync start time: " + str(ts1))
+    print_starttime(lprint, ts1)
+    
+    lprint.screenprint("info", f"Preparing to load JSON files into MongoDB.")
+    lprint.screenprint("info", f"Loading JSON files into MongoDB.")
+    successful, failed , st= load_json_to_mongodb(mongo_updater, directories, updateList, updateDict, lprint)
+    lprint.screenprint("info", f"JSON loading complete. Successful: {successful}, Failed: {failed}")
+    lprint.screenprint("info", f"Stats: {st}")
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+

yaml_ddl/tableddl_from_yaml.py

@@ -1,138 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-from pymongo import MongoClient
-import yaml
-import json
-import psycopg2
-
-class ColumnMeta:
-    def __init__(self, name, type, description=None):
-        self.name = name
-        self.type = type
-        self.description = description
-
-    def from_dict(value: dict):
-        return ColumnMeta(name=value['name'], type=value['type'], description=value.get('description'))
-
-
-class TableMeta:
-    def __init__(self, name, columns, description=None):
-        self.name = name
-        self.columns = columns 
-        self.description = description
-        
-
-    def from_dict(value):
-        columns = [ColumnMeta.from_dict(v) for v in value['columns']]
-        return TableMeta(name=value['name'], columns=columns, description=value.get('description'))
-
-    def from_file(path):
-        file = open(path, 'r')
-        yaml_to_dict = yaml.safe_load(file.read())
-        try:
-            file.close()
-        except Exception:
-            pass
-        return TableMeta.from_dict(yaml_to_dict)
-
-
-try:
-    connection_string = (
-        (
-            'postgres://postgres:Optimus0329@192.168.128.7:5432/media_dbsync'
-            '?options=-csearch_path%3Ddbo,public,updates'
-        )
-    )
-    pgclient = psycopg2.connect(connection_string)
-    pgcursor = pgclient.cursor()
-    print(pgclient.get_dsn_parameters(), "\n")
-    print("Connected to postgresql database server.")
-    pgcursor.execute("SELECT version();")
-    record = pgcursor.fetchone()
-    print("You are connected to - ", record, "\n")
-except Exception as e:
-    print(f"Unable to connect to postgresql database server:{e}\nExitting!")
-    exit()
-
-
-host = '192.168.128.8'
-port = 27017
-print(f"Connecting to mongodb at {host}:{port}")
-
-try:
-    connection_string = (
-        f"mongodb://{host}:{port}"
-    )
-    mgclient = MongoClient(connection_string)
-    mgdb = mgclient['test2']
-except BaseException as e:
-    print(f"Unable to connect to mongodb.{e}\nExitting!")
-    exit()
-else:
-    mgseries = mgdb["series"]
-    mgseries.create_index('id', unique=True)
-    mgepisodes = mgdb["episodes"]
-    mgepisodes.create_index('id', unique=True)
-    mgactors = mgdb["actors"]
-    mgactors.create_index('id', unique=True)
-    mgcharacters = mgdb["characters"]
-    mgcharacters.create_index('id', unique=True)
-    mgcrew = mgdb["crew"]
-    mgcrew.create_index('id', unique=True)
-    print(
-        f"Connection to mongodb at {host}:"
-        f"{port} successful."
-    )
-os.chdir('C:\\Projects\\new_dbsync\\db\\ddl')
-# Create the database and tables in PostgreSQL
-# Check if the database exists, if not create it
-# Check if the tables exist, if not create them
-# possibly add , 'genre_ddl.yaml', 'language_ddl.yaml' down the road
-ddl_list = [
-    # 'seriesdata_ddl.yaml',
-    # 'epdata_ddl.yaml',
-    # 'actordata_ddl.yaml',
-    # 'castdata_ddl.yaml',
-    # 'crewdata_ddl.yaml',
-    # 'countrydata_ddl.yaml',
-    # 'networkdata_ddl.yaml',
-    # 'webchanneldata_ddl.yaml',
-    # 'temp_seriesdata_ddl.yaml',
-    # 'temp_epdata_ddl.yaml',
-    # 'temp_actordata_ddl.yaml',
-    # 'temp_castdata_ddl.yaml',
-    # 'temp_crewdata_ddl.yaml',
-    # 'temp_countrydata_ddl.yaml',
-    # 'temp_networkdata_ddl.yaml',
-    # 'temp_webchanneldata_ddl.yaml',
-    # 'tvupdates_ddl.yaml'
-    'db_schema.yaml'
-]
-for ddl in ddl_list:
-    path = os.path.join(os.getcwd(), ddl)
-    table_meta = TableMeta.from_file(path)
-
-    columnlist = []
-    for column in table_meta.columns:
-        columnlist.append(f"{column.name} {column.type}")
-        table_meta.columns = ', '.join(columnlist)
-        #print(column.name, column.type)
-
-    sql_command = f"DROP TABLE IF EXISTS {table_meta.name};CREATE TABLE IF NOT EXISTS {table_meta.name} ({table_meta.columns})"
-    #sql_command = f"CREATE TABLE IF NOT EXISTS {table_meta.name} ({table_meta.columns})"
-
-    try:
-        pgcursor.execute(sql_command)
-        pgclient.commit()
-    except Exception as e:
-        print(f"Unable to create table {table_meta.name} in postgresql database server:{e}\nExitting!")
-        exit()
-        #print(sql_command)
-
-pgcursor.close()
-pgclient.close()
-print("PostgreSQL connection is closed")
-mgclient.close()
-print("MongoDB connection is closed")
-exit()

yaml_ddl/test_tableddl_from_yaml.py

@@ -1,118 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-from pymongo import MongoClient
-import yaml
-import json
-import psycopg2
-
-class ColumnMeta:
-    def __init__(self, name, type, description=None):
-        self.name = name
-        self.type = type
-        self.description = description
-
-    def from_dict(value: dict):
-        return ColumnMeta(name=value['name'], type=value['type'], description=value.get('description'))
-
-
-class TableMeta:
-    def __init__(self, name, columns, description=None):
-        self.name = name
-        self.columns = columns 
-        self.description = description
-
-    def from_dict(value):
-        columns = [ColumnMeta.from_dict(v) for v in value['columns']]
-        return TableMeta(name=value['name'], columns=columns, description=value.get('description'))
-
-    def from_file(path):
-        file = open(path, 'r')
-        yaml_to_dict = yaml.safe_load(file.read())
-        try:
-            file.close()
-        except Exception:
-            pass
-        return TableMeta.from_dict(yaml_to_dict)
-
-
-try:
-    connection_string = (
-        (
-            'postgres://postgres:Optimus0329@192.168.128.7:5432/media_dbsync'
-            '?options=-csearch_path%3Ddbo,public,updates'
-        )
-    )
-    pgclient = psycopg2.connect(connection_string)
-    pgcursor = pgclient.cursor()
-    print(pgclient.get_dsn_parameters(), "\n")
-    print("Connected to postgresql database server.")
-    pgcursor.execute("SELECT version();")
-    record = pgcursor.fetchone()
-    print("You are connected to - ", record, "\n")
-except Exception as e:
-    print(f"Unable to connect to postgresql database server:{e}\nExitting!")
-    exit()
-
-
-host = '192.168.128.8'
-port = 27017
-print(f"Connecting to mongodb at {host}:{port}")
-
-try:
-    connection_string = (
-        f"mongodb://{host}:{port}"
-    )
-    mgclient = MongoClient(connection_string)
-    mgdb = mgclient['test2']
-except BaseException as e:
-    print(f"Unable to connect to mongodb.{e}\nExitting!")
-    exit()
-else:
-    mgseries = mgdb["series"]
-    mgseries.create_index('id', unique=True)
-    mgepisodes = mgdb["episodes"]
-    mgepisodes.create_index('id', unique=True)
-    mgactors = mgdb["actors"]
-    mgactors.create_index('id', unique=True)
-    mgcharacters = mgdb["characters"]
-    mgcharacters.create_index('id', unique=True)
-    mgcrew = mgdb["crew"]
-    mgcrew.create_index('id', unique=True)
-    print(
-        f"Connection to mongodb at {host}:"
-        f"{port} successful."
-    )
-os.chdir('C:\\Projects\\new_dbsync\\db\\ddl')
-# Create the database and tables in PostgreSQL
-# Check if the database exists, if not create it
-# Check if the tables exist, if not create them
-# possibly add , 'genre_ddl.yaml', 'language_ddl.yaml' down the road
-ddl_list = os.listdir(os.getcwd())
-for ddl in ddl_list:
-    path = os.path.join(os.getcwd(), ddl)
-    table_meta = TableMeta.from_file(path)
-
-    columnlist = []
-    for column in table_meta.columns:
-        columnlist.append(f"{column.name} {column.type}")
-        table_meta.columns = ', '.join(columnlist)
-        #print(column.name, column.type)
-
-    sql_command = f"DROP TABLE IF EXISTS {table_meta.name};CREATE TABLE IF NOT EXISTS {table_meta.name} ({table_meta.columns})"
-    #sql_command = f"CREATE TABLE IF NOT EXISTS {table_meta.name} ({table_meta.columns})"
-
-    try:
-        pgcursor.execute(sql_command)
-        pgclient.commit()
-    except Exception as e:
-        print(f"Unable to create table {table_meta.name} in postgresql database server:{e}\nExitting!")
-        exit()
-        #print(sql_command)
-
-pgcursor.close()
-pgclient.close()
-print("PostgreSQL connection is closed")
-mgclient.close()
-print("MongoDB connection is closed")
-exit()