Merge pull request 'development' (#4) from development into main
Reviewed-on: #4
This commit is contained in:
+10
-2
@@ -5,11 +5,19 @@
|
||||
/.vs/new_dbsync/FileContentIndex
|
||||
/.vs
|
||||
*.pyc
|
||||
*.json
|
||||
/cache/series/*.json
|
||||
/cache/episodes/*.json
|
||||
/cache/cast/*.json
|
||||
/cache/crew/*.json
|
||||
/cache/guestcast/*.json
|
||||
/cache/guestcrew/*.json
|
||||
/cache/aliases/*.json
|
||||
*.DS_Store
|
||||
*.log
|
||||
/pyproject.toml
|
||||
dbsync.log.*
|
||||
.idea
|
||||
/log
|
||||
venv
|
||||
/venv
|
||||
/temp/post_tvmaze_updates.json.*
|
||||
/archived
|
||||
@@ -1,15 +0,0 @@
|
||||
CC = gcc
|
||||
CFLAGS = -fPIC -O2 -I/usr/include
|
||||
LDFLAGS = -shared
|
||||
LIBS = -lpq
|
||||
|
||||
TARGET = pg_vtab.so
|
||||
SRC = pg_vtab.c
|
||||
|
||||
all: $(TARGET)
|
||||
|
||||
$(TARGET): $(SRC)
|
||||
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)
|
||||
|
||||
clean:
|
||||
rm -f $(TARGET)
|
||||
@@ -1,38 +0,0 @@
|
||||
# SQLite -> PostgreSQL virtual table (minimal)
|
||||
|
||||
This directory contains a minimal SQLite extension that implements a read-only virtual table backed directly by PostgreSQL using libpq.
|
||||
|
||||
Build
|
||||
-----
|
||||
You need `libpq` (Postgres client) and a C compiler. On Debian/Ubuntu:
|
||||
|
||||
```bash
|
||||
sudo apt-get install libpq-dev build-essential
|
||||
make
|
||||
```
|
||||
|
||||
This produces `pg_vtab.so`.
|
||||
|
||||
Usage
|
||||
-----
|
||||
Start `sqlite3`, load the extension and create a virtual table pointing at a Postgres table:
|
||||
|
||||
```sql
|
||||
.load ./pg_vtab.so
|
||||
CREATE VIRTUAL TABLE remote_tbl USING pg_vtab('host=localhost dbname=mydb user=me password=secret', 'public.mytable');
|
||||
SELECT * FROM remote_tbl LIMIT 10;
|
||||
```
|
||||
|
||||
Limitations
|
||||
-----------
|
||||
- Minimal proof-of-concept: supports only full table scans.
|
||||
- All columns are presented as `TEXT`.
|
||||
- No constraint pushdown or type mapping sophistication.
|
||||
- Error handling and SQL injection hygiene are minimal; do not use on untrusted inputs without review.
|
||||
|
||||
Next steps (optional)
|
||||
---------------------
|
||||
- Add support for WHERE constraints and indexed access (`xBestIndex`).
|
||||
- Map Postgres types more accurately to SQLite storage classes.
|
||||
- Implement server-side cursors for large tables.
|
||||
- Harden error reporting and resource cleanup.
|
||||
+6
-7
@@ -250,7 +250,6 @@ class CreateTempTables():
|
||||
pass
|
||||
self.create(Base, engine, metadata, session, self.tablelist)
|
||||
session.commit()
|
||||
#self.send(tablelist)
|
||||
|
||||
def settables(self, metadata, engine):
|
||||
tablelist = []
|
||||
@@ -379,12 +378,12 @@ class CreateTempTables():
|
||||
schema = "updates",
|
||||
extend_existing=True)
|
||||
tablelist.append(countries)
|
||||
tvupdates = Table('tvupdates', metadata,
|
||||
Column('seriesid', BIGINT, primary_key = True),
|
||||
Column('timestamp', VARCHAR(15)),
|
||||
schema = "updates",
|
||||
extend_existing=True)
|
||||
tablelist.append(tvupdates)
|
||||
# tvupdates = Table('tvupdates', metadata,
|
||||
# Column('seriesid', BIGINT, primary_key = True),
|
||||
# Column('timestamp', VARCHAR(15)),
|
||||
# schema = "updates",
|
||||
# extend_existing=True)
|
||||
# tablelist.append(tvupdates)
|
||||
return(tablelist)
|
||||
|
||||
def create(self, Base, engine, metadata, session, tablelist):
|
||||
|
||||
@@ -24,10 +24,14 @@ class Dbexec():
|
||||
)
|
||||
]
|
||||
|
||||
# Commented out code for applying updates for now. Will re-enable later after downloads are verified.
|
||||
print('Applying updates to table.')
|
||||
with engine.connect() as conn:
|
||||
conn.execute(text(f"truncate table {tablename}"))
|
||||
conn.execute(tablename.insert(), updatelist)
|
||||
for key, value in available_updates.items():
|
||||
seriesid = str(key)
|
||||
timestamp = value
|
||||
conn.execute(text(f"INSERT INTO {tablename} (seriesid, timestamp) VALUES (:seriesid, :timestamp)"), {'seriesid': seriesid, 'timestamp': timestamp})
|
||||
conn.commit()
|
||||
print('Update of tvupdates table is complete.')
|
||||
|
||||
|
||||
@@ -1,105 +0,0 @@
|
||||
__author__ = 'Wendell Jones'
|
||||
import json
|
||||
import progressbar
|
||||
from url.process import Loadurl
|
||||
|
||||
class Jsonload():
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def load(self, rawdata):
|
||||
jdata = {}
|
||||
jdata = json.loads(rawdata)
|
||||
newdata = self.dump(jdata)
|
||||
return(newdata)
|
||||
|
||||
def dump(self, rawdata):
|
||||
jdata = json.dumps(rawdata, indent=4)
|
||||
return(jdata)
|
||||
|
||||
def jconvert(self, inputdata):
|
||||
outputdata = json.loads(inputdata)
|
||||
return(outputdata)
|
||||
|
||||
def populate_cache(self, jdata, jfile):
|
||||
try:
|
||||
with open(jfile, 'w') as datafile:
|
||||
datafile.write(jdata)
|
||||
except:
|
||||
print('Unable to save cache file.')
|
||||
else:
|
||||
datafile.close()
|
||||
|
||||
def fetch_json(self, weburl, seriesid, cachefile):
|
||||
uprocess = Loadurl()
|
||||
try:
|
||||
webdata = uprocess.get_data(weburl)
|
||||
jdata = self.load(webdata)
|
||||
except Exception as e:
|
||||
print('Unable to get JSON data for ' + str(seriesid) + ': ' + str(e))
|
||||
else:
|
||||
if jdata == '[]':
|
||||
pass
|
||||
else:
|
||||
self.populate_cache(jdata, cachefile)
|
||||
|
||||
def processSeriesJson(self, seriesdata):
|
||||
seriesresults = {}
|
||||
networkchannel = {}
|
||||
webchannel = {}
|
||||
country = {}
|
||||
|
||||
seriesresults['id'] = seriesdata['id']
|
||||
seriesresults['url'] = seriesdata['url']
|
||||
seriesresults['name'] = seriesdata['name']
|
||||
seriesresults['type'] = seriesdata['type']
|
||||
seriesresults['language'] = seriesdata['language']
|
||||
seriesresults['genres'] = ', '.join(seriesdata['genres'])
|
||||
seriesresults['status'] = seriesdata['status']
|
||||
seriesresults['runtime'] = seriesdata['runtime']
|
||||
seriesresults['premiered'] = seriesdata['premiered']
|
||||
seriesresults['officalsite'] = seriesdata['officialSite']
|
||||
if len(seriesdata['schedule']['days']) > 0:
|
||||
seriesresults['days'] = seriesdata['schedule']['days'][0]
|
||||
seriesresults['time'] = seriesdata['schedule']['time']
|
||||
seriesresults['rating'] = seriesdata['rating']['average']
|
||||
seriesresults['weight'] = seriesdata['weight']
|
||||
seriesresults['networkid'] = seriesdata['id']
|
||||
seriesresults['webchannel'] = seriesdata['id']
|
||||
seriesresults['tvrage'] = seriesdata['externals']['tvrage']
|
||||
seriesresults['thetvdb'] = seriesdata['externals']['thetvdb']
|
||||
seriesresults['imdb'] = seriesdata['externals']['imdb']
|
||||
if seriesdata['image']:
|
||||
seriesresults['medium'] = seriesdata['image']['medium']
|
||||
seriesresults['original'] = seriesdata['image']['original']
|
||||
seriesresults['summary'] = seriesdata['summary']
|
||||
seriesresults['serieslink'] = seriesdata['_links']['self']['href']
|
||||
if 'previousepisode' in seriesdata['_links'].keys():
|
||||
seriesresults['previousepisode'] = seriesdata['_links']['previousepisode']['href']
|
||||
if 'nextepisode' in seriesdata['_links'].keys():
|
||||
seriesresults['nextepisode'] = seriesdata['_links']['nextepisode']['href']
|
||||
|
||||
|
||||
if seriesdata['network'] and seriesdata['network'] != None:
|
||||
networkchannel['id'] = seriesdata['network']['id']
|
||||
networkchannel['name'] = seriesdata['network']['name']
|
||||
#networkchannel['countryid'] = country['id']
|
||||
if seriesdata['network']['country']:
|
||||
# country['id'] = "auto incremented id"
|
||||
country['name'] = seriesdata['network']['country']['name']
|
||||
country['code'] = seriesdata['network']['country']['code']
|
||||
country['timezone'] = seriesdata['network']['country']['timezone']
|
||||
if seriesdata['webChannel'] and seriesdata['webChannel'] != None:
|
||||
webchannel['id'] = seriesdata['webChannel']['id']
|
||||
webchannel['name'] = seriesdata['webChannel']['name']
|
||||
#webchannel['countryid'] = country['id']
|
||||
if seriesdata['webChannel']['country']:
|
||||
# country['id'] = "auto incremented id"
|
||||
country['name'] = seriesdata['webchannel']['country']['name']
|
||||
country['code'] = seriesdata['webchannel']['country']['code']
|
||||
country['timezone'] = seriesdata['webchannel']['country']['timezone']
|
||||
|
||||
return(seriesresults, networkchannel, country, webchannel)
|
||||
|
||||
def __del__(self):
|
||||
pass
|
||||
+5
-84302
File diff suppressed because it is too large
Load Diff
@@ -79,6 +79,23 @@ class LPrint:
|
||||
else:
|
||||
logging.error(f'Unknown log level: {loglevel}. Message: {logmessage}')
|
||||
|
||||
def screenprint(self, loglevel, logmessage):
|
||||
"""Log a message with a timestamp."""
|
||||
logstamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||
log_message = f'{logstamp}: {logmessage}'
|
||||
loglevel = loglevel.lower()
|
||||
if loglevel == 'debug':
|
||||
logging.debug(log_message)
|
||||
print(f"{log_message}")
|
||||
elif loglevel == 'warning':
|
||||
logging.warning(log_message)
|
||||
print(f"{log_message}")
|
||||
elif loglevel == 'info':
|
||||
logging.info(log_message)
|
||||
print(f"{log_message}")
|
||||
else:
|
||||
logging.error(f'Unknown log level: {loglevel}. Message: {logmessage}')
|
||||
|
||||
def create_rotating_log(self, logpath, max_bytes=10_000_000, backup_count=5):
|
||||
"""Create a rotating log handler."""
|
||||
logger = logging.getLogger("Rotating Log")
|
||||
|
||||
+205
@@ -0,0 +1,205 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Convert a MongoDB collection into a PostgreSQL table.
|
||||
|
||||
Contains `convert_mongo_to_pg(mongo_uri, mongo_db, collection_name, pg_dsn, pg_table=None, ...)`
|
||||
and a small CLI.
|
||||
|
||||
Notes:
|
||||
- Uses `pymongo` to read documents and `psycopg2` to write to Postgres.
|
||||
- Infers basic column types from a sample of documents; nested objects/arrays
|
||||
are stored as `JSONB`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
import pymongo
|
||||
import psycopg2
|
||||
import psycopg2.extras as pgextras
|
||||
import psycopg2.sql as sql
|
||||
|
||||
LOG = logging.getLogger("mongo2pgsql")
|
||||
|
||||
|
||||
def _detect_type(value: Any) -> str:
|
||||
if value is None:
|
||||
return "null"
|
||||
if isinstance(value, bool):
|
||||
return "boolean"
|
||||
if isinstance(value, int) and not isinstance(value, bool):
|
||||
return "integer"
|
||||
if isinstance(value, float):
|
||||
return "real"
|
||||
if isinstance(value, (dict, list)):
|
||||
return "jsonb"
|
||||
return "text"
|
||||
|
||||
|
||||
def _merge_types(types: List[str]) -> str:
|
||||
# priority: jsonb > text > real > integer > boolean > null
|
||||
s = set(types)
|
||||
if "jsonb" in s:
|
||||
return "jsonb"
|
||||
if "text" in s:
|
||||
return "text"
|
||||
if "real" in s:
|
||||
return "real"
|
||||
if "integer" in s:
|
||||
return "integer"
|
||||
if "boolean" in s:
|
||||
return "boolean"
|
||||
return "text"
|
||||
|
||||
|
||||
def _pg_type_for(kind: str) -> str:
|
||||
return {
|
||||
"integer": "BIGINT",
|
||||
"real": "DOUBLE PRECISION",
|
||||
"boolean": "BOOLEAN",
|
||||
"jsonb": "JSONB",
|
||||
"text": "TEXT",
|
||||
}.get(kind, "TEXT")
|
||||
|
||||
|
||||
def convert_mongo_to_pg(
|
||||
mongo_uri: str,
|
||||
mongo_db: str,
|
||||
collection_name: str,
|
||||
pg_dsn: str,
|
||||
pg_table: Optional[str] = None,
|
||||
sample_size: int = 500,
|
||||
batch_size: int = 1000,
|
||||
create_table: bool = True,
|
||||
pk_field: str = "_id",
|
||||
) -> None:
|
||||
"""Copy a MongoDB collection to PostgreSQL.
|
||||
|
||||
- `pg_table` defaults to `collection_name` if omitted.
|
||||
- Nested documents/arrays are stored as JSONB.
|
||||
- `_id` is stored as TEXT by default.
|
||||
"""
|
||||
if pg_table is None:
|
||||
pg_table = collection_name
|
||||
|
||||
client = pymongo.MongoClient(mongo_uri)
|
||||
coll = client[mongo_db][collection_name]
|
||||
|
||||
sample = list(coll.find({}, projection=None, limit=sample_size))
|
||||
if not sample:
|
||||
raise ValueError("collection is empty or not found")
|
||||
|
||||
# gather all field names and observed types
|
||||
field_types: Dict[str, List[str]] = {}
|
||||
for doc in sample:
|
||||
for k, v in doc.items():
|
||||
t = _detect_type(v)
|
||||
field_types.setdefault(k, []).append(t)
|
||||
|
||||
# finalize types
|
||||
final_types: Dict[str, str] = {}
|
||||
for k, types in field_types.items():
|
||||
merged = _merge_types(types)
|
||||
final_types[k] = _pg_type_for(merged)
|
||||
|
||||
# ensure primary key present
|
||||
if pk_field not in final_types:
|
||||
final_types[pk_field] = "TEXT"
|
||||
|
||||
cols = list(final_types.items())
|
||||
|
||||
# create table
|
||||
if create_table:
|
||||
# Build CREATE TABLE statement using psycopg2.sql for safe identifiers
|
||||
col_defs = []
|
||||
for name, ptype in cols:
|
||||
col_defs.append(sql.SQL("{} {}").format(sql.Identifier(name), sql.SQL(ptype)))
|
||||
|
||||
create_body = sql.SQL(", ").join(col_defs)
|
||||
if pk_field in final_types:
|
||||
pk_clause = sql.SQL(", PRIMARY KEY ({})").format(sql.SQL(",").join(map(sql.Identifier, [pk_field])))
|
||||
create_body = create_body + pk_clause
|
||||
|
||||
create_stmt = sql.SQL("CREATE TABLE IF NOT EXISTS {} ({})").format(sql.Identifier(pg_table), create_body)
|
||||
with psycopg2.connect(pg_dsn) as pgconn:
|
||||
with pgconn.cursor() as cur:
|
||||
LOG.info("Creating table %s", pg_table)
|
||||
cur.execute(create_stmt)
|
||||
pgconn.commit()
|
||||
|
||||
# stream documents and insert
|
||||
insert_cols = [n for n, _ in cols]
|
||||
|
||||
with psycopg2.connect(pg_dsn) as pgconn:
|
||||
with pgconn.cursor() as cur:
|
||||
# build INSERT statement for execute_values: it expects a single %s
|
||||
base_insert = sql.SQL("INSERT INTO {} ({}) VALUES %s").format(
|
||||
sql.Identifier(pg_table),
|
||||
sql.SQL(', ').join(map(sql.Identifier, insert_cols)),
|
||||
).as_string(pgconn)
|
||||
batch = []
|
||||
# Use an explicit session when using no_cursor_timeout to avoid pymongo warning
|
||||
with client.start_session() as session:
|
||||
cursor = coll.find({}, no_cursor_timeout=True, session=session).batch_size(batch_size)
|
||||
try:
|
||||
for doc in cursor:
|
||||
row = []
|
||||
for col_name in insert_cols:
|
||||
v = doc.get(col_name)
|
||||
if v is None:
|
||||
row.append(None)
|
||||
continue
|
||||
# convert ObjectId and other non-JSON types
|
||||
if isinstance(v, (dict, list)):
|
||||
row.append(pgextras.Json(v))
|
||||
else:
|
||||
row.append(str(v) if col_name == pk_field else v)
|
||||
batch.append(tuple(row))
|
||||
if len(batch) >= batch_size:
|
||||
pgextras.execute_values(cur, base_insert, batch, template=None, page_size=batch_size)
|
||||
pgconn.commit()
|
||||
batch = []
|
||||
finally:
|
||||
try:
|
||||
cursor.close()
|
||||
except Exception:
|
||||
pass
|
||||
if batch:
|
||||
pgextras.execute_values(cur, base_insert, batch, template=None, page_size=len(batch))
|
||||
pgconn.commit()
|
||||
|
||||
LOG.info("Finished copying collection %s.%s to %s", mongo_db, collection_name, pg_table)
|
||||
|
||||
|
||||
def _cli() -> None:
|
||||
parser = argparse.ArgumentParser(description="Copy a MongoDB collection to PostgreSQL")
|
||||
parser.add_argument("mongo_uri", help="MongoDB connection URI, e.g. mongodb://user:pw@host:27017")
|
||||
parser.add_argument("mongo_db", help="MongoDB database name")
|
||||
parser.add_argument("collection", help="Collection name to copy")
|
||||
parser.add_argument("pg_dsn", help="Postgres DSN, e.g. \"host=.. dbname=.. user=.. password=..\"")
|
||||
parser.add_argument("--pg-table", help="Target Postgres table name (defaults to collection name)")
|
||||
parser.add_argument("--sample", type=int, default=500, help="Sample size to infer schema")
|
||||
parser.add_argument("--batch", type=int, default=1000, help="Insert batch size")
|
||||
parser.add_argument("--no-create", dest="create", action="store_false", help="Don't attempt to create table in PG")
|
||||
parser.add_argument("--verbose", "-v", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
|
||||
|
||||
convert_mongo_to_pg(
|
||||
args.mongo_uri,
|
||||
args.mongo_db,
|
||||
args.collection,
|
||||
args.pg_dsn,
|
||||
pg_table=args.pg_table,
|
||||
sample_size=args.sample,
|
||||
batch_size=args.batch,
|
||||
create_table=args.create,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
_cli()
|
||||
@@ -1,259 +0,0 @@
|
||||
/*
|
||||
* Minimal SQLite virtual table module backed by PostgreSQL via libpq.
|
||||
*
|
||||
* Usage (after building as shared object):
|
||||
* sqlite3> .load ./pg_vtab.so
|
||||
* sqlite> CREATE VIRTUAL TABLE t1 USING pg_vtab('host=... dbname=... user=... password=...', 'schema.table');
|
||||
* sqlite> SELECT * FROM t1 LIMIT 10;
|
||||
*
|
||||
* Notes:
|
||||
* - This is a minimal read-only implementation.
|
||||
* - It only supports full table scans (no constraint pushdown).
|
||||
* - All values are returned as TEXT.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <assert.h>
|
||||
|
||||
#include <libpq-fe.h>
|
||||
#include <sqlite3.h>
|
||||
|
||||
/* vtab object */
|
||||
typedef struct PgVTab {
|
||||
sqlite3_vtab base; /* must be first */
|
||||
char *conninfo;
|
||||
char *schema_table; /* qualified table name */
|
||||
int ncol;
|
||||
char **colnames;
|
||||
} PgVTab;
|
||||
|
||||
/* cursor object */
|
||||
typedef struct PgVTabCursor {
|
||||
sqlite3_vtab_cursor base; /* must be first */
|
||||
PGconn *pgconn;
|
||||
PGresult *res;
|
||||
int row; /* current row index in res */
|
||||
sqlite3_int64 rowid; /* generated rowid */
|
||||
} PgVTabCursor;
|
||||
|
||||
/* forward declarations */
|
||||
static int pgConnect(sqlite3 *db, void *pAux, int argc, const char *const *argv, sqlite3_vtab **ppVTab, char **pzErr);
|
||||
static int pgDisconnect(sqlite3_vtab *pVTab);
|
||||
static int pgBestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pIdx);
|
||||
static int pgOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor);
|
||||
static int pgClose(sqlite3_vtab_cursor *cur);
|
||||
static int pgFilter(sqlite3_vtab_cursor *pVtabCursor, int idxNum, const char *idxStr, int argc, sqlite3_value **argv);
|
||||
static int pgNext(sqlite3_vtab_cursor *cur);
|
||||
static int pgEof(sqlite3_vtab_cursor *cur);
|
||||
static int pgColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i);
|
||||
static int pgRowid(sqlite3_vtab_cursor *cur, sqlite3_int64 *pRowid);
|
||||
|
||||
static sqlite3_module PgModule = {
|
||||
0, /* iVersion */
|
||||
pgConnect,
|
||||
pgConnect, /* xCreate == xConnect for this module */
|
||||
pgBestIndex,
|
||||
pgDisconnect,
|
||||
pgDisconnect, /* xDestroy == xDisconnect */
|
||||
pgOpen,
|
||||
pgClose,
|
||||
pgFilter,
|
||||
pgNext,
|
||||
pgEof,
|
||||
pgColumn,
|
||||
pgRowid,
|
||||
NULL, NULL, NULL, NULL, NULL, NULL, NULL
|
||||
};
|
||||
|
||||
/* helper: strdup-like */
|
||||
static char *str_dup(const char *s){
|
||||
if(!s) return NULL;
|
||||
size_t n = strlen(s)+1;
|
||||
char *r = (char*)malloc(n);
|
||||
if(r) memcpy(r,s,n);
|
||||
return r;
|
||||
}
|
||||
|
||||
/* read column names from POSTGRES by executing SELECT * FROM table LIMIT 0 */
|
||||
static int fetch_pg_columns(const char *conninfo, const char *schema_table, int *pncol, char ***pnames, char **pErr){
|
||||
PGconn *conn = PQconnectdb(conninfo);
|
||||
if (PQstatus(conn) != CONNECTION_OK) {
|
||||
if(pErr) *pErr = str_dup(PQerrorMessage(conn));
|
||||
PQfinish(conn);
|
||||
return SQLITE_ERROR;
|
||||
}
|
||||
char q[1024];
|
||||
snprintf(q, sizeof(q), "SELECT * FROM %s LIMIT 0", schema_table);
|
||||
PGresult *res = PQexec(conn, q);
|
||||
if (PQresultStatus(res) != PGRES_TUPLES_OK && PQresultStatus(res) != PGRES_COMMAND_OK) {
|
||||
if(pErr) *pErr = str_dup(PQerrorMessage(conn));
|
||||
PQclear(res);
|
||||
PQfinish(conn);
|
||||
return SQLITE_ERROR;
|
||||
}
|
||||
int n = PQnfields(res);
|
||||
char **names = (char**)malloc(sizeof(char*)*n);
|
||||
for(int i=0;i<n;i++){
|
||||
names[i] = str_dup(PQfname(res,i));
|
||||
}
|
||||
PQclear(res);
|
||||
PQfinish(conn);
|
||||
*pncol = n;
|
||||
*pnames = names;
|
||||
return SQLITE_OK;
|
||||
}
|
||||
|
||||
static int pgConnect(sqlite3 *db, void *pAux, int argc, const char *const *argv, sqlite3_vtab **ppVTab, char **pzErr){
|
||||
/* argv: module name, db name, conninfo, schema.table */
|
||||
if(argc < 4){
|
||||
if(pzErr) *pzErr = str_dup("pg_vtab requires connection string and qualified table name");
|
||||
return SQLITE_ERROR;
|
||||
}
|
||||
const char *conninfo = argv[2];
|
||||
const char *schema_table = argv[3];
|
||||
|
||||
int ncol = 0;
|
||||
char **names = NULL;
|
||||
int rc = fetch_pg_columns(conninfo, schema_table, &ncol, &names, pzErr);
|
||||
if(rc != SQLITE_OK) return rc;
|
||||
|
||||
/* build CREATE TABLE declaration with TEXT columns */
|
||||
size_t needed = 64;
|
||||
for(int i=0;i<ncol;i++) needed += strlen(names[i]) + 8;
|
||||
char *decl = (char*)malloc(needed);
|
||||
if(!decl){
|
||||
if(pzErr) *pzErr = str_dup("out of memory");
|
||||
return SQLITE_NOMEM;
|
||||
}
|
||||
strcpy(decl, "CREATE TABLE x(");
|
||||
for(int i=0;i<ncol;i++){
|
||||
if(i) strcat(decl, ",");
|
||||
strcat(decl, names[i]);
|
||||
strcat(decl, " TEXT");
|
||||
}
|
||||
strcat(decl, ")");
|
||||
|
||||
rc = sqlite3_declare_vtab(db, decl);
|
||||
free(decl);
|
||||
if(rc != SQLITE_OK){
|
||||
if(pzErr) *pzErr = str_dup("sqlite3_declare_vtab failed");
|
||||
for(int i=0;i<ncol;i++) free(names[i]); free(names);
|
||||
return rc;
|
||||
}
|
||||
|
||||
PgVTab *vtab = (PgVTab*)sqlite3_malloc(sizeof(PgVTab));
|
||||
if(!vtab) return SQLITE_NOMEM;
|
||||
memset(vtab,0,sizeof(PgVTab));
|
||||
vtab->conninfo = str_dup(conninfo);
|
||||
vtab->schema_table = str_dup(schema_table);
|
||||
vtab->ncol = ncol;
|
||||
vtab->colnames = names;
|
||||
|
||||
*ppVTab = (sqlite3_vtab*)vtab;
|
||||
return SQLITE_OK;
|
||||
}
|
||||
|
||||
static int pgDisconnect(sqlite3_vtab *pVTab){
|
||||
PgVTab *v = (PgVTab*)pVTab;
|
||||
if(!v) return SQLITE_OK;
|
||||
if(v->conninfo) free(v->conninfo);
|
||||
if(v->schema_table) free(v->schema_table);
|
||||
for(int i=0;i<v->ncol;i++) free(v->colnames[i]);
|
||||
if(v->colnames) free(v->colnames);
|
||||
sqlite3_free(v);
|
||||
return SQLITE_OK;
|
||||
}
|
||||
|
||||
static int pgBestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pIdx){
|
||||
/* do not support indexes or constraints; full table scan */
|
||||
pIdx->estimatedCost = 1e6;
|
||||
return SQLITE_OK;
|
||||
}
|
||||
|
||||
static int pgOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
|
||||
PgVTabCursor *cur = (PgVTabCursor*)sqlite3_malloc(sizeof(PgVTabCursor));
|
||||
if(!cur) return SQLITE_NOMEM;
|
||||
memset(cur,0,sizeof(PgVTabCursor));
|
||||
*ppCursor = &cur->base;
|
||||
cur->pgconn = NULL;
|
||||
cur->res = NULL;
|
||||
cur->row = 0;
|
||||
cur->rowid = 0;
|
||||
return SQLITE_OK;
|
||||
}
|
||||
|
||||
static int pgClose(sqlite3_vtab_cursor *curp){
|
||||
PgVTabCursor *cur = (PgVTabCursor*)curp;
|
||||
if(cur->res) PQclear(cur->res);
|
||||
if(cur->pgconn) PQfinish(cur->pgconn);
|
||||
sqlite3_free(cur);
|
||||
return SQLITE_OK;
|
||||
}
|
||||
|
||||
static int pgFilter(sqlite3_vtab_cursor *pVtabCursor, int idxNum, const char *idxStr, int argc, sqlite3_value **argv){
|
||||
PgVTabCursor *cur = (PgVTabCursor*)pVtabCursor;
|
||||
PgVTab *vtab = (PgVTab*)(pVtabCursor->pVtab);
|
||||
/* connect to pg */
|
||||
if(cur->res){ PQclear(cur->res); cur->res = NULL; }
|
||||
if(cur->pgconn){ PQfinish(cur->pgconn); cur->pgconn = NULL; }
|
||||
cur->pgconn = PQconnectdb(vtab->conninfo);
|
||||
if(PQstatus(cur->pgconn) != CONNECTION_OK){
|
||||
/* can't report error via sqlite3_errmsg here, return error */
|
||||
return SQLITE_ERROR;
|
||||
}
|
||||
|
||||
/* simple select all */
|
||||
char q[1024];
|
||||
snprintf(q, sizeof(q), "SELECT * FROM %s", vtab->schema_table);
|
||||
cur->res = PQexec(cur->pgconn, q);
|
||||
if(PQresultStatus(cur->res) != PGRES_TUPLES_OK && PQresultStatus(cur->res) != PGRES_COMMAND_OK){
|
||||
PQclear(cur->res); cur->res = NULL; return SQLITE_ERROR;
|
||||
}
|
||||
cur->row = 0;
|
||||
cur->rowid = 0;
|
||||
return SQLITE_OK;
|
||||
}
|
||||
|
||||
static int pgNext(sqlite3_vtab_cursor *curp){
|
||||
PgVTabCursor *cur = (PgVTabCursor*)curp;
|
||||
cur->row++;
|
||||
cur->rowid++;
|
||||
return SQLITE_OK;
|
||||
}
|
||||
|
||||
static int pgEof(sqlite3_vtab_cursor *curp){
|
||||
PgVTabCursor *cur = (PgVTabCursor*)curp;
|
||||
if(!cur->res) return 1;
|
||||
int nt = PQntuples(cur->res);
|
||||
return cur->row >= nt;
|
||||
}
|
||||
|
||||
static int pgColumn(sqlite3_vtab_cursor *curp, sqlite3_context *ctx, int i){
|
||||
PgVTabCursor *cur = (PgVTabCursor*)curp;
|
||||
if(!cur->res) { sqlite3_result_null(ctx); return SQLITE_OK; }
|
||||
if(PQgetisnull(cur->res, cur->row, i)){
|
||||
sqlite3_result_null(ctx);
|
||||
} else {
|
||||
const char *val = PQgetvalue(cur->res, cur->row, i);
|
||||
sqlite3_result_text(ctx, val, -1, SQLITE_TRANSIENT);
|
||||
}
|
||||
return SQLITE_OK;
|
||||
}
|
||||
|
||||
static int pgRowid(sqlite3_vtab_cursor *curp, sqlite3_int64 *pRowid){
|
||||
PgVTabCursor *cur = (PgVTabCursor*)curp;
|
||||
*pRowid = cur->rowid;
|
||||
return SQLITE_OK;
|
||||
}
|
||||
|
||||
/* Entry point called by sqlite3 when loading extension */
|
||||
int sqlite3_pg_vtab_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi){
|
||||
SQLITE_EXTENSION_INIT2(pApi)}
|
||||
int rc = sqlite3_create_module(db, "pg_vtab", &PgModule, NULL);
|
||||
if(rc != SQLITE_OK){
|
||||
if(pzErrMsg) *pzErrMsg = str_dup("failed to register pg_vtab module");
|
||||
}
|
||||
return rc;
|
||||
}
|
||||
+2
-2
@@ -1,11 +1,11 @@
|
||||
logrotater==1.3
|
||||
numpy==2.2.3
|
||||
progressbar33==2.4
|
||||
psycopg2==2.9.10
|
||||
psycopg2-binary==2.9.10
|
||||
pymongo==4.11.2
|
||||
pymssql==2.3.2
|
||||
Requests==2.32.3
|
||||
SQLAlchemy==2.0.39
|
||||
pymysql
|
||||
tqdm
|
||||
pandas
|
||||
pandas
|
||||
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
cd /srv/Projects/new_dbsync
|
||||
/srv/Projects/new_dbsync/update_json.py >> /home/wjones/update_mongo.log
|
||||
Executable
+266
@@ -0,0 +1,266 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
import os, time, sys, datetime
|
||||
from db.functions import settype, dbmongo
|
||||
import json
|
||||
|
||||
|
||||
def set_apienv(urls, uprocess, dbengine, dbExec, updatesBase, lprint):
|
||||
"""Populate updates table from the API and return available updates.
|
||||
|
||||
Args:
|
||||
urls: URL configuration with an "updatesurl" key.
|
||||
uprocess: URL loader instance with `get_data`.
|
||||
jget: JSON processor with `jconvert`.
|
||||
dbengine: database engine dict (expects key "engine").
|
||||
dbExec: database execution helper with `update_tvupdates` and `rawsql_select`.
|
||||
updatesBase: list-like containing tables (uses index 8 for updates table).
|
||||
lprint: logger instance with `logprint`.
|
||||
|
||||
Returns:
|
||||
List of tuples (seriesid, timestamp) representing available updates.
|
||||
"""
|
||||
#updatetable = updatesBase[8]
|
||||
#inputdata = uprocess.get_data(urls["updatesurl"])
|
||||
#lprint.logprint("info", f"Retrieved {len(availableupdates)} rows for processing......")
|
||||
#
|
||||
newupdates = dbExec.rawsql_select(
|
||||
dbengine["engine"],
|
||||
"select seriesid,timestamp from updates.tvupdates",
|
||||
lprint
|
||||
)
|
||||
return newupdates
|
||||
|
||||
|
||||
def print_starttime(lprint, ts1):
|
||||
lprint.logprint("info", f"Sync start time: {str(ts1)}")
|
||||
|
||||
|
||||
def find_file_in_multiple_dirs(filename, directories):
|
||||
"""
|
||||
Checks if a file exists in any of the provided directories.
|
||||
|
||||
Args:
|
||||
filename (str): The name of the file to search for.
|
||||
directories (list): A list of directory paths to check.
|
||||
|
||||
Returns:
|
||||
Path or None: The full path to the file if found, otherwise None.
|
||||
"""
|
||||
directories_found = []
|
||||
for directory in directories:
|
||||
if os.path.isfile(f'{directory}{filename}'):
|
||||
directories_found.append(directory)
|
||||
|
||||
return directories_found
|
||||
|
||||
def load_json_to_mongodb(mongo_updater, directories, update_list, updateDict, lprint, track_changes=True):
|
||||
"""
|
||||
Load downloaded JSON files into MongoDB collections with change detection.
|
||||
|
||||
Processes series, episodes, cast, and crew data from cached JSON files
|
||||
and inserts or updates them in the appropriate MongoDB collections.
|
||||
Automatically detects if documents are new, updated, or unchanged by comparing
|
||||
updateDict timestamps with existing document 'updated' fields.
|
||||
|
||||
Args:
|
||||
mongo_updater: dbmongo instance for MongoDB operations
|
||||
directories: dict containing cache directory paths for different data types
|
||||
update_list: list of series IDs to process
|
||||
updateDict: dict with seriesid as key and timestamp as value
|
||||
lprint: logger instance for logging operations
|
||||
track_changes: bool, if True tracks inserted vs updated documents (default: True)
|
||||
|
||||
Returns:
|
||||
tuple: (successful_count, failed_count, stats_dict) for series processed
|
||||
stats_dict contains: {'inserted': count, 'updated': count}
|
||||
"""
|
||||
successful_count = 0
|
||||
failed_count = 0
|
||||
stats = {'series_inserted': 0, 'series_updated': 0, 'series_skipped': 0,
|
||||
'episodes_inserted': 0, 'episodes_updated': 0, 'episodes_skipped': 0,
|
||||
'cast_inserted': 0, 'cast_updated': 0, 'cast_skipped': 0,
|
||||
'crew_inserted': 0, 'crew_updated': 0, 'crew_skipped': 0}
|
||||
|
||||
lprint.logprint("info", f"Starting to load {len(update_list)} series into MongoDB")
|
||||
|
||||
for seriesid in tqdm(update_list, desc="Loading series data into MongoDB", unit="series"):
|
||||
try:
|
||||
# Load series data
|
||||
series_file = f"{directories['SERIESDIR']}{seriesid}.json"
|
||||
if os.path.exists(series_file):
|
||||
try:
|
||||
with open(series_file, 'r') as f:
|
||||
series_data = json.load(f)
|
||||
|
||||
# Insert or update series in MongoDB
|
||||
if series_data:
|
||||
if track_changes:
|
||||
existing = mongo_updater.mgseries.find_one({"id": series_data.get("id")})
|
||||
seriesid_str = str(seriesid)
|
||||
update_timestamp = updateDict.get(seriesid_str)
|
||||
if existing:
|
||||
# Check if data has changed by comparing update timestamp
|
||||
if update_timestamp and int(update_timestamp) > existing.get("updated", 0):
|
||||
mongo_updater.bulk_upsert_by_id(mongo_updater.mgseries, [series_data])
|
||||
stats['series_updated'] += 1
|
||||
lprint.logprint("debug", f"Updated series data for ID {seriesid}")
|
||||
else:
|
||||
lprint.logprint("debug", f"Series {seriesid} unchanged, skipping")
|
||||
stats['series_skipped'] += 1
|
||||
pass # Document unchanged
|
||||
else:
|
||||
mongo_updater.bulk_upsert_by_id(mongo_updater.mgseries, [series_data])
|
||||
stats['series_inserted'] += 1
|
||||
lprint.logprint("debug", f"Inserted new series data for ID {seriesid}")
|
||||
# Load episodes data
|
||||
episodes_file = f"{directories['EPISODEDIR']}{seriesid}.json"
|
||||
if os.path.exists(episodes_file):
|
||||
try:
|
||||
with open(episodes_file, 'r') as f:
|
||||
episodes_data = json.load(f)
|
||||
|
||||
if isinstance(episodes_data, list) and episodes_data:
|
||||
# Add seriesid to each episode
|
||||
for episode in episodes_data:
|
||||
episode['seriesid'] = seriesid
|
||||
|
||||
if track_changes:
|
||||
for episode in episodes_data:
|
||||
existing = mongo_updater.mgepisodes.find_one({"id": episode.get("id")})
|
||||
seriesid_str = str(seriesid)
|
||||
update_timestamp = updateDict.get(seriesid_str)
|
||||
if existing:
|
||||
if update_timestamp and int(update_timestamp) > existing.get("updated", 0):
|
||||
mongo_updater.bulk_upsert_by_id(mongo_updater.mgepisodes, episodes_data)
|
||||
stats['episodes_updated'] += 1
|
||||
else:
|
||||
lprint.logprint("debug", f"Episode Data for Series {seriesid} unchanged, skipping")
|
||||
pass # Document unchanged
|
||||
else:
|
||||
mongo_updater.bulk_upsert_by_id(mongo_updater.mgepisodes, episodes_data)
|
||||
stats['episodes_inserted'] += 1
|
||||
|
||||
#mongo_updater.bulk_upsert_by_id(mongo_updater.mgepisodes, episodes_data)
|
||||
lprint.logprint("debug", f"Loaded {len(episodes_data)} episodes for series {seriesid}")
|
||||
except Exception as e:
|
||||
lprint.logprint("warning", f"Failed to load episodes file for {seriesid}: {e}")
|
||||
|
||||
# Load cast data
|
||||
cast_file = f"{directories['CASTDIR']}{seriesid}.json"
|
||||
if os.path.exists(cast_file):
|
||||
try:
|
||||
with open(cast_file, 'r') as f:
|
||||
cast_data = json.load(f)
|
||||
|
||||
if isinstance(cast_data, list) and cast_data:
|
||||
actors = []
|
||||
characters = []
|
||||
for item in cast_data:
|
||||
if 'person' in item and item['person']:
|
||||
item['person']['seriesid'] = seriesid
|
||||
actors.append(item['person'])
|
||||
if 'character' in item and item['character']:
|
||||
item['character']['seriesid'] = seriesid
|
||||
characters.append(item['character'])
|
||||
|
||||
if track_changes:
|
||||
for actor in actors:
|
||||
existing = mongo_updater.mgactors.find_one({"id": actor.get("id")})
|
||||
seriesid_str = str(seriesid)
|
||||
update_timestamp = updateDict.get(seriesid_str)
|
||||
if existing:
|
||||
if update_timestamp and int(update_timestamp) > existing.get("updated", 0):
|
||||
stats['cast_updated'] += 1
|
||||
else:
|
||||
stats['cast_inserted'] += 1
|
||||
|
||||
if actors:
|
||||
mongo_updater.bulk_upsert_by_id(mongo_updater.mgactors, actors)
|
||||
if characters:
|
||||
mongo_updater.bulk_upsert_by_id(mongo_updater.mgcharacters, characters)
|
||||
lprint.logprint("debug", f"Loaded cast data for series {seriesid}")
|
||||
except Exception as e:
|
||||
lprint.logprint("warning", f"Failed to load cast file for {seriesid}: {e}")
|
||||
|
||||
# Load crew data
|
||||
crew_file = f"{directories['CREWDIR']}{seriesid}.json"
|
||||
if os.path.exists(crew_file):
|
||||
try:
|
||||
with open(crew_file, 'r') as f:
|
||||
crew_data = json.load(f)
|
||||
|
||||
if isinstance(crew_data, list) and crew_data:
|
||||
crew_list = []
|
||||
for item in crew_data:
|
||||
if 'person' in item and item['person']:
|
||||
item['person']['seriesid'] = seriesid
|
||||
item['person']['crew_type'] = item.get('type', 'unknown')
|
||||
crew_list.append(item['person'])
|
||||
|
||||
if track_changes:
|
||||
for crew in crew_list:
|
||||
existing = mongo_updater.mgcrew.find_one({"id": crew.get("id")})
|
||||
seriesid_str = str(seriesid)
|
||||
update_timestamp = updateDict.get(seriesid_str)
|
||||
if existing:
|
||||
if update_timestamp and int(update_timestamp) > existing.get("updated", 0):
|
||||
stats['crew_updated'] += 1
|
||||
else:
|
||||
stats['crew_inserted'] += 1
|
||||
|
||||
if crew_list:
|
||||
mongo_updater.bulk_upsert_by_id(mongo_updater.mgcrew, crew_list)
|
||||
lprint.logprint("debug", f"Loaded crew data for series {seriesid}")
|
||||
except Exception as e:
|
||||
lprint.logprint("warning", f"Failed to load crew file for {seriesid}: {e}")
|
||||
|
||||
successful_count += 1
|
||||
lprint.logprint("info", f"Successfully processed series {seriesid}")
|
||||
#mongo_updater.bulk_upsert_by_id(mongo_updater.mgseries, [series_data])
|
||||
except Exception as e:
|
||||
lprint.logprint("warning", f"Failed to load series file for {seriesid}: {e}")
|
||||
failed_count += 1
|
||||
continue
|
||||
except Exception as e:
|
||||
failed_count += 1
|
||||
lprint.logprint("error", f"Unexpected error processing series {seriesid}: {e}")
|
||||
|
||||
lprint.logprint("info", f"MongoDB loading complete.")
|
||||
return successful_count, failed_count, stats
|
||||
|
||||
def main() -> int:
|
||||
ROOTDIR = os.getcwd()
|
||||
directories = {}
|
||||
# Use cross-platform path builders and ensure directories exist
|
||||
LOGDIR = os.path.join(ROOTDIR, "log")
|
||||
directories["SERIESDIR"] = os.path.join(ROOTDIR, "cache", "series") + os.sep
|
||||
directories["EPISODEDIR"] = os.path.join(ROOTDIR, "cache", "episodes") + os.sep
|
||||
directories["CASTDIR"] = os.path.join(ROOTDIR, "cache", "cast") + os.sep
|
||||
directories["GUESTCASTDIR"] = os.path.join(ROOTDIR, "cache", "guestcast") + os.sep
|
||||
directories["CREWDIR"] = os.path.join(ROOTDIR, "cache", "crew") + os.sep
|
||||
#directories["GUESTCREWDIR"] = os.path.join(ROOTDIR, "cache", "guestcrew") + os.sep
|
||||
#directories["CREDITSDIR"] = os.path.join(ROOTDIR, "cache", "credits") + os.sep
|
||||
#directories["ALIASDIR"] = os.path.join(ROOTDIR, "cache", "aliases") + os.sep
|
||||
tempdir = os.path.join(ROOTDIR, 'temp') + os.sep
|
||||
|
||||
import settings.config
|
||||
|
||||
updateList = []
|
||||
updateDict = {}
|
||||
needed_updates = []
|
||||
|
||||
config_options = settings.config.Config(ROOTDIR)
|
||||
options = config_options.config_options
|
||||
mongo_query = dbmongo(options)
|
||||
|
||||
seriesid = 4
|
||||
query = { "id": int(seriesid)}
|
||||
for doc in mongo_query.mgseries.find(query):
|
||||
print(doc)
|
||||
query = { "seriesid": str(seriesid)}
|
||||
for doc in mongo_query.mgepisodes.find(query):
|
||||
print(doc)
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
[global]
|
||||
loglevel=debug
|
||||
loglevel=info
|
||||
|
||||
[dbsettings]
|
||||
sqlserver_driver=ODBC Driver 13 for SQL Server
|
||||
hostname=192.168.128.7
|
||||
hostname=192.168.128.3
|
||||
mghostname=192.168.128.24
|
||||
dbname=media_dbsync
|
||||
mgdbname=tvdata
|
||||
|
||||
-193
@@ -1,193 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Simple utility to copy a single table from SQLite to PostgreSQL.
|
||||
|
||||
Usage example:
|
||||
python sqlite2pgsql /path/to/sqlite.db "host=... dbname=... user=... password=..." mytable
|
||||
|
||||
Implements `convert_table_sqlite_to_pg(sqlite_path, pg_dsn, table_name, create_table=True, batch_size=1000)`
|
||||
which copies schema (basic types) and data in batches.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sqlite3
|
||||
from typing import List, Tuple
|
||||
|
||||
import psycopg2
|
||||
from psycopg2 import sql
|
||||
|
||||
LOG = logging.getLogger("sqlite2pgsql")
|
||||
|
||||
|
||||
def _map_sqlite_type_to_pg(sqlite_type: str) -> str:
|
||||
t = (sqlite_type or "").upper()
|
||||
if "INT" in t:
|
||||
return "INTEGER"
|
||||
if any(x in t for x in ("CHAR", "CLOB", "TEXT")):
|
||||
return "TEXT"
|
||||
if "BLOB" in t:
|
||||
return "BYTEA"
|
||||
if any(x in t for x in ("REAL", "FLOA", "DOUB")):
|
||||
return "REAL"
|
||||
if "NUM" in t or "DEC" in t:
|
||||
return "NUMERIC"
|
||||
# default fallback
|
||||
return "TEXT"
|
||||
|
||||
|
||||
def _get_table_info_sqlite(conn: sqlite3.Connection, table: str) -> List[Tuple]:
|
||||
cur = conn.execute(f"PRAGMA table_info('{table}')")
|
||||
rows = cur.fetchall()
|
||||
return rows
|
||||
|
||||
|
||||
def list_tables_sqlite(sqlite_path: str) -> List[str]:
|
||||
"""Return a list of non-system table names from the SQLite database."""
|
||||
conn = sqlite3.connect(sqlite_path)
|
||||
try:
|
||||
cur = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name;")
|
||||
return [r[0] for r in cur.fetchall()]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def convert_table_sqlite_to_pg(
|
||||
sqlite_path: str,
|
||||
pg_dsn: str,
|
||||
table_name: str,
|
||||
create_table: bool = True,
|
||||
batch_size: int = 1000,
|
||||
) -> None:
|
||||
"""Copy a table from a SQLite file to PostgreSQL.
|
||||
|
||||
Arguments:
|
||||
sqlite_path: path to the .db file
|
||||
pg_dsn: libpq connection string for psycopg2
|
||||
table_name: table to copy
|
||||
create_table: if True, attempt to create the table in Postgres (basic mapping)
|
||||
batch_size: number of rows per insert batch
|
||||
"""
|
||||
LOG.info("Opening SQLite DB %s", sqlite_path)
|
||||
sconn = sqlite3.connect(sqlite_path)
|
||||
sconn.row_factory = sqlite3.Row
|
||||
|
||||
try:
|
||||
info = _get_table_info_sqlite(sconn, table_name)
|
||||
if not info:
|
||||
raise ValueError(f"Table {table_name} not found in {sqlite_path}")
|
||||
|
||||
columns = []
|
||||
pk_cols = []
|
||||
for col in info:
|
||||
# PRAGMA table_info: cid, name, type, notnull, dflt_value, pk
|
||||
name = col[1]
|
||||
ctype = col[2]
|
||||
notnull = bool(col[3])
|
||||
dflt = col[4]
|
||||
pk = bool(col[5])
|
||||
pgtype = _map_sqlite_type_to_pg(ctype)
|
||||
columns.append((name, pgtype, notnull, dflt))
|
||||
if pk:
|
||||
pk_cols.append(name)
|
||||
|
||||
if create_table:
|
||||
LOG.info("Connecting to Postgres to create table %s", table_name)
|
||||
with psycopg2.connect(pg_dsn) as pgconn:
|
||||
with pgconn.cursor() as cur:
|
||||
col_defs = []
|
||||
for name, pgtype, notnull, _dflt in columns:
|
||||
part = sql.SQL("{} {}{}") .format(
|
||||
sql.Identifier(name),
|
||||
sql.SQL(pgtype),
|
||||
sql.SQL(" NOT NULL") if notnull else sql.SQL(""),
|
||||
)
|
||||
col_defs.append(part)
|
||||
|
||||
if pk_cols:
|
||||
pk_clause = sql.SQL(", PRIMARY KEY ({})").format(sql.SQL(",").join(map(sql.Identifier, pk_cols)))
|
||||
create_body = sql.SQL(", ").join(col_defs) + pk_clause
|
||||
else:
|
||||
create_body = sql.SQL(", ").join(col_defs)
|
||||
|
||||
create_stmt = sql.SQL("CREATE TABLE IF NOT EXISTS {} ({})").format(
|
||||
sql.Identifier(table_name), create_body
|
||||
)
|
||||
cur.execute(create_stmt)
|
||||
pgconn.commit()
|
||||
|
||||
# Now copy data
|
||||
LOG.info("Copying data for table %s", table_name)
|
||||
s_cur = sconn.cursor()
|
||||
s_cur.execute(f"SELECT * FROM '{table_name}'")
|
||||
|
||||
cols = [d[0] for d in s_cur.description]
|
||||
placeholders = ",".join(["%s"] * len(cols))
|
||||
insert_sql = sql.SQL("INSERT INTO {} ({}) VALUES ({})").format(
|
||||
sql.Identifier(table_name),
|
||||
sql.SQL(",").join(map(sql.Identifier, cols)),
|
||||
sql.SQL(placeholders),
|
||||
)
|
||||
|
||||
with psycopg2.connect(pg_dsn) as pgconn:
|
||||
with pgconn.cursor() as pgcur:
|
||||
batch = []
|
||||
for row in s_cur:
|
||||
batch.append(tuple(row))
|
||||
if len(batch) >= batch_size:
|
||||
pgcur.executemany(insert_sql.as_string(pgconn), batch)
|
||||
pgconn.commit()
|
||||
batch = []
|
||||
if batch:
|
||||
pgcur.executemany(insert_sql.as_string(pgconn), batch)
|
||||
pgconn.commit()
|
||||
|
||||
finally:
|
||||
sconn.close()
|
||||
|
||||
|
||||
def _cli() -> None:
|
||||
parser = argparse.ArgumentParser(description="Copy one table from SQLite to PostgreSQL")
|
||||
parser.add_argument("sqlite_db", help="Path to SQLite database file")
|
||||
parser.add_argument("pg_dsn", help="Postgres DSN, e.g. \"host=.. dbname=.. user=.. password=..\"")
|
||||
parser.add_argument("table", nargs="?", help="Table name to copy (optional if --tables or --all used)")
|
||||
parser.add_argument("--tables", help="Comma-separated list of tables to copy")
|
||||
parser.add_argument("--all", action="store_true", help="Copy all non-system tables from the SQLite DB")
|
||||
parser.add_argument("--no-create", dest="create", action="store_false", help="Don't attempt to create table in PG")
|
||||
parser.add_argument("--list", dest="list_tables", action="store_true", help="List tables in the SQLite DB and exit")
|
||||
parser.add_argument("--batch", type=int, default=1000, help="Batch size for inserts")
|
||||
parser.add_argument("--verbose", "-v", action="store_true", help="Verbose logging")
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
|
||||
if getattr(args, "list_tables", False):
|
||||
for t in list_tables_sqlite(args.sqlite_db):
|
||||
print(t)
|
||||
return
|
||||
|
||||
# Determine tables to process
|
||||
table_names = []
|
||||
if args.all:
|
||||
table_names = list_tables_sqlite(args.sqlite_db)
|
||||
elif args.tables:
|
||||
table_names = [t.strip() for t in args.tables.split(",") if t.strip()]
|
||||
elif args.table:
|
||||
table_names = [args.table]
|
||||
else:
|
||||
parser.error("must specify a table, --tables, or --all unless --list is used")
|
||||
|
||||
if not args.pg_dsn:
|
||||
parser.error("pg_dsn is required to copy tables to Postgres")
|
||||
|
||||
# Process tables sequentially
|
||||
for tbl in table_names:
|
||||
try:
|
||||
LOG.info("Processing table %s", tbl)
|
||||
convert_table_sqlite_to_pg(args.sqlite_db, args.pg_dsn, tbl, create_table=args.create, batch_size=args.batch)
|
||||
except Exception as exc:
|
||||
LOG.exception("Failed to process table %s: %s", tbl, exc)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
_cli()
|
||||
@@ -1,154 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Utility to expose a PostgreSQL table to SQLite by copying it into a sqlite file.
|
||||
|
||||
This is a pragmatic "plugin": it creates a local SQLite table (read-only by default)
|
||||
that mirrors a PostgreSQL table so SQLite queries can run locally against the data.
|
||||
|
||||
Usage examples:
|
||||
# copy single table into sqlite file
|
||||
python sqlite_pg_plugin.py copy --pg "host=... dbname=... user=... password=..." public.mytable /tmp/my.db
|
||||
|
||||
# list tables in Postgres
|
||||
python sqlite_pg_plugin.py list --pg "host=... dbname=... user=... password=..."
|
||||
|
||||
This avoids needing to build a native virtual-table extension. For a direct virtual
|
||||
table (no copy) a C extension using SQLite virtual table API + libpq would be needed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sqlite3
|
||||
from typing import List, Tuple
|
||||
|
||||
import psycopg2
|
||||
from psycopg2 import sql
|
||||
|
||||
LOG = logging.getLogger("sqlite_pg_plugin")
|
||||
|
||||
|
||||
def list_tables_pg(pg_dsn: str) -> List[str]:
|
||||
q = "SELECT table_schema||'.'||table_name FROM information_schema.tables WHERE table_type='BASE TABLE' AND table_schema NOT IN ('pg_catalog','information_schema') ORDER BY 1;"
|
||||
with psycopg2.connect(pg_dsn) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(q)
|
||||
return [r[0] for r in cur.fetchall()]
|
||||
|
||||
|
||||
def fetch_table_schema_pg(pg_dsn: str, qualified_table: str) -> List[Tuple[str, str, bool]]:
|
||||
"""Return list of (colname, pgtype, notnull) for the given qualified table (schema.table)."""
|
||||
if '.' in qualified_table:
|
||||
schema, table = qualified_table.split('.', 1)
|
||||
else:
|
||||
schema, table = 'public', qualified_table
|
||||
|
||||
q = sql.SQL("SELECT column_name, data_type, is_nullable FROM information_schema.columns WHERE table_schema=%s AND table_name=%s ORDER BY ordinal_position;")
|
||||
with psycopg2.connect(pg_dsn) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(q, (schema, table))
|
||||
out = []
|
||||
for name, dtype, is_nullable in cur.fetchall():
|
||||
out.append((name, dtype, is_nullable == 'NO'))
|
||||
return out
|
||||
|
||||
|
||||
def _map_pg_to_sqlite(pg_type: str) -> str:
|
||||
t = (pg_type or '').lower()
|
||||
if 'int' in t or t in ('bigserial', 'serial'):
|
||||
return 'INTEGER'
|
||||
if any(x in t for x in ('char', 'text', 'uuid')):
|
||||
return 'TEXT'
|
||||
if any(x in t for x in ('timestamp', 'date', 'time')):
|
||||
return 'TEXT'
|
||||
if any(x in t for x in ('numeric', 'decimal', 'real', 'double', 'float')):
|
||||
return 'REAL'
|
||||
if 'bool' in t:
|
||||
return 'INTEGER'
|
||||
return 'TEXT'
|
||||
|
||||
|
||||
def copy_table_pg_to_sqlite(pg_dsn: str, qualified_table: str, sqlite_path: str, sqlite_table: str | None = None, batch_size: int = 1000, overwrite: bool = True) -> None:
|
||||
"""Copy schema and data from Postgres table into a SQLite file.
|
||||
|
||||
- `qualified_table` can be 'schema.table' or 'table'
|
||||
- `sqlite_table` is optional name to use in sqlite; defaults to table part
|
||||
"""
|
||||
if '.' in qualified_table and not sqlite_table:
|
||||
sqlite_table = qualified_table.split('.', 1)[1]
|
||||
elif not sqlite_table:
|
||||
sqlite_table = qualified_table
|
||||
|
||||
LOG.info('Fetching schema for %s', qualified_table)
|
||||
cols = fetch_table_schema_pg(pg_dsn, qualified_table)
|
||||
if not cols:
|
||||
raise ValueError(f'No columns found for {qualified_table}')
|
||||
|
||||
# create sqlite table
|
||||
sconn = sqlite3.connect(sqlite_path)
|
||||
try:
|
||||
s_cur = sconn.cursor()
|
||||
if overwrite:
|
||||
s_cur.execute(f"DROP TABLE IF EXISTS '{sqlite_table}'")
|
||||
|
||||
col_defs = []
|
||||
for name, pgtype, notnull in cols:
|
||||
stype = _map_pg_to_sqlite(pgtype)
|
||||
col_defs.append(f"{name} {stype}{' NOT NULL' if notnull else ''}")
|
||||
|
||||
create_sql = f"CREATE TABLE IF NOT EXISTS '{sqlite_table}' ({', '.join(col_defs)})"
|
||||
s_cur.execute(create_sql)
|
||||
sconn.commit()
|
||||
|
||||
# stream rows from Postgres
|
||||
with psycopg2.connect(pg_dsn) as pgconn:
|
||||
with pgconn.cursor(name='stream_cursor') as pgcur:
|
||||
pgcur.itersize = batch_size
|
||||
pgcur.execute(sql.SQL('SELECT {} FROM {}').format(sql.SQL(',').join(sql.Identifier(c[0]) for c in cols), sql.Identifier(*qualified_table.split('.', 1)) if '.' in qualified_table else sql.Identifier(qualified_table)))
|
||||
|
||||
placeholders = ','.join('?' for _ in cols)
|
||||
insert_sql = f"INSERT INTO '{sqlite_table}' ({', '.join(n for n,_,_ in cols)}) VALUES ({placeholders})"
|
||||
batch = []
|
||||
for row in pgcur:
|
||||
batch.append(tuple(row))
|
||||
if len(batch) >= batch_size:
|
||||
s_cur.executemany(insert_sql, batch)
|
||||
sconn.commit()
|
||||
batch = []
|
||||
if batch:
|
||||
s_cur.executemany(insert_sql, batch)
|
||||
sconn.commit()
|
||||
finally:
|
||||
sconn.close()
|
||||
|
||||
|
||||
def _cli() -> None:
|
||||
parser = argparse.ArgumentParser(description='Expose Postgres table(s) to SQLite by copying them')
|
||||
sub = parser.add_subparsers(dest='cmd', required=True)
|
||||
|
||||
p_list = sub.add_parser('list', help='List tables in Postgres')
|
||||
p_list.add_argument('--pg', required=True, help='Postgres DSN')
|
||||
|
||||
p_copy = sub.add_parser('copy', help='Copy a Postgres table into sqlite file')
|
||||
p_copy.add_argument('--pg', required=True, help='Postgres DSN')
|
||||
p_copy.add_argument('table', help='Qualified table name (schema.table or table)')
|
||||
p_copy.add_argument('sqlite', help='Path to sqlite file to write to')
|
||||
p_copy.add_argument('--target', help='Target sqlite table name (defaults to original table)')
|
||||
p_copy.add_argument('--batch', type=int, default=1000)
|
||||
p_copy.add_argument('--no-overwrite', dest='overwrite', action='store_false')
|
||||
p_copy.add_argument('--verbose', '-v', action='store_true')
|
||||
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(level=logging.DEBUG if getattr(args, 'verbose', False) else logging.INFO)
|
||||
|
||||
if args.cmd == 'list':
|
||||
for t in list_tables_pg(args.pg):
|
||||
print(t)
|
||||
return
|
||||
|
||||
if args.cmd == 'copy':
|
||||
copy_table_pg_to_sqlite(args.pg, args.table, args.sqlite, sqlite_table=args.target, batch_size=args.batch, overwrite=args.overwrite)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
_cli()
|
||||
-71381
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
-71381
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,986 @@
|
||||
137;http://api.tvmaze.com/people/137/castcredits
|
||||
145;http://api.tvmaze.com/people/145/castcredits
|
||||
137;http://api.tvmaze.com/people/137/castcredits
|
||||
145;http://api.tvmaze.com/people/145/castcredits
|
||||
211;http://api.tvmaze.com/people/211/castcredits
|
||||
214;http://api.tvmaze.com/people/214/castcredits
|
||||
315;http://api.tvmaze.com/people/315/castcredits
|
||||
368;http://api.tvmaze.com/people/368/castcredits
|
||||
416;http://api.tvmaze.com/people/416/castcredits
|
||||
473;http://api.tvmaze.com/people/473/castcredits
|
||||
557;http://api.tvmaze.com/people/557/castcredits
|
||||
577;http://api.tvmaze.com/people/577/castcredits
|
||||
583;http://api.tvmaze.com/people/583/castcredits
|
||||
635;http://api.tvmaze.com/people/635/castcredits
|
||||
688;http://api.tvmaze.com/people/688/castcredits
|
||||
743;http://api.tvmaze.com/people/743/castcredits
|
||||
771;http://api.tvmaze.com/people/771/castcredits
|
||||
811;http://api.tvmaze.com/people/811/castcredits
|
||||
817;http://api.tvmaze.com/people/817/castcredits
|
||||
823;http://api.tvmaze.com/people/823/castcredits
|
||||
864;http://api.tvmaze.com/people/864/castcredits
|
||||
1008;http://api.tvmaze.com/people/1008/castcredits
|
||||
1037;http://api.tvmaze.com/people/1037/castcredits
|
||||
1100;http://api.tvmaze.com/people/1100/castcredits
|
||||
1238;http://api.tvmaze.com/people/1238/castcredits
|
||||
1267;http://api.tvmaze.com/people/1267/castcredits
|
||||
1314;http://api.tvmaze.com/people/1314/castcredits
|
||||
1358;http://api.tvmaze.com/people/1358/castcredits
|
||||
1375;http://api.tvmaze.com/people/1375/castcredits
|
||||
1394;http://api.tvmaze.com/people/1394/castcredits
|
||||
1555;http://api.tvmaze.com/people/1555/castcredits
|
||||
1611;http://api.tvmaze.com/people/1611/castcredits
|
||||
1749;http://api.tvmaze.com/people/1749/castcredits
|
||||
1813;http://api.tvmaze.com/people/1813/castcredits
|
||||
2002;http://api.tvmaze.com/people/2002/castcredits
|
||||
2041;http://api.tvmaze.com/people/2041/castcredits
|
||||
2101;http://api.tvmaze.com/people/2101/castcredits
|
||||
2191;http://api.tvmaze.com/people/2191/castcredits
|
||||
2233;http://api.tvmaze.com/people/2233/castcredits
|
||||
2280;http://api.tvmaze.com/people/2280/castcredits
|
||||
2292;http://api.tvmaze.com/people/2292/castcredits
|
||||
2307;http://api.tvmaze.com/people/2307/castcredits
|
||||
2339;http://api.tvmaze.com/people/2339/castcredits
|
||||
2342;http://api.tvmaze.com/people/2342/castcredits
|
||||
2408;http://api.tvmaze.com/people/2408/castcredits
|
||||
2428;http://api.tvmaze.com/people/2428/castcredits
|
||||
2466;http://api.tvmaze.com/people/2466/castcredits
|
||||
2542;http://api.tvmaze.com/people/2542/castcredits
|
||||
2545;http://api.tvmaze.com/people/2545/castcredits
|
||||
2654;http://api.tvmaze.com/people/2654/castcredits
|
||||
2692;http://api.tvmaze.com/people/2692/castcredits
|
||||
2735;http://api.tvmaze.com/people/2735/castcredits
|
||||
2736;http://api.tvmaze.com/people/2736/castcredits
|
||||
2737;http://api.tvmaze.com/people/2737/castcredits
|
||||
2743;http://api.tvmaze.com/people/2743/castcredits
|
||||
2745;http://api.tvmaze.com/people/2745/castcredits
|
||||
2747;http://api.tvmaze.com/people/2747/castcredits
|
||||
2963;http://api.tvmaze.com/people/2963/castcredits
|
||||
2972;http://api.tvmaze.com/people/2972/castcredits
|
||||
3020;http://api.tvmaze.com/people/3020/castcredits
|
||||
3032;http://api.tvmaze.com/people/3032/castcredits
|
||||
3046;http://api.tvmaze.com/people/3046/castcredits
|
||||
3092;http://api.tvmaze.com/people/3092/castcredits
|
||||
3261;http://api.tvmaze.com/people/3261/castcredits
|
||||
3302;http://api.tvmaze.com/people/3302/castcredits
|
||||
3346;http://api.tvmaze.com/people/3346/castcredits
|
||||
3428;http://api.tvmaze.com/people/3428/castcredits
|
||||
3454;http://api.tvmaze.com/people/3454/castcredits
|
||||
3464;http://api.tvmaze.com/people/3464/castcredits
|
||||
3505;http://api.tvmaze.com/people/3505/castcredits
|
||||
3581;http://api.tvmaze.com/people/3581/castcredits
|
||||
3777;http://api.tvmaze.com/people/3777/castcredits
|
||||
3841;http://api.tvmaze.com/people/3841/castcredits
|
||||
3878;http://api.tvmaze.com/people/3878/castcredits
|
||||
3909;http://api.tvmaze.com/people/3909/castcredits
|
||||
3942;http://api.tvmaze.com/people/3942/castcredits
|
||||
3944;http://api.tvmaze.com/people/3944/castcredits
|
||||
3946;http://api.tvmaze.com/people/3946/castcredits
|
||||
3948;http://api.tvmaze.com/people/3948/castcredits
|
||||
3950;http://api.tvmaze.com/people/3950/castcredits
|
||||
3952;http://api.tvmaze.com/people/3952/castcredits
|
||||
3954;http://api.tvmaze.com/people/3954/castcredits
|
||||
3965;http://api.tvmaze.com/people/3965/castcredits
|
||||
4046;http://api.tvmaze.com/people/4046/castcredits
|
||||
4193;http://api.tvmaze.com/people/4193/castcredits
|
||||
4261;http://api.tvmaze.com/people/4261/castcredits
|
||||
4276;http://api.tvmaze.com/people/4276/castcredits
|
||||
4379;http://api.tvmaze.com/people/4379/castcredits
|
||||
4395;http://api.tvmaze.com/people/4395/castcredits
|
||||
4406;http://api.tvmaze.com/people/4406/castcredits
|
||||
4445;http://api.tvmaze.com/people/4445/castcredits
|
||||
4492;http://api.tvmaze.com/people/4492/castcredits
|
||||
4548;http://api.tvmaze.com/people/4548/castcredits
|
||||
4585;http://api.tvmaze.com/people/4585/castcredits
|
||||
4601;http://api.tvmaze.com/people/4601/castcredits
|
||||
4677;http://api.tvmaze.com/people/4677/castcredits
|
||||
4820;http://api.tvmaze.com/people/4820/castcredits
|
||||
4841;http://api.tvmaze.com/people/4841/castcredits
|
||||
4851;http://api.tvmaze.com/people/4851/castcredits
|
||||
4908;http://api.tvmaze.com/people/4908/castcredits
|
||||
4911;http://api.tvmaze.com/people/4911/castcredits
|
||||
4941;http://api.tvmaze.com/people/4941/castcredits
|
||||
4983;http://api.tvmaze.com/people/4983/castcredits
|
||||
5010;http://api.tvmaze.com/people/5010/castcredits
|
||||
5013;http://api.tvmaze.com/people/5013/castcredits
|
||||
5019;http://api.tvmaze.com/people/5019/castcredits
|
||||
5051;http://api.tvmaze.com/people/5051/castcredits
|
||||
5066;http://api.tvmaze.com/people/5066/castcredits
|
||||
5198;http://api.tvmaze.com/people/5198/castcredits
|
||||
5205;http://api.tvmaze.com/people/5205/castcredits
|
||||
5208;http://api.tvmaze.com/people/5208/castcredits
|
||||
5209;http://api.tvmaze.com/people/5209/castcredits
|
||||
5210;http://api.tvmaze.com/people/5210/castcredits
|
||||
5211;http://api.tvmaze.com/people/5211/castcredits
|
||||
5212;http://api.tvmaze.com/people/5212/castcredits
|
||||
5233;http://api.tvmaze.com/people/5233/castcredits
|
||||
5277;http://api.tvmaze.com/people/5277/castcredits
|
||||
5425;http://api.tvmaze.com/people/5425/castcredits
|
||||
5450;http://api.tvmaze.com/people/5450/castcredits
|
||||
5584;http://api.tvmaze.com/people/5584/castcredits
|
||||
5615;http://api.tvmaze.com/people/5615/castcredits
|
||||
5622;http://api.tvmaze.com/people/5622/castcredits
|
||||
5758;http://api.tvmaze.com/people/5758/castcredits
|
||||
5760;http://api.tvmaze.com/people/5760/castcredits
|
||||
5777;http://api.tvmaze.com/people/5777/castcredits
|
||||
5885;http://api.tvmaze.com/people/5885/castcredits
|
||||
5934;http://api.tvmaze.com/people/5934/castcredits
|
||||
5936;http://api.tvmaze.com/people/5936/castcredits
|
||||
5950;http://api.tvmaze.com/people/5950/castcredits
|
||||
5952;http://api.tvmaze.com/people/5952/castcredits
|
||||
5954;http://api.tvmaze.com/people/5954/castcredits
|
||||
5992;http://api.tvmaze.com/people/5992/castcredits
|
||||
5994;http://api.tvmaze.com/people/5994/castcredits
|
||||
5996;http://api.tvmaze.com/people/5996/castcredits
|
||||
5998;http://api.tvmaze.com/people/5998/castcredits
|
||||
6000;http://api.tvmaze.com/people/6000/castcredits
|
||||
6002;http://api.tvmaze.com/people/6002/castcredits
|
||||
6004;http://api.tvmaze.com/people/6004/castcredits
|
||||
6006;http://api.tvmaze.com/people/6006/castcredits
|
||||
6008;http://api.tvmaze.com/people/6008/castcredits
|
||||
6010;http://api.tvmaze.com/people/6010/castcredits
|
||||
6012;http://api.tvmaze.com/people/6012/castcredits
|
||||
6014;http://api.tvmaze.com/people/6014/castcredits
|
||||
6016;http://api.tvmaze.com/people/6016/castcredits
|
||||
6018;http://api.tvmaze.com/people/6018/castcredits
|
||||
6071;http://api.tvmaze.com/people/6071/castcredits
|
||||
6130;http://api.tvmaze.com/people/6130/castcredits
|
||||
6158;http://api.tvmaze.com/people/6158/castcredits
|
||||
6185;http://api.tvmaze.com/people/6185/castcredits
|
||||
6197;http://api.tvmaze.com/people/6197/castcredits
|
||||
6199;http://api.tvmaze.com/people/6199/castcredits
|
||||
6201;http://api.tvmaze.com/people/6201/castcredits
|
||||
6203;http://api.tvmaze.com/people/6203/castcredits
|
||||
6205;http://api.tvmaze.com/people/6205/castcredits
|
||||
6207;http://api.tvmaze.com/people/6207/castcredits
|
||||
6463;http://api.tvmaze.com/people/6463/castcredits
|
||||
6471;http://api.tvmaze.com/people/6471/castcredits
|
||||
6642;http://api.tvmaze.com/people/6642/castcredits
|
||||
6693;http://api.tvmaze.com/people/6693/castcredits
|
||||
6719;http://api.tvmaze.com/people/6719/castcredits
|
||||
6741;http://api.tvmaze.com/people/6741/castcredits
|
||||
6801;http://api.tvmaze.com/people/6801/castcredits
|
||||
6943;http://api.tvmaze.com/people/6943/castcredits
|
||||
7063;http://api.tvmaze.com/people/7063/castcredits
|
||||
7251;http://api.tvmaze.com/people/7251/castcredits
|
||||
7282;http://api.tvmaze.com/people/7282/castcredits
|
||||
7320;http://api.tvmaze.com/people/7320/castcredits
|
||||
7574;http://api.tvmaze.com/people/7574/castcredits
|
||||
7591;http://api.tvmaze.com/people/7591/castcredits
|
||||
7686;http://api.tvmaze.com/people/7686/castcredits
|
||||
7730;http://api.tvmaze.com/people/7730/castcredits
|
||||
7740;http://api.tvmaze.com/people/7740/castcredits
|
||||
7815;http://api.tvmaze.com/people/7815/castcredits
|
||||
7834;http://api.tvmaze.com/people/7834/castcredits
|
||||
7848;http://api.tvmaze.com/people/7848/castcredits
|
||||
7876;http://api.tvmaze.com/people/7876/castcredits
|
||||
7950;http://api.tvmaze.com/people/7950/castcredits
|
||||
8071;http://api.tvmaze.com/people/8071/castcredits
|
||||
8081;http://api.tvmaze.com/people/8081/castcredits
|
||||
8089;http://api.tvmaze.com/people/8089/castcredits
|
||||
8171;http://api.tvmaze.com/people/8171/castcredits
|
||||
8218;http://api.tvmaze.com/people/8218/castcredits
|
||||
8382;http://api.tvmaze.com/people/8382/castcredits
|
||||
8450;http://api.tvmaze.com/people/8450/castcredits
|
||||
8497;http://api.tvmaze.com/people/8497/castcredits
|
||||
8524;http://api.tvmaze.com/people/8524/castcredits
|
||||
8525;http://api.tvmaze.com/people/8525/castcredits
|
||||
8527;http://api.tvmaze.com/people/8527/castcredits
|
||||
8560;http://api.tvmaze.com/people/8560/castcredits
|
||||
8574;http://api.tvmaze.com/people/8574/castcredits
|
||||
8581;http://api.tvmaze.com/people/8581/castcredits
|
||||
8585;http://api.tvmaze.com/people/8585/castcredits
|
||||
8587;http://api.tvmaze.com/people/8587/castcredits
|
||||
8589;http://api.tvmaze.com/people/8589/castcredits
|
||||
8590;http://api.tvmaze.com/people/8590/castcredits
|
||||
8591;http://api.tvmaze.com/people/8591/castcredits
|
||||
8593;http://api.tvmaze.com/people/8593/castcredits
|
||||
8692;http://api.tvmaze.com/people/8692/castcredits
|
||||
8747;http://api.tvmaze.com/people/8747/castcredits
|
||||
8752;http://api.tvmaze.com/people/8752/castcredits
|
||||
8789;http://api.tvmaze.com/people/8789/castcredits
|
||||
8848;http://api.tvmaze.com/people/8848/castcredits
|
||||
9010;http://api.tvmaze.com/people/9010/castcredits
|
||||
9037;http://api.tvmaze.com/people/9037/castcredits
|
||||
9162;http://api.tvmaze.com/people/9162/castcredits
|
||||
9212;http://api.tvmaze.com/people/9212/castcredits
|
||||
9217;http://api.tvmaze.com/people/9217/castcredits
|
||||
9219;http://api.tvmaze.com/people/9219/castcredits
|
||||
9221;http://api.tvmaze.com/people/9221/castcredits
|
||||
9223;http://api.tvmaze.com/people/9223/castcredits
|
||||
9225;http://api.tvmaze.com/people/9225/castcredits
|
||||
9237;http://api.tvmaze.com/people/9237/castcredits
|
||||
9286;http://api.tvmaze.com/people/9286/castcredits
|
||||
9317;http://api.tvmaze.com/people/9317/castcredits
|
||||
9320;http://api.tvmaze.com/people/9320/castcredits
|
||||
9321;http://api.tvmaze.com/people/9321/castcredits
|
||||
9322;http://api.tvmaze.com/people/9322/castcredits
|
||||
9323;http://api.tvmaze.com/people/9323/castcredits
|
||||
9324;http://api.tvmaze.com/people/9324/castcredits
|
||||
9325;http://api.tvmaze.com/people/9325/castcredits
|
||||
9326;http://api.tvmaze.com/people/9326/castcredits
|
||||
9327;http://api.tvmaze.com/people/9327/castcredits
|
||||
9328;http://api.tvmaze.com/people/9328/castcredits
|
||||
9329;http://api.tvmaze.com/people/9329/castcredits
|
||||
9330;http://api.tvmaze.com/people/9330/castcredits
|
||||
9331;http://api.tvmaze.com/people/9331/castcredits
|
||||
9332;http://api.tvmaze.com/people/9332/castcredits
|
||||
9334;http://api.tvmaze.com/people/9334/castcredits
|
||||
9335;http://api.tvmaze.com/people/9335/castcredits
|
||||
9336;http://api.tvmaze.com/people/9336/castcredits
|
||||
9355;http://api.tvmaze.com/people/9355/castcredits
|
||||
9666;http://api.tvmaze.com/people/9666/castcredits
|
||||
9754;http://api.tvmaze.com/people/9754/castcredits
|
||||
9772;http://api.tvmaze.com/people/9772/castcredits
|
||||
9801;http://api.tvmaze.com/people/9801/castcredits
|
||||
9851;http://api.tvmaze.com/people/9851/castcredits
|
||||
9891;http://api.tvmaze.com/people/9891/castcredits
|
||||
9913;http://api.tvmaze.com/people/9913/castcredits
|
||||
9929;http://api.tvmaze.com/people/9929/castcredits
|
||||
9931;http://api.tvmaze.com/people/9931/castcredits
|
||||
9933;http://api.tvmaze.com/people/9933/castcredits
|
||||
9935;http://api.tvmaze.com/people/9935/castcredits
|
||||
9937;http://api.tvmaze.com/people/9937/castcredits
|
||||
9939;http://api.tvmaze.com/people/9939/castcredits
|
||||
9948;http://api.tvmaze.com/people/9948/castcredits
|
||||
9971;http://api.tvmaze.com/people/9971/castcredits
|
||||
10118;http://api.tvmaze.com/people/10118/castcredits
|
||||
10119;http://api.tvmaze.com/people/10119/castcredits
|
||||
10120;http://api.tvmaze.com/people/10120/castcredits
|
||||
10121;http://api.tvmaze.com/people/10121/castcredits
|
||||
10122;http://api.tvmaze.com/people/10122/castcredits
|
||||
10123;http://api.tvmaze.com/people/10123/castcredits
|
||||
10124;http://api.tvmaze.com/people/10124/castcredits
|
||||
10125;http://api.tvmaze.com/people/10125/castcredits
|
||||
10126;http://api.tvmaze.com/people/10126/castcredits
|
||||
10127;http://api.tvmaze.com/people/10127/castcredits
|
||||
137;http://api.tvmaze.com/people/137/castcredits
|
||||
145;http://api.tvmaze.com/people/145/castcredits
|
||||
211;http://api.tvmaze.com/people/211/castcredits
|
||||
214;http://api.tvmaze.com/people/214/castcredits
|
||||
315;http://api.tvmaze.com/people/315/castcredits
|
||||
368;http://api.tvmaze.com/people/368/castcredits
|
||||
416;http://api.tvmaze.com/people/416/castcredits
|
||||
473;http://api.tvmaze.com/people/473/castcredits
|
||||
557;http://api.tvmaze.com/people/557/castcredits
|
||||
577;http://api.tvmaze.com/people/577/castcredits
|
||||
583;http://api.tvmaze.com/people/583/castcredits
|
||||
635;http://api.tvmaze.com/people/635/castcredits
|
||||
688;http://api.tvmaze.com/people/688/castcredits
|
||||
743;http://api.tvmaze.com/people/743/castcredits
|
||||
771;http://api.tvmaze.com/people/771/castcredits
|
||||
10148;http://api.tvmaze.com/people/10148/castcredits
|
||||
10149;http://api.tvmaze.com/people/10149/castcredits
|
||||
10162;http://api.tvmaze.com/people/10162/castcredits
|
||||
10177;http://api.tvmaze.com/people/10177/castcredits
|
||||
10213;http://api.tvmaze.com/people/10213/castcredits
|
||||
10271;http://api.tvmaze.com/people/10271/castcredits
|
||||
10283;http://api.tvmaze.com/people/10283/castcredits
|
||||
10309;http://api.tvmaze.com/people/10309/castcredits
|
||||
10333;http://api.tvmaze.com/people/10333/castcredits
|
||||
10355;http://api.tvmaze.com/people/10355/castcredits
|
||||
10383;http://api.tvmaze.com/people/10383/castcredits
|
||||
10390;http://api.tvmaze.com/people/10390/castcredits
|
||||
10455;http://api.tvmaze.com/people/10455/castcredits
|
||||
10462;http://api.tvmaze.com/people/10462/castcredits
|
||||
10477;http://api.tvmaze.com/people/10477/castcredits
|
||||
10513;http://api.tvmaze.com/people/10513/castcredits
|
||||
10529;http://api.tvmaze.com/people/10529/castcredits
|
||||
10532;http://api.tvmaze.com/people/10532/castcredits
|
||||
10534;http://api.tvmaze.com/people/10534/castcredits
|
||||
10616;http://api.tvmaze.com/people/10616/castcredits
|
||||
10647;http://api.tvmaze.com/people/10647/castcredits
|
||||
10659;http://api.tvmaze.com/people/10659/castcredits
|
||||
10720;http://api.tvmaze.com/people/10720/castcredits
|
||||
10862;http://api.tvmaze.com/people/10862/castcredits
|
||||
10864;http://api.tvmaze.com/people/10864/castcredits
|
||||
10865;http://api.tvmaze.com/people/10865/castcredits
|
||||
10866;http://api.tvmaze.com/people/10866/castcredits
|
||||
10867;http://api.tvmaze.com/people/10867/castcredits
|
||||
10868;http://api.tvmaze.com/people/10868/castcredits
|
||||
10970;http://api.tvmaze.com/people/10970/castcredits
|
||||
11039;http://api.tvmaze.com/people/11039/castcredits
|
||||
11047;http://api.tvmaze.com/people/11047/castcredits
|
||||
11079;http://api.tvmaze.com/people/11079/castcredits
|
||||
11150;http://api.tvmaze.com/people/11150/castcredits
|
||||
11178;http://api.tvmaze.com/people/11178/castcredits
|
||||
11312;http://api.tvmaze.com/people/11312/castcredits
|
||||
11402;http://api.tvmaze.com/people/11402/castcredits
|
||||
11453;http://api.tvmaze.com/people/11453/castcredits
|
||||
11457;http://api.tvmaze.com/people/11457/castcredits
|
||||
11493;http://api.tvmaze.com/people/11493/castcredits
|
||||
11546;http://api.tvmaze.com/people/11546/castcredits
|
||||
11547;http://api.tvmaze.com/people/11547/castcredits
|
||||
11548;http://api.tvmaze.com/people/11548/castcredits
|
||||
11549;http://api.tvmaze.com/people/11549/castcredits
|
||||
11550;http://api.tvmaze.com/people/11550/castcredits
|
||||
11551;http://api.tvmaze.com/people/11551/castcredits
|
||||
11593;http://api.tvmaze.com/people/11593/castcredits
|
||||
11621;http://api.tvmaze.com/people/11621/castcredits
|
||||
11759;http://api.tvmaze.com/people/11759/castcredits
|
||||
11790;http://api.tvmaze.com/people/11790/castcredits
|
||||
11952;http://api.tvmaze.com/people/11952/castcredits
|
||||
11971;http://api.tvmaze.com/people/11971/castcredits
|
||||
11989;http://api.tvmaze.com/people/11989/castcredits
|
||||
12043;http://api.tvmaze.com/people/12043/castcredits
|
||||
12066;http://api.tvmaze.com/people/12066/castcredits
|
||||
12096;http://api.tvmaze.com/people/12096/castcredits
|
||||
12097;http://api.tvmaze.com/people/12097/castcredits
|
||||
12098;http://api.tvmaze.com/people/12098/castcredits
|
||||
12099;http://api.tvmaze.com/people/12099/castcredits
|
||||
12100;http://api.tvmaze.com/people/12100/castcredits
|
||||
12101;http://api.tvmaze.com/people/12101/castcredits
|
||||
12102;http://api.tvmaze.com/people/12102/castcredits
|
||||
12141;http://api.tvmaze.com/people/12141/castcredits
|
||||
12146;http://api.tvmaze.com/people/12146/castcredits
|
||||
12147;http://api.tvmaze.com/people/12147/castcredits
|
||||
12148;http://api.tvmaze.com/people/12148/castcredits
|
||||
12149;http://api.tvmaze.com/people/12149/castcredits
|
||||
12150;http://api.tvmaze.com/people/12150/castcredits
|
||||
12201;http://api.tvmaze.com/people/12201/castcredits
|
||||
12379;http://api.tvmaze.com/people/12379/castcredits
|
||||
12445;http://api.tvmaze.com/people/12445/castcredits
|
||||
12575;http://api.tvmaze.com/people/12575/castcredits
|
||||
12650;http://api.tvmaze.com/people/12650/castcredits
|
||||
12666;http://api.tvmaze.com/people/12666/castcredits
|
||||
12842;http://api.tvmaze.com/people/12842/castcredits
|
||||
12872;http://api.tvmaze.com/people/12872/castcredits
|
||||
12929;http://api.tvmaze.com/people/12929/castcredits
|
||||
13169;http://api.tvmaze.com/people/13169/castcredits
|
||||
13232;http://api.tvmaze.com/people/13232/castcredits
|
||||
13262;http://api.tvmaze.com/people/13262/castcredits
|
||||
13277;http://api.tvmaze.com/people/13277/castcredits
|
||||
13509;http://api.tvmaze.com/people/13509/castcredits
|
||||
13513;http://api.tvmaze.com/people/13513/castcredits
|
||||
13516;http://api.tvmaze.com/people/13516/castcredits
|
||||
13518;http://api.tvmaze.com/people/13518/castcredits
|
||||
13519;http://api.tvmaze.com/people/13519/castcredits
|
||||
13520;http://api.tvmaze.com/people/13520/castcredits
|
||||
13679;http://api.tvmaze.com/people/13679/castcredits
|
||||
13760;http://api.tvmaze.com/people/13760/castcredits
|
||||
13774;http://api.tvmaze.com/people/13774/castcredits
|
||||
13922;http://api.tvmaze.com/people/13922/castcredits
|
||||
13957;http://api.tvmaze.com/people/13957/castcredits
|
||||
13968;http://api.tvmaze.com/people/13968/castcredits
|
||||
13997;http://api.tvmaze.com/people/13997/castcredits
|
||||
14057;http://api.tvmaze.com/people/14057/castcredits
|
||||
14099;http://api.tvmaze.com/people/14099/castcredits
|
||||
14132;http://api.tvmaze.com/people/14132/castcredits
|
||||
14159;http://api.tvmaze.com/people/14159/castcredits
|
||||
14162;http://api.tvmaze.com/people/14162/castcredits
|
||||
14307;http://api.tvmaze.com/people/14307/castcredits
|
||||
14309;http://api.tvmaze.com/people/14309/castcredits
|
||||
14310;http://api.tvmaze.com/people/14310/castcredits
|
||||
14397;http://api.tvmaze.com/people/14397/castcredits
|
||||
14436;http://api.tvmaze.com/people/14436/castcredits
|
||||
14472;http://api.tvmaze.com/people/14472/castcredits
|
||||
14501;http://api.tvmaze.com/people/14501/castcredits
|
||||
14525;http://api.tvmaze.com/people/14525/castcredits
|
||||
14541;http://api.tvmaze.com/people/14541/castcredits
|
||||
14566;http://api.tvmaze.com/people/14566/castcredits
|
||||
14572;http://api.tvmaze.com/people/14572/castcredits
|
||||
14625;http://api.tvmaze.com/people/14625/castcredits
|
||||
14627;http://api.tvmaze.com/people/14627/castcredits
|
||||
14671;http://api.tvmaze.com/people/14671/castcredits
|
||||
14672;http://api.tvmaze.com/people/14672/castcredits
|
||||
14673;http://api.tvmaze.com/people/14673/castcredits
|
||||
14674;http://api.tvmaze.com/people/14674/castcredits
|
||||
14853;http://api.tvmaze.com/people/14853/castcredits
|
||||
14864;http://api.tvmaze.com/people/14864/castcredits
|
||||
14872;http://api.tvmaze.com/people/14872/castcredits
|
||||
14960;http://api.tvmaze.com/people/14960/castcredits
|
||||
14977;http://api.tvmaze.com/people/14977/castcredits
|
||||
14996;http://api.tvmaze.com/people/14996/castcredits
|
||||
15000;http://api.tvmaze.com/people/15000/castcredits
|
||||
15021;http://api.tvmaze.com/people/15021/castcredits
|
||||
15028;http://api.tvmaze.com/people/15028/castcredits
|
||||
15029;http://api.tvmaze.com/people/15029/castcredits
|
||||
15032;http://api.tvmaze.com/people/15032/castcredits
|
||||
15036;http://api.tvmaze.com/people/15036/castcredits
|
||||
15043;http://api.tvmaze.com/people/15043/castcredits
|
||||
15048;http://api.tvmaze.com/people/15048/castcredits
|
||||
15074;http://api.tvmaze.com/people/15074/castcredits
|
||||
15098;http://api.tvmaze.com/people/15098/castcredits
|
||||
15108;http://api.tvmaze.com/people/15108/castcredits
|
||||
15114;http://api.tvmaze.com/people/15114/castcredits
|
||||
15128;http://api.tvmaze.com/people/15128/castcredits
|
||||
15147;http://api.tvmaze.com/people/15147/castcredits
|
||||
15150;http://api.tvmaze.com/people/15150/castcredits
|
||||
15172;http://api.tvmaze.com/people/15172/castcredits
|
||||
15188;http://api.tvmaze.com/people/15188/castcredits
|
||||
15189;http://api.tvmaze.com/people/15189/castcredits
|
||||
15196;http://api.tvmaze.com/people/15196/castcredits
|
||||
15219;http://api.tvmaze.com/people/15219/castcredits
|
||||
15220;http://api.tvmaze.com/people/15220/castcredits
|
||||
15234;http://api.tvmaze.com/people/15234/castcredits
|
||||
15257;http://api.tvmaze.com/people/15257/castcredits
|
||||
15270;http://api.tvmaze.com/people/15270/castcredits
|
||||
15276;http://api.tvmaze.com/people/15276/castcredits
|
||||
15279;http://api.tvmaze.com/people/15279/castcredits
|
||||
15281;http://api.tvmaze.com/people/15281/castcredits
|
||||
15285;http://api.tvmaze.com/people/15285/castcredits
|
||||
15292;http://api.tvmaze.com/people/15292/castcredits
|
||||
15295;http://api.tvmaze.com/people/15295/castcredits
|
||||
15315;http://api.tvmaze.com/people/15315/castcredits
|
||||
15344;http://api.tvmaze.com/people/15344/castcredits
|
||||
15369;http://api.tvmaze.com/people/15369/castcredits
|
||||
15371;http://api.tvmaze.com/people/15371/castcredits
|
||||
15426;http://api.tvmaze.com/people/15426/castcredits
|
||||
15443;http://api.tvmaze.com/people/15443/castcredits
|
||||
15448;http://api.tvmaze.com/people/15448/castcredits
|
||||
15449;http://api.tvmaze.com/people/15449/castcredits
|
||||
15450;http://api.tvmaze.com/people/15450/castcredits
|
||||
15451;http://api.tvmaze.com/people/15451/castcredits
|
||||
15454;http://api.tvmaze.com/people/15454/castcredits
|
||||
15473;http://api.tvmaze.com/people/15473/castcredits
|
||||
15474;http://api.tvmaze.com/people/15474/castcredits
|
||||
15496;http://api.tvmaze.com/people/15496/castcredits
|
||||
15509;http://api.tvmaze.com/people/15509/castcredits
|
||||
15520;http://api.tvmaze.com/people/15520/castcredits
|
||||
15534;http://api.tvmaze.com/people/15534/castcredits
|
||||
15535;http://api.tvmaze.com/people/15535/castcredits
|
||||
15574;http://api.tvmaze.com/people/15574/castcredits
|
||||
15587;http://api.tvmaze.com/people/15587/castcredits
|
||||
15613;http://api.tvmaze.com/people/15613/castcredits
|
||||
15630;http://api.tvmaze.com/people/15630/castcredits
|
||||
15631;http://api.tvmaze.com/people/15631/castcredits
|
||||
15643;http://api.tvmaze.com/people/15643/castcredits
|
||||
15644;http://api.tvmaze.com/people/15644/castcredits
|
||||
15645;http://api.tvmaze.com/people/15645/castcredits
|
||||
15647;http://api.tvmaze.com/people/15647/castcredits
|
||||
15654;http://api.tvmaze.com/people/15654/castcredits
|
||||
15676;http://api.tvmaze.com/people/15676/castcredits
|
||||
15677;http://api.tvmaze.com/people/15677/castcredits
|
||||
15678;http://api.tvmaze.com/people/15678/castcredits
|
||||
15688;http://api.tvmaze.com/people/15688/castcredits
|
||||
15689;http://api.tvmaze.com/people/15689/castcredits
|
||||
15693;http://api.tvmaze.com/people/15693/castcredits
|
||||
15698;http://api.tvmaze.com/people/15698/castcredits
|
||||
15701;http://api.tvmaze.com/people/15701/castcredits
|
||||
15706;http://api.tvmaze.com/people/15706/castcredits
|
||||
15716;http://api.tvmaze.com/people/15716/castcredits
|
||||
15731;http://api.tvmaze.com/people/15731/castcredits
|
||||
15751;http://api.tvmaze.com/people/15751/castcredits
|
||||
15767;http://api.tvmaze.com/people/15767/castcredits
|
||||
15771;http://api.tvmaze.com/people/15771/castcredits
|
||||
15778;http://api.tvmaze.com/people/15778/castcredits
|
||||
15812;http://api.tvmaze.com/people/15812/castcredits
|
||||
15824;http://api.tvmaze.com/people/15824/castcredits
|
||||
15826;http://api.tvmaze.com/people/15826/castcredits
|
||||
15847;http://api.tvmaze.com/people/15847/castcredits
|
||||
15855;http://api.tvmaze.com/people/15855/castcredits
|
||||
15866;http://api.tvmaze.com/people/15866/castcredits
|
||||
15882;http://api.tvmaze.com/people/15882/castcredits
|
||||
15909;http://api.tvmaze.com/people/15909/castcredits
|
||||
15928;http://api.tvmaze.com/people/15928/castcredits
|
||||
16316;http://api.tvmaze.com/people/16316/castcredits
|
||||
16317;http://api.tvmaze.com/people/16317/castcredits
|
||||
16331;http://api.tvmaze.com/people/16331/castcredits
|
||||
16332;http://api.tvmaze.com/people/16332/castcredits
|
||||
16340;http://api.tvmaze.com/people/16340/castcredits
|
||||
16341;http://api.tvmaze.com/people/16341/castcredits
|
||||
16343;http://api.tvmaze.com/people/16343/castcredits
|
||||
16348;http://api.tvmaze.com/people/16348/castcredits
|
||||
16352;http://api.tvmaze.com/people/16352/castcredits
|
||||
16362;http://api.tvmaze.com/people/16362/castcredits
|
||||
16363;http://api.tvmaze.com/people/16363/castcredits
|
||||
16367;http://api.tvmaze.com/people/16367/castcredits
|
||||
16368;http://api.tvmaze.com/people/16368/castcredits
|
||||
16383;http://api.tvmaze.com/people/16383/castcredits
|
||||
16386;http://api.tvmaze.com/people/16386/castcredits
|
||||
16387;http://api.tvmaze.com/people/16387/castcredits
|
||||
16394;http://api.tvmaze.com/people/16394/castcredits
|
||||
16395;http://api.tvmaze.com/people/16395/castcredits
|
||||
16398;http://api.tvmaze.com/people/16398/castcredits
|
||||
16399;http://api.tvmaze.com/people/16399/castcredits
|
||||
16400;http://api.tvmaze.com/people/16400/castcredits
|
||||
16404;http://api.tvmaze.com/people/16404/castcredits
|
||||
16407;http://api.tvmaze.com/people/16407/castcredits
|
||||
16417;http://api.tvmaze.com/people/16417/castcredits
|
||||
16424;http://api.tvmaze.com/people/16424/castcredits
|
||||
16435;http://api.tvmaze.com/people/16435/castcredits
|
||||
16440;http://api.tvmaze.com/people/16440/castcredits
|
||||
16442;http://api.tvmaze.com/people/16442/castcredits
|
||||
16443;http://api.tvmaze.com/people/16443/castcredits
|
||||
16448;http://api.tvmaze.com/people/16448/castcredits
|
||||
16449;http://api.tvmaze.com/people/16449/castcredits
|
||||
16469;http://api.tvmaze.com/people/16469/castcredits
|
||||
16470;http://api.tvmaze.com/people/16470/castcredits
|
||||
16473;http://api.tvmaze.com/people/16473/castcredits
|
||||
16474;http://api.tvmaze.com/people/16474/castcredits
|
||||
16475;http://api.tvmaze.com/people/16475/castcredits
|
||||
16476;http://api.tvmaze.com/people/16476/castcredits
|
||||
16481;http://api.tvmaze.com/people/16481/castcredits
|
||||
16492;http://api.tvmaze.com/people/16492/castcredits
|
||||
16493;http://api.tvmaze.com/people/16493/castcredits
|
||||
16496;http://api.tvmaze.com/people/16496/castcredits
|
||||
16497;http://api.tvmaze.com/people/16497/castcredits
|
||||
16506;http://api.tvmaze.com/people/16506/castcredits
|
||||
16507;http://api.tvmaze.com/people/16507/castcredits
|
||||
16510;http://api.tvmaze.com/people/16510/castcredits
|
||||
16511;http://api.tvmaze.com/people/16511/castcredits
|
||||
16512;http://api.tvmaze.com/people/16512/castcredits
|
||||
16514;http://api.tvmaze.com/people/16514/castcredits
|
||||
16515;http://api.tvmaze.com/people/16515/castcredits
|
||||
16522;http://api.tvmaze.com/people/16522/castcredits
|
||||
16523;http://api.tvmaze.com/people/16523/castcredits
|
||||
16528;http://api.tvmaze.com/people/16528/castcredits
|
||||
16534;http://api.tvmaze.com/people/16534/castcredits
|
||||
16535;http://api.tvmaze.com/people/16535/castcredits
|
||||
16539;http://api.tvmaze.com/people/16539/castcredits
|
||||
16540;http://api.tvmaze.com/people/16540/castcredits
|
||||
16542;http://api.tvmaze.com/people/16542/castcredits
|
||||
16547;http://api.tvmaze.com/people/16547/castcredits
|
||||
16548;http://api.tvmaze.com/people/16548/castcredits
|
||||
16549;http://api.tvmaze.com/people/16549/castcredits
|
||||
16551;http://api.tvmaze.com/people/16551/castcredits
|
||||
16552;http://api.tvmaze.com/people/16552/castcredits
|
||||
16559;http://api.tvmaze.com/people/16559/castcredits
|
||||
16563;http://api.tvmaze.com/people/16563/castcredits
|
||||
16564;http://api.tvmaze.com/people/16564/castcredits
|
||||
16567;http://api.tvmaze.com/people/16567/castcredits
|
||||
16568;http://api.tvmaze.com/people/16568/castcredits
|
||||
16569;http://api.tvmaze.com/people/16569/castcredits
|
||||
16574;http://api.tvmaze.com/people/16574/castcredits
|
||||
16578;http://api.tvmaze.com/people/16578/castcredits
|
||||
16579;http://api.tvmaze.com/people/16579/castcredits
|
||||
16580;http://api.tvmaze.com/people/16580/castcredits
|
||||
16586;http://api.tvmaze.com/people/16586/castcredits
|
||||
16587;http://api.tvmaze.com/people/16587/castcredits
|
||||
16588;http://api.tvmaze.com/people/16588/castcredits
|
||||
16593;http://api.tvmaze.com/people/16593/castcredits
|
||||
16596;http://api.tvmaze.com/people/16596/castcredits
|
||||
16597;http://api.tvmaze.com/people/16597/castcredits
|
||||
16598;http://api.tvmaze.com/people/16598/castcredits
|
||||
16601;http://api.tvmaze.com/people/16601/castcredits
|
||||
16602;http://api.tvmaze.com/people/16602/castcredits
|
||||
16605;http://api.tvmaze.com/people/16605/castcredits
|
||||
16617;http://api.tvmaze.com/people/16617/castcredits
|
||||
16618;http://api.tvmaze.com/people/16618/castcredits
|
||||
16626;http://api.tvmaze.com/people/16626/castcredits
|
||||
16627;http://api.tvmaze.com/people/16627/castcredits
|
||||
16628;http://api.tvmaze.com/people/16628/castcredits
|
||||
16633;http://api.tvmaze.com/people/16633/castcredits
|
||||
16634;http://api.tvmaze.com/people/16634/castcredits
|
||||
16635;http://api.tvmaze.com/people/16635/castcredits
|
||||
16642;http://api.tvmaze.com/people/16642/castcredits
|
||||
16643;http://api.tvmaze.com/people/16643/castcredits
|
||||
16649;http://api.tvmaze.com/people/16649/castcredits
|
||||
16651;http://api.tvmaze.com/people/16651/castcredits
|
||||
16657;http://api.tvmaze.com/people/16657/castcredits
|
||||
16664;http://api.tvmaze.com/people/16664/castcredits
|
||||
16665;http://api.tvmaze.com/people/16665/castcredits
|
||||
16666;http://api.tvmaze.com/people/16666/castcredits
|
||||
16670;http://api.tvmaze.com/people/16670/castcredits
|
||||
16671;http://api.tvmaze.com/people/16671/castcredits
|
||||
16672;http://api.tvmaze.com/people/16672/castcredits
|
||||
16678;http://api.tvmaze.com/people/16678/castcredits
|
||||
16680;http://api.tvmaze.com/people/16680/castcredits
|
||||
16685;http://api.tvmaze.com/people/16685/castcredits
|
||||
16690;http://api.tvmaze.com/people/16690/castcredits
|
||||
16691;http://api.tvmaze.com/people/16691/castcredits
|
||||
16692;http://api.tvmaze.com/people/16692/castcredits
|
||||
16696;http://api.tvmaze.com/people/16696/castcredits
|
||||
16697;http://api.tvmaze.com/people/16697/castcredits
|
||||
16698;http://api.tvmaze.com/people/16698/castcredits
|
||||
16704;http://api.tvmaze.com/people/16704/castcredits
|
||||
16705;http://api.tvmaze.com/people/16705/castcredits
|
||||
16706;http://api.tvmaze.com/people/16706/castcredits
|
||||
16708;http://api.tvmaze.com/people/16708/castcredits
|
||||
16712;http://api.tvmaze.com/people/16712/castcredits
|
||||
16718;http://api.tvmaze.com/people/16718/castcredits
|
||||
16719;http://api.tvmaze.com/people/16719/castcredits
|
||||
16720;http://api.tvmaze.com/people/16720/castcredits
|
||||
16726;http://api.tvmaze.com/people/16726/castcredits
|
||||
16727;http://api.tvmaze.com/people/16727/castcredits
|
||||
16737;http://api.tvmaze.com/people/16737/castcredits
|
||||
16738;http://api.tvmaze.com/people/16738/castcredits
|
||||
16741;http://api.tvmaze.com/people/16741/castcredits
|
||||
16751;http://api.tvmaze.com/people/16751/castcredits
|
||||
16753;http://api.tvmaze.com/people/16753/castcredits
|
||||
16759;http://api.tvmaze.com/people/16759/castcredits
|
||||
16766;http://api.tvmaze.com/people/16766/castcredits
|
||||
16769;http://api.tvmaze.com/people/16769/castcredits
|
||||
16771;http://api.tvmaze.com/people/16771/castcredits
|
||||
16787;http://api.tvmaze.com/people/16787/castcredits
|
||||
16788;http://api.tvmaze.com/people/16788/castcredits
|
||||
16797;http://api.tvmaze.com/people/16797/castcredits
|
||||
16798;http://api.tvmaze.com/people/16798/castcredits
|
||||
16802;http://api.tvmaze.com/people/16802/castcredits
|
||||
16803;http://api.tvmaze.com/people/16803/castcredits
|
||||
16804;http://api.tvmaze.com/people/16804/castcredits
|
||||
16808;http://api.tvmaze.com/people/16808/castcredits
|
||||
16813;http://api.tvmaze.com/people/16813/castcredits
|
||||
16814;http://api.tvmaze.com/people/16814/castcredits
|
||||
16817;http://api.tvmaze.com/people/16817/castcredits
|
||||
16824;http://api.tvmaze.com/people/16824/castcredits
|
||||
16825;http://api.tvmaze.com/people/16825/castcredits
|
||||
16826;http://api.tvmaze.com/people/16826/castcredits
|
||||
16827;http://api.tvmaze.com/people/16827/castcredits
|
||||
16828;http://api.tvmaze.com/people/16828/castcredits
|
||||
16834;http://api.tvmaze.com/people/16834/castcredits
|
||||
16835;http://api.tvmaze.com/people/16835/castcredits
|
||||
16836;http://api.tvmaze.com/people/16836/castcredits
|
||||
16837;http://api.tvmaze.com/people/16837/castcredits
|
||||
16838;http://api.tvmaze.com/people/16838/castcredits
|
||||
16839;http://api.tvmaze.com/people/16839/castcredits
|
||||
16844;http://api.tvmaze.com/people/16844/castcredits
|
||||
16845;http://api.tvmaze.com/people/16845/castcredits
|
||||
16854;http://api.tvmaze.com/people/16854/castcredits
|
||||
16858;http://api.tvmaze.com/people/16858/castcredits
|
||||
16859;http://api.tvmaze.com/people/16859/castcredits
|
||||
16971;http://api.tvmaze.com/people/16971/castcredits
|
||||
17044;http://api.tvmaze.com/people/17044/castcredits
|
||||
17049;http://api.tvmaze.com/people/17049/castcredits
|
||||
17306;http://api.tvmaze.com/people/17306/castcredits
|
||||
17334;http://api.tvmaze.com/people/17334/castcredits
|
||||
17371;http://api.tvmaze.com/people/17371/castcredits
|
||||
17445;http://api.tvmaze.com/people/17445/castcredits
|
||||
17470;http://api.tvmaze.com/people/17470/castcredits
|
||||
17505;http://api.tvmaze.com/people/17505/castcredits
|
||||
17558;http://api.tvmaze.com/people/17558/castcredits
|
||||
17588;http://api.tvmaze.com/people/17588/castcredits
|
||||
17640;http://api.tvmaze.com/people/17640/castcredits
|
||||
17657;http://api.tvmaze.com/people/17657/castcredits
|
||||
17719;http://api.tvmaze.com/people/17719/castcredits
|
||||
17736;http://api.tvmaze.com/people/17736/castcredits
|
||||
17783;http://api.tvmaze.com/people/17783/castcredits
|
||||
17818;http://api.tvmaze.com/people/17818/castcredits
|
||||
17890;http://api.tvmaze.com/people/17890/castcredits
|
||||
17898;http://api.tvmaze.com/people/17898/castcredits
|
||||
17954;http://api.tvmaze.com/people/17954/castcredits
|
||||
18043;http://api.tvmaze.com/people/18043/castcredits
|
||||
18080;http://api.tvmaze.com/people/18080/castcredits
|
||||
18086;http://api.tvmaze.com/people/18086/castcredits
|
||||
18308;http://api.tvmaze.com/people/18308/castcredits
|
||||
18469;http://api.tvmaze.com/people/18469/castcredits
|
||||
18508;http://api.tvmaze.com/people/18508/castcredits
|
||||
18676;http://api.tvmaze.com/people/18676/castcredits
|
||||
18880;http://api.tvmaze.com/people/18880/castcredits
|
||||
18882;http://api.tvmaze.com/people/18882/castcredits
|
||||
19047;http://api.tvmaze.com/people/19047/castcredits
|
||||
19107;http://api.tvmaze.com/people/19107/castcredits
|
||||
19194;http://api.tvmaze.com/people/19194/castcredits
|
||||
19205;http://api.tvmaze.com/people/19205/castcredits
|
||||
19214;http://api.tvmaze.com/people/19214/castcredits
|
||||
19228;http://api.tvmaze.com/people/19228/castcredits
|
||||
19441;http://api.tvmaze.com/people/19441/castcredits
|
||||
19502;http://api.tvmaze.com/people/19502/castcredits
|
||||
19504;http://api.tvmaze.com/people/19504/castcredits
|
||||
19536;http://api.tvmaze.com/people/19536/castcredits
|
||||
19559;http://api.tvmaze.com/people/19559/castcredits
|
||||
19606;http://api.tvmaze.com/people/19606/castcredits
|
||||
19611;http://api.tvmaze.com/people/19611/castcredits
|
||||
19619;http://api.tvmaze.com/people/19619/castcredits
|
||||
19640;http://api.tvmaze.com/people/19640/castcredits
|
||||
19686;http://api.tvmaze.com/people/19686/castcredits
|
||||
19689;http://api.tvmaze.com/people/19689/castcredits
|
||||
19690;http://api.tvmaze.com/people/19690/castcredits
|
||||
19691;http://api.tvmaze.com/people/19691/castcredits
|
||||
19699;http://api.tvmaze.com/people/19699/castcredits
|
||||
19769;http://api.tvmaze.com/people/19769/castcredits
|
||||
19788;http://api.tvmaze.com/people/19788/castcredits
|
||||
19840;http://api.tvmaze.com/people/19840/castcredits
|
||||
20084;http://api.tvmaze.com/people/20084/castcredits
|
||||
20213;http://api.tvmaze.com/people/20213/castcredits
|
||||
20287;http://api.tvmaze.com/people/20287/castcredits
|
||||
20289;http://api.tvmaze.com/people/20289/castcredits
|
||||
20327;http://api.tvmaze.com/people/20327/castcredits
|
||||
20385;http://api.tvmaze.com/people/20385/castcredits
|
||||
20402;http://api.tvmaze.com/people/20402/castcredits
|
||||
20421;http://api.tvmaze.com/people/20421/castcredits
|
||||
20463;http://api.tvmaze.com/people/20463/castcredits
|
||||
20468;http://api.tvmaze.com/people/20468/castcredits
|
||||
20540;http://api.tvmaze.com/people/20540/castcredits
|
||||
20570;http://api.tvmaze.com/people/20570/castcredits
|
||||
20580;http://api.tvmaze.com/people/20580/castcredits
|
||||
20647;http://api.tvmaze.com/people/20647/castcredits
|
||||
20668;http://api.tvmaze.com/people/20668/castcredits
|
||||
20671;http://api.tvmaze.com/people/20671/castcredits
|
||||
20797;http://api.tvmaze.com/people/20797/castcredits
|
||||
20811;http://api.tvmaze.com/people/20811/castcredits
|
||||
20812;http://api.tvmaze.com/people/20812/castcredits
|
||||
20813;http://api.tvmaze.com/people/20813/castcredits
|
||||
20814;http://api.tvmaze.com/people/20814/castcredits
|
||||
20822;http://api.tvmaze.com/people/20822/castcredits
|
||||
20830;http://api.tvmaze.com/people/20830/castcredits
|
||||
20862;http://api.tvmaze.com/people/20862/castcredits
|
||||
20886;http://api.tvmaze.com/people/20886/castcredits
|
||||
20896;http://api.tvmaze.com/people/20896/castcredits
|
||||
20905;http://api.tvmaze.com/people/20905/castcredits
|
||||
20907;http://api.tvmaze.com/people/20907/castcredits
|
||||
20929;http://api.tvmaze.com/people/20929/castcredits
|
||||
21001;http://api.tvmaze.com/people/21001/castcredits
|
||||
21003;http://api.tvmaze.com/people/21003/castcredits
|
||||
21035;http://api.tvmaze.com/people/21035/castcredits
|
||||
21057;http://api.tvmaze.com/people/21057/castcredits
|
||||
21062;http://api.tvmaze.com/people/21062/castcredits
|
||||
21075;http://api.tvmaze.com/people/21075/castcredits
|
||||
21077;http://api.tvmaze.com/people/21077/castcredits
|
||||
21078;http://api.tvmaze.com/people/21078/castcredits
|
||||
21084;http://api.tvmaze.com/people/21084/castcredits
|
||||
21085;http://api.tvmaze.com/people/21085/castcredits
|
||||
21128;http://api.tvmaze.com/people/21128/castcredits
|
||||
21165;http://api.tvmaze.com/people/21165/castcredits
|
||||
21170;http://api.tvmaze.com/people/21170/castcredits
|
||||
21171;http://api.tvmaze.com/people/21171/castcredits
|
||||
21176;http://api.tvmaze.com/people/21176/castcredits
|
||||
21177;http://api.tvmaze.com/people/21177/castcredits
|
||||
21181;http://api.tvmaze.com/people/21181/castcredits
|
||||
21185;http://api.tvmaze.com/people/21185/castcredits
|
||||
21186;http://api.tvmaze.com/people/21186/castcredits
|
||||
21189;http://api.tvmaze.com/people/21189/castcredits
|
||||
21190;http://api.tvmaze.com/people/21190/castcredits
|
||||
21191;http://api.tvmaze.com/people/21191/castcredits
|
||||
21192;http://api.tvmaze.com/people/21192/castcredits
|
||||
21193;http://api.tvmaze.com/people/21193/castcredits
|
||||
21196;http://api.tvmaze.com/people/21196/castcredits
|
||||
21198;http://api.tvmaze.com/people/21198/castcredits
|
||||
21199;http://api.tvmaze.com/people/21199/castcredits
|
||||
21203;http://api.tvmaze.com/people/21203/castcredits
|
||||
21204;http://api.tvmaze.com/people/21204/castcredits
|
||||
21205;http://api.tvmaze.com/people/21205/castcredits
|
||||
21206;http://api.tvmaze.com/people/21206/castcredits
|
||||
21208;http://api.tvmaze.com/people/21208/castcredits
|
||||
21209;http://api.tvmaze.com/people/21209/castcredits
|
||||
21210;http://api.tvmaze.com/people/21210/castcredits
|
||||
21212;http://api.tvmaze.com/people/21212/castcredits
|
||||
21213;http://api.tvmaze.com/people/21213/castcredits
|
||||
21214;http://api.tvmaze.com/people/21214/castcredits
|
||||
21218;http://api.tvmaze.com/people/21218/castcredits
|
||||
21219;http://api.tvmaze.com/people/21219/castcredits
|
||||
21220;http://api.tvmaze.com/people/21220/castcredits
|
||||
21222;http://api.tvmaze.com/people/21222/castcredits
|
||||
21223;http://api.tvmaze.com/people/21223/castcredits
|
||||
21224;http://api.tvmaze.com/people/21224/castcredits
|
||||
21225;http://api.tvmaze.com/people/21225/castcredits
|
||||
21226;http://api.tvmaze.com/people/21226/castcredits
|
||||
21227;http://api.tvmaze.com/people/21227/castcredits
|
||||
21228;http://api.tvmaze.com/people/21228/castcredits
|
||||
21232;http://api.tvmaze.com/people/21232/castcredits
|
||||
21233;http://api.tvmaze.com/people/21233/castcredits
|
||||
21234;http://api.tvmaze.com/people/21234/castcredits
|
||||
21235;http://api.tvmaze.com/people/21235/castcredits
|
||||
21236;http://api.tvmaze.com/people/21236/castcredits
|
||||
21237;http://api.tvmaze.com/people/21237/castcredits
|
||||
21238;http://api.tvmaze.com/people/21238/castcredits
|
||||
21239;http://api.tvmaze.com/people/21239/castcredits
|
||||
21242;http://api.tvmaze.com/people/21242/castcredits
|
||||
21243;http://api.tvmaze.com/people/21243/castcredits
|
||||
21244;http://api.tvmaze.com/people/21244/castcredits
|
||||
21245;http://api.tvmaze.com/people/21245/castcredits
|
||||
21246;http://api.tvmaze.com/people/21246/castcredits
|
||||
21249;http://api.tvmaze.com/people/21249/castcredits
|
||||
21250;http://api.tvmaze.com/people/21250/castcredits
|
||||
21251;http://api.tvmaze.com/people/21251/castcredits
|
||||
21253;http://api.tvmaze.com/people/21253/castcredits
|
||||
21254;http://api.tvmaze.com/people/21254/castcredits
|
||||
21255;http://api.tvmaze.com/people/21255/castcredits
|
||||
21256;http://api.tvmaze.com/people/21256/castcredits
|
||||
21262;http://api.tvmaze.com/people/21262/castcredits
|
||||
21263;http://api.tvmaze.com/people/21263/castcredits
|
||||
21264;http://api.tvmaze.com/people/21264/castcredits
|
||||
21265;http://api.tvmaze.com/people/21265/castcredits
|
||||
21266;http://api.tvmaze.com/people/21266/castcredits
|
||||
21267;http://api.tvmaze.com/people/21267/castcredits
|
||||
21270;http://api.tvmaze.com/people/21270/castcredits
|
||||
21272;http://api.tvmaze.com/people/21272/castcredits
|
||||
21273;http://api.tvmaze.com/people/21273/castcredits
|
||||
21275;http://api.tvmaze.com/people/21275/castcredits
|
||||
21276;http://api.tvmaze.com/people/21276/castcredits
|
||||
21277;http://api.tvmaze.com/people/21277/castcredits
|
||||
21278;http://api.tvmaze.com/people/21278/castcredits
|
||||
21279;http://api.tvmaze.com/people/21279/castcredits
|
||||
21280;http://api.tvmaze.com/people/21280/castcredits
|
||||
21281;http://api.tvmaze.com/people/21281/castcredits
|
||||
21282;http://api.tvmaze.com/people/21282/castcredits
|
||||
21283;http://api.tvmaze.com/people/21283/castcredits
|
||||
21284;http://api.tvmaze.com/people/21284/castcredits
|
||||
21285;http://api.tvmaze.com/people/21285/castcredits
|
||||
21287;http://api.tvmaze.com/people/21287/castcredits
|
||||
21288;http://api.tvmaze.com/people/21288/castcredits
|
||||
21289;http://api.tvmaze.com/people/21289/castcredits
|
||||
21290;http://api.tvmaze.com/people/21290/castcredits
|
||||
21291;http://api.tvmaze.com/people/21291/castcredits
|
||||
21292;http://api.tvmaze.com/people/21292/castcredits
|
||||
21293;http://api.tvmaze.com/people/21293/castcredits
|
||||
21294;http://api.tvmaze.com/people/21294/castcredits
|
||||
21295;http://api.tvmaze.com/people/21295/castcredits
|
||||
21296;http://api.tvmaze.com/people/21296/castcredits
|
||||
21297;http://api.tvmaze.com/people/21297/castcredits
|
||||
21298;http://api.tvmaze.com/people/21298/castcredits
|
||||
21299;http://api.tvmaze.com/people/21299/castcredits
|
||||
21300;http://api.tvmaze.com/people/21300/castcredits
|
||||
21301;http://api.tvmaze.com/people/21301/castcredits
|
||||
21303;http://api.tvmaze.com/people/21303/castcredits
|
||||
21304;http://api.tvmaze.com/people/21304/castcredits
|
||||
21305;http://api.tvmaze.com/people/21305/castcredits
|
||||
21307;http://api.tvmaze.com/people/21307/castcredits
|
||||
21308;http://api.tvmaze.com/people/21308/castcredits
|
||||
21313;http://api.tvmaze.com/people/21313/castcredits
|
||||
21314;http://api.tvmaze.com/people/21314/castcredits
|
||||
21317;http://api.tvmaze.com/people/21317/castcredits
|
||||
21319;http://api.tvmaze.com/people/21319/castcredits
|
||||
21320;http://api.tvmaze.com/people/21320/castcredits
|
||||
21321;http://api.tvmaze.com/people/21321/castcredits
|
||||
21322;http://api.tvmaze.com/people/21322/castcredits
|
||||
21323;http://api.tvmaze.com/people/21323/castcredits
|
||||
21327;http://api.tvmaze.com/people/21327/castcredits
|
||||
21328;http://api.tvmaze.com/people/21328/castcredits
|
||||
21329;http://api.tvmaze.com/people/21329/castcredits
|
||||
21330;http://api.tvmaze.com/people/21330/castcredits
|
||||
21331;http://api.tvmaze.com/people/21331/castcredits
|
||||
21332;http://api.tvmaze.com/people/21332/castcredits
|
||||
21335;http://api.tvmaze.com/people/21335/castcredits
|
||||
21338;http://api.tvmaze.com/people/21338/castcredits
|
||||
21339;http://api.tvmaze.com/people/21339/castcredits
|
||||
21342;http://api.tvmaze.com/people/21342/castcredits
|
||||
21343;http://api.tvmaze.com/people/21343/castcredits
|
||||
21344;http://api.tvmaze.com/people/21344/castcredits
|
||||
21345;http://api.tvmaze.com/people/21345/castcredits
|
||||
21410;http://api.tvmaze.com/people/21410/castcredits
|
||||
21416;http://api.tvmaze.com/people/21416/castcredits
|
||||
21420;http://api.tvmaze.com/people/21420/castcredits
|
||||
21435;http://api.tvmaze.com/people/21435/castcredits
|
||||
21438;http://api.tvmaze.com/people/21438/castcredits
|
||||
21447;http://api.tvmaze.com/people/21447/castcredits
|
||||
21449;http://api.tvmaze.com/people/21449/castcredits
|
||||
21458;http://api.tvmaze.com/people/21458/castcredits
|
||||
21459;http://api.tvmaze.com/people/21459/castcredits
|
||||
21465;http://api.tvmaze.com/people/21465/castcredits
|
||||
21466;http://api.tvmaze.com/people/21466/castcredits
|
||||
21467;http://api.tvmaze.com/people/21467/castcredits
|
||||
21468;http://api.tvmaze.com/people/21468/castcredits
|
||||
21481;http://api.tvmaze.com/people/21481/castcredits
|
||||
21488;http://api.tvmaze.com/people/21488/castcredits
|
||||
21501;http://api.tvmaze.com/people/21501/castcredits
|
||||
21502;http://api.tvmaze.com/people/21502/castcredits
|
||||
21509;http://api.tvmaze.com/people/21509/castcredits
|
||||
21510;http://api.tvmaze.com/people/21510/castcredits
|
||||
21519;http://api.tvmaze.com/people/21519/castcredits
|
||||
21520;http://api.tvmaze.com/people/21520/castcredits
|
||||
21521;http://api.tvmaze.com/people/21521/castcredits
|
||||
21522;http://api.tvmaze.com/people/21522/castcredits
|
||||
21523;http://api.tvmaze.com/people/21523/castcredits
|
||||
21534;http://api.tvmaze.com/people/21534/castcredits
|
||||
21535;http://api.tvmaze.com/people/21535/castcredits
|
||||
21539;http://api.tvmaze.com/people/21539/castcredits
|
||||
21545;http://api.tvmaze.com/people/21545/castcredits
|
||||
21549;http://api.tvmaze.com/people/21549/castcredits
|
||||
21572;http://api.tvmaze.com/people/21572/castcredits
|
||||
21581;http://api.tvmaze.com/people/21581/castcredits
|
||||
21584;http://api.tvmaze.com/people/21584/castcredits
|
||||
21585;http://api.tvmaze.com/people/21585/castcredits
|
||||
21592;http://api.tvmaze.com/people/21592/castcredits
|
||||
21599;http://api.tvmaze.com/people/21599/castcredits
|
||||
21609;http://api.tvmaze.com/people/21609/castcredits
|
||||
21614;http://api.tvmaze.com/people/21614/castcredits
|
||||
21622;http://api.tvmaze.com/people/21622/castcredits
|
||||
21623;http://api.tvmaze.com/people/21623/castcredits
|
||||
21627;http://api.tvmaze.com/people/21627/castcredits
|
||||
21628;http://api.tvmaze.com/people/21628/castcredits
|
||||
21642;http://api.tvmaze.com/people/21642/castcredits
|
||||
21643;http://api.tvmaze.com/people/21643/castcredits
|
||||
21644;http://api.tvmaze.com/people/21644/castcredits
|
||||
21645;http://api.tvmaze.com/people/21645/castcredits
|
||||
21648;http://api.tvmaze.com/people/21648/castcredits
|
||||
21649;http://api.tvmaze.com/people/21649/castcredits
|
||||
21652;http://api.tvmaze.com/people/21652/castcredits
|
||||
21653;http://api.tvmaze.com/people/21653/castcredits
|
||||
21666;http://api.tvmaze.com/people/21666/castcredits
|
||||
21667;http://api.tvmaze.com/people/21667/castcredits
|
||||
21671;http://api.tvmaze.com/people/21671/castcredits
|
||||
21674;http://api.tvmaze.com/people/21674/castcredits
|
||||
21695;http://api.tvmaze.com/people/21695/castcredits
|
||||
21709;http://api.tvmaze.com/people/21709/castcredits
|
||||
21710;http://api.tvmaze.com/people/21710/castcredits
|
||||
21719;http://api.tvmaze.com/people/21719/castcredits
|
||||
21720;http://api.tvmaze.com/people/21720/castcredits
|
||||
21721;http://api.tvmaze.com/people/21721/castcredits
|
||||
21739;http://api.tvmaze.com/people/21739/castcredits
|
||||
21744;http://api.tvmaze.com/people/21744/castcredits
|
||||
21745;http://api.tvmaze.com/people/21745/castcredits
|
||||
21752;http://api.tvmaze.com/people/21752/castcredits
|
||||
21755;http://api.tvmaze.com/people/21755/castcredits
|
||||
21756;http://api.tvmaze.com/people/21756/castcredits
|
||||
21770;http://api.tvmaze.com/people/21770/castcredits
|
||||
21776;http://api.tvmaze.com/people/21776/castcredits
|
||||
21783;http://api.tvmaze.com/people/21783/castcredits
|
||||
21789;http://api.tvmaze.com/people/21789/castcredits
|
||||
21790;http://api.tvmaze.com/people/21790/castcredits
|
||||
21799;http://api.tvmaze.com/people/21799/castcredits
|
||||
21806;http://api.tvmaze.com/people/21806/castcredits
|
||||
21807;http://api.tvmaze.com/people/21807/castcredits
|
||||
21821;http://api.tvmaze.com/people/21821/castcredits
|
||||
21822;http://api.tvmaze.com/people/21822/castcredits
|
||||
21836;http://api.tvmaze.com/people/21836/castcredits
|
||||
21840;http://api.tvmaze.com/people/21840/castcredits
|
||||
21841;http://api.tvmaze.com/people/21841/castcredits
|
||||
21842;http://api.tvmaze.com/people/21842/castcredits
|
||||
21844;http://api.tvmaze.com/people/21844/castcredits
|
||||
21845;http://api.tvmaze.com/people/21845/castcredits
|
||||
21854;http://api.tvmaze.com/people/21854/castcredits
|
||||
21855;http://api.tvmaze.com/people/21855/castcredits
|
||||
21872;http://api.tvmaze.com/people/21872/castcredits
|
||||
21873;http://api.tvmaze.com/people/21873/castcredits
|
||||
21874;http://api.tvmaze.com/people/21874/castcredits
|
||||
21883;http://api.tvmaze.com/people/21883/castcredits
|
||||
21884;http://api.tvmaze.com/people/21884/castcredits
|
||||
21885;http://api.tvmaze.com/people/21885/castcredits
|
||||
21886;http://api.tvmaze.com/people/21886/castcredits
|
||||
21901;http://api.tvmaze.com/people/21901/castcredits
|
||||
21902;http://api.tvmaze.com/people/21902/castcredits
|
||||
21905;http://api.tvmaze.com/people/21905/castcredits
|
||||
21906;http://api.tvmaze.com/people/21906/castcredits
|
||||
21911;http://api.tvmaze.com/people/21911/castcredits
|
||||
21919;http://api.tvmaze.com/people/21919/castcredits
|
||||
21920;http://api.tvmaze.com/people/21920/castcredits
|
||||
21925;http://api.tvmaze.com/people/21925/castcredits
|
||||
21928;http://api.tvmaze.com/people/21928/castcredits
|
||||
21930;http://api.tvmaze.com/people/21930/castcredits
|
||||
21936;http://api.tvmaze.com/people/21936/castcredits
|
||||
21939;http://api.tvmaze.com/people/21939/castcredits
|
||||
21942;http://api.tvmaze.com/people/21942/castcredits
|
||||
21943;http://api.tvmaze.com/people/21943/castcredits
|
||||
22007;http://api.tvmaze.com/people/22007/castcredits
|
||||
22059;http://api.tvmaze.com/people/22059/castcredits
|
||||
22063;http://api.tvmaze.com/people/22063/castcredits
|
||||
22064;http://api.tvmaze.com/people/22064/castcredits
|
||||
22069;http://api.tvmaze.com/people/22069/castcredits
|
||||
22076;http://api.tvmaze.com/people/22076/castcredits
|
||||
22078;http://api.tvmaze.com/people/22078/castcredits
|
||||
22086;http://api.tvmaze.com/people/22086/castcredits
|
||||
22095;http://api.tvmaze.com/people/22095/castcredits
|
||||
22099;http://api.tvmaze.com/people/22099/castcredits
|
||||
22104;http://api.tvmaze.com/people/22104/castcredits
|
||||
22146;http://api.tvmaze.com/people/22146/castcredits
|
||||
22147;http://api.tvmaze.com/people/22147/castcredits
|
||||
22172;http://api.tvmaze.com/people/22172/castcredits
|
||||
22176;http://api.tvmaze.com/people/22176/castcredits
|
||||
22193;http://api.tvmaze.com/people/22193/castcredits
|
||||
22198;http://api.tvmaze.com/people/22198/castcredits
|
||||
22199;http://api.tvmaze.com/people/22199/castcredits
|
||||
22204;http://api.tvmaze.com/people/22204/castcredits
|
||||
22211;http://api.tvmaze.com/people/22211/castcredits
|
||||
22214;http://api.tvmaze.com/people/22214/castcredits
|
||||
22215;http://api.tvmaze.com/people/22215/castcredits
|
||||
22222;http://api.tvmaze.com/people/22222/castcredits
|
||||
22223;http://api.tvmaze.com/people/22223/castcredits
|
||||
22243;http://api.tvmaze.com/people/22243/castcredits
|
||||
22245;http://api.tvmaze.com/people/22245/castcredits
|
||||
22250;http://api.tvmaze.com/people/22250/castcredits
|
||||
22321;http://api.tvmaze.com/people/22321/castcredits
|
||||
22395;http://api.tvmaze.com/people/22395/castcredits
|
||||
22458;http://api.tvmaze.com/people/22458/castcredits
|
||||
22522;http://api.tvmaze.com/people/22522/castcredits
|
||||
22536;http://api.tvmaze.com/people/22536/castcredits
|
||||
22697;http://api.tvmaze.com/people/22697/castcredits
|
||||
22792;http://api.tvmaze.com/people/22792/castcredits
|
||||
22798;http://api.tvmaze.com/people/22798/castcredits
|
||||
22803;http://api.tvmaze.com/people/22803/castcredits
|
||||
22899;http://api.tvmaze.com/people/22899/castcredits
|
||||
23153;http://api.tvmaze.com/people/23153/castcredits
|
||||
23177;http://api.tvmaze.com/people/23177/castcredits
|
||||
23198;http://api.tvmaze.com/people/23198/castcredits
|
||||
23204;http://api.tvmaze.com/people/23204/castcredits
|
||||
23262;http://api.tvmaze.com/people/23262/castcredits
|
||||
File diff suppressed because it is too large
Load Diff
Executable
+370
@@ -0,0 +1,370 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
import time
|
||||
import sys
|
||||
import url.urls
|
||||
import url.process
|
||||
from tqdm import tqdm
|
||||
import db.sql
|
||||
import logs.Logger
|
||||
from db.functions import settype, dbmongo
|
||||
from datetime import datetime
|
||||
import json
|
||||
import json_downloader
|
||||
|
||||
response = None
|
||||
|
||||
|
||||
def set_apienv(urls, uprocess, dbengine, dbExec, updatesBase, lprint):
|
||||
"""Populate updates table from the API and return available updates.
|
||||
|
||||
Args:
|
||||
urls: URL configuration with an "updatesurl" key.
|
||||
uprocess: URL loader instance with `get_data`.
|
||||
jget: JSON processor with `jconvert`.
|
||||
dbengine: database engine dict (expects key "engine").
|
||||
dbExec: database execution helper with `update_tvupdates` and `rawsql_select`.
|
||||
updatesBase: list-like containing tables (uses index 8 for updates table).
|
||||
lprint: logger instance with `logprint`.
|
||||
|
||||
Returns:
|
||||
List of tuples (seriesid, timestamp) representing available updates.
|
||||
"""
|
||||
#updatetable = updatesBase[8]
|
||||
#inputdata = uprocess.get_data(urls["updatesurl"])
|
||||
#lprint.logprint("info", f"Retrieved {len(availableupdates)} rows for processing......")
|
||||
#
|
||||
newupdates = dbExec.rawsql_select(
|
||||
dbengine["engine"],
|
||||
"select seriesid,timestamp from updates.tvupdates",
|
||||
lprint
|
||||
)
|
||||
return newupdates
|
||||
|
||||
|
||||
def print_starttime(lprint, ts1):
|
||||
lprint.logprint("info", f"Sync start time: {str(ts1)}")
|
||||
|
||||
|
||||
def get_tvdb(tvdburls):
|
||||
return tvdburls.get_data()
|
||||
|
||||
|
||||
def get_tvmaze(tvmazeurls):
|
||||
return tvmazeurls.get_data()
|
||||
|
||||
def find_file_in_multiple_dirs(filename, directories):
|
||||
"""
|
||||
Checks if a file exists in any of the provided directories.
|
||||
|
||||
Args:
|
||||
filename (str): The name of the file to search for.
|
||||
directories (list): A list of directory paths to check.
|
||||
|
||||
Returns:
|
||||
Path or None: The full path to the file if found, otherwise None.
|
||||
"""
|
||||
directories_found = []
|
||||
for directory in directories:
|
||||
if os.path.isfile(f'{directory}{filename}'):
|
||||
directories_found.append(directory)
|
||||
|
||||
return directories_found
|
||||
|
||||
def main() -> int:
|
||||
ROOTDIR = os.getcwd()
|
||||
directories = {}
|
||||
# Use cross-platform path builders and ensure directories exist
|
||||
LOGDIR = os.path.join(ROOTDIR, "log")
|
||||
directories["SERIESDIR"] = os.path.join(ROOTDIR, "cache", "series") + os.sep
|
||||
directories["EPISODEDIR"] = os.path.join(ROOTDIR, "cache", "episodes") + os.sep
|
||||
directories["CASTDIR"] = os.path.join(ROOTDIR, "cache", "cast") + os.sep
|
||||
directories["GUESTCASTDIR"] = os.path.join(ROOTDIR, "cache", "guestcast") + os.sep
|
||||
directories["CREWDIR"] = os.path.join(ROOTDIR, "cache", "crew") + os.sep
|
||||
#directories["GUESTCREWDIR"] = os.path.join(ROOTDIR, "cache", "guestcrew") + os.sep
|
||||
#directories["CREDITSDIR"] = os.path.join(ROOTDIR, "cache", "credits") + os.sep
|
||||
#directories["ALIASDIR"] = os.path.join(ROOTDIR, "cache", "aliases") + os.sep
|
||||
tempdir = os.path.join(ROOTDIR, 'temp') + os.sep
|
||||
updatetable = 'updates.tvupdates'
|
||||
skip_file = f"{tempdir}skip_ids.txt"
|
||||
|
||||
# Create directories if missing so later code doesn't fail
|
||||
try:
|
||||
os.makedirs(LOGDIR, exist_ok=True)
|
||||
os.makedirs(os.path.join(ROOTDIR, "cache", "series"), exist_ok=True)
|
||||
os.makedirs(os.path.join(ROOTDIR, "cache", "episodes"), exist_ok=True)
|
||||
os.makedirs(os.path.join(ROOTDIR, "cache", "cast"), exist_ok=True)
|
||||
os.makedirs(os.path.join(ROOTDIR, "cache", "guestcast"), exist_ok=True)
|
||||
os.makedirs(os.path.join(ROOTDIR, "cache", "crew"), exist_ok=True)
|
||||
os.makedirs(os.path.join(ROOTDIR, "cache", "guestcrew"), exist_ok=True)
|
||||
except Exception:
|
||||
# If directory creation fails, let the logging initialization surface the error
|
||||
pass
|
||||
|
||||
lprint = logs.Logger.LPrint(LOGDIR, False)
|
||||
lprint.start_log()
|
||||
import settings.config
|
||||
|
||||
config_options = settings.config.Config(ROOTDIR)
|
||||
options = config_options.config_options
|
||||
mongo_updater = dbmongo(options) # Create an instance of dbmongo
|
||||
dbtype, apitype = options["dbtype"], options["apitype"]
|
||||
dbs = settype(dbtype, apitype, options)
|
||||
dbclass, dbengine, apiengine = dbs.dbclass, dbs.dbengine, dbs.apiengine
|
||||
uprocess = url.process.Loadurl()
|
||||
TVMAZEURLS = url.urls.Tvmazeurls()
|
||||
TVMAZE_URLS = get_tvmaze(TVMAZEURLS)
|
||||
TVDBURLS = url.urls.TVDBurls()
|
||||
TVDB_URLS = get_tvdb(TVDBURLS)
|
||||
updateRange = None
|
||||
|
||||
lprint.logprint("info", "Creating JSON instance for data retrieval.")
|
||||
lprint.logprint("info", "JSON instance created.")
|
||||
lprint.logprint("info", f"Update type is {options['updatetype']}")
|
||||
timedifference = options["updatetype"]
|
||||
currenttimestamp = time.time()
|
||||
if timedifference.lower() == "full":
|
||||
timedifference = int(currenttimestamp)
|
||||
basetime = str(currenttimestamp).split(".")
|
||||
currenttimestamp = int(basetime[0])
|
||||
ts1 = str(datetime.now()).split(".")[0]
|
||||
lprint.logprint("info", "Sync start time: " + str(ts1))
|
||||
print_starttime(lprint, ts1)
|
||||
dbExec = db.sql.Dbexec()
|
||||
createTempTables = dbclass["CreateTemptables"]
|
||||
new_updates = {}
|
||||
|
||||
ct = createTempTables(
|
||||
dbengine["session"],
|
||||
dbengine["metadata"],
|
||||
dbengine["engine"]
|
||||
)
|
||||
tempTableList = ct.tablelist
|
||||
|
||||
new_updates = []
|
||||
if options["apitype"].lower() == "tvmaze":
|
||||
try:
|
||||
new_updates = set_apienv(
|
||||
TVMAZE_URLS,
|
||||
uprocess,
|
||||
dbengine,
|
||||
dbExec,
|
||||
tempTableList,
|
||||
lprint,
|
||||
)
|
||||
except Exception as exc: # noqa: PLC0301 - broad handler to log failures
|
||||
lprint.logprint("error", f"Failed to set API env: {exc}")
|
||||
return 1
|
||||
|
||||
timeDifference = options["updatetype"]
|
||||
currentTimestamp = time.time()
|
||||
if timeDifference.lower() == "full" or options["initload"] == "1":
|
||||
tdifference = 0
|
||||
else:
|
||||
tdifference = int(currentTimestamp) - int(
|
||||
options[options["updatetype"]]
|
||||
)
|
||||
needed_updates = []
|
||||
updateList = []
|
||||
for result in new_updates:
|
||||
updateid = result[0]
|
||||
updatestamp = int(result[1])
|
||||
tdif = int(tdifference)
|
||||
if updatestamp > tdif:
|
||||
needed_updates.append(updateid)
|
||||
updateList.append(updateid)
|
||||
|
||||
lprint.logprint("info", "Update Type: " + str(options["updatetype"]))
|
||||
updateRange = len(updateList)
|
||||
lprint.logprint("info", "Number of updates: " + str(updateRange))
|
||||
|
||||
|
||||
# Save a downloaded copy of the updates JSON for debugging/inspection
|
||||
try:
|
||||
os.makedirs(tempdir, exist_ok=True)
|
||||
out_file = 'tvmaze_updates.json'
|
||||
prev_file = f'post_tvmaze_updates.json.{currentTimestamp}'
|
||||
previous_updates_file = f"{tempdir}{prev_file}"
|
||||
# TVMAZE_URLS is a dict of urls; use the updatesurl if available
|
||||
updates_url = TVMAZE_URLS.get('updatesurl') if isinstance(TVMAZE_URLS, dict) else None
|
||||
if updates_url:
|
||||
json_downloader.download_json_to_file(updates_url, tempdir, out_file, lprint)
|
||||
# load and convert saved JSON into a dict (id -> timestamp)
|
||||
try:
|
||||
downloaded_updates_dict = {}
|
||||
downloaded_updates_list = []
|
||||
if os.path.exists(f"{tempdir}{out_file}"):
|
||||
with open(f"{tempdir}{out_file}", 'r') as jf:
|
||||
loaded = json.load(jf)
|
||||
|
||||
if isinstance(loaded, dict):
|
||||
for k, v in loaded.items():
|
||||
try:
|
||||
kid = int(k)
|
||||
except Exception:
|
||||
kid = k
|
||||
try:
|
||||
vt = int(v)
|
||||
except Exception:
|
||||
vt = v
|
||||
downloaded_updates_dict[kid] = vt
|
||||
downloaded_updates_list = list(downloaded_updates_dict.items())
|
||||
|
||||
elif isinstance(loaded, list):
|
||||
for item in loaded:
|
||||
if isinstance(item, list) and len(item) >= 2:
|
||||
k = item[0]; v = item[1]
|
||||
elif isinstance(item, dict):
|
||||
if 'id' in item and 'timestamp' in item:
|
||||
k = item['id']; v = item['timestamp']
|
||||
elif 'seriesid' in item and 'timestamp' in item:
|
||||
k = item['seriesid']; v = item['timestamp']
|
||||
else:
|
||||
downloaded_updates_list.append(item)
|
||||
continue
|
||||
else:
|
||||
k = item; v = None
|
||||
try:
|
||||
kid = int(k)
|
||||
except Exception:
|
||||
kid = k
|
||||
try:
|
||||
vt = int(v) if v is not None and str(v).isdigit() else v
|
||||
except Exception:
|
||||
vt = v
|
||||
downloaded_updates_dict[kid] = vt
|
||||
downloaded_updates_list.append((kid, vt))
|
||||
|
||||
else:
|
||||
downloaded_updates_list = [loaded]
|
||||
lprint.logprint("info", f"Loaded {len(downloaded_updates_list)} entries from tvmaze_updates.json")
|
||||
else:
|
||||
downloaded_updates_dict = {}
|
||||
downloaded_updates_list = []
|
||||
except Exception:
|
||||
downloaded_updates_dict = {}
|
||||
downloaded_updates_list = []
|
||||
except Exception:
|
||||
# non-fatal; continue
|
||||
lprint.logprint("warning", "Failed to save or load downloaded updates JSON.")
|
||||
previouslisting = dbExec.rawsql_select(
|
||||
dbengine["engine"],
|
||||
"select seriesid, timestamp from updates.tvupdates order by seriesid",
|
||||
lprint
|
||||
)
|
||||
print(f"Saving previous updates to file.")
|
||||
with open(f"{previous_updates_file}", 'w') as pjf:
|
||||
for pl in previouslisting:
|
||||
pjf.write(f"{pl[0]}: {pl[1]}\n")
|
||||
pjf.close()
|
||||
# convert previouslisting (list of tuples) into a dict: row_id -> country_name
|
||||
try:
|
||||
_previouslisting_list = previouslisting
|
||||
previous_listing_dict = {row[0]: row[1] for row in _previouslisting_list}
|
||||
except Exception:
|
||||
# leave previouslisting as-is on any failure
|
||||
pass
|
||||
# Ensure dictionaries exist
|
||||
try:
|
||||
downloaded_updates_dict
|
||||
except NameError:
|
||||
downloaded_updates_dict = {}
|
||||
try:
|
||||
previous_listing_dict
|
||||
except NameError:
|
||||
previous_listing_dict = {}
|
||||
files_needed = []
|
||||
directory_list = []
|
||||
for value in directories.values():
|
||||
directory_list.append(value)
|
||||
skip_ids = {}
|
||||
skipfile = open(skip_file, 'r')
|
||||
for line in skipfile:
|
||||
field = line.split(';')
|
||||
skip_ids[field[0]] = field[1]
|
||||
print(f"Updating update table.")
|
||||
dbExec.update_tvupdates(
|
||||
dbengine["engine"],
|
||||
downloaded_updates_dict,
|
||||
"updates.tvupdates"
|
||||
)
|
||||
print("update complete.")
|
||||
print(f"Processing {len(downloaded_updates_dict)} downloaded updates against {len(previous_listing_dict)} previous updates for files needed.")
|
||||
for seriesid, ts in downloaded_updates_dict.items():
|
||||
if seriesid in previous_listing_dict:
|
||||
prev_ts = previous_listing_dict[seriesid]
|
||||
# Check to see if all json files are present for seriesid
|
||||
try:
|
||||
if int(ts) == int(prev_ts):
|
||||
found_path = find_file_in_multiple_dirs(f"{seriesid}.json", directory_list)
|
||||
if len(found_path) == 5:
|
||||
continue
|
||||
else:
|
||||
for directory in directory_list:
|
||||
if directory not in found_path:
|
||||
files_needed.append([f"{directory}", f"{seriesid}.json"])
|
||||
elif int(ts) > int(prev_ts):
|
||||
for directory in directory_list:
|
||||
files_needed.append([f"{directory}", f"{seriesid}.json"])
|
||||
except Exception:
|
||||
lprint.logprint("debug", f"Unable to process Series {seriesid} timestamps: downloaded {ts} vs previous {prev_ts}")
|
||||
else:
|
||||
# Add needed json files to needed file list.
|
||||
for directory in directory_list:
|
||||
files_needed.append([f"{directory}", f"{seriesid}.json"])
|
||||
lprint.logprint("debug", f"Series {seriesid} present in downloaded updates but missing in previous listing")
|
||||
|
||||
lprint.logprint("info", f"Downloadinng JSON for {len(files_needed)} series IDs.")
|
||||
print(f"Downloading JSON for {len(files_needed)} files.")
|
||||
for i in tqdm(range(len(files_needed)), desc="Downloading JSON files", unit="filename"):
|
||||
for directory, file in files_needed[i:i+1]:
|
||||
if 'credits' in directory:
|
||||
if file.replace('.json', '') in skip_ids and 'castcredits' in skip_ids[file.replace('.json', '')].strip():
|
||||
lprint.logprint("info", f"Skipping credits download for series ID {file.replace('.json', '')} as per skip_ids.txt")
|
||||
continue
|
||||
else:
|
||||
url_template = TVMAZE_URLS["creditsurl"]
|
||||
elif 'crew' in directory:
|
||||
url_template = TVMAZE_URLS["crewurl"]
|
||||
elif 'cast' in directory:
|
||||
url_template = TVMAZE_URLS["casturl"]
|
||||
elif 'episodes' in directory:
|
||||
url_template = TVMAZE_URLS["episodesurl"]
|
||||
elif 'aliases' in directory:
|
||||
url_template = TVMAZE_URLS["aliasurl"]
|
||||
else:
|
||||
url_template = TVMAZE_URLS["showurl"]
|
||||
seriesid = file.replace('.json', '')
|
||||
json_downloader.download_json_to_file(url_template.replace("<seriesid>", str(seriesid)), directory, f'{seriesid}.json', lprint)
|
||||
# dbExec.update_tvupdates(dbengine["engine"], downloaded_updates_dict, updatetable)
|
||||
|
||||
|
||||
|
||||
# countrylisting = dbExec.rawsql_select(
|
||||
# dbengine["engine"],
|
||||
# "select row_id, country_name from dbo.countrydata",
|
||||
# lprint
|
||||
# )
|
||||
# for country in countrylisting:
|
||||
# country_name = country[1]
|
||||
# country_id = country[0]
|
||||
# countrydictionary[country_name] = country_id
|
||||
# mongo_updater = dbmongo(options)
|
||||
# mongo_updater.update_mongo(
|
||||
# updateList,
|
||||
# ROOTDIR,
|
||||
# directories,
|
||||
# jget,
|
||||
# countrydictionary,
|
||||
# dbExec,
|
||||
# dbengine,
|
||||
# dboBase,
|
||||
# lprint,
|
||||
# )
|
||||
# return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
||||
-207
@@ -1,207 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
import time
|
||||
import sys
|
||||
import url.urls
|
||||
import url.process
|
||||
import api.tvmaze.cast as mazecast
|
||||
import api.tvmaze.crew as mazecrew
|
||||
import api.tvmaze.episodes as mazeepisodes
|
||||
import api.tvmaze.series as mazeseries
|
||||
import db.sql
|
||||
import jprocessor.process
|
||||
import logs.Logger
|
||||
from db.functions import settype, dbmongo
|
||||
from datetime import datetime
|
||||
|
||||
response = None
|
||||
|
||||
|
||||
def set_apienv(urls, uprocess, jget, dbengine, dbExec, updatesBase, lprint):
|
||||
"""Populate updates table from the API and return available updates.
|
||||
|
||||
Args:
|
||||
urls: URL configuration with an "updatesurl" key.
|
||||
uprocess: URL loader instance with `get_data`.
|
||||
jget: JSON processor with `jconvert`.
|
||||
dbengine: database engine dict (expects key "engine").
|
||||
dbExec: database execution helper with `update_tvupdates` and `rawsql_select`.
|
||||
updatesBase: list-like containing tables (uses index 8 for updates table).
|
||||
lprint: logger instance with `logprint`.
|
||||
|
||||
Returns:
|
||||
List of tuples (seriesid, timestamp) representing available updates.
|
||||
"""
|
||||
updatetable = updatesBase[8]
|
||||
inputdata = uprocess.get_data(urls["updatesurl"])
|
||||
availableupdates = jget.jconvert(inputdata)
|
||||
lprint.logprint("info", f"Retrieved {len(availableupdates)} rows for processing......")
|
||||
dbExec.update_tvupdates(dbengine["engine"], availableupdates, updatetable)
|
||||
newupdates = dbExec.rawsql_select(
|
||||
dbengine["engine"],
|
||||
"select seriesid,timestamp from updates.tvupdates",
|
||||
lprint
|
||||
)
|
||||
return newupdates
|
||||
|
||||
|
||||
def print_starttime(lprint, ts1):
|
||||
lprint.logprint("info", f"Sync start time: {str(ts1)}")
|
||||
|
||||
|
||||
def get_tvdb(tvdburls):
|
||||
return tvdburls.get_data()
|
||||
|
||||
|
||||
def get_tvmaze(tvmazeurls):
|
||||
return tvmazeurls.get_data()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ROOTDIR = os.getcwd()
|
||||
directories = {}
|
||||
# Use cross-platform path builders and ensure directories exist
|
||||
LOGDIR = os.path.join(ROOTDIR, "log")
|
||||
directories["SERIESDIR"] = os.path.join(ROOTDIR, "cache", "series") + os.sep
|
||||
directories["EPISODEDIR"] = os.path.join(ROOTDIR, "cache", "episodes") + os.sep
|
||||
directories["CASTDIR"] = os.path.join(ROOTDIR, "cache", "cast") + os.sep
|
||||
directories["CREWDIR"] = os.path.join(ROOTDIR, "cache", "crew") + os.sep
|
||||
|
||||
# Create directories if missing so later code doesn't fail
|
||||
try:
|
||||
os.makedirs(LOGDIR, exist_ok=True)
|
||||
os.makedirs(os.path.join(ROOTDIR, "cache", "series"), exist_ok=True)
|
||||
os.makedirs(os.path.join(ROOTDIR, "cache", "episodes"), exist_ok=True)
|
||||
os.makedirs(os.path.join(ROOTDIR, "cache", "cast"), exist_ok=True)
|
||||
os.makedirs(os.path.join(ROOTDIR, "cache", "crew"), exist_ok=True)
|
||||
except Exception:
|
||||
# If directory creation fails, let the logging initialization surface the error
|
||||
pass
|
||||
|
||||
lprint = logs.Logger.LPrint(LOGDIR, False)
|
||||
lprint.start_log()
|
||||
import settings.config
|
||||
|
||||
config_options = settings.config.Config(ROOTDIR)
|
||||
options = config_options.config_options
|
||||
dbtype, apitype = options["dbtype"], options["apitype"]
|
||||
dbs = settype(dbtype, apitype, options)
|
||||
dbclass, dbengine, apiengine = dbs.dbclass, dbs.dbengine, dbs.apiengine
|
||||
uprocess = url.process.Loadurl()
|
||||
TVMAZEURLS = url.urls.Tvmazeurls()
|
||||
TVMAZE_URLS = get_tvmaze(TVMAZEURLS)
|
||||
TVDBURLS = url.urls.TVDBurls()
|
||||
TVDB_URLS = get_tvdb(TVDBURLS)
|
||||
#mseries = mazeseries.tvshow()
|
||||
#mcast = mazecast.cast()
|
||||
#mepisode = mazeepisodes.episodes()
|
||||
#mcrew = mazecrew.crew()
|
||||
countrydictionary = {}
|
||||
updateRange = None
|
||||
|
||||
lprint.logprint("info", "Creating JSON instance for data retrieval.")
|
||||
jget = jprocessor.process.Jsonload()
|
||||
lprint.logprint("info", "JSON instance created.")
|
||||
lprint.logprint("info", f"Update type is {options['updatetype']}")
|
||||
timedifference = options["updatetype"]
|
||||
currenttimestamp = time.time()
|
||||
if timedifference.lower() == "full":
|
||||
timedifference = int(currenttimestamp)
|
||||
basetime = str(currenttimestamp).split(".")
|
||||
currenttimestamp = int(basetime[0])
|
||||
ts1 = str(datetime.now()).split(".")[0]
|
||||
lprint.logprint("info", "Sync start time: " + str(ts1))
|
||||
print_starttime(lprint, ts1)
|
||||
dboBase = dbengine["dboBase"]
|
||||
dbExec = db.sql.Dbexec()
|
||||
createTables = dbclass["Createtables"]
|
||||
createTempTables = dbclass["CreateTemptables"]
|
||||
new_updates = {}
|
||||
|
||||
ct = createTempTables(
|
||||
dbengine["session"],
|
||||
dbengine["metadata"],
|
||||
dbengine["engine"]
|
||||
)
|
||||
tempTableList = ct.tablelist
|
||||
|
||||
if options["apitype"].lower() == "tvmaze":
|
||||
try:
|
||||
new_updates = set_apienv(
|
||||
TVMAZE_URLS,
|
||||
uprocess,
|
||||
jget,
|
||||
dbengine,
|
||||
dbExec,
|
||||
tempTableList,
|
||||
lprint,
|
||||
)
|
||||
except Exception as exc: # noqa: PLC0301 - broad handler to log failures
|
||||
lprint.logprint("error", f"Failed to set API env: {exc}")
|
||||
return 1
|
||||
|
||||
from cache.process import Cacher
|
||||
|
||||
cacher = Cacher()
|
||||
try:
|
||||
cacher.process(options, jget, new_updates, ROOTDIR, TVMAZE_URLS)
|
||||
except Exception as exc:
|
||||
lprint.logprint("error", f"Cache processing failed: {exc}")
|
||||
return 1
|
||||
|
||||
timeDifference = options["updatetype"]
|
||||
currentTimestamp = time.time()
|
||||
if timeDifference.lower() == "full" or options["initload"] == "1":
|
||||
tdifference = 0
|
||||
else:
|
||||
tdifference = int(currentTimestamp) - int(
|
||||
options[options["updatetype"]]
|
||||
)
|
||||
needed_updates = []
|
||||
updateList = []
|
||||
for result in new_updates:
|
||||
updateid = result[0]
|
||||
updatestamp = int(result[1])
|
||||
tdif = int(tdifference)
|
||||
if updatestamp > tdif:
|
||||
needed_updates.append(updateid)
|
||||
updateList.append(updateid)
|
||||
|
||||
lprint.logprint("info", "Update Type: " + str(options["updatetype"]))
|
||||
updateRange = len(updateList)
|
||||
lprint.logprint("info", "Number of updates: " + str(updateRange))
|
||||
try:
|
||||
cacher.update_cache(
|
||||
jget, updateList, ROOTDIR, TVMAZE_URLS, options, lprint
|
||||
)
|
||||
except Exception as exc:
|
||||
lprint.logprint("error", f"update_cache failed: {exc}")
|
||||
return 1
|
||||
|
||||
countrylisting = dbExec.rawsql_select(
|
||||
dbengine["engine"],
|
||||
"select row_id, country_name from dbo.countrydata",
|
||||
lprint
|
||||
)
|
||||
for country in countrylisting:
|
||||
country_name = country[1]
|
||||
country_id = country[0]
|
||||
countrydictionary[country_name] = country_id
|
||||
mongo_updater = dbmongo(options)
|
||||
mongo_updater.update_mongo(
|
||||
updateList,
|
||||
ROOTDIR,
|
||||
directories,
|
||||
jget,
|
||||
countrydictionary,
|
||||
dbExec,
|
||||
dbengine,
|
||||
dboBase,
|
||||
lprint,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+305
@@ -0,0 +1,305 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
import time
|
||||
import sys
|
||||
from tqdm import tqdm
|
||||
import logs.Logger
|
||||
from db.functions import settype, dbmongo
|
||||
from datetime import datetime
|
||||
import json
|
||||
|
||||
|
||||
def set_apienv(urls, uprocess, dbengine, dbExec, updatesBase, lprint):
|
||||
"""Populate updates table from the API and return available updates.
|
||||
|
||||
Args:
|
||||
urls: URL configuration with an "updatesurl" key.
|
||||
uprocess: URL loader instance with `get_data`.
|
||||
jget: JSON processor with `jconvert`.
|
||||
dbengine: database engine dict (expects key "engine").
|
||||
dbExec: database execution helper with `update_tvupdates` and `rawsql_select`.
|
||||
updatesBase: list-like containing tables (uses index 8 for updates table).
|
||||
lprint: logger instance with `logprint`.
|
||||
|
||||
Returns:
|
||||
List of tuples (seriesid, timestamp) representing available updates.
|
||||
"""
|
||||
#updatetable = updatesBase[8]
|
||||
#inputdata = uprocess.get_data(urls["updatesurl"])
|
||||
#lprint.logprint("info", f"Retrieved {len(availableupdates)} rows for processing......")
|
||||
#
|
||||
newupdates = dbExec.rawsql_select(
|
||||
dbengine["engine"],
|
||||
"select seriesid,timestamp from updates.tvupdates",
|
||||
lprint
|
||||
)
|
||||
return newupdates
|
||||
|
||||
|
||||
def print_starttime(lprint, ts1):
|
||||
lprint.logprint("info", f"Sync start time: {str(ts1)}")
|
||||
|
||||
|
||||
def find_file_in_multiple_dirs(filename, directories):
|
||||
"""
|
||||
Checks if a file exists in any of the provided directories.
|
||||
|
||||
Args:
|
||||
filename (str): The name of the file to search for.
|
||||
directories (list): A list of directory paths to check.
|
||||
|
||||
Returns:
|
||||
Path or None: The full path to the file if found, otherwise None.
|
||||
"""
|
||||
directories_found = []
|
||||
for directory in directories:
|
||||
if os.path.isfile(f'{directory}{filename}'):
|
||||
directories_found.append(directory)
|
||||
|
||||
return directories_found
|
||||
|
||||
def load_json_to_mongodb(mongo_updater, directories, update_list, updateDict, lprint, track_changes=True):
|
||||
"""
|
||||
Load downloaded JSON files into MongoDB collections with change detection.
|
||||
|
||||
Processes series, episodes, cast, and crew data from cached JSON files
|
||||
and inserts or updates them in the appropriate MongoDB collections.
|
||||
Automatically detects if documents are new, updated, or unchanged by comparing
|
||||
updateDict timestamps with existing document 'updated' fields.
|
||||
|
||||
Args:
|
||||
mongo_updater: dbmongo instance for MongoDB operations
|
||||
directories: dict containing cache directory paths for different data types
|
||||
update_list: list of series IDs to process
|
||||
updateDict: dict with seriesid as key and timestamp as value
|
||||
lprint: logger instance for logging operations
|
||||
track_changes: bool, if True tracks inserted vs updated documents (default: True)
|
||||
|
||||
Returns:
|
||||
tuple: (successful_count, failed_count, stats_dict) for series processed
|
||||
stats_dict contains: {'inserted': count, 'updated': count}
|
||||
"""
|
||||
successful_count = 0
|
||||
failed_count = 0
|
||||
stats = {'series_inserted': 0, 'series_updated': 0, 'series_skipped': 0,
|
||||
'episodes_inserted': 0, 'episodes_updated': 0, 'episodes_skipped': 0,
|
||||
'cast_inserted': 0, 'cast_updated': 0, 'cast_skipped': 0,
|
||||
'crew_inserted': 0, 'crew_updated': 0, 'crew_skipped': 0}
|
||||
|
||||
lprint.logprint("info", f"Starting to load {len(update_list)} series into MongoDB")
|
||||
|
||||
for seriesid in tqdm(update_list[1:-1], desc="Loading series data into MongoDB", unit="series"):
|
||||
seriesid_str = int(seriesid)
|
||||
try:
|
||||
# Load series data
|
||||
series_file = f"{directories['SERIESDIR']}{seriesid}.json"
|
||||
if os.path.exists(series_file):
|
||||
try:
|
||||
with open(series_file, 'r') as f:
|
||||
series_data = json.load(f)
|
||||
|
||||
# Insert or update series in MongoDB
|
||||
if series_data:
|
||||
if track_changes:
|
||||
existing = mongo_updater.mgseries.find_one({"id": series_data.get("id")})
|
||||
#seriesid_str = str(seriesid)
|
||||
update_timestamp = updateDict.get(seriesid_str)
|
||||
if existing:
|
||||
# Check if data has changed by comparing update timestamp
|
||||
if update_timestamp and int(update_timestamp) > existing.get("updated", 0):
|
||||
mongo_updater.bulk_upsert_by_id(mongo_updater.mgseries, [series_data])
|
||||
stats['series_updated'] += 1
|
||||
lprint.logprint("debug", f"Updated series data for ID {seriesid}")
|
||||
else:
|
||||
lprint.logprint("debug", f"Series {seriesid} unchanged, skipping")
|
||||
stats['series_skipped'] += 1
|
||||
pass # Document unchanged
|
||||
else:
|
||||
mongo_updater.bulk_upsert_by_id(mongo_updater.mgseries, [series_data])
|
||||
stats['series_inserted'] += 1
|
||||
lprint.logprint("debug", f"Inserted new series data for ID {seriesid}")
|
||||
# Load episodes data
|
||||
episodes_file = f"{directories['EPISODEDIR']}{seriesid}.json"
|
||||
if os.path.exists(episodes_file):
|
||||
try:
|
||||
with open(episodes_file, 'r') as f:
|
||||
episodes_data = json.load(f)
|
||||
|
||||
if isinstance(episodes_data, list) and episodes_data:
|
||||
# Add seriesid to each episode
|
||||
for episode in episodes_data:
|
||||
episode['seriesid'] = seriesid_str
|
||||
|
||||
if track_changes:
|
||||
for episode in episodes_data:
|
||||
existing = mongo_updater.mgepisodes.find_one({"id": episode.get("id")})
|
||||
#seriesid_str = str(seriesid)
|
||||
update_timestamp = updateDict.get(seriesid_str)
|
||||
if existing:
|
||||
if update_timestamp and int(update_timestamp) > existing.get("updated", 0):
|
||||
mongo_updater.bulk_upsert_by_id(mongo_updater.mgepisodes, episodes_data)
|
||||
stats['episodes_updated'] += 1
|
||||
else:
|
||||
lprint.logprint("debug", f"Episode Data for Series {seriesid} unchanged, skipping")
|
||||
pass # Document unchanged
|
||||
else:
|
||||
mongo_updater.bulk_upsert_by_id(mongo_updater.mgepisodes, episodes_data)
|
||||
stats['episodes_inserted'] += 1
|
||||
|
||||
#mongo_updater.bulk_upsert_by_id(mongo_updater.mgepisodes, episodes_data)
|
||||
lprint.logprint("debug", f"Loaded {len(episodes_data)} episodes for series {seriesid}")
|
||||
except Exception as e:
|
||||
lprint.logprint("warning", f"Failed to load episodes file for {seriesid}: {e}")
|
||||
|
||||
# Load cast data
|
||||
cast_file = f"{directories['CASTDIR']}{seriesid}.json"
|
||||
if os.path.exists(cast_file):
|
||||
try:
|
||||
with open(cast_file, 'r') as f:
|
||||
cast_data = json.load(f)
|
||||
|
||||
if isinstance(cast_data, list) and cast_data:
|
||||
actors = []
|
||||
characters = []
|
||||
for item in cast_data:
|
||||
if 'person' in item and item['person']:
|
||||
item['person']['seriesid'] = seriesid_str
|
||||
actors.append(item['person'])
|
||||
if 'character' in item and item['character']:
|
||||
item['character']['seriesid'] = seriesid_str
|
||||
characters.append(item['character'])
|
||||
|
||||
if track_changes:
|
||||
for actor in actors:
|
||||
existing = mongo_updater.mgactors.find_one({"id": actor.get("id")})
|
||||
#seriesid_str = str(seriesid)
|
||||
update_timestamp = updateDict.get(seriesid_str)
|
||||
if existing:
|
||||
if update_timestamp and int(update_timestamp) > existing.get("updated", 0):
|
||||
stats['cast_updated'] += 1
|
||||
else:
|
||||
stats['cast_inserted'] += 1
|
||||
|
||||
if actors:
|
||||
mongo_updater.bulk_upsert_by_id(mongo_updater.mgactors, actors)
|
||||
if characters:
|
||||
mongo_updater.bulk_upsert_by_id(mongo_updater.mgcharacters, characters)
|
||||
lprint.logprint("debug", f"Loaded cast data for series {seriesid}")
|
||||
except Exception as e:
|
||||
lprint.logprint("warning", f"Failed to load cast file for {seriesid}: {e}")
|
||||
|
||||
# Load crew data
|
||||
crew_file = f"{directories['CREWDIR']}{seriesid}.json"
|
||||
if os.path.exists(crew_file):
|
||||
try:
|
||||
with open(crew_file, 'r') as f:
|
||||
crew_data = json.load(f)
|
||||
|
||||
if isinstance(crew_data, list) and crew_data:
|
||||
crew_list = []
|
||||
for item in crew_data:
|
||||
if 'person' in item and item['person']:
|
||||
item['person']['seriesid'] = seriesid_str
|
||||
item['person']['crew_type'] = item.get('type', 'unknown')
|
||||
crew_list.append(item['person'])
|
||||
|
||||
if track_changes:
|
||||
for crew in crew_list:
|
||||
existing = mongo_updater.mgcrew.find_one({"id": crew.get("id")})
|
||||
#seriesid_str = str(seriesid)
|
||||
update_timestamp = updateDict.get(seriesid_str)
|
||||
if existing:
|
||||
if update_timestamp and int(update_timestamp) > existing.get("updated", 0):
|
||||
stats['crew_updated'] += 1
|
||||
else:
|
||||
stats['crew_inserted'] += 1
|
||||
|
||||
if crew_list:
|
||||
mongo_updater.bulk_upsert_by_id(mongo_updater.mgcrew, crew_list)
|
||||
lprint.logprint("debug", f"Loaded crew data for series {seriesid}")
|
||||
except Exception as e:
|
||||
lprint.logprint("warning", f"Failed to load crew file for {seriesid}: {e}")
|
||||
|
||||
successful_count += 1
|
||||
lprint.logprint("info", f"Successfully processed series {seriesid}")
|
||||
#mongo_updater.bulk_upsert_by_id(mongo_updater.mgseries, [series_data])
|
||||
except Exception as e:
|
||||
lprint.logprint("warning", f"Failed to load series file for {seriesid}: {e}")
|
||||
failed_count += 1
|
||||
continue
|
||||
except Exception as e:
|
||||
failed_count += 1
|
||||
lprint.logprint("error", f"Unexpected error processing series {seriesid}: {e}")
|
||||
|
||||
lprint.logprint("info", f"MongoDB loading complete.")
|
||||
return successful_count, failed_count, stats
|
||||
|
||||
def main() -> int:
|
||||
ROOTDIR = os.getcwd()
|
||||
directories = {}
|
||||
# Use cross-platform path builders and ensure directories exist
|
||||
LOGDIR = os.path.join(ROOTDIR, "log")
|
||||
directories["SERIESDIR"] = os.path.join(ROOTDIR, "cache", "series") + os.sep
|
||||
directories["EPISODEDIR"] = os.path.join(ROOTDIR, "cache", "episodes") + os.sep
|
||||
directories["CASTDIR"] = os.path.join(ROOTDIR, "cache", "cast") + os.sep
|
||||
directories["GUESTCASTDIR"] = os.path.join(ROOTDIR, "cache", "guestcast") + os.sep
|
||||
directories["CREWDIR"] = os.path.join(ROOTDIR, "cache", "crew") + os.sep
|
||||
#directories["GUESTCREWDIR"] = os.path.join(ROOTDIR, "cache", "guestcrew") + os.sep
|
||||
#directories["CREDITSDIR"] = os.path.join(ROOTDIR, "cache", "credits") + os.sep
|
||||
#directories["ALIASDIR"] = os.path.join(ROOTDIR, "cache", "aliases") + os.sep
|
||||
tempdir = os.path.join(ROOTDIR, 'temp') + os.sep
|
||||
|
||||
lprint = logs.Logger.LPrint(LOGDIR, False)
|
||||
lprint.start_log()
|
||||
import settings.config
|
||||
|
||||
updateList = []
|
||||
updateDict = {}
|
||||
needed_updates = []
|
||||
|
||||
new_updates = open(f'{tempdir}tvmaze_updates.json', 'r').readlines()
|
||||
for line in new_updates:
|
||||
line = line.strip().replace('\n', '').replace('\r', '')
|
||||
linefield = line.split(":")
|
||||
updateList.append(linefield[0].replace('"',''))
|
||||
try:
|
||||
updateDict[linefield[0].replace('"','')] = linefield[1].replace(',','').strip()
|
||||
except IndexError:
|
||||
lprint.logprint("warning", f"Failed to parse line: {line}")
|
||||
# for result in new_updates:
|
||||
# updateid = result[0]
|
||||
# updatestamp = int(result[1])
|
||||
# tdif = int(tdifference)
|
||||
# if updatestamp > tdif:
|
||||
# needed_updates.append(updateid)
|
||||
# updateList.append(updateid)
|
||||
|
||||
config_options = settings.config.Config(ROOTDIR)
|
||||
options = config_options.config_options
|
||||
mongo_updater = dbmongo(options)
|
||||
dbtype, apitype = options["dbtype"], options["apitype"]
|
||||
|
||||
lprint.logprint("info", "Creating JSON instance for data retrieval.")
|
||||
lprint.logprint("info", "JSON instance created.")
|
||||
lprint.logprint("info", f"Update type is {options['updatetype']}")
|
||||
timedifference = options["updatetype"]
|
||||
currenttimestamp = time.time()
|
||||
if timedifference.lower() == "full":
|
||||
timedifference = int(currenttimestamp)
|
||||
basetime = str(currenttimestamp).split(".")
|
||||
currenttimestamp = int(basetime[0])
|
||||
ts1 = str(datetime.now()).split(".")[0]
|
||||
lprint.logprint("info", "Sync start time: " + str(ts1))
|
||||
print_starttime(lprint, ts1)
|
||||
|
||||
lprint.screenprint("info", f"Preparing to load JSON files into MongoDB.")
|
||||
lprint.screenprint("info", f"Loading JSON files into MongoDB.")
|
||||
successful, failed , st= load_json_to_mongodb(mongo_updater, directories, updateList, updateDict, lprint)
|
||||
lprint.screenprint("info", f"JSON loading complete. Successful: {successful}, Failed: {failed}")
|
||||
lprint.screenprint("info", f"Stats: {st}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
||||
@@ -1,138 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
from pymongo import MongoClient
|
||||
import yaml
|
||||
import json
|
||||
import psycopg2
|
||||
|
||||
class ColumnMeta:
|
||||
def __init__(self, name, type, description=None):
|
||||
self.name = name
|
||||
self.type = type
|
||||
self.description = description
|
||||
|
||||
def from_dict(value: dict):
|
||||
return ColumnMeta(name=value['name'], type=value['type'], description=value.get('description'))
|
||||
|
||||
|
||||
class TableMeta:
|
||||
def __init__(self, name, columns, description=None):
|
||||
self.name = name
|
||||
self.columns = columns
|
||||
self.description = description
|
||||
|
||||
|
||||
def from_dict(value):
|
||||
columns = [ColumnMeta.from_dict(v) for v in value['columns']]
|
||||
return TableMeta(name=value['name'], columns=columns, description=value.get('description'))
|
||||
|
||||
def from_file(path):
|
||||
file = open(path, 'r')
|
||||
yaml_to_dict = yaml.safe_load(file.read())
|
||||
try:
|
||||
file.close()
|
||||
except Exception:
|
||||
pass
|
||||
return TableMeta.from_dict(yaml_to_dict)
|
||||
|
||||
|
||||
try:
|
||||
connection_string = (
|
||||
(
|
||||
'postgres://postgres:Optimus0329@192.168.128.7:5432/media_dbsync'
|
||||
'?options=-csearch_path%3Ddbo,public,updates'
|
||||
)
|
||||
)
|
||||
pgclient = psycopg2.connect(connection_string)
|
||||
pgcursor = pgclient.cursor()
|
||||
print(pgclient.get_dsn_parameters(), "\n")
|
||||
print("Connected to postgresql database server.")
|
||||
pgcursor.execute("SELECT version();")
|
||||
record = pgcursor.fetchone()
|
||||
print("You are connected to - ", record, "\n")
|
||||
except Exception as e:
|
||||
print(f"Unable to connect to postgresql database server:{e}\nExitting!")
|
||||
exit()
|
||||
|
||||
|
||||
host = '192.168.128.8'
|
||||
port = 27017
|
||||
print(f"Connecting to mongodb at {host}:{port}")
|
||||
|
||||
try:
|
||||
connection_string = (
|
||||
f"mongodb://{host}:{port}"
|
||||
)
|
||||
mgclient = MongoClient(connection_string)
|
||||
mgdb = mgclient['test2']
|
||||
except BaseException as e:
|
||||
print(f"Unable to connect to mongodb.{e}\nExitting!")
|
||||
exit()
|
||||
else:
|
||||
mgseries = mgdb["series"]
|
||||
mgseries.create_index('id', unique=True)
|
||||
mgepisodes = mgdb["episodes"]
|
||||
mgepisodes.create_index('id', unique=True)
|
||||
mgactors = mgdb["actors"]
|
||||
mgactors.create_index('id', unique=True)
|
||||
mgcharacters = mgdb["characters"]
|
||||
mgcharacters.create_index('id', unique=True)
|
||||
mgcrew = mgdb["crew"]
|
||||
mgcrew.create_index('id', unique=True)
|
||||
print(
|
||||
f"Connection to mongodb at {host}:"
|
||||
f"{port} successful."
|
||||
)
|
||||
os.chdir('C:\\Projects\\new_dbsync\\db\\ddl')
|
||||
# Create the database and tables in PostgreSQL
|
||||
# Check if the database exists, if not create it
|
||||
# Check if the tables exist, if not create them
|
||||
# possibly add , 'genre_ddl.yaml', 'language_ddl.yaml' down the road
|
||||
ddl_list = [
|
||||
# 'seriesdata_ddl.yaml',
|
||||
# 'epdata_ddl.yaml',
|
||||
# 'actordata_ddl.yaml',
|
||||
# 'castdata_ddl.yaml',
|
||||
# 'crewdata_ddl.yaml',
|
||||
# 'countrydata_ddl.yaml',
|
||||
# 'networkdata_ddl.yaml',
|
||||
# 'webchanneldata_ddl.yaml',
|
||||
# 'temp_seriesdata_ddl.yaml',
|
||||
# 'temp_epdata_ddl.yaml',
|
||||
# 'temp_actordata_ddl.yaml',
|
||||
# 'temp_castdata_ddl.yaml',
|
||||
# 'temp_crewdata_ddl.yaml',
|
||||
# 'temp_countrydata_ddl.yaml',
|
||||
# 'temp_networkdata_ddl.yaml',
|
||||
# 'temp_webchanneldata_ddl.yaml',
|
||||
# 'tvupdates_ddl.yaml'
|
||||
'db_schema.yaml'
|
||||
]
|
||||
for ddl in ddl_list:
|
||||
path = os.path.join(os.getcwd(), ddl)
|
||||
table_meta = TableMeta.from_file(path)
|
||||
|
||||
columnlist = []
|
||||
for column in table_meta.columns:
|
||||
columnlist.append(f"{column.name} {column.type}")
|
||||
table_meta.columns = ', '.join(columnlist)
|
||||
#print(column.name, column.type)
|
||||
|
||||
sql_command = f"DROP TABLE IF EXISTS {table_meta.name};CREATE TABLE IF NOT EXISTS {table_meta.name} ({table_meta.columns})"
|
||||
#sql_command = f"CREATE TABLE IF NOT EXISTS {table_meta.name} ({table_meta.columns})"
|
||||
|
||||
try:
|
||||
pgcursor.execute(sql_command)
|
||||
pgclient.commit()
|
||||
except Exception as e:
|
||||
print(f"Unable to create table {table_meta.name} in postgresql database server:{e}\nExitting!")
|
||||
exit()
|
||||
#print(sql_command)
|
||||
|
||||
pgcursor.close()
|
||||
pgclient.close()
|
||||
print("PostgreSQL connection is closed")
|
||||
mgclient.close()
|
||||
print("MongoDB connection is closed")
|
||||
exit()
|
||||
@@ -1,118 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
from pymongo import MongoClient
|
||||
import yaml
|
||||
import json
|
||||
import psycopg2
|
||||
|
||||
class ColumnMeta:
|
||||
def __init__(self, name, type, description=None):
|
||||
self.name = name
|
||||
self.type = type
|
||||
self.description = description
|
||||
|
||||
def from_dict(value: dict):
|
||||
return ColumnMeta(name=value['name'], type=value['type'], description=value.get('description'))
|
||||
|
||||
|
||||
class TableMeta:
|
||||
def __init__(self, name, columns, description=None):
|
||||
self.name = name
|
||||
self.columns = columns
|
||||
self.description = description
|
||||
|
||||
def from_dict(value):
|
||||
columns = [ColumnMeta.from_dict(v) for v in value['columns']]
|
||||
return TableMeta(name=value['name'], columns=columns, description=value.get('description'))
|
||||
|
||||
def from_file(path):
|
||||
file = open(path, 'r')
|
||||
yaml_to_dict = yaml.safe_load(file.read())
|
||||
try:
|
||||
file.close()
|
||||
except Exception:
|
||||
pass
|
||||
return TableMeta.from_dict(yaml_to_dict)
|
||||
|
||||
|
||||
try:
|
||||
connection_string = (
|
||||
(
|
||||
'postgres://postgres:Optimus0329@192.168.128.7:5432/media_dbsync'
|
||||
'?options=-csearch_path%3Ddbo,public,updates'
|
||||
)
|
||||
)
|
||||
pgclient = psycopg2.connect(connection_string)
|
||||
pgcursor = pgclient.cursor()
|
||||
print(pgclient.get_dsn_parameters(), "\n")
|
||||
print("Connected to postgresql database server.")
|
||||
pgcursor.execute("SELECT version();")
|
||||
record = pgcursor.fetchone()
|
||||
print("You are connected to - ", record, "\n")
|
||||
except Exception as e:
|
||||
print(f"Unable to connect to postgresql database server:{e}\nExitting!")
|
||||
exit()
|
||||
|
||||
|
||||
host = '192.168.128.8'
|
||||
port = 27017
|
||||
print(f"Connecting to mongodb at {host}:{port}")
|
||||
|
||||
try:
|
||||
connection_string = (
|
||||
f"mongodb://{host}:{port}"
|
||||
)
|
||||
mgclient = MongoClient(connection_string)
|
||||
mgdb = mgclient['test2']
|
||||
except BaseException as e:
|
||||
print(f"Unable to connect to mongodb.{e}\nExitting!")
|
||||
exit()
|
||||
else:
|
||||
mgseries = mgdb["series"]
|
||||
mgseries.create_index('id', unique=True)
|
||||
mgepisodes = mgdb["episodes"]
|
||||
mgepisodes.create_index('id', unique=True)
|
||||
mgactors = mgdb["actors"]
|
||||
mgactors.create_index('id', unique=True)
|
||||
mgcharacters = mgdb["characters"]
|
||||
mgcharacters.create_index('id', unique=True)
|
||||
mgcrew = mgdb["crew"]
|
||||
mgcrew.create_index('id', unique=True)
|
||||
print(
|
||||
f"Connection to mongodb at {host}:"
|
||||
f"{port} successful."
|
||||
)
|
||||
os.chdir('C:\\Projects\\new_dbsync\\db\\ddl')
|
||||
# Create the database and tables in PostgreSQL
|
||||
# Check if the database exists, if not create it
|
||||
# Check if the tables exist, if not create them
|
||||
# possibly add , 'genre_ddl.yaml', 'language_ddl.yaml' down the road
|
||||
ddl_list = os.listdir(os.getcwd())
|
||||
for ddl in ddl_list:
|
||||
path = os.path.join(os.getcwd(), ddl)
|
||||
table_meta = TableMeta.from_file(path)
|
||||
|
||||
columnlist = []
|
||||
for column in table_meta.columns:
|
||||
columnlist.append(f"{column.name} {column.type}")
|
||||
table_meta.columns = ', '.join(columnlist)
|
||||
#print(column.name, column.type)
|
||||
|
||||
sql_command = f"DROP TABLE IF EXISTS {table_meta.name};CREATE TABLE IF NOT EXISTS {table_meta.name} ({table_meta.columns})"
|
||||
#sql_command = f"CREATE TABLE IF NOT EXISTS {table_meta.name} ({table_meta.columns})"
|
||||
|
||||
try:
|
||||
pgcursor.execute(sql_command)
|
||||
pgclient.commit()
|
||||
except Exception as e:
|
||||
print(f"Unable to create table {table_meta.name} in postgresql database server:{e}\nExitting!")
|
||||
exit()
|
||||
#print(sql_command)
|
||||
|
||||
pgcursor.close()
|
||||
pgclient.close()
|
||||
print("PostgreSQL connection is closed")
|
||||
mgclient.close()
|
||||
print("MongoDB connection is closed")
|
||||
exit()
|
||||
Reference in New Issue
Block a user