new_dbsync/update_mongo.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import time
import url.urls
import url.process
import api.tvmaze.cast as mazecast
import api.tvmaze.crew as mazecrew
import api.tvmaze.episodes as mazeepisodes
import api.tvmaze.series as mazeseries
import db.sql
import jprocessor.process
import logs.Logger
from db.functions import settype, dbmongo
from datetime import datetime

response = None


def set_apienv(urls, dbExec, updatesBase):
    updatetable = updatesBase[8]
    inputdata = uprocess.get_data(urls["updatesurl"])
    availableupdates = jget.jconvert(inputdata)
    print(f"Retrieved {len(availableupdates)} rows for processing......")
    dbExec.update_tvupdates(dbengine["engine"], availableupdates, updatetable)
    newupdates = dbExec.rawsql_select(
        dbengine["engine"],
        "select seriesid,timestamp from updates.tvupdates",
        lprint
    )
    return newupdates


def print_starttime(lprint, ts1):
    lprint.logprint("info", f"Sync start time: {str(ts1)}")


def get_tvdb():
    tvdb_urls = TVDBURLS.get_data()
    return tvdb_urls


def get_tvmaze():
    tvmaze_urls = TVMAZEURLS.get_data()
    return tvmaze_urls


if __name__ == "__main__":
    ROOTDIR = os.getcwd()
    directories = {}
    if os.name == "nt":
        LOGDIR = f"{ROOTDIR}\\log"
        directories["SERIESDIR"] = f"{ROOTDIR}\\cache\\series\\"
        directories["EPISODEDIR"] = f"{ROOTDIR}\\cache\\episodes\\"
        directories["CASTDIR"] = f"{ROOTDIR}\\cache\\cast\\"
        directories["CREWDIR"] = f"{ROOTDIR}\\cache\\crew\\"
    else:
        LOGDIR = f"{ROOTDIR}/log"
        directories["SERIESDIR"] = f"{ROOTDIR}/cache/series/"
        directories["EPISODEDIR"] = f"{ROOTDIR}/cache/episodes/"
        directories["CASTDIR"] = f"{ROOTDIR}/cache/cast/"
        directories["CREWDIR"] = f"{ROOTDIR}/cache/crew/"
    lprint = logs.Logger.LPrint(LOGDIR, False)
    lprint.start_log()
    import settings.config

    config_options = settings.config.Config(ROOTDIR)
    options = config_options.config_options
    dbtype, apitype = options["dbtype"], options["apitype"]
    dbs = settype(dbtype, apitype, options)
    dbclass, dbengine, apiengine = dbs.dbclass, dbs.dbengine, dbs.apiengine
    uprocess = url.process.Loadurl()
    TVMAZEURLS = url.urls.Tvmazeurls()
    TVMAZE_URLS = get_tvmaze()
    TVDBURLS = url.urls.TVDBurls()
    TVDB_URLS = get_tvdb()
    mseries = mazeseries.tvshow()
    mcast = mazecast.cast()
    mepisode = mazeepisodes.episodes()
    mcrew = mazecrew.crew()
    countrydictionary = {}
    updateRange = None

    lprint.logprint("info", "Creating JSON instance for data retrieval.")
    jget = jprocessor.process.Jsonload()
    lprint.logprint("info", "JSON instance created.")
    print(f"Update type is {options['updatetype']}")
    timedifference = options["updatetype"]
    currenttimestamp = time.time()
    if timedifference.lower() == "full":
        timedifference = int(currenttimestamp)
    basetime = str(currenttimestamp).split(".")
    currenttimestamp = int(basetime[0])
    ts1 = str(datetime.now()).split(".")[0]
    print("Sync start time: " + str(ts1))
    print_starttime(lprint, ts1)
    updatesBase = dbengine["updatesBase"]
    dboBase = dbengine["dboBase"]
    session = dbengine["session"]
    dbExec = db.sql.Dbexec()
    createTables = dbclass["Createtables"]
    Tables = dbclass["Tables"]
    createTempTables = dbclass["CreateTemptables"]
    new_updates = {}
    sqlexec = db.sql.SQLGenerate()
    mt = createTables(
        dbengine["session"],
        dbengine["metadata"],
        dbengine["engine"]
    )
    ct = createTempTables(
        dbengine["session"],
        dbengine["metadata"],
        dbengine["engine"]
    )
    tempTableList = ct.tablelist

    if options["apitype"].lower() == "tvmaze":
        new_updates = set_apienv(TVMAZE_URLS, dbExec, tempTableList)
    from cache.process import Cacher

    cacher = Cacher()
    cacher.process(options, jget, new_updates, ROOTDIR, TVMAZE_URLS)
    timeDifference = options["updatetype"]
    currentTimestamp = time.time()
    baseTime = str(currentTimestamp).split(".")
    if timeDifference.lower() == "full" or options["initload"] == "1":
        tdifference = 0
    else:
        tdifference = int(currentTimestamp) - int(
            options[options["updatetype"]]
        )
    needed_updates = []
    updateList = []
    for result in new_updates:
        updateid = result[0]
        updatestamp = int(result[1])
        tdif = int(tdifference)
        if updatestamp > tdif:
            needed_updates.append(updateid)
            updateList.append(updateid)

    print("Update Type: " + str(options["updatetype"]))
    updateRange = len(updateList)
    print("Number of updates: " + str(updateRange))
    cacher.update_cache(
        jget, updateList, ROOTDIR, TVMAZE_URLS, options, lprint
        )
    countrylisting = dbExec.rawsql_select(
        dbengine["engine"],
        "select row_id, country_name from dbo.countrydata",
        lprint
    )
    for country in countrylisting:
        country_name = country[1]
        country_id = country[0]
        countrydictionary[country_name] = country_id
    mongo_updater = dbmongo(options)
    mongo_updater.update_mongo(
        updateList,
        ROOTDIR,
        directories,
        jget,
        countrydictionary,
        dbExec,
        dbengine,
        dboBase,
        lprint,
    )
exit(0)