commit 29814092b719906d00e0a2429ff970e1f9bb0690 Author: Wendell Jones Date: Tue May 20 12:02:07 2025 -0400 repo migration diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..7fe70d7 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.json filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7c0fea6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +################################################################################ +# This .gitignore file was automatically created by Microsoft(R) Visual Studio. +################################################################################ + +/.vs/new_dbsync/FileContentIndex +/.vs +*.pyc +*.json +*.DS_Store +*.log +/pyproject.toml +dbsync.log.* +.idea +/log +venv diff --git a/README.md b/README.md new file mode 100644 index 0000000..047477f --- /dev/null +++ b/README.md @@ -0,0 +1,8 @@ +# Sample GitLab Project + +This sample project shows how a project in GitLab looks for demonstration purposes. It contains issues, merge requests and Markdown files in many branches, +named and filled with lorem ipsum. + +You can look around to get an idea how to structure your project and, when done, you can safely delete this project. + +[Learn more about creating GitLab projects.](https://docs.gitlab.com/ee/gitlab-basics/create-project.html) diff --git a/api/__init__.py b/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/thetvdb/__init__.py b/api/thetvdb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/thetvdb/cast.py b/api/thetvdb/cast.py new file mode 100644 index 0000000..483fd02 --- /dev/null +++ b/api/thetvdb/cast.py @@ -0,0 +1,14 @@ + +class cast(): + + def __init__(self): + pass + + def listcast(self, seriesid): + pass + + def listcredits(self, seriesid): + pass + + def __del__(self): + pass \ No newline at end of file diff --git a/api/thetvdb/crew.py b/api/thetvdb/crew.py new file mode 100644 index 0000000..bd86034 --- /dev/null +++ b/api/thetvdb/crew.py @@ -0,0 +1,15 @@ + +class crew(): + + def __init__(self): + pass + + def listcrew(self, seriesid): + pass + + def listcredits(self, seriesid): + pass + + def __del__(self): + pass + \ No newline at end of file diff --git a/api/thetvdb/episodes.py b/api/thetvdb/episodes.py new file mode 100644 index 0000000..cac6af5 --- /dev/null +++ b/api/thetvdb/episodes.py @@ -0,0 +1,18 @@ + +class episodes(): + + def __init__(self): + pass + + def fulllisting(self, seriesid): + pass + + def seasonlist(self, seriesid, seasonnumber): + pass + + def eplist(self, seriesid, seasonnumber, episodenumber): + pass + + def __del__(self): + pass + \ No newline at end of file diff --git a/api/thetvdb/process.py b/api/thetvdb/process.py new file mode 100644 index 0000000..68f679e --- /dev/null +++ b/api/thetvdb/process.py @@ -0,0 +1,256 @@ + +class show(): + + def __init__(self): + pass + + def process(self, metadata, showload, seriesid): + dataresult = [] + tableresult = [] + if showload == None: + pass + else: + networkid = None + try: + network = {} + for key, value in showload['network'].items(): + if key == 'country': + for key, value in showload['network']['country'].items(): + if key == 'name': + newkey = 'country' + network[newkey] = showload['network']['country'][key] + else: + newkey = 'country' + key + network[newkey] = showload['network']['country'][key] + else: + newkey = 'network' + key + network[newkey] = showload['network'][key] + except: + #print('Failed to insert network information.') + showload['networkid'] = 0 + else: + showload.pop('network') + showload['networkid'] = network['networkid'] + dataresult.append(network) + tableresult.append(metadata.tables['[updates].tvnetworknewvals']) + #insertresult = insert_data(network, tables[4]) + try: + webchannel = {} + for key, value in showload['webChannel'].items(): + if key == 'country': + if showload['webChannel'][key] is None: + pass + else: + for key, value in showload['webChannel']['country'].items(): + newkey = 'webChannelcountry'+ key + webchannel[newkey] = value + else: + webchannel[key] = value + if key == 'name': + newkey = 'webChannel' + key + webhchannel[newkey] = value + if key == 'timezone': + newkey = 'webChannelcountry'+ key + webchannel[newkey] = value + except: + #print('Failed to insert webchannel information.') + showload['webchannelid'] = '0' + else: + showload.pop('webChannel') + showload['webchannelid'] = webchannel['id'] + webchannel['webchannelname'] = webchannel['name'] + webchannel['networkid'] = webchannel['id'] + try: + dataresult.append(webchannel) + tableresult.append(metadata.tables['[updates].tvwebchannelnewvals']) + #insertresult = insert_data(webchannel, tables[5]) + except sqlalchemy.exc.IntegrityError as e: + pass + except Exception as e: + print('Failed to insert webchannel information.') + try: + showload['seriesid'] = showload['id'] + except: + pass + + #print('TV Show: ' + showload['seriesname']) + + try: + if showload['image']: + for key, value in showload['image'].items(): + newkey = key + 'image' + showload[newkey] = value + showload.pop('image') + else: + showload.pop('image') + except: + pass + + try: + if showload['externals']: + for key, value in showload['externals'].items(): + showload[key] = value + showload.pop('externals') + else: + showload.pop('externals') + except KeyError as e: + pass + + try: + if showload['rating']: + for key, value in showload['rating'].items(): + showload['rating'] = value + + if showload['schedule']: + showload['time'] = showload['schedule']['time'] + showload['days'] = ", ".join(showload['schedule']['days']) + showload.pop('schedule') + except KeyError as e: + pass + + try: + for key, value in showload['_links']['self'].items(): + showload[key] = value + showload.pop('_links') + except KeyError as e: + pass + else: + showload.pop('id') + + showload['genres'] = ", ".join(showload['genres']) + + # Finish transforming to match table columns + showload['seriesname'] = showload.pop('name') + showload['tvrageid'] = showload.pop('tvrage') + showload['tvdbid'] = showload.pop('thetvdb') + showload['imdbid'] = showload.pop('imdb') + showload['serieslanguage'] = showload.pop('language') + showload['seriesstatus'] = showload.pop('status') + showload['airdays'] = showload.pop('days') + showload['airtime'] = showload.pop('time') + showload['showtype'] = showload.pop('type') + showload['overview'] = showload.pop('summary') + showload['showurl'] = showload.pop('url') + showload['apilink'] = showload.pop('href') + showload['officialsite'] = showload.pop('officialSite') + + results = [] + results.append([dataresult, tableresult, showload]) + #results.append(tableresult) + #results.append(showload) + return (results) + + + def __del__(self): + pass + +class cast(): + def __init__(self): + pass + + def process(self, cdata, seriesid): + datalist = {} + datalist['actors'] = [] + datalist['characters'] = [] + try: + castload = json.loads(cdata) + for idx, castdata in enumerate(castload): + actordata = {} + characterdata = {} + tcharacterdata = castload[idx]['character'] + tactordata = castload[idx]['person'] + try: + for key, value in tactordata.items(): + if tactordata[key] is None: + pass + else: + if key == 'image': + for key, value in tactordata['image'].items(): + newkey = key + 'image' + actordata[newkey] = tactordata['image'][key] + elif key == '_links': + for key, value in tactordata['_links']['self'].items(): + actordata['apilink'] = tactordata['_links']['self'][key] + elif key == 'country': + for key, value in tactordata['country'].items(): + newkey = 'country' + key + actordata[newkey] = tactordata['country'][key] + else: + actordata[key] = tactordata[key] + actordata['actorid'] = tactordata['id'] + actordata['actorname'] = tactordata['name'] + except: + print('Unable to process actor information.') + else: + actordata.pop('id') + + try: + if tcharacterdata: + for key, value in tcharacterdata.items(): + if key is None: + pass + else: + if tcharacterdata['image']: + characterdata['mediumimage'] = tcharacterdata['image']['medium'] + characterdata['originalimage'] = tcharacterdata['image']['original'] + characterdata['apilink'] = tcharacterdata['_links']['self']['href'] + characterdata['tvmazeilink'] = tcharacterdata['url'] + characterdata['characterid'] = tcharacterdata['id'] + characterdata['charactername'] = tcharacterdata['name'] + characterdata['seriesid'] = seriesid + characterdata['actorid'] = actordata['actorid'] + except Exception as e: + print('Unable to process character information. -> ' + str(e)) + + datalist['characters'].append(characterdata) + datalist['actors'].append(actordata) + linkdata = {} + linkdata['actorid'] = actordata['actorid'] + linkdata['characterid'] = characterdata['characterid'] + #return(datalist) + except Exception as e: + #print('Line 389: ' + str(e)) + pass + else: + return(datalist) + def __del__(self): + pass + +class Episode(): + + def __init__(self): + pass + + def process(self, epload, seriesid): + if epload: + eplist = [] + try: + for epdata in epload: + epdata['seriesid'] = seriesid + if epdata['image']: + for key, value in epdata['image'].items(): + newkey = key + 'image' + epdata[newkey] = value + epdata.pop('image') + else: + epdata['mediumimage'] = 'None' + epdata['originalimage'] = 'None' + epdata.pop('image') + + for key, value in epdata['_links']['self'].items(): + epdata['eplink'] = value + epdata.pop('_links') + + epdata['episodeid'] = epdata.pop('id') + epdata['episodename'] = epdata.pop('name') + epdata['episodesummary'] = epdata.pop('summary') + epdata['episodenumber'] = epdata.pop('number') + eplist.append(epdata) + return eplist + except: + print('Unable to import episode data for for seriesid: ' + str(seriesid)) + else: + return None + + def __del__(self): + pass diff --git a/api/thetvdb/series.py b/api/thetvdb/series.py new file mode 100644 index 0000000..d0b96a2 --- /dev/null +++ b/api/thetvdb/series.py @@ -0,0 +1,18 @@ + +class series(): + + def __init__(self): + pass + + def search(self, searchquery): + pass + + def singlesearch(self, searchquery): + pass + + def idsearch(self, searchid): + pass + + def __del__(self): + pass + diff --git a/api/thetvdb/thetvdb.py b/api/thetvdb/thetvdb.py new file mode 100644 index 0000000..bf0d225 --- /dev/null +++ b/api/thetvdb/thetvdb.py @@ -0,0 +1,231 @@ +import requests +import json + +class Processor(): + def __init__(self): + global loginurl + global refreshurl + global seriesurl + global actorurl + global episodeurl + global episodequeryurl + global episodeparamurl + global episodesummaryurl + global filterurl + global filterparamurl + global imageurl + global imagequeryurl + global imageparamurl + global username + global apikey + global userkey + global jwtoken + global refreshtoken + loginurl = 'https://api.thetvdb.com/login' + refreshurl = 'https://api.thetvdb.com/refresh_token' + seriesurl = 'https://api.thetvdb.com/series/{id}' + episodeidurl = 'https://api.thetvdb.com/episodes/{id}' + actorurl='https://api.thetvdb.com/series/{id}/actors' + episodeurl='https://api.thetvdb.com/series/{id}/episodes' + episodequeryurl='https://api.thetvdb.com/series/{id}/episodes/query' + episodeparamurl='https://api.thetvdb.com/series/{id}/episodes/query/params' + episodesummaryurl='https://api.thetvdb.com/series/{id}/episodes/summary' + filterurl='https://api.thetvdb.com/series/{id}/filter' + filterparamurl='https://api.thetvdb.com/series/{id}/filter/params' + imageurl='https://api.thetvdb.com/series/{id}/images' + imagequeryurl='https://api.thetvdb.com/series/{id}/images/query' + imageparamurl='https://api.thetvdb.com/series/{id}/images/query/params' + username = 'wpjonesnhcjc' + apikey = 'NACCS2LUWCFEFABX' + userkey = 'UUSZHUQ8CY7N31CF' + #jwtoken = login.run(self) + + def __del__(self): + pass + +class login(): + def __init__(self): + pass + + def run(self): + with requests.post(loginurl, json={"username": username, "apikey": apikey, "userkey": userkey}) as r: + response = r.json() + jwtoken = response['token'] + #refreshtoken = self.refresh(jwtoken) + return(jwtoken) + + def refresh(self, token): + token= "Bearer " + token + with requests.get(refreshurl, headers={'Authorization': token}) as r: + response = r.json() + refreshtoken = response['token'] + return(refreshtoken) + +class loadjson(): + def __init__(self): + pass + + def loadseries(seriesid, token): + url = seriesurl.replace('{id}', seriesid) + #print(str(url)) + seriesdata = loadjson.load(url, token) + return(seriesdata) + + def loadepisodes(seriesid, token): + url = episodeurl.replace('{id}', seriesid) + #print(str(url)) + seriesdata = loadjson.load(url, token) + return(seriesdata) + + def load(url, token): + token= "Bearer " + token + with requests.get(url, headers={'Authorization': token}) as r: + response = r.json() + return(response) + + def insert(self, updater, tablename): + for key, value in updater.items(): + ins = tablename.insert().values( key = value ) + print(str(ins)) + result = engine.execute(ins) + + def populate(self, jsonurl, seriesid): + try: + with urllib.request.urlopen(jsonurl) as response: + html = response.read() + jsoninput = html + return jsoninput + except: + pass + + def __del__(self): + pass + +class showprocess(): + + def __init__(self): + pass + + def process(self, metadata, tempshowload): + dataresult = [] + tableresult = [] + if tempshowload == None: + pass + else: + showload = json.loads(tempshowload) + networkid = None + try: + network = {} + for key, value in showload['network'].items(): + if key == 'country': + for key, value in showload['network']['country'].items(): + if key == 'name': + newkey = 'country' + network[newkey] = showload['network']['country'][key] + else: + newkey = 'country' + key + network[newkey] = showload['network']['country'][key] + else: + newkey = 'network' + key + network[newkey] = showload['network'][key] + except: + #print('Failed to insert network information.') + showload['networkid'] = '0' + else: + showload.pop('network') + showload['networkid'] = network['networkid'] + dataresult.append(network) + tableresult.append(metadata.tables['[updates].tvnetworknewvals']) + #insertresult = insert_data(network, tables[4]) + try: + webchannel = {} + for key, value in showload['webChannel'].items(): + if key == 'country': + if showload['webChannel'][key] is None: + pass + else: + for key, value in showload['webChannel']['country'].items(): + newkey = 'webChannelcountry'+ key + webchannel[newkey] = value + else: + webchannel[key] = value + if key == 'name': + newkey = 'webChannel' + key + webhchannel[newkey] = value + if key == 'timezone': + newkey = 'webChannelcountry'+ key + webchannel[newkey] = value + except: + #print('Failed to insert webchannel information.') + showload['webchannelid'] = '0' + else: + showload.pop('webChannel') + showload['webchannelid'] = webchannel['id'] + webchannel['webchannelname'] = webchannel['name'] + try: + dataresult.append(webchannel) + tableresult.append(metadata.tables['[updates].tvwebchannelnewvals']) + #insertresult = insert_data(webchannel, tables[5]) + except sqlalchemy.exc.IntegrityError as e: + pass + except Exception as e: + print('Failed to insert webchannel information.') + try: + showload['seriesid'] = showload['id'] + except: + pass + + #print('TV Show: ' + showload['seriesname']) + + if showload['image']: + for key, value in showload['image'].items(): + newkey = key + 'image' + showload[newkey] = value + showload.pop('image') + else: + showload.pop('image') + + if showload['externals']: + for key, value in showload['externals'].items(): + showload[key] = value + showload.pop('externals') + else: + showload.pop('externals') + + if showload['rating']: + for key, value in showload['rating'].items(): + showload['rating'] = value + + if showload['schedule']: + showload['time'] = showload['schedule']['time'] + showload['days'] = ", ".join(showload['schedule']['days']) + showload.pop('schedule') + + for key, value in showload['_links']['self'].items(): + showload[key] = value + showload.pop('_links') + + showload.pop('id') + + showload['genres'] = ", ".join(showload['genres']) + + # Finish transforming to match table columns + showload['seriesname'] = showload.pop('name') + showload['tvrageid'] = showload.pop('tvrage') + showload['tvdbid'] = showload.pop('thetvdb') + showload['imdbid'] = showload.pop('imdb') + showload['serieslanguage'] = showload.pop('language') + showload['seriesstatus'] = showload.pop('status') + showload['airdays'] = showload.pop('days') + showload['airtime'] = showload.pop('time') + showload['showtype'] = showload.pop('type') + showload['overview'] = showload.pop('summary') + showload['showurl'] = showload.pop('url') + showload['apilink'] = showload.pop('href') + showload['officialsite'] = showload.pop('officialSite') + + results = [] + results.append(dataresult) + results.append(tableresult) + results.append(showload) + return (results) diff --git a/api/tvmaze/__init__.py b/api/tvmaze/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/tvmaze/cast.py b/api/tvmaze/cast.py new file mode 100644 index 0000000..7e08137 --- /dev/null +++ b/api/tvmaze/cast.py @@ -0,0 +1,105 @@ +from sqlalchemy.sql.expression import false + +class cast(): + + def __init__(self): + pass + + def process(self, castdata, seriesid): + datalist = {} + linkdata = {} + datalist['actors'] = [] + datalist['characters'] = [] + linklist = [] + for castmember in castdata: + actordata = {} + characterdata = {} + tcharacterdata = castmember['character'] + tactordata = castmember['person'] + tselfdata = castmember['self'] + tvoicedata = castmember['voice'] + try: + for key, value in tactordata.items(): + if tactordata[key] is None: + pass + else: + if key == 'image': + for key, value in tactordata['image'].items(): + newkey = key + 'image' + actordata[newkey] = tactordata['image'][key] + elif key == '_links': + for key, value in tactordata['_links']['self'].items(): + actordata['apilink'] = tactordata['_links']['self'][key] + elif key == 'country': + for key, value in tactordata['country'].items(): + newkey = 'country' + key + actordata[newkey] = tactordata['country'][key] + else: + actordata[key] = tactordata[key] + actordata['actorid'] = tactordata['id'] + actordata['actorname'] = tactordata['name'] + if tactordata['country'] == None: + actordata['countryname'] = None + actordata['countrytimezone'] = None + actordata['countrycode'] = None + if tactordata['birthday'] == None: + actordata['birthday'] = None + if tactordata['deathday'] == None: + actordata['deathday'] = None + if tactordata['gender'] == None: + actordata['gender'] = None + if tactordata['image'] == None: + actordata['mediumimage'] = None + actordata['originalimage'] = None + if tactordata['_links'] == None: + actordata['apilink'] = None + except: + print('Unable to process actor information.') + else: + actordata.pop('id') + + try: + if tcharacterdata: + for key, value in tcharacterdata.items(): + if key is None: + pass + else: + if tcharacterdata['image']: + characterdata['mediumimage'] = tcharacterdata['image']['medium'] + characterdata['originalimage'] = tcharacterdata['image']['original'] + characterdata['apilink'] = tcharacterdata['_links']['self']['href'] + characterdata['tvmazeilink'] = tcharacterdata['url'] + characterdata['characterid'] = tcharacterdata['id'] + characterdata['charactername'] = tcharacterdata['name'] + characterdata['seriesid'] = seriesid + characterdata['actorid'] = actordata['actorid'] + if tcharacterdata['image'] == None: + characterdata['mediumimage'] = None + characterdata['originalimage'] = None + if tactordata['_links'] == None: + actordata['apilink'] = None + if not tselfdata: + characterdata['self'] = 0 + else: + characterdata['self'] = 1 + if not tvoicedata: + characterdata['voice'] = 0 + else: + characterdata['voice'] = 1 + except Exception as e: + print('Unable to process character information. -> ' + str(e)) + datalist['characters'].append(characterdata) + datalist['actors'].append(actordata) + linkdata = {'actorid': actordata['actorid'], 'characterid': characterdata['characterid']} + linklist.append(linkdata) + return(datalist, linklist) + # except Exception as e: + # print('Unable to process cast data: ' + str(e)) + # else: + # return(datalist, linklist) + + def listcredits(self, seriesid): + pass + + def __del__(self): + pass \ No newline at end of file diff --git a/api/tvmaze/crew.py b/api/tvmaze/crew.py new file mode 100644 index 0000000..1ffc017 --- /dev/null +++ b/api/tvmaze/crew.py @@ -0,0 +1,42 @@ + +class crew(): + + def __init__(self): + pass + + def process(self, crewdata, seriesid): + crewlist = [] + try: + for crewmember in crewdata: + cmember = {} + cmember['seriesid'] = seriesid + cmember['crewid'] = crewmember['person']['id'] + cmember['type'] = crewmember['type'] + cmember['url'] = crewmember['person']['url'] + cmember['name'] = crewmember['person']['name'] + cmember['countryid'] = '0' + cmember['birthday'] = crewmember['person']['birthday'] + cmember['deathday'] = crewmember['person']['deathday'] + cmember['gender'] = crewmember['person']['gender'] + if crewmember['person']['image']: + cmember['medium'] = crewmember['person']['image']['medium'] + cmember['original'] = crewmember['person']['image']['original'] + else: + cmember['medium'] = None + cmember['original'] = None + if crewmember['person']['_links']: + cmember['apilink'] = crewmember['person']['_links']['self']['href'] + else: + cmember['apilink'] = None + crewlist.append(cmember) + except Exception as e: + print('Unable to process crew information for seriesid: ' + str(seriesid)) + else: + return(crewlist) + + def listcredits(self, seriesid): + pass + + def __del__(self): + pass + \ No newline at end of file diff --git a/api/tvmaze/episodes.py b/api/tvmaze/episodes.py new file mode 100644 index 0000000..d8a4be6 --- /dev/null +++ b/api/tvmaze/episodes.py @@ -0,0 +1,77 @@ +import csv +class episodes(): + + + def __init__(self): + pass + + def update_episodes(self, epload, seriesid): + if epload: + eplist = [] + try: + for epdata in epload: + epdata['seriesid'] = seriesid + if epdata['image']: + for key, value in epdata['image'].items(): + newkey = key + 'image' + epdata[newkey] = value + epdata.pop('image') + else: + epdata['mediumimage'] = 'None' + epdata['originalimage'] = 'None' + epdata.pop('image') + + for key, value in epdata['_links']['self'].items(): + epdata['eplink'] = value + epdata.pop('_links') + + epdata['episodeid'] = epdata.pop('id') + epdata['episodename'] = epdata.pop('name') + epdata['episodesummary'] = epdata.pop('summary') + epdata['episodenumber'] = epdata.pop('number') + eplist.append(epdata) + except: + print('Unable to import episode data for for seriesid: ' + str(seriesid)) + else: + return(eplist) + + def fulllisting(self, seriesid): + pass + + def seasonlist(self, seriesid, seasonnumber): + pass + + def process_episodes(self, episoderesults, seriesid, rootdir): + queryrows = [] + if episoderesults: + resultlength = len(episoderesults) + if resultlength > 0: + for idx, episode in enumerate(episoderesults): + if episoderesults[idx]['episodesummary'] is None: + episoderesults[idx]['episodesummary'] = 'No summary at this time.' + querylist = '''INSERT INTO updates.newepdata (episodeid, seriesid, season, episodenumber, episodename, airdate, airtime, episodesummary, airstamp, runtime, eplink, mediumimage, url, originalimage) VALUES\n''' + filename = rootdir + '/temp/' + str(seriesid) + '_episodes.sql' + sqlfile = open(filename, 'w', encoding="utf-8") + sqlfile.write(querylist) + try: + SQL = '''( '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s' )''' % (episoderesults[idx]['episodeid'], + episoderesults[idx]['seriesid'], episoderesults[idx]['season'], episoderesults[idx]['episodenumber'], episoderesults[idx]['episodename'].replace("'", "''"), + episoderesults[idx]['airdate'], episoderesults[idx]['airtime'], episoderesults[idx]['episodesummary'].replace("'", "''"), episoderesults[idx]['airstamp'], + episoderesults[idx]['runtime'], episoderesults[idx]['eplink'], episoderesults[idx]['mediumimage'], episoderesults[idx]['url'], episoderesults[idx]['originalimage']) + #tv.insert_data(engine, episode, metadata.tables['[updates].newepdata']) + queryrows.append(SQL) + except Exception as e: + lprint.logprint('info', 'Unable to process episodes for seriesid: ' + str(cseriesid)) + else: + qlength = len(queryrows) - 1 + for rowidx, queryrow in enumerate(queryrows): + if rowidx == qlength and idx == len(episoderesults) -1: + sqlfile.write(queryrow + ';\n') + else: + sqlfile.write(queryrow + ',\n') + sqlfile.close() + return(filename, resultlength, querylist) + + def __del__(self): + pass + \ No newline at end of file diff --git a/api/tvmaze/process.py b/api/tvmaze/process.py new file mode 100644 index 0000000..68f679e --- /dev/null +++ b/api/tvmaze/process.py @@ -0,0 +1,256 @@ + +class show(): + + def __init__(self): + pass + + def process(self, metadata, showload, seriesid): + dataresult = [] + tableresult = [] + if showload == None: + pass + else: + networkid = None + try: + network = {} + for key, value in showload['network'].items(): + if key == 'country': + for key, value in showload['network']['country'].items(): + if key == 'name': + newkey = 'country' + network[newkey] = showload['network']['country'][key] + else: + newkey = 'country' + key + network[newkey] = showload['network']['country'][key] + else: + newkey = 'network' + key + network[newkey] = showload['network'][key] + except: + #print('Failed to insert network information.') + showload['networkid'] = 0 + else: + showload.pop('network') + showload['networkid'] = network['networkid'] + dataresult.append(network) + tableresult.append(metadata.tables['[updates].tvnetworknewvals']) + #insertresult = insert_data(network, tables[4]) + try: + webchannel = {} + for key, value in showload['webChannel'].items(): + if key == 'country': + if showload['webChannel'][key] is None: + pass + else: + for key, value in showload['webChannel']['country'].items(): + newkey = 'webChannelcountry'+ key + webchannel[newkey] = value + else: + webchannel[key] = value + if key == 'name': + newkey = 'webChannel' + key + webhchannel[newkey] = value + if key == 'timezone': + newkey = 'webChannelcountry'+ key + webchannel[newkey] = value + except: + #print('Failed to insert webchannel information.') + showload['webchannelid'] = '0' + else: + showload.pop('webChannel') + showload['webchannelid'] = webchannel['id'] + webchannel['webchannelname'] = webchannel['name'] + webchannel['networkid'] = webchannel['id'] + try: + dataresult.append(webchannel) + tableresult.append(metadata.tables['[updates].tvwebchannelnewvals']) + #insertresult = insert_data(webchannel, tables[5]) + except sqlalchemy.exc.IntegrityError as e: + pass + except Exception as e: + print('Failed to insert webchannel information.') + try: + showload['seriesid'] = showload['id'] + except: + pass + + #print('TV Show: ' + showload['seriesname']) + + try: + if showload['image']: + for key, value in showload['image'].items(): + newkey = key + 'image' + showload[newkey] = value + showload.pop('image') + else: + showload.pop('image') + except: + pass + + try: + if showload['externals']: + for key, value in showload['externals'].items(): + showload[key] = value + showload.pop('externals') + else: + showload.pop('externals') + except KeyError as e: + pass + + try: + if showload['rating']: + for key, value in showload['rating'].items(): + showload['rating'] = value + + if showload['schedule']: + showload['time'] = showload['schedule']['time'] + showload['days'] = ", ".join(showload['schedule']['days']) + showload.pop('schedule') + except KeyError as e: + pass + + try: + for key, value in showload['_links']['self'].items(): + showload[key] = value + showload.pop('_links') + except KeyError as e: + pass + else: + showload.pop('id') + + showload['genres'] = ", ".join(showload['genres']) + + # Finish transforming to match table columns + showload['seriesname'] = showload.pop('name') + showload['tvrageid'] = showload.pop('tvrage') + showload['tvdbid'] = showload.pop('thetvdb') + showload['imdbid'] = showload.pop('imdb') + showload['serieslanguage'] = showload.pop('language') + showload['seriesstatus'] = showload.pop('status') + showload['airdays'] = showload.pop('days') + showload['airtime'] = showload.pop('time') + showload['showtype'] = showload.pop('type') + showload['overview'] = showload.pop('summary') + showload['showurl'] = showload.pop('url') + showload['apilink'] = showload.pop('href') + showload['officialsite'] = showload.pop('officialSite') + + results = [] + results.append([dataresult, tableresult, showload]) + #results.append(tableresult) + #results.append(showload) + return (results) + + + def __del__(self): + pass + +class cast(): + def __init__(self): + pass + + def process(self, cdata, seriesid): + datalist = {} + datalist['actors'] = [] + datalist['characters'] = [] + try: + castload = json.loads(cdata) + for idx, castdata in enumerate(castload): + actordata = {} + characterdata = {} + tcharacterdata = castload[idx]['character'] + tactordata = castload[idx]['person'] + try: + for key, value in tactordata.items(): + if tactordata[key] is None: + pass + else: + if key == 'image': + for key, value in tactordata['image'].items(): + newkey = key + 'image' + actordata[newkey] = tactordata['image'][key] + elif key == '_links': + for key, value in tactordata['_links']['self'].items(): + actordata['apilink'] = tactordata['_links']['self'][key] + elif key == 'country': + for key, value in tactordata['country'].items(): + newkey = 'country' + key + actordata[newkey] = tactordata['country'][key] + else: + actordata[key] = tactordata[key] + actordata['actorid'] = tactordata['id'] + actordata['actorname'] = tactordata['name'] + except: + print('Unable to process actor information.') + else: + actordata.pop('id') + + try: + if tcharacterdata: + for key, value in tcharacterdata.items(): + if key is None: + pass + else: + if tcharacterdata['image']: + characterdata['mediumimage'] = tcharacterdata['image']['medium'] + characterdata['originalimage'] = tcharacterdata['image']['original'] + characterdata['apilink'] = tcharacterdata['_links']['self']['href'] + characterdata['tvmazeilink'] = tcharacterdata['url'] + characterdata['characterid'] = tcharacterdata['id'] + characterdata['charactername'] = tcharacterdata['name'] + characterdata['seriesid'] = seriesid + characterdata['actorid'] = actordata['actorid'] + except Exception as e: + print('Unable to process character information. -> ' + str(e)) + + datalist['characters'].append(characterdata) + datalist['actors'].append(actordata) + linkdata = {} + linkdata['actorid'] = actordata['actorid'] + linkdata['characterid'] = characterdata['characterid'] + #return(datalist) + except Exception as e: + #print('Line 389: ' + str(e)) + pass + else: + return(datalist) + def __del__(self): + pass + +class Episode(): + + def __init__(self): + pass + + def process(self, epload, seriesid): + if epload: + eplist = [] + try: + for epdata in epload: + epdata['seriesid'] = seriesid + if epdata['image']: + for key, value in epdata['image'].items(): + newkey = key + 'image' + epdata[newkey] = value + epdata.pop('image') + else: + epdata['mediumimage'] = 'None' + epdata['originalimage'] = 'None' + epdata.pop('image') + + for key, value in epdata['_links']['self'].items(): + epdata['eplink'] = value + epdata.pop('_links') + + epdata['episodeid'] = epdata.pop('id') + epdata['episodename'] = epdata.pop('name') + epdata['episodesummary'] = epdata.pop('summary') + epdata['episodenumber'] = epdata.pop('number') + eplist.append(epdata) + return eplist + except: + print('Unable to import episode data for for seriesid: ' + str(seriesid)) + else: + return None + + def __del__(self): + pass diff --git a/api/tvmaze/series.py b/api/tvmaze/series.py new file mode 100644 index 0000000..1a181b9 --- /dev/null +++ b/api/tvmaze/series.py @@ -0,0 +1,156 @@ + +class tvshow(): + + def __init__(self): + pass + + def process(self, metadata, seriesdata, seriesid): + webresult = [] + networkresult = [] + webworkchannel = [] + networkchannel = [] + if seriesdata == None: + pass + else: + networkid = None + try: + network = {} + for key, value in seriesdata['network'].items(): + if key == 'country': + for key, value in seriesdata['network']['country'].items(): + if key == 'name': + newkey = 'country' + network[newkey] = seriesdata['network']['country'][key] + else: + newkey = 'country' + key + network[newkey] = seriesdata['network']['country'][key] + else: + newkey = 'network' + key + network[newkey] = seriesdata['network'][key] + except: + #print('Failed to insert network information.') + #seriesdata['networkid'] = '0' + pass + else: + seriesdata.pop('network') + seriesdata['networkid'] = network['networkid'] + networkresult.append(network) + networkchannel.append(metadata.tables['updates.tvnetworknewvals']) + #insertresult = insert_data(network, tables[4]) + try: + webchannel = {} + if seriesdata['webChannel']: + for key, value in seriesdata['webChannel'].items(): + if key == 'country': + if seriesdata['webChannel'][key] is None: + pass + else: + for key, value in seriesdata['webChannel']['country'].items(): + newkey = 'webChannelcountry'+ key + webchannel[newkey] = value + else: + webchannel[key] = value + if key == 'name': + newkey = 'webChannel' + key + webchannel[newkey] = value + if key == 'timezone': + newkey = 'webChannelcountry'+ key + webchannel[newkey] = value + except: + #print('Failed to insert webchannel information.') + seriesdata['webchannelid'] = '0' + pass + else: + #seriesdata.pop('webChannel') + webchannel['id'] = '0' + seriesdata['webchannelid'] = webchannel['id'] + webchannel['name'] = 'Not Applicable' + webchannel['webchannelname'] = webchannel['name'] + #webchannel['networkid'] = webchannel['id'] + try: + webresult.append(webchannel) + webworkchannel.append(metadata.tables['updates.tvwebchannelnewvals']) + #insertresult = insert_data(webchannel, tables[5]) + except sqlalchemy.exc.IntegrityError as e: + pass + except Exception as e: + #print('Failed to insert webchannel information.') + pass + try: + seriesdata['seriesid'] = seriesdata['id'] + except: + pass + + #print('TV Show: ' + seriesdata['seriesname']) + + try: + if seriesdata['image']: + for key, value in seriesdata['image'].items(): + newkey = key + 'image' + seriesdata[newkey] = value + seriesdata.pop('image') + else: + seriesdata.pop('image') + except: + pass + + try: + if seriesdata['externals']: + for key, value in seriesdata['externals'].items(): + seriesdata[key] = value + seriesdata.pop('externals') + else: + seriesdata.pop('externals') + except KeyError as e: + pass + + try: + if seriesdata['rating']: + for key, value in seriesdata['rating'].items(): + seriesdata['rating'] = value + + if seriesdata['schedule']: + seriesdata['time'] = seriesdata['schedule']['time'] + seriesdata['days'] = ", ".join(seriesdata['schedule']['days']) + seriesdata.pop('schedule') + except KeyError as e: + pass + + try: + for key, value in seriesdata['_links']['self'].items(): + seriesdata[key] = value + seriesdata.pop('_links') + except KeyError as e: + pass + else: + seriesdata.pop('id') + + if seriesdata['genres']: + seriesdata['genres'] = ", ".join(seriesdata['genres']) + + # Finish transforming to match table columns + seriesdata['seriesname'] = seriesdata.pop('name') + seriesdata['tvrageid'] = seriesdata.pop('tvrage') + seriesdata['tvdbid'] = seriesdata.pop('thetvdb') + seriesdata['imdbid'] = seriesdata.pop('imdb') + seriesdata['serieslanguage'] = seriesdata.pop('language') + seriesdata['seriesstatus'] = seriesdata.pop('status') + seriesdata['airdays'] = seriesdata.pop('days') + seriesdata['airtime'] = seriesdata.pop('time') + seriesdata['showtype'] = seriesdata.pop('type') + seriesdata['overview'] = seriesdata.pop('summary') + seriesdata['showurl'] = seriesdata.pop('url') + seriesdata['apilink'] = seriesdata.pop('href') + seriesdata['officialsite'] = seriesdata.pop('officialSite') + + results = {'networkdata': networkresult, 'networktable': networkchannel, + 'webdata': webresult, 'webtable': webchannel, + 'seriesdata': seriesdata} + #results.append(tableresult) + #results.append(seriesdata) + return (results) + + + def __del__(self): + pass + diff --git a/cache.zip b/cache.zip new file mode 100644 index 0000000..3e0f2bc Binary files /dev/null and b/cache.zip differ diff --git a/cache/__init__.py b/cache/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cache/process.py b/cache/process.py new file mode 100644 index 0000000..08a3770 --- /dev/null +++ b/cache/process.py @@ -0,0 +1,89 @@ +import os +import progressbar +from pathlib import Path +from time import sleep +from tqdm import tqdm + +class Cacher(object): + + def __init__(self): + pass + + def process(self, options, jget, new_updates, ROOTDIR, URLS): + SERIESCACHE = str(ROOTDIR) + '/cache/series/' + EPISODECACHE = str(ROOTDIR) + '/cache/episodes/' + CASTCACHE = str(ROOTDIR) + '/cache/cast/' + CREWCACHE = str(ROOTDIR) + '/cache/crew/' + TEMPDIR = ROOTDIR + '/temp/' + available_updates = {} + updateList = [] + if options['refreshcache'] == '1': + print('Refreshing cache directory.') + print('Checking cache directory for missing files.') + for update in new_updates: + available_updates[update[0]] = update[1] + updateList = [str(key) for key in available_updates.keys()] + for series in tqdm(range(len(new_updates)), desc='Checking cache', unit='series'): + seriesid = new_updates[series][0] + cachefile = str(seriesid) + '.json' + seriescachefile = f'{SERIESCACHE}{cachefile}' + episodecachefile = f'{EPISODECACHE}{cachefile}' + castcachefile = f'{CASTCACHE}{cachefile}' + crewcachefile = f'{CREWCACHE}{cachefile}' + self.check_and_download(jget, URLS['showurl'], str(seriesid), new_updates[0], seriescachefile, 1500) + sleep(1) + self.check_and_download(jget, URLS['episodesurl'], str(seriesid), new_updates[0], episodecachefile, 1500) + sleep(1) + self.check_and_download(jget, URLS['casturl'], str(seriesid), new_updates[0], castcachefile, 500) + sleep(1) + self.check_and_download(jget, URLS['crewurl'], str(seriesid), new_updates[0], crewcachefile, 500) + sleep(1) + print('Refresh completed.') + + def check_and_download(self, jget, url, SERIESID, series_id, cachefile, size_limit): + try: + my_abs_path = Path(cachefile).resolve(strict=True) + file_stats = os.stat(cachefile) + if file_stats.st_size <= size_limit: + #print(f'File Size for {cachefile} in Bytes is {file_stats.st_size}. File too small. Attempting to download.') + jget.fetch_json(url.replace(SERIESID, str(series_id)), series_id, cachefile) + except FileNotFoundError: + #print(f'{cachefile} does not exist. Attempting to download.') + jget.fetch_json(url.replace(SERIESID, str(series_id)), series_id, cachefile) + except Exception as e: + print(f'Unable to download {cachefile}. {e}') + + def update_cache(self, jget, updateList, ROOTDIR, URLS, options, lprint): + if options['skipcache'] == '0': + SERIESCACHE = str(ROOTDIR) + '/cache/series/' + EPISODECACHE = str(ROOTDIR) + '/cache/episodes/' + CASTCACHE = str(ROOTDIR) + '/cache/cast/' + CREWCACHE = str(ROOTDIR) + '/cache/crew/' + TEMPDIR = ROOTDIR + '/temp/' + print('Now downloading updates.') + updaterange = len(updateList) + for seriesid in tqdm(range(updaterange), desc='Downloading updates', unit='seriesid'): + cachefile = f'{updateList[seriesid]}.json' + seriescachefile = SERIESCACHE + cachefile + episodecachefile = EPISODECACHE + cachefile + castcachefile = CASTCACHE + cachefile + crewcachefile = CREWCACHE + cachefile + self.download_json(jget, URLS['showurl'], updateList[seriesid], seriescachefile, lprint) + sleep(1) + self.download_json(jget, URLS['episodesurl'], updateList[seriesid], episodecachefile, lprint) + sleep(1) + self.download_json(jget, URLS['casturl'], updateList[seriesid], castcachefile, lprint) + sleep(1) + self.download_json(jget, URLS['crewurl'], updateList[seriesid], crewcachefile, lprint) + sleep(1) + + def download_json(self, jget, url, seriesid, cachefile, lprint): + try: + jget.fetch_json(url.replace('', str(seriesid)), seriesid, cachefile) + except Exception as e: + lprint.logprint('info', f'Unable to acquire {cachefile}. {e}') + + def __del__(self): + pass + + diff --git a/db/Tables.py b/db/Tables.py new file mode 100644 index 0000000..5cae3a3 --- /dev/null +++ b/db/Tables.py @@ -0,0 +1,401 @@ +import sqlalchemy +from sqlalchemy import create_engine, MetaData, Table, Column, Integer, String, Sequence, text, UniqueConstraint +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.schema import MetaData +from sqlalchemy.orm import sessionmaker +from sqlalchemy.types import * + +class Tables(): + def __init__(self): + global tvnetworknewvals + global tvwebchannelnewvals + global characterjsonnewvals + global seriesjsonnewvals + global episodejsonnewvals + global metadata + global engine + global session + global tvnetworknewvals + global tvwebchannelnewvals + global characterjsonnewvals + global seriesjsonnewvals + global episodejsonnewvals + global tvupdates + global newseriesdata + global actorlinks + global tablelist + + def __del__(self): + pass + + +class Createtables(): + + def __init__(self, session, metadata, engine): + Base = declarative_base(metadata = metadata) + self.tablelist = self.settables(metadata, engine) + try: + self.drop(session, self.tablelist) + except: + pass + self.create(Base, engine, metadata, session, self.tablelist) + session.commit() + #self.send(tablelist) + + def settables(self, metadata, engine): + tablelist = [] + newseriesdata = Table('seriesdata', metadata, + Column('id', BIGINT, Sequence('seriesdata_id_seq'), primary_key = True), + Column('seriesid',INTEGER, unique=True), + Column('tvdbid', String(25)), + Column('tvrageid', String(25)), + Column('imdbid', String(25)), + Column('serieslanguage', String(25)), + Column('genres', String(75)), + Column('seriesname', String(150)), + Column('seriesstatus', String(25)), + Column('premiered', String(25)), + Column('airdays', String(72)), + Column('airtime', String(25)), + Column('runtime', String(25)), + Column('rating', String(25)), + Column('showtype',TEXT), + Column('overview',TEXT), + Column('mediumimage',TEXT), + Column('originalimage',TEXT), + Column('weight',INTEGER), + Column('networkid', BIGINT), + Column('webchannel', BIGINT), + Column('officalsite',TEXT), + Column('showurl',TEXT), + Column('apilink',TEXT), + Column('previousepisode',TEXT), + Column('nextepisode',TEXT), + schema = "dbo") + tablelist.append(newseriesdata) + newactordata = Table('actordata', metadata, + Column('id', BIGINT, Sequence('actordata_id_seq'), primary_key = True), + Column('actorid', INTEGER, unique=True), + Column('actorname', VARCHAR(72)), + Column('countryname', VARCHAR(144)), + Column('countrytimezone', VARCHAR(72)), + Column('birthday', DATE), + Column('deathday', DATE), + Column('gender', VARCHAR(6)), + Column('mediumimage', TEXT), + Column('originalimage', TEXT), + Column('apilink', TEXT), + Column('url', TEXT), + Column('countrycode', VARCHAR(12)), + schema = "dbo") + tablelist.append(newactordata) + newepdata = Table('epdata', metadata, + Column('id', BIGINT, Sequence('epdata_id_seq'), primary_key = True), + Column('episodeid', INTEGER), + Column('seriesid', INTEGER), + Column('season', INTEGER), + Column('episodenumber', INTEGER), + Column('episodename', TEXT), + Column('airdate', VARCHAR(32)), + Column('airtime', VARCHAR(32)), + Column('episodesummary', TEXT), + Column('airstamp', VARCHAR(32)), + Column('runtime', VARCHAR(6)), + Column('eplink', TEXT), + Column('mediumimage', TEXT), + Column('url', TEXT), + Column('originalimage', TEXT), + schema = "dbo") + tablelist.append(newepdata) + newcastdata = Table('castdata', metadata, + Column('id', BIGINT, Sequence('castdata_id_seq'), primary_key = True), + Column('seriesid', INTEGER), + Column('characterid', INTEGER), + Column('actorid', INTEGER), + Column('charactername', INTEGER), + Column('mediumimage', TEXT), + Column('originalimage', TEXT), + Column('apilink', TEXT), + Column('tvmazeilink', TEXT), + schema = "dbo") + tablelist.append(newcastdata) + tvnetworks = Table('tvnetworkdata', metadata, + Column('id', Integer, primary_key=True), + Column('networkid', String(12)), + Column('networkname', String(255)), + Column('country', String(255)), + Column('countrycode', String(4)), + Column('countrytimezone', String(25)), + schema = "dbo") + tablelist.append(tvnetworks) + tvwebchannels = Table('tvwebchanneldata', metadata, + Column('id', Integer, primary_key=True), + Column('webchannelname', String(255)), + Column('webchannelcountrycode', String(4)), + Column('webchannelcountryname', String(255)), + Column('webchannelcountrytimezone', String(4)), + schema = "dbo") + tablelist.append(tvwebchannels) + countries = Table('countrydata', metadata, + Column('id', Integer, primary_key=True), + Column('name', String(255)), + Column('code', String(255)), + Column('timezone', String(255)), + UniqueConstraint('name', 'timezone', name='tz_name_uk'), + schema = "dbo") + tablelist.append(countries) + + return(tablelist) + + def create(self, Base, engine, metadata, session, tablelist): + print('Creating Data Tables') + Base.metadata.create_all(engine, tables=tablelist) + + def drop(self, session, tablelist): + for table in tablelist: + try: + #print('Dropping ' + str(table)) + table.drop() + except sqlalchemy.exc.ProgrammingError as se: + pass + + def __del__(self): + pass + + +class TempTables(): + + def __init__(self, session, metadata, engine, tablelist): + Base = declarative_base(metadata = metadata) + self.tablelist = self.settables(metadata, engine) + try: + self.drop(session, self.tablelist) + except: + pass + self.create(Base, engine, metadata, session, self.tablelist) + #session.commit() + + def settables(self, metadata, engine): + tablelist = [] + tvnetworknewvals = Table('tvnetworknewvals', metadata, + Column('id', BIGINT, Sequence('network_id_seq'), primary_key = True), + Column('name', String(255)), + Column('countryid', String(255)), + schema = "updates") + tablelist.append(tvnetworknewvals) + tvwebchannelnewvals = Table('tvwebchannelnewvals', metadata, + Column('id', BIGINT, Sequence('webchannel_id_seq'), primary_key = True), + Column('name', String(255)), + Column('countryid', String(4)), + schema = "updates") + tablelist.append(tvwebchannelnewvals) + tvupdates = Table('tvupdates', metadata, + Column('seriesid', BIGINT, primary_key = True), + Column('timestamp', VARCHAR(15)), + schema = "updates") + tablelist.append(tvupdates) + tvupdatedata = Table('tvupdatedata', metadata, + Column('seriesid', BIGINT, primary_key = True), + Column('timestamp', VARCHAR(15)), + schema = "updates") + tablelist.append(tvupdatedata) + newactorlinks = Table('newactorlinks', metadata, + Column('id', BIGINT, Sequence('tempepisodejson_id_seq'), primary_key = True), + Column('actorid', INTEGER), + Column('characterid', INTEGER), + schema = "updates") + tablelist.append(newactorlinks) + return(tablelist) + + def create(self, Base, engine, metadata, session, tablelist): + tablelist.remove(tablelist[6]) + print('Creating Temporary JSON Tables') + try: + Base.metadata.create_all(engine, tables=tablelist) + except sqlalchemy.exc.ProgrammingError as pe: + pass + + def drop(self, session, tablelist): + for table in tablelist: + if 'tvupdatedata' in table.description: + pass + else: + try: + #print('Dropping ' + str(table)) + table.drop() + except sqlalchemy.exc.ProgrammingError as se: + pass + #session.commit() + +class CreateTempTables(): + + global tvnetworknewvals + global tvwebchannelnewvals + global characterjsonnewvals + global seriesjsonnewvals + global episodejsonnewvals + global metadata + global engine + global session + + global tablelist + + def __init__(self, session, metadata, engine): + # Need to create schema if it does not exist + Base = declarative_base(metadata = metadata) + self.tablelist = self.settables(metadata, engine) + try: + self.drop(session, self.tablelist) + except: + pass + self.create(Base, engine, metadata, session, self.tablelist) + session.commit() + #self.send(tablelist) + + def settables(self, metadata, engine): + tablelist = [] + newseriesdata = Table('newseriesdata', metadata, + Column('id', Integer, primary_key=True), + Column('seriesid',INTEGER, unique=True), + Column('thetvdb', String(25)), + Column('tvrage', String(25)), + Column('imdb', String(25)), + Column('language', String(25)), + Column('genres', String(75)), + Column('name', String(150)), + Column('status', String(25)), + Column('premiered', String(25)), + Column('days', String(72)), + Column('time', String(25)), + Column('runtime', String(25)), + Column('rating', String(25)), + Column('type',TEXT), + Column('summary',TEXT), + Column('medium',TEXT), + Column('original',TEXT), + Column('weight',INTEGER), + Column('networkid', BIGINT), + Column('webchannelid', BIGINT), + Column('officalsite',TEXT), + Column('url',TEXT), + Column('serieslink',TEXT), + Column('previousepisode',TEXT), + Column('nextepisode',TEXT), + schema="updates", + extend_existing=True) + tablelist.append(newseriesdata) + newactordata = Table('newactordata', metadata, + Column('id', Integer, primary_key=True), + Column('actorid', INTEGER), + Column('actorname', VARCHAR(72)), + Column('name', VARCHAR(72)), + Column('countryname', VARCHAR(72)), + Column('countrytimezone', VARCHAR(72)), + Column('birthday', DATE), + Column('deathday', DATE), + Column('gender', VARCHAR(6)), + Column('mediumimage', TEXT), + Column('originalimage', TEXT), + Column('apilink', TEXT), + Column('url', TEXT), + Column('countrycode', VARCHAR(12)), + schema="updates", + extend_existing=True) + tablelist.append(newactordata) + NEWEPDATA_ID = Sequence('newepdata_id_seq', start=1) + newepdata = Table('newepdata', metadata, + Column('id', Integer, primary_key=True), + Column('episodeid', INTEGER), + Column('seriesid', INTEGER), + Column('season', INTEGER), + Column('episodenumber', INTEGER), + Column('episodename', TEXT), + Column('airdate', VARCHAR(32)), + Column('airtime', VARCHAR(32)), + Column('episodesummary', TEXT), + Column('airstamp', VARCHAR(32)), + Column('runtime', VARCHAR(6)), + Column('eplink', TEXT), + Column('mediumimage', TEXT), + Column('url', TEXT), + Column('originalimage', TEXT), + schema="updates", + extend_existing=True) + tablelist.append(newepdata) + newcastdata = Table('newcastdata', metadata, + Column('id', Integer, primary_key=True), + Column('seriesid', INTEGER), + Column('characterid', INTEGER), + Column('actorid', INTEGER), + Column('url', TEXT), + Column('charactername', VARCHAR(128)), + Column('mediumimage', TEXT), + Column('originalimage', TEXT), + Column('apilink', TEXT), + Column('tvmazeilink', TEXT), + Column('playingself', CHAR), + Column('voice', CHAR), + schema="updates", + extend_existing=True) + tablelist.append(newcastdata) + newcrewdata = Table('newcrewdata', metadata, + Column('id', Integer, primary_key=True), + Column('seriesid', INTEGER), + Column('crewid', INTEGER), + Column('type', VARCHAR(128)), + Column('url', TEXT), + Column('name', VARCHAR(256)), + Column('countryid', INTEGER), + Column('birthday', DATE), + Column('deathday', DATE), + Column('gender', VARCHAR(24)), + Column('medium', TEXT), + Column('original', TEXT), + Column('apilink', TEXT), + UniqueConstraint('seriesid', 'crewid', 'type', name='crew_unique'), + schema="updates", + extend_existing=True) + tablelist.append(newcrewdata) + tvnetworks = Table('tvnetworkdata', metadata, + Column('id', Integer, primary_key=True), + Column('name', String(255)), + Column('countryid', String(255)), + schema = "updates", + extend_existing=True) + tablelist.append(tvnetworks) + tvwebchannels = Table('tvwebchanneldata', metadata, + Column('id', Integer, primary_key=True), + Column('name', String(255)), + Column('countryid', String(255)), + schema = "updates", + extend_existing=True) + tablelist.append(tvwebchannels) + countries = Table('newcountrydata', metadata, + Column('id', Integer, primary_key=True), + Column('name', String(255)), + Column('code', String(255)), + Column('timezone', String(255)), + UniqueConstraint('name', 'timezone', name='tz_name_uk'), + schema = "updates", + extend_existing=True) + tablelist.append(countries) + tvupdates = Table('tvupdates', metadata, + Column('seriesid', BIGINT, primary_key = True), + Column('timestamp', VARCHAR(15)), + schema = "updates", + extend_existing=True) + tablelist.append(tvupdates) + return(tablelist) + + def create(self, Base, engine, metadata, session, tablelist): + print('Creating Temporary Data Tables') + Base.metadata.create_all(engine, tables=tablelist) + + def drop(self, session, tablelist): + print('Dropping temporary update tables.') + for table in tablelist: + try: + print('Dropping ' + str(table)) + table.drop() + except sqlalchemy.exc.ProgrammingError as se: + pass \ No newline at end of file diff --git a/db/__init__.py b/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/db/ddl/create_schema.sql b/db/ddl/create_schema.sql new file mode 100644 index 0000000..01b5c49 --- /dev/null +++ b/db/ddl/create_schema.sql @@ -0,0 +1,566 @@ +CREATE SCHEMA dbo; + +ALTER SCHEMA dbo OWNER TO postgres; + +CREATE SCHEMA updates; + +ALTER SCHEMA updates OWNER TO postgres; + +CREATE SEQUENCE dbo.actordata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE dbo.actordata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE dbo.castdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE dbo.castdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE dbo.countrydata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE dbo.countrydata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE dbo.crewdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE dbo.crewdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.epdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.epdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE dbo.epdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE dbo.epdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE dbo.seriesdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE dbo.seriesdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE dbo.tvnetworkdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE dbo.tvnetworkdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE dbo.tvwebchanneldata_id_seq + AS integer START WITH 1 + INCREMENT BY 1 + NO MINVALUE + MAXVALUE 2147483647 + CACHE 1; + +ALTER SEQUENCE dbo.tvwebchanneldata_id_seq OWNER TO postgres; + +CREATE SEQUENCE dbo.webchanneldata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE dbo.webchanneldata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.actordata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.actordata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.castdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.castdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.countrydata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.countrydata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.crewdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.crewdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.seriesdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.seriesdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.tvnetworkdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.tvnetworkdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.tvupdates_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.tvupdates_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.tvwebchanneldata_id_seq + AS integer START WITH 1 + INCREMENT BY 1 + NO MINVALUE + MAXVALUE 2147483647 + CACHE 1; + +ALTER SEQUENCE updates.tvwebchanneldata_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.webchanneldata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.webchanneldata_row_id_seq OWNER TO postgres; + +CREATE TABLE dbo.actordata ( + row_id bigint NOT NULL DEFAULT nextval('dbo.actordata_row_id_seq'::regclass), + actorid integer, + actorname character varying(72), + countryname character varying(144), + countrytimezone character varying(72), + birthday date, + deathday date, + gender character varying(6), + mediumimage text, + originalimage text, + apilink text, + url text, + countrycode character varying(12)); + +ALTER TABLE dbo.actordata OWNER TO postgres; + +ALTER SEQUENCE dbo.actordata_row_id_seq OWNED BY dbo.actordata.row_id; + +CREATE TABLE dbo.castdata ( + row_id bigint NOT NULL DEFAULT nextval('dbo.castdata_row_id_seq'::regclass), + seriesid integer, + characterid integer, + actorid integer, + charactername integer, + mediumimage text, + originalimage text, + apilink text, + tvmazeilink text); + +ALTER TABLE dbo.castdata OWNER TO postgres; + +ALTER SEQUENCE dbo.castdata_row_id_seq OWNED BY dbo.castdata.row_id; + +CREATE TABLE dbo.countrydata ( + row_id bigint NOT NULL DEFAULT nextval('dbo.countrydata_row_id_seq'::regclass), + country_name character varying(255), + country_code character varying(255), + country_timezone character varying(255)); + +ALTER TABLE dbo.countrydata OWNER TO postgres; + +ALTER SEQUENCE dbo.countrydata_row_id_seq OWNED BY dbo.countrydata.row_id; + +CREATE TABLE dbo.crewdata ( + row_id bigint NOT NULL DEFAULT nextval('dbo.crewdata_row_id_seq'::regclass), + crewid integer, + crew_type character varying(128), + name character varying(255), + _links_self_href text, + country_name text, + country_code text, + country_timezone text, + birthday date, + deathday date, + gender character varying(24), + image_medium text, + image_original text, + updated text, + url text); + +ALTER TABLE dbo.crewdata OWNER TO postgres; + +ALTER SEQUENCE dbo.crewdata_row_id_seq OWNED BY dbo.crewdata.row_id; + +CREATE TABLE dbo.epdata ( + row_id bigint NOT NULL DEFAULT nextval('dbo.epdata_row_id_seq'::regclass), + episodeid integer, + seriesid integer, + season integer, + episode_number integer, + episode_name text, + airdate character varying(32), + airtime character varying(32), + summary text, + airstamp character varying(32), + runtime character varying(6), + eplink text, + image_medium text, + url text, + image_original text, + episode_type text, + rating_average text, + _links_self_href text, + _links_show_href text, + _links_show_name text); + +ALTER TABLE dbo.epdata OWNER TO postgres; + +ALTER SEQUENCE dbo.epdata_row_id_seq OWNED BY dbo.epdata.row_id; + +CREATE TABLE dbo.seriesdata ( + row_id bigint NOT NULL DEFAULT nextval('dbo.seriesdata_row_id_seq'::regclass), + _links_previousepisode_href text, + _links_previousepisode_name text, + _links_nextepisode_href text, + _links_nextepisode_name text, + _links_self_href text, + averageruntime text, + dvdcountry text, + series_name text, + ended date, + externals_imdb text, + externals_thetvdb text, + externals_tvrage text, + genres text, + seriesid text, + image_medium text, + image_original text, + language_name text, + network_id text, + network_name text, + officialsite text, + network_officialsite text, + network_country_code text, + network_country_name text, + network_country_timezone text, + premiered text, + rating_average text, + runtime text, + schedule_days text, + schedule_time text, + status text, + summary text, + series_type text, + updated text, + url text, + webchannel_id integer, + webchannel_name text, + webchannel_country_name text, + webchannel_country_code text, + webchannel_country_timezone text, + webchannel_officialsite text, + weight text); + +ALTER TABLE dbo.seriesdata OWNER TO postgres; + +ALTER SEQUENCE dbo.seriesdata_row_id_seq OWNED BY dbo.seriesdata.row_id; + +CREATE TABLE dbo.tvnetworkdata ( + row_id bigint NOT NULL DEFAULT nextval('dbo.tvnetworkdata_row_id_seq'::regclass), + networkid character varying(12), + networkname character varying(255), + country character varying(255), + countrycode character varying(4), + countrytimezone character varying(25)); + +ALTER TABLE dbo.tvnetworkdata OWNER TO postgres; + +ALTER SEQUENCE dbo.tvnetworkdata_row_id_seq OWNED BY dbo.tvnetworkdata.row_id; + +CREATE TABLE dbo.tvwebchanneldata ( + id integer NOT NULL DEFAULT nextval('dbo.tvwebchanneldata_id_seq'::regclass), + webchannelname character varying(255), + webchannelcountrycode character varying(4), + webchannelcountryname character varying(255), + webchannelcountrytimezone character varying(4)); + +ALTER TABLE dbo.tvwebchanneldata OWNER TO postgres; + +ALTER SEQUENCE dbo.tvwebchanneldata_id_seq OWNED BY dbo.tvwebchanneldata.id; + +CREATE TABLE dbo.webchanneldata ( + row_id bigint NOT NULL DEFAULT nextval('dbo.webchanneldata_row_id_seq'::regclass), + webchannelname character varying(255), + webchannelcountrycode character varying(4), + webchannelcountryname character varying(255), + webchannelcountrytimezone character varying(4)); + +ALTER TABLE dbo.webchanneldata OWNER TO postgres; + +ALTER SEQUENCE dbo.webchanneldata_row_id_seq OWNED BY dbo.webchanneldata.row_id; + +CREATE TABLE updates.actordata ( + row_id bigint NOT NULL DEFAULT nextval('updates.actordata_row_id_seq'::regclass), + actorid integer, + actorname character varying(72), + countryname character varying(144), + countrytimezone character varying(72), + birthday date, + deathday date, + gender character varying(6), + mediumimage text, + originalimage text, + apilink text, + url text, + countrycode character varying(12)); + +ALTER TABLE updates.actordata OWNER TO postgres; + +ALTER SEQUENCE updates.actordata_row_id_seq OWNED BY updates.actordata.row_id; + +CREATE TABLE updates.castdata ( + row_id bigint NOT NULL DEFAULT nextval('updates.castdata_row_id_seq'::regclass), + seriesid integer, + characterid integer, + actorid integer, + charactername integer, + mediumimage text, + originalimage text, + apilink text, + tvmazeilink text); + +ALTER TABLE updates.castdata OWNER TO postgres; + +ALTER SEQUENCE updates.castdata_row_id_seq OWNED BY updates.castdata.row_id; + +CREATE TABLE updates.countrydata ( + row_id bigint NOT NULL DEFAULT nextval('updates.countrydata_row_id_seq'::regclass), + country_name character varying(255), + country_code character varying(255), + country_timezone character varying(255)); + +ALTER TABLE updates.countrydata OWNER TO postgres; + +ALTER SEQUENCE updates.countrydata_row_id_seq OWNED BY updates.countrydata.row_id; + +CREATE TABLE updates.crewdata ( + row_id bigint NOT NULL DEFAULT nextval('updates.crewdata_row_id_seq'::regclass), + crewid integer, + crew_type character varying(128), + name character varying(255), + _links_self_href text, + country_name text, + country_code text, + country_timezone text, + birthday date, + deathday date, + gender character varying(24), + image_medium text, + image_original text, + updated text, + url text); + +ALTER TABLE updates.crewdata OWNER TO postgres; + +ALTER SEQUENCE updates.crewdata_row_id_seq OWNED BY updates.crewdata.row_id; + +CREATE TABLE updates.epdata ( + row_id bigint NOT NULL DEFAULT nextval('updates.epdata_row_id_seq'::regclass), + episodeid integer, + seriesid integer, + season integer, + episode_number integer, + episode_name text, + airdate character varying(32), + airtime character varying(32), + summary text, + airstamp character varying(32), + runtime character varying(6), + eplink text, + image_medium text, + url text, + image_original text, + episode_type text, + rating_average text, + _links_self_href text, + _links_show_href text, + _links_show_name text); + +ALTER TABLE updates.epdata OWNER TO postgres; + +ALTER SEQUENCE updates.epdata_row_id_seq OWNED BY updates.epdata.row_id; + +CREATE TABLE updates.seriesdata ( + row_id bigint NOT NULL DEFAULT nextval('updates.seriesdata_row_id_seq'::regclass), + _links_previousepisode_href text, + _links_previousepisode_name text, + _links_nextepisode_href text, + _links_nextepisode_name text, + _links_self_href text, + averageruntime text, + dvdcountry text, + series_name text, + ended date, + externals_imdb text, + externals_thetvdb text, + externals_tvrage text, + genres text, + seriesid text, + image_medium text, + image_original text, + language_name text, + network_id text, + network_name text, + officialsite text, + network_officialsite text, + network_country_code text, + network_country_name text, + network_country_timezone text, + premiered text, + rating_average text, + runtime text, + schedule_days text, + schedule_time text, + status text, + summary text, + series_type text, + updated text, + url text, + webchannel_id integer, + webchannel_name text, + webchannel_country_name text, + webchannel_country_code text, + webchannel_country_timezone text, + webchannel_officialsite text, + weight text); + +ALTER TABLE updates.seriesdata OWNER TO postgres; + +ALTER SEQUENCE updates.seriesdata_row_id_seq OWNED BY updates.seriesdata.row_id; + +CREATE TABLE updates.tvnetworkdata ( + row_id bigint NOT NULL DEFAULT nextval('updates.tvnetworkdata_row_id_seq'::regclass), + networkid character varying(12), + networkname character varying(255), + country character varying(255), + countrycode character varying(4), + countrytimezone character varying(25)); + +ALTER TABLE updates.tvnetworkdata OWNER TO postgres; + +ALTER SEQUENCE updates.tvnetworkdata_row_id_seq OWNED BY updates.tvnetworkdata.row_id; + +CREATE TABLE updates.tvupdates ( + row_id bigint NOT NULL DEFAULT nextval('updates.tvupdates_row_id_seq'::regclass), + seriesid integer, + "timestamp" character varying(15)); + +ALTER TABLE updates.tvupdates OWNER TO postgres; + +ALTER SEQUENCE updates.tvupdates_row_id_seq OWNED BY updates.tvupdates.row_id; + +CREATE TABLE updates.tvwebchanneldata ( + row_id integer NOT NULL DEFAULT nextval('updates.tvwebchanneldata_id_seq'::regclass), + name character varying(255), + countryid character varying(255)); + +ALTER TABLE updates.tvwebchanneldata OWNER TO postgres; + +ALTER SEQUENCE updates.tvwebchanneldata_id_seq OWNED BY updates.tvwebchanneldata.row_id; + +CREATE TABLE updates.webchanneldata ( + row_id bigint NOT NULL DEFAULT nextval('updates.webchanneldata_row_id_seq'::regclass), + webchannel_name character varying(255), + webchannel_country_code character varying(4), + webchannel_country_name character varying(255), + webchannelcountry_timezone character varying(4)); + +ALTER TABLE updates.webchanneldata OWNER TO postgres; + +ALTER SEQUENCE updates.webchanneldata_row_id_seq OWNED BY updates.webchanneldata.row_id; + +ALTER TABLE dbo.actordata ADD CONSTRAINT actordata_pkey PRIMARY KEY (row_id); + +ALTER TABLE dbo.castdata ADD CONSTRAINT castdata_pkey PRIMARY KEY (row_id); + +ALTER TABLE dbo.countrydata ADD CONSTRAINT countrydata_pkey PRIMARY KEY (row_id); + +ALTER TABLE dbo.epdata ADD CONSTRAINT epdata_pkey PRIMARY KEY (row_id); + +ALTER TABLE dbo.seriesdata ADD CONSTRAINT seriesdata_pkey PRIMARY KEY (row_id); + +ALTER TABLE dbo.tvnetworkdata ADD CONSTRAINT tvnetworkdata_pkey PRIMARY KEY (row_id); + +ALTER TABLE dbo.tvwebchanneldata ADD CONSTRAINT tvwebchanneldata_pkey PRIMARY KEY (id); + +ALTER TABLE dbo.webchanneldata ADD CONSTRAINT webchanneldata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.actordata ADD CONSTRAINT actordata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.actordata ADD CONSTRAINT actorid UNIQUE (actorid); + +ALTER TABLE updates.castdata ADD CONSTRAINT castdata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.castdata ADD CONSTRAINT series_actor_character_index UNIQUE (seriesid, characterid, actorid); + +ALTER TABLE updates.countrydata ADD CONSTRAINT country_code_idx UNIQUE (country_code); + +ALTER TABLE updates.countrydata ADD CONSTRAINT countrydata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.crewdata ADD CONSTRAINT crewdata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.crewdata ADD CONSTRAINT crewid_crewid_idx UNIQUE (crewid); + +ALTER TABLE updates.epdata ADD CONSTRAINT epdata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.epdata ADD CONSTRAINT episodeie UNIQUE (episodeid); + +ALTER TABLE updates.seriesdata ADD CONSTRAINT seriesdata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.seriesdata ADD CONSTRAINT seriesid UNIQUE (seriesid); + +ALTER TABLE updates.tvnetworkdata ADD CONSTRAINT networkid UNIQUE (networkid); + +ALTER TABLE updates.tvnetworkdata ADD CONSTRAINT tvnetworkdata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.tvupdates ADD CONSTRAINT tvupdates_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.tvwebchanneldata ADD CONSTRAINT tvwebchanneldata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.webchanneldata ADD CONSTRAINT webchanneldata_pkey PRIMARY KEY (row_id); + +ALTER SCHEMA updates OWNER TO postgres; diff --git a/db/ddl/recreate_schema.sql b/db/ddl/recreate_schema.sql new file mode 100644 index 0000000..9274220 --- /dev/null +++ b/db/ddl/recreate_schema.sql @@ -0,0 +1,570 @@ +DROP SCHEMA dbo CASCADE; + +DROP SCHEMA updates CASCADE; + +CREATE SCHEMA dbo; + +ALTER SCHEMA dbo OWNER TO postgres; + +CREATE SCHEMA updates; + +ALTER SCHEMA updates OWNER TO postgres; + +CREATE SEQUENCE dbo.actordata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE dbo.actordata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE dbo.castdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE dbo.castdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE dbo.countrydata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE dbo.countrydata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE dbo.crewdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE dbo.crewdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.epdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.epdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE dbo.epdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE dbo.epdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE dbo.seriesdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE dbo.seriesdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE dbo.tvnetworkdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE dbo.tvnetworkdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE dbo.tvwebchanneldata_id_seq + AS integer START WITH 1 + INCREMENT BY 1 + NO MINVALUE + MAXVALUE 2147483647 + CACHE 1; + +ALTER SEQUENCE dbo.tvwebchanneldata_id_seq OWNER TO postgres; + +CREATE SEQUENCE dbo.webchanneldata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE dbo.webchanneldata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.actordata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.actordata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.castdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.castdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.countrydata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.countrydata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.crewdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.crewdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.seriesdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.seriesdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.tvnetworkdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.tvnetworkdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.tvupdates_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.tvupdates_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.tvwebchanneldata_id_seq + AS integer START WITH 1 + INCREMENT BY 1 + NO MINVALUE + MAXVALUE 2147483647 + CACHE 1; + +ALTER SEQUENCE updates.tvwebchanneldata_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.webchanneldata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.webchanneldata_row_id_seq OWNER TO postgres; + +CREATE TABLE dbo.actordata ( + row_id bigint NOT NULL DEFAULT nextval('dbo.actordata_row_id_seq'::regclass), + actorid integer, + actorname character varying(72), + countryname character varying(144), + countrytimezone character varying(72), + birthday date, + deathday date, + gender character varying(6), + mediumimage text, + originalimage text, + apilink text, + url text, + countrycode character varying(12)); + +ALTER TABLE dbo.actordata OWNER TO postgres; + +ALTER SEQUENCE dbo.actordata_row_id_seq OWNED BY dbo.actordata.row_id; + +CREATE TABLE dbo.castdata ( + row_id bigint NOT NULL DEFAULT nextval('dbo.castdata_row_id_seq'::regclass), + seriesid integer, + characterid integer, + actorid integer, + charactername integer, + mediumimage text, + originalimage text, + apilink text, + tvmazeilink text); + +ALTER TABLE dbo.castdata OWNER TO postgres; + +ALTER SEQUENCE dbo.castdata_row_id_seq OWNED BY dbo.castdata.row_id; + +CREATE TABLE dbo.countrydata ( + row_id bigint NOT NULL DEFAULT nextval('dbo.countrydata_row_id_seq'::regclass), + country_name character varying(255), + country_code character varying(255), + country_timezone character varying(255)); + +ALTER TABLE dbo.countrydata OWNER TO postgres; + +ALTER SEQUENCE dbo.countrydata_row_id_seq OWNED BY dbo.countrydata.row_id; + +CREATE TABLE dbo.crewdata ( + row_id bigint NOT NULL DEFAULT nextval('dbo.crewdata_row_id_seq'::regclass), + crewid integer, + crew_type character varying(128), + name character varying(255), + _links_self_href text, + country_name text, + country_code text, + country_timezone text, + birthday date, + deathday date, + gender character varying(24), + image_medium text, + image_original text, + updated text, + url text); + +ALTER TABLE dbo.crewdata OWNER TO postgres; + +ALTER SEQUENCE dbo.crewdata_row_id_seq OWNED BY dbo.crewdata.row_id; + +CREATE TABLE dbo.epdata ( + row_id bigint NOT NULL DEFAULT nextval('dbo.epdata_row_id_seq'::regclass), + episodeid integer, + seriesid integer, + season integer, + episode_number integer, + episode_name text, + airdate character varying(32), + airtime character varying(32), + summary text, + airstamp character varying(32), + runtime character varying(6), + eplink text, + image_medium text, + url text, + image_original text, + episode_type text, + rating_average text, + _links_self_href text, + _links_show_href text, + _links_show_name text); + +ALTER TABLE dbo.epdata OWNER TO postgres; + +ALTER SEQUENCE dbo.epdata_row_id_seq OWNED BY dbo.epdata.row_id; + +CREATE TABLE dbo.seriesdata ( + row_id bigint NOT NULL DEFAULT nextval('dbo.seriesdata_row_id_seq'::regclass), + _links_previousepisode_href text, + _links_previousepisode_name text, + _links_nextepisode_href text, + _links_nextepisode_name text, + _links_self_href text, + averageruntime text, + dvdcountry text, + series_name text, + ended date, + externals_imdb text, + externals_thetvdb text, + externals_tvrage text, + genres text, + seriesid text, + image_medium text, + image_original text, + language_name text, + network_id text, + network_name text, + officialsite text, + network_officialsite text, + network_country_code text, + network_country_name text, + network_country_timezone text, + premiered text, + rating_average text, + runtime text, + schedule_days text, + schedule_time text, + status text, + summary text, + series_type text, + updated text, + url text, + webchannel_id integer, + webchannel_name text, + webchannel_country_name text, + webchannel_country_code text, + webchannel_country_timezone text, + webchannel_officialsite text, + weight text); + +ALTER TABLE dbo.seriesdata OWNER TO postgres; + +ALTER SEQUENCE dbo.seriesdata_row_id_seq OWNED BY dbo.seriesdata.row_id; + +CREATE TABLE dbo.tvnetworkdata ( + row_id bigint NOT NULL DEFAULT nextval('dbo.tvnetworkdata_row_id_seq'::regclass), + networkid character varying(12), + networkname character varying(255), + country character varying(255), + countrycode character varying(4), + countrytimezone character varying(25)); + +ALTER TABLE dbo.tvnetworkdata OWNER TO postgres; + +ALTER SEQUENCE dbo.tvnetworkdata_row_id_seq OWNED BY dbo.tvnetworkdata.row_id; + +CREATE TABLE dbo.tvwebchanneldata ( + id integer NOT NULL DEFAULT nextval('dbo.tvwebchanneldata_id_seq'::regclass), + webchannelname character varying(255), + webchannelcountrycode character varying(4), + webchannelcountryname character varying(255), + webchannelcountrytimezone character varying(4)); + +ALTER TABLE dbo.tvwebchanneldata OWNER TO postgres; + +ALTER SEQUENCE dbo.tvwebchanneldata_id_seq OWNED BY dbo.tvwebchanneldata.id; + +CREATE TABLE dbo.webchanneldata ( + row_id bigint NOT NULL DEFAULT nextval('dbo.webchanneldata_row_id_seq'::regclass), + webchannelname character varying(255), + webchannelcountrycode character varying(4), + webchannelcountryname character varying(255), + webchannelcountrytimezone character varying(4)); + +ALTER TABLE dbo.webchanneldata OWNER TO postgres; + +ALTER SEQUENCE dbo.webchanneldata_row_id_seq OWNED BY dbo.webchanneldata.row_id; + +CREATE TABLE updates.actordata ( + row_id bigint NOT NULL DEFAULT nextval('updates.actordata_row_id_seq'::regclass), + actorid integer, + actorname character varying(72), + countryname character varying(144), + countrytimezone character varying(72), + birthday date, + deathday date, + gender character varying(6), + mediumimage text, + originalimage text, + apilink text, + url text, + countrycode character varying(12)); + +ALTER TABLE updates.actordata OWNER TO postgres; + +ALTER SEQUENCE updates.actordata_row_id_seq OWNED BY updates.actordata.row_id; + +CREATE TABLE updates.castdata ( + row_id bigint NOT NULL DEFAULT nextval('updates.castdata_row_id_seq'::regclass), + seriesid integer, + characterid integer, + actorid integer, + charactername integer, + mediumimage text, + originalimage text, + apilink text, + tvmazeilink text); + +ALTER TABLE updates.castdata OWNER TO postgres; + +ALTER SEQUENCE updates.castdata_row_id_seq OWNED BY updates.castdata.row_id; + +CREATE TABLE updates.countrydata ( + row_id bigint NOT NULL DEFAULT nextval('updates.countrydata_row_id_seq'::regclass), + country_name character varying(255), + country_code character varying(255), + country_timezone character varying(255)); + +ALTER TABLE updates.countrydata OWNER TO postgres; + +ALTER SEQUENCE updates.countrydata_row_id_seq OWNED BY updates.countrydata.row_id; + +CREATE TABLE updates.crewdata ( + row_id bigint NOT NULL DEFAULT nextval('updates.crewdata_row_id_seq'::regclass), + crewid integer, + crew_type character varying(128), + name character varying(255), + _links_self_href text, + country_name text, + country_code text, + country_timezone text, + birthday date, + deathday date, + gender character varying(24), + image_medium text, + image_original text, + updated text, + url text); + +ALTER TABLE updates.crewdata OWNER TO postgres; + +ALTER SEQUENCE updates.crewdata_row_id_seq OWNED BY updates.crewdata.row_id; + +CREATE TABLE updates.epdata ( + row_id bigint NOT NULL DEFAULT nextval('updates.epdata_row_id_seq'::regclass), + episodeid integer, + seriesid integer, + season integer, + episode_number integer, + episode_name text, + airdate character varying(32), + airtime character varying(32), + summary text, + airstamp character varying(32), + runtime character varying(6), + eplink text, + image_medium text, + url text, + image_original text, + episode_type text, + rating_average text, + _links_self_href text, + _links_show_href text, + _links_show_name text); + +ALTER TABLE updates.epdata OWNER TO postgres; + +ALTER SEQUENCE updates.epdata_row_id_seq OWNED BY updates.epdata.row_id; + +CREATE TABLE updates.seriesdata ( + row_id bigint NOT NULL DEFAULT nextval('updates.seriesdata_row_id_seq'::regclass), + _links_previousepisode_href text, + _links_previousepisode_name text, + _links_nextepisode_href text, + _links_nextepisode_name text, + _links_self_href text, + averageruntime text, + dvdcountry text, + series_name text, + ended date, + externals_imdb text, + externals_thetvdb text, + externals_tvrage text, + genres text, + seriesid text, + image_medium text, + image_original text, + language_name text, + network_id text, + network_name text, + officialsite text, + network_officialsite text, + network_country_code text, + network_country_name text, + network_country_timezone text, + premiered text, + rating_average text, + runtime text, + schedule_days text, + schedule_time text, + status text, + summary text, + series_type text, + updated text, + url text, + webchannel_id integer, + webchannel_name text, + webchannel_country_name text, + webchannel_country_code text, + webchannel_country_timezone text, + webchannel_officialsite text, + weight text); + +ALTER TABLE updates.seriesdata OWNER TO postgres; + +ALTER SEQUENCE updates.seriesdata_row_id_seq OWNED BY updates.seriesdata.row_id; + +CREATE TABLE updates.tvnetworkdata ( + row_id bigint NOT NULL DEFAULT nextval('updates.tvnetworkdata_row_id_seq'::regclass), + networkid character varying(12), + networkname character varying(255), + country character varying(255), + countrycode character varying(4), + countrytimezone character varying(25)); + +ALTER TABLE updates.tvnetworkdata OWNER TO postgres; + +ALTER SEQUENCE updates.tvnetworkdata_row_id_seq OWNED BY updates.tvnetworkdata.row_id; + +CREATE TABLE updates.tvupdates ( + row_id bigint NOT NULL DEFAULT nextval('updates.tvupdates_row_id_seq'::regclass), + seriesid integer, + "timestamp" character varying(15)); + +ALTER TABLE updates.tvupdates OWNER TO postgres; + +ALTER SEQUENCE updates.tvupdates_row_id_seq OWNED BY updates.tvupdates.row_id; + +CREATE TABLE updates.tvwebchanneldata ( + row_id integer NOT NULL DEFAULT nextval('updates.tvwebchanneldata_id_seq'::regclass), + name character varying(255), + countryid character varying(255)); + +ALTER TABLE updates.tvwebchanneldata OWNER TO postgres; + +ALTER SEQUENCE updates.tvwebchanneldata_id_seq OWNED BY updates.tvwebchanneldata.row_id; + +CREATE TABLE updates.webchanneldata ( + row_id bigint NOT NULL DEFAULT nextval('updates.webchanneldata_row_id_seq'::regclass), + webchannel_name character varying(255), + webchannel_country_code character varying(4), + webchannel_country_name character varying(255), + webchannelcountry_timezone character varying(4)); + +ALTER TABLE updates.webchanneldata OWNER TO postgres; + +ALTER SEQUENCE updates.webchanneldata_row_id_seq OWNED BY updates.webchanneldata.row_id; + +ALTER TABLE dbo.actordata ADD CONSTRAINT actordata_pkey PRIMARY KEY (row_id); + +ALTER TABLE dbo.castdata ADD CONSTRAINT castdata_pkey PRIMARY KEY (row_id); + +ALTER TABLE dbo.countrydata ADD CONSTRAINT countrydata_pkey PRIMARY KEY (row_id); + +ALTER TABLE dbo.epdata ADD CONSTRAINT epdata_pkey PRIMARY KEY (row_id); + +ALTER TABLE dbo.seriesdata ADD CONSTRAINT seriesdata_pkey PRIMARY KEY (row_id); + +ALTER TABLE dbo.tvnetworkdata ADD CONSTRAINT tvnetworkdata_pkey PRIMARY KEY (row_id); + +ALTER TABLE dbo.tvwebchanneldata ADD CONSTRAINT tvwebchanneldata_pkey PRIMARY KEY (id); + +ALTER TABLE dbo.webchanneldata ADD CONSTRAINT webchanneldata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.actordata ADD CONSTRAINT actordata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.actordata ADD CONSTRAINT actorid UNIQUE (actorid); + +ALTER TABLE updates.castdata ADD CONSTRAINT castdata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.castdata ADD CONSTRAINT series_actor_character_index UNIQUE (seriesid, characterid, actorid); + +ALTER TABLE updates.countrydata ADD CONSTRAINT country_code_idx UNIQUE (country_code); + +ALTER TABLE updates.countrydata ADD CONSTRAINT countrydata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.crewdata ADD CONSTRAINT crewdata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.crewdata ADD CONSTRAINT crewid_crewid_idx UNIQUE (crewid); + +ALTER TABLE updates.epdata ADD CONSTRAINT epdata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.epdata ADD CONSTRAINT episodeie UNIQUE (episodeid); + +ALTER TABLE updates.seriesdata ADD CONSTRAINT seriesdata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.seriesdata ADD CONSTRAINT seriesid UNIQUE (seriesid); + +ALTER TABLE updates.tvnetworkdata ADD CONSTRAINT networkid UNIQUE (networkid); + +ALTER TABLE updates.tvnetworkdata ADD CONSTRAINT tvnetworkdata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.tvupdates ADD CONSTRAINT tvupdates_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.tvwebchanneldata ADD CONSTRAINT tvwebchanneldata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.webchanneldata ADD CONSTRAINT webchanneldata_pkey PRIMARY KEY (row_id); + +ALTER SCHEMA updates OWNER TO postgres; diff --git a/db/ddl/recreate_updates_schema.sql b/db/ddl/recreate_updates_schema.sql new file mode 100644 index 0000000..00573f7 --- /dev/null +++ b/db/ddl/recreate_updates_schema.sql @@ -0,0 +1,299 @@ +DROP SCHEMA updates CASCADE; + +CREATE SCHEMA updates; + +ALTER SCHEMA updates OWNER TO postgres; + +CREATE SEQUENCE updates.actordata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.actordata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.castdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.castdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.countrydata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.countrydata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.crewdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.crewdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.epdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.epdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.seriesdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.seriesdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.tvnetworkdata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.tvnetworkdata_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.tvupdates_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.tvupdates_row_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.tvwebchanneldata_id_seq + AS integer START WITH 1 + INCREMENT BY 1 + NO MINVALUE + MAXVALUE 2147483647 + CACHE 1; + +ALTER SEQUENCE updates.tvwebchanneldata_id_seq OWNER TO postgres; + +CREATE SEQUENCE updates.webchanneldata_row_id_seq START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE updates.webchanneldata_row_id_seq OWNER TO postgres; + +CREATE TABLE updates.actordata ( + row_id bigint NOT NULL DEFAULT nextval('updates.actordata_row_id_seq'::regclass), + actorid integer, + actorname character varying(72), + countryname character varying(144), + countrytimezone character varying(72), + birthday date, + deathday date, + gender character varying(6), + mediumimage text, + originalimage text, + apilink text, + url text, + countrycode character varying(12)); + +ALTER TABLE updates.actordata OWNER TO postgres; + +ALTER SEQUENCE updates.actordata_row_id_seq OWNED BY updates.actordata.row_id; + +CREATE TABLE updates.castdata ( + row_id bigint NOT NULL DEFAULT nextval('updates.castdata_row_id_seq'::regclass), + seriesid integer, + characterid integer, + actorid integer, + charactername integer, + mediumimage text, + originalimage text, + apilink text, + tvmazeilink text); + +ALTER TABLE updates.castdata OWNER TO postgres; + +ALTER SEQUENCE updates.castdata_row_id_seq OWNED BY updates.castdata.row_id; + +CREATE TABLE updates.countrydata ( + row_id bigint NOT NULL DEFAULT nextval('updates.countrydata_row_id_seq'::regclass), + country_name character varying(255), + country_code character varying(255), + country_timezone character varying(255)); + +ALTER TABLE updates.countrydata OWNER TO postgres; + +ALTER SEQUENCE updates.countrydata_row_id_seq OWNED BY updates.countrydata.row_id; + +CREATE TABLE updates.crewdata ( + row_id bigint NOT NULL DEFAULT nextval('updates.crewdata_row_id_seq'::regclass), + crewid integer, + crew_type character varying(128), + name character varying(255), + _links_self_href text, + country_name text, + country_code text, + country_timezone text, + birthday date, + deathday date, + gender character varying(24), + image_medium text, + image_original text, + updated text, + url text); +ALTER TABLE updates.crewdata OWNER TO postgres; + +ALTER SEQUENCE updates.crewdata_row_id_seq OWNED BY updates.crewdata.row_id; + +CREATE TABLE updates.epdata ( + row_id bigint NOT NULL DEFAULT nextval('updates.epdata_row_id_seq'::regclass), + episodeid integer, + seriesid integer, + season integer, + episode_number integer, + episode_name text, + airdate character varying(32), + airtime character varying(32), + summary text, + airstamp character varying(32), + runtime character varying(6), + eplink text, + image_medium text, + url text, + image_original text, + episode_type text, + rating_average text, + _links_self_href text, + _links_show_href text, + _links_show_name text); + +ALTER TABLE updates.epdata OWNER TO postgres; + +ALTER SEQUENCE updates.epdata_row_id_seq OWNED BY updates.epdata.row_id; + +CREATE TABLE updates.seriesdata ( + row_id bigint NOT NULL DEFAULT nextval('updates.seriesdata_row_id_seq'::regclass), + _links_previousepisode_href text, + _links_previousepisode_name text, + _links_nextepisode_href text, + _links_nextepisode_name text, + _links_self_href text, + averageruntime text, + dvdcountry text, + series_name text, + ended date, + externals_imdb text, + externals_thetvdb text, + externals_tvrage text, + genres text, + seriesid text, + image_medium text, + image_original text, + language_name text, + network_id text, + network_name text, + officialsite text, + network_officialsite text, + network_country_code text, + network_country_name text, + network_country_timezone text, + premiered text, + rating_average text, + runtime text, + schedule_days text, + schedule_time text, + status text, + summary text, + series_type text, + updated text, + url text, + webchannel_id integer, + webchannel_name text, + webchannel_country_name text, + webchannel_country_code text, + webchannel_country_timezone text, + webchannel_officialsite text, + weight text); + +ALTER TABLE updates.seriesdata OWNER TO postgres; + +ALTER SEQUENCE updates.seriesdata_row_id_seq OWNED BY updates.seriesdata.row_id; + +CREATE TABLE updates.tvnetworkdata ( + row_id bigint NOT NULL DEFAULT nextval('updates.tvnetworkdata_row_id_seq'::regclass), + networkid character varying(12), + networkname character varying(255), + country character varying(255), + countrycode character varying(4), + countrytimezone character varying(25)); + +ALTER TABLE updates.tvnetworkdata OWNER TO postgres; + +ALTER SEQUENCE updates.tvnetworkdata_row_id_seq OWNED BY updates.tvnetworkdata.row_id; + +CREATE TABLE updates.tvupdates ( + row_id bigint NOT NULL DEFAULT nextval('updates.tvupdates_row_id_seq'::regclass), + seriesid integer, + "timestamp" character varying(15)); + +ALTER TABLE updates.tvupdates OWNER TO postgres; + +ALTER SEQUENCE updates.tvupdates_row_id_seq OWNED BY updates.tvupdates.row_id; + +CREATE TABLE updates.tvwebchanneldata ( + row_id integer NOT NULL DEFAULT nextval('updates.tvwebchanneldata_id_seq'::regclass), + name character varying(255), + countryid character varying(255)); + +ALTER TABLE updates.tvwebchanneldata OWNER TO postgres; + +ALTER SEQUENCE updates.tvwebchanneldata_id_seq OWNED BY updates.tvwebchanneldata.row_id; + +CREATE TABLE updates.webchanneldata ( + row_id bigint NOT NULL DEFAULT nextval('updates.webchanneldata_row_id_seq'::regclass), + webchannel_name character varying(255), + webchannel_country_code character varying(4), + webchannel_country_name character varying(255), + webchannelcountry_timezone character varying(4)); + +ALTER TABLE updates.webchanneldata OWNER TO postgres; + +ALTER SEQUENCE updates.webchanneldata_row_id_seq OWNED BY updates.webchanneldata.row_id; + +ALTER TABLE updates.actordata ADD CONSTRAINT actordata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.actordata ADD CONSTRAINT actorid UNIQUE (actorid); + +ALTER TABLE updates.castdata ADD CONSTRAINT castdata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.castdata ADD CONSTRAINT series_actor_character_index UNIQUE (seriesid, characterid, actorid); + +ALTER TABLE updates.countrydata ADD CONSTRAINT country_code_idx UNIQUE (country_code); + +ALTER TABLE updates.countrydata ADD CONSTRAINT countrydata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.crewdata ADD CONSTRAINT crewdata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.crewdata ADD CONSTRAINT seriesid_crewid_idx UNIQUE (crewid); + +ALTER TABLE updates.epdata ADD CONSTRAINT epdata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.epdata ADD CONSTRAINT episodeie UNIQUE (episodeid); + +ALTER TABLE updates.seriesdata ADD CONSTRAINT seriesdata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.seriesdata ADD CONSTRAINT seriesid UNIQUE (seriesid); + +ALTER TABLE updates.tvnetworkdata ADD CONSTRAINT networkid UNIQUE (networkid); + +ALTER TABLE updates.tvnetworkdata ADD CONSTRAINT tvnetworkdata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.tvupdates ADD CONSTRAINT tvupdates_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.tvwebchanneldata ADD CONSTRAINT tvwebchanneldata_pkey PRIMARY KEY (row_id); + +ALTER TABLE updates.webchanneldata ADD CONSTRAINT webchanneldata_pkey PRIMARY KEY (row_id); diff --git a/db/functions.py b/db/functions.py new file mode 100644 index 0000000..80fe9f4 --- /dev/null +++ b/db/functions.py @@ -0,0 +1,605 @@ +__author__ = "Wendell Jones" + +import os +import sqlalchemy +from sqlalchemy import text +from sqlalchemy import MetaData +from sqlalchemy.ext.automap import automap_base +from sqlalchemy.orm import Session +from db.Tables import Createtables, CreateTempTables, Tables as dbTables +from db.mssql.Merger import Charactermerge as MSCharactermerge +from db.mssql.Merger import Jsonmerge as MSJsonmerge +from db.mssql.Merger import SQLGenerate as MSSQLGenerate +from db.mssql.Merger import Episodemerge as MSEpisodemerge +from db.pgsql.Merger import Charactermerge as PGCharactermerge +from db.pgsql.Merger import Updatemerge as PGUpdatemerge +from db.mssql.Merger import Jsonmerge as PGJsonmerge +from db.pgsql.Merger import SQLGenerate as PGSQLGenerate +from db.pgsql.Merger import Episodemerge as PGEpisodemerge +import api.tvmaze.episodes +import api.tvmaze.cast +import api.tvmaze.series +from pymongo import MongoClient +from tqdm import tqdm + +Createtables = Createtables +CreateTempTables = CreateTempTables +dbTables = dbTables + + +class settype(object): + + def __init__(self, dbtype, apitype, options): + self.dbclass = {} + self.dbengine = {} + self.apiengine = {} + self.load(dbtype) + self.dbcreate(dbtype, options) + self.setapi(apitype) + self.get_data() + + def load(self, dbtype): + dbtype = dbtype.lower() + if dbtype in ["pgsql", "mysql"]: + Updatemerge = PGUpdatemerge + Jsonmerge = PGJsonmerge + Episodemerge = PGEpisodemerge + Charactermerge = PGCharactermerge + merger = PGSQLGenerate + elif dbtype == "mssql": + Jsonmerge = MSJsonmerge + Charactermerge = MSCharactermerge + merger = MSSQLGenerate + Episodemerge = MSEpisodemerge + + self.dbclass["jmerge"] = Jsonmerge() + self.dbclass["cmerge"] = Charactermerge() + self.dbclass["Createtables"] = Createtables + self.dbclass["CreateTemptables"] = CreateTempTables + self.dbclass["Tables"] = Createtables + self.dbclass["merger"] = merger() + + if dbtype == "pgsql": + self.dbclass["Updatemerge"] = Updatemerge + self.dbclass["Episodemerge"] = Episodemerge + elif dbtype == "mssql": + self.dbclass["Charactermerge"] = Charactermerge + + def dbcreate(self, dbtype, options): + updatesBase = automap_base() + dboBase = automap_base() + connection_string = None + if dbtype.lower() == "mssql": + connection_string = ( + "mssql+pymssql://" + + options["mssqlusername"] + + ":" + + options["mssqlpassword"] + + "@" + + options["hostname"] + + ":" + + options["mssqlport"] + + "/" + + options["dbname"] + ) + elif dbtype.lower() == "pgsql": + connection_string = ( + "postgresql+psycopg2://" + + options["pgsqlusername"] + + ":" + + options["pgsqlpassword"] + + "@" + + options["hostname"] + + ":" + + options["pgsqlport"] + + "/" + + options["dbname"] + ) + elif dbtype.lower() == "mysql": + connection_string = ( + "mysql+pymysql://" + + options["mysqlusername"] + + ":" + + options["mysqlpassword"] + + "@" + + options["hostname"] + + ":" + + options["mysqlport"] + + "/" + + options["dbname"] + ) + engine = sqlalchemy.create_engine( + connection_string, pool_size=20, max_overflow=20, pool_recycle=15 + ) + currentDir = os.path.dirname(os.path.abspath(__file__)) + currentDir = os.path.dirname(currentDir) + ddl_file_path = f"{currentDir}/db/ddl/recreate_updates_schema.sql" + with open(ddl_file_path, "r") as file: + sql = file.read() + file + with engine.connect() as conn: + with conn as cursor: + cursor.execute(text(sql)) + # conn.commit() + # with engine.connect() as conn: + # conn.execute(text("CREATE SCHEMA IF NOT EXISTS updates")) + # conn.execute(text("CREATE SCHEMA IF NOT EXISTS dbo")) + # conn.commit() + updatesBase.prepare(engine, reflect=True, schema="updates") + dboBase.prepare(engine, reflect=True, schema="dbo") + metadata = sqlalchemy.MetaData("updates") + metadata.reflect(bind=engine) + dboMetadata = sqlalchemy.MetaData("dbo") + dboMetadata.reflect(bind=engine) + session = Session(engine) + self.dbengine["engine"] = engine + self.dbengine["metadata"] = metadata + self.dbengine["dbometadata"] = dboMetadata + self.dbengine["updatesBase"] = updatesBase + self.dbengine["dboBase"] = dboBase + self.dbengine["session"] = session + return self.dbengine + + def setapi(self, apitype): + if apitype.lower() == "tvmaze": + self.apiengine["sprocess"] = api.tvmaze.series + self.apiengine["cprocess"] = api.tvmaze.cast + self.apiengine["eprocess"] = api.tvmaze.episodes + return self.apiengine + elif apitype.lower() == "thetvdb": + pass + + def reflect_tables(self, engine, Metadata): + meta = MetaData() + meta.reflect(bind=engine) + return meta.tables + + def get_data(self): + return self.dbclass, self.dbengine, self.apiengine + + def __del__(self): + pass + + +class dbmongo(object): + + global mgclient + global mgseries + global mgepisodes + global mgactors + global mgcharacters + global mgcrew + + def __init__(self, options): + host = options["mghostname"] + port = options["mgport"] + print(f"Connecting to mongodb at {host}:{port}") + try: + self.connection_string = ( + f"mongodb://{options['mghostname']}:{options['mgport']}" + ) + self.mgclient = MongoClient(self.connection_string) + self.mgdb = self.mgclient[options["mgdbname"]] + except BaseException as e: + print(f"Unable to connect to mongodb.{e}\nExitting!") + exit() + else: + self.mgseries = self.mgdb["series"] + self.mgseries.create_index("id", unique=True) + self.mgepisodes = self.mgdb["episodes"] + self.mgepisodes.create_index("id", unique=True) + self.mgactors = self.mgdb["actors"] + self.mgactors.create_index("id", unique=True) + self.mgcharacters = self.mgdb["characters"] + self.mgcharacters.create_index("id", unique=True) + self.mgcrew = self.mgdb["crew"] + self.mgcrew.create_index("id", unique=True) + print( + f"Connection to mongodb at {options['mghostname']}:" + f"{options['mgport']} successful." + ) + + def load_json_file(self, cachefile, jget, lprint, seriesid, datatype): + try: + with open(cachefile, "r") as file: + jsondata = file.read() + datastring = jget.load(jsondata) + data = jget.jconvert(datastring) + return data + except Exception: + lprint.logprint( + "info", f"Unable to open {datatype} file for {seriesid}" + ) + # print(f'Unable to open {datatype} file for seriesid {seriesid}') + return None + + def process_series_data( + self, + seriesdata, + jget, + dbExec, + dbengine, + dboBase, + lprint, + countrydictionary, + seriesid, + ): + try: + seriesresults, networkchannel, country, webchannel = ( + jget.processSeriesJson(seriesdata) + ) + except Exception: + lprint.logprint( + "info", f"Unable to process series file for {seriesid}" + ) + else: + self.update_country_data( + country, + countrydictionary, + dbExec, + dbengine, + dboBase, + dbengine["dbometadata"], + lprint, + ) + self.update_network_data( + networkchannel, + countrydictionary, + seriesresults, + dbExec, + dbengine, + dboBase, + lprint, + ) + self.update_webchannel_data( + webchannel, + countrydictionary, + seriesresults, + dbExec, + dbengine, + dboBase, + lprint, + ) + self.insert_or_update_mongo( + self.mgseries, seriesdata, "id" + ) + + def process_cast_data(self, persondata, characterdata, seriesid, lprint): + try: + aupserts = [ + self.mgactors.update_many( + {"id": x["id"]}, {"$setOnInsert": x}, upsert=True + ) + for x in persondata + ] + bupserts = [ + self.mgcharacters.update_many( + {"id": x["id"]}, {"$setOnInsert": x}, upsert=True + ) + for x in characterdata + ] + except Exception as e: + print(f"Skipping cast data.Reason: {str(e)}") + else: + [ + self.mgactors.update_many( + {"id": x["id"]}, {"$setOnInsert": x}, upsert=True + ) + for x in aupserts + ] + [ + self.mgcharacters.insert_many( + {"id": x["id"]}, {"$setOnInsert": x}, upsert=True + ) + for x in bupserts + ] + + def process_crew_data(self, crewdata, seriesid, jget, lprint): + try: + upserts = [ + self.mgcrew.update_many( + {"id": x["id"]}, {"$setOnInsert": x}, upsert=True + ) + for x in crewdata + ] + except Exception as e: + print(f"Skipping crew data.Reason: {str(e)}") + else: + [ + self.mgcrew.insert_many( + {"id": x["id"]}, {"$setOnInsert": x}, upsert=True + ) + for x in upserts + ] + + def process_episode_data(self, episodedata, seriesid, lprint): + try: + upserts = [ + self.mgepisodes.update_many( + {"id": x["id"]}, {"$setOnInsert": x}, upsert=True + ) + for x in episodedata + ] + except Exception as e: + print( + f"file does not contain the correct json. " + f"Skipping. {str(e)}" + ) + else: + for episode in upserts: + [ + self.mgepisodes.insert_many( + {"id": x["id"]}, {"$setOnInsert": x}, upsert=True + ) + for x in upserts + ] + + def update_country_data( + self, country, countrydictionary, dbExec, dbengine, dbBase, + metadata, lprint + ): + if len(country) > 0: + try: + selectquery = ( + f"select row_id, country_name from dbo.countrydata where " + f"country_name = '{country['name'].replace("'", "''")}'" + ) + countrylisting = dbExec.rawsql_select( + dbengine["engine"], selectquery, lprint + ) + for countryname in countrylisting: + countrydictionary[countryname[1]] = countryname[0] + except Exception as e: + lprint.logprint( + "info", ( + ( + f'Unable to process country data for ' + f'{country["name"]}. {e}' + ) + ) + ) + else: + if len(countrylisting) > 0: + pass + else: + insert_sql = ( + ( + f"insert into dbo.countrydata " + "(country_name, country_code, country_timezone) " + f"values ('{country['name'].replace("'", "''")}', " + f"'{country['code'].replace("'", "''")}', " + f"'{country['timezone'].replace("'", "''")}')" + ) + ) + dbExec.rawsql_insert( + dbengine["engine"], insert_sql, lprint + ) + # seriesresults['countryid'] = 'No Country' + # return seriesresults['countryid'] + + def update_network_data( + self, + networkchannel, + countrydictionary, + seriesresults, + dbExec, + dbengine, + dboBase, + lprint, + ): + if len(networkchannel) > 0: + try: + networkchannel["countryid"] = countrydictionary[ + networkchannel["country"]["name"] + ] + seriesresults["networkid"] = networkchannel["id"] + except Exception: + pass + + def update_webchannel_data( + self, + webchannel, + countrydictionary, + seriesresults, + dbExec, + dbengine, + dboBase, + lprint, + ): + if len(webchannel) > 0: + try: + webchannel["countryid"] = countrydictionary[ + webchannel["country"]["name"] + ] + seriesresults["webchannelid"] = webchannel["id"] + except Exception: + pass + seriesresults["seriesid"] = seriesresults["id"] + del seriesresults["webchannel"] + + def insert_or_update_mongo(self, collection, data, unique_key): + try: + collection.insert_one(data) + except Exception: + filter_query = {unique_key: data[unique_key]} + new_values = {"$set": data} + del new_values["$set"]["_id"] + collection.update_one(filter_query, new_values) + + def insert_or_update_many_mongo(self, collection, data, unique_key): + try: + collection.insert_many(data) + except Exception: + filter_query = {unique_key: data[unique_key]} + new_values = {"$set": data} + del new_values["$set"]["_id"] + collection.update_one(filter_query, new_values) + + def update_mongo( + self, + updateList, + ROOTDIR, + directories, + jget, + countrydictionary, + dbExec, + dbengine, + dboBase, + lprint, + ): + print("Now inserting data into tables.") + series_failed_count = 0 + episode_failed_count = 0 + cast_failed_count = 0 + crew_failed_count = 0 + processed_count = 0 + failed_file_count = 0 + + def process_data( + file_path, data_type, process_method, failed_file_count, lprint + ): + """Helper function to load and process data.""" + data = self.load_json_file( + file_path, jget, lprint, seriesid, data_type + ) + if data: + if "episode" in file_path: + try: + for item in data: + item["seriesid"] = seriesid + except Exception as e: + lprint.logprint( + "info", f"Unable to add {data_type} to data. {e}" + ) + if "cast" in file_path: + persondata = [] + characterdata = [] + for item in data: + if "person" in item: + persondata.append(item["person"]) + if "character" in item: + characterdata.append(item["character"]) + try: + for item in persondata: + item["seriesid"] = seriesid + for item in characterdata: + item["seriesid"] = seriesid + except Exception as e: + lprint.logprint( + "info", f"Unable to add {data_type} data. {e}" + ) + + try: + process_method( + persondata, characterdata, seriesid, lprint + ) + failed_file_count = "N/A" + return failed_file_count + except Exception as e: + failed_file_count = 1 + lprint.logprint( + "info", + f"Unable to process {data_type} data. {e}" + ) + return failed_file_count + if "crew" in file_path: + try: + crewdata = [] + except Exception as e: + lprint.logprint( + "info", + f"Unable to add {data_type} data. {e}" + ) + else: + for item in data: + item["seriesid"] = seriesid + if "person" in item: + crewdata.append(item["person"]) + try: + process_method(crewdata, seriesid, jget, lprint) + failed_file_count = "N/A" + return failed_file_count + except Exception as e: + failed_file_count = 1 + lprint.logprint( + "info", + f"Unable to process {data_type} data.{e}" + ) + return failed_file_count + + for series in tqdm( + range(len(updateList)), desc="Update MongoDB", unit="series" + ): + seriesid = updateList[series] + seriescachefile = f'{directories["SERIESDIR"]}{seriesid}.json' + episodecachefile = f'{directories["EPISODEDIR"]}{seriesid}.json' + castcachefile = f'{directories["CASTDIR"]}{seriesid}.json' + crewcachefile = f'{directories["CREWDIR"]}{seriesid}.json' + + # Process series data + seriesdata = self.load_json_file( + seriescachefile, jget, lprint, seriesid, "series" + ) + if seriesdata: + try: + self.process_series_data( + seriesdata, + jget, + dbExec, + dbengine, + dboBase, + lprint, + countrydictionary, + seriesid, + ) + self.insert_or_update_mongo( + self.mgseries, seriesdata, "id" + ) + except Exception as e: + series_failed_count += 1 + lprint.logprint( + "info", + ( + f"Unable to process series data for seriesid " + f"{seriesid}. {e}" + ), + ) + else: + processed_count += 1 + + # Process episode, cast, and crew data using the helper function + fcount = process_data( + episodecachefile, + "episode", + self.process_episode_data, + failed_file_count, + lprint, + ) + if isinstance(fcount, int): + episode_failed_count = episode_failed_count + fcount + fcount = process_data( + castcachefile, "cast", self.process_cast_data, + failed_file_count, lprint + ) + if isinstance(fcount, int): + cast_failed_count = cast_failed_count + fcount + try: + fcount = process_data( + crewcachefile, "crew", self.process_crew_data, + failed_file_count, lprint + ) + except Exception: + crew_failed_count = crew_failed_count + 1 + else: + if isinstance(fcount, int): + crew_failed_count = crew_failed_count + fcount + processed_count += 1 + + print(f"Processing complete. {processed_count} series processed.") + print(f"Failed to load {series_failed_count} series files.") + print(f"Failed to load {episode_failed_count} episode files.") + print(f"Failed to load {crew_failed_count} crew files.") + print(f"Failed to load {cast_failed_count} cast files.") + + def __del__(self): + pass diff --git a/db/mssql/Merger.py b/db/mssql/Merger.py new file mode 100644 index 0000000..05ae6b1 --- /dev/null +++ b/db/mssql/Merger.py @@ -0,0 +1,327 @@ +from sqlalchemy import text + +class SQLGenerate(): + + def __init__(self): + pass + + def merge_updates(self): + updatesql = text(''' + MERGE updates.tvupdatedata + USING updates.tvupdates + ON updates.tvupdatedata.seriesid = updates.tvupdates.seriesid + WHEN MATCHED THEN + UPDATE SET updates.tvupdatedata.seriesid = updates.tvupdates.seriesid,updates.tvupdatedata.timestamp = updates.tvupdates.timestamp + WHEN NOT MATCHED THEN + INSERT (seriesid, timestamp) + VALUES (updates.tvupdates.seriesid, updates.tvupdates.timestamp); + ''') + return(updatesql) + + def merge_series(self): + seriessql = text(''' + MERGE dbo.seriesdata + USING updates.newseriesdata ON + dbo.seriesdata.seriesid = updates.newseriesdata.seriesid + WHEN MATCHED THEN UPDATE + SET + dbo.seriesdata.seriesid = updates.newseriesdata.seriesid, + dbo.seriesdata.thetvdb = updates.newseriesdata.thetvdb, + dbo.seriesdata.tvrage = updates.newseriesdata.tvrage, + dbo.seriesdata.imdb = updates.newseriesdata.imdb, + dbo.seriesdata.[language] = updates.newseriesdata.[language], + dbo.seriesdata.genres = updates.newseriesdata.genres, + dbo.seriesdata.name = updates.newseriesdata.name, + dbo.seriesdata.status = updates.newseriesdata.status, + dbo.seriesdata.premiered = updates.newseriesdata.premiered, + dbo.seriesdata.days = updates.newseriesdata.days, + dbo.seriesdata.[time] = updates.newseriesdata.[time], + dbo.seriesdata.runtime = updates.newseriesdata.runtime, + dbo.seriesdata.rating = updates.newseriesdata.rating, + dbo.seriesdata.[type] = updates.newseriesdata.[type], + dbo.seriesdata.summary = updates.newseriesdata.summary, + dbo.seriesdata.medium = updates.newseriesdata.medium, + dbo.seriesdata.original = updates.newseriesdata.original, + dbo.seriesdata.[weight] = updates.newseriesdata.[weight], + dbo.seriesdata.networkid = updates.newseriesdata.networkid, + dbo.seriesdata.webchannelid = updates.newseriesdata.webchannelid, + dbo.seriesdata.officialsite = updates.newseriesdata.officalsite, + dbo.seriesdata.url = updates.newseriesdata.url, + dbo.seriesdata.serieslink = updates.newseriesdata.serieslink, + dbo.seriesdata.updated_on = GETDATE() + WHEN NOT MATCHED THEN INSERT + (seriesid, + thetvdb, + tvrage, + imdb, + [language], + genres, + name, + status, + premiered, + days, + [time], + runtime, + rating, + [type], + summary, + medium, + original, + weight, + networkid, + webchannelid, + officialsite, + url, + serieslink, + created_on) + VALUES (updates.newseriesdata.seriesid, + updates.newseriesdata.thetvdb, + updates.newseriesdata.tvrage, + updates.newseriesdata.imdb, + updates.newseriesdata.[language], + updates.newseriesdata.genres, + updates.newseriesdata.name, + updates.newseriesdata.status, + updates.newseriesdata.premiered, + updates.newseriesdata.days, + updates.newseriesdata.[time], + updates.newseriesdata.runtime, + updates.newseriesdata.rating, + updates.newseriesdata.[type], + updates.newseriesdata.summary, + updates.newseriesdata.medium, + updates.newseriesdata.original, + updates.newseriesdata.weight, + updates.newseriesdata.networkid, + updates.newseriesdata.webchannelid, + updates.newseriesdata.officalsite, + updates.newseriesdata.url, + updates.newseriesdata.serieslink, + GETDATE()); + ''') + return(seriessql) + + def merge_episodes(self): + episodesql = text(''' + MERGE dbo.epdata + USING updates.newepdata ON + dbo.epdata.episodeid = updates.newepdata.episodeid + WHEN MATCHED THEN UPDATE + SET + dbo.epdata.episodeid = updates.newepdata.episodeid, + dbo.epdata.seriesid = updates.newepdata.seriesid, + dbo.epdata.season = updates.newepdata.season, + dbo.epdata.episodenumber = updates.newepdata.episodenumber, + dbo.epdata.episodename = updates.newepdata.episodename, + dbo.epdata.airdate = updates.newepdata.airdate, + dbo.epdata.airtime = updates.newepdata.airtime, + dbo.epdata.episodesummary = updates.newepdata.episodesummary, + dbo.epdata.airstamp = updates.newepdata.airstamp, + dbo.epdata.runtime = updates.newepdata.runtime, + dbo.epdata.eplink = updates.newepdata.eplink, + dbo.epdata.mediumimage = updates.newepdata.mediumimage, + dbo.epdata.originalimage = updates.newepdata.originalimage, + dbo.epdata.[url] = updates.newepdata.[url], + dbo.epdata.updated_on = GETDATE() + WHEN NOT MATCHED THEN INSERT + (episodeid, + seriesid, + season, + episodenumber, + episodename, + airdate, + airtime, + episodesummary, + airstamp, + runtime, + eplink, + mediumimage, + originalimage, + [url], + created_on) + VALUES (updates.newepdata.episodeid, + updates.newepdata.seriesid, + updates.newepdata.season, + updates.newepdata.episodenumber, + updates.newepdata.episodename, + updates.newepdata.airdate, + updates.newepdata.airtime, + updates.newepdata.episodesummary, + updates.newepdata.airstamp, + updates.newepdata.runtime, + updates.newepdata.eplink, + updates.newepdata.mediumimage, + updates.newepdata.originalimage, + updates.newepdata.[url], + GETDATE()); + ''') + return(episodesql) + + def merge_cast(self): + castsql = text(''' + MERGE castdata as N + USING updates.newcastdata as O + ON N.seriesid = O.seriesid and N.characterid = O.characterid and N.actorid = O.actorid + WHEN MATCHED THEN + UPDATE SET N.seriesid = O.seriesid, N.characterid = O.characterid,N.actorid = O.actorid,N.charactername = O.charactername,N.mediumimage = O.mediumimage,N.originalimage = O.originalimage,N.apilink = O.apilink,N.tvmazelink = O.tvmazelink, N.updated_on = GETDATE() + WHEN NOT MATCHED THEN + INSERT (seriesid,characterid,actorid,charactername,mediumimage,originalimage,apilink,tvmazelink, created_on) + VALUES + (O.seriesid,O.characterid,O.actorid,O.charactername,O.mediumimage,O.originalimage,O.apilink,O.tvmazelink, GETDATE()); + ''') + return(castsql) + + def merge_actors(self): + actorsql = text(''' + MERGE actors as N + USING updates.newactordata as O + ON N.actorid = O.actorid and N.actorname = O.actorname + WHEN MATCHED THEN + UPDATE SET N.actorid = O.actorid, N.actorname = O.actorname, N.countryname = O.countryname, N.countrytimezone = O.countrytimezone, N.birthday = O.birthday, N.deathday = O.deathday, N.gender = O.gender, N.mediumimage = O.mediumimage, N.originalimage = O.originalimage, N.apilink = O.apilink, N.url = O.url, N.countrycode = O.countrycode, N.updated_on = GETDATE() + WHEN NOT MATCHED THEN + INSERT (actorid,actorname,countryname,countrytimezone,birthday,deathday,gender,mediumimage,originalimage,apilink,url,countrycode,created_on) + VALUES + (O.actorid,O.actorname,O.countryname,O.countrytimezone,O.birthday,O.deathday,O.gender,O.mediumimage,O.originalimage,O.apilink,O.url,O.countrycode, GETDATE()); + ''') + return(actorsql) + + def merge_crew(self): + crewsql = text(''' + MERGE dbo.crewdata as N + USING updates.newcrewdata as O ON + N.[type] = O.[type] + and N.seriesid = O.seriesid + and N.crewid = O.crewid + WHEN MATCHED THEN UPDATE + SET + N.seriesid = O.seriesid, + N.crewid = O.crewid, + N.countryid = O.countryid, + N.type = O.type, + N.name = O.name, + N.medium = O.medium, + N.original = O.original, + N.apilink = O.apilink, + N.gender = O.gender, + N.birthday = O.birthday, + N.deathday = O.deathday, + N.updated_on = GETDATE() + WHEN NOT MATCHED THEN INSERT + (seriesid, + crewid, + countryid, + type, + name, + medium, + original, + apilink, + gender, + birthday, + deathday, + created_on) + VALUES (O.seriesid, + O.crewid, + O.countryid, + O.type, + O.name, + O.medium, + O.original, + O.apilink, + O.gender, + O.birthday, + O.deathday, + GETDATE()); + ''') + return(crewsql) + + def merge_country(self): + countrysql = text(''' + MERGE countrydata + USING updates.newcountrydata + ON countrydata.id = updates.newcountrydata.id + WHEN MATCHED THEN + UPDATE SET countrydata.name = updates.newcountrydata.name, countrydata.code = updates.newcountrydata.code, countrydata.timezone = updates.newcountrydata.timezone, countrydata.updated_on = GETDATE() + WHEN NOT MATCHED THEN + INSERT (name, code, timezone, created_on) + VALUES + (updates.newcountrydata.name,updates.newcountrydata.code,updates.newcountrydata.timezone, GETDATE()); + ''') + return(countrysql) + + def __del__(self): + pass + +class Charactermerge(): + + def __init__(self): + pass + + def __del__(self): + pass + +class Episodemerge(): + def __init_(self): + pass + + def __del__(self): + pass + +class Jsonmerge(): + + def __init__(self): + pass + + def merge(self, engine): + seriessql = text('''BEGIN TRANSACTION; + MERGE dbo.seriesjson as N + USING updates.seriesjsonnewvals as O + ON N.seriesid = O.seriesid and N.jdata = O.jdata + WHEN MATCHED THEN + UPDATE SET N.seriesid = O.seriesid, N.jdata = O.jdata, N.updated_on = GETDATE() + WHEN NOT MATCHED THEN + INSERT (seriesid, jdata, created_on) + VALUES + (O.seriesid, O.jdata, GETDATE()); + COMMIT; + ''') + with engine.connect() as con: + rs = con.execute(seriessql) + #engine.commit() + + episodesql = text('''BEGIN TRANSACTION; + MERGE updates.tvmaze_episodejson as N + USING updates.episodejsonnewvals as O + ON N.seriesid = O.seriesid + WHEN MATCHED THEN + UPDATE SET N.seriesid = O.seriesid, N.jdata = O.jdata, N.updated_on = GETDATE() + WHEN NOT MATCHED THEN + INSERT (seriesid, jdata, created_on) + VALUES + (O.seriesid, O.jdata, GETDATE()); + SELECT * FROM updates.tvmaze_episodejson;COMMIT; + ''') + with engine.connect() as con: + rs = con.execute(episodesql) + #engine.commit() + + charactersql = text('''BEGIN TRANSACTION; + MERGE updates.tvmaze_characterjson as N + USING updates.characterjsonnewvals as O + ON N.seriesid = O.seriesid + WHEN MATCHED THEN + UPDATE SET N.seriesid = O.seriesid, N.jdata = O.jdata, N.updated_on = GETDATE() + WHEN NOT MATCHED THEN + INSERT (seriesid, jdata, created_on) + VALUES + (O.seriesid, O.jdata, GETDATE()); + SELECT * FROM updates.tvmaze_characterjson;COMMIT; + ''') + with engine.connect() as con: + rs = con.execute(charactersql) + #engine.commit() + + def __del__(self): + pass + + + \ No newline at end of file diff --git a/db/mssql/Tables.py b/db/mssql/Tables.py new file mode 100644 index 0000000..3234090 --- /dev/null +++ b/db/mssql/Tables.py @@ -0,0 +1,376 @@ +import sqlalchemy +from sqlalchemy import create_engine, MetaData, Table, Column, Integer, String, Sequence, text, UniqueConstraint +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.schema import MetaData +from sqlalchemy.orm import sessionmaker +from sqlalchemy.types import * + +class Tables(): + def __init__(self): + global tvnetworknewvals + global tvwebchannelnewvals + global characterjsonnewvals + global seriesjsonnewvals + global episodejsonnewvals + global metadata + global engine + global session + global tvnetworknewvals + global tvwebchannelnewvals + global characterjsonnewvals + global seriesjsonnewvals + global episodejsonnewvals + global tvupdates + global newseriesdata + global actorlinks + global tablelist + + def __del__(self): + pass + + +class Createtables(): + + def __init__(self, session, metadata, engine, tablelist): + Base = declarative_base(metadata = metadata) + self.tablelist = self.settables(metadata, engine) + try: + self.drop(session, self.tablelist) + except: + pass + self.create(Base, engine, metadata, session, self.tablelist) + session.commit() + #self.send(tablelist) + + def settables(self, metadata, engine): + tablelist = [] + newseriesdata = Table('newseriesdata', metadata, + Column('id', BIGINT, Sequence('newseriesdata_id_seq'), primary_key = True), + Column('seriesid',INTEGER, unique=True), + Column('tvdbid', String(25)), + Column('tvrageid', String(25)), + Column('imdbid', String(25)), + Column('serieslanguage', String(25)), + Column('genres', String(75)), + Column('seriesname', String(150)), + Column('seriesstatus', String(25)), + Column('premiered', String(25)), + Column('airdays', String(72)), + Column('airtime', String(25)), + Column('runtime', String(25)), + Column('rating', String(25)), + Column('showtype',TEXT), + Column('overview',TEXT), + Column('mediumimage',TEXT), + Column('originalimage',TEXT), + Column('weight',INTEGER), + Column('networkid', String(25)), + Column('webchannelid', String(25)), + Column('officialsite',TEXT), + Column('showurl',TEXT), + Column('apilink',TEXT), + schema = "updates") + tablelist.append(newseriesdata) + newactordata = Table('newactordata', metadata, + Column('id', BIGINT, Sequence('newactordata_id_seq'), primary_key = True), + Column('actorid', INTEGER, unique=True), + Column('actorname', VARCHAR(72)), + Column('countryname', VARCHAR(144)), + Column('countrytimezone', VARCHAR(72)), + Column('birthday', DATE), + Column('deathday', DATE), + Column('gender', VARCHAR(6)), + Column('mediumimage', TEXT), + Column('originalimage', TEXT), + Column('apilink', TEXT), + Column('url', TEXT), + Column('countrycode', VARCHAR(12)), + schema = "updates") + tablelist.append(newactordata) + newepdata = Table('newepdata', metadata, + Column('id', BIGINT, Sequence('newepdata_id_seq'), primary_key = True), + Column('episodeid', INTEGER), + Column('seriesid', INTEGER), + Column('season', INTEGER), + Column('episodenumber', INTEGER), + Column('episodename', TEXT), + Column('airdate', VARCHAR(32)), + Column('airtime', VARCHAR(32)), + Column('episodesummary', TEXT), + Column('airstamp', VARCHAR(32)), + Column('runtime', VARCHAR(6)), + Column('eplink', TEXT), + Column('mediumimage', TEXT), + Column('url', TEXT), + Column('originalimage', TEXT), + schema = "updates") + tablelist.append(newepdata) + newcastdata = Table('newcastdata', metadata, + Column('id', BIGINT, Sequence('newcastdata_id_seq'), primary_key = True), + Column('seriesid', INTEGER), + Column('characterid', INTEGER), + Column('actorid', INTEGER), + Column('charactername', VARCHAR(250)), + Column('mediumimage', TEXT), + Column('originalimage', TEXT), + Column('apilink', TEXT), + Column('tvmazelink', TEXT), + schema = "updates") + tablelist.append(newcastdata) + tvnetworks = Table('tvnetworkdata', metadata, + Column('id', BIGINT, Sequence('network_id_seq'), primary_key = True), + Column('networkid', String(12)), + Column('networkname', String(255)), + Column('country', String(255)), + Column('countrycode', String(4)), + Column('countrytimezone', String(25)), + schema = "updates") + tablelist.append(tvnetworks) + tvwebchannels = Table('tvwebchanneldata', metadata, + Column('id', BIGINT, Sequence('webchannel_id_seq'), primary_key = True), + Column('webchannelname', String(255)), + Column('webchannelcountrycode', String(4)), + Column('webchannelcountryname', String(255)), + Column('webchannelcountrytimezone', String(4)), + schema = "updates") + tablelist.append(tvwebchannels) + return(tablelist) + + def create(self, Base, engine, metadata, session, tablelist): + print('Creating Temporary Data Tables') + Base.metadata.create_all(engine, tables=tablelist) + + def drop(self, session, tablelist): + for table in tablelist: + try: + #print('Dropping ' + str(table)) + table.drop() + except sqlalchemy.exc.ProgrammingError as se: + pass + + def __del__(self): + pass + + +class Temptables(): + + def __init__(self, session, metadata, engine, tablelist): + Base = declarative_base(metadata = metadata) + self.tablelist = self.settables(metadata, engine) + try: + self.drop(session, self.tablelist) + except: + pass + self.create(Base, engine, metadata, session, self.tablelist) + #session.commit() + + def settables(self, metadata, engine): + tablelist = [] + tvnetworknewvals = Table('tvnetworknewvals', metadata, + Column('id', BIGINT, Sequence('network_id_seq'), primary_key = True), + Column('name', String(255)), + Column('countryid', String(255)), + schema = "updates") + tablelist.append(tvnetworknewvals) + tvwebchannelnewvals = Table('tvwebchannelnewvals', metadata, + Column('id', BIGINT, Sequence('webchannel_id_seq'), primary_key = True), + Column('name', String(255)), + Column('countryid', String(4)), + schema = "updates") + tablelist.append(tvwebchannelnewvals) + tvupdates = Table('tvupdates', metadata, + Column('seriesid', BIGINT, primary_key = True), + Column('timestamp', VARCHAR(15)), + schema = "updates") + tablelist.append(tvupdates) + tvupdatedata = Table('tvupdatedata', metadata, + Column('seriesid', BIGINT, primary_key = True), + Column('timestamp', VARCHAR(15)), + schema = "updates") + tablelist.append(tvupdatedata) + newactorlinks = Table('newactorlinks', metadata, + Column('id', BIGINT, Sequence('tempepisodejson_id_seq'), primary_key = True), + Column('actorid', INTEGER), + Column('characterid', INTEGER), + schema = "updates") + tablelist.append(newactorlinks) + return(tablelist) + + def create(self, Base, engine, metadata, session, tablelist): + tablelist.remove(tablelist[6]) + print('Creating Temporary JSON Tables') + try: + Base.metadata.create_all(engine, tables=tablelist) + except sqlalchemy.exc.ProgrammingError as pe: + pass + + def drop(self, session, tablelist): + for table in tablelist: + if 'tvupdatedata' in table.description: + pass + else: + try: + #print('Dropping ' + str(table)) + table.drop() + except sqlalchemy.exc.ProgrammingError as se: + pass + #session.commit() + +class CreateTempTables(): + + global tvnetworknewvals + global tvwebchannelnewvals + global characterjsonnewvals + global seriesjsonnewvals + global episodejsonnewvals + global metadata + global engine + global session + + global tablelist + + def __init__(self, session, metadata, engine, tablelist): + Base = declarative_base(metadata = metadata) + self.tablelist = self.settables(metadata, engine) + try: + self.drop(session, self.tablelist) + except: + pass + self.create(Base, engine, metadata, session, self.tablelist) + session.commit() + #self.send(tablelist) + + def settables(self, metadata, engine): + tablelist = [] + newseriesdata = Table('newseriesdata', metadata, + Column('id', BIGINT, Sequence('newseriesdata_id_seq'), primary_key = True), + Column('seriesid',INTEGER, unique=True), + Column('thetvdb', String(25)), + Column('tvrage', String(25)), + Column('imdb', String(25)), + Column('language', String(25)), + Column('genres', String(75)), + Column('name', String(150)), + Column('status', String(25)), + Column('premiered', String(25)), + Column('days', String(72)), + Column('time', String(25)), + Column('runtime', String(25)), + Column('rating', String(25)), + Column('type',TEXT), + Column('summary',TEXT), + Column('medium',TEXT), + Column('original',TEXT), + Column('weight',INTEGER), + Column('networkid', String(25)), + Column('webchannelid', String(25)), + Column('officialsite',TEXT), + Column('url',TEXT), + Column('serieslink',TEXT), + schema="updates") + tablelist.append(newseriesdata) + newactordata = Table('newactordata', metadata, + Column('id', BIGINT, Sequence('newactordata_id_seq'), primary_key = True), + Column('actorid', INTEGER, unique=True), + Column('actorname', VARCHAR(72)), + Column('countryname', VARCHAR(36)), + Column('countrytimezone', VARCHAR(72)), + Column('birthday', DATE), + Column('deathday', DATE), + Column('gender', VARCHAR(6)), + Column('mediumimage', TEXT), + Column('originalimage', TEXT), + Column('apilink', TEXT), + Column('url', TEXT), + Column('countrycode', VARCHAR(12)), + schema="updates") + tablelist.append(newactordata) + newepdata = Table('newepdata', metadata, + Column('id', BIGINT, Sequence('newepdata_id_seq'), primary_key = True), + Column('episodeid', INTEGER), + Column('seriesid', INTEGER), + Column('season', INTEGER), + Column('episodenumber', INTEGER), + Column('episodename', TEXT), + Column('airdate', VARCHAR(32)), + Column('airtime', VARCHAR(32)), + Column('episodesummary', TEXT), + Column('airstamp', VARCHAR(32)), + Column('runtime', VARCHAR(6)), + Column('eplink', TEXT), + Column('mediumimage', TEXT), + Column('url', TEXT), + Column('originalimage', TEXT), + schema="updates") + tablelist.append(newepdata) + newcastdata = Table('newcastdata', metadata, + Column('id', BIGINT, Sequence('newcastdata_id_seq'), primary_key = True), + Column('seriesid', INTEGER), + Column('characterid', VARCHAR(24)), + Column('actorid', VARCHAR(24)), + Column('url', TEXT), + Column('charactername', VARCHAR(128)), + Column('mediumimage', TEXT), + Column('originalimage', TEXT), + Column('apilink', TEXT), + Column('tvmazelink', TEXT), + Column('self', CHAR), + Column('voice', CHAR), + schema="updates") + tablelist.append(newcastdata) + newcrewdata = Table('newcrewdata', metadata, + Column('id', BIGINT, Sequence('newcrewdata_id_seq'), primary_key = True), + Column('seriesid', INTEGER), + Column('crewid', VARCHAR(128)), + Column('type', VARCHAR(128)), + Column('url', TEXT), + Column('name', VARCHAR(256)), + Column('countryid', INTEGER), + Column('birthday', DATE), + Column('deathday', DATE), + Column('gender', VARCHAR(24)), + Column('medium', TEXT), + Column('original', TEXT), + Column('apilink', TEXT), + UniqueConstraint('seriesid', 'crewid', 'type', name='crew_unique'), + schema="updates") + tablelist.append(newcrewdata) + tvnetworks = Table('tvnetworkdata', metadata, + Column('id', BIGINT, Sequence('network_id_seq'), primary_key = True), + Column('name', String(255)), + Column('countryid', String(255)), + schema = "updates") + tablelist.append(tvnetworks) + tvwebchannels = Table('tvwebchanneldata', metadata, + Column('id', BIGINT, Sequence('webchannel_id_seq'), primary_key = True), + Column('name', String(255)), + Column('countryid', String(255)), + schema = "updates") + tablelist.append(tvwebchannels) + countries = Table('newcountrydata', metadata, + Column('id', BIGINT, Sequence('country_id_seq'), primary_key = True), + Column('name', String(255)), + Column('code', String(255)), + Column('timezone', String(255)), + UniqueConstraint('name', 'timezone', name='tz_name_uk'), + schema = "updates") + tablelist.append(countries) + tvupdates = Table('tvupdates', metadata, + Column('seriesid', BIGINT, primary_key = True), + Column('timestamp', VARCHAR(15)), + schema = "updates") + tablelist.append(tvupdates) + return(tablelist) + + def create(self, Base, engine, metadata, session, tablelist): + print('Creating Temporary Data Tables') + Base.metadata.create_all(engine, tables=tablelist) + + def drop(self, session, tablelist): + print('Dropping temporary update tables.') + for table in tablelist: + try: + print('Dropping ' + str(table)) + table.drop() + except sqlalchemy.exc.ProgrammingError as se: + pass \ No newline at end of file diff --git a/db/mssql/Updater.py b/db/mssql/Updater.py new file mode 100644 index 0000000..7163ab8 --- /dev/null +++ b/db/mssql/Updater.py @@ -0,0 +1,39 @@ +import sqlalchemy +import progressbar +from tvmaze.Processor import loadjson + +class Updater(): + def __init__(self): + global jprocess + + def __del__(self): + pass + +class tvupdates(): + def __init__(self): + pass + + def update_tables(engine, updatesapi, tablename): + jprocess = loadjson() + print('Now retrieving available updates.') + print('Creating update transaction.') + ins = tablename.insert() + seriesupdates = jprocess.load(updatesapi) + updatelist = [] + seriesrange = len(seriesupdates) + with progressbar.ProgressBar(maxval=seriesrange, redirect_stdout=True) as p: + for key, value in seriesupdates.items(): + i = int(key) + updater = {} + updater['seriesid'] = key + updater['timestamp'] = value + updatelist.append(updater) + if i <= seriesrange: + p.update(i) + #time.sleep(0.1) + print('Update of tvupdates table is complete.') + print('Done.') + return(ins, updatelist) + + def __del__(self): + pass diff --git a/db/mssql/__init__.py b/db/mssql/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/db/mssql/execute.py b/db/mssql/execute.py new file mode 100644 index 0000000..7eed615 --- /dev/null +++ b/db/mssql/execute.py @@ -0,0 +1,80 @@ +import sqlalchemy +import progressbar +import psycopg2.errors + +class Update(): + def __init__(self): + global jprocess + + def rawexec(self, sqlquery): + connection = engine.connect() + try: + trans = connection.begin() + connection.execute(str(sqlquery)) + except: + trans.rollback() + else: + trans.commit() + connection.close() + + def fileexec(self, sqlfile): + connection = engine.connect() + trans = connection.begin() + for line in sqlfile: + sqlquery = sqlquery + str(line) + try: + connection.execute(str(sqlquery)) + except: + trans.rollback() + lprint.logprint('error', 'Rollback for filename: ' + str(sqlfile)) + sqlfile.close() + connection.close() + else: + trans.commit() + sqlfile.close() + connection.close() + + def ormexec(self, listing, tablename): + try: + ins = tablename.insert() + result = engine.execute(ins, listing) + except sqlalchemy.exc.IntegrityError as ie: + if 'UNIQUE KEY' in ie.args[0]: + return True + except psycopg2.errors.UniqueViolation as ie2: + if 'unique constraint' in ie.args[0]: + return True + else: + return False + + def __del__(self): + pass + +class tvupdates(): + def __init__(self): + pass + + def update_tables(engine, updatesapi, tablename): + jprocess = loadjson() + print('Now retrieving available updates.') + print('Creating update transaction.') + ins = tablename.insert() + seriesupdates = jprocess.load(updatesapi) + updatelist = [] + seriesrange = len(seriesupdates) + with progressbar.ProgressBar(maxval=seriesrange, redirect_stdout=True) as p: + for key, value in seriesupdates.items(): + i = int(key) + updater = {} + updater['seriesid'] = key + updater['timestamp'] = value + updatelist.append(updater) + if i <= seriesrange: + p.update(i) + #time.sleep(0.1) + print('Update of tvupdates table is complete.') + print('Done.') + return(ins, updatelist) + + def __del__(self): + pass diff --git a/db/pgsql/Merger.py b/db/pgsql/Merger.py new file mode 100644 index 0000000..1afea8e --- /dev/null +++ b/db/pgsql/Merger.py @@ -0,0 +1,309 @@ +import datetime +from sqlalchemy import text +from sqlalchemy import DateTime + +class SQLGenerate(): + def __init__(self): + pass + + def merge_series(self): + seriessql = text(''' + INSERT INTO dbo.seriesdata (serieslink, url, thetvdb, tvrage, imdb, + genres, seriesid, medium,original, language, name, webchannelid, networkid, premiered, rating, runtime, days, time, + status, summary, type, weight) SELECT serieslink, url, thetvdb, tvrage, imdb, + genres, seriesid, medium,original, language, name, webchannelid, networkid, premiered, rating, runtime, days, time, + status, summary, type, weight from updates.newseriesdata + ON CONFLICT(seriesid) DO UPDATE + SET serieslink = excluded.serieslink, + url = excluded.url, + thetvdb = excluded.thetvdb, + tvrage = excluded.tvrage, + genres = excluded.genres, + seriesid = excluded.seriesid, + medium = excluded.medium, + original = excluded.original, + language = excluded.language, + name = excluded.name, + webchannelid = excluded.webchannelid, + networkid = excluded.networkid, + premiered = excluded.premiered, + rating = excluded.rating, + runtime = excluded.runtime, + days = excluded.days, + time = excluded.time, + status = excluded.status, + summary = excluded.summary, + type = excluded.type, + weight = excluded.weight; + ''') + return(seriessql) + + def merge_episodes(self): + episodesql = text(''' + INSERT INTO dbo.epdata (episodeid, + seriesid, + season, + episodenumber, + episodename, + airdate, + airtime, + episodesummary, + airstamp, + runtime, + eplink, + mediumimage, + originalimage, + "url") + SELECT episodeid, + seriesid, + season, + episodenumber, + episodename, + airdate, + airtime, + episodesummary, + airstamp, + runtime, + eplink, + mediumimage, + originalimage, + "url" + FROM updates.newepdata + ON CONFLICT (seriesid, episodeid) do UPDATE + SET episodeid = excluded.episodeid, + seriesid = excluded.seriesid, + season = excluded.season, + episodenumber = excluded.episodenumber, + episodename = excluded.episodename, + airdate = excluded.airdate, + airtime = excluded.airtime, + episodesummary = excluded.episodesummary, + airstamp = excluded.airstamp, + runtime = excluded.runtime, + eplink = excluded.eplink, + mediumimage = excluded.mediumimage, + originalimage = excluded.originalimage, + "url" = excluded."url"; + ''') + return(episodesql) + + def merge_updates(self): + updatesql = text(''' + INSERT INTO updates.tvupdatedata(seriesid, timestamp) + SELECT seriesid, timestamp from updates.tvupdates + ON CONFLICT(seriesid) DO UPDATE + SET seriesid = excluded.seriesid, timestamp = excluded.timestamp; + ''') + return(updatesql) + + def merge_cast(self): + datetimestamp = DateTime(datetime.datetime.utcnow) + castsql = text(''' + INSERT INTO dbo.castdata (seriesid,characterid,actorid,charactername,mediumimage,originalimage,apilink,tvmazeilink) + SELECT seriesid,characterid,actorid,charactername,mediumimage,originalimage,apilink,tvmazeilink + FROM updates.newcastdata + ON CONFLICT (seriesid, characterid, actorid) DO UPDATE + SET seriesid = excluded.seriesid, + characterid = excluded.characterid, + actorid = excluded.actorid, + charactername = excluded.charactername, + mediumimage = excluded.mediumimage, + originalimage = excluded.originalimage, + apilink = excluded.apilink, + tvmazeilink = excluded.tvmazeilink + ''') + return(castsql) + + def merge_actors(self): + actorsql = text(''' + INSERT INTO dbo.actors (actorid,actorname,countryname,countrytimezone,birthday,deathday,gender,mediumimage,originalimage,apilink,"url",countrycode) + SELECT actorid,actorname,countryname,countrytimezone,birthday,deathday,gender,mediumimage,originalimage,apilink,"url",countrycode + FROM updates.newactordata + ON CONFLICT (actorid) DO UPDATE + SET actorid = excluded.actorid, + actorname = excluded.actorname, + countryname = excluded.countryname, + countrytimezone = excluded.countrytimezone, + birthday = excluded.birthday, + deathday = excluded.deathday, + gender = excluded.gender, + mediumimage = excluded.mediumimage, + originalimage = excluded.originalimage, + apilink = excluded.apilink, + "url" = excluded."url", + countrycode = excluded.countrycode; + ''') + return(actorsql) + + def merge_crew(self): + crewsql = text(''' + INSERT INTO dbo.crewdata + (seriesid, + crewid, + countryid, + type, + name, + medium, + original, + apilink, + gender, + birthday, + deathday) + SELECT seriesid, + crewid, + countryid, + type, + name, + medium, + original, + apilink, + gender, + birthday, + deathday + FROM updates.newcrewdata + ON CONFLICT (seriesid, crewid, type) DO UPDATE + SET seriesid = excluded.seriesid, + crewid = excluded.crewid, + countryid = excluded.countryid, + type = excluded.type, + name = excluded.name, + medium = excluded.medium, + original = excluded.original, + apilink = excluded.apilink, + gender = excluded.gender, + birthday = excluded.birthday, + deathday = excluded.deathday; + ''') + return(crewsql) + + def __del__(self): + pass +class Merger(): + + def __init__(self): + pass + + def __del__(self): + pass + +class Charactermerge(): + + def __init__(self): + pass + + def merge(self, engine): + castsql = text('''MERGE dbo.castdata as N + USING updates.newcastdata as O + ON N.seriesid = O.seriesid and N.characterid = O.characterid and N.actorid = O.actorid + WHEN MATCHED THEN + UPDATE SET N.seriesid = O.seriesid, N.characterid = O.characterid,N.actorid = O.actorid,N.charactername = O.charactername,N.mediumimage = O.mediumimage,N.originalimage = O.originalimage,N.apilink = O.apilink,N.tvmazelink = O.tvmazelink, N.updated_on = GETDATE() + WHEN NOT MATCHED THEN + INSERT (seriesid,characterid,actorid,charactername,mediumimage,originalimage,apilink,tvmazelink, created_on) + VALUES + (O.seriesid,O.characterid,O.actorid,O.charactername,O.mediumimage,O.originalimage,O.apilink,O.tvmazelink, GETDATE()); + SELECT * FROM updates.newcastdata; + ''') + with engine.connect() as con: + rs = con.execute(castsql) + #engine.commit() + + actorsql = text('''MERGE dbo.actors as N + USING updates.newactordata as O + ON N.actorid = O.actorid + WHEN MATCHED THEN + UPDATE SET N.actorid = O.actorid, N.actorname = O.actorname, N.countryname = O.countryname, N.countrytimezone = O.countrytimezone, N.birthday = O.birthday, N.deathday = O.deathday, N.gender = O.gender, N.mediumimage = O.mediumimage, N.originalimage = O.originalimage, N.apilink = O.apilink, N.url = O.url, N.countrycode = O.countrycode, N.updated_on = GETDATE() + WHEN NOT MATCHED THEN + INSERT (actorid,actorname,countryname,countrytimezone,birthday,deathday,gender,mediumimage,originalimage,apilink,url,countrycode,created_on) + VALUES + (O.actorid,O.actorname,O.countryname,O.countrytimezone,O.birthday,O.deathday,O.gender,O.mediumimage,O.originalimage,O.apilink,O.url,O.countrycode, GETDATE()); + SELECT * FROM updates.newactordata; + ''') + with engine.connect() as con: + rs = con.execute(actorsql) + #engine.commit() + + def __del__(self): + pass + +class Updatemerge(): + + def __init__(self): + pass + + def merge(self, engine): + updatesql = text('''INSERT INTO updates.tvupdatedata (seriesid, timestamp) + select seriesid, timestamp from updates.tvupdates + ON CONFLICT (seriesid) + DO + UPDATE + SET seriesid = EXCLUDED.seriesid, timestamp = EXCLUDED.timestamp;''') + with engine.connect() as con: + rs = con.execute(updatesql) + #engine.commit() + + def __del__(self): + pass +class Jsonmerge(): + + def __init__(self): + pass + + def merge(self, engine): + seriessql = text('''INSERT INTO "[updates]".tvmaze_seriesjson (seriesid, jdata, created_on) + select seriesid, jdata, date_trunc('minute', current_timestamp) FROM "[updates]".seriesjsonnewvals + ON CONFLICT (seriesid) + DO + update + SET seriesid = EXCLUDED.seriesid, jdata = EXCLUDED.jdata, updated_on = date_trunc('minute', current_timestamp); + ''') + with engine.connect() as con: + rs = con.execute(seriessql) + #engine.commit() + + episodesql = text('''INSERT INTO "[updates]".tvmaze_episodejson (seriesid, jdata, created_on) + select seriesid, jdata, date_trunc('minute', current_timestamp) FROM "[updates]".episodejsonnewvals + ON CONFLICT (seriesid) + DO + update + SET seriesid = EXCLUDED.seriesid, jdata = EXCLUDED.jdata, updated_on = date_trunc('minute', current_timestamp); + ''') + with engine.connect() as con: + rs = con.execute(episodesql) + #engine.commit() + + charactersql = text('''INSERT INTO "[updates]".tvmaze_characterjson (seriesid, jdata, created_on) + select seriesid, jdata, date_trunc('minute', current_timestamp) FROM "[updates]".characterjsonnewvals + ON CONFLICT (seriesid) + DO + update + SET seriesid = EXCLUDED.seriesid, jdata = EXCLUDED.jdata, updated_on = date_trunc('minute', current_timestamp); + ''') + with engine.connect() as con: + rs = con.execute(charactersql) + #engine.commit() + + def __del__(self): + pass + +class Episodemerge(): + + def __init__(self): + pass + + def merge(self, engine): + episodesql = text('''INSERT INTO dbo.epdata (episodeid,seriesid,season,episodenumber,episodename,airdate,airtime,episodesummary,airstamp,runtime,eplink,mediumimage,originalimage,url, created_on) + select episodeid,seriesid,season,episodenumber,episodename,airdate,airtime,episodesummary,airstamp,runtime,eplink,mediumimage,originalimage,url, date_trunc('minute', current_timestamp) FROM "[updates]".newepdata + ON CONFLICT (episodeid) + DO + UPDATE + SET episodeid = EXCLUDED.episodeid, seriesid = EXCLUDED.seriesid, season = EXCLUDED.season, episodenumber = EXCLUDED.episodenumber, episodename = EXCLUDED.episodename, + airdate = EXCLUDED.airdate, airtime = EXCLUDED.airtime, episodesummary = EXCLUDED.episodesummary, airstamp = EXCLUDED.airstamp, runtime = EXCLUDED.runtime, + eplink = EXCLUDED.eplink, mediumimage = EXCLUDED.mediumimage, originalimage = EXCLUDED.originalimage, url = EXCLUDED.url, updated_on = date_trunc('minute', current_timestamp);''') + with engine.connect() as con: + rs = con.execute(episodesql) + #engine.commit() + + def __del__(self): + pass + + \ No newline at end of file diff --git a/db/pgsql/Tables.py b/db/pgsql/Tables.py new file mode 100644 index 0000000..62e500e --- /dev/null +++ b/db/pgsql/Tables.py @@ -0,0 +1,376 @@ +import sqlalchemy +from sqlalchemy import create_engine, MetaData, Table, Column, Integer, String, Sequence, text, UniqueConstraint +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.schema import MetaData +from sqlalchemy.orm import sessionmaker +from sqlalchemy.types import * + +class Tables(): + def __init__(self): + global tvnetworknewvals + global tvwebchannelnewvals + global characterjsonnewvals + global seriesjsonnewvals + global episodejsonnewvals + global metadata + global engine + global session + global tvnetworknewvals + global tvwebchannelnewvals + global characterjsonnewvals + global seriesjsonnewvals + global episodejsonnewvals + global tvupdates + global newseriesdata + global actorlinks + global tablelist + + def __del__(self): + pass + + +class Createtables(): + + def __init__(self, session, metadata, engine, tablelist): + Base = declarative_base(metadata = metadata) + self.tablelist = self.settables(metadata, engine) + try: + self.drop(session, self.tablelist) + except: + pass + self.create(Base, engine, metadata, session, self.tablelist) + session.commit() + #self.send(tablelist) + + def settables(self, metadata, engine): + tablelist = [] + newseriesdata = Table('newseriesdata', metadata, + Column('id', BIGINT, Sequence('newseriesdata_id_seq'), primary_key = True), + Column('seriesid',INTEGER, unique=True), + Column('tvdbid', String(25)), + Column('tvrageid', String(25)), + Column('imdbid', String(25)), + Column('serieslanguage', String(25)), + Column('genres', String(75)), + Column('seriesname', String(150)), + Column('seriesstatus', String(25)), + Column('premiered', String(25)), + Column('airdays', String(72)), + Column('airtime', String(25)), + Column('runtime', String(25)), + Column('rating', String(25)), + Column('showtype',TEXT), + Column('overview',TEXT), + Column('mediumimage',TEXT), + Column('originalimage',TEXT), + Column('weight',INTEGER), + Column('networkid', BIGINT), + Column('webchannelid', BIGINT), + Column('officialsite',TEXT), + Column('showurl',TEXT), + Column('apilink',TEXT), + schema = "updates") + tablelist.append(newseriesdata) + newactordata = Table('newactordata', metadata, + Column('id', BIGINT, Sequence('newactordata_id_seq'), primary_key = True), + Column('actorid', INTEGER, unique=True), + Column('actorname', VARCHAR(72)), + Column('countryname', VARCHAR(144)), + Column('countrytimezone', VARCHAR(72)), + Column('birthday', DATE), + Column('deathday', DATE), + Column('gender', VARCHAR(6)), + Column('mediumimage', TEXT), + Column('originalimage', TEXT), + Column('apilink', TEXT), + Column('url', TEXT), + Column('countrycode', VARCHAR(12)), + schema = "updates") + tablelist.append(newactordata) + newepdata = Table('newepdata', metadata, + #Column('id', BIGINT, Sequence('newepdata_id_seq'), primary_key = True), + Column('episodeid', INTEGER), + Column('seriesid', INTEGER), + Column('season', INTEGER), + Column('episodenumber', INTEGER), + Column('episodename', TEXT), + Column('airdate', VARCHAR(32)), + Column('airtime', VARCHAR(32)), + Column('episodesummary', TEXT), + Column('airstamp', VARCHAR(32)), + Column('runtime', VARCHAR(6)), + Column('eplink', TEXT), + Column('mediumimage', TEXT), + Column('url', TEXT), + Column('originalimage', TEXT), + schema = "updates") + tablelist.append(newepdata) + newcastdata = Table('newcastdata', metadata, + Column('id', BIGINT, Sequence('newcastdata_id_seq'), primary_key = True), + Column('seriesid', INTEGER), + Column('characterid', INTEGER), + Column('actorid', INTEGER), + Column('charactername', VARCHAR(250)), + Column('mediumimage', TEXT), + Column('originalimage', TEXT), + Column('apilink', TEXT), + Column('tvmazelink', TEXT), + schema = "updates") + tablelist.append(newcastdata) + tvnetworks = Table('tvnetworkdata', metadata, + Column('id', BIGINT, Sequence('network_id_seq'), primary_key = True), + Column('networkid', String(12)), + Column('networkname', String(255)), + Column('country', String(255)), + Column('countrycode', String(4)), + Column('countrytimezone', String(25)), + schema = "updates") + tablelist.append(tvnetworks) + tvwebchannels = Table('tvwebchanneldata', metadata, + Column('id', BIGINT, Sequence('webchannel_id_seq'), primary_key = True), + Column('webchannelname', String(255)), + Column('webchannelcountrycode', String(4)), + Column('webchannelcountryname', String(255)), + Column('webchannelcountrytimezone', String(4)), + schema = "updates") + tablelist.append(tvwebchannels) + return(tablelist) + + def create(self, Base, engine, metadata, session, tablelist): + print('Creating Temporary Data Tables') + Base.metadata.create_all(engine, tables=tablelist) + + def drop(self, session, tablelist): + for table in tablelist: + try: + #print('Dropping ' + str(table)) + table.drop() + except sqlalchemy.exc.ProgrammingError as se: + pass + + def __del__(self): + pass + + +class Temptables(): + + def __init__(self, session, metadata, engine, tablelist): + Base = declarative_base(metadata = metadata) + self.tablelist = self.settables(metadata, engine) + try: + self.drop(session, self.tablelist) + except: + pass + self.create(Base, engine, metadata, session, self.tablelist) + #session.commit() + + def settables(self, metadata, engine): + tablelist = [] + tvnetworknewvals = Table('tvnetworknewvals', metadata, + Column('id', BIGINT, Sequence('network_id_seq'), primary_key = True), + Column('name', String(255)), + Column('countryid', String(255)), + schema = "updates") + tablelist.append(tvnetworknewvals) + tvwebchannelnewvals = Table('tvwebchannelnewvals', metadata, + Column('id', BIGINT, Sequence('webchannel_id_seq'), primary_key = True), + Column('name', String(255)), + Column('countryid', String(4)), + schema = "updates") + tablelist.append(tvwebchannelnewvals) + tvupdates = Table('tvupdates', metadata, + Column('seriesid', BIGINT, primary_key = True), + Column('timestamp', VARCHAR(15)), + schema = "updates") + tablelist.append(tvupdates) + tvupdatedata = Table('tvupdatedata', metadata, + Column('seriesid', BIGINT, primary_key = True), + Column('timestamp', VARCHAR(15)), + schema = "updates") + tablelist.append(tvupdatedata) + newactorlinks = Table('newactorlinks', metadata, + Column('id', BIGINT, Sequence('tempepisodejson_id_seq'), primary_key = True), + Column('actorid', INTEGER), + Column('characterid', INTEGER), + schema = "updates") + tablelist.append(newactorlinks) + return(tablelist) + + def create(self, Base, engine, metadata, session, tablelist): + tablelist.remove(tablelist[6]) + print('Creating Temporary JSON Tables') + try: + Base.metadata.create_all(engine, tables=tablelist) + except sqlalchemy.exc.ProgrammingError as pe: + pass + + def drop(self, session, tablelist): + for table in tablelist: + if 'tvupdatedata' in table.description: + pass + else: + try: + #print('Dropping ' + str(table)) + table.drop() + except sqlalchemy.exc.ProgrammingError as se: + pass + #session.commit() + +class CreateTempTables(): + + global tvnetworknewvals + global tvwebchannelnewvals + global characterjsonnewvals + global seriesjsonnewvals + global episodejsonnewvals + global metadata + global engine + global session + + global tablelist + + def __init__(self, session, metadata, engine, tablelist): + Base = declarative_base(metadata = metadata) + self.tablelist = self.settables(metadata, engine) + try: + self.drop(session, self.tablelist) + except: + pass + self.create(Base, engine, metadata, session, self.tablelist) + session.commit() + #self.send(tablelist) + + def settables(self, metadata, engine): + tablelist = [] + newseriesdata = Table('newseriesdata', metadata, + Column('id', BIGINT, Sequence('newseriesdata_id_seq'), primary_key = True), + Column('seriesid',INTEGER, unique=True), + Column('thetvdb', String(25)), + Column('tvrage', String(25)), + Column('imdb', String(25)), + Column('language', String(25)), + Column('genres', String(75)), + Column('name', String(150)), + Column('status', String(25)), + Column('premiered', String(25)), + Column('days', String(72)), + Column('time', String(25)), + Column('runtime', String(25)), + Column('rating', String(25)), + Column('type',TEXT), + Column('summary',TEXT), + Column('medium',TEXT), + Column('original',TEXT), + Column('weight',INTEGER), + Column('networkid', BIGINT), + Column('webchannelid', BIGINT), + Column('officialsite',TEXT), + Column('url',TEXT), + Column('serieslink',TEXT), + schema="updates") + tablelist.append(newseriesdata) + newactordata = Table('newactordata', metadata, + Column('id', BIGINT, Sequence('newactordata_id_seq'), primary_key = True), + Column('actorid', INTEGER, unique=True), + Column('actorname', VARCHAR(72)), + Column('countryname', VARCHAR(36)), + Column('countrytimezone', VARCHAR(72)), + Column('birthday', DATE), + Column('deathday', DATE), + Column('gender', VARCHAR(6)), + Column('mediumimage', TEXT), + Column('originalimage', TEXT), + Column('apilink', TEXT), + Column('url', TEXT), + Column('countrycode', VARCHAR(12)), + schema="updates") + tablelist.append(newactordata) + newepdata = Table('newepdata', metadata, + #Column('id', BIGINT, Sequence('newepdata_id_seq'), primary_key = True), + Column('episodeid', INTEGER), + Column('seriesid', INTEGER), + Column('season', INTEGER), + Column('episodenumber', INTEGER), + Column('episodename', TEXT), + Column('airdate', VARCHAR(32)), + Column('airtime', VARCHAR(32)), + Column('episodesummary', TEXT), + Column('airstamp', VARCHAR(32)), + Column('runtime', VARCHAR(6)), + Column('eplink', TEXT), + Column('mediumimage', TEXT), + Column('url', TEXT), + Column('originalimage', TEXT), + schema="updates") + tablelist.append(newepdata) + newcastdata = Table('newcastdata', metadata, + Column('id', BIGINT, Sequence('newcastdata_id_seq'), primary_key = True), + Column('seriesid', INTEGER), + Column('characterid', INTEGER), + Column('actorid', INTEGER), + Column('url', TEXT), + Column('charactername', VARCHAR(128)), + Column('mediumimage', TEXT), + Column('originalimage', TEXT), + Column('apilink', TEXT), + Column('tvmazelink', TEXT), + Column('self', CHAR), + Column('voice', CHAR), + schema="updates") + tablelist.append(newcastdata) + newcrewdata = Table('newcrewdata', metadata, + Column('id', BIGINT, Sequence('newcrewdata_id_seq'), primary_key = True), + Column('seriesid', INTEGER), + Column('crewid', INTEGER), + Column('type', VARCHAR(128)), + Column('url', TEXT), + Column('name', VARCHAR(256)), + Column('countryid', INTEGER), + Column('birthday', DATE), + Column('deathday', DATE), + Column('gender', VARCHAR(24)), + Column('medium', TEXT), + Column('original', TEXT), + Column('apilink', TEXT), + UniqueConstraint('seriesid', 'crewid', 'type', name='crew_unique'), + schema="updates") + tablelist.append(newcrewdata) + tvnetworks = Table('tvnetworkdata', metadata, + Column('id', BIGINT, Sequence('network_id_seq'), primary_key = True), + Column('name', String(255)), + Column('countryid', String(255)), + schema = "updates") + tablelist.append(tvnetworks) + tvwebchannels = Table('tvwebchanneldata', metadata, + Column('id', BIGINT, Sequence('webchannel_id_seq'), primary_key = True), + Column('name', String(255)), + Column('countryid', String(255)), + schema = "updates") + tablelist.append(tvwebchannels) + countries = Table('newcountrydata', metadata, + Column('id', BIGINT, Sequence('country_id_seq'), primary_key = True), + Column('name', String(255)), + Column('code', String(255)), + Column('timezone', String(255)), + UniqueConstraint('name', 'timezone', name='tz_name_uk'), + schema = "updates") + tablelist.append(countries) + tvupdates = Table('tvupdates', metadata, + Column('seriesid', BIGINT, primary_key = True), + Column('timestamp', VARCHAR(15)), + schema = "updates") + tablelist.append(tvupdates) + return(tablelist) + + def create(self, Base, engine, metadata, session, tablelist): + print('Creating Temporary Data Tables') + Base.metadata.create_all(engine, tables=tablelist) + + def drop(self, session, tablelist): + print('Dropping temporary update tables.') + for table in tablelist: + try: + print('Dropping ' + str(table)) + table.drop() + except sqlalchemy.exc.ProgrammingError as se: + pass \ No newline at end of file diff --git a/db/pgsql/Updater.py b/db/pgsql/Updater.py new file mode 100644 index 0000000..7163ab8 --- /dev/null +++ b/db/pgsql/Updater.py @@ -0,0 +1,39 @@ +import sqlalchemy +import progressbar +from tvmaze.Processor import loadjson + +class Updater(): + def __init__(self): + global jprocess + + def __del__(self): + pass + +class tvupdates(): + def __init__(self): + pass + + def update_tables(engine, updatesapi, tablename): + jprocess = loadjson() + print('Now retrieving available updates.') + print('Creating update transaction.') + ins = tablename.insert() + seriesupdates = jprocess.load(updatesapi) + updatelist = [] + seriesrange = len(seriesupdates) + with progressbar.ProgressBar(maxval=seriesrange, redirect_stdout=True) as p: + for key, value in seriesupdates.items(): + i = int(key) + updater = {} + updater['seriesid'] = key + updater['timestamp'] = value + updatelist.append(updater) + if i <= seriesrange: + p.update(i) + #time.sleep(0.1) + print('Update of tvupdates table is complete.') + print('Done.') + return(ins, updatelist) + + def __del__(self): + pass diff --git a/db/pgsql/__init__.py b/db/pgsql/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/db/sql.py b/db/sql.py new file mode 100644 index 0000000..09c40a3 --- /dev/null +++ b/db/sql.py @@ -0,0 +1,71 @@ +__author__ = 'Wendell Jones' + +from tqdm import tqdm +from sqlalchemy import text +from sqlalchemy.orm import Session + + +class Dbexec(): + def __init__(self): + pass + + def update_tvupdates(self, engine, available_updates, tablename): + print('Creating update transaction.') + + updatelist = [ + { + 'seriesid': seriesid, + 'timestamp': available_updates[seriesid] + } + for seriesid in tqdm( + available_updates, + desc='Updating tvupdates table', + unit='record' + ) + ] + + print('Applying updates to table.') + with engine.connect() as conn: + conn.execute(text(f"truncate table {tablename}")) + conn.execute(tablename.insert(), updatelist) + conn.commit() + print('Update of tvupdates table is complete.') + + def rawsql_select(self, engine, sqlquery, lprint): + session = Session(engine) + resultset = [] + rs = session.execute(text(sqlquery)) + for row in rs: + resultset.append(row) + session.close() + return (resultset) + + def rawsql_insert(self, dbengine, sqlquery, lprint): + session = Session(dbengine) + try: + session.execute(text(sqlquery)) + except Exception as e: + print(str(e)) + lprint.logprint('error', 'Rollback for query: ' + sqlquery) + session.rollback() + else: + session.commit() + session.close() + + def __del__(self): + pass + + +class SQLGenerate(): + + def __init__(self): + pass + + def read_updatetable(self, engine, lprint): + seriessql = text(''' + SELECT seriesid, timestamp from updates.tvupdatedata; + ''') + Dbexec.execute_rawsql(self, engine, seriessql, lprint) + + def __del__(self): + pass diff --git a/jprocessor/__init__.py b/jprocessor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/jprocessor/process.py b/jprocessor/process.py new file mode 100644 index 0000000..e3a72f5 --- /dev/null +++ b/jprocessor/process.py @@ -0,0 +1,105 @@ +__author__ = 'Wendell Jones' +import json +import progressbar +from url.process import Loadurl + +class Jsonload(): + def __init__(self): + pass + + def load(self, rawdata): + jdata = {} + jdata = json.loads(rawdata) + newdata = self.dump(jdata) + return(newdata) + + def dump(self, rawdata): + jdata = json.dumps(rawdata, indent=4) + return(jdata) + + def jconvert(self, inputdata): + outputdata = json.loads(inputdata) + return(outputdata) + + def populate_cache(self, jdata, jfile): + try: + with open(jfile, 'w') as datafile: + datafile.write(jdata) + except: + print('Unable to save cache file.') + else: + datafile.close() + + def fetch_json(self, weburl, seriesid, cachefile): + uprocess = Loadurl() + try: + webdata = uprocess.get_data(weburl) + jdata = self.load(webdata) + except Exception as e: + print('Unable to get JSON data for ' + str(seriesid) + ': ' + str(e)) + else: + if jdata == '[]': + pass + else: + self.populate_cache(jdata, cachefile) + + def processSeriesJson(self, seriesdata): + seriesresults = {} + networkchannel = {} + webchannel = {} + country = {} + + seriesresults['id'] = seriesdata['id'] + seriesresults['url'] = seriesdata['url'] + seriesresults['name'] = seriesdata['name'] + seriesresults['type'] = seriesdata['type'] + seriesresults['language'] = seriesdata['language'] + seriesresults['genres'] = ', '.join(seriesdata['genres']) + seriesresults['status'] = seriesdata['status'] + seriesresults['runtime'] = seriesdata['runtime'] + seriesresults['premiered'] = seriesdata['premiered'] + seriesresults['officalsite'] = seriesdata['officialSite'] + if len(seriesdata['schedule']['days']) > 0: + seriesresults['days'] = seriesdata['schedule']['days'][0] + seriesresults['time'] = seriesdata['schedule']['time'] + seriesresults['rating'] = seriesdata['rating']['average'] + seriesresults['weight'] = seriesdata['weight'] + seriesresults['networkid'] = seriesdata['id'] + seriesresults['webchannel'] = seriesdata['id'] + seriesresults['tvrage'] = seriesdata['externals']['tvrage'] + seriesresults['thetvdb'] = seriesdata['externals']['thetvdb'] + seriesresults['imdb'] = seriesdata['externals']['imdb'] + if seriesdata['image']: + seriesresults['medium'] = seriesdata['image']['medium'] + seriesresults['original'] = seriesdata['image']['original'] + seriesresults['summary'] = seriesdata['summary'] + seriesresults['serieslink'] = seriesdata['_links']['self']['href'] + if 'previousepisode' in seriesdata['_links'].keys(): + seriesresults['previousepisode'] = seriesdata['_links']['previousepisode']['href'] + if 'nextepisode' in seriesdata['_links'].keys(): + seriesresults['nextepisode'] = seriesdata['_links']['nextepisode']['href'] + + + if seriesdata['network'] and seriesdata['network'] != None: + networkchannel['id'] = seriesdata['network']['id'] + networkchannel['name'] = seriesdata['network']['name'] + #networkchannel['countryid'] = country['id'] + if seriesdata['network']['country']: + # country['id'] = "auto incremented id" + country['name'] = seriesdata['network']['country']['name'] + country['code'] = seriesdata['network']['country']['code'] + country['timezone'] = seriesdata['network']['country']['timezone'] + if seriesdata['webChannel'] and seriesdata['webChannel'] != None: + webchannel['id'] = seriesdata['webChannel']['id'] + webchannel['name'] = seriesdata['webChannel']['name'] + #webchannel['countryid'] = country['id'] + if seriesdata['webChannel']['country']: + # country['id'] = "auto incremented id" + country['name'] = seriesdata['webchannel']['country']['name'] + country['code'] = seriesdata['webchannel']['country']['code'] + country['timezone'] = seriesdata['webchannel']['country']['timezone'] + + return(seriesresults, networkchannel, country, webchannel) + + def __del__(self): + pass diff --git a/jprocessor/process_view.py b/jprocessor/process_view.py new file mode 100644 index 0000000..d3e78ac --- /dev/null +++ b/jprocessor/process_view.py @@ -0,0 +1,80 @@ +__author__ = 'Wendell Jones' +from ast import Global +import json +import progressbar + +class Process_View(object): + + global seriesinfo + global episodeinfo + + def __init__(self): + pass + + def process_seriesdata(self, seriesdata): + seriesinfo = {} + seriesinfo['seriesid'] = seriesdata['id'] + seriesinfo['seriesurl'] = seriesdata['url'] + seriesinfo['seriesname'] = seriesdata['name'] + seriesinfo['seriestype'] = seriesdata['type'] + seriesinfo['series_language'] = seriesdata['language'] + seriesinfo['series_genres'] = ', '.join(seriesdata['genres']) + seriesinfo['series_status'] = seriesdata['status'] + seriesinfo['series_runtime'] = seriesdata['runtime'] + seriesinfo['series_premiered'] = seriesdata['premiered'] + seriesinfo['offcialsite'] = seriesdata['officialSite'] + seriesinfo['air_day'] = ', '.join(seriesdata['schedule']['days']) + seriesinfo['air_time'] = seriesdata['schedule']['time'] + seriesinfo['rating'] = seriesdata['rating']['average'] + seriesinfo['weight'] = seriesdata['weight'] + seriesinfo['networkid'] = seriesdata['network']['id'] + seriesinfo['networkname'] = seriesdata['network']['name'] + seriesinfo['tvrageid'] = seriesdata['externals']['tvrage'] + seriesinfo['tvdbid'] = seriesdata['externals']['thetvdb'] + seriesinfo['imdbid'] = seriesdata['externals']['imdb'] + seriesinfo['countryname'] = seriesdata['network']['country']['name'] + seriesinfo['countrycode'] = seriesdata['network']['country']['code'] + seriesinfo['timezone'] = seriesdata['network']['country']['timezone'] + self.print_seriesdata(seriesinfo) + + def print_seriesdata(self, seriesinfo): + print(f"Series ID: {seriesinfo['seriesid']}") + print(f"Series URL: {seriesinfo['seriesurl']}") + print(f"Series Name: {seriesinfo['seriesname']}") + print(f"Type of Series: {seriesinfo['seriestype']}") + print(f"Lanuguage: {seriesinfo['series_language']}") + print(f"Genres: {seriesinfo['series_genres']}") + print(f"Series Status: {seriesinfo['series_status']}") + print(f"Run Time: {seriesinfo['series_runtime']}") + print(f"Premier Date: {seriesinfo['series_premiered']}") + print(f"Official Website: {seriesinfo['offcialsite']}") + print(f"Day Aired: {seriesinfo['air_day']}") + print(f"Time Aired: {seriesinfo['air_time']}") + print(f"Rating: {seriesinfo['rating']}") + print(f"Weight: {seriesinfo['weight']}") + print(f"Network ID: {seriesinfo['networkid']}") + print(f"Network Name: {seriesinfo['networkname']}") + print(f"TVRage ID: {seriesinfo['tvrageid']}") + print(f"TVDB ID: {seriesinfo['tvdbid']}") + print(f"IMDB ID: {seriesinfo['imdbid']}") + print(f"Country: {seriesinfo['countryname']}") + print(f"Country Code: {seriesinfo['countrycode']}") + print(f"Timezone: {seriesinfo['timezone']}") + + def process_episodedata(self, episodeinfo): + episodeinfo = {} + + def print_episodedata(self, episodeinfo): + for episode in episodeinfo: + print(f"Air Date: {episode['airdate']}") + print(f"Air Time: {episode['airtime']}") + print(f"Season: {episode['season']}") + print(f"Episode: {episode['number']}") + print(f"Episode Name: {episode['name']}") + print(f"Summary: {episode['summary']}") + print(f"Rating: {episode['rating']['average']}") + print(f"Medium Image: {episode['image']['medium']}") + print(f"Original Image: {episode['image']['original']}") + + def __del__(self): + pass diff --git a/json_downloader.py b/json_downloader.py new file mode 100644 index 0000000..e72d242 --- /dev/null +++ b/json_downloader.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 + +import os +from pathlib import Path +from time import sleep +from tqdm import tqdm +import json +import requests + +def load(self, rawdata): + jdata = {} + jdata = json.loads(rawdata) + newdata = self.dump(jdata) + return(newdata) + +def dump(self, rawdata): + jdata = json.dumps(rawdata, indent=4) + return(jdata) + +def jconvert(self, inputdata): + outputdata = json.loads(inputdata) + return(outputdata) + +def populate_cache(self, jdata, jfile): + try: + with open(jfile, 'w') as datafile: + datafile.write(jdata) + except: + print('Unable to save cache file.') + else: + datafile.close() + +def fetch_json(self, weburl, seriesid, cachefile): + try: + webdata = get_data(weburl) + jdata = json.load(webdata) + except Exception as e: + print('Unable to get JSON data for ' + str(seriesid) + ': ' + str(e)) + else: + if jdata == '[]': + pass + else: + populate_cache(jdata, cachefile) + + def get_data(self,weburl): + try: + showresponse = requests.get(weburl) + except requests.exceptions.ConnectionError as e: + print('Unable to connect and get JSON data for url ' + str(weburl)) + else: + return(showresponse.text) + +def check_and_download(url, series_id, cachefile): + try: + mazeurl = URL.replace('', str(series_id)) + response = requests.get(mazeurl) + data = json.loads(response.text) + except FileNotFoundError: + #print(f'{cachefile} does not exist. Attempting to download.') + jget.fetch_json(url.replace(SERIESID, str(series_id)), series_id, cachefile) + except Exception as e: + print(f'Unable to download {cachefile}. {e}') + else: + with open(cachefile, 'w') as f: + json.dump(data, f, indent=4) + f.close() + +new_updates = [] +listing = open ('D:/Temp/filelist.txt', 'w') + +os.chdir('D:/Projects/new_dbsync/cache/series') +directory_path = 'D:/Projects/new_dbsync/cache/series' # replace with your directory path +files = [f for f in os.listdir(directory_path) if os.path.isfile(os.path.join(directory_path, f))] +for filename in files: + with open(filename, 'r') as file: + content = file.read() + if 'Not Found' in content or 'Too Many Requests' in content: + seriesid = filename.split('.')[0] + new_updates.append(seriesid) + listing.write(seriesid + '\n') +listing.close() + +exit() + +# new_updates_2 = [] +# listing = open ('D:/Temp/filelist.txt', 'r') +# for seriesid in listing: +# seriesid = seriesid.strip() +# new_updates_2.append(seriesid) +# listing.close() + +# for seriesid in tqdm(range(len(new_updates_2)), desc='Checking cache', unit='seriesid'): +# # cachefile = str(new_updates_2[seriesid]) + '.json' +# # URL ='http://api.tvmaze.com/shows//cast' +# # check_and_download(URL, str(new_updates_2[seriesid]), f'{new_updates_2[seriesid]}.json') +# os.remove(f'D:/Projects/new_dbsync/cache/cast/{new_updates_2[seriesid]}.json') \ No newline at end of file diff --git a/logs/Logger.py b/logs/Logger.py new file mode 100644 index 0000000..845e8b7 --- /dev/null +++ b/logs/Logger.py @@ -0,0 +1,100 @@ +__author__ = 'Wendell Jones' + +import logging +from logging.handlers import RotatingFileHandler +import time +from datetime import datetime, timedelta +import os +from glob import glob +import math as m + + +class Logger: + def __init__(self): + pass + + def __del__(self): + pass + + +class LPrint: + def __init__(self, log_dir, force_rotate): + self.log_dir = log_dir + self.logfile = os.path.join(log_dir, 'dbsync.log') + self.setup_logfile() + self.cleanup_old_logs() + self.check_log_size(force_rotate) + + def setup_logfile(self): + """Ensure the logfile exists and is ready for use.""" + os.makedirs(self.log_dir, exist_ok=True) + with open(self.logfile, 'a'): + os.utime(self.logfile, None) + + def cleanup_old_logs(self): + """Remove log files older than 14 days.""" + logfile_list = glob(f'{self.logfile}.*') + date_limit = datetime.today() - timedelta(days=14) + for log in logfile_list: + timestamp = datetime.fromtimestamp(os.path.getmtime(log)) + if timestamp < date_limit: + os.remove(log) + + def check_log_size(self, force_rotate): + """Check the log file size and rotate if necessary.""" + log_size = os.path.getsize(self.logfile) + print(f'Logfile Size: {log_size} ({self.humanbytes(log_size)})') + if force_rotate or log_size > 10_000_000: # 10 MB + self.rotate_logs() + + def rotate_logs(self): + """Rotate the log file.""" + from logrotater import LogRotate # Assuming logrotater is a custom module + rotater = LogRotate(prefix=self.logfile, verbose=True) + try: + rotater.rotate() + except FileExistsError as e: + print(f'Unable to overwrite archive log: {e}. Deleting old logs.') + for log in glob(f'{self.logfile}.*'): + try: + os.remove(log) + except Exception as e: + print(f'Unable to delete {log}: {e}') + + def start_log(self): + """Initialize logging.""" + logging.basicConfig(filename=self.logfile, level=logging.DEBUG) + + def logprint(self, loglevel, logmessage): + """Log a message with a timestamp.""" + logstamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + log_message = f'{logstamp}: {logmessage}' + loglevel = loglevel.lower() + if loglevel == 'debug': + logging.debug(log_message) + elif loglevel == 'warning': + logging.warning(log_message) + elif loglevel == 'info': + logging.info(log_message) + else: + logging.error(f'Unknown log level: {loglevel}. Message: {logmessage}') + + def create_rotating_log(self, logpath, max_bytes=10_000_000, backup_count=5): + """Create a rotating log handler.""" + logger = logging.getLogger("Rotating Log") + logger.setLevel(logging.INFO) + handler = RotatingFileHandler(logpath, maxBytes=max_bytes, backupCount=backup_count) + logger.addHandler(handler) + + @staticmethod + def humanbytes(size, binary=False, precision=2): + """Convert bytes to a human-readable format.""" + multiples = ["B", "k{}B", "M{}B", "G{}B", "T{}B", "P{}B", "E{}B", "Z{}B", "Y{}B"] + base = 1024 if binary else 1000 + multiple = m.trunc(m.log2(size) / m.log2(base)) if size > 0 else 0 + value = size / m.pow(base, multiple) if size > 0 else 0 + suffix = multiples[multiple].format("i" if binary else "") + return f"{value:.{precision}f} {suffix}" + + def __del__(self): + pass \ No newline at end of file diff --git a/logs/__init__.py b/logs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3b39954 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +logrotater==1.3 +numpy==2.2.3 +progressbar33==2.4 +psycopg2==2.9.10 +pymongo==4.11.2 +pymssql==2.3.2 +Requests==2.32.3 +SQLAlchemy==2.0.39 +pymysql +tqdm diff --git a/settings/__init__.py b/settings/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/settings/config.py b/settings/config.py new file mode 100644 index 0000000..1a97c93 --- /dev/null +++ b/settings/config.py @@ -0,0 +1,63 @@ +__author__ = 'Wendell Jones' +import configparser +import os +import time +class Config(object): + global ROOTDIR + def __init__(self, ROOTDIR): + self.config_options = {} + self.configurer = configparser.ConfigParser() + self.read_settings(ROOTDIR) + + def read_settings(self, ROOTDIR): + if os.name == 'nt': + self.configurer.read(ROOTDIR + '\\settings\\tvsync_settings.cfg') + else: + self.configurer.read(ROOTDIR + '/settings/tvsync_settings.cfg') + print(f"Reading configuration file from {ROOTDIR}/settings") + self.set_data() + self.get_data() + + def set_data(self): + self.config_options['loglevel'] = self.configurer.get('global', 'loglevel') + self.config_options['hostname'] = self.configurer.get('dbsettings', 'hostname') + self.config_options['mghostname'] = self.configurer.get('dbsettings', 'mghostname') + self.config_options['dbname'] = self.configurer.get('dbsettings', 'dbname') + self.config_options['mgdbname'] = self.configurer.get('dbsettings', 'mgdbname') + self.config_options['mysqlusername'] = self.configurer.get('dbsettings', 'mysqlusername') + self.config_options['pgsqlusername'] = self.configurer.get('dbsettings', 'pgsqlusername') + self.config_options['mssqlusername'] = self.configurer.get('dbsettings', 'mssqlusername') + self.config_options['mysqlpassword'] = self.configurer.get('dbsettings', 'mysqlpassword') + self.config_options['pgsqlpassword'] = self.configurer.get('dbsettings', 'pgsqlpassword') + self.config_options['mssqlpassword'] = self.configurer.get('dbsettings', 'mssqlpassword') + self.config_options['pgsqlport'] = self.configurer.get('dbsettings', 'pgsqlport') + self.config_options['mysqlport'] = self.configurer.get('dbsettings', 'mysqlport') + self.config_options['mssqlport'] = self.configurer.get('dbsettings', 'mssqlport') + self.config_options['mgport'] = self.configurer.get('dbsettings', 'mgport') + self.config_options['updatetype'] = self.configurer.get('updatetype', 'type') + self.config_options['apitype'] = self.configurer.get('api', 'type') + self.config_options['updateschema'] = self.configurer.get('database', 'updateschema') + self.config_options['dbtype'] = self.configurer.get('database', 'dbtype') + self.config_options['initdb'] = self.configurer.get('database', 'initialize') + self.config_options['initload'] = self.configurer.get('database', 'initload') + self.config_options['skipcache'] = self.configurer.get('updatetype', 'cacheskip') + self.config_options['refreshcache'] = self.configurer.get('updatetype', 'refreshcache') + self.config_options['full'] = self.configurer.get('updatetype', 'full') + self.config_options['monthly'] = self.configurer.get('updatetype', 'monthly') + self.config_options['weekly'] = self.configurer.get('updatetype', 'weekly') + self.config_options['daily'] = self.configurer.get('updatetype', 'daily') + self.config_options['12h'] = self.configurer.get('updatetype', '12h') + self.config_options['6h'] = self.configurer.get('updatetype', '6h') + self.config_options['custom'] = self.configurer.get('updatetype', 'custom') + self.config_options['serieslist'] = list() + self.config_options['timedifference'] = self.configurer.get('updatetype', self.config_options['updatetype']) + self.config_options['current_timestamp'] = time.time() + if self.config_options['timedifference'].lower() == 'full': + self.config_options['timedifference'] = int(self.config_options['current_timestamp']) + + def get_data(self): + return self.config_options + + def __del__(self): + + pass diff --git a/settings/tvsync_settings.cfg b/settings/tvsync_settings.cfg new file mode 100644 index 0000000..c961186 --- /dev/null +++ b/settings/tvsync_settings.cfg @@ -0,0 +1,86 @@ +[global] +loglevel=debug + +[dbsettings] +sqlserver_driver=ODBC Driver 13 for SQL Server +hostname=192.168.128.7 +mghostname=192.168.128.8 +dbname=media_dbsync +mgdbname=test2 +mysqlusername=root +pgsqlusername=postgres +mssqlusername=sa +mysqlpassword=Optimus0329 +pgsqlpassword=Optimus0329 +mssqlpassword=Optimus0329 +pgsqlport=5432 +mysqlport=3306 +mssqlport=1433 +mgport=27017 + +[api] +type=tvmaze + +[json] +seriesapi=http://api.tvmaze.com/shows?page= +showapi=http://api.tvmaze.com/shows/ +episodeapi=http://api.tvmaze.com/shows//episodes +aliasapi=http://api.tvmaze.com/shows//akas +castapi=http://api.tvmaze.com/shows//cast +crewapi=http://api.tvmaze.com/shows//crew +creditsapi=http://api.tvmaze.com/people//castcredits +updatesapi=http://api.tvmaze.com/updates/shows + +[database] +dbtype=pgsql +updateschema=updates +initialize=0 +initload=0 + +[updatetype] +#valid values are: daily, weekly, monthly, full, custom +type=full +full=full +monthly=1219200 +weekly=604800 +daily=86400 +12h=23200 +6h = 16600 +# If settype = custom set below line to the number of seconds you want updates for. +custom=4150 +cacheskip = 1 +refreshcache = 0 + +# Field names for series insert query +# field01: nextepisode, +# field02: previousepisode, +# field03: mazeurl, +# field04: tvdbid, +# field05: tvrageid, +# field06: genres, +# field07: tvmazeid, +# field08: mediumimage, +# field09: originalimage, +# field10: showlanguage, +# field11: seriesname, +# field12: webchannelcountrycode, +# field13: webchannelcountryname, +# field14: webchannelcountrytimezone, +# field15: webchannelid, +# field16: webchannelname, +# field17: networkcountrycode, +# field18: networkcountryname, +# field19: networkcountrytimezone, +# field20: networkid, +# field21: networkname, +# field22: premierdate, +# field23: showrating, +# field24: showruntime, +# field25: airdays, +# field26: scheduletime, +# field27: showstatus, +# field28: showoverview, +# field29: showtype, +# field30: showupdated, +# field31: apilink, +# field32: showweight diff --git a/tabledata_from_json.py b/tabledata_from_json.py new file mode 100644 index 0000000..ecdd3ac --- /dev/null +++ b/tabledata_from_json.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 + +from pymongo import MongoClient +import json +import psycopg2 +import pandas as pd +from collections.abc import MutableMapping +from tqdm import tqdm + + +def flatten_dict(d: MutableMapping, sep: str = '.') -> MutableMapping: + [flat_dict] = pd.json_normalize(d, sep=sep).to_dict(orient='records') + return flat_dict + +def delete_none(_dict): + """Delete None values recursively from all of the dictionaries, tuples, lists, sets""" + if isinstance(_dict, dict): + for key, value in list(_dict.items()): + if isinstance(value, (list, dict, tuple, set)): + _dict[key] = delete_none(value) + elif value is None or key is None: + del _dict[key] + + elif isinstance(_dict, (list, set, tuple)): + _dict = type(_dict)(delete_none(item) for item in _dict if item is not None) + + return _dict + +def process_data(data, tablename): + inserts = [] + failed_count = 0 + seriesconflict = "ON CONFLICT (seriesid) DO UPDATE SET" + epconflict = "ON CONFLICT (episodeid) DO UPDATE SET" + crewconflict = "ON CONFLICT (crewid) DO UPDATE SET" + for document in data: + if document['name'].lower() == 'too many requests': + print("Skipping document due to 'Too Many Requests' in " + "document data") + continue + else: + new_document = delete_none(document) + if '_id' in new_document: + del new_document['_id'] + # if tablename == 'epdata' and 'image' in document: + # del document['image'] + # if 'network' in document: + # if document['network'] is None: + # del document['network'] + # del document['image'] + try: + flatten_data = flatten_dict(new_document) + except TypeError as e: + print( + ( + f"Unable to flatten data for current document. {e}\n" + "Exitting!" + ) + ) + failed_count += 1 + else: + column_list = [] + data_list = [] + for key, value in flatten_data.items(): + if tablename == 'seriesdata' and ( + key.lower() == 'name' or key.lower() == 'type' + ): + key = f"series_{key}" + if tablename == 'seriesdata' and key.lower() == 'id': + key = "seriesid" + if tablename == 'crewdata' and key.lower() == 'id': + key = "crewid" + if ( + tablename == 'epdata' and ( + key.lower() == 'name' or key.lower() == 'number' or + key.lower() == 'type' + ) + ): + key = f"episode_{key}" + if key.lower() == 'language': + key = "language_name" + if tablename == 'epdata' and key.lower() == 'id': + key = "episodeid" + if ( + '_links' in key.lower() or + 'image' in key.lower() or + 'schedule' in key.lower() or + 'externals' in key.lower() or + 'network' in key.lower() or + 'webchannel' in key.lower() or + 'country' in key.lower() or + 'rating' in key.lower() + ): + key = key.replace('.', '_') + column_list.append(key) + if 'genres' in key.lower() or 'schedule' in key.lower(): + if isinstance(value, list): + value = ', '.join(value) + else: + value = str(value).replace("'", "''") + if isinstance(value, str): + value = value.replace("'", "''") + if isinstance(value, int): + value = f"'{value}'" + elif isinstance(value, float): + value = f"'{json.dumps(value)}'" + elif isinstance(value, dict): + value = f"'{json.dumps(value)}'" + elif isinstance(value, bool): + value = str(value).lower() + elif value is None: + value = 'NULL' + else: + value = str(f"'{value}'") + data_list.append(value) + columns = ", ".join(column_list) + column_data = ", ".join(data_list) + conflict_list = [] + for indexer, column in enumerate(column_list): + conflict_list.append(f"{column_list[indexer]} = {data_list[indexer]}") + conflict_data = ', '.join(conflict_list) + conflict_clause = '' + if tablename == 'seriesdata': + conflict_clause = seriesconflict + elif tablename == 'epdata': + conflict_clause = epconflict + elif tablename == 'crewdata': + conflict_clause = crewconflict + insert_command = ( + ( + f"Insert Into updates.{tablename} ({columns}) " + f"values ({column_data}) " + f"{conflict_clause} {conflict_data};" + ) + ) + inserts.append(insert_command) + return inserts, failed_count + + +try: + connection_string = ( + ( + 'postgres://postgres:Optimus0329@192.168.128.7:5432/media_dbsync' + '?options=-csearch_path%3Ddbo,public,updates' + ) + ) + pgclient = psycopg2.connect(connection_string) + pgcursor = pgclient.cursor() + #print(pgclient.get_dsn_parameters(), "\n") + print("Connected to postgresql database server.") + pgcursor.execute("SELECT version();") + record = pgcursor.fetchone() + print("You are connected to - ", record, "\n") + pgclient.set_isolation_level(0) +except Exception as e: + print(f"Unable to connect to postgresql database server:{e}\nExitting!") + exit() + + +host = '192.168.128.8' +port = 27017 +print(f"Connecting to mongodb at {host}:{port}") + +try: + connection_string = ( + f"mongodb://{host}:{port}" + ) + mgclient = MongoClient(connection_string) + mgdb = mgclient['test2'] +except BaseException as e: + print(f"Unable to connect to mongodb.{e}\nExitting!") + exit() +else: + mgseries = mgdb["series"] + mgseries.create_index('id', unique=True) + mgepisodes = mgdb["episodes"] + mgepisodes.create_index('id', unique=True) + mgactors = mgdb["actors"] + mgactors.create_index('id', unique=True) + mgcharacters = mgdb["characters"] + mgcharacters.create_index('id', unique=True) + mgcrew = mgdb["crew"] + mgcrew.create_index('id', unique=True) + print( + f"Connection to mongodb at {host}:" + f"{port} successful." + ) + +print("Creating Insert commands from collection data.") + +# print("Collecting series data for series inserts.") +# series_list = mgseries.find({}) +# inserts, failed_count = process_data(series_list, 'seriesdata') +# print(f"Total number of series inserts: {len(inserts)}") +# for i in tqdm(range(len(inserts))): +# try: +# insert = inserts[i] +# pgcursor.execute(insert) +# except Exception as e: +# # print(f"Unable to insert data into seriesdata table. {e}") +# failed_count += 1 +# print(f"Total number of failed series inserts: {failed_count}") +# pgclient.commit() + +# epinserts = [] +# failures = 0 +# episode_array = mgepisodes.find({}) +# print("Collecting episode data for episode inserts.") +# inserts, failed_count = process_data(episode_array, 'epdata') +# print(f"Total number of episode inserts: {len(inserts)}") +# for i in tqdm(range(len(inserts))): +# try: +# insert = inserts[i] +# pgcursor.execute(insert) +# except Exception as e: +# # print(f"Unable to insert data into epdata table. {e}") +# failed_count += 1 +# print(f"Total number of failed episode inserts: {failed_count}") +# pgclient.commit() + +failures = 0 +crew_array = mgcrew.find({}) +print("Collecting crew data for crew inserts.") +inserts, failed_count = process_data(crew_array, 'crewdata') +print(f"Total number of crew inserts: {len(inserts)}") +for i in tqdm(range(len(inserts))): + try: + insert = inserts[i] + pgcursor.execute(insert) + except Exception as e: + print(f"Unable to insert data into epdata table. {e}") + failed_count += 1 +print(f"Total number of failed crew inserts: {failed_count}") +pgclient.commit() + +pgclient.close() +print("PostgreSQL connection is closed") + + +exit() diff --git a/tableddl_from_yaml.py b/tableddl_from_yaml.py new file mode 100644 index 0000000..0d630d6 --- /dev/null +++ b/tableddl_from_yaml.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 + +import os +from pymongo import MongoClient +import yaml +import json +import psycopg2 + +class ColumnMeta: + def __init__(self, name, type, description=None): + self.name = name + self.type = type + self.description = description + + def from_dict(value: dict): + return ColumnMeta(name=value['name'], type=value['type'], description=value.get('description')) + + +class TableMeta: + def __init__(self, name, columns, description=None): + self.name = name + self.columns = columns + self.description = description + + + def from_dict(value): + columns = [ColumnMeta.from_dict(v) for v in value['columns']] + return TableMeta(name=value['name'], columns=columns, description=value.get('description')) + + def from_file(path): + file = open(path, 'r') + yaml_to_dict = yaml.safe_load(file.read()) + try: + file.close() + except Exception: + pass + return TableMeta.from_dict(yaml_to_dict) + + +try: + connection_string = ( + ( + 'postgres://postgres:Optimus0329@192.168.128.7:5432/media_dbsync' + '?options=-csearch_path%3Ddbo,public,updates' + ) + ) + pgclient = psycopg2.connect(connection_string) + pgcursor = pgclient.cursor() + print(pgclient.get_dsn_parameters(), "\n") + print("Connected to postgresql database server.") + pgcursor.execute("SELECT version();") + record = pgcursor.fetchone() + print("You are connected to - ", record, "\n") +except Exception as e: + print(f"Unable to connect to postgresql database server:{e}\nExitting!") + exit() + + +host = '192.168.128.8' +port = 27017 +print(f"Connecting to mongodb at {host}:{port}") + +try: + connection_string = ( + f"mongodb://{host}:{port}" + ) + mgclient = MongoClient(connection_string) + mgdb = mgclient['test2'] +except BaseException as e: + print(f"Unable to connect to mongodb.{e}\nExitting!") + exit() +else: + mgseries = mgdb["series"] + mgseries.create_index('id', unique=True) + mgepisodes = mgdb["episodes"] + mgepisodes.create_index('id', unique=True) + mgactors = mgdb["actors"] + mgactors.create_index('id', unique=True) + mgcharacters = mgdb["characters"] + mgcharacters.create_index('id', unique=True) + mgcrew = mgdb["crew"] + mgcrew.create_index('id', unique=True) + print( + f"Connection to mongodb at {host}:" + f"{port} successful." + ) +os.chdir('C:\\Projects\\new_dbsync\\db\\ddl') +# Create the database and tables in PostgreSQL +# Check if the database exists, if not create it +# Check if the tables exist, if not create them +# possibly add , 'genre_ddl.yaml', 'language_ddl.yaml' down the road +ddl_list = [ + # 'seriesdata_ddl.yaml', + # 'epdata_ddl.yaml', + # 'actordata_ddl.yaml', + # 'castdata_ddl.yaml', + # 'crewdata_ddl.yaml', + # 'countrydata_ddl.yaml', + # 'networkdata_ddl.yaml', + # 'webchanneldata_ddl.yaml', + # 'temp_seriesdata_ddl.yaml', + # 'temp_epdata_ddl.yaml', + # 'temp_actordata_ddl.yaml', + # 'temp_castdata_ddl.yaml', + # 'temp_crewdata_ddl.yaml', + # 'temp_countrydata_ddl.yaml', + # 'temp_networkdata_ddl.yaml', + # 'temp_webchanneldata_ddl.yaml', + # 'tvupdates_ddl.yaml' + 'db_schema.yaml' +] +for ddl in ddl_list: + path = os.path.join(os.getcwd(), ddl) + table_meta = TableMeta.from_file(path) + + columnlist = [] + for column in table_meta.columns: + columnlist.append(f"{column.name} {column.type}") + table_meta.columns = ', '.join(columnlist) + #print(column.name, column.type) + + sql_command = f"DROP TABLE IF EXISTS {table_meta.name};CREATE TABLE IF NOT EXISTS {table_meta.name} ({table_meta.columns})" + #sql_command = f"CREATE TABLE IF NOT EXISTS {table_meta.name} ({table_meta.columns})" + + try: + pgcursor.execute(sql_command) + pgclient.commit() + except Exception as e: + print(f"Unable to create table {table_meta.name} in postgresql database server:{e}\nExitting!") + exit() + #print(sql_command) + +pgcursor.close() +pgclient.close() +print("PostgreSQL connection is closed") +mgclient.close() +print("MongoDB connection is closed") +exit() diff --git a/test_tableddl_from_yaml.py b/test_tableddl_from_yaml.py new file mode 100644 index 0000000..9240194 --- /dev/null +++ b/test_tableddl_from_yaml.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 + +import os +from pymongo import MongoClient +import yaml +import json +import psycopg2 + +class ColumnMeta: + def __init__(self, name, type, description=None): + self.name = name + self.type = type + self.description = description + + def from_dict(value: dict): + return ColumnMeta(name=value['name'], type=value['type'], description=value.get('description')) + + +class TableMeta: + def __init__(self, name, columns, description=None): + self.name = name + self.columns = columns + self.description = description + + def from_dict(value): + columns = [ColumnMeta.from_dict(v) for v in value['columns']] + return TableMeta(name=value['name'], columns=columns, description=value.get('description')) + + def from_file(path): + file = open(path, 'r') + yaml_to_dict = yaml.safe_load(file.read()) + try: + file.close() + except Exception: + pass + return TableMeta.from_dict(yaml_to_dict) + + +try: + connection_string = ( + ( + 'postgres://postgres:Optimus0329@192.168.128.7:5432/media_dbsync' + '?options=-csearch_path%3Ddbo,public,updates' + ) + ) + pgclient = psycopg2.connect(connection_string) + pgcursor = pgclient.cursor() + print(pgclient.get_dsn_parameters(), "\n") + print("Connected to postgresql database server.") + pgcursor.execute("SELECT version();") + record = pgcursor.fetchone() + print("You are connected to - ", record, "\n") +except Exception as e: + print(f"Unable to connect to postgresql database server:{e}\nExitting!") + exit() + + +host = '192.168.128.8' +port = 27017 +print(f"Connecting to mongodb at {host}:{port}") + +try: + connection_string = ( + f"mongodb://{host}:{port}" + ) + mgclient = MongoClient(connection_string) + mgdb = mgclient['test2'] +except BaseException as e: + print(f"Unable to connect to mongodb.{e}\nExitting!") + exit() +else: + mgseries = mgdb["series"] + mgseries.create_index('id', unique=True) + mgepisodes = mgdb["episodes"] + mgepisodes.create_index('id', unique=True) + mgactors = mgdb["actors"] + mgactors.create_index('id', unique=True) + mgcharacters = mgdb["characters"] + mgcharacters.create_index('id', unique=True) + mgcrew = mgdb["crew"] + mgcrew.create_index('id', unique=True) + print( + f"Connection to mongodb at {host}:" + f"{port} successful." + ) +os.chdir('C:\\Projects\\new_dbsync\\db\\ddl') +# Create the database and tables in PostgreSQL +# Check if the database exists, if not create it +# Check if the tables exist, if not create them +# possibly add , 'genre_ddl.yaml', 'language_ddl.yaml' down the road +ddl_list = os.listdir(os.getcwd()) +for ddl in ddl_list: + path = os.path.join(os.getcwd(), ddl) + table_meta = TableMeta.from_file(path) + + columnlist = [] + for column in table_meta.columns: + columnlist.append(f"{column.name} {column.type}") + table_meta.columns = ', '.join(columnlist) + #print(column.name, column.type) + + sql_command = f"DROP TABLE IF EXISTS {table_meta.name};CREATE TABLE IF NOT EXISTS {table_meta.name} ({table_meta.columns})" + #sql_command = f"CREATE TABLE IF NOT EXISTS {table_meta.name} ({table_meta.columns})" + + try: + pgcursor.execute(sql_command) + pgclient.commit() + except Exception as e: + print(f"Unable to create table {table_meta.name} in postgresql database server:{e}\nExitting!") + exit() + #print(sql_command) + +pgcursor.close() +pgclient.close() +print("PostgreSQL connection is closed") +mgclient.close() +print("MongoDB connection is closed") +exit() diff --git a/update.py b/update.py new file mode 100644 index 0000000..b9a6ebf --- /dev/null +++ b/update.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import os +import time +import url.urls +import url.process +import api.tvmaze.cast as mazecast +import api.tvmaze.crew as mazecrew +import api.tvmaze.episodes as mazeepisodes +import api.tvmaze.series as mazeseries +import db.sql +import jprocessor.process +import logs.Logger +from db.functions import settype, dbmongo +from datetime import datetime + +response = None + + +def set_apienv(urls, dbExec, updatesBase): + updatetable = updatesBase[8] + inputdata = uprocess.get_data(urls["updatesurl"]) + availableupdates = jget.jconvert(inputdata) + print(f"Retrieved {len(availableupdates)} rows for processing......") + dbExec.update_tvupdates(dbengine["engine"], availableupdates, updatetable) + newupdates = dbExec.rawsql_select( + dbengine["engine"], + "select seriesid,timestamp from updates.tvupdates", + lprint + ) + return newupdates + + +def print_starttime(lprint, ts1): + lprint.logprint("info", f"Sync start time: {str(ts1)}") + + +def get_tvdb(): + tvdb_urls = TVDBURLS.get_data() + return tvdb_urls + + +def get_tvmaze(): + tvmaze_urls = TVMAZEURLS.get_data() + return tvmaze_urls + + +if __name__ == "__main__": + ROOTDIR = os.getcwd() + directories = {} + if os.name == "nt": + LOGDIR = f"{ROOTDIR}\\log" + directories["SERIESDIR"] = f"{ROOTDIR}\\cache\\series\\" + directories["EPISODEDIR"] = f"{ROOTDIR}\\cache\\episodes\\" + directories["CASTDIR"] = f"{ROOTDIR}\\cache\\cast\\" + directories["CREWDIR"] = f"{ROOTDIR}\\cache\\crew\\" + else: + LOGDIR = f"{ROOTDIR}/log" + directories["SERIESDIR"] = f"{ROOTDIR}/cache/series/" + directories["EPISODEDIR"] = f"{ROOTDIR}/cache/episodes/" + directories["CASTDIR"] = f"{ROOTDIR}/cache/cast/" + directories["CREWDIR"] = f"{ROOTDIR}/cache/crew/" + lprint = logs.Logger.LPrint(LOGDIR, False) + lprint.start_log() + import settings.config + + config_options = settings.config.Config(ROOTDIR) + options = config_options.config_options + dbtype, apitype = options["dbtype"], options["apitype"] + dbs = settype(dbtype, apitype, options) + dbclass, dbengine, apiengine = dbs.dbclass, dbs.dbengine, dbs.apiengine + uprocess = url.process.Loadurl() + TVMAZEURLS = url.urls.Tvmazeurls() + TVMAZE_URLS = get_tvmaze() + TVDBURLS = url.urls.TVDBurls() + TVDB_URLS = get_tvdb() + mseries = mazeseries.tvshow() + mcast = mazecast.cast() + mepisode = mazeepisodes.episodes() + mcrew = mazecrew.crew() + countrydictionary = {} + updateRange = None + + lprint.logprint("info", "Creating JSON instance for data retrieval.") + jget = jprocessor.process.Jsonload() + lprint.logprint("info", "JSON instance created.") + print(f"Update type is {options['updatetype']}") + timedifference = options["updatetype"] + currenttimestamp = time.time() + if timedifference.lower() == "full": + timedifference = int(currenttimestamp) + basetime = str(currenttimestamp).split(".") + currenttimestamp = int(basetime[0]) + ts1 = str(datetime.now()).split(".")[0] + print("Sync start time: " + str(ts1)) + print_starttime(lprint, ts1) + updatesBase = dbengine["updatesBase"] + dboBase = dbengine["dboBase"] + session = dbengine["session"] + dbExec = db.sql.Dbexec() + createTables = dbclass["Createtables"] + Tables = dbclass["Tables"] + createTempTables = dbclass["CreateTemptables"] + new_updates = {} + sqlexec = db.sql.SQLGenerate() + mt = createTables( + dbengine["session"], + dbengine["metadata"], + dbengine["engine"] + ) + ct = createTempTables( + dbengine["session"], + dbengine["metadata"], + dbengine["engine"] + ) + tempTableList = ct.tablelist + + if options["apitype"].lower() == "tvmaze": + new_updates = set_apienv(TVMAZE_URLS, dbExec, tempTableList) + from cache.process import Cacher + + cacher = Cacher() + cacher.process(options, jget, new_updates, ROOTDIR, TVMAZE_URLS) + timeDifference = options["updatetype"] + currentTimestamp = time.time() + baseTime = str(currentTimestamp).split(".") + if timeDifference.lower() == "full" or options["initload"] == "1": + tdifference = 0 + else: + tdifference = int(currentTimestamp) - int( + options[options["updatetype"]] + ) + needed_updates = [] + updateList = [] + for result in new_updates: + updateid = result[0] + updatestamp = int(result[1]) + tdif = int(tdifference) + if updatestamp > tdif: + needed_updates.append(updateid) + updateList.append(updateid) + + print("Update Type: " + str(options["updatetype"])) + updateRange = len(updateList) + print("Number of updates: " + str(updateRange)) + cacher.update_cache( + jget, updateList, ROOTDIR, TVMAZE_URLS, options, lprint + ) + countrylisting = dbExec.rawsql_select( + dbengine["engine"], + "select row_id, country_name from dbo.countrydata", + lprint + ) + for country in countrylisting: + country_name = country[1] + country_id = country[0] + countrydictionary[country_name] = country_id + mongo_updater = dbmongo(options) + mongo_updater.update_mongo( + updateList, + ROOTDIR, + directories, + jget, + countrydictionary, + dbExec, + dbengine, + dboBase, + lprint, + ) +exit(0) diff --git a/url/__init__.py b/url/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/url/process.py b/url/process.py new file mode 100644 index 0000000..1334e57 --- /dev/null +++ b/url/process.py @@ -0,0 +1,17 @@ +__author__ = 'Wendell Jones' +import requests + +class Loadurl(): + def __init__(self): + pass + + def get_data(self,weburl): + try: + showresponse = requests.get(weburl) + except requests.exceptions.ConnectionError as e: + print('Unable to connect and get JSON data for url ' + str(weburl)) + else: + return(showresponse.text) + + def __del__(self): + pass diff --git a/url/urls.py b/url/urls.py new file mode 100644 index 0000000..b114b9c --- /dev/null +++ b/url/urls.py @@ -0,0 +1,60 @@ +__author__ = 'Wendell Jones' +global urls + + +class Tvmazeurls(object): + + def __init__(self): + self.urls = {} + self.set_data() + + + def set_data(self): + self.urls['showurl'] = 'http://api.tvmaze.com/shows/' + self.urls['serieslisturl'] = 'http://api.tvmaze.com/shows?page=' + self.urls['casturl'] = 'http://api.tvmaze.com/shows//cast' + self.urls['crewurl'] = 'https://api.thetvdb.com/series//crew' + self.urls['creditsurl'] = 'https://api.thetvdb.com/series//castcredits' + self.urls['episodesurl'] = 'http://api.tvmaze.com/shows//episodes' + self.urls['aliasurl'] = 'http://api.tvmaze.com/shows//akas' + self.urls['creditsurl'] = 'http://api.tvmaze.com/people//castcredits' + self.urls['updatesurl'] = 'http://api.tvmaze.com/updates/shows' + self.urls['crewurl'] = 'http://api.tvmaze.com/shows//crew' + + + def get_data(self): + return self.urls + + def __del__(self): + pass + +class TVDBurls(object): + + def __init__(self): + self.urls = {} + self.set_data() + + def set_data(self): + self.urls['loginurl'] = 'https://api.thetvdb.com/login' + self.urls['refreshurl'] = 'https://api.thetvdb.com/refresh_token' + self.urls['showurl'] = 'https://api.thetvdb.com/series/{id}' + self.urls['episodeurl'] = 'https://api.thetvdb.com/episodes/{id}' + self.urls['casturl'] = 'https://api.thetvdb.com/series/{id}/actors' + self.urls['crewurl'] = 'https://api.thetvdb.com/series/{id}/crew' + self.urls['creditsurl'] = 'https://api.thetvdb.com/series/{id}/castcredits' + self.urls['episodesurl'] = 'https://api.thetvdb.com/series/{id}/episodes' + self.urls['episodequeryurl'] = 'https://api.thetvdb.com/series/{id}/episodes/query' + self.urls['episodeparamurl'] = 'https://api.thetvdb.com/series/{id}/episodes/query/params' + self.urls['episodesummaryurl'] = 'https://api.thetvdb.com/series/{id}/episodes/summary' + self.urls['filterurl'] = 'https://api.thetvdb.com/series/{id}/filter' + self.urls['filterparamurl'] = 'https://api.thetvdb.com/series/{id}/filter/params' + self.urls['imageurl'] = 'https://api.thetvdb.com/series/{id}/images' + self.urls['imagequeryurl'] = 'https://api.thetvdb.com/series/{id}/images/query' + self.urls['imageparamurl'] = 'https://api.thetvdb.com/series/{id}/images/query/params' + self.urls['username'] = 'wpjonesnhcjc' + self.urls['apikey'] = 'NACCS2LUWCFEFABX' + self.urls['userkey'] = 'UUSZHUQ8CY7N31CF' + + + def get_data(self): + return self.urls \ No newline at end of file diff --git a/view.py b/view.py new file mode 100644 index 0000000..46cd809 --- /dev/null +++ b/view.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import os +import time +import json +from pathlib import Path +import db.sql +from jprocessor.process import Jsonload +from jprocessor.process_view import Process_View +import logs.Logger +from db.functions import settype, dbmongo + +if __name__ == "__main__": + seriesid = input('Please enter seriesid: ') + #seriesname = input('Please enter series name: ') + ROOTDIR = os.getcwd() + directories = {} + if os.name == 'nt': + LOGDIR = f'{ROOTDIR}\\log' + directories['SERIESDIR'] = f'{ROOTDIR}\\cache\\series\\' + directories['EPISODEDIR'] =f'{ROOTDIR}\\cache\\episodes\\' + directories['CASTDIR'] = f'{ROOTDIR}\\cache\\cast\\' + directories['CREWDIR'] = f'{ROOTDIR}\\cache\\crew\\' + else: + LOGDIR = f'{ROOTDIR}/log' + directories['SERIESDIR'] = f'{ROOTDIR}/cache/series/' + directories['EPISODEDIR'] =f'{ROOTDIR}/cache/episodes/' + directories['CASTDIR'] = f'{ROOTDIR}/cache/cast/' + directories['CREWDIR'] = f'{ROOTDIR}/cache/crew/' + lprint = logs.Logger.lprint(LOGDIR, False) + lprint.startlog() + import settings.config + config_options = settings.config.Config(ROOTDIR) + options = config_options.config_options + mongodb = dbmongo(options) + jload = Jsonload() + viewprocess = Process_View() + #serieslist = mongodb.mgseries.find() + #for series in serieslist: + # print(series) + series = mongodb.mgseries.find_one({"id": int(seriesid)}) + episodecursor = mongodb.mgepisodes.find({"seriesid": int(seriesid)}) + episodes = [] + for episode in episodecursor: + episodes.append(episode) + viewprocess.process_seriesdata(series) + viewprocess.print_episodedata(episodes) + exit() + \ No newline at end of file