repo migration

.gitattributes

@@ -0,0 +1 @@
+*.json filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,15 @@
+################################################################################
+# This .gitignore file was automatically created by Microsoft(R) Visual Studio.
+################################################################################
+
+/.vs/new_dbsync/FileContentIndex
+/.vs
+*.pyc
+*.json
+*.DS_Store
+*.log
+/pyproject.toml
+dbsync.log.*
+.idea
+/log
+venv

README.md

@@ -0,0 +1,8 @@
+# Sample GitLab Project
+
+This sample project shows how a project in GitLab looks for demonstration purposes. It contains issues, merge requests and Markdown files in many branches,
+named and filled with lorem ipsum.
+
+You can look around to get an idea how to structure your project and, when done, you can safely delete this project.
+
+[Learn more about creating GitLab projects.](https://docs.gitlab.com/ee/gitlab-basics/create-project.html)

api/thetvdb/cast.py

@@ -0,0 +1,14 @@
+
+class cast():
+    
+    def __init__(self):
+        pass
+
+    def listcast(self, seriesid):
+        pass
+    
+    def listcredits(self, seriesid):
+        pass
+    
+    def __del__(self):
+        pass

api/thetvdb/crew.py

@@ -0,0 +1,15 @@
+
+class crew():
+    
+    def __init__(self):
+        pass
+
+    def listcrew(self, seriesid):
+        pass
+    
+    def listcredits(self, seriesid):
+        pass
+    
+    def __del__(self):
+        pass
+    

api/thetvdb/episodes.py

@@ -0,0 +1,18 @@
+
+class episodes():
+    
+    def __init__(self):
+        pass
+
+    def fulllisting(self, seriesid):
+        pass
+    
+    def seasonlist(self, seriesid, seasonnumber):
+        pass
+    
+    def eplist(self, seriesid, seasonnumber, episodenumber):
+        pass
+    
+    def __del__(self):
+        pass
+    

api/thetvdb/process.py

@@ -0,0 +1,256 @@
+
+class show():
+    
+    def __init__(self):
+        pass
+    
+    def process(self, metadata, showload, seriesid):
+        dataresult = []
+        tableresult = []
+        if showload == None:
+            pass
+        else:
+            networkid = None
+            try:
+                network = {}
+                for key, value in showload['network'].items():
+                    if key == 'country':
+                        for key, value in showload['network']['country'].items():
+                            if key == 'name':
+                                newkey = 'country'
+                                network[newkey] = showload['network']['country'][key]
+                            else:
+                                newkey = 'country' + key
+                                network[newkey] = showload['network']['country'][key]
+                    else:
+                        newkey = 'network' + key
+                        network[newkey] = showload['network'][key]
+            except:
+                #print('Failed to insert network information.')
+                showload['networkid'] = 0
+            else:    
+                showload.pop('network')
+                showload['networkid'] = network['networkid']
+                dataresult.append(network)
+                tableresult.append(metadata.tables['[updates].tvnetworknewvals'])
+                #insertresult = insert_data(network, tables[4])
+            try:
+                webchannel = {}
+                for key, value in showload['webChannel'].items():
+                    if key == 'country':
+                            if showload['webChannel'][key] is None:
+                                pass
+                            else:
+                                for key, value in showload['webChannel']['country'].items():
+                                    newkey = 'webChannelcountry'+ key
+                                    webchannel[newkey] = value 
+                    else:
+                        webchannel[key] = value
+                if key == 'name':
+                    newkey = 'webChannel' + key
+                    webhchannel[newkey] = value 
+                if key == 'timezone':
+                    newkey = 'webChannelcountry'+ key
+                    webchannel[newkey] = value
+            except:
+                #print('Failed to insert webchannel information.')
+                showload['webchannelid'] = '0'
+            else:
+                showload.pop('webChannel')
+                showload['webchannelid'] = webchannel['id']
+                webchannel['webchannelname'] = webchannel['name']
+                webchannel['networkid'] = webchannel['id']
+                try:
+                    dataresult.append(webchannel)
+                    tableresult.append(metadata.tables['[updates].tvwebchannelnewvals'])
+                    #insertresult = insert_data(webchannel, tables[5])
+                except sqlalchemy.exc.IntegrityError as e:
+                    pass
+                except Exception as e:
+                    print('Failed to insert webchannel information.')
+            try:
+                showload['seriesid'] = showload['id']
+            except:
+                pass
+            
+            #print('TV Show: ' + showload['seriesname'])
+            
+            try:
+                if showload['image']:
+                    for key, value in showload['image'].items():
+                        newkey = key + 'image' 
+                        showload[newkey] = value
+                    showload.pop('image')
+                else:
+                    showload.pop('image')
+            except:
+                pass
+            
+            try:
+                if showload['externals']:
+                    for key, value in showload['externals'].items():
+                        showload[key] = value
+                    showload.pop('externals')
+                else:
+                    showload.pop('externals')
+            except KeyError as e:
+                pass
+            
+            try:
+                if showload['rating']:
+                    for key, value in showload['rating'].items():
+                        showload['rating'] = value
+                
+                if showload['schedule']:
+                    showload['time'] = showload['schedule']['time']
+                    showload['days'] = ", ".join(showload['schedule']['days'])
+                    showload.pop('schedule')
+            except KeyError as e:
+                pass
+                   
+            try:
+                for key, value in showload['_links']['self'].items():
+                    showload[key] = value    
+                showload.pop('_links')
+            except KeyError as e:
+                pass
+            else:
+                showload.pop('id')
+            
+            showload['genres'] = ", ".join(showload['genres'])
+            
+            # Finish transforming to match table columns
+            showload['seriesname'] = showload.pop('name')
+            showload['tvrageid'] = showload.pop('tvrage')
+            showload['tvdbid'] = showload.pop('thetvdb')
+            showload['imdbid'] = showload.pop('imdb')
+            showload['serieslanguage'] = showload.pop('language')
+            showload['seriesstatus'] = showload.pop('status')
+            showload['airdays'] = showload.pop('days')
+            showload['airtime'] = showload.pop('time')
+            showload['showtype'] = showload.pop('type')
+            showload['overview'] = showload.pop('summary')
+            showload['showurl'] = showload.pop('url')
+            showload['apilink'] = showload.pop('href')
+            showload['officialsite'] = showload.pop('officialSite')
+            
+            results = []
+            results.append([dataresult, tableresult, showload])
+            #results.append(tableresult)
+            #results.append(showload)
+            return (results)
+
+        
+    def __del__(self):
+        pass
+
+class cast():
+    def __init__(self):
+        pass
+        
+    def process(self, cdata, seriesid):
+        datalist = {}
+        datalist['actors'] = []
+        datalist['characters'] = []
+        try:
+            castload = json.loads(cdata)
+            for idx, castdata in enumerate(castload):
+                actordata = {}
+                characterdata = {}
+                tcharacterdata = castload[idx]['character']
+                tactordata = castload[idx]['person']
+                try:
+                    for key, value in tactordata.items():    
+                        if tactordata[key] is None:
+                            pass
+                        else:
+                            if key == 'image':
+                                for key, value in tactordata['image'].items():
+                                    newkey = key + 'image' 
+                                    actordata[newkey] = tactordata['image'][key]
+                            elif key == '_links':
+                                for key, value in tactordata['_links']['self'].items():
+                                    actordata['apilink'] = tactordata['_links']['self'][key]    
+                            elif key == 'country':
+                                for key, value in tactordata['country'].items():
+                                    newkey = 'country' + key
+                                    actordata[newkey] = tactordata['country'][key]    
+                            else:
+                                actordata[key] = tactordata[key]
+                            actordata['actorid'] = tactordata['id']
+                            actordata['actorname'] = tactordata['name']
+                except:
+                    print('Unable to process actor information.')
+                else:
+                    actordata.pop('id')
+                
+                try:
+                    if tcharacterdata:
+                        for key, value in tcharacterdata.items():    
+                            if key is None:
+                                pass
+                            else:
+                                if tcharacterdata['image']:
+                                    characterdata['mediumimage'] = tcharacterdata['image']['medium']
+                                    characterdata['originalimage'] = tcharacterdata['image']['original']
+                                characterdata['apilink'] = tcharacterdata['_links']['self']['href']    
+                                characterdata['tvmazeilink'] = tcharacterdata['url']
+                                characterdata['characterid'] = tcharacterdata['id']
+                                characterdata['charactername'] = tcharacterdata['name']    
+                                characterdata['seriesid'] = seriesid
+                                characterdata['actorid'] = actordata['actorid']
+                except Exception as e:
+                    print('Unable to process character information. -> ' + str(e))
+                    
+                datalist['characters'].append(characterdata)
+                datalist['actors'].append(actordata)
+                linkdata = {}
+                linkdata['actorid'] = actordata['actorid']
+                linkdata['characterid'] = characterdata['characterid']
+                #return(datalist)
+        except Exception as e:
+            #print('Line 389: ' + str(e))
+            pass
+        else:
+            return(datalist)
+    def __del__(self):
+        pass
+
+class Episode():
+    
+    def __init__(self):
+        pass
+        
+    def process(self, epload, seriesid):
+        if epload:
+            eplist = []
+            try:
+                for epdata in epload:
+                    epdata['seriesid'] = seriesid
+                    if epdata['image']:
+                        for key, value in epdata['image'].items():
+                            newkey = key + 'image' 
+                            epdata[newkey] = value
+                        epdata.pop('image')
+                    else:
+                        epdata['mediumimage'] = 'None'
+                        epdata['originalimage'] = 'None'
+                        epdata.pop('image')
+                    
+                    for key, value in epdata['_links']['self'].items():
+                        epdata['eplink'] = value    
+                    epdata.pop('_links')
+                    
+                    epdata['episodeid'] = epdata.pop('id')
+                    epdata['episodename'] = epdata.pop('name')
+                    epdata['episodesummary'] = epdata.pop('summary')
+                    epdata['episodenumber'] = epdata.pop('number')
+                    eplist.append(epdata)
+                return eplist
+            except:
+                print('Unable to import episode data for for seriesid: ' + str(seriesid))
+        else:
+            return None
+
+    def __del__(self):
+        pass

api/thetvdb/series.py

@@ -0,0 +1,18 @@
+
+class series():
+    
+    def __init__(self):
+        pass
+    
+    def search(self, searchquery):
+        pass
+    
+    def singlesearch(self, searchquery):
+        pass
+    
+    def idsearch(self, searchid):
+        pass
+    
+    def __del__(self):
+        pass
+

api/thetvdb/thetvdb.py

@@ -0,0 +1,231 @@
+import requests
+import json
+
+class Processor():
+    def __init__(self):
+        global loginurl
+        global refreshurl
+        global seriesurl
+        global actorurl
+        global episodeurl
+        global episodequeryurl
+        global episodeparamurl
+        global episodesummaryurl
+        global filterurl
+        global filterparamurl
+        global imageurl
+        global imagequeryurl
+        global imageparamurl
+        global username
+        global apikey
+        global userkey
+        global jwtoken
+        global refreshtoken
+        loginurl = 'https://api.thetvdb.com/login'
+        refreshurl = 'https://api.thetvdb.com/refresh_token'
+        seriesurl = 'https://api.thetvdb.com/series/{id}'
+        episodeidurl = 'https://api.thetvdb.com/episodes/{id}'
+        actorurl='https://api.thetvdb.com/series/{id}/actors'
+        episodeurl='https://api.thetvdb.com/series/{id}/episodes'
+        episodequeryurl='https://api.thetvdb.com/series/{id}/episodes/query'
+        episodeparamurl='https://api.thetvdb.com/series/{id}/episodes/query/params'
+        episodesummaryurl='https://api.thetvdb.com/series/{id}/episodes/summary'
+        filterurl='https://api.thetvdb.com/series/{id}/filter'
+        filterparamurl='https://api.thetvdb.com/series/{id}/filter/params'
+        imageurl='https://api.thetvdb.com/series/{id}/images'
+        imagequeryurl='https://api.thetvdb.com/series/{id}/images/query'
+        imageparamurl='https://api.thetvdb.com/series/{id}/images/query/params'
+        username = 'wpjonesnhcjc'
+        apikey = 'NACCS2LUWCFEFABX'
+        userkey = 'UUSZHUQ8CY7N31CF'
+        #jwtoken = login.run(self)
+        
+    def __del__(self):
+        pass
+
+class login():
+    def __init__(self):
+        pass
+    
+    def run(self):    
+        with requests.post(loginurl,  json={"username": username, "apikey": apikey, "userkey": userkey}) as r:
+            response = r.json()
+        jwtoken = response['token']
+        #refreshtoken = self.refresh(jwtoken)
+        return(jwtoken)
+    
+    def refresh(self, token):
+        token= "Bearer " + token
+        with requests.get(refreshurl, headers={'Authorization': token}) as r:
+            response = r.json()
+        refreshtoken = response['token']
+        return(refreshtoken)
+    
+class loadjson():
+    def __init__(self):
+        pass
+    
+    def loadseries(seriesid, token):
+        url = seriesurl.replace('{id}', seriesid)
+        #print(str(url))
+        seriesdata = loadjson.load(url, token)
+        return(seriesdata)
+
+    def loadepisodes(seriesid, token):
+        url = episodeurl.replace('{id}', seriesid)
+        #print(str(url))
+        seriesdata = loadjson.load(url, token)
+        return(seriesdata)
+    
+    def load(url, token):
+        token= "Bearer " + token
+        with requests.get(url, headers={'Authorization': token}) as r:
+            response = r.json()
+        return(response)
+        
+    def insert(self, updater, tablename):
+        for key, value in updater.items():
+            ins = tablename.insert().values( key = value )
+            print(str(ins))
+        result = engine.execute(ins)
+    
+    def populate(self, jsonurl, seriesid):
+        try:
+            with urllib.request.urlopen(jsonurl) as response:
+                html = response.read()
+            jsoninput = html
+            return jsoninput
+        except:
+           pass
+        
+    def __del__(self):
+        pass
+
+class showprocess():
+    
+    def __init__(self):
+        pass
+    
+    def process(self, metadata, tempshowload):
+        dataresult = []
+        tableresult = []
+        if tempshowload == None:
+            pass
+        else:
+            showload = json.loads(tempshowload)
+            networkid = None
+            try:
+                network = {}
+                for key, value in showload['network'].items():
+                    if key == 'country':
+                        for key, value in showload['network']['country'].items():
+                            if key == 'name':
+                                newkey = 'country'
+                                network[newkey] = showload['network']['country'][key]
+                            else:
+                                newkey = 'country' + key
+                                network[newkey] = showload['network']['country'][key]
+                    else:
+                        newkey = 'network' + key
+                        network[newkey] = showload['network'][key]
+            except:
+                #print('Failed to insert network information.')
+                showload['networkid'] = '0'
+            else:    
+                showload.pop('network')
+                showload['networkid'] = network['networkid']
+                dataresult.append(network)
+                tableresult.append(metadata.tables['[updates].tvnetworknewvals'])
+                #insertresult = insert_data(network, tables[4])
+            try:
+                webchannel = {}
+                for key, value in showload['webChannel'].items():
+                    if key == 'country':
+                            if showload['webChannel'][key] is None:
+                                pass
+                            else:
+                                for key, value in showload['webChannel']['country'].items():
+                                    newkey = 'webChannelcountry'+ key
+                                    webchannel[newkey] = value 
+                    else:
+                        webchannel[key] = value
+                if key == 'name':
+                    newkey = 'webChannel' + key
+                    webhchannel[newkey] = value 
+                if key == 'timezone':
+                    newkey = 'webChannelcountry'+ key
+                    webchannel[newkey] = value
+            except:
+                #print('Failed to insert webchannel information.')
+                showload['webchannelid'] = '0'
+            else:
+                showload.pop('webChannel')
+                showload['webchannelid'] = webchannel['id']
+                webchannel['webchannelname'] = webchannel['name']
+                try:
+                    dataresult.append(webchannel)
+                    tableresult.append(metadata.tables['[updates].tvwebchannelnewvals'])
+                    #insertresult = insert_data(webchannel, tables[5])
+                except sqlalchemy.exc.IntegrityError as e:
+                    pass
+                except Exception as e:
+                    print('Failed to insert webchannel information.')
+            try:
+                showload['seriesid'] = showload['id']
+            except:
+                pass
+            
+            #print('TV Show: ' + showload['seriesname'])
+            
+            if showload['image']:
+                for key, value in showload['image'].items():
+                    newkey = key + 'image' 
+                    showload[newkey] = value
+                showload.pop('image')
+            else:
+                showload.pop('image')
+            
+            if showload['externals']:
+                for key, value in showload['externals'].items():
+                    showload[key] = value
+                showload.pop('externals')
+            else:
+                showload.pop('externals')
+            
+            if showload['rating']:
+                for key, value in showload['rating'].items():
+                    showload['rating'] = value
+            
+            if showload['schedule']:
+                showload['time'] = showload['schedule']['time']
+                showload['days'] = ", ".join(showload['schedule']['days'])
+                showload.pop('schedule')
+                
+            for key, value in showload['_links']['self'].items():
+                showload[key] = value    
+            showload.pop('_links')
+            
+            showload.pop('id')
+            
+            showload['genres'] = ", ".join(showload['genres'])
+            
+            # Finish transforming to match table columns
+            showload['seriesname'] = showload.pop('name')
+            showload['tvrageid'] = showload.pop('tvrage')
+            showload['tvdbid'] = showload.pop('thetvdb')
+            showload['imdbid'] = showload.pop('imdb')
+            showload['serieslanguage'] = showload.pop('language')
+            showload['seriesstatus'] = showload.pop('status')
+            showload['airdays'] = showload.pop('days')
+            showload['airtime'] = showload.pop('time')
+            showload['showtype'] = showload.pop('type')
+            showload['overview'] = showload.pop('summary')
+            showload['showurl'] = showload.pop('url')
+            showload['apilink'] = showload.pop('href')
+            showload['officialsite'] = showload.pop('officialSite')
+            
+            results = []
+            results.append(dataresult)
+            results.append(tableresult)
+            results.append(showload)
+            return (results)

api/tvmaze/cast.py

@@ -0,0 +1,105 @@
+from sqlalchemy.sql.expression import false
+
+class cast():
+    
+    def __init__(self):
+        pass
+        
+    def process(self, castdata, seriesid):
+        datalist = {}
+        linkdata = {}
+        datalist['actors'] = []
+        datalist['characters'] = []
+        linklist = []
+        for castmember in castdata:
+            actordata = {}
+            characterdata = {}
+            tcharacterdata = castmember['character']
+            tactordata = castmember['person']
+            tselfdata = castmember['self']
+            tvoicedata = castmember['voice']
+            try:
+                for key, value in tactordata.items():    
+                    if tactordata[key] is None:
+                        pass
+                    else:
+                        if key == 'image':
+                            for key, value in tactordata['image'].items():
+                                newkey = key + 'image' 
+                                actordata[newkey] = tactordata['image'][key]
+                        elif key == '_links':
+                            for key, value in tactordata['_links']['self'].items():
+                                actordata['apilink'] = tactordata['_links']['self'][key]    
+                        elif key == 'country':
+                            for key, value in tactordata['country'].items():
+                                newkey = 'country' + key
+                                actordata[newkey] = tactordata['country'][key]    
+                        else:
+                            actordata[key] = tactordata[key]
+                        actordata['actorid'] = tactordata['id']
+                        actordata['actorname'] = tactordata['name']
+                    if tactordata['country'] == None:
+                        actordata['countryname'] = None
+                        actordata['countrytimezone'] = None
+                        actordata['countrycode'] = None
+                    if tactordata['birthday'] == None:
+                        actordata['birthday'] = None
+                    if tactordata['deathday'] == None:
+                        actordata['deathday'] = None
+                    if tactordata['gender'] == None:
+                        actordata['gender'] = None
+                    if tactordata['image'] == None:
+                        actordata['mediumimage'] = None
+                        actordata['originalimage'] = None
+                    if tactordata['_links'] == None:
+                        actordata['apilink'] = None
+            except:
+                print('Unable to process actor information.')
+            else:
+                actordata.pop('id')
+            
+            try:
+                if tcharacterdata:
+                    for key, value in tcharacterdata.items():    
+                        if key is None:
+                            pass
+                        else:
+                            if tcharacterdata['image']:
+                                characterdata['mediumimage'] = tcharacterdata['image']['medium']
+                                characterdata['originalimage'] = tcharacterdata['image']['original']
+                            characterdata['apilink'] = tcharacterdata['_links']['self']['href']    
+                            characterdata['tvmazeilink'] = tcharacterdata['url']
+                            characterdata['characterid'] = tcharacterdata['id']
+                            characterdata['charactername'] = tcharacterdata['name']    
+                            characterdata['seriesid'] = seriesid
+                            characterdata['actorid'] = actordata['actorid']
+                        if tcharacterdata['image'] == None:
+                            characterdata['mediumimage'] = None
+                            characterdata['originalimage'] = None
+                        if tactordata['_links'] == None:
+                            actordata['apilink'] = None
+                        if not tselfdata:
+                            characterdata['self'] = 0
+                        else:
+                            characterdata['self'] = 1
+                        if not tvoicedata:
+                            characterdata['voice'] = 0
+                        else:
+                            characterdata['voice'] = 1                                              
+            except Exception as e:
+                print('Unable to process character information. -> ' + str(e))
+            datalist['characters'].append(characterdata)
+            datalist['actors'].append(actordata)
+            linkdata = {'actorid': actordata['actorid'], 'characterid': characterdata['characterid']}
+            linklist.append(linkdata)
+        return(datalist, linklist)
+        # except Exception as e:
+        #     print('Unable to process cast data: ' + str(e))
+        # else:
+        #     return(datalist, linklist)
+    
+    def listcredits(self, seriesid):
+        pass
+    
+    def __del__(self):
+        pass

api/tvmaze/crew.py

@@ -0,0 +1,42 @@
+
+class crew():
+    
+    def __init__(self):
+        pass
+
+    def process(self, crewdata, seriesid):
+        crewlist = []
+        try:
+            for crewmember in crewdata:
+                cmember = {}
+                cmember['seriesid'] = seriesid
+                cmember['crewid'] = crewmember['person']['id']
+                cmember['type'] = crewmember['type']
+                cmember['url'] = crewmember['person']['url']
+                cmember['name'] = crewmember['person']['name']
+                cmember['countryid'] = '0'
+                cmember['birthday'] = crewmember['person']['birthday']
+                cmember['deathday'] = crewmember['person']['deathday']
+                cmember['gender'] = crewmember['person']['gender']
+                if crewmember['person']['image']:
+                    cmember['medium'] = crewmember['person']['image']['medium']
+                    cmember['original'] = crewmember['person']['image']['original']
+                else:
+                    cmember['medium'] = None
+                    cmember['original'] = None
+                if crewmember['person']['_links']:
+                    cmember['apilink'] = crewmember['person']['_links']['self']['href']
+                else:
+                    cmember['apilink'] = None
+                crewlist.append(cmember)
+        except Exception as e:
+            print('Unable to process crew information for seriesid: ' + str(seriesid))
+        else:
+            return(crewlist)
+    
+    def listcredits(self, seriesid):
+        pass
+    
+    def __del__(self):
+        pass
+    

api/tvmaze/episodes.py

@@ -0,0 +1,77 @@
+import csv
+class episodes():
+    
+    
+    def __init__(self):
+        pass
+        
+    def update_episodes(self, epload, seriesid):
+        if epload:
+            eplist = []
+            try:
+                for epdata in epload:
+                    epdata['seriesid'] = seriesid
+                    if epdata['image']:
+                        for key, value in epdata['image'].items():
+                            newkey = key + 'image' 
+                            epdata[newkey] = value
+                        epdata.pop('image')
+                    else:
+                        epdata['mediumimage'] = 'None'
+                        epdata['originalimage'] = 'None'
+                        epdata.pop('image')
+                    
+                    for key, value in epdata['_links']['self'].items():
+                        epdata['eplink'] = value    
+                    epdata.pop('_links')
+                    
+                    epdata['episodeid'] = epdata.pop('id')
+                    epdata['episodename'] = epdata.pop('name')
+                    epdata['episodesummary'] = epdata.pop('summary')
+                    epdata['episodenumber'] = epdata.pop('number')
+                    eplist.append(epdata)
+            except:
+                print('Unable to import episode data for for seriesid: ' + str(seriesid))
+            else:
+                return(eplist)
+        
+    def fulllisting(self, seriesid):
+        pass
+    
+    def seasonlist(self, seriesid, seasonnumber):
+        pass
+    
+    def process_episodes(self, episoderesults, seriesid, rootdir):
+        queryrows = []
+        if episoderesults:
+            resultlength = len(episoderesults)
+            if resultlength > 0:
+                for idx, episode in enumerate(episoderesults):
+                    if episoderesults[idx]['episodesummary'] is None:
+                        episoderesults[idx]['episodesummary'] = 'No summary at this time.'
+                    querylist = '''INSERT INTO updates.newepdata (episodeid, seriesid, season, episodenumber, episodename, airdate, airtime, episodesummary, airstamp, runtime, eplink, mediumimage, url, originalimage) VALUES\n'''
+                    filename = rootdir + '/temp/' + str(seriesid) + '_episodes.sql'
+                    sqlfile = open(filename, 'w', encoding="utf-8")
+                    sqlfile.write(querylist)
+                    try:
+                        SQL = '''( '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s' )''' % (episoderesults[idx]['episodeid'],
+                        episoderesults[idx]['seriesid'], episoderesults[idx]['season'], episoderesults[idx]['episodenumber'], episoderesults[idx]['episodename'].replace("'", "''"),
+                        episoderesults[idx]['airdate'], episoderesults[idx]['airtime'], episoderesults[idx]['episodesummary'].replace("'", "''"), episoderesults[idx]['airstamp'],
+                        episoderesults[idx]['runtime'], episoderesults[idx]['eplink'], episoderesults[idx]['mediumimage'], episoderesults[idx]['url'], episoderesults[idx]['originalimage'])
+                            #tv.insert_data(engine, episode, metadata.tables['[updates].newepdata'])
+                        queryrows.append(SQL)
+                    except Exception as e:
+                        lprint.logprint('info', 'Unable to process episodes for seriesid: ' + str(cseriesid))
+                    else:
+                        qlength = len(queryrows) - 1
+                        for rowidx, queryrow in enumerate(queryrows):
+                            if rowidx == qlength and idx == len(episoderesults) -1:
+                                sqlfile.write(queryrow + ';\n')
+                            else:
+                                sqlfile.write(queryrow + ',\n')
+                        sqlfile.close()
+                return(filename, resultlength, querylist)
+
+    def __del__(self):
+        pass
+    

api/tvmaze/process.py

@@ -0,0 +1,256 @@
+
+class show():
+    
+    def __init__(self):
+        pass
+    
+    def process(self, metadata, showload, seriesid):
+        dataresult = []
+        tableresult = []
+        if showload == None:
+            pass
+        else:
+            networkid = None
+            try:
+                network = {}
+                for key, value in showload['network'].items():
+                    if key == 'country':
+                        for key, value in showload['network']['country'].items():
+                            if key == 'name':
+                                newkey = 'country'
+                                network[newkey] = showload['network']['country'][key]
+                            else:
+                                newkey = 'country' + key
+                                network[newkey] = showload['network']['country'][key]
+                    else:
+                        newkey = 'network' + key
+                        network[newkey] = showload['network'][key]
+            except:
+                #print('Failed to insert network information.')
+                showload['networkid'] = 0
+            else:    
+                showload.pop('network')
+                showload['networkid'] = network['networkid']
+                dataresult.append(network)
+                tableresult.append(metadata.tables['[updates].tvnetworknewvals'])
+                #insertresult = insert_data(network, tables[4])
+            try:
+                webchannel = {}
+                for key, value in showload['webChannel'].items():
+                    if key == 'country':
+                            if showload['webChannel'][key] is None:
+                                pass
+                            else:
+                                for key, value in showload['webChannel']['country'].items():
+                                    newkey = 'webChannelcountry'+ key
+                                    webchannel[newkey] = value 
+                    else:
+                        webchannel[key] = value
+                if key == 'name':
+                    newkey = 'webChannel' + key
+                    webhchannel[newkey] = value 
+                if key == 'timezone':
+                    newkey = 'webChannelcountry'+ key
+                    webchannel[newkey] = value
+            except:
+                #print('Failed to insert webchannel information.')
+                showload['webchannelid'] = '0'
+            else:
+                showload.pop('webChannel')
+                showload['webchannelid'] = webchannel['id']
+                webchannel['webchannelname'] = webchannel['name']
+                webchannel['networkid'] = webchannel['id']
+                try:
+                    dataresult.append(webchannel)
+                    tableresult.append(metadata.tables['[updates].tvwebchannelnewvals'])
+                    #insertresult = insert_data(webchannel, tables[5])
+                except sqlalchemy.exc.IntegrityError as e:
+                    pass
+                except Exception as e:
+                    print('Failed to insert webchannel information.')
+            try:
+                showload['seriesid'] = showload['id']
+            except:
+                pass
+            
+            #print('TV Show: ' + showload['seriesname'])
+            
+            try:
+                if showload['image']:
+                    for key, value in showload['image'].items():
+                        newkey = key + 'image' 
+                        showload[newkey] = value
+                    showload.pop('image')
+                else:
+                    showload.pop('image')
+            except:
+                pass
+            
+            try:
+                if showload['externals']:
+                    for key, value in showload['externals'].items():
+                        showload[key] = value
+                    showload.pop('externals')
+                else:
+                    showload.pop('externals')
+            except KeyError as e:
+                pass
+            
+            try:
+                if showload['rating']:
+                    for key, value in showload['rating'].items():
+                        showload['rating'] = value
+                
+                if showload['schedule']:
+                    showload['time'] = showload['schedule']['time']
+                    showload['days'] = ", ".join(showload['schedule']['days'])
+                    showload.pop('schedule')
+            except KeyError as e:
+                pass
+                   
+            try:
+                for key, value in showload['_links']['self'].items():
+                    showload[key] = value    
+                showload.pop('_links')
+            except KeyError as e:
+                pass
+            else:
+                showload.pop('id')
+            
+            showload['genres'] = ", ".join(showload['genres'])
+            
+            # Finish transforming to match table columns
+            showload['seriesname'] = showload.pop('name')
+            showload['tvrageid'] = showload.pop('tvrage')
+            showload['tvdbid'] = showload.pop('thetvdb')
+            showload['imdbid'] = showload.pop('imdb')
+            showload['serieslanguage'] = showload.pop('language')
+            showload['seriesstatus'] = showload.pop('status')
+            showload['airdays'] = showload.pop('days')
+            showload['airtime'] = showload.pop('time')
+            showload['showtype'] = showload.pop('type')
+            showload['overview'] = showload.pop('summary')
+            showload['showurl'] = showload.pop('url')
+            showload['apilink'] = showload.pop('href')
+            showload['officialsite'] = showload.pop('officialSite')
+            
+            results = []
+            results.append([dataresult, tableresult, showload])
+            #results.append(tableresult)
+            #results.append(showload)
+            return (results)
+
+        
+    def __del__(self):
+        pass
+
+class cast():
+    def __init__(self):
+        pass
+        
+    def process(self, cdata, seriesid):
+        datalist = {}
+        datalist['actors'] = []
+        datalist['characters'] = []
+        try:
+            castload = json.loads(cdata)
+            for idx, castdata in enumerate(castload):
+                actordata = {}
+                characterdata = {}
+                tcharacterdata = castload[idx]['character']
+                tactordata = castload[idx]['person']
+                try:
+                    for key, value in tactordata.items():    
+                        if tactordata[key] is None:
+                            pass
+                        else:
+                            if key == 'image':
+                                for key, value in tactordata['image'].items():
+                                    newkey = key + 'image' 
+                                    actordata[newkey] = tactordata['image'][key]
+                            elif key == '_links':
+                                for key, value in tactordata['_links']['self'].items():
+                                    actordata['apilink'] = tactordata['_links']['self'][key]    
+                            elif key == 'country':
+                                for key, value in tactordata['country'].items():
+                                    newkey = 'country' + key
+                                    actordata[newkey] = tactordata['country'][key]    
+                            else:
+                                actordata[key] = tactordata[key]
+                            actordata['actorid'] = tactordata['id']
+                            actordata['actorname'] = tactordata['name']
+                except:
+                    print('Unable to process actor information.')
+                else:
+                    actordata.pop('id')
+                
+                try:
+                    if tcharacterdata:
+                        for key, value in tcharacterdata.items():    
+                            if key is None:
+                                pass
+                            else:
+                                if tcharacterdata['image']:
+                                    characterdata['mediumimage'] = tcharacterdata['image']['medium']
+                                    characterdata['originalimage'] = tcharacterdata['image']['original']
+                                characterdata['apilink'] = tcharacterdata['_links']['self']['href']    
+                                characterdata['tvmazeilink'] = tcharacterdata['url']
+                                characterdata['characterid'] = tcharacterdata['id']
+                                characterdata['charactername'] = tcharacterdata['name']    
+                                characterdata['seriesid'] = seriesid
+                                characterdata['actorid'] = actordata['actorid']
+                except Exception as e:
+                    print('Unable to process character information. -> ' + str(e))
+                    
+                datalist['characters'].append(characterdata)
+                datalist['actors'].append(actordata)
+                linkdata = {}
+                linkdata['actorid'] = actordata['actorid']
+                linkdata['characterid'] = characterdata['characterid']
+                #return(datalist)
+        except Exception as e:
+            #print('Line 389: ' + str(e))
+            pass
+        else:
+            return(datalist)
+    def __del__(self):
+        pass
+
+class Episode():
+    
+    def __init__(self):
+        pass
+        
+    def process(self, epload, seriesid):
+        if epload:
+            eplist = []
+            try:
+                for epdata in epload:
+                    epdata['seriesid'] = seriesid
+                    if epdata['image']:
+                        for key, value in epdata['image'].items():
+                            newkey = key + 'image' 
+                            epdata[newkey] = value
+                        epdata.pop('image')
+                    else:
+                        epdata['mediumimage'] = 'None'
+                        epdata['originalimage'] = 'None'
+                        epdata.pop('image')
+                    
+                    for key, value in epdata['_links']['self'].items():
+                        epdata['eplink'] = value    
+                    epdata.pop('_links')
+                    
+                    epdata['episodeid'] = epdata.pop('id')
+                    epdata['episodename'] = epdata.pop('name')
+                    epdata['episodesummary'] = epdata.pop('summary')
+                    epdata['episodenumber'] = epdata.pop('number')
+                    eplist.append(epdata)
+                return eplist
+            except:
+                print('Unable to import episode data for for seriesid: ' + str(seriesid))
+        else:
+            return None
+
+    def __del__(self):
+        pass

api/tvmaze/series.py

@@ -0,0 +1,156 @@
+
+class tvshow():
+    
+    def __init__(self):
+        pass
+    
+    def process(self, metadata, seriesdata, seriesid):
+        webresult = []
+        networkresult = []
+        webworkchannel = []
+        networkchannel = []
+        if seriesdata == None:
+            pass
+        else:
+            networkid = None
+            try:
+                network = {}
+                for key, value in seriesdata['network'].items():
+                    if key == 'country':
+                        for key, value in seriesdata['network']['country'].items():
+                            if key == 'name':
+                                newkey = 'country'
+                                network[newkey] = seriesdata['network']['country'][key]
+                            else:
+                                newkey = 'country' + key
+                                network[newkey] = seriesdata['network']['country'][key]
+                    else:
+                        newkey = 'network' + key
+                        network[newkey] = seriesdata['network'][key]
+            except:
+                #print('Failed to insert network information.')
+                #seriesdata['networkid'] = '0'
+                pass
+            else:    
+                seriesdata.pop('network')
+                seriesdata['networkid'] = network['networkid']
+                networkresult.append(network)
+                networkchannel.append(metadata.tables['updates.tvnetworknewvals'])
+                #insertresult = insert_data(network, tables[4])
+            try:
+                webchannel = {}
+                if seriesdata['webChannel']:
+                    for key, value in seriesdata['webChannel'].items():
+                        if key == 'country':
+                                if seriesdata['webChannel'][key] is None:
+                                    pass
+                                else:
+                                    for key, value in seriesdata['webChannel']['country'].items():
+                                        newkey = 'webChannelcountry'+ key
+                                        webchannel[newkey] = value 
+                        else:
+                            webchannel[key] = value
+                    if key == 'name':
+                        newkey = 'webChannel' + key
+                        webchannel[newkey] = value 
+                    if key == 'timezone':
+                        newkey = 'webChannelcountry'+ key
+                        webchannel[newkey] = value
+            except:
+                #print('Failed to insert webchannel information.')
+                seriesdata['webchannelid'] = '0'
+                pass
+            else:
+                #seriesdata.pop('webChannel')
+                webchannel['id'] = '0'
+                seriesdata['webchannelid'] = webchannel['id']
+                webchannel['name'] = 'Not Applicable'
+                webchannel['webchannelname'] = webchannel['name']
+                #webchannel['networkid'] = webchannel['id']
+                try:
+                    webresult.append(webchannel)
+                    webworkchannel.append(metadata.tables['updates.tvwebchannelnewvals'])
+                    #insertresult = insert_data(webchannel, tables[5])
+                except sqlalchemy.exc.IntegrityError as e:
+                    pass
+                except Exception as e:
+                    #print('Failed to insert webchannel information.')
+                    pass
+            try:
+                seriesdata['seriesid'] = seriesdata['id']
+            except:
+                pass
+            
+            #print('TV Show: ' + seriesdata['seriesname'])
+            
+            try:
+                if seriesdata['image']:
+                    for key, value in seriesdata['image'].items():
+                        newkey = key + 'image' 
+                        seriesdata[newkey] = value
+                    seriesdata.pop('image')
+                else:
+                    seriesdata.pop('image')
+            except:
+                pass
+            
+            try:
+                if seriesdata['externals']:
+                    for key, value in seriesdata['externals'].items():
+                        seriesdata[key] = value
+                    seriesdata.pop('externals')
+                else:
+                    seriesdata.pop('externals')
+            except KeyError as e:
+                pass
+            
+            try:
+                if seriesdata['rating']:
+                    for key, value in seriesdata['rating'].items():
+                        seriesdata['rating'] = value
+                
+                if seriesdata['schedule']:
+                    seriesdata['time'] = seriesdata['schedule']['time']
+                    seriesdata['days'] = ", ".join(seriesdata['schedule']['days'])
+                    seriesdata.pop('schedule')
+            except KeyError as e:
+                pass
+                   
+            try:
+                for key, value in seriesdata['_links']['self'].items():
+                    seriesdata[key] = value    
+                seriesdata.pop('_links')
+            except KeyError as e:
+                pass
+            else:
+                seriesdata.pop('id')
+            
+            if seriesdata['genres']:
+                seriesdata['genres'] = ", ".join(seriesdata['genres'])
+            
+            # Finish transforming to match table columns
+            seriesdata['seriesname'] = seriesdata.pop('name')
+            seriesdata['tvrageid'] = seriesdata.pop('tvrage')
+            seriesdata['tvdbid'] = seriesdata.pop('thetvdb')
+            seriesdata['imdbid'] = seriesdata.pop('imdb')
+            seriesdata['serieslanguage'] = seriesdata.pop('language')
+            seriesdata['seriesstatus'] = seriesdata.pop('status')
+            seriesdata['airdays'] = seriesdata.pop('days')
+            seriesdata['airtime'] = seriesdata.pop('time')
+            seriesdata['showtype'] = seriesdata.pop('type')
+            seriesdata['overview'] = seriesdata.pop('summary')
+            seriesdata['showurl'] = seriesdata.pop('url')
+            seriesdata['apilink'] = seriesdata.pop('href')
+            seriesdata['officialsite'] = seriesdata.pop('officialSite')
+            
+            results = {'networkdata': networkresult, 'networktable': networkchannel,
+                       'webdata': webresult, 'webtable': webchannel,
+                       'seriesdata': seriesdata}
+            #results.append(tableresult)
+            #results.append(seriesdata)
+            return (results)
+
+    
+    def __del__(self):
+        pass
+

cache/process.py

@@ -0,0 +1,89 @@
+import os
+import progressbar
+from pathlib import Path
+from time import sleep
+from tqdm import tqdm
+
+class Cacher(object):
+
+    def __init__(self):
+        pass
+
+    def process(self, options, jget, new_updates, ROOTDIR, URLS):
+        SERIESCACHE = str(ROOTDIR) + '/cache/series/'
+        EPISODECACHE = str(ROOTDIR) + '/cache/episodes/'
+        CASTCACHE = str(ROOTDIR) + '/cache/cast/'
+        CREWCACHE = str(ROOTDIR) + '/cache/crew/'
+        TEMPDIR = ROOTDIR + '/temp/'
+        available_updates = {}
+        updateList = []
+        if options['refreshcache'] == '1':
+            print('Refreshing cache directory.')
+            print('Checking cache directory for missing files.')
+            for update in new_updates:
+                available_updates[update[0]] = update[1]
+            updateList = [str(key) for key in available_updates.keys()]
+            for series in tqdm(range(len(new_updates)), desc='Checking cache', unit='series'):
+                seriesid = new_updates[series][0]
+                cachefile = str(seriesid) + '.json'
+                seriescachefile = f'{SERIESCACHE}{cachefile}'
+                episodecachefile = f'{EPISODECACHE}{cachefile}'
+                castcachefile = f'{CASTCACHE}{cachefile}'
+                crewcachefile = f'{CREWCACHE}{cachefile}'
+                self.check_and_download(jget, URLS['showurl'], str(seriesid), new_updates[0], seriescachefile, 1500)
+                sleep(1)
+                self.check_and_download(jget, URLS['episodesurl'], str(seriesid), new_updates[0], episodecachefile, 1500)
+                sleep(1)
+                self.check_and_download(jget, URLS['casturl'], str(seriesid), new_updates[0], castcachefile, 500)
+                sleep(1)
+                self.check_and_download(jget, URLS['crewurl'], str(seriesid), new_updates[0], crewcachefile, 500)
+                sleep(1)
+        print('Refresh completed.')
+
+    def check_and_download(self, jget, url, SERIESID, series_id, cachefile, size_limit):
+        try:
+            my_abs_path = Path(cachefile).resolve(strict=True)
+            file_stats = os.stat(cachefile)
+            if file_stats.st_size <= size_limit:
+                #print(f'File Size for {cachefile} in Bytes is {file_stats.st_size}. File too small. Attempting to download.')
+                jget.fetch_json(url.replace(SERIESID, str(series_id)), series_id, cachefile)
+        except FileNotFoundError:
+            #print(f'{cachefile} does not exist. Attempting to download.')
+            jget.fetch_json(url.replace(SERIESID, str(series_id)), series_id, cachefile)
+        except Exception as e:
+            print(f'Unable to download {cachefile}. {e}')
+
+    def update_cache(self, jget, updateList, ROOTDIR, URLS, options, lprint):
+        if options['skipcache'] == '0':
+            SERIESCACHE = str(ROOTDIR) + '/cache/series/'
+            EPISODECACHE = str(ROOTDIR) + '/cache/episodes/'
+            CASTCACHE = str(ROOTDIR) + '/cache/cast/'
+            CREWCACHE = str(ROOTDIR) + '/cache/crew/'
+            TEMPDIR = ROOTDIR + '/temp/'
+            print('Now downloading updates.')
+            updaterange = len(updateList)
+            for seriesid in tqdm(range(updaterange), desc='Downloading updates', unit='seriesid'):
+                cachefile = f'{updateList[seriesid]}.json'
+                seriescachefile = SERIESCACHE + cachefile
+                episodecachefile = EPISODECACHE + cachefile
+                castcachefile = CASTCACHE + cachefile
+                crewcachefile = CREWCACHE + cachefile
+                self.download_json(jget, URLS['showurl'], updateList[seriesid], seriescachefile, lprint)
+                sleep(1)
+                self.download_json(jget, URLS['episodesurl'], updateList[seriesid], episodecachefile, lprint)
+                sleep(1)
+                self.download_json(jget, URLS['casturl'], updateList[seriesid], castcachefile, lprint)
+                sleep(1)
+                self.download_json(jget, URLS['crewurl'], updateList[seriesid], crewcachefile, lprint)
+                sleep(1)
+                
+    def download_json(self, jget, url, seriesid, cachefile, lprint):
+        try:
+            jget.fetch_json(url.replace('<seriesid>', str(seriesid)), seriesid, cachefile)
+        except Exception as e:
+            lprint.logprint('info', f'Unable to acquire {cachefile}. {e}')
+
+    def __del__(self):
+        pass
+
+

db/Tables.py

@@ -0,0 +1,401 @@
+import sqlalchemy
+from sqlalchemy import create_engine, MetaData, Table, Column, Integer, String, Sequence, text, UniqueConstraint
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.schema import MetaData
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.types import *
+
+class Tables():
+    def __init__(self):
+        global tvnetworknewvals
+        global tvwebchannelnewvals
+        global characterjsonnewvals
+        global seriesjsonnewvals
+        global episodejsonnewvals
+        global metadata
+        global engine
+        global session
+        global tvnetworknewvals
+        global tvwebchannelnewvals
+        global characterjsonnewvals
+        global seriesjsonnewvals
+        global episodejsonnewvals
+        global tvupdates
+        global newseriesdata
+        global actorlinks
+        global tablelist
+        
+    def __del__(self):
+        pass
+
+
+class Createtables():
+    
+    def __init__(self, session, metadata, engine):
+        Base = declarative_base(metadata = metadata)
+        self.tablelist = self.settables(metadata, engine)
+        try:
+            self.drop(session, self.tablelist)
+        except:
+            pass
+        self.create(Base, engine, metadata, session, self.tablelist)
+        session.commit()
+        #self.send(tablelist)
+    
+    def settables(self, metadata, engine):
+        tablelist = []
+        newseriesdata = Table('seriesdata', metadata,
+            Column('id', BIGINT, Sequence('seriesdata_id_seq'), primary_key = True),
+            Column('seriesid',INTEGER, unique=True),
+            Column('tvdbid', String(25)),
+            Column('tvrageid', String(25)),
+            Column('imdbid', String(25)),
+            Column('serieslanguage', String(25)),
+            Column('genres', String(75)),
+            Column('seriesname', String(150)),
+            Column('seriesstatus', String(25)),
+            Column('premiered', String(25)),
+            Column('airdays', String(72)),
+            Column('airtime', String(25)),
+            Column('runtime', String(25)),
+            Column('rating', String(25)),
+            Column('showtype',TEXT),
+            Column('overview',TEXT),
+            Column('mediumimage',TEXT),
+            Column('originalimage',TEXT),
+            Column('weight',INTEGER),
+            Column('networkid', BIGINT),
+            Column('webchannel', BIGINT),
+            Column('officalsite',TEXT),
+            Column('showurl',TEXT),
+            Column('apilink',TEXT),
+            Column('previousepisode',TEXT),
+            Column('nextepisode',TEXT),
+            schema = "dbo")
+        tablelist.append(newseriesdata)
+        newactordata = Table('actordata', metadata,
+            Column('id', BIGINT, Sequence('actordata_id_seq'), primary_key = True),
+            Column('actorid', INTEGER, unique=True),
+            Column('actorname', VARCHAR(72)),
+            Column('countryname', VARCHAR(144)),
+            Column('countrytimezone', VARCHAR(72)),
+            Column('birthday', DATE),
+            Column('deathday', DATE),
+            Column('gender', VARCHAR(6)),
+            Column('mediumimage', TEXT),
+            Column('originalimage', TEXT),
+            Column('apilink', TEXT),
+            Column('url', TEXT),
+            Column('countrycode', VARCHAR(12)),
+            schema = "dbo")
+        tablelist.append(newactordata)
+        newepdata = Table('epdata', metadata,
+        Column('id', BIGINT, Sequence('epdata_id_seq'), primary_key = True),
+            Column('episodeid', INTEGER),
+            Column('seriesid', INTEGER),
+            Column('season', INTEGER),
+            Column('episodenumber', INTEGER),
+            Column('episodename', TEXT),
+            Column('airdate', VARCHAR(32)),
+            Column('airtime', VARCHAR(32)),
+            Column('episodesummary', TEXT),
+            Column('airstamp', VARCHAR(32)),
+            Column('runtime', VARCHAR(6)),
+            Column('eplink', TEXT),
+            Column('mediumimage', TEXT),
+            Column('url', TEXT),
+            Column('originalimage', TEXT),
+            schema = "dbo")
+        tablelist.append(newepdata)
+        newcastdata = Table('castdata', metadata,
+        Column('id', BIGINT, Sequence('castdata_id_seq'), primary_key = True),
+            Column('seriesid', INTEGER),
+            Column('characterid', INTEGER),
+            Column('actorid', INTEGER),
+            Column('charactername', INTEGER),
+            Column('mediumimage', TEXT),
+            Column('originalimage', TEXT),
+            Column('apilink', TEXT),
+            Column('tvmazeilink', TEXT),
+            schema = "dbo")
+        tablelist.append(newcastdata)
+        tvnetworks = Table('tvnetworkdata', metadata,
+          Column('id', Integer, primary_key=True),
+          Column('networkid', String(12)),
+          Column('networkname', String(255)),
+          Column('country', String(255)),
+          Column('countrycode', String(4)),
+          Column('countrytimezone', String(25)),
+          schema = "dbo")
+        tablelist.append(tvnetworks)
+        tvwebchannels = Table('tvwebchanneldata', metadata,
+          Column('id', Integer, primary_key=True),
+          Column('webchannelname', String(255)),
+          Column('webchannelcountrycode', String(4)),
+          Column('webchannelcountryname', String(255)),
+          Column('webchannelcountrytimezone', String(4)),
+          schema = "dbo")
+        tablelist.append(tvwebchannels)
+        countries = Table('countrydata', metadata,
+          Column('id', Integer, primary_key=True),
+          Column('name', String(255)),
+          Column('code', String(255)),
+          Column('timezone', String(255)),
+          UniqueConstraint('name', 'timezone', name='tz_name_uk'),
+          schema = "dbo")
+        tablelist.append(countries)
+        
+        return(tablelist)
+    
+    def create(self, Base, engine, metadata, session, tablelist):
+        print('Creating Data Tables')
+        Base.metadata.create_all(engine, tables=tablelist)
+
+    def drop(self, session, tablelist):
+        for table in tablelist:
+            try:
+                #print('Dropping ' + str(table))
+                table.drop()
+            except sqlalchemy.exc.ProgrammingError as se:
+                pass
+            
+        def __del__(self):
+            pass
+            
+
+class TempTables():
+    
+    def __init__(self, session, metadata, engine, tablelist):
+        Base = declarative_base(metadata = metadata)
+        self.tablelist = self.settables(metadata, engine)
+        try:
+            self.drop(session, self.tablelist)
+        except:
+            pass
+        self.create(Base, engine, metadata, session, self.tablelist)
+        #session.commit()
+        
+    def settables(self, metadata, engine):
+        tablelist = []
+        tvnetworknewvals = Table('tvnetworknewvals', metadata,
+          Column('id', BIGINT, Sequence('network_id_seq'), primary_key = True),
+          Column('name', String(255)),
+          Column('countryid', String(255)),
+          schema = "updates")
+        tablelist.append(tvnetworknewvals)
+        tvwebchannelnewvals = Table('tvwebchannelnewvals', metadata,
+          Column('id', BIGINT, Sequence('webchannel_id_seq'), primary_key = True),
+          Column('name', String(255)),
+          Column('countryid', String(4)),
+          schema = "updates")
+        tablelist.append(tvwebchannelnewvals)
+        tvupdates = Table('tvupdates', metadata,
+        Column('seriesid', BIGINT, primary_key = True),
+            Column('timestamp', VARCHAR(15)),
+          schema = "updates")
+        tablelist.append(tvupdates)
+        tvupdatedata = Table('tvupdatedata', metadata,
+        Column('seriesid', BIGINT, primary_key = True),
+            Column('timestamp', VARCHAR(15)),
+          schema = "updates")
+        tablelist.append(tvupdatedata)
+        newactorlinks = Table('newactorlinks', metadata,
+          Column('id', BIGINT, Sequence('tempepisodejson_id_seq'), primary_key = True),
+          Column('actorid', INTEGER),
+          Column('characterid', INTEGER),
+          schema = "updates")
+        tablelist.append(newactorlinks)
+        return(tablelist)
+    
+    def create(self, Base, engine, metadata, session, tablelist):
+        tablelist.remove(tablelist[6])
+        print('Creating Temporary JSON Tables')
+        try:
+            Base.metadata.create_all(engine, tables=tablelist)
+        except sqlalchemy.exc.ProgrammingError as pe:
+            pass 
+
+    def drop(self, session, tablelist):
+        for table in tablelist:
+            if 'tvupdatedata' in table.description:
+                pass
+            else:
+                try:
+                    #print('Dropping ' + str(table))
+                    table.drop()
+                except sqlalchemy.exc.ProgrammingError as se:
+                    pass
+        #session.commit()
+
+class CreateTempTables():
+    
+    global tvnetworknewvals
+    global tvwebchannelnewvals
+    global characterjsonnewvals
+    global seriesjsonnewvals
+    global episodejsonnewvals
+    global metadata
+    global engine
+    global session
+    
+    global tablelist
+    
+    def __init__(self, session, metadata, engine):
+        # Need to create schema if it does not exist
+        Base = declarative_base(metadata = metadata)
+        self.tablelist = self.settables(metadata, engine)
+        try:
+            self.drop(session, self.tablelist)
+        except:
+            pass
+        self.create(Base, engine, metadata, session, self.tablelist)
+        session.commit()
+        #self.send(tablelist)
+    
+    def settables(self, metadata, engine):
+        tablelist = []
+        newseriesdata = Table('newseriesdata', metadata,
+                Column('id', Integer, primary_key=True),
+                Column('seriesid',INTEGER, unique=True),
+                Column('thetvdb', String(25)),
+                Column('tvrage', String(25)),
+                Column('imdb', String(25)),
+                Column('language', String(25)),
+                Column('genres', String(75)),
+                Column('name', String(150)),
+                Column('status', String(25)),
+                Column('premiered', String(25)),
+                Column('days', String(72)),
+                Column('time', String(25)),
+                Column('runtime', String(25)),
+                Column('rating', String(25)),
+                Column('type',TEXT),
+                Column('summary',TEXT),
+                Column('medium',TEXT),
+                Column('original',TEXT),
+                Column('weight',INTEGER),
+                Column('networkid', BIGINT),
+                Column('webchannelid', BIGINT),
+                Column('officalsite',TEXT),
+                Column('url',TEXT),
+                Column('serieslink',TEXT),
+                Column('previousepisode',TEXT),
+                Column('nextepisode',TEXT),
+                schema="updates",
+                extend_existing=True)
+        tablelist.append(newseriesdata)
+        newactordata = Table('newactordata', metadata,
+                Column('id', Integer, primary_key=True),
+                Column('actorid', INTEGER),
+                Column('actorname', VARCHAR(72)),
+                Column('name', VARCHAR(72)),
+                Column('countryname', VARCHAR(72)),
+                Column('countrytimezone', VARCHAR(72)),
+                Column('birthday', DATE),
+                Column('deathday', DATE),
+                Column('gender', VARCHAR(6)),
+                Column('mediumimage', TEXT),
+                Column('originalimage', TEXT),
+                Column('apilink', TEXT),
+                Column('url', TEXT),
+                Column('countrycode', VARCHAR(12)),
+                schema="updates",
+                extend_existing=True)
+        tablelist.append(newactordata)
+        NEWEPDATA_ID = Sequence('newepdata_id_seq', start=1)
+        newepdata = Table('newepdata', metadata,
+                Column('id', Integer, primary_key=True),
+                Column('episodeid', INTEGER),
+                Column('seriesid', INTEGER),
+                Column('season', INTEGER),
+                Column('episodenumber', INTEGER),
+                Column('episodename', TEXT),
+                Column('airdate', VARCHAR(32)),
+                Column('airtime', VARCHAR(32)),
+                Column('episodesummary', TEXT),
+                Column('airstamp', VARCHAR(32)),
+                Column('runtime', VARCHAR(6)),
+                Column('eplink', TEXT),
+                Column('mediumimage', TEXT),
+                Column('url', TEXT),
+                Column('originalimage', TEXT),
+                schema="updates",
+                extend_existing=True)
+        tablelist.append(newepdata)
+        newcastdata = Table('newcastdata', metadata,
+                Column('id', Integer, primary_key=True),
+                Column('seriesid', INTEGER),
+                Column('characterid', INTEGER),
+                Column('actorid', INTEGER),
+                Column('url', TEXT),
+                Column('charactername', VARCHAR(128)),
+                Column('mediumimage', TEXT),
+                Column('originalimage', TEXT),
+                Column('apilink', TEXT),
+                Column('tvmazeilink', TEXT),
+                Column('playingself', CHAR),
+                Column('voice', CHAR),
+                schema="updates",
+                extend_existing=True)
+        tablelist.append(newcastdata)
+        newcrewdata = Table('newcrewdata', metadata,
+                Column('id', Integer, primary_key=True),
+                Column('seriesid', INTEGER),
+                Column('crewid', INTEGER),
+                Column('type', VARCHAR(128)),
+                Column('url', TEXT),
+                Column('name', VARCHAR(256)),
+                Column('countryid', INTEGER),
+                Column('birthday', DATE),
+                Column('deathday', DATE),
+                Column('gender', VARCHAR(24)),
+                Column('medium', TEXT),
+                Column('original', TEXT),
+                Column('apilink', TEXT),
+                UniqueConstraint('seriesid', 'crewid', 'type', name='crew_unique'),
+                schema="updates",
+                extend_existing=True)
+        tablelist.append(newcrewdata)
+        tvnetworks = Table('tvnetworkdata', metadata,
+                Column('id', Integer, primary_key=True),
+                Column('name', String(255)),
+                Column('countryid', String(255)),
+                schema = "updates",
+                extend_existing=True)
+        tablelist.append(tvnetworks)
+        tvwebchannels = Table('tvwebchanneldata', metadata,
+                Column('id', Integer, primary_key=True),
+                Column('name', String(255)),
+                Column('countryid', String(255)),
+                schema = "updates",
+                extend_existing=True)
+        tablelist.append(tvwebchannels)
+        countries = Table('newcountrydata', metadata,
+                Column('id', Integer, primary_key=True),
+                Column('name', String(255)),
+                Column('code', String(255)),
+                Column('timezone', String(255)),
+                UniqueConstraint('name', 'timezone', name='tz_name_uk'),
+                schema = "updates",
+                extend_existing=True)
+        tablelist.append(countries)
+        tvupdates = Table('tvupdates', metadata,
+                Column('seriesid', BIGINT, primary_key = True),
+                Column('timestamp', VARCHAR(15)),
+                schema = "updates",
+                extend_existing=True)
+        tablelist.append(tvupdates)
+        return(tablelist)
+    
+    def create(self, Base, engine, metadata, session, tablelist):
+        print('Creating Temporary Data Tables')
+        Base.metadata.create_all(engine, tables=tablelist)
+
+    def drop(self, session, tablelist):
+        print('Dropping temporary update tables.')
+        for table in tablelist:
+            try:
+                print('Dropping ' + str(table))
+                table.drop()
+            except sqlalchemy.exc.ProgrammingError as se:
+                pass

db/ddl/create_schema.sql

@@ -0,0 +1,566 @@
+CREATE SCHEMA dbo;
+
+ALTER SCHEMA dbo OWNER TO postgres;
+
+CREATE SCHEMA updates;
+
+ALTER SCHEMA updates OWNER TO postgres;
+
+CREATE SEQUENCE dbo.actordata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE dbo.actordata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE dbo.castdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE dbo.castdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE dbo.countrydata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE dbo.countrydata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE dbo.crewdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE dbo.crewdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.epdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.epdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE dbo.epdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE dbo.epdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE dbo.seriesdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE dbo.seriesdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE dbo.tvnetworkdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE dbo.tvnetworkdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE dbo.tvwebchanneldata_id_seq
+    AS integer    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    MAXVALUE 2147483647
+    CACHE 1;
+
+ALTER SEQUENCE dbo.tvwebchanneldata_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE dbo.webchanneldata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE dbo.webchanneldata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.actordata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.actordata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.castdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.castdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.countrydata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.countrydata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.crewdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.crewdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.seriesdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.seriesdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.tvnetworkdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.tvnetworkdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.tvupdates_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.tvupdates_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.tvwebchanneldata_id_seq
+    AS integer    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    MAXVALUE 2147483647
+    CACHE 1;
+
+ALTER SEQUENCE updates.tvwebchanneldata_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.webchanneldata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.webchanneldata_row_id_seq OWNER TO postgres;
+
+CREATE TABLE dbo.actordata (
+    row_id bigint NOT NULL DEFAULT nextval('dbo.actordata_row_id_seq'::regclass),
+    actorid integer,
+    actorname character varying(72),
+    countryname character varying(144),
+    countrytimezone character varying(72),
+    birthday date,
+    deathday date,
+    gender character varying(6),
+    mediumimage text,
+    originalimage text,
+    apilink text,
+    url text,
+    countrycode character varying(12));
+
+ALTER TABLE dbo.actordata OWNER TO postgres;
+
+ALTER SEQUENCE dbo.actordata_row_id_seq OWNED BY dbo.actordata.row_id;
+
+CREATE TABLE dbo.castdata (
+    row_id bigint NOT NULL DEFAULT nextval('dbo.castdata_row_id_seq'::regclass),
+    seriesid integer,
+    characterid integer,
+    actorid integer,
+    charactername integer,
+    mediumimage text,
+    originalimage text,
+    apilink text,
+    tvmazeilink text);
+
+ALTER TABLE dbo.castdata OWNER TO postgres;
+
+ALTER SEQUENCE dbo.castdata_row_id_seq OWNED BY dbo.castdata.row_id;
+
+CREATE TABLE dbo.countrydata (
+    row_id bigint NOT NULL DEFAULT nextval('dbo.countrydata_row_id_seq'::regclass),
+    country_name character varying(255),
+    country_code character varying(255),
+    country_timezone character varying(255));
+
+ALTER TABLE dbo.countrydata OWNER TO postgres;
+
+ALTER SEQUENCE dbo.countrydata_row_id_seq OWNED BY dbo.countrydata.row_id;
+
+CREATE TABLE dbo.crewdata (
+    row_id bigint NOT NULL DEFAULT nextval('dbo.crewdata_row_id_seq'::regclass),
+    crewid integer,
+    crew_type character varying(128),
+    name character varying(255),
+    _links_self_href text,
+    country_name text,
+    country_code text,
+    country_timezone text,
+    birthday date,
+    deathday date,
+    gender character varying(24),
+    image_medium text,
+    image_original text,
+    updated text,
+    url text);
+
+ALTER TABLE dbo.crewdata OWNER TO postgres;
+
+ALTER SEQUENCE dbo.crewdata_row_id_seq OWNED BY dbo.crewdata.row_id;
+
+CREATE TABLE dbo.epdata (
+    row_id bigint NOT NULL DEFAULT nextval('dbo.epdata_row_id_seq'::regclass),
+    episodeid integer,
+    seriesid integer,
+    season integer,
+    episode_number integer,
+    episode_name text,
+    airdate character varying(32),
+    airtime character varying(32),
+    summary text,
+    airstamp character varying(32),
+    runtime character varying(6),
+    eplink text,
+    image_medium text,
+    url text,
+    image_original text,
+    episode_type text,
+    rating_average text,
+    _links_self_href text,
+    _links_show_href text,
+    _links_show_name text);
+
+ALTER TABLE dbo.epdata OWNER TO postgres;
+
+ALTER SEQUENCE dbo.epdata_row_id_seq OWNED BY dbo.epdata.row_id;
+
+CREATE TABLE dbo.seriesdata (
+    row_id bigint NOT NULL DEFAULT nextval('dbo.seriesdata_row_id_seq'::regclass),
+    _links_previousepisode_href text,
+    _links_previousepisode_name text,
+    _links_nextepisode_href text,
+    _links_nextepisode_name text,
+    _links_self_href text,
+    averageruntime text,
+    dvdcountry text,
+    series_name text,
+    ended date,
+    externals_imdb text,
+    externals_thetvdb text,
+    externals_tvrage text,
+    genres text,
+    seriesid text,
+    image_medium text,
+    image_original text,
+    language_name text,
+    network_id text,
+    network_name text,
+    officialsite text,
+    network_officialsite text,
+    network_country_code text,
+    network_country_name text,
+    network_country_timezone text,
+    premiered text,
+    rating_average text,
+    runtime text,
+    schedule_days text,
+    schedule_time text,
+    status text,
+    summary text,
+    series_type text,
+    updated text,
+    url text,
+    webchannel_id integer,
+    webchannel_name text,
+	webchannel_country_name text,
+	webchannel_country_code text,
+	webchannel_country_timezone text,
+    webchannel_officialsite text,
+    weight text);
+
+ALTER TABLE dbo.seriesdata OWNER TO postgres;
+
+ALTER SEQUENCE dbo.seriesdata_row_id_seq OWNED BY dbo.seriesdata.row_id;
+
+CREATE TABLE dbo.tvnetworkdata (
+    row_id bigint NOT NULL DEFAULT nextval('dbo.tvnetworkdata_row_id_seq'::regclass),
+    networkid character varying(12),
+    networkname character varying(255),
+    country character varying(255),
+    countrycode character varying(4),
+    countrytimezone character varying(25));
+
+ALTER TABLE dbo.tvnetworkdata OWNER TO postgres;
+
+ALTER SEQUENCE dbo.tvnetworkdata_row_id_seq OWNED BY dbo.tvnetworkdata.row_id;
+
+CREATE TABLE dbo.tvwebchanneldata (
+    id integer NOT NULL DEFAULT nextval('dbo.tvwebchanneldata_id_seq'::regclass),
+    webchannelname character varying(255),
+    webchannelcountrycode character varying(4),
+    webchannelcountryname character varying(255),
+    webchannelcountrytimezone character varying(4));
+
+ALTER TABLE dbo.tvwebchanneldata OWNER TO postgres;
+
+ALTER SEQUENCE dbo.tvwebchanneldata_id_seq OWNED BY dbo.tvwebchanneldata.id;
+
+CREATE TABLE dbo.webchanneldata (
+    row_id bigint NOT NULL DEFAULT nextval('dbo.webchanneldata_row_id_seq'::regclass),
+    webchannelname character varying(255),
+    webchannelcountrycode character varying(4),
+    webchannelcountryname character varying(255),
+    webchannelcountrytimezone character varying(4));
+
+ALTER TABLE dbo.webchanneldata OWNER TO postgres;
+
+ALTER SEQUENCE dbo.webchanneldata_row_id_seq OWNED BY dbo.webchanneldata.row_id;
+
+CREATE TABLE updates.actordata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.actordata_row_id_seq'::regclass),
+    actorid integer,
+    actorname character varying(72),
+    countryname character varying(144),
+    countrytimezone character varying(72),
+    birthday date,
+    deathday date,
+    gender character varying(6),
+    mediumimage text,
+    originalimage text,
+    apilink text,
+    url text,
+    countrycode character varying(12));
+
+ALTER TABLE updates.actordata OWNER TO postgres;
+
+ALTER SEQUENCE updates.actordata_row_id_seq OWNED BY updates.actordata.row_id;
+
+CREATE TABLE updates.castdata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.castdata_row_id_seq'::regclass),
+    seriesid integer,
+    characterid integer,
+    actorid integer,
+    charactername integer,
+    mediumimage text,
+    originalimage text,
+    apilink text,
+    tvmazeilink text);
+
+ALTER TABLE updates.castdata OWNER TO postgres;
+
+ALTER SEQUENCE updates.castdata_row_id_seq OWNED BY updates.castdata.row_id;
+
+CREATE TABLE updates.countrydata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.countrydata_row_id_seq'::regclass),
+    country_name character varying(255),
+    country_code character varying(255),
+    country_timezone character varying(255));
+
+ALTER TABLE updates.countrydata OWNER TO postgres;
+
+ALTER SEQUENCE updates.countrydata_row_id_seq OWNED BY updates.countrydata.row_id;
+
+CREATE TABLE updates.crewdata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.crewdata_row_id_seq'::regclass),
+    crewid integer,
+    crew_type character varying(128),
+    name character varying(255),
+    _links_self_href text,
+    country_name text,
+    country_code text,
+    country_timezone text,
+    birthday date,
+    deathday date,
+    gender character varying(24),
+    image_medium text,
+    image_original text,
+    updated text,
+    url text);
+
+ALTER TABLE updates.crewdata OWNER TO postgres;
+
+ALTER SEQUENCE updates.crewdata_row_id_seq OWNED BY updates.crewdata.row_id;
+
+CREATE TABLE updates.epdata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.epdata_row_id_seq'::regclass),
+    episodeid integer,
+    seriesid integer,
+    season integer,
+    episode_number integer,
+    episode_name text,
+    airdate character varying(32),
+    airtime character varying(32),
+    summary text,
+    airstamp character varying(32),
+    runtime character varying(6),
+    eplink text,
+    image_medium text,
+    url text,
+    image_original text,
+    episode_type text,
+    rating_average text,
+    _links_self_href text,
+    _links_show_href text,
+    _links_show_name text);
+
+ALTER TABLE updates.epdata OWNER TO postgres;
+
+ALTER SEQUENCE updates.epdata_row_id_seq OWNED BY updates.epdata.row_id;
+
+CREATE TABLE updates.seriesdata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.seriesdata_row_id_seq'::regclass),
+    _links_previousepisode_href text,
+    _links_previousepisode_name text,
+    _links_nextepisode_href text,
+    _links_nextepisode_name text,
+    _links_self_href text,
+    averageruntime text,
+    dvdcountry text,
+    series_name text,
+    ended date,
+    externals_imdb text,
+    externals_thetvdb text,
+    externals_tvrage text,
+    genres text,
+    seriesid text,
+    image_medium text,
+    image_original text,
+    language_name text,
+    network_id text,
+    network_name text,
+    officialsite text,
+    network_officialsite text,
+    network_country_code text,
+    network_country_name text,
+    network_country_timezone text,
+    premiered text,
+    rating_average text,
+    runtime text,
+    schedule_days text,
+    schedule_time text,
+    status text,
+    summary text,
+    series_type text,
+    updated text,
+    url text,
+    webchannel_id integer,
+    webchannel_name text,
+	webchannel_country_name text,
+	webchannel_country_code text,
+	webchannel_country_timezone text,
+    webchannel_officialsite text,
+    weight text);
+
+ALTER TABLE updates.seriesdata OWNER TO postgres;
+
+ALTER SEQUENCE updates.seriesdata_row_id_seq OWNED BY updates.seriesdata.row_id;
+
+CREATE TABLE updates.tvnetworkdata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.tvnetworkdata_row_id_seq'::regclass),
+    networkid character varying(12),
+    networkname character varying(255),
+    country character varying(255),
+    countrycode character varying(4),
+    countrytimezone character varying(25));
+
+ALTER TABLE updates.tvnetworkdata OWNER TO postgres;
+
+ALTER SEQUENCE updates.tvnetworkdata_row_id_seq OWNED BY updates.tvnetworkdata.row_id;
+
+CREATE TABLE updates.tvupdates (
+    row_id bigint NOT NULL DEFAULT nextval('updates.tvupdates_row_id_seq'::regclass),
+    seriesid integer,
+    "timestamp" character varying(15));
+
+ALTER TABLE updates.tvupdates OWNER TO postgres;
+
+ALTER SEQUENCE updates.tvupdates_row_id_seq OWNED BY updates.tvupdates.row_id;
+
+CREATE TABLE updates.tvwebchanneldata (
+    row_id integer NOT NULL DEFAULT nextval('updates.tvwebchanneldata_id_seq'::regclass),
+    name character varying(255),
+    countryid character varying(255));
+
+ALTER TABLE updates.tvwebchanneldata OWNER TO postgres;
+
+ALTER SEQUENCE updates.tvwebchanneldata_id_seq OWNED BY updates.tvwebchanneldata.row_id;
+
+CREATE TABLE updates.webchanneldata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.webchanneldata_row_id_seq'::regclass),
+    webchannel_name character varying(255),
+    webchannel_country_code character varying(4),
+    webchannel_country_name character varying(255),
+    webchannelcountry_timezone character varying(4));
+
+ALTER TABLE updates.webchanneldata OWNER TO postgres;
+
+ALTER SEQUENCE updates.webchanneldata_row_id_seq OWNED BY updates.webchanneldata.row_id;
+
+ALTER TABLE dbo.actordata ADD CONSTRAINT actordata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE dbo.castdata ADD CONSTRAINT castdata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE dbo.countrydata ADD CONSTRAINT countrydata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE dbo.epdata ADD CONSTRAINT epdata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE dbo.seriesdata ADD CONSTRAINT seriesdata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE dbo.tvnetworkdata ADD CONSTRAINT tvnetworkdata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE dbo.tvwebchanneldata ADD CONSTRAINT tvwebchanneldata_pkey PRIMARY KEY (id);
+
+ALTER TABLE dbo.webchanneldata ADD CONSTRAINT webchanneldata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.actordata ADD CONSTRAINT actordata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.actordata ADD CONSTRAINT actorid UNIQUE (actorid);
+
+ALTER TABLE updates.castdata ADD CONSTRAINT castdata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.castdata ADD CONSTRAINT series_actor_character_index UNIQUE (seriesid, characterid, actorid);
+
+ALTER TABLE updates.countrydata ADD CONSTRAINT country_code_idx UNIQUE (country_code);
+
+ALTER TABLE updates.countrydata ADD CONSTRAINT countrydata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.crewdata ADD CONSTRAINT crewdata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.crewdata ADD CONSTRAINT crewid_crewid_idx UNIQUE (crewid);
+
+ALTER TABLE updates.epdata ADD CONSTRAINT epdata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.epdata ADD CONSTRAINT episodeie UNIQUE (episodeid);
+
+ALTER TABLE updates.seriesdata ADD CONSTRAINT seriesdata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.seriesdata ADD CONSTRAINT seriesid UNIQUE (seriesid);
+
+ALTER TABLE updates.tvnetworkdata ADD CONSTRAINT networkid UNIQUE (networkid);
+
+ALTER TABLE updates.tvnetworkdata ADD CONSTRAINT tvnetworkdata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.tvupdates ADD CONSTRAINT tvupdates_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.tvwebchanneldata ADD CONSTRAINT tvwebchanneldata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.webchanneldata ADD CONSTRAINT webchanneldata_pkey PRIMARY KEY (row_id);
+
+ALTER SCHEMA updates OWNER TO postgres;

db/ddl/recreate_schema.sql

@@ -0,0 +1,570 @@
+DROP SCHEMA dbo CASCADE;
+
+DROP SCHEMA updates CASCADE;
+
+CREATE SCHEMA dbo;
+
+ALTER SCHEMA dbo OWNER TO postgres;
+
+CREATE SCHEMA updates;
+
+ALTER SCHEMA updates OWNER TO postgres;
+
+CREATE SEQUENCE dbo.actordata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE dbo.actordata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE dbo.castdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE dbo.castdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE dbo.countrydata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE dbo.countrydata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE dbo.crewdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE dbo.crewdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.epdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.epdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE dbo.epdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE dbo.epdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE dbo.seriesdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE dbo.seriesdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE dbo.tvnetworkdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE dbo.tvnetworkdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE dbo.tvwebchanneldata_id_seq
+    AS integer    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    MAXVALUE 2147483647
+    CACHE 1;
+
+ALTER SEQUENCE dbo.tvwebchanneldata_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE dbo.webchanneldata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE dbo.webchanneldata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.actordata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.actordata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.castdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.castdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.countrydata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.countrydata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.crewdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.crewdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.seriesdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.seriesdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.tvnetworkdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.tvnetworkdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.tvupdates_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.tvupdates_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.tvwebchanneldata_id_seq
+    AS integer    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    MAXVALUE 2147483647
+    CACHE 1;
+
+ALTER SEQUENCE updates.tvwebchanneldata_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.webchanneldata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.webchanneldata_row_id_seq OWNER TO postgres;
+
+CREATE TABLE dbo.actordata (
+    row_id bigint NOT NULL DEFAULT nextval('dbo.actordata_row_id_seq'::regclass),
+    actorid integer,
+    actorname character varying(72),
+    countryname character varying(144),
+    countrytimezone character varying(72),
+    birthday date,
+    deathday date,
+    gender character varying(6),
+    mediumimage text,
+    originalimage text,
+    apilink text,
+    url text,
+    countrycode character varying(12));
+
+ALTER TABLE dbo.actordata OWNER TO postgres;
+
+ALTER SEQUENCE dbo.actordata_row_id_seq OWNED BY dbo.actordata.row_id;
+
+CREATE TABLE dbo.castdata (
+    row_id bigint NOT NULL DEFAULT nextval('dbo.castdata_row_id_seq'::regclass),
+    seriesid integer,
+    characterid integer,
+    actorid integer,
+    charactername integer,
+    mediumimage text,
+    originalimage text,
+    apilink text,
+    tvmazeilink text);
+
+ALTER TABLE dbo.castdata OWNER TO postgres;
+
+ALTER SEQUENCE dbo.castdata_row_id_seq OWNED BY dbo.castdata.row_id;
+
+CREATE TABLE dbo.countrydata (
+    row_id bigint NOT NULL DEFAULT nextval('dbo.countrydata_row_id_seq'::regclass),
+    country_name character varying(255),
+    country_code character varying(255),
+    country_timezone character varying(255));
+
+ALTER TABLE dbo.countrydata OWNER TO postgres;
+
+ALTER SEQUENCE dbo.countrydata_row_id_seq OWNED BY dbo.countrydata.row_id;
+
+CREATE TABLE dbo.crewdata (
+    row_id bigint NOT NULL DEFAULT nextval('dbo.crewdata_row_id_seq'::regclass),
+    crewid integer,
+    crew_type character varying(128),
+    name character varying(255),
+    _links_self_href text,
+    country_name text,
+    country_code text,
+    country_timezone text,
+    birthday date,
+    deathday date,
+    gender character varying(24),
+    image_medium text,
+    image_original text,
+    updated text,
+    url text);
+
+ALTER TABLE dbo.crewdata OWNER TO postgres;
+
+ALTER SEQUENCE dbo.crewdata_row_id_seq OWNED BY dbo.crewdata.row_id;
+
+CREATE TABLE dbo.epdata (
+    row_id bigint NOT NULL DEFAULT nextval('dbo.epdata_row_id_seq'::regclass),
+    episodeid integer,
+    seriesid integer,
+    season integer,
+    episode_number integer,
+    episode_name text,
+    airdate character varying(32),
+    airtime character varying(32),
+    summary text,
+    airstamp character varying(32),
+    runtime character varying(6),
+    eplink text,
+    image_medium text,
+    url text,
+    image_original text,
+    episode_type text,
+    rating_average text,
+    _links_self_href text,
+    _links_show_href text,
+    _links_show_name text);
+
+ALTER TABLE dbo.epdata OWNER TO postgres;
+
+ALTER SEQUENCE dbo.epdata_row_id_seq OWNED BY dbo.epdata.row_id;
+
+CREATE TABLE dbo.seriesdata (
+    row_id bigint NOT NULL DEFAULT nextval('dbo.seriesdata_row_id_seq'::regclass),
+    _links_previousepisode_href text,
+    _links_previousepisode_name text,
+    _links_nextepisode_href text,
+    _links_nextepisode_name text,
+    _links_self_href text,
+    averageruntime text,
+    dvdcountry text,
+    series_name text,
+    ended date,
+    externals_imdb text,
+    externals_thetvdb text,
+    externals_tvrage text,
+    genres text,
+    seriesid text,
+    image_medium text,
+    image_original text,
+    language_name text,
+    network_id text,
+    network_name text,
+    officialsite text,
+    network_officialsite text,
+    network_country_code text,
+    network_country_name text,
+    network_country_timezone text,
+    premiered text,
+    rating_average text,
+    runtime text,
+    schedule_days text,
+    schedule_time text,
+    status text,
+    summary text,
+    series_type text,
+    updated text,
+    url text,
+    webchannel_id integer,
+    webchannel_name text,
+	webchannel_country_name text,
+	webchannel_country_code text,
+	webchannel_country_timezone text,
+    webchannel_officialsite text,
+    weight text);
+
+ALTER TABLE dbo.seriesdata OWNER TO postgres;
+
+ALTER SEQUENCE dbo.seriesdata_row_id_seq OWNED BY dbo.seriesdata.row_id;
+
+CREATE TABLE dbo.tvnetworkdata (
+    row_id bigint NOT NULL DEFAULT nextval('dbo.tvnetworkdata_row_id_seq'::regclass),
+    networkid character varying(12),
+    networkname character varying(255),
+    country character varying(255),
+    countrycode character varying(4),
+    countrytimezone character varying(25));
+
+ALTER TABLE dbo.tvnetworkdata OWNER TO postgres;
+
+ALTER SEQUENCE dbo.tvnetworkdata_row_id_seq OWNED BY dbo.tvnetworkdata.row_id;
+
+CREATE TABLE dbo.tvwebchanneldata (
+    id integer NOT NULL DEFAULT nextval('dbo.tvwebchanneldata_id_seq'::regclass),
+    webchannelname character varying(255),
+    webchannelcountrycode character varying(4),
+    webchannelcountryname character varying(255),
+    webchannelcountrytimezone character varying(4));
+
+ALTER TABLE dbo.tvwebchanneldata OWNER TO postgres;
+
+ALTER SEQUENCE dbo.tvwebchanneldata_id_seq OWNED BY dbo.tvwebchanneldata.id;
+
+CREATE TABLE dbo.webchanneldata (
+    row_id bigint NOT NULL DEFAULT nextval('dbo.webchanneldata_row_id_seq'::regclass),
+    webchannelname character varying(255),
+    webchannelcountrycode character varying(4),
+    webchannelcountryname character varying(255),
+    webchannelcountrytimezone character varying(4));
+
+ALTER TABLE dbo.webchanneldata OWNER TO postgres;
+
+ALTER SEQUENCE dbo.webchanneldata_row_id_seq OWNED BY dbo.webchanneldata.row_id;
+
+CREATE TABLE updates.actordata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.actordata_row_id_seq'::regclass),
+    actorid integer,
+    actorname character varying(72),
+    countryname character varying(144),
+    countrytimezone character varying(72),
+    birthday date,
+    deathday date,
+    gender character varying(6),
+    mediumimage text,
+    originalimage text,
+    apilink text,
+    url text,
+    countrycode character varying(12));
+
+ALTER TABLE updates.actordata OWNER TO postgres;
+
+ALTER SEQUENCE updates.actordata_row_id_seq OWNED BY updates.actordata.row_id;
+
+CREATE TABLE updates.castdata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.castdata_row_id_seq'::regclass),
+    seriesid integer,
+    characterid integer,
+    actorid integer,
+    charactername integer,
+    mediumimage text,
+    originalimage text,
+    apilink text,
+    tvmazeilink text);
+
+ALTER TABLE updates.castdata OWNER TO postgres;
+
+ALTER SEQUENCE updates.castdata_row_id_seq OWNED BY updates.castdata.row_id;
+
+CREATE TABLE updates.countrydata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.countrydata_row_id_seq'::regclass),
+    country_name character varying(255),
+    country_code character varying(255),
+    country_timezone character varying(255));
+
+ALTER TABLE updates.countrydata OWNER TO postgres;
+
+ALTER SEQUENCE updates.countrydata_row_id_seq OWNED BY updates.countrydata.row_id;
+
+CREATE TABLE updates.crewdata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.crewdata_row_id_seq'::regclass),
+    crewid integer,
+    crew_type character varying(128),
+    name character varying(255),
+    _links_self_href text,
+    country_name text,
+    country_code text,
+    country_timezone text,
+    birthday date,
+    deathday date,
+    gender character varying(24),
+    image_medium text,
+    image_original text,
+    updated text,
+    url text);
+
+ALTER TABLE updates.crewdata OWNER TO postgres;
+
+ALTER SEQUENCE updates.crewdata_row_id_seq OWNED BY updates.crewdata.row_id;
+
+CREATE TABLE updates.epdata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.epdata_row_id_seq'::regclass),
+    episodeid integer,
+    seriesid integer,
+    season integer,
+    episode_number integer,
+    episode_name text,
+    airdate character varying(32),
+    airtime character varying(32),
+    summary text,
+    airstamp character varying(32),
+    runtime character varying(6),
+    eplink text,
+    image_medium text,
+    url text,
+    image_original text,
+    episode_type text,
+    rating_average text,
+    _links_self_href text,
+    _links_show_href text,
+    _links_show_name text);
+
+ALTER TABLE updates.epdata OWNER TO postgres;
+
+ALTER SEQUENCE updates.epdata_row_id_seq OWNED BY updates.epdata.row_id;
+
+CREATE TABLE updates.seriesdata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.seriesdata_row_id_seq'::regclass),
+    _links_previousepisode_href text,
+    _links_previousepisode_name text,
+    _links_nextepisode_href text,
+    _links_nextepisode_name text,
+    _links_self_href text,
+    averageruntime text,
+    dvdcountry text,
+    series_name text,
+    ended date,
+    externals_imdb text,
+    externals_thetvdb text,
+    externals_tvrage text,
+    genres text,
+    seriesid text,
+    image_medium text,
+    image_original text,
+    language_name text,
+    network_id text,
+    network_name text,
+    officialsite text,
+    network_officialsite text,
+    network_country_code text,
+    network_country_name text,
+    network_country_timezone text,
+    premiered text,
+    rating_average text,
+    runtime text,
+    schedule_days text,
+    schedule_time text,
+    status text,
+    summary text,
+    series_type text,
+    updated text,
+    url text,
+    webchannel_id integer,
+    webchannel_name text,
+	webchannel_country_name text,
+	webchannel_country_code text,
+	webchannel_country_timezone text,
+    webchannel_officialsite text,
+    weight text);
+
+ALTER TABLE updates.seriesdata OWNER TO postgres;
+
+ALTER SEQUENCE updates.seriesdata_row_id_seq OWNED BY updates.seriesdata.row_id;
+
+CREATE TABLE updates.tvnetworkdata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.tvnetworkdata_row_id_seq'::regclass),
+    networkid character varying(12),
+    networkname character varying(255),
+    country character varying(255),
+    countrycode character varying(4),
+    countrytimezone character varying(25));
+
+ALTER TABLE updates.tvnetworkdata OWNER TO postgres;
+
+ALTER SEQUENCE updates.tvnetworkdata_row_id_seq OWNED BY updates.tvnetworkdata.row_id;
+
+CREATE TABLE updates.tvupdates (
+    row_id bigint NOT NULL DEFAULT nextval('updates.tvupdates_row_id_seq'::regclass),
+    seriesid integer,
+    "timestamp" character varying(15));
+
+ALTER TABLE updates.tvupdates OWNER TO postgres;
+
+ALTER SEQUENCE updates.tvupdates_row_id_seq OWNED BY updates.tvupdates.row_id;
+
+CREATE TABLE updates.tvwebchanneldata (
+    row_id integer NOT NULL DEFAULT nextval('updates.tvwebchanneldata_id_seq'::regclass),
+    name character varying(255),
+    countryid character varying(255));
+
+ALTER TABLE updates.tvwebchanneldata OWNER TO postgres;
+
+ALTER SEQUENCE updates.tvwebchanneldata_id_seq OWNED BY updates.tvwebchanneldata.row_id;
+
+CREATE TABLE updates.webchanneldata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.webchanneldata_row_id_seq'::regclass),
+    webchannel_name character varying(255),
+    webchannel_country_code character varying(4),
+    webchannel_country_name character varying(255),
+    webchannelcountry_timezone character varying(4));
+
+ALTER TABLE updates.webchanneldata OWNER TO postgres;
+
+ALTER SEQUENCE updates.webchanneldata_row_id_seq OWNED BY updates.webchanneldata.row_id;
+
+ALTER TABLE dbo.actordata ADD CONSTRAINT actordata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE dbo.castdata ADD CONSTRAINT castdata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE dbo.countrydata ADD CONSTRAINT countrydata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE dbo.epdata ADD CONSTRAINT epdata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE dbo.seriesdata ADD CONSTRAINT seriesdata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE dbo.tvnetworkdata ADD CONSTRAINT tvnetworkdata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE dbo.tvwebchanneldata ADD CONSTRAINT tvwebchanneldata_pkey PRIMARY KEY (id);
+
+ALTER TABLE dbo.webchanneldata ADD CONSTRAINT webchanneldata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.actordata ADD CONSTRAINT actordata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.actordata ADD CONSTRAINT actorid UNIQUE (actorid);
+
+ALTER TABLE updates.castdata ADD CONSTRAINT castdata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.castdata ADD CONSTRAINT series_actor_character_index UNIQUE (seriesid, characterid, actorid);
+
+ALTER TABLE updates.countrydata ADD CONSTRAINT country_code_idx UNIQUE (country_code);
+
+ALTER TABLE updates.countrydata ADD CONSTRAINT countrydata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.crewdata ADD CONSTRAINT crewdata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.crewdata ADD CONSTRAINT crewid_crewid_idx UNIQUE (crewid);
+
+ALTER TABLE updates.epdata ADD CONSTRAINT epdata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.epdata ADD CONSTRAINT episodeie UNIQUE (episodeid);
+
+ALTER TABLE updates.seriesdata ADD CONSTRAINT seriesdata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.seriesdata ADD CONSTRAINT seriesid UNIQUE (seriesid);
+
+ALTER TABLE updates.tvnetworkdata ADD CONSTRAINT networkid UNIQUE (networkid);
+
+ALTER TABLE updates.tvnetworkdata ADD CONSTRAINT tvnetworkdata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.tvupdates ADD CONSTRAINT tvupdates_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.tvwebchanneldata ADD CONSTRAINT tvwebchanneldata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.webchanneldata ADD CONSTRAINT webchanneldata_pkey PRIMARY KEY (row_id);
+
+ALTER SCHEMA updates OWNER TO postgres;

db/ddl/recreate_updates_schema.sql

@@ -0,0 +1,299 @@
+DROP SCHEMA updates CASCADE;
+
+CREATE SCHEMA updates;
+
+ALTER SCHEMA updates OWNER TO postgres;
+
+CREATE SEQUENCE updates.actordata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.actordata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.castdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.castdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.countrydata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.countrydata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.crewdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.crewdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.epdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.epdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.seriesdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.seriesdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.tvnetworkdata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.tvnetworkdata_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.tvupdates_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.tvupdates_row_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.tvwebchanneldata_id_seq
+    AS integer    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    MAXVALUE 2147483647
+    CACHE 1;
+
+ALTER SEQUENCE updates.tvwebchanneldata_id_seq OWNER TO postgres;
+
+CREATE SEQUENCE updates.webchanneldata_row_id_seq    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE updates.webchanneldata_row_id_seq OWNER TO postgres;
+
+CREATE TABLE updates.actordata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.actordata_row_id_seq'::regclass),
+    actorid integer,
+    actorname character varying(72),
+    countryname character varying(144),
+    countrytimezone character varying(72),
+    birthday date,
+    deathday date,
+    gender character varying(6),
+    mediumimage text,
+    originalimage text,
+    apilink text,
+    url text,
+    countrycode character varying(12));
+
+ALTER TABLE updates.actordata OWNER TO postgres;
+
+ALTER SEQUENCE updates.actordata_row_id_seq OWNED BY updates.actordata.row_id;
+
+CREATE TABLE updates.castdata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.castdata_row_id_seq'::regclass),
+    seriesid integer,
+    characterid integer,
+    actorid integer,
+    charactername integer,
+    mediumimage text,
+    originalimage text,
+    apilink text,
+    tvmazeilink text);
+
+ALTER TABLE updates.castdata OWNER TO postgres;
+
+ALTER SEQUENCE updates.castdata_row_id_seq OWNED BY updates.castdata.row_id;
+
+CREATE TABLE updates.countrydata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.countrydata_row_id_seq'::regclass),
+    country_name character varying(255),
+    country_code character varying(255),
+    country_timezone character varying(255));
+
+ALTER TABLE updates.countrydata OWNER TO postgres;
+
+ALTER SEQUENCE updates.countrydata_row_id_seq OWNED BY updates.countrydata.row_id;
+
+CREATE TABLE updates.crewdata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.crewdata_row_id_seq'::regclass),
+    crewid integer,
+    crew_type character varying(128),
+    name character varying(255),
+    _links_self_href text,
+    country_name text,
+    country_code text,
+    country_timezone text,
+    birthday date,
+    deathday date,
+    gender character varying(24),
+    image_medium text,
+    image_original text,
+    updated text,
+    url text);
+ALTER TABLE updates.crewdata OWNER TO postgres;
+
+ALTER SEQUENCE updates.crewdata_row_id_seq OWNED BY updates.crewdata.row_id;
+
+CREATE TABLE updates.epdata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.epdata_row_id_seq'::regclass),
+    episodeid integer,
+    seriesid integer,
+    season integer,
+    episode_number integer,
+    episode_name text,
+    airdate character varying(32),
+    airtime character varying(32),
+    summary text,
+    airstamp character varying(32),
+    runtime character varying(6),
+    eplink text,
+    image_medium text,
+    url text,
+    image_original text,
+    episode_type text,
+    rating_average text,
+    _links_self_href text,
+    _links_show_href text,
+    _links_show_name text);
+
+ALTER TABLE updates.epdata OWNER TO postgres;
+
+ALTER SEQUENCE updates.epdata_row_id_seq OWNED BY updates.epdata.row_id;
+
+CREATE TABLE updates.seriesdata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.seriesdata_row_id_seq'::regclass),
+    _links_previousepisode_href text,
+    _links_previousepisode_name text,
+    _links_nextepisode_href text,
+    _links_nextepisode_name text,
+   _links_self_href text,
+    averageruntime text,
+    dvdcountry text,
+    series_name text,
+    ended date,
+    externals_imdb text,
+    externals_thetvdb text,
+    externals_tvrage text,
+    genres text,
+    seriesid text,
+    image_medium text,
+    image_original text,
+    language_name text,
+    network_id text,
+    network_name text,
+    officialsite text,
+    network_officialsite text,
+    network_country_code text,
+    network_country_name text,
+    network_country_timezone text,
+    premiered text,
+    rating_average text,
+    runtime text,
+    schedule_days text,
+    schedule_time text,
+    status text,
+    summary text,
+    series_type text,
+    updated text,
+    url text,
+    webchannel_id integer,
+    webchannel_name text,
+	webchannel_country_name text,
+	webchannel_country_code text,
+	webchannel_country_timezone text,
+    webchannel_officialsite text,
+    weight text);
+
+ALTER TABLE updates.seriesdata OWNER TO postgres;
+
+ALTER SEQUENCE updates.seriesdata_row_id_seq OWNED BY updates.seriesdata.row_id;
+
+CREATE TABLE updates.tvnetworkdata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.tvnetworkdata_row_id_seq'::regclass),
+    networkid character varying(12),
+    networkname character varying(255),
+    country character varying(255),
+    countrycode character varying(4),
+    countrytimezone character varying(25));
+
+ALTER TABLE updates.tvnetworkdata OWNER TO postgres;
+
+ALTER SEQUENCE updates.tvnetworkdata_row_id_seq OWNED BY updates.tvnetworkdata.row_id;
+
+CREATE TABLE updates.tvupdates (
+    row_id bigint NOT NULL DEFAULT nextval('updates.tvupdates_row_id_seq'::regclass),
+    seriesid integer,
+    "timestamp" character varying(15));
+
+ALTER TABLE updates.tvupdates OWNER TO postgres;
+
+ALTER SEQUENCE updates.tvupdates_row_id_seq OWNED BY updates.tvupdates.row_id;
+
+CREATE TABLE updates.tvwebchanneldata (
+    row_id integer NOT NULL DEFAULT nextval('updates.tvwebchanneldata_id_seq'::regclass),
+    name character varying(255),
+    countryid character varying(255));
+
+ALTER TABLE updates.tvwebchanneldata OWNER TO postgres;
+
+ALTER SEQUENCE updates.tvwebchanneldata_id_seq OWNED BY updates.tvwebchanneldata.row_id;
+
+CREATE TABLE updates.webchanneldata (
+    row_id bigint NOT NULL DEFAULT nextval('updates.webchanneldata_row_id_seq'::regclass),
+    webchannel_name character varying(255),
+    webchannel_country_code character varying(4),
+    webchannel_country_name character varying(255),
+    webchannelcountry_timezone character varying(4));
+
+ALTER TABLE updates.webchanneldata OWNER TO postgres;
+
+ALTER SEQUENCE updates.webchanneldata_row_id_seq OWNED BY updates.webchanneldata.row_id;
+
+ALTER TABLE updates.actordata ADD CONSTRAINT actordata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.actordata ADD CONSTRAINT actorid UNIQUE (actorid);
+
+ALTER TABLE updates.castdata ADD CONSTRAINT castdata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.castdata ADD CONSTRAINT series_actor_character_index UNIQUE (seriesid, characterid, actorid);
+
+ALTER TABLE updates.countrydata ADD CONSTRAINT country_code_idx UNIQUE (country_code);
+
+ALTER TABLE updates.countrydata ADD CONSTRAINT countrydata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.crewdata ADD CONSTRAINT crewdata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.crewdata ADD CONSTRAINT seriesid_crewid_idx UNIQUE (crewid);
+
+ALTER TABLE updates.epdata ADD CONSTRAINT epdata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.epdata ADD CONSTRAINT episodeie UNIQUE (episodeid);
+
+ALTER TABLE updates.seriesdata ADD CONSTRAINT seriesdata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.seriesdata ADD CONSTRAINT seriesid UNIQUE (seriesid);
+
+ALTER TABLE updates.tvnetworkdata ADD CONSTRAINT networkid UNIQUE (networkid);
+
+ALTER TABLE updates.tvnetworkdata ADD CONSTRAINT tvnetworkdata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.tvupdates ADD CONSTRAINT tvupdates_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.tvwebchanneldata ADD CONSTRAINT tvwebchanneldata_pkey PRIMARY KEY (row_id);
+
+ALTER TABLE updates.webchanneldata ADD CONSTRAINT webchanneldata_pkey PRIMARY KEY (row_id);

db/functions.py

@@ -0,0 +1,605 @@
+__author__ = "Wendell Jones"
+
+import os
+import sqlalchemy
+from sqlalchemy import text
+from sqlalchemy import MetaData
+from sqlalchemy.ext.automap import automap_base
+from sqlalchemy.orm import Session
+from db.Tables import Createtables, CreateTempTables, Tables as dbTables
+from db.mssql.Merger import Charactermerge as MSCharactermerge
+from db.mssql.Merger import Jsonmerge as MSJsonmerge
+from db.mssql.Merger import SQLGenerate as MSSQLGenerate
+from db.mssql.Merger import Episodemerge as MSEpisodemerge
+from db.pgsql.Merger import Charactermerge as PGCharactermerge
+from db.pgsql.Merger import Updatemerge as PGUpdatemerge
+from db.mssql.Merger import Jsonmerge as PGJsonmerge
+from db.pgsql.Merger import SQLGenerate as PGSQLGenerate
+from db.pgsql.Merger import Episodemerge as PGEpisodemerge
+import api.tvmaze.episodes
+import api.tvmaze.cast
+import api.tvmaze.series
+from pymongo import MongoClient
+from tqdm import tqdm
+
+Createtables = Createtables
+CreateTempTables = CreateTempTables
+dbTables = dbTables
+
+
+class settype(object):
+
+    def __init__(self, dbtype, apitype, options):
+        self.dbclass = {}
+        self.dbengine = {}
+        self.apiengine = {}
+        self.load(dbtype)
+        self.dbcreate(dbtype, options)
+        self.setapi(apitype)
+        self.get_data()
+
+    def load(self, dbtype):
+        dbtype = dbtype.lower()
+        if dbtype in ["pgsql", "mysql"]:
+            Updatemerge = PGUpdatemerge
+            Jsonmerge = PGJsonmerge
+            Episodemerge = PGEpisodemerge
+            Charactermerge = PGCharactermerge
+            merger = PGSQLGenerate
+        elif dbtype == "mssql":
+            Jsonmerge = MSJsonmerge
+            Charactermerge = MSCharactermerge
+            merger = MSSQLGenerate
+            Episodemerge = MSEpisodemerge
+
+        self.dbclass["jmerge"] = Jsonmerge()
+        self.dbclass["cmerge"] = Charactermerge()
+        self.dbclass["Createtables"] = Createtables
+        self.dbclass["CreateTemptables"] = CreateTempTables
+        self.dbclass["Tables"] = Createtables
+        self.dbclass["merger"] = merger()
+
+        if dbtype == "pgsql":
+            self.dbclass["Updatemerge"] = Updatemerge
+            self.dbclass["Episodemerge"] = Episodemerge
+        elif dbtype == "mssql":
+            self.dbclass["Charactermerge"] = Charactermerge
+
+    def dbcreate(self, dbtype, options):
+        updatesBase = automap_base()
+        dboBase = automap_base()
+        connection_string = None
+        if dbtype.lower() == "mssql":
+            connection_string = (
+                "mssql+pymssql://"
+                + options["mssqlusername"]
+                + ":"
+                + options["mssqlpassword"]
+                + "@"
+                + options["hostname"]
+                + ":"
+                + options["mssqlport"]
+                + "/"
+                + options["dbname"]
+            )
+        elif dbtype.lower() == "pgsql":
+            connection_string = (
+                "postgresql+psycopg2://"
+                + options["pgsqlusername"]
+                + ":"
+                + options["pgsqlpassword"]
+                + "@"
+                + options["hostname"]
+                + ":"
+                + options["pgsqlport"]
+                + "/"
+                + options["dbname"]
+            )
+        elif dbtype.lower() == "mysql":
+            connection_string = (
+                "mysql+pymysql://"
+                + options["mysqlusername"]
+                + ":"
+                + options["mysqlpassword"]
+                + "@"
+                + options["hostname"]
+                + ":"
+                + options["mysqlport"]
+                + "/"
+                + options["dbname"]
+            )
+        engine = sqlalchemy.create_engine(
+            connection_string, pool_size=20, max_overflow=20, pool_recycle=15
+        )
+        currentDir = os.path.dirname(os.path.abspath(__file__))
+        currentDir = os.path.dirname(currentDir)
+        ddl_file_path = f"{currentDir}/db/ddl/recreate_updates_schema.sql"
+        with open(ddl_file_path, "r") as file:
+            sql = file.read()
+        file
+        with engine.connect() as conn:
+            with conn as cursor:
+                cursor.execute(text(sql))
+                # conn.commit()
+        # with engine.connect() as conn:
+        #     conn.execute(text("CREATE SCHEMA IF NOT EXISTS updates"))
+        #     conn.execute(text("CREATE SCHEMA IF NOT EXISTS dbo"))
+        #     conn.commit()
+        updatesBase.prepare(engine, reflect=True, schema="updates")
+        dboBase.prepare(engine, reflect=True, schema="dbo")
+        metadata = sqlalchemy.MetaData("updates")
+        metadata.reflect(bind=engine)
+        dboMetadata = sqlalchemy.MetaData("dbo")
+        dboMetadata.reflect(bind=engine)
+        session = Session(engine)
+        self.dbengine["engine"] = engine
+        self.dbengine["metadata"] = metadata
+        self.dbengine["dbometadata"] = dboMetadata
+        self.dbengine["updatesBase"] = updatesBase
+        self.dbengine["dboBase"] = dboBase
+        self.dbengine["session"] = session
+        return self.dbengine
+
+    def setapi(self, apitype):
+        if apitype.lower() == "tvmaze":
+            self.apiengine["sprocess"] = api.tvmaze.series
+            self.apiengine["cprocess"] = api.tvmaze.cast
+            self.apiengine["eprocess"] = api.tvmaze.episodes
+            return self.apiengine
+        elif apitype.lower() == "thetvdb":
+            pass
+
+    def reflect_tables(self, engine, Metadata):
+        meta = MetaData()
+        meta.reflect(bind=engine)
+        return meta.tables
+
+    def get_data(self):
+        return self.dbclass, self.dbengine, self.apiengine
+
+    def __del__(self):
+        pass
+
+
+class dbmongo(object):
+
+    global mgclient
+    global mgseries
+    global mgepisodes
+    global mgactors
+    global mgcharacters
+    global mgcrew
+
+    def __init__(self, options):
+        host = options["mghostname"]
+        port = options["mgport"]
+        print(f"Connecting to mongodb at {host}:{port}")
+        try:
+            self.connection_string = (
+                f"mongodb://{options['mghostname']}:{options['mgport']}"
+            )
+            self.mgclient = MongoClient(self.connection_string)
+            self.mgdb = self.mgclient[options["mgdbname"]]
+        except BaseException as e:
+            print(f"Unable to connect to mongodb.{e}\nExitting!")
+            exit()
+        else:
+            self.mgseries = self.mgdb["series"]
+            self.mgseries.create_index("id", unique=True)
+            self.mgepisodes = self.mgdb["episodes"]
+            self.mgepisodes.create_index("id", unique=True)
+            self.mgactors = self.mgdb["actors"]
+            self.mgactors.create_index("id", unique=True)
+            self.mgcharacters = self.mgdb["characters"]
+            self.mgcharacters.create_index("id", unique=True)
+            self.mgcrew = self.mgdb["crew"]
+            self.mgcrew.create_index("id", unique=True)
+            print(
+                f"Connection to mongodb at {options['mghostname']}:"
+                f"{options['mgport']} successful."
+            )
+
+    def load_json_file(self, cachefile, jget, lprint, seriesid, datatype):
+        try:
+            with open(cachefile, "r") as file:
+                jsondata = file.read()
+                datastring = jget.load(jsondata)
+                data = jget.jconvert(datastring)
+                return data
+        except Exception:
+            lprint.logprint(
+                "info", f"Unable to open {datatype} file for {seriesid}"
+            )
+            # print(f'Unable to open {datatype} file for seriesid {seriesid}')
+            return None
+
+    def process_series_data(
+        self,
+        seriesdata,
+        jget,
+        dbExec,
+        dbengine,
+        dboBase,
+        lprint,
+        countrydictionary,
+        seriesid,
+    ):
+        try:
+            seriesresults, networkchannel, country, webchannel = (
+                jget.processSeriesJson(seriesdata)
+            )
+        except Exception:
+            lprint.logprint(
+                "info", f"Unable to process series file for {seriesid}"
+            )
+        else:
+            self.update_country_data(
+                country,
+                countrydictionary,
+                dbExec,
+                dbengine,
+                dboBase,
+                dbengine["dbometadata"],
+                lprint,
+            )
+            self.update_network_data(
+                networkchannel,
+                countrydictionary,
+                seriesresults,
+                dbExec,
+                dbengine,
+                dboBase,
+                lprint,
+            )
+            self.update_webchannel_data(
+                webchannel,
+                countrydictionary,
+                seriesresults,
+                dbExec,
+                dbengine,
+                dboBase,
+                lprint,
+            )
+            self.insert_or_update_mongo(
+                self.mgseries, seriesdata, "id"
+            )
+
+    def process_cast_data(self, persondata, characterdata, seriesid, lprint):
+        try:
+            aupserts = [
+                self.mgactors.update_many(
+                    {"id": x["id"]}, {"$setOnInsert": x}, upsert=True
+                )
+                for x in persondata
+            ]
+            bupserts = [
+                self.mgcharacters.update_many(
+                    {"id": x["id"]}, {"$setOnInsert": x}, upsert=True
+                )
+                for x in characterdata
+            ]
+        except Exception as e:
+            print(f"Skipping cast data.Reason: {str(e)}")
+        else:
+            [
+                self.mgactors.update_many(
+                    {"id": x["id"]}, {"$setOnInsert": x}, upsert=True
+                )
+                for x in aupserts
+            ]
+            [
+                self.mgcharacters.insert_many(
+                    {"id": x["id"]}, {"$setOnInsert": x}, upsert=True
+                )
+                for x in bupserts
+            ]
+
+    def process_crew_data(self, crewdata, seriesid, jget, lprint):
+        try:
+            upserts = [
+                self.mgcrew.update_many(
+                    {"id": x["id"]}, {"$setOnInsert": x}, upsert=True
+                )
+                for x in crewdata
+            ]
+        except Exception as e:
+            print(f"Skipping crew data.Reason: {str(e)}")
+        else:
+            [
+                self.mgcrew.insert_many(
+                    {"id": x["id"]}, {"$setOnInsert": x}, upsert=True
+                )
+                for x in upserts
+            ]
+
+    def process_episode_data(self, episodedata, seriesid, lprint):
+        try:
+            upserts = [
+                self.mgepisodes.update_many(
+                    {"id": x["id"]}, {"$setOnInsert": x}, upsert=True
+                )
+                for x in episodedata
+            ]
+        except Exception as e:
+            print(
+                f"file does not contain the correct json. "
+                f"Skipping. {str(e)}"
+            )
+        else:
+            for episode in upserts:
+                [
+                    self.mgepisodes.insert_many(
+                        {"id": x["id"]}, {"$setOnInsert": x}, upsert=True
+                    )
+                    for x in upserts
+                ]
+
+    def update_country_data(
+        self, country, countrydictionary, dbExec, dbengine, dbBase,
+        metadata, lprint
+    ):
+        if len(country) > 0:
+            try:
+                selectquery = (
+                    f"select row_id, country_name from dbo.countrydata where "
+                    f"country_name = '{country['name'].replace("'", "''")}'"
+                )
+                countrylisting = dbExec.rawsql_select(
+                    dbengine["engine"], selectquery, lprint
+                )
+                for countryname in countrylisting:
+                    countrydictionary[countryname[1]] = countryname[0]
+            except Exception as e:
+                lprint.logprint(
+                    "info", (
+                        (
+                            f'Unable to process country data for '
+                            f'{country["name"]}. {e}'
+                        )
+                    )
+                )
+            else:
+                if len(countrylisting) > 0:
+                    pass
+                else:
+                    insert_sql = (
+                        (
+                            f"insert into dbo.countrydata "
+                            "(country_name, country_code, country_timezone) "
+                            f"values ('{country['name'].replace("'", "''")}', "
+                            f"'{country['code'].replace("'", "''")}', "
+                            f"'{country['timezone'].replace("'", "''")}')"
+                        )
+                    )
+                    dbExec.rawsql_insert(
+                        dbengine["engine"], insert_sql, lprint
+                    )
+                    # seriesresults['countryid'] = 'No Country'
+                    # return seriesresults['countryid']
+
+    def update_network_data(
+        self,
+        networkchannel,
+        countrydictionary,
+        seriesresults,
+        dbExec,
+        dbengine,
+        dboBase,
+        lprint,
+    ):
+        if len(networkchannel) > 0:
+            try:
+                networkchannel["countryid"] = countrydictionary[
+                    networkchannel["country"]["name"]
+                ]
+                seriesresults["networkid"] = networkchannel["id"]
+            except Exception:
+                pass
+
+    def update_webchannel_data(
+        self,
+        webchannel,
+        countrydictionary,
+        seriesresults,
+        dbExec,
+        dbengine,
+        dboBase,
+        lprint,
+    ):
+        if len(webchannel) > 0:
+            try:
+                webchannel["countryid"] = countrydictionary[
+                    webchannel["country"]["name"]
+                ]
+                seriesresults["webchannelid"] = webchannel["id"]
+            except Exception:
+                pass
+            seriesresults["seriesid"] = seriesresults["id"]
+            del seriesresults["webchannel"]
+
+    def insert_or_update_mongo(self, collection, data, unique_key):
+        try:
+            collection.insert_one(data)
+        except Exception:
+            filter_query = {unique_key: data[unique_key]}
+            new_values = {"$set": data}
+            del new_values["$set"]["_id"]
+            collection.update_one(filter_query, new_values)
+
+    def insert_or_update_many_mongo(self, collection, data, unique_key):
+        try:
+            collection.insert_many(data)
+        except Exception:
+            filter_query = {unique_key: data[unique_key]}
+            new_values = {"$set": data}
+            del new_values["$set"]["_id"]
+            collection.update_one(filter_query, new_values)
+
+    def update_mongo(
+        self,
+        updateList,
+        ROOTDIR,
+        directories,
+        jget,
+        countrydictionary,
+        dbExec,
+        dbengine,
+        dboBase,
+        lprint,
+    ):
+        print("Now inserting data into tables.")
+        series_failed_count = 0
+        episode_failed_count = 0
+        cast_failed_count = 0
+        crew_failed_count = 0
+        processed_count = 0
+        failed_file_count = 0
+
+        def process_data(
+            file_path, data_type, process_method, failed_file_count, lprint
+        ):
+            """Helper function to load and process data."""
+            data = self.load_json_file(
+                file_path, jget, lprint, seriesid, data_type
+            )
+            if data:
+                if "episode" in file_path:
+                    try:
+                        for item in data:
+                            item["seriesid"] = seriesid
+                    except Exception as e:
+                        lprint.logprint(
+                            "info", f"Unable to add {data_type} to data. {e}"
+                        )
+                if "cast" in file_path:
+                    persondata = []
+                    characterdata = []
+                    for item in data:
+                        if "person" in item:
+                            persondata.append(item["person"])
+                        if "character" in item:
+                            characterdata.append(item["character"])
+                    try:
+                        for item in persondata:
+                            item["seriesid"] = seriesid
+                        for item in characterdata:
+                            item["seriesid"] = seriesid
+                    except Exception as e:
+                        lprint.logprint(
+                            "info", f"Unable to add {data_type} data. {e}"
+                        )
+
+                    try:
+                        process_method(
+                            persondata, characterdata, seriesid, lprint
+                        )
+                        failed_file_count = "N/A"
+                        return failed_file_count
+                    except Exception as e:
+                        failed_file_count = 1
+                        lprint.logprint(
+                            "info",
+                            f"Unable to process {data_type} data. {e}"
+                        )
+                        return failed_file_count
+                if "crew" in file_path:
+                    try:
+                        crewdata = []
+                    except Exception as e:
+                        lprint.logprint(
+                            "info",
+                            f"Unable to add {data_type} data. {e}"
+                        )
+                    else:
+                        for item in data:
+                            item["seriesid"] = seriesid
+                            if "person" in item:
+                                crewdata.append(item["person"])
+                        try:
+                            process_method(crewdata, seriesid, jget, lprint)
+                            failed_file_count = "N/A"
+                            return failed_file_count
+                        except Exception as e:
+                            failed_file_count = 1
+                            lprint.logprint(
+                                "info",
+                                f"Unable to process {data_type} data.{e}"
+                            )
+                            return failed_file_count
+
+        for series in tqdm(
+            range(len(updateList)), desc="Update MongoDB", unit="series"
+        ):
+            seriesid = updateList[series]
+            seriescachefile = f'{directories["SERIESDIR"]}{seriesid}.json'
+            episodecachefile = f'{directories["EPISODEDIR"]}{seriesid}.json'
+            castcachefile = f'{directories["CASTDIR"]}{seriesid}.json'
+            crewcachefile = f'{directories["CREWDIR"]}{seriesid}.json'
+
+            # Process series data
+            seriesdata = self.load_json_file(
+                seriescachefile, jget, lprint, seriesid, "series"
+            )
+            if seriesdata:
+                try:
+                    self.process_series_data(
+                        seriesdata,
+                        jget,
+                        dbExec,
+                        dbengine,
+                        dboBase,
+                        lprint,
+                        countrydictionary,
+                        seriesid,
+                    )
+                    self.insert_or_update_mongo(
+                        self.mgseries, seriesdata, "id"
+                    )
+                except Exception as e:
+                    series_failed_count += 1
+                    lprint.logprint(
+                        "info",
+                        (
+                            f"Unable to process series data for seriesid "
+                            f"{seriesid}. {e}"
+                        ),
+                    )
+                else:
+                    processed_count += 1
+
+            # Process episode, cast, and crew data using the helper function
+            fcount = process_data(
+                episodecachefile,
+                "episode",
+                self.process_episode_data,
+                failed_file_count,
+                lprint,
+            )
+            if isinstance(fcount, int):
+                episode_failed_count = episode_failed_count + fcount
+            fcount = process_data(
+                castcachefile, "cast", self.process_cast_data,
+                failed_file_count, lprint
+            )
+            if isinstance(fcount, int):
+                cast_failed_count = cast_failed_count + fcount
+            try:
+                fcount = process_data(
+                    crewcachefile, "crew", self.process_crew_data,
+                    failed_file_count, lprint
+                )
+            except Exception:
+                crew_failed_count = crew_failed_count + 1
+            else:
+                if isinstance(fcount, int):
+                    crew_failed_count = crew_failed_count + fcount
+                processed_count += 1
+
+        print(f"Processing complete. {processed_count} series processed.")
+        print(f"Failed to load {series_failed_count} series files.")
+        print(f"Failed to load {episode_failed_count} episode files.")
+        print(f"Failed to load {crew_failed_count} crew files.")
+        print(f"Failed to load {cast_failed_count} cast files.")
+
+    def __del__(self):
+        pass

db/mssql/Merger.py

@@ -0,0 +1,327 @@
+from sqlalchemy import text
+
+class SQLGenerate():
+    
+    def __init__(self):
+        pass
+    
+    def merge_updates(self):
+        updatesql = text('''
+            MERGE updates.tvupdatedata
+            USING updates.tvupdates
+            ON updates.tvupdatedata.seriesid = updates.tvupdates.seriesid
+            WHEN MATCHED THEN
+              UPDATE  SET updates.tvupdatedata.seriesid = updates.tvupdates.seriesid,updates.tvupdatedata.timestamp = updates.tvupdates.timestamp
+            WHEN NOT MATCHED THEN  
+              INSERT (seriesid, timestamp) 
+              VALUES (updates.tvupdates.seriesid, updates.tvupdates.timestamp);
+            ''')
+        return(updatesql)
+    
+    def merge_series(self):
+        seriessql = text('''
+            MERGE dbo.seriesdata
+              USING updates.newseriesdata ON
+            dbo.seriesdata.seriesid = updates.newseriesdata.seriesid
+            WHEN MATCHED THEN UPDATE
+            SET
+              dbo.seriesdata.seriesid = updates.newseriesdata.seriesid,
+              dbo.seriesdata.thetvdb = updates.newseriesdata.thetvdb,
+              dbo.seriesdata.tvrage = updates.newseriesdata.tvrage,
+              dbo.seriesdata.imdb = updates.newseriesdata.imdb,
+              dbo.seriesdata.[language] = updates.newseriesdata.[language],
+              dbo.seriesdata.genres = updates.newseriesdata.genres,
+              dbo.seriesdata.name = updates.newseriesdata.name,
+              dbo.seriesdata.status = updates.newseriesdata.status,
+              dbo.seriesdata.premiered = updates.newseriesdata.premiered,
+              dbo.seriesdata.days = updates.newseriesdata.days,
+              dbo.seriesdata.[time] = updates.newseriesdata.[time],
+              dbo.seriesdata.runtime = updates.newseriesdata.runtime,
+              dbo.seriesdata.rating = updates.newseriesdata.rating,
+              dbo.seriesdata.[type] = updates.newseriesdata.[type],
+              dbo.seriesdata.summary = updates.newseriesdata.summary,
+              dbo.seriesdata.medium = updates.newseriesdata.medium,
+              dbo.seriesdata.original = updates.newseriesdata.original,
+              dbo.seriesdata.[weight] = updates.newseriesdata.[weight],
+              dbo.seriesdata.networkid = updates.newseriesdata.networkid,
+              dbo.seriesdata.webchannelid = updates.newseriesdata.webchannelid,
+              dbo.seriesdata.officialsite = updates.newseriesdata.officalsite,
+              dbo.seriesdata.url = updates.newseriesdata.url,
+              dbo.seriesdata.serieslink = updates.newseriesdata.serieslink,
+              dbo.seriesdata.updated_on = GETDATE()
+              WHEN NOT MATCHED THEN INSERT
+                (seriesid,
+                thetvdb,
+                tvrage,
+                imdb,
+                [language],
+                genres,
+                name,
+                status,
+                premiered,
+                days,
+                [time],
+                runtime,
+                rating,
+                [type],
+                summary,
+                medium,
+                original,
+                weight,
+                networkid,
+                webchannelid,
+                officialsite,
+                url,
+                serieslink,
+                created_on)
+              VALUES (updates.newseriesdata.seriesid,
+              updates.newseriesdata.thetvdb,
+              updates.newseriesdata.tvrage,
+              updates.newseriesdata.imdb,
+              updates.newseriesdata.[language],
+              updates.newseriesdata.genres,
+              updates.newseriesdata.name,
+              updates.newseriesdata.status,
+              updates.newseriesdata.premiered,
+              updates.newseriesdata.days,
+              updates.newseriesdata.[time],
+              updates.newseriesdata.runtime,
+              updates.newseriesdata.rating,
+              updates.newseriesdata.[type],
+              updates.newseriesdata.summary,
+              updates.newseriesdata.medium,
+              updates.newseriesdata.original,
+              updates.newseriesdata.weight,
+              updates.newseriesdata.networkid,
+              updates.newseriesdata.webchannelid,
+              updates.newseriesdata.officalsite,
+              updates.newseriesdata.url,
+              updates.newseriesdata.serieslink,
+              GETDATE());
+            ''')
+        return(seriessql)
+
+    def merge_episodes(self):
+        episodesql = text('''
+            MERGE dbo.epdata
+              USING updates.newepdata ON
+            dbo.epdata.episodeid = updates.newepdata.episodeid
+            WHEN MATCHED THEN UPDATE
+            SET
+              dbo.epdata.episodeid = updates.newepdata.episodeid,
+              dbo.epdata.seriesid = updates.newepdata.seriesid,
+              dbo.epdata.season = updates.newepdata.season,
+              dbo.epdata.episodenumber = updates.newepdata.episodenumber,
+              dbo.epdata.episodename = updates.newepdata.episodename,
+              dbo.epdata.airdate = updates.newepdata.airdate,
+              dbo.epdata.airtime = updates.newepdata.airtime,
+              dbo.epdata.episodesummary = updates.newepdata.episodesummary,
+              dbo.epdata.airstamp = updates.newepdata.airstamp,
+              dbo.epdata.runtime = updates.newepdata.runtime,
+              dbo.epdata.eplink = updates.newepdata.eplink,
+              dbo.epdata.mediumimage = updates.newepdata.mediumimage,
+              dbo.epdata.originalimage = updates.newepdata.originalimage,
+              dbo.epdata.[url] = updates.newepdata.[url],
+              dbo.epdata.updated_on = GETDATE()
+              WHEN NOT MATCHED THEN INSERT
+                (episodeid,
+                seriesid,
+                season,
+                episodenumber,
+                episodename,
+                airdate,
+                airtime,
+                episodesummary,
+                airstamp,
+                runtime,
+                eplink,
+                mediumimage,
+                originalimage,
+                [url],
+                created_on)
+              VALUES (updates.newepdata.episodeid,
+              updates.newepdata.seriesid,
+              updates.newepdata.season,
+              updates.newepdata.episodenumber,
+              updates.newepdata.episodename,
+              updates.newepdata.airdate,
+              updates.newepdata.airtime,
+              updates.newepdata.episodesummary,
+              updates.newepdata.airstamp,
+              updates.newepdata.runtime,
+              updates.newepdata.eplink,
+              updates.newepdata.mediumimage,
+              updates.newepdata.originalimage,
+              updates.newepdata.[url],
+              GETDATE());
+            ''')
+        return(episodesql)   
+
+    def merge_cast(self):
+        castsql = text('''
+        MERGE castdata as N
+        USING updates.newcastdata as O
+        ON N.seriesid = O.seriesid and N.characterid = O.characterid and N.actorid = O.actorid
+         WHEN MATCHED THEN
+          UPDATE  SET N.seriesid = O.seriesid, N.characterid = O.characterid,N.actorid = O.actorid,N.charactername = O.charactername,N.mediumimage = O.mediumimage,N.originalimage = O.originalimage,N.apilink = O.apilink,N.tvmazelink = O.tvmazelink, N.updated_on = GETDATE()
+        WHEN NOT MATCHED THEN  
+          INSERT (seriesid,characterid,actorid,charactername,mediumimage,originalimage,apilink,tvmazelink, created_on) 
+          VALUES 
+          (O.seriesid,O.characterid,O.actorid,O.charactername,O.mediumimage,O.originalimage,O.apilink,O.tvmazelink, GETDATE());
+        ''')
+        return(castsql)
+    
+    def merge_actors(self):
+        actorsql = text('''
+        MERGE actors as N
+        USING updates.newactordata as O
+        ON N.actorid = O.actorid and N.actorname = O.actorname
+         WHEN MATCHED THEN
+          UPDATE  SET N.actorid = O.actorid, N.actorname = O.actorname, N.countryname = O.countryname, N.countrytimezone = O.countrytimezone, N.birthday = O.birthday, N.deathday = O.deathday, N.gender = O.gender, N.mediumimage = O.mediumimage, N.originalimage = O.originalimage, N.apilink = O.apilink, N.url = O.url, N.countrycode = O.countrycode, N.updated_on = GETDATE()
+        WHEN NOT MATCHED THEN  
+          INSERT (actorid,actorname,countryname,countrytimezone,birthday,deathday,gender,mediumimage,originalimage,apilink,url,countrycode,created_on) 
+          VALUES 
+          (O.actorid,O.actorname,O.countryname,O.countrytimezone,O.birthday,O.deathday,O.gender,O.mediumimage,O.originalimage,O.apilink,O.url,O.countrycode, GETDATE());
+        ''')
+        return(actorsql)
+    
+    def merge_crew(self):
+      crewsql = text('''
+      MERGE dbo.crewdata as N
+      USING updates.newcrewdata as O ON
+      N.[type] = O.[type]
+      and N.seriesid = O.seriesid
+      and N.crewid = O.crewid
+      WHEN MATCHED THEN UPDATE
+      SET
+        N.seriesid = O.seriesid,
+        N.crewid = O.crewid,
+        N.countryid = O.countryid,
+        N.type = O.type,
+        N.name = O.name,
+        N.medium = O.medium,
+        N.original = O.original,
+        N.apilink = O.apilink,
+        N.gender = O.gender,
+        N.birthday = O.birthday,
+        N.deathday = O.deathday,
+        N.updated_on = GETDATE()
+        WHEN NOT MATCHED THEN INSERT
+          (seriesid,
+          crewid,
+          countryid,
+          type,
+          name,
+          medium,
+          original,
+          apilink,
+          gender,
+          birthday,
+          deathday,
+          created_on)
+        VALUES (O.seriesid,
+          O.crewid,
+          O.countryid,
+          O.type,
+          O.name,
+          O.medium,
+          O.original,
+          O.apilink,
+          O.gender,
+          O.birthday,
+          O.deathday,
+        GETDATE());
+        ''')
+      return(crewsql)
+    
+    def merge_country(self):
+        countrysql = text('''
+            MERGE countrydata
+            USING updates.newcountrydata
+            ON countrydata.id = updates.newcountrydata.id
+            WHEN MATCHED THEN
+              UPDATE  SET countrydata.name = updates.newcountrydata.name, countrydata.code = updates.newcountrydata.code, countrydata.timezone = updates.newcountrydata.timezone, countrydata.updated_on = GETDATE()
+            WHEN NOT MATCHED THEN  
+              INSERT (name, code, timezone, created_on) 
+              VALUES 
+              (updates.newcountrydata.name,updates.newcountrydata.code,updates.newcountrydata.timezone, GETDATE());
+            ''')
+        return(countrysql)   
+                
+    def __del__(self):
+        pass
+        
+class Charactermerge():
+    
+    def __init__(self):
+        pass
+    
+    def __del__(self):
+        pass
+
+class Episodemerge():
+    def __init_(self):
+        pass
+
+    def __del__(self):
+        pass
+
+class Jsonmerge():
+    
+    def __init__(self):
+        pass
+    
+    def merge(self, engine):
+        seriessql = text('''BEGIN TRANSACTION;
+            MERGE dbo.seriesjson as N
+            USING updates.seriesjsonnewvals as O
+            ON N.seriesid = O.seriesid and N.jdata = O.jdata
+            WHEN MATCHED THEN
+              UPDATE  SET N.seriesid = O.seriesid, N.jdata = O.jdata, N.updated_on = GETDATE()
+            WHEN NOT MATCHED THEN  
+              INSERT (seriesid, jdata, created_on) 
+              VALUES 
+              (O.seriesid, O.jdata, GETDATE());
+            COMMIT;
+            ''')
+        with engine.connect() as con:
+            rs = con.execute(seriessql)
+        #engine.commit()
+            
+        episodesql = text('''BEGIN TRANSACTION;
+            MERGE updates.tvmaze_episodejson as N
+            USING updates.episodejsonnewvals as O
+            ON N.seriesid = O.seriesid
+             WHEN MATCHED THEN
+              UPDATE  SET N.seriesid = O.seriesid, N.jdata = O.jdata, N.updated_on = GETDATE()
+            WHEN NOT MATCHED THEN  
+              INSERT (seriesid, jdata, created_on) 
+              VALUES 
+              (O.seriesid, O.jdata, GETDATE());
+            SELECT * FROM updates.tvmaze_episodejson;COMMIT;
+            ''')
+        with engine.connect() as con:
+            rs = con.execute(episodesql)
+        #engine.commit()
+            
+        charactersql = text('''BEGIN TRANSACTION;
+            MERGE updates.tvmaze_characterjson as N
+            USING updates.characterjsonnewvals as O
+            ON N.seriesid = O.seriesid
+             WHEN MATCHED THEN
+              UPDATE  SET N.seriesid = O.seriesid, N.jdata = O.jdata, N.updated_on = GETDATE()
+            WHEN NOT MATCHED THEN  
+              INSERT (seriesid, jdata, created_on) 
+              VALUES 
+              (O.seriesid, O.jdata, GETDATE());
+            SELECT * FROM updates.tvmaze_characterjson;COMMIT;
+            ''')
+        with engine.connect() as con:
+            rs = con.execute(charactersql)
+        #engine.commit()
+    
+    def __del__(self):
+        pass
+
+    
+    

db/mssql/Tables.py

@@ -0,0 +1,376 @@
+import sqlalchemy
+from sqlalchemy import create_engine, MetaData, Table, Column, Integer, String, Sequence, text, UniqueConstraint
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.schema import MetaData
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.types import *
+
+class Tables():
+    def __init__(self):
+        global tvnetworknewvals
+        global tvwebchannelnewvals
+        global characterjsonnewvals
+        global seriesjsonnewvals
+        global episodejsonnewvals
+        global metadata
+        global engine
+        global session
+        global tvnetworknewvals
+        global tvwebchannelnewvals
+        global characterjsonnewvals
+        global seriesjsonnewvals
+        global episodejsonnewvals
+        global tvupdates
+        global newseriesdata
+        global actorlinks
+        global tablelist
+        
+    def __del__(self):
+        pass
+
+
+class Createtables():
+    
+    def __init__(self, session, metadata, engine, tablelist):
+        Base = declarative_base(metadata = metadata)
+        self.tablelist = self.settables(metadata, engine)
+        try:
+            self.drop(session, self.tablelist)
+        except:
+            pass
+        self.create(Base, engine, metadata, session, self.tablelist)
+        session.commit()
+        #self.send(tablelist)
+    
+    def settables(self, metadata, engine):
+        tablelist = []
+        newseriesdata = Table('newseriesdata', metadata,
+            Column('id', BIGINT, Sequence('newseriesdata_id_seq'), primary_key = True),
+            Column('seriesid',INTEGER, unique=True),
+            Column('tvdbid', String(25)),
+            Column('tvrageid', String(25)),
+            Column('imdbid', String(25)),
+            Column('serieslanguage', String(25)),
+            Column('genres', String(75)),
+            Column('seriesname', String(150)),
+            Column('seriesstatus', String(25)),
+            Column('premiered', String(25)),
+            Column('airdays', String(72)),
+            Column('airtime', String(25)),
+            Column('runtime', String(25)),
+            Column('rating', String(25)),
+            Column('showtype',TEXT),
+            Column('overview',TEXT),
+            Column('mediumimage',TEXT),
+            Column('originalimage',TEXT),
+            Column('weight',INTEGER),
+            Column('networkid', String(25)),
+            Column('webchannelid', String(25)),
+            Column('officialsite',TEXT),
+            Column('showurl',TEXT),
+            Column('apilink',TEXT),
+            schema = "updates")
+        tablelist.append(newseriesdata)
+        newactordata = Table('newactordata', metadata,
+            Column('id', BIGINT, Sequence('newactordata_id_seq'), primary_key = True),
+            Column('actorid', INTEGER, unique=True),
+            Column('actorname', VARCHAR(72)),
+            Column('countryname', VARCHAR(144)),
+            Column('countrytimezone', VARCHAR(72)),
+            Column('birthday', DATE),
+            Column('deathday', DATE),
+            Column('gender', VARCHAR(6)),
+            Column('mediumimage', TEXT),
+            Column('originalimage', TEXT),
+            Column('apilink', TEXT),
+            Column('url', TEXT),
+            Column('countrycode', VARCHAR(12)),
+            schema = "updates")
+        tablelist.append(newactordata)
+        newepdata = Table('newepdata', metadata,
+        Column('id', BIGINT, Sequence('newepdata_id_seq'), primary_key = True),
+            Column('episodeid', INTEGER),
+            Column('seriesid', INTEGER),
+            Column('season', INTEGER),
+            Column('episodenumber', INTEGER),
+            Column('episodename', TEXT),
+            Column('airdate', VARCHAR(32)),
+            Column('airtime', VARCHAR(32)),
+            Column('episodesummary', TEXT),
+            Column('airstamp', VARCHAR(32)),
+            Column('runtime', VARCHAR(6)),
+            Column('eplink', TEXT),
+            Column('mediumimage', TEXT),
+            Column('url', TEXT),
+            Column('originalimage', TEXT),
+            schema = "updates")
+        tablelist.append(newepdata)
+        newcastdata = Table('newcastdata', metadata,
+        Column('id', BIGINT, Sequence('newcastdata_id_seq'), primary_key = True),
+            Column('seriesid', INTEGER),
+            Column('characterid', INTEGER),
+            Column('actorid', INTEGER),
+            Column('charactername', VARCHAR(250)),
+            Column('mediumimage', TEXT),
+            Column('originalimage', TEXT),
+            Column('apilink', TEXT),
+            Column('tvmazelink', TEXT),
+            schema = "updates")
+        tablelist.append(newcastdata)
+        tvnetworks = Table('tvnetworkdata', metadata,
+          Column('id', BIGINT, Sequence('network_id_seq'), primary_key = True),
+          Column('networkid', String(12)),
+          Column('networkname', String(255)),
+          Column('country', String(255)),
+          Column('countrycode', String(4)),
+          Column('countrytimezone', String(25)),
+          schema = "updates")
+        tablelist.append(tvnetworks)
+        tvwebchannels = Table('tvwebchanneldata', metadata,
+          Column('id', BIGINT, Sequence('webchannel_id_seq'), primary_key = True),
+          Column('webchannelname', String(255)),
+          Column('webchannelcountrycode', String(4)),
+          Column('webchannelcountryname', String(255)),
+          Column('webchannelcountrytimezone', String(4)),
+          schema = "updates")
+        tablelist.append(tvwebchannels)
+        return(tablelist)
+    
+    def create(self, Base, engine, metadata, session, tablelist):
+        print('Creating Temporary Data Tables')
+        Base.metadata.create_all(engine, tables=tablelist)
+
+    def drop(self, session, tablelist):
+        for table in tablelist:
+            try:
+                #print('Dropping ' + str(table))
+                table.drop()
+            except sqlalchemy.exc.ProgrammingError as se:
+                pass
+            
+        def __del__(self):
+            pass
+            
+
+class Temptables():
+    
+    def __init__(self, session, metadata, engine, tablelist):
+        Base = declarative_base(metadata = metadata)
+        self.tablelist = self.settables(metadata, engine)
+        try:
+            self.drop(session, self.tablelist)
+        except:
+            pass
+        self.create(Base, engine, metadata, session, self.tablelist)
+        #session.commit()
+        
+    def settables(self, metadata, engine):
+        tablelist = []
+        tvnetworknewvals = Table('tvnetworknewvals', metadata,
+          Column('id', BIGINT, Sequence('network_id_seq'), primary_key = True),
+          Column('name', String(255)),
+          Column('countryid', String(255)),
+          schema = "updates")
+        tablelist.append(tvnetworknewvals)
+        tvwebchannelnewvals = Table('tvwebchannelnewvals', metadata,
+          Column('id', BIGINT, Sequence('webchannel_id_seq'), primary_key = True),
+          Column('name', String(255)),
+          Column('countryid', String(4)),
+          schema = "updates")
+        tablelist.append(tvwebchannelnewvals)
+        tvupdates = Table('tvupdates', metadata,
+        Column('seriesid', BIGINT, primary_key = True),
+            Column('timestamp', VARCHAR(15)),
+          schema = "updates")
+        tablelist.append(tvupdates)
+        tvupdatedata = Table('tvupdatedata', metadata,
+        Column('seriesid', BIGINT, primary_key = True),
+            Column('timestamp', VARCHAR(15)),
+          schema = "updates")
+        tablelist.append(tvupdatedata)
+        newactorlinks = Table('newactorlinks', metadata,
+          Column('id', BIGINT, Sequence('tempepisodejson_id_seq'), primary_key = True),
+          Column('actorid', INTEGER),
+          Column('characterid', INTEGER),
+          schema = "updates")
+        tablelist.append(newactorlinks)
+        return(tablelist)
+    
+    def create(self, Base, engine, metadata, session, tablelist):
+        tablelist.remove(tablelist[6])
+        print('Creating Temporary JSON Tables')
+        try:
+            Base.metadata.create_all(engine, tables=tablelist)
+        except sqlalchemy.exc.ProgrammingError as pe:
+            pass 
+
+    def drop(self, session, tablelist):
+        for table in tablelist:
+            if 'tvupdatedata' in table.description:
+                pass
+            else:
+                try:
+                    #print('Dropping ' + str(table))
+                    table.drop()
+                except sqlalchemy.exc.ProgrammingError as se:
+                    pass
+        #session.commit()
+
+class CreateTempTables():
+    
+    global tvnetworknewvals
+    global tvwebchannelnewvals
+    global characterjsonnewvals
+    global seriesjsonnewvals
+    global episodejsonnewvals
+    global metadata
+    global engine
+    global session
+    
+    global tablelist
+    
+    def __init__(self, session, metadata, engine, tablelist):
+        Base = declarative_base(metadata = metadata)
+        self.tablelist = self.settables(metadata, engine)
+        try:
+            self.drop(session, self.tablelist)
+        except:
+            pass
+        self.create(Base, engine, metadata, session, self.tablelist)
+        session.commit()
+        #self.send(tablelist)
+    
+    def settables(self, metadata, engine):
+        tablelist = []
+        newseriesdata = Table('newseriesdata', metadata,
+            Column('id', BIGINT, Sequence('newseriesdata_id_seq'), primary_key = True),
+            Column('seriesid',INTEGER, unique=True),
+            Column('thetvdb', String(25)),
+            Column('tvrage', String(25)),
+            Column('imdb', String(25)),
+            Column('language', String(25)),
+            Column('genres', String(75)),
+            Column('name', String(150)),
+            Column('status', String(25)),
+            Column('premiered', String(25)),
+            Column('days', String(72)),
+            Column('time', String(25)),
+            Column('runtime', String(25)),
+            Column('rating', String(25)),
+            Column('type',TEXT),
+            Column('summary',TEXT),
+            Column('medium',TEXT),
+            Column('original',TEXT),
+            Column('weight',INTEGER),
+            Column('networkid', String(25)),
+            Column('webchannelid', String(25)),
+            Column('officialsite',TEXT),
+            Column('url',TEXT),
+            Column('serieslink',TEXT),
+            schema="updates")
+        tablelist.append(newseriesdata)
+        newactordata = Table('newactordata', metadata,
+            Column('id', BIGINT, Sequence('newactordata_id_seq'), primary_key = True),
+            Column('actorid', INTEGER, unique=True),
+            Column('actorname', VARCHAR(72)),
+            Column('countryname', VARCHAR(36)),
+            Column('countrytimezone', VARCHAR(72)),
+            Column('birthday', DATE),
+            Column('deathday', DATE),
+            Column('gender', VARCHAR(6)),
+            Column('mediumimage', TEXT),
+            Column('originalimage', TEXT),
+            Column('apilink', TEXT),
+            Column('url', TEXT),
+            Column('countrycode', VARCHAR(12)),
+            schema="updates")
+        tablelist.append(newactordata)
+        newepdata = Table('newepdata', metadata,
+        Column('id', BIGINT, Sequence('newepdata_id_seq'), primary_key = True),
+            Column('episodeid', INTEGER),
+            Column('seriesid', INTEGER),
+            Column('season', INTEGER),
+            Column('episodenumber', INTEGER),
+            Column('episodename', TEXT),
+            Column('airdate', VARCHAR(32)),
+            Column('airtime', VARCHAR(32)),
+            Column('episodesummary', TEXT),
+            Column('airstamp', VARCHAR(32)),
+            Column('runtime', VARCHAR(6)),
+            Column('eplink', TEXT),
+            Column('mediumimage', TEXT),
+            Column('url', TEXT),
+            Column('originalimage', TEXT),
+            schema="updates")
+        tablelist.append(newepdata)
+        newcastdata = Table('newcastdata', metadata,
+        Column('id', BIGINT, Sequence('newcastdata_id_seq'), primary_key = True),
+            Column('seriesid', INTEGER),
+            Column('characterid', VARCHAR(24)),
+            Column('actorid', VARCHAR(24)),
+            Column('url', TEXT),
+            Column('charactername', VARCHAR(128)),
+            Column('mediumimage', TEXT),
+            Column('originalimage', TEXT),
+            Column('apilink', TEXT),
+            Column('tvmazelink', TEXT),
+            Column('self', CHAR),
+            Column('voice', CHAR),
+            schema="updates")
+        tablelist.append(newcastdata)
+        newcrewdata = Table('newcrewdata', metadata,
+        Column('id', BIGINT, Sequence('newcrewdata_id_seq'), primary_key = True),
+            Column('seriesid', INTEGER),
+            Column('crewid', VARCHAR(128)),
+            Column('type', VARCHAR(128)),
+            Column('url', TEXT),
+            Column('name', VARCHAR(256)),
+            Column('countryid', INTEGER),
+            Column('birthday', DATE),
+            Column('deathday', DATE),
+            Column('gender', VARCHAR(24)),
+            Column('medium', TEXT),
+            Column('original', TEXT),
+            Column('apilink', TEXT),
+            UniqueConstraint('seriesid', 'crewid', 'type', name='crew_unique'),
+            schema="updates")
+        tablelist.append(newcrewdata)
+        tvnetworks = Table('tvnetworkdata', metadata,
+          Column('id', BIGINT, Sequence('network_id_seq'), primary_key = True),
+          Column('name', String(255)),
+          Column('countryid', String(255)),
+          schema = "updates")
+        tablelist.append(tvnetworks)
+        tvwebchannels = Table('tvwebchanneldata', metadata,
+          Column('id', BIGINT, Sequence('webchannel_id_seq'), primary_key = True),
+          Column('name', String(255)),
+          Column('countryid', String(255)),
+          schema = "updates")
+        tablelist.append(tvwebchannels)
+        countries = Table('newcountrydata', metadata,
+          Column('id', BIGINT, Sequence('country_id_seq'), primary_key = True),
+          Column('name', String(255)),
+          Column('code', String(255)),
+          Column('timezone', String(255)),
+          UniqueConstraint('name', 'timezone', name='tz_name_uk'),
+          schema = "updates")
+        tablelist.append(countries)
+        tvupdates = Table('tvupdates', metadata,
+        Column('seriesid', BIGINT, primary_key = True),
+            Column('timestamp', VARCHAR(15)),
+          schema = "updates")
+        tablelist.append(tvupdates)
+        return(tablelist)
+    
+    def create(self, Base, engine, metadata, session, tablelist):
+        print('Creating Temporary Data Tables')
+        Base.metadata.create_all(engine, tables=tablelist)
+
+    def drop(self, session, tablelist):
+        print('Dropping temporary update tables.')
+        for table in tablelist:
+            try:
+                print('Dropping ' + str(table))
+                table.drop()
+            except sqlalchemy.exc.ProgrammingError as se:
+                pass

db/mssql/Updater.py

@@ -0,0 +1,39 @@
+import sqlalchemy
+import progressbar
+from tvmaze.Processor import loadjson
+
+class Updater():
+    def __init__(self):
+        global jprocess
+        
+    def __del__(self):
+        pass
+
+class tvupdates():
+    def __init__(self):
+        pass        
+
+    def update_tables(engine, updatesapi, tablename):
+        jprocess = loadjson()
+        print('Now retrieving available updates.')
+        print('Creating update transaction.')
+        ins = tablename.insert()
+        seriesupdates = jprocess.load(updatesapi)
+        updatelist = []
+        seriesrange = len(seriesupdates) 
+        with progressbar.ProgressBar(maxval=seriesrange, redirect_stdout=True) as p:
+            for key, value in seriesupdates.items():
+                i = int(key)
+                updater = {}
+                updater['seriesid'] = key
+                updater['timestamp'] = value
+                updatelist.append(updater)
+                if i <= seriesrange:
+                    p.update(i)
+                #time.sleep(0.1) 
+        print('Update of tvupdates table is complete.')
+        print('Done.')
+        return(ins, updatelist)
+
+    def __del__(self):
+        pass

db/mssql/execute.py

@@ -0,0 +1,80 @@
+import sqlalchemy
+import progressbar
+import psycopg2.errors
+
+class Update():
+    def __init__(self):
+        global jprocess
+
+    def rawexec(self, sqlquery):
+        connection = engine.connect()
+        try:    
+            trans = connection.begin()
+            connection.execute(str(sqlquery))
+        except:
+            trans.rollback()
+        else:
+            trans.commit()
+            connection.close()
+                
+    def fileexec(self, sqlfile):
+        connection = engine.connect()
+        trans = connection.begin()
+        for line in sqlfile:
+            sqlquery = sqlquery + str(line)
+        try:
+            connection.execute(str(sqlquery))
+        except:
+            trans.rollback()
+            lprint.logprint('error', 'Rollback for filename: ' + str(sqlfile))
+            sqlfile.close()
+            connection.close()
+        else:
+            trans.commit()
+            sqlfile.close()
+            connection.close()
+        
+    def ormexec(self, listing, tablename):
+        try:
+            ins = tablename.insert()
+            result = engine.execute(ins, listing)
+        except sqlalchemy.exc.IntegrityError as ie:
+            if 'UNIQUE KEY' in ie.args[0]:
+                return True
+        except psycopg2.errors.UniqueViolation as ie2:
+            if 'unique constraint' in ie.args[0]:
+                return True
+        else:
+            return False        
+        
+    def __del__(self):
+        pass
+
+class tvupdates():
+    def __init__(self):
+        pass        
+
+    def update_tables(engine, updatesapi, tablename):
+        jprocess = loadjson()
+        print('Now retrieving available updates.')
+        print('Creating update transaction.')
+        ins = tablename.insert()
+        seriesupdates = jprocess.load(updatesapi)
+        updatelist = []
+        seriesrange = len(seriesupdates) 
+        with progressbar.ProgressBar(maxval=seriesrange, redirect_stdout=True) as p:
+            for key, value in seriesupdates.items():
+                i = int(key)
+                updater = {}
+                updater['seriesid'] = key
+                updater['timestamp'] = value
+                updatelist.append(updater)
+                if i <= seriesrange:
+                    p.update(i)
+                #time.sleep(0.1) 
+        print('Update of tvupdates table is complete.')
+        print('Done.')
+        return(ins, updatelist)
+
+    def __del__(self):
+        pass

db/pgsql/Merger.py

@@ -0,0 +1,309 @@
+import datetime
+from sqlalchemy import text
+from sqlalchemy import DateTime
+
+class SQLGenerate():
+    def __init__(self):
+        pass
+
+    def merge_series(self):
+        seriessql = text('''
+        INSERT INTO dbo.seriesdata (serieslink, url, thetvdb, tvrage, imdb, 
+            genres, seriesid, medium,original, language, name, webchannelid, networkid, premiered, rating, runtime, days, time,
+            status, summary, type, weight) SELECT serieslink, url, thetvdb, tvrage, imdb, 
+            genres, seriesid, medium,original, language, name, webchannelid, networkid, premiered, rating, runtime, days, time,
+            status, summary, type, weight from updates.newseriesdata
+            ON CONFLICT(seriesid) DO UPDATE
+            SET serieslink = excluded.serieslink,
+            url = excluded.url,
+            thetvdb = excluded.thetvdb,
+            tvrage = excluded.tvrage,
+            genres = excluded.genres,
+            seriesid = excluded.seriesid,
+            medium = excluded.medium,
+            original = excluded.original,
+            language = excluded.language,
+            name = excluded.name,
+            webchannelid = excluded.webchannelid,
+            networkid = excluded.networkid,
+            premiered = excluded.premiered,
+            rating = excluded.rating,
+            runtime = excluded.runtime,
+            days = excluded.days,
+            time = excluded.time,
+            status = excluded.status,
+            summary = excluded.summary,
+            type = excluded.type,
+            weight = excluded.weight;
+            ''')
+        return(seriessql)
+
+    def merge_episodes(self):
+        episodesql = text('''
+            INSERT INTO dbo.epdata (episodeid,
+                seriesid,
+                season,
+                episodenumber,
+                episodename,
+                airdate,
+                airtime,
+                episodesummary,
+                airstamp,
+                runtime,
+                eplink,
+                mediumimage,
+                originalimage,
+                "url")
+			SELECT episodeid,
+                seriesid,
+                season,
+                episodenumber,
+                episodename,
+                airdate,
+                airtime,
+                episodesummary,
+                airstamp,
+                runtime,
+                eplink,
+                mediumimage,
+                originalimage,
+                "url"
+			FROM updates.newepdata
+			ON CONFLICT (seriesid, episodeid) do UPDATE
+			SET episodeid = excluded.episodeid,
+                seriesid = excluded.seriesid,
+                season = excluded.season,
+                episodenumber = excluded.episodenumber,
+                episodename = excluded.episodename,
+                airdate = excluded.airdate,
+                airtime = excluded.airtime,
+                episodesummary = excluded.episodesummary,
+                airstamp = excluded.airstamp,
+                runtime = excluded.runtime,
+                eplink = excluded.eplink,
+                mediumimage = excluded.mediumimage,
+                originalimage = excluded.originalimage,
+                "url" = excluded."url";
+            ''')
+        return(episodesql)
+
+    def merge_updates(self):
+        updatesql = text('''
+        INSERT INTO updates.tvupdatedata(seriesid, timestamp)
+        SELECT seriesid, timestamp from updates.tvupdates
+        ON CONFLICT(seriesid) DO UPDATE
+        SET seriesid = excluded.seriesid, timestamp = excluded.timestamp;
+        ''')
+        return(updatesql)
+
+    def merge_cast(self):
+        datetimestamp = DateTime(datetime.datetime.utcnow)
+        castsql = text('''
+        INSERT INTO dbo.castdata (seriesid,characterid,actorid,charactername,mediumimage,originalimage,apilink,tvmazeilink)
+        SELECT seriesid,characterid,actorid,charactername,mediumimage,originalimage,apilink,tvmazeilink
+        FROM updates.newcastdata
+        ON CONFLICT (seriesid, characterid, actorid) DO UPDATE
+        SET seriesid = excluded.seriesid,
+        characterid = excluded.characterid,
+        actorid = excluded.actorid,
+        charactername = excluded.charactername,
+        mediumimage = excluded.mediumimage,
+        originalimage = excluded.originalimage,
+        apilink = excluded.apilink,
+        tvmazeilink = excluded.tvmazeilink
+        ''')
+        return(castsql)
+    
+    def merge_actors(self):
+        actorsql = text('''
+        INSERT INTO dbo.actors (actorid,actorname,countryname,countrytimezone,birthday,deathday,gender,mediumimage,originalimage,apilink,"url",countrycode)
+		SELECT actorid,actorname,countryname,countrytimezone,birthday,deathday,gender,mediumimage,originalimage,apilink,"url",countrycode
+		FROM updates.newactordata
+		ON CONFLICT (actorid) DO UPDATE
+		SET actorid = excluded.actorid,
+		actorname = excluded.actorname,
+		countryname = excluded.countryname,
+		countrytimezone = excluded.countrytimezone,
+		birthday = excluded.birthday,
+		deathday = excluded.deathday,
+		gender = excluded.gender,
+		mediumimage = excluded.mediumimage,
+		originalimage = excluded.originalimage,
+		apilink = excluded.apilink,
+		"url" = excluded."url",
+		countrycode = excluded.countrycode;
+		''')
+        return(actorsql)
+
+    def merge_crew(self):
+      crewsql = text('''
+        INSERT INTO dbo.crewdata
+	    (seriesid,
+          crewid,
+          countryid,
+          type,
+          name,
+          medium,
+          original,
+          apilink,
+          gender,
+          birthday,
+          deathday)
+        SELECT seriesid,
+          crewid,
+          countryid,
+          type,
+          name,
+          medium,
+          original,
+          apilink,
+          gender,
+          birthday,
+          deathday
+		FROM updates.newcrewdata
+		ON CONFLICT (seriesid, crewid, type) DO UPDATE
+		SET seriesid = excluded.seriesid,
+          crewid = excluded.crewid,
+          countryid = excluded.countryid,
+          type = excluded.type,
+          name = excluded.name,
+          medium = excluded.medium,
+          original = excluded.original,
+          apilink = excluded.apilink,
+          gender = excluded.gender,
+          birthday = excluded.birthday,
+          deathday = excluded.deathday;
+        ''')
+      return(crewsql)
+
+    def __del__(self):
+        pass
+class Merger():
+    
+    def __init__(self):
+        pass
+
+    def __del__(self):
+        pass
+    
+class Charactermerge():
+    
+    def __init__(self):
+        pass
+    
+    def merge(self, engine):
+        castsql = text('''MERGE dbo.castdata as N
+        USING updates.newcastdata as O
+        ON N.seriesid = O.seriesid and N.characterid = O.characterid and N.actorid = O.actorid
+         WHEN MATCHED THEN
+          UPDATE  SET N.seriesid = O.seriesid, N.characterid = O.characterid,N.actorid = O.actorid,N.charactername = O.charactername,N.mediumimage = O.mediumimage,N.originalimage = O.originalimage,N.apilink = O.apilink,N.tvmazelink = O.tvmazelink, N.updated_on = GETDATE()
+        WHEN NOT MATCHED THEN  
+          INSERT (seriesid,characterid,actorid,charactername,mediumimage,originalimage,apilink,tvmazelink, created_on) 
+          VALUES 
+          (O.seriesid,O.characterid,O.actorid,O.charactername,O.mediumimage,O.originalimage,O.apilink,O.tvmazelink, GETDATE());
+        SELECT * FROM updates.newcastdata;
+        ''')
+        with engine.connect() as con:
+            rs = con.execute(castsql)
+        #engine.commit()
+    
+        actorsql = text('''MERGE dbo.actors as N
+            USING updates.newactordata as O
+            ON N.actorid = O.actorid
+             WHEN MATCHED THEN
+              UPDATE  SET N.actorid = O.actorid, N.actorname = O.actorname, N.countryname = O.countryname, N.countrytimezone = O.countrytimezone, N.birthday = O.birthday, N.deathday = O.deathday, N.gender = O.gender, N.mediumimage = O.mediumimage, N.originalimage = O.originalimage, N.apilink = O.apilink, N.url = O.url, N.countrycode = O.countrycode, N.updated_on = GETDATE()
+            WHEN NOT MATCHED THEN  
+              INSERT (actorid,actorname,countryname,countrytimezone,birthday,deathday,gender,mediumimage,originalimage,apilink,url,countrycode,created_on) 
+              VALUES 
+              (O.actorid,O.actorname,O.countryname,O.countrytimezone,O.birthday,O.deathday,O.gender,O.mediumimage,O.originalimage,O.apilink,O.url,O.countrycode, GETDATE());
+            SELECT * FROM updates.newactordata;
+            ''')
+        with engine.connect() as con:
+            rs = con.execute(actorsql)
+        #engine.commit()
+    
+    def __del__(self):
+        pass
+
+class Updatemerge():
+    
+    def __init__(self):
+        pass
+    
+    def merge(self, engine):
+        updatesql = text('''INSERT INTO updates.tvupdatedata (seriesid, timestamp)
+            select seriesid, timestamp from updates.tvupdates
+            ON CONFLICT (seriesid) 
+            DO
+             UPDATE
+               SET seriesid = EXCLUDED.seriesid, timestamp = EXCLUDED.timestamp;''')
+        with engine.connect() as con:
+            rs = con.execute(updatesql)
+        #engine.commit()
+        
+    def __del__(self):
+        pass
+class Jsonmerge():
+    
+    def __init__(self):
+        pass
+    
+    def merge(self, engine):
+        seriessql = text('''INSERT INTO "[updates]".tvmaze_seriesjson (seriesid, jdata, created_on)
+            select seriesid, jdata, date_trunc('minute', current_timestamp) FROM "[updates]".seriesjsonnewvals
+            ON CONFLICT (seriesid)
+            DO
+              update
+                SET seriesid = EXCLUDED.seriesid, jdata = EXCLUDED.jdata, updated_on = date_trunc('minute', current_timestamp);
+            ''')
+        with engine.connect() as con:
+            rs = con.execute(seriessql)
+        #engine.commit()
+            
+        episodesql = text('''INSERT INTO "[updates]".tvmaze_episodejson (seriesid, jdata, created_on)
+            select seriesid, jdata, date_trunc('minute', current_timestamp) FROM "[updates]".episodejsonnewvals
+            ON CONFLICT (seriesid)
+            DO
+              update
+                SET seriesid = EXCLUDED.seriesid, jdata = EXCLUDED.jdata, updated_on = date_trunc('minute', current_timestamp);
+            ''')
+        with engine.connect() as con:
+            rs = con.execute(episodesql)
+        #engine.commit()
+            
+        charactersql = text('''INSERT INTO "[updates]".tvmaze_characterjson (seriesid, jdata, created_on)
+            select seriesid, jdata, date_trunc('minute', current_timestamp) FROM "[updates]".characterjsonnewvals
+            ON CONFLICT (seriesid)
+            DO
+              update
+                SET seriesid = EXCLUDED.seriesid, jdata = EXCLUDED.jdata, updated_on = date_trunc('minute', current_timestamp);
+            ''')
+        with engine.connect() as con:
+            rs = con.execute(charactersql)
+        #engine.commit()
+    
+    def __del__(self):
+        pass
+     
+class Episodemerge():
+    
+    def __init__(self):
+        pass
+    
+    def merge(self, engine):
+        episodesql = text('''INSERT INTO dbo.epdata (episodeid,seriesid,season,episodenumber,episodename,airdate,airtime,episodesummary,airstamp,runtime,eplink,mediumimage,originalimage,url, created_on)
+            select episodeid,seriesid,season,episodenumber,episodename,airdate,airtime,episodesummary,airstamp,runtime,eplink,mediumimage,originalimage,url, date_trunc('minute', current_timestamp) FROM "[updates]".newepdata
+            ON CONFLICT (episodeid)
+            DO
+              UPDATE
+                SET episodeid = EXCLUDED.episodeid, seriesid = EXCLUDED.seriesid, season = EXCLUDED.season, episodenumber = EXCLUDED.episodenumber, episodename = EXCLUDED.episodename,
+                airdate = EXCLUDED.airdate, airtime = EXCLUDED.airtime, episodesummary = EXCLUDED.episodesummary, airstamp = EXCLUDED.airstamp, runtime = EXCLUDED.runtime,
+                eplink = EXCLUDED.eplink, mediumimage = EXCLUDED.mediumimage, originalimage = EXCLUDED.originalimage, url = EXCLUDED.url, updated_on = date_trunc('minute', current_timestamp);''')
+        with engine.connect() as con:
+            rs = con.execute(episodesql)
+        #engine.commit()   
+    
+    def __del__(self):
+        pass
+    
+    

db/pgsql/Tables.py

@@ -0,0 +1,376 @@
+import sqlalchemy
+from sqlalchemy import create_engine, MetaData, Table, Column, Integer, String, Sequence, text, UniqueConstraint
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.schema import MetaData
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.types import *
+
+class Tables():
+    def __init__(self):
+        global tvnetworknewvals
+        global tvwebchannelnewvals
+        global characterjsonnewvals
+        global seriesjsonnewvals
+        global episodejsonnewvals
+        global metadata
+        global engine
+        global session
+        global tvnetworknewvals
+        global tvwebchannelnewvals
+        global characterjsonnewvals
+        global seriesjsonnewvals
+        global episodejsonnewvals
+        global tvupdates
+        global newseriesdata
+        global actorlinks
+        global tablelist
+        
+    def __del__(self):
+        pass
+
+
+class Createtables():
+    
+    def __init__(self, session, metadata, engine, tablelist):
+        Base = declarative_base(metadata = metadata)
+        self.tablelist = self.settables(metadata, engine)
+        try:
+            self.drop(session, self.tablelist)
+        except:
+            pass
+        self.create(Base, engine, metadata, session, self.tablelist)
+        session.commit()
+        #self.send(tablelist)
+    
+    def settables(self, metadata, engine):
+        tablelist = []
+        newseriesdata = Table('newseriesdata', metadata,
+            Column('id', BIGINT, Sequence('newseriesdata_id_seq'), primary_key = True),
+            Column('seriesid',INTEGER, unique=True),
+            Column('tvdbid', String(25)),
+            Column('tvrageid', String(25)),
+            Column('imdbid', String(25)),
+            Column('serieslanguage', String(25)),
+            Column('genres', String(75)),
+            Column('seriesname', String(150)),
+            Column('seriesstatus', String(25)),
+            Column('premiered', String(25)),
+            Column('airdays', String(72)),
+            Column('airtime', String(25)),
+            Column('runtime', String(25)),
+            Column('rating', String(25)),
+            Column('showtype',TEXT),
+            Column('overview',TEXT),
+            Column('mediumimage',TEXT),
+            Column('originalimage',TEXT),
+            Column('weight',INTEGER),
+            Column('networkid', BIGINT),
+            Column('webchannelid', BIGINT),
+            Column('officialsite',TEXT),
+            Column('showurl',TEXT),
+            Column('apilink',TEXT),
+            schema = "updates")
+        tablelist.append(newseriesdata)
+        newactordata = Table('newactordata', metadata,
+            Column('id', BIGINT, Sequence('newactordata_id_seq'), primary_key = True),
+            Column('actorid', INTEGER, unique=True),
+            Column('actorname', VARCHAR(72)),
+            Column('countryname', VARCHAR(144)),
+            Column('countrytimezone', VARCHAR(72)),
+            Column('birthday', DATE),
+            Column('deathday', DATE),
+            Column('gender', VARCHAR(6)),
+            Column('mediumimage', TEXT),
+            Column('originalimage', TEXT),
+            Column('apilink', TEXT),
+            Column('url', TEXT),
+            Column('countrycode', VARCHAR(12)),
+            schema = "updates")
+        tablelist.append(newactordata)
+        newepdata = Table('newepdata', metadata,
+        #Column('id', BIGINT, Sequence('newepdata_id_seq'), primary_key = True),
+            Column('episodeid', INTEGER),
+            Column('seriesid', INTEGER),
+            Column('season', INTEGER),
+            Column('episodenumber', INTEGER),
+            Column('episodename', TEXT),
+            Column('airdate', VARCHAR(32)),
+            Column('airtime', VARCHAR(32)),
+            Column('episodesummary', TEXT),
+            Column('airstamp', VARCHAR(32)),
+            Column('runtime', VARCHAR(6)),
+            Column('eplink', TEXT),
+            Column('mediumimage', TEXT),
+            Column('url', TEXT),
+            Column('originalimage', TEXT),
+            schema = "updates")
+        tablelist.append(newepdata)
+        newcastdata = Table('newcastdata', metadata,
+        Column('id', BIGINT, Sequence('newcastdata_id_seq'), primary_key = True),
+            Column('seriesid', INTEGER),
+            Column('characterid', INTEGER),
+            Column('actorid', INTEGER),
+            Column('charactername', VARCHAR(250)),
+            Column('mediumimage', TEXT),
+            Column('originalimage', TEXT),
+            Column('apilink', TEXT),
+            Column('tvmazelink', TEXT),
+            schema = "updates")
+        tablelist.append(newcastdata)
+        tvnetworks = Table('tvnetworkdata', metadata,
+          Column('id', BIGINT, Sequence('network_id_seq'), primary_key = True),
+          Column('networkid', String(12)),
+          Column('networkname', String(255)),
+          Column('country', String(255)),
+          Column('countrycode', String(4)),
+          Column('countrytimezone', String(25)),
+          schema = "updates")
+        tablelist.append(tvnetworks)
+        tvwebchannels = Table('tvwebchanneldata', metadata,
+          Column('id', BIGINT, Sequence('webchannel_id_seq'), primary_key = True),
+          Column('webchannelname', String(255)),
+          Column('webchannelcountrycode', String(4)),
+          Column('webchannelcountryname', String(255)),
+          Column('webchannelcountrytimezone', String(4)),
+          schema = "updates")
+        tablelist.append(tvwebchannels)
+        return(tablelist)
+    
+    def create(self, Base, engine, metadata, session, tablelist):
+        print('Creating Temporary Data Tables')
+        Base.metadata.create_all(engine, tables=tablelist)
+
+    def drop(self, session, tablelist):
+        for table in tablelist:
+            try:
+                #print('Dropping ' + str(table))
+                table.drop()
+            except sqlalchemy.exc.ProgrammingError as se:
+                pass
+            
+        def __del__(self):
+            pass
+            
+
+class Temptables():
+    
+    def __init__(self, session, metadata, engine, tablelist):
+        Base = declarative_base(metadata = metadata)
+        self.tablelist = self.settables(metadata, engine)
+        try:
+            self.drop(session, self.tablelist)
+        except:
+            pass
+        self.create(Base, engine, metadata, session, self.tablelist)
+        #session.commit()
+        
+    def settables(self, metadata, engine):
+        tablelist = []
+        tvnetworknewvals = Table('tvnetworknewvals', metadata,
+          Column('id', BIGINT, Sequence('network_id_seq'), primary_key = True),
+          Column('name', String(255)),
+          Column('countryid', String(255)),
+          schema = "updates")
+        tablelist.append(tvnetworknewvals)
+        tvwebchannelnewvals = Table('tvwebchannelnewvals', metadata,
+          Column('id', BIGINT, Sequence('webchannel_id_seq'), primary_key = True),
+          Column('name', String(255)),
+          Column('countryid', String(4)),
+          schema = "updates")
+        tablelist.append(tvwebchannelnewvals)
+        tvupdates = Table('tvupdates', metadata,
+        Column('seriesid', BIGINT, primary_key = True),
+            Column('timestamp', VARCHAR(15)),
+          schema = "updates")
+        tablelist.append(tvupdates)
+        tvupdatedata = Table('tvupdatedata', metadata,
+        Column('seriesid', BIGINT, primary_key = True),
+            Column('timestamp', VARCHAR(15)),
+          schema = "updates")
+        tablelist.append(tvupdatedata)
+        newactorlinks = Table('newactorlinks', metadata,
+          Column('id', BIGINT, Sequence('tempepisodejson_id_seq'), primary_key = True),
+          Column('actorid', INTEGER),
+          Column('characterid', INTEGER),
+          schema = "updates")
+        tablelist.append(newactorlinks)
+        return(tablelist)
+    
+    def create(self, Base, engine, metadata, session, tablelist):
+        tablelist.remove(tablelist[6])
+        print('Creating Temporary JSON Tables')
+        try:
+            Base.metadata.create_all(engine, tables=tablelist)
+        except sqlalchemy.exc.ProgrammingError as pe:
+            pass 
+
+    def drop(self, session, tablelist):
+        for table in tablelist:
+            if 'tvupdatedata' in table.description:
+                pass
+            else:
+                try:
+                    #print('Dropping ' + str(table))
+                    table.drop()
+                except sqlalchemy.exc.ProgrammingError as se:
+                    pass
+        #session.commit()
+
+class CreateTempTables():
+    
+    global tvnetworknewvals
+    global tvwebchannelnewvals
+    global characterjsonnewvals
+    global seriesjsonnewvals
+    global episodejsonnewvals
+    global metadata
+    global engine
+    global session
+    
+    global tablelist
+    
+    def __init__(self, session, metadata, engine, tablelist):
+        Base = declarative_base(metadata = metadata)
+        self.tablelist = self.settables(metadata, engine)
+        try:
+            self.drop(session, self.tablelist)
+        except:
+            pass
+        self.create(Base, engine, metadata, session, self.tablelist)
+        session.commit()
+        #self.send(tablelist)
+    
+    def settables(self, metadata, engine):
+        tablelist = []
+        newseriesdata = Table('newseriesdata', metadata,
+            Column('id', BIGINT, Sequence('newseriesdata_id_seq'), primary_key = True),
+            Column('seriesid',INTEGER, unique=True),
+            Column('thetvdb', String(25)),
+            Column('tvrage', String(25)),
+            Column('imdb', String(25)),
+            Column('language', String(25)),
+            Column('genres', String(75)),
+            Column('name', String(150)),
+            Column('status', String(25)),
+            Column('premiered', String(25)),
+            Column('days', String(72)),
+            Column('time', String(25)),
+            Column('runtime', String(25)),
+            Column('rating', String(25)),
+            Column('type',TEXT),
+            Column('summary',TEXT),
+            Column('medium',TEXT),
+            Column('original',TEXT),
+            Column('weight',INTEGER),
+            Column('networkid', BIGINT),
+            Column('webchannelid', BIGINT),
+            Column('officialsite',TEXT),
+            Column('url',TEXT),
+            Column('serieslink',TEXT),
+            schema="updates")
+        tablelist.append(newseriesdata)
+        newactordata = Table('newactordata', metadata,
+            Column('id', BIGINT, Sequence('newactordata_id_seq'), primary_key = True),
+            Column('actorid', INTEGER, unique=True),
+            Column('actorname', VARCHAR(72)),
+            Column('countryname', VARCHAR(36)),
+            Column('countrytimezone', VARCHAR(72)),
+            Column('birthday', DATE),
+            Column('deathday', DATE),
+            Column('gender', VARCHAR(6)),
+            Column('mediumimage', TEXT),
+            Column('originalimage', TEXT),
+            Column('apilink', TEXT),
+            Column('url', TEXT),
+            Column('countrycode', VARCHAR(12)),
+            schema="updates")
+        tablelist.append(newactordata)
+        newepdata = Table('newepdata', metadata,
+        #Column('id', BIGINT, Sequence('newepdata_id_seq'), primary_key = True),
+            Column('episodeid', INTEGER),
+            Column('seriesid', INTEGER),
+            Column('season', INTEGER),
+            Column('episodenumber', INTEGER),
+            Column('episodename', TEXT),
+            Column('airdate', VARCHAR(32)),
+            Column('airtime', VARCHAR(32)),
+            Column('episodesummary', TEXT),
+            Column('airstamp', VARCHAR(32)),
+            Column('runtime', VARCHAR(6)),
+            Column('eplink', TEXT),
+            Column('mediumimage', TEXT),
+            Column('url', TEXT),
+            Column('originalimage', TEXT),
+            schema="updates")
+        tablelist.append(newepdata)
+        newcastdata = Table('newcastdata', metadata,
+        Column('id', BIGINT, Sequence('newcastdata_id_seq'), primary_key = True),
+            Column('seriesid', INTEGER),
+            Column('characterid', INTEGER),
+            Column('actorid', INTEGER),
+            Column('url', TEXT),
+            Column('charactername', VARCHAR(128)),
+            Column('mediumimage', TEXT),
+            Column('originalimage', TEXT),
+            Column('apilink', TEXT),
+            Column('tvmazelink', TEXT),
+            Column('self', CHAR),
+            Column('voice', CHAR),
+            schema="updates")
+        tablelist.append(newcastdata)
+        newcrewdata = Table('newcrewdata', metadata,
+        Column('id', BIGINT, Sequence('newcrewdata_id_seq'), primary_key = True),
+            Column('seriesid', INTEGER),
+            Column('crewid', INTEGER),
+            Column('type', VARCHAR(128)),
+            Column('url', TEXT),
+            Column('name', VARCHAR(256)),
+            Column('countryid', INTEGER),
+            Column('birthday', DATE),
+            Column('deathday', DATE),
+            Column('gender', VARCHAR(24)),
+            Column('medium', TEXT),
+            Column('original', TEXT),
+            Column('apilink', TEXT),
+            UniqueConstraint('seriesid', 'crewid', 'type', name='crew_unique'),
+            schema="updates")
+        tablelist.append(newcrewdata)
+        tvnetworks = Table('tvnetworkdata', metadata,
+          Column('id', BIGINT, Sequence('network_id_seq'), primary_key = True),
+          Column('name', String(255)),
+          Column('countryid', String(255)),
+          schema = "updates")
+        tablelist.append(tvnetworks)
+        tvwebchannels = Table('tvwebchanneldata', metadata,
+          Column('id', BIGINT, Sequence('webchannel_id_seq'), primary_key = True),
+          Column('name', String(255)),
+          Column('countryid', String(255)),
+          schema = "updates")
+        tablelist.append(tvwebchannels)
+        countries = Table('newcountrydata', metadata,
+          Column('id', BIGINT, Sequence('country_id_seq'), primary_key = True),
+          Column('name', String(255)),
+          Column('code', String(255)),
+          Column('timezone', String(255)),
+          UniqueConstraint('name', 'timezone', name='tz_name_uk'),
+          schema = "updates")
+        tablelist.append(countries)
+        tvupdates = Table('tvupdates', metadata,
+        Column('seriesid', BIGINT, primary_key = True),
+            Column('timestamp', VARCHAR(15)),
+          schema = "updates")
+        tablelist.append(tvupdates)
+        return(tablelist)
+    
+    def create(self, Base, engine, metadata, session, tablelist):
+        print('Creating Temporary Data Tables')
+        Base.metadata.create_all(engine, tables=tablelist)
+
+    def drop(self, session, tablelist):
+        print('Dropping temporary update tables.')
+        for table in tablelist:
+            try:
+                print('Dropping ' + str(table))
+                table.drop()
+            except sqlalchemy.exc.ProgrammingError as se:
+                pass

db/pgsql/Updater.py

@@ -0,0 +1,39 @@
+import sqlalchemy
+import progressbar
+from tvmaze.Processor import loadjson
+
+class Updater():
+    def __init__(self):
+        global jprocess
+        
+    def __del__(self):
+        pass
+
+class tvupdates():
+    def __init__(self):
+        pass        
+
+    def update_tables(engine, updatesapi, tablename):
+        jprocess = loadjson()
+        print('Now retrieving available updates.')
+        print('Creating update transaction.')
+        ins = tablename.insert()
+        seriesupdates = jprocess.load(updatesapi)
+        updatelist = []
+        seriesrange = len(seriesupdates) 
+        with progressbar.ProgressBar(maxval=seriesrange, redirect_stdout=True) as p:
+            for key, value in seriesupdates.items():
+                i = int(key)
+                updater = {}
+                updater['seriesid'] = key
+                updater['timestamp'] = value
+                updatelist.append(updater)
+                if i <= seriesrange:
+                    p.update(i)
+                #time.sleep(0.1) 
+        print('Update of tvupdates table is complete.')
+        print('Done.')
+        return(ins, updatelist)
+
+    def __del__(self):
+        pass

db/sql.py

@@ -0,0 +1,71 @@
+__author__ = 'Wendell Jones'
+
+from tqdm import tqdm
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+
+class Dbexec():
+    def __init__(self):
+        pass
+
+    def update_tvupdates(self, engine, available_updates, tablename):
+        print('Creating update transaction.')
+
+        updatelist = [
+            {
+                'seriesid': seriesid,
+                'timestamp': available_updates[seriesid]
+            }
+            for seriesid in tqdm(
+                available_updates,
+                desc='Updating tvupdates table',
+                unit='record'
+            )
+        ]
+
+        print('Applying updates to table.')
+        with engine.connect() as conn:
+            conn.execute(text(f"truncate table {tablename}"))
+            conn.execute(tablename.insert(), updatelist)
+            conn.commit()
+        print('Update of tvupdates table is complete.')
+
+    def rawsql_select(self, engine, sqlquery, lprint):
+        session = Session(engine)
+        resultset = []
+        rs = session.execute(text(sqlquery))
+        for row in rs:
+            resultset.append(row)
+        session.close()
+        return (resultset)
+
+    def rawsql_insert(self, dbengine, sqlquery, lprint):
+        session = Session(dbengine)
+        try:
+            session.execute(text(sqlquery))
+        except Exception as e:
+            print(str(e))
+            lprint.logprint('error', 'Rollback for query: ' + sqlquery)
+            session.rollback()
+        else:
+            session.commit()
+            session.close()
+
+    def __del__(self):
+        pass
+
+
+class SQLGenerate():
+
+    def __init__(self):
+        pass
+
+    def read_updatetable(self, engine, lprint):
+        seriessql = text('''
+            SELECT seriesid, timestamp from updates.tvupdatedata;
+            ''')
+        Dbexec.execute_rawsql(self, engine, seriessql, lprint)
+
+    def __del__(self):
+        pass

jprocessor/process.py

@@ -0,0 +1,105 @@
+__author__ = 'Wendell Jones'
+import json
+import progressbar
+from url.process import Loadurl
+
+class Jsonload():
+    def __init__(self):
+        pass
+    
+    def load(self, rawdata):
+        jdata = {}
+        jdata = json.loads(rawdata)
+        newdata = self.dump(jdata)
+        return(newdata)
+    
+    def dump(self, rawdata):
+        jdata = json.dumps(rawdata, indent=4)
+        return(jdata)
+
+    def jconvert(self, inputdata):
+        outputdata = json.loads(inputdata)
+        return(outputdata)
+    
+    def populate_cache(self, jdata, jfile):
+        try:
+            with open(jfile, 'w') as datafile:
+                datafile.write(jdata)
+        except:
+            print('Unable to save cache file.')
+        else:
+            datafile.close()
+    
+    def fetch_json(self, weburl, seriesid, cachefile):
+        uprocess = Loadurl()
+        try:
+            webdata = uprocess.get_data(weburl)
+            jdata = self.load(webdata)
+        except Exception as e:
+            print('Unable to get JSON data for ' + str(seriesid) + ': ' + str(e))
+        else:
+            if jdata == '[]':
+                pass
+            else:
+                self.populate_cache(jdata, cachefile)
+
+    def processSeriesJson(self, seriesdata):
+        seriesresults = {}
+        networkchannel = {}
+        webchannel = {}
+        country = {}
+
+        seriesresults['id'] = seriesdata['id']
+        seriesresults['url'] = seriesdata['url']
+        seriesresults['name'] = seriesdata['name']
+        seriesresults['type'] = seriesdata['type']
+        seriesresults['language'] = seriesdata['language']
+        seriesresults['genres'] = ', '.join(seriesdata['genres'])
+        seriesresults['status'] = seriesdata['status']
+        seriesresults['runtime'] = seriesdata['runtime']
+        seriesresults['premiered'] = seriesdata['premiered']
+        seriesresults['officalsite'] = seriesdata['officialSite']
+        if len(seriesdata['schedule']['days']) > 0:
+            seriesresults['days'] = seriesdata['schedule']['days'][0]
+            seriesresults['time'] = seriesdata['schedule']['time']
+        seriesresults['rating'] = seriesdata['rating']['average']
+        seriesresults['weight'] = seriesdata['weight']
+        seriesresults['networkid'] = seriesdata['id']
+        seriesresults['webchannel'] = seriesdata['id']
+        seriesresults['tvrage'] = seriesdata['externals']['tvrage']
+        seriesresults['thetvdb'] = seriesdata['externals']['thetvdb']
+        seriesresults['imdb'] = seriesdata['externals']['imdb']
+        if seriesdata['image']:
+            seriesresults['medium'] = seriesdata['image']['medium']
+            seriesresults['original'] = seriesdata['image']['original']
+        seriesresults['summary'] = seriesdata['summary']
+        seriesresults['serieslink'] = seriesdata['_links']['self']['href']
+        if 'previousepisode' in seriesdata['_links'].keys():
+            seriesresults['previousepisode'] = seriesdata['_links']['previousepisode']['href']
+        if 'nextepisode' in seriesdata['_links'].keys():
+            seriesresults['nextepisode'] = seriesdata['_links']['nextepisode']['href']
+
+        
+        if seriesdata['network'] and seriesdata['network'] != None:
+            networkchannel['id'] = seriesdata['network']['id']
+            networkchannel['name'] = seriesdata['network']['name']
+            #networkchannel['countryid'] = country['id']
+            if seriesdata['network']['country']:
+            #    country['id'] = "auto incremented id"
+                country['name'] = seriesdata['network']['country']['name']
+                country['code'] = seriesdata['network']['country']['code']
+                country['timezone'] = seriesdata['network']['country']['timezone']
+        if seriesdata['webChannel'] and seriesdata['webChannel'] != None:
+            webchannel['id'] = seriesdata['webChannel']['id']
+            webchannel['name'] = seriesdata['webChannel']['name']
+            #webchannel['countryid'] = country['id']    
+            if seriesdata['webChannel']['country']:
+                #    country['id'] = "auto incremented id"
+                country['name'] = seriesdata['webchannel']['country']['name']
+                country['code'] = seriesdata['webchannel']['country']['code']
+                country['timezone'] = seriesdata['webchannel']['country']['timezone']
+        
+        return(seriesresults, networkchannel, country, webchannel)
+    
+    def __del__(self):
+        pass

jprocessor/process_view.py

@@ -0,0 +1,80 @@
+__author__ = 'Wendell Jones'
+from ast import Global
+import json
+import progressbar
+
+class Process_View(object):
+
+    global seriesinfo
+    global episodeinfo
+
+    def __init__(self):
+        pass
+
+    def process_seriesdata(self, seriesdata):
+        seriesinfo = {}
+        seriesinfo['seriesid'] = seriesdata['id']
+        seriesinfo['seriesurl'] = seriesdata['url']
+        seriesinfo['seriesname'] = seriesdata['name']
+        seriesinfo['seriestype'] = seriesdata['type']
+        seriesinfo['series_language'] = seriesdata['language']
+        seriesinfo['series_genres'] = ', '.join(seriesdata['genres'])
+        seriesinfo['series_status'] = seriesdata['status']
+        seriesinfo['series_runtime'] = seriesdata['runtime']
+        seriesinfo['series_premiered'] = seriesdata['premiered']
+        seriesinfo['offcialsite'] = seriesdata['officialSite']
+        seriesinfo['air_day'] = ', '.join(seriesdata['schedule']['days'])
+        seriesinfo['air_time'] = seriesdata['schedule']['time']
+        seriesinfo['rating'] = seriesdata['rating']['average']
+        seriesinfo['weight'] = seriesdata['weight']
+        seriesinfo['networkid'] = seriesdata['network']['id']
+        seriesinfo['networkname'] = seriesdata['network']['name']
+        seriesinfo['tvrageid'] = seriesdata['externals']['tvrage']
+        seriesinfo['tvdbid'] = seriesdata['externals']['thetvdb']
+        seriesinfo['imdbid'] = seriesdata['externals']['imdb']
+        seriesinfo['countryname'] = seriesdata['network']['country']['name']
+        seriesinfo['countrycode'] = seriesdata['network']['country']['code']
+        seriesinfo['timezone'] = seriesdata['network']['country']['timezone']
+        self.print_seriesdata(seriesinfo)
+    
+    def print_seriesdata(self, seriesinfo):
+        print(f"Series ID: {seriesinfo['seriesid']}")        
+        print(f"Series URL: {seriesinfo['seriesurl']}")
+        print(f"Series Name: {seriesinfo['seriesname']}")
+        print(f"Type of Series: {seriesinfo['seriestype']}")
+        print(f"Lanuguage: {seriesinfo['series_language']}")
+        print(f"Genres: {seriesinfo['series_genres']}")
+        print(f"Series Status: {seriesinfo['series_status']}")
+        print(f"Run Time: {seriesinfo['series_runtime']}")
+        print(f"Premier Date: {seriesinfo['series_premiered']}")
+        print(f"Official Website: {seriesinfo['offcialsite']}")
+        print(f"Day Aired: {seriesinfo['air_day']}")
+        print(f"Time Aired: {seriesinfo['air_time']}")
+        print(f"Rating: {seriesinfo['rating']}")
+        print(f"Weight: {seriesinfo['weight']}")
+        print(f"Network ID: {seriesinfo['networkid']}")
+        print(f"Network Name: {seriesinfo['networkname']}")
+        print(f"TVRage ID: {seriesinfo['tvrageid']}")
+        print(f"TVDB ID: {seriesinfo['tvdbid']}")
+        print(f"IMDB ID: {seriesinfo['imdbid']}")
+        print(f"Country: {seriesinfo['countryname']}")
+        print(f"Country Code: {seriesinfo['countrycode']}")
+        print(f"Timezone: {seriesinfo['timezone']}")
+
+    def process_episodedata(self, episodeinfo):
+        episodeinfo = {}
+
+    def print_episodedata(self, episodeinfo):
+        for episode in episodeinfo:
+            print(f"Air Date: {episode['airdate']}")
+            print(f"Air Time: {episode['airtime']}")
+            print(f"Season: {episode['season']}")
+            print(f"Episode: {episode['number']}")
+            print(f"Episode Name: {episode['name']}")
+            print(f"Summary: {episode['summary']}")
+            print(f"Rating: {episode['rating']['average']}")
+            print(f"Medium Image: {episode['image']['medium']}")
+            print(f"Original Image: {episode['image']['original']}")
+        
+    def __del__(self):
+        pass

json_downloader.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+
+import os
+from pathlib import Path
+from time import sleep
+from tqdm import tqdm
+import json
+import requests
+
+def load(self, rawdata):
+    jdata = {}
+    jdata = json.loads(rawdata)
+    newdata = self.dump(jdata)
+    return(newdata)
+
+def dump(self, rawdata):
+    jdata = json.dumps(rawdata, indent=4)
+    return(jdata)
+
+def jconvert(self, inputdata):
+    outputdata = json.loads(inputdata)
+    return(outputdata)
+
+def populate_cache(self, jdata, jfile):
+    try:
+        with open(jfile, 'w') as datafile:
+            datafile.write(jdata)
+    except:
+        print('Unable to save cache file.')
+    else:
+        datafile.close()
+
+def fetch_json(self, weburl, seriesid, cachefile):
+    try:
+        webdata = get_data(weburl)
+        jdata = json.load(webdata)
+    except Exception as e:
+        print('Unable to get JSON data for ' + str(seriesid) + ': ' + str(e))
+    else:
+        if jdata == '[]':
+            pass
+        else:
+            populate_cache(jdata, cachefile)
+
+    def get_data(self,weburl):
+        try:
+            showresponse = requests.get(weburl)
+        except requests.exceptions.ConnectionError as e:
+            print('Unable to connect and get JSON data for url ' + str(weburl))
+        else:    
+            return(showresponse.text)
+
+def check_and_download(url, series_id, cachefile):
+    try:
+        mazeurl = URL.replace('<seriesid>', str(series_id))
+        response = requests.get(mazeurl)
+        data = json.loads(response.text)
+    except FileNotFoundError:
+        #print(f'{cachefile} does not exist. Attempting to download.')
+        jget.fetch_json(url.replace(SERIESID, str(series_id)), series_id, cachefile)
+    except Exception as e:
+        print(f'Unable to download {cachefile}. {e}')
+    else:
+        with open(cachefile, 'w') as f:
+            json.dump(data, f, indent=4)
+        f.close()
+
+new_updates = []
+listing = open ('D:/Temp/filelist.txt', 'w')
+
+os.chdir('D:/Projects/new_dbsync/cache/series')
+directory_path = 'D:/Projects/new_dbsync/cache/series'  # replace with your directory path
+files = [f for f in os.listdir(directory_path) if os.path.isfile(os.path.join(directory_path, f))]
+for filename in files:
+    with open(filename, 'r') as file:
+        content = file.read()
+        if 'Not Found' in content or 'Too Many Requests' in content:
+            seriesid = filename.split('.')[0]
+            new_updates.append(seriesid)
+            listing.write(seriesid + '\n')
+listing.close()
+
+exit()
+
+# new_updates_2 = []
+# listing = open ('D:/Temp/filelist.txt', 'r')
+# for seriesid in listing:
+#     seriesid = seriesid.strip()
+#     new_updates_2.append(seriesid)
+# listing.close()
+
+# for seriesid in tqdm(range(len(new_updates_2)), desc='Checking cache', unit='seriesid'):
+#     # cachefile = str(new_updates_2[seriesid]) + '.json'
+#     # URL ='http://api.tvmaze.com/shows/<seriesid>/cast'
+#     # check_and_download(URL, str(new_updates_2[seriesid]), f'{new_updates_2[seriesid]}.json')
+#     os.remove(f'D:/Projects/new_dbsync/cache/cast/{new_updates_2[seriesid]}.json')

logs/Logger.py

@@ -0,0 +1,100 @@
+__author__ = 'Wendell Jones'
+
+import logging
+from logging.handlers import RotatingFileHandler
+import time
+from datetime import datetime, timedelta
+import os
+from glob import glob
+import math as m
+
+
+class Logger:
+    def __init__(self):
+        pass
+
+    def __del__(self):
+        pass
+
+
+class LPrint:
+    def __init__(self, log_dir, force_rotate):
+        self.log_dir = log_dir
+        self.logfile = os.path.join(log_dir, 'dbsync.log')
+        self.setup_logfile()
+        self.cleanup_old_logs()
+        self.check_log_size(force_rotate)
+
+    def setup_logfile(self):
+        """Ensure the logfile exists and is ready for use."""
+        os.makedirs(self.log_dir, exist_ok=True)
+        with open(self.logfile, 'a'):
+            os.utime(self.logfile, None)
+
+    def cleanup_old_logs(self):
+        """Remove log files older than 14 days."""
+        logfile_list = glob(f'{self.logfile}.*')
+        date_limit = datetime.today() - timedelta(days=14)
+        for log in logfile_list:
+            timestamp = datetime.fromtimestamp(os.path.getmtime(log))
+            if timestamp < date_limit:
+                os.remove(log)
+
+    def check_log_size(self, force_rotate):
+        """Check the log file size and rotate if necessary."""
+        log_size = os.path.getsize(self.logfile)
+        print(f'Logfile Size: {log_size} ({self.humanbytes(log_size)})')
+        if force_rotate or log_size > 10_000_000:  # 10 MB
+            self.rotate_logs()
+
+    def rotate_logs(self):
+        """Rotate the log file."""
+        from logrotater import LogRotate  # Assuming logrotater is a custom module
+        rotater = LogRotate(prefix=self.logfile, verbose=True)
+        try:
+            rotater.rotate()
+        except FileExistsError as e:
+            print(f'Unable to overwrite archive log: {e}. Deleting old logs.')
+            for log in glob(f'{self.logfile}.*'):
+                try:
+                    os.remove(log)
+                except Exception as e:
+                    print(f'Unable to delete {log}: {e}')
+
+    def start_log(self):
+        """Initialize logging."""
+        logging.basicConfig(filename=self.logfile, level=logging.DEBUG)
+
+    def logprint(self, loglevel, logmessage):
+        """Log a message with a timestamp."""
+        logstamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+        log_message = f'{logstamp}: {logmessage}'
+        loglevel = loglevel.lower()
+        if loglevel == 'debug':
+            logging.debug(log_message)
+        elif loglevel == 'warning':
+            logging.warning(log_message)
+        elif loglevel == 'info':
+            logging.info(log_message)
+        else:
+            logging.error(f'Unknown log level: {loglevel}. Message: {logmessage}')
+
+    def create_rotating_log(self, logpath, max_bytes=10_000_000, backup_count=5):
+        """Create a rotating log handler."""
+        logger = logging.getLogger("Rotating Log")
+        logger.setLevel(logging.INFO)
+        handler = RotatingFileHandler(logpath, maxBytes=max_bytes, backupCount=backup_count)
+        logger.addHandler(handler)
+
+    @staticmethod
+    def humanbytes(size, binary=False, precision=2):
+        """Convert bytes to a human-readable format."""
+        multiples = ["B", "k{}B", "M{}B", "G{}B", "T{}B", "P{}B", "E{}B", "Z{}B", "Y{}B"]
+        base = 1024 if binary else 1000
+        multiple = m.trunc(m.log2(size) / m.log2(base)) if size > 0 else 0
+        value = size / m.pow(base, multiple) if size > 0 else 0
+        suffix = multiples[multiple].format("i" if binary else "")
+        return f"{value:.{precision}f} {suffix}"
+
+    def __del__(self):
+        pass

requirements.txt

@@ -0,0 +1,10 @@
+logrotater==1.3
+numpy==2.2.3
+progressbar33==2.4
+psycopg2==2.9.10
+pymongo==4.11.2
+pymssql==2.3.2
+Requests==2.32.3
+SQLAlchemy==2.0.39
+pymysql
+tqdm

settings/config.py

@@ -0,0 +1,63 @@
+__author__ = 'Wendell Jones'
+import configparser
+import os
+import time
+class Config(object):
+    global ROOTDIR
+    def __init__(self, ROOTDIR):
+        self.config_options = {}
+        self.configurer = configparser.ConfigParser()
+        self.read_settings(ROOTDIR)
+
+    def read_settings(self, ROOTDIR):
+        if os.name == 'nt':
+            self.configurer.read(ROOTDIR + '\\settings\\tvsync_settings.cfg')
+        else:
+            self.configurer.read(ROOTDIR + '/settings/tvsync_settings.cfg')
+        print(f"Reading configuration file from {ROOTDIR}/settings")
+        self.set_data()
+        self.get_data()
+
+    def set_data(self):
+        self.config_options['loglevel'] = self.configurer.get('global', 'loglevel')
+        self.config_options['hostname'] = self.configurer.get('dbsettings', 'hostname')
+        self.config_options['mghostname'] = self.configurer.get('dbsettings', 'mghostname')
+        self.config_options['dbname'] = self.configurer.get('dbsettings', 'dbname')
+        self.config_options['mgdbname'] = self.configurer.get('dbsettings', 'mgdbname')
+        self.config_options['mysqlusername'] = self.configurer.get('dbsettings', 'mysqlusername')
+        self.config_options['pgsqlusername'] = self.configurer.get('dbsettings', 'pgsqlusername')
+        self.config_options['mssqlusername'] = self.configurer.get('dbsettings', 'mssqlusername')
+        self.config_options['mysqlpassword'] = self.configurer.get('dbsettings', 'mysqlpassword')
+        self.config_options['pgsqlpassword'] = self.configurer.get('dbsettings', 'pgsqlpassword')
+        self.config_options['mssqlpassword'] = self.configurer.get('dbsettings', 'mssqlpassword')
+        self.config_options['pgsqlport'] = self.configurer.get('dbsettings', 'pgsqlport')
+        self.config_options['mysqlport'] = self.configurer.get('dbsettings', 'mysqlport')
+        self.config_options['mssqlport'] = self.configurer.get('dbsettings', 'mssqlport')
+        self.config_options['mgport'] = self.configurer.get('dbsettings', 'mgport')
+        self.config_options['updatetype'] = self.configurer.get('updatetype', 'type')
+        self.config_options['apitype'] = self.configurer.get('api', 'type')
+        self.config_options['updateschema'] = self.configurer.get('database', 'updateschema')
+        self.config_options['dbtype'] = self.configurer.get('database', 'dbtype')
+        self.config_options['initdb'] = self.configurer.get('database', 'initialize')
+        self.config_options['initload'] = self.configurer.get('database', 'initload')
+        self.config_options['skipcache'] = self.configurer.get('updatetype', 'cacheskip')
+        self.config_options['refreshcache'] = self.configurer.get('updatetype', 'refreshcache')
+        self.config_options['full'] = self.configurer.get('updatetype', 'full')
+        self.config_options['monthly'] = self.configurer.get('updatetype', 'monthly')
+        self.config_options['weekly'] = self.configurer.get('updatetype', 'weekly')
+        self.config_options['daily'] = self.configurer.get('updatetype', 'daily')
+        self.config_options['12h'] = self.configurer.get('updatetype', '12h')
+        self.config_options['6h'] = self.configurer.get('updatetype', '6h')
+        self.config_options['custom'] = self.configurer.get('updatetype', 'custom')        
+        self.config_options['serieslist'] = list()
+        self.config_options['timedifference'] = self.configurer.get('updatetype', self.config_options['updatetype'])
+        self.config_options['current_timestamp'] = time.time()
+        if self.config_options['timedifference'].lower() == 'full':
+            self.config_options['timedifference'] = int(self.config_options['current_timestamp'])
+        
+    def get_data(self):
+        return self.config_options
+
+    def __del__(self):
+
+        pass

settings/tvsync_settings.cfg

@@ -0,0 +1,86 @@
+[global]
+loglevel=debug
+
+[dbsettings]
+sqlserver_driver=ODBC Driver 13 for SQL Server
+hostname=192.168.128.7
+mghostname=192.168.128.8
+dbname=media_dbsync
+mgdbname=test2
+mysqlusername=root
+pgsqlusername=postgres
+mssqlusername=sa
+mysqlpassword=Optimus0329
+pgsqlpassword=Optimus0329
+mssqlpassword=Optimus0329
+pgsqlport=5432
+mysqlport=3306
+mssqlport=1433
+mgport=27017
+
+[api]
+type=tvmaze
+
+[json]
+seriesapi=http://api.tvmaze.com/shows?page=<pagenum>
+showapi=http://api.tvmaze.com/shows/<tvmazeid>
+episodeapi=http://api.tvmaze.com/shows/<tvmazeid>/episodes
+aliasapi=http://api.tvmaze.com/shows/<tvmazeid>/akas
+castapi=http://api.tvmaze.com/shows/<tvmazeid>/cast
+crewapi=http://api.tvmaze.com/shows/<tvmazeid>/crew
+creditsapi=http://api.tvmaze.com/people/<actorid>/castcredits
+updatesapi=http://api.tvmaze.com/updates/shows
+
+[database]
+dbtype=pgsql
+updateschema=updates
+initialize=0
+initload=0
+        
+[updatetype]
+#valid values are: daily, weekly, monthly, full, custom
+type=full
+full=full
+monthly=1219200
+weekly=604800
+daily=86400
+12h=23200
+6h = 16600
+# If settype = custom set below line to the number of seconds you want updates for.
+custom=4150
+cacheskip = 1
+refreshcache = 0
+
+# Field names for series insert query
+# field01: nextepisode,
+# field02: previousepisode,
+# field03: mazeurl,
+# field04: tvdbid,
+# field05: tvrageid,
+# field06: genres,
+# field07: tvmazeid,
+# field08: mediumimage,
+# field09: originalimage,
+# field10: showlanguage,
+# field11: seriesname,
+# field12: webchannelcountrycode,
+# field13: webchannelcountryname,
+# field14: webchannelcountrytimezone,
+# field15: webchannelid,
+# field16: webchannelname,
+# field17: networkcountrycode,
+# field18: networkcountryname,
+# field19: networkcountrytimezone,
+# field20: networkid,
+# field21: networkname,
+# field22: premierdate,
+# field23: showrating,
+# field24: showruntime,
+# field25: airdays,
+# field26: scheduletime,
+# field27: showstatus,
+# field28: showoverview,
+# field29: showtype,
+# field30: showupdated,
+# field31: apilink,
+# field32: showweight

tabledata_from_json.py

@@ -0,0 +1,239 @@
+#!/usr/bin/env python3
+
+from pymongo import MongoClient
+import json
+import psycopg2
+import pandas as pd
+from collections.abc import MutableMapping
+from tqdm import tqdm
+
+
+def flatten_dict(d: MutableMapping, sep: str = '.') -> MutableMapping:
+    [flat_dict] = pd.json_normalize(d, sep=sep).to_dict(orient='records')
+    return flat_dict
+
+def delete_none(_dict):
+    """Delete None values recursively from all of the dictionaries, tuples, lists, sets"""
+    if isinstance(_dict, dict):
+        for key, value in list(_dict.items()):
+            if isinstance(value, (list, dict, tuple, set)):
+                _dict[key] = delete_none(value)
+            elif value is None or key is None:
+                del _dict[key]
+
+    elif isinstance(_dict, (list, set, tuple)):
+        _dict = type(_dict)(delete_none(item) for item in _dict if item is not None)
+
+    return _dict
+
+def process_data(data, tablename):
+    inserts = []
+    failed_count = 0
+    seriesconflict = "ON CONFLICT (seriesid) DO UPDATE SET"
+    epconflict = "ON CONFLICT (episodeid) DO UPDATE SET"
+    crewconflict = "ON CONFLICT (crewid) DO UPDATE SET"
+    for document in data:
+        if document['name'].lower() == 'too many requests':
+            print("Skipping document due to 'Too Many Requests' in "
+                  "document data")
+            continue
+        else:
+            new_document = delete_none(document)
+            if '_id' in new_document:
+                del new_document['_id']
+            # if tablename == 'epdata' and 'image' in document:
+            #     del document['image']
+            # if 'network' in document:
+            #     if document['network'] is None:
+            #         del document['network']
+            #     del document['image']
+            try:
+                flatten_data = flatten_dict(new_document)
+            except TypeError as e:
+                print(
+                    (
+                        f"Unable to flatten data for current document. {e}\n"
+                        "Exitting!"
+                    )
+                )
+                failed_count += 1
+            else:
+                column_list = []
+                data_list = []
+                for key, value in flatten_data.items():
+                    if tablename == 'seriesdata' and (
+                        key.lower() == 'name' or key.lower() == 'type'
+                    ):
+                        key = f"series_{key}"
+                    if tablename == 'seriesdata' and key.lower() == 'id':
+                        key = "seriesid"
+                    if tablename == 'crewdata' and key.lower() == 'id':
+                        key = "crewid"
+                    if (
+                        tablename == 'epdata' and (
+                            key.lower() == 'name' or key.lower() == 'number' or
+                            key.lower() == 'type'
+                        )
+                    ):
+                        key = f"episode_{key}"
+                    if key.lower() == 'language':
+                        key = "language_name"
+                    if tablename == 'epdata' and key.lower() == 'id':
+                        key = "episodeid"
+                    if (
+                        '_links' in key.lower() or
+                        'image' in key.lower() or
+                        'schedule' in key.lower() or
+                        'externals' in key.lower() or
+                        'network' in key.lower() or
+                        'webchannel' in key.lower() or
+                        'country' in key.lower() or
+                        'rating' in key.lower()
+                    ):
+                        key = key.replace('.', '_')
+                    column_list.append(key)
+                    if 'genres' in key.lower() or 'schedule' in key.lower():
+                        if isinstance(value, list):
+                            value = ', '.join(value)
+                        else:
+                            value = str(value).replace("'", "''")
+                    if isinstance(value, str):
+                        value = value.replace("'", "''")
+                    if isinstance(value, int):
+                        value = f"'{value}'"
+                    elif isinstance(value, float):
+                        value = f"'{json.dumps(value)}'"
+                    elif isinstance(value, dict):
+                        value = f"'{json.dumps(value)}'"
+                    elif isinstance(value, bool):
+                        value = str(value).lower()
+                    elif value is None:
+                        value = 'NULL'
+                    else:
+                        value = str(f"'{value}'")
+                    data_list.append(value)
+                columns = ", ".join(column_list)
+                column_data = ", ".join(data_list)
+                conflict_list = []
+                for indexer, column in enumerate(column_list):
+                    conflict_list.append(f"{column_list[indexer]} = {data_list[indexer]}")
+                conflict_data = ', '.join(conflict_list) 
+                conflict_clause = ''
+                if tablename == 'seriesdata': 
+                    conflict_clause = seriesconflict
+                elif tablename == 'epdata':
+                    conflict_clause = epconflict
+                elif tablename == 'crewdata':
+                    conflict_clause = crewconflict
+                insert_command = (
+                    (
+                        f"Insert Into updates.{tablename} ({columns}) "
+                        f"values ({column_data}) "
+                        f"{conflict_clause} {conflict_data};"
+                    )
+                )
+                inserts.append(insert_command)
+    return inserts, failed_count
+
+
+try:
+    connection_string = (
+        (
+            'postgres://postgres:Optimus0329@192.168.128.7:5432/media_dbsync'
+            '?options=-csearch_path%3Ddbo,public,updates'
+        )
+    )
+    pgclient = psycopg2.connect(connection_string)
+    pgcursor = pgclient.cursor()
+    #print(pgclient.get_dsn_parameters(), "\n")
+    print("Connected to postgresql database server.")
+    pgcursor.execute("SELECT version();")
+    record = pgcursor.fetchone()
+    print("You are connected to - ", record, "\n")
+    pgclient.set_isolation_level(0)
+except Exception as e:
+    print(f"Unable to connect to postgresql database server:{e}\nExitting!")
+    exit()
+
+
+host = '192.168.128.8'
+port = 27017
+print(f"Connecting to mongodb at {host}:{port}")
+
+try:
+    connection_string = (
+        f"mongodb://{host}:{port}"
+    )
+    mgclient = MongoClient(connection_string)
+    mgdb = mgclient['test2']
+except BaseException as e:
+    print(f"Unable to connect to mongodb.{e}\nExitting!")
+    exit()
+else:
+    mgseries = mgdb["series"]
+    mgseries.create_index('id', unique=True)
+    mgepisodes = mgdb["episodes"]
+    mgepisodes.create_index('id', unique=True)
+    mgactors = mgdb["actors"]
+    mgactors.create_index('id', unique=True)
+    mgcharacters = mgdb["characters"]
+    mgcharacters.create_index('id', unique=True)
+    mgcrew = mgdb["crew"]
+    mgcrew.create_index('id', unique=True)
+    print(
+        f"Connection to mongodb at {host}:"
+        f"{port} successful."
+    )
+
+print("Creating Insert commands from collection data.")
+
+# print("Collecting series data for series inserts.")
+# series_list = mgseries.find({})
+# inserts, failed_count = process_data(series_list, 'seriesdata')
+# print(f"Total number of series inserts: {len(inserts)}")
+# for i in tqdm(range(len(inserts))):
+#     try:
+#         insert = inserts[i]
+#         pgcursor.execute(insert)
+#     except Exception as e:
+#         # print(f"Unable to insert data into seriesdata table. {e}")
+#         failed_count += 1
+# print(f"Total number of failed series inserts: {failed_count}")
+# pgclient.commit()
+
+# epinserts = []
+# failures = 0
+# episode_array = mgepisodes.find({})
+# print("Collecting episode data for episode inserts.")
+# inserts, failed_count = process_data(episode_array, 'epdata')
+# print(f"Total number of episode inserts: {len(inserts)}")
+# for i in tqdm(range(len(inserts))):
+#     try:
+#         insert = inserts[i]
+#         pgcursor.execute(insert)
+#     except Exception as e:
+#         # print(f"Unable to insert data into epdata table. {e}")
+#         failed_count += 1
+# print(f"Total number of failed episode inserts: {failed_count}")
+# pgclient.commit()
+
+failures = 0
+crew_array = mgcrew.find({})
+print("Collecting crew data for crew inserts.")
+inserts, failed_count = process_data(crew_array, 'crewdata')
+print(f"Total number of crew inserts: {len(inserts)}")
+for i in tqdm(range(len(inserts))):
+    try:
+        insert = inserts[i]
+        pgcursor.execute(insert)
+    except Exception as e:
+        print(f"Unable to insert data into epdata table. {e}")
+        failed_count += 1
+print(f"Total number of failed crew inserts: {failed_count}")
+pgclient.commit()
+
+pgclient.close()
+print("PostgreSQL connection is closed")
+
+
+exit()

tableddl_from_yaml.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+
+import os
+from pymongo import MongoClient
+import yaml
+import json
+import psycopg2
+
+class ColumnMeta:
+    def __init__(self, name, type, description=None):
+        self.name = name
+        self.type = type
+        self.description = description
+
+    def from_dict(value: dict):
+        return ColumnMeta(name=value['name'], type=value['type'], description=value.get('description'))
+
+
+class TableMeta:
+    def __init__(self, name, columns, description=None):
+        self.name = name
+        self.columns = columns 
+        self.description = description
+        
+
+    def from_dict(value):
+        columns = [ColumnMeta.from_dict(v) for v in value['columns']]
+        return TableMeta(name=value['name'], columns=columns, description=value.get('description'))
+
+    def from_file(path):
+        file = open(path, 'r')
+        yaml_to_dict = yaml.safe_load(file.read())
+        try:
+            file.close()
+        except Exception:
+            pass
+        return TableMeta.from_dict(yaml_to_dict)
+
+
+try:
+    connection_string = (
+        (
+            'postgres://postgres:Optimus0329@192.168.128.7:5432/media_dbsync'
+            '?options=-csearch_path%3Ddbo,public,updates'
+        )
+    )
+    pgclient = psycopg2.connect(connection_string)
+    pgcursor = pgclient.cursor()
+    print(pgclient.get_dsn_parameters(), "\n")
+    print("Connected to postgresql database server.")
+    pgcursor.execute("SELECT version();")
+    record = pgcursor.fetchone()
+    print("You are connected to - ", record, "\n")
+except Exception as e:
+    print(f"Unable to connect to postgresql database server:{e}\nExitting!")
+    exit()
+
+
+host = '192.168.128.8'
+port = 27017
+print(f"Connecting to mongodb at {host}:{port}")
+
+try:
+    connection_string = (
+        f"mongodb://{host}:{port}"
+    )
+    mgclient = MongoClient(connection_string)
+    mgdb = mgclient['test2']
+except BaseException as e:
+    print(f"Unable to connect to mongodb.{e}\nExitting!")
+    exit()
+else:
+    mgseries = mgdb["series"]
+    mgseries.create_index('id', unique=True)
+    mgepisodes = mgdb["episodes"]
+    mgepisodes.create_index('id', unique=True)
+    mgactors = mgdb["actors"]
+    mgactors.create_index('id', unique=True)
+    mgcharacters = mgdb["characters"]
+    mgcharacters.create_index('id', unique=True)
+    mgcrew = mgdb["crew"]
+    mgcrew.create_index('id', unique=True)
+    print(
+        f"Connection to mongodb at {host}:"
+        f"{port} successful."
+    )
+os.chdir('C:\\Projects\\new_dbsync\\db\\ddl')
+# Create the database and tables in PostgreSQL
+# Check if the database exists, if not create it
+# Check if the tables exist, if not create them
+# possibly add , 'genre_ddl.yaml', 'language_ddl.yaml' down the road
+ddl_list = [
+    # 'seriesdata_ddl.yaml',
+    # 'epdata_ddl.yaml',
+    # 'actordata_ddl.yaml',
+    # 'castdata_ddl.yaml',
+    # 'crewdata_ddl.yaml',
+    # 'countrydata_ddl.yaml',
+    # 'networkdata_ddl.yaml',
+    # 'webchanneldata_ddl.yaml',
+    # 'temp_seriesdata_ddl.yaml',
+    # 'temp_epdata_ddl.yaml',
+    # 'temp_actordata_ddl.yaml',
+    # 'temp_castdata_ddl.yaml',
+    # 'temp_crewdata_ddl.yaml',
+    # 'temp_countrydata_ddl.yaml',
+    # 'temp_networkdata_ddl.yaml',
+    # 'temp_webchanneldata_ddl.yaml',
+    # 'tvupdates_ddl.yaml'
+    'db_schema.yaml'
+]
+for ddl in ddl_list:
+    path = os.path.join(os.getcwd(), ddl)
+    table_meta = TableMeta.from_file(path)
+
+    columnlist = []
+    for column in table_meta.columns:
+        columnlist.append(f"{column.name} {column.type}")
+        table_meta.columns = ', '.join(columnlist)
+        #print(column.name, column.type)
+
+    sql_command = f"DROP TABLE IF EXISTS {table_meta.name};CREATE TABLE IF NOT EXISTS {table_meta.name} ({table_meta.columns})"
+    #sql_command = f"CREATE TABLE IF NOT EXISTS {table_meta.name} ({table_meta.columns})"
+
+    try:
+        pgcursor.execute(sql_command)
+        pgclient.commit()
+    except Exception as e:
+        print(f"Unable to create table {table_meta.name} in postgresql database server:{e}\nExitting!")
+        exit()
+        #print(sql_command)
+
+pgcursor.close()
+pgclient.close()
+print("PostgreSQL connection is closed")
+mgclient.close()
+print("MongoDB connection is closed")
+exit()

test_tableddl_from_yaml.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+
+import os
+from pymongo import MongoClient
+import yaml
+import json
+import psycopg2
+
+class ColumnMeta:
+    def __init__(self, name, type, description=None):
+        self.name = name
+        self.type = type
+        self.description = description
+
+    def from_dict(value: dict):
+        return ColumnMeta(name=value['name'], type=value['type'], description=value.get('description'))
+
+
+class TableMeta:
+    def __init__(self, name, columns, description=None):
+        self.name = name
+        self.columns = columns 
+        self.description = description
+
+    def from_dict(value):
+        columns = [ColumnMeta.from_dict(v) for v in value['columns']]
+        return TableMeta(name=value['name'], columns=columns, description=value.get('description'))
+
+    def from_file(path):
+        file = open(path, 'r')
+        yaml_to_dict = yaml.safe_load(file.read())
+        try:
+            file.close()
+        except Exception:
+            pass
+        return TableMeta.from_dict(yaml_to_dict)
+
+
+try:
+    connection_string = (
+        (
+            'postgres://postgres:Optimus0329@192.168.128.7:5432/media_dbsync'
+            '?options=-csearch_path%3Ddbo,public,updates'
+        )
+    )
+    pgclient = psycopg2.connect(connection_string)
+    pgcursor = pgclient.cursor()
+    print(pgclient.get_dsn_parameters(), "\n")
+    print("Connected to postgresql database server.")
+    pgcursor.execute("SELECT version();")
+    record = pgcursor.fetchone()
+    print("You are connected to - ", record, "\n")
+except Exception as e:
+    print(f"Unable to connect to postgresql database server:{e}\nExitting!")
+    exit()
+
+
+host = '192.168.128.8'
+port = 27017
+print(f"Connecting to mongodb at {host}:{port}")
+
+try:
+    connection_string = (
+        f"mongodb://{host}:{port}"
+    )
+    mgclient = MongoClient(connection_string)
+    mgdb = mgclient['test2']
+except BaseException as e:
+    print(f"Unable to connect to mongodb.{e}\nExitting!")
+    exit()
+else:
+    mgseries = mgdb["series"]
+    mgseries.create_index('id', unique=True)
+    mgepisodes = mgdb["episodes"]
+    mgepisodes.create_index('id', unique=True)
+    mgactors = mgdb["actors"]
+    mgactors.create_index('id', unique=True)
+    mgcharacters = mgdb["characters"]
+    mgcharacters.create_index('id', unique=True)
+    mgcrew = mgdb["crew"]
+    mgcrew.create_index('id', unique=True)
+    print(
+        f"Connection to mongodb at {host}:"
+        f"{port} successful."
+    )
+os.chdir('C:\\Projects\\new_dbsync\\db\\ddl')
+# Create the database and tables in PostgreSQL
+# Check if the database exists, if not create it
+# Check if the tables exist, if not create them
+# possibly add , 'genre_ddl.yaml', 'language_ddl.yaml' down the road
+ddl_list = os.listdir(os.getcwd())
+for ddl in ddl_list:
+    path = os.path.join(os.getcwd(), ddl)
+    table_meta = TableMeta.from_file(path)
+
+    columnlist = []
+    for column in table_meta.columns:
+        columnlist.append(f"{column.name} {column.type}")
+        table_meta.columns = ', '.join(columnlist)
+        #print(column.name, column.type)
+
+    sql_command = f"DROP TABLE IF EXISTS {table_meta.name};CREATE TABLE IF NOT EXISTS {table_meta.name} ({table_meta.columns})"
+    #sql_command = f"CREATE TABLE IF NOT EXISTS {table_meta.name} ({table_meta.columns})"
+
+    try:
+        pgcursor.execute(sql_command)
+        pgclient.commit()
+    except Exception as e:
+        print(f"Unable to create table {table_meta.name} in postgresql database server:{e}\nExitting!")
+        exit()
+        #print(sql_command)
+
+pgcursor.close()
+pgclient.close()
+print("PostgreSQL connection is closed")
+mgclient.close()
+print("MongoDB connection is closed")
+exit()

update.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import os
+import time
+import url.urls
+import url.process
+import api.tvmaze.cast as mazecast
+import api.tvmaze.crew as mazecrew
+import api.tvmaze.episodes as mazeepisodes
+import api.tvmaze.series as mazeseries
+import db.sql
+import jprocessor.process
+import logs.Logger
+from db.functions import settype, dbmongo
+from datetime import datetime
+
+response = None
+
+
+def set_apienv(urls, dbExec, updatesBase):
+    updatetable = updatesBase[8]
+    inputdata = uprocess.get_data(urls["updatesurl"])
+    availableupdates = jget.jconvert(inputdata)
+    print(f"Retrieved {len(availableupdates)} rows for processing......")
+    dbExec.update_tvupdates(dbengine["engine"], availableupdates, updatetable)
+    newupdates = dbExec.rawsql_select(
+        dbengine["engine"],
+        "select seriesid,timestamp from updates.tvupdates",
+        lprint
+    )
+    return newupdates
+
+
+def print_starttime(lprint, ts1):
+    lprint.logprint("info", f"Sync start time: {str(ts1)}")
+
+
+def get_tvdb():
+    tvdb_urls = TVDBURLS.get_data()
+    return tvdb_urls
+
+
+def get_tvmaze():
+    tvmaze_urls = TVMAZEURLS.get_data()
+    return tvmaze_urls
+
+
+if __name__ == "__main__":
+    ROOTDIR = os.getcwd()
+    directories = {}
+    if os.name == "nt":
+        LOGDIR = f"{ROOTDIR}\\log"
+        directories["SERIESDIR"] = f"{ROOTDIR}\\cache\\series\\"
+        directories["EPISODEDIR"] = f"{ROOTDIR}\\cache\\episodes\\"
+        directories["CASTDIR"] = f"{ROOTDIR}\\cache\\cast\\"
+        directories["CREWDIR"] = f"{ROOTDIR}\\cache\\crew\\"
+    else:
+        LOGDIR = f"{ROOTDIR}/log"
+        directories["SERIESDIR"] = f"{ROOTDIR}/cache/series/"
+        directories["EPISODEDIR"] = f"{ROOTDIR}/cache/episodes/"
+        directories["CASTDIR"] = f"{ROOTDIR}/cache/cast/"
+        directories["CREWDIR"] = f"{ROOTDIR}/cache/crew/"
+    lprint = logs.Logger.LPrint(LOGDIR, False)
+    lprint.start_log()
+    import settings.config
+
+    config_options = settings.config.Config(ROOTDIR)
+    options = config_options.config_options
+    dbtype, apitype = options["dbtype"], options["apitype"]
+    dbs = settype(dbtype, apitype, options)
+    dbclass, dbengine, apiengine = dbs.dbclass, dbs.dbengine, dbs.apiengine
+    uprocess = url.process.Loadurl()
+    TVMAZEURLS = url.urls.Tvmazeurls()
+    TVMAZE_URLS = get_tvmaze()
+    TVDBURLS = url.urls.TVDBurls()
+    TVDB_URLS = get_tvdb()
+    mseries = mazeseries.tvshow()
+    mcast = mazecast.cast()
+    mepisode = mazeepisodes.episodes()
+    mcrew = mazecrew.crew()
+    countrydictionary = {}
+    updateRange = None
+
+    lprint.logprint("info", "Creating JSON instance for data retrieval.")
+    jget = jprocessor.process.Jsonload()
+    lprint.logprint("info", "JSON instance created.")
+    print(f"Update type is {options['updatetype']}")
+    timedifference = options["updatetype"]
+    currenttimestamp = time.time()
+    if timedifference.lower() == "full":
+        timedifference = int(currenttimestamp)
+    basetime = str(currenttimestamp).split(".")
+    currenttimestamp = int(basetime[0])
+    ts1 = str(datetime.now()).split(".")[0]
+    print("Sync start time: " + str(ts1))
+    print_starttime(lprint, ts1)
+    updatesBase = dbengine["updatesBase"]
+    dboBase = dbengine["dboBase"]
+    session = dbengine["session"]
+    dbExec = db.sql.Dbexec()
+    createTables = dbclass["Createtables"]
+    Tables = dbclass["Tables"]
+    createTempTables = dbclass["CreateTemptables"]
+    new_updates = {}
+    sqlexec = db.sql.SQLGenerate()
+    mt = createTables(
+        dbengine["session"],
+        dbengine["metadata"],
+        dbengine["engine"]
+    )
+    ct = createTempTables(
+        dbengine["session"],
+        dbengine["metadata"],
+        dbengine["engine"]
+    )
+    tempTableList = ct.tablelist
+
+    if options["apitype"].lower() == "tvmaze":
+        new_updates = set_apienv(TVMAZE_URLS, dbExec, tempTableList)
+    from cache.process import Cacher
+
+    cacher = Cacher()
+    cacher.process(options, jget, new_updates, ROOTDIR, TVMAZE_URLS)
+    timeDifference = options["updatetype"]
+    currentTimestamp = time.time()
+    baseTime = str(currentTimestamp).split(".")
+    if timeDifference.lower() == "full" or options["initload"] == "1":
+        tdifference = 0
+    else:
+        tdifference = int(currentTimestamp) - int(
+            options[options["updatetype"]]
+        )
+    needed_updates = []
+    updateList = []
+    for result in new_updates:
+        updateid = result[0]
+        updatestamp = int(result[1])
+        tdif = int(tdifference)
+        if updatestamp > tdif:
+            needed_updates.append(updateid)
+            updateList.append(updateid)
+
+    print("Update Type: " + str(options["updatetype"]))
+    updateRange = len(updateList)
+    print("Number of updates: " + str(updateRange))
+    cacher.update_cache(
+        jget, updateList, ROOTDIR, TVMAZE_URLS, options, lprint
+        )
+    countrylisting = dbExec.rawsql_select(
+        dbengine["engine"],
+        "select row_id, country_name from dbo.countrydata",
+        lprint
+    )
+    for country in countrylisting:
+        country_name = country[1]
+        country_id = country[0]
+        countrydictionary[country_name] = country_id
+    mongo_updater = dbmongo(options)
+    mongo_updater.update_mongo(
+        updateList,
+        ROOTDIR,
+        directories,
+        jget,
+        countrydictionary,
+        dbExec,
+        dbengine,
+        dboBase,
+        lprint,
+    )
+exit(0)

url/process.py

@@ -0,0 +1,17 @@
+__author__ = 'Wendell Jones'
+import requests
+
+class Loadurl():
+    def __init__(self):
+        pass
+    
+    def get_data(self,weburl):
+        try:
+            showresponse = requests.get(weburl)
+        except requests.exceptions.ConnectionError as e:
+            print('Unable to connect and get JSON data for url ' + str(weburl))
+        else:    
+            return(showresponse.text)
+
+    def __del__(self):
+        pass

url/urls.py

@@ -0,0 +1,60 @@
+__author__ = 'Wendell Jones'
+global urls
+
+
+class Tvmazeurls(object):
+    
+    def __init__(self):
+        self.urls = {}
+        self.set_data()
+
+
+    def set_data(self):
+        self.urls['showurl'] = 'http://api.tvmaze.com/shows/<seriesid>'
+        self.urls['serieslisturl'] = 'http://api.tvmaze.com/shows?page=<pagenum>'
+        self.urls['casturl'] = 'http://api.tvmaze.com/shows/<seriesid>/cast'
+        self.urls['crewurl'] = 'https://api.thetvdb.com/series/<seriesid>/crew'
+        self.urls['creditsurl'] = 'https://api.thetvdb.com/series/<seriesid>/castcredits'
+        self.urls['episodesurl'] = 'http://api.tvmaze.com/shows/<seriesid>/episodes'
+        self.urls['aliasurl'] = 'http://api.tvmaze.com/shows/<seriesid>/akas'
+        self.urls['creditsurl'] = 'http://api.tvmaze.com/people/<seriesid>/castcredits'
+        self.urls['updatesurl'] = 'http://api.tvmaze.com/updates/shows'
+        self.urls['crewurl'] = 'http://api.tvmaze.com/shows/<seriesid>/crew'
+
+
+    def get_data(self):
+        return self.urls
+
+    def __del__(self):
+        pass
+
+class TVDBurls(object):
+
+    def __init__(self):
+        self.urls = {}
+        self.set_data()
+        
+    def set_data(self):
+        self.urls['loginurl'] = 'https://api.thetvdb.com/login'
+        self.urls['refreshurl'] = 'https://api.thetvdb.com/refresh_token'
+        self.urls['showurl'] = 'https://api.thetvdb.com/series/{id}'
+        self.urls['episodeurl'] = 'https://api.thetvdb.com/episodes/{id}'
+        self.urls['casturl'] = 'https://api.thetvdb.com/series/{id}/actors'
+        self.urls['crewurl'] = 'https://api.thetvdb.com/series/{id}/crew'
+        self.urls['creditsurl'] = 'https://api.thetvdb.com/series/{id}/castcredits'
+        self.urls['episodesurl'] = 'https://api.thetvdb.com/series/{id}/episodes'
+        self.urls['episodequeryurl'] = 'https://api.thetvdb.com/series/{id}/episodes/query'
+        self.urls['episodeparamurl'] = 'https://api.thetvdb.com/series/{id}/episodes/query/params'
+        self.urls['episodesummaryurl'] = 'https://api.thetvdb.com/series/{id}/episodes/summary'
+        self.urls['filterurl'] = 'https://api.thetvdb.com/series/{id}/filter'
+        self.urls['filterparamurl'] = 'https://api.thetvdb.com/series/{id}/filter/params'
+        self.urls['imageurl'] = 'https://api.thetvdb.com/series/{id}/images'
+        self.urls['imagequeryurl'] = 'https://api.thetvdb.com/series/{id}/images/query'
+        self.urls['imageparamurl'] = 'https://api.thetvdb.com/series/{id}/images/query/params'
+        self.urls['username'] = 'wpjonesnhcjc'
+        self.urls['apikey'] = 'NACCS2LUWCFEFABX'
+        self.urls['userkey'] = 'UUSZHUQ8CY7N31CF'
+
+
+    def get_data(self):
+        return self.urls

view.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import os
+import time
+import json
+from pathlib import Path
+import db.sql
+from jprocessor.process import Jsonload
+from jprocessor.process_view import Process_View
+import logs.Logger
+from db.functions import settype, dbmongo
+
+if __name__ == "__main__":
+    seriesid = input('Please enter seriesid: ')
+    #seriesname = input('Please enter series name: ')
+    ROOTDIR = os.getcwd()
+    directories = {}
+    if os.name == 'nt':
+        LOGDIR = f'{ROOTDIR}\\log'
+        directories['SERIESDIR'] = f'{ROOTDIR}\\cache\\series\\'
+        directories['EPISODEDIR'] =f'{ROOTDIR}\\cache\\episodes\\'
+        directories['CASTDIR'] = f'{ROOTDIR}\\cache\\cast\\'
+        directories['CREWDIR'] = f'{ROOTDIR}\\cache\\crew\\'
+    else:
+        LOGDIR = f'{ROOTDIR}/log'
+        directories['SERIESDIR'] = f'{ROOTDIR}/cache/series/'
+        directories['EPISODEDIR'] =f'{ROOTDIR}/cache/episodes/'
+        directories['CASTDIR'] = f'{ROOTDIR}/cache/cast/'
+        directories['CREWDIR'] = f'{ROOTDIR}/cache/crew/'
+    lprint = logs.Logger.lprint(LOGDIR, False)
+    lprint.startlog()
+    import settings.config
+    config_options = settings.config.Config(ROOTDIR)
+    options = config_options.config_options
+    mongodb = dbmongo(options)
+    jload = Jsonload()
+    viewprocess = Process_View()
+    #serieslist = mongodb.mgseries.find()
+    #for series in serieslist:
+    #    print(series)
+    series = mongodb.mgseries.find_one({"id": int(seriesid)})
+    episodecursor = mongodb.mgepisodes.find({"seriesid": int(seriesid)})
+    episodes = []
+    for episode in episodecursor:
+        episodes.append(episode)
+    viewprocess.process_seriesdata(series)
+    viewprocess.print_episodedata(episodes)
+    exit()
+