repo migration
This commit is contained in:
@@ -0,0 +1,309 @@
|
||||
import datetime
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy import DateTime
|
||||
|
||||
class SQLGenerate():
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def merge_series(self):
|
||||
seriessql = text('''
|
||||
INSERT INTO dbo.seriesdata (serieslink, url, thetvdb, tvrage, imdb,
|
||||
genres, seriesid, medium,original, language, name, webchannelid, networkid, premiered, rating, runtime, days, time,
|
||||
status, summary, type, weight) SELECT serieslink, url, thetvdb, tvrage, imdb,
|
||||
genres, seriesid, medium,original, language, name, webchannelid, networkid, premiered, rating, runtime, days, time,
|
||||
status, summary, type, weight from updates.newseriesdata
|
||||
ON CONFLICT(seriesid) DO UPDATE
|
||||
SET serieslink = excluded.serieslink,
|
||||
url = excluded.url,
|
||||
thetvdb = excluded.thetvdb,
|
||||
tvrage = excluded.tvrage,
|
||||
genres = excluded.genres,
|
||||
seriesid = excluded.seriesid,
|
||||
medium = excluded.medium,
|
||||
original = excluded.original,
|
||||
language = excluded.language,
|
||||
name = excluded.name,
|
||||
webchannelid = excluded.webchannelid,
|
||||
networkid = excluded.networkid,
|
||||
premiered = excluded.premiered,
|
||||
rating = excluded.rating,
|
||||
runtime = excluded.runtime,
|
||||
days = excluded.days,
|
||||
time = excluded.time,
|
||||
status = excluded.status,
|
||||
summary = excluded.summary,
|
||||
type = excluded.type,
|
||||
weight = excluded.weight;
|
||||
''')
|
||||
return(seriessql)
|
||||
|
||||
def merge_episodes(self):
|
||||
episodesql = text('''
|
||||
INSERT INTO dbo.epdata (episodeid,
|
||||
seriesid,
|
||||
season,
|
||||
episodenumber,
|
||||
episodename,
|
||||
airdate,
|
||||
airtime,
|
||||
episodesummary,
|
||||
airstamp,
|
||||
runtime,
|
||||
eplink,
|
||||
mediumimage,
|
||||
originalimage,
|
||||
"url")
|
||||
SELECT episodeid,
|
||||
seriesid,
|
||||
season,
|
||||
episodenumber,
|
||||
episodename,
|
||||
airdate,
|
||||
airtime,
|
||||
episodesummary,
|
||||
airstamp,
|
||||
runtime,
|
||||
eplink,
|
||||
mediumimage,
|
||||
originalimage,
|
||||
"url"
|
||||
FROM updates.newepdata
|
||||
ON CONFLICT (seriesid, episodeid) do UPDATE
|
||||
SET episodeid = excluded.episodeid,
|
||||
seriesid = excluded.seriesid,
|
||||
season = excluded.season,
|
||||
episodenumber = excluded.episodenumber,
|
||||
episodename = excluded.episodename,
|
||||
airdate = excluded.airdate,
|
||||
airtime = excluded.airtime,
|
||||
episodesummary = excluded.episodesummary,
|
||||
airstamp = excluded.airstamp,
|
||||
runtime = excluded.runtime,
|
||||
eplink = excluded.eplink,
|
||||
mediumimage = excluded.mediumimage,
|
||||
originalimage = excluded.originalimage,
|
||||
"url" = excluded."url";
|
||||
''')
|
||||
return(episodesql)
|
||||
|
||||
def merge_updates(self):
|
||||
updatesql = text('''
|
||||
INSERT INTO updates.tvupdatedata(seriesid, timestamp)
|
||||
SELECT seriesid, timestamp from updates.tvupdates
|
||||
ON CONFLICT(seriesid) DO UPDATE
|
||||
SET seriesid = excluded.seriesid, timestamp = excluded.timestamp;
|
||||
''')
|
||||
return(updatesql)
|
||||
|
||||
def merge_cast(self):
|
||||
datetimestamp = DateTime(datetime.datetime.utcnow)
|
||||
castsql = text('''
|
||||
INSERT INTO dbo.castdata (seriesid,characterid,actorid,charactername,mediumimage,originalimage,apilink,tvmazeilink)
|
||||
SELECT seriesid,characterid,actorid,charactername,mediumimage,originalimage,apilink,tvmazeilink
|
||||
FROM updates.newcastdata
|
||||
ON CONFLICT (seriesid, characterid, actorid) DO UPDATE
|
||||
SET seriesid = excluded.seriesid,
|
||||
characterid = excluded.characterid,
|
||||
actorid = excluded.actorid,
|
||||
charactername = excluded.charactername,
|
||||
mediumimage = excluded.mediumimage,
|
||||
originalimage = excluded.originalimage,
|
||||
apilink = excluded.apilink,
|
||||
tvmazeilink = excluded.tvmazeilink
|
||||
''')
|
||||
return(castsql)
|
||||
|
||||
def merge_actors(self):
|
||||
actorsql = text('''
|
||||
INSERT INTO dbo.actors (actorid,actorname,countryname,countrytimezone,birthday,deathday,gender,mediumimage,originalimage,apilink,"url",countrycode)
|
||||
SELECT actorid,actorname,countryname,countrytimezone,birthday,deathday,gender,mediumimage,originalimage,apilink,"url",countrycode
|
||||
FROM updates.newactordata
|
||||
ON CONFLICT (actorid) DO UPDATE
|
||||
SET actorid = excluded.actorid,
|
||||
actorname = excluded.actorname,
|
||||
countryname = excluded.countryname,
|
||||
countrytimezone = excluded.countrytimezone,
|
||||
birthday = excluded.birthday,
|
||||
deathday = excluded.deathday,
|
||||
gender = excluded.gender,
|
||||
mediumimage = excluded.mediumimage,
|
||||
originalimage = excluded.originalimage,
|
||||
apilink = excluded.apilink,
|
||||
"url" = excluded."url",
|
||||
countrycode = excluded.countrycode;
|
||||
''')
|
||||
return(actorsql)
|
||||
|
||||
def merge_crew(self):
|
||||
crewsql = text('''
|
||||
INSERT INTO dbo.crewdata
|
||||
(seriesid,
|
||||
crewid,
|
||||
countryid,
|
||||
type,
|
||||
name,
|
||||
medium,
|
||||
original,
|
||||
apilink,
|
||||
gender,
|
||||
birthday,
|
||||
deathday)
|
||||
SELECT seriesid,
|
||||
crewid,
|
||||
countryid,
|
||||
type,
|
||||
name,
|
||||
medium,
|
||||
original,
|
||||
apilink,
|
||||
gender,
|
||||
birthday,
|
||||
deathday
|
||||
FROM updates.newcrewdata
|
||||
ON CONFLICT (seriesid, crewid, type) DO UPDATE
|
||||
SET seriesid = excluded.seriesid,
|
||||
crewid = excluded.crewid,
|
||||
countryid = excluded.countryid,
|
||||
type = excluded.type,
|
||||
name = excluded.name,
|
||||
medium = excluded.medium,
|
||||
original = excluded.original,
|
||||
apilink = excluded.apilink,
|
||||
gender = excluded.gender,
|
||||
birthday = excluded.birthday,
|
||||
deathday = excluded.deathday;
|
||||
''')
|
||||
return(crewsql)
|
||||
|
||||
def __del__(self):
|
||||
pass
|
||||
class Merger():
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def __del__(self):
|
||||
pass
|
||||
|
||||
class Charactermerge():
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def merge(self, engine):
|
||||
castsql = text('''MERGE dbo.castdata as N
|
||||
USING updates.newcastdata as O
|
||||
ON N.seriesid = O.seriesid and N.characterid = O.characterid and N.actorid = O.actorid
|
||||
WHEN MATCHED THEN
|
||||
UPDATE SET N.seriesid = O.seriesid, N.characterid = O.characterid,N.actorid = O.actorid,N.charactername = O.charactername,N.mediumimage = O.mediumimage,N.originalimage = O.originalimage,N.apilink = O.apilink,N.tvmazelink = O.tvmazelink, N.updated_on = GETDATE()
|
||||
WHEN NOT MATCHED THEN
|
||||
INSERT (seriesid,characterid,actorid,charactername,mediumimage,originalimage,apilink,tvmazelink, created_on)
|
||||
VALUES
|
||||
(O.seriesid,O.characterid,O.actorid,O.charactername,O.mediumimage,O.originalimage,O.apilink,O.tvmazelink, GETDATE());
|
||||
SELECT * FROM updates.newcastdata;
|
||||
''')
|
||||
with engine.connect() as con:
|
||||
rs = con.execute(castsql)
|
||||
#engine.commit()
|
||||
|
||||
actorsql = text('''MERGE dbo.actors as N
|
||||
USING updates.newactordata as O
|
||||
ON N.actorid = O.actorid
|
||||
WHEN MATCHED THEN
|
||||
UPDATE SET N.actorid = O.actorid, N.actorname = O.actorname, N.countryname = O.countryname, N.countrytimezone = O.countrytimezone, N.birthday = O.birthday, N.deathday = O.deathday, N.gender = O.gender, N.mediumimage = O.mediumimage, N.originalimage = O.originalimage, N.apilink = O.apilink, N.url = O.url, N.countrycode = O.countrycode, N.updated_on = GETDATE()
|
||||
WHEN NOT MATCHED THEN
|
||||
INSERT (actorid,actorname,countryname,countrytimezone,birthday,deathday,gender,mediumimage,originalimage,apilink,url,countrycode,created_on)
|
||||
VALUES
|
||||
(O.actorid,O.actorname,O.countryname,O.countrytimezone,O.birthday,O.deathday,O.gender,O.mediumimage,O.originalimage,O.apilink,O.url,O.countrycode, GETDATE());
|
||||
SELECT * FROM updates.newactordata;
|
||||
''')
|
||||
with engine.connect() as con:
|
||||
rs = con.execute(actorsql)
|
||||
#engine.commit()
|
||||
|
||||
def __del__(self):
|
||||
pass
|
||||
|
||||
class Updatemerge():
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def merge(self, engine):
|
||||
updatesql = text('''INSERT INTO updates.tvupdatedata (seriesid, timestamp)
|
||||
select seriesid, timestamp from updates.tvupdates
|
||||
ON CONFLICT (seriesid)
|
||||
DO
|
||||
UPDATE
|
||||
SET seriesid = EXCLUDED.seriesid, timestamp = EXCLUDED.timestamp;''')
|
||||
with engine.connect() as con:
|
||||
rs = con.execute(updatesql)
|
||||
#engine.commit()
|
||||
|
||||
def __del__(self):
|
||||
pass
|
||||
class Jsonmerge():
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def merge(self, engine):
|
||||
seriessql = text('''INSERT INTO "[updates]".tvmaze_seriesjson (seriesid, jdata, created_on)
|
||||
select seriesid, jdata, date_trunc('minute', current_timestamp) FROM "[updates]".seriesjsonnewvals
|
||||
ON CONFLICT (seriesid)
|
||||
DO
|
||||
update
|
||||
SET seriesid = EXCLUDED.seriesid, jdata = EXCLUDED.jdata, updated_on = date_trunc('minute', current_timestamp);
|
||||
''')
|
||||
with engine.connect() as con:
|
||||
rs = con.execute(seriessql)
|
||||
#engine.commit()
|
||||
|
||||
episodesql = text('''INSERT INTO "[updates]".tvmaze_episodejson (seriesid, jdata, created_on)
|
||||
select seriesid, jdata, date_trunc('minute', current_timestamp) FROM "[updates]".episodejsonnewvals
|
||||
ON CONFLICT (seriesid)
|
||||
DO
|
||||
update
|
||||
SET seriesid = EXCLUDED.seriesid, jdata = EXCLUDED.jdata, updated_on = date_trunc('minute', current_timestamp);
|
||||
''')
|
||||
with engine.connect() as con:
|
||||
rs = con.execute(episodesql)
|
||||
#engine.commit()
|
||||
|
||||
charactersql = text('''INSERT INTO "[updates]".tvmaze_characterjson (seriesid, jdata, created_on)
|
||||
select seriesid, jdata, date_trunc('minute', current_timestamp) FROM "[updates]".characterjsonnewvals
|
||||
ON CONFLICT (seriesid)
|
||||
DO
|
||||
update
|
||||
SET seriesid = EXCLUDED.seriesid, jdata = EXCLUDED.jdata, updated_on = date_trunc('minute', current_timestamp);
|
||||
''')
|
||||
with engine.connect() as con:
|
||||
rs = con.execute(charactersql)
|
||||
#engine.commit()
|
||||
|
||||
def __del__(self):
|
||||
pass
|
||||
|
||||
class Episodemerge():
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def merge(self, engine):
|
||||
episodesql = text('''INSERT INTO dbo.epdata (episodeid,seriesid,season,episodenumber,episodename,airdate,airtime,episodesummary,airstamp,runtime,eplink,mediumimage,originalimage,url, created_on)
|
||||
select episodeid,seriesid,season,episodenumber,episodename,airdate,airtime,episodesummary,airstamp,runtime,eplink,mediumimage,originalimage,url, date_trunc('minute', current_timestamp) FROM "[updates]".newepdata
|
||||
ON CONFLICT (episodeid)
|
||||
DO
|
||||
UPDATE
|
||||
SET episodeid = EXCLUDED.episodeid, seriesid = EXCLUDED.seriesid, season = EXCLUDED.season, episodenumber = EXCLUDED.episodenumber, episodename = EXCLUDED.episodename,
|
||||
airdate = EXCLUDED.airdate, airtime = EXCLUDED.airtime, episodesummary = EXCLUDED.episodesummary, airstamp = EXCLUDED.airstamp, runtime = EXCLUDED.runtime,
|
||||
eplink = EXCLUDED.eplink, mediumimage = EXCLUDED.mediumimage, originalimage = EXCLUDED.originalimage, url = EXCLUDED.url, updated_on = date_trunc('minute', current_timestamp);''')
|
||||
with engine.connect() as con:
|
||||
rs = con.execute(episodesql)
|
||||
#engine.commit()
|
||||
|
||||
def __del__(self):
|
||||
pass
|
||||
|
||||
|
||||
@@ -0,0 +1,376 @@
|
||||
import sqlalchemy
|
||||
from sqlalchemy import create_engine, MetaData, Table, Column, Integer, String, Sequence, text, UniqueConstraint
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.schema import MetaData
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy.types import *
|
||||
|
||||
class Tables():
|
||||
def __init__(self):
|
||||
global tvnetworknewvals
|
||||
global tvwebchannelnewvals
|
||||
global characterjsonnewvals
|
||||
global seriesjsonnewvals
|
||||
global episodejsonnewvals
|
||||
global metadata
|
||||
global engine
|
||||
global session
|
||||
global tvnetworknewvals
|
||||
global tvwebchannelnewvals
|
||||
global characterjsonnewvals
|
||||
global seriesjsonnewvals
|
||||
global episodejsonnewvals
|
||||
global tvupdates
|
||||
global newseriesdata
|
||||
global actorlinks
|
||||
global tablelist
|
||||
|
||||
def __del__(self):
|
||||
pass
|
||||
|
||||
|
||||
class Createtables():
|
||||
|
||||
def __init__(self, session, metadata, engine, tablelist):
|
||||
Base = declarative_base(metadata = metadata)
|
||||
self.tablelist = self.settables(metadata, engine)
|
||||
try:
|
||||
self.drop(session, self.tablelist)
|
||||
except:
|
||||
pass
|
||||
self.create(Base, engine, metadata, session, self.tablelist)
|
||||
session.commit()
|
||||
#self.send(tablelist)
|
||||
|
||||
def settables(self, metadata, engine):
|
||||
tablelist = []
|
||||
newseriesdata = Table('newseriesdata', metadata,
|
||||
Column('id', BIGINT, Sequence('newseriesdata_id_seq'), primary_key = True),
|
||||
Column('seriesid',INTEGER, unique=True),
|
||||
Column('tvdbid', String(25)),
|
||||
Column('tvrageid', String(25)),
|
||||
Column('imdbid', String(25)),
|
||||
Column('serieslanguage', String(25)),
|
||||
Column('genres', String(75)),
|
||||
Column('seriesname', String(150)),
|
||||
Column('seriesstatus', String(25)),
|
||||
Column('premiered', String(25)),
|
||||
Column('airdays', String(72)),
|
||||
Column('airtime', String(25)),
|
||||
Column('runtime', String(25)),
|
||||
Column('rating', String(25)),
|
||||
Column('showtype',TEXT),
|
||||
Column('overview',TEXT),
|
||||
Column('mediumimage',TEXT),
|
||||
Column('originalimage',TEXT),
|
||||
Column('weight',INTEGER),
|
||||
Column('networkid', BIGINT),
|
||||
Column('webchannelid', BIGINT),
|
||||
Column('officialsite',TEXT),
|
||||
Column('showurl',TEXT),
|
||||
Column('apilink',TEXT),
|
||||
schema = "updates")
|
||||
tablelist.append(newseriesdata)
|
||||
newactordata = Table('newactordata', metadata,
|
||||
Column('id', BIGINT, Sequence('newactordata_id_seq'), primary_key = True),
|
||||
Column('actorid', INTEGER, unique=True),
|
||||
Column('actorname', VARCHAR(72)),
|
||||
Column('countryname', VARCHAR(144)),
|
||||
Column('countrytimezone', VARCHAR(72)),
|
||||
Column('birthday', DATE),
|
||||
Column('deathday', DATE),
|
||||
Column('gender', VARCHAR(6)),
|
||||
Column('mediumimage', TEXT),
|
||||
Column('originalimage', TEXT),
|
||||
Column('apilink', TEXT),
|
||||
Column('url', TEXT),
|
||||
Column('countrycode', VARCHAR(12)),
|
||||
schema = "updates")
|
||||
tablelist.append(newactordata)
|
||||
newepdata = Table('newepdata', metadata,
|
||||
#Column('id', BIGINT, Sequence('newepdata_id_seq'), primary_key = True),
|
||||
Column('episodeid', INTEGER),
|
||||
Column('seriesid', INTEGER),
|
||||
Column('season', INTEGER),
|
||||
Column('episodenumber', INTEGER),
|
||||
Column('episodename', TEXT),
|
||||
Column('airdate', VARCHAR(32)),
|
||||
Column('airtime', VARCHAR(32)),
|
||||
Column('episodesummary', TEXT),
|
||||
Column('airstamp', VARCHAR(32)),
|
||||
Column('runtime', VARCHAR(6)),
|
||||
Column('eplink', TEXT),
|
||||
Column('mediumimage', TEXT),
|
||||
Column('url', TEXT),
|
||||
Column('originalimage', TEXT),
|
||||
schema = "updates")
|
||||
tablelist.append(newepdata)
|
||||
newcastdata = Table('newcastdata', metadata,
|
||||
Column('id', BIGINT, Sequence('newcastdata_id_seq'), primary_key = True),
|
||||
Column('seriesid', INTEGER),
|
||||
Column('characterid', INTEGER),
|
||||
Column('actorid', INTEGER),
|
||||
Column('charactername', VARCHAR(250)),
|
||||
Column('mediumimage', TEXT),
|
||||
Column('originalimage', TEXT),
|
||||
Column('apilink', TEXT),
|
||||
Column('tvmazelink', TEXT),
|
||||
schema = "updates")
|
||||
tablelist.append(newcastdata)
|
||||
tvnetworks = Table('tvnetworkdata', metadata,
|
||||
Column('id', BIGINT, Sequence('network_id_seq'), primary_key = True),
|
||||
Column('networkid', String(12)),
|
||||
Column('networkname', String(255)),
|
||||
Column('country', String(255)),
|
||||
Column('countrycode', String(4)),
|
||||
Column('countrytimezone', String(25)),
|
||||
schema = "updates")
|
||||
tablelist.append(tvnetworks)
|
||||
tvwebchannels = Table('tvwebchanneldata', metadata,
|
||||
Column('id', BIGINT, Sequence('webchannel_id_seq'), primary_key = True),
|
||||
Column('webchannelname', String(255)),
|
||||
Column('webchannelcountrycode', String(4)),
|
||||
Column('webchannelcountryname', String(255)),
|
||||
Column('webchannelcountrytimezone', String(4)),
|
||||
schema = "updates")
|
||||
tablelist.append(tvwebchannels)
|
||||
return(tablelist)
|
||||
|
||||
def create(self, Base, engine, metadata, session, tablelist):
|
||||
print('Creating Temporary Data Tables')
|
||||
Base.metadata.create_all(engine, tables=tablelist)
|
||||
|
||||
def drop(self, session, tablelist):
|
||||
for table in tablelist:
|
||||
try:
|
||||
#print('Dropping ' + str(table))
|
||||
table.drop()
|
||||
except sqlalchemy.exc.ProgrammingError as se:
|
||||
pass
|
||||
|
||||
def __del__(self):
|
||||
pass
|
||||
|
||||
|
||||
class Temptables():
|
||||
|
||||
def __init__(self, session, metadata, engine, tablelist):
|
||||
Base = declarative_base(metadata = metadata)
|
||||
self.tablelist = self.settables(metadata, engine)
|
||||
try:
|
||||
self.drop(session, self.tablelist)
|
||||
except:
|
||||
pass
|
||||
self.create(Base, engine, metadata, session, self.tablelist)
|
||||
#session.commit()
|
||||
|
||||
def settables(self, metadata, engine):
|
||||
tablelist = []
|
||||
tvnetworknewvals = Table('tvnetworknewvals', metadata,
|
||||
Column('id', BIGINT, Sequence('network_id_seq'), primary_key = True),
|
||||
Column('name', String(255)),
|
||||
Column('countryid', String(255)),
|
||||
schema = "updates")
|
||||
tablelist.append(tvnetworknewvals)
|
||||
tvwebchannelnewvals = Table('tvwebchannelnewvals', metadata,
|
||||
Column('id', BIGINT, Sequence('webchannel_id_seq'), primary_key = True),
|
||||
Column('name', String(255)),
|
||||
Column('countryid', String(4)),
|
||||
schema = "updates")
|
||||
tablelist.append(tvwebchannelnewvals)
|
||||
tvupdates = Table('tvupdates', metadata,
|
||||
Column('seriesid', BIGINT, primary_key = True),
|
||||
Column('timestamp', VARCHAR(15)),
|
||||
schema = "updates")
|
||||
tablelist.append(tvupdates)
|
||||
tvupdatedata = Table('tvupdatedata', metadata,
|
||||
Column('seriesid', BIGINT, primary_key = True),
|
||||
Column('timestamp', VARCHAR(15)),
|
||||
schema = "updates")
|
||||
tablelist.append(tvupdatedata)
|
||||
newactorlinks = Table('newactorlinks', metadata,
|
||||
Column('id', BIGINT, Sequence('tempepisodejson_id_seq'), primary_key = True),
|
||||
Column('actorid', INTEGER),
|
||||
Column('characterid', INTEGER),
|
||||
schema = "updates")
|
||||
tablelist.append(newactorlinks)
|
||||
return(tablelist)
|
||||
|
||||
def create(self, Base, engine, metadata, session, tablelist):
|
||||
tablelist.remove(tablelist[6])
|
||||
print('Creating Temporary JSON Tables')
|
||||
try:
|
||||
Base.metadata.create_all(engine, tables=tablelist)
|
||||
except sqlalchemy.exc.ProgrammingError as pe:
|
||||
pass
|
||||
|
||||
def drop(self, session, tablelist):
|
||||
for table in tablelist:
|
||||
if 'tvupdatedata' in table.description:
|
||||
pass
|
||||
else:
|
||||
try:
|
||||
#print('Dropping ' + str(table))
|
||||
table.drop()
|
||||
except sqlalchemy.exc.ProgrammingError as se:
|
||||
pass
|
||||
#session.commit()
|
||||
|
||||
class CreateTempTables():
|
||||
|
||||
global tvnetworknewvals
|
||||
global tvwebchannelnewvals
|
||||
global characterjsonnewvals
|
||||
global seriesjsonnewvals
|
||||
global episodejsonnewvals
|
||||
global metadata
|
||||
global engine
|
||||
global session
|
||||
|
||||
global tablelist
|
||||
|
||||
def __init__(self, session, metadata, engine, tablelist):
|
||||
Base = declarative_base(metadata = metadata)
|
||||
self.tablelist = self.settables(metadata, engine)
|
||||
try:
|
||||
self.drop(session, self.tablelist)
|
||||
except:
|
||||
pass
|
||||
self.create(Base, engine, metadata, session, self.tablelist)
|
||||
session.commit()
|
||||
#self.send(tablelist)
|
||||
|
||||
def settables(self, metadata, engine):
|
||||
tablelist = []
|
||||
newseriesdata = Table('newseriesdata', metadata,
|
||||
Column('id', BIGINT, Sequence('newseriesdata_id_seq'), primary_key = True),
|
||||
Column('seriesid',INTEGER, unique=True),
|
||||
Column('thetvdb', String(25)),
|
||||
Column('tvrage', String(25)),
|
||||
Column('imdb', String(25)),
|
||||
Column('language', String(25)),
|
||||
Column('genres', String(75)),
|
||||
Column('name', String(150)),
|
||||
Column('status', String(25)),
|
||||
Column('premiered', String(25)),
|
||||
Column('days', String(72)),
|
||||
Column('time', String(25)),
|
||||
Column('runtime', String(25)),
|
||||
Column('rating', String(25)),
|
||||
Column('type',TEXT),
|
||||
Column('summary',TEXT),
|
||||
Column('medium',TEXT),
|
||||
Column('original',TEXT),
|
||||
Column('weight',INTEGER),
|
||||
Column('networkid', BIGINT),
|
||||
Column('webchannelid', BIGINT),
|
||||
Column('officialsite',TEXT),
|
||||
Column('url',TEXT),
|
||||
Column('serieslink',TEXT),
|
||||
schema="updates")
|
||||
tablelist.append(newseriesdata)
|
||||
newactordata = Table('newactordata', metadata,
|
||||
Column('id', BIGINT, Sequence('newactordata_id_seq'), primary_key = True),
|
||||
Column('actorid', INTEGER, unique=True),
|
||||
Column('actorname', VARCHAR(72)),
|
||||
Column('countryname', VARCHAR(36)),
|
||||
Column('countrytimezone', VARCHAR(72)),
|
||||
Column('birthday', DATE),
|
||||
Column('deathday', DATE),
|
||||
Column('gender', VARCHAR(6)),
|
||||
Column('mediumimage', TEXT),
|
||||
Column('originalimage', TEXT),
|
||||
Column('apilink', TEXT),
|
||||
Column('url', TEXT),
|
||||
Column('countrycode', VARCHAR(12)),
|
||||
schema="updates")
|
||||
tablelist.append(newactordata)
|
||||
newepdata = Table('newepdata', metadata,
|
||||
#Column('id', BIGINT, Sequence('newepdata_id_seq'), primary_key = True),
|
||||
Column('episodeid', INTEGER),
|
||||
Column('seriesid', INTEGER),
|
||||
Column('season', INTEGER),
|
||||
Column('episodenumber', INTEGER),
|
||||
Column('episodename', TEXT),
|
||||
Column('airdate', VARCHAR(32)),
|
||||
Column('airtime', VARCHAR(32)),
|
||||
Column('episodesummary', TEXT),
|
||||
Column('airstamp', VARCHAR(32)),
|
||||
Column('runtime', VARCHAR(6)),
|
||||
Column('eplink', TEXT),
|
||||
Column('mediumimage', TEXT),
|
||||
Column('url', TEXT),
|
||||
Column('originalimage', TEXT),
|
||||
schema="updates")
|
||||
tablelist.append(newepdata)
|
||||
newcastdata = Table('newcastdata', metadata,
|
||||
Column('id', BIGINT, Sequence('newcastdata_id_seq'), primary_key = True),
|
||||
Column('seriesid', INTEGER),
|
||||
Column('characterid', INTEGER),
|
||||
Column('actorid', INTEGER),
|
||||
Column('url', TEXT),
|
||||
Column('charactername', VARCHAR(128)),
|
||||
Column('mediumimage', TEXT),
|
||||
Column('originalimage', TEXT),
|
||||
Column('apilink', TEXT),
|
||||
Column('tvmazelink', TEXT),
|
||||
Column('self', CHAR),
|
||||
Column('voice', CHAR),
|
||||
schema="updates")
|
||||
tablelist.append(newcastdata)
|
||||
newcrewdata = Table('newcrewdata', metadata,
|
||||
Column('id', BIGINT, Sequence('newcrewdata_id_seq'), primary_key = True),
|
||||
Column('seriesid', INTEGER),
|
||||
Column('crewid', INTEGER),
|
||||
Column('type', VARCHAR(128)),
|
||||
Column('url', TEXT),
|
||||
Column('name', VARCHAR(256)),
|
||||
Column('countryid', INTEGER),
|
||||
Column('birthday', DATE),
|
||||
Column('deathday', DATE),
|
||||
Column('gender', VARCHAR(24)),
|
||||
Column('medium', TEXT),
|
||||
Column('original', TEXT),
|
||||
Column('apilink', TEXT),
|
||||
UniqueConstraint('seriesid', 'crewid', 'type', name='crew_unique'),
|
||||
schema="updates")
|
||||
tablelist.append(newcrewdata)
|
||||
tvnetworks = Table('tvnetworkdata', metadata,
|
||||
Column('id', BIGINT, Sequence('network_id_seq'), primary_key = True),
|
||||
Column('name', String(255)),
|
||||
Column('countryid', String(255)),
|
||||
schema = "updates")
|
||||
tablelist.append(tvnetworks)
|
||||
tvwebchannels = Table('tvwebchanneldata', metadata,
|
||||
Column('id', BIGINT, Sequence('webchannel_id_seq'), primary_key = True),
|
||||
Column('name', String(255)),
|
||||
Column('countryid', String(255)),
|
||||
schema = "updates")
|
||||
tablelist.append(tvwebchannels)
|
||||
countries = Table('newcountrydata', metadata,
|
||||
Column('id', BIGINT, Sequence('country_id_seq'), primary_key = True),
|
||||
Column('name', String(255)),
|
||||
Column('code', String(255)),
|
||||
Column('timezone', String(255)),
|
||||
UniqueConstraint('name', 'timezone', name='tz_name_uk'),
|
||||
schema = "updates")
|
||||
tablelist.append(countries)
|
||||
tvupdates = Table('tvupdates', metadata,
|
||||
Column('seriesid', BIGINT, primary_key = True),
|
||||
Column('timestamp', VARCHAR(15)),
|
||||
schema = "updates")
|
||||
tablelist.append(tvupdates)
|
||||
return(tablelist)
|
||||
|
||||
def create(self, Base, engine, metadata, session, tablelist):
|
||||
print('Creating Temporary Data Tables')
|
||||
Base.metadata.create_all(engine, tables=tablelist)
|
||||
|
||||
def drop(self, session, tablelist):
|
||||
print('Dropping temporary update tables.')
|
||||
for table in tablelist:
|
||||
try:
|
||||
print('Dropping ' + str(table))
|
||||
table.drop()
|
||||
except sqlalchemy.exc.ProgrammingError as se:
|
||||
pass
|
||||
@@ -0,0 +1,39 @@
|
||||
import sqlalchemy
|
||||
import progressbar
|
||||
from tvmaze.Processor import loadjson
|
||||
|
||||
class Updater():
|
||||
def __init__(self):
|
||||
global jprocess
|
||||
|
||||
def __del__(self):
|
||||
pass
|
||||
|
||||
class tvupdates():
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def update_tables(engine, updatesapi, tablename):
|
||||
jprocess = loadjson()
|
||||
print('Now retrieving available updates.')
|
||||
print('Creating update transaction.')
|
||||
ins = tablename.insert()
|
||||
seriesupdates = jprocess.load(updatesapi)
|
||||
updatelist = []
|
||||
seriesrange = len(seriesupdates)
|
||||
with progressbar.ProgressBar(maxval=seriesrange, redirect_stdout=True) as p:
|
||||
for key, value in seriesupdates.items():
|
||||
i = int(key)
|
||||
updater = {}
|
||||
updater['seriesid'] = key
|
||||
updater['timestamp'] = value
|
||||
updatelist.append(updater)
|
||||
if i <= seriesrange:
|
||||
p.update(i)
|
||||
#time.sleep(0.1)
|
||||
print('Update of tvupdates table is complete.')
|
||||
print('Done.')
|
||||
return(ins, updatelist)
|
||||
|
||||
def __del__(self):
|
||||
pass
|
||||
Reference in New Issue
Block a user