diff --git a/db/functions.py b/db/functions.py index 5d39bb7..e21473f 100644 --- a/db/functions.py +++ b/db/functions.py @@ -123,9 +123,9 @@ class settype(object): cursor.execute(text(sql)) updatesBase.prepare(engine, reflect=True, schema="updates") dboBase.prepare(engine, reflect=True, schema="dbo") - metadata = sqlalchemy.MetaData("updates") + metadata = sqlalchemy.MetaData(schema="updates") metadata.reflect(bind=engine) - dboMetadata = sqlalchemy.MetaData("dbo") + dboMetadata = sqlalchemy.MetaData(schema="dbo") dboMetadata.reflect(bind=engine) session = Session(engine) self.dbengine["engine"] = engine diff --git a/db/sql.py b/db/sql.py index c9a9e4c..efa3ef2 100644 --- a/db/sql.py +++ b/db/sql.py @@ -12,27 +12,32 @@ class Dbexec(): def update_tvupdates(self, engine, available_updates, tablename): print('Creating update transaction.') + # Build batch insert data updatelist = [ { - 'seriesid': seriesid, + 'seriesid': str(seriesid), 'timestamp': available_updates[seriesid] } for seriesid in tqdm( available_updates, - desc='Updating tvupdates table', + desc='Preparing tvupdates records', unit='record' ) ] - # Commented out code for applying updates for now. Will re-enable later after downloads are verified. print('Applying updates to table.') - with engine.connect() as conn: - conn.execute(text(f"truncate table {tablename}")) - for key, value in available_updates.items(): - seriesid = str(key) - timestamp = value - conn.execute(text(f"INSERT INTO {tablename} (seriesid, timestamp) VALUES (:seriesid, :timestamp)"), {'seriesid': seriesid, 'timestamp': timestamp}) - conn.commit() + # Use a transactional context; SQLAlchemy Connection.execute accepts + # a list of parameter mappings for bulk inserts (it will use the DBAPI + # executemany under the hood). + with engine.begin() as conn: + print(f'Truncating table {tablename} before insert.') + conn.execute(text(f"TRUNCATE TABLE {tablename}")) + print(f'Inserting {len(updatelist)} records into {tablename}.') + if updatelist: + conn.execute( + text(f"INSERT INTO {tablename} (seriesid, timestamp) VALUES (:seriesid, :timestamp)"), + updatelist, + ) print('Update of tvupdates table is complete.') def rawsql_select(self, engine, sqlquery, lprint): diff --git a/settings/tvsync_settings.cfg b/settings/tvsync_settings.cfg index 633f231..cafbc24 100644 --- a/settings/tvsync_settings.cfg +++ b/settings/tvsync_settings.cfg @@ -5,7 +5,7 @@ loglevel=info sqlserver_driver=ODBC Driver 13 for SQL Server hostname=192.168.128.3 mghostname=192.168.128.24 -dbname=media_dbsync +dbname=media_dbsync2 mgdbname=tvdata mysqlusername=root pgsqlusername=postgres diff --git a/temp/tvmaze_updates.json b/temp/tvmaze_updates.json index e1def04..e99a469 100644 --- a/temp/tvmaze_updates.json +++ b/temp/tvmaze_updates.json @@ -654,7 +654,7 @@ "672": 1770369680, "673": 1704794564, "674": 1756390245, - "675": 1770370407, + "675": 1770390069, "676": 1769358070, "677": 1739576696, "678": 1753836211, @@ -813,7 +813,7 @@ "836": 1764338899, "837": 1704794646, "838": 1732255155, - "839": 1770059557, + "839": 1770380111, "840": 1704794606, "841": 1728159086, "842": 1770083276, @@ -2043,7 +2043,7 @@ "2128": 1765286378, "2129": 1759734792, "2130": 1682854620, - "2131": 1770202701, + "2131": 1770389935, "2132": 1754081874, "2133": 1712680616, "2134": 1762462723, @@ -3305,7 +3305,7 @@ "3506": 1720796277, "3507": 1770263891, "3508": 1770366267, - "3509": 1768197125, + "3509": 1770384015, "3510": 1754868332, "3511": 1620925576, "3512": 1761558735, @@ -4748,7 +4748,7 @@ "5044": 1626604012, "5045": 1649023262, "5046": 1758134797, - "5047": 1770362960, + "5047": 1770389581, "5048": 1704795049, "5049": 1704794626, "5050": 1724707540, @@ -5284,7 +5284,7 @@ "5596": 1689448796, "5597": 1704795366, "5598": 1756382872, - "5599": 1766682039, + "5599": 1770383718, "5600": 1661599579, "5601": 1704795607, "5602": 1707581168, @@ -8058,7 +8058,7 @@ "8563": 1667774933, "8564": 1632672464, "8565": 1573754123, - "8566": 1770299022, + "8566": 1770383882, "8567": 1631188853, "8569": 1770238950, "8570": 1704795431, @@ -8977,7 +8977,7 @@ "9561": 1728600078, "9562": 1704794430, "9563": 1738102434, - "9564": 1770299127, + "9564": 1770383751, "9565": 1629446527, "9566": 1687701865, "9567": 1629446836, @@ -9061,7 +9061,7 @@ "9656": 1632926726, "9657": 1705154797, "9658": 1504835414, - "9659": 1770335206, + "9659": 1770380870, "9660": 1770342212, "9661": 1756100645, "9662": 1629180069, @@ -14343,8 +14343,8 @@ "15305": 1704795571, "15306": 1463223641, "15307": 1704795215, - "15308": 1770289880, - "15309": 1770307975, + "15308": 1770383287, + "15309": 1770390049, "15310": 1498199696, "15311": 1509849357, "15312": 1766427955, @@ -20300,7 +20300,7 @@ "21735": 1704795457, "21736": 1704793712, "21737": 1768578747, - "21738": 1770298180, + "21738": 1770383374, "21739": 1716253548, "21740": 1731204469, "21741": 1522661460, @@ -21807,7 +21807,7 @@ "23433": 1628631843, "23434": 1628637597, "23435": 1628637613, - "23436": 1481042161, + "23436": 1770382530, "23437": 1481041026, "23438": 1694104850, "23439": 1481049428, @@ -21928,7 +21928,7 @@ "23574": 1481573752, "23575": 1704795447, "23576": 1747320054, - "23577": 1724876760, + "23577": 1770390077, "23578": 1765392742, "23579": 1760210421, "23580": 1647202950, @@ -27247,7 +27247,7 @@ "29255": 1628522102, "29256": 1497558293, "29257": 1539116084, - "29259": 1770367925, + "29259": 1770387734, "29260": 1497572565, "29261": 1745446976, "29262": 1512674977, @@ -35175,7 +35175,7 @@ "37753": 1704793343, "37754": 1531699967, "37755": 1531748329, - "37756": 1770189666, + "37756": 1770380703, "37757": 1642223724, "37758": 1650994724, "37759": 1696148307, @@ -36423,7 +36423,7 @@ "39104": 1760378772, "39105": 1754132426, "39106": 1598397158, - "39107": 1764733647, + "39107": 1770381399, "39108": 1765038117, "39109": 1543156919, "39110": 1554972064, @@ -36474,7 +36474,7 @@ "39160": 1698029461, "39161": 1539905543, "39162": 1707811760, - "39163": 1769784522, + "39163": 1770389612, "39165": 1648137690, "39166": 1688320797, "39167": 1591725578, @@ -43816,7 +43816,7 @@ "46943": 1770243343, "46944": 1700089243, "46945": 1602438703, - "46946": 1735443941, + "46946": 1770381831, "46947": 1701103174, "46948": 1726280227, "46949": 1628841278, @@ -44823,7 +44823,7 @@ "47997": 1768856143, "47998": 1657728175, "47999": 1621762284, - "48000": 1760255385, + "48000": 1770384225, "48001": 1661583522, "48002": 1769958866, "48003": 1721922404, @@ -45172,7 +45172,7 @@ "48359": 1622723450, "48360": 1590765031, "48361": 1698407992, - "48362": 1769890084, + "48362": 1770389538, "48363": 1590860239, "48364": 1590674096, "48365": 1635803910, @@ -46082,7 +46082,7 @@ "49297": 1639764935, "49298": 1726141404, "49299": 1769555022, - "49300": 1770301348, + "49300": 1770386654, "49301": 1597529622, "49302": 1767610269, "49303": 1595897249, @@ -47584,7 +47584,7 @@ "50859": 1655261753, "50860": 1726997000, "50862": 1686923254, - "50863": 1770379277, + "50863": 1770379294, "50864": 1717432866, "50866": 1645129729, "50867": 1645316574, @@ -49699,7 +49699,7 @@ "53060": 1612246381, "53061": 1621109361, "53062": 1694741345, - "53063": 1770375548, + "53063": 1770384991, "53064": 1764157618, "53065": 1640595088, "53066": 1728813262, @@ -55856,7 +55856,7 @@ "59405": 1706814683, "59406": 1647860158, "59407": 1639197253, - "59408": 1745052135, + "59408": 1770380959, "59409": 1719835262, "59410": 1722798177, "59411": 1693816976, @@ -56047,7 +56047,7 @@ "59601": 1640111791, "59602": 1640108248, "59603": 1640110326, - "59604": 1770239960, + "59604": 1770379352, "59605": 1659182435, "59606": 1764671835, "59607": 1752655773, @@ -56664,7 +56664,7 @@ "60243": 1767887932, "60244": 1761404543, "60245": 1744968779, - "60246": 1768652340, + "60246": 1770386931, "60247": 1698265230, "60248": 1748359385, "60249": 1716059235, @@ -56821,7 +56821,7 @@ "60406": 1691253620, "60407": 1644413859, "60408": 1765306589, - "60409": 1764020578, + "60409": 1770380036, "60410": 1766837754, "60411": 1730521941, "60412": 1644607682, @@ -57968,7 +57968,7 @@ "61602": 1649850192, "61603": 1765317022, "61604": 1649860658, - "61605": 1770365336, + "61605": 1770383926, "61606": 1733481464, "61607": 1744813312, "61608": 1761742933, @@ -60007,7 +60007,7 @@ "63709": 1661465758, "63710": 1766237079, "63711": 1661730024, - "63712": 1672145630, + "63712": 1770379986, "63713": 1662157637, "63714": 1726920075, "63715": 1662157134, @@ -60861,7 +60861,7 @@ "64590": 1754943885, "64591": 1761422838, "64592": 1736339460, - "64593": 1770362466, + "64593": 1770386107, "64594": 1675519996, "64595": 1721990770, "64596": 1737673546, @@ -62313,7 +62313,7 @@ "66081": 1672763556, "66082": 1672325952, "66083": 1674572636, - "66084": 1770330926, + "66084": 1770384812, "66085": 1749380610, "66087": 1672339301, "66088": 1725004398, @@ -62544,7 +62544,7 @@ "66316": 1695678339, "66317": 1674291993, "66318": 1739522228, - "66319": 1770299300, + "66319": 1770383945, "66320": 1673467669, "66321": 1680546872, "66323": 1729544285, @@ -64946,7 +64946,7 @@ "68799": 1704793557, "68800": 1768128357, "68801": 1766505280, - "68802": 1761928983, + "68802": 1770389875, "68803": 1752508624, "68804": 1718623922, "68805": 1700369742, @@ -65842,7 +65842,7 @@ "69717": 1769721254, "69718": 1688666206, "69719": 1693388160, - "69720": 1770376501, + "69720": 1770388104, "69721": 1704709157, "69722": 1746639900, "69723": 1688686825, @@ -65920,7 +65920,7 @@ "69796": 1710435465, "69797": 1692859855, "69798": 1707370233, - "69799": 1694346042, + "69799": 1770387958, "69800": 1691237888, "69802": 1688922786, "69803": 1699391479, @@ -67614,7 +67614,7 @@ "71521": 1704794680, "71522": 1762797711, "71523": 1696364305, - "71524": 1770225202, + "71524": 1770385531, "71525": 1697874991, "71526": 1726402987, "71527": 1770225880, @@ -67925,7 +67925,7 @@ "71843": 1766886700, "71844": 1727895790, "71845": 1699105768, - "71846": 1766242116, + "71846": 1770380430, "71847": 1701450871, "71848": 1701105079, "71849": 1727140020, @@ -68533,7 +68533,7 @@ "72463": 1750671873, "72464": 1699107113, "72465": 1699108042, - "72466": 1770303379, + "72466": 1770388117, "72467": 1760248398, "72468": 1699190921, "72469": 1699122789, @@ -68568,7 +68568,7 @@ "72499": 1763421745, "72500": 1769685541, "72501": 1714488170, - "72503": 1770365206, + "72503": 1770389446, "72504": 1768137892, "72505": 1699650170, "72506": 1738182845, @@ -69249,7 +69249,7 @@ "73205": 1701776542, "73206": 1704576025, "73207": 1701877011, - "73208": 1770327598, + "73208": 1770386825, "73209": 1731774599, "73211": 1719693165, "73212": 1701801487, @@ -72914,7 +72914,7 @@ "76966": 1768761465, "76967": 1768758519, "76968": 1769954755, - "76969": 1766257916, + "76969": 1770381586, "76970": 1755010600, "76971": 1755019739, "76972": 1739111156, @@ -73142,7 +73142,7 @@ "77203": 1746727348, "77204": 1755015647, "77205": 1752312979, - "77206": 1766259087, + "77206": 1770382671, "77207": 1746807200, "77208": 1746539572, "77209": 1746633383, @@ -73841,7 +73841,7 @@ "77916": 1743014955, "77917": 1760026625, "77918": 1718830237, - "77919": 1770374260, + "77919": 1770380750, "77920": 1718877983, "77921": 1734218388, "77922": 1718880242, @@ -75837,7 +75837,7 @@ "79981": 1735325096, "79982": 1763020185, "79983": 1728159496, - "79984": 1770374849, + "79984": 1770387819, "79985": 1729468424, "79986": 1746436252, "79987": 1728239280, @@ -76501,7 +76501,7 @@ "80666": 1745811599, "80667": 1761644015, "80668": 1737713346, - "80669": 1770015814, + "80669": 1770386507, "80670": 1731086096, "80671": 1764146510, "80672": 1731692207, @@ -76912,7 +76912,7 @@ "81083": 1742124425, "81084": 1764461940, "81085": 1770082562, - "81086": 1769792537, + "81086": 1770388358, "81087": 1760112115, "81088": 1770148494, "81089": 1770147915, @@ -77045,7 +77045,7 @@ "81218": 1734746700, "81219": 1743943483, "81220": 1733188497, - "81221": 1746017882, + "81221": 1770388233, "81222": 1754669600, "81223": 1733646901, "81224": 1733688682, @@ -77132,7 +77132,7 @@ "81306": 1769476047, "81307": 1743013970, "81308": 1738312206, - "81309": 1770064272, + "81309": 1770380362, "81310": 1770180582, "81311": 1758417980, "81312": 1762305124, @@ -77382,8 +77382,8 @@ "81565": 1734595294, "81566": 1734595676, "81567": 1743703666, - "81568": 1769770251, - "81569": 1770326396, + "81568": 1770389726, + "81569": 1770388981, "81570": 1759413046, "81571": 1734598275, "81572": 1746684532, @@ -78247,7 +78247,7 @@ "82451": 1738831592, "82452": 1738832974, "82453": 1770188859, - "82454": 1769518011, + "82454": 1770386319, "82455": 1738854177, "82456": 1756458272, "82457": 1765148055, @@ -80323,7 +80323,7 @@ "84567": 1746894728, "84568": 1748265563, "84569": 1746877112, - "84570": 1750181288, + "84570": 1770383969, "84571": 1752166593, "84572": 1768731011, "84573": 1746943367, @@ -80608,7 +80608,7 @@ "84863": 1751387076, "84864": 1752068734, "84865": 1748011964, - "84867": 1750525495, + "84867": 1770382699, "84868": 1749156589, "84869": 1748014475, "84870": 1748326620, @@ -81265,7 +81265,7 @@ "85536": 1750608288, "85537": 1767478722, "85538": 1765910274, - "85539": 1769517959, + "85539": 1770386286, "85540": 1750633652, "85541": 1750629832, "85542": 1768666861, @@ -81917,7 +81917,7 @@ "86198": 1753625431, "86199": 1753721893, "86200": 1756201319, - "86201": 1753643053, + "86201": 1770382711, "86202": 1753643223, "86203": 1756041317, "86204": 1754558819, @@ -82258,7 +82258,7 @@ "86546": 1755495978, "86547": 1755491967, "86548": 1756799789, - "86549": 1769738565, + "86549": 1770386260, "86550": 1768752858, "86551": 1755513073, "86552": 1755513106, @@ -82655,7 +82655,7 @@ "86954": 1760947445, "86956": 1757246884, "86957": 1765576391, - "86958": 1770102835, + "86958": 1770386411, "86959": 1757263721, "86960": 1768084646, "86961": 1760804995, @@ -82850,7 +82850,7 @@ "87154": 1757981408, "87155": 1757996181, "87156": 1766798833, - "87157": 1769738465, + "87157": 1770386976, "87158": 1761729663, "87159": 1766675860, "87160": 1758010168, @@ -83011,7 +83011,7 @@ "87317": 1759126381, "87318": 1770352035, "87319": 1761950006, - "87320": 1769738498, + "87320": 1770386217, "87321": 1764720650, "87322": 1758795932, "87323": 1766660961, @@ -83037,7 +83037,7 @@ "87344": 1765138889, "87345": 1758883690, "87346": 1761241280, - "87347": 1770188277, + "87347": 1770386464, "87348": 1763664142, "87349": 1758885260, "87350": 1758885266, @@ -84304,7 +84304,7 @@ "88640": 1768330728, "88641": 1767045759, "88642": 1765734444, - "88643": 1770057169, + "88643": 1770380166, "88644": 1769594243, "88645": 1768640171, "88646": 1764262894, @@ -84901,7 +84901,7 @@ "89247": 1766623135, "89248": 1766624188, "89249": 1770125566, - "89250": 1769965178, + "89250": 1770386346, "89251": 1766671413, "89252": 1766694552, "89253": 1767337981, @@ -85309,7 +85309,7 @@ "89663": 1770376457, "89664": 1767778012, "89665": 1769937659, - "89666": 1770221040, + "89666": 1770386379, "89667": 1768024493, "89668": 1767817646, "89669": 1767797039, @@ -85404,7 +85404,7 @@ "89762": 1769791747, "89763": 1768170314, "89764": 1770289534, - "89765": 1770298563, + "89765": 1770380209, "89766": 1768413494, "89767": 1769473263, "89768": 1769362187, @@ -85599,7 +85599,7 @@ "89963": 1768903757, "89964": 1768903900, "89965": 1768826000, - "89966": 1769020186, + "89966": 1770379684, "89967": 1768750815, "89968": 1768753730, "89969": 1769616921, @@ -85660,7 +85660,7 @@ "90024": 1769186410, "90025": 1769186332, "90026": 1769172583, - "90027": 1770033309, + "90027": 1770380036, "90028": 1770284294, "90029": 1769204606, "90030": 1769187207, @@ -85788,7 +85788,7 @@ "90153": 1770040645, "90154": 1770057517, "90155": 1770044140, - "90156": 1770366182, + "90156": 1770381576, "90157": 1770059065, "90158": 1770058997, "90159": 1770366240, @@ -85822,7 +85822,7 @@ "90187": 1770239660, "90188": 1770251847, "90189": 1770226358, - "90190": 1770378900, + "90190": 1770379946, "90191": 1770372888, "90192": 1770297108, "90193": 1770281201, diff --git a/update_json.py b/update_json.py index 1cf1ae6..1dca40d 100755 --- a/update_json.py +++ b/update_json.py @@ -16,7 +16,7 @@ import json_downloader response = None -def set_apienv(urls, uprocess, dbengine, dbExec, updatesBase, lprint): +def set_apienv(dbengine, dbExec, lprint): """Populate updates table from the API and return available updates. Args: @@ -86,7 +86,6 @@ def main() -> int: #directories["CREDITSDIR"] = os.path.join(ROOTDIR, "cache", "credits") + os.sep #directories["ALIASDIR"] = os.path.join(ROOTDIR, "cache", "aliases") + os.sep tempdir = os.path.join(ROOTDIR, 'temp') + os.sep - updatetable = 'updates.tvupdates' skip_file = f"{tempdir}skip_ids.txt" # Create directories if missing so later code doesn't fail @@ -108,10 +107,9 @@ def main() -> int: config_options = settings.config.Config(ROOTDIR) options = config_options.config_options - mongo_updater = dbmongo(options) # Create an instance of dbmongo dbtype, apitype = options["dbtype"], options["apitype"] dbs = settype(dbtype, apitype, options) - dbclass, dbengine, apiengine = dbs.dbclass, dbs.dbengine, dbs.apiengine + dbclass, dbengine = dbs.dbclass, dbs.dbengine uprocess = url.process.Loadurl() TVMAZEURLS = url.urls.Tvmazeurls() TVMAZE_URLS = get_tvmaze(TVMAZEURLS) @@ -146,12 +144,9 @@ def main() -> int: if options["apitype"].lower() == "tvmaze": try: new_updates = set_apienv( - TVMAZE_URLS, - uprocess, dbengine, dbExec, - tempTableList, - lprint, + lprint ) except Exception as exc: # noqa: PLC0301 - broad handler to log failures lprint.logprint("error", f"Failed to set API env: {exc}") @@ -337,32 +332,7 @@ def main() -> int: url_template = TVMAZE_URLS["showurl"] seriesid = file.replace('.json', '') json_downloader.download_json_to_file(url_template.replace("", str(seriesid)), directory, f'{seriesid}.json', lprint) - # dbExec.update_tvupdates(dbengine["engine"], downloaded_updates_dict, updatetable) - - - - # countrylisting = dbExec.rawsql_select( - # dbengine["engine"], - # "select row_id, country_name from dbo.countrydata", - # lprint - # ) - # for country in countrylisting: - # country_name = country[1] - # country_id = country[0] - # countrydictionary[country_name] = country_id - # mongo_updater = dbmongo(options) - # mongo_updater.update_mongo( - # updateList, - # ROOTDIR, - # directories, - # jget, - # countrydictionary, - # dbExec, - # dbengine, - # dboBase, - # lprint, - # ) - # return 0 + lprint.logprint("info", "JSON download complete.") if __name__ == "__main__":