From c947f7a6db6177cf72336add6e8fa3a38f0d8b5d Mon Sep 17 00:00:00 2001 From: Wendell Jones Date: Wed, 25 Jun 2025 16:13:34 -0400 Subject: [PATCH] fix code for all tables in json2postgres.py so all tables get processed now. Update requirements.txt to include pandas. --- json2postgres.py | 143 ++++++++++++++++++++++------------------------- requirements.txt | 1 + 2 files changed, 67 insertions(+), 77 deletions(-) diff --git a/json2postgres.py b/json2postgres.py index dacbfc9..d34d78d 100644 --- a/json2postgres.py +++ b/json2postgres.py @@ -36,10 +36,10 @@ def process_data(data, tablename): inserts = [] failed_count = 0 seriesconflict = "ON CONFLICT (seriesid) DO UPDATE SET" - epconflict = "ON CONFLICT (episodeid) DO UPDATE SET" + epconflict = "ON CONFLICT (episodeid, seriesid) DO UPDATE SET" crewconflict = "ON CONFLICT (crewid) DO UPDATE SET" actorconflict = "ON CONFLICT (actorid) DO UPDATE SET" - characterconflict = "ON CONFLICT (characterid, actorid) DO UPDATE SET" + characterconflict = "ON CONFLICT (seriesid, characterid) DO UPDATE SET" for document in data: if document['name'].lower() == 'too many requests': print("Skipping document due to 'Too Many Requests' in " @@ -63,8 +63,14 @@ def process_data(data, tablename): column_list = [] data_list = [] for key, value in flatten_data.items(): + if isinstance(value, str): + value = value.replace('"', "'").replace("'", "''") if key.lower() == '_links.self.href': key = 'apilink' + if key.lower() == '_links.show.href': + key = 'apishowlink' + if key.lower() == '_links.show.name': + key = 'apishowname' if '.' in key.lower(): key = key.replace('.', '') if tablename == 'seriesdata' and ( @@ -73,6 +79,8 @@ def process_data(data, tablename): key = f"series_{key}" if tablename == 'seriesdata' and key.lower() == 'id': key = "seriesid" + if tablename == 'crewdata' and key.lower() == 'id': + key = "crewid" if ( tablename == 'actordata' and ( key.lower() == 'name' or key.lower() == 'number' or @@ -89,8 +97,12 @@ def process_data(data, tablename): if tablename == 'characterdata' and key.lower() == 'id': key = "seriesid, characterid" if ( - (tablename == 'epdata' or tablename == 'actordata') and ( - key.lower() == 'name' or key.lower() == 'number' or + ( + tablename == 'epdata' or + tablename == 'actordata' + ) and ( + key.lower() == 'name' or + key.lower() == 'number' or key.lower() == 'type' ) ): @@ -99,41 +111,19 @@ def process_data(data, tablename): key = "language_name" if tablename == 'epdata' and key.lower() == 'id': key = "episodeid" - if ( - '_links' in key.lower() or - 'image' in key.lower() or - 'schedule' in key.lower() or - 'externals' in key.lower() or - 'network' in key.lower() or - 'webchannel' in key.lower() or - 'country' in key.lower() or - 'rating' in key.lower() - ): - key = key.replace('.', '_') if tablename == 'actordata' and 'seriesid' in key.lower(): continue else: column_list.append(key) - if 'genres' in key.lower() or 'schedule' in key.lower(): + if ( + 'genres' in key.lower() or + 'schedule' in key.lower() + ): if isinstance(value, list): value = ', '.join(value) - else: - value = str(value).replace("'", "''") - if isinstance(value, str): - value = value.replace("'", "''") - if isinstance(value, int): - value = f"'{value}'" - elif isinstance(value, float): - value = f"'{json.dumps(value)}'" - elif isinstance(value, dict): - value = f"'{json.dumps(value)}'" - elif isinstance(value, bool): - value = str(value).lower() - elif value is None: - value = 'NULL' - else: - value = str(f"'{value}'") - data_list.append(value) + if isinstance(value, int) or isinstance(value, float): + value = str(json.dumps(value)) + data_list.append(f"'{value}'") columns = ", ".join(column_list) column_data = ", ".join(data_list) conflict_list = [] @@ -141,7 +131,7 @@ def process_data(data, tablename): conflict_list.append( f"{column_list[indexer]} = {data_list[indexer]}" ) - conflict_data = ', '.join(conflict_list) + conflict_data = ', '.join(conflict_list).replace("\\", "") conflict_clause = '' if tablename == 'seriesdata': conflict_clause = seriesconflict @@ -160,7 +150,6 @@ def process_data(data, tablename): f"{conflict_clause} {conflict_data};" ) ) - inserts.append(insert_command) return inserts, failed_count @@ -216,35 +205,35 @@ else: print("Creating Insert commands from collection data.") -# print("Collecting series data for series inserts.") -# series_list = mgseries.find({}) -# inserts, failed_count = process_data(series_list, 'seriesdata') -# print(f"Total number of series inserts: {len(inserts)}") -# for i in tqdm(range(len(inserts))): -# try: -# insert = inserts[i] -# pgcursor.execute(insert) -# except Exception: -# # print(f"Unable to insert data into seriesdata table. {e}") -# failed_count += 1 -# print(f"Total number of failed series inserts: {failed_count}") -# pgclient.commit() +print("Collecting series data for series inserts.") +series_list = mgseries.find({}) +inserts, failed_count = process_data(series_list, 'seriesdata') +print(f"Total number of series inserts: {len(inserts)}") +for i in tqdm(range(len(inserts))): + try: + insert = inserts[i] + pgcursor.execute(insert) + except Exception as e: + print(f"Unable to insert data into seriesdata table. {e}") + failed_count += 1 +print(f"Total number of failed series inserts: {failed_count}") +pgclient.commit() -# epinserts = [] -# failures = 0 -# episode_array = mgepisodes.find({}) -# print("Collecting episode data for episode inserts.") -# inserts, failed_count = process_data(episode_array, 'epdata') -# print(f"Total number of episode inserts: {len(inserts)}") -# for i in tqdm(range(len(inserts))): -# try: -# insert = inserts[i] -# pgcursor.execute(insert) -# except Exception: -# # print(f"Unable to insert data into epdata table. {e}") -# failed_count += 1 -# print(f"Total number of failed episode inserts: {failed_count}") -# pgclient.commit() +epinserts = [] +failures = 0 +episode_array = mgepisodes.find({}) +print("Collecting episode data for episode inserts.") +inserts, failed_count = process_data(episode_array, 'epdata') +print(f"Total number of episode inserts: {len(inserts)}") +for i in tqdm(range(len(inserts))): + try: + insert = inserts[i] + pgcursor.execute(insert) + except Exception as e: + print(f"Unable to insert data into epdata table. {e}") + failed_count += 1 +print(f"Total number of failed episode inserts: {failed_count}") +pgclient.commit() failures = 0 actor_array = mgactors.find({}) @@ -261,20 +250,20 @@ for i in tqdm(range(len(inserts))): print(f"Total number of failed actor inserts: {failed_count}") pgclient.commit() -# failures = 0 -# character_array = mgcharacters.find({}) -# print("Collecting actor data for actor inserts.") -# inserts, failed_count = process_data(character_array, 'characterdata') -# print(f"Total number of character inserts: {len(inserts)}") -# for i in tqdm(range(len(inserts))): -# try: -# insert = inserts[i] -# pgcursor.execute(insert) -# except Exception as e: -# print(f"Unable to insert data into character table. {e}") -# failed_count += 1 -# print(f"Total number of failed character inserts: {failed_count}") -# pgclient.commit() +failures = 0 +character_array = mgcharacters.find({}) +print("Collecting actor data for actor inserts.") +inserts, failed_count = process_data(character_array, 'characterdata') +print(f"Total number of character inserts: {len(inserts)}") +for i in tqdm(range(len(inserts))): + try: + insert = inserts[i] + pgcursor.execute(insert) + except Exception as e: + print(f"Unable to insert data into character table. {e}") + failed_count += 1 +print(f"Total number of failed character inserts: {failed_count}") +pgclient.commit() failures = 0 crew_array = mgcrew.find({}) diff --git a/requirements.txt b/requirements.txt index 3b39954..665441d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ Requests==2.32.3 SQLAlchemy==2.0.39 pymysql tqdm +pandas \ No newline at end of file