fix code for all tables in json2postgres.py so all tables get processed now. Update requirements.txt to include pandas.

2025-06-25 16:13:34 -04:00
parent f719155a35
commit c947f7a6db
2 changed files with 67 additions and 77 deletions
@@ -36,10 +36,10 @@ def process_data(data, tablename):
    inserts = []
    failed_count = 0
    seriesconflict = "ON CONFLICT (seriesid) DO UPDATE SET"
-    epconflict = "ON CONFLICT (episodeid) DO UPDATE SET"
+    epconflict = "ON CONFLICT (episodeid, seriesid) DO UPDATE SET"
    crewconflict = "ON CONFLICT (crewid) DO UPDATE SET"
    actorconflict = "ON CONFLICT (actorid) DO UPDATE SET"
-    characterconflict = "ON CONFLICT (characterid, actorid) DO UPDATE SET"
+    characterconflict = "ON CONFLICT (seriesid, characterid) DO UPDATE SET"
    for document in data:
        if document['name'].lower() == 'too many requests':
            print("Skipping document due to 'Too Many Requests' in "
@@ -63,8 +63,14 @@ def process_data(data, tablename):
                column_list = []
                data_list = []
                for key, value in flatten_data.items():
+                    if isinstance(value, str):
+                        value = value.replace('"', "'").replace("'", "''")
                    if key.lower() == '_links.self.href':
                        key = 'apilink'
+                    if key.lower() == '_links.show.href':
+                        key = 'apishowlink'
+                    if key.lower() == '_links.show.name':
+                        key = 'apishowname'
                    if '.' in key.lower():
                        key = key.replace('.', '')
                    if tablename == 'seriesdata' and (
@@ -73,6 +79,8 @@ def process_data(data, tablename):
                        key = f"series_{key}"
                    if tablename == 'seriesdata' and key.lower() == 'id':
                        key = "seriesid"
+                    if tablename == 'crewdata' and key.lower() == 'id':
+                        key = "crewid"
                    if (
                        tablename == 'actordata' and (
                            key.lower() == 'name' or key.lower() == 'number' or
@@ -89,8 +97,12 @@ def process_data(data, tablename):
                    if tablename == 'characterdata' and key.lower() == 'id':
                        key = "seriesid, characterid"
                    if (
-                        (tablename == 'epdata' or tablename == 'actordata') and (
-                            key.lower() == 'name' or key.lower() == 'number' or
+                        (
+                            tablename == 'epdata' or
+                            tablename == 'actordata'
+                        ) and (
+                            key.lower() == 'name' or
+                            key.lower() == 'number' or
                            key.lower() == 'type'
                        )
                    ):
@@ -99,41 +111,19 @@ def process_data(data, tablename):
                        key = "language_name"
                    if tablename == 'epdata' and key.lower() == 'id':
                        key = "episodeid"
-                    if (
-                        '_links' in key.lower() or
-                        'image' in key.lower() or
-                        'schedule' in key.lower() or
-                        'externals' in key.lower() or
-                        'network' in key.lower() or
-                        'webchannel' in key.lower() or
-                        'country' in key.lower() or
-                        'rating' in key.lower()
-                    ):
-                        key = key.replace('.', '_')
                    if tablename == 'actordata' and 'seriesid' in key.lower():
                        continue
                    else:
                        column_list.append(key)
-                        if 'genres' in key.lower() or 'schedule' in key.lower():
+                        if (
+                            'genres' in key.lower() or
+                            'schedule' in key.lower()
+                        ):
                            if isinstance(value, list):
                                value = ', '.join(value)
-                            else:
-                                value = str(value).replace("'", "''")
-                        if isinstance(value, str):
-                            value = value.replace("'", "''")
-                        if isinstance(value, int):
-                            value = f"'{value}'"
-                        elif isinstance(value, float):
-                            value = f"'{json.dumps(value)}'"
-                        elif isinstance(value, dict):
-                            value = f"'{json.dumps(value)}'"
-                        elif isinstance(value, bool):
-                            value = str(value).lower()
-                        elif value is None:
-                            value = 'NULL'
-                        else:
-                            value = str(f"'{value}'")
-                        data_list.append(value)
+                        if isinstance(value, int) or isinstance(value, float):
+                            value = str(json.dumps(value))
+                        data_list.append(f"'{value}'")
                columns = ", ".join(column_list)
                column_data = ", ".join(data_list)
                conflict_list = []
@@ -141,7 +131,7 @@ def process_data(data, tablename):
                    conflict_list.append(
                        f"{column_list[indexer]} = {data_list[indexer]}"
                    )
-                conflict_data = ', '.join(conflict_list)
+                conflict_data = ', '.join(conflict_list).replace("\\", "")
                conflict_clause = ''
                if tablename == 'seriesdata': 
                    conflict_clause = seriesconflict
@@ -160,7 +150,6 @@ def process_data(data, tablename):
                        f"{conflict_clause} {conflict_data};"
                    )
                )
-                
                inserts.append(insert_command)
    return inserts, failed_count

@@ -216,35 +205,35 @@ else:

 print("Creating Insert commands from collection data.")

-# print("Collecting series data for series inserts.")
-# series_list = mgseries.find({})
-# inserts, failed_count = process_data(series_list, 'seriesdata')
-# print(f"Total number of series inserts: {len(inserts)}")
-# for i in tqdm(range(len(inserts))):
-#     try:
-#         insert = inserts[i]
-#         pgcursor.execute(insert)
-#     except Exception:
-#         # print(f"Unable to insert data into seriesdata table. {e}")
-#         failed_count += 1
-# print(f"Total number of failed series inserts: {failed_count}")
-# pgclient.commit()
+print("Collecting series data for series inserts.")
+series_list = mgseries.find({})
+inserts, failed_count = process_data(series_list, 'seriesdata')
+print(f"Total number of series inserts: {len(inserts)}")
+for i in tqdm(range(len(inserts))):
+    try:
+        insert = inserts[i]
+        pgcursor.execute(insert)
+    except Exception as e:
+        print(f"Unable to insert data into seriesdata table. {e}")
+        failed_count += 1
+print(f"Total number of failed series inserts: {failed_count}")
+pgclient.commit()

-# epinserts = []
-# failures = 0
-# episode_array = mgepisodes.find({})
-# print("Collecting episode data for episode inserts.")
-# inserts, failed_count = process_data(episode_array, 'epdata')
-# print(f"Total number of episode inserts: {len(inserts)}")
-# for i in tqdm(range(len(inserts))):
-#     try:
-#         insert = inserts[i]
-#         pgcursor.execute(insert)
-#     except Exception:
-#         # print(f"Unable to insert data into epdata table. {e}")
-#         failed_count += 1
-# print(f"Total number of failed episode inserts: {failed_count}")
-# pgclient.commit()
+epinserts = []
+failures = 0
+episode_array = mgepisodes.find({})
+print("Collecting episode data for episode inserts.")
+inserts, failed_count = process_data(episode_array, 'epdata')
+print(f"Total number of episode inserts: {len(inserts)}")
+for i in tqdm(range(len(inserts))):
+    try:
+        insert = inserts[i]
+        pgcursor.execute(insert)
+    except Exception as e:
+        print(f"Unable to insert data into epdata table. {e}")
+        failed_count += 1
+print(f"Total number of failed episode inserts: {failed_count}")
+pgclient.commit()

 failures = 0
 actor_array = mgactors.find({})
@@ -261,20 +250,20 @@ for i in tqdm(range(len(inserts))):
 print(f"Total number of failed actor inserts: {failed_count}")
 pgclient.commit()

-# failures = 0
-# character_array = mgcharacters.find({})
-# print("Collecting actor data for actor inserts.")
-# inserts, failed_count = process_data(character_array, 'characterdata')
-# print(f"Total number of character inserts: {len(inserts)}")
-# for i in tqdm(range(len(inserts))):
-#     try:
-#         insert = inserts[i]
-#         pgcursor.execute(insert)
-#     except Exception as e:
-#         print(f"Unable to insert data into character table. {e}")
-#         failed_count += 1
-# print(f"Total number of failed character inserts: {failed_count}")
-# pgclient.commit()
+failures = 0
+character_array = mgcharacters.find({})
+print("Collecting actor data for actor inserts.")
+inserts, failed_count = process_data(character_array, 'characterdata')
+print(f"Total number of character inserts: {len(inserts)}")
+for i in tqdm(range(len(inserts))):
+    try:
+        insert = inserts[i]
+        pgcursor.execute(insert)
+    except Exception as e:
+        print(f"Unable to insert data into character table. {e}")
+        failed_count += 1
+print(f"Total number of failed character inserts: {failed_count}")
+pgclient.commit()

 failures = 0
 crew_array = mgcrew.find({})
@@ -8,3 +8,4 @@ Requests==2.32.3
 SQLAlchemy==2.0.39
 pymysql
 tqdm
+pandas