adding work scripts
This commit is contained in:
@@ -0,0 +1,40 @@
|
||||
"""
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
|
||||
|
||||
class Output:
|
||||
def __init__(self):
|
||||
self.scoutput = self
|
||||
|
||||
def process_table(
|
||||
self, directional, sfenv, tablename, startdate, use_daterange, enddate
|
||||
):
|
||||
if use_daterange == "0":
|
||||
print(
|
||||
f"Retreiving rows from {directional} table ({sfenv}) {tablename} from {startdate}."
|
||||
)
|
||||
else:
|
||||
print(
|
||||
f"Retreiving rows from {directional} table ({sfenv}) {tablename} from {startdate} through {enddate}."
|
||||
)
|
||||
|
||||
def display_rowcount(self, rowcount):
|
||||
|
||||
print(f"{rowcount} rows received.")
|
||||
|
||||
def output_status(self, message):
|
||||
print(f"{message}")
|
||||
|
||||
def __del__(self):
|
||||
pass
|
||||
@@ -0,0 +1,13 @@
|
||||
"""
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
@@ -0,0 +1,185 @@
|
||||
"""
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
import sqlite3
|
||||
import pandas as pd
|
||||
import csv
|
||||
import re
|
||||
import os
|
||||
import psycopg2
|
||||
|
||||
from sqlalchemy import over
|
||||
from recovery.lib.overrides.Overrider import overrider
|
||||
|
||||
|
||||
class csvwrite:
|
||||
def __init__(self):
|
||||
self.override = overrider()
|
||||
|
||||
def create_csv(self, csvfilename, csvcolumns, csvresults):
|
||||
csvcolumnlist = csvcolumns.upper().replace('"', "").split(",")
|
||||
with open(csvfilename, "w", newline="", encoding="utf8") as f:
|
||||
write = csv.writer(f, delimiter=",", quoting=csv.QUOTE_ALL)
|
||||
write.writerow(csvcolumnlist)
|
||||
write.writerows(csvresults)
|
||||
f.close()
|
||||
|
||||
def create_tables(
|
||||
self,
|
||||
utilconn,
|
||||
utilcur,
|
||||
fromconn,
|
||||
toconn,
|
||||
from_schema,
|
||||
tableinfo,
|
||||
getddl,
|
||||
session,
|
||||
):
|
||||
tablestatement = (
|
||||
getddl
|
||||
.replace("create or replace TABLE", "CREATE TABLE")
|
||||
.replace("BINARY(256)", "BYTEA")
|
||||
.replace("BINARY(16000)", "BYTEA")
|
||||
.replace("BINARY(7000)", "BYTEA")
|
||||
.replace("BINARY(7168000)", "BYTEA")
|
||||
.replace("NUMBER", "DECIMAL")
|
||||
.replace("TIMESTAMP_NTZ", "TIMESTAMP")
|
||||
.replace("'en-ci'", "pg_catalog.\"default\"")
|
||||
.replace("\"", "")
|
||||
)
|
||||
|
||||
sourcedb = session['sourcedb']
|
||||
targetdb = session['targetdb']
|
||||
|
||||
i = 0
|
||||
tableinfo = self.override.override_table(tableinfo, minustable_config=0)
|
||||
table_ddls = {}
|
||||
actualtable = tableinfo
|
||||
fromtable = f"\"{from_schema}\".{sourcedb}_{tableinfo}"
|
||||
totable = f"\"{from_schema}\".{targetdb}_{tableinfo}"
|
||||
minustable = f"\"{from_schema}\".{tableinfo}_MINUSTABLE"
|
||||
recovertable = f"\"{from_schema}\".RECOVERED_{tableinfo}"
|
||||
data_columns = []
|
||||
all_columns = []
|
||||
exclude_columns = []
|
||||
|
||||
utilcur.execute(f"SELECT * FROM public.excluded_columns ORDER BY id ASC ;")
|
||||
excluderesults = utilcur.fetchall()
|
||||
for excludecolumn in excluderesults:
|
||||
exclude_columns.append(excludecolumn[1])
|
||||
session['exclude_list'] = ",".join(exclude_columns)
|
||||
|
||||
utilcur.execute(f"DROP TABLE IF EXISTS {fromtable};")
|
||||
utilcur.execute(f"DROP TABLE IF EXISTS {totable};")
|
||||
# utilcur.execute(f"DROP TABLE IF EXISTS {minustable};")
|
||||
# utilcur.execute(f"DROP TABLE IF EXISTS {recovertable};")
|
||||
utilconn.commit()
|
||||
primary_key = ""
|
||||
match = re.search(
|
||||
r'primary key [_\(\)\w",\s]+', tablestatement, re.MULTILINE
|
||||
)
|
||||
|
||||
if match:
|
||||
result = match.group()
|
||||
primary_key = result.replace("primary key (", "").replace(")", "")
|
||||
else:
|
||||
primary_key = "NOT DEFINED"
|
||||
tempcreatetable = tablestatement.replace(f"CREATE TABLE {actualtable}",f"CREATE TABLE {fromtable}")
|
||||
createtable = re.sub(
|
||||
r'constraint\s[\w_]+\s', f"CONSTRAINT {sourcedb}_{tableinfo}_PKY " , tempcreatetable, flags=re.IGNORECASE
|
||||
)
|
||||
i = i + 1
|
||||
utilcur.execute(createtable)
|
||||
utilconn.commit()
|
||||
tempcreatetable = tablestatement.replace(f"CREATE TABLE {actualtable}",f"CREATE TABLE {totable}")
|
||||
createtable = re.sub(
|
||||
r'constraint\s[\w_]+\s', f"CONSTRAINT {targetdb}_{tableinfo}_PKY " , tempcreatetable, flags=re.IGNORECASE
|
||||
)
|
||||
i = i + 1
|
||||
utilcur.execute(createtable)
|
||||
utilconn.commit()
|
||||
tempcreatetable = tablestatement.replace(f"CREATE TABLE {actualtable}",f"CREATE TABLE {minustable}")
|
||||
createtable1 = re.sub(" COLLATE pg_catalog.default", "", tempcreatetable, flags=re.IGNORECASE)
|
||||
createtable2 = re.sub(r'constraint\s[\w_]+\s', "" , createtable1, flags=re.IGNORECASE)
|
||||
createtable3 = re.sub(r'NOT NULL', "" , createtable2, flags=re.IGNORECASE)
|
||||
createtable4 = re.sub(r'primary key\s[_\(\),\w\s]+', "" , createtable3, flags=re.IGNORECASE)
|
||||
createtable5 = re.sub("ODS_AUDIT_ID DECIMAL(18,0),","ODS_AUDIT_ID DECIMAL(18,0))", createtable4, flags=re.IGNORECASE)
|
||||
createtable6 = re.sub("LM_BESTTIMETOCONTACTCODE VARCHAR(40) COLLATE pg_catalog.default,", "LM_BESTTIMETOCONTACTCODE VARCHAR(40)", createtable5, flags=re.IGNORECASE).rstrip()
|
||||
createtable = re.sub(";", ");", createtable6, flags=re.IGNORECASE).strip()
|
||||
# try:
|
||||
# utilcur.execute(createtable)
|
||||
# except Exception as err:
|
||||
# createtable4 = createtable3.replace("ODS_AUDIT_ID DECIMAL(18,0))", "ODS_AUDIT_ID DECIMAL(18,0),")
|
||||
# createtable5 = createtable4.replace("ADMIN_X VARCHAR(1) COLLATE pg_catalog.default,", "ADMIN_X VARCHAR(1) COLLATE pg_catalog.default)")
|
||||
# createtable6 = createtable5.replace("LM_BESTTIMETOCONTACTCODE VARCHAR(40) COLLATE pg_catalog.default,", "LM_BESTTIMETOCONTACTCODE VARCHAR(40) COLLATE pg_catalog.default").rstrip()
|
||||
# createtable7 = createtable6.replace(" COLLATE pg_catalog.default", "")
|
||||
# createtable8 = re.sub(r'primary key\s[_\(\),\w\s]+', "" , createtable7, flags=re.IGNORECASE)
|
||||
# createtable = createtable8.replace(";", ");")
|
||||
# utilcur.execute(createtable)
|
||||
tempcreatetable = tablestatement.replace(f"CREATE TABLE {actualtable}",f"CREATE TABLE {recovertable}")
|
||||
createtable1 = re.sub(r'constraint\s[\w_]+\s', "" , tempcreatetable, flags=re.IGNORECASE)
|
||||
createtable2 = re.sub(r'NOT NULL', "" , createtable1, flags=re.IGNORECASE)
|
||||
createtable3 = re.sub(r'primary key\s[_\(\),\w\s]+', "" , createtable2, flags=re.IGNORECASE)
|
||||
createtable4 = createtable3.replace("ODS_AUDIT_ID DECIMAL(18,0),","ODS_AUDIT_ID DECIMAL(18,0))")
|
||||
createtable = createtable4.replace(";", ");")
|
||||
# utilcur.execute(createtable)
|
||||
utilconn.commit()
|
||||
|
||||
return (
|
||||
fromtable,
|
||||
totable,
|
||||
minustable,
|
||||
recovertable,
|
||||
)
|
||||
|
||||
def primary_whereclause(self, primary_key):
|
||||
whereclause = ""
|
||||
if "," in primary_key:
|
||||
keyfield = primary_key.split(",")
|
||||
whereclause = (
|
||||
"A."
|
||||
+ keyfield[0]
|
||||
+ " = B."
|
||||
+ keyfield[0]
|
||||
+ " AND A."
|
||||
+ keyfield[1]
|
||||
+ " = B."
|
||||
+ keyfield[1]
|
||||
)
|
||||
else:
|
||||
whereclause = "A." + primary_key + " = B." + primary_key
|
||||
return whereclause
|
||||
|
||||
def data_whereclause(self, data_columns):
|
||||
whereclauselist = []
|
||||
whereclause = ""
|
||||
for columnname in data_columns:
|
||||
whereclauselist.append("A." + columnname + " = B." + columnname)
|
||||
whereclause = ",".join(whereclauselist)
|
||||
return whereclause
|
||||
|
||||
def generate_impactedtables(
|
||||
self, impacted_tables, from_database, to_database, schema
|
||||
):
|
||||
table_ddl = []
|
||||
for tablename in impacted_tables:
|
||||
table_ddl.append(
|
||||
[
|
||||
tablename,
|
||||
from_database + "_" + tablename,
|
||||
to_database + "_" + tablename,
|
||||
]
|
||||
)
|
||||
|
||||
def __del__(self):
|
||||
pass
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,96 @@
|
||||
"""
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
|
||||
|
||||
class Settings(db.Model):
|
||||
id = db.Column(db.Integer(), primary_key=True)
|
||||
keyname = db.Column(db.String(128))
|
||||
valuename = db.Column(db.String(128))
|
||||
|
||||
def __init__(self, keyname, valuename):
|
||||
self.keyname = keyname
|
||||
self.valuename = valuename
|
||||
|
||||
def __repr__(self):
|
||||
return "<key '{}': value '{}'>".format(self.keyname, self.valuename)
|
||||
|
||||
|
||||
class Targets(db.Model):
|
||||
id = db.Column(db.Integer(), primary_key=True)
|
||||
targetname = db.Column(db.String(128))
|
||||
username = db.Column(db.String(128))
|
||||
password = db.Column(db.String(128))
|
||||
account = db.Column(db.String(128))
|
||||
warehouse = db.Column(db.String(128))
|
||||
database = db.Column(db.String(128))
|
||||
schemaname = db.Column(db.String(128))
|
||||
|
||||
def __init__(
|
||||
self, targetname, username, password, account, warehouse, database, schemaname
|
||||
):
|
||||
self.targetname = targetname
|
||||
self.username = username
|
||||
self.password = password
|
||||
self.account = account
|
||||
self.warehouse = warehouse
|
||||
self.database = database
|
||||
self.schemaname = schemaname
|
||||
|
||||
def __repr__(self):
|
||||
return "<target: '{}, {}, {}, {}, {}, {}, {}'>".format(
|
||||
self.targetname,
|
||||
self.username,
|
||||
self.password,
|
||||
self.account,
|
||||
self.warehouse,
|
||||
self.database,
|
||||
self.schemaname,
|
||||
)
|
||||
|
||||
|
||||
class Queries(db.Model):
|
||||
id = db.Column(db.Integer(), primary_key=True)
|
||||
keyname = db.Column(db.String(128))
|
||||
valuename = db.Column(db.String(128))
|
||||
|
||||
def __init__(self, keyname, valuename):
|
||||
self.keyname = keyname
|
||||
self.valuename = valuename
|
||||
|
||||
def __repr__(self):
|
||||
return "<key '{}': value '{}'>".format(self.keyname, self.valuename)
|
||||
|
||||
|
||||
class Sync_params(db.Model):
|
||||
id = db.Column(db.Integer(), primary_key=True)
|
||||
keyname = db.Column(db.String(128))
|
||||
valuename = db.Column(db.String(128))
|
||||
|
||||
def __init__(self, keyname, valuename):
|
||||
self.keyname = keyname
|
||||
self.valuename = valuename
|
||||
|
||||
def __repr__(self):
|
||||
return "<key '{}': value '{}'>".format(self.keyname, self.valuename)
|
||||
|
||||
|
||||
class Excluded_columns(db.Model):
|
||||
id = db.Column(db.Integer(), primary_key=True)
|
||||
columnname = db.Column(db.String(128))
|
||||
|
||||
def __init__(self, columnname):
|
||||
self.columnname = columnname
|
||||
|
||||
def __repr__(self):
|
||||
return "<columnname '{}'>".format(self.columnname)
|
||||
@@ -0,0 +1,190 @@
|
||||
import os
|
||||
from io import StringIO
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import psycopg2
|
||||
import psycopg2.extras as extras
|
||||
|
||||
|
||||
def execute_many(conn, df, table):
|
||||
"""
|
||||
Using cursor.executemany() to insert the dataframe
|
||||
"""
|
||||
# Create a list of tupples from the dataframe values
|
||||
tuples = [tuple(x) for x in df.to_numpy()]
|
||||
# Comma-separated dataframe columns
|
||||
cols = ','.join(list(df.columns))
|
||||
# SQL quert to execute
|
||||
query = "INSERT INTO %s(%s) VALUES(%%s,%%s,%%s)" % (table, cols)
|
||||
cursor = conn.cursor()
|
||||
try:
|
||||
cursor.executemany(query, tuples)
|
||||
conn.commit()
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
print("Error: %s" % error)
|
||||
conn.rollback()
|
||||
cursor.close()
|
||||
return 1
|
||||
print("execute_many() done")
|
||||
cursor.close()
|
||||
|
||||
|
||||
def execute_batch(conn, table, df, page_size):
|
||||
# csvdata = open(csvfile, 'r')
|
||||
# newcvsfile = csvfile.replace('data', 'data_filtered')
|
||||
# newcsvdata = open(newcvsfile, 'w')
|
||||
# for line in csvdata:
|
||||
# newline = line.replace("bytearray(b'\\xff\\xff\\xff\\x80')","").replace('"','')
|
||||
# newcsvdata.write(newline.replace('""',''))
|
||||
# csvdata.close()
|
||||
# newcsvdata.close()
|
||||
# df = pd.read_csv(newcvsfile, dtype=object, low_memory=False)
|
||||
"""
|
||||
Using psycopg2.extras.execute_batch() to insert the dataframe
|
||||
"""
|
||||
# Create a list of tupples from the dataframe values
|
||||
tuples = [tuple(x) for x in df.to_numpy()]
|
||||
# Comma-separated dataframe columns
|
||||
cols = ','.join(list(df.columns))
|
||||
data = ''
|
||||
for col in df.columns:
|
||||
data = data + ',%%s'
|
||||
rowdata = data[1:]
|
||||
dynamicquery = f"INSERT INTO %s(%s) VALUES({rowdata})"
|
||||
# SQL quert to execute
|
||||
query = dynamicquery % (table, cols)
|
||||
cursor = conn.cursor()
|
||||
try:
|
||||
extras.execute_batch(cursor, query, tuples, page_size)
|
||||
conn.commit()
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
print("Error: %s" % error)
|
||||
conn.rollback()
|
||||
cursor.close()
|
||||
return 1
|
||||
print("execute_batch() done")
|
||||
cursor.close()
|
||||
|
||||
|
||||
def execute_values(conn, df, table):
|
||||
"""
|
||||
Using psycopg2.extras.execute_values() to insert the dataframe
|
||||
"""
|
||||
# Create a list of tupples from the dataframe values
|
||||
tuples = [tuple(x) for x in df.to_numpy()]
|
||||
# Comma-separated dataframe columns
|
||||
cols = ','.join(list(df.columns))
|
||||
# SQL quert to execute
|
||||
query = "INSERT INTO %s(%s) VALUES %%s" % (table, cols)
|
||||
cursor = conn.cursor()
|
||||
try:
|
||||
extras.execute_values(cursor, query, tuples)
|
||||
conn.commit()
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
print("Error: %s" % error)
|
||||
conn.rollback()
|
||||
cursor.close()
|
||||
return 1
|
||||
print("execute_values() done")
|
||||
cursor.close()
|
||||
|
||||
|
||||
def execute_mogrify(conn, df, table):
|
||||
"""
|
||||
Using cursor.mogrify() to build the bulk insert query
|
||||
then cursor.execute() to execute the query
|
||||
"""
|
||||
# Create a list of tupples from the dataframe values
|
||||
tuples = [tuple(x) for x in df.to_numpy()]
|
||||
# Comma-separated dataframe columns
|
||||
cols = ','.join(list(df.columns))
|
||||
# SQL quert to execute
|
||||
cursor = conn.cursor()
|
||||
values = [cursor.mogrify("(%s,%s,%s)", tup).decode('utf8') for tup in tuples]
|
||||
query = "INSERT INTO %s(%s) VALUES " % (table, cols) + ",".join(values)
|
||||
|
||||
try:
|
||||
cursor.execute(query, tuples)
|
||||
conn.commit()
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
print("Error: %s" % error)
|
||||
conn.rollback()
|
||||
cursor.close()
|
||||
return 1
|
||||
print("execute_mogrify() done")
|
||||
cursor.close()
|
||||
|
||||
|
||||
def copy_from_file(conn, df, table):
|
||||
"""
|
||||
Here we are going save the dataframe on disk as
|
||||
a csv file, load the csv file
|
||||
and use copy_from() to copy it to the table
|
||||
"""
|
||||
# Save the dataframe to disk
|
||||
tmp_df = "./tmp_dataframe.csv"
|
||||
df.to_csv(tmp_df, index_label='id', header=False)
|
||||
f = open(tmp_df, 'r')
|
||||
cursor = conn.cursor()
|
||||
try:
|
||||
cursor.copy_from(f, table, sep=",")
|
||||
conn.commit()
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
os.remove(tmp_df)
|
||||
print("Error: %s" % error)
|
||||
conn.rollback()
|
||||
cursor.close()
|
||||
return 1
|
||||
print("copy_from_file() done")
|
||||
cursor.close()
|
||||
os.remove(tmp_df)
|
||||
|
||||
|
||||
def copy_from_stringio(conn, df, table):
|
||||
"""
|
||||
Here we are going save the dataframe in memory
|
||||
and use copy_from() to copy it to the table
|
||||
"""
|
||||
# save dataframe to an in memory buffer
|
||||
buffer = StringIO()
|
||||
df.to_csv(buffer, index_label='id', header=False)
|
||||
buffer.seek(0)
|
||||
|
||||
cursor = conn.cursor()
|
||||
try:
|
||||
cursor.copy_from(buffer, table, sep=",")
|
||||
conn.commit()
|
||||
except (Exception, psycopg2.DatabaseError) as error:
|
||||
print("Error: %s" % error)
|
||||
conn.rollback()
|
||||
cursor.close()
|
||||
return 1
|
||||
print("copy_from_stringio() done")
|
||||
cursor.close()
|
||||
|
||||
|
||||
#----------------------------------------------------------------
|
||||
# SqlAlchemy Only
|
||||
#----------------------------------------------------------------
|
||||
# from sqlalchemy import create_engine
|
||||
|
||||
# connect = "postgresql+psycopg2://%s:%s@%s:5432/%s" % (
|
||||
# param_dic['user'],
|
||||
# param_dic['password'],
|
||||
# param_dic['host'],
|
||||
# param_dic['database']
|
||||
# )
|
||||
|
||||
# def to_alchemy(df):
|
||||
# """
|
||||
# Using a dummy table to test this call library
|
||||
# """
|
||||
# engine = create_engine(connect)
|
||||
# df.to_sql(
|
||||
# 'test_table',
|
||||
# con=engine,
|
||||
# index=False,
|
||||
# if_exists='replace'
|
||||
# )
|
||||
# print("to_sql() done (sqlalchemy)")
|
||||
Reference in New Issue
Block a user