312 lines
13 KiB
Python
312 lines
13 KiB
Python
import os
|
|
from flask import Blueprint, render_template, render_template_string, redirect, request, session
|
|
from flask_wtf import FlaskForm
|
|
from wtforms.fields import DateTimeField, SubmitField, StringField, SelectMultipleField
|
|
from datetime import datetime
|
|
from recovery.lib.common.connecter import Connecter
|
|
from recovery.lib.function.Outputter import Output
|
|
from recovery.lib.function.Writer import csvwrite
|
|
import pandas as pd
|
|
|
|
class MyForm(FlaskForm):
|
|
startdate = DateTimeField(id='startdate',format='%Y-%m-%d %H:%M:%S')
|
|
enddate = DateTimeField(id='enddate', format='%Y-%m-%d %H:%M:%S')
|
|
submit = SubmitField("Submit")
|
|
|
|
class SchemaForm(FlaskForm):
|
|
source = StringField("source")
|
|
target = StringField("target")
|
|
submit = SubmitField("Submit")
|
|
|
|
class TableForm(FlaskForm):
|
|
startdate = DateTimeField("Start Date", format='%Y-%m-%d %H:%M:%S')
|
|
enddate = DateTimeField("End Date", format='%Y-%m-%d %H:%M:%S')
|
|
source = StringField("source")
|
|
target = StringField("target")
|
|
tables = SelectMultipleField("tables")
|
|
submit = SubmitField("Submit")
|
|
|
|
class SubmitForm(FlaskForm):
|
|
startdate = DateTimeField("Start Date", format='%Y-%m-%d %H:%M:%S')
|
|
enddate = DateTimeField("End Date", format='%Y-%m-%d %H:%M:%S')
|
|
source = StringField("source")
|
|
target = StringField("target")
|
|
tables = SelectMultipleField("tables")
|
|
submit = SubmitField("Submit")
|
|
|
|
recover = Blueprint('recover', __name__, template_folder='templates')
|
|
|
|
@recover.route('/daterange', methods=["GET", "POST"])
|
|
def daterange():
|
|
session.clear()
|
|
form = MyForm()
|
|
if request.method == "POST":
|
|
tempdate = datetime.strptime(request.form['startdate'], '%m/%d/%Y %I:%M %p')
|
|
timestamp = datetime.timestamp(tempdate)
|
|
startdate = datetime.fromtimestamp(timestamp)#.strftime('%Y-%m-%d %H:%M:%S')
|
|
session["startdate"] = startdate
|
|
tempdate = datetime.strptime(request.form['enddate'], '%m/%d/%Y %I:%M %p')
|
|
timestamp = datetime.timestamp(tempdate)
|
|
print(timestamp)
|
|
enddate = datetime.fromtimestamp(timestamp)#.strftime('%Y-%m-%d %H:%M:%S')
|
|
print(enddate)
|
|
session["enddate"] = enddate
|
|
return redirect("schemas")
|
|
return render_template('recover/daterange.html', form=form)
|
|
|
|
@recover.route('/schemas', methods=["GET", "POST"])
|
|
def schemas():
|
|
form = SchemaForm()
|
|
connector = Connecter()
|
|
getschemas = "SELECT dbname,schemaname,dbuser,dbpassword,sfaccount,sfwarehouse FROM public.targets"
|
|
(pgconn,pgcur) = connector.open_pgsql('public')
|
|
pgcur.execute(getschemas)
|
|
schemarows = pgcur.fetchall()
|
|
if request.method == "POST":
|
|
sourcefield = request.form['source'].split(',')
|
|
session["source"] = sourcefield[0]
|
|
session["sourceuser"] = sourcefield[1]
|
|
session["sourcepassword"] = sourcefield[2]
|
|
session["sourceaccount"] = sourcefield[3]
|
|
session["sourcewarehouse"] = sourcefield[4]
|
|
targetfield = request.form['target'].split(',')
|
|
session["target"] = targetfield[0]
|
|
session["targetuser"] = targetfield[1]
|
|
session["targetpassword"] = targetfield[2]
|
|
session["targetaccount"] = targetfield[3]
|
|
session["targetwarehouse"] = targetfield[4]
|
|
return redirect("tables")
|
|
return render_template('recover/schemas.html', form=form, schemarows=schemarows)
|
|
|
|
|
|
@recover.route('/tables', methods=["GET", "POST"])
|
|
def tables():
|
|
form = TableForm()
|
|
connector = Connecter()
|
|
dbfield = session["source"].split('.')
|
|
session['sourcedb'] = dbfield[0]
|
|
session['sourceschema'] = dbfield[1]
|
|
gettables = f"select distinct(table_name) from information_schema.columns where table_schema = '{dbfield[1]}' order by table_name;"
|
|
session["sffrominfo"] = [
|
|
session["sourceaccount"],
|
|
session["sourceuser"],
|
|
session["sourcepassword"],
|
|
session["sourcewarehouse"],
|
|
session['sourcedb'],
|
|
session['sourceschema'],
|
|
]
|
|
(database, schema, conn) = connector.connect(session["sffrominfo"])
|
|
cur = conn.cursor()
|
|
cur.execute(gettables)
|
|
cur.get_results_from_sfqid(cur.sfqid)
|
|
results = cur.fetchall()
|
|
tablecount = len(results)
|
|
if request.method == "POST":
|
|
session['tablelist'] = request.form.getlist('tables')
|
|
return redirect("verify")
|
|
return render_template('recover/tables.html', form=form, results=results, tablecount=tablecount)
|
|
|
|
@recover.route('/verify')
|
|
def verify():
|
|
form = SubmitForm()
|
|
if request.method == "POST":
|
|
return redirect('sfexec')
|
|
return render_template('recover/submit.html', form=form)
|
|
|
|
@recover.route('/sfexec')
|
|
def sfexec():
|
|
failed_tables = []
|
|
discrepancylist = []
|
|
mismatchlist = []
|
|
minuscountlist = []
|
|
outputter = Output()
|
|
writer = csvwrite()
|
|
fromdata = './recovery/data/from_data.csv'
|
|
#fromdata = fromdatafile.valuename
|
|
todata = './recovery/data/to_data.csv'
|
|
#todata = todatafile.valuename
|
|
recovereddata = './recovery/data/recovered_data.csv'
|
|
#recovereddata = recovereddatafile.valuename
|
|
table_ddl = []
|
|
table_ddls = {}
|
|
connecter = Connecter()
|
|
dbfield = session["source"].split('.')
|
|
session["sffrominfo"] = [
|
|
session["sourceaccount"],
|
|
session["sourceuser"],
|
|
session["sourcepassword"],
|
|
session["sourcewarehouse"],
|
|
session['sourcedb'],
|
|
session['sourceschema'],
|
|
]
|
|
(fromdatabase, fromschema, fromconn) = connecter.connect(session["sffrominfo"])
|
|
fromcur = fromconn.cursor()
|
|
|
|
(pgconn, pgcur) = connecter.open_pgsql(session['sourceschema'])
|
|
|
|
dbfield = session["target"].split('.')
|
|
session['targetdb'] = dbfield[0]
|
|
session['targetschema'] = session['sourceschema']
|
|
session["sftoinfo"] = [
|
|
session["targetaccount"],
|
|
session["targetuser"],
|
|
session["targetpassword"],
|
|
session["targetwarehouse"],
|
|
session['targetdb'],
|
|
session['targetschema'],
|
|
]
|
|
(todatabase, toschema, toconn) = connecter.connect(session["sftoinfo"])
|
|
tocur = toconn.cursor()
|
|
|
|
for tablename in session["tablelist"]:
|
|
table_ddl.append(
|
|
[
|
|
tablename,
|
|
fromdatabase + "_" + tablename,
|
|
todatabase + "_" + tablename,
|
|
]
|
|
)
|
|
for tablename in session['tablelist']:
|
|
get_ddl = f"SELECT get_ddl('TABLE','{tablename}');"
|
|
fromcur.execute(get_ddl)
|
|
fromcur.get_results_from_sfqid(fromcur.sfqid)
|
|
results = fromcur.fetchall()
|
|
|
|
( fromtable,
|
|
totable,
|
|
minustable,
|
|
recovertable,
|
|
) = writer.create_tables(
|
|
pgconn, pgcur, fromconn, toconn, session['sourceschema'], tablename, results[0][0], session
|
|
)
|
|
session['fromtable'] = fromtable
|
|
session['totable'] = totable
|
|
session['minustable'] = minustable
|
|
session['recovertable'] = recovertable
|
|
|
|
columnfield = session['fromtable'].split('.')
|
|
tvar = session['exclude_list'].replace(',', "','")
|
|
excluded_columns = f"'{tvar}'"
|
|
get_columns = f"SELECT column_name FROM information_schema.columns WHERE table_schema = '{session['targetschema']}' AND table_name = '{columnfield[1].lower()}';"
|
|
get_datacolumns = f"SELECT column_name FROM information_schema.columns WHERE table_schema = '{session['targetschema']}' AND table_name = '{columnfield[1].lower()}' AND column_name NOT IN ({excluded_columns});"
|
|
|
|
pgcur.execute(get_columns)
|
|
columnlist = []
|
|
columnlistquery = pgcur.fetchall()
|
|
for row in columnlistquery:
|
|
columnlist.append(row[0])
|
|
|
|
session['all_columns'] = ",".join(columnlist)
|
|
pgcur.execute(get_datacolumns)
|
|
columnlist = []
|
|
columnlistquery = pgcur.fetchall()
|
|
for row in columnlistquery:
|
|
columnlist.append(row[0])
|
|
|
|
session['data_columns'] = ",".join(columnlist)
|
|
|
|
session['get_data'] = f" SELECT * FROM {tablename} WHERE ODS_EFF_ROW_DTM >= '{session['startdate']}' AND ODS_EFF_ROW_DTM <= '{session['enddate']}';"
|
|
fromcur.execute(session['get_data'])
|
|
fromcur.get_results_from_sfqid(fromcur.sfqid)
|
|
fromresults = fromcur.fetchall()
|
|
session['from_rows'] = len(fromresults)
|
|
if session['from_rows'] > 0:
|
|
valuelist = []
|
|
for column in fromresults[0]:
|
|
valuelist.append('%s')
|
|
valueslist = ','.join(valuelist)
|
|
fromargs = ','.join(pgcur.mogrify(f"({valueslist})", i).decode('utf-8')
|
|
for i in fromresults)
|
|
pgcur.execute(f"INSERT INTO {fromtable} ({session['all_columns']}) VALUES {fromargs}")
|
|
else:
|
|
print("0 Rows received.")
|
|
pgconn.commit()
|
|
|
|
tocur.execute(session['get_data'])
|
|
tocur.get_results_from_sfqid(tocur.sfqid)
|
|
toresults = tocur.fetchall()
|
|
session['to_rows'] = len(toresults)
|
|
toargs = ','.join(pgcur.mogrify(f"({valueslist})", i).decode('utf-8')
|
|
for i in toresults)
|
|
if len(toresults) > 0:
|
|
pgcur.execute(f"INSERT INTO {totable} ({session['all_columns']}) VALUES {toargs}")
|
|
else:
|
|
print("0 Rows received.")
|
|
pgconn.commit()
|
|
|
|
session['minusquery'] = f"SELECT {session['data_columns']} FROM {fromtable} EXCEPT SELECT {session['data_columns']} FROM {totable};"
|
|
pgcur.execute(session['minusquery'])
|
|
minusresults = pgcur.fetchall()
|
|
session['minus_rows'] = len(minusresults)
|
|
if session['minus_rows'] > 0:
|
|
valuelist = []
|
|
for column in minusresults[0]:
|
|
valuelist.append('%s')
|
|
valueslist = ','.join(valuelist)
|
|
minusargs = ','.join(pgcur.mogrify(f"({valueslist})", i).decode('utf-8')
|
|
for i in minusresults)
|
|
pgcur.execute(f"INSERT INTO {minustable} ({session['data_columns']}) VALUES {minusargs}")
|
|
pgconn.commit()
|
|
valuelist = []
|
|
for column in fromresults[0]:
|
|
valuelist.append('%s')
|
|
valueslist = ','.join(valuelist)
|
|
whereclausecolumns = session['data_columns'].split(',')
|
|
whereclauselist = []
|
|
for column_name in whereclausecolumns:
|
|
pgcur.execute(f"SELECT COUNT(*) FROM {minustable} WHERE {column_name} IS NULL;")
|
|
result = pgcur.fetchone()
|
|
if result[0] == 0:
|
|
whereclauselist.append(f"A.{column_name} = B.{column_name}")
|
|
recoverclause = " AND ".join(whereclauselist)
|
|
datacolumns = session['all_columns'].split(',')
|
|
selectcolumns = f"A.{',A.'.join(datacolumns)}"
|
|
recoveryquery = f"INSERT INTO {recovertable} ({session['all_columns']}) SELECT {selectcolumns} FROM {fromtable} A, {minustable} B WHERE {recoverclause};"
|
|
print(recoveryquery)
|
|
pgcur.execute(recoveryquery)
|
|
pgconn.commit()
|
|
#pgcur.execute(f'SELECT * from {recovertable}')
|
|
#recoverresults = pgcur.fetchall()
|
|
#session['recovered_rows'] = len(recoverresults)
|
|
#if session['recovered_rows'] > 0:
|
|
# resultdf = pd.DataFrame()
|
|
meta_columns = ['change_seq','change_oper','change_mask','stream_position','operation','transaction_id','timestamp']
|
|
selectstatement = f'SELECT * from {recovertable}'
|
|
recover_query = pd.read_sql_query(selectstatement, pgconn)
|
|
resultdf = pd.DataFrame(recover_query)
|
|
resultdf.to_csv('results.csv', index=False)
|
|
recoveredcsv = open(f'{tablename}.csv', 'w')
|
|
resultcsv = open('results.csv', 'r')
|
|
for index, row in enumerate(resultcsv):
|
|
line = row.replace('UNKNOWN', '')
|
|
linefield = line.split(',')
|
|
for idx, fieldname in enumerate(linefield):
|
|
if "memory at" in fieldname:
|
|
linefield[idx] = ''
|
|
if index == 0:
|
|
for index1, lfield in enumerate(linefield):
|
|
if lfield in meta_columns:
|
|
newfield = f'"{lfield}"'
|
|
linefield[index1] = newfield
|
|
line = ",".join(linefield)
|
|
elif index > 0:
|
|
linefield[2] = ''
|
|
line = ",".join(linefield)
|
|
recoveredcsv.write(f'{line}')
|
|
resultcsv.close()
|
|
recoveredcsv.close()
|
|
os.unlink('results.csv')
|
|
try:
|
|
print("Transferring file.......")
|
|
tocur.execute(f"PUT file://{tablename}.csv @%{tablename} AUTO_COMPRESS=TRUE OVERWRITE=TRUE;")
|
|
except Exception as e:
|
|
print(e)
|
|
else:
|
|
print('Copying data.......')
|
|
tocur.execute(f"copy into {tablename} file_format = (type = csv field_delimiter = ',' skip_header = 1 error_on_column_count_mismatch=false) ENFORCE_LENGTH = FALSE;");
|
|
else:
|
|
print('0 rows to process.')
|
|
pgconn.commit()
|
|
|
|
return render_template('recover/sfexec.html')
|