adding work scripts
This commit is contained in:
@@ -0,0 +1,162 @@
|
||||
#!/usr/bin/python3
|
||||
"""
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
|
||||
from config.Reader import reader
|
||||
from sfconnect.Connecter import connecter
|
||||
from sql.Queries import sfquery
|
||||
from function.Writer import csvwrite
|
||||
import os, datetime, socket
|
||||
import base64
|
||||
import smtplib
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
|
||||
|
||||
def emailsupport(messagetype, messagefile):
|
||||
"""Send to Email"""
|
||||
me = options["sender"]
|
||||
# you = ['HTTP_509@LibertyMutual.com', 'PMEDW_Prod_Support@LibertyMutual.com']
|
||||
you = options["receivers"].split()
|
||||
|
||||
# Open a plain text file for reading. For this example, assume that
|
||||
# the text file contains only ASCII characters.
|
||||
message_part = open(messagefile, "r")
|
||||
part2 = MIMEText(message_part.read(), "html")
|
||||
# Create a text/plain message
|
||||
message_part.close()
|
||||
|
||||
msg = MIMEMultipart("alternative")
|
||||
|
||||
msg["Subject"] = messagetype
|
||||
|
||||
msg["From"] = me
|
||||
msg["To"] = ",".join(you)
|
||||
|
||||
msg.attach(part2)
|
||||
|
||||
# Send the message via our own SMTP server, but don't include the envelope header.
|
||||
sendmail(me, you, msg.as_string())
|
||||
|
||||
|
||||
# Set header automation statement for email
|
||||
def openemail(emailfile):
|
||||
emailfile.write(
|
||||
"<p>THIS IS AN AUTOMATED NOTIFICATION. PLEASE DO NOT RESPOND TO THIS E-MAIL.</p>"
|
||||
)
|
||||
emailfile.write("<html>\n")
|
||||
emailfile.write("<table border='1'>\n")
|
||||
|
||||
|
||||
# Set footer automation statement for email
|
||||
def closeemail(emailfile):
|
||||
emailfile.write("</table>\n")
|
||||
emailfile.write(
|
||||
"<p>THIS IS AN AUTOMATED NOTIFICATION. PLEASE DO NOT RESPOND TO THIS E-MAIL.</p>"
|
||||
)
|
||||
emailfile.write("\t</body>")
|
||||
emailfile.write("</html>\n")
|
||||
|
||||
|
||||
def sendmail(me, you, mailmsg):
|
||||
# Send the message via our own SMTP server, but don't include the envelope header.
|
||||
try:
|
||||
s = smtplib.SMTP(options["mailhost"])
|
||||
s.sendmail(me, you, mailmsg)
|
||||
s.quit()
|
||||
except smtplib.SMTPException:
|
||||
print(
|
||||
"Email was drafted but I was unable to send it. Unable to connect to mailserver"
|
||||
)
|
||||
return 1
|
||||
else:
|
||||
return 0
|
||||
|
||||
|
||||
def get_options():
|
||||
creader = reader()
|
||||
options = creader.retrieve("./settings/sync_settings.ini")
|
||||
return options
|
||||
|
||||
|
||||
def sfconnect(sfparams):
|
||||
connect = connecter()
|
||||
conn = connect.connect(sfparams)
|
||||
return conn
|
||||
|
||||
|
||||
def get_sql(options):
|
||||
sfretrieve = sfquery()
|
||||
queries = sfretrieve.retrieve(options)
|
||||
return queries
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
options = get_options()
|
||||
sfqueries = get_sql(options)
|
||||
nowstamp = datetime.datetime.now()
|
||||
script_name = os.path.basename(__file__)
|
||||
|
||||
for login in options["logins"]:
|
||||
(database, schema, conn) = sfconnect(options["logins"][login])
|
||||
cur = conn.cursor()
|
||||
checkquery = options["check_suspended"]
|
||||
cur.execute(checkquery)
|
||||
|
||||
rows = 0
|
||||
|
||||
cur.get_results_from_sfqid(cur.sfqid)
|
||||
results = cur.fetchall()
|
||||
conn.close()
|
||||
if len(results) == 0:
|
||||
print("No suspended tables found.")
|
||||
else:
|
||||
timestamp = nowstamp.strftime("%m/%d/%Y, %H:%M:%S")
|
||||
html_report = open("email_template.html", "r")
|
||||
suspended_list = []
|
||||
suspended = open("suspended.html", "w")
|
||||
openemail(suspended)
|
||||
for line in html_report:
|
||||
suspended.write(line.replace("DateNow", timestamp))
|
||||
suspended.write(
|
||||
' <table class="basic_lines" summary="Table Listing">\n'
|
||||
)
|
||||
suspended.write(
|
||||
f'<tr><th colspan="8">Qlik Task(s) found with suspended tables:</th></tr>\n'
|
||||
)
|
||||
suspended.write(
|
||||
f'\t\t\t\t<tr><th class="odd_row_data">ENVIRONMENT</th><th class="odd_row_data">SERVER NAME</th><th class="odd_row_data" colspan="2">TASK NAME</th><th class="odd_row_data">OWNER</th><th class="odd_row_data">TABLE NAME</th><th class="odd_row_data">SUSPEND REASON</th><th class="odd_row_data">SUSPEND TIMESTAMP</th></tr>\n'
|
||||
)
|
||||
print(f"{len(results)} suspended tables detected.")
|
||||
for idx, sfobject in enumerate(results):
|
||||
numcheck = idx + 2
|
||||
if numcheck % 2 == 0:
|
||||
suspended.write(
|
||||
f'\t\t\t\t<tr><td class="even_row_data">Environment</td><td class="even_row_data">{results[idx][0]}</td><td class="even_row_data block" colspan="2">{results[idx][1]}</td><td class="even_row_data">{results[idx][2]}</td><td class="even_row_data">{results[idx][3]}</td><td class="even_row_data">{results[idx][4]}</td><td class="even_row_data block" >{results[idx][5]}</td></tr>\n'
|
||||
)
|
||||
else:
|
||||
suspended.write(
|
||||
f'\t\t\t\t<tr><td class="odd_row_data">Environment</td><td class="odd_row_data">{results[idx][0]}</td><td class="odd_row_data block" colspan="2">{results[idx][1]}</td><td class="odd_row_data">{results[idx][2]}</td><td class="odd_row_data">{results[idx][3]}</td><td class="odd_row_data">{results[idx][4]}</td><td class="odd_row_data block">{results[idx][5]}</td></tr>\n'
|
||||
)
|
||||
suspended.write(" </table>\n")
|
||||
closeemail(suspended)
|
||||
suspended.close()
|
||||
emailsupport(
|
||||
"SUSPENDED TABLES DETECTED IN QLIK REPLICATE", "suspended.html"
|
||||
)
|
||||
nowstamp = datetime.datetime.now()
|
||||
os.remove("suspended.html")
|
||||
print(f"Endtime: {nowstamp}")
|
||||
|
||||
exit()
|
||||
@@ -0,0 +1,53 @@
|
||||
"""
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
|
||||
import configparser
|
||||
from base64 import b64decode
|
||||
from encodings import utf_8
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class reader:
|
||||
def __init__(self):
|
||||
self.CONFIG = configparser.RawConfigParser(allow_no_value=True)
|
||||
self.options = []
|
||||
self.parameters = {}
|
||||
|
||||
def retrieve(self, filename):
|
||||
self.CONFIG.read(filename)
|
||||
self.parameters["check_suspended"] = self.CONFIG.get(
|
||||
"query_template", "check_suspended"
|
||||
)
|
||||
self.parameters["sender"] = self.CONFIG.get("global", "sender")
|
||||
self.parameters["receivers"] = self.CONFIG.get("global", "receivers")
|
||||
self.parameters["mailhost"] = self.CONFIG.get("global", "mailhost")
|
||||
self.sf_logins = {}
|
||||
for option, value in self.CONFIG.items("sf_targets"):
|
||||
loginname = option
|
||||
itemlist = value.split(",")
|
||||
pw = bytes(itemlist[2], "UTF8")
|
||||
npw = b64decode(pw).decode("UTF8")
|
||||
self.sf_logins[loginname] = (
|
||||
loginname,
|
||||
[itemlist[0], itemlist[1], npw, itemlist[3], itemlist[4], itemlist[5]],
|
||||
itemlist[6],
|
||||
)
|
||||
self.parameters["logins"] = self.sf_logins
|
||||
now = datetime.now()
|
||||
currenttime = now.strftime("%m%d%Y_%H%M%S")
|
||||
self.parameters["currenttime"] = currenttime
|
||||
return self.parameters
|
||||
|
||||
def __del__(self):
|
||||
pass
|
||||
@@ -0,0 +1,116 @@
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="en">
|
||||
|
||||
<head>
|
||||
<title>Email Template</title>
|
||||
<style type="text/css">
|
||||
table.basic_lines {
|
||||
height: 300px;
|
||||
width: 900px;
|
||||
border: 3px solid #000;
|
||||
}
|
||||
|
||||
.header {
|
||||
height: 40px;
|
||||
color: white;
|
||||
background: #022664;
|
||||
text-align: center;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
#logo {
|
||||
float: right;
|
||||
margin-top: -45px;
|
||||
}
|
||||
|
||||
.row_label {
|
||||
height: 45px;
|
||||
}
|
||||
|
||||
.odd_row_data {
|
||||
background: #B6DCFC;
|
||||
font-size: 12px;
|
||||
font-weight: bold;
|
||||
padding-left: 2%;
|
||||
}
|
||||
|
||||
.even_row_data {
|
||||
background: #80C3FA;
|
||||
font-size: 12px;
|
||||
font-weight: bold;
|
||||
padding-left: 2%;
|
||||
}
|
||||
|
||||
.col_label {
|
||||
width: 175px;
|
||||
color: white;
|
||||
font-weight: bold;
|
||||
background: #2967AE;
|
||||
padding-left: 2%;
|
||||
}
|
||||
|
||||
#contact_info {
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
#footer {
|
||||
height: 35px;
|
||||
text-align: center;
|
||||
font: bold;
|
||||
font-size: 10px;
|
||||
color: white;
|
||||
background: #022664;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<table class="basic_lines">
|
||||
<caption>Technology Alert</caption>
|
||||
<th>
|
||||
<tr>
|
||||
<td class="header" colspan="2">GRM US - Information Management Technology Alert<br />Qlik Replicate Performance Alert</td>
|
||||
</tr>
|
||||
<tr class="row_label">
|
||||
<td class="col_label">Date</td>
|
||||
<td class="odd_row_data">DateNow</td>
|
||||
</tr>
|
||||
<tr class="row_label">
|
||||
<td class="col_label">Affected Application(s)</td>
|
||||
<td class="even_row_data">Attunity Qlik Replicate</td>
|
||||
</tr>
|
||||
<tr class="row_label">
|
||||
<td class="col_label">Event</td>
|
||||
<td class="odd_row_data">Suspended tables found for the following objects below:</td>
|
||||
</tr>
|
||||
<tr class="row_label">
|
||||
<td class="col_label">Impact</td>
|
||||
<td class="even_row_data">Affected objects listed below are currently showing as having suspended tables.
|
||||
</td>
|
||||
</tr>
|
||||
<tr class="row_label">
|
||||
<td class="col_label">Action Required</td>
|
||||
<td class="odd_row_data">This is an informational notification and no action is required by the users.</td>
|
||||
</tr>
|
||||
<tr class="row_label">
|
||||
<td class="col_label">Contact & Additional Information</td>
|
||||
<td class="even_row_data">Support teams have been notified of the issue and are working to address
|
||||
it.<br />If you have any questions, please email: PM EDW Prod Support</td>
|
||||
</tr>
|
||||
<tr class="row_label">
|
||||
<td id="footer" colspan="2">Please do not reply to this message. All replies to this message are routed
|
||||
to an unmonitored mailbox.</td>
|
||||
</tr>
|
||||
<tr class="row_label">
|
||||
<td id="footer" colspan="2"><a href="https://libertymutual.sharepoint.com/:x:/r/sites/InformationManagement/_layouts/15/Doc.aspx?sourcedoc=%7BCDD784EF-E954-41AC-9A1E-3E0AC1BB095B%7D&file=Data%20Recovery%20Master%20Runbook.xlsx&wdOrigin=TEAMS-ELECTRON.p2p.mw&action=default&mobileredirect=true">Support Link: Data Recovery Master Runbook</a></td>
|
||||
</tr>
|
||||
<tr class="row_label">
|
||||
<td id="footer" colspan="2"><a href="https://libertymutual.sharepoint.com/:w:/r/sites/InformationManagement/_layouts/15/Doc.aspx?sourcedoc=%7BCF7C233A-A02C-4511-9C21-1F7DD62E658E%7D&file=TRANS%20layer%20Data-recovery%20Runbook.docx&wdOrigin=TEAMS-ELECTRON.p2p.mw&action=default&mobileredirect=true">Support Link: TRANS Layer Data Recovery Runbook</a></td>
|
||||
</tr>
|
||||
</th>
|
||||
|
||||
</table>
|
||||
<!-- Affected Tables -->
|
||||
</body>
|
||||
|
||||
</html>
|
||||
@@ -0,0 +1,76 @@
|
||||
[global]
|
||||
mailhost = smtprelay.lmig.com
|
||||
sender = Qlik_Admin@LibertyMutual.com
|
||||
#receivers = HTTP_509@LibertyMutual.com, PMEDW_Prod_Support@LibertyMutual.com
|
||||
receivers = Wendell.Jones@LibertyMutual.com
|
||||
cleanup_tempdata=0
|
||||
# valid options for utility database are "snowflake" or "sqlite"
|
||||
utilitydb=sqlite
|
||||
|
||||
[log_table]
|
||||
DEV=PL_DEV.PM_EDW_META_D.TRANS_RECOVER_LOG
|
||||
QA=PL_QA.PM_EDW_META_D.TRANS_RECOVER_FROM_LOG
|
||||
PROD=PL_PROD.PM_EDW_META_D.TRANS_RECOVER_FROM_LOG
|
||||
|
||||
#Need all logins added here with passwords encrypted
|
||||
[sf_targets]
|
||||
PL_DEV_QLIK_PM_EDW_PSA_D=PL_DEV,PM_EDW_PSA_D,UWtDZzczNkg=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_EDW_PSA_D,PM_EDW_PSA_D_BASE
|
||||
; (Disabled due to permissions issue) PL_DEV_QLIK_PM_EDW_TRANS_D=PL_DEV,PM_EDW_TRANS_NRT_SD_QLIK_CT_D,TU5uSHE2WVA=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_EDW_TRANS_NRT_SD_QLIK_CT_D,PM_EDW_TRANS_D_BASE
|
||||
PL_DEV_QLIK_PM_EDW_TRANS_NRT_SD_D=PL_DEV,PM_EDW_TRANS_QLIK_CT_D,UXBBaFd5Nno=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_EDW_TRANS_QLIK_CT_D,PM_EDW_TRANS_NRT_SD_D_BASE
|
||||
PL_DEV_QLIK_PM_ODS_FIN_FINX_D=PL_DEV,PM_ODS_FIN_FINX_QLIK_D,VTZuWTdBR3k=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_ODS_FIN_FINX_QLIK_D,PM_ODS_FIN_FINX_D_BASE
|
||||
PL_DEV_QLIK_PM_ODS_VISION_D=PL_DEV,PM_ODS_VISION_QLIK_CT_D,NUgzWGFuVE0=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_ODS_VISION_QLIK_CT_D,PM_ODS_VISION_D_BASE
|
||||
PL_DEV_QLIK_PM_ODS_VISION_TRANS_D=PL_DEV,PM_ODS_VISION_TRANS_QLIK_CT_D,ejY1S3RIZk0=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_ODS_VISION_TRANS_QLIK_CT_D,PM_ODS_VISION_TRANS_D_BASE
|
||||
; (Disabled due to no base tables) PL_DEV_QLIK_PM_ODS_XTRNL_INCMNG_D=PL_DEV,PM_ODS_XTRNL_INCMING_D,UGJaZ3FVeDU=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_ODS_XTRNL_INCMING_D,PM_ODS_XTRNL_INCMNG_D_BASE
|
||||
; (Disabled due to no base tables) PL_DEV_QLIK_PM_TRANS_AVAYA_CTIEMAIL_D=PL_DEV,PM_TRANS_AVAYA_CTIEMAIL_QLIK_CT_D,WjgzSm83QzQ=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_TRANS_AVAYA_CTIEMAIL_QLIK_CT_D,PM_TRANS_AVAYA_CTIEMAIL_D_BASE
|
||||
PL_DEV_QLIK_PM_TRANS_AVAYA_REPO_D=PL_DEV,PM_TRANS_AVAYA_REPO_QLIK_CT_D,WjgzSm83QzQ=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_TRANS_AVAYA_REPO_QLIK_CT_D,PM_TRANS_AVAYA_REPO_D_BASE
|
||||
PL_DEV_QLIK_PM_TRANS_CL_AB_D=PL_DEV,PM_TRANS_CL_AB_ALT_QLIK_CT_D,aDdBSE9Vclc=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_TRANS_CL_AB_ALT_QLIK_CT_D,PM_TRANS_CL_AB_D_BASE
|
||||
PL_DEV_QLIK_PM_TRANS_CL_CC_PRL_D=PL_DEV,PM_TRANS_CL_CC_PRL_QLIK_CT_D,S2o5OEVmWEo=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_TRANS_CL_CC_PRL_QLIK_CT_D,PM_TRANS_CL_CC_PRL_D_BASE
|
||||
PL_DEV_QLIK_PM_TRANS_CL_EPALARM_D=PL_DEV,PM_TRANS_CL_EPALARM_QLIK_CT_D,dFdRNWdXeUM=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_TRANS_CL_EPALARM_QLIK_CT_D,PM_TRANS_CL_EPALARM_D_BASE
|
||||
PL_DEV_QLIK_PM_TRANS_CL_EPCUST_D=PL_DEV,PM_TRANS_CL_EPCUST_QLIK_CT_D,RXJRUjlZS0o=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_TRANS_CL_EPCUST_QLIK_CT_D,PM_TRANS_CL_EPCUST_D_BASE
|
||||
PL_DEV_QLIK_PM_TRANS_CL_ICREPO_D=PL_DEV,PM_TRANS_CL_ICREPO_QLIK_CT_D,cDJTWldwQXE=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_TRANS_CL_ICREPO_QLIK_CT_D,PM_TRANS_CL_ICREPO_D_BASE
|
||||
PL_DEV_QLIK_PM_TRANS_CL_OAHIST_D=PL_DEV,PM_TRANS_CL_OAHIST_QLIK_CT_D,NlJ5eFhBNXE=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_TRANS_CL_OAHIST_QLIK_CT_D,PM_TRANS_CL_OAHIST_D_BASE
|
||||
PL_DEV_QLIK_PM_TRANS_CMS_D=PL_DEV,PM_TRANS_CMS_QLIK_CT_D,ZDdkODc0V1I=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_TRANS_CMS_QLIK_CT_D,PM_TRANS_CMS_D_BASE
|
||||
; (Disabled due to no base tables) PL_DEV_QLIK_PM_TRANS_CTI_D=PL_DEV,PM_TRANS_CTI_QLIK_CT_D,Um5IOEVLRmU=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_TRANS_CTI_QLIK_CT_D,PM_TRANS_CTI_D_BASE
|
||||
PL_DEV_QLIK_PM_TRANS_SALES_COMP_D=PL_DEV,PM_TRANS_SALES_COMP_QLIK_CT_D,S3U4djJFUWI=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_TRANS_SALES_COMP_QLIK_CT_D,PM_TRANS_SALES_COMP_D_BASE
|
||||
PL_DEV_QLIK_PM_TRANS_VH_D=PL_DEV,PM_TRANS_VH_QLIK_CT_D,Y0JVSHltOWU=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_TRANS_VH_QLIK_CT_D,PM_TRANS_VH_D_BASE
|
||||
PL_QA_QLIK_PM_EDW_PSA_D=PL_QA,PL_QA_QLIK_PM_EDW_PSA_D,cjlMUXRQRE4=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_EDW_PSA_D,PM_EDW_PSA_D_BASE
|
||||
PL_QA_QLIK_PM_EDW_TRANS_D=PL_QA,PL_QA_QLIK_PM_EDW_TRANS_D,T01JN3RDYko=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_EDW_TRANS_NRT_SD_QLIK_CT_D,PM_EDW_TRANS_D_BASE
|
||||
PL_QA_QLIK_PM_EDW_TRANS_NRT_SD_D=PL_QA,PL_QA_QLIK_PM_EDW_TRANS_NRT_SD_D,bTc5QjdTRmk=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_EDW_TRANS_QLIK_CT_D,PM_EDW_TRANS_NRT_SD_D_BASE
|
||||
PL_QA_QLIK_PM_ODS_FIN_FINX_D=PL_QA,PL_QA_QLIK_PM_ODS_FIN_FINX_D,dXZhZE04OXk=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_ODS_FIN_FINX_QLIK_D,PM_ODS_FIN_FINX_D_BASE
|
||||
PL_QA_QLIK_PM_ODS_VISION_D=PL_QA,PL_QA_QLIK_PM_ODS_VISION_D,c3FoNVZ4R1Y=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_ODS_VISION_QLIK_CT_D,PM_ODS_VISION_D_BASE
|
||||
PL_QA_QLIK_PM_ODS_VISION_TRANS_D=PL_QA,PL_QA_QLIK_PM_ODS_VISION_TRANS_D,dndBSjQwSks=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_ODS_VISION_TRANS_QLIK_CT_D,PM_ODS_VISION_TRANS_D_BASE
|
||||
; (Disabled due to no base tables) PL_QA_QLIK_PM_ODS_XTRNL_INCMNG_D=PL_QA,PL_QA_QLIK_PM_ODS_XTRNL_INCMNG_D,Z0NSWGp0NVE=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_ODS_XTRNL_INCMING_D,PM_ODS_XTRNL_INCMNG_D_BASE
|
||||
; (Disabled due to no base tables) PL_QA_QLIK_PM_TRANS_AVAYA_CTIEMAIL_D=PL_QA,PL_QA_QLIK_PM_TRANS_AVAYA_CTIEMAIL_D,VUcxZVJ4a00=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_TRANS_AVAYA_CTIEMAIL_QLIK_CT_D,PM_TRANS_AVAYA_CTIEMAIL_D_BASE
|
||||
PL_QA_QLIK_PM_TRANS_AVAYA_REPO_D=PL_QA,PL_QA_QLIK_PM_TRANS_AVAYA_REPO_D,WVI2N2VlVWQ=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_TRANS_AVAYA_REPO_QLIK_CT_D,PM_TRANS_AVAYA_REPO_D_BASE
|
||||
PL_QA_QLIK_PM_TRANS_CL_AB_D=PL_QA,PL_QA_QLIK_PM_TRANS_CL_AB_D,aXI2WFFjYUQ=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_TRANS_CL_AB_ALT_QLIK_CT_D,PM_TRANS_CL_AB_D_BASE
|
||||
PL_QA_QLIK_PM_TRANS_CL_CC_PRL_D=PL_QA,PL_QA_QLIK_PM_TRANS_CL_CC_PRL_D,WlE1alpGeXU=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_TRANS_CL_CC_PRL_QLIK_CT_D,PM_TRANS_CL_CC_PRL_D_BASE
|
||||
PL_QA_QLIK_PM_TRANS_CL_EPALARM_D=PL_QA,PL_QA_QLIK_PM_TRANS_CL_EPALARM_D,YjZTbzFpSVE=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_TRANS_CL_EPALARM_QLIK_CT_D,PM_TRANS_CL_EPALARM_D_BASE
|
||||
PL_QA_QLIK_PM_TRANS_CL_EPCUST_D=PL_QA,PL_QA_QLIK_PM_TRANS_CL_EPCUST_D,VDZQU0hEeko=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_TRANS_CL_EPCUST_QLIK_CT_D,PM_TRANS_CL_EPCUST_D_BASE
|
||||
PL_QA_QLIK_PM_TRANS_CL_ICREPO_D=PL_QA,PL_QA_QLIK_PM_TRANS_CL_ICREPO_D,STZUNnFkMVA=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_TRANS_CL_ICREPO_QLIK_CT_D,PM_TRANS_CL_ICREPO_D_BASE
|
||||
PL_QA_QLIK_PM_TRANS_CL_OAHIST_D=PL_QA,PL_QA_QLIK_PM_TRANS_CL_OAHIST_D,QW1Id0oxQTM=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_TRANS_CL_OAHIST_QLIK_CT_D,PM_TRANS_CL_OAHIST_D_BASE
|
||||
PL_QA_QLIK_PM_TRANS_CMS_D=PL_QA,PL_QA_QLIK_PM_TRANS_CMS_D,bHFUenNMM1A=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_TRANS_CMS_QLIK_CT_D,PM_TRANS_CMS_D_BASE
|
||||
; (Disabled due to no base tables) PL_QA_QLIK_PM_TRANS_CTI_D=PL_QA,PL_QA_QLIK_PM_TRANS_CTI_D,SE5jSVNJSTE=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_TRANS_CTI_QLIK_CT_D,PM_TRANS_CTI_D_BASE
|
||||
PL_QA_QLIK_PM_TRANS_SALES_COMP_D=PL_QA,PL_QA_QLIK_PM_TRANS_SALES_COMP_D,aER1cDRraVA=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_TRANS_SALES_COMP_QLIK_CT_D,PM_TRANS_SALES_COMP_D_BASE
|
||||
PL_QA_QLIK_PM_TRANS_VH_D=PL_QA,PL_QA_QLIK_PM_TRANS_VH_D,S0hxakxhOUQ=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_TRANS_VH_QLIK_CT_D,PM_TRANS_VH_D_BASE
|
||||
PL_PROD_QLIK_PM_EDW_PSA_D=PL_PROD,PL_PROD_QLIK_PM_EDW_PSA_D,QVE0eEFkaEI=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_EDW_PSA_D,PM_EDW_PSA_D_BASE
|
||||
PL_PROD_QLIK_PM_EDW_TRANS_D=PL_PROD,PL_PROD_QLIK_PM_EDW_TRANS_D,bUR4MjV5Ulk=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_EDW_TRANS_NRT_SD_QLIK_CT_D,PM_EDW_TRANS_D_BASE
|
||||
PL_PROD_QLIK_PM_EDW_TRANS_NRT_SD_D=PL_PROD,PL_PROD_QLIK_PM_EDW_TRANS_NRT_SD_D,b1EzZFdTVHg=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_EDW_TRANS_QLIK_CT_D,PM_EDW_TRANS_NRT_SD_D_BASE
|
||||
PL_PROD_QLIK_PM_ODS_FIN_FINX_D=PL_PROD,PL_PROD_QLIK_PM_ODS_FIN_FINX_D,aERweDNiaVg=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_ODS_FIN_FINX_QLIK_D,PM_ODS_FIN_FINX_D_BASE
|
||||
PL_PROD_QLIK_PM_ODS_VISION_D=PL_PROD,PL_PROD_QLIK_PM_ODS_VISION_D,U0tsOWpzVEU=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_ODS_VISION_QLIK_CT_D,PM_ODS_VISION_D_BASE
|
||||
PL_PROD_QLIK_PM_ODS_VISION_TRANS_D=PL_PROD,PL_PROD_QLIK_PM_ODS_VISION_TRANS_D,aTZmMVB4TU8=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_ODS_VISION_TRANS_QLIK_CT_D,PM_ODS_VISION_TRANS_D_BASE
|
||||
; (Disabled due to no base tables) PL_PROD_QLIK_PM_ODS_XTRNL_INCMNG_D=PL_PROD,PL_PROD_QLIK_PM_ODS_XTRNL_INCMNG_D,eDh4RFlPQTc=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_ODS_XTRNL_INCMING_D,PM_ODS_XTRNL_INCMNG_D_BASE
|
||||
; (Disabled due to no base tables) PL_PROD_QLIK_PM_TRANS_AVAYA_CTIEMAIL_D=PL_PROD,PL_PROD_QLIK_PM_TRANS_AVAYA_CTIEMAIL_D,cnJacWxMTjE=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_TRANS_AVAYA_CTIEMAIL_QLIK_CT_D,PM_TRANS_AVAYA_CTIEMAIL_D_BASE
|
||||
PL_PROD_QLIK_PM_TRANS_AVAYA_REPO_D=PL_PROD,PL_PROD_QLIK_PM_TRANS_AVAYA_REPO_D,bjdQQWtvcm4=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_TRANS_AVAYA_REPO_QLIK_CT_D,PM_TRANS_AVAYA_REPO_D_BASE
|
||||
PL_PROD_QLIK_PM_TRANS_CL_AB_D=PL_PROD,PL_PROD_QLIK_PM_TRANS_CL_AB_D,bHc5RmRSTTI=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_TRANS_CL_AB_ALT_QLIK_CT_D,PM_TRANS_CL_AB_D_BASE
|
||||
PL_PROD_QLIK_PM_TRANS_CL_CC_PRL_D=PL_PROD,PL_PROD_QLIK_PM_TRANS_CL_CC_PRL_D,YmdDYTRNWE8=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_TRANS_CL_CC_PRL_QLIK_CT_D,PM_TRANS_CL_CC_PRL_D_BASE
|
||||
PL_PROD_QLIK_PM_TRANS_CL_EPALARM_D=PL_PROD,PL_PROD_QLIK_PM_TRANS_CL_EPALARM_D,cjlDbm12Yjg=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_TRANS_CL_EPALARM_QLIK_CT_D,PM_TRANS_CL_EPALARM_D_BASE
|
||||
PL_PROD_QLIK_PM_TRANS_CL_EPCUST_D=PL_PROD,PL_PROD_QLIK_PM_TRANS_CL_EPCUST_D,WjVqTDlySHU=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_TRANS_CL_EPCUST_QLIK_CT_D,PM_TRANS_CL_EPCUST_D_BASE
|
||||
PL_PROD_QLIK_PM_TRANS_CL_ICREPO_D=PL_PROD,PL_PROD_QLIK_PM_TRANS_CL_ICREPO_D,TTB6VTNVcGQ=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_TRANS_CL_ICREPO_QLIK_CT_D,PM_TRANS_CL_ICREPO_D_BASE
|
||||
PL_PROD_QLIK_PM_TRANS_CL_OAHIST_D=PL_PROD,PL_PROD_QLIK_PM_TRANS_CL_OAHIST_D,Tko5UlZRelc=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_TRANS_CL_OAHIST_QLIK_CT_D,PM_TRANS_CL_OAHIST_D_BASE
|
||||
PL_PROD_QLIK_PM_TRANS_CMS_D=PL_PROD,PL_PROD_QLIK_PM_TRANS_CMS_D,Wjd2OE9EaVc=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_TRANS_CMS_QLIK_CT_D,PM_TRANS_CMS_D_BASE
|
||||
; (Disabled due to no base tables) PL_PROD_QLIK_PM_TRANS_CTI_D=PL_PROD,PL_PROD_QLIK_PM_TRANS_CTI_D,RlFsU3N2QzY=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_TRANS_CTI_QLIK_CT_D,PM_TRANS_CTI_D_BASE
|
||||
PL_PROD_QLIK_PM_TRANS_SALES_COMP_D=PL_PROD,PL_PROD_QLIK_PM_TRANS_SALES_COMP_D,NnFNYjNydjA=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_TRANS_SALES_COMP_QLIK_CT_D,PM_TRANS_SALES_COMP_D_BASE
|
||||
PL_PROD_QLIK_PM_TRANS_VH_D=PL_PROD,PL_PROD_QLIK_PM_TRANS_VH_D,bzQ3YWVaaEU=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_TRANS_VH_QLIK_CT_D,PM_TRANS_VH_D_BASE
|
||||
|
||||
[query_template]
|
||||
check_suspended = SELECT "server_name", "task_name", "table_owner", "table_name", "suspend_reason", "suspend_timestamp" FROM "attrep_suspended_tables";
|
||||
@@ -0,0 +1,53 @@
|
||||
"""
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
|
||||
from snowflake import connector
|
||||
import sqlite3
|
||||
|
||||
|
||||
class connecter(object):
|
||||
def __init__(self):
|
||||
self.cobject = []
|
||||
|
||||
def connect(self, sfparams):
|
||||
user = sfparams[0].rstrip()
|
||||
password = sfparams[1][2]
|
||||
account = sfparams[1][3]
|
||||
warehouse = sfparams[1][4]
|
||||
database = sfparams[1][0]
|
||||
schema = sfparams[2]
|
||||
try:
|
||||
conn = connector.connect(
|
||||
user=user,
|
||||
password=password,
|
||||
account=account,
|
||||
warehouse=warehouse,
|
||||
database=database,
|
||||
schema=schema,
|
||||
)
|
||||
conn._paramstyle = "qmark"
|
||||
except Exception as e:
|
||||
print(f"Unable to connect to schema: {database}.{schema}:")
|
||||
print(str(e))
|
||||
else:
|
||||
print(f"Connected to {database}.{schema} in warehouse: {warehouse}.")
|
||||
return (database, schema, conn)
|
||||
|
||||
def open_sqlite(self, dbfile):
|
||||
utilconn = sqlite3.connect(dbfile)
|
||||
utilcur = utilconn.cursor()
|
||||
return (utilconn, utilcur)
|
||||
|
||||
def __del__(self):
|
||||
pass
|
||||
@@ -0,0 +1,85 @@
|
||||
"""
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
import configparser
|
||||
from base64 import b64decode
|
||||
from encodings import utf_8
|
||||
|
||||
|
||||
class sfquery(object):
|
||||
def __init__(self):
|
||||
self.cobject = []
|
||||
|
||||
def retrieve(self, options):
|
||||
queries = {}
|
||||
queries["check_suspended"] = options["check_suspended"]
|
||||
return queries
|
||||
|
||||
def generate_query(
|
||||
options,
|
||||
schema,
|
||||
tablename,
|
||||
start_date,
|
||||
end_date,
|
||||
use_daterange,
|
||||
database_name,
|
||||
columnlist,
|
||||
):
|
||||
if use_daterange == "0":
|
||||
query = (
|
||||
options["get_data"]
|
||||
.replace("r_impacted_tables.SCHEM", schema)
|
||||
.replace("r_impacted_tables.TBL", tablename)
|
||||
.replace("startdate", start_date)
|
||||
.replace("sfenv", database_name)
|
||||
.replace("v_column_list", columnlist)
|
||||
)
|
||||
else:
|
||||
query = (
|
||||
options["get_daterange"]
|
||||
.replace("r_impacted_tables.SCHEM", schema)
|
||||
.replace("r_impacted_tables.TBL", tablename)
|
||||
.replace("startdate", start_date)
|
||||
.replace("enddate", end_date)
|
||||
.replace("sfenv", database_name)
|
||||
.replace("v_column_list", columnlist)
|
||||
)
|
||||
return query
|
||||
|
||||
def generate_minusquery(object, minustable, data_columns, fromtable, totable):
|
||||
query = (
|
||||
object.replace("minustable", minustable)
|
||||
.replace("v_column_list", ",".join(data_columns))
|
||||
.replace(
|
||||
"fromsfenv.r_impacted_tables.SCHEM.r_impacted_tables.TBL", fromtable
|
||||
)
|
||||
.replace("tosfenv.r_impacted_tables.SCHEM.r_impacted_tables.TBL", totable)
|
||||
)
|
||||
return query
|
||||
|
||||
def generate_recoveryquery(
|
||||
object, recovertable, columnlist, fromtable, minustable, whereclause
|
||||
):
|
||||
query = (
|
||||
object.replace("recoverytable", recovertable)
|
||||
.replace("v_column_list", columnlist)
|
||||
.replace(
|
||||
"fromsfenv.r_impacted_tables.SCHEM.r_impacted_tables.TBL", fromtable
|
||||
)
|
||||
.replace("minustable", minustable)
|
||||
.replace("columnlist", whereclause)
|
||||
)
|
||||
return query
|
||||
|
||||
def __del__(self):
|
||||
pass
|
||||
Reference in New Issue
Block a user