adding work scripts
This commit is contained in:
Binary file not shown.
@@ -0,0 +1,107 @@
|
||||
<!DOCTYPE HTML>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<title>Email Template</title>
|
||||
<style type="text/css">
|
||||
table.basic_lines {
|
||||
height:300px;
|
||||
width:900px;
|
||||
border: 3px solid #000;
|
||||
}
|
||||
|
||||
.header {
|
||||
height: 40px;
|
||||
color: white;
|
||||
background: #022664;
|
||||
text-align: center;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
#logo {
|
||||
float: right;
|
||||
margin-top: -45px;
|
||||
}
|
||||
|
||||
.row_label {
|
||||
height: 45px;
|
||||
}
|
||||
|
||||
.odd_row_data {
|
||||
background: #B6DCFC;
|
||||
font-size: 12px;
|
||||
font-weight: bold;
|
||||
padding-left: 2%;
|
||||
}
|
||||
|
||||
a {
|
||||
color: white;
|
||||
font-size: 12px;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.col_label {
|
||||
width: 175px;
|
||||
color: white;
|
||||
font-weight: bold;
|
||||
background: #2967AE;
|
||||
padding-left: 2%;
|
||||
}
|
||||
|
||||
#contact_info {
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
#footer {
|
||||
height: 35px;
|
||||
text-align: center;
|
||||
font: bold;
|
||||
font-size: 10px;
|
||||
color: white;
|
||||
background: #022664;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<table class="basic_lines">
|
||||
<caption>Qlik Replicate Performance Alert</caption>
|
||||
<th>
|
||||
<tr>
|
||||
<td class="header" colspan="2">GRM US - Information Management Technology Alert<br/>Qlik Replicate Performance Alert</td>
|
||||
</tr>
|
||||
<tr class="row_label">
|
||||
<td class="col_label">Date</td>
|
||||
<td class = "odd_row_data">DateNow</td>
|
||||
</tr>
|
||||
<tr class="row_label">
|
||||
<td class="col_label">Affected Application(s)</td>
|
||||
<td class = "even_row_data">Attunity Qlik Replicate</td>
|
||||
</tr>
|
||||
<tr class="row_label">
|
||||
<td class="col_label">Event</td>
|
||||
<td class = "odd_row_data">Latency Performance Issue for the following objects below:</td>
|
||||
</tr>
|
||||
<tr class="row_label">
|
||||
<td class="col_label">Impact</td>
|
||||
<td class = "even_row_data">Affected objects listed below are currently showing high latency.</td>
|
||||
</tr>
|
||||
<tr class="row_label">
|
||||
<td class="col_label">Action Required</td>
|
||||
<td class = "odd_row_data">This is an informational notification and no action is required by the users.</td>
|
||||
</tr>
|
||||
<tr class="row_label">
|
||||
<td class="col_label">Contact & Additional Information</td>
|
||||
<td class = "even_row_data">Support teams have been notified of the issue and are working to address it.<br/>If you have any questions, please email: PM EDW Prod Support</td>
|
||||
</tr>
|
||||
<tr class="row_label">
|
||||
<td id="footer" colspan="2">Please do not reply to this message. All replies to this message are routed to an unmonitored mailbox.</td>
|
||||
</tr>
|
||||
<tr class="row_label">
|
||||
<td id="footer" colspan="2"><a href="https://libertymutual.sharepoint.com/:x:/r/sites/InformationManagement/_layouts/15/Doc.aspx?sourcedoc=%7BCDD784EF-E954-41AC-9A1E-3E0AC1BB095B%7D&file=Data%20Recovery%20Master%20Runbook.xlsx&wdOrigin=TEAMS-ELECTRON.p2p.mw&action=default&mobileredirect=true">Support Link: Data Recovery Master Runbook</a></td>
|
||||
</tr>
|
||||
<tr class="row_label">
|
||||
<td id="footer" colspan="2"><a href="https://libertymutual.sharepoint.com/:w:/r/sites/InformationManagement/_layouts/15/Doc.aspx?sourcedoc=%7BCF7C233A-A02C-4511-9C21-1F7DD62E658E%7D&file=TRANS%20layer%20Data-recovery%20Runbook.docx&wdOrigin=TEAMS-ELECTRON.p2p.mw&action=default&mobileredirect=true">Support Link: TRANS Layer Data Recovery Runbook</a></td>
|
||||
</tr>
|
||||
</th>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,12 @@
|
||||
[global]
|
||||
mailhost = smtprelay.lmig.com
|
||||
sender = Qlik_Admin@LibertyMutual.com
|
||||
receivers = HTTP_509@LibertyMutual.com, PMEDW_Prod_Support@LibertyMutual.com, QLIK_ADMIN@LibertyMutual.com
|
||||
#receivers = Wendell.Jones@LibertyMutual.com
|
||||
minutes = 180
|
||||
query = SELECT T.EXT_NAME, T.REP_HOST, T.REP_NAME, T.SOURCE_DTM FROM EDW_GOLDENGATE_LATENCY_TABLE T inner join (SELECT REP_NAME, MAX(SOURCE_DTM) AS MAX_SOURCE_DTM FROM EDW_GOLDENGATE_LATENCY_TABLE GROUP BY REP_NAME) A on t.REP_NAME = a.REP_NAME and t.SOURCE_DTM = a.MAX_SOURCE_DTM ORDER BY REP_NAME
|
||||
|
||||
[sf_targets]
|
||||
#0=DEV,PL_DEV_QLIK_PM_EDW_TRANS_D,TU5uSHE2WVA=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PL_DEV,PM_EDW_META_D
|
||||
2=QA,PL_QA_QLIK_PM_EDW_TRANS_D,T01JN3RDYko=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PL_QA,PM_EDW_META_D
|
||||
3=PROD,PL_PROD_QLIK_PM_EDW_TRANS_D,bUR4MjV5Ulk=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PL_PROD,PM_EDW_META_D
|
||||
@@ -0,0 +1,176 @@
|
||||
"""
|
||||
Copyright 2018 Attunity
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
|
||||
# coding: utf-8
|
||||
import os, datetime, socket, sys, fileinput
|
||||
from snowflake import connector
|
||||
import base64
|
||||
import progressbar
|
||||
import smtplib
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
import configparser
|
||||
|
||||
def emailsupport(messagetype, messagefile):
|
||||
"""Send to Email"""
|
||||
me = GGENV = CONFIG.get('global', 'sender')
|
||||
#you = ['HTTP_509@LibertyMutual.com', 'PMEDW_Prod_Support@LibertyMutual.com']
|
||||
you = CONFIG.get('global', 'receivers').split()
|
||||
|
||||
# Open a plain text file for reading. For this example, assume that
|
||||
# the text file contains only ASCII characters.
|
||||
message_part = open(messagefile, 'r')
|
||||
part2 = MIMEText(message_part.read(), 'html')
|
||||
# Create a text/plain message
|
||||
message_part.close()
|
||||
|
||||
msg = MIMEMultipart('alternative')
|
||||
|
||||
msg['Subject'] = messagetype
|
||||
|
||||
msg['From'] = me
|
||||
msg['To'] = ",".join(you)
|
||||
|
||||
msg.attach(part2)
|
||||
|
||||
# Send the message via our own SMTP server, but don't include the envelope header.
|
||||
sendmail(me, you, msg.as_string())
|
||||
|
||||
# Set header automation statement for email
|
||||
def openemail(emailfile):
|
||||
emailfile.write("<p>THIS IS AN AUTOMATED NOTIFICATION. PLEASE DO NOT RESPOND TO THIS E-MAIL.</p>")
|
||||
emailfile.write("<html>\n")
|
||||
emailfile.write("<table border='1'>\n")
|
||||
|
||||
# Set footer automation statement for email
|
||||
def closeemail(emailfile):
|
||||
emailfile.write("</table>\n")
|
||||
emailfile.write("<p>THIS IS AN AUTOMATED NOTIFICATION. PLEASE DO NOT RESPOND TO THIS E-MAIL.</p>")
|
||||
emailfile.write("\t</body>")
|
||||
emailfile.write("</html>\n")
|
||||
|
||||
def sendmail(me, you, mailmsg):
|
||||
# Send the message via our own SMTP server, but don't include the envelope header.
|
||||
try:
|
||||
s = smtplib.SMTP(CONFIG.get('global', 'mailhost'))
|
||||
s.sendmail(me, you, mailmsg)
|
||||
s.quit()
|
||||
except smtplib.SMTPException:
|
||||
print('Email was drafted but I was unable to send it. Unable to connect to mailserver')
|
||||
return 1
|
||||
else:
|
||||
return 0
|
||||
|
||||
CONFIG = configparser.RawConfigParser(allow_no_value=True)
|
||||
CONFIG.read('settings.ini')
|
||||
sf_logins = []
|
||||
for option, value in CONFIG.items('sf_targets'):
|
||||
itemlist = value.split(',')
|
||||
pw = bytes(itemlist[2], 'UTF8')
|
||||
npw = base64.b64decode(pw).decode('UTF8')
|
||||
sf_logins.append([itemlist[0],itemlist[1],npw,itemlist[3],itemlist[4],itemlist[5],itemlist[6]])
|
||||
|
||||
nowstamp = datetime.datetime.now()
|
||||
script_name = os.path.basename(__file__)
|
||||
object_list = []
|
||||
latency_detected = False
|
||||
latency_minutes = CONFIG.get('global', 'minutes')
|
||||
latency_seconds = int(latency_minutes) * 60
|
||||
sfquery = CONFIG.get('global', 'query')
|
||||
sfdict = {}
|
||||
|
||||
print(f'Scanning for objects with latency more than {latency_minutes} minute(s) ({latency_seconds} seconds).')
|
||||
print(f'Starttime: {nowstamp}')
|
||||
updaterange = len(sf_logins)
|
||||
with progressbar.ProgressBar(maxval=updaterange, redirect_stdout=True) as p:
|
||||
for i in range(updaterange):
|
||||
login_name=sf_logins[i][1]
|
||||
login_password=sf_logins[i][2]
|
||||
sf_account=sf_logins[i][3]
|
||||
sf_warehouse=sf_logins[i][4]
|
||||
sf_database=sf_logins[i][5]
|
||||
sf_schema=sf_logins[i][6]
|
||||
counter = 0
|
||||
try:
|
||||
conn = connector.connect(
|
||||
user=login_name,
|
||||
password=str(login_password),
|
||||
account=sf_account,
|
||||
database=sf_database,
|
||||
schema=sf_schema,
|
||||
)
|
||||
conn._paramstyle='qmark'
|
||||
except Exception as e:
|
||||
print('Unable to connect to Snowflake')
|
||||
print(str(e))
|
||||
|
||||
|
||||
#print(f'Connection to Snowflake established as {login_name}.')
|
||||
|
||||
rootdir = os.getcwd()
|
||||
|
||||
server_name = socket.gethostname()
|
||||
|
||||
current_time = datetime.datetime.now()
|
||||
cur = conn.cursor()
|
||||
cur.execute(sfquery)
|
||||
#print(cur.sfqid)
|
||||
cur.get_results_from_sfqid(cur.sfqid)
|
||||
results = cur.fetchall()
|
||||
#print(f'result retrieval successful.')
|
||||
for result in results:
|
||||
#print(result)
|
||||
time_difference = current_time - result[3]
|
||||
if time_difference.total_seconds() > int(latency_seconds):
|
||||
if result[2] == '$AR_V_TASK_NAME':
|
||||
pass
|
||||
elif result not in object_list:
|
||||
latency_detected = True
|
||||
object_list.append(result)
|
||||
sfdict[counter] = {'ENV': sf_logins[i][0], 'EXT_NAME': result[0], 'REP_HOST': result[1], 'REP_NAME': result[2], 'SRC_DTM': result[3]}
|
||||
counter += 1
|
||||
conn.close()
|
||||
p.update(i)
|
||||
|
||||
if len(sfdict) > 0:
|
||||
timestamp = nowstamp.strftime("%m/%d/%Y, %H:%M:%S")
|
||||
html_report = open('email_template.html', 'r')
|
||||
latency_list = []
|
||||
latency = open('latency.html', 'w')
|
||||
openemail(latency)
|
||||
for line in html_report:
|
||||
latency.write(line.replace('DateNow', timestamp))
|
||||
latency.write(' <table class="basic_lines" summary="Table Listing">\n')
|
||||
latency.write(f'<tr><th colspan="7">Objects with more than {int(latency_minutes)} minute(s) of latency:</th></tr>\n')
|
||||
latency.write(f'\t\t\t\t<tr><th class="odd_row_data">ENVIRONMENT</th><th class="odd_row_data">TARGET DATABASE</th><th class="odd_row_data" colspan="2">REPLICATE HOST</th><th class="odd_row_data">TASK OBJECT</th><th class="odd_row_data">LAST CANARY UPDATE</th><th class="odd_row_data">LATENCY (MINUTES)</th></tr>\n')
|
||||
print(f'{len(sfdict)} objects detected with latency.')
|
||||
for idx, sfobject in enumerate(sfdict):
|
||||
time_difference = nowstamp - sfdict[idx]["SRC_DTM"]
|
||||
time_in_minutes = round(int(time_difference.total_seconds() / 60))
|
||||
numcheck = idx + 2
|
||||
if numcheck % 2 == 0:
|
||||
latency.write(f'\t\t\t\t<tr><td class="even_row_data">{sfdict[idx]["ENV"]}</td><td class="even_row_data">{sfdict[idx]["EXT_NAME"]}</td><td class="even_row_data block" colspan="2">{sfdict[idx]["REP_HOST"].strip()}</td><td class="even_row_data">{sfdict[idx]["REP_NAME"]}</td><td class="even_row_data">{str(sfdict[idx]["SRC_DTM"])}</td><td class="even_row_data block">{str(time_in_minutes)} minutes behind</td></tr>\n')
|
||||
else:
|
||||
latency.write(f'\t\t\t\t<tr><td class="odd_row_data">{sfdict[idx]["ENV"]}</td><td class="odd_row_data">{sfdict[idx]["EXT_NAME"]}</td><td class="odd_row_data block" colspan="2">{sfdict[idx]["REP_HOST"].strip()}</td><td class="odd_row_data">{sfdict[idx]["REP_NAME"]}</td><td class="odd_row_data">{str(sfdict[idx]["SRC_DTM"])}</td><td class="odd_row_data block">{str(time_in_minutes)} minutes behind</td></tr>\n')
|
||||
latency.write(' </table>\n')
|
||||
closeemail(latency)
|
||||
latency.close()
|
||||
emailsupport('LATENCY DETECTED IN QLIK REPLICATE', 'latency.html')
|
||||
nowstamp = datetime.datetime.now()
|
||||
os.remove('latency.html')
|
||||
else:
|
||||
print('Zero objects detected with latency.')
|
||||
print(f'Endtime: {nowstamp}')
|
||||
print('latency_detected: ' + str(latency_detected))
|
||||
Reference in New Issue
Block a user