177 lines
7.3 KiB
Python
177 lines
7.3 KiB
Python
"""
|
|
Copyright 2018 Attunity
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
"""
|
|
|
|
# coding: utf-8
|
|
import os, datetime, socket, sys, fileinput
|
|
from snowflake import connector
|
|
import base64
|
|
import progressbar
|
|
import smtplib
|
|
from email.mime.multipart import MIMEMultipart
|
|
from email.mime.text import MIMEText
|
|
import configparser
|
|
|
|
def emailsupport(messagetype, messagefile):
|
|
"""Send to Email"""
|
|
me = GGENV = CONFIG.get('global', 'sender')
|
|
#you = ['HTTP_509@LibertyMutual.com', 'PMEDW_Prod_Support@LibertyMutual.com']
|
|
you = CONFIG.get('global', 'receivers').split()
|
|
|
|
# Open a plain text file for reading. For this example, assume that
|
|
# the text file contains only ASCII characters.
|
|
message_part = open(messagefile, 'r')
|
|
part2 = MIMEText(message_part.read(), 'html')
|
|
# Create a text/plain message
|
|
message_part.close()
|
|
|
|
msg = MIMEMultipart('alternative')
|
|
|
|
msg['Subject'] = messagetype
|
|
|
|
msg['From'] = me
|
|
msg['To'] = ",".join(you)
|
|
|
|
msg.attach(part2)
|
|
|
|
# Send the message via our own SMTP server, but don't include the envelope header.
|
|
sendmail(me, you, msg.as_string())
|
|
|
|
# Set header automation statement for email
|
|
def openemail(emailfile):
|
|
emailfile.write("<p>THIS IS AN AUTOMATED NOTIFICATION. PLEASE DO NOT RESPOND TO THIS E-MAIL.</p>")
|
|
emailfile.write("<html>\n")
|
|
emailfile.write("<table border='1'>\n")
|
|
|
|
# Set footer automation statement for email
|
|
def closeemail(emailfile):
|
|
emailfile.write("</table>\n")
|
|
emailfile.write("<p>THIS IS AN AUTOMATED NOTIFICATION. PLEASE DO NOT RESPOND TO THIS E-MAIL.</p>")
|
|
emailfile.write("\t</body>")
|
|
emailfile.write("</html>\n")
|
|
|
|
def sendmail(me, you, mailmsg):
|
|
# Send the message via our own SMTP server, but don't include the envelope header.
|
|
try:
|
|
s = smtplib.SMTP(CONFIG.get('global', 'mailhost'))
|
|
s.sendmail(me, you, mailmsg)
|
|
s.quit()
|
|
except smtplib.SMTPException:
|
|
print('Email was drafted but I was unable to send it. Unable to connect to mailserver')
|
|
return 1
|
|
else:
|
|
return 0
|
|
|
|
CONFIG = configparser.RawConfigParser(allow_no_value=True)
|
|
CONFIG.read('settings.ini')
|
|
sf_logins = []
|
|
for option, value in CONFIG.items('sf_targets'):
|
|
itemlist = value.split(',')
|
|
pw = bytes(itemlist[2], 'UTF8')
|
|
npw = base64.b64decode(pw).decode('UTF8')
|
|
sf_logins.append([itemlist[0],itemlist[1],npw,itemlist[3],itemlist[4],itemlist[5],itemlist[6]])
|
|
|
|
nowstamp = datetime.datetime.now()
|
|
script_name = os.path.basename(__file__)
|
|
object_list = []
|
|
latency_detected = False
|
|
latency_minutes = CONFIG.get('global', 'minutes')
|
|
latency_seconds = int(latency_minutes) * 60
|
|
sfquery = CONFIG.get('global', 'query')
|
|
sfdict = {}
|
|
|
|
print(f'Scanning for objects with latency more than {latency_minutes} minute(s) ({latency_seconds} seconds).')
|
|
print(f'Starttime: {nowstamp}')
|
|
updaterange = len(sf_logins)
|
|
with progressbar.ProgressBar(maxval=updaterange, redirect_stdout=True) as p:
|
|
for i in range(updaterange):
|
|
login_name=sf_logins[i][1]
|
|
login_password=sf_logins[i][2]
|
|
sf_account=sf_logins[i][3]
|
|
sf_warehouse=sf_logins[i][4]
|
|
sf_database=sf_logins[i][5]
|
|
sf_schema=sf_logins[i][6]
|
|
counter = 0
|
|
try:
|
|
conn = connector.connect(
|
|
user=login_name,
|
|
password=str(login_password),
|
|
account=sf_account,
|
|
database=sf_database,
|
|
schema=sf_schema,
|
|
)
|
|
conn._paramstyle='qmark'
|
|
except Exception as e:
|
|
print('Unable to connect to Snowflake')
|
|
print(str(e))
|
|
|
|
|
|
#print(f'Connection to Snowflake established as {login_name}.')
|
|
|
|
rootdir = os.getcwd()
|
|
|
|
server_name = socket.gethostname()
|
|
|
|
current_time = datetime.datetime.now()
|
|
cur = conn.cursor()
|
|
cur.execute(sfquery)
|
|
#print(cur.sfqid)
|
|
cur.get_results_from_sfqid(cur.sfqid)
|
|
results = cur.fetchall()
|
|
#print(f'result retrieval successful.')
|
|
for result in results:
|
|
#print(result)
|
|
time_difference = current_time - result[3]
|
|
if time_difference.total_seconds() > int(latency_seconds):
|
|
if result[2] == '$AR_V_TASK_NAME':
|
|
pass
|
|
elif result not in object_list:
|
|
latency_detected = True
|
|
object_list.append(result)
|
|
sfdict[counter] = {'ENV': sf_logins[i][0], 'EXT_NAME': result[0], 'REP_HOST': result[1], 'REP_NAME': result[2], 'SRC_DTM': result[3]}
|
|
counter += 1
|
|
conn.close()
|
|
p.update(i)
|
|
|
|
if len(sfdict) > 0:
|
|
timestamp = nowstamp.strftime("%m/%d/%Y, %H:%M:%S")
|
|
html_report = open('email_template.html', 'r')
|
|
latency_list = []
|
|
latency = open('latency.html', 'w')
|
|
openemail(latency)
|
|
for line in html_report:
|
|
latency.write(line.replace('DateNow', timestamp))
|
|
latency.write(' <table class="basic_lines" summary="Table Listing">\n')
|
|
latency.write(f'<tr><th colspan="7">Objects with more than {int(latency_minutes)} minute(s) of latency:</th></tr>\n')
|
|
latency.write(f'\t\t\t\t<tr><th class="odd_row_data">ENVIRONMENT</th><th class="odd_row_data">TARGET DATABASE</th><th class="odd_row_data" colspan="2">REPLICATE HOST</th><th class="odd_row_data">TASK OBJECT</th><th class="odd_row_data">LAST CANARY UPDATE</th><th class="odd_row_data">LATENCY (MINUTES)</th></tr>\n')
|
|
print(f'{len(sfdict)} objects detected with latency.')
|
|
for idx, sfobject in enumerate(sfdict):
|
|
time_difference = nowstamp - sfdict[idx]["SRC_DTM"]
|
|
time_in_minutes = round(int(time_difference.total_seconds() / 60))
|
|
numcheck = idx + 2
|
|
if numcheck % 2 == 0:
|
|
latency.write(f'\t\t\t\t<tr><td class="even_row_data">{sfdict[idx]["ENV"]}</td><td class="even_row_data">{sfdict[idx]["EXT_NAME"]}</td><td class="even_row_data block" colspan="2">{sfdict[idx]["REP_HOST"].strip()}</td><td class="even_row_data">{sfdict[idx]["REP_NAME"]}</td><td class="even_row_data">{str(sfdict[idx]["SRC_DTM"])}</td><td class="even_row_data block">{str(time_in_minutes)} minutes behind</td></tr>\n')
|
|
else:
|
|
latency.write(f'\t\t\t\t<tr><td class="odd_row_data">{sfdict[idx]["ENV"]}</td><td class="odd_row_data">{sfdict[idx]["EXT_NAME"]}</td><td class="odd_row_data block" colspan="2">{sfdict[idx]["REP_HOST"].strip()}</td><td class="odd_row_data">{sfdict[idx]["REP_NAME"]}</td><td class="odd_row_data">{str(sfdict[idx]["SRC_DTM"])}</td><td class="odd_row_data block">{str(time_in_minutes)} minutes behind</td></tr>\n')
|
|
latency.write(' </table>\n')
|
|
closeemail(latency)
|
|
latency.close()
|
|
emailsupport('LATENCY DETECTED IN QLIK REPLICATE', 'latency.html')
|
|
nowstamp = datetime.datetime.now()
|
|
os.remove('latency.html')
|
|
else:
|
|
print('Zero objects detected with latency.')
|
|
print(f'Endtime: {nowstamp}')
|
|
print('latency_detected: ' + str(latency_detected))
|