adding work scripts
This commit is contained in:
@@ -0,0 +1,176 @@
|
||||
"""
|
||||
Copyright 2018 Attunity
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
|
||||
# coding: utf-8
|
||||
import os, datetime, socket, sys, fileinput
|
||||
from snowflake import connector
|
||||
import base64
|
||||
import progressbar
|
||||
import smtplib
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
import configparser
|
||||
|
||||
def emailsupport(messagetype, messagefile):
|
||||
"""Send to Email"""
|
||||
me = GGENV = CONFIG.get('global', 'sender')
|
||||
#you = ['HTTP_509@LibertyMutual.com', 'PMEDW_Prod_Support@LibertyMutual.com']
|
||||
you = CONFIG.get('global', 'receivers').split()
|
||||
|
||||
# Open a plain text file for reading. For this example, assume that
|
||||
# the text file contains only ASCII characters.
|
||||
message_part = open(messagefile, 'r')
|
||||
part2 = MIMEText(message_part.read(), 'html')
|
||||
# Create a text/plain message
|
||||
message_part.close()
|
||||
|
||||
msg = MIMEMultipart('alternative')
|
||||
|
||||
msg['Subject'] = messagetype
|
||||
|
||||
msg['From'] = me
|
||||
msg['To'] = ",".join(you)
|
||||
|
||||
msg.attach(part2)
|
||||
|
||||
# Send the message via our own SMTP server, but don't include the envelope header.
|
||||
sendmail(me, you, msg.as_string())
|
||||
|
||||
# Set header automation statement for email
|
||||
def openemail(emailfile):
|
||||
emailfile.write("<p>THIS IS AN AUTOMATED NOTIFICATION. PLEASE DO NOT RESPOND TO THIS E-MAIL.</p>")
|
||||
emailfile.write("<html>\n")
|
||||
emailfile.write("<table border='1'>\n")
|
||||
|
||||
# Set footer automation statement for email
|
||||
def closeemail(emailfile):
|
||||
emailfile.write("</table>\n")
|
||||
emailfile.write("<p>THIS IS AN AUTOMATED NOTIFICATION. PLEASE DO NOT RESPOND TO THIS E-MAIL.</p>")
|
||||
emailfile.write("\t</body>")
|
||||
emailfile.write("</html>\n")
|
||||
|
||||
def sendmail(me, you, mailmsg):
|
||||
# Send the message via our own SMTP server, but don't include the envelope header.
|
||||
try:
|
||||
s = smtplib.SMTP(CONFIG.get('global', 'mailhost'))
|
||||
s.sendmail(me, you, mailmsg)
|
||||
s.quit()
|
||||
except smtplib.SMTPException:
|
||||
print('Email was drafted but I was unable to send it. Unable to connect to mailserver')
|
||||
return 1
|
||||
else:
|
||||
return 0
|
||||
|
||||
CONFIG = configparser.RawConfigParser(allow_no_value=True)
|
||||
CONFIG.read('settings.ini')
|
||||
sf_logins = []
|
||||
for option, value in CONFIG.items('sf_targets'):
|
||||
itemlist = value.split(',')
|
||||
pw = bytes(itemlist[2], 'UTF8')
|
||||
npw = base64.b64decode(pw).decode('UTF8')
|
||||
sf_logins.append([itemlist[0],itemlist[1],npw,itemlist[3],itemlist[4],itemlist[5],itemlist[6]])
|
||||
|
||||
nowstamp = datetime.datetime.now()
|
||||
script_name = os.path.basename(__file__)
|
||||
object_list = []
|
||||
latency_detected = False
|
||||
latency_minutes = CONFIG.get('global', 'minutes')
|
||||
latency_seconds = int(latency_minutes) * 60
|
||||
sfquery = CONFIG.get('global', 'query')
|
||||
sfdict = {}
|
||||
|
||||
print(f'Scanning for objects with latency more than {latency_minutes} minute(s) ({latency_seconds} seconds).')
|
||||
print(f'Starttime: {nowstamp}')
|
||||
updaterange = len(sf_logins)
|
||||
with progressbar.ProgressBar(maxval=updaterange, redirect_stdout=True) as p:
|
||||
for i in range(updaterange):
|
||||
login_name=sf_logins[i][1]
|
||||
login_password=sf_logins[i][2]
|
||||
sf_account=sf_logins[i][3]
|
||||
sf_warehouse=sf_logins[i][4]
|
||||
sf_database=sf_logins[i][5]
|
||||
sf_schema=sf_logins[i][6]
|
||||
counter = 0
|
||||
try:
|
||||
conn = connector.connect(
|
||||
user=login_name,
|
||||
password=str(login_password),
|
||||
account=sf_account,
|
||||
database=sf_database,
|
||||
schema=sf_schema,
|
||||
)
|
||||
conn._paramstyle='qmark'
|
||||
except Exception as e:
|
||||
print('Unable to connect to Snowflake')
|
||||
print(str(e))
|
||||
|
||||
|
||||
#print(f'Connection to Snowflake established as {login_name}.')
|
||||
|
||||
rootdir = os.getcwd()
|
||||
|
||||
server_name = socket.gethostname()
|
||||
|
||||
current_time = datetime.datetime.now()
|
||||
cur = conn.cursor()
|
||||
cur.execute(sfquery)
|
||||
#print(cur.sfqid)
|
||||
cur.get_results_from_sfqid(cur.sfqid)
|
||||
results = cur.fetchall()
|
||||
#print(f'result retrieval successful.')
|
||||
for result in results:
|
||||
#print(result)
|
||||
time_difference = current_time - result[3]
|
||||
if time_difference.total_seconds() > int(latency_seconds):
|
||||
if result[2] == '$AR_V_TASK_NAME':
|
||||
pass
|
||||
elif result not in object_list:
|
||||
latency_detected = True
|
||||
object_list.append(result)
|
||||
sfdict[counter] = {'ENV': sf_logins[i][0], 'EXT_NAME': result[0], 'REP_HOST': result[1], 'REP_NAME': result[2], 'SRC_DTM': result[3]}
|
||||
counter += 1
|
||||
conn.close()
|
||||
p.update(i)
|
||||
|
||||
if len(sfdict) > 0:
|
||||
timestamp = nowstamp.strftime("%m/%d/%Y, %H:%M:%S")
|
||||
html_report = open('email_template.html', 'r')
|
||||
latency_list = []
|
||||
latency = open('latency.html', 'w')
|
||||
openemail(latency)
|
||||
for line in html_report:
|
||||
latency.write(line.replace('DateNow', timestamp))
|
||||
latency.write(' <table class="basic_lines" summary="Table Listing">\n')
|
||||
latency.write(f'<tr><th colspan="7">Objects with more than {int(latency_minutes)} minute(s) of latency:</th></tr>\n')
|
||||
latency.write(f'\t\t\t\t<tr><th class="odd_row_data">ENVIRONMENT</th><th class="odd_row_data">TARGET DATABASE</th><th class="odd_row_data" colspan="2">REPLICATE HOST</th><th class="odd_row_data">TASK OBJECT</th><th class="odd_row_data">LAST CANARY UPDATE</th><th class="odd_row_data">LATENCY (MINUTES)</th></tr>\n')
|
||||
print(f'{len(sfdict)} objects detected with latency.')
|
||||
for idx, sfobject in enumerate(sfdict):
|
||||
time_difference = nowstamp - sfdict[idx]["SRC_DTM"]
|
||||
time_in_minutes = round(int(time_difference.total_seconds() / 60))
|
||||
numcheck = idx + 2
|
||||
if numcheck % 2 == 0:
|
||||
latency.write(f'\t\t\t\t<tr><td class="even_row_data">{sfdict[idx]["ENV"]}</td><td class="even_row_data">{sfdict[idx]["EXT_NAME"]}</td><td class="even_row_data block" colspan="2">{sfdict[idx]["REP_HOST"].strip()}</td><td class="even_row_data">{sfdict[idx]["REP_NAME"]}</td><td class="even_row_data">{str(sfdict[idx]["SRC_DTM"])}</td><td class="even_row_data block">{str(time_in_minutes)} minutes behind</td></tr>\n')
|
||||
else:
|
||||
latency.write(f'\t\t\t\t<tr><td class="odd_row_data">{sfdict[idx]["ENV"]}</td><td class="odd_row_data">{sfdict[idx]["EXT_NAME"]}</td><td class="odd_row_data block" colspan="2">{sfdict[idx]["REP_HOST"].strip()}</td><td class="odd_row_data">{sfdict[idx]["REP_NAME"]}</td><td class="odd_row_data">{str(sfdict[idx]["SRC_DTM"])}</td><td class="odd_row_data block">{str(time_in_minutes)} minutes behind</td></tr>\n')
|
||||
latency.write(' </table>\n')
|
||||
closeemail(latency)
|
||||
latency.close()
|
||||
emailsupport('LATENCY DETECTED IN QLIK REPLICATE', 'latency.html')
|
||||
nowstamp = datetime.datetime.now()
|
||||
os.remove('latency.html')
|
||||
else:
|
||||
print('Zero objects detected with latency.')
|
||||
print(f'Endtime: {nowstamp}')
|
||||
print('latency_detected: ' + str(latency_detected))
|
||||
Reference in New Issue
Block a user