""" Copyright 2018 Attunity Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ # coding: utf-8 import os, datetime, socket, sys, fileinput from snowflake import connector import base64 import progressbar import smtplib from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText import configparser def emailsupport(messagetype, messagefile): """Send to Email""" me = GGENV = CONFIG.get('global', 'sender') #you = ['HTTP_509@LibertyMutual.com', 'PMEDW_Prod_Support@LibertyMutual.com'] you = CONFIG.get('global', 'receivers').split() # Open a plain text file for reading. For this example, assume that # the text file contains only ASCII characters. message_part = open(messagefile, 'r') part2 = MIMEText(message_part.read(), 'html') # Create a text/plain message message_part.close() msg = MIMEMultipart('alternative') msg['Subject'] = messagetype msg['From'] = me msg['To'] = ",".join(you) msg.attach(part2) # Send the message via our own SMTP server, but don't include the envelope header. sendmail(me, you, msg.as_string()) # Set header automation statement for email def openemail(emailfile): emailfile.write("

THIS IS AN AUTOMATED NOTIFICATION. PLEASE DO NOT RESPOND TO THIS E-MAIL.

") emailfile.write("\n") emailfile.write("\n") # Set footer automation statement for email def closeemail(emailfile): emailfile.write("
\n") emailfile.write("

THIS IS AN AUTOMATED NOTIFICATION. PLEASE DO NOT RESPOND TO THIS E-MAIL.

") emailfile.write("\t") emailfile.write("\n") def sendmail(me, you, mailmsg): # Send the message via our own SMTP server, but don't include the envelope header. try: s = smtplib.SMTP(CONFIG.get('global', 'mailhost')) s.sendmail(me, you, mailmsg) s.quit() except smtplib.SMTPException: print('Email was drafted but I was unable to send it. Unable to connect to mailserver') return 1 else: return 0 CONFIG = configparser.RawConfigParser(allow_no_value=True) CONFIG.read('settings.ini') sf_logins = [] for option, value in CONFIG.items('sf_targets'): itemlist = value.split(',') pw = bytes(itemlist[2], 'UTF8') npw = base64.b64decode(pw).decode('UTF8') sf_logins.append([itemlist[0],itemlist[1],npw,itemlist[3],itemlist[4],itemlist[5],itemlist[6]]) nowstamp = datetime.datetime.now() script_name = os.path.basename(__file__) object_list = [] latency_detected = False latency_minutes = CONFIG.get('global', 'minutes') latency_seconds = int(latency_minutes) * 60 sfquery = CONFIG.get('global', 'query') sfdict = {} print(f'Scanning for objects with latency more than {latency_minutes} minute(s) ({latency_seconds} seconds).') print(f'Starttime: {nowstamp}') updaterange = len(sf_logins) with progressbar.ProgressBar(maxval=updaterange, redirect_stdout=True) as p: for i in range(updaterange): login_name=sf_logins[i][1] login_password=sf_logins[i][2] sf_account=sf_logins[i][3] sf_warehouse=sf_logins[i][4] sf_database=sf_logins[i][5] sf_schema=sf_logins[i][6] counter = 0 try: conn = connector.connect( user=login_name, password=str(login_password), account=sf_account, database=sf_database, schema=sf_schema, ) conn._paramstyle='qmark' except Exception as e: print('Unable to connect to Snowflake') print(str(e)) #print(f'Connection to Snowflake established as {login_name}.') rootdir = os.getcwd() server_name = socket.gethostname() current_time = datetime.datetime.now() cur = conn.cursor() cur.execute(sfquery) #print(cur.sfqid) cur.get_results_from_sfqid(cur.sfqid) results = cur.fetchall() #print(f'result retrieval successful.') for result in results: #print(result) time_difference = current_time - result[3] if time_difference.total_seconds() > int(latency_seconds): if result[2] == '$AR_V_TASK_NAME': pass elif result not in object_list: latency_detected = True object_list.append(result) sfdict[counter] = {'ENV': sf_logins[i][0], 'EXT_NAME': result[0], 'REP_HOST': result[1], 'REP_NAME': result[2], 'SRC_DTM': result[3]} counter += 1 conn.close() p.update(i) if len(sfdict) > 0: timestamp = nowstamp.strftime("%m/%d/%Y, %H:%M:%S") html_report = open('email_template.html', 'r') latency_list = [] latency = open('latency.html', 'w') openemail(latency) for line in html_report: latency.write(line.replace('DateNow', timestamp)) latency.write(' \n') latency.write(f'\n') latency.write(f'\t\t\t\t\n') print(f'{len(sfdict)} objects detected with latency.') for idx, sfobject in enumerate(sfdict): time_difference = nowstamp - sfdict[idx]["SRC_DTM"] time_in_minutes = round(int(time_difference.total_seconds() / 60)) numcheck = idx + 2 if numcheck % 2 == 0: latency.write(f'\t\t\t\t\n') else: latency.write(f'\t\t\t\t\n') latency.write('
Objects with more than {int(latency_minutes)} minute(s) of latency:
ENVIRONMENTTARGET DATABASEREPLICATE HOSTTASK OBJECTLAST CANARY UPDATELATENCY (MINUTES)
{sfdict[idx]["ENV"]}{sfdict[idx]["EXT_NAME"]}{sfdict[idx]["REP_HOST"].strip()}{sfdict[idx]["REP_NAME"]}{str(sfdict[idx]["SRC_DTM"])}{str(time_in_minutes)} minutes behind
{sfdict[idx]["ENV"]}{sfdict[idx]["EXT_NAME"]}{sfdict[idx]["REP_HOST"].strip()}{sfdict[idx]["REP_NAME"]}{str(sfdict[idx]["SRC_DTM"])}{str(time_in_minutes)} minutes behind
\n') closeemail(latency) latency.close() emailsupport('LATENCY DETECTED IN QLIK REPLICATE', 'latency.html') nowstamp = datetime.datetime.now() os.remove('latency.html') else: print('Zero objects detected with latency.') print(f'Endtime: {nowstamp}') print('latency_detected: ' + str(latency_detected))