434 lines
15 KiB
Python
434 lines
15 KiB
Python
#!/usr/bin/python3
|
|
"""
|
|
Copyright 2018 Attunity
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
"""
|
|
|
|
# coding: utf-8
|
|
import os, sys, time, datetime, re
|
|
import difflib
|
|
from os import listdir
|
|
from os.path import isfile, join
|
|
from shutil import copyfile
|
|
from time import sleep
|
|
import smtplib
|
|
from email.mime.multipart import MIMEMultipart
|
|
from email.mime.text import MIMEText
|
|
import configparser
|
|
from unicodedata import name
|
|
|
|
# import AEM-Client
|
|
from aem_client import *
|
|
|
|
if sys.version_info > (3,):
|
|
print('Using environment Python 3.X\n')
|
|
else:
|
|
print('Using environment Python 2.X\n')
|
|
|
|
ENDPOINT_ERRORS = 'endpoint_errors.html'
|
|
|
|
# Utility class for Examples
|
|
class Utils(object):
|
|
@staticmethod
|
|
def dollar_type(obj):
|
|
"""
|
|
get attribute '$type'
|
|
"""
|
|
return getattr(obj, '$type')
|
|
# Utils.dollar_type
|
|
|
|
@staticmethod
|
|
def enum_name(enum_val):
|
|
return enum_val.name
|
|
# Utils.enum_name
|
|
|
|
@staticmethod
|
|
def print_action_title(action_text):
|
|
print('\n############################################################')
|
|
print('\n{0}\n'.format(str(action_text) ) )
|
|
# Utils.print_action_title
|
|
|
|
@staticmethod
|
|
def print_action_title_without_separator(action_text):
|
|
print('\n{0}\n'.format(str(action_text) ) )
|
|
# Utils.print_action_title_without_separator
|
|
|
|
@staticmethod
|
|
def time_sleep(duration=3, show_msg=True):
|
|
"""
|
|
sleep for duration (in seconds)
|
|
"""
|
|
if show_msg:
|
|
print('Sleeping for: {0} seconds'.format(duration))
|
|
time.sleep(duration)
|
|
# Utils.time_sleep
|
|
|
|
@staticmethod
|
|
def get_b64_user_pass(username, password):
|
|
"""
|
|
return a base64 of 'username:password'
|
|
"""
|
|
username_password_str = str.encode('{0}:{1}'.format(username, password))
|
|
return base64.b64encode(username_password_str).decode('ascii')
|
|
# Utils.get_b64_user_pass
|
|
|
|
@staticmethod
|
|
def changed_initial(current_value, predefine_value, param_name):
|
|
if current_value == predefine_value:
|
|
raise RuntimeError('Please replace param: "{0}", value:"{1}", to the real value.'.format(param_name, predefine_value ) )
|
|
# Utils.changed_initial
|
|
|
|
@staticmethod
|
|
def print_dict_attr(obj, attr_name):
|
|
for key in obj.__dict__:
|
|
item = obj.__dict__[key]
|
|
if hasattr(item, attr_name):
|
|
inner_list = getattr(item,attr_name)
|
|
for inner_item in inner_list:
|
|
print('\t{0}:\t{1}'.format(inner_item.name, key))
|
|
# Utils.print_dict_attr
|
|
# END of class Util
|
|
|
|
class PythonProgram(object):
|
|
def __init__(self, ):
|
|
|
|
### Information for connecting to AEM (replace placeholders with actual connection details)
|
|
domain = 'LM'
|
|
username = 'sapipqlikadmin'
|
|
password = base64.b64decode(b'UHJpbWU1OTY2TG9naWMyMw==').decode('utf-8')
|
|
aem_machine_name = 'vmpip-q525jzty.lm.lmig.com'
|
|
|
|
Utils.changed_initial(domain, 'DOMAIN', 'domain')
|
|
Utils.changed_initial(username, 'USER', 'username')
|
|
Utils.changed_initial(password, 'PASSWORD', 'password')
|
|
Utils.changed_initial(aem_machine_name, 'some-host', 'aem_machine_name')
|
|
# Combine domain and user name - domain\\username
|
|
domain_username = '{0}\\{1}'.format(domain, username)
|
|
|
|
print('Connecting to AemClient with user "{0}" on machine name: "{1}"...'.format(domain_username, aem_machine_name))
|
|
try:
|
|
# The format of credentials AemClient must get is domain\\username:password base64 encoded
|
|
b64_username_password = Utils.get_b64_user_pass(domain_username, password)
|
|
# Create an instance of AemClient
|
|
self.aem_client = AemClient(b64_username_password, aem_machine_name, verify_certificate=False)
|
|
print('Done.')
|
|
print('\n############################################################')
|
|
except Exception as ex:
|
|
print('Failed connecting to AemClient')
|
|
print(ex)
|
|
self.aem_client = None
|
|
# end function __init__ for ExamplePythonProgram
|
|
|
|
def get_server(self, name):
|
|
try:
|
|
resp = self.aem_client.get_server(name)
|
|
return resp
|
|
except Exception as ex:
|
|
print(ex)
|
|
return None
|
|
# end function get_server_acl
|
|
|
|
def get_server_list(self):
|
|
resp = self.aem_client.get_server_list()
|
|
# serverList, tasklist, endpointList - are the only-one in camelCase
|
|
if resp and resp.serverList:
|
|
return resp.serverList
|
|
else:
|
|
print('0 server found')
|
|
return []
|
|
# end function get_server_list_and_print
|
|
|
|
def get_server_details(self, name):
|
|
resp = self.aem_client.get_server_details(name)
|
|
# serverList, tasklist, endpointList - are the only-one in camelCase
|
|
if resp:
|
|
return resp
|
|
else:
|
|
print('0 server found')
|
|
return []
|
|
# end function get_server_list_and_print
|
|
|
|
def is_task_in_server(self, task_name, server_name):
|
|
tasklist_i = self.get_task_list_and_print(server_name, print_list=False)
|
|
### serverList, tasklist, endpointList - are the only-one in camelCase
|
|
if not tasklist_i:
|
|
return False
|
|
task_matches = [task for task in tasklist_i if task.name == task_name]
|
|
return len(task_matches) > 0
|
|
# end of function is_task_in_server
|
|
|
|
def get_task_list(self, server_name, print_list=False):
|
|
resp = self.aem_client.get_task_list(server_name)
|
|
if print_list and resp:
|
|
print(str( len(resp.tasklist) ) +' task(s) found')
|
|
print('For each task show [Name, State, Stop Reason]:\n')
|
|
for task in resp.tasklist:
|
|
print('\t[{0}, {1}, {2}]'.format(task.name, Utils.enum_name(task.state), Utils.enum_name(task.stop_reason) ) )
|
|
return resp.tasklist
|
|
# end function get_task_list_and_print
|
|
|
|
def get_task_list_and_print(self, server_name, print_list=False):
|
|
resp = self.aem_client.get_task_list(server_name)
|
|
if print_list and resp:
|
|
print(str( len(resp.tasklist) ) +' task(s) found')
|
|
print('For each task show [Name, State, Stop Reason]:\n')
|
|
for task in resp.tasklist:
|
|
print('\t[{0}, {1}, {2}]'.format(task.name, Utils.enum_name(task.state), Utils.enum_name(task.stop_reason) ) )
|
|
return resp.tasklist
|
|
# end function get_task_list_and_print
|
|
|
|
#SET that executing import, override and then export task
|
|
|
|
def export_task_json(self, server, task, withendpoints=True):
|
|
if self.is_task_in_server(task, server):
|
|
try:
|
|
resp_stream = self.aem_client.export_task(server, task, withendpoints)
|
|
str_decoded = resp_stream.decode('utf-8').rstrip()
|
|
return str_decoded
|
|
except Exception as ex:
|
|
print(ex)
|
|
print('Error: aem_export_task\n')
|
|
jdata = "{\t\"result\": \"SYS-E-HTTPFAIL, SYS-E-HTTPFAIL, Cannot load replication definition...\"\n}\n"
|
|
return jdata
|
|
else:
|
|
print('Error: Task "{0}" is NOT in server: "{1}"\n...'.format(task, server))
|
|
def export_task_json_no_endpoints(self, server, task, withendpoints=False):
|
|
if self.is_task_in_server(task, server):
|
|
try:
|
|
resp_stream = self.aem_client.export_task(server, task, withendpoints)
|
|
str_decoded = resp_stream.decode('utf-8').rstrip()
|
|
return str_decoded
|
|
except Exception as ex:
|
|
print(ex)
|
|
print('Error: aem_export_task\n')
|
|
jdata = "{\t\"result\": \"SYS-E-HTTPFAIL, SYS-E-HTTPFAIL, Cannot load replication definition...\"\n}\n"
|
|
return jdata
|
|
else:
|
|
print('Error: Task "{0}" is NOT in server: "{1}"\n...'.format(task, server))
|
|
|
|
def get_task_details(self, server, task):
|
|
resp_stream = self.aem_client.get_task_details(server, task)
|
|
return resp_stream
|
|
|
|
def get_endpoint_list(self, server):
|
|
resp_stream = self.aem_client.get_endpoint_list(server)
|
|
return resp_stream
|
|
|
|
def test_endpoint(self, server, endpoint):
|
|
resp_stream = self.aem_client.test_endpoint(server, endpoint)
|
|
return resp_stream
|
|
|
|
# END class PythonProgram
|
|
|
|
def diffmessage(server1, server2, filename):
|
|
print('Checking for differences between ' + server1 + ' and ' + server2 + ' for filename ' + filename)
|
|
|
|
def filediff(filename1,filename2, differences):
|
|
founddiffs = False
|
|
tofiledata = filename1
|
|
filedata = open(tofiledata, 'r')
|
|
filelines1 = filedata.readlines()
|
|
filedata.close()
|
|
for line in filelines1:
|
|
if "\"description\":" in line or "\"task_uuid\":" in line:
|
|
filelines1.remove(line)
|
|
if "\"name\":" in line and re.search(r"[-\d]+", line):
|
|
filelines1.remove(line)
|
|
fromfiledata = filename2
|
|
filedata = open(fromfiledata, 'r')
|
|
filelines2 = filedata.readlines()
|
|
filedata.close()
|
|
for line in filelines2:
|
|
if "\"description\":" in line or "\"task_uuid\":" in line:
|
|
filelines2.remove(line)
|
|
if "\"name\":" in line and re.search(r"[-\d]+", line):
|
|
filelines2.remove(line)
|
|
for line in difflib.unified_diff(filelines1, filelines2, fromfile=fromfiledata, tofile=tofiledata, lineterm=''):
|
|
differences.write(line + '\n')
|
|
founddiffs = True
|
|
return(founddiffs)
|
|
|
|
founddiffs = False
|
|
|
|
#################################
|
|
# example_main #
|
|
#################################
|
|
|
|
CONFIG = configparser.RawConfigParser(allow_no_value=True)
|
|
CONFIG.read('D:\\Qlik\\Scripts\\task_check\\settings.ini')
|
|
|
|
|
|
def emailsupport(messagetype, messagefile):
|
|
"""Send to Email"""
|
|
me = CONFIG.get('global', 'sender')
|
|
you = CONFIG.get('global', 'receivers').split()
|
|
|
|
# Open a plain text file for reading. For this example, assume that
|
|
# the text file contains only ASCII characters.
|
|
message_part = open(messagefile, 'r')
|
|
part2 = MIMEText(message_part.read(), 'html')
|
|
# Create a text/plain message
|
|
message_part.close()
|
|
|
|
msg = MIMEMultipart('alternative')
|
|
|
|
msg['Subject'] = messagetype
|
|
|
|
msg['From'] = me
|
|
msg['To'] = ",".join(you)
|
|
|
|
msg.attach(part2)
|
|
|
|
# Send the message via our own SMTP server, but don't include the envelope header.
|
|
sendmail(me, you, msg.as_string())
|
|
|
|
# Set header automation statement for email
|
|
def openemail(emailfile):
|
|
emailfile.write("<p>THIS IS AN AUTOMATED NOTIFICATION. PLEASE DO NOT RESPOND TO THIS E-MAIL.</p>")
|
|
emailfile.write("<html>\n")
|
|
emailfile.write("<table border='1'>\n")
|
|
|
|
# Set footer automation statement for email
|
|
def closeemail(emailfile):
|
|
emailfile.write("</table>\n")
|
|
emailfile.write("<p>THIS IS AN AUTOMATED NOTIFICATION. PLEASE DO NOT RESPOND TO THIS E-MAIL.</p>")
|
|
emailfile.write("\t</body>")
|
|
emailfile.write("</html>\n")
|
|
|
|
def sendmail(me, you, mailmsg):
|
|
try:
|
|
s = smtplib.SMTP(CONFIG.get('global', 'mailhost'))
|
|
s.sendmail(me, you, mailmsg)
|
|
s.quit()
|
|
except smtplib.SMTPException:
|
|
print('Email was drafted but I was unable to send it. Unable to connect to mailserver')
|
|
return 1
|
|
else:
|
|
return 0
|
|
|
|
|
|
def get_endpoint_list(counter, error_list, program, servername, task_list):
|
|
if servername == 'DEVQRP-EDWKDC':
|
|
print(f'getting endpoint list for {servername}')
|
|
endpoint_list = program.get_endpoint_list(servername)
|
|
print('Testing server endpoints........')
|
|
for endpoint in endpoint_list.endpointList:
|
|
test_endpoint(counter, endpoint, error_list, program, servername, task_list)
|
|
|
|
|
|
def test_endpoint(counter, endpoint, error_list, program, servername, task_list):
|
|
testresult = program.test_endpoint(servername, endpoint.name)
|
|
if testresult.status.name != 'CONNECTED':
|
|
result = 'FAILED'
|
|
tasknamelist = []
|
|
taskname = ''
|
|
for tname in task_list:
|
|
if endpoint.name in task_list[tname]['source_endpoint'] or endpoint.name in \
|
|
task_list[tname]['target_endpoint']:
|
|
tasknamelist.append(task_list[tname]['task_name'])
|
|
taskname = ','.join(tasknamelist)
|
|
connection_name = 'Server Name: ' + servername + ' Endpoint Name: ' + endpoint.name
|
|
error_list[counter] = {'cname': connection_name, 'cvalue': result, 'taskname': taskname}
|
|
counter += 1
|
|
|
|
|
|
def get_server_taskinfo(program, server, server_list, task_list, unmonitored):
|
|
try:
|
|
program.get_task_list(server)
|
|
tasks = program.get_task_list(server)
|
|
for task in tasks:
|
|
field = task.name.split()
|
|
server_list[server].append(field[0])
|
|
details = program.get_task_details(server, task.name)
|
|
task_list[task.name] = {'task_name': task.name, 'source_endpoint': details.source_endpoint.name,
|
|
'target_endpoint': details.target_endpoint.name}
|
|
except Exception:
|
|
Utils.print_action_title_without_separator('Server ' + server + ' currently not being monitored.......')
|
|
unmonitored = unmonitored + 1
|
|
|
|
def check_env(replicateservers, server, server_list):
|
|
replicateservers.append(server.name)
|
|
if CONFIG.get('global', 'skip_dev') == '1':
|
|
if 'SANDBOX' in server.name or 'DEV' in server.name:
|
|
print(f'Skipping sandbox server {server.name}')
|
|
else:
|
|
server_list[server.name] = []
|
|
else:
|
|
server_list[server.name] = []
|
|
|
|
def check_endpoints():
|
|
global main
|
|
|
|
error_list = {}
|
|
task_list = {}
|
|
server_list = {}
|
|
replicateservers = []
|
|
unmonitored = 0
|
|
counter = 0
|
|
|
|
# setup_aem_client
|
|
program = PythonProgram()
|
|
if program.aem_client is None:
|
|
print('Error: in aem_client,\nExiting.')
|
|
return
|
|
|
|
Utils.print_action_title_without_separator('Getting AEM server list......')
|
|
mainserver_list = program.get_server_list()
|
|
for server in mainserver_list:
|
|
check_env(replicateservers, server, server_list)
|
|
for server in replicateservers:
|
|
get_server_taskinfo(program, server, server_list, task_list, unmonitored)
|
|
Utils.print_action_title_without_separator('Checking status of all tasks......')
|
|
for servername in server_list:
|
|
get_endpoint_list(counter, error_list, program, servername, task_list)
|
|
|
|
|
|
nowstamp = datetime.datetime.now()
|
|
|
|
if len(error_list) > 0:
|
|
nowstamp = datetime.datetime.now()
|
|
timestamp = nowstamp.strftime("%m/%d/%Y, %H:%M:%S")
|
|
html_report = open('email_template.html', 'r')
|
|
task_errors = open(ENDPOINT_ERRORS, 'w')
|
|
openemail(task_errors)
|
|
for line in html_report:
|
|
task_errors.write(line.replace('DateNow', timestamp))
|
|
task_errors.write(' <table class="basic_lines" summary="Table Listing">\n')
|
|
task_errors.write('<tr><th colspan="3">Endpoints that failed validation:</th></tr>\n')
|
|
task_errors.write(
|
|
'\t\t\t\t<tr><th class="odd_row_data" width="100">ENDPOINT</th><th class="odd_row_data" width="50">CONNECTION RESULT</th><th class="odd_row_data" width="25">TASK(s) POSSIBLY AFFECTED</th></tr>\n')
|
|
print(f'{len(error_list)} objects detected with errors.')
|
|
for idx, sfobject in enumerate(error_list):
|
|
numcheck = idx + 2
|
|
if numcheck % 2 == 0:
|
|
task_errors.write(
|
|
f'\t\t\t\t<tr><td class="even_row_data">{error_list[idx]["cname"]}</td><td class="even_row_data">{error_list[idx]["cvalue"]}</td><td class="even_row_data">{error_list[idx]["taskname"]}</td></tr>\n')
|
|
else:
|
|
task_errors.write(
|
|
f'\t\t\t\t<tr><td class="odd_row_data">{error_list[idx]["cname"]}</td><td class="odd_row_data">{error_list[idx]["cvalue"]}</td><td class="even_row_data">{error_list[idx]["taskname"]}</td></tr>\n')
|
|
task_errors.write(' </table>\n')
|
|
closeemail(task_errors)
|
|
task_errors.close()
|
|
emailsupport('ENDPOINT VALIDATION FAILURES DETECTED IN QLIK REPLICATE', ENDPOINT_ERRORS)
|
|
nowstamp = datetime.datetime.now()
|
|
os.remove(ENDPOINT_ERRORS)
|
|
else:
|
|
print('Zero objects found in error.')
|
|
print(f'Endtime: {nowstamp}')
|
|
print('Errored tasks detected: ' + str(len(error_list)))
|
|
|
|
|
|
|
|
check_endpoints()
|
|
|