adding work scripts
This commit is contained in:
@@ -0,0 +1,433 @@
|
||||
#!/usr/bin/python3
|
||||
"""
|
||||
Copyright 2018 Attunity
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
|
||||
# coding: utf-8
|
||||
import os, sys, time, datetime, re
|
||||
import difflib
|
||||
from os import listdir
|
||||
from os.path import isfile, join
|
||||
from shutil import copyfile
|
||||
from time import sleep
|
||||
import smtplib
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
import configparser
|
||||
from unicodedata import name
|
||||
|
||||
# import AEM-Client
|
||||
from aem_client import *
|
||||
|
||||
if sys.version_info > (3,):
|
||||
print('Using environment Python 3.X\n')
|
||||
else:
|
||||
print('Using environment Python 2.X\n')
|
||||
|
||||
ENDPOINT_ERRORS = 'endpoint_errors.html'
|
||||
|
||||
# Utility class for Examples
|
||||
class Utils(object):
|
||||
@staticmethod
|
||||
def dollar_type(obj):
|
||||
"""
|
||||
get attribute '$type'
|
||||
"""
|
||||
return getattr(obj, '$type')
|
||||
# Utils.dollar_type
|
||||
|
||||
@staticmethod
|
||||
def enum_name(enum_val):
|
||||
return enum_val.name
|
||||
# Utils.enum_name
|
||||
|
||||
@staticmethod
|
||||
def print_action_title(action_text):
|
||||
print('\n############################################################')
|
||||
print('\n{0}\n'.format(str(action_text) ) )
|
||||
# Utils.print_action_title
|
||||
|
||||
@staticmethod
|
||||
def print_action_title_without_separator(action_text):
|
||||
print('\n{0}\n'.format(str(action_text) ) )
|
||||
# Utils.print_action_title_without_separator
|
||||
|
||||
@staticmethod
|
||||
def time_sleep(duration=3, show_msg=True):
|
||||
"""
|
||||
sleep for duration (in seconds)
|
||||
"""
|
||||
if show_msg:
|
||||
print('Sleeping for: {0} seconds'.format(duration))
|
||||
time.sleep(duration)
|
||||
# Utils.time_sleep
|
||||
|
||||
@staticmethod
|
||||
def get_b64_user_pass(username, password):
|
||||
"""
|
||||
return a base64 of 'username:password'
|
||||
"""
|
||||
username_password_str = str.encode('{0}:{1}'.format(username, password))
|
||||
return base64.b64encode(username_password_str).decode('ascii')
|
||||
# Utils.get_b64_user_pass
|
||||
|
||||
@staticmethod
|
||||
def changed_initial(current_value, predefine_value, param_name):
|
||||
if current_value == predefine_value:
|
||||
raise RuntimeError('Please replace param: "{0}", value:"{1}", to the real value.'.format(param_name, predefine_value ) )
|
||||
# Utils.changed_initial
|
||||
|
||||
@staticmethod
|
||||
def print_dict_attr(obj, attr_name):
|
||||
for key in obj.__dict__:
|
||||
item = obj.__dict__[key]
|
||||
if hasattr(item, attr_name):
|
||||
inner_list = getattr(item,attr_name)
|
||||
for inner_item in inner_list:
|
||||
print('\t{0}:\t{1}'.format(inner_item.name, key))
|
||||
# Utils.print_dict_attr
|
||||
# END of class Util
|
||||
|
||||
class PythonProgram(object):
|
||||
def __init__(self, ):
|
||||
|
||||
### Information for connecting to AEM (replace placeholders with actual connection details)
|
||||
domain = 'LM'
|
||||
username = 'sapipqlikadmin'
|
||||
password = base64.b64decode(b'UHJpbWU1OTY2TG9naWMyMw==').decode('utf-8')
|
||||
aem_machine_name = 'vmpip-q525jzty.lm.lmig.com'
|
||||
|
||||
Utils.changed_initial(domain, 'DOMAIN', 'domain')
|
||||
Utils.changed_initial(username, 'USER', 'username')
|
||||
Utils.changed_initial(password, 'PASSWORD', 'password')
|
||||
Utils.changed_initial(aem_machine_name, 'some-host', 'aem_machine_name')
|
||||
# Combine domain and user name - domain\\username
|
||||
domain_username = '{0}\\{1}'.format(domain, username)
|
||||
|
||||
print('Connecting to AemClient with user "{0}" on machine name: "{1}"...'.format(domain_username, aem_machine_name))
|
||||
try:
|
||||
# The format of credentials AemClient must get is domain\\username:password base64 encoded
|
||||
b64_username_password = Utils.get_b64_user_pass(domain_username, password)
|
||||
# Create an instance of AemClient
|
||||
self.aem_client = AemClient(b64_username_password, aem_machine_name, verify_certificate=False)
|
||||
print('Done.')
|
||||
print('\n############################################################')
|
||||
except Exception as ex:
|
||||
print('Failed connecting to AemClient')
|
||||
print(ex)
|
||||
self.aem_client = None
|
||||
# end function __init__ for ExamplePythonProgram
|
||||
|
||||
def get_server(self, name):
|
||||
try:
|
||||
resp = self.aem_client.get_server(name)
|
||||
return resp
|
||||
except Exception as ex:
|
||||
print(ex)
|
||||
return None
|
||||
# end function get_server_acl
|
||||
|
||||
def get_server_list(self):
|
||||
resp = self.aem_client.get_server_list()
|
||||
# serverList, tasklist, endpointList - are the only-one in camelCase
|
||||
if resp and resp.serverList:
|
||||
return resp.serverList
|
||||
else:
|
||||
print('0 server found')
|
||||
return []
|
||||
# end function get_server_list_and_print
|
||||
|
||||
def get_server_details(self, name):
|
||||
resp = self.aem_client.get_server_details(name)
|
||||
# serverList, tasklist, endpointList - are the only-one in camelCase
|
||||
if resp:
|
||||
return resp
|
||||
else:
|
||||
print('0 server found')
|
||||
return []
|
||||
# end function get_server_list_and_print
|
||||
|
||||
def is_task_in_server(self, task_name, server_name):
|
||||
tasklist_i = self.get_task_list_and_print(server_name, print_list=False)
|
||||
### serverList, tasklist, endpointList - are the only-one in camelCase
|
||||
if not tasklist_i:
|
||||
return False
|
||||
task_matches = [task for task in tasklist_i if task.name == task_name]
|
||||
return len(task_matches) > 0
|
||||
# end of function is_task_in_server
|
||||
|
||||
def get_task_list(self, server_name, print_list=False):
|
||||
resp = self.aem_client.get_task_list(server_name)
|
||||
if print_list and resp:
|
||||
print(str( len(resp.tasklist) ) +' task(s) found')
|
||||
print('For each task show [Name, State, Stop Reason]:\n')
|
||||
for task in resp.tasklist:
|
||||
print('\t[{0}, {1}, {2}]'.format(task.name, Utils.enum_name(task.state), Utils.enum_name(task.stop_reason) ) )
|
||||
return resp.tasklist
|
||||
# end function get_task_list_and_print
|
||||
|
||||
def get_task_list_and_print(self, server_name, print_list=False):
|
||||
resp = self.aem_client.get_task_list(server_name)
|
||||
if print_list and resp:
|
||||
print(str( len(resp.tasklist) ) +' task(s) found')
|
||||
print('For each task show [Name, State, Stop Reason]:\n')
|
||||
for task in resp.tasklist:
|
||||
print('\t[{0}, {1}, {2}]'.format(task.name, Utils.enum_name(task.state), Utils.enum_name(task.stop_reason) ) )
|
||||
return resp.tasklist
|
||||
# end function get_task_list_and_print
|
||||
|
||||
#SET that executing import, override and then export task
|
||||
|
||||
def export_task_json(self, server, task, withendpoints=True):
|
||||
if self.is_task_in_server(task, server):
|
||||
try:
|
||||
resp_stream = self.aem_client.export_task(server, task, withendpoints)
|
||||
str_decoded = resp_stream.decode('utf-8').rstrip()
|
||||
return str_decoded
|
||||
except Exception as ex:
|
||||
print(ex)
|
||||
print('Error: aem_export_task\n')
|
||||
jdata = "{\t\"result\": \"SYS-E-HTTPFAIL, SYS-E-HTTPFAIL, Cannot load replication definition...\"\n}\n"
|
||||
return jdata
|
||||
else:
|
||||
print('Error: Task "{0}" is NOT in server: "{1}"\n...'.format(task, server))
|
||||
def export_task_json_no_endpoints(self, server, task, withendpoints=False):
|
||||
if self.is_task_in_server(task, server):
|
||||
try:
|
||||
resp_stream = self.aem_client.export_task(server, task, withendpoints)
|
||||
str_decoded = resp_stream.decode('utf-8').rstrip()
|
||||
return str_decoded
|
||||
except Exception as ex:
|
||||
print(ex)
|
||||
print('Error: aem_export_task\n')
|
||||
jdata = "{\t\"result\": \"SYS-E-HTTPFAIL, SYS-E-HTTPFAIL, Cannot load replication definition...\"\n}\n"
|
||||
return jdata
|
||||
else:
|
||||
print('Error: Task "{0}" is NOT in server: "{1}"\n...'.format(task, server))
|
||||
|
||||
def get_task_details(self, server, task):
|
||||
resp_stream = self.aem_client.get_task_details(server, task)
|
||||
return resp_stream
|
||||
|
||||
def get_endpoint_list(self, server):
|
||||
resp_stream = self.aem_client.get_endpoint_list(server)
|
||||
return resp_stream
|
||||
|
||||
def test_endpoint(self, server, endpoint):
|
||||
resp_stream = self.aem_client.test_endpoint(server, endpoint)
|
||||
return resp_stream
|
||||
|
||||
# END class PythonProgram
|
||||
|
||||
def diffmessage(server1, server2, filename):
|
||||
print('Checking for differences between ' + server1 + ' and ' + server2 + ' for filename ' + filename)
|
||||
|
||||
def filediff(filename1,filename2, differences):
|
||||
founddiffs = False
|
||||
tofiledata = filename1
|
||||
filedata = open(tofiledata, 'r')
|
||||
filelines1 = filedata.readlines()
|
||||
filedata.close()
|
||||
for line in filelines1:
|
||||
if "\"description\":" in line or "\"task_uuid\":" in line:
|
||||
filelines1.remove(line)
|
||||
if "\"name\":" in line and re.search(r"[-\d]+", line):
|
||||
filelines1.remove(line)
|
||||
fromfiledata = filename2
|
||||
filedata = open(fromfiledata, 'r')
|
||||
filelines2 = filedata.readlines()
|
||||
filedata.close()
|
||||
for line in filelines2:
|
||||
if "\"description\":" in line or "\"task_uuid\":" in line:
|
||||
filelines2.remove(line)
|
||||
if "\"name\":" in line and re.search(r"[-\d]+", line):
|
||||
filelines2.remove(line)
|
||||
for line in difflib.unified_diff(filelines1, filelines2, fromfile=fromfiledata, tofile=tofiledata, lineterm=''):
|
||||
differences.write(line + '\n')
|
||||
founddiffs = True
|
||||
return(founddiffs)
|
||||
|
||||
founddiffs = False
|
||||
|
||||
#################################
|
||||
# example_main #
|
||||
#################################
|
||||
|
||||
CONFIG = configparser.RawConfigParser(allow_no_value=True)
|
||||
CONFIG.read('D:\\Qlik\\Scripts\\task_check\\settings.ini')
|
||||
|
||||
|
||||
def emailsupport(messagetype, messagefile):
|
||||
"""Send to Email"""
|
||||
me = CONFIG.get('global', 'sender')
|
||||
you = CONFIG.get('global', 'receivers').split()
|
||||
|
||||
# Open a plain text file for reading. For this example, assume that
|
||||
# the text file contains only ASCII characters.
|
||||
message_part = open(messagefile, 'r')
|
||||
part2 = MIMEText(message_part.read(), 'html')
|
||||
# Create a text/plain message
|
||||
message_part.close()
|
||||
|
||||
msg = MIMEMultipart('alternative')
|
||||
|
||||
msg['Subject'] = messagetype
|
||||
|
||||
msg['From'] = me
|
||||
msg['To'] = ",".join(you)
|
||||
|
||||
msg.attach(part2)
|
||||
|
||||
# Send the message via our own SMTP server, but don't include the envelope header.
|
||||
sendmail(me, you, msg.as_string())
|
||||
|
||||
# Set header automation statement for email
|
||||
def openemail(emailfile):
|
||||
emailfile.write("<p>THIS IS AN AUTOMATED NOTIFICATION. PLEASE DO NOT RESPOND TO THIS E-MAIL.</p>")
|
||||
emailfile.write("<html>\n")
|
||||
emailfile.write("<table border='1'>\n")
|
||||
|
||||
# Set footer automation statement for email
|
||||
def closeemail(emailfile):
|
||||
emailfile.write("</table>\n")
|
||||
emailfile.write("<p>THIS IS AN AUTOMATED NOTIFICATION. PLEASE DO NOT RESPOND TO THIS E-MAIL.</p>")
|
||||
emailfile.write("\t</body>")
|
||||
emailfile.write("</html>\n")
|
||||
|
||||
def sendmail(me, you, mailmsg):
|
||||
try:
|
||||
s = smtplib.SMTP(CONFIG.get('global', 'mailhost'))
|
||||
s.sendmail(me, you, mailmsg)
|
||||
s.quit()
|
||||
except smtplib.SMTPException:
|
||||
print('Email was drafted but I was unable to send it. Unable to connect to mailserver')
|
||||
return 1
|
||||
else:
|
||||
return 0
|
||||
|
||||
|
||||
def get_endpoint_list(counter, error_list, program, servername, task_list):
|
||||
if servername == 'DEVQRP-EDWKDC':
|
||||
print(f'getting endpoint list for {servername}')
|
||||
endpoint_list = program.get_endpoint_list(servername)
|
||||
print('Testing server endpoints........')
|
||||
for endpoint in endpoint_list.endpointList:
|
||||
test_endpoint(counter, endpoint, error_list, program, servername, task_list)
|
||||
|
||||
|
||||
def test_endpoint(counter, endpoint, error_list, program, servername, task_list):
|
||||
testresult = program.test_endpoint(servername, endpoint.name)
|
||||
if testresult.status.name != 'CONNECTED':
|
||||
result = 'FAILED'
|
||||
tasknamelist = []
|
||||
taskname = ''
|
||||
for tname in task_list:
|
||||
if endpoint.name in task_list[tname]['source_endpoint'] or endpoint.name in \
|
||||
task_list[tname]['target_endpoint']:
|
||||
tasknamelist.append(task_list[tname]['task_name'])
|
||||
taskname = ','.join(tasknamelist)
|
||||
connection_name = 'Server Name: ' + servername + ' Endpoint Name: ' + endpoint.name
|
||||
error_list[counter] = {'cname': connection_name, 'cvalue': result, 'taskname': taskname}
|
||||
counter += 1
|
||||
|
||||
|
||||
def get_server_taskinfo(program, server, server_list, task_list, unmonitored):
|
||||
try:
|
||||
program.get_task_list(server)
|
||||
tasks = program.get_task_list(server)
|
||||
for task in tasks:
|
||||
field = task.name.split()
|
||||
server_list[server].append(field[0])
|
||||
details = program.get_task_details(server, task.name)
|
||||
task_list[task.name] = {'task_name': task.name, 'source_endpoint': details.source_endpoint.name,
|
||||
'target_endpoint': details.target_endpoint.name}
|
||||
except Exception:
|
||||
Utils.print_action_title_without_separator('Server ' + server + ' currently not being monitored.......')
|
||||
unmonitored = unmonitored + 1
|
||||
|
||||
def check_env(replicateservers, server, server_list):
|
||||
replicateservers.append(server.name)
|
||||
if CONFIG.get('global', 'skip_dev') == '1':
|
||||
if 'SANDBOX' in server.name or 'DEV' in server.name:
|
||||
print(f'Skipping sandbox server {server.name}')
|
||||
else:
|
||||
server_list[server.name] = []
|
||||
else:
|
||||
server_list[server.name] = []
|
||||
|
||||
def check_endpoints():
|
||||
global main
|
||||
|
||||
error_list = {}
|
||||
task_list = {}
|
||||
server_list = {}
|
||||
replicateservers = []
|
||||
unmonitored = 0
|
||||
counter = 0
|
||||
|
||||
# setup_aem_client
|
||||
program = PythonProgram()
|
||||
if program.aem_client is None:
|
||||
print('Error: in aem_client,\nExiting.')
|
||||
return
|
||||
|
||||
Utils.print_action_title_without_separator('Getting AEM server list......')
|
||||
mainserver_list = program.get_server_list()
|
||||
for server in mainserver_list:
|
||||
check_env(replicateservers, server, server_list)
|
||||
for server in replicateservers:
|
||||
get_server_taskinfo(program, server, server_list, task_list, unmonitored)
|
||||
Utils.print_action_title_without_separator('Checking status of all tasks......')
|
||||
for servername in server_list:
|
||||
get_endpoint_list(counter, error_list, program, servername, task_list)
|
||||
|
||||
|
||||
nowstamp = datetime.datetime.now()
|
||||
|
||||
if len(error_list) > 0:
|
||||
nowstamp = datetime.datetime.now()
|
||||
timestamp = nowstamp.strftime("%m/%d/%Y, %H:%M:%S")
|
||||
html_report = open('email_template.html', 'r')
|
||||
task_errors = open(ENDPOINT_ERRORS, 'w')
|
||||
openemail(task_errors)
|
||||
for line in html_report:
|
||||
task_errors.write(line.replace('DateNow', timestamp))
|
||||
task_errors.write(' <table class="basic_lines" summary="Table Listing">\n')
|
||||
task_errors.write('<tr><th colspan="3">Endpoints that failed validation:</th></tr>\n')
|
||||
task_errors.write(
|
||||
'\t\t\t\t<tr><th class="odd_row_data" width="100">ENDPOINT</th><th class="odd_row_data" width="50">CONNECTION RESULT</th><th class="odd_row_data" width="25">TASK(s) POSSIBLY AFFECTED</th></tr>\n')
|
||||
print(f'{len(error_list)} objects detected with errors.')
|
||||
for idx, sfobject in enumerate(error_list):
|
||||
numcheck = idx + 2
|
||||
if numcheck % 2 == 0:
|
||||
task_errors.write(
|
||||
f'\t\t\t\t<tr><td class="even_row_data">{error_list[idx]["cname"]}</td><td class="even_row_data">{error_list[idx]["cvalue"]}</td><td class="even_row_data">{error_list[idx]["taskname"]}</td></tr>\n')
|
||||
else:
|
||||
task_errors.write(
|
||||
f'\t\t\t\t<tr><td class="odd_row_data">{error_list[idx]["cname"]}</td><td class="odd_row_data">{error_list[idx]["cvalue"]}</td><td class="even_row_data">{error_list[idx]["taskname"]}</td></tr>\n')
|
||||
task_errors.write(' </table>\n')
|
||||
closeemail(task_errors)
|
||||
task_errors.close()
|
||||
emailsupport('ENDPOINT VALIDATION FAILURES DETECTED IN QLIK REPLICATE', ENDPOINT_ERRORS)
|
||||
nowstamp = datetime.datetime.now()
|
||||
os.remove(ENDPOINT_ERRORS)
|
||||
else:
|
||||
print('Zero objects found in error.')
|
||||
print(f'Endtime: {nowstamp}')
|
||||
print('Errored tasks detected: ' + str(len(error_list)))
|
||||
|
||||
|
||||
|
||||
check_endpoints()
|
||||
|
||||
Reference in New Issue
Block a user