#!/usr/bin/python3
"""
Copyright 2018 Attunity
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

# coding: utf-8
import os, sys, time, datetime, json, re
#from collections import OrderedDict
import difflib
from os import listdir
from os.path import isfile, join
import pandas
#import numpy
import csv
import xlsxwriter
from shutil import copyfile
from time import sleep

# import AEM-Client
from aem_client import *

if sys.version_info > (3,):
	print('Using environment Python 3.X\n')
else:
	print('Using environment Python 2.X\n')

# Utility class for Examples
class Utils(object):
	@staticmethod
	def dollar_type(obj):
		"""
		get attribute '$type'
		"""
		return getattr(obj, '$type')
	# Utils.dollar_type

	@staticmethod
	def enum_name(enum_val):
		return enum_val.name
	# Utils.enum_name

	@staticmethod
	def print_action_title(action_text):
		print('\n############################################################')
		print('\n{0}\n'.format(str(action_text) ) )
	# Utils.print_action_title
	
	@staticmethod
	def print_action_title_without_separator(action_text):
		print('\n{0}\n'.format(str(action_text) ) )
	# Utils.print_action_title_without_separator
	
	@staticmethod
	def time_sleep(duration=3, show_msg=True):
		"""
		sleep for duration (in seconds)
		"""
		if show_msg:
			print('Sleeping for: {0} seconds'.format(duration))
		time.sleep(duration)
	# Utils.time_sleep

	@staticmethod
	def get_b64_user_pass(username, password):
		"""
		return a base64 of 'username:password'
		"""
		username_password_str = str.encode('{0}:{1}'.format(username, password))
		return base64.b64encode(username_password_str).decode('ascii')
	# Utils.get_b64_user_pass

	@staticmethod
	def changed_initial(current_value, predefine_value, param_name):	
		if current_value == predefine_value:
			raise Exception('Please replace param: "{0}", value:"{1}", to the real value.'.format(param_name, predefine_value ) )
	# Utils.changed_initial

	@staticmethod
	def print_dict_attr(obj, attr_name):
		for key in obj.__dict__:
			item = obj.__dict__[key]					
			if hasattr(item, attr_name):
				inner_list = getattr(item,attr_name)
				for inner_item in inner_list:
					print('\t{0}:\t{1}'.format(inner_item.name, key))
	# Utils.print_dict_attr
# END of class Util

class PythonProgram(object):
	def __init__(self, ):
		
		### Information for connecting to AEM (replace placeholders with actual connection details)
		domain = 'LM'
		aem_machine_name = input('servername: \n')
		username = input('username: \n')
		password = input('password: \n')

		Utils.changed_initial(domain, 'DOMAIN', 'domain')
		Utils.changed_initial(username, 'USER', 'username')
		Utils.changed_initial(password, 'PASSWORD', 'password')
		Utils.changed_initial(aem_machine_name, 'some-host', 'aem_machine_name')
		# Combine domain and user name - domain\\username
		domain_username = '{0}\\{1}'.format(domain, username)	

		print('Connecting to AemClient with user "{0}" on machine name: "{1}"...'.format(domain_username, aem_machine_name))
		try:
			# The format of credentials AemClient must get is domain\\username:password base64 encoded
			b64_username_password = Utils.get_b64_user_pass(domain_username, password)
			# Create an instance of AemClient
			self.aem_client = AemClient(b64_username_password, aem_machine_name, verify_certificate=False)
			print('Done.')			
			print('\n############################################################')
		except Exception as ex:
			print('Failed connecting to AemClient')
			print(ex)	
			self.aem_client = None
	# end function __init__ for ExamplePythonProgram

	def get_server(self, name):
		#resp = AemAuthorizationAcl
		try:
			resp = self.aem_client.get_server(name)
			return resp
		except Exception as ex:
			print(ex)	
			return None
	# end function get_server_acl

	def get_server_list(self, print_list = False):		
		resp = self.aem_client.get_server_list()		
		# serverList, taskList, endpointList - are the only-one in camelCase
		if resp and resp.serverList:			
			return resp.serverList
		else:
			print('0 server found')
			return []		
	# end function get_server_list_and_print

	def is_task_in_server(self, task_name, server_name):
		taskList_i = self.get_task_list_and_print(server_name, print_list=False)		
		### serverList, taskList, endpointList - are the only-one in camelCase
		if not taskList_i:
			return False
		task_matches = [task for task in taskList_i if task.name == task_name]
		return len(task_matches) > 0
	# end of function is_task_in_server

	def get_task_list(self, server_name, print_list=False):
		resp = self.aem_client.get_task_list(server_name)	
		if print_list and resp:
			print(str( len(resp.taskList) ) +' task(s) found')
			print('For each task show [Name, State, Stop Reason]:\n')			
			for task in resp.taskList:
				print('\t[{0}, {1}, {2}]'.format(task.name, Utils.enum_name(task.state), Utils.enum_name(task.stop_reason) ) )
		return resp.taskList
	# end function get_task_list_and_print
	
	def get_task_list_and_print(self, server_name, print_list=False):
		resp = self.aem_client.get_task_list(server_name)	
		if print_list and resp:
			print(str( len(resp.taskList) ) +' task(s) found')
			print('For each task show [Name, State, Stop Reason]:\n')			
			for task in resp.taskList:
				print('\t[{0}, {1}, {2}]'.format(task.name, Utils.enum_name(task.state), Utils.enum_name(task.stop_reason) ) )
		return resp.taskList
	# end function get_task_list_and_print

	#SET that executing import, override and then export task
	
	def export_task_json(self, server, task, withendpoints=True):
		if self.is_task_in_server(task, server):
			try:
				resp_stream = self.aem_client.export_task(server, task, withendpoints)
				str_decoded = resp_stream.decode('utf-8').rstrip()
				#print('Done.\n')
				return str_decoded
			except Exception as ex:
				#print(ex)
				#print('Error: aem_export_task\n')
				jdata = "{\t\"result\": \"SYS-E-HTTPFAIL, SYS-E-HTTPFAIL, Cannot load replication definition...\"\n}\n"				
				return jdata			
		else:			
			print('Error: Task "{0}" is NOT in server: "{1}"\n...'.format(task, server))			
	def export_task_json_no_endpoints(self, server, task, withendpoints=False):
		if self.is_task_in_server(task, server):
			try:
				resp_stream = self.aem_client.export_task(server, task, withendpoints)
				str_decoded = resp_stream.decode('utf-8').rstrip()
				#print('Done.\n')
				return str_decoded
			except Exception as ex:
				#print(ex)
				#print('Error: aem_export_task\n')
				jdata = "{\t\"result\": \"SYS-E-HTTPFAIL, SYS-E-HTTPFAIL, Cannot load replication definition...\"\n}\n"				
				return jdata			
		else:			
			print('Error: Task "{0}" is NOT in server: "{1}"\n...'.format(task, server))
# END class PythonProgram

def diffmessage(server1, server2, filename):
	print('Checking for differences between ' + server1 + ' and ' + server2 + ' for filename ' + filename)

def filediff(filename1,filename2, differences):
	founddiffs = False
	tofiledata = filename1
	filedata = open(tofiledata, 'r')
	filelines1 = filedata.readlines()
	filedata.close()
	for line in filelines1:
			if "\"description\":" in line or "\"task_uuid\":" in line:
					filelines1.remove(line)
			if "\"name\":" in line:
					if re.search(r"[-\d]+", line):
							filelines1.remove(line)
	fromfiledata = filename2
	filedata = open(fromfiledata, 'r')
	filelines2 = filedata.readlines()
	filedata.close()
	for line in filelines2:
			if "\"description\":" in line or "\"task_uuid\":" in line:
					filelines2.remove(line)
			if "\"name\":" in line:
					if re.search(r"[-\d]+", line):
							filelines2.remove(line)
	for line in difflib.unified_diff(filelines1, filelines2, fromfile=fromfiledata, tofile=tofiledata, lineterm=''):
		differences.write(line + '\n')
		founddiffs = True
	return(founddiffs)

founddiffs = False
	
#################################
#		example_main			#
#################################
def main():

	#script_dir = os.getcwd() + '/taskinfo_reader'  #This line used for testing in vscode
	script_dir = os.getcwd()
	export_dir = script_dir + '/exports'
	site_diff = False
	reverse_diff = False
	with_dev = False
	stampfile = '.lastrun'
	task_list = {}
	replicateservers = []
	unmonitored = 0

	# setup_aem_client
	program = PythonProgram()
	if program.aem_client == None:
		print('Error: in aem_client,\nExiting.')
		return
	
	### print get_server_list
	Utils.print_action_title_without_separator('Getting AEM server list......')
	server_list = program.get_server_list()
	for server in server_list:
		replicateservers.append(server.name)
		task_list[server.name] = []
	for server in replicateservers:
		try:
			serverinfo = program.get_server(server)
			program.get_task_list(server)
			tasks = program.get_task_list(server)
			for task in tasks:
				field = task.name.split()
				task_list[server].append(field[0])
		except Exception as e:
			Utils.print_action_title_without_separator('Server ' + server + ' currently not being monitored. Unable to export JSON at this time......')
			unmonitored = unmonitored + 1
	Utils.print_action_title_without_separator('Exporting JSON......')
	ts = str(datetime.datetime.now())#.split('.')[0]
	tsfield = ts.split()
	datestamp = tsfield[0].replace('-', '_')
	timestamp = tsfield[1].replace(':', '_').replace('.', '_')
	datetimestamp = datestamp + '_' + timestamp
	copyfile(script_dir + r'.\global_settings.csv', script_dir + r'\exports\global_settings.csv')
	os.chdir(script_dir + '/exports')
	if os.path.isfile(stampfile):
		lastrunfile = open(stampfile, 'r')
		lastrun = lastrunfile.readline()
	else:
		lastrun = None
	newrun = open(stampfile, 'w')
	newrun.write(datetimestamp)
	newrun.close()
	for server in task_list:
		for taskname in task_list[server]:
			jsondata = ''
			try:
				jdata = program.export_task_json(server, taskname)
				tempfile = open('temp.json', 'w')
				tempfile.write(jdata)
				tempfile.close()
				with open('temp.json') as json_file:
					tempjsondata = json.load(json_file)
				jsondata = json.dumps(tempjsondata, indent=4)

				jsonfile = open('AEM_' + server + '_' + taskname + '.json', 'w')
				jsonfile.write(jsondata)
				jsonfile.close()
			except:
				jdata = program.export_task_json_no_endpoints(server, taskname)
				try:
					tempfile = open('temp.json', 'w')
					tempfile.write(jdata)
					tempfile.close()
					with open('temp.json') as json_file:
						tempjsondata = json.load(json_file)
					jsondata = json.dumps(tempjsondata, indent=4)

					jsonfile = open('AEM_' + server + '_' + taskname + '.json', 'w')
					jsonfile.write(jsondata)
					jsonfile.close()
					os.unlink('temp.json')
				except Exception:
					print(f'Empty json string returned for {taskname}. Skipping')

	print('Compiling data for spreadsheet')

	filelist = {}
	qlik_object = re.compile(r"[A-Z\d]+[0-9A-Z_-]+")
	objectname_remover = re.compile(r"AEM[A-Z\d_-]+DC_")
	qlik_env = re.compile(r"[A-Z]+QRP")
	qlik_server = re.compile(r"EDW[A-Z]+")

	onlyfiles = [f for f in os.listdir(export_dir) if os.path.isfile(os.path.join(export_dir, f))]
	for onlyfile in onlyfiles:
		if 'temp.json' in onlyfile or 'SANDBOX' in onlyfile:
			os.remove(onlyfile)
		elif '.json' in onlyfile:
			try:
				match = qlik_object.search(onlyfile)
				object_name = match.group()
				filelist[object_name] = onlyfile
			except Exception:
				print(f'{onlyfile} is empty. Skipping.')
	
	sourcetable_rows = []
	endpoint_rows = []
	tablefilter_rows = []
	tableexpression_rows = []
	columntransformation_rows = []
	transformer_rows = []
	endpoint_parameters = []
	endpoint_names = []
	
	print(f"processing {len(filelist)} files")
		
	for file_name in filelist:
		qlikenv = None
		qlikserver = None
		objectname = None
		fname = f'{file_name}.json'
		try:
			match = qlik_object.search(filelist[file_name])
			tmpobjectname = match.group()
			match = objectname_remover.search(tmpobjectname)
			if match:
				removed = match.group()
				objectname = tmpobjectname.replace(removed,'')
			match = qlik_env.search(filelist[file_name])
			qlikenv = match.group()
			match = qlik_server.search(filelist[file_name])
			qlikserver = match.group()
			filename = open(fname, 'r', encoding="utf-8")
			filedata = json.load(filename)
			if 'CC10' in file_name:
				sleep(2)
			try:
				if 'explicit_included_tables' in filedata['cmd.replication_definition']['tasks'][0]['source']['source_tables']:
					tabledata = filedata['cmd.replication_definition']['tasks'][0]['source']['source_tables']['explicit_included_tables']
					for tableitem in tabledata:
						sourcetable_rows.append([qlikenv, qlikserver, objectname, tableitem['owner'], tableitem['name']])
				if 'manipulations' in filedata['cmd.replication_definition']['tasks'][0]:
					manipulator = filedata['cmd.replication_definition']['tasks'][0]['manipulations']
					for manipulation in manipulator:
						if manipulation['table_manipulation']:
							texpression = ''
							try:
								texpression = manipulation['table_manipulation']['expression']
							except Exception as err:
								pass
							tableexpression_rows.append([qlikenv, qlikserver, objectname, manipulation['table_manipulation']['owner'], manipulation['table_manipulation']['name'], texpression])
							if 'transforms_columns' in manipulation['table_manipulation']:
								if manipulation['table_manipulation']['transforms_columns']:
									columnname = manipulation['table_manipulation']['transforms_columns']
									try:
										for columnname in manipulation['table_manipulation']:
											columntransformation_rows.append([qlikenv, qlikserver, objectname, columnname['owner'], columnname['name'], columnname['new_column_name'],
																			columnname['action'], columnname['new_data_type'], columnname['length'], columnname['expression']])
									except KeyError as e:
										print(e)
							if 'filter_columns' in manipulation['table_manipulation']:
								try:
									for columnname in manipulation['table_manipulation']['filter_columns']:
										tablefilter_rows.append([qlikenv, qlikserver, objectname, manipulation['table_manipulation']['owner'], manipulation['table_manipulation']['name'], columnname['column_name'],columnname['ranges']])
								except KeyError as e:
									print(e)
			except Exception as err:
				print(f'Failure processing table manipulations: {str(err)}')
		except Exception as err:
			print(f'Skipping empty resultset: {str(err)}')
		else:
			try:
				if 'databases' in filedata['cmd.replication_definition']:
					for database in filedata['cmd.replication_definition']['databases']:
						if [qlikenv, qlikserver, database["name"]] in endpoint_names:
							pass
						else:
							if 'DB2ZOS_NATIVE_COMPONENT_TYPE' in database["type_id"]:
								if "SetDataCaptureChanges" in database["db_settings"]:
									endpoint_parameters.append([qlikenv, qlikserver,database["name"],database["role"],database["is_licensed"],database["type_id"],database["db_settings"]["$type"],database["db_settings"]["username"],database["db_settings"]["server"],
									database["db_settings"]["port"],database["db_settings"]["databaseName"],'','','','',database["db_settings"]["SetDataCaptureChanges"],database["db_settings"]["connectMode"],database["db_settings"]["ifi306SpName"], '', ''
									,'','','','','',''])
								else:
									endpoint_parameters.append([qlikenv, qlikserver,database["name"],database["role"],database["is_licensed"],database["type_id"],database["db_settings"]["$type"],database["db_settings"]["username"],database["db_settings"]["server"],
									database["db_settings"]["port"],database["db_settings"]["databaseName"],'','','', '', '', database["db_settings"]["connectMode"],database["db_settings"]["ifi306SpName"], '', '','','','','','',''])
							elif 'SNOWFLAKE_COMPONENT_TYPE' in database["type_id"]:
								if "maxFileSize" in database["db_settings"]:
									if 'loadTimeout' in database['db_settings'] and 'executeTimeout' in database['db_settings']:
										endpoint_parameters.append([qlikenv, qlikserver,database["name"],database["role"],database["is_licensed"],database["type_id"],database["db_settings"]["$type"],database["db_settings"]["username"],database["db_settings"]["server"],
										'',database["db_settings"]["database"],database["db_settings"]["warehouse"],database["db_settings"]["maxFileSize"],database["db_settings"]["metadataschema"],database["db_settings"]["stagingtype"], '', '', '',
										str(database["db_settings"]['loadTimeout']), str(database["db_settings"]['executeTimeout']),'','','','','',''])
									if 'loadTimeout' in database['db_settings'] and 'executeTimeout' in database['db_settings']:
										endpoint_parameters.append([qlikenv, qlikserver,database["name"],database["role"],database["is_licensed"],database["type_id"],database["db_settings"]["$type"],database["db_settings"]["username"],database["db_settings"]["server"],
										'',database["db_settings"]["database"],database["db_settings"]["warehouse"],'',database["db_settings"]["metadataschema"],database["db_settings"]["stagingtype"], '', '', '',
										str(database["db_settings"]['loadTimeout']), str(database["db_settings"]['executeTimeout']),'','','','','',''])
									elif 'loadTimeout' in database['db_settings']:
										endpoint_parameters.append([qlikenv, qlikserver,database["name"],database["role"],database["is_licensed"],database["type_id"],database["db_settings"]["$type"],database["db_settings"]["username"],database["db_settings"]["server"],
										'',database["db_settings"]["database"],database["db_settings"]["warehouse"],'',database["db_settings"]["metadataschema"],database["db_settings"]["stagingtype"], '', '', '',
										str(database["db_settings"]['loadTimeout']), '','','','','','',''])
									elif 'executeTimeout' in database['db_settings']:
										endpoint_parameters.append([qlikenv, qlikserver,database["name"],database["role"],database["is_licensed"],database["type_id"],database["db_settings"]["$type"],database["db_settings"]["username"],database["db_settings"]["server"],
										'',database["db_settings"]["database"],database["db_settings"]["warehouse"],'',database["db_settings"]["metadataschema"],database["db_settings"]["stagingtype"], '', '', '',
										'', str(database["db_settings"]['executeTimeout']),'','','','','',''])
									else:
										endpoint_parameters.append([qlikenv, qlikserver,database["name"],database["role"],database["is_licensed"],database["type_id"],database["db_settings"]["$type"],database["db_settings"]["username"],database["db_settings"]["server"],
										'',database["db_settings"]["database"],database["db_settings"]["warehouse"],'',database["db_settings"]["metadataschema"],database["db_settings"]["stagingtype"],
										'', '', '', '', '','','','','','',''])
							elif 'ORACLE_COMPONENT_TYPE' in database["type_id"]:
								if 'addSupplementalLogging' in database['db_settings'] and 'parallelASMReadThreads' in database['db_settings']:
									endpoint_parameters.append([qlikenv, qlikserver,database["name"],database["role"],database["is_licensed"],database["type_id"],database["db_settings"]["$type"],database["db_settings"]["username"],database["db_settings"]["server"],
									'','','','','','','', '', '', '', '',database["db_settings"]["addSupplementalLogging"],database["db_settings"]["accessAlternateDirectly"],str(database["db_settings"]["parallelASMReadThreads"]),'','',''])
								elif 'addSupplementalLogging' in database['db_settings']:
									endpoint_parameters.append([qlikenv, qlikserver,database["name"],database["role"],database["is_licensed"],database["type_id"],database["db_settings"]["$type"],database["db_settings"]["username"],database["db_settings"]["server"],
									'','','','','','','', '', '', '', '',database["db_settings"]["addSupplementalLogging"],database["db_settings"]["accessAlternateDirectly"],'',''])
								elif 'parallelASMReadThreads' in database['db_settings']:
									endpoint_parameters.append([qlikenv, qlikserver,database["name"],database["role"],database["is_licensed"],database["type_id"],database["db_settings"]["$type"],database["db_settings"]["username"],database["db_settings"]["server"],
									'','','','','','','', '', '', '', '','',database["db_settings"]["accessAlternateDirectly"],str(database["db_settings"]["parallelASMReadThreads"]),'','',''])
								else:
									endpoint_parameters.append([qlikenv, qlikserver,database["name"],database["role"],database["is_licensed"],database["type_id"],database["db_settings"]["$type"],database["db_settings"]["username"],database["db_settings"]["server"],
									'','','','','','','', '', '', '', '','',database["db_settings"]["accessAlternateDirectly"],'',''])
							elif 'LOG_STREAM_COMPONENT_TYPE' in database["type_id"]:
								try:
									if database["db_settings"]["retentionmaxagehours"]:
										maxdays = '( ' + str(int(database["db_settings"]["retentionmaxagehours"]/24)) + ' Days )'
										maxhours = str(database["db_settings"]["retentionmaxagehours"]) + ' Hours'
									endpoint_parameters.append([qlikenv, qlikserver,database["name"],database["role"],database["is_licensed"],database["type_id"],database["db_settings"]["$type"],'','','','','','','','','', '', '', '', '','','','',
									database["db_settings"]["path"],maxhours, maxdays])
								except Exception as err:
									pass
						endpoint_names.append([qlikenv, qlikserver, database["name"]])
			except Exception as err:
				print(f'{str(err)}')
			try:
				if 'global_manipulation' in filedata['cmd.replication_definition']['tasks'][0]:
					for rowitem in filedata['cmd.replication_definition']['tasks'][0]['global_manipulation']['rules']:
						for i in rowitem:
							try:
								if rowitem['column']['where_column_name']:
									pass
							except:
								rowitem['column']['where_column_name'] = ''
							try:
								if rowitem['column']['new_data_type']:
									pass
							except:
								rowitem['column']['new_data_type'] = ''
							try:
								if rowitem['column']['new_length']:
									pass
							except:
								rowitem['column']['new_length'] = ''
							try:
								if rowitem['column']['new_precision']:
									pass
							except:
								rowitem['column']['new_precision'] = ''
							try:
								if rowitem['column']['new_scale']:
									pass
							except:
								rowitem['column']['new_scale'] = ''
							try:
								if rowitem['column']['computation_expression']:
									pass
							except:
								rowitem['column']['computation_expression'] = ''
						transformer_rows.append([qlikenv, qlikserver, objectname,rowitem['action'],rowitem['name'],rowitem['expression'],rowitem['column']['where_column_name'],
												rowitem['column']['new_data_type'],rowitem['column']['new_length'],rowitem['column']['new_precision'],rowitem['column']['new_scale'],
												rowitem['column']['computation_expression'],rowitem['sub_action']])
				if 'source' in filedata['cmd.replication_definition']['tasks'][0]:
						fullload = ''
						applychanges = ''
						savechanges = ''
						batchapply = ''
						minbatchapply = ''

						try:
							fullload = filedata['cmd.replication_definition']['tasks'][0]['task_settings']['common_settings']['full_load_enabled']
						except:
							pass
						try:
							applychanges =  filedata['cmd.replication_definition']['tasks'][0]['task_settings']['common_settings']['apply_changes_enabled']
						except:
							pass
						try:
							savechanges = filedata['cmd.replication_definition']['tasks'][0]['task_settings']['common_settings']['save_changes_enabled']
						except:
							pass
						try:
							batchapply = filedata['cmd.replication_definition']['tasks'][0]['task_settings']['common_settings']['batch_apply_timeout']
						except:
							pass
						try:
							minbatchapply = filedata['cmd.replication_definition']['tasks'][0]['task_settings']['common_settings']['batch_apply_timeout_min']
						except:
							pass
						endpoint_rows.append([qlikenv, qlikserver, objectname, filedata['cmd.replication_definition']['tasks'][0]['source']['rep_source']['source_name'], filedata['cmd.replication_definition']['tasks'][0]['source']['rep_source']['database_name'],
											filedata['cmd.replication_definition']['tasks'][0]['targets'][0]['rep_target']['target_name'], filedata['cmd.replication_definition']['tasks'][0]['targets'][0]['rep_target']['target_state'],
											filedata['cmd.replication_definition']['tasks'][0]['targets'][0]['rep_target']['database_name'], fullload, applychanges, savechanges, batchapply, minbatchapply])
			except Exception as err:
				print(f'{str(err)}')
			filename.close()

	for file_name in filelist:
		file_name = file_name + '.json'
		try:
			os.unlink(file_name)
		except:
			print(f"Unable to remove {file_name}")

	print('Compiling spreadsheet')


	sourcetables = 'source_tables.csv'
	with open(sourcetables, 'w', newline='') as csvfile:
		csvwriter = csv.writer(csvfile)
		csvwriter.writerows(sourcetable_rows)
	
	endpoinitdata = 'endpoints.csv'
	with open(endpoinitdata, 'w', newline='') as csvfile:
		csvwriter = csv.writer(csvfile)
		csvwriter.writerows(endpoint_rows)
	
	tablefilters = 'tablefilters.csv'
	with open(tablefilters, 'w', newline='') as csvfile:
		csvwriter = csv.writer(csvfile)
		csvwriter.writerows(tablefilter_rows)
	
	tableexpressions = 'table_expressions.csv'
	with open(tableexpressions, 'w', newline='') as csvfile:
		csvwriter = csv.writer(csvfile)
		csvwriter.writerows(tableexpression_rows)
	
	columntransformations = 'column_transformations.csv'
	with open(columntransformations, 'w', newline='') as csvfile:
		csvwriter = csv.writer(csvfile)
		csvwriter.writerows(columntransformation_rows)
	
	transformerdata = 'transformer.csv'
	with open(transformerdata, 'w', newline='') as csvfile:
		csvwriter = csv.writer(csvfile)
		csvwriter.writerows(transformer_rows)

	parameterdata = 'endpoint_parameters.csv'
	with open(parameterdata, 'w', newline='') as csvfile:
		csvwriter = csv.writer(csvfile)
		csvwriter.writerows(endpoint_parameters)

	workbook  = xlsxwriter.Workbook('qlik_settings.xlsx')
	worksheet1 = workbook.add_worksheet('Source Tables')
	worksheet2 = workbook.add_worksheet('Global Settings')
	worksheet3 = workbook.add_worksheet('Global Transformations')
	worksheet4 = workbook.add_worksheet('Table Expressions')
	worksheet5 = workbook.add_worksheet('Task Information')
	worksheet6 = workbook.add_worksheet('Table Filters')
	worksheet7 = workbook.add_worksheet('Endpoint Parameters')
	header_format = workbook.add_format({'bg_color': '#000000', 'font_color': '#FFFFFF', 'bold': True, 'align': 'center'}) 
	false_format = workbook.add_format({'bg_color': '#FF0000', 'font_color': '#FFFFFF'})
	true_format = workbook.add_format({'bg_color': '#3EA055', 'font_color': '#FFFFFF'})
	greyline = workbook.add_format({'bg_color': '#98AFC7'})
	maroon_bg = workbook.add_format({'bg_color': '#728FCE'})
	yellow_bg = workbook.add_format({'bg_color': '#FFFF00'})
	lime_bg = workbook.add_format({'bg_color': '#00FF00'})
	
	worksheet1.write('A1', 'Environment', header_format)
	worksheet1.write('B1', 'Server', header_format)
	worksheet1.write('C1', 'Object_Name', header_format)
	worksheet1.write('D1', 'Owner', header_format)
	worksheet1.write('E1', 'Name', header_format)         
	worksheet1.write('F1', 'Reviewed', header_format)
	worksheet1.write('G1', 'Match', header_format)         
	
	worksheet2.write('A1', 'Environment', header_format)
	worksheet2.write('B1', 'Server', header_format)
	worksheet2.write('C1', 'Category', header_format)
	worksheet2.write('D1', 'Sub-Category', header_format)
	worksheet2.write('E1', 'Option', header_format)
	worksheet2.write('F1', 'Enabled', header_format)
	worksheet2.write('G1', 'Value', header_format)
	worksheet2.write('H1', 'Condition', header_format)
	worksheet2.write('I1', 'Send Message To', header_format)
	worksheet2.write('J1', 'Tasks', header_format)
	worksheet2.write('K1', 'Active', header_format)
	worksheet2.write('L1', 'Reviewed', header_format)
	worksheet2.write('M1', 'Match', header_format)         
	
	worksheet3.write('A1', 'Environment', header_format)
	worksheet3.write('B1', 'Server', header_format)
	worksheet3.write('C1', 'Object Name', header_format)
	worksheet3.write('D1', 'Action', header_format)
	worksheet3.write('E1', 'Name', header_format)
	worksheet3.write('F1', 'Expression', header_format)
	worksheet3.write('G1', 'Where_column_name', header_format)
	worksheet3.write('H1', 'new_data_type', header_format)
	worksheet3.write('I1', 'new_length', header_format)
	worksheet3.write('J1', 'new_precision', header_format)
	worksheet3.write('K1', 'new_scale', header_format)
	worksheet3.write('L1', 'computation_expression', header_format)
	worksheet3.write('M1', 'sub_action', header_format)
	worksheet3.write('N1', 'Reviewed', header_format)
	worksheet3.write('O1', 'Match', header_format)         
	
	worksheet4.write('A1', 'Environment', header_format)
	worksheet4.write('B1', 'Server', header_format)
	worksheet4.write('C1', 'Object Name', header_format)
	worksheet4.write('D1', 'Owner', header_format)
	worksheet4.write('E1', 'Name', header_format)
	worksheet4.write('F1', 'Expression', header_format)
	worksheet4.write('G1', 'Reviewed', header_format)
	worksheet4.write('H1', 'Match', header_format)         
	
	worksheet5.write('A1', 'Environment', header_format)
	worksheet5.write('B1', 'Server', header_format)
	worksheet5.write('C1', 'Object Name', header_format)
	worksheet5.write('D1', 'Source_name', header_format)
	worksheet5.write('E1', 'Source_database_name', header_format)
	worksheet5.write('F1', 'Target_name', header_format)
	worksheet5.write('G1', 'Target_state', header_format)
	worksheet5.write('H1', 'Target_database_name', header_format)
	worksheet5.write('I1', 'Full_load_enabled', header_format)
	worksheet5.write('J1', 'Apply_changes_enabled', header_format)
	worksheet5.write('K1', 'Save_changes_enabled', header_format)
	worksheet5.write('L1', 'Batch_apply_timeout', header_format)
	worksheet5.write('M1', 'Batch_apply_timeout_min', header_format)
	worksheet5.write('N1', 'Reviewed', header_format)
	worksheet5.write('O1', 'Match', header_format)         
	
	worksheet6.write('A1', 'Environment', header_format)
	worksheet6.write('B1', 'Server', header_format)
	worksheet6.write('C1', 'Object Name', header_format)
	worksheet6.write('D1', 'Owner', header_format)
	worksheet6.write('E1', 'Name', header_format)
	worksheet6.write('F1', 'Column_name', header_format)
	worksheet6.write('G1', 'Ranges', header_format)
	worksheet6.write('H1', 'Reviewed', header_format)
	worksheet6.write('I1', 'Match', header_format)         
	
	worksheet7.write('A1', 'environment', header_format)
	worksheet7.write('B1', 'server', header_format)
	worksheet7.write('C1', 'name', header_format)
	worksheet7.write('D1', 'role', header_format)
	worksheet7.write('E1', 'is_licensed', header_format)
	worksheet7.write('F1', 'type_id', header_format)
	worksheet7.write('G1', 'type', header_format)
	worksheet7.write('H1', 'username', header_format)
	worksheet7.write('I1', 'server', header_format)
	worksheet7.write('J1', 'port', header_format)
	worksheet7.write('K1', 'database', header_format)
	worksheet7.write('L1', 'warehouse', header_format)
	worksheet7.write('M1', 'maxFileSize', header_format)
	worksheet7.write('N1', 'metadataschema', header_format)
	worksheet7.write('O1', 'stagingtype', header_format)
	worksheet7.write('P1', 'SetDataCaptureChanges', header_format)
	worksheet7.write('Q1', 'connectMode', header_format)
	worksheet7.write('R1', 'ifi306SpName', header_format)
	worksheet7.write('S1', 'loadTimeout', header_format)
	worksheet7.write('T1', 'executeTimeout', header_format)
	worksheet7.write('U1', 'addSupplementalLogging', header_format)
	worksheet7.write('V1', 'accessAlternateDirectly', header_format)
	worksheet7.write('W1', 'parallelASMReadThreads', header_format)
	worksheet7.write('X1', 'path', header_format)
	worksheet7.write('Y1', 'retentionmaxagehours', header_format)
	worksheet7.write('Z1', 'retentionmaxagedays', header_format)
	worksheet7.write('AA1', 'Reviewed', header_format)
	worksheet7.write('AB1', 'Match', header_format)         
		
	with open('source_tables.csv', 'rt', encoding='utf8') as f:
			reader = csv.reader(f)
			for r, row in enumerate(reader):
				r = r + 1
				remainder = r % 2
				for c, col in enumerate(row):
					if remainder == 0:
						worksheet1.write(r, c, col, greyline)
					else:
						worksheet1.write(r, c, col)
    
	worksheet1.set_column('A:A', 17)	
	worksheet1.set_column('B:B', 11)
	worksheet1.set_column('C:C', 35)	
	worksheet1.set_column('D:D', 23)
	worksheet1.set_column('E:E', 37)
	worksheet1.set_column('F:F', 14)
	worksheet1.set_column('G:G', 14)
	# Set the autofilter.
	worksheet1.autofilter(0, 0,3000, 6)
	worksheet1.conditional_format('A1:A3000', {'type': 'text', 'criteria': 'containing', 'value': 'DEVQRP', 'format': maroon_bg})
	worksheet1.conditional_format('A1:A3000', {'type': 'text', 'criteria': 'containing', 'value': 'QAQRP', 'format': yellow_bg})
	worksheet1.conditional_format('A1:A3000', {'type': 'text', 'criteria': 'containing', 'value': 'PRDQRP', 'format': lime_bg})
	
	with open('global_settings.csv', 'rt', encoding='utf8') as f:
			reader = csv.reader(f)
			for r, row in enumerate(reader):
				r = r + 1
				remainder = r % 2
				for c, col in enumerate(row):
					if remainder == 0:
						worksheet2.write(r, c, col, greyline)
					else:
						worksheet2.write(r, c, col)
	
	worksheet2.set_column('A:A', 17)	
	worksheet2.set_column('B:B', 11)
	worksheet2.set_column('C:C', 20)	
	worksheet2.set_column('D:D', 32)
	worksheet2.set_column('E:E', 71)
	worksheet2.set_column('F:F', 17)
	worksheet2.set_column('G:G', 43)
	worksheet2.set_column('H:H', 99)
	worksheet2.set_column('I:I', 23)
	worksheet2.set_column('J:J', 48)
	worksheet2.set_column('K:K', 26)
	worksheet2.set_column('L:L', 14)
	worksheet2.set_column('M:M', 14)
	worksheet2.autofilter(0, 0,3000, 12)
	worksheet2.conditional_format('A1:A3000', {'type': 'text', 'criteria': 'containing', 'value': 'DEVQRP', 'format': maroon_bg})
	worksheet2.conditional_format('A1:A3000', {'type': 'text', 'criteria': 'containing', 'value': 'QAQRP', 'format': yellow_bg})
	worksheet2.conditional_format('A1:A3000', {'type': 'text', 'criteria': 'containing', 'value': 'PRDQRP', 'format': lime_bg})

	with open('transformer.csv', 'rt', encoding='utf8') as f:
			reader = csv.reader(f)
			for r, row in enumerate(reader):
				r = r + 1
				remainder = r % 2
				for c, col in enumerate(row):
					if remainder == 0:
						worksheet3.write(r, c, col, greyline)
					else:
						worksheet3.write(r, c, col)
	
	worksheet3.set_column('A:A', 18)	
	worksheet3.set_column('B:B', 11)
	worksheet3.set_column('C:C', 18)	
	worksheet3.set_column('D:D', 32)
	worksheet3.set_column('E:E', 26)
	worksheet3.set_column('F:F', 36)
	worksheet3.set_column('G:G', 25)
	worksheet3.set_column('H:H', 27)
	worksheet3.set_column('I:I', 18)
	worksheet3.set_column('J:J', 18)
	worksheet3.set_column('K:K', 18)
	worksheet3.set_column('L:L', 80)
	worksheet3.set_column('M:M', 20)
	worksheet3.set_column('N:N', 14)
	worksheet3.set_column('O:O', 14)
	worksheet3.autofilter(0, 0, 3000, 14)
	worksheet3.conditional_format('A1:A3000', {'type': 'text', 'criteria': 'containing', 'value': 'DEVQRP', 'format': maroon_bg})
	worksheet3.conditional_format('A1:A3000', {'type': 'text', 'criteria': 'containing', 'value': 'QAQRP', 'format': yellow_bg})
	worksheet3.conditional_format('A1:A3000', {'type': 'text', 'criteria': 'containing', 'value': 'PRDQRP', 'format': lime_bg})
	
	with open('table_expressions.csv', 'rt', encoding='utf8') as f:
			reader = csv.reader(f)
			for r, row in enumerate(reader):
				r = r + 1
				remainder = r % 2
				for c, col in enumerate(row):
					if remainder == 0:
						worksheet4.write(r, c, col, greyline)
					else:
						worksheet4.write(r, c, col)
	
	worksheet4.set_column('A:A', 18)	
	worksheet4.set_column('B:B', 11)
	worksheet4.set_column('C:C', 18)	
	worksheet4.set_column('D:D', 23)
	worksheet4.set_column('E:E', 36)
	worksheet4.set_column('F:F', 255)
	worksheet4.set_column('G:G', 14)
	worksheet4.set_column('H:H', 14)
	worksheet4.autofilter(0, 0,  3000, 7)
	worksheet4.conditional_format('A1:A3000', {'type': 'text', 'criteria': 'containing', 'value': 'DEVQRP', 'format': maroon_bg})
	worksheet4.conditional_format('A1:A3000', {'type': 'text', 'criteria': 'containing', 'value': 'QAQRP', 'format': yellow_bg})
	worksheet4.conditional_format('A1:A3000', {'type': 'text', 'criteria': 'containing', 'value': 'PRDQRP', 'format': lime_bg})
	
	with open('endpoints.csv', 'rt', encoding='utf8') as f:
			reader = csv.reader(f)
			for r, row in enumerate(reader):
				r = r + 1
				remainder = r % 2
				for c, col in enumerate(row):
					if remainder == 0:
						worksheet5.write(r, c, col, greyline)
					else:
						worksheet5.write(r, c, col)
	
	worksheet5.set_column('A:A', 18)	
	worksheet5.set_column('B:B', 11)
	worksheet5.set_column('C:C', 35)	
	worksheet5.set_column('D:E', 39)
	worksheet5.set_column('F:F', 36)
	worksheet5.set_column('G:G', 16)
	worksheet5.set_column('H:H', 36)
	worksheet5.set_column('I:I', 22)
	worksheet5.set_column('J:J', 27)
	worksheet5.set_column('K:K', 26)
	worksheet5.set_column('L:L', 24)
	worksheet5.set_column('M:M', 29)
	worksheet5.set_column('N:N', 14)
	worksheet5.set_column('O:O', 14)
	worksheet5.set_row(0, None, header_format)
	worksheet5.conditional_format('G1:G3000', {'type': 'text', 'criteria': 'containing', 'value': 'DISABLED', 'format': false_format})
	worksheet5.conditional_format('G1:G3000', {'type': 'text', 'criteria': 'containing', 'value': 'ENABLED', 'format': true_format})
	worksheet5.conditional_format('I1:K3000', {'type': 'text', 'criteria': 'containing', 'value': 'FALSE', 'format': false_format})
	worksheet5.conditional_format('I1:K3000', {'type': 'text', 'criteria': 'containing', 'value': 'TRUE', 'format': true_format})
	worksheet5.conditional_format('A1:A3000', {'type': 'text', 'criteria': 'containing', 'value': 'DEVQRP', 'format': maroon_bg})
	worksheet5.conditional_format('A1:A3000', {'type': 'text', 'criteria': 'containing', 'value': 'QAQRP', 'format': yellow_bg})
	worksheet5.conditional_format('A1:A3000', {'type': 'text', 'criteria': 'containing', 'value': 'PRDQRP', 'format': lime_bg})
	worksheet5.autofilter(0, 0,  3000, 14)
	
	with open('tablefilters.csv', 'rt', encoding='utf8') as f:
			reader = csv.reader(f)
			for r, row in enumerate(reader):
				r = r + 1
				remainder = r % 2
				for c, col in enumerate(row):
					if remainder == 0:
						worksheet6.write(r, c, col, greyline)
					else:
						worksheet6.write(r, c, col)

	worksheet6.set_column('A:A', 18)	
	worksheet6.set_column('B:B', 11)
	worksheet6.set_column('C:C', 18)	
	worksheet6.set_column('D:D', 11)
	worksheet6.set_column('E:E', 36)
	worksheet6.set_column('F:F', 18)
	worksheet6.set_column('G:G', 36)
	worksheet6.set_column('H:H', 14)
	worksheet6.set_column('I:I', 14)
	worksheet6.autofilter(0, 0,  3000, 7)
	worksheet6.conditional_format('A1:A3000', {'type': 'text', 'criteria': 'containing', 'value': 'DEVQRP', 'format': maroon_bg})
	worksheet6.conditional_format('A1:A3000', {'type': 'text', 'criteria': 'containing', 'value': 'QAQRP', 'format': yellow_bg})
	worksheet6.conditional_format('A1:A3000', {'type': 'text', 'criteria': 'containing', 'value': 'PRDQRP', 'format': lime_bg})

	with open('endpoint_parameters.csv', 'rt', encoding='utf8') as f:
			reader = csv.reader(f)
			for r, row in enumerate(reader):
				r = r + 1
				remainder = r % 2
				for c, col in enumerate(row):
					if remainder == 0:
						worksheet7.write(r, c, col, greyline)
					else:
						worksheet7.write(r, c, col)

	worksheet7.set_column('A:A', 17)	
	worksheet7.set_column('B:B', 11)
	worksheet7.set_column('C:C', 35)	
	worksheet7.set_column('D:D', 34)
	worksheet7.set_column('E:E', 23)
	worksheet7.set_column('F:F', 34)
	worksheet7.set_column('G:G', 22)
	worksheet7.set_column('H:H', 38)
	worksheet7.set_column('I:I', 203)
	worksheet7.set_column('J:J', 10)
	worksheet7.set_column('K:K', 26)
	worksheet7.set_column('L:L', 24)
	worksheet7.set_column('M:M', 29)
	worksheet7.set_column('N:N', 26)
	worksheet7.set_column('O:O', 24)
	worksheet7.set_column('P:P', 29)
	worksheet7.set_column('Q:Q', 15)
	worksheet7.set_column('R:R', 15)
	worksheet7.set_column('S:S', 24)
	worksheet7.set_column('T:T', 24)
	worksheet7.set_column('U:U', 24)
	worksheet7.set_column('V:V', 24)
	worksheet7.set_column('W:W', 24)
	worksheet7.set_column('X:X', 24)
	worksheet7.set_column('Y:Y', 24)
	worksheet7.set_column('Z:Z', 25)
	worksheet7.set_column('Y:Y', 24)
	worksheet7.set_column('Z:Z', 25)
	worksheet7.set_column('AA:AA', 14)
	worksheet7.set_column('AB:AB', 14)
	worksheet7.autofilter(0, 0,  3000, 27)
	worksheet7.conditional_format('A1:A3000', {'type': 'text', 'criteria': 'containing', 'value': 'DEVQRP', 'format': maroon_bg})
	worksheet7.conditional_format('A1:A3000', {'type': 'text', 'criteria': 'containing', 'value': 'QAQRP', 'format': yellow_bg})
	worksheet7.conditional_format('A1:A3000', {'type': 'text', 'criteria': 'containing', 'value': 'PRDQRP', 'format': lime_bg})
	worksheet7.conditional_format('D1:D3000', {'type': 'text', 'criteria': 'containing', 'value': 'SOURCE', 'format': true_format})
	worksheet7.conditional_format('D1:D3000', {'type': 'text', 'criteria': 'containing', 'value': 'TARGET', 'format': yellow_bg})
	worksheet7.conditional_format('E1:E3000', {'type': 'text', 'criteria': 'containing', 'value': 'TRUE', 'format': true_format})
	worksheet7.conditional_format('E1:E3000', {'type': 'text', 'criteria': 'containing', 'value': 'FALSE', 'format': false_format})	
		
	workbook.close()
	
	os.unlink('source_tables.csv')
	os.unlink('transformer.csv')
	os.unlink('table_expressions.csv')
	os.unlink('endpoints.csv')
	os.unlink('tablefilters.csv')
	os.unlink('column_transformations.csv')
	os.unlink('global_settings.csv')
	os.unlink('endpoint_parameters.csv')

	print('\nDONE.')
### END of function MAIN_USE
##############################################################


if __name__ == '__main__':
	main()