adding work scripts

work/size_check/settings.ini

@@ -0,0 +1,55 @@
+[global]
+mailhost = smtprelay.lmig.com
+sender = Qlik_Admin@LibertyMutual.com
+receivers = wendell.jones@LibertyMutual.com
+query = SELECT TABLE_SCHEMA, TABLE_NAME, CONCAT(BYTES || ' BYTES'), CONCAT(CAST(ROUND(BYTES/(1024*1024*1024)) AS INTEGER) || ' GB') FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA LIKE 'PM_%' GROUP BY TABLE_SCHEMA, TABLE_NAME, BYTES
+#query = SELECT TABLE_SCHEMA, TABLE_NAME, CONCAT(BYTES || ' BYTES'), CONCAT(CAST(ROUND(BYTES/(1024*1024*1024)) AS INTEGER) || ' GB') FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA LIKE 'PM_%' and BYTES > 1200000000 GROUP BY TABLE_SCHEMA, TABLE_NAME, BYTES
+
+[sf_targets]
+;01=PL_DEV,PL_DEV_QLIK_PM_EDW_PSA_D,UWtDZzczNkg=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_EDW_PSA_D,PM_EDW_PSA_D
+;02=PL_DEV,PL_DEV_QLIK_PM_EDW_TRANS_D,UXBBaFd5Nno=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_EDW_TRANS_QLIK_CT_D,PM_EDW_TRANS_NRT_SD_D
+;03=PL_DEV,PL_DEV_QLIK_PM_ODS_FIN_FINX_D,VTZuWTdBR3k=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_ODS_FIN_FINX_QLIK_D,PM_ODS_FIN_FINX_D
+;04=PL_DEV,PL_DEV_QLIK_PM_ODS_VISION_D,NUgzWGFuVE0=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_ODS_VISION_QLIK_CT_D,PM_ODS_VISION_D
+;05=PL_DEV,PL_DEV_QLIK_PM_ODS_VISION_TRANS_D,ejY1S3RIZk0=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_ODS_VISION_TRANS_QLIK_CT_D,PM_ODS_VISION_TRANS_D
+;06=PL_DEV,PL_DEV_QLIK_PM_TRANS_AVAYA_REPO_D,WjgzSm83QzQ=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_TRANS_AVAYA_REPO_QLIK_CT_D,PM_TRANS_AVAYA_REPO_D
+;07=PL_DEV,PM_TRANS_CL_AB_ALT_QLIK_CT_D,aDdBSE9Vclc=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_TRANS_CL_AB_ALT_QLIK_CT_D,PM_TRANS_CL_AB_D
+;08=PL_DEV,PM_TRANS_CL_CC_PRL_QLIK_CT_D,S2o5OEVmWEo=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_TRANS_CL_CC_PRL_QLIK_CT_D,PM_TRANS_CL_CC_PRL_D
+;09=PL_DEV,PM_TRANS_CL_EPALARM_QLIK_CT_D,dFdRNWdXeUM=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_TRANS_CL_EPALARM_QLIK_CT_D,PM_TRANS_CL_EPALARM_D
+;10=PL_DEV,PM_TRANS_CL_EPCUST_QLIK_CT_D,RXJRUjlZS0o=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_TRANS_CL_EPCUST_QLIK_CT_D,PM_TRANS_CL_EPCUST_D
+;11=PL_DEV,PM_TRANS_CL_ICREPO_QLIK_CT_D,cDJTWldwQXE=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_TRANS_CL_ICREPO_QLIK_CT_D,PM_TRANS_CL_ICREPO_D
+;12=PL_DEV,PM_TRANS_CL_OAHIST_QLIK_CT_D,NlJ5eFhBNXE=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_TRANS_CL_OAHIST_QLIK_CT_D,PM_TRANS_CL_OAHIST_D
+13=DEV,PL_DEV_QLIK_PM_TRANS_CMS_D,ZDdkODc0V1I=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_TRANS_CMS_QLIK_CT_D
+14=DEV,PL_DEV_QLIK_PM_TRANS_SALES_COMP_D,S3U4djJFUWI=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_TRANS_SALES_COMP_QLIK_CT_D
+15=DEV,PL_DEV_QLIK_PM_TRANS_VH_D,Y0JVSHltOWU=,libertymutual_grm_us_edw.us-east-1,PL_DEV_QLIK_LOAD_WH,PM_TRANS_VH_QLIK_CT_D
+16=QA,PL_QA_QLIK_PM_EDW_PSA_D,cjlMUXRQRE4=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_EDW_PSA_D
+17=QA,PL_QA_QLIK_PM_EDW_TRANS_D,T01JN3RDYko=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_EDW_TRANS_NRT_SD_QLIK_CT_D
+18=QA,PL_QA_QLIK_PM_EDW_TRANS_NRT_SD_D,bTc5QjdTRmk=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_EDW_TRANS_QLIK_CT_D
+19=QA,PL_QA_QLIK_PM_ODS_FIN_FINX_D,dXZhZE04OXk=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_ODS_FIN_FINX_QLIK_D
+20=QA,PL_QA_QLIK_PM_ODS_VISION_D,c3FoNVZ4R1Y=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_ODS_VISION_QLIK_CT_D
+21=QA,PL_QA_QLIK_PM_ODS_VISION_TRANS_D,dndBSjQwSks=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_ODS_VISION_TRANS_QLIK_CT_D
+22=QA,PL_QA_QLIK_PM_TRANS_AVAYA_REPO_D,WVI2N2VlVWQ=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_TRANS_AVAYA_REPO_QLIK_CT_D
+23=QA,PL_QA_QLIK_PM_TRANS_CL_AB_D,aXI2WFFjYUQ=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_TRANS_CL_AB_ALT_QLIK_CT_D
+24=QA,PL_QA_QLIK_PM_TRANS_CL_CC_PRL_D,WlE1alpGeXU=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_TRANS_CL_CC_PRL_QLIK_CT_D
+25=QA,PL_QA_QLIK_PM_TRANS_CL_EPALARM_D,YjZTbzFpSVE=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_TRANS_CL_EPALARM_QLIK_CT_D
+26=QA,PL_QA_QLIK_PM_TRANS_CL_EPCUST_D,VDZQU0hEeko=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_TRANS_CL_EPCUST_QLIK_CT_D
+27=QA,PL_QA_QLIK_PM_TRANS_CL_ICREPO_D,STZUNnFkMVA=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_TRANS_CL_ICREPO_QLIK_CT_D
+28=QA,PL_QA_QLIK_PM_TRANS_CL_OAHIST_D,QW1Id0oxQTM=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_TRANS_CL_OAHIST_QLIK_CT_D
+29=QA,PL_QA_QLIK_PM_TRANS_CMS_D,bHFUenNMM1A=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_TRANS_CMS_QLIK_CT_D
+31=QA,PL_QA_QLIK_PM_TRANS_SALES_COMP_D,aER1cDRraVA=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_TRANS_SALES_COMP_QLIK_CT_D
+32=QA,PL_QA_QLIK_PM_TRANS_VH_D,S0hxakxhOUQ=,libertymutual_grm_us_edw.us-east-1,PL_QA_QLIK_LOAD_WH,PM_TRANS_VH_QLIK_CT_D
+33=PROD,PL_PROD_QLIK_PM_EDW_PSA_D,QVE0eEFkaEI=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_EDW_PSA_D
+34=PROD,PL_PROD_QLIK_PM_EDW_TRANS_D,bUR4MjV5Ulk=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_EDW_TRANS_NRT_SD_QLIK_CT_D
+35=PROD,PL_PROD_QLIK_PM_EDW_TRANS_NRT_SD_D,b1EzZFdTVHg=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_EDW_TRANS_QLIK_CT_D
+36=PROD,PL_PROD_QLIK_PM_ODS_FIN_FINX_D,aERweDNiaVg=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_ODS_FIN_FINX_QLIK_D
+37=PROD,PL_PROD_QLIK_PM_ODS_VISION_D,U0tsOWpzVEU=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_ODS_VISION_QLIK_CT_D
+38=PROD,PL_PROD_QLIK_PM_ODS_VISION_TRANS_D,aTZmMVB4TU8=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_ODS_VISION_TRANS_QLIK_CT_D
+39=PROD,PL_PROD_QLIK_PM_TRANS_AVAYA_REPO_D,bjdQQWtvcm4=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_TRANS_AVAYA_REPO_QLIK_CT_D
+40=PROD,PL_PROD_QLIK_PM_TRANS_CL_AB_D,bHc5RmRSTTI=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_TRANS_CL_AB_ALT_QLIK_CT_D
+41=PROD,PL_PROD_QLIK_PM_TRANS_CL_CC_PRL_D,YmdDYTRNWE8=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_TRANS_CL_CC_PRL_QLIK_CT_D
+42=PROD,PL_PROD_QLIK_PM_TRANS_CL_EPALARM_D,cjlDbm12Yjg=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_TRANS_CL_EPALARM_QLIK_CT_D
+43=PROD,PL_PROD_QLIK_PM_TRANS_CL_EPCUST_D,WjVqTDlySHU=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_TRANS_CL_EPCUST_QLIK_CT_D
+44=PROD,PL_PROD_QLIK_PM_TRANS_CL_ICREPO_D,TTB6VTNVcGQ=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_TRANS_CL_ICREPO_QLIK_CT_D
+45=PROD,PL_PROD_QLIK_PM_TRANS_CL_OAHIST_D,Tko5UlZRelc=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_TRANS_CL_OAHIST_QLIK_CT_D
+46=PROD,PL_PROD_QLIK_PM_TRANS_CMS_D,Wjd2OE9EaVc=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_TRANS_CMS_QLIK_CT_D
+47=PROD,PL_PROD_QLIK_PM_TRANS_SALES_COMP_D,NnFNYjNydjA=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_TRANS_SALES_COMP_QLIK_CT_D
+48=PROD,PL_PROD_QLIK_PM_TRANS_VH_D,bzQ3YWVaaEU=,libertymutual_grm_us_edw.us-east-1,PL_PROD_QLIK_LOAD_WH,PM_TRANS_VH_QLIK_CT_D

work/size_check/sf_size_check.py

@@ -0,0 +1,89 @@
+"""
+Copyright 2018 Attunity
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+# coding: utf-8
+import os, datetime, socket
+from snowflake import connector
+import base64
+import configparser
+
+CONFIG = configparser.RawConfigParser(allow_no_value=True)
+CONFIG.read('settings.ini')
+sf_logins = []
+for option, value in CONFIG.items('sf_targets'):
+    itemlist = value.split(',')
+    pw = bytes(itemlist[2], 'UTF8')
+    npw = base64.b64decode(pw).decode('UTF8')
+    sf_logins.append([itemlist[0],itemlist[1],npw,itemlist[3],itemlist[4],itemlist[5]])
+
+nowstamp = datetime.datetime.now()
+script_name = os.path.basename(__file__)
+object_list = []
+sfquery = CONFIG.get('global', 'query')
+sfdict = {}
+        
+print(f'Starttime: {nowstamp}')
+updaterange = len(sf_logins)
+total_bytes = 0
+seperator = ";"
+csvfile = open('results.csv', 'w')
+for i in range(updaterange):
+    login_name=sf_logins[i][1]
+    login_password=sf_logins[i][2]
+    sf_account=sf_logins[i][3]
+    sf_warehouse=sf_logins[i][4]
+    sf_database=f'PL_{sf_logins[i][0]}'
+    sf_schema=sf_logins[i][5]
+    sf_env = sf_logins[i][0]
+    counter = 0
+    try:
+        conn = connector.connect(
+            user=login_name,
+            password=str(login_password),
+            account=sf_account,
+            database=sf_database,
+            schema=sf_schema,
+            )
+        conn._paramstyle='qmark'
+    except Exception as e:
+        print(f'Unable to connect to Snowflake schema {sf_schema}  in {sf_env} using {login_name}')
+        #print(str(e))
+    else:
+        rootdir = os.getcwd()
+
+        server_name = socket.gethostname()
+
+        current_time = datetime.datetime.now()
+        cur = conn.cursor()
+        #cur.execute(f'USE DATABASE {sf_env}')
+        cur.execute(sfquery)
+        cur.get_results_from_sfqid(cur.sfqid)
+        results = cur.fetchall()
+        print(f'result retrieval successful.')
+        for result in results:
+            try:
+                line_string = f'{sf_env};{seperator.join(result)}'
+            except Exception as e:
+                print(f'result set skipped for {result[1]}')
+            else:
+                csvfile.write(f"{line_string}\n")
+                result_bytes = int(result[3].replace(' GB', ''))
+                total_bytes = total_bytes + result_bytes
+
+conn.close()
+csvfile.close()
+print(f"{str(total_bytes)} GB")
+print(f'Endtime: {nowstamp}')
+