Add Nginx-Logic to Log_reporter

log_reporter.py

@@ -30,10 +30,12 @@ def args_read(l_search_parameter):
 
     try:
         opts, args = getopt.getopt(l_args,"d:;",["dir=",
+                                                 "mode="
                                                     ])
     except getopt.GetoptError as err_det:
         print ("Error in reading parameters:" + str(err_det))
         print ("Call with --dir= or without any parameters")
+        print ("--mode=recycle|allnew|nginx to ignore files <> this mode")
         sys.exit("Abgebrochen wegen Fehler in Aufrufparametern")
     if opts:
         for opt, arg in opts:
@@ -72,6 +74,9 @@ class read_csv():
         # Default values for All-Calls, All-Duration, Absolute Min, Absolute Max)
         self.attribs["API-AVG"] = [0,0,0,9999,0]
         self.attribs["API-Calls"] = []
+        # They're used later to create the variable header of all API-Calls, that were executed
+        # so that we can print Headers for them
+        self.api_detail_headers={}
     def load(self):
         with codecs.open(filename=self.csv, mode='r', encoding="utf-8", errors="ignore") as l_csvfile:
             l_reader = csv.reader(l_csvfile)
@@ -162,6 +167,7 @@ class read_csv():
         l_max = float(l_array[4])
         l_last = float(l_array[5])
         self.attribs["API-Calls"].append([l_api, l_count, l_avg, l_total, l_min, l_max, l_last])
+        self.api_detail_headers[l_api] = ""
         self.attribs["API-AVG"][0] = self.attribs["API-AVG"][0] + l_count
         self.attribs["API-AVG"][1] = round(self.attribs["API-AVG"][1] + ( l_count * l_avg ),2)
         self.attribs["API-AVG"][2] = round(self.attribs["API-AVG"][1] / self.attribs["API-AVG"][0],4)
@@ -171,16 +177,28 @@ class read_csv():
             self.attribs["API-AVG"][4] = l_max
     def get_all_attribs(self):
         return self.attribs
+    def get_all_api_names(self):
+        return self.api_detail_headers
 
 def l_export_csvs(l_csvs, l_export_filename):
     with codecs.open(filename=l_export_filename, mode='wb', encoding='utf-8-sig') as l_csv:
         # write header
         l_csv_writer = csv.writer(l_csv, delimiter=";", quoting = csv.QUOTE_ALL)
         l_header = []
-        # read a random file to get attribs for header:
+        l_api_header = {}
+
+        # read all files to get attribs for header:
         for l_csv_item in l_csvs.values():
-            break
-        
+            # If this is not one of the file-modes we're searching for, then ignore also the header
+            if args_read(l_search_parameter="--mode"):
+                l_temp = l_csv_item.get_all_attribs()
+                if args_read(l_search_parameter="--mode") != l_temp["mode"]:
+                    continue
+
+            for key in l_csv_item.get_all_api_names().keys():
+                l_api_header[key] = ""
+
+
         # Write header to CSV
         for l_key in l_csv_item.get_all_attribs().keys():
             if l_key == "API-AVG":
@@ -189,22 +207,49 @@ def l_export_csvs(l_csvs, l_export_filename):
                 l_header.append("Total Average")
                 l_header.append("Total Min")
                 l_header.append("Total Max")
+            elif l_key == "API-Calls":
+                # That's the last column - enter here all our collected Headers:
+                for h_key in l_api_header.keys():
+                    # append to Header-Line and save position within the header-array, so that we can
+                    # later find the position for a specific API-Name and write into this column and the next ones
+                    l_header.append(h_key+" #")
+                    l_api_header[h_key] = len(l_header)-1
+                    l_header.append(h_key + " Avg")
+                    l_header.append(h_key + " Tot")
             else:
                 l_header.append(l_key)
+
         l_csv_writer.writerow(l_header)
         
         # Write all concatenated infos to CSV
         for l_csv_in in l_csvs.values():
+            # If this is not one of the file-modes we're searching for, then ignore also the header
+            if args_read(l_search_parameter="--mode"):
+                l_temp = l_csv_in.get_all_attribs()
+                if args_read(l_search_parameter="--mode") != l_temp["mode"]:
+                    continue
+
             l_row_out = []
+            l_pos = 0
             for l_row_key, l_row_item in l_csv_in.get_all_attribs().items():
                 if l_row_key == "API-AVG":
                     for n in range(0, 5):
-                        #if isinstance(l_row_item[n], int):
                         l_row_out.append(str(l_row_item[n]).replace(".", ","))
-                        #elif 
-                
+                        l_pos += 1
+                elif l_row_key == "API-Calls":
+                    if len(l_row_item) > 0:
+                        # We need to find our destination column (hopefully in dictionary)
+                        # find out, now many columns we must skip and place the value there
+                        for l_api_row in l_row_item:
+                            for n in range(0,l_api_header[l_row_item[0][0]] - l_pos):
+                                l_row_out.append(";")
+                            l_row_out.append(l_api_row[1])     # Number of calls
+                            l_row_out.append(str(l_api_row[2]).replace(".", ","))     # AVG / call
+                            l_row_out.append(str(l_api_row[3]).replace(".", ","))     # Total duration
+                            l_pos+=3
                 else:
                     l_row_out.append(l_row_item)
+                    l_pos+=1
             l_csv_writer.writerow(l_row_out)
             
 
@@ -227,8 +272,3 @@ for l_logfile in f_get_logfiles_from_dir(l_dir):
 l_export_csvs(l_csvs, "log_summary.csv")
 print("----")
 print("Log_summary.csv created")
-
-                    
-#l_test_csv = read_csv("C:\\Users\\Bernhard\\cs_earthsquad\\CloudStation\\pwa\\Testcases\\API\\ES_API_TEST\\pyAPITest\\dist\\log_api_calls_20180625_075708.log")
-#l_test_csv.load()
-#l_test_csv.report()