Nested loops done along with repeat, rlp, commenting & docs

baangt/TestSteps/TestStepMaster.py

@@ -12,6 +12,7 @@ from baangt.base.Faker import Faker as baangtFaker
 from baangt.base.Utils import utils
 from baangt.base.RuntimeStatistics import Statistic
 import random
+import itertools
 
 logger = logging.getLogger("pyC")
 
@@ -29,10 +30,10 @@ class TestStepMaster:
         self.elseLis = [self.elseIsTrue]
         self.ifConditions = 0
         self.repeatIsTrue = [False]
-        self.repeatDict = []
-        self.repeatData = []
-        self.repeatCount = []
-        self.repeatActive = 0
+        self.repeatDict = [] # used to store steps command of repeat data
+        self.repeatData = [] # used to store RLP_ data to be looped in repeat
+        self.repeatCount = [] # Used to store count of randomdata in loop
+        self.repeatActive = 0 # to sync active repeat counts with repeat done and will be execute when both are equal
         self.repeatDone = 0
         self.baangtFaker = None
         self.statistics = Statistic()
@@ -101,7 +102,7 @@ class TestStepMaster:
         self.ifIsTrue = self.ifLis[-1]
         self.elseIsTrue = self.elseLis[-1]
 
-    def executeDirectSingle(self, commandNumber, command):
+    def executeDirectSingle(self, commandNumber, command, replaceFromDict=None):
         """
         This will execute a single instruction
         """
@@ -111,42 +112,50 @@ class TestStepMaster:
         if not self.ifIsTrue and not self.elseIsTrue:
             if command["Activity"].upper() != "ELSE" and command["Activity"].upper() != "ENDIF":
                 return
-        if self.repeatIsTrue[-1]:
-            if command["Activity"].upper() == "REPEAT":
+        if self.repeatIsTrue[-1]: # If repeat statement is active then execute this
+            if command["Activity"].upper() == "REPEAT": # To sync active repeat with repeat done
                 self.repeatActive += 1
-            if command["Activity"].upper() != "REPEAT-DONE":
+            if command["Activity"].upper() != "REPEAT-DONE": # store command in repeatDict
                 self.repeatDict[-1][commandNumber] = command
                 return
             else:
-                self.repeatDone += 1
-                if self.repeatDone <= self.repeatActive:
+                self.repeatDone += 1 # to sync repeat done with active repeat
+                if self.repeatDone < self.repeatActive: # if all repeat-done are not synced with repeat store the data
                     self.repeatDict[-1][commandNumber] = command
                     logger.info(command)
                     return
                 self.repeatIsTrue[-1] = False
-                logger.info(self.repeatIsTrue)
-                if self.repeatData[-1] or self.repeatData[-1] == "":
-                    data_list = self.repeatData[-1]
-                    if len(self.repeatCount) > 0 and self.repeatCount[-1]:
+                if self.repeatData[-1]:
+                    data_list = []
+                    if type(self.repeatData[-1]) is list:
+                        for data_dic in self.repeatData[-1]:
+                            keys, values = zip(*data_dic.items())
+                            final_values = []
+                            for value in values: # coverting none list values to list. Useful in make all possible data using itertools
+                                if type(value) is not list:
+                                    final_values.append([value])
+                                else:
+                                    final_values.append(value)
+                            data_l = [dict(zip(keys, v)) for v in itertools.product(*final_values)] # itertools to make all possible combinations
+                            data_list.extend(data_l)
+                    else:
+                        data_list = [self.repeatData[-1]]
+                    if len(self.repeatCount) > 0 and self.repeatCount[-1]: # get random data from list of data
                         try:
                             data_list = random.sample(data_list, int(self.repeatCount[-1]))
                         except:
                             pass
-                    if data_list != "":
+                    if data_list:
                         for data in data_list:
                             temp_dic = dict(self.repeatDict[-1])
+                            processed_data = {}
                             for key in temp_dic:
                                 try:
                                     processed_data = dict(temp_dic[key])
                                 except Exception as ex:
-                                    logger.info(ex)
-                                for ky in processed_data:
-                                    try:
-                                        processed_data[ky] = self.replaceVariables(processed_data[ky], data)
-                                    except Exception as ex:
-                                        logger.info(ex)
+                                    logger.debug(ex)
                                 try:
-                                    self.executeDirectSingle(key, processed_data)
+                                    self.executeDirectSingle(key, processed_data, replaceFromDict=data)
                                 except Exception as ex:
                                     logger.info(ex)
                     else:
@@ -211,8 +220,8 @@ class TestStepMaster:
         logger.info(
             f"Executing TestStepDetail {commandNumber} with parameters: act={lActivity}, lType={lLocatorType}, loc={lLocator}, "
             f"Val1={lValue}, comp={lComparison}, Val2={lValue2}, Optional={lOptional}, timeout={lTimeout}")
-
-        lValue, lValue2 = self.replaceAllVariables(lValue, lValue2)
+        original_value = lValue # used as key to make rlp json dict and will be used further to make sheet name
+        lValue, lValue2 = self.replaceAllVariables(lValue, lValue2, replaceFromDict=replaceFromDict)
 
         if not TestStepMaster.ifQualifyForExecution(self.globalRelease, lRelease):
             logger.debug(f"we skipped this line due to {lRelease} disqualifies according to {self.globalRelease} ")
@@ -270,8 +279,12 @@ class TestStepMaster:
             self.ifIsTrue = True
             self.elseIsTrue = False
         elif lActivity == "REPEAT":
+            self.repeatActive += 1
             self.repeatIsTrue.append(True)
             self.repeatDict.append({})
+            if original_value not in self.testRunInstance.json_dict:
+                self.testRunInstance.json_dict[original_value] = []
+            self.testRunInstance.json_dict[original_value].append(lValue)
             self.repeatData.append(lValue)
             self.repeatCount.append(lValue2)
         elif lActivity == 'GOBACK':
@@ -343,13 +356,13 @@ class TestStepMaster:
             self.testcaseDataDict[lFieldnameForResults + "_Status"] = lDict[""].Status
             self.testcaseDataDict[lFieldnameForResults + "_Results"] = lDict[""].StatusText
 
-    def replaceAllVariables(self, lValue, lValue2):
+    def replaceAllVariables(self, lValue, lValue2, replaceFromDict=None):
         # Replace variables from data file
         try:
             if len(lValue) > 0:
-                lValue = self.replaceVariables(lValue)
+                lValue = self.replaceVariables(lValue, replaceFromDict=replaceFromDict)
             if len(lValue2) > 0:
-                lValue2 = self.replaceVariables(lValue2)
+                lValue2 = self.replaceVariables(lValue2, replaceFromDict=replaceFromDict)
         except Exception as ex:
             logger.info(ex)
         return lValue, lValue2
@@ -493,7 +506,7 @@ class TestStepMaster:
 
         self.timing.takeTime(self.timingName)
 
-    def replaceVariables(self, expression, data=None):
+    def replaceVariables(self, expression, replaceFromDict=None):
         """
         The syntax for variables is currently $(<column_name_from_data_file>). Multiple variables can be assigned
         in one cell, for instance perfectly fine: "http://$(BASEURL)/$(ENDPOINT)"
@@ -519,8 +532,17 @@ class TestStepMaster:
             center = center.split(")")[0]
 
             right_part = expression[len(left_part) + len(center) + 3:]
+            centerValue = ""
+
+            if replaceFromDict: # json is supplied with repeat tag, that json is used here to get main data
+                    dic = replaceFromDict
+                    for key in center.split('.')[-1:]:
+                        dic = self.iterate_json(dic, key)
+                    centerValue = dic
 
-            if "." not in center:
+            if centerValue: # if we got value from the passed json then bypass this if else conditions
+                pass
+            elif "." not in center:
                 # Replace the variable with the value from data structure
                 centerValue = self.testcaseDataDict.get(center)
             else:
@@ -528,20 +550,16 @@ class TestStepMaster:
                 dictVariable = center.split(".")[0]
                 dictValue = center.split(".")[1]
 
-                if data:
-                    if dictValue in data:
-                        startingValue = data
-                        for dt in center.split('.')[1:]:
-                            try:
-                                startingValue = startingValue.get(dt)
-                            except:
-                                raise KeyError(f"{data} key not inside {startingValue}")
-                        centerValue = startingValue
-                elif dictVariable == 'ANSWER_CONTENT':
+                if dictVariable == 'ANSWER_CONTENT':
                     centerValue = self.apiSession.session[1].answerJSON.get(dictValue, "Empty")
                 elif dictVariable == 'FAKER':
                     # This is to call Faker Module with the Method, that is given after the .
                     centerValue = self.__getFakerData(dictValue)
+                elif self.testcaseDataDict.get(dictVariable):
+                    dic = self.testcaseDataDict.get(dictVariable)
+                    for key in center.split('.')[1:]:
+                        dic = self.iterate_json(dic, key)
+                    centerValue = dic
                 else:
                     raise BaseException(f"Missing code to replace value for: {center}")
 
@@ -559,6 +577,18 @@ class TestStepMaster:
                     expression = str(int(expression))
         return expression
 
+    def iterate_json(self, data, key):
+        # itereate through list of json and create list of data from dictionary inside those list
+        if type(data) is list:
+            lis = []
+            for dt in data:
+                dt = self.iterate_json(dt, key)
+                lis.append(dt)
+            return lis
+        elif type(data) is dict:
+            return data.get(key)
+
+
     def __getFakerData(self, fakerMethod):
         if not self.baangtFaker:
             self.baangtFaker = baangtFaker()

baangt/base/BrowserHandling/BrowserHandling.py

@@ -217,17 +217,18 @@ class BrowserDriver:
         self.statistics.update_teststep()
         try:
             if self.browserData.driver:
+                try:
+                    if len(self.bpid) > 0:
+                        for bpid in self.bpid:
+                            os.kill(bpid, signal.SIGINT)
+                except:
+                    pass
                 self.browserData.driver.close()
                 self.browserData.driver.quit()
-                self.browserData.driver = None
-                if len(self.bpid) > 0:
-                    for bpid in self.bpid:
-                        os.kill(bpid, signal.SIGINT)
-
         except Exception as ex:
             logger.info(ex)
             pass  # If the driver is already dead, it's fine.
-
+        self.browserData.driver = None
 
     def refresh(self):
         self.browserData.driver.execute_script("window.location.reload()")

baangt/base/DataBaseORM.py

@@ -47,6 +47,7 @@ class TestrunLog(base):
 	statusOk = Column(Integer, nullable=False)
 	statusFailed = Column(Integer, nullable=False)
 	statusPaused = Column(Integer, nullable=False)
+	RLPJson = Column(String, nullable=True)
 	# relationships
 	globalVars = relationship('GlobalAttribute')
 	testcase_sequences = relationship('TestCaseSequenceLog')

baangt/base/ExportResults/ExportResults.py

@@ -78,6 +78,7 @@ class ExportResults:
             self.exportResultExcel()
             self.exportJsonExcel()
             self.exportAdditionalData()
+            self.write_json_sheet()
             self.exportTiming = ExportTiming(self.dataRecords,
                                              self.timingSheet)
             if self.networkInfo:
@@ -171,6 +172,11 @@ class ExportResults:
         self.statistics.update_attribute_with_value("TestCaseFailed", error)
         self.statistics.update_attribute_with_value("TestCasePaused", waiting)
         self.statistics.update_attribute_with_value("TestCaseExecuted", success + error + waiting)
+        try:
+            json_data = json.loads(self.testRunInstance.json_dict)
+        except Exception as ex:
+            logger.info(f"RLP Json error while updating in db : {str(ex)}")
+            json_data = ""
         # get documents
         datafiles = self.fileName
 
@@ -185,6 +191,7 @@ class ExportResults:
             statusFailed=error,
             statusPaused=waiting,
             dataFile=self.fileName,
+            RLPJson=json_data,
         )
         # add to DataBase
         session.add(tr_log)
@@ -328,6 +335,55 @@ class ExportResults:
         for n in range(len(headers)):
             ExcelSheetHelperFunctions.set_column_autowidth(self.jsonSheet, n)
 
+    def write_json_sheet(self):
+        # Used to write rlp_ json in individual sheets
+        dic = self.testRunInstance.json_dict
+        for js in dic:
+            if not js:
+                continue
+            elif js[:2] == "$(":
+                name = js[2:-1]
+            else:
+                name = js
+            jsonSheet = self.workbook.add_worksheet(f"{self.stage}_{name}")
+            if type(dic[js][0]) == dict: # Condition to get dictionary or dictionary inside list to write headers
+                data_dic = dic[js][0]
+            elif type(dic[js][0][0]) == dict:
+                data_dic = dic[js][0][0]
+            else:
+                logger.debug(f"{dic[js]} is not json convertible.")
+                continue
+            remove_header = []
+            for key in data_dic: # Removing headers which consist nested data
+                if type(data_dic[key]) == list or type(data_dic[key]) == dict:
+                    remove_header.append(key)
+            for key in remove_header:
+                del data_dic[key]
+            headers = []
+            for index, header in enumerate(data_dic):
+                jsonSheet.write(0, index, header)
+                headers.append(header)
+            row = 1
+            for data in dic[js]:
+                if not data:
+                    continue
+                dt = {}
+                for y, dt in enumerate(data):
+                    if type(dt) != dict: # for single dictionary data
+                        jsonSheet.write(row, y, data[dt])
+                    else: # if dictionaries are inside list
+                        column = 0 # used to update individual column
+                        for d in dt:
+                            if d not in headers:
+                                continue
+                            try:
+                                jsonSheet.write(row, column, dt[d])
+                            except Exception as ex:
+                                print(ex)
+                            column += 1
+                        row += 1
+                if type(dt) != dict:
+                    row += 1
 
     def makeSummaryExcel(self):
 

baangt/base/HandleDatabase.py

@@ -148,9 +148,7 @@ class HandleDatabase:
                     for data in rre_data:
                         new_data_dic[data] = rre_data[data]
                 elif temp_dic[keys][:4] == "RLP_":
-                    rlp_string = self.__process_rlp_string(temp_dic[keys])[5:-1]
-                    rlp_data = self.__rlp_string_to_python(rlp_string, fileName)
-                    temp_dic[keys] = rlp_data
+                    temp_dic[keys] = self.rlp_process(temp_dic[keys], fileName)
                 else:
                     try:
                         js = json.loads(temp_dic[keys])
@@ -160,6 +158,34 @@ class HandleDatabase:
             for key in new_data_dic:
                 temp_dic[key] = new_data_dic[key]
 
+    def rlp_process(self, string, fileName):
+        # Will get real data from rlp_ prefix string
+        rlp_string = self.__process_rlp_string(string)[5:-1]
+        rlp_data = self.__rlp_string_to_python(rlp_string, fileName)
+        data = rlp_data
+        self.rlp_iterate(data, fileName)
+        return data
+
+    def rlp_iterate(self, data, fileName):
+        # Rlp datas are stored in either json or list. This function will loop on every data and convert every
+        # Rlp string to data
+        if type(data) is list:
+            for dt in data:
+                if type(dt) is str:
+                    dt = self.rlp_iterate(dt, fileName)
+                elif type(dt) is dict or type(dt) is list:
+                    dt = self.rlp_iterate(dt, fileName)
+        elif type(data) is dict:
+            for key in data:
+                if type(data[key]) is list or type(data[key]) is dict:
+                    data[key] = self.rlp_iterate(data[key], fileName)
+                elif type(data[key]) is str:
+                    data[key] = self.rlp_iterate(data[key], fileName)
+        elif type(data) is str:
+            if data[:4] == "RLP_":
+                data = self.rlp_process(data, fileName)
+        return data
+
     def __compareEqualStageInGlobalsAndDataRecord(self, currentNewRecordDict:dict) -> bool:
         """
         As method name says, compares, whether Stage in Global-settings is equal to stage in Data Record,
@@ -271,6 +297,7 @@ class HandleDatabase:
         return processed_datas[randint(0, len(processed_datas)-1)]
 
     def __rlp_string_to_python(self, raw_data, fileName):
+        # will convert rlp string to python
         sheetName = raw_data.split(',')[0].strip()
         headerName = raw_data.split(',')[1].strip().split('=')[0].strip()
         headerValue = raw_data.split(',')[1].strip().split('=')[1].strip()
@@ -360,6 +387,8 @@ class HandleDatabase:
         else:
             assert sheet_name in sheet_dict, f"Excel file doesn't contain {sheet_name} sheet. Please recheck."
             base_sheet = sheet_dict[sheet_name]
+        self.sheet_dict = sheet_dict
+        self.base_sheet = base_sheet
         return sheet_dict, base_sheet
 
     def readNextRecord(self):

baangt/base/TestRun/TestRun.py

@@ -60,6 +60,7 @@ class TestRun:
         self.kwargs = {}
         self.dataRecords = {}
         self.globalSettings = {}
+        self.json_dict = {} # Used to maintain records of RLP_ data which will be used will exporting results
         self.managedPaths = ManagedPaths()
         self.classesForObjects = ClassesForObjects()         # Dynamically loaded classes
         self.timing = Timing()

docs/NestedLoops.rst

@@ -0,0 +1,44 @@
+***********
+NestedLoops
+***********
+While working on different TestCases we might need to repeat some TestSteps with a set of data and data might be big.
+To counter this repeat issue we have ``REPEAT`` & ``REPEAT-DONE`` activity tags, everything between this two tags will
+go on a loop. And to counter big & complex data structure we have ``RLP_`` prefix which will take written in another
+sheet, hence it will be much easier to enter and maintain data in whole sheet rather than a tab.
+
+RLP
+===
+``RLP_`` is a prefix which is used in data sheet. Its structure is ``RLP_(sheetName, lineReference=Number)`` here
+sheetName will hold the name of sheet where main data is stored. LineRefernce holds the header name of the column which
+contains Reference Number in target sheet, with the help of this reference number a list of data matching same reference
+number is made which is then looped inside ``REPEAT`` loop. We can even use ``RLP_`` statement in a sheet previously
+targeted by an other ``RLP_`` prefix. Hence, we can create a nested data and the data called by another ``RLP_`` data
+will become its child and the caller become its parent.
+
+.. image:: RLP_Nested_Loops.png
+
+.. image:: NestedLoopsSheet.png
+
+Here in image_1 inside ``RLP_`` LoopDataExampleTab is the sheetName of the target sheet containing main data and
+line_reference is the header which we are looking inside targetSheet, you can also see the header in image_2 which is of
+targetSheet. Then the value of header is matched. Like in first ``RLP_`` data value of line_reference is 0, which means
+it will create a list of data matching same in targetSheet, in this case their are 2 matching data so repeat statement
+will execute one after another.
+
+Repeat
+======
+``REPEAT`` tag is used in testrun file. It must be written inside "Activity" header and along with it, reference to the
+node referring to the data must be written inside "Value" tag. Format of node must be ``$(parent)`` we have to replace
+parent with the header name where rlp data is stored. In image_1 we can see "textarea2" is the header containing main
+data, so it will look like ``($textarea2)``. Now, if need to use child node we can use ``$(parent.child)`` here child is
+the name of header of the sheet which is replaced by ``RLP_`` prefix. In image_2 we can see their is "input_text" header
+the sheet in image_2 replaces ``RLP_`` data of image_1 so it will look like ``$(textarea2.input_text)``. This node will
+be replaced by the value inside input_text and a loop will start on all the matching line_reference. And lastly we have
+a option to choose number of times loop is run. When we have a large number of data and we don't need to run loop on all
+we can input number in value2 tab, then the loop will be run on the same number of randomly selected data. If we
+want to run loop on all data we can simply leave the value2 tab empty or put 0. Also remember we need to use
+``REPEAT-DONE`` at the other point of the loop (i.e. from where we need to run again from the starting of the loop.
+Please see below image for further reference.
+
+.. image:: RepeatTag.png
+

tests/test_ServiceTest.py

@@ -251,3 +251,16 @@ def test_NestedIfElse_with_NoBrowser():
     output_file = output_dir.joinpath(new_file[0][0]).as_posix()
     check_output(output_file)
     os.remove(output_file)
+
+def test_NestedLoops_and_repeat():
+    run_file = str(input_dir.joinpath("CompleteBaangtWebdemo_nested.xlsx"))
+    execute(run_file, globals_file=Path(input_dir).joinpath("globalsNoBrowser.json"))
+    new_file = folder_monitor.getNewFiles()
+    assert new_file
+    output_file = output_dir.joinpath(new_file[0][0]).as_posix()
+    wb = xlrd.open_workbook(output_file)
+    sheet1 = wb.sheet_by_name("Test_textarea2")
+    sheet2 = wb.sheet_by_name("Test_textarea2.nested")
+    assert sheet1.nrows == 3
+    assert sheet2.nrows == 3
+    os.remove(output_file)