Merge branch 'master' into random_generator

# Conflicts:
#	baangt/TestSteps/TestStepMaster.py

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.1.5
+current_version = 1.2.5
 commit = True
 tag = True
 

.gitignore

@@ -19,4 +19,7 @@ baangt_*.xlsx
 /venv3_6/
 /tests/proxies.csv
 
-baangt/reports/*.html
+0TestInput
+1TestResults
+2DBResults
+3Reports

README.md

@@ -16,4 +16,13 @@ Please find the full documentation on [ReadTheDocs](https://baangt.readthedocs.i
 * Telegram Channel: [https://t.me/baangt](https://t.me/baangt)
 * Official repository for download: [https://github.com/Athos1972/baangt](https://github.com/Athos1972/baangt)
 * Nightly builds: [https://gogs.earthsquad.global/Athos/baangt](https://gogs.earthsquad.global/Athos/baangt) 
-* Binary repositories: [https://github.com/Athos1972/baangt-executables](https://github.com/Athos1972/baangt-executables)
+* Binary repositories: [https://github.com/Athos1972/baangt-executables](https://github.com/Athos1972/baangt-executables)
+
+# UPDATE: 24 Aug 2020  
+The `ResultsDB` structure was updatetd to support `ResultsBrowser` module:
+* field `number` was inrouduced to tables `TestCaseSequences` and `TestCases`
+
+To update an older DB, run
+```bash
+python db_update.py
+```

TechSpec/ResultsBrowser.md

@@ -0,0 +1,45 @@
+Results Browser
+===============
+**Results Browser** is a `baangt` feature that allows to view the results of the completed TestRuns from the database. It comprises two modules: 
+* *ResultQuery.py* -- the back-end module
+* *ResultsBrowser.py* -- the front-end module
+
+ResultsQuery.py
+---------------
+**_ResultsQuery.py_** implements the back-end of the *Results Browser* feature. Its main functionality includes the follow:
+* make queries to the database
+* return the query results as a `dict` object that is handy to show/visualize
+
+### Queries
+* summary on a particular test run:
+  * name
+  * stage
+  * execution date
+  * number of all/successful/failed/paused test cases
+  * test run duration
+  * duration and execution status of the test cases within the test run
+  * path to logfile
+  * path to results file  
+
+* summary on a set of test runs:
+  * name
+  * stage
+  * execution date
+  * test run duration
+  * average test run duration (for sets defined by both name and stage)
+  * average duration of the test cases within the TestRun (for sets defined by both name and stage)
+
+  the variants of the sets:
+  * all
+  * filtered by name
+  * filtered by stage
+  * filtered by name and stage
+  * all above within a certain time gap
+
+* test case status history of a TestRun executed on a certain stage (shows how often and which test case within a TestRun was executed OK (and when) and how often it failed)
+* test case duration history of a TestRun executed on a certain stage
+
+ResultsBrowser.py
+-----------------
+**_ResultsBrowser.py_** implements the front-end (GUI) of the *Results Browser* feature. It extends the current `baangt` GUI with additional screen.
+![Results Browser Screen](resultsbrowser.png "Results Browser Screen")

baangt/TestCase/TestCaseMaster.py

@@ -2,6 +2,7 @@ from baangt.base import GlobalConstants as GC
 from baangt.base.Timing.Timing import Timing
 from baangt.TestSteps.Exceptions import *
 from baangt.base.RuntimeStatistics import Statistic
+from baangt.base.Utils import utils
 
 
 class TestCaseMaster:
@@ -31,7 +32,13 @@ class TestCaseMaster:
         # In Unit-Tests this is a problem. When we run within the main loop of TestRun we are expected to directly
         # execute on __init__.
         if executeDirect:
-            self.executeTestCase()
+            try:
+                self.executeTestCase()
+            except Exception as e:
+                logger.warning(f"Uncought exception {e}")
+                utils.traceback(exception_in=e)
+                self.kwargs[GC.KWARGS_DATA][GC.TESTCASESTATUS_STOPERROR] = True
+                self.kwargs[GC.KWARGS_DATA][GC.TESTCASESTATUS] = GC.TESTCASESTATUS_ERROR
 
     def executeTestCase(self):
         self.timingName = self.timing.takeTime(self.__class__.__name__, forceNew=True)

baangt/TestCaseSequence/TestCaseSequenceMaster.py

@@ -13,6 +13,7 @@ import gevent
 import gevent.queue
 import gevent.pool
 from baangt.base.RuntimeStatistics import Statistic
+from CloneXls import CloneXls
 
 logger = logging.getLogger("pyC")
 
@@ -33,14 +34,20 @@ class TestCaseSequenceMaster:
         self.testCases = self.testSequenceData[GC.STRUCTURE_TESTCASE]
         self.kwargs = kwargs
         self.timingName = self.timing.takeTime(self.__class__.__name__, forceNew=True)
-        self.statistics = Statistic()
-        self.prepareExecution()
-        if int(self.testSequenceData.get(GC.EXECUTION_PARALLEL, 0)) > 1:
-            self.execute_parallel(self.testSequenceData.get(GC.EXECUTION_PARALLEL, 0))
-        else:
-            self.execute()
+
+        try:
+            self.statistics = Statistic()
+            self.prepareExecution()
+            if int(self.testSequenceData.get(GC.EXECUTION_PARALLEL, 0)) > 1:
+                self.execute_parallel(self.testSequenceData.get(GC.EXECUTION_PARALLEL, 0))
+            else:
+                self.execute()
+        except Exception as e:
+            logger.warning(f"Uncought exception {e}")
+            utils.traceback(exception_in=e)
 
     def prepareExecution(self):
+        logger.info("Preparing Test Records...")
         if self.testSequenceData.get(GC.DATABASE_FROM_LINE) and not self.testSequenceData.get(GC.DATABASE_LINES, None):
             # Change old line selection format into new format:
             self.testSequenceData[
@@ -57,6 +64,10 @@ class TestCaseSequenceMaster:
                 recordPointer -= 1
                 break
             recordPointer += 1
+        self.testdataDataBase.update_datarecords(self.dataRecords, fileName=utils.findFileAndPathFromPath(
+            self.testSequenceData[GC.DATABASE_FILENAME],
+            basePath=str(Path(self.testRunInstance.globalSettingsFileNameAndPath).parent)),
+            sheetName=self.testSequenceData[GC.DATABASE_SHEETNAME], noCloneXls=self.testRunInstance.noCloneXls)
         logger.info(f"{recordPointer + 1} test records read for processing")
         self.statistics.total_testcases(recordPointer + 1)
 
@@ -134,10 +145,16 @@ class TestCaseSequenceMaster:
         if not self.testdataDataBase:
             self.testdataDataBase = HandleDatabase(globalSettings=self.testRunInstance.globalSettings,
                                                    linesToRead=self.testSequenceData.get(GC.DATABASE_LINES))
-            self.testdataDataBase.read_excel(
-                fileName=utils.findFileAndPathFromPath(
-                    self.testSequenceData[GC.DATABASE_FILENAME],
-                    basePath=str(Path(self.testRunInstance.globalSettingsFileNameAndPath).parent)),
+        testDataFile = utils.findFileAndPathFromPath(
+            self.testSequenceData[GC.DATABASE_FILENAME],
+            basePath=str(Path(self.testRunInstance.globalSettingsFileNameAndPath).parent))
+        if not self.testRunInstance.noCloneXls and not "_baangt" in testDataFile:
+            cloneXls = CloneXls(testDataFile)  # , logger=logger)
+            testDataFile = cloneXls.update_or_make_clone(
+                ignore_headers=["TestResult", "UseCount"])
+        self.testSequenceData[GC.DATABASE_FILENAME] = testDataFile
+        self.testdataDataBase.read_excel(
+                fileName=testDataFile,
                 sheetName=self.testSequenceData[GC.DATABASE_SHEETNAME])
         return self.testdataDataBase
 

baangt/TestCaseSequence/TestCaseSequenceParallel.py

@@ -1,6 +1,7 @@
 import logging
 from baangt.TestSteps import Exceptions
 from baangt.base import GlobalConstants as GC
+from baangt.base.Utils import utils
 from datetime import datetime
 import time
 import gevent.queue
@@ -38,6 +39,7 @@ class TestCaseSequenceParallel:
 
         except Exceptions.baangtTestStepException as e:
             logger.critical(f"Unhandled Error happened in parallel run {parallelizationSequenceNumber}: " + str(e))
+            utils.traceback(exception_in=e)
             dataRecord[GC.TESTCASESTATUS] = GC.TESTCASESTATUS_ERROR
         finally:
             d_t = datetime.fromtimestamp(time.time())

baangt/TestDataGenerator/TestDataGenerator.py

@@ -1,19 +1,83 @@
-import csv
 import itertools
-import xlsxwriter
-import xl2dict
-import xlrd
 import errno
 import os
 import logging
 import faker
-from random import sample, randint
+from random import choice
 import baangt.base.GlobalConstants as GC
 import re
+import sys
+import pandas as pd
+from CloneXls import CloneXls
+import json
 
 logger = logging.getLogger("pyC")
 
 
+class PrefixDataManager:
+    def __init__(self, dataList: list, prefix: str, tdg_object=None):
+        """
+        This class manages data list as per functionality of their prefix
+        :param dataList: List of data to be managed
+        :param prefix: RRE, RRD, FKR, RND are the prefixs managed by this class
+        :param tdg_object: TestDataGenerator object, only useful in RRE, RRD prefix to update usecount
+        """
+        self.dataList = dataList        # Iterable object to be managed
+        self.prefix = prefix            # Prefix data
+        self.tdg_object = tdg_object    # used to update usecount data, only for rre and rrd
+        self.process()
+
+    def process(self):
+        """
+        It processes the data list with respect to their prefix and save it in dataList attribute.
+        :return: 
+        """
+        if self.prefix.lower() == "rrd" or self.prefix.lower() == "rre":
+            self.dataList = [
+                data for data in self.dataList if not self.tdg_object.usecountDataRecords[repr(data)]["limit"] or \
+                self.tdg_object.usecountDataRecords[repr(data)]['use'] < self.tdg_object.usecountDataRecords[repr(data)]['limit']
+               ]    # removing data with reached use limit
+
+        elif self.prefix.lower() == "fkr":
+            fake = faker.Faker(self.dataList[1])               # Creating faker class object
+            fake_lis = []                                                       
+            if len(self.dataList) == 3:         # Checks if their is any predefined number of fake data to be generated
+                if int(self.dataList[2]) == 0:                 # if number of predefined data is 0 then we need to 
+                    fake_lis.append([fake, self.dataList[0]])  # generate new data for each call so the required info
+                    fake_lis = tuple(fake_lis)                 # is saved as tuple so program can distinguish
+                else:
+                    for x in range(int(self.dataList[2])):  # if greater than 0, list is created with defined number of data
+                        fake_lis.append(getattr(fake, self.dataList[0])())  # on every call random data is sent from list
+            else:
+                for x in range(5):          # if their is no predefined number of data than a list of 5 fake data
+                    fake_lis.append(getattr(fake, self.dataList[0])())  # is generated
+            self.dataList = fake_lis
+
+    def return_random(self):
+        """
+        It returns data as per their prefix
+        :return: 
+        """
+        if self.prefix == "rre" or self.prefix == "rrd":
+            if not len(self.dataList):
+                raise BaseException(f"Not enough data, please verify if data is present or usecount limit" \
+                                    "has reached!!")
+            data = choice(self.dataList)
+            self.tdg_object.usecountDataRecords[repr(data)]['use'] += 1  # updates usecount in TDG object
+            if self.tdg_object.usecountDataRecords[repr(data)]['limit'] and \
+                self.tdg_object.usecountDataRecords[repr(data)]['use'] >= self.tdg_object.usecountDataRecords[repr(data)]['limit']:
+                self.dataList.remove(data)  # checks usecount after using a data and removes if limit is reached
+            return data
+
+        elif self.prefix.lower() == "fkr":
+            if type(self.dataList) == tuple:  # if type is tuple then we need generate new data on every call
+                return getattr(self.dataList[0][0], self.dataList[0][1])()
+            return choice(self.dataList)  # else it is a list and we can send any random data from it
+
+        elif self.prefix == 'rnd':
+            return choice(self.dataList)
+
+
 class TestDataGenerator:
     """
     TestDataGenerator Class is to used to create a TestData file from raw excel file containing all possible values.
@@ -27,21 +91,31 @@ class TestDataGenerator:
     6. List of header    = ``[<title1>, <title2>, <title3>]``
     7. Faker Prefix      = ``FKR_(<type>, <locale>, <number_of_data>)``
     8. RRD Prefix        = ``RRD_(<sheetName>,<TargetData>,[<Header1>:[<Value1>],<Header2>:[<Value1>,<Value2>]])``
+    9. RRE Prefix        = ``RRE_(<fileName>,<sheetName>,<TargetData>,[<Header1>:[<Value1>],<Header2>:[<Value1>,<Value2>]])``
+    10. Renv Prefix      = ``RENV_(<environmentVariable>,<default>)
 
     :param rawExcelPath: Takes input path for xlsx file containing input data.
     :param sheetName: Name of sheet where all base data is located.
     :method write: Will write the final processed data in excel/csv file.
     """
-    def __init__(self, rawExcelPath=GC.TESTDATAGENERATOR_INPUTFILE, sheetName=""):
-        self.path = os.path.abspath(rawExcelPath)
+    def __init__(self, rawExcelPath=GC.TESTDATAGENERATOR_INPUTFILE, sheetName="",
+                 from_handleDatabase=False, noUpdate=True):
+        self.fileNameAndPath = os.path.abspath(rawExcelPath)
         self.sheet_name = sheetName
-        if not os.path.isfile(self.path):
-            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), self.path)
-        self.sheet_dict, self.raw_data_json = self.__read_excel(self.path, self.sheet_name)
-        self.remove_header = []
-        self.processed_datas = self.__process_data(self.raw_data_json)
-        self.headers = [x for x in list(self.processed_datas[0].keys()) if x not in self.remove_header]
-        self.final_data = self.__generateFinalData(self.processed_datas)
+        if not os.path.isfile(self.fileNameAndPath):
+            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), self.fileNameAndPath)
+        self.allDataFramesDict, self.mainDataFrame = self.read_excel(self.fileNameAndPath, self.sheet_name)
+        self.openedSheetsDataFrame = {}  # stores filenames which contains sheetname: sheetDataFrame
+        self.usecountHeaderName = {}   # contains filenames which contains sheetname: usecountHeaderName
+        self.usecountDataRecords = {}  # used to maintain usecount limit record and verify non of the data cross limit
+        self.rreRrdProcessedData = {}  # stores rre processed data with its criteria as key and used when same criteria
+        self.noUpdateFiles = noUpdate
+        if not from_handleDatabase:
+            self.processed_data = self.processDataFrame(self.mainDataFrame)
+            self.finalDataFrame = self.updatePrefixData(self.processed_data)
+            if self.usecountHeaderName:
+                if not self.noUpdateFiles:
+                    self.save_usecount()  # saving source input file once everything is done
 
     def write(self, OutputFormat=GC.TESTDATAGENERATOR_OUTPUT_FORMAT, batch_size=0, outputfile=None):
         """
@@ -51,269 +125,86 @@ class TestDataGenerator:
         :param outputfile: name and path of outputfile.
         :return:
         """
+        if batch_size > 0:
+            if len(self.finalDataFrame) > batch_size:
+                data_lis = self.finalDataFrame.sample(n=batch_size)
+            else:
+                data_lis = self.finalDataFrame
+                logger.debug("Total final data is smaller than batch size.")
+        else:
+            data_lis = self.finalDataFrame
+
         if OutputFormat.lower() == "xlsx":
             if outputfile == None:
                 outputfile = GC.TESTDATAGENERATOR_OUTPUTFILE_XLSX
-            self.__write_excel(batch_size=batch_size, outputfile=outputfile)
+            with pd.ExcelWriter(outputfile) as writer:
+                data_lis.to_excel(writer, index=False)
+                writer.save()
+
         elif OutputFormat.lower() == "csv":
             if outputfile == None:
                 outputfile = GC.TESTDATAGENERATOR_OUTPUTFILE_CSV
-            self.__write_csv(batch_size=batch_size, outputfile=outputfile)
-        else:
-            logger.debug("Incorrect file format")
+            data_lis.to_csv(outputfile)
 
-    def __write_excel(self, outputfile=GC.TESTDATAGENERATOR_OUTPUTFILE_XLSX, batch_size=0):
-        """
-        Writes TestData file with final processsed data.
-        :param outputfile: Name and path for output file.
-        :param batch_size: No. of data to be randomly selected and written in output file.
-        :return: None
-        """
-        if batch_size > 0:
-            if len(self.final_data) > batch_size:
-                data_lis = sample(self.final_data, batch_size)
-            else:
-                data_lis = self.final_data
-                logger.debug("Total final data is smaller than batch size.")
         else:
-            data_lis = self.final_data
-        with xlsxwriter.Workbook(outputfile) as workbook:
-            worksheet = workbook.add_worksheet()
-            worksheet.write_row(0, 0, self.headers)
-            for row_num, data in enumerate(data_lis):
-                worksheet.write_row(row_num+1, 0, data)
-
-    def __write_csv(self, outputfile=GC.TESTDATAGENERATOR_OUTPUTFILE_CSV, batch_size=0):
-        """
-        Writes final data in csv
-        :param outputfile: Name and path of output file
-        :param batch_size: No. of data to be randomly selected and written in output file.
-        :return:
-        """
-        if batch_size > 0:
-            if len(self.final_data) > batch_size:
-                data_lis = sample(self.final_data, batch_size)
-            else:
-                data_lis = self.final_data
-        else:
-            data_lis = self.final_data
-        with open(outputfile, 'w', newline='\n', encoding='utf-8-sig') as file:
-            fl = csv.writer(file)
-            fl.writerow(self.headers)
-            for dt in data_lis:
-                fl.writerow(list(dt))
+            logger.debug("Incorrect file format")
 
-    def __generateFinalData(self, processed_data):
+    def updatePrefixData(self, processed_data):
         """
-        This method will do the final process on the processed_data. Processed_data contains list of dictionary, each
-        dictionary is the row from input file which are processed to be interact able in python as per the requirement.
-
-        First loop is of processed_data
-        Second loop is of the dictionary(row) and each key:value of that dictionary is header:processed_data
-
-        Method will first check the data type of value.
-        If it is a string than method will put it inside a list(i.e. ["string"])
-        If it is a tuple than it is a data with prefix so it will be sent to ``__prefix_data_processing`` method for
-        further processing.
-        Else the value is of type list.
-
-        Then we store all this lists in a list(can be treat as row). This list contains value of cells. They are evaluted
-        i.e. ranges are converted to list, strings are converted to list & list are all ready list. So to generate all
-        possible combinations from it we use ``iterable`` module.
-
-        Once this list of lists which contains all possible combinations is created we will call
-        ``__update_prefix_data_in_final_list`` method. This method will insert the processed prefix data along with the
-        data of all combinations list in the final list with the correct position of every value.
-
-        Finally it will return the list of lists which is completely processed and ready to be written in output file.
 
         :param processed_data:
         :return: Final_data_list
         """
-        final_data = []
-        for lis in processed_data:
-            index = {}
-            data_lis = []
-            for key in lis:
-                if type(lis[key]) == str:
-                    data = [lis[key]]
-                elif type(lis[key]) == tuple:
-                    if len(lis[key]) > 0:
-                        self.__prefix_data_processing(lis, key, index)
-                        continue
+        for dic in processed_data:
+            for key in dic.copy():
+                if type(dic[key]) == PrefixDataManager:
+                    data = dic[key].return_random()
+                    if type(data) == dict:
+                        del dic[key]
+                        dic.update(data)
                     else:
-                        data = ['']
-                else:
-                    data = lis[key]
-                data_lis.append(data)
-            datas = list(itertools.product(*data_lis))
-            self.__update_prefix_data_in_final_list(datas, index, final_data)
-        logger.info(f"Total generated data = {len(final_data)}")
+                        dic[key] = data
+        final_data = pd.DataFrame(processed_data)
         return final_data
 
-    def __update_prefix_data_in_final_list(self, data_list, dictionary, final_list):
-        """
-        This method will insert the data from the dictionary to final_list. So further it can be written in the output.
-
-        ``data_list`` is the list where all possible combinations generated by the input lists and ranges are stored.
-        ``dictionary`` is where the data with prefix are stored with their index value(which will be used to place data)
-        ``final_list`` is the list where final data will be stored after merging values from data_list and dictionary
-
-        First it will iterate through data_list which is list of lists(Also you can take it as list of rows).
-        Second loop will go through dictionary and check the data type of each value. Pairings inside dictionary are of
-        ``index: value`` here index is the position from where this value was picked. So it will be used in placing the
-        data in their correct position.
-
-        List:
-        =====
-            If value is list then it is a data with ``FKR_`` prefix with number of data 0(i.e. create new fake data for
-            every output). So we will create the faker module instance as per the input and will generate fake data for
-            every row. And will insert them in the position of index(key of dictionary).
-
-
-        Dictionary:
-        ==========
-            If it is not of type list then we will check that if it is of type dict. If yes then this is a data with
-            "RRD_" prefix and we have to select the random data here. So we will start looping through this dictionary.
-            Remember this is the third loop. This dictionary contains header:value(TargetDatas from matched data) pair.
-            On every loop it will first check that if the same header is stored in done dictionary. If yes then it will
-            get value of it from done dictionary. Then it will create a list from the TargetData list. This new list
-            will contain only data which has same value for same header stored in done dictionary.
-            i.e. If matching Header has value x then from TargetDatas list only data where header=x will be
-            considered for random pick.
-
-            Then the random value is selected from that list.
-            If none of the header is processed before(for the same row). Then it will get random data from the list and
-            will store header: value pair in done dictionary so it is used in the checking process as above.
-
-            It will also check if the ``self.header`` list contains all the header which are in the random selected data.
-            If not then it will add the header there.
-
-            At last we will index the position of header inside self.headers list and will insert the value in same
-            position and append the row in final_data list.
-
-        Tuple:
-        ======
-            If the type is tuple then we need to simply pick a random value from it and insert it in the same position of
-            index(key of dictionary for current value).
-
-        :param data_list:
-        :param dictionary:
-        :param final_list:
-        :return: None
-        """
-        for data in data_list:
-            data = list(data)
-            done = {}
-            for ind in dictionary:
-                if type(dictionary[ind]) == list:
-                    fake = faker.Faker(dictionary[ind][2])
-                    data.insert(ind, getattr(fake, dictionary[ind][1])())
-                else:
-                    if type(dictionary[ind][0]) == dict:
-                        sorted_data = False
-                        for header in dictionary[ind][0]:
-                            if header in done:
-                                match = done[header]
-                                sorted_data = [x for x in dictionary[ind] if x[header] == match]
-                                break
-                        if not sorted_data:
-                            sorted_data = dictionary[ind]
-                        data_to_insert = sorted_data[randint(0, len(sorted_data) - 1)]
-                        for keys in data_to_insert:
-                            if keys not in self.headers:
-                                self.headers.append(keys)
-                            if keys not in done:
-                                data.insert(self.headers.index(keys), data_to_insert[keys])
-                                done[keys] = data_to_insert[keys]
-                    else:
-                        data_to_insert = dictionary[ind][randint(0, len(dictionary[ind]) - 1)]
-                        data.insert(ind, data_to_insert)
-            final_list.append(data)
-
-    def __prefix_data_processing(self, dic, key, dictionary: dict):
-        """
-        This method will process the datas with prefix.
-
-        ``dic`` the dictionary where all data which are in final process is stored
-        ``key`` the header of the current data which will be used now to call the data.
-        ``dictionary`` in which the values will be inserted after performing their process.
-
-        First it will check the first value of tuple.
-        If it is ``Faker`` then in will continue the process and will check the 4th value of tuple.
-        If the 4th value(which is used to determine the number of fake data to be generated and store inside a list)
-        is ``0`` then the method will store the values as it is in a list, because ``0`` value means we have to generate
-        new fake data for every output data, so it will be done later.
-        If it is greater than ``0`` then this method will create tuple with the given number of fake data and store it.
-        (If no number is given then default number is 5.)
-
-        If first value is not ``Faker`` then no process will be done.
-
-        Finally the data will be inserted in the dictionary.
-
-        :param dic:
-        :param key:
-        :param dictionary:
-        :return:
-        """
-        ltuple = dic[key]
-        if ltuple[0] == "Faker":
-            fake = faker.Faker(ltuple[2])
-            fake_lis = []
-            if len(ltuple) == 4:
-                if int(ltuple[3]) == 0:
-                    dictionary[list(dic.keys()).index(key)] = list(ltuple)
-                    return True
-                else:
-                    for x in range(int(ltuple[3])):
-                        fake_lis.append(getattr(fake, ltuple[1])())
-            else:
-                for x in range(5):
-                    fake_lis.append(getattr(fake, ltuple[1])())
-            dictionary[list(dic.keys()).index(key)] = tuple(fake_lis)
-            return True
-        else:
-            dictionary[list(dic.keys()).index(key)] = ltuple
-            return True
-
-    def __process_data(self, raw_json):
+    def processDataFrame(self, dataFrame):
         """
         This method is used to Process all the raw unprocessed data read from the excel file.
 
-        It will first send the header to ``__data_generator`` so that if it is a list then it will get converted in
+        It will first send the header to ``__splitList`` so that if it is a list then it will get converted in
         individual header.
 
         Later it will process the values using ``__data_generator``.
 
-        It will then check returned iterable type, if it is a tuple that mean input value was with prefix, so, it will
-        further check if the tuple contains dict. If True than prefix was RRD_. In that case we will have to deal with
-        the original header of the input value. Because if the original value's header is not in the TargetData then this
-        header will contain no value in the output file and my cause errors too. So the header will added in
-        ``self.remove_header`` list which will be further used to remove it from main header list.
+        It will then check returned data type, and convert it into list if it is not already. Then from every row which
+        are dict contains value/values inside list it will use itertools to create all possible combination, now every
+        data in the value list is converted in a new row with value as string.
 
-        Finally it will return list of dictionarys. Each dictionary contains processed data of a row of input file.
-        Processed data are the raw data converted into python data type and iterables. Ranges are converted into list.
+        Finally it will return list of dictionary. Each dictionary contains processed data of a row of input file.
+        Processed data are the raw data converted into python data type and class. Ranges are converted into list.
 
-        :param raw_json:
+        :param dataFrame:
         :return:
         """
         processed_datas = []
-        for raw_data in raw_json:
+        json_df = json.loads(dataFrame.to_json(orient="records"))
+        for raw_data in json_df:
             if not list(raw_data.values())[0]:
                 continue
             processed_data = {}
             for key in raw_data:
-                keys = self.__data_generators(key)
+                keys = self.__splitList(key)
                 for ke in keys:
-                    processed_data[ke] = self.__data_generators(raw_data[key])
-                    if type(processed_data[ke]) == tuple and len(processed_data[ke])>0:
-                        if type(processed_data[ke][0]) == dict:
-                            if ke not in processed_data[ke][0]:
-                                self.remove_header.append(ke)
-            processed_datas.append(processed_data)
+                    data = self.data_generators(raw_data[key])
+                    if type(data) != list:
+                        processed_data[ke] = [data]
+                    else:
+                        processed_data[ke] = data
+            product = list(self.product_dict(**processed_data))
+            processed_datas += product
         return processed_datas
 
-    def __data_generators(self, raw_data):
+    def data_generators(self, raw_data_old):
         """
         This method first send the data to ``__raw_data_string_process`` method to split the data and remove the unwanted
         spaces.
@@ -324,71 +215,62 @@ class TestDataGenerator:
 
         Later according to the prefix of data and the data_type assigned it will convert them.
         Simple list and strings are converted in to ``list`` type.
-        Data with prefix will converted in to ``tuple`` type so further it will be helpful in distinguishing. Also it will
-        insert the prefix name in first value of tuple if the prefix is ``FKR_`` so it will be helpful in further process.
+        Data with prefix will be processed and the data list generated by them is used to create PrefixDataManager class
+        so later when we need to get the final data this class is used to get the data as per prefix and requirements.
 
-        Finally it will return the iterable for further process.
+        Finally it will return the data for further process.
 
         :param raw_data:
         :return: List or Tuple containing necessary data
         """
-        raw_data, prefix, data_type = self.__raw_data_string_process(raw_data)
+        raw_data, prefix, data_type = self.__raw_data_string_process(raw_data_old)
         if len(raw_data)<=1:
             return [""]
-        if raw_data[0] == "[" and raw_data[-1] == "]" and prefix == "":
-            processed_datas = self.__splitList(raw_data)
-            processed_datas = data_type(processed_datas)
+
+        if prefix == "Rnd":
+            if "-" in raw_data:
+                raw_data = raw_data.split('-')
+                start = raw_data[0].strip()
+                end = raw_data[1].strip()
+                step = 1
+                if "," in end:
+                    raw_data = end.split(",")
+                    end = raw_data[0].strip()
+                    step = raw_data[1].strip()
+                processed_datas = [x for x in range(int(start), int(end) + 1, int(step))]
+            else:
+                processed_datas = self.__splitList(raw_data)
+            processed_datas = PrefixDataManager(processed_datas, 'rnd')
 
         elif prefix == "Faker":
-                processed_datas = [data.strip() for data in raw_data[1:-1].split(",")]
-                processed_datas.insert(0, "Faker")
-                processed_datas = data_type(processed_datas)
+                dataList = [data.strip() for data in raw_data[1:-1].split(",")]
+                processed_datas = PrefixDataManager(dataList, prefix="fkr")
 
         elif prefix == "Rrd":
-            first_value = raw_data[1:-1].split(',')[0].strip()
-            second_value = raw_data[1:-1].split(',')[1].strip()
-            if second_value[0] == "[":
-                second_value = ','.join(raw_data[1:-1].split(',')[1:]).strip()
-                second_value = second_value[:second_value.index(']')+1]
-                third_value = [x.strip() for x in ']'.join(raw_data[1:-1].split(']')[1:]).split(',')[1:]]
-            else:
-                third_value = [x.strip() for x in raw_data[1:-1].split(',')[2:]]
-            evaluated_list = ']],'.join(','.join(third_value)[1:-1].strip().split('],')).split('],')
-            if evaluated_list[0] == "":
-                evaluated_dict = {}
-            else:
-                evaluated_dict = {
-                    splited_data.split(':')[0]: self.__splitList(splited_data.split(':')[1])  for splited_data in evaluated_list
-                }
-            if second_value[0] == "[" and second_value[-1] == "]":
-                second_value = self.__splitList(second_value)
-            processed_datas = self.__processRrd(first_value, second_value,evaluated_dict)
-            processed_datas = data_type(processed_datas)
+            sheet_name, data_looking_for, data_to_match = self.extractDataFromRrd(raw_data)
+            try:
+                dataList = self.__processRrdRre(sheet_name, data_looking_for, data_to_match)
+                processed_datas = PrefixDataManager(dataList, prefix='rrd', tdg_object=self)
+            except KeyError:
+                sys.exit(f"Please check that source files contains all the headers mentioned in : {raw_data_old}")
 
         elif prefix == "Rre":
             file_name = raw_data[1:-1].split(',')[0].strip()
-            sheet_dict, _ = self.__read_excel(file_name)
-            first_value = raw_data[1:-1].split(',')[1].strip()
-            second_value = raw_data[1:-1].split(',')[2].strip()
-            if second_value[0] == "[":
-                second_value = ','.join(raw_data[1:-1].split(',')[2:]).strip()
-                second_value = second_value[:second_value.index(']')+1]
-                third_value = [x.strip() for x in ']'.join(raw_data[1:-1].split(']')[1:]).split(',')[1:]]
-            else:
-                third_value = [x.strip() for x in raw_data[1:-1].split(',')[3:]]
-            evaluated_list = ']],'.join(','.join(third_value)[1:-1].strip().split('],')).split('],')
-            if evaluated_list[0] == "":
-                evaluated_dict = {}
-            else:
-                evaluated_dict = {
-                    splited_data.split(':')[0]: self.__splitList(splited_data.split(':')[1])  for splited_data in evaluated_list
-                }
-            if second_value[0] == "[" and second_value[-1] == "]":
-                second_value = self.__splitList(second_value)
-            processed_datas = self.__processRrd(first_value, second_value, evaluated_dict, sheet_dict)
-            processed_datas = data_type(processed_datas)
+            sheet_name, data_looking_for, data_to_match = self.extractDataFromRrd(raw_data, index=1)
+            try:
+                dataList = self.__processRrdRre(sheet_name, data_looking_for, data_to_match, filename=file_name)
+                processed_datas = PrefixDataManager(dataList, prefix="rre", tdg_object=self)
+            except KeyError:
+                sys.exit(f"Please check that source files contains all the headers mentioned in : {raw_data_old}")
+
+        elif prefix == "Renv":
+            processed_datas = self.get_env_variable(raw_data)
+
+        elif raw_data[0] == "[" and raw_data[-1] == "]":
+            processed_datas = self.__splitList(raw_data)
 
         elif "-" in raw_data:
+            raw_data_original = raw_data[:]
             raw_data = raw_data.split('-')
             start = raw_data[0].strip()
             end = raw_data[1].strip()
@@ -397,14 +279,138 @@ class TestDataGenerator:
                 raw_data = end.split(",")
                 end = raw_data[0].strip()
                 step = raw_data[1].strip()
-            processed_datas = [x for x in range(int(start), int(end)+1, int(step))]
-            processed_datas = data_type(processed_datas)
+            try:
+                processed_datas = [x for x in range(int(start), int(end)+1, int(step))]
+            except:
+                processed_datas = [raw_data_original.strip()]
 
         else:
-            processed_datas = [raw_data.strip()]
-            processed_datas = data_type(processed_datas)
+            processed_datas = raw_data.strip()
         return processed_datas
 
+    def extractDataFromRrd(self, raw_data, index=0):
+        """
+        Splits rrd/rre string and used them according to their position
+        :param raw_data:
+        :param index:
+        :return:
+        """
+        first_value = raw_data[1:-1].split(',')[0+index].strip()
+        second_value = raw_data[1:-1].split(',')[1+index].strip()
+        if second_value[0] == "[":
+            second_value = ','.join(raw_data[1:-1].split(',')[1+index:]).strip()
+            second_value = second_value[:second_value.index(']') + 1]
+            third_value = [x.strip() for x in ']'.join(raw_data[1:-1].split(']')[1:]).split(',')[1:]]
+        else:
+            third_value = [x.strip() for x in raw_data[1:-1].split(',')[2+index:]]
+        evaluated_list = ']],'.join(','.join(third_value)[1:-1].strip().split('],')).split('],')
+        if evaluated_list[0] == "":
+            evaluated_dict = {}
+        else:
+            evaluated_dict = {
+                splited_data.split(':')[0]: self.__splitList(splited_data.split(':')[1]) for splited_data in
+                evaluated_list
+            }
+        if second_value[0] == "[" and second_value[-1] == "]":
+            second_value = self.__splitList(second_value)
+        return first_value, second_value, evaluated_dict
+
+    def __processRrdRre(self, sheet_name, data_looking_for, data_to_match: dict, filename=None):
+        if filename:  # if filename than it is an rre file so we have to use filename instead of base file
+            filename = os.path.join(os.path.dirname(self.fileNameAndPath), filename)
+            if not self.noUpdateFiles:
+                file_name = ".".join(filename.split(".")[:-1])
+                file_extension = filename.split(".")[-1]
+                file = file_name + "_baangt" + "." + file_extension
+            else:
+                file = filename
+            if not file in self.openedSheetsDataFrame:
+                logger.debug(f"Creating clone file of: {filename}")
+                if not self.noUpdateFiles:
+                    filename = CloneXls(filename).update_or_make_clone()
+                self.openedSheetsDataFrame[filename] = {}
+            filename = file
+            if sheet_name in self.openedSheetsDataFrame[filename]:
+                df = self.openedSheetsDataFrame[filename][sheet_name]
+            else:
+                df = pd.read_excel(filename, sheet_name, dtype=str)
+                df.fillna("", inplace=True)
+                self.openedSheetsDataFrame[filename][sheet_name] = df
+        else:
+            df = self.allDataFramesDict[sheet_name]
+            if not self.fileNameAndPath in self.openedSheetsDataFrame:
+                self.openedSheetsDataFrame[self.fileNameAndPath] = {}
+            if not sheet_name in self.openedSheetsDataFrame[self.fileNameAndPath]:
+                self.openedSheetsDataFrame[self.fileNameAndPath][sheet_name] = df
+        df1 = df.copy()
+        for key, value in data_to_match.items():
+            if not isinstance(value, list):
+                value = [value]
+            df1 = df1.loc[df1[key].isin(value)]
+        data_lis = []
+
+        if type(data_looking_for) == str:
+            data_looking_for = data_looking_for.split(",")
+        data_new_header = {}
+        for header in data_looking_for:
+            if ":" in header:
+                old_header = header.split(":")[0].strip()
+                new_header = header.split(":")[1].strip()
+            else:
+                old_header = header
+                new_header = header
+            data_new_header[old_header] = new_header
+
+        key_name = repr(sheet_name) + repr(data_looking_for) + repr(data_to_match) + repr(filename)
+        if key_name in self.rreRrdProcessedData:
+            logger.debug(f"Data Gathered from previously saved data.")
+            return self.rreRrdProcessedData[key_name]
+
+        usecount, limit, usecount_header = self.check_usecount(df.columns.values.tolist())
+        if not filename:
+            if self.fileNameAndPath not in self.usecountHeaderName:
+                self.usecountHeaderName[self.fileNameAndPath] = {}
+            if sheet_name not in self.usecountHeaderName[self.fileNameAndPath] and usecount_header:
+                self.usecountHeaderName[self.fileNameAndPath][sheet_name] = usecount_header
+        else:
+            if filename not in self.usecountHeaderName:
+                self.usecountHeaderName[filename] = {}
+            if sheet_name not in self.usecountHeaderName[filename]:
+                self.usecountHeaderName[filename][sheet_name] = usecount_header
+        df1_dict = df1.to_dict(orient="index")
+        for index in df1_dict:
+            data = df1_dict[index]
+            if usecount_header:
+                try:
+                    used_limit = int(data[usecount_header])
+                except:
+                    used_limit = 0
+            else:
+                used_limit = 0
+
+            if data_looking_for[0] == "*":
+                if usecount_header:
+                    del data[usecount_header]
+                data_lis.append(data)
+                self.usecountDataRecords[repr(data)] = {
+                    "use": used_limit, "limit": limit, "index": index,
+                    "sheet_name": sheet_name, "file_name": filename
+                }
+            else:
+                dt = {data_new_header[header]: data[header] for header in data_new_header}
+                data_lis.append(dt)
+                self.usecountDataRecords[repr(dt)] = {
+                    "use": used_limit, "limit": limit, "index": index,
+                    "sheet_name": sheet_name, "file_name": filename
+                }
+
+        if len(data_lis) == 0:
+            logger.info(f"No data matching: {data_to_match}")
+            sys.exit(f"No data matching: {data_to_match}")
+        logger.debug(f"New Data Gathered.")
+        self.rreRrdProcessedData[key_name] = data_lis
+        return data_lis
+
     def __raw_data_string_process(self, raw_string):
         """
         Returns ``String, prefix, data_type`` which are later used to decided the process to perform on string.
@@ -430,7 +436,8 @@ class TestDataGenerator:
         prefix = ""
         if len(raw_string)>4:
             if raw_string[3] == "_":
-                if raw_string[:4].lower() == "rnd_":           # Random
+                if raw_string[:4].lower() == "rnd_":
+                    prefix = "Rnd"
                     raw_string = raw_string[4:]
                     data_type = tuple
                 elif raw_string[:4].lower() == "fkr_":
@@ -450,13 +457,28 @@ class TestDataGenerator:
                 else:
                     data_type = list
             else:
+                if raw_string[:5].lower() == "renv_":
+                    prefix = "Renv"
+                    raw_string = raw_string[5:]
                 data_type = list
         else:
             data_type = list
         return raw_string, prefix, data_type
 
-    @staticmethod
-    def __read_excel(path, sheet_name=""):
+    def get_str_sheet(self, excel, sheet):
+        """
+        Returns dataFrame with all data treated as string
+        :param excel:
+        :param sheet:
+        :return:
+        """
+        columns = excel.parse(sheet).columns
+        converters = {column: str for column in columns}
+        data = excel.parse(sheet, converters=converters)
+        data.fillna("", inplace=True)
+        return data
+
+    def read_excel(self, path, sheet_name="", return_json=False):
         """
         This method will read the input excel file.
         It will read all the sheets inside this excel file and will create a dictionary of dictionary containing all data
@@ -472,19 +494,21 @@ class TestDataGenerator:
         :param sheet_name: Name of base sheet sheet where main input data is located. Default will be the first sheet.
         :return: Dictionary of all sheets and data, Dictionary of base sheet.
         """
-        wb = xlrd.open_workbook(path)
-        sheet_lis = wb.sheet_names()
-        sheet_dict = {}
+        wb = pd.ExcelFile(path)
+        sheet_lis = wb.sheet_names
+        sheet_df = {}
         for sheet in sheet_lis:
-            xl_obj = xl2dict.XlToDict()
-            data = xl_obj.fetch_data_by_column_by_sheet_name(path,sheet_name=sheet)
-            sheet_dict[sheet] = data
+            sheet_df[sheet] = self.get_str_sheet(wb, sheet)
+            sheet_df[sheet].fillna("", inplace=True)
+        if return_json:
+            for df in sheet_df.keys():
+                sheet_df[df] = json.loads(sheet_df[df].to_json(orient="records"))
         if sheet_name == "":
-            base_sheet = sheet_dict[sheet_lis[0]]
+            base_sheet = sheet_df[sheet_lis[0]]
         else:
-            assert sheet_name in sheet_dict, f"Excel file doesn't contain {sheet_name} sheet. Please recheck."
-            base_sheet = sheet_dict[sheet_name]
-        return sheet_dict, base_sheet
+            assert sheet_name in sheet_df, f"Excel file doesn't contain {sheet_name} sheet. Please recheck."
+            base_sheet = sheet_df[sheet_name]
+        return sheet_df, base_sheet
 
     @staticmethod
     def __splitList(raw_data):
@@ -494,55 +518,71 @@ class TestDataGenerator:
         :param raw_data: string of list
         :return: Python list
         """
-        proccesed_datas = [data.strip() for data in raw_data[1:-1].split(",")]
+        if raw_data[0] == "[" and raw_data[-1] == "]":
+            data = raw_data[1:-1]
+        else:
+            data = raw_data
+        proccesed_datas = [data.strip() for data in data.split(",")]
         return proccesed_datas
 
-    def __processRrd(self, sheet_name, data_looking_for, data_to_match: dict, sheet_dict=None, caller="RRD_"):
+    def check_usecount(self, data):
+        # used to find and return if their is usecount header and limit in input file
+        usecount = False
+        limit = 0
+        usecount_header = None
+        for header in data:
+            if "usecount" in header.lower():
+                usecount = True
+                usecount_header = header
+                if "usecount_" in header.lower():
+                    try:
+                        limit = int(header.lower().strip().split("count_")[1])
+                    except:
+                        limit = 0
+        return usecount, limit, usecount_header
+
+    def save_usecount(self):
         """
-        This function is internal function to process the data wil RRD_ prefix.
-        The General input in excel file is like ``RRD_[sheetName,TargetData,[Header:[values**],Header:[values**]]]``
-        So this program will take already have processed input i.e. strings converted as python string and list to
-        python list, vice versa.
-
-        ``sheet_name`` is the python string referring to TargetData containing string.
-
-        ``data_looking_for`` is expected to be a string or list of TargetData. When there are multiple values then the
-        previous process will send list else it will send string. When a string is received it will be automatically
-        converted in list so the program will alwaus have to deal with list. If input is "*" then this method will take
-        all value in the matched row as TargetData.
-
-        ``data_to_match`` is a python dictionary created by the previous process. It will contain the key value pair
-        same as given in input but just converted in python dict. Then all possible combinations will be generated inside
-        this method. If an empty list is given by the user in the excel file then this list will get emptu dictionary from
-        the previous process. Thus then this method will pick TargetData from all rows of the target sheet.
-
-        :param sheet_name:
-        :param data_looking_for:
-        :param data_to_match:
-        :return: dictionary of TargetData
+        Saves the excel file in which have usecount to be updated
+        :return:
         """
-        sheet_dict = self.sheet_dict if sheet_dict is None else sheet_dict
-        matching_data = [list(x) for x in itertools.product(*[data_to_match[key] for key in data_to_match])]
-        assert sheet_name in sheet_dict, \
-            f"Excel file doesn't contain {sheet_name} sheet. Please recheck. Called in '{caller}'"
-        base_sheet = sheet_dict[sheet_name]
-        data_lis = []
-        if type(data_looking_for) == str:
-            data_looking_for = data_looking_for.split(",")
-
-        for data in base_sheet:
-            if len(matching_data) == 1 and len(matching_data[0]) == 0:
-                if data_looking_for[0] == "*":
-                    data_lis.append(data)
-                else:
-                    data_lis.append({keys: data[keys] for keys in data_looking_for})
-            else:
-                if [data[key] for key in data_to_match] in matching_data:
-                    if data_looking_for[0] == "*":
-                        data_lis.append(data)
-                    else:
-                        data_lis.append({keys: data[keys] for keys in data_looking_for})
-        return data_lis
+        if self.noUpdateFiles:
+            return 
+        for filename in self.usecountHeaderName:
+            logger.debug(f"Updating file {filename} with usecounts.")
+            sheet_dict = self.openedSheetsDataFrame[filename]
+            ex = pd.ExcelFile(filename)
+            for sheet in ex.sheet_names:
+                if sheet in sheet_dict:
+                    continue
+                df = self.get_str_sheet(ex, sheet)
+                sheet_dict[sheet] = df
+            with pd.ExcelWriter(filename) as writer:
+                for sheetname in sheet_dict:
+                    sheet_dict[sheetname].to_excel(writer, sheetname, index=False)
+                writer.save()
+            logger.debug(f"File updated {filename}.")
+
+    def update_usecount_in_source(self, data):
+        """
+        Updates usecount in the dataframe in correct position. These function only updates not save the file.
+        :param data:
+        :return:
+        """
+        if self.noUpdateFiles:
+            return 
+        filename = self.usecountDataRecords[repr(data)]["file_name"]
+        sheetName = self.usecountDataRecords[repr(data)]["sheet_name"]
+        if not filename:
+            filename = self.fileNameAndPath
+        if filename not in self.usecountHeaderName:
+            return
+        elif sheetName not in self.usecountHeaderName[filename]:
+            return
+        if not self.usecountHeaderName[filename][sheetName]:
+            return
+        self.openedSheetsDataFrame[filename][sheetName][self.usecountHeaderName[filename][sheetName]][
+            self.usecountDataRecords[repr(data)]["index"]] = self.usecountDataRecords[repr(data)]["use"]
 
     def __process_rrd_string(self, rrd_string):
         """
@@ -594,6 +634,35 @@ class TestDataGenerator:
         assert match, err_string
         return processed_string
 
+    @staticmethod
+    def get_env_variable(string):
+        """
+        Returns environment variable or the default value predefined.
+        :param string:
+        :return:
+        """
+        variable = string[1:-1].strip().split(',')[0].strip()
+        data = os.environ.get(variable)
+        try:
+            if not data:
+                data = string[1:-1].strip().split(',')[1].strip()
+                logger.info(f"{variable} not found in environment, using {data} instead")
+        except:
+            raise BaseException(f"Can't find {variable} in envrionment & default value is also not set")
+        return data
+
+    @staticmethod
+    def product_dict(**kwargs):
+        """
+        :param kwargs: Dictionary containing values in a list
+        :return: yield dictionaries with individual value in dictionary from value list
+        """
+        keys = kwargs.keys()
+        vals = kwargs.values()
+        for instance in itertools.product(*vals):
+            yield dict(zip(keys, instance))
+
+
 if __name__ == "__main__":
     lTestDataGenerator = TestDataGenerator("../../tests/0TestInput/RawTestData.xlsx")
     lTestDataGenerator.write()

baangt/TestSteps/TestStepMaster.py

@@ -60,17 +60,21 @@ class TestStepMaster:
         self.testStep = self.testRunUtil.getTestStepByNumber(self.testCase, self.testStepNumber)
         self.randomValues = RandomValues()
 
-        if self.testStep and len(self.testStep) > 1:
-            if not isinstance(self.testStep[1], str) and executeDirect:
-                # This TestStepMaster-Instance should actually do something - activitites are described
-                # in the TestExecutionSteps.
-                # Otherwise there's only a classname in TestStep[0]
-                self.executeDirect(self.testStep[1][GC.STRUCTURE_TESTSTEPEXECUTION])
-
-                # Teardown makes only sense, when we actually executed something directly in here
-                # Otherwise (if it was 1 or 2 Tab-stops more to the left) we'd take execution time without
-                # having done anything
-                self.teardown()
+        try:
+            if self.testStep and len(self.testStep) > 1:
+                if not isinstance(self.testStep[1], str) and executeDirect:
+                    # This TestStepMaster-Instance should actually do something - activitites are described
+                    # in the TestExecutionSteps.
+                    # Otherwise there's only a classname in TestStep[0]
+                    self.executeDirect(self.testStep[1][GC.STRUCTURE_TESTSTEPEXECUTION])
+
+                    # Teardown makes only sense, when we actually executed something directly in here
+                    # Otherwise (if it was 1 or 2 Tab-stops more to the left) we'd take execution time without
+                    # having done anything
+                    self.teardown()
+        except Exception as e:
+            logger.warning(f"Uncought exception {e}")
+            utils.traceback(exception_in=e)
 
         self.statistics.update_teststep_sequence()
 
@@ -84,9 +88,13 @@ class TestStepMaster:
             try:
                 self.executeDirectSingle(index, command)
             except Exception as ex:
-                logger.info(ex)
-            # self.statistics.update_teststep()  --> Moved to BrowserHandling into the activities themselves,
-            # so that we can also count for externally (subclassed) activitis
+                # 2020-07-16: This Exception is cought, printed and then nothing. That's not good. The test run
+                # continues forever, despite this exception. Correct way would be to set test case to error and stop
+                # this test case
+                # Before we change something here we should check, why the calling function raises an error.
+                logger.critical(ex)
+                self.testcaseDataDict[GC.TESTCASESTATUS] = GC.TESTCASESTATUS_ERROR
+                return
 
     def manageNestedCondition(self, condition="", ifis=False):
         if condition.upper() == "IF":
@@ -113,8 +121,11 @@ class TestStepMaster:
         # when we have an IF-condition and it's condition was not TRUE, then skip whatever comes here until we
         # reach Endif
         if not self.ifIsTrue and not self.elseIsTrue:
+            if command["Activity"].upper() == "IF":
+                self.manageNestedCondition(condition=command["Activity"].upper(), ifis=self.ifIsTrue)
+                return True
             if command["Activity"].upper() != "ELSE" and command["Activity"].upper() != "ENDIF":
-                return
+                return True
         if self.repeatIsTrue[-1]: # If repeat statement is active then execute this
             if command["Activity"].upper() == "REPEAT": # To sync active repeat with repeat done
                 self.repeatActive += 1
@@ -148,33 +159,16 @@ class TestStepMaster:
                             data_list = random.sample(data_list, int(self.repeatCount[-1]))
                         except:
                             pass
-                    if data_list:
-                        for data in data_list:
-                            temp_dic = dict(self.repeatDict[-1])
-                            processed_data = {}
-                            for key in temp_dic:
-                                try:
-                                    processed_data = dict(temp_dic[key])
-                                except Exception as ex:
-                                    logger.debug(ex)
-                                try:
-                                    self.executeDirectSingle(key, processed_data, replaceFromDict=data)
-                                except Exception as ex:
-                                    logger.info(ex)
-                    else:
+                    for data in data_list:
                         temp_dic = dict(self.repeatDict[-1])
+                        processed_data = {}
                         for key in temp_dic:
                             try:
                                 processed_data = dict(temp_dic[key])
                             except Exception as ex:
                                 logger.debug(ex)
-                            for ky in processed_data:
-                                try:
-                                    processed_data[ky] = self.replaceVariables(processed_data[ky])
-                                except Exception as ex:
-                                    logger.info(ex)
                             try:
-                                self.executeDirectSingle(key, processed_data)
+                                self.executeDirectSingle(key, processed_data, replaceFromDict=data)
                             except Exception as ex:
                                 logger.info(ex)
                 del self.repeatIsTrue[-1]
@@ -184,20 +178,11 @@ class TestStepMaster:
                 self.repeatDone -= 1
                 self.repeatActive -= 1
                 return
-        lActivity = command["Activity"].upper()
-        if lActivity == "COMMENT":
-            return  # Comment's are ignored
 
-        lLocatorType = command["LocatorType"].upper()
-        try:
-            lLocator = self.replaceVariables(command["Locator"])
-        except Exception as ex:
-            logger.info(ex)
+        css, id, lActivity, lLocator, lLocatorType, xpath = self._extractAllSingleValues(command)
 
-        if lLocator and not lLocatorType:  # If locatorType is empty, default it to XPATH
-            lLocatorType = 'XPATH'
-
-        xpath, css, id = self.__setLocator(lLocatorType, lLocator)
+        if lActivity == "COMMENT":
+            return  # Comment's are ignored
 
         if self.anchor and xpath:
             if xpath[0:3] == '///':         # Xpath doesn't want to use Anchor
@@ -216,7 +201,7 @@ class TestStepMaster:
         lRelease = command["Release"]
 
         # Timeout defaults to 20 seconds, if not set otherwise.
-        lTimeout = TestStepMaster.__setTimeout(command["Timeout"])
+        lTimeout = TestStepMaster._setTimeout(command["Timeout"])
 
         lTimingString = f"TS {commandNumber} {lActivity.lower()}"
         self.timing.takeTime(lTimingString)
@@ -230,14 +215,30 @@ class TestStepMaster:
             logger.debug(f"we skipped this line due to {lRelease} disqualifies according to {self.globalRelease} ")
             return
         if lActivity == "GOTOURL":
-            self.browserSession.goToUrl(lValue)
+            if lValue:
+                self.browserSession.goToUrl(lValue)
+            elif lLocator:
+                self.browserSession.goToUrl(lLocator)
+            else:
+                logger.critical("GotoURL without URL called. Aborting. "
+                                "Please provide URL either in Value or Locator columns")
         elif lActivity == "SETTEXT":
-            self.browserSession.findByAndSetText(xpath=xpath, css=css, id=id, value=lValue, timeout=lTimeout)
+            self.browserSession.findByAndSetText(xpath=xpath, css=css, id=id, value=lValue, timeout=lTimeout,
+                                                 optional=lOptional)
         elif lActivity == "SETTEXTIF":
             self.browserSession.findByAndSetTextIf(xpath=xpath, css=css, id=id, value=lValue, timeout=lTimeout,
                                                    optional=lOptional)
         elif lActivity == "FORCETEXT":
-            self.browserSession.findByAndForceText(xpath=xpath, css=css, id=id, value=lValue, timeout=lTimeout)
+            self.browserSession.findByAndForceText(xpath=xpath, css=css, id=id, value=lValue, timeout=lTimeout,
+                                                   optional=lOptional)
+        elif lActivity == "FORCETEXTIF":
+            if lValue:
+                self.browserSession.findByAndForceText(xpath=xpath, css=css, id=id, value=lValue, timeout=lTimeout,
+                                                       optional=lOptional)
+        elif lActivity == "FORCETEXTJS":
+            if lValue:
+                self.browserSession.findByAndForceViaJS(xpath=xpath, css=css, id=id, value=lValue, timeout=lTimeout,
+                                                        optional=lOptional)
         elif lActivity == "SETANCHOR":
             if not lLocator:
                 self.anchor = None
@@ -261,9 +262,10 @@ class TestStepMaster:
                 lWindow = int(lWindow)
             self.browserSession.handleWindow(windowNumber=lWindow, timeout=lTimeout)
         elif lActivity == "CLICK":
-            self.browserSession.findByAndClick(xpath=xpath, css=css, id=id, timeout=lTimeout)
+            self.browserSession.findByAndClick(xpath=xpath, css=css, id=id, timeout=lTimeout, optional=lOptional)
         elif lActivity == "CLICKIF":
-            self.browserSession.findByAndClickIf(xpath=xpath, css=css, id=id, timeout=lTimeout, value=lValue)
+            self.browserSession.findByAndClickIf(xpath=xpath, css=css, id=id, timeout=lTimeout, value=lValue,
+                                                 optional=lOptional)
         elif lActivity == "PAUSE":
             self.browserSession.sleep(seconds=float(lValue))
         elif lActivity == "IF":
@@ -273,14 +275,16 @@ class TestStepMaster:
                                                     timeout=lTimeout)
 
             self.__doComparisons(lComparison=lComparison, value1=lValue, value2=lValue2)
-            logger.debug(f"IF-condition {lValue} {lComparison} {lValue2} evaluated to: {self.ifIsTrue} ")
+            logger.debug(f"IF-condition original Value: {original_value} (transformed: {lValue}) {lComparison} {lValue2} "
+                         f"evaluated to: {self.ifIsTrue} ")
         elif lActivity == "ELSE":
             if not self.ifIsTrue:
                 self.manageNestedCondition(condition=lActivity)
                 logger.debug("Executing ELSE-condition")
+            else:
+                self.ifIsTrue = False
         elif lActivity == "ENDIF":
-            self.ifIsTrue = True
-            self.elseIsTrue = False
+            self.manageNestedCondition(condition=lActivity)
         elif lActivity == "REPEAT":
             self.repeatActive += 1
             self.repeatIsTrue.append(True)
@@ -323,6 +327,7 @@ class TestStepMaster:
             value_found = self.browserSession.findByAndWaitForValue(xpath=xpath, css=css, id=id, optional=lOptional,
                                                                     timeout=lTimeout)
             if not self.__doComparisons(lComparison=lComparison, value1=value_found, value2=lValue):
+                logger.error(f"Condition {value_found} {lComparison} {lValue} was not met during assert.")
                 raise baangtTestStepException(f"Expected Value: {lValue}, Value found :{value_found} ")
         elif lActivity == 'IBAN':
             # Create Random IBAN. Value1 = Input-Parameter for IBAN-Function. Value2=Fieldname
@@ -337,11 +342,26 @@ class TestStepMaster:
             self.testcaseDataDict[GC.TESTCASESTATUS_STOP] = "X"              # will stop the test case
         elif lActivity == GC.TESTCASESTATUS_STOPERROR.upper():
             self.testcaseDataDict[GC.TESTCASESTATUS_STOPERROR] = "X"         # will stop the test case and set error
+        elif lActivity[0:3] == "ZZ_":
+            # custom command. Do nothing and return
+            return
         else:
             raise BaseException(f"Unknown command in TestStep {lActivity}")
 
         self.timing.takeTime(lTimingString)
 
+    def _extractAllSingleValues(self, command):
+        lActivity = command["Activity"].upper()
+        lLocatorType = command["LocatorType"].upper()
+        try:
+            lLocator = self.replaceVariables(command["Locator"])
+        except Exception as ex:
+            logger.info(ex)
+        if lLocator and not lLocatorType:  # If locatorType is empty, default it to XPATH
+            lLocatorType = 'XPATH'
+        xpath, css, id = self.__setLocator(lLocatorType, lLocator)
+        return css, id, lActivity, lLocator, lLocatorType, xpath
+
     def doPDFComparison(self, lValue, lFieldnameForResults="DOC_Compare"):
         lFiles = self.browserSession.findNewFiles()
         if len(lFiles) > 1:
@@ -361,18 +381,13 @@ class TestStepMaster:
 
     def replaceAllVariables(self, lValue, lValue2, replaceFromDict=None):
         # Replace variables from data file
-        if len(lValue) > 0:
-            try:
+        try:
+            if len(lValue) > 0:
                 lValue = self.replaceVariables(lValue, replaceFromDict=replaceFromDict)
-            except Exception as ex:
-                logger.info(f"Exception in {lValue}")
-                logger.info(ex)
-        if len(lValue2) > 0:
-            try:
+            if len(lValue2) > 0:
                 lValue2 = self.replaceVariables(lValue2, replaceFromDict=replaceFromDict)
-            except Exception as ex:
-                logger.info(f"Exception in {lValue2}")
-                logger.info(ex)
+        except Exception as ex:
+            logger.warning(f"During replacement of variables an error happened: {ex}")
         return lValue, lValue2
 
     def __getIBAN(self, lValue, lValue2):
@@ -460,7 +475,7 @@ class TestStepMaster:
         return utils.setLocatorFromLocatorType(lLocatorType, lLocator)
 
     @staticmethod
-    def __setTimeout(lTimeout):
+    def _setTimeout(lTimeout):
         return 20 if not lTimeout else float(lTimeout)
 
     def __doComparisons(self, lComparison, value1, value2):
@@ -471,13 +486,18 @@ class TestStepMaster:
         if value2 == 'None':
             value2 = None
 
+        logger.debug(f"Evaluating IF-Condition: Value1 = {value1}, comparison={lComparison}, value2={value2}")
+
         if lComparison == "=":
             if value1 == value2:
                 self.manageNestedCondition(condition="IF", ifis=True)
             else:
                 self.manageNestedCondition(condition="IF", ifis=False)
         elif lComparison == "!=":
-            self.ifIsTrue = False if value1 == value2 else True
+            if value1 != value2:
+                self.manageNestedCondition(condition="IF", ifis=True)
+            else:
+                self.manageNestedCondition(condition="IF", ifis=False)
         elif lComparison == ">":
             if value1 > value2:
                 self.manageNestedCondition(condition="IF", ifis=True)
@@ -488,6 +508,26 @@ class TestStepMaster:
                 self.manageNestedCondition(condition="IF", ifis=True)
             else:
                 self.manageNestedCondition(condition="IF", ifis=False)
+        elif lComparison == ">=":
+            if value1 >= value2:
+                self.manageNestedCondition(condition="IF", ifis=True)
+            else:
+                self.manageNestedCondition(condition="IF", ifis=False)
+        elif lComparison == "<=":
+            if value1 <= value2:
+                self.manageNestedCondition(condition="IF", ifis=True)
+            else:
+                self.manageNestedCondition(condition="IF", ifis=False)
+        elif lComparison.upper() == "IP":     # Is Part of (Value 1 is part of Value 2)
+            if value1 in value2:
+                self.manageNestedCondition(condition="IF", ifis=True)
+            else:
+                self.manageNestedCondition(condition="IF", ifis=False)
+        elif lComparison.upper() == 'CO':      # COntains (Value 1 contains Value 2)
+            if value2 in value1:
+                self.manageNestedCondition(condition="IF", ifis=True)
+            else:
+                self.manageNestedCondition(condition="IF", ifis=False)
         elif not lComparison:  # Check only, if Value1 has a value.
             val = True if value1 else False
             self.manageNestedCondition(condition="IF", ifis=val)
@@ -573,7 +613,7 @@ class TestStepMaster:
                     centerValue = self.apiSession.session[1].answerJSON.get(dictValue, "Empty")
                 elif dictVariable == 'FAKER':
                     # This is to call Faker Module with the Method, that is given after the .
-                    centerValue = self.__getFakerData(dictValue)
+                    centerValue = self._getFakerData(dictValue)
                 elif self.testcaseDataDict.get(dictVariable):
                     dic = self.testcaseDataDict.get(dictVariable)
                     for key in center.split('.')[1:]:
@@ -585,17 +625,15 @@ class TestStepMaster:
             if not centerValue:
                 if center in self.testcaseDataDict.keys():
                     # The variable exists, but has no value.
-                    return None
+                    centerValue = ""
                 else:
                     raise BaseException(f"Variable not found: {center}, input parameter was: {expression}")
-            try:
-                expression = "".join([left_part, centerValue, right_part])
-            except:
+            if not isinstance(centerValue, list) and not isinstance(centerValue, dict):
+                expression = "".join([left_part, str(centerValue), right_part])
+            else:
                 expression = centerValue
-                if type(expression) == int:
+                if type(expression) == float or type(expression) == int:
                     expression = str(int(expression))
-                elif type(expression) == float:
-                    expression = str(float(expression))
         return expression
 
     def iterate_json(self, data, key):
@@ -610,7 +648,7 @@ class TestStepMaster:
             return data.get(key)
 
 
-    def __getFakerData(self, fakerMethod):
+    def _getFakerData(self, fakerMethod):
         if not self.baangtFaker:
             self.baangtFaker = baangtFaker()
 

baangt/base/BrowserFactory.py

@@ -28,6 +28,8 @@ class BrowserFactory:
             if self.globalSettings.get('TC.' + GC.EXECUTION_NETWORK_INFO) == True else None
         self.browsersMobProxies = {}
 
+        self.callsPerBrowserInstance = {}
+
         self.__startRotatingProxies()
 
     def __startRotatingProxies(self):
@@ -75,9 +77,12 @@ class BrowserFactory:
                 self.browser[browserInstance].slowExecutionToggle()
             return self.browser[browserInstance]
         else:
-            if self.globalSettings.get("TC.RestartBrowser"):
+
+            lRestartBrowserSession = self.checkMaxBrowserInstanceCallsUsedToRestart(browserInstance)
+
+            if self.globalSettings.get("TC.RestartBrowser") or lRestartBrowserSession:
                 if browserInstance in self.browser.keys():
-                    logger.debug(f"Instance {browserInstance}: TC.RestartBrowser was set. Quitting old browser.")
+                    logger.debug(f"Instance {browserInstance}: TC.RestartBrowser was set or Threshold-Limit for Testcases reached. Quitting old browser.")
                     lBrowser = self.browser[browserInstance]
                     lBrowser.closeBrowser()
                     del self.browser[browserInstance]
@@ -116,6 +121,24 @@ class BrowserFactory:
                 logger.debug(f"Using existing instance of browser {browserInstance}")
             return self.browser[browserInstance]
 
+    def checkMaxBrowserInstanceCallsUsedToRestart(self, browserInstance):
+        """
+        for each browserInstance we record how often it was used by a testcase.
+        If there's a Number n RestartBrowserAfter Global parameter and we're higher than that,
+        we restart the browser.
+        :param browserInstance:
+        :return:
+        """
+        lRestartBrowserSession = False
+        self.callsPerBrowserInstance[browserInstance] = self.callsPerBrowserInstance.get(browserInstance, 0) + 1
+        if self.callsPerBrowserInstance[browserInstance] > int(
+                self.globalSettings.get("TC.RestartBrowserAfter", 99999999)):
+            lRestartBrowserSession = True
+            logger.debug(f"Reached threshold for browser restart in instance {browserInstance}. Restarting Browser.")
+            self.callsPerBrowserInstance[browserInstance] = 0
+
+        return lRestartBrowserSession
+
     @staticmethod
     def setBrowserWindowSize(lBrowserInstance: BrowserDriver, browserWindowSize):
         lBrowserInstance.setBrowserWindowSize(browserWindowSize)

baangt/base/BrowserHandling/BrowserHandling.py

@@ -86,7 +86,7 @@ class BrowserDriver:
         self.takeTime("Browser Start")
         self.randomProxy = randomProxy
         self.browserName = browserName
-        self.bpid = []
+        self.browserProcessID = []
         lCurPath = Path(self.managedPaths.getOrSetDriverPath())
 
         if browserName in webDrv.BROWSER_DRIVERS:
@@ -99,14 +99,14 @@ class BrowserDriver:
             elif GC.BROWSER_FIREFOX == browserName:
                 self.browserData.driver = self._browserFirefoxRun(browserName, lCurPath, browserProxy, randomProxy, desiredCapabilities)
                 helper.browserHelper_startBrowsermobProxy(browserName=browserName, browserInstance=browserInstance, browserProxy=browserProxy)
-                self.bpid.append(self.browserData.driver.capabilities.get("moz:processID"))
+                self.browserProcessID.append(self.browserData.driver.capabilities.get("moz:processID"))
             elif GC.BROWSER_CHROME == browserName:
                 self.browserData.driver = self._browserChromeRun(browserName, lCurPath, browserProxy, randomProxy, desiredCapabilities)
                 helper.browserHelper_startBrowsermobProxy(browserName=browserName, browserInstance=browserInstance, browserProxy=browserProxy)
                 try:
                     port = self.browserData.driver.capabilities['goog:chromeOptions']["debuggerAddress"].split(":")[1]
                     fp = os.popen(f"lsof -nP -iTCP:{port} | grep LISTEN")
-                    self.bpid.append(int(fp.readlines()[-1].split()[1]))
+                    self.browserProcessID.append(int(fp.readlines()[-1].split()[1]))
                 except Exception as ex:
                     logger.info(ex)
             elif GC.BROWSER_EDGE == browserName:
@@ -123,8 +123,8 @@ class BrowserDriver:
                                                         command_executor=GC.REMOTE_EXECUTE_URL,
                                                         desired_capabilities=desiredCapabilities)
             else:
-                # TODO add exception, this code should never be reached
-                pass
+                logger.critical(f"Browsername not found: {browserName}. Cancelling test run")
+                raise SystemError(f"Browsername not found: {browserName}. Cancelling test run")
         elif GC.BROWSER_REMOTE_V4 == browserName:
             desired_capabilities, seleniumGridIp, seleniumGridPort = helper.browserHelper_setSettingsRemoteV4(desiredCapabilities)
 
@@ -218,8 +218,8 @@ class BrowserDriver:
         try:
             if self.browserData.driver:
                 try:
-                    if len(self.bpid) > 0:
-                        for bpid in self.bpid:
+                    if len(self.browserProcessID) > 0:
+                        for bpid in self.browserProcessID:
                             os.kill(bpid, signal.SIGINT)
                 except:
                     pass
@@ -356,27 +356,29 @@ class BrowserDriver:
         returnValue = None
         start = time.time()
         duration = 0
+        retry = True
 
-        while not self.element and duration < timeout:
+        while retry and duration < timeout:
             self.element, self.html = self.findBy(id=id, css=css, xpath=xpath, class_name=class_name, iframe=iframe, timeout=timeout / 3,
                             optional=optional)
             time.sleep(0.5)
             duration = time.time() - start
 
-        if self.element:
-            try:
-                if len(self.element.text) > 0:
-                    returnValue = self.element.text
-                elif self.element.tag_name == 'input':
-                    #  element is of type <input />
-                    returnValue = self.element.get_property('value')
-                else:
-                    returnValue = None
-            except Exception as e:
-                logger.debug(f"Exception during findByAndWaitForValue, but continuing {str(e)}, "
-                             f"Locator: {self.browserData.locatorType} = {self.browserData.locator}")
-        else:
-            logger.info(f"Couldn't find value for element {self.browserData.locatorType}:{self.browserData.locator}")
+            if self.element:
+                try:
+                    if len(self.element.text) > 0:
+                        returnValue = self.element.text.strip()
+                    elif self.element.tag_name == 'input':
+                        #  element is of type <input />
+                        returnValue = self.element.get_property('value').strip()
+                except Exception as e:
+                    logger.debug(f"Exception during findByAndWaitForValue, but continuing {str(e)}, "
+                                 f"Locator: {self.browserData.locatorType} = {self.browserData.locator}")
+            else:
+                logger.info(f"Couldn't find value for element {self.browserData.locatorType}:{self.browserData.locator}")
+
+            if returnValue and len(returnValue.strip()) > 0:
+                return returnValue
 
         return returnValue
 
@@ -385,7 +387,7 @@ class BrowserDriver:
         """
         Please see documentation in findBy and __doSomething
         """
-        self.element, self.html = self.findBy(id=id, css=css, xpath=xpath, class_name=class_name, iframe=iframe, timeout=timeout)
+        self.element, self.html = self.findBy(id=id, css=css, xpath=xpath, class_name=class_name, iframe=iframe, timeout=timeout, optional=optional)
         if not self.element:
             return False
 
@@ -460,8 +462,8 @@ class BrowserDriver:
         @return wasSuccessful says, whether the element was found.
         """
 
-        self.element, self.html = self.findBy(id=id, css=css, xpath=xpath, class_name=class_name, iframe=iframe, timeout=timeout,
-                                    optional=optional)
+        self.element, self.html = self.findBy(id=id, css=css, xpath=xpath, class_name=class_name, iframe=iframe,
+                                              timeout=timeout, optional=optional)
 
         if not self.element:
             logger.debug("findBy didn't work in findByAndClick")
@@ -481,7 +483,7 @@ class BrowserDriver:
         isValid = False
         if not value:
             pass
-        elif len(value) == 0 or str(value) == "0":
+        elif len(str(value)) == 0 or str(value) == "0":
             pass
         else:
             isValid = True
@@ -500,7 +502,8 @@ class BrowserDriver:
         If value is evaluated to "True", the click-event is executed.
         """
         if self._isValidKeyValue(value):
-            return self.findByAndClick(id=id, css=css, xpath=xpath, class_name=class_name, iframe=iframe, timeout=timeout, optional=optional)
+            return self.findByAndClick(id=id, css=css, xpath=xpath, class_name=class_name, iframe=iframe,
+                                       timeout=timeout, optional=optional)
         else:
             return False
 
@@ -511,13 +514,39 @@ class BrowserDriver:
 
         """
 
-        self.element, self.html = self.findBy(id=id, css=css, xpath=xpath, class_name=class_name, iframe=iframe, timeout=timeout)
+        self.element, self.html = self.findBy(id=id, css=css, xpath=xpath, class_name=class_name, iframe=iframe,
+                                              timeout=timeout, optional=optional)
 
         if not self.element:
             return False
 
         return webDrv.webdriver_doSomething(GC.CMD_FORCETEXT, self.element, value=value, timeout=timeout, optional=optional, browserData = self.browserData)
 
+    def findByAndForceViaJS(self, id=None, css=None, xpath:str=None, class_name=None, value=None,
+                           iframe=None, timeout=60, optional=False):
+        """
+        Identifies the object via JS and set's the desired value via JS
+        """
+        # element, html = self.findBy(id=id ,css=css, xpath=xpath, class_name=class_name, iframe=iframe,
+        # timeout=timeout, optional=optional)
+        # didn't work to give the element to JavaScript-method
+
+        xpath = xpath.replace('"', "'")
+        xpath = xpath.replace("'", "\\'")
+        lJSText = "\n".join(
+            [f"var zzbaangt = document.evaluate('{xpath}', document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);",
+             "if (zzbaangt.snapshotLength > 0) { ",
+             f"zzbaangt[0].value='{value}';",
+             "};",
+             f""
+            ]
+        )
+        logger.debug(f"Setting element using JS-Text: {lJSText}")
+
+        # lJSText = f"arguments[0].value='{value}';" --> Didn't work with Element.
+
+        self.javaScript(lJSText)
+
     def setBrowserWindowSize(self, browserWindowSize: str):
         """
         Resized the browser Window to a fixed size

baangt/base/BrowserHandling/BrowserHelperFunction.py

@@ -158,27 +158,38 @@ class BrowserHelperFunction:
         gecko = response.json()
         gecko = gecko['assets']
         gecko_length_results = len(gecko)
-        drivers_url_dict = []
+        drivers_urls = []
 
         for i in range(gecko_length_results):
-            drivers_url_dict.append(gecko[i]['browser_download_url'])
+            drivers_urls.append(gecko[i]['browser_download_url'])
 
+        # New approach August 2018
         isTarFile = True
-        zipbObj = zip(GC.OS_list, drivers_url_dict)
-        geckoDriversDict = dict(zipbObj)
-        if platform.system().lower() == GC.WIN_PLATFORM:
+
+        if ctypes.sizeof(ctypes.c_voidp) == GC.BIT_64:
+            searchBits = '64'
+        else:
+            searchBits = '32'
+
+        searchString = None
+        if platform.system().lower() == GC.PLATFORM_WINDOWS:
+            searchString = f"win{searchBits}.zip"
             isTarFile = False
-            if ctypes.sizeof(ctypes.c_voidp) == GC.BIT_64:
-                url = geckoDriversDict[GC.OS_list[4]]
-            else:
-                url = geckoDriversDict[GC.OS_list[3]]
-        elif platform.system().lower() == GC.LINUX_PLATFORM:
-            if ctypes.sizeof(ctypes.c_voidp) == GC.BIT_64:
-                url = geckoDriversDict[GC.OS_list[1]]
-            else:
-                url = geckoDriversDict[GC.OS_list[0]]
+        elif platform.system().lower() == GC.PLATFORM_LINUX:
+            searchString = f"linux{searchBits}.tar.gz"
+        elif platform.system().lower() == GC.PLATFORM_MAC:
+            searchString = "macos.tar.gz"
         else:
-            url = geckoDriversDict[GC.OS_list[2]]
+            logger.critical(f"Don't know how to download drivers for this OS: {platform.system().lower()}. "
+                            f"Please download and put in folder /browserDrivers")
+            return None, None
+
+        url = [x for x in drivers_urls if searchString in x][0]
+        if not url:
+            logger.critical(f"Could not find driver for {searchString} in this "
+                            f"list of available drivers: {','.join(drivers_urls)}")
+
+        logger.debug(f"Downloading Geckodriver for Firefox from here {url}")
 
         return url, isTarFile
 
@@ -196,13 +207,15 @@ class BrowserHelperFunction:
 
         zipbObjChrome = zip(GC.OS_list, chromedriver_url_dict)
         chromeDriversDict = dict(zipbObjChrome)
-        if platform.system().lower() == GC.WIN_PLATFORM:
+        if platform.system().lower() == GC.PLATFORM_WINDOWS:
             url = chromeDriversDict[GC.OS_list[3]]
-        elif platform.system().lower() == GC.LINUX_PLATFORM:
+        elif platform.system().lower() == GC.PLATFORM_LINUX:
             url = chromeDriversDict[GC.OS_list[1]]
         else:
             url = chromeDriversDict[GC.OS_list[2]]
 
+        logger.debug(f"Downloading Chromedriver from here: {url}")
+
         return url
   
     @staticmethod
@@ -214,7 +227,7 @@ class BrowserHelperFunction:
         with zipfile.ZipFile(path_zip, 'r') as zip_ref:
             zip_ref.extractall(path)
 
-        if platform.system().lower() != GC.WIN_PLATFORM:
+        if platform.system().lower() != GC.PLATFORM_WINDOWS:
             file_path = path.joinpath(driverName.replace('.exe', ''))
             os.chmod(file_path, 0o777)
         os.remove(path_zip)

baangt/base/BrowserHandling/WebdriverFunctions.py

@@ -296,10 +296,12 @@ class WebdriverFunctions:
                     element.click()
                 elif command.upper() == GC.CMD_FORCETEXT:
                     element.clear()
-                    for i in range(0, NUMBER_OF_SEND_KEY_BACKSPACE):
-                        element.send_keys(keys.Keys.BACKSPACE)
-                    time.sleep(0.1)
+                    element.click()
+                    # element.send_keys(keys.Keys.CONTROL+"A")
+                    # for i in range(0, NUMBER_OF_SEND_KEY_BACKSPACE):
+                    #     element.send_keys(keys.Keys.BACKSPACE)
                     element.send_keys(value)
+                    element.send_keys(keys.Keys.TAB)
                 didWork = True
             except ElementClickInterceptedException as e:
                 logger.debug("doSomething: Element intercepted - retry")
@@ -310,7 +312,8 @@ class WebdriverFunctions:
                 if counter < COUNTER_LIMIT_RETRY:
                     time.sleep(0.2)
                 elif counter < COUNTER_LIMIT_ELEMENT_REF:
-                    begin, element = WebdriverFunctions.webdriver_refindElementAfterError(browserData, timeout)
+                    begin, element = WebdriverFunctions.webdriver_refindElementAfterError(browserData, timeout,
+                                                                                          optional=optional)
                 else:
                     raise Exceptions.baangtTestStepException(e)
             except NoSuchElementException as e:
@@ -325,7 +328,8 @@ class WebdriverFunctions:
                     time.sleep(0.2)
                 elif counter < COUNTER_LIMIT_ELEMENT_NOT_INTERACT:
                     logger.debug(f"Element not interactable {browserData.locatorType} {browserData.locator}, re-finding element")
-                    begin, element = WebdriverFunctions.webdriver_refindElementAfterError(browserData, timeout)
+                    begin, element = WebdriverFunctions.webdriver_refindElementAfterError(browserData, timeout,
+                                                                                          optional=optional)
                 else:
                     helper.browserHelper_log(logging.ERROR, f"Element not interactable {e}", browserData)
                     raise Exceptions.baangtTestStepException(e)
@@ -346,13 +350,18 @@ class WebdriverFunctions:
         return didWork
 
     @staticmethod
-    def webdriver_refindElementAfterError(browserData, timeout):
+    def webdriver_refindElementAfterError(browserData, timeout, optional=None):
+        if not optional:
+            optional = False
         element, _ = WebdriverFunctions.webdriver_tryAndRetry(browserData, timeout=timeout / 2, optional=True)
         if element:
             logger.debug(f"Re-Found element {browserData.locatorType}: {browserData.locator}, will retry ")
             begin = time.time()
         else:
-            raise Exceptions.baangtTestStepException(
-                f"Element {browserData.locatorType} {browserData.locator} couldn't be found. "
-                f"Tried to re-find it, but not element was not found")
+            if not optional:
+                raise Exceptions.baangtTestStepException(
+                    f"Element {browserData.locatorType} {browserData.locator} couldn't be found. "
+                    f"Tried to re-find it, but not element was not found")
+            else:
+                return None, None
         return begin, element

baangt/base/Cleanup.py

@@ -5,6 +5,7 @@ from pathlib import Path
 from datetime import datetime
 from baangt.base.PathManagement import ManagedPaths
 from baangt.base.DownloadFolderMonitoring import DownloadFolderMonitoring
+from baangt.reports import Report
 
 logger = logging.getLogger("pyC")
 
@@ -93,7 +94,32 @@ class Cleanup:
         else:
             logger.info(f"{str(len(removed))} empty folder deleted")
 
+    def clean_reports(self):
+        #
+        # cleanup reports
+        #
+
+        logger.info(f"Removing reports older than {str(self.threshold)} days")
+        reports_dir = self.managedPaths.getOrSetReportPath()
+        files = Path(reports_dir).glob('**/*')
+        removed = []
+        for file in files:
+            timedelta = datetime.now() - datetime.fromtimestamp(DownloadFolderMonitoring.creation_date(file))
+            if timedelta.total_seconds()/86400 > self.threshold:
+                try:
+                    os.remove(str(file))
+                    removed.append(file)
+                except:
+                    logger.debug(f"Cannot remove {str(file)}")
+                    continue
+                logger.debug(f"{str(file)} deleted")
+        if len(removed) == 0:
+            logger.info(f"No report older than {str(self.threshold)} days found")
+        else:
+            logger.info(f"{str(len(removed))} reports deleted")
+
     def clean_all(self):
         self.clean_logs()
         self.clean_screenshots()
         self.clean_downloads()
+        self.clean_reports()

baangt/base/CustGlobalConstants.py

@@ -1,6 +1,6 @@
 CUST_TOASTS = "Toasts"
 # CUST_TOASTS_ERROR = "ToastsError" --> Replaced by GC.TESTCASEERRORLOG
-VIGOGFNUMMER = "GF#"
+VIGOGFNUMMER = "VIGO-GF"
 SAPPOLNR = "SAP Polizzennr"
 PRAEMIE = "Prämie"
 POLNRHOST = "PolNR Host"

baangt/base/DataBaseORM.py

@@ -1,12 +1,11 @@
 from sqlalchemy import Column, String, Integer, DateTime, Boolean, Table, ForeignKey
-#from sqlalchemy.types import Binary(16), TypeDecorator
 from sqlalchemy.orm import relationship
 from sqlalchemy import create_engine
 from sqlalchemy.ext.declarative import declarative_base
-import os
-import re
-import uuid
+import os, re, uuid
+from datetime import timedelta
 from baangt.base.PathManagement import ManagedPaths
+from baangt.base.GlobalConstants import EXECUTION_STAGE, TIMING_DURATION, TESTCASESTATUS
 
 
 managedPaths = ManagedPaths()
@@ -47,7 +46,7 @@ class TestrunLog(base):
 	statusOk = Column(Integer, nullable=False)
 	statusFailed = Column(Integer, nullable=False)
 	statusPaused = Column(Integer, nullable=False)
-	RLPJson = Column(String, nullable=True)
+	RLPJson = Column(String, nullable=True) # ------------------------> New feature: comment for old DB
 	# relationships
 	globalVars = relationship('GlobalAttribute')
 	testcase_sequences = relationship('TestCaseSequenceLog')
@@ -59,7 +58,14 @@ class TestrunLog(base):
 	@property
 	def duration(self):
 		return (self.endTime - self.startTime).seconds
-	
+
+	@property
+	def stage(self):
+		for gv in self.globalVars:
+			if gv.name == EXECUTION_STAGE:
+				return gv.value
+
+		return None
 
 	def __str__(self):
 		return str(uuid.UUID(bytes=self.id))
@@ -110,11 +116,21 @@ class TestCaseSequenceLog(base):
 	__tablename__ = 'testCaseSequences'
 	# columns
 	id = Column(Binary(16), primary_key=True, default=uuidAsBytes)
+	number = Column(Integer, nullable=False)
 	testrun_id = Column(Binary(16), ForeignKey('testruns.id'), nullable=False)
 	# relationships
 	testrun = relationship('TestrunLog', foreign_keys=[testrun_id])
 	testcases = relationship('TestCaseLog')
 
+	@property
+	def duration(self):
+		#
+		# duration in seconds
+		#
+
+		return sum([tc.duration for tc in self.testcases if tc.duration])
+
+
 	def __str__(self):
 		return str(uuid.UUID(bytes=self.id))
 
@@ -136,15 +152,53 @@ class TestCaseLog(base):
 	__tablename__ = 'testCases'
 	# columns
 	id = Column(Binary(16), primary_key=True, default=uuidAsBytes)
+	number = Column(Integer, nullable=False)
 	testcase_sequence_id = Column(Binary(16), ForeignKey('testCaseSequences.id'), nullable=False)
 	# relationships
 	testcase_sequence = relationship('TestCaseSequenceLog', foreign_keys=[testcase_sequence_id])
-	fields = relationship('TestCaseField')
+	fields = relationship('TestCaseField', lazy='select')
 	networkInfo = relationship('TestCaseNetworkInfo')
 
+	@property
+	def status(self):
+		#
+		# testcase status
+		#
+
+		for field in self.fields:
+			if field.name == TESTCASESTATUS:
+				return field.value
+
+		return None	
+
+	@property
+	def duration(self):
+		#
+		# duration in seconds
+		#
+
+		for field in self.fields:
+			if field.name == TIMING_DURATION:
+				# parse value from H:M:S.microseconds
+				m = re.search(r'(?P<hours>\d+):(?P<minutes>\d+):(?P<seconds>\d[\.\d+]*)', field.value)
+				if m:
+					factors = {
+						'hours': 3600,
+						'minutes': 60,
+						'seconds': 1,
+					}
+					#duration = {key: float(value) for key, value in m.groupdict().items()}
+					#return timedelta(**duration)
+					return sum([factors[key]*float(value) for key, value in m.groupdict().items()])
+
+		return None
+
 	def __str__(self):
 		return str(uuid.UUID(bytes=self.id))
 
+	def fields_as_dict(self):
+		return {pr.name: pr.value for pr in self.fields}
+
 	def to_json(self):
 		return {
 			'id': str(self),

baangt/base/ExportResults/Append2BaseXLS.py

@@ -0,0 +1,54 @@
+from logging import getLogger
+import baangt.base.GlobalConstants as GC
+from icopy2xls import Mover
+from baangt.base.PathManagement import ManagedPaths
+
+logger = getLogger("pyC")
+
+
+class Append2BaseXLS:
+    """
+    If in the globals of the current testrun the parameter AR2BXLS (Append Results to Base XLS) is set,
+    we execute baangt Move-Corresponding module accordingly.
+    """
+    def __init__(self, testRunInstance, resultsFileName: str=None):
+        self.testRunInstance = testRunInstance
+        self.resultsFileName = resultsFileName
+        self.mp = ManagedPaths()
+
+        self._append2BaseXLS()
+
+    def _append2BaseXLS(self):
+        lGlobals = self.testRunInstance.globalSettings
+        if not lGlobals.get("AR2BXLS"):
+            logger.debug("No request to save to further destinations. Exiting.")
+            return
+
+        # Format: fileAndPath,Sheet;fileAndPath,sheet
+        fileTuples = []
+        if ";" in lGlobals.get("AR2BXLS"):
+            files = lGlobals.get("AR2BXLS").split(";")
+            for file in files:
+                fileTuples.append(self.checkAppend(file))
+        else:
+            fileTuples.append(self.checkAppend(lGlobals["AR2BXLS"]))
+
+        for fileTuple in fileTuples:
+            if not fileTuple:
+                logger.critical("File to append results to not found (see message above")
+                break
+            logger.info(f"Starting to append results to: {str(fileTuple)}")
+            lMover = Mover(source_file_path=self.resultsFileName,
+                           source_sheet="Output",
+                           destination_file_path=fileTuple[0],
+                           destination_sheet=fileTuple[1].strip())
+            lMover.move(filters={GC.TESTCASESTATUS:GC.TESTCASESTATUS_SUCCESS}, add_missing_columns=False)
+            logger.debug(f"Appending results to {str(fileTuple)} finished")
+
+    def checkAppend(self, file):
+        lFileAndPath = self.mp.findFileInAnyPath(filename=file.split(",")[0])
+        if lFileAndPath:
+            return [lFileAndPath, file.split(",")[1]]
+        else:
+            logger.critical(f"File not found anywhere: {file.split(',')[0]}")
+            return None

baangt/base/ExportResults/ExportConfluence.py

@@ -0,0 +1,130 @@
+from atlassian import Confluence
+from html import escape
+import xlrd3 as xlrd
+import os
+
+
+class ExportConfluence:
+    def __init__(self, url, space, pageTitle, fileNameAndPathToResultXLSX, username, password, rootPage=None,
+                 remove_headers=["json"], uploadOriginalFile=False, CreateSubPagesForEachXXEntries=0):
+        self.url = url
+        self.space = space
+        self.rootPage = rootPage
+        self.pageTitle = pageTitle
+        self.fileNameAndPathToResultXLSX = fileNameAndPathToResultXLSX
+        self.username = username
+        self.password = password
+        self.remove_headers = [headers.lower() for headers in remove_headers]
+        self.uploadOriginalFile = uploadOriginalFile
+        self.CreateSubPagesForEachXXEntries = CreateSubPagesForEachXXEntries
+        self.html = self.makeBody()
+        self.update_confluence()
+
+    def makeBody(self):
+        summary = self.xlsx2html(self.fileNameAndPathToResultXLSX, sheet="Summary")
+        if not self.CreateSubPagesForEachXXEntries:  # If not subPages then create a single page
+            output = self.xlsx2html(self.fileNameAndPathToResultXLSX, sheet="Output")
+            html = "<h1>Summary</h1>" + summary + "<br /><br /><h1>Output</h1>" + output  # joining output in main page
+        else:
+            html = "<h1>Summary</h1>" + summary + "<br />"  # Main page without output tab data
+        html = html.replace('\\',  '\\\\')
+        return html
+
+    def update_confluence(self):
+        confluence = Confluence(url=self.url, username=self.username, password=self.password)  # Confluence login
+        if self.uploadOriginalFile:  # if original xlsx_file is to be attach on the page
+            file = self.attach_file(confluence)
+            html = file + "<br /><br />" + self.html
+        else:
+            html = self.html
+        new_page = confluence.create_page(
+            self.space, self.pageTitle, html, parent_id=self.rootPage, type='page', representation='storage'
+        )  # creating main page
+        if self.CreateSubPagesForEachXXEntries:  # if we want subpages for output
+            try:  # getting page if of main page. With help of this id, sub pages are created
+                parent_page = new_page["id"]
+            except KeyError: # if key is not present then it is most probably inside a list with key "results"
+                parent_page = new_page["results"]["id"]
+            self.create_child_pages(confluence, parent_page)  # creating child pages
+
+    def xlsx2html(self, filePath, sheet):
+        wb = xlrd.open_workbook(filePath)
+        sht = wb.sheet_by_name(sheet)
+        data = []  # used to store rows, which are later joined to make complete html
+        remove_index = []  # used to store columns which are removable
+        header = []
+        for row in range(sht.nrows):
+            dt = []
+            for column in range(sht.ncols):
+                value = sht.cell_value(row, column)
+                if type(value) == float:
+                    if repr(value)[-2:] == '.0':
+                        value = int(value)
+                value = str(value)
+                if row == 0:
+                    if value.lower() in self.remove_headers:
+                        remove_index.append(column)  # storing column number of removable headers in a list
+                    else:
+                        header.append(escape(value))
+                else:
+                    if column not in remove_index:  # if column is not in remove_header list than add the data in html
+                        dt.append(escape(value))
+            data.append('<td>' + '</td>\n<td>'.join(dt) + '</td>')  # joining individual data of a row in single row
+        header = '<tr><th>' + '</th>\n<th>'.join(header) + '</th></tr>'
+        html = '<table><tbody>' + header + '<tr>' + '</tr>\n<tr>'.join(data) + '</tr>' + '</tbody></table>'  # joining list of rows to make html
+        return html
+
+    def attach_file(self, confluence):
+        fileName = os.path.basename(self.fileNameAndPathToResultXLSX)  # getting basename of xlsx file for title
+        # Attaching basefile
+        attach = confluence.attach_file(self.fileNameAndPathToResultXLSX, name=fileName, content_type=None,
+                               page_id=self.rootPage, title=self.pageTitle, space=self.space, comment=None)
+        try:
+            url = "/confluence"+attach["_links"]["download"].split(".xlsx?")[0] + ".xlsx"
+        except KeyError:  # if key is not present then it is most probably inside a list with key "results"
+            url = "/confluence"+attach["results"][0]["_links"]["download"].split(".xlsx?")[0] + ".xlsx"
+        link = f'<a href="{url}">{fileName}</a>'
+        html = "<h1>Original file</h1>"+link
+        return html
+
+    def create_child_pages(self, confluence, parent_page):
+        # Creating child pages if required
+        wb = xlrd.open_workbook(self.fileNameAndPathToResultXLSX)
+        output_xlsx = wb.sheet_by_name("Output")
+        # Getting starting points for each subpage
+        starting_points = [x for x in range(1, output_xlsx.nrows, self.CreateSubPagesForEachXXEntries)]
+        header = []  # header used in every subpage
+        remove_index = []  # headers column number which are not to be used are stored in it.
+        for x in range(output_xlsx.ncols):
+            value = output_xlsx.cell_value(0, x)
+            if value.lower() not in self.remove_headers:
+                header.append(value)
+            else:
+                remove_index.append(x)  # if header is to be removed its column number is stored here
+        header = '<tr><th>' + '</th>\n<th>'.join(header) + '</th></tr>'  # html table headers are formatted here
+        for starting in starting_points:
+            if starting + self.CreateSubPagesForEachXXEntries < output_xlsx.nrows:  # if it is not last sub page
+                ending = starting + self.CreateSubPagesForEachXXEntries - 1  # ending point is only used for title
+            else:
+                ending = output_xlsx.nrows
+            title = ((len(str(output_xlsx.nrows)) - len(str(starting))) * "0") + str(starting
+                    ) + " - " + ((len(str(output_xlsx.nrows)) - len(str(ending))) * "0") + str(ending
+                    ) + " " + self.pageTitle  # generating title for subpage
+            data = []
+            for row in range(starting, ending):
+                dt = []
+                for column in range(output_xlsx.ncols):
+                    value = output_xlsx.cell_value(row, column)
+                    if type(value) == float:  # coverting int and float into string
+                        if repr(value)[-2:] == '.0':
+                            value = int(value)
+                    value = str(value)
+                    if column not in remove_index:  # if column is not in remove_header list than add the data in html
+                        dt.append(escape(value))
+                data.append('<td>' + '</td>\n<td>'.join(dt) + '</td>')  # generating html table row and appending it in list
+            # html table row tag is added for every row which are stored in list
+            html = '<table><tbody>' + header + '<tr>' + '</tr>\n<tr>'.join(data) + '</tr>' + '</tbody></table>'
+            confluence.create_page(
+                self.space, title, html, parent_id=parent_page, type='page', representation='storage'
+            )  # subpage is created here with the help of parent page id
+

baangt/base/ExportResults/ExportResults.py

@@ -4,6 +4,7 @@ import json
 import baangt.base.GlobalConstants as GC
 from baangt.base.Timing.Timing import Timing
 from baangt.base.Utils import utils
+from baangt.base.ExportResults.Append2BaseXLS import Append2BaseXLS
 from pathlib import Path
 from typing import Optional
 from xlsxwriter.worksheet import (
@@ -22,6 +23,7 @@ from uuid import uuid4
 from pathlib import Path
 from baangt.base.ExportResults.SendStatistics import Statistics
 from baangt.base.RuntimeStatistics import Statistic
+from openpyxl import load_workbook
 
 logger = logging.getLogger("pyC")
 
@@ -30,6 +32,7 @@ class ExportResults:
     def __init__(self, **kwargs):
         self.kwargs = kwargs
         self.testList = []
+        self.fileName = None
         self.testRunInstance = kwargs.get(GC.KWARGS_TESTRUNINSTANCE)
         self.testCasesEndDateTimes_1D = kwargs.get('testCasesEndDateTimes_1D')
         self.testCasesEndDateTimes_2D = kwargs.get('testCasesEndDateTimes_2D')
@@ -52,9 +55,22 @@ class ExportResults:
         except KeyError:
             self.exportFormat = GC.EXP_XLSX
 
-        self.fileName = self.__getOutputFileName()
+        try:
+            if kwargs.get(GC.KWARGS_TESTRUNATTRIBUTES).get(GC.STRUCTURE_TESTCASESEQUENCE)[1][1].get(GC.EXPORT_FILENAME):
+                self.fileName = kwargs.get(GC.KWARGS_TESTRUNATTRIBUTES).get(GC.STRUCTURE_TESTCASESEQUENCE)[1][1].get(GC.EXPORT_FILENAME)
+        except Exception as e:
+            # fixme: I don't know, why this error came. When a Filename is set, then the above works.
+            #        No time now to debug.
+            pass
+
+        if not self.fileName:
+            self.fileName = self.__getOutputFileName()
+
         logger.info("Export-Sheet for results: " + self.fileName)
 
+        self.__removeUnwantedFields()  # Will remove Password-Contents AND fields from data records, that came from
+                                       # Globals-File.
+
         # export results to DB
         self.testcase_uuids = []
         self.exportToDataBase()
@@ -89,18 +105,45 @@ class ExportResults:
                                                    self.workbook,
                                                    self.networkSheet)
             self.closeExcel()
+
+            # Call functionality for potentially exporting data to other sheets/databases
+            Append2BaseXLS(self.testRunInstance, self.fileName)
+
         elif self.exportFormat == GC.EXP_CSV:
             self.export2CSV()
+
         if self.testRunInstance.globalSettings.get("DeactivateStatistics") == "True":
-            logger.debug("Send Statistics to server is deactive")
+            logger.debug("Send Statistics to server is deactivated. Not sending.")
         elif self.testRunInstance.globalSettings.get("DeactivateStatistics") is True:
-            logger.debug("Send Statistics to server is deactive")
+            logger.debug("Send Statistics to server is deactivated. Not sending.")
         else:
             try:
                 self.statistics.send_statistics()
             except Exception as ex:
                 logger.debug(ex)
-        #self.exportToDataBase()
+        if not self.testRunInstance.noCloneXls:
+            self.update_result_in_testrun()
+
+    def __removeUnwantedFields(self):
+        lListPasswordFieldNames = ["PASSWORD", "PASSWORT", "PASSW"]
+        if not self.testRunInstance.globalSettings.get("LetPasswords"):
+            # If there's a password in GlobalSettings, remove the value:
+            for key, value in self.testRunInstance.globalSettings.items():
+                if key.upper() in lListPasswordFieldNames:
+                    self.testRunInstance.globalSettings[key] = "*" * 8
+
+            # If there's a password in the datafile, remove the value
+            # Also remove all columns, that are anyway included in the global settings
+            for key, fields in self.dataRecords.items():
+                fieldsToPop = []
+                for field, value in fields.items():
+                    if field.upper() in lListPasswordFieldNames:
+                        self.dataRecords[key][field] = "*" * 8
+                    if field in self.testRunInstance.globalSettings.keys():
+                        fieldsToPop.append(field)
+                for field in fieldsToPop:
+                    if field != 'Screenshots' and field != 'Stage':   # Stage and Screenshot are needed in output file
+                        fields.pop(field)
 
     def exportAdditionalData(self):
         # Runs only, when KWARGS-Parameter is set.
@@ -119,7 +162,7 @@ class ExportResults:
         we shall take it from GlobalSettings. If also not there, take the default Value GC.EXECUTIN_STAGE_TEST
         :return:
         """
-        value = None
+        value = {}
         for key, value in self.dataRecords.items():
             break
         if not value.get(GC.EXECUTION_STAGE):
@@ -212,16 +255,17 @@ class ExportResults:
         self.__save_commit(session)
 
         # create testcase sequence instance
-        tcs_log = TestCaseSequenceLog(testrun=tr_log)
+        tcs_log = TestCaseSequenceLog(testrun=tr_log, number=1)
 
         # create testcases
-        for tc in self.dataRecords.values():
+        for tc_number, tc in enumerate(self.dataRecords.values(), 1):
             # get uuid
             uuid = uuid4()
             # create TestCaseLog instances
             tc_log = TestCaseLog(
                 id=uuid.bytes,
-                testcase_sequence=tcs_log
+                testcase_sequence=tcs_log,
+                number=tc_number,
             )
             # store uuid
             self.testcase_uuids.append(uuid)
@@ -422,6 +466,7 @@ class ExportResults:
         # Timing
         timing: Timing = self.testRunInstance.timing
         start, end, duration = timing.returnTimeSegment(GC.TIMING_TESTRUN)
+        self.testRun_end = end  # used while updating timestamp in source file
         self.statistics.update_attribute_with_value("Duration", duration)
         self.statistics.update_attribute_with_value("TestRunUUID", str(self.testRunInstance.uuid))
         self.__writeSummaryCell("Starttime", start, row=10)
@@ -435,6 +480,8 @@ class ExportResults:
         # Globals:
         self.__writeSummaryCell("Global settings for this testrun", "", format=self.cellFormatBold, row=15)
         for key, value in self.testRunInstance.globalSettings.items():
+            if key.upper() in ["PASSWORD", "PASSWORT", "CONFLUENCE-PASSWORD"]:
+                continue
             self.__writeSummaryCell(key, str(value))
             # get global data my
             self.testList.append(str(value))
@@ -545,6 +592,34 @@ class ExportResults:
         for n in range(0, len(self.fieldListExport)):
             ExcelSheetHelperFunctions.set_column_autowidth(self.worksheet, n)
 
+    def update_result_in_testrun(self):
+        # To update source testrun file
+        logger.debug("TestResult updating")
+        try:
+            testrun_column = self.dataRecords[0]["testcase_column"]
+        except:
+            return
+        if testrun_column:  # if testrun_column is greater than 0 that means testresult header is present in source file
+            logger.debug(f'Header for result update is {self.dataRecords[0]["testcase_column"]} in sheet ' \
+                         f'{self.dataRecords[0]["testcase_sheet"]} of file {self.dataRecords[0]["testcase_file"]}')
+            testrun_file = load_workbook(self.dataRecords[0]["testcase_file"])
+            testrun_sheet = testrun_file.get_sheet_by_name(self.dataRecords[0]["testcase_sheet"])
+            for key, value in self.dataRecords.items():
+                data = f"TestCaseStatus: {value['TestCaseStatus']}\r\n" \
+                       f"Timestamp: {self.testRun_end}\r\n" \
+                       f"Duration: {value['Duration']}\r\n" \
+                       f"TCErrorLog: {value['TCErrorLog']}\r\n" \
+                       f"TestRun_UUID: {str(self.testRunInstance.uuid)}\r\n" \
+                       f"TestCase_UUID: {str(self.testcase_uuids[key])}\r\n\r\n"
+                old_value = testrun_sheet.cell(value["testcase_row"] + 1, value["testcase_column"]).value or ""
+                testrun_sheet.cell(value["testcase_row"] + 1, value["testcase_column"]).value = data + old_value
+                logger.debug(f'Result written in row {value["testcase_row"]} column {value["testcase_column"]}')
+            logger.debug("Saving Source TestRun file.")
+            testrun_file.save(self.dataRecords[0]["testcase_file"])
+            logger.info(f"Source TestRun file {self.dataRecords[0]['testcase_file']} updated.")
+        else:
+            logger.debug(f"No TestResult column found")
+
     def __writeCell(self, line, cellNumber, testRecordDict, fieldName, strip=False):
         if fieldName in testRecordDict.keys() and testRecordDict[fieldName]:
             # Convert boolean for Output
@@ -552,8 +627,10 @@ class ExportResults:
                 testRecordDict[fieldName] = "True" if testRecordDict[fieldName] else "False"
 
             # Remove leading New-Line:
-            if '\n' in testRecordDict[fieldName][0:5] or strip:
-                testRecordDict[fieldName] = testRecordDict[fieldName].strip()
+            if isinstance(testRecordDict[fieldName], str):
+                if '\n' in testRecordDict[fieldName][0:5] or strip:
+                    testRecordDict[fieldName] = testRecordDict[fieldName].strip()
+
             # Do different stuff for Dicts and Lists:
             if isinstance(testRecordDict[fieldName], dict):
                 self.worksheet.write(line, cellNumber, testRecordDict[fieldName])
@@ -566,13 +643,13 @@ class ExportResults:
             else:
                 if fieldName == GC.TESTCASESTATUS:
                     if testRecordDict[GC.TESTCASESTATUS] == GC.TESTCASESTATUS_SUCCESS:
-                        self.worksheet.write(line, cellNumber, testRecordDict[fieldName], self.cellFormatGreen)
+                        self.worksheet.write(line, cellNumber, str(testRecordDict[fieldName]), self.cellFormatGreen)
                     elif testRecordDict[GC.TESTCASESTATUS] == GC.TESTCASESTATUS_ERROR:
-                        self.worksheet.write(line, cellNumber, testRecordDict[fieldName], self.cellFormatRed)
+                        self.worksheet.write(line, cellNumber, str(testRecordDict[fieldName]), self.cellFormatRed)
                 elif fieldName == GC.SCREENSHOTS:
                     self.__attachScreenshotsToExcelCells(cellNumber, fieldName, line, testRecordDict)
                 else:
-                    self.worksheet.write(line, cellNumber, testRecordDict[fieldName])
+                    self.worksheet.write(line, cellNumber, str(testRecordDict[fieldName]))
 
     def __attachScreenshotsToExcelCells(self, cellNumber, fieldName, line, testRecordDict):
         # Place the screenshot images "on" the appropriate cell
@@ -688,8 +765,8 @@ class ExportNetWork:
                  testCasesEndDateTimes_2D: list, workbook: xlsxwriter.Workbook, sheet: xlsxwriter.worksheet):
 
         self.networkInfo = networkInfo
-        #self.testCasesEndDateTimes_1D = testCasesEndDateTimes_1D
-        #self.testCasesEndDateTimes_2D = testCasesEndDateTimes_2D
+        self.testCasesEndDateTimes_1D = testCasesEndDateTimes_1D
+        self.testCasesEndDateTimes_2D = testCasesEndDateTimes_2D
         self.workbook = workbook
         self.sheet = sheet
         header_style = self.get_header_style()

baangt/base/Faker.py

@@ -1,6 +1,7 @@
 import logging
 from faker import Faker as FakerBase
 from random import randint
+from datetime import datetime
 
 logger = logging.getLogger("pyC")
 
@@ -19,6 +20,12 @@ class Faker:
         :return: the value, that was delivered by Faker.
         """
         lValue = None
+
+        if fakerMethod == "birthdate":
+            fakerMethod = 'date_between_dates'
+            kwargs["date_start"] = datetime(1960,1,1)
+            kwargs["date_end"] = datetime(2000,1,1)
+            # fake.date_between_dates(date_start=datetime(1960, 1, 1), date_end=datetime(2000, 1, 1))
         try:
             lCallFakerMethod = getattr(self.faker, fakerMethod)
             lValue = lCallFakerMethod(**kwargs)
@@ -26,3 +33,6 @@ class Faker:
             logging.error(f"Error during Faker-Call. Method was: {fakerMethod}, kwargs were: {kwargs}, Exception: {e}")
 
         return lValue
+
+    def fakerProxyIBAN(self, country_or_ListOfCountries):
+        return self.faker.Code.Iban.iban(country_or_ListOfCountries)

baangt/base/FilesOpen.py

@@ -2,6 +2,10 @@ import platform
 import os
 import subprocess
 import xl2dict
+import baangt.base.GlobalConstants as GC
+from logging import getLogger
+
+logger = getLogger("pyC")
 
 def open(filenameAndPath: str):
     """Will open the given file with the Operating system default program.
@@ -16,10 +20,13 @@ def open(filenameAndPath: str):
         filenameAndPath = str(filenameAndPath)
 
     filenameAndPath = os.path.abspath(filenameAndPath)
+    logger.debug(f"Trying to open file with it's application: {filenameAndPath}")
 
     if not os.path.exists(filenameAndPath):
+        logger.warning(f"Filename doesn't exist and can't be opened: {filenameAndPath}")
         return False
-    elif platform.system() == "Windows":
+
+    elif platform.system().lower() == GC.PLATFORM_WINDOWS:
         try:
             filenameAndPath = f'"{filenameAndPath}"'
             os.startfile(filenameAndPath)
@@ -31,14 +38,15 @@ def open(filenameAndPath: str):
             else:
                 return False
 
-    elif platform.system() == "Linux":
-        filenameAndPath = f'"{filenameAndPath}"'
+    elif platform.system().lower() == GC.PLATFORM_LINUX:
+        logger.debug(f"In Linux trying to call xdg-open with filename: {str(filenameAndPath)}")
         status = subprocess.call(["xdg-open", str(filenameAndPath)])
         if status == 0:
             return True
         else:
             return False
-    elif platform.system() == "Darwin":
+
+    elif platform.system().lower() == GC.PLATFORM_MAC:
         filenameAndPath = f'"{filenameAndPath}"'
         status = os.system("open " + str(filenameAndPath))
         if status == 0:

baangt/base/GlobalConstants.py

@@ -81,10 +81,13 @@ EXECUTION_STAGE_QA = 'Quality Assurance'
 
 EXPORT_FORMAT = "ExportFormat"
 EXPORT_ADDITIONAL_DATA = "AdditionalExportTabs"
+EXPORT_FILENAME = "ExportFilenameAndPath"
 EXP_FIELDLIST = "Fieldlist"
 EXP_XLSX = "XLSX"
 EXP_CSV = "CSV"
 
+PATH_REPORT = '3Reports'
+PATH_DB_EXPORT = '2DBResults'
 PATH_EXPORT = '1TestResults'
 PATH_IMPORT = '0TestInput'
 PATH_SCREENSHOTS = 'Screenshots'
@@ -106,8 +109,9 @@ MOBILE_APP_PACKAGE = 'appPackage'
 MOBILE_APP_ACTIVITY = 'appActivity'
 MOBILE_APP_BROWSER_PATH = 'mobileAppBrowserPath'   # Path to Browser on Mobile device
 
-WIN_PLATFORM = 'windows'
-LINUX_PLATFORM = 'linux'
+PLATFORM_WINDOWS = 'windows'
+PLATFORM_LINUX = 'linux'
+PLATFORM_MAC = 'darwin'
 
 BIT_64 = 8
 BIT_32 = 4

baangt/base/HandleDatabase.py

@@ -1,19 +1,46 @@
 import logging
-from xlrd import open_workbook
-import itertools
+from xlrd3 import open_workbook
 import json
 import baangt.base.CustGlobalConstants as CGC
 import baangt.base.GlobalConstants as GC
 from baangt.base.Utils import utils
-import baangt.TestSteps.Exceptions
-from pathlib import Path
-import xl2dict
 import re
 from random import randint
+from openpyxl import load_workbook
+from baangt.TestDataGenerator.TestDataGenerator import TestDataGenerator
 
 logger = logging.getLogger("pyC")
 
 
+class Writer:
+    """
+    This class is made to update existing excel file.
+    First it will open the file in python and then we can do multiple writes and once everything is update we can use
+    save method in order to save the updated excel file. Hence, this class is very useful is saving time while updating
+    excel files.
+    """
+    def __init__(self, path):
+        self.path = path
+        self.workbook = load_workbook(self.path)
+
+    def write(self, row, data, sht):
+        # Update the values using row and col number.
+        # Note :- We are using openpyxl so row & column index will start from 1 instead of 0
+        column = 0
+        sheet = self.workbook[sht]
+        headers = next(sheet.rows)
+        for header in headers:  # finds the header position
+            if "usecount" in str(header.value).lower():
+                column = headers.index(header) + 1
+        if column:
+            sheet.cell(row, column).value = data
+
+    def save(self):
+        # Call this method to save the file once every updates are written
+        self.workbook.save(self.path)
+        self.workbook.close()
+
+
 class HandleDatabase:
     def __init__(self, linesToRead, globalSettings=None):
         self.lineNumber = 3
@@ -41,6 +68,7 @@ class HandleDatabase:
         self.dataDict = []
         self.recordPointer = 0
         self.sheet_dict = {}
+        self.usecount = False
 
     def __buildRangeDict(self):
         """
@@ -106,13 +134,20 @@ class HandleDatabase:
         if not fileName:
             logger.critical(f"Can't open file: {fileName}")
             return
-
+        logger.debug(f"Reading excel file {fileName}...")
         book = open_workbook(fileName)
         sheet = book.sheet_by_name(sheetName)
 
         # read header values into the list
         keys = [sheet.cell(0, col_index).value for col_index in range(sheet.ncols)]
 
+        # if testresult header is present then taking its index, which is later used as column number
+        testrun_index = [keys.index(x) for x in keys if str(x).lower() == "testresult"]
+        if testrun_index:
+            testrun_index = testrun_index[0] + 1  # adding +1 value which is the correct column position
+        else:  # if list is empty that means their is no testresult header
+            testrun_index = 0
+
         for row_index in range(1, sheet.nrows):
             temp_dic = {}
             for col_index in range(sheet.ncols):
@@ -121,10 +156,19 @@ class HandleDatabase:
                     temp_dic[keys[col_index]] = repr(temp_dic[keys[col_index]])
                     if temp_dic[keys[col_index]][-2:] == ".0":
                         temp_dic[keys[col_index]] = temp_dic[keys[col_index]][:-2]
-
+            # row, column, sheetName & fileName which are later used in updating source testrun file
+            temp_dic["testcase_row"] = row_index
+            temp_dic["testcase_sheet"] = sheetName
+            temp_dic["testcase_file"] = fileName
+            temp_dic["testcase_column"] = testrun_index
             self.dataDict.append(temp_dic)
 
-        for temp_dic in self.dataDict:
+    def update_datarecords(self, dataDict, fileName, sheetName, noCloneXls):
+        logger.debug("Updating prefix data...")
+        self.testDataGenerator = TestDataGenerator(fileName, sheetName=sheetName,
+                                                   from_handleDatabase=True, noUpdate=noCloneXls)
+        for td in dataDict:
+            temp_dic = dataDict[td]
             new_data_dic = {}
             for keys in temp_dic:
                 if type(temp_dic[keys]) != str:
@@ -137,18 +181,27 @@ class HandleDatabase:
                         temp_dic[keys] = temp_dic[keys].replace(
                             temp_dic[keys][start_index:end_index+1], data_to_replace_with
                         )
-                if temp_dic[keys][:4] == "RRD_":
-                    rrd_string = self.__process_rrd_string(temp_dic[keys])
-                    rrd_data = self.__rrd_string_to_python(rrd_string[4:], fileName)
+
+                if str(temp_dic[keys])[:4].upper() == "RRD_":
+                    logger.debug(f"Processing rrd data - {temp_dic[keys]}")
+                    rrd_data = self.get_data_from_tdg(temp_dic[keys])
+                    self.testDataGenerator.usecountDataRecords[repr(rrd_data)]["use"] += 1
+                    self.testDataGenerator.update_usecount_in_source(rrd_data)
                     for data in rrd_data:
                         new_data_dic[data] = rrd_data[data]
-                elif temp_dic[keys][:4] == "RRE_":
-                    rre_string = self.__process_rre_string(temp_dic[keys])
-                    rre_data = self.__rre_string_to_python(rre_string[4:])
+                    logger.debug(f"Data processed - {temp_dic[keys]}")
+                elif str(temp_dic[keys])[:4].upper() == "RRE_":
+                    logger.debug(f"Processing rre data - {temp_dic[keys]}")
+                    rre_data = self.get_data_from_tdg(temp_dic[keys])
+                    self.testDataGenerator.usecountDataRecords[repr(rre_data)]["use"] += 1
+                    self.testDataGenerator.update_usecount_in_source(rre_data)
                     for data in rre_data:
                         new_data_dic[data] = rre_data[data]
-                elif temp_dic[keys][:4] == "RLP_":
+                    logger.debug(f"Data processed - {temp_dic[keys]}")
+                elif str(temp_dic[keys])[:4].upper() == "RLP_":
                     temp_dic[keys] = self.rlp_process(temp_dic[keys], fileName)
+                elif str(temp_dic[keys])[:5].upper() == "RENV_":
+                    temp_dic[keys] = str(TestDataGenerator.get_env_variable(temp_dic[keys][5:]))
                 else:
                     try:
                         js = json.loads(temp_dic[keys])
@@ -157,6 +210,16 @@ class HandleDatabase:
                         pass
             for key in new_data_dic:
                 temp_dic[key] = new_data_dic[key]
+        self.testDataGenerator.save_usecount()
+
+    def get_data_from_tdg(self, string):
+        data = self.testDataGenerator.data_generators(string)
+        try:
+            data = data.return_random()
+        except BaseException:
+            raise BaseException(f"Not enough data {string}, please verify if data is present or usecount limit" \
+                                "has reached!!")
+        return data
 
     def rlp_process(self, string, fileName):
         # Will get real data from rlp_ prefix string
@@ -186,122 +249,12 @@ class HandleDatabase:
                 data = self.rlp_process(data, fileName)
         return data
 
-    def __compareEqualStageInGlobalsAndDataRecord(self, currentNewRecordDict:dict) -> bool:
-        """
-        As method name says, compares, whether Stage in Global-settings is equal to stage in Data Record,
-        so that this record might be excluded, if it's for the wrong Stage.
-
-        :param currentNewRecordDict: The current Record
-        :return: Boolean
-        """
-        lAppend = True
-        if self.globals.get(GC.EXECUTION_STAGE):
-            if currentNewRecordDict.get(GC.EXECUTION_STAGE):
-                if currentNewRecordDict[GC.EXECUTION_STAGE] != self.globals[GC.EXECUTION_STAGE]:
-                    lAppend = False
-        return lAppend
-
-    def __processRrd(self, sheet_name, data_looking_for, data_to_match: dict, sheet_dict=None, caller="RRD_"):
-        """
-        For more detail please refer to TestDataGenerator.py
-        :param sheet_name:
-        :param data_looking_for:
-        :param data_to_match:
-        :return: dictionary of TargetData
-        """
-        sheet_dict = self.sheet_dict if sheet_dict is None else sheet_dict
-        matching_data = [list(x) for x in itertools.product(*[data_to_match[key] for key in data_to_match])]
-        assert sheet_name in sheet_dict, \
-            f"Excel file doesn't contain {sheet_name} sheet. Please recheck. Called in '{caller}'"
-        base_sheet = sheet_dict[sheet_name]
-        data_lis = []
-        if type(data_looking_for) == str:
-            data_looking_for = data_looking_for.split(",")
-
-        for data in base_sheet:
-            if len(matching_data) == 1 and len(matching_data[0]) == 0:
-                if data_looking_for[0] == "*":
-                    data_lis.append(data)
-                else:
-                    data_lis.append({keys: data[keys] for keys in data_looking_for})
-            else:
-                if [data[key] for key in data_to_match] in matching_data:
-                    if data_looking_for[0] == "*":
-                        data_lis.append(data)
-                    else:
-                        data_lis.append({keys: data[keys] for keys in data_looking_for})
-        return data_lis
-
-    def __rrd_string_to_python(self, raw_data, fileName):
-        """
-        Convert string to python data types
-        :param raw_data:
-        :return:
-        """
-        first_value = raw_data[1:-1].split(',')[0].strip()
-        second_value = raw_data[1:-1].split(',')[1].strip()
-        if second_value[0] == "[":
-            second_value = ','.join(raw_data[1:-1].split(',')[1:]).strip()
-            second_value = second_value[:second_value.index(']') + 1]
-            third_value = [x.strip() for x in ']'.join(raw_data[1:-1].split(']')[1:]).split(',')[1:]]
-        else:
-            third_value = [x.strip() for x in raw_data[1:-1].split(',')[2:]]
-        evaluated_list = ']],'.join(','.join(third_value)[1:-1].strip().split('],')).split('],')
-        if evaluated_list[0] == "":
-            evaluated_dict = {}
-        else:
-            evaluated_dict = {
-                splited_data.split(':')[0]: self.__splitList(splited_data.split(':')[1]) for splited_data in
-                evaluated_list
-            }
-        if second_value[0] == "[" and second_value[-1] == "]":
-            second_value = self.__splitList(second_value)
-        if first_value not in self.sheet_dict:
-            self.sheet_dict, _ = self.__read_excel(path=fileName)
-        processed_datas = self.__processRrd(first_value, second_value, evaluated_dict)
-        assert len(processed_datas)>0, f"No matching data for RRD_. Please check the input file. Was searching for " \
-                                       f"{first_value}, {second_value} and {str(evaluated_dict)} " \
-                                       f"but didn't find anything"
-        return processed_datas[randint(0, len(processed_datas)-1)]
-
-    def __rre_string_to_python(self, raw_data):
-        """
-        Convert string to python data types
-        :param raw_data:
-        :return:
-        """
-        file_name = raw_data[1:-1].split(',')[0].strip()
-        sheet_dict, _ = self.__read_excel(file_name)
-        first_value = raw_data[1:-1].split(',')[1].strip()
-        second_value = raw_data[1:-1].split(',')[2].strip()
-        if second_value[0] == "[":
-            second_value = ','.join(raw_data[1:-1].split(',')[2:]).strip()
-            second_value = second_value[:second_value.index(']') + 1]
-            third_value = [x.strip() for x in ']'.join(raw_data[1:-1].split(']')[1:]).split(',')[1:]]
-        else:
-            third_value = [x.strip() for x in raw_data[1:-1].split(',')[3:]]
-        evaluated_list = ']],'.join(','.join(third_value)[1:-1].strip().split('],')).split('],')
-        if evaluated_list[0] == "":
-            evaluated_dict = {}
-        else:
-            evaluated_dict = {
-                splited_data.split(':')[0]: self.__splitList(splited_data.split(':')[1]) for splited_data in
-                evaluated_list
-            }
-        if second_value[0] == "[" and second_value[-1] == "]":
-            second_value = self.__splitList(second_value)
-        processed_datas = self.__processRrd(first_value, second_value, evaluated_dict, sheet_dict, caller="RRE_")
-        assert len(processed_datas)>0, f"No matching data for RRD_. Please check the input file. Was searching for " \
-                                       f"{first_value}, {second_value} and {str(evaluated_dict)} " \
-                                       f"but didn't find anything"
-        return processed_datas[randint(0, len(processed_datas)-1)]
-
     def __rlp_string_to_python(self, raw_data, fileName):
         # will convert rlp string to python
         sheetName = raw_data.split(',')[0].strip()
         headerName = raw_data.split(',')[1].strip().split('=')[0].strip()
         headerValue = raw_data.split(',')[1].strip().split('=')[1].strip()
-        all_sheets, main_sheet = self.__read_excel(path=fileName, sheet_name=sheetName)
+        all_sheets, main_sheet = self.testDataGenerator.read_excel(path=fileName, sheet_name=sheetName, return_json=True)
         data_list = []
         for data in main_sheet:
             main_value = data[headerName]
@@ -317,37 +270,6 @@ class HandleDatabase:
                 data_list.append(data)
         return data_list
 
-    def __process_rre_string(self, rre_string):
-        """
-        For more detail please refer to TestDataGenerator.py
-        :param rre_string:
-        :return:
-        """
-        processed_string = ','.join([word.strip() for word in rre_string.split(', ')])
-        match = re.match(
-            r"(RRE_(\(|\[))[\w\d\s\-./\\]+\.(xlsx|xls),[a-zA-z0-9\s]+,(\[?[a-zA-z\s,]+\]?|)|\*,\[([a-zA-z0-9\s]+:\[[a-zA-z0-9,\s]+\](,?))*\]",
-            processed_string)
-        err_string = f"{rre_string} not matching pattern RRE_(fileName, sheetName, TargetData," \
-                     f"[Header1:[Value1],Header2:[Value1,Value2]])"
-        assert match, err_string
-        return processed_string
-
-    def __process_rrd_string(self, rrd_string):
-        """
-        For more detail please refer to TestDataGenerator.py
-        :param rrd_string:
-        :return:
-        """
-        processed_string = ','.join([word.strip() for word in rrd_string.split(', ')])
-        match = re.match(
-            r"(RRD_(\(|\[))[a-zA-z0-9\s]+,(\[?[a-zA-z\s,]+\]?|)|\*,\[([a-zA-z0-9\s]+:\[[a-zA-z0-9,\s]+\](,?))*\]",
-            processed_string
-        )
-        err_string = f"{rrd_string} not matching pattern RRD_(sheetName,TargetData," \
-                     f"[Header1:[Value1],Header2:[Value1,Value2]])"
-        assert match, err_string
-        return processed_string
-
     def __process_rlp_string(self, rlp_string):
         processed_string = ','.join([word.strip() for word in rlp_string.split(', ')])
         match = re.match(
@@ -358,38 +280,20 @@ class HandleDatabase:
         assert match, err_string
         return processed_string
 
-    def __splitList(self, raw_data):
-        """
-        Will convert string list to python list.
-        i.e. "[value1,value2,value3]" ==> ["value1","value2","value3"]
-        :param raw_data: string of list
-        :return: Python list
+    def __compareEqualStageInGlobalsAndDataRecord(self, currentNewRecordDict:dict) -> bool:
         """
-        proccesed_datas = [data.strip() for data in raw_data[1:-1].split(",")]
-        return proccesed_datas
+        As method name says, compares, whether Stage in Global-settings is equal to stage in Data Record,
+        so that this record might be excluded, if it's for the wrong Stage.
 
-    def __read_excel(self, path, sheet_name=""):
-        """
-        For more detail please refer to TestDataGenerator.py
-        :param path: Path to raw data xlsx file.
-        :param sheet_name: Name of base sheet sheet where main input data is located. Default will be the first sheet.
-        :return: Dictionary of all sheets and data, Dictionary of base sheet.
+        :param currentNewRecordDict: The current Record
+        :return: Boolean
         """
-        wb = open_workbook(path)
-        sheet_lis = wb.sheet_names()
-        sheet_dict = {}
-        for sheet in sheet_lis:
-            xl_obj = xl2dict.XlToDict()
-            data = xl_obj.fetch_data_by_column_by_sheet_name(path,sheet_name=sheet)
-            sheet_dict[sheet] = data
-        if sheet_name == "":
-            base_sheet = sheet_dict[sheet_lis[0]]
-        else:
-            assert sheet_name in sheet_dict, f"Excel file doesn't contain {sheet_name} sheet. Please recheck."
-            base_sheet = sheet_dict[sheet_name]
-        self.sheet_dict = sheet_dict
-        self.base_sheet = base_sheet
-        return sheet_dict, base_sheet
+        lAppend = True
+        if self.globals.get(GC.EXECUTION_STAGE):
+            if currentNewRecordDict.get(GC.EXECUTION_STAGE):
+                if currentNewRecordDict[GC.EXECUTION_STAGE] != self.globals[GC.EXECUTION_STAGE]:
+                    lAppend = False
+        return lAppend
 
     def readNextRecord(self):
         """

baangt/base/IBAN.py

@@ -1,4 +1,8 @@
-#import schwifty
+try:
+    import schwifty
+except Exception as ex:
+    pass
+
 from baangt.base.Faker import Faker
 import random
 
@@ -26,11 +30,15 @@ class IBAN:
         for n in range(laenge):
             digits.append(random.randrange(0, 10))
         digits = "".join(str(x) for x in digits)
-        #return str(schwifty.IBAN.generate(country_code=self.bankLand,
-        #                                  bank_code=self.bankLeitZahl,
-        #                                  account_code=digits))
-        return Faker().fakerProxy(fakerMethod="iban")
+        # Schwifty doesn't work on Ubuntu
+        try:
+            lReturn = str(schwifty.IBAN.generate(country_code=self.bankLand,
+                                              bank_code=self.bankLeitZahl,
+                                              account_code=digits))
+        except Exception:
+            lReturn = Faker().fakerProxyIBAN("AT")
 
+        return lReturn
 
 if __name__ == '__main__':
     l = IBAN()

baangt/base/PathManagement.py

@@ -25,9 +25,11 @@ class ManagedPaths(metaclass=Singleton):
     - getOrSetScreenshotsPath: Will return a path for Screenshots directory. You can also set path.
     - getOrSetDownloadsPath: Will return a path for Download directory. You can also set path.
     - getOrSetAttachmentDownloadPath: Will return a path for Attachment Download directory. You can also set path.
-    - getOrSetDriverPath: Will return path were webdriver is located. You can also set path.
-    - getOrSetImportPath: Will return path were input files are located. You can also set path.
-    - getOrSetExportPath: Will return path were files will be save. You can also set path.
+    - getOrSetDriverPath: Will return path where webdriver is located. You can also set path.
+    - getOrSetImportPath: Will return path where input files are located. You can also set path.
+    - getOrSetExportPath: Will return path where files will be save. You can also set path.
+    - getOrSetDBExportPath: Will return path where DB query outputs should be saved. You can also set path.
+    - getOrSetReportPath: Will return path where reports should be saved. You can also set path.
     - getOrSetRootPath: Will return root directory path. You can also set path.
     - getOrSetIni: Will return path of directory where ini files are managed.
     """
@@ -42,6 +44,8 @@ class ManagedPaths(metaclass=Singleton):
         self.RootPath = ""
         self.ExportPath = ""
         self.ImportPath = ""
+        self.DBExportPath = ""
+        self.ReportPath = ""
         self.IniPath = ""
 
     def getLogfilePath(self):
@@ -200,6 +204,42 @@ class ManagedPaths(metaclass=Singleton):
         self.__makeAndCheckDir(self.ImportPath)
         return self.ImportPath
 
+    def getOrSetDBExportPath(self, path=None):
+        """
+        Will return path where DB queriy outputs should be saved.
+
+        :param path: Path to be set for DB exports.
+        :return: DB Export path
+        """
+        if self.DBExportPath != "":
+            return self.DBExportPath
+
+        if path:
+            self.DBExportPath = path
+        else:
+            self.DBExportPath = self.__combineBasePathWithObjectPath(GC.PATH_DB_EXPORT)
+
+        self.__makeAndCheckDir(self.DBExportPath)
+        return self.DBExportPath
+
+    def getOrSetReportPath(self, path=None):
+        """
+        Will return path where reports should be saved.
+
+        :param path: Path to be set for reports.
+        :return: Report path
+        """
+        if self.ReportPath != "":
+            return self.ReportPath
+
+        if path:
+            self.ReportPath = path
+        else:
+            self.ReportPath = self.__combineBasePathWithObjectPath(GC.PATH_REPORT)
+
+        self.__makeAndCheckDir(self.ReportPath)
+        return self.ReportPath
+
     def __combineBasePathWithObjectPath(self, objectPath: str):
         """
 
@@ -220,7 +260,7 @@ class ManagedPaths(metaclass=Singleton):
 
         :return: base Path
         """
-        if platform.system() == "Windows":
+        if platform.system().lower() == GC.PLATFORM_WINDOWS:
             path = os.path.join(os.path.expanduser("~"), "baangt")
             if os.path.exists(path):
                 return Path(path)
@@ -257,3 +297,18 @@ class ManagedPaths(metaclass=Singleton):
 
         self.__makeAndCheckDir(self.IniPath)
         return self.IniPath
+
+    def findFileInAnyPath(self, filename):
+        if Path(filename).exists():
+            return filename
+
+        if Path(self.getOrSetExportPath()).joinpath(filename).exists():
+            return Path(self.getOrSetExportPath()).joinpath(filename)
+
+        if Path(self.getOrSetImportPath()).joinpath(filename).exists():
+            return Path(self.getOrSetImportPath()).joinpath(filename)
+
+        if Path(self.getOrSetRootPath()).joinpath(filename).exists():
+            return Path(self.getOrSetRootPath()).joinpath(filename)
+
+        return None

baangt/base/ResultsBrowser.py

@@ -0,0 +1,595 @@
+from sqlalchemy import create_engine, desc, and_
+from sqlalchemy.orm import sessionmaker
+from baangt.base.DataBaseORM import DATABASE_URL, TestrunLog, GlobalAttribute, TestCaseLog, TestCaseSequenceLog, TestCaseField
+from baangt.base.ExportResults.ExportResults import ExcelSheetHelperFunctions
+from baangt.base.PathManagement import ManagedPaths
+import baangt.base.GlobalConstants as GC
+import uuid
+from datetime import datetime
+from xlsxwriter import Workbook
+import logging
+import os
+import json
+import re
+import uuid
+import time # time tracker
+
+logger = logging.getLogger("pyC")
+
+class QuerySet:
+
+    # flags
+    SIZE_TCS = 0
+    SIZE_TC = 1
+
+    def __init__(self):
+        self.data = None
+
+    def __del__(self):
+        del self.data
+
+    @property
+    def length(self):
+        return len(self.data)
+
+    def set(self, data):
+        self.data = list(data)
+
+    def names(self):
+        return {tr.testrunName for tr in self.data}
+
+    def all(self):
+        return (tr for tr in self.data)
+
+    def filter(self, tr_name, tcs_index=None, tc_index=None):
+        #
+        # returns generator of filtered query_set
+        #
+
+        if tcs_index is None:
+            return filter(lambda tr: tr.testrunName == tr_name, self.data)
+
+        if tc_index is None:
+            return filter(lambda tr: tr.testrunName == tr_name and len(tr.testcase_sequences) > tcs_index, self.data)
+
+        return filter(
+            lambda tr: tr.testrunName == tr_name and len(tr.testcase_sequences) > tcs_index \
+                and len(tr.testcase_sequences[tcs_index].testcases) > tc_index,
+            self.data,
+        )
+
+    def max_size(self, tr_name, tcs_index=None):
+        #
+        # returns the max number of 
+        # tcs_index is None: testcase sequences within the testruns in query_set
+        # tcs_index is a number: testcases within specified testcase sequence of all testruns in db
+        #
+
+        # test case sequences
+        if tcs_index is None:
+            return max(map(lambda tr: len(tr.testcase_sequences), self.filter(tr_name=tr_name)))
+
+        # test cases
+        return max(map(lambda tr: len(tr.testcase_sequences[tcs_index].testcases), self.filter(tr_name=tr_name, tcs_index=tcs_index)))
+
+    def tr_avg_duration(self, tr_name):
+        duration = 0
+        quantity = 0
+        for d in (tr.duration for tr in self.filter(tr_name) if tr.duration):
+            duration += d
+            quantity += 1
+
+        if quantity > 0:
+            return round(duration/quantity, 2)
+
+        return None
+
+    def tc_avg_duration(self, tr_name, indexes):
+        durations = map(
+            lambda tr: tr.testcase_sequences[indexes[0]].testcases[indexes[1]].duration,
+            self.filter(tr_name, tcs_index=indexes[0], tc_index=indexes[1]),
+        )
+
+        duration = 0
+        quantity = 0
+        for d in durations:
+            if d:
+                duration += d
+                quantity += 1
+
+        if quantity > 0:
+            return round(duration/quantity, 2)
+
+        return None
+
+
+class ExportSheet:
+
+    def __init__(self, sheet, header_format):
+        self.sheet = sheet
+        self.line = 1 # current line
+        self.column = 0
+        self.header_map = {}
+        self.header_format = header_format
+
+    def set_header_format(self, header_format):
+        self.header_format = header_format
+
+    def hr(self):
+        self.line += 1
+
+    def header(self, headers):
+        for header in headers:
+            self.sheet.write(0, self.column, header, self.header_format)
+            self.header_map[header] = self.column
+            self.column += 1
+
+    def add_header(self, header):
+        if not header in self.header_map:
+            self.sheet.write(0, self.column, header, self.header_format)
+            self.header_map[header] = self.column
+            self.column += 1
+
+    def row(self, row_data):
+        for col, data in enumerate(row_data):
+            self.sheet.write(self.line, col, data.get('value'), data.get('format'))
+        self.line += 1
+
+    def new_row(self, row_data):
+        self.line += 1
+        for col, data in enumerate(row_data):
+            self.sheet.write(self.line, col, data.get('value'), data.get('format'))
+
+    def by_header(self, header, value):
+        self.add_header(header)
+        self.sheet.write(self.line, self.header_map[header], value)
+
+
+
+
+class ResultsBrowser:
+
+    def __init__(self, db_url=None):
+        # setup db engine
+        if db_url:
+            engine = create_engine(db_url)
+        else:
+            engine = create_engine(DATABASE_URL)
+        self.db = sessionmaker(bind=engine)()
+        # result query set
+        self.query_set = QuerySet()
+        # query filters
+        self.filters = {}
+        # path management
+        self.managedPaths = ManagedPaths()
+        logger.info(f'Initiated with DATABASE_URL: {db_url if db_url else DATABASE_URL}')
+
+    def __del__(self):
+        self.db.close()
+
+    def name_list(self):
+        names = self.db.query(TestrunLog.testrunName).group_by(TestrunLog.testrunName).order_by(TestrunLog.testrunName)
+        return [x[0] for x in names]
+
+    def stage_list(self):
+        stages = self.db.query(GlobalAttribute.value).filter_by(name=GC.EXECUTION_STAGE)\
+            .group_by(GlobalAttribute.value).order_by(GlobalAttribute.value)
+        return [x[0] for x in stages]
+
+    def globals_names(self):
+        names = self.db.query(GlobalAttribute.name).group_by(GlobalAttribute.name).order_by(GlobalAttribute.name)
+        return [x[0] for x in names]
+
+    def globals_values(self, name):
+        values = self.db.query(GlobalAttribute.value).filter_by(name=name).group_by(GlobalAttribute.value).order_by(GlobalAttribute.value)
+        return [x[0] for x in values]
+
+    def query(self, name=None, stage=None, start_date=None, end_date=None, global_settings={}):
+        #
+        # get TestrunLogs from db
+        #
+
+        # store filters
+        self.filters = {
+            'name': name,
+            'stage': stage,
+            'start_date': start_date,
+            'end_date': end_date,
+            'globals': global_settings,
+        }
+
+        # get records
+        logger.info(f'Quering: name={name}, stage={stage}, dates=[{start_date} - {end_date}], globals={global_settings}')
+        records = self.db.query(TestrunLog).order_by(TestrunLog.startTime)
+
+        # filter by name
+        if name:
+            records = records.filter_by(testrunName=name)
+
+        # filter bay dates
+        if start_date:
+            records = records.filter(TestrunLog.startTime >= start_date)
+
+        if end_date:
+            records = records.filter(TestrunLog.endTime <= end_date)
+
+        # filter by globals
+        if stage:
+            records = records.filter(TestrunLog.globalVars.any(and_(
+                GlobalAttribute.name == GC.EXECUTION_STAGE,
+                GlobalAttribute.value == stage,
+            )))
+
+        for key, value in global_settings.items():
+            records = records.filter(TestrunLog.globalVars.any(and_(
+                GlobalAttribute.name == key,
+                GlobalAttribute.value == value,
+            )))
+
+        self.query_set.set(records)
+
+        logger.info(f'Number of found records: {self.query_set.length}')
+
+    def filter_to_str(self, key):
+        #
+        # get filter value as str
+        #
+
+        value = self.filters.get(key)
+        if value:
+            return str(value)
+
+        # undefined values
+        if key == 'end_date':
+            return datetime.now().strftime('%Y-%m-%d')
+        if key == 'start_date':
+            return 'None'
+        return 'All'
+
+    def filename(self, extention):
+        #
+        # generate filename for exports
+        #
+
+        filename = '_'.join([
+            'TestrunLogs',
+            self.filter_to_str('name'),
+            self.filter_to_str('stage'),
+            self.filter_to_str('start_date'),
+            self.filter_to_str('end_date'),
+            'globals' if self.filters.get('globals') else 'None',
+        ])
+        return f'{filename}.{extention}'
+            
+    def export(self):
+        #
+        # exports the query set to xlsx
+        #
+
+        # initialize workbook
+        path_to_file = self.managedPaths.getOrSetDBExportPath().joinpath(self.filename('xlsx'))
+        workbook = Workbook(str(path_to_file))
+
+        # set labels
+        labels = {
+            'testrun': 'TestRun',
+            'testcase_sequence': 'Test Case Sequence',
+            'testcase': 'Test Case',
+            'stage': 'Stage',
+            'duration': 'Avg. Duration',
+            'globals': 'Global Settings',
+        }
+
+        # set output headers
+        base_headers = [
+            'Testrun ID',
+            'TestCase ID',
+            'TestCase Number',
+        ]
+
+        base_fields = [
+            GC.EXECUTION_STAGE,
+            GC.TESTCASESTATUS,
+            GC.TIMING_DURATION,
+        ]
+       
+        # define cell formats
+        cformats = {
+            'bg_red': workbook.add_format({'bg_color': 'red'}),
+            'bg_green': workbook.add_format({'bg_color': 'green'}),
+            'bg_yellow': workbook.add_format({'bg_color': 'yellow'}),
+            'font_bold': workbook.add_format({'bold': True}),
+            'font_bold_italic': workbook.add_format({'bold': True, 'italic': True}),
+        }
+
+        # map status styles
+        status_style = {
+            GC.TESTCASESTATUS_SUCCESS: cformats.get('bg_green'),
+            GC.TESTCASESTATUS_ERROR: cformats.get('bg_red'),
+            GC.TESTCASESTATUS_WAITING: None,
+        }
+
+        # create sheets
+        sheets = {
+            'summary': ExportSheet(workbook.add_worksheet('Summary'), cformats.get('font_bold')),
+            'output': ExportSheet(workbook.add_worksheet('Output'), cformats.get('font_bold')),
+        }
+
+        # write summary titles
+        time_start = time.time() # -------------------------> time tracker
+        # title
+        sheets['summary'].sheet.set_column(first_col=0, last_col=0, width=18)
+        sheets['summary'].sheet.set_column(first_col=1, last_col=1, width=10)
+        sheets['summary'].header([f'{labels.get("testrun")}s Summary'])
+        
+        # parameters
+        for key in self.filters:
+            if key != 'globals':
+                sheets['summary'].row([
+                    {
+                        'value': key,
+                        'format': cformats.get('font_bold'),
+                    },
+                    {
+                        'value': self.filter_to_str(key),
+                    },
+                ])
+        # global settings
+        if self.filters.get('globals'):
+            sheets['summary'].hr()
+            sheets['summary'].row([
+                {
+                    'value': f"{labels.get('globals')}:",
+                    'format': cformats.get('font_bold'),
+                },
+            ])
+            for name, value in self.filters.get('globals').items():
+                sheets['summary'].row([
+                    {
+                        'value': name,
+                    },
+                    {
+                        'value': value,
+                    },
+                ])
+
+        # write output titles
+        sheets['output'].header(base_headers + base_fields)
+
+        # write items
+        # testruns
+        for tr_name in self.query_set.names():
+            logger.info(f'Exporting Tetsrun "{tr_name}": {len(list(self.query_set.filter(tr_name)))} records')
+            # testrun name
+            sheets['summary'].hr()
+            sheets['summary'].row([
+                {
+                    'value': labels.get('testrun'),
+                    'format': cformats.get('font_bold'),
+                },
+                {
+                    'value': tr_name,
+                }
+            ])
+            
+            # average duration
+            sheets['summary'].row([
+                {
+                    'value': labels.get('duration'),
+                    'format': cformats.get('font_bold'),
+                },
+                {
+                    'value': self.query_set.tr_avg_duration(tr_name),
+                }
+            ])
+
+            #
+
+            # testcase sequences
+            for tcs_index in range(self.query_set.max_size(tr_name)):
+                tcs_number = tcs_index+1
+                # testcase sequence
+                sheets['summary'].hr()
+                sheets['summary'].row([
+                    {
+                        'value': labels.get('testcase_sequence'),
+                        'format': cformats.get('font_bold'),
+                    },
+                    {
+                        'value': tcs_number,
+                    }
+                ])
+
+                # test cases
+                # header
+                tc_num_max = self.query_set.max_size(tr_name, tcs_index=tcs_index)
+                sheets['summary'].hr()
+                sheets['summary'].row(
+                    [
+                        {
+                            'value': 'Run Date',
+                            'format': cformats.get('font_bold_italic'),
+                        },
+                        {
+                            'value': labels.get('testcase'),
+                            'format': cformats.get('font_bold_italic'),
+                        },
+                    ] + [{} for i in range(1, tc_num_max)] +[
+                        {
+                            'value': labels.get('stage'),
+                            'format': cformats.get('font_bold_italic'),
+                        },
+                        {
+                            'value': f'{labels.get("testrun")} ID',
+                            'format': cformats.get('font_bold_italic'),
+                        },
+                    ]
+                )
+                sheets['summary'].row(
+                    [{}] + [{'value': i} for i in range(tc_num_max)]
+                )
+
+                durations =  [[.0,0] for i in range(tc_num_max)]
+                for tr_index, tr in enumerate(self.query_set.filter(tr_name, tcs_index)):
+                    tc_num = len(tr.testcase_sequences[tcs_index].testcases)
+                    status_row = [{'value': tr.startTime.strftime('%Y-%m-%d %H:%M:%S')}]
+
+                    # query fields
+                    data = self.db.query(
+                        TestCaseField.name,
+                        TestCaseField.value,
+                        TestCaseLog.number,
+                        TestCaseLog.id,
+                    ).join(TestCaseField.testcase).join(TestCaseLog.testcase_sequence).join(TestCaseSequenceLog.testrun)\
+                    .filter(and_(TestrunLog.id == tr.id, TestCaseSequenceLog.number == tcs_number)).order_by(TestCaseLog.number)
+                    tc_id_cur = None
+                    tr_stage = None
+                    for name, value, tc_index, tc_id in data.yield_per(500):
+                        # summary data
+                        if name == GC.TESTCASESTATUS:
+                            status_row.append({
+                                'value': value,
+                                'format': status_style.get(value),
+                            })
+                        elif name == GC.EXECUTION_STAGE:
+                            tr_stage = value
+                        elif name == GC.TIMING_DURATION:
+                            m = re.search(r'(?P<hours>\d+):(?P<minutes>\d+):(?P<seconds>\d[\.\d+]*)', value)
+                            if m:
+                                factors = {
+                                    'hours': 3600,
+                                    'minutes': 60,
+                                    'seconds': 1,
+                                }
+                                durations[tc_index-1][0] += sum([factors[key]*float(value) for key, value in m.groupdict().items()])
+                                durations[tc_index-1][1] += 1
+                            
+                        # write to output
+                        # write testcase
+                        if tc_id != tc_id_cur:
+                            tc_id_cur = tc_id
+                            sheets['output'].new_row([
+                                {
+                                    'value': str(tr),
+                                },
+                                {
+                                    'value': str(uuid.UUID(bytes=tc_id)),
+                                },
+                                {
+                                    'value': tc_index,
+                                },
+                            ])
+
+                        #field
+                        sheets['output'].by_header(name, value)
+
+                    # write state row to summary sheet
+                    sheets['summary'].row(
+                        status_row + [{} for i in range(tc_num, tc_num_max)] + [
+                            {
+                                'value': tr_stage,
+                            },
+                            {
+                                'value': str(tr),
+                            },
+                        ]
+                    )
+
+                # avg durations
+                sheets['summary'].row(
+                    [
+                        {
+                            'value': labels.get('duration'),
+                            'format': cformats.get('font_bold_italic'),
+                        },
+                    ] + [{'value': d} for d in map(lambda d: round(d[0]/d[1], 2) if d[1] > 0 else .0, durations)]
+                )
+        
+        # autowidth output columns
+        for col in range(len(base_headers)+len(base_fields)):
+            ExcelSheetHelperFunctions.set_column_autowidth(sheets['output'].sheet, col)
+
+        workbook.close()
+
+        execution_time = round(time.time()-time_start, 2)
+        logger.info(f'Query successfully exported to {path_to_file} in {execution_time} seconds')
+
+        return path_to_file
+    
+
+    def export_txt(self):
+        #
+        # export to txt
+        #
+
+        path_to_file = self.managedPaths.getOrSetDBExportPath().joinpath(self.filename('txt'))
+
+        # set labels
+        labels = {
+            'testrun': 'TestRun',
+            'testcase_sequence': 'Test Case Sequence',
+            'testcase': 'Test Case',
+            'stage': 'Stage',
+            'duration': 'Avg. Duration',
+        }
+
+        # write data
+        with open(path_to_file, 'w') as f:
+            # title
+            f.write(f'{labels.get("testrun")}s Summary\n\n')
+            
+            # parameters
+            for key in self.filters:
+                f.write(f'{key}\t{self.filter_to_str(key)}\n')
+
+            # testruns
+            for tr_name in self.query_set.names():
+                print(f'*** Tetsrun "{tr_name}"')
+                # testrun name
+                f.write(f'\n{labels.get("testrun")}: {tr_name}\n')
+                
+                # average duration
+                f.write(f'{labels.get("duration")}: {self.query_set.tr_avg_duration(tr_name)}\n')
+
+                # testcase sequences
+                for tcs_index in range(self.query_set.max_size(tr_name)):
+                    print(f'**** TestCaseSequence-{tcs_index}')
+                    # testcase sequence
+                    f.write(f'\n{labels.get("testcase_sequence")}: {tcs_index}\n\n')
+
+                    # test cases
+                    # header
+                    tc_num = self.query_set.max_size(tr_name, tcs_index=tcs_index)
+                    f.write(f'{"Run Date":20}{labels.get("testcase"):8}')
+                    f.write(' '*7)
+                    f.write((' '*8)*(tc_num-2))
+                    f.write(f'{labels.get("stage"):8}{labels.get("testcase")} ID\n')
+                    f.write(' '*20)            
+                    for tc_index in range(tc_num):
+                        f.write(f'{tc_index:<8}')
+                    f.write('\n')
+
+                    # testcase status
+                    tr_counter = 1 
+                    for tr in self.query_set.filter(tr_name, tcs_index):
+                        print(f'***** TestRun {tr_counter}: {len(tr.testcase_sequences[tcs_index].testcases)} testcases')
+                        tr_counter += 1
+                        f.write(f'{tr.startTime.strftime("%Y-%m-%d %H:%M:%S"):20}')
+                        for tc in tr.testcase_sequences[tcs_index].testcases:
+                            f.write(f'{tc.status:8}')
+                        # tail
+                        print(f'{tc_num} - {len(tr.testcase_sequences[tcs_index].testcases)} = {(tc_num-len(tr.testcase_sequences[tcs_index].testcases))}')
+                        f.write((' '*8)*(tc_num-len(tr.testcase_sequences[tcs_index].testcases)))
+                        f.write(f'{tr.stage:8}{tr}\n')
+
+                    # average durations
+                    f.write(f'{labels.get("duration"):20}')
+                    for tc_index in range(tc_num):
+                        f.write(f'{self.query_set.tc_avg_duration(tr_name, (tcs_index, tc_index)):8}')
+
+        logger.info(f'Query successfully exported to {path_to_file}')
+
+        return path_to_file
+
+
+

baangt/base/SendReports/__init__.py

@@ -1,6 +1,6 @@
 import os
 import csv
-import xlrd
+import xlrd3 as xlrd
 import json
 import xlsxwriter
 import logging
@@ -10,6 +10,7 @@ import configparser
 from slack_webhook import Slack
 from baangt.base.PathManagement import ManagedPaths
 from baangt.base.SendReports.mailer import SendMail
+from baangt.base.ExportResults.ExportConfluence import ExportConfluence
 
 logger = logging.getLogger('pyC')
 
@@ -79,6 +80,22 @@ class Sender:
             if test:
                 return messages
 
+    def sendConfluence(self):
+        self.set_globalSettings(self.defaultSettings, "Confluence-Base-Url")
+        if self.globalSettings["Confluence-Base-Url"]:
+            args = {}
+            args['url'] = self.globalSettings["Confluence-Base-Url"]
+            args['space'] = self.globalSettings["Confluence-Space"]
+            args['pageTitle'] = self.globalSettings["Confluence-Pagetitle"]
+            args['fileNameAndPathToResultXLSX'] = self.xlsx_file
+            args['username'] = self.globalSettings["Confluence-Username"]
+            args['password'] = self.globalSettings["Confluence-Password"]
+            args['rootPage'] = self.globalSettings["Confluence-Rootpage"]
+            args['remove_headers'] = [x.strip() for x in self.globalSettings["Confluence-Remove_Headers"].split(",")]
+            args['uploadOriginalFile'] = self.globalSettings["Confluence-Uploadoriginalfile"]
+            args['CreateSubPagesForEachXXEntries'] = int(self.globalSettings["Confluence-Createsubpagesforeachxxentries"])
+            ExportConfluence(**args)
+
     def set_globalSettings(self, setting, word_to_look):
         if word_to_look in setting:
             self.globalSettings = setting
@@ -108,6 +125,8 @@ class Sender:
         self.globalSettings["TelegramChannel"] = config["Default"].get("TelegramChannel"
                                                                     ) or self.write_config(config_file, "TelegramChannel",
                                                                                            "")
+        self.globalSettings["Confluence-Base-Url"] = config["Default"].get("Confluence-Base-Url") or self.write_config(
+            config_file, "Confluence-Base-Url", "")
 
     def write_config(self, config_file, key=None, value=None):
         if key:
@@ -184,21 +203,30 @@ class Sender:
             os.remove(temp_file)
         else:
             send_stats = Sender(globalSettings, results.fileName)
+
         try:
             send_stats.sendMail()
         except Exception as ex:
-            logger.debug(ex)
+            logger.error(f"Error when sending mail: {ex}")
+
         try:
             send_stats.sendMsTeam()
         except Exception as ex:
-            logger.debug(ex)
+            logger.error(f"Error when sending MS Teams: {ex}")
+
         try:
             send_stats.sendSlack()
         except Exception as ex:
-            logger.debug(ex)
+            logger.error(f"Error when sending Slack: {ex}")
+
         try:
             send_stats.sendTelegram()
         except Exception as ex:
-            logger.debug(ex)
+            logger.error(f"Error when sending Telegram: {ex}")
+
+        try:
+            send_stats.sendConfluence()
+        except Exception as ex:
+            logger.error(f"Error when saving in Confluence: {ex}")
 
 

baangt/base/TestRun/TestRun.py

@@ -26,6 +26,8 @@ from uuid import uuid4
 from baangt.base.RuntimeStatistics import Statistic
 from baangt.base.SendReports import Sender
 import signal
+from baangt.TestDataGenerator.TestDataGenerator import TestDataGenerator
+from CloneXls import CloneXls
 
 logger = logging.getLogger("pyC")
 
@@ -37,7 +39,7 @@ class TestRun:
     """
 
     def __init__(self, testRunName, globalSettingsFileNameAndPath=None,
-                 testRunDict=None, uuid=uuid4(), executeDirect=True):  # -- API support: testRunDict --
+                 testRunDict=None, uuid=uuid4(), executeDirect=True, noCloneXls=False):  # -- API support: testRunDict --
         """
         @param testRunName: The name of the TestRun to be executed.
         @param globalSettingsFileNameAndPath: from where to read the <globals>.json
@@ -49,7 +51,7 @@ class TestRun:
         self.testRunDict = testRunDict
         self.globalSettingsFileNameAndPath = globalSettingsFileNameAndPath
         self.testRunName, self.testRunFileName = \
-            self._sanitizeTestRunNameAndFileName(testRunName)
+            self._sanitizeTestRunNameAndFileName(testRunName, executeDirect)
 
         # Initialize everything else
         self.apiInstance = None
@@ -72,6 +74,7 @@ class TestRun:
         # from anywhere within your custom code base.
         self.additionalExportTabs = {}
         self.statistics = Statistic()
+        self.noCloneXls = noCloneXls
         signal.signal(signal.SIGINT, self.exit_signal_handler)
         signal.signal(signal.SIGTERM, self.exit_signal_handler)
 
@@ -99,7 +102,7 @@ class TestRun:
         try:
             Sender.send_all(self.results, self.globalSettings)
         except Exception as ex:
-            logger.debug(ex)
+            logger.error(f"Error from SendAll: {ex}")
 
     def append1DTestCaseEndDateTimes(self, dt):
         self.testCasesEndDateTimes_1D.append(dt)
@@ -322,13 +325,17 @@ class TestRun:
             if isinstance(value, str):
                 if value.lower() in ("false", "true", "no", "x"):
                     self.globalSettings[key] = utils.anything2Boolean(value)
-
+                elif "renv_" in value.lower():
+                    self.globalSettings[key] = TestDataGenerator.get_env_variable(value[5:])
             if isinstance(value, dict):
                 if "default" in value:
                     # This happens in the new UI, if a value was added manually,
                     # but is not part of the globalSetting.json. In this case there's the whole shebang in a dict. We
                     # are only interested in the actual value, which is stored in "default":
                     self.globalSettings[key] = value["default"]
+                    if isinstance(self.globalSettings[key], str):
+                        if "renv_" in self.globalSettings[key].lower():
+                            self.globalSettings[key] = TestDataGenerator.get_env_variable(self.globalSettings[key][5:])
                     continue
                 else:
                     # This could be the "old" way of the globals-file (with {"HEADLESS":"True"})
@@ -375,12 +382,16 @@ class TestRun:
         return utils.dynamicImportOfClasses(fullQualifiedImportName=fullQualifiedImportName)
 
     @staticmethod
-    def _sanitizeTestRunNameAndFileName(TestRunNameInput):
+    def _sanitizeTestRunNameAndFileName(TestRunNameInput, direct):
         """
         @param TestRunNameInput: The complete File and Path of the TestRun definition (JSON or XLSX).
         @return: TestRunName and FileName (if definition of testrun comes from a file (JSON or XLSX)
         """
-        if ".XLSX" in TestRunNameInput.upper() or ".JSON" in TestRunNameInput.upper():
+        if ".XLSX" in TestRunNameInput.upper():
+            cloneXls = CloneXls(TestRunNameInput)
+            lFileName = cloneXls.update_or_make_clone(ignore_headers=["TestResult", "UseCount"], clone=direct)
+            lRunName = utils.extractFileNameFromFullPath(lFileName)
+        elif ".JSON" in TestRunNameInput.upper():
             lRunName = utils.extractFileNameFromFullPath(TestRunNameInput)
             lFileName = TestRunNameInput
         else:

baangt/base/TestRunExcelImporter.py

@@ -2,7 +2,7 @@ from baangt.base.Utils import utils
 from baangt.base.TestRunUtils import TestRunUtils
 import baangt.base.GlobalConstants as GC
 import baangt.base.CustGlobalConstants as CGC
-import xlrd
+import xlrd3 as xlrd
 import logging
 
 logger = logging.getLogger("pyC")
@@ -81,14 +81,16 @@ class TestRunExcelImporter:
                 "TestDataFileName": self.fileName,
                 "Sheetname": "data",
                 "ParallelRuns": 1,
-                "Lines": "0-999999"
+                "Lines": "0-999999",
+                GC.EXPORT_FILENAME: ""
                 },
             }
         for key, sequence in lSequenceDict.items():
             testrunSequence[key] = [sequence["SequenceClass"], {}]
             for field, value in sequence.items():
                 testRunAttributes[GC.STRUCTURE_TESTCASESEQUENCE][key][1][field] = value
-
+            if not testRunAttributes[GC.STRUCTURE_TESTCASESEQUENCE][key][1].get(GC.EXPORT_FILENAME):
+                testRunAttributes[GC.STRUCTURE_TESTCASESEQUENCE][key][1][GC.EXPORT_FILENAME]=None
 
         xlsTab = self._getTab("TestCase")
         # if Tab "TestCase" exists, then take the definitions from there. Otherwise (means simpleFormat)
@@ -305,6 +307,15 @@ class TestRunExcelImporter:
         @param value: potentially convertable value (e.g. GC.BROWSER)
         @return: potentially converted value (e.g. "Browser")
         """
+
+        # with change to Pandas all Cells are treated as string. Which is good otherwise in baangt.
+        # Here it's not good as we need int-values for the keys in the dicts.
+        if isinstance(value, str):
+            if value.isnumeric():
+                value = int(value)
+                return value
+
+        # Old implementation came with sometimes with floats for ints (Excel..)
         if isinstance(value, float):
             if value % 1 == 0:
                 return int(value)

baangt/base/Utils.py

@@ -6,6 +6,7 @@ import ntpath
 import logging
 import json
 import sys
+import traceback
 from pathlib import Path
 from baangt.base.PathManagement import ManagedPaths
 
@@ -66,6 +67,14 @@ class utils:
         return lDictOut
 
     @staticmethod
+    def traceback(exception_in):
+
+        ex_traceback = exception_in.__traceback__
+        tb_lines = "\n".join([line.rstrip('\n') for line in
+                    traceback.format_exception(exception_in.__class__, exception_in, ex_traceback)])
+        logger.info(tb_lines)
+
+    @staticmethod
     def _loopList(listIn):
         listOut = []
         for item in listIn:

baangt/reports.py

@@ -3,6 +3,7 @@ from sqlalchemy.orm import sessionmaker
 from sqlalchemy import desc, and_
 from baangt.base.DataBaseORM import engine, TestrunLog, GlobalAttribute, TestCaseLog, TestCaseSequenceLog, TestCaseField
 import baangt.base.GlobalConstants as GC
+from baangt.base.PathManagement import ManagedPaths
 from jinja2 import Environment, FileSystemLoader
 import json
 import os
@@ -21,11 +22,12 @@ class Report:
 
 	def __init__(self):
 		self.created = datetime.now()
+		self.managedPaths = ManagedPaths()
 		self.generate();
 
 	@property
 	def path(self):
-		return ''	
+		return ''
 
 	def generate(self):
 		#
@@ -47,7 +49,7 @@ class Report:
 		# returns jinja2 template
 		#
 
-		file_loader = FileSystemLoader(os.path.join(os.path.dirname(__file__), GC.REPORT_PATH, template_dir))
+		file_loader = FileSystemLoader(os.path.join(os.path.dirname(__file__), template_dir))
 		env = Environment(loader=file_loader)
 
 		return env.get_template(template_name)
@@ -306,11 +308,8 @@ class Dashboard(Report):
 		#
 		# path to report
 		#
-		return os.path.join(
-			os.path.dirname(__file__),
-			GC.REPORT_PATH,
-			self.created.strftime('dashboard-%Y%m%d-%H%M%S.html'),
-		)
+
+		return self.managedPaths.getOrSetReportPath().joinpath(self.created.strftime('dashboard-%Y%m%d-%H%M%S.html'))
 
 
 	def generate(self):
@@ -450,11 +449,8 @@ class Summary(Report):
 		#
 		# path to report
 		#
-		return os.path.join(
-			os.path.dirname(__file__),
-			GC.REPORT_PATH,
-			self.created.strftime('summary-%Y%m%d-%H%M%S.html'),
-		)
+
+		return self.managedPaths.getOrSetReportPath().joinpath(self.created.strftime('summary-%Y%m%d-%H%M%S.html'))
 
 	def generate(self):
 		#

baangt/reports/templates/dashboard.html

@@ -16,7 +16,7 @@
     {% if data.names %}
     <div class="row mx-2 pt-3 rounded shadow">
         <div class="col-md-1 h4">Filters:</div>
-        <div class="col-md-3 mb-3 input-group">
+        <div class="col-lg-3 mb-3 input-group">
             <div class="input-group-prepend">
                 <label class="input-group-text px-4" for="selectName">Name</label>
             </div>
@@ -27,7 +27,7 @@
                 {% endfor %}
             </select>
         </div>
-        <div class="col-md-3 mb-3 input-group">
+        <div class="col-lg-3 mb-3 input-group">
             <div class="input-group-prepend">
                 <label class="input-group-text px-4" for="selectStage">Stage</label>
             </div>
@@ -43,53 +43,53 @@
 
     <!-- header -->
     <div class="row mx-2 mb-2 mt-3">
-        <div class="col-md-3 h3">Testrun</div>
-        <div class="col-md-3 h3">Summary</div>
-        <div class="col-md-3 h3">Results</div>
-        <div class="col-md-3 h3">Duration</div>
+        <div class="col-lg-3 h3">Testrun</div>
+        <div class="col-lg-3 h3">Summary</div>
+        <div class="col-lg-3 h3">Results</div>
+        <div class="col-lg-3 h3">Duration</div>
     </div>
 
     <!-- testrun list -->
     {% for item in data.records %}
         <div class="row hovered-item border-top py-3 mx-2" id="{{ item.id }}" data-name="{{ item.name }}" data-stage="{{ item.stage }}">
-            <div class="col-md-3 text-break">
+            <div class="col-lg-3 text-break">
                 <p class="text-primary">{{ item.name }}</p>
                 <p class="text-secondary">{{ item.stage }}</p> 
             </div>
-            <div class="col-md-3">
-                <div class="row">
-                    <div class="d-flex col-3">
+            <div class="col-lg-3">
+                <div class="row my-auto">
+                    <div class="col-4 p-0">
                         <canvas class="statusChart" width="100" height="100"></canvas>
                     </div>
-                    <div class="d-flex flex-column col-2 border-right px-0">
-                        <h2 class="text-center">{{ item.figures.records }}</h2>
-                        <h6 class="text-center">TESTS</h6>
+                    <div class="col-2 border-right px-0 figure my-auto">
+                        <div class="test-digit">{{ item.figures.records }}</div>
+                        <div class="test-status">TESTS</div>
                     </div>
                     {% if item.figures.successful > 0  %}
-                        <div class="d-flex flex-column col-2 text-success px-0">
-                            <h2 class="text-center">{{ item.figures.successful }}</h2>
-                            <h6 class="text-center">PASSED</h6>
+                        <div class="col-2 text-success px-0 figure">
+                            <div class="test-digit">{{ item.figures.successful }}</div>
+                            <div class="test-status">PASSED</div>
                         </div>
                     {% endif %}
                     {% if item.figures.error > 0  %}
-                        <div class="d-flex flex-column col-2 text-danger px-0">
-                            <h2 class="text-center">{{ item.figures.error }}</h2>
-                            <h6 class="text-center">FAILED</h6>
+                        <div class="col-2 text-danger px-0 figure">
+                            <div class="test-digit">{{ item.figures.error }}</div>
+                            <div class="test-status">FAILED</div>
                         </div>
                     {% endif %}
                     {% if item.figures.paused > 0  %}
-                        <div class="d-flex flex-column col-2 text-secondary px-0">
-                            <h2 class="text-center">{{ item.figures.paused }}</h2>
-                            <h6 class="text-center">PAUSED</h6>
+                        <div class="col-2 text-secondary px-0 figure">
+                            <div class="test-digit">{{ item.figures.paused }}</div>
+                            <div class="test-status">PAUSED</div>
                         </div>
                     {% endif %}
                 </div>
             </div>
-            <div class="col-md-3">
-                <canvas class="resultsChart" height="60"></canvas>
+            <div class="col-lg-3 p-0 pl-2">
+                <canvas class="resultsChart timeChart" height="100"></canvas>
             </div>
-            <div class="col-md-3 text-break">
-                <canvas class="durationChart" height="60"></canvas>
+            <div class="col-lg-3 p-0 pl-2">
+                <canvas class="durationChart timeChart" height="100"></canvas>
             </div>
         </div>
 
@@ -98,6 +98,34 @@
 </div>
 </main>
 
+<style>
+    .figure {
+      min-width: 4rem;
+      margin-bottom: auto;
+      margin-top: auto;
+      padding: 0;
+    }
+
+    .statusChart {
+        min-width: 6rem;
+        min-height: 6rem;
+    }
+
+    .timeChart {
+        min-height: 6rem;
+    } 
+
+    .test-digit {
+        font-size: 2.7rem;
+        text-align: center !important;
+    }
+
+    .test-state {
+        font-size: 1rem;
+        text-align: center !important;
+    }
+</style>
+
 <script src="https://cdn.jsdelivr.net/npm/chart.js@2.8.0"></script>
 <script type="text/javascript">
 

baangt/ui/pyqt/baangtResource.qrc

@@ -9,6 +9,7 @@
         <file alias="katalonicon">katalon.svg</file>
         <file alias="reporticon">report.svg</file>
         <file alias="cleanupicon">cleanup.svg</file>
+        <file alias="dbqueryicon">queryresults.svg</file>
         <file alias="exiticon">exit.svg</file>
         <file alias="tdgicon">testdatagenerator.svg</file>
     </qresource>

baangt/ui/pyqt/globalSetting.json

@@ -176,7 +176,7 @@
             "hint": "Name of telegram channel where you want to send the report. If multiple channels then seperate them from comma.\nE.g. = channel_1, channel_2",
             "type": "text",
             "default": "",
-            "displayText": "Telgram Channel(s)"
+            "displayText": "Telegram Channel(s)"
         },
         "DeactivateStatistics": {
             "hint": "We send statistics to our server containing only type of run to know where we should focus & improve more.\nWe don't send any personal data. You can see the data sent to server in logs(debug) for further reference.\nTrue for deactivating it.",
@@ -191,6 +191,18 @@
             "default": "",
             "options": ["", "Debug", "Info", "Warning", "Error"],
             "displayText": "LogLevel"
+        },
+        "AR2BXLS": {
+            "hint": "Append output of this test run to one or more Base-Excel sheets, e.g. for statistics or input for other\ntest runs. <Fname>,<sheet>;<Fname2>,<sheet>",
+            "type": "text",
+            "default": "",
+            "displayText": "AppRes2BaseXLS"
+        },
+        "TC.RestartBrowserAfter": {
+            "hint": "Enter a number of Testcases, after that the browser shall be restartet. Valid: e.g. '2'. Instead of '1' you could set the flag RestartBrowser.",
+            "type": "text",
+            "default": "",
+            "displayText": "RestBrowserAfter"
         }
     }
 }

baangt/ui/pyqt/uiDesign.py

@@ -9,11 +9,14 @@
 from PyQt5 import QtCore, QtGui, QtWidgets
 import logging
 
+GLOBALS_FILTER_NUMBER = 2
+
 
 class Ui_MainWindow(QtCore.QObject):
     def setupUi(self, MainWindow):
         MainWindow.setObjectName("MainWindow")
         MainWindow.resize(920, 480)
+        MainWindow.setMinimumSize(900, 400)
         font = QtGui.QFont()
         font.setFamily("Arial")
         font.setPointSize(11)
@@ -422,6 +425,235 @@ class Ui_MainWindow(QtCore.QObject):
         self.verticalLayout_9.addLayout(self.horizontalLayout_23)
         self.gridLayout_7.addWidget(self.groupBox, 0, 0, 1, 1)
         self.stackedWidget.addWidget(self.settingPage)
+
+        # query screen
+        self.queryPage = QtWidgets.QWidget()
+        self.queryPage.setObjectName("queryPage")
+
+        # fonts
+        titleFont = QtGui.QFont()
+        titleFont.setFamily("Arial")
+        titleFont.setPointSize(32)
+        titleFont.setBold(True)
+        titleFont.setItalic(False)
+        titleFont.setWeight(18)
+        titleFont.setKerning(False)
+
+        labelFont = QtGui.QFont()
+        labelFont.setFamily("Arial")
+        labelFont.setPointSize(11)
+        labelFont.setBold(False)
+        labelFont.setItalic(False)
+        labelFont.setWeight(9)
+        labelFont.setKerning(False)
+
+        # main layout
+        self.queryMainLayout = QtWidgets.QVBoxLayout(self.queryPage)
+        self.queryMainLayout.setObjectName("queryMainLayout")
+
+        # page title
+        self.queryTitleLabel = QtWidgets.QLabel(self.queryPage)
+        self.queryTitleLabel.setAlignment(QtCore.Qt.AlignTop)
+        self.queryTitleLabel.setFont(titleFont)
+        self.queryTitleLabel.setMinimumSize(QtCore.QSize(0, 0))
+        self.queryTitleLabel.setMaximumSize(QtCore.QSize(200, 15))
+        self.queryTitleLabel.setObjectName("queryTitleLabel")
+        self.queryMainLayout.addWidget(self.queryTitleLabel)
+
+        # action layout
+        self.queryActionLayout = QtWidgets.QHBoxLayout()
+        self.queryActionLayout.setSizeConstraint(QtWidgets.QLayout.SetMaximumSize)
+        #self.queryActionLayout.setContentsMargins(5, 5, 5, 5)
+        #self.queryActionLayout.setSpacing(15)
+        self.queryActionLayout.setObjectName("queryActionLayout")
+        self.queryMainLayout.addLayout(self.queryActionLayout)
+
+        # input group
+        self.queryGroupBox = QtWidgets.QGroupBox(self.queryPage)
+        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Preferred)
+        sizePolicy.setHorizontalStretch(0)
+        sizePolicy.setVerticalStretch(0)
+        sizePolicy.setHeightForWidth(self.queryGroupBox.sizePolicy().hasHeightForWidth())
+        self.queryGroupBox.setSizePolicy(sizePolicy)
+        self.queryGroupBox.setMinimumSize(QtCore.QSize(500, 0))
+        self.queryGroupBox.setFlat(False)
+        self.queryGroupBox.setAlignment(QtCore.Qt.AlignTop)
+        self.queryGroupBox.setObjectName("queryGroupBox")
+        self.queryActionLayout.addWidget(self.queryGroupBox)
+
+        self.queryFormLayout = QtWidgets.QVBoxLayout(self.queryGroupBox)
+        self.queryFormLayout.setSpacing(15)
+        self.queryFormLayout.setObjectName("queryFormLayout")
+        
+        self.queryInputLayout = QtWidgets.QGridLayout()
+        self.queryInputLayout.setObjectName("queryInputLayout")
+        self.queryInputLayout.setSpacing(20)
+        self.queryInputLayout.setColumnStretch(0, 1)
+        self.queryInputLayout.setColumnStretch(1, 8)
+        self.queryInputLayout.setColumnStretch(2, 2)
+        self.queryInputLayout.setColumnStretch(3, 5)
+
+        self.queryGlobalsLayout = QtWidgets.QGridLayout()
+        self.queryGlobalsLayout.setObjectName("queryGlobalsLayout")
+        self.queryGlobalsLayout.setSpacing(5)
+        self.queryGlobalsLayout.setColumnStretch(0, 8)
+        self.queryGlobalsLayout.setColumnStretch(1, 8)
+
+        self.queryButtonLayout = QtWidgets.QHBoxLayout()
+        self.queryButtonLayout.setSizeConstraint(QtWidgets.QLayout.SetMaximumSize)
+        self.queryButtonLayout.setContentsMargins(5, 5, 5, 10)
+        self.queryButtonLayout.setSpacing(15)
+
+        self.queryFormLayout.addLayout(self.queryInputLayout)
+        self.queryFormLayout.addLayout(self.queryGlobalsLayout)
+        self.queryFormLayout.addLayout(self.queryButtonLayout)
+
+        # name
+        self.nameComboBoxLabel = QtWidgets.QLabel(self.queryGroupBox)
+        self.nameComboBoxLabel.setFont(labelFont)
+        self.nameComboBoxLabel.setStyleSheet("color: rgb(32, 74, 135);")
+        self.nameComboBoxLabel.setObjectName("nameComboBoxLabel")
+        self.queryInputLayout.addWidget(self.nameComboBoxLabel)
+        self.nameComboBox = QtWidgets.QComboBox(self.queryGroupBox)
+        self.nameComboBox.setStyleSheet("color: rgb(46, 52, 54); background-color: rgb(255, 255, 255);")
+        self.nameComboBox.setObjectName("nameComboBox")
+        #self.nameComboBox.addItems(["", "Debug", "Info", "Warning", "Error"])
+        self.queryInputLayout.addWidget(self.nameComboBox)
+
+        # date from
+        self.dateFromInputLabel = QtWidgets.QLabel(self.queryGroupBox)
+        self.dateFromInputLabel.setFont(labelFont)
+        self.dateFromInputLabel.setStyleSheet("color: rgb(32, 74, 135);")
+        self.dateFromInputLabel.setObjectName("dateFromInputLabel")
+        self.queryInputLayout.addWidget(self.dateFromInputLabel)
+        self.dateFromInput = QtWidgets.QDateEdit(self.queryGroupBox, calendarPopup=True)
+        self.dateFromInput.setDateTime(QtCore.QDateTime.currentDateTime())
+        self.dateFromInput.setDisplayFormat("dd.MM.yyyy")
+        self.dateFromInput.setStyleSheet("color: rgb(46, 52, 54); background-color: rgb(255, 255, 255);")
+        self.dateFromInput.setObjectName("dateFromInput")
+        self.queryInputLayout.addWidget(self.dateFromInput)
+
+        # stage
+        self.stageComboBoxLabel = QtWidgets.QLabel(self.queryGroupBox)
+        self.stageComboBoxLabel.setFont(labelFont)
+        self.stageComboBoxLabel.setStyleSheet("color: rgb(32, 74, 135);")
+        self.stageComboBoxLabel.setObjectName("stageComboBoxLabel")
+        self.queryInputLayout.addWidget(self.stageComboBoxLabel)
+        self.stageComboBox = QtWidgets.QComboBox(self.queryGroupBox)
+        self.stageComboBox.setStyleSheet("color: rgb(46, 52, 54); background-color: rgb(255, 255, 255);")
+        self.stageComboBox.setObjectName("stageComboBox")
+        self.queryInputLayout.addWidget(self.stageComboBox)
+
+        # date to
+        self.dateToInputLabel = QtWidgets.QLabel(self.queryGroupBox)
+        self.dateToInputLabel.setFont(labelFont)
+        self.dateToInputLabel.setStyleSheet("color: rgb(32, 74, 135);")
+        self.dateToInputLabel.setObjectName("dateToInputLabel")
+        self.queryInputLayout.addWidget(self.dateToInputLabel)
+        self.dateToInput = QtWidgets.QDateEdit(self.queryGroupBox, calendarPopup=True)
+        self.dateToInput.setDateTime(QtCore.QDateTime.currentDateTime())
+        self.dateToInput.setDisplayFormat("dd.MM.yyyy")
+        self.dateToInput.setStyleSheet("color: rgb(46, 52, 54); background-color: rgb(255, 255, 255);")
+        self.dateToInput.setObjectName("dateToInput")
+        self.queryInputLayout.addWidget(self.dateToInput)
+
+        # Globals
+        # title
+        self.globalsTile = QtWidgets.QLabel(self.queryGroupBox)
+        self.globalsTile.setFont(labelFont)
+        self.globalsTile.setStyleSheet("color: rgb(32, 74, 135);")
+        self.globalsTile.setObjectName("globalsTile")
+        self.queryGlobalsLayout.addWidget(self.globalsTile, 0, 0)
+        # options
+        self.globalsOptions = []
+        for index in range(GLOBALS_FILTER_NUMBER):
+            # name
+            globalsNameComboBox = QtWidgets.QComboBox(self.queryGroupBox)
+            globalsNameComboBox.setStyleSheet("color: rgb(46, 52, 54); background-color: rgb(255, 255, 255);")
+            globalsNameComboBox.setObjectName(f"globalsNameComboBox{index}")
+            self.queryGlobalsLayout.addWidget(globalsNameComboBox, index+1, 0)
+            # value
+            globalsValueComboBox = QtWidgets.QComboBox(self.queryGroupBox)
+            globalsValueComboBox.setStyleSheet("color: rgb(46, 52, 54); background-color: rgb(255, 255, 255);")
+            globalsValueComboBox.setObjectName(f"globalsValueComboBox{index}")
+            self.queryGlobalsLayout.addWidget(globalsValueComboBox, index+1, 1)
+            # store
+            self.globalsOptions.append((globalsNameComboBox, globalsValueComboBox))
+
+        # query buttons
+        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Fixed, QtWidgets.QSizePolicy.Fixed)
+        sizePolicy.setHorizontalStretch(0)
+        sizePolicy.setVerticalStretch(0)
+        #sizePolicy.setHeightForWidth(self.queryMakePushButton.sizePolicy().hasHeightForWidth())
+        # make query
+        self.queryMakePushButton = QtWidgets.QPushButton(self.queryGroupBox)
+        #sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Fixed, QtWidgets.QSizePolicy.Fixed)
+        #sizePolicy.setHorizontalStretch(0)
+        #sizePolicy.setVerticalStretch(0)
+        sizePolicy.setHeightForWidth(self.queryMakePushButton.sizePolicy().hasHeightForWidth())
+        self.queryMakePushButton.setSizePolicy(sizePolicy)
+        self.queryMakePushButton.setMinimumSize(QtCore.QSize(120, 0))
+        self.queryMakePushButton.setStyleSheet("color: rgb(255, 255, 255); background-color: rgb(114, 159, 207);")
+        self.queryMakePushButton.setObjectName("queryMakePushButton")
+        self.queryButtonLayout.addWidget(self.queryMakePushButton)
+        # export results
+        self.queryExportPushButton = QtWidgets.QPushButton(self.queryGroupBox)
+        #sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Fixed, QtWidgets.QSizePolicy.Fixed)
+        #sizePolicy.setHorizontalStretch(0)
+        #sizePolicy.setVerticalStretch(0)
+        sizePolicy.setHeightForWidth(self.queryExportPushButton.sizePolicy().hasHeightForWidth())
+        self.queryExportPushButton.setSizePolicy(sizePolicy)
+        self.queryExportPushButton.setMinimumSize(QtCore.QSize(120, 0))
+        self.queryExportPushButton.setStyleSheet("color: rgb(255, 255, 255); background-color: rgb(138, 226, 52);")
+        self.queryExportPushButton.setObjectName("queryExportPushButton")
+        self.queryButtonLayout.addWidget(self.queryExportPushButton)
+        # open recent export result file
+        self.openExportPushButton = QtWidgets.QPushButton(self.queryGroupBox)
+        #sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Fixed, QtWidgets.QSizePolicy.Fixed)
+        #sizePolicy.setHorizontalStretch(0)
+        #sizePolicy.setVerticalStretch(0)
+        sizePolicy.setHeightForWidth(self.openExportPushButton.sizePolicy().hasHeightForWidth())
+        self.openExportPushButton.setSizePolicy(sizePolicy)
+        self.openExportPushButton.setMinimumSize(QtCore.QSize(120, 0))
+        self.openExportPushButton.setStyleSheet("color: rgb(255, 255, 255); background-color: rgb(138, 226, 52);")
+        self.openExportPushButton.setObjectName("openExportPushButton")
+        self.queryButtonLayout.addWidget(self.openExportPushButton)
+        # exit to main screen
+        self.queryExitPushButton = QtWidgets.QPushButton(self.queryGroupBox)
+        #sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Fixed, QtWidgets.QSizePolicy.Fixed)
+        #sizePolicy.setHorizontalStretch(0)
+        #sizePolicy.setVerticalStretch(0)
+        sizePolicy.setHeightForWidth(self.queryExitPushButton.sizePolicy().hasHeightForWidth())
+        self.queryExitPushButton.setSizePolicy(sizePolicy)
+        self.queryExitPushButton.setMinimumSize(QtCore.QSize(120, 0))
+        self.queryExitPushButton.setStyleSheet("color: rgb(255, 255, 255); background-color: rgb(204, 0, 0);")
+        self.queryExitPushButton.setObjectName("queryExitPushButton")
+        self.queryButtonLayout.addWidget(self.queryExitPushButton)
+
+        # status message
+        self.queryStatusLabel = QtWidgets.QLabel(self.queryGroupBox)
+        self.queryStatusLabel.setAlignment(QtCore.Qt.AlignTop)
+        self.queryStatusLabel.setObjectName("queryStatusLabel")
+        self.queryMainLayout.addWidget(self.queryStatusLabel)
+
+        # logo
+        self.queryLogo = QtWidgets.QLabel(self.queryPage)
+        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Expanding)
+        sizePolicy.setHorizontalStretch(0)
+        sizePolicy.setVerticalStretch(0)
+        sizePolicy.setHeightForWidth(self.queryLogo.sizePolicy().hasHeightForWidth())
+        self.queryLogo.setSizePolicy(sizePolicy)
+        self.queryLogo.setMinimumSize(QtCore.QSize(0, 0))
+        self.queryLogo.setMaximumSize(QtCore.QSize(300, 120))
+        self.queryLogo.setBaseSize(QtCore.QSize(600, 240))
+        self.queryLogo.setText("")
+        self.queryLogo.setScaledContents(True)
+        self.queryLogo.setObjectName("queryLogo")
+        self.queryActionLayout.addWidget(self.queryLogo)
+
+        # END of query screen
+
+
         self.katalonPage = QtWidgets.QWidget()
         self.katalonPage.setObjectName("katalonPage")
         self.gridLayout_8 = QtWidgets.QGridLayout(self.katalonPage)
@@ -484,6 +716,12 @@ class Ui_MainWindow(QtCore.QObject):
         self.actionImport_Katalon.setObjectName("actionImport_Katalon")
         self.actionImport_KatalonIcon = QtGui.QIcon(":/baangt/katalonicon")
         self.actionImport_Katalon.setIcon(self.actionImport_KatalonIcon)
+        # Browse Results
+        self.actionQuery = QtWidgets.QAction(MainWindow)
+        self.actionQuery.setObjectName("actionQuery")
+        self.actionQueryIcon = QtGui.QIcon(":/baangt/dbqueryicon")
+        self.actionQuery.setIcon(self.actionQueryIcon)
+        # END: Browse Results
         self.actionReport = QtWidgets.QAction(MainWindow)
         self.actionReport.setObjectName("actionReport")
         self.actionReportIcon = QtGui.QIcon(":/baangt/reporticon")
@@ -497,6 +735,7 @@ class Ui_MainWindow(QtCore.QObject):
         TestDataGenIcon = QtGui.QIcon(":/baangt/tdgicon")
         self.actionTestDataGen.setIcon(TestDataGenIcon)
         self.toolBar.addAction(self.actionImport_Katalon)
+        self.toolBar.addAction(self.actionQuery)
         self.toolBar.addAction(self.actionReport)
         self.toolBar.addAction(self.actionCleanup)
         self.toolBar.addAction(self.actionTestDataGen)
@@ -687,6 +926,9 @@ class Ui_MainWindow(QtCore.QObject):
         self.gridLayout_9.addLayout(self.verticalLayout_12, 0, 0, 1, 1)
         self.stackedWidget.addWidget(self.TDGPage)
 
+        # add query page
+        self.stackedWidget.addWidget(self.queryPage)
+
         self.retranslateUi(MainWindow)
         self.stackedWidget.setCurrentIndex(0)
         QtCore.QMetaObject.connectSlotsByName(MainWindow)
@@ -755,6 +997,21 @@ class Ui_MainWindow(QtCore.QObject):
             "MainWindow", "Number of records output file\nSet 0 or blank for all possible records"))
         self.exitPushButton_4.setText(_translate("MainWindow", "Exit"))
 
+        # query page
+        self.actionQuery.setText(_translate("MainWindow", "Query Results"))
+        self.actionQuery.setToolTip(_translate("MainWindow", "Query Results"))
+        self.queryTitleLabel.setText(_translate("MainWindow","Browse Results"))
+        self.nameComboBoxLabel.setText(_translate("MainWindow", "Name"))
+        self.stageComboBoxLabel.setText(_translate("MainWindow", "Stage"))
+        self.dateFromInputLabel.setText(_translate("MainWindow", "Date from"))
+        self.dateToInputLabel.setText(_translate("MainWindow", "Date to"))
+        self.globalsTile.setText(_translate("MainWindow", "Global Settings"))
+        self.queryMakePushButton.setText(_translate("MainWindow", "Query"))
+        self.queryExportPushButton.setText(_translate("MainWindow", "Export"))
+        self.openExportPushButton.setText(_translate("MainWindow", "Open Recent"))
+        self.queryExitPushButton.setText(_translate("MainWindow", "Exit"))
+        self.queryStatusLabel.setText(_translate("MainWindow", "Make a query"))
+
 
 if __name__ == "__main__":
     import sys

baangt/ui/pyqt/uimain.py

@@ -25,19 +25,25 @@ from baangt.base.PathManagement import ManagedPaths
 from uuid import uuid4
 from baangt.base.FilesOpen import FilesOpen
 from baangt.base.RuntimeStatistics import Statistic
-from baangt.base.PathManagement import ManagedPaths
+#from baangt.base.PathManagement import ManagedPaths
 from baangt.base.DownloadFolderMonitoring import DownloadFolderMonitoring
 from baangt.base.Cleanup import Cleanup
-import xlrd
+from baangt.base.ResultsBrowser import ResultsBrowser
+import xlrd3 as xlrd
 from baangt.reports import Dashboard, Summary
 from baangt.TestDataGenerator.TestDataGenerator import TestDataGenerator
 from baangt.base.Utils import utils
 from threading import Thread
 from time import sleep
 import signal
+from datetime import datetime
+
 
 logger = logging.getLogger("pyC")
 
+NAME_PLACEHOLDER = '< Name >'
+VALUE_PLACEHOLDER = '< Value >'
+
 
 class PyqtKatalonUI(ImportKatalonRecorder):
     """ Subclass of ImportKatalonRecorder :
@@ -90,6 +96,7 @@ class MainWindow(Ui_MainWindow):
         self.__log_file = ""
         self.__open_files = 0
         self.TDGResult = ""
+        self.dataFile = ""
 
         # self.refreshNew()
         # self.setupBasePath(self.directory)
@@ -130,6 +137,13 @@ class MainWindow(Ui_MainWindow):
         self.openTestFilePushButton_4.clicked.connect(self.openTestFile)
         self.InputFileOpen.clicked.connect(self.openInputFile)
 
+        # Result Browser buttons
+        self.actionQuery.triggered.connect(self.showQueryPage)
+        self.queryMakePushButton.clicked.connect(self.makeResultQuery)
+        self.queryExportPushButton.clicked.connect(self.exportResultQuery)
+        self.openExportPushButton.clicked.connect(self.openRecentQueryResults)
+        self.queryExitPushButton.clicked.connect(self.mainPageView)
+
         # Quit Event
         self.actionExit.triggered.connect(self.quitApplication)
 
@@ -238,6 +252,7 @@ class MainWindow(Ui_MainWindow):
         logo_pixmap = QtGui.QPixmap(":/baangt/baangtlogo")
         logo_pixmap.scaled(300, 120, QtCore.Qt.KeepAspectRatio)
         self.logo_4.setPixmap(logo_pixmap)
+        self.queryLogo.setPixmap(logo_pixmap)
         icon = QtGui.QIcon()
         icon.addPixmap(
                 QtGui.QPixmap(":/baangt/baangticon"),
@@ -442,9 +457,8 @@ class MainWindow(Ui_MainWindow):
     def updateRunFile(self):
         """ this file will update the testRunFile selection
         """
-        self.testRunFile = os.path.join(self.directory,
-                                  self.testRunComboBox_4.currentText())
-        self.statusMessage("Test Run Changed to: {}".format(self.testRunFile))
+        self.testRunFile = self.testRunComboBox_4.currentText()
+        self.statusMessage("Test Run Changed to: {}".format(self.testRunFile), 3000)
         self.saveInteractiveGuiConfig()
 
     @pyqtSlot()
@@ -456,7 +470,7 @@ class MainWindow(Ui_MainWindow):
                                  self.settingComboBox_4.currentText())
 
         self.saveInteractiveGuiConfig()
-        self.statusMessage("Settings changed to: {}".format(self.configFile))
+        self.statusMessage("Settings changed to: {}".format(self.configFile), 3000)
         self.readContentofGlobals()
 
     @pyqtSlot()
@@ -464,7 +478,7 @@ class MainWindow(Ui_MainWindow):
         """ this file will update the testRunFile selection
         """
         self.selectedSheet = self.SheetCombo.currentText()
-        self.statusMessage("Selected Sheet: {}".format(self.selectedSheet))
+        self.statusMessage("Selected Sheet: {}".format(self.selectedSheet), 3000)
 
     def executeButtonClicked(self):
         self.__result_file = ""
@@ -516,7 +530,7 @@ class MainWindow(Ui_MainWindow):
                     str(self.run_process.readAllStandardError().data().decode('iso-8859-1'))))
             self.run_process.finished.connect(self.processFinished)
             self.run_process.start(runCmd)
-            self.statusbar.showMessage("Running.....")
+            self.statusbar.showMessage("Running.....",4000)
 
     @pyqtSlot()
     def stopButtonPressed(self):
@@ -534,6 +548,39 @@ class MainWindow(Ui_MainWindow):
             self.run_process.waitForFinished(3000)
             self.run_process.kill()
 
+    def update_datafile(self):
+        from baangt.base.TestRun.TestRun import TestRun
+        testRunFile = f"{Path(self.directory).joinpath(self.testRunFile)}"
+        globalsFile = self.configFile
+        uu = uuid4()
+        tr = TestRun(testRunFile, globalsFile, uuid=uu, executeDirect=False)
+        tr._initTestRunSettingsFromFile()
+        if "TC.TestDataFileName" in tr.globalSettings:
+            self.dataFile = tr.globalSettings["TC.TestDataFileName"]
+        else:
+            tr._loadJSONTestRunDefinitions()
+            tr._loadExcelTestRunDefinitions()
+            self.dataFile = self.findKeyFromDict(tr.testRunUtils.testRunAttributes, "TestDataFileName")
+
+    def findKeyFromDict(self, dic, key):
+        if isinstance(dic, list):
+            for data in dic:
+                if isinstance(dic, list) or isinstance(dic, dict):
+                    result = self.findKeyFromDict(data, key)
+                    if result:
+                        return result
+        elif isinstance(dic, dict):
+            for k in dic:
+                if k == key:
+                    return dic[k]
+                elif isinstance(dic[k], list) or isinstance(dic[k], dict):
+                    result = self.findKeyFromDict(dic[k], key)
+                    if result:
+                        return result
+        return ""
+
+
+
     def signalCtrl(self, qProcess, ctrlEvent=None):
         import win32console, win32process, win32api, win32con
         if ctrlEvent is None:
@@ -629,7 +676,10 @@ class MainWindow(Ui_MainWindow):
                 else:
                     break
             if result_file[-5:] == ".xlsx":
-                self.__result_file =result_file
+                logger.debug(f"Found result_file in the logs: {result_file}")
+                self.__result_file = result_file
+        elif "Logfile used: " in text:
+            self.__log_file = text.split("used: ")[1].strip()
         if "||Statistic:" in text:
             lis = text.split('||')
             stat = lis[1][10:]
@@ -922,7 +972,7 @@ class MainWindow(Ui_MainWindow):
                 if self.directory:
                     fullpath = os.path.join(self.directory, self.configFile)
                 else:
-                    self.directory = os.getcwd()#.managedPaths.derivePathForOSAndInstallationOption()
+                    self.directory = os.getcwd()   # .managedPaths.derivePathForOSAndInstallationOption()
                     fullpath = os.path.join(self.directory, self.configFile)
 
             with open(fullpath, 'w') as f:
@@ -1205,6 +1255,7 @@ class MainWindow(Ui_MainWindow):
         """ Uses Files Open class to open Result file """
         try:
             if self.__result_file != "":
+                logger.debug(f"Opening ResultFile: {self.__result_file}")
                 filePathName = self.__result_file
                 fileName = os.path.basename(filePathName)
                 self.statusMessage(f"Opening file {fileName}", 3000)
@@ -1244,6 +1295,12 @@ class MainWindow(Ui_MainWindow):
             fileName = os.path.basename(filePathName)
             self.statusMessage(f"Opening file {fileName}", 3000)
             FilesOpen.openResultFile(filePathName)
+            self.update_datafile()
+            if self.dataFile:
+                self.statusMessage(f"Opening file {self.dataFile}", 3000)
+                PathName = f"{Path(self.directory).joinpath(self.dataFile)}"
+                Name = os.path.basename(PathName)
+                FilesOpen.openResultFile(PathName)
         except:
             self.statusMessage("No file found!", 3000)
 
@@ -1293,6 +1350,129 @@ class MainWindow(Ui_MainWindow):
             self.__open_files = 0
         self.saveInteractiveGuiConfig()
 
+
+    #
+    # Browse Results actions
+    #
+
+    @pyqtSlot()
+    def showQueryPage(self):
+        #
+        # display Query Page
+        #
+
+        # setup query object
+        self.queryResults = ResultsBrowser()
+
+        # setup combo box lists
+        self.nameComboBox.clear()
+        self.nameComboBox.addItems([''] + self.queryResults.name_list())
+        self.stageComboBox.clear()
+        self.stageComboBox.addItems([''] + self.queryResults.stage_list())
+        # globals
+        for nameComboBox, valueComboBox in self.globalsOptions:
+            nameComboBox.clear()
+            nameComboBox.addItems([NAME_PLACEHOLDER] + self.queryResults.globals_names())
+            nameComboBox.currentIndexChanged.connect(self.onGlobalsNameChange)
+            valueComboBox.clear()
+            valueComboBox.addItems([VALUE_PLACEHOLDER])
+
+        # show the page
+        self.stackedWidget.setCurrentIndex(4)
+        self.statusMessage("Query Page is triggered", 1000)
+
+
+    @pyqtSlot(int)
+    def onGlobalsNameChange(self, index):
+        #
+        # loads globals values on name changed
+        #
+
+        combo = self.sender()
+        for nameComboBox, valueComboBox in self.globalsOptions:
+            if combo == nameComboBox:
+                valueComboBox.clear()
+                valueComboBox.addItems([VALUE_PLACEHOLDER] + self.queryResults.globals_values(combo.currentText()))
+
+
+    @pyqtSlot()
+    def makeResultQuery(self):
+        #
+        # makes query to results db
+        #
+
+        # get field data
+        name = self.nameComboBox.currentText() or None
+        stage = self.stageComboBox.currentText() or None
+        #date_from = datetime.strptime(self.dateFromInput.date().toString("yyyyMMdd"), "%Y%m%d")
+        #date_to = datetime.strptime(self.dateToInput.date().toString("yyyyMMdd"), "%Y%m%d")
+        date_from = self.dateFromInput.date().toString("yyyy-MM-dd")
+        date_to = self.dateToInput.date().toString("yyyy-MM-dd")
+        # get globals
+        globals_value = lambda v: v if v != VALUE_PLACEHOLDER else None
+
+        global_settings = { 
+            nameComboBox.currentText(): globals_value(valueComboBox.currentText()) for nameComboBox, valueComboBox in filter(
+                lambda g: g[0].currentText() != NAME_PLACEHOLDER,
+                self.globalsOptions,
+            )
+        }
+
+        # make query
+        self.queryResults.query(
+            name=name,
+            stage=stage,
+            start_date=date_from,
+            end_date=date_to,
+            global_settings=global_settings,
+        )
+
+        # display status
+        if self.queryResults.query_set:
+            if self.queryResults.query_set.length > 1:
+                status = f'Found: {self.queryResults.query_set.length} records'
+            else:
+                status = f'Found: {self.queryResults.query_set.length} record'
+        else:
+            status = 'No record found'
+        self.queryStatusLabel.setText(QtCore.QCoreApplication.translate("MainWindow", status))
+
+
+    @pyqtSlot()
+    def exportResultQuery(self):
+        #
+        # exports query results
+        #
+
+        _translate = QtCore.QCoreApplication.translate
+
+        if self.queryResults.query_set:
+            #self.queryStatusLabel.setText(_translate("MainWindow", "Exporting results..."))
+            #sleep(1)
+            path_to_export = self.queryResults.export()
+            self.queryStatusLabel.setText(_translate("MainWindow", f"Exported to: {path_to_export}"))
+            FilesOpen.openResultFile(path_to_export)
+        else:
+            self.queryStatusLabel.setText(_translate("MainWindow", f"ERROR: No data to export"))
+
+    @pyqtSlot()
+    def openRecentQueryResults(self):
+        #
+        # opens recent query result file
+        #
+
+        try:
+            # get recent file
+            recent_export = max(
+                list(Path(self.managedPaths.getOrSetDBExportPath()).glob('*.xlsx')),
+                key=os.path.getctime,
+            )
+            FilesOpen.openResultFile(recent_export)
+        except Exception:
+            # no export file exists
+            self.statusMessage(f"No Result Export File to Show", 3000)
+
+
     @pyqtSlot()
     def cleanup_dialog(self):
         self.clean_dialog = QtWidgets.QDialog(self.centralwidget)
@@ -1307,6 +1487,9 @@ class MainWindow(Ui_MainWindow):
         self.cleanup_downloads = QtWidgets.QCheckBox()
         self.cleanup_downloads.setText("Downloads")
         self.cleanup_downloads.setChecked(True)
+        self.cleanup_reports = QtWidgets.QCheckBox() # cleanup reports
+        self.cleanup_reports.setText("Reports") # cleanup reports
+        self.cleanup_reports.setChecked(True) # cleanup reports
         hlay = QtWidgets.QHBoxLayout()
         label = QtWidgets.QLabel()
         label.setText("Days: ")
@@ -1321,6 +1504,7 @@ class MainWindow(Ui_MainWindow):
         vlay.addWidget(self.cleanup_logs)
         vlay.addWidget(self.cleanup_screenshots)
         vlay.addWidget(self.cleanup_downloads)
+        vlay.addWidget(self.cleanup_reports) # cleanup reports
         vlay.addLayout(hlay)
         vlay.addWidget(button)
         vlay.addWidget(self.cleanup_status)
@@ -1343,6 +1527,9 @@ class MainWindow(Ui_MainWindow):
             c.clean_screenshots()
         if self.cleanup_downloads.isChecked():
             c.clean_downloads()
+        # cleanup reports
+        if self.cleanup_reports.isChecked():
+            c.clean_reports()
         self.cleanup_status.showMessage("Cleaning Complete!")
 
     def showReport(self):

db_update.py

@@ -0,0 +1,48 @@
+from sqlalchemy import create_engine, Column, Integer
+from sqlalchemy.orm import sessionmaker
+from baangt.base.DataBaseORM import DATABASE_URL, TestrunLog, TestCaseLog, TestCaseSequenceLog
+
+def add_column(engine, table_name, column):
+    column_name = column.compile(dialect=engine.dialect)
+    column_type = column.type.compile(engine.dialect)
+    #column_nullable = column.nullable.compile(engine.dialect)
+    #column_default = column.default.compile(engine.dialect)
+    sql = f'''
+    	ALTER TABLE {table_name}
+    	ADD COLUMN {column_name} {column_type} {"NULL" if column.nullable else "NOT NULL"}
+    	{"DEFAULT " if column.default else ""}{column.default.arg if column.default else ""}
+    '''
+    print(f'*** CREATING new column in table {table_name}:\n{sql}')
+    engine.execute(sql)
+
+if __name__ == '__main__':
+	engine = create_engine(DATABASE_URL)
+	
+	# add new columns
+	number_column = Column('number', Integer, nullable=False, default=0)
+	add_column(engine, TestCaseSequenceLog.__table__, number_column)
+	add_column(engine, TestCaseLog.__table__, number_column)
+
+	# populate with numbers
+	print('\n*** POPULATING new columns')
+	session = sessionmaker(bind=engine)()
+
+	# fetch testruns
+	print('\nSetting numbers of TestCaseSequeces')
+	testruns = session.query(TestrunLog)
+	for tr_index, tr in enumerate(testruns):
+		for tcs_index, tcs in enumerate(tr.testcase_sequences, 1):
+			tcs.number = tcs_index
+		if tr_index and not tr_index%50:
+			print(f'{tr_index} TestrunLogs processed')
+	session.commit()
+
+	# fetch testcase sequences
+	print('\nSetting numbers of TestCases')
+	testcase_sequences = session.query(TestCaseSequenceLog)
+	for tcs_index, tcs in enumerate(testcase_sequences):
+		for tc_index, tc in enumerate(tcs.testcases, 1):
+			tc.number = tc_index
+		if tcs_index and not tcs_index%50:
+			print(f'{tcs_index} TestCaseSequenceLogs processed')
+	session.commit()

docs/Datagenerator.rst

@@ -22,6 +22,7 @@ This image is an example input file. Different types of data types supported are
   8. ``FKR_`` prefix is used here with a new integer value 0 in end.
   9. ``RRD_`` prefix is used here.
   10. ``RRE_`` prefix is used here.
+  11. ``RENV_`` prefix is used here.
 
 Using these data type we will generate all possible values.
 Here is a simple example with simple value and value of list.
@@ -83,11 +84,14 @@ We will use the reference of above image and assigned number to learn about it i
      Now this will generate new email for every data in the output.
   9. ``RRD_`` is used when we have multiple sheets in a input file and we need to take value which are matching conditions
      from that sheet.
-     Format:- ``RRD_(<sheetName>,<TargetData>,[Header1:[Value1],Header2:[Value1,Value2]])``
+     Format:- ``RRD_(<sheetName>,<TargetData>,[Header1:[Value1],Header2:[Value1,Value2]])`` or
+              ``RRD_(<sheetName>,<TargetData>:<HeaderName>,[Header1:[Value1],Header2:[Value1,Value2]])``
      Here sheetName is the name of the sheet where our TargetData is located. A dictionary of TargetData is generated with all
      the data which are matching from our Header: Value pair. A header with multiple value list is than converted to all
      possible value as mentioned in above explanation. At last a random value is selected from TargetData dictionary for every
      output data.
+     If TargetData:HeaderName then the header of the targetdata from the target file will be replaced with HeaderName in
+        the output file. Same applies in ``RRE_`` prefix.
      If TargetData = ``*`` then all the values of the matched row will be treated as TargetData.
      If Header:Value List = ``[]`` then the defined TargetData will be collected from every row of the defined sheet.
      i.e.
@@ -100,9 +104,19 @@ We will use the reference of above image and assigned number to learn about it i
      i.e. First ``RRD_`` cell has value "x" for the header while selected randomly, then the second cell will select data
      randomly only from the rows which have "x" value for the same header.
   10. ``RRE_`` is same as ``RRD_`` only change is that in rrd we take data from same file and different sheet but, in
-     this ``RRE_`` prefix we can take data from another file. The only change in its structure is that filename comes
-     before sheetname.
-     i.e. ``RRE_[fileName,sheetName,Targetdata,[key:value]]``
+      this ``RRE_`` prefix we can take data from another file. The only change in its structure is that filename comes
+      before sheetname.
+      i.e. ``RRE_[fileName,sheetName,Targetdata,[key:value]]``
+  11. ``RENV_`` prefix is used to get environment varaible from your system. Their might be sometimes when you don't
+      want to input sensitive data like password, username, etc. directly inside TestRun file or Globals file, at that
+      time this prefix will be very useful and it will get the data from your system environment.
+      Its structure is ``RENV_(<env_variable>,<default>)`` here "<env_variable>" holds the place of variable name which
+      contains data and "<default>" holds the default value which is used in case given variable is not present in
+      environment. If "<default>" value is not supplied and given variable is also not present in environment then
+      it will raise an error.
+      e.g.- ``RENV(USERNAME, My_Pc)``
+      Here it will first look for "USERNAME" variable in environment, if it is not present then it will use "My_Pc"
+
 
 
 All Data Types Format
@@ -116,4 +130,5 @@ All Data Types Format
 6. List of header    = ``[<title1>, <title2>, <title3>]``
 7. Faker Prefix      = ``FKR_(<type>, <locale>, <number_of_data>)``
 8. RRD Prefix        = ``RRD_(<sheetName>,<TargetData>,[<Header1>:[<Value1>],<Header2>:[<Value1>,<Value2>]])``
-9. RRE Prefix        = ``RRE_(<fileName>,<sheetName>,<TargetData>,[<Header1>:[<Value1>],<Header2>:[<Value1>,<Value2>]])``
+9. RRE Prefix        = ``RRE_(<fileName>,<sheetName>,<TargetData>,[<Header1>:[<Value1>],<Header2>:[<Value1>,<Value2>]])``
+10. RENV Prefix      = ``RENV_(<env_variable>,<default>)``

docs/ParametersConfigFile.rst

@@ -74,4 +74,9 @@ for instance to slowly retest a single testrecord or to not close the browser af
      - Set the LogLevel to a different value. In baangt standard the file-logger is set to ``debug`` while the console
        output is set to ``info``. Using this setting you'll set both logger channels to whatever value you provide.
        In the new UI you'll see a dropdown menu.
+   * - ``Append Excel``
+     - When you use this parameter, you can append the corresponding columns of the output sheet to existing Excel-Sheet(s).
+       We will compare column header names and append records (lines) to all Excel-Workbooks/Sheets, that you want.
+       This feature is very handy when you create test data, that you want to use later in other and/or multiple test cases.
+       e.g. if you create customer master data, that you want to use for multiple orders/deliveries.
 

docs/PlannedFeatures.rst

@@ -5,10 +5,10 @@ We implement all features for 3 operating Systems (Mac, Windows, Ubuntu and Ubun
 Short/Medium term features
 ---------------------------
 * Improve template for TestCaseDefinitions
-* DataFiles and TestDataGenerator: Read remote data sources (e.g. other sheets or SQL-Databases)
-* DataFiles: Nested data structures per line item (e.g. Sales order header --> Sales order item)
 * Better support for scraping
-* ELSE-Activity and nested IF/ELSE/ENDIF-Activities
+* Improved reporting on test runs/Test cases, etc.
+* Improved handling of test data base entities (e.g. use-counters for each object)
+
 
 Features for later
 ------------------
@@ -19,7 +19,6 @@ Features for later
 * Improved support for Mass testing APIs
 * Katalon Importer/Converter as Webservice
 * Integration with Atlassian Confluence (for Testcase and Testrun definitions)
-* Integration with Atlassian Confluence (to publish results of testruns)
 * Better support for oData V4.0 (similar to SOAP)
 * Support for GraphQL via Graphene
 * Multi-Language interface (I18n)

docs/SendStatistics.rst

@@ -13,6 +13,7 @@ Their are 4 different services where we can send test reports. They are:
 2. Ms Teams
 3. Slack
 4. Telegram
+5. Confluence
 
 Lets first discuss things we need to use this services in our program one by one.
 
@@ -124,4 +125,68 @@ globals file, if it is not their then it will take settings from ``mail.ini``. H
 Now as we can see the we have override Mail, Ms Teams & Slack settings. So now our program will take mails from globals
 and as the ``NotificationWithAttachment`` parameter is False it won't attach the xlsx file. ``MsWebHook`` & ``SlackWebHook``
 are empty so no report will be sent on those platforms. Here we haven't declared any setting for **Telegram** so the
-program will now look for those settings in ``mail.ini`` and send the report as per that settings.
+program will now look for those settings in ``mail.ini`` and send the report as per that settings.
+
+Confluence
+==========
+We also have functionality to update report in confluence. Reports are updated as page. Along with it we can also attach
+original report(.xlsx) file in main page. Main page can consist link to original file, data from "Summary" tab & data
+from "Output" tab. Their might also be cases when "Output" tab has too many datas, so we have solution for that too. We
+have given you an option to create subpages of data from "Output" tab. You can use this functionality with the help of
+global files. Lets see the keywords needed for global file in order to update the report in confluence.
+
+| **globals.json**
+| {
+|    "Confluence-Base-Url" : "",
+|    "Confluence-Space" : "",
+|    "Confluence-Username" : "",
+|    "Confluence-Password" : "",
+|    "Confluence-Rootpage" : "",
+|    "Confluence-Pagetitle" : "",
+|    "Confluence-Remove_Headers" : "",
+|    "Confluence-Uploadoriginalfile" : "",
+|    "Confluence-Createsubpagesforeachxxentries" : 0
+| }
+
+**Confluence-Base-Url**
+
+``Confluence-Base-Url`` contains the url for your confluence.
+
+**Confluence-Space**
+
+``Confluence-Space`` contains the name of space where page
+is to be created.
+
+**Confluence-Username**
+
+``Confluence-Username`` contains your username.
+
+**Confluence-Password**
+
+``Confluence-Password`` contains your password.
+
+**Confluence-Rootpage**
+
+``Confluence-Rootpage`` contains parent page id, this option is optional and in most of the case is not usable, you must
+use this option if you want to create the report page as a sub-page to another main page.
+
+**Confluence-Pagetitle**
+
+``Confluence-Pagetitle`` title for the report page.
+
+**Confluence-Remove_Headers**
+
+``Confluence-Remove_Headers`` contains headers from "Output" tab which are not to be considered
+while generating report page. Multiple headers should be sperated by comma, e.g. - "header1, header2, header3".
+
+**Confluence-Uploadoriginalfile**
+
+``Confluence-Uploadoriginalfile`` value must be true if you want to upload original xlsx file in main report page.
+
+**Confluence-Createsubpagesforeachxxentries**
+
+``Confluence-Createsubpagesforeachxxentries`` contain integer, when we want to create sub-pages for "Output" tab data
+we should input the number of rows present in a subpage, multiple sub-pages are created with the number of rows which
+are defined here. e.g. - "Confluence-Createsubpagesforeachxxentries" : 100, here we have given input of maximum 100 data
+in a sub-page and suppose if total number of data is 288, then their will be 3 pages containing 1-100, 101-200, 201-288
+data.

docs/changelog.rst

@@ -1,7 +1,40 @@
 Change log
 ==========
 
-1.1.4
+1.2.x
+^^^^^
+
+New features:
++++++++++++++
+* Completely new concept for file handling. Original Excel-Files are not overwritten. Including TestDataGenerator.
+* ``Usecount`` columns will show, how often qualitifed test data was selected in all test runs.
+* ``TestResult``: if this column is in a test data sheet, we'll write the test result of each test run (condensed) into this cell
+* Change Log: In all copied files there's a sheet ``Change Log`` which shows per each timestamp, which changes were made to the original sheet. Incredibly helpful feature!
+* Teststeps: New comparison operators ``CO``ntains and ``IP`` (Is Part of) to compare strings
+
+Refactoring:
+++++++++++++
+* Great speed improvements with large files (read and write, also in remote read)
+* Teststeps: ``GoToURL`` parameter now also possible in ``Locator`` (makes more sense). ``Value`` still works fine.
+
+Bugfixing:
+++++++++++
+
+
+1.1.15
+^^^^^^^
+
+Summary:
+++++++++
+
+* Export (all channels): Passwords are replaced by ``*******``
+* ELSE-Activity and nested IF/ELSE/ENDIF-Activities
+* DataFiles: Nested data structures per line item (e.g. Sales order header --> Sales order item)
+* DataFiles and TestDataGenerator: Read remote data sources (e.g. other sheets or SQL-Databases)
+* Integration with Atlassian Confluence to export test run results into Confluence WIKI-Pages (and Sub-pages!)
+* Export results to multiple Excel-Sheets (e.g. when collecting reusable master data like customer master records)
+
+1.1.5
 ^^^^^^^
 This is the version, that was released as first publicly downloadable version.
 

docs/conf.py

@@ -24,7 +24,7 @@ copyright = '2020, Bernhard Buhl'
 author = 'Bernhard Buhl'
 
 # The full version, including alpha/beta/rc tags
-release = '1.1.5'
+release = '1.2.5'
 
 
 # -- General configuration ---------------------------------------------------

examples/globals.json

@@ -1,10 +1,10 @@
 {
-    "TC.Lines": "",
+    "TC.Lines": "1",
     "TC.dontCloseBrowser": "False",
     "TC.slowExecution": "False",
     "TC.NetworkInfo": "False",
-    "TX.DEBUG": "False",
-    "TC.Browser": "Chrome",
+    "TX.DEBUG": "True",
+    "TC.Browser": "FF",
     "TC.BrowserAttributes": "",
     "TC.ParallelRuns": "1",
     "TC.BrowserWindowSize": "1024x768",
@@ -16,5 +16,7 @@
     "SlackWebHook": "",
     "TelegramBot": "",
     "TelegramChannel": "",
-    "DeactivateStatistics": "False"
+    "DeactivateStatistics": "False",
+    "Password": "Franzi1234",
+    "AR2BXLS": "CompleteBaangtWebdemo_ResultsCombined.xlsx,CombinedResults"
 }

examples/globalsConfluence.json

@@ -0,0 +1,31 @@
+{
+    "TC.Lines": "",
+    "TC.dontCloseBrowser": "False",
+    "TC.slowExecution": "False",
+    "TC.NetworkInfo": "False",
+    "TX.DEBUG": "False",
+    "TC.Browser": "FF",
+    "TC.ParallelRuns": "1",
+    "TC.BrowserWindowSize": "1024x768",
+    "TC.LogLevel": "",
+    "TC.BrowserAttributes": "",
+    "Stage": "Test",
+    "SendMailTo": "",
+    "NotificationWithAttachment": "True",
+    "MsWebHook": "",
+    "SlackWebHook": "",
+    "TelegramBot": "",
+    "TelegramChannel": "",
+    "DeactivateStatistics": "False",
+    "TC.UseRotatingProxies": "False",
+    "TC.ReReadProxies": "False",
+    "Confluence-Base-Url" : "",
+    "Confluence-Space" : "",
+    "Confluence-Username" : "",
+    "Confluence-Password" : "",
+    "Confluence-Rootpage" : "",
+    "Confluence-Pagetitle" : "",
+    "Confluence-Remove_Headers" : "",
+    "Confluence-Uploadoriginalfile" : "",
+    "Confluence-Createsubpagesforeachxxentries" : 0
+}

ini/mail.ini

@@ -1,8 +1,9 @@
 [Default]
-telegramchannel = 
+confluence-base-url = 
 sendmailto = info@baangt.org
 notificationwithattachment = True
 mswebhook = 
 slackwebhook = 
 telegrambot = 
+telegramchannel = 
 

requirements.txt

@@ -1,5 +1,5 @@
 Appium-Python-Client==0.52
-beautifulsoup4==4.8.2
+beautifulsoup4>=4.8.2
 browsermob-proxy>=0.8.0
 dataclasses>=0.6
 dataclasses-json>=0.4.2
@@ -7,22 +7,26 @@ faker>=4.0.2
 gevent>=1.5.0
 lxml>=4.5.0
 openpyxl>=3.0.3
+pandas>=1.1.1
 Pillow>=7.1.2
 pyperclip>=1.8.0
 pluggy>=0.13.1
-PyQt5>=5.14.2
+PyQt5==5.14.2
 requests>=2.23.0
 requests-toolbelt>=0.9.1
-schwifty>=2020.5.2
+schwifty>=2020.8.0
 selenium>=3.141.0
 SQLAlchemy>=1.3.13
 urllib3>=1.25.7
 wheel>=0.34.2
 xl2dict>=0.1.5
-xlrd>=1.2.0
+xlrd3>=1.0.0
 XlsxWriter>=1.2.7
-numpy>=1.18.4
-jinja2>=2.11
-pymsteams>=0.1.12
+numpy>=1.19.2
+jinja2>=2.11.2
+pymsteams>=0.1.14
 slack-webhook>=1.0.3
-psutil>=5.7.0
+psutil>=5.7.2
+atlassian-python-api>=1.17.3
+icopy2xls>=0.0.4
+xlsclone>=0.0.7

setup.py

@@ -6,7 +6,7 @@ if __name__ == '__main__':
 
     setuptools.setup(
         name="baangt",
-        version="1.1.5",
+        version="1.2.5",
         author="Bernhard Buhl",
         author_email="info@baangt.org",
         description="Open source Test Automation Suite for MacOS, Windows, Linux",
@@ -18,14 +18,14 @@ if __name__ == '__main__':
         package_data={"baangt.ui.pyqt": ['globalSetting.json',]},
         install_requires=["Appium-Python-Client", "beautifulsoup4", "browsermob-proxy",
                           "dataclasses", "dataclasses-json",
-                          "faker",  "gevent", "lxml",
+                          "faker",  "gevent", "jinja2", "lxml",
                           "openpyxl",
-                          "Pillow", "pluggy", "pyperclip",  "pyQT5",
+                          "pandas", "Pillow", "pluggy", "pyperclip",  "pyQT5==5.14.2",
                           "requests", "requests-toolbelt",
                           "schwifty", "selenium", "sqlalchemy",
-                          "urllib3",
-                          "xl2dict", "xlrd", "xlsxwriter",
-                           ],
+                          "urllib3", "psutil", "pymsteams", "slack-webhook",
+                          "xl2dict", "xlrd3", "xlsxwriter", "atlassian-python-api",
+                          "icopy2xls", "xlsclone"],
         classifiers=[
             "Programming Language :: Python :: 3",
             "License :: OSI Approved :: MIT License",

test_browser.py

@@ -0,0 +1,100 @@
+from baangt.base.ResultsBrowser import ResultsBrowser
+from datetime import datetime
+import json
+
+#DATABASE_URL = 'sqlite:///testrun_my.db'
+
+# filter parameters
+name = 'heartbeat.json'
+#name = 'RSantragAll.json'
+stage = 'HF'
+#stage = 'PQA'
+start_time = datetime.strptime("2020-06-10 00:00", "%Y-%m-%d %H:%M")
+end_time = datetime.strptime("2020-07-11 00:00", "%Y-%m-%d %H:%M")
+
+def print_logs(logs):
+	for index, log in enumerate(logs):
+		#data = log.to_json()
+		print(f"{index:^4}{log.testrunName:20}{log.stage:10}{log.startTime}\t{log}")
+
+
+#r = ResultsBrowser(db_url=DATABASE_URL)
+r = ResultsBrowser()
+#r.getTestCases(name=name, stage=stage)
+
+'''
+print('\n***** Get All Records')
+logs = r.getResults()
+print_logs(logs)
+
+print('\n***** Filter By Name')
+logs = r.getResults(name=name)
+print_logs(logs)
+
+print('\n***** Filter By Stage')
+logs = r.getResults(stage=stage)
+print_logs(logs)
+
+print('\n***** Filter By Name & Stage')
+logs = r.getResults(name=name, stage=stage)
+print_logs(logs)
+
+print('\n***** Filter By Name and Date')
+logs = r.getResults(name=name, start_date=start_time, end_date=end_time)
+print_logs(logs)
+'''
+'''
+id = 'eff78fa9-83b7-484a-a8ab-5e30cf0f12cc'
+print(f'\n****** GET BY ID: {id}')
+print_logs([r.get(id)])
+'''
+'''
+def draw_seconds(t):
+	if t is None:
+		return '\033[35mnan\033[0m'
+
+	n = t.seconds
+	if n < 20:
+		return f'\033[45m{n:3}\033[0m'
+	elif n < 50:
+		return f'\033[44m{n:3}\033[0m'
+	elif n < 100:
+		return f'\033[42m{n:3}\033[0m'
+	elif n < 150:
+		return f'\033[43m{n:3}\033[0m'
+	else:
+		return f'\033[41m{n:3}\033[0m'
+
+print(f'\n****** TestCase details for {name}')
+status = {
+	"OK": '\033[92mK\033[0m',
+	"Failed": '\033[91mF\033[0m',
+	"Paused": '\033[90mP\033[0m',
+}
+logs = r.getResults(name=name, stage=stage)
+
+print(f'\n{"Date":^20}\tTest Case Status')
+for log in logs:
+	print(log.startTime, end='\t')
+	for tc in log.testcase_sequences[0].testcases:
+		print(status[tc.status], end=' ')
+	print()
+
+print(f'\n{"Date":^20}\tTest Case Duration (s)')
+for log in logs:
+	print(log.startTime, end='\t')
+	for tc in log.testcase_sequences[0].testcases:
+		print(draw_seconds(tc.duration), end=' ')
+	print()
+'''
+
+r.query(name=name, stage=stage, start_date=start_time, end_date=end_time)
+#r.query(name=name, start_date=start_time)
+f = r.export()
+
+
+#print(f'Exported to: {f}')
+
+
+
+

tests/0TestInput/ServiceTestInput/example_googleImages.json

@@ -0,0 +1,37 @@
+{"TESTRUNEXECUTIONPARAMETERS":
+   {"ExportFormat":  {
+     "ExportFormat": "XLSX",
+     "Fieldlist": ["TestCaseStatus", "Duration", "timelog"]},
+     "TESTSEQUENCE": {
+       "1": ["baangt.TestCaseSequence.TestCaseSequenceMaster.TestCaseSequenceMaster", {
+         "SequenceClass": "baangt.TestCaseSequence.TestCaseSequenceMaster.TestCaseSequenceMaster",
+         "TestDataFileName": "example_googleImages.xlsx",
+         "Sheetname": "data",
+         "ParallelRuns": 1,
+         "FromLine": 0,
+         "ToLine": 1,
+         "TESTCASE": {
+           "1": ["baangt.TestCase.TestCaseMaster.TestCaseMaster", {
+             "TestCaseType": "Browser",
+             "Browser": "FF",
+             "BrowserAttributes": ""
+           },
+             {"TestStep": {
+               "1": [{"TestStepClass": "baangt.TestSteps.TestStepMaster"}, {
+                 "TestStepExecutionParameters": {
+                   "1": {"Activity": "GOTOURL", "LocatorType": "", "Locator": "", "Value": "https://google.com", "Comparison": "", "Value2": "", "Timeout": "", "Optional": "", "Release": ""},
+                   "2": {"Activity": "COMMENT", "LocatorType": "", "Locator": "Open Google.com and search for terms from the data tab", "Value": "", "Comparison": "", "Value2": "", "Timeout": "", "Optional": "", "Release": ""},
+                   "3": {"Activity": "SETTEXT", "LocatorType": "xpath", "Locator": "//input[@role='combobox']", "Value": "$(searchterm)", "Comparison": "", "Value2": "", "Timeout": "", "Optional": "", "Release": ""},
+                   "4": {"Activity": "CLICK", "LocatorType": "xpath", "Locator": "//input[contains(@type,'submit')]", "Value": "", "Comparison": "", "Value2": "", "Timeout": "", "Optional": "", "Release": ""}
+                 }
+               }
+               ]
+             }
+             }
+           ]
+         }
+       }
+       ]
+     }
+   }
+}

tests/0TestInput/ServiceTestInput/globals.json

@@ -0,0 +1,22 @@
+{
+    "TC.Lines": "0-1",
+    "TC.dontCloseBrowser": "False",
+    "TC.slowExecution": "False",
+    "TC.NetworkInfo": "False",
+    "TX.DEBUG": "False",
+    "TC.Browser": "FF",
+    "TC.BrowserAttributes": "",
+    "TC.ParallelRuns": "1",
+    "TC.BrowserWindowSize": "1024x768",
+    "TC.LogLevel": "Debug",
+    "TC.Stage": "Test",
+    "SendMailTo": "",
+    "NotificationWithAttachment": "True",
+    "MsWebHook": "",
+    "SlackWebHook": "",
+    "TelegramBot": "",
+    "TelegramChannel": "",
+    "DeactivateStatistics": "False",
+    "Password": "Franzi1234",
+    "AR2BXLS": "CompleteBaangtWebdemo_ResultsCombined.xlsx,CombinedResults"
+}

tests/0TestInput/ServiceTestInput/globalsNoBrowser.json

@@ -9,7 +9,6 @@
     "TC.BrowserWindowSize": "1024x768",
     "TC.LogLevel": "Debug",
     "Stage": "Test",
-    "SendMailTo": "randomthings2206@gmail.com, buhl@buhl-consulting.com.cy",
     "MsWebHook": "",
     "SlackWebHook": "",
     "TelegramBot": "",

tests/test_AddressCreate.py

@@ -1,19 +1,17 @@
 import pytest
+import baangt.base.GlobalConstants as GC
+from baangt.base.AddressCreate import AddressCreate
 
 
-def test_getRandomAddress():
-    import baangt.base.GlobalConstants as GC
-    from baangt.base.AddressCreate import AddressCreate
+@pytest.mark.parametrize("CountryCode, PostalCode", [("AT", "1020"), ("CY", "6020")])
+def test_getRandomAddress(CountryCode, PostalCode):
+    addressCreate = AddressCreate(addressFilterCriteria={GC.ADDRESS_COUNTRYCODE:CountryCode})
+    address = addressCreate.returnAddress()
+    assert address[GC.ADDRESS_COUNTRYCODE] == CountryCode
+    assert address[GC.ADDRESS_POSTLCODE] == PostalCode
 
-    addressCreate = AddressCreate(addressFilterCriteria={GC.ADDRESS_COUNTRYCODE:"AT"})
 
-    assert addressCreate.returnAddress()[GC.ADDRESS_COUNTRYCODE] == "AT"
-    assert addressCreate.returnAddress()[GC.ADDRESS_POSTLCODE] == "1020"
-
-    addressCreate = AddressCreate(addressFilterCriteria={GC.ADDRESS_COUNTRYCODE:"CY"})
-
-    assert addressCreate.returnAddress()[GC.ADDRESS_COUNTRYCODE] == "CY"
-    assert addressCreate.returnAddress()[GC.ADDRESS_POSTLCODE] == "6020"
-
-if __name__ == '__main__':
-    test_getRandomAddress()
+def test_testStepAddressCreation():
+    from baangt.TestSteps.AddressCreation import AddressCreate
+    address = AddressCreate.returnAddress()
+    assert type(address) == dict

tests/test_ApiHandling.py

@@ -0,0 +1,75 @@
+import pytest
+from unittest.mock import patch
+from baangt.TestSteps.Exceptions import baangtTestStepException
+from baangt.base.ApiHandling import ApiHandling
+
+
+class fake_response:
+    def __init__(self, **kwargs):
+        self.status_code = 200
+        self.headers = ""
+        self.process()
+
+    def get(self, **kwargs):
+        return self
+
+    def post(self, **kwargs):
+        return self
+
+    def process(self):
+        return {"success": 200, "headers": ""}
+
+    def json(self):
+        return '{"success": 200, "headers": ""}'
+
+    def close(self):
+        pass
+
+
+@pytest.fixture(scope="module")
+def apiHandling():
+    return ApiHandling()
+
+
+@patch("requests.Session", fake_response)
+def test_getSession(apiHandling):
+    apiHandling.getSession()
+    assert 1 == 1
+
+
+@pytest.mark.parametrize("sessionNumber", [(None), (1)])
+def test_getNewSession(sessionNumber, apiHandling):
+    if sessionNumber:
+        apiHandling.getNewSession(sessionNumber=sessionNumber)
+    else:
+        with pytest.raises(baangtTestStepException):
+            apiHandling.getNewSession()
+    assert 1 in apiHandling.session
+
+
+@patch.object(ApiHandling, "getSession", fake_response)
+def test_getURL(apiHandling):
+    apiHandling.setBaseURL("")
+    apiHandling.setEndPoint("")
+    apiHandling.getURL()
+    assert 1 == 1
+
+
+@pytest.mark.parametrize("status", [(200), (300)])
+def test_returnTestCaseStatus(status, apiHandling):
+    result = apiHandling.returnTestCaseStatus(status)
+    assert result == "OK" or result == "Failed"
+
+
+@patch.object(ApiHandling, "getSession", fake_response)
+def test_postURL(apiHandling):
+    apiHandling.setBaseURL("")
+    apiHandling.setEndPoint("")
+    apiHandling.postURL(content="{}", url="url")
+    assert 1 == 1
+
+
+def test_setLoginData(apiHandling):
+    apiHandling.setLoginData("user", "pass")
+    assert apiHandling.session[1].auth == ("user", "pass")
+    apiHandling.tearDown()

tests/test_Append2BaseXls.py

@@ -0,0 +1,20 @@
+from baangt.base.ExportResults.Append2BaseXLS import Append2BaseXLS
+import pytest
+from unittest.mock import patch
+import icopy2xls
+
+
+class testRun:
+    def __init__(self):
+        self.globalSettings = {"AR2BXLS": "examples/CompleteBaangtWebdemo.xlsx,1;/fake/path/test.xlsx,1"}
+
+
+@pytest.fixture(scope="module")
+def testRunInstance():
+    return testRun()
+
+
+@patch.object(icopy2xls.Mover, "move")
+def test_A2BX(mock_mover, testRunInstance):
+    Append2BaseXLS(testRunInstance)
+    assert 1 == 1

tests/test_ExportResults.py

@@ -0,0 +1,20 @@
+from baangt.base.ExportResults.ExportResults import ExportAdditionalDataIntoTab, ExportNetWork
+from unittest.mock import MagicMock
+import datetime
+import pytest
+
+
+def test_ExportAdditionalDataIntoTab():
+    ExportAdditionalDataIntoTab("temp", {1: {"header": "value"}}, MagicMock()).export()
+    assert 1 == 1
+
+
+@pytest.mark.parametrize("d1, d2", [
+    ([datetime.datetime.now() + datetime.timedelta(hours=1)], []),
+    ([], [[[0, datetime.datetime.now() + datetime.timedelta(hours=1)]]]),
+    ([], []),
+])
+def test_get_test_case_num(d1, d2):
+    en = ExportNetWork({}, d1, d2, MagicMock(), MagicMock())
+    en._get_test_case_num(str(datetime.datetime.now()), "chrome")
+    assert 1 == 1

tests/test_SendReports.py

@@ -1,9 +1,9 @@
 from baangt.base.SendReports import Sender
 from pathlib import Path
 import configparser
-import requests
-import json
 import os
+from unittest.mock import patch
+from atlassian import Confluence
 
 
 def readConfig():
@@ -25,3 +25,44 @@ def test_telegram():
     messages = send_stats.sendTelegram(test=True)
     text = subject + "\n\n" + body
     assert text in messages
+
+
+def fake_response(text):
+    return '{"test@pytest": "Success"}'
+
+@patch.object(Confluence, "create_page")
+@patch.object(Confluence, "attach_file")
+def test_confluence(mock_attach, mock_create):
+    send_stats.globalSettings["Confluence-Base-Url"] = "xxxx"
+    send_stats.globalSettings["Confluence-Space"] = "xxxx"
+    send_stats.globalSettings["Confluence-Pagetitle"] = "xxxx"
+    send_stats.globalSettings["Confluence-Username"] = "xxxx"
+    send_stats.globalSettings["Confluence-Password"] = "xxxx"
+    send_stats.globalSettings["Confluence-Rootpage"] = "xxxx"
+    send_stats.globalSettings["Confluence-Remove_Headers"] = "xxxx"
+    send_stats.globalSettings["Confluence-Uploadoriginalfile"] = "xxxx"
+    send_stats.globalSettings["Confluence-Createsubpagesforeachxxentries"] = 11
+    send_stats.sendConfluence()
+    assert 1 == 1
+
+
+@patch("json.loads", fake_response)
+@patch("requests.post")
+def test_SendMail(mock_request):
+    send_stats.defaultSettings["SendMailTo"] = "test@pytest"
+    send_stats.sendMail()
+    assert mock_request.call_count == 1
+
+
+@patch("pymsteams.connectorcard")
+def test_SendMsTeams(mock_conn):
+    send_stats.defaultSettings["MsWebHook"] = "xxxx"
+    send_stats.sendMsTeam()
+    assert mock_conn.call_count == 1
+
+
+@patch("slack_webhook.Slack")
+def test_SendSlack(mock_conn):
+    send_stats.defaultSettings["SlackWebHook"] = "xxxx"
+    send_stats.sendSlack()
+    assert 1 == 1

tests/test_ServiceTest.py

@@ -1,6 +1,6 @@
 import os
 import glob
-import xlrd
+import xlrd3 as xlrd
 import subprocess
 from pathlib import Path
 from baangt.base.DownloadFolderMonitoring import DownloadFolderMonitoring
@@ -108,7 +108,6 @@ def test_regular_firefox():
     assert new_file
     output_file = output_dir.joinpath(new_file[0][0]).as_posix()
     check_output(output_file)
-    os.remove(output_file)
     return "Firefox regular test succeed output file =", new_file[0][0]
 
 
@@ -121,7 +120,6 @@ def test_parellel_firefox():
     assert new_file
     output_file = output_dir.joinpath(new_file[0][0]).as_posix()
     check_output(output_file)
-    os.remove(output_file)
     return "Firefox parellel test succeed output file =", new_file[0][0]
 
 
@@ -135,7 +133,6 @@ def test_browsermob_proxy_firefox():
     output_file = output_dir.joinpath(new_file[0][0]).as_posix()
     check_output(output_file)
     check_browsermob_output(output_file)
-    os.remove(output_file)
     return "Firefox Browsermob test succeed output file =", new_file[0][0]
 
 
@@ -148,7 +145,6 @@ def test_headless_firefox():
     assert new_file
     output_file = output_dir.joinpath(new_file[0][0]).as_posix()
     check_output(output_file)
-    os.remove(output_file)
     return "Firefox headless test succeed output file =", new_file[0][0]
 
 
@@ -160,8 +156,6 @@ def test_csv_firefox():
     new_file = folder_monitor.getNewFiles()
     assert new_file
     assert ".csv" in new_file[0][0]
-    output_file = output_dir.joinpath(new_file[0][0]).as_posix()
-    os.remove(output_file)
     return "Firefox Output Format test succeed output file =", new_file[0][0]
 
 
@@ -176,7 +170,6 @@ def test_regular_chrome():
     assert new_file
     output_file = output_dir.joinpath(new_file[0][0]).as_posix()
     check_output(output_file)
-    os.remove(output_file)
     return "Chrome regular test succeed output file =", new_file[0][0]
 
 
@@ -190,7 +183,6 @@ def test_parellel_chrome():
     assert new_file
     output_file = output_dir.joinpath(new_file[0][0]).as_posix()
     check_output(output_file)
-    os.remove(output_file)
     return "Chrome parellel test succeed output file =", new_file[0][0]
 
 
@@ -205,7 +197,6 @@ def test_browsermob_proxy_chrome():
     output_file = output_dir.joinpath(new_file[0][0]).as_posix()
     check_output(output_file)
     check_browsermob_output(output_file)
-    os.remove(output_file)
     return "Chrome Browsermob test succeed output file =", new_file[0][0]
 
 
@@ -223,7 +214,6 @@ def test_headless_chrome():
     assert new_file
     output_file = output_dir.joinpath(new_file[0][0]).as_posix()
     check_output(output_file)
-    os.remove(output_file)
     return "Chrome headless test succeed output file =", new_file[0][0]
 
 
@@ -246,7 +236,6 @@ def test_full_BaangtWebDemo():
     assert new_file
     output_file = output_dir.joinpath(new_file[0][0]).as_posix()
     check_output(output_file)
-    os.remove(output_file)
 
 def test_NestedIfElse_with_NoBrowser():
     run_file = str(input_dir.joinpath("CompleteBaangtWebdemo_else.xlsx"))
@@ -255,7 +244,14 @@ def test_NestedIfElse_with_NoBrowser():
     assert new_file
     output_file = output_dir.joinpath(new_file[0][0]).as_posix()
     check_output(output_file)
-    os.remove(output_file)
+
+def test_NestedIfElse_with_greater_endif():
+    run_file = str(input_dir.joinpath("CompleteBaangtWebdemo_else_error.xlsx"))
+    try:
+        execute(run_file, globals_file=Path(input_dir).joinpath("globalsNoBrowser.json"))
+        assert 1 == 0
+    except BaseException:
+        assert 1 == 1
 
 def test_NestedLoops_and_repeat():
     run_file = str(input_dir.joinpath("CompleteBaangtWebdemo_nested.xlsx"))
@@ -278,4 +274,10 @@ def test_NestedLoops_and_repeat():
     s = Session()
     data = s.query(TestrunLog).get(uuid.UUID(TestRunUUID).bytes)
     assert "textarea2" in json.loads(data.RLPJson)
-    os.remove(output_file)
+
+
+def test_JsonTestRun():
+    run_file = str(input_dir.joinpath("example_googleImages.json"))
+    execute(run_file, globals_file=Path(input_dir).joinpath("globals.json"))
+    new_file = folder_monitor.getNewFiles()
+    assert new_file

tests/test_TestDataGenerator.py

@@ -1,5 +1,5 @@
 from baangt.TestDataGenerator.TestDataGenerator import TestDataGenerator
-import xlrd
+import xlrd3 as xlrd
 import os
 import csv
 from pathlib import Path
@@ -83,56 +83,54 @@ def test_write_to_wrong_Path():
 
 def test_rrd_simple_input():
     # Checks __processRrd to get dict to target data
-    rrd_output_dict = testDataGenerator._TestDataGenerator__processRrd(
+    rrd_output_dict = testDataGenerator._TestDataGenerator__processRrdRre(
         'Customers', 'Customer', {'Age group': ['30s', '40s'], 'Employment_status': ['employed']}
     )
     assert len(rrd_output_dict) > 0
-    for data in rrd_output_dict:
-        print(data)
 
 
 def test_rrd_target_data_all():
     # Checks __processRrd to get dict to for all data of matching values
-    rrd_output_dict = testDataGenerator._TestDataGenerator__processRrd(
+    rrd_output_dict = testDataGenerator._TestDataGenerator__processRrdRre(
         'Customers', '*', {'Age group': ['30s', '40s'], 'Employment_status': ['employed']}
     )
     assert len(rrd_output_dict) > 0
-    for data in rrd_output_dict:
-        print(data)
 
 
 def test_rrd_no_data_to_match():
     # Checks __processRrd to get dict to for all data of when no value of matching is given
-    rrd_output_dict = testDataGenerator._TestDataGenerator__processRrd('PaymentTerms', '*', {})
+    rrd_output_dict = testDataGenerator._TestDataGenerator__processRrdRre('PaymentTerms', '*', {})
     assert len(rrd_output_dict) > 0
-    for data in rrd_output_dict:
-        print(data)
 
 def test_rre_simple_input():
     # Checks __processRrd to get dict to target data
-    rrd_output_dict = testDataGenerator._TestDataGenerator__data_generators(
-        "RRE_[examples/CompleteBaangtWebdemo.xlsx,CustomerData,[NameFirst,NameLast],[Stage:[Test]]]"
+    rrd_output_dict = testDataGenerator.data_generators(
+        "RRE_[../../examples/CompleteBaangtWebdemo.xlsx,CustomerData,[NameFirst,NameLast],[Stage:[Test]]]"
     )
-    assert len(rrd_output_dict) == 5
-    for data in rrd_output_dict:
-        print(data)
+    assert len(rrd_output_dict.dataList) == 5
 
 
 def test_rre_target_data_all():
     # Checks __processRrd to get dict to for all data of matching values
-    rrd_output_dict = testDataGenerator._TestDataGenerator__data_generators(
-        "RRE_[examples/CompleteBaangtWebdemo.xlsx,CustomerData,*,[Stage:[Test]]]"
+    rrd_output_dict = testDataGenerator.data_generators(
+        "RRE_[../../examples/CompleteBaangtWebdemo.xlsx,CustomerData,*,[Stage:[Test]]]"
     )
-    assert len(rrd_output_dict) == 5
-    for data in rrd_output_dict:
-        print(data)
+    assert len(rrd_output_dict.dataList) == 5
 
 
 def test_rre_no_data_to_match():
     # Checks __processRrd to get dict to for all data of when no value of matching is given
-    rrd_output_dict = testDataGenerator._TestDataGenerator__data_generators(
-        "RRE_[examples/CompleteBaangtWebdemo.xlsx,CustomerData,*,[]"
+    rrd_output_dict = testDataGenerator.data_generators(
+        "RRE_[../../examples/CompleteBaangtWebdemo.xlsx,CustomerData,*,[]"
     )
-    assert len(rrd_output_dict) == 10
-    for data in rrd_output_dict:
-        print(data)
+    assert len(rrd_output_dict.dataList) == 10
+
+
+def test_renv():
+    data = TestDataGenerator.get_env_variable("(USERNAME, test)")
+    assert data
+
+
+def test_renv_without_default():
+    with pytest.raises(BaseException):
+        TestDataGenerator.get_env_variable("(URNAMEfh)")

tests/test_TestStepMaster.py

@@ -0,0 +1,47 @@
+from baangt.TestSteps.TestStepMaster import TestStepMaster
+from baangt.base.Timing.Timing import Timing
+import pytest
+from unittest.mock import patch, MagicMock
+
+
+class FakeTestRun:
+    def __init__(self):
+        self.testRunName = ""
+        self.testRunUtils = MagicMock()
+        self.globalSettings = {}
+
+
+@pytest.fixture(scope="module")
+def testStepMaster():
+    return TestStepMaster(TimingClassInstance=Timing(), TESTRUNINSTANCE=FakeTestRun(), data={})
+
+
+@pytest.mark.parametrize("lComparison, value1, value2",[("=", True, True), ("=", "True", "None"), ("!=", True, True),
+    ("!=", True, "None"), (">", 1, 2), (">", 2, 1), ("<", 1, 2), ("<", 2, 1), (">=", 1, 2), (">=", 2, 1), ("<=", 1, 2),
+    ("<=", 2, 1), ("!!", True, True), ("", True, True)])
+def test_doComparison(lComparison, value1, value2, testStepMaster):
+    if lComparison != "!!":
+        result = testStepMaster._TestStepMaster__doComparisons(lComparison, value1, value2)
+        assert type(result) == bool
+    else:
+        with pytest.raises(BaseException):
+            testStepMaster._TestStepMaster__doComparisons(lComparison, value1, value2)
+        assert 1 == 1
+
+
+@pytest.mark.parametrize("command, locator", [("SETTEXTIF", ""), ("FORCETEXT", ""), ("FORCETEXTIF", ""), ("FORCETEXTJS", ""),
+                                     ("SETANCHOR", ""), ("HANDLEIFRAME", ""), ("APIURL", ""), ("ENDPOINT", ""),
+                                     ("POST", ""), ("GET", ""), ("HEADER", ""), ("SAVE", ""), ("CLEAR", ""),
+                                     ("ADDRESS_CREATE", ""), ("ASSERT", ""), ("PDFCOMPARE", ""), ("CHECKLINKS", ""),
+                                     ("ZZ_", ""), ("TCStopTestCase".upper(), ""), ("TCStopTestCaseError".upper(), ""),
+                                     ("SETANCHOR", "temp")])
+def test_executeDirectSingle(command, locator, testStepMaster):
+    if command == "SAVE":
+        testStepMaster.doSaveData = MagicMock()
+    testStepMaster.browserSession = MagicMock()
+    testStepMaster.apiSession = MagicMock()
+    testStepMaster.executeDirectSingle(0, {
+        "Activity": command, "LocatorType": "", "Locator": locator, "Value": 'temp',
+        "Value2": '', "Comparison": '', "Optional": '', 'Release': '', "Timeout": ''
+    })
+    assert 1 == 1

tests/test_browserHandling.py

@@ -1,6 +1,22 @@
 import pytest
 import platform
 from baangt.base import GlobalConstants as GC
+from baangt.base.BrowserHandling.BrowserHandling import BrowserDriver
+from unittest.mock import patch, MagicMock
+from baangt.TestSteps.Exceptions import baangtTestStepException
+from baangt.base.BrowserHandling.WebdriverFunctions import WebdriverFunctions
+from baangt.base.Utils import utils
+
+
+browserName = "FIREFOX"
+desired_app = [{GC.MOBILE_PLATFORM_NAME: "Android"}, {GC.MOBILE_PLATFORM_NAME: "iOS"}]
+mobileApp = ["True", "False"]
+mobile_app_setting = {
+    GC.MOBILE_APP_URL: "test",
+    GC.MOBILE_APP_PACKAGE: "com.baangt.pytest",
+    GC.MOBILE_APP_ACTIVITY: "baangt.test",
+    GC.MOBILE_APP_BROWSER_PATH: "temp"
+}
 
 
 @pytest.fixture
@@ -8,7 +24,6 @@ def getdriver():
     """ This will return BrowserDriver instance
         for below test function
     """
-    from baangt.base.BrowserHandling.BrowserHandling import BrowserDriver
     return BrowserDriver()
 
 
@@ -20,6 +35,7 @@ def test_setZoomFactor(getdriver):
     getdriver.createNewBrowser()
     getdriver.goToUrl("https://www.baangt.org")
     # FInd element by class
+    getdriver.zoomFactorDesired = True
     getdriver.setZoomFactor(lZoomFactor=200)
 
     # TODO add check
@@ -193,3 +209,123 @@ def test_setBrowserWindowSizeWithX(getdriver):
     result = getdriver.setBrowserWindowSize("--800x600")
     getdriver.closeBrowser()
     assert isinstance(result, dict)
+
+
+@pytest.mark.parametrize(
+    "browserName, desired_app, mobileApp, mobile_app_setting",
+    [
+        (browserName, desired_app[0], mobileApp[0], mobile_app_setting),
+        (browserName, desired_app[0], mobileApp[1], mobile_app_setting),
+        (browserName, desired_app[1], mobileApp[0], mobile_app_setting),
+        (browserName, desired_app[1], mobileApp[1], mobile_app_setting)
+    ]
+)
+def test_mobileConnectAppium(browserName, desired_app, mobileApp, mobile_app_setting):
+    wdf = WebdriverFunctions
+    with patch.dict(wdf.BROWSER_DRIVERS, {GC.BROWSER_APPIUM: MagicMock}):
+        BrowserDriver._mobileConnectAppium(browserName, desired_app, mobileApp, mobile_app_setting)
+    assert 1 == 1
+
+
+def test_handleIframe(getdriver):
+    getdriver.browserData = MagicMock()
+    getdriver.handleIframe("test")
+    assert 1 == 1
+    getdriver.iFrame = "test"
+    getdriver.handleIframe()
+    assert 1 == 1
+
+
+def test_checkLinks(getdriver):
+    getdriver.browserData = MagicMock()
+    getdriver.browserData.driver.find_elements_by_css_selector.return_value = [MagicMock()]
+    getdriver.checkLinks()
+    assert 1 == 1
+
+
+def test_waitForPageLoadAfterButtonClick(getdriver):
+    getdriver.browserData = MagicMock()
+    getdriver.html = "test"
+    getdriver.waitForPageLoadAfterButtonClick()
+    assert 1 == 1
+
+
+@pytest.mark.parametrize("function", [("close"), ("closeall-0"), ("")])
+def test_handleWindow(function, getdriver):
+    getdriver.browserData = MagicMock()
+    getdriver.browserData.driver.window_handles = ["test", "test"]
+    if function == "":
+        with pytest.raises(baangtTestStepException):
+            getdriver.handleWindow(function=function)
+        getdriver.refresh()
+    else:
+        getdriver.handleWindow(function=function)
+    assert 1 == 1
+
+
+def test_findNewFiles(getdriver):
+    from baangt.base.DownloadFolderMonitoring import DownloadFolderMonitoring
+    getdriver.downloadFolderMonitoring = DownloadFolderMonitoring(getdriver.downloadFolder)
+    result = getdriver.findNewFiles()
+    assert type(result) == list
+
+
+def test_getURL(getdriver):
+    result = getdriver.getURL()
+    assert result
+
+
+@pytest.mark.parametrize("css, xpath, id", [("test", None, None), (None, "test", None), (None, None, "test")])
+def test_findWaitNotVisible(css, xpath, id, getdriver):
+    getdriver.browserData = MagicMock()
+    result = getdriver.findWaitNotVisible(css, xpath, id)
+    assert type(result) == bool
+
+
+@pytest.mark.parametrize("browser", ["firefox", "chrome"])
+def test_createNewBrowser(browser, getdriver):
+    with patch.dict(WebdriverFunctions.BROWSER_DRIVERS, {GC.BROWSER_REMOTE: MagicMock}):
+        getdriver.createNewBrowser(browserName="REMOTE_V4", desiredCapabilities={"browserName": browser})
+    assert 1 == 1
+
+
+@patch.object(utils, "setLocatorFromLocatorType")
+def test_waitForElementChangeAfterButtonClick(mock_utils, getdriver):
+    mock_utils.return_value = (1, 2, 3)
+    getdriver.findBy = MagicMock()
+    id = MagicMock()
+    getdriver.findBy.return_value = (id, '1234')
+    getdriver.element = MagicMock()
+    getdriver.element.id = "xxxx"
+    id.id = "1234"
+    getdriver.waitForElementChangeAfterButtonClick(timeout=0.1)
+    assert 1 == 1
+
+
+@pytest.mark.parametrize("css, xpath, id", [("x", None, None), (None, "x", None), (None, None, "x")])
+def test_findWaitNotVisible(css, xpath, id, getdriver):
+    getdriver.browserData = MagicMock()
+    getdriver.findWaitNotVisible(css, xpath, id, timeout=1)
+    assert 1 == 1
+
+
+def test_findByAndForceViaJS(getdriver):
+    getdriver.browserData = MagicMock()
+    getdriver.findByAndForceViaJS(xpath="Test")
+    assert 1 == 1
+
+
+def test_findByAndForceText(getdriver):
+    getdriver.findBy = MagicMock()
+    getdriver.findBy.return_value = False, True
+    getdriver.findByAndForceText()
+    assert 1 == 1
+
+
+def test_findByAndSetTextValidated(getdriver):
+    getdriver.findBy = MagicMock()
+    getdriver.findByAndForceText = MagicMock()
+    getdriver.findBy.return_value = MagicMock(), "def"
+    getdriver.findBy.return_value[0].text = "abc"
+    getdriver.findBy.return_value[0].get_property.return_value = "abc"
+    getdriver.findByAndSetTextValidated(value="123")

tests/test_browserHelperFunction.py

@@ -0,0 +1,22 @@
+import pytest
+import logging
+from unittest.mock import MagicMock, patch
+from baangt.base.BrowserHandling.BrowserHelperFunction import BrowserHelperFunction
+
+
+@pytest.mark.parametrize("desiredCapabilities", [({}), ({"seleniumGridIp": "0.0.0.0", "seleniumGridPort": "4444"})])
+def test_browserHelper_setSettingsRemoteV4(desiredCapabilities):
+    result = BrowserHelperFunction.browserHelper_setSettingsRemoteV4(desiredCapabilities)
+    assert len(result) == 3
+
+
+@pytest.mark.parametrize("logType", [(logging.ERROR), (logging.WARN), ("")])
+def test_browserHelper_log(logType):
+    BrowserHelperFunction.browserHelper_log(logType, "Log Text", MagicMock(), MagicMock, extra="test")
+    assert 1 == 1
+
+
+@patch("baangt.base.ProxyRotate.ProxyRotate.remove_proxy", MagicMock)
+def test_browserHelper_setProxyError():
+    BrowserHelperFunction.browserHelper_setProxyError({"ip": "127.0.0.1", "port": "4444"})
+    assert 1 == 1

tests/test_testrun.py

@@ -1,17 +1,27 @@
 import pytest
+from unittest.mock import patch
+from baangt.base.TestRun.TestRun import TestRun
+
+
+class subprocess_communicate:
+    def __init__(self, *stdout, **stderr):
+        pass
+
+    def communicate(self):
+        return b"firefox\n", 1
+
 
 @pytest.fixture
 def testrun_obj():
     """ This function return instance of TestRun object
         which will be used by other test methods
     """
-    from baangt.base.TestRun.TestRun import TestRun
-    return TestRun("SimpleTheInternet.xlsx","globals.json", executeDirect=False)
+    return TestRun("examples/SimpleTheInternet.xlsx","globals.json", executeDirect=False)
 
 
 def test_filenotfound():
     """ To check if function raises File not found Error """
-    with pytest.raises(Exception) as e:
+    with pytest.raises(BaseException) as e:
         TestRun("SimpleTheInternet.xlsx","global.json")
 
 
@@ -46,6 +56,23 @@ def test_setResult(testrun_obj):
     # check the result
     new_result = set(old_result[0]+1, old_result[1])
     assert new_result == testrun_obj.getSuccessAndError()
+
+
+@pytest.mark.parametrize("system, os_version", [
+    ("Linux", ["redhat-release"]),
+    ("Linux", ["debian_version"]),
+    ("Darwin", ["firefox"]),
+    ("Darwin", ["chrome"])
+])
+@patch("subprocess.Popen", subprocess_communicate)
+@patch("os.listdir")
+@patch("platform.system")
+def test_ExcelImporter(mock_plat, mock_list, system, os_version):
+    from baangt.base.TestRunExcelImporter import TestRunExcelImporter
+    mock_plat.return_value = system
+    mock_list.return_value = os_version
+    TestRunExcelImporter.get_browser(TestRunExcelImporter)
+    assert mock_plat.call_count > 0
     
 
 

tests/test_testrun_jsonimport.py

@@ -10,11 +10,11 @@ def testrun_obj():
         which will be used by other test methods
     """
     from baangt.base.TestRun.TestRun import TestRun
-    return TestRun("SimpleTheInternet.xlsx", "globals.json", executeDirect=False)
+    return TestRun("examples/SimpleTheInternet.xlsx", "globals.json", executeDirect=False)
 
 
 def test_with_globalsHeadless(testrun_obj):
-    lTestRun = TestRun("SimpleTheInternet.xlsx",
+    lTestRun = TestRun("examples/SimpleTheInternet.xlsx",
                        globalSettingsFileNameAndPath= \
                            Path(os.getcwd()).joinpath("tests").joinpath("jsons").joinpath("globals_headless.json"),
                        executeDirect=False)
@@ -25,7 +25,7 @@ def test_with_globalsHeadless(testrun_obj):
 
 
 def test_with_globalsHeadlessVersion2(testrun_obj):
-    lTestRun = TestRun("SimpleTheInternet.xlsx",
+    lTestRun = TestRun("examples/SimpleTheInternet.xlsx",
                        globalSettingsFileNameAndPath= \
                            Path(os.getcwd()).joinpath("tests").joinpath("jsons").joinpath("globals_headless2.json"),
                        executeDirect=False)
@@ -36,7 +36,7 @@ def test_with_globalsHeadlessVersion2(testrun_obj):
     assert isinstance(lTestRun.globalSettings["TC.BrowserAttributes"], dict)
 
 def tests_with_fullGlobalsFile(testrun_obj):
-    lTestRun = TestRun("SimpleTheInternet.xlsx",
+    lTestRun = TestRun("examples/SimpleTheInternet.xlsx",
                        globalSettingsFileNameAndPath= \
                            Path(os.getcwd()).joinpath("tests").joinpath("jsons").joinpath("globalsFullExample.json"),
                        executeDirect=False)