Coverage for app/utils/xlsx_handler.py : 94%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import os
2import re
3import xlrd
4#from app.utils.testrun_tree import TestrunTree, TestrunNode
5from app import app, models, db
6from app.utils import idAsBytes
7from app.utils.datafile_handler import uploadDataFile
8from flask_login import current_user
9from sqlalchemy import func
10from datetime import datetime
13#
14# XLSX definition
15#
16class TestrunXLSXDefinition:
17 #
18 # defines required sheets and fields
19 #
21 COMPLEX_HEADERS = {
22 'TestCaseSequence': [
23 None,
24 'Number',
25 'SequenceClass',
26 'TestDataFileName',
27 'Sheetname',
28 ],
29 'TestCase': [
30 'TestCaseSequenceNumber',
31 'TestCaseNumber',
32 'TestCaseClass',
33 'TestCaseType',
34 'Browser',
35 ],
36 'TestStep': [
37 'TestCaseNumber',
38 'TestStepNumber',
39 'TestStepClass',
40 ],
41 'TestStepExecution': [
42 'TestStepNumber',
43 'TestStepExecutionNumber',
44 'Activity',
45 'LocatorType',
46 'Locator',
47 'Value',
48 'Comparison',
49 'Value2',
50 'Timeout',
51 'Optional',
52 'Release',
53 ],
54 }
56 SIMPLE_HEADERS = {
57 'TestStepExecution': [
58 'Activity',
59 'LocatorType',
60 'Locator',
61 'Value',
62 'Comparison',
63 'Value2',
64 'Timeout',
65 'Optional',
66 'Release',
67 ],
68 }
71class TestrunXLSXParser:
72 #
73 # parses XLSX definition to TestrunNodes
74 #
76 COMPLEX_SHEETS = [key for key in TestrunXLSXDefinition.COMPLEX_HEADERS]
77 SIMPLE_SHEET = next(iter(TestrunXLSXDefinition.SIMPLE_HEADERS))
79 def __init__(self, content):
80 # filename
81 self.filename = os.path.basename(content.filename)
82 # get workbook
83 self.workbook = xlrd.open_workbook(file_contents=content.read())
84 # check definition format
85 self.is_simple = not all(sheet in self.workbook.sheet_names() for sheet in self.COMPLEX_SHEETS)
86 self.assert_workbook()
88 def assert_workbook(self):
89 if self.is_simple:
90 if not self.SIMPLE_SHEET in self.workbook.sheet_names():
91 raise Exception(f'Invalid definition format: sheet "{self.SIMPLE_SHEET}" is absent')
92 if not all(field in self.workbook.sheet_by_name(self.SIMPLE_SHEET).row_values(0) for field in TestrunXLSXDefinition.SIMPLE_HEADERS[self.SIMPLE_SHEET]):
93 raise Exception(
94 f'Invalid definition format: sheet "{self.SIMPLE_SHEET}" defines not all required fields {TestrunXLSXDefinition.SIMPLE_HEADERS[self.SIMPLE_SHEET]}'
95 )
96 else:
97 for sheet, fields in TestrunXLSXDefinition.COMPLEX_HEADERS.items():
98 if not all(field in self.workbook.sheet_by_name(sheet).row_values(0) for field in fields if field):
99 raise Exception(f'Invalid definition format: sheet "{sheet}" defines not all required fields {fields}')
101 def simple_nodes(self):
102 #
103 # generator
104 # parses simple definition file
105 # yields TestStepExecution nodes
106 #
108 # get TestStepExecution sheet
109 sheet = self.workbook.sheet_by_name(self.SIMPLE_SHEET)
111 # parse TestSteps
112 sheet_headers = {h: i for i, h in enumerate(sheet.row_values(0))}
113 for row in range(1, sheet.nrows):
114 # create new TestStep Node
115 teststep = TestrunNode(TestrunNode.TS, row)
116 # set attributes
117 for field in TestrunXLSXDefinition.SIMPLE_HEADERS[self.SIMPLE_SHEET]:
118 teststep.set_attr(field, sheet.cell(row, sheet_headers[field]).value or None)
120 yield teststep
122 def simple_datafile(self):
123 #
124 # returns tuple
125 # (datafile name, data sheet)
126 # for simple definition
127 #
129 if 'data' in self.workbook.sheet_names():
130 return self.filename, 'data'
133 def complex_nodes(self):
134 #
135 # generator
136 # parses complex definition file
137 # yields tuple of 2 elements:
138 # [0]: parent id
139 # [1]: child node
140 #
142 # parse sheets
143 for sheet_name, headers in TestrunXLSXDefinition.COMPLEX_HEADERS.items():
144 # get sheet
145 sheet = self.workbook.sheet_by_name(sheet_name)
146 # parse sheet
147 sheet_headers = {h: i for i, h in enumerate(sheet.row_values(0))}
148 layer = TestrunNode.label_to_layer(sheet_name)
149 for row in range(1, sheet.nrows):
150 # create new child
151 child = TestrunNode(layer, int(sheet.cell(row, sheet_headers[headers[1]]).value))
152 # set child's attributes
153 for field in headers[2:]:
154 child.set_attr(field, sheet.cell(row, sheet_headers[field]).value or None)
155 # get parent id
156 parent_id = int(sheet.cell(row, sheet_headers[headers[0]]).value) if headers[0] else 1
158 yield parent_id, child
161#
162# Testrun Tree
163#
164class TestrunNode:
165 # layers
166 TR = 0
167 TCS = 1 # TestCaseSequence
168 TC = 2 # TestCase
169 TSS = 3 # TestStepSequence
170 TS = 4 # TestStep
172 # node labels
173 LABELS = {'Testrun': 0, **{key: index+1 for index, key in enumerate(TestrunXLSXDefinition.COMPLEX_HEADERS.keys())} }
175 def __init__(self, layer, node_id):
176 self.layer = layer
177 self.id = node_id
178 self.attrs = {}
179 self.children = []
181 def __str__(self):
182 return f'{self.label}-{self.id}: {self.attrs}'
183 #return f'{self.layer}-{self.id}'
185 @classmethod
186 def label_to_layer(cls, label):
187 return cls.LABELS.get(label)
189 @property
190 def label(self):
191 return list(self.LABELS.keys())[self.layer]
194 def pprint(self, indent='|'):
195 print(f'{indent}{self}')
196 for child in self.children:
197 child.pprint(indent=f'{indent}-')
199 def add_child(self, child):
200 #
201 # inserts child node according to its id
202 #
203 self.children.append(child)
204 self.children.sort(key=lambda child: child.id)
206 def set_datafile(self, data):
207 #
208 # sets datafile and sheet attributes of TCS node
209 #
210 if self.layer != self.TCS:
211 return False
213 self.set_attr(TestrunXLSXDefinition.COMPLEX_HEADERS['TestCaseSequence'][3], data[0])
214 self.set_attr(TestrunXLSXDefinition.COMPLEX_HEADERS['TestCaseSequence'][4], data[1])
215 return True
217 def set_attr(self, key, value):
218 #
219 # sets attribute to {key: value}
220 #
221 self.attrs[key] = value
223 def get_attr(self, key):
224 #
225 # returns key atrribute
226 #
227 return self.attrs.get(key)
229 def get_node(self, layer, node_id):
230 if layer < self.layer:
231 return None
232 if layer == self.layer:
233 if self.id == node_id:
234 return self
235 return None
236 if layer > self.layer:
237 for child in self.children:
238 #print(child.layer)
239 node = child.get_node(layer, node_id)
240 if node:
241 return node
243class TestrunTree:
244 #
245 # Default attributes
246 #
247 CLASSNAME_TESTCASESEQUENCE = 'GC.CLASSES_TESTCASESEQUENCE'
248 CLASSNAME_TESTCASE = 'GC.CLASSES_TESTCASE'
249 CLASSNAME_TESTSTEP = 'GC.CLASSES_TESTSTEPMASTER'
250 BROWSER_TYPE = 'FF'
251 TESTCASE_TYPE = 'Browser'
252 ACTIVITY_TYPE = 'GOTOURL'
254 def __init__(self, simple=False):
255 if simple:
256 # create default tree
257 child_node = None
258 for layer in reversed(range(TestrunNode.TS)):
259 node = TestrunNode(layer, 1)
260 if child_node:
261 node.add_child(child_node)
262 child_node = node
263 self.root = node
264 self.set_defaults()
265 else:
266 self.root = TestrunNode(TestrunNode.TR, 1)
269 def set_defaults(self):
270 # TestCaseSequence
271 node = self.root.children[0]
272 node.set_attr(TestrunXLSXDefinition.COMPLEX_HEADERS['TestCaseSequence'][2], self.CLASSNAME_TESTCASESEQUENCE)
273 # TestCase
274 node = node.children[0]
275 node.set_attr(TestrunXLSXDefinition.COMPLEX_HEADERS['TestCase'][2], self.CLASSNAME_TESTCASE)
276 node.set_attr(TestrunXLSXDefinition.COMPLEX_HEADERS['TestCase'][3], self.TESTCASE_TYPE)
277 node.set_attr(TestrunXLSXDefinition.COMPLEX_HEADERS['TestCase'][4], self.BROWSER_TYPE)
278 # TestStepSequence
279 node = node.children[0]
280 node.set_attr(TestrunXLSXDefinition.COMPLEX_HEADERS['TestStep'][2], self.CLASSNAME_TESTSTEP)
282 def get_node(self, layer, node_id):
283 return self.root.get_node(layer, node_id)
285 def pprint(self):
286 self.root.pprint()
288 @classmethod
289 def from_xlsx(cls, content):
290 parser = TestrunXLSXParser(content)
291 if parser.is_simple:
292 tree = cls(simple=True)
293 # set datafile
294 tcs = tree.get_node(TestrunNode.TCS, 1)
295 tcs.set_datafile(parser.simple_datafile())
296 # set steps
297 tss = tree.get_node(TestrunNode.TSS, 1)
298 for node in parser.simple_nodes():
299 tss.add_child(node)
300 else:
301 tree = cls()
302 for parent_id, child in parser.complex_nodes():
303 parent = tree.get_node(child.layer-1, parent_id)
304 parent.add_child(child)
306 return tree
309#
310# import Testrun from XLSX definition
311#
312def importXLSX(xlsx_content, datafiles=None, testrun_id=None):
313 #
314 # imports testrun from xlsx file
315 #
317 # get Testrun name
318 testrun_name = os.path.basename(xlsx_content.filename)
320 # check for import option: create or update
321 if testrun_id:
322 # get item
323 testrun = models.Testrun.query.get(idAsBytes(testrun_id))
324 if testrun is None:
325 # item does not exist
326 raise Exception(f'Testrun {testrun_id} does not exists.')
327 else:
328 # check if imported Testrun already exists
329 testrun = models.Testrun.query.filter_by(name=testrun_name).first()
330 if testrun:
331 testrun_id = testrun.uuid
333 # parse xlsx
334 testrun_tree = TestrunTree.from_xlsx(xlsx_content)
335 app.logger.info(f'Testrun successfully read from {xlsx_content.filename} by {current_user}.')
337 if testrun:
338 # update mode
339 app.logger.info(f'Updating Testrun {testrun.uuid} from {xlsx_content.filename} by {current_user}.')
341 testrun.description = f'Updated from "{testrun_name}"'
342 testrun.editor = current_user
343 testrun.edited = datetime.utcnow()
344 else:
345 # create mode
346 app.logger.info(f'Importing Testrun from {xlsx_content.filename} by {current_user}.')
348 # create Testrun
349 testrun = models.Testrun(
350 name=testrun_name,
351 description=f'Imported from "{testrun_name}"',
352 creator=current_user,
353 )
354 db.session.add(testrun)
356 # TestCaseSequences
357 for tcs_index, tcs in enumerate(testrun_tree.root.children):
358 # ClassName
359 field_name = TestrunXLSXDefinition.COMPLEX_HEADERS['TestCaseSequence'][2]
360 classname = getOrCreateClassNameByName(tcs.get_attr(field_name), f'Imported from "{testrun_name}"')
361 # DataFile
362 datafile_name = tcs.get_attr(TestrunXLSXDefinition.COMPLEX_HEADERS['TestCaseSequence'][3])
363 datafile_sheet = tcs.get_attr(TestrunXLSXDefinition.COMPLEX_HEADERS['TestCaseSequence'][4])
364 # get datafile
365 try:
366 datafile = testrun.testcase_sequences[tcs_index].datafiles[0]
367 except Exception:
368 datafile = None
369 # upload datafile
370 datafile_id = None
371 if datafile_name == testrun_name:
372 xlsx_content.seek(0)
373 datafile_id = uploadDataFile(xlsx_content, datafile.uuid if datafile else None)
374 else:
375 try:
376 new_datafile = next(filter(lambda df: df.data and os.path.basename(df.data.filename) == datafile_name, datafiles)).data
377 except Exception:
378 new_datafile = None
379 if new_datafile:
380 datafile_id = uploadDataFile(new_datafile, datafile.uuid if datafile else None)
381 if datafile_id and datafile:
382 datafile.filename = datafile_name
383 datafile.sheet = datafile_sheet
384 elif datafile_id:
385 datafile = models.DataFile(
386 id=idAsBytes(datafile_id),
387 filename=datafile_name,
388 sheet=datafile_sheet,
389 creator=current_user,
390 )
391 db.session.add(datafile)
393 if tcs_index < len(testrun.testcase_sequences):
394 # update mode
395 testcase_sequence = testrun.testcase_sequences[tcs_index]
396 testcase_sequence.description = f'Updated from "{testrun_name}"'
397 testcase_sequence.editor = current_user
398 testcase_sequence.edited = datetime.utcnow()
399 testcase_sequence.classname = classname
400 testcase_sequence.datafiles = [datafile] if datafile else []
401 else:
402 # create mode
403 testcase_sequence = models.TestCaseSequence(
404 name=f'{testrun_name}_{tcs.id}',
405 description=f'Imported from "{testrun_name}"',
406 creator=current_user,
407 classname=classname,
408 datafiles=[datafile] if datafile else [],
409 testrun=[testrun],
410 )
411 db.session.add(testcase_sequence)
413 # TestCases
414 for tc_index, tc in enumerate(tcs.children):
415 # ClassName
416 field_name = TestrunXLSXDefinition.COMPLEX_HEADERS['TestCase'][2]
417 classname = getOrCreateClassNameByName(tc.get_attr(field_name), f'Imported from "{testrun_name}"')
418 # TestCaseType
419 field_name = TestrunXLSXDefinition.COMPLEX_HEADERS['TestCase'][3]
420 testcase_type=getTestCaseTypeByName(tc.get_attr(field_name))
421 # BrowserType
422 field_name = TestrunXLSXDefinition.COMPLEX_HEADERS['TestCase'][4]
423 browser_type = getBrowserTypeByName(tc.get_attr(field_name))
424 if tc_index < len(testcase_sequence.testcases):
425 # update mode
426 testcase = testcase_sequence.testcases[tc_index]
427 testcase.description = f'Updated from "{testrun_name}"'
428 testcase.editor = current_user
429 testcase.edited = datetime.utcnow()
430 testcase.classname = classname
431 testcase.testcase_type = testcase_type
432 testcase.browser_type = browser_type
433 else:
434 # create mode
435 testcase = models.TestCase(
436 name=f'{testrun_name}_{tc.id}',
437 description=f'Imported from "{testrun_name}"',
438 creator=current_user,
439 classname=classname,
440 browser_type=browser_type,
441 testcase_type=testcase_type,
442 testcase_sequence=[testcase_sequence],
443 )
444 db.session.add(testcase)
446 # TestStepSequences
447 for tss_index, tss in enumerate(tc.children):
448 # ClassName
449 field_name = TestrunXLSXDefinition.COMPLEX_HEADERS['TestStep'][2]
450 classname = getOrCreateClassNameByName(tss.get_attr(field_name), f'Imported from "{testrun_name}"')
451 if tss_index < len(testcase.teststep_sequences):
452 # update mode
453 teststep_sequence = testcase.teststep_sequences[tss_index]
454 teststep_sequence.description = f'Updated from "{testrun_name}"'
455 teststep_sequence.editor = current_user
456 teststep_sequence.edited = datetime.utcnow()
457 teststep_sequence.classname = classname
458 else:
459 # create mode
460 teststep_sequence = models.TestStepSequence(
461 name=f'{testrun_name}_{tss.id}',
462 description=f'Imported from "{testrun_name}"',
463 creator=current_user,
464 classname=classname,
465 testcase=[testcase],
466 )
467 db.session.add(teststep_sequence)
469 # TestStep
470 for ts_index, ts in enumerate(tss.children):
471 # ClassName
472 field_name = TestrunXLSXDefinition.COMPLEX_HEADERS['TestStepExecution'][2]
473 activity_type = getActivityTypeByName(ts.get_attr(field_name))
474 # TestCaseType
475 field_name = TestrunXLSXDefinition.COMPLEX_HEADERS['TestStepExecution'][3]
476 locator_type=getLocatorTypeByName(ts.get_attr(field_name))
477 # value
478 int_parser = lambda value: int(value) if re.match(r'\d+\.0+$', str(value)) else value
479 field_name = TestrunXLSXDefinition.COMPLEX_HEADERS['TestStepExecution'][5]
480 value = int_parser(ts.get_attr(field_name))
481 # value2
482 field_name = TestrunXLSXDefinition.COMPLEX_HEADERS['TestStepExecution'][7]
483 value2 = int_parser(ts.get_attr(field_name))
484 # optional
485 field_name = TestrunXLSXDefinition.COMPLEX_HEADERS['TestStepExecution'][9]
486 optional=bool(ts.get_attr(field_name))
487 if ts_index < len(teststep_sequence.teststeps):
488 # update mode
489 teststep = teststep_sequence.teststeps[ts_index]
490 teststep.description = f'Updated from "{testrun_name}"'
491 teststep.editor = current_user
492 teststep.edited = datetime.utcnow()
493 teststep.activity_type = activity_type
494 teststep.locator_type = locator_type
495 teststep.locator = ts.get_attr(TestrunXLSXDefinition.COMPLEX_HEADERS['TestStepExecution'][4])
496 teststep.value = value
497 teststep.comparison = ts.get_attr(TestrunXLSXDefinition.COMPLEX_HEADERS['TestStepExecution'][6])
498 teststep.value2 = value2
499 teststep.timeout = ts.get_attr(TestrunXLSXDefinition.COMPLEX_HEADERS['TestStepExecution'][8])
500 teststep.optional = optional
501 teststep.release = ts.get_attr(TestrunXLSXDefinition.COMPLEX_HEADERS['TestStepExecution'][10])
502 else:
503 # create mode
504 teststep = models.TestStepExecution(
505 name=f'{testrun_name}_{ts.id}',
506 description=f'Imported from "{testrun_name}"',
507 creator=current_user,
508 teststep_sequence=teststep_sequence,
509 activity_type=activity_type,
510 locator_type=locator_type,
511 locator=ts.get_attr(TestrunXLSXDefinition.COMPLEX_HEADERS['TestStepExecution'][4]),
512 value=value,
513 comparison=ts.get_attr(TestrunXLSXDefinition.COMPLEX_HEADERS['TestStepExecution'][6]),
514 value2=value2,
515 timeout=ts.get_attr(TestrunXLSXDefinition.COMPLEX_HEADERS['TestStepExecution'][8]),
516 optional=optional,
517 release=ts.get_attr(TestrunXLSXDefinition.COMPLEX_HEADERS['TestStepExecution'][10]),
518 )
519 db.session.add(teststep)
520 # remove old extra items
521 teststep_sequence.teststeps = teststep_sequence.teststeps[:len(tss.children)]
522 testcase.teststep_sequences = testcase.teststep_sequences[:len(tc.children)]
523 testcase_sequence.testcases = testcase_sequence.testcases[:len(tcs.children)]
524 testrun.testcase_sequences = testrun.testcase_sequences[:len(testrun_tree.root.children)]
526 db.session.commit()
529#
530# support instances getters
531#
532def getOrCreateClassNameByName(name, description):
533 # get ClassName from DB
534 classname = models.ClassName.query.filter_by(name=name).first()
535 if classname is None:
536 # create ClassName if it doesn't exist
537 classname = models.ClassName(
538 name=name,
539 description=description,
540 )
541 db.session.add(classname)
542 db.session.commit()
543 app.logger.info(f'Created ClassName ID #{classname.id} by {current_user}.')
545 return classname
548def getBrowserTypeByName(name):
549 # browser mapper
550 bm = {
551 'BROWSER_FIREFOX': "FF",
552 'BROWSER_CHROME': "Chrome",
553 'BROWSER_SAFARI': "Safari",
554 'BROWSER_EDGE': "Edge",
555 }
556 if bm.get(name.split('.')[-1]):
557 return models.BrowserType.query.filter_by(name=bm[name.split('.')[-1]]).first()
558 return models.BrowserType.query.filter_by(name=name).first()
560def getTestCaseTypeByName(name):
561 return models.TestCaseType.query.filter_by(name=name).first()
563def getActivityTypeByName(name):
564 return models.ActivityType.query.filter(func.upper(models.ActivityType.name) == name.upper()).first()
566def getLocatorTypeByName(name):
567 if name:
568 #return models.LocatorType.query.filter(func.upper(models.LocatorType.name) == name.upper()).first()
569 return models.LocatorType.query.filter_by(name=name).first()
570 else:
571 return None