baangt-statistics-service/StatisticServer/routes.py

from StatisticServer import app, db
from flask import request, jsonify, abort, make_response, Response
from StatisticServer.models import TestRunStatistics, ActivityStatistics
from sqlalchemy import exc
from threading import Thread
import json


def update_activities(activity_headers):
    try:
        activity_stats = db.session.query(ActivityStatistics).all()
    except exc.OperationalError as ex:
        if "no such table:" in str(ex):
            db.session.rollback()
            db.create_all()
            activity_stats = db.session.query(ActivityStatistics).all()
    for activity_stat in activity_stats:
        activity_json = activity_stat.to_json()
        for key in activity_json:
            if key != "TestRunUUID":
                if key not in activity_headers:
                    activity_headers.append(key)

testrun_headers = [
    'TestRunUUID', 'Browser', 'api', 'soap', 'web', 'TestCaseExecuted', 'TestCasePassed', 'TestCaseFailed',
    'TestCasePaused', 'SequenceClass', 'TestCaseClass', 'TestStepMaster', 'TestStepSequences', 'TestSteps',
    'TestCaseSequences', 'XPATH', 'CSS', 'ID', 'equalTo', 'notEqualTo', 'smallerThan', 'greaterThan', 'timeout',
    'optional', 'release', 'Duration', 'ip', 'sendingTime'
    ]
activity_headers = [] # To keep records of every new activity field so we don't have to manage it while creating csv
t = Thread(target=update_activities, args=[activity_headers,])
t.daemon = True
t.start()


@app.route('/', methods=['POST'])
def main():
    try:
        dic = request.json
        if "TestRunUUID" not in dic:
            abort(400)
        if dic["TestRunUUID"] == "PYTEST":
            return make_response(jsonify(dic), 200)
        if request.environ.get('HTTP_X_FORWARDED_FOR') is None:
            ip = request.environ['REMOTE_ADDR']
        else:
            ip = request.environ['HTTP_X_FORWARDED_FOR']
        process_data(dic, ip)
    except Exception as ex:
        print(ex)
        return abort(400)
    return jsonify(success=True)


@app.route('/stats', methods=['GET'])
def stats_download(): # Will convert database data in csv and response csv file to request
    global activity_headers, testrun_headers
    test_stats = db.session.query(TestRunStatistics).all()
    def generate_data():
        for _, test_stat in enumerate(test_stats):
            data = []
            activities = {}
            # Iter through every data in TestRunStatistics the get the data for same UUID from ActivityStatistics
            activity_stats = db.session.query(ActivityStatistics
                                              ).filter(ActivityStatistics.TestRunUUID == test_stat.TestRunUUID).all()
            test_json = test_stat.to_json()
            for key in testrun_headers:
                data.append(f'"{str(test_json[key])}"')
            for activity_stat in activity_stats:
                activity_json = activity_stat.to_json()
                for key in activity_json:
                    if key == "TestRunUUID":
                        continue
                    activities[key] = activity_json[key]
            for key in activity_headers:
                if key not in activities:
                    data.append(str(0))
                else:
                    data.append(f'"{str(activities[key])}"')
            if _ == 0:
                headers = []
                for key in testrun_headers:
                    headers.append(key)
                for key in activity_headers:
                    headers.append(key)
                yield ','.join(headers)+'\n'
            yield ','.join(data)+'\n'
    resp = Response(
                    generate_data(),
                    mimetype="text/csv",
                    headers={"Content-disposition":
                                 "attachment; filename=statistics_database.csv"})
    return resp


def process_data(data, ip):
    global activity_headers
    dict = {}
    for key in data:
        if "Activity_" in key:
            AS = ActivityStatistics()
            Activity_dict = {}
            Activity_dict["TestRunUUID"] = data["TestRunUUID"]
            Activity_dict["Key"] = key
            Activity_dict["Value"] = data[key]
            AS.update(Activity_dict)
            db.session.add(AS)
            if key not in activity_headers:
                activity_headers.append(key)
        elif "TestStepMaster" == key:
            dict[key] = json.dumps(data[key])
        else:
            dict[key] = data[key]
    dict["ip"] = ip
    TRS = TestRunStatistics()
    TRS.update(dict)
    db.session.add(TRS)
    try:
        db.session.commit()
    except exc.OperationalError as ex:
        if "no such table:" in str(ex):
            db.session.rollback()
            db.create_all()
            process_data(data, ip)