Fixes for resilient calls

API_Calls.py

@@ -25,6 +25,7 @@ import math
 import time
 from requests.packages.urllib3.util.retry import Retry
 from requests.adapters import HTTPAdapter
+from requests import Session
 
 
 def f_args_read(l_search_parameter):
@@ -118,7 +119,11 @@ def f_post(url, json=None, headers=None, data=None, files=None):
                 # We'll create an empty session, because we're a new login
                 f_create_session()
                 resp = g_session.post(url, json=json)
-                g_session.headers.update({"X-CSRFToken": resp.json()["key"]})
+                if resp.status_code <= 205:
+                    g_session.headers.update({"X-CSRFToken": resp.json()["key"]})
+                else:
+                    logger.critical("Error when calling server: " + resp.text)
+                    sys.exit("Error when calling server: " + resp.text)
             except requests.exceptions.RequestException as e:
                 logger.error("Connection error raised: " + str(e))
         else:
@@ -194,7 +199,10 @@ class MyMockResponse:
     
 
 def f_check_status(resp):
-    if resp.status_code!=200 and resp.status_code != 201:
+    if not resp:
+        logger.info("There was no output from the Call - empty response")
+        return False
+    if resp.status_code != 200 and resp.status_code != 201:
         if resp.status_code == 400 or resp.status_code == 404:
             logger.debug(resp.status_code)
             logger.debug(resp)
@@ -1230,13 +1238,43 @@ def f_products_import():
     return l_iteration
 
 
+class MyResilientSession(Session):
+    """
+    This class is supposed to retry requests that return temporary errors.
+    At this moment it supports: 502, 503, 504
+    """
+
+    def request(self, method, url, **kwargs):
+        counter = 0
+
+        while counter < 5:
+            counter += 1
+
+            r = super(MyResilientSession, self).request(method, url, **kwargs)
+
+            if r.status_code in [500, 502, 503, 504]:
+                delay = 1 * counter
+                logger.warning("Got recoverable error [%s] from %s %s, retry #%s in %ss" %
+                               (r.status_code, method, url, counter, delay))
+                logger.debug("Error was:" + str(r.content))
+                time.sleep(delay)
+                continue
+            elif r.status_code <= 499:
+                break
+
+        if counter == 5:
+            raise requests.exceptions.RequestException
+
+        return r
+
+
 def f_create_session():
     global g_session
-    g_session = requests.Session()
-    retries = Retry(total=15,
-                    backoff_factor=0.1,
+    g_session = MyResilientSession() # requests.Session()
+    retries = Retry(total=2,
+                    backoff_factor=1,
                     status_forcelist=[500, 502, 503, 504])
-    g_session.mount("https://", HTTPAdapter(max_retries=retries))
+    g_session.mount("https://", HTTPAdapter())     # HTTPAdapter(max_retries=retries))
 
 ##################################################################
 # Here starts the program ########################################

dist/_mysql_recycle.cmd

@@ -1 +1 @@
-api_calls.exe --users=1000 --products=1000 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
+api_calls.exe --users=1000 --products=500 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"

dist/_mysql_short_recycle.cmd

@@ -1 +1 @@
-api_calls.exe --users=100 --products=50 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
+api_calls.exe --users=200 --products=10 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"

dist/_mysql_start_mass.cmd

@@ -1,19 +1,106 @@
-start cmd /c api_calls.exe --users=100 --products=40 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
+start cmd /c api_calls.exe --users=120 --products=40 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
+ping 127.0.0.1 -n 5
+start cmd /c api_calls.exe --users=120 --products=40 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
+ping 127.0.0.1 -n 5
+start cmd /c api_calls.exe --users=120 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
 
-ping 127.0.0.1 -n 6 
-start cmd /c api_calls.exe --users=100 --products=40 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
+ping 127.0.0.1 -n 350
+start cmd /c api_calls.exe --users=120 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
+ping 127.0.0.1 -n 6
+start cmd /c api_calls.exe --users=200 --products=500 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
+
+ping 127.0.0.1 -n 350
+start cmd /c api_calls.exe --users=120 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
+ping 127.0.0.1 -n 6
+start cmd /c api_calls.exe --users=200 --products=500 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
+
+ping 127.0.0.1 -n 350
+start cmd /c api_calls.exe --users=120 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
+ping 127.0.0.1 -n 6
+start cmd /c api_calls.exe --users=200 --products=500 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
+
+ping 127.0.0.1 -n 350
+start cmd /c api_calls.exe --users=120 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
+ping 127.0.0.1 -n 6
+start cmd /c api_calls.exe --users=200 --products=500 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
+
+ping 127.0.0.1 -n 350
+start cmd /c api_calls.exe --users=120 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
+ping 127.0.0.1 -n 6
+start cmd /c api_calls.exe --users=200 --products=500 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
+
+ping 127.0.0.1 -n 350
+start cmd /c api_calls.exe --users=120 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
+ping 127.0.0.1 -n 6
+start cmd /c api_calls.exe --users=200 --products=500 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
+
+ping 127.0.0.1 -n 350
+start cmd /c api_calls.exe --users=120 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
+ping 127.0.0.1 -n 6
+start cmd /c api_calls.exe --users=200 --products=500 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
+
+ping 127.0.0.1 -n 350
+start cmd /c api_calls.exe --users=120 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
+ping 127.0.0.1 -n 6
+start cmd /c api_calls.exe --users=200 --products=500 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
+
+ping 127.0.0.1 -n 350
+start cmd /c api_calls.exe --users=120 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
+ping 127.0.0.1 -n 6
+start cmd /c api_calls.exe --users=200 --products=500 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
+
+ping 127.0.0.1 -n 350
+start cmd /c api_calls.exe --users=120 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
+ping 127.0.0.1 -n 6
+start cmd /c api_calls.exe --users=200 --products=500 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
+
+ping 127.0.0.1 -n 350
+start cmd /c api_calls.exe --users=120 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
+ping 127.0.0.1 -n 6
+start cmd /c api_calls.exe --users=200 --products=500 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
+
+ping 127.0.0.1 -n 350
+start cmd /c api_calls.exe --users=120 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
+ping 127.0.0.1 -n 6
+start cmd /c api_calls.exe --users=200 --products=500 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
+
+ping 127.0.0.1 -n 350
+start cmd /c api_calls.exe --users=120 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
+ping 127.0.0.1 -n 6
+start cmd /c api_calls.exe --users=200 --products=500 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
+
+ping 127.0.0.1 -n 350
+start cmd /c api_calls.exe --users=120 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
+ping 127.0.0.1 -n 6
+start cmd /c api_calls.exe --users=200 --products=500 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
+
+ping 127.0.0.1 -n 350
+start cmd /c api_calls.exe --users=120 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
+ping 127.0.0.1 -n 6
+start cmd /c api_calls.exe --users=200 --products=500 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
+
+ping 127.0.0.1 -n 350
+start cmd /c api_calls.exe --users=120 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
+ping 127.0.0.1 -n 6
+start cmd /c api_calls.exe --users=200 --products=500 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
 
-start cmd /c api_calls.exe --users=100 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
+ping 127.0.0.1 -n 350
+start cmd /c api_calls.exe --users=120 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
 ping 127.0.0.1 -n 6
+start cmd /c api_calls.exe --users=200 --products=500 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
 
-start cmd /c api_calls.exe --users=100 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
+ping 127.0.0.1 -n 350
+start cmd /c api_calls.exe --users=120 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
 ping 127.0.0.1 -n 6
+start cmd /c api_calls.exe --users=200 --products=500 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
 
-start cmd /c api_calls.exe --users=100 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
-ping 127.0.0.1 -n 6  
+ping 127.0.0.1 -n 350
+start cmd /c api_calls.exe --users=120 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
+ping 127.0.0.1 -n 6
+start cmd /c api_calls.exe --users=200 --products=500 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
 
-start cmd /c api_calls.exe --users=100 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
+ping 127.0.0.1 -n 350
+start cmd /c api_calls.exe --users=120 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
 ping 127.0.0.1 -n 6
+start cmd /c api_calls.exe --users=200 --products=500 --server=https://app-test.earthsquad.global/api --mode=recycling --csv="auth_user_testserver.csv"
 
-start cmd /c api_calls.exe --users=100 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
-ping 127.0.0.1 -n 6   

dist/pp_mysql.cmd

@@ -1 +1 @@
-api_calls.exe --users=100 --products=1000 --noimage=yes --server=https://app-test.earthsquad.global/api
+api_calls.exe --users=100 --products=1000 --noimage=yes --server=https://app-pp.earthsquad.global/api

dist/pp_mysql_short.cmd

@@ -1 +1 @@
-api_calls.exe --users=10 --products=100 --noimage=yes --server=https://app-test.earthsquad.global/api
+api_calls.exe --users=10 --products=100 --noimage=yes --server=https://app-pp.earthsquad.global/api

dist/pp_mysql_start_mass.cmd

@@ -1,19 +1,80 @@
-start cmd /c api_calls.exe --users=100 --products=40 --noimage=yes --server=https://app-pp.earthsquad.global/api --mode=recycling --csv="auth_user_pp.csv"
-
-ping 127.0.0.1 -n 60 
-start cmd /c api_calls.exe --users=100 --products=40 --noimage=yes --server=https://app-pp.earthsquad.global/api --mode=recycling --csv="auth_user_pp.csv"
+start cmd /c api_calls.exe --users=100 --products=100 --noimage=yes --server=https://app-pp.earthsquad.global/api --mode=recycling --csv="auth_user_pp.csv"
+ping 127.0.0.1 -n 5
 
+start cmd /c api_calls.exe --users=100 --products=100 --noimage=yes --server=https://app-pp.earthsquad.global/api --mode=recycling --csv="auth_user_pp.csv"
+ping 127.0.0.1 -n 5
 start cmd /c api_calls.exe --users=100 --products=1000 --noimage=yes --server=https://app-pp.earthsquad.global/api
-ping 127.0.0.1 -n 1200
+ping 127.0.0.1 -n 600
 
-start cmd /c api_calls.exe --users=100 --products=1000 --noimage=yes --server=https://app-pp.earthsquad.global/api
-ping 127.0.0.1 -n 1200
+start cmd /c api_calls.exe --users=200 --products=500 --noimage=yes --server=https://app-pp.earthsquad.global/api --mode=recycling --csv="auth_user_pp.csv"
+ping 127.0.0.1 -n 5
+start cmd /c api_calls.exe --users=150 --products=1200 --noimage=yes --server=https://app-pp.earthsquad.global/api
+ping 127.0.0.1 -n 600
 
-start cmd /c api_calls.exe --users=100 --products=1000 --noimage=yes --server=https://app-pp.earthsquad.global/api
-ping 127.0.0.1 -n 1200  
+start cmd /c api_calls.exe --users=200 --products=500 --noimage=yes --server=https://app-pp.earthsquad.global/api --mode=recycling --csv="auth_user_pp.csv"
+ping 127.0.0.1 -n 5
+start cmd /c api_calls.exe --users=150 --products=1200 --noimage=yes --server=https://app-pp.earthsquad.global/api
+ping 127.0.0.1 -n 600
 
-start cmd /c api_calls.exe --users=100 --products=1000 --noimage=yes --server=https://app-pp.earthsquad.global/api
-ping 127.0.0.1 -n 1200
+start cmd /c api_calls.exe --users=200 --products=500 --noimage=yes --server=https://app-pp.earthsquad.global/api --mode=recycling --csv="auth_user_pp.csv"
+ping 127.0.0.1 -n 5
+start cmd /c api_calls.exe --users=150 --products=1200 --noimage=yes --server=https://app-pp.earthsquad.global/api
+ping 127.0.0.1 -n 600
+
+start cmd /c api_calls.exe --users=200 --products=500 --noimage=yes --server=https://app-pp.earthsquad.global/api --mode=recycling --csv="auth_user_pp.csv"
+ping 127.0.0.1 -n 5
+start cmd /c api_calls.exe --users=150 --products=1200 --noimage=yes --server=https://app-pp.earthsquad.global/api
+ping 127.0.0.1 -n 600
+
+start cmd /c api_calls.exe --users=200 --products=500 --noimage=yes --server=https://app-pp.earthsquad.global/api --mode=recycling --csv="auth_user_pp.csv"
+ping 127.0.0.1 -n 5
+start cmd /c api_calls.exe --users=150 --products=1200 --noimage=yes --server=https://app-pp.earthsquad.global/api
+ping 127.0.0.1 -n 600
+
+start cmd /c api_calls.exe --users=200 --products=500 --noimage=yes --server=https://app-pp.earthsquad.global/api --mode=recycling --csv="auth_user_pp.csv"
+ping 127.0.0.1 -n 5
+start cmd /c api_calls.exe --users=150 --products=1200 --noimage=yes --server=https://app-pp.earthsquad.global/api
+ping 127.0.0.1 -n 600
+
+
+start cmd /c api_calls.exe --users=200 --products=500 --noimage=yes --server=https://app-pp.earthsquad.global/api --mode=recycling --csv="auth_user_pp.csv"
+ping 127.0.0.1 -n 5
+start cmd /c api_calls.exe --users=150 --products=1200 --noimage=yes --server=https://app-pp.earthsquad.global/api
+ping 127.0.0.1 -n 600
+
+
+start cmd /c api_calls.exe --users=200 --products=500 --noimage=yes --server=https://app-pp.earthsquad.global/api --mode=recycling --csv="auth_user_pp.csv"
+ping 127.0.0.1 -n 5
+start cmd /c api_calls.exe --users=150 --products=1200 --noimage=yes --server=https://app-pp.earthsquad.global/api
+ping 127.0.0.1 -n 600
+
+
+start cmd /c api_calls.exe --users=200 --products=500 --noimage=yes --server=https://app-pp.earthsquad.global/api --mode=recycling --csv="auth_user_pp.csv"
+ping 127.0.0.1 -n 5
+start cmd /c api_calls.exe --users=150 --products=1200 --noimage=yes --server=https://app-pp.earthsquad.global/api
+ping 127.0.0.1 -n 600
+
+
+start cmd /c api_calls.exe --users=200 --products=500 --noimage=yes --server=https://app-pp.earthsquad.global/api --mode=recycling --csv="auth_user_pp.csv"
+ping 127.0.0.1 -n 5
+start cmd /c api_calls.exe --users=150 --products=1200 --noimage=yes --server=https://app-pp.earthsquad.global/api
+ping 127.0.0.1 -n 600
+
+
+start cmd /c api_calls.exe --users=200 --products=500 --noimage=yes --server=https://app-pp.earthsquad.global/api --mode=recycling --csv="auth_user_pp.csv"
+ping 127.0.0.1 -n 5
+start cmd /c api_calls.exe --users=150 --products=1200 --noimage=yes --server=https://app-pp.earthsquad.global/api
+ping 127.0.0.1 -n 600
+
+
+start cmd /c api_calls.exe --users=200 --products=500 --noimage=yes --server=https://app-pp.earthsquad.global/api --mode=recycling --csv="auth_user_pp.csv"
+ping 127.0.0.1 -n 5
+start cmd /c api_calls.exe --users=150 --products=1200 --noimage=yes --server=https://app-pp.earthsquad.global/api
+ping 127.0.0.1 -n 600
+
+
+start cmd /c api_calls.exe --users=200 --products=500 --noimage=yes --server=https://app-pp.earthsquad.global/api --mode=recycling --csv="auth_user_pp.csv"
+ping 127.0.0.1 -n 5
+start cmd /c api_calls.exe --users=150 --products=1200 --noimage=yes --server=https://app-pp.earthsquad.global/api
+ping 127.0.0.1 -n 600
 
-start cmd /c api_calls.exe --users=100 --products=1000 --noimage=yes --server=https://app-pp.earthsquad.global/api
-ping 127.0.0.1 -n 1200