improve tests

tests-ng/tests-launcher.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# author: deadc0de6 (https://github.com/deadc0de6)
+# Copyright (c) 2020, deadc0de6
+#
+# tests launcher
+#
+
+
+import os
+import sys
+import subprocess
+from concurrent import futures
+
+
+MAX_JOBS = 10
+
+
+def run_test(path):
+    cur = os.path.dirname(sys.argv[0])
+    name = os.path.basename(path)
+    path = os.path.join(cur, name)
+
+    p = subprocess.Popen(path, shell=False,
+                         stdout=subprocess.PIPE,
+                         stderr=subprocess.STDOUT)
+    out, _ = p.communicate()
+    out = out.decode()
+    r = p.returncode == 0
+    reason = 'returncode'
+    if 'Traceback' in out:
+        r = False
+        reason = 'traceback'
+    return r, reason, path, out
+
+
+def get_tests():
+    tests = []
+    cur = os.path.dirname(sys.argv[0])
+    for (_, _, filenames) in os.walk(cur):
+        for path in filenames:
+            if not path.endswith('.sh'):
+                continue
+            tests.append(path)
+        break
+    return tests
+
+
+def main():
+    global MAX_JOBS
+    if len(sys.argv) > 1:
+        MAX_JOBS = int(sys.argv[1])
+
+    tests = get_tests()
+
+    with futures.ThreadPoolExecutor(max_workers=MAX_JOBS) as ex:
+        wait_for = []
+        for test in tests:
+            j = ex.submit(run_test, test)
+            wait_for.append(j)
+
+        for f in futures.as_completed(wait_for):
+            r, reason, p, log = f.result()
+            if not r:
+                ex.shutdown(wait=False)
+                for x in wait_for:
+                    x.cancel()
+                print()
+                print(log)
+                print('test {} failed ({})'.format(p, reason))
+                return False
+            else:
+                sys.stdout.write('.')
+                sys.stdout.flush()
+        sys.stdout.write('\n')
+    return True
+
+
+if __name__ == '__main__':
+    if not main():
+        sys.exit(1)
+    sys.exit(0)

tests.sh

@@ -31,16 +31,15 @@ export DOTDROP_FORCE_NODEBUG=yes
 
 # coverage file location
 cur=`dirname $(readlink -f "${0}")`
-export COVERAGE_FILE="${cur}/.coverage"
 
 # execute tests with coverage
 if [ -z ${GITHUB_WORKFLOW} ]; then
   ## local
-  #PYTHONPATH="dotdrop" ${nosebin} --processes=0 --with-coverage --cover-package=dotdrop
+  export COVERAGE_FILE=
   PYTHONPATH="dotdrop" ${nosebin} -s --processes=-1 --with-coverage --cover-package=dotdrop
 else
   ## CI/CD
-  #PYTHONPATH="dotdrop" ${nosebin} --processes=-1 --with-coverage --cover-package=dotdrop
+  export COVERAGE_FILE="${cur}/.coverage"
   PYTHONPATH="dotdrop" ${nosebin} --processes=0 --with-coverage --cover-package=dotdrop
 fi
 #PYTHONPATH="dotdrop" python3 -m pytest tests
@@ -52,34 +51,16 @@ unset DOTDROP_FORCE_NODEBUG
 #export DOTDROP_FORCE_NODEBUG=yes
 export DOTDROP_WORKDIR=/tmp/dotdrop-tests-workdir
 
-## execute bash script tests
+# run bash tests
-[ "$1" = '--python-only' ] || {
+if [ -z ${GITHUB_WORKFLOW} ]; then
-  echo "doing extended tests"
+  ## local
-  logdir=`mktemp -d`
+  export COVERAGE_FILE=
-  tot=`ls -1 tests-ng/*.sh | wc -l`
+  tests-ng/tests-launcher.py
-  cnt=0
+else
-  for scr in tests-ng/*.sh; do
+  ## CI/CD
-    cnt=$((cnt + 1))
+  export COVERAGE_FILE="${cur}/.coverage"
-    logfile="${logdir}/`basename ${scr}`.log"
+  tests-ng/tests-launcher.py 1
-    echo "-> (${cnt}/${tot}) running test ${scr} (logfile:${logfile})"
+fi
-    set +e
-    ${scr} > "${logfile}" 2>&1
-    if [ "$?" -ne 0 ]; then
-      cat ${logfile}
-      echo "test ${scr} finished with error"
-      rm -rf ${logdir}
-      exit 1
-    elif grep Traceback ${logfile}; then
-      cat ${logfile}
-      echo "test ${scr} crashed"
-      rm -rf ${logdir}
-      exit 1
-    fi
-    set -e
-    echo "test ${scr} ok"
-  done
-  rm -rf ${logdir}
-}
 
 ## test the doc with remark
 ## https://github.com/remarkjs/remark-validate-links