run_tests

#!/bin/bash

# Copyright (c) 2008-2021 the MRtrix3 contributors.
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Covered Software is provided under this License on an "as is"
# basis, without warranty of any kind, either expressed, implied, or
# statutory, including, without limitation, warranties that the
# Covered Software is free of defects, merchantable, fit for a
# particular purpose or non-infringing.
# See the Mozilla Public License v. 2.0 for more details.
#
# For more details, see http://www.mrtrix.org/.



# when testing a binary release, you can use the MRTRIX_BINDIR environment
# variable to point to the location of the executables you want to test. You'll
# need to ensure the same commit/tag is checked out on this repo for the
# results to be valid.


if test "$#" -ne 1; then
  echo "Script must be provided with single input argument: \"binaries\", \"scripts\", \"units\", or the name of a specific test to be run."
  exit 1
fi


if [ -d testing/data/ ]; then
  echo -n "Deleting old testing data directory testing/data/"
  rm -rf testing/data/
  echo OK
fi


datadir=""
testsdir=""
testlist=""
teststring=""
case $1 in
  binaries)
    datadir="testing/binaries/data/"
    testsdir="testing/binaries/tests/"
    teststring="binaries"
    ;;
  scripts)
    datadir="testing/scripts/data/"
    testsdir="testing/scripts/tests/"
    teststring="scripts"
    ;;
  units)
    testsdir="testing/tests/"
    teststring="unit tests"
    ;;
  *)
    for item in testing/binaries/tests/* testing/scripts/tests/* testing/tests/*; do
      if [[ $(basename $item) == $1 ]]; then
        testlist=$item
        if [[ $item == testing/binaries/tests/* ]]; then
          datadir="testing/binaries/data/"
          teststring="binary \""$1"\""
        elif [[ $item == testing/scripts/tests/* ]]; then
          datadir="testing/scripts/data/"
          teststring="script \""$1"\""
        else
          teststring="unit test \""$1"\""
        fi
        break
      fi
    done
    if [ -z $testlist ]; then
      echo "Unable to find requested test \""$1"\""
      exit 1
    fi
    ;;
esac


LOGFILE="testing_"$1".log"
echo logging to \""$LOGFILE"\"
cat > $LOGFILE <<EOD
------------------------------------------
  Testing MRtrix3 $teststring
------------------------------------------

EOD


if [ ! -z $datadir ]; then
  echo -n "checking for requisite test data... "
  git submodule update --init $datadir >> $LOGFILE 2>&1
  if [ $? != 0 ]; then
    if [[ -e $datadir && ! -z $(ls -A $datadir) ]]; then
      echo "Found; UNABLE TO VERIFY VERSION MATCH"
    else
      echo "ERROR!"
      exit 1
    fi
  else
    echo "Synchronized OK"
  fi
fi


if [ ! -z $testsdir ]; then
  echo -n "building list of tests... "
  for item in $testsdir/*; do
    testlist="$testlist $(basename $item)"
  done
  if [ -z "$testlist" ]; then
    echo ERROR!
    exit 1
  fi
  echo OK
  cat >> $LOGFILE <<EOD
## Tests: ${testlist}

EOD
fi

echo -n "building testing commands... "
cat >> $LOGFILE <<EOD

-------------------------------------------

## building test commands...

EOD
(
  cd testing
  if [ -z "$PYTHON" ]; then
    ../build
  else
    $PYTHON ../build
  fi
) >> $LOGFILE 2>&1
if [ $? != 0 ]; then
  echo ERROR!
  exit 1
else
  echo OK
fi


if [[ $datadir == "testing/scripts/data/" && -d "testing/scripts/tmp/" ]]; then
  echo -n "Erasing old script test output directory... "
  rm -rf testing/scripts/tmp/
  echo OK
fi


a_test_has_failed=false

cat >> $LOGFILE <<EOD

PATH is set to $PATH
EOD

if [[ -z "$MRTRIX_BINDIR" ]]; then
  MRTRIX_BINDIR="$(pwd)/bin"
else
  MRTRIX_BINDIR="${MRTRIX_BINDIR}"
fi
cat >> $LOGFILE <<EOD

Testing executables in: $MRTRIX_BINDIR
EOD


for testitem in $testlist; do

  cat >> $LOGFILE <<EOD
-------------------------------------------

## running "${testitem}"...

EOD

  echo -n 'running "'${testitem}'"... '
  [[ -n "$datadir" ]] && rm -rf $datadir/tmp* $datadir/*-tmp-*
  ((ntests=0))
  ((success=0))
  while IFS='' read -r cmd || [[ -n "$cmd" ]]; do
    cmd="${cmd%\#*}"
    [[ -n "$cmd" ]] || continue
    echo -n '# command: '$cmd >> $LOGFILE
    (
      export PATH="$(pwd)/testing/bin:${MRTRIX_BINDIR}:$PATH";
      cd $datadir
      eval $cmd
    ) > .__tmp.log 2>&1

    if [[ $? -ne 0 ]]; then
      echo " [ ERROR ]" >> $LOGFILE
    else
      echo " [ ok ]" >> $LOGFILE
      ((success++));
    fi
    cat .__tmp.log >> $LOGFILE
    echo "" >> $LOGFILE
    ((ntests++))
  done < $testsdir$testitem

  echo -n $success of $ntests passed
  if [[ $success -lt $ntests ]]; then
    echo "    <-------- ERROR"
    a_test_has_failed=true
    cat >> $LOGFILE <<EOD

## ERROR: $(($ntests-$success)) tests failed for "${testitem}"

EOD
  else
    echo ""
    cat >> $LOGFILE <<EOD
## $ntests tests completed OK for "${testitem}"

EOD
  fi

done


if [[ ! -z $testsdir && ! -z $datadir ]]; then
  echo -n "Cleaning up temporary files... "
  rm -rf $datadir/tmp* $datadir/*-tmp-*/
  echo OK
fi


if [ ${a_test_has_failed} = true ];
then
  exit 1
fi