#!/bin/bash
# By default, this script runs in testing mode
# Although it allows for a server debugging mode, which attaches gdb to
# the `cvmfs_server publish` process and allows for both online and
# failure debugging.
#
# To allow for online debugging, export CVMFS_TEST_SRVDEBUG like so:
#
# `export CVMFS_TEST_SRVDEBUG=fail`    - for crash debugging with gdb
# `export CVMFS_TEST_SRVDEBUG=startup` - for interactive debugging with gdb

CVMFS_SERVER_APACHE_RELOAD_IS_RESTART=true
CVMFS_TEST_USE_MACFUSE_KEXT=${CVMFS_TEST_USE_MACFUSE_KEXT:=no}
CVMFS_TEST_DEBUGLOG=${CVMFS_TEST_DEBUGLOG:=}
CVMFS_TEST_PROXY=${CVMFS_TEST_PROXY:=http://ca-proxy.cern.ch:3128}
CVMFS_TEST_SCRATCH=${CVMFS_TEST_SCRATCH:=/tmp/cvmfs-test}
CVMFS_TEST_SUITE_NAME=${CVMFS_TEST_SUITE_NAME:=Integration Tests}
CVMFS_TEST_CLASS_NAME=${CVMFS_TEST_CLASS_NAME:=IntegrationTests}
CVMFS_TEST_EXCLUDE=${CVMFS_TEST_EXCLUDE:=}
CVMFS_TEST_SYSLOG_FACILITY=${CVMFS_TEST_SYSLOG_FACILITY:=5}
CVMFS_TEST_SYSLOG_TARGET=${CVMFS_TEST_SYSLOG_TARGET:=/var/log/cvmfs-testing.log}

CVMFS_TEST_REPO=${CVMFS_TEST_REPO:=test.cern.ch}
CVMFS_TEST_REPO_MORE=${CVMFS_TEST_REPO_MORE:=test-more.cern.ch}
CVMFS_TEST_USER=${CVMFS_TEST_USER:=sftnight}   # user and group are used to over-
CVMFS_TEST_GROUP=${CVMFS_TEST_GROUP:=sftnight} # write the owner of files for testing
CVMFS_TEST_GEO_LICENSE_KEY=${CVMFS_TEST_GEO_LICENSE_KEY:=}
CVMFS_TEST_GEO_ACCOUNT_ID=${CVMFS_TEST_GEO_ACCOUNT_ID:=}

CVMFS_TEST_UNIONFS=${CVMFS_TEST_UNIONFS:=} # union filesystem type to test
CVMFS_TEST_SRVDEBUG=${CVMFS_TEST_SRVDEBUG:=}
CVMFS_TEST_HASHALGO=${CVMFS_TEST_HASHALGO:=sha1}
CVMFS_TEST_S3_CONFIG=${CVMFS_TEST_S3_CONFIG:=}
CVMFS_TEST_S3_STORAGE=${CVMFS_TEST_S3_STORAGE:=}
CVMFS_TEST_HTTP_BASE=${CVMFS_TEST_HTTP_BASE:=}
CVMFS_TEST_STRATUM0=${CVMFS_TEST_STRATUM0:=}   # backward compatibility! use CVMFS_TEST_HTTP_BASE instead

# These OPT variables are for the benchmark options under the benchmarks folder
CVMFS_OPT_WARM_CACHE=${CVMFS_OPT_WARM_CACHE:=yes}
CVMFS_OPT_HOT_CACHE=${CVMFS_OPT_HOT_CACHE:=no}
CVMFS_OPT_TALK_STATISTICS=${CVMFS_OPT_TALK_STATISTICS:=yes}
CVMFS_OPT_VALGRIND=${CVMFS_OPT_VALGRIND:=yes}
CVMFS_OPT_ITERATIONS=${CVMFS_OPT_ITERATIONS:=1}
CVMFS_OPT_TEST_TYPE=${CVMFS_OPT_TEST_TYPE:=callgrind} # callgrind / memcheck
CVMFS_OPT_CONFIG_FILE=${CVMFS_OPT_CONFIG_FILE:=}
CVMFS_OPT_OUTPUT_DIR=${CVMFS_OPT_OUTPUT_DIR:=/tmp/cvmfs_benchmarks}
CVMFS_OPT_CACHEDIR=${CVMFS_OPT_CACHEDIR:=~/.benchmark}
CVMFS_OPT_PARALLEL_RUNS=${CVMFS_OPT_PARALLEL_RUNS:=1}
CVMFS_OPT_MTAB_MODIFIED="false"

# backward compatibility.
#     CVMFS_TEST_STRATUM0 was replaced by CVMFS_TEST_HTTP_BASE to better reflect
#     the purpose of this variable. Please don't use CVMFS_TEST_STRATUM0 anymore
if [ -z "$CVMFS_TEST_HTTP_BASE" ] && [ ! -z "$CVMFS_TEST_STRATUM0" ]; then
  echo "Warning: You are using the deprecated CVMFS_TEST_STRATUM0 variable."
  echo "         Please consider switching to CVMFS_TEST_HTTP_BASE"
  CVMFS_TEST_HTTP_BASE="$CVMFS_TEST_STRATUM0"
fi

if [ ! -z "$CVMFS_TEST_GEO_LICENSE_KEY" ]; then
  echo "CVMFS_GEO_LICENSE_KEY=$CVMFS_TEST_GEO_LICENSE_KEY" | sudo tee /etc/cvmfs/server.local > /dev/null
fi
if [ ! -z "$CVMFS_TEST_GEO_ACCOUNT_ID" ]; then
  echo "CVMFS_GEO_ACCOUNT_ID=$CVMFS_TEST_GEO_ACCOUNT_ID" | sudo tee -a /etc/cvmfs/server.local > /dev/null
fi

if [ -f /.dockerinit ]; then
  CVMFS_TEST_DOCKER=yes
else
  CVMFS_TEST_DOCKER=no
fi

sys_arch=$(uname)

die() {
  echo -e $1 >&2
  exit 1
}


running_on_osx() {
  [ "x$(uname)" == "xDarwin" ]
}

if running_on_osx; then
  CVMFS_PYTHON2="python"
else
  CVMFS_PYTHON2="python2"
fi

CVMFS_SYS_PYTHON="python"
if ! $CVMFS_SYS_PYTHON -V >/dev/null 2>&1; then
  CVMFS_SYS_PYTHON="python3"
  if ! $CVMFS_SYS_PYTHON -V >/dev/null 2>&1; then
    CVMFS_SYS_PYTHON="python2"
  fi
fi

running_on_s3() {
  [ "x$CVMFS_TEST_S3_CONFIG" != "x" ]
}

running_with_external_cache() {
  local repo="$1"
  sudo cvmfs_talk -i $repo cache instance | head -1 | grep -q External
}

get_xattr() {
  local extended_attr="$1"
  local path="$2"
  if running_on_osx; then
    # For some reason xattr -p sometimes works after xattr -l
    if $(xattr -l "$path" > /dev/null 2>&1); then
      xattr -p "user.$extended_attr" "$path"
    fi
  else
    attr -qg $extended_attr "$path"
  fi
}

list_xattrs() {
  local repo="$1"
  if running_on_osx; then
    xattr "$repo"
  else
    attr -ql "$repo"
  fi
}

show_mounted() {
  if running_on_osx; then
    mount
  else
    cat /proc/mounts
  fi
}

get_user_data() {
  local user=$1
  if running_on_osx; then
    id -P "$user"
  else
    getent passwd "$user"
  fi
}

stat_wrapper() {
  local format=$1
  local repo=$2
  if running_on_osx; then
    stat -f $format $repo
  else
    stat -c $format $repo
  fi
}

# Find the service binary (or detect systemd)
SERVICE_BIN="false"
if ! ps -p 1 -o comm= > /dev/null 2>&1 || ! [ -d /run/systemd/system ]; then
  if [ -x /sbin/service ]; then
    SERVICE_BIN="/sbin/service"
  elif [ -x /usr/sbin/service ]; then
    SERVICE_BIN="/usr/sbin/service" # Ubuntu
  elif [ -x /sbin/rc-service ]; then
    SERVICE_BIN="/sbin/rc-service" # OpenRC
  elif running_on_osx; then
    SERVICE_BIN="false" # we don't need to run any service on mac
  else
    die "Neither systemd nor service binary detected"
  fi
fi

is_systemd() {
  [ x"$SERVICE_BIN" = x"false" ]
}

is_redhat() {
  [ -e /etc/redhat-release ]
}

is_el7() {
  is_redhat && which lsb_release > /dev/null 2>&1 && \
  [ x"$(lsb_release -rs | cut -f1 -d'.')" = x"7" ]
}

is_el6() {
  is_redhat && which lsb_release > /dev/null 2>&1 && \
  [ x"$(lsb_release -rs | cut -f1 -d'.')" = x"6" ]
}

is_el5() {
  is_redhat && which lsb_release > /dev/null 2>&1 && \
  [ x"$(lsb_release -rs | cut -f1 -d'.')" = x"5" ]
}

# find the name of the httpd service


# figure out the installed apache platform
if which httpd2 >/dev/null 2>&1; then #SLES/OpenSuSE
  APACHE_SERVICE="apache2"
  APACHE_CONF=${APACHE_SERVICE}
  APACHE_BIN=$(which httpd2)
elif which apache2 >/dev/null 2>&1; then # Debian based
  APACHE_SERVICE="apache2"
  APACHE_CONF=${APACHE_SERVICE}
  APACHE_BIN=$(which apache2)
elif which /usr/sbin/apache2 >/dev/null 2>&1; then # newer Ubuntu
  APACHE_SERVICE="apache2"
  APACHE_CONF=${APACHE_SERVICE}
  APACHE_BIN=/usr/sbin/apache2
else
  APACHE_SERVICE="httpd" # EL based
  APACHE_CONF=${APACHE_SERVICE}
  APACHE_BIN="/usr/sbin/httpd"
fi

normalize_version() {
  local version_string="$1"
  while [ $(echo "$version_string" | grep -o '\.' | wc -l) -lt 2 ]; do
    version_string="${version_string}.0"
  done
  echo "$version_string"
}

to_syslog() {
  logger -t cvmfs_test "$1"
}

version_major() { echo $1 | cut --delimiter=. --fields=1; }
version_minor() { echo $1 | cut --delimiter=. --fields=2; }
version_patch() { echo $1 | cut --delimiter=. --fields=3; }
prepend_zeros() { printf %05d "$1"; }
compare_versions() {
  local lhs="$(normalize_version $1)"
  local comparison_operator=$2
  local rhs="$(normalize_version $3)"

  local lhs1=$(prepend_zeros $(version_major $lhs))
  local lhs2=$(prepend_zeros $(version_minor $lhs))
  local lhs3=$(prepend_zeros $(version_patch $lhs))
  local rhs1=$(prepend_zeros $(version_major $rhs))
  local rhs2=$(prepend_zeros $(version_minor $rhs))
  local rhs3=$(prepend_zeros $(version_patch $rhs))

  [ $lhs1$lhs2$lhs3 $comparison_operator $rhs1$rhs2$rhs3 ]
}

service_request() {
  local service_name=$1
  local request_verb=$2
  if is_systemd; then
    echo "using systemd"
    # Prevent "service restarted too often" failures
    sudo systemctl reset-failed $service_name
    if [ "x$request_verb" = "xrestart" ]; then
      sudo systemctl stop $service_name
      if ! sudo systemctl start $service_name; then
        sleep 20
        sudo systemctl start $service_name
      fi
    else
      if ! sudo systemctl $request_verb $service_name; then
        sleep 20
        sudo systemctl $request_verb $service_name
      fi
    fi
  else
    echo "using traditional service binary"
    sudo $SERVICE_BIN $service_name $request_verb
  fi
}

# ensures that a generic service is running
# @param service  the name of the service
# @param state    the desired state of the service (on|off)
# @return         0 on success
service_switch() {
  local service_name=$1
  local state=$2

  running_on_osx && return 0

  case $state in
    on)
      echo "starting $service_name..."
      service_request $service_name start || return 100
      ;;
    off)
      echo "stopping $service_name..."
      service_request $service_name stop || return 101
      ;;
    restart)
      echo "restarting $service_name..."
      service_request $service_name restart || return 102
      ;;
    *)
      echo "unrecognized state switching for $service_name"
      return 102
  esac

  return 0
}


# checks if a service needs to be switched on or off
# @param desired_state  the state the service should be switched to
# @param state          the state the service is currently in
# @return  0 if a switch is needed
service_should_switch() {
  local desired_state=$1
  local state=$2
  if [ $state -eq 0 ]; then # is running and...
    if [ "$desired_state" = "off" ]; then # should be switched off
      return 0
    fi
  else                 # is NOT running and...
    if [ "$desired_state" = "on" ]; then # should be switched on
      return 0
    fi
  fi

  return 1
}


# checks if autofs is running on /cvmfs
# @return   0 when autofs is mounted on /cvmfs
autofs_check() {
  if running_on_osx; then
    return 0
  else
    cat /proc/mounts | grep -q "^/etc/\(autofs/\)\?auto.cvmfs /cvmfs "
  fi
}


# ensures that autofs is on or off
# @param state   the desired state of autofs (on|off)
# @return        0 on success
autofs_switch() {
  autofs_check
  if service_should_switch $1 $?; then
    service_switch autofs $1
  fi
}


# switches apache on or off
# @param state   the desired state of autofs (on|off)
# @return        0 on success
apache_switch() {
  service_request $APACHE_SERVICE status > /dev/null 2>&1
  if service_should_switch $1 $?; then
    service_switch $APACHE_SERVICE $1
  fi
}

# Checks whether the given test case is marked with any of the given
# test suite tags.  If there are no labels, execute the test
# @return        0 on success
is_in_suite() {
  local testdir=$1
  shift
  local labels=$@

  if [ "x$labels" = "x" ]; then
    return 0;
  fi

  (
    unset cvmfs_test_suites
    . $t/main
    for wanted_label in $labels; do
      for test_label in $cvmfs_test_suites; do
        if [ "$wanted_label" = "$test_label" ]; then
          exit 0;
        fi
      done
    done
    exit 1
  )
  return $?
}


contains() {
  local haystack="$1"
  local needle=$2

  for elem in $haystack
  do
    if [ $(basename $elem) = $(basename $needle) ]; then
      return 0
    fi
  done

  return 1
}


uses_aufs() {
  local repo_name="$1"
  load_repo_config $repo_name
  [ x"$CVMFS_UNION_FS_TYPE" = x"aufs" ]
}


uses_overlayfs() {
  local repo_name="$1"
  load_repo_config $repo_name
  [ x"$CVMFS_UNION_FS_TYPE" = x"overlayfs" ]
}


is_linux() {
  [ x"$(uname)" = x"Linux" ]
}


is_macos() {
  [ x"$(uname)" = x"Darwin" ]
}

is_macfuse_enabled() {
  if [ x"$CVMFS_TEST_USE_MACFUSE_KEXT" = x ]; then
    return 1
  fi
  local value=$(echo "$CVMFS_TEST_USE_MACFUSE_KEXT" | tr '[:upper:]' '[:lower:]')
  case "$value" in
    yes|on|1|true)
      return 0 ;;
    no|off|0|false)
      return 1 ;;
    *)
      echo "Error: invalid value of CVMFS_USE_MACFUSE_KEXT: $CVMFS_TEST_USE_MACFUSE_KEXT"
      echo "Valid values are: yes, on, 1, true, no, off, 0, false"
      return 1 ;;
  esac
}


get_number_of_cpu_cores() {
  if is_linux; then
    cat /proc/cpuinfo | grep -e '^processor' | wc -l
  elif is_macos; then
    sysctl -n hw.ncpu
  else
    echo "1"
  fi
}


break_hardlink() {
  local path="$1"
  local tmp_file="$(mktemp)"
  local linkcount="$(stat --format=%h $path)"

  echo "Note: breaking up hardlink to '$path' linkcount: $linkcount"
  cp -f $path $tmp_file
  rm -f $path
  mv    $tmp_file $path
}

wait_for_umount() {
  local timeout=60
  while $(pgrep -u cvmfs cvmfs2 > /dev/null); do
    if [ $timeout -eq 0 ]; then
      return 1
    fi
    timeout=$(($timeout-1))
    sleep 1
  done
  return 0
}

cvmfs_clean() {
  sudo env CVMFS_USE_MACFUSE_KEXT="$CVMFS_TEST_USE_MACFUSE_KEXT" cvmfs_config umount > /dev/null  || return 100
  sudo sh -c "rm -rf /var/lib/cvmfs/*"                  || return 101
  sudo rm -f /etc/cvmfs/default.local                    || return 102
  sudo sh -c "rm -f /etc/cvmfs/config.d/*.local"         || return 104
  sudo sh -c "cat /dev/null > $CVMFS_TEST_SYSLOG_TARGET" || return 105

  wait_for_umount || return 99

  return 0
}

cvmfs_disable_config_repository() {
  sudo sh -c "echo 'CVMFS_CONFIG_REPOSITORY=' > /etc/cvmfs/default.d/99_no_config_repo.conf" \
    || return $?
}

cvmfs_enable_config_repository() {
  sudo rm -f /etc/cvmfs/default.d/99_no_config_repo.conf || return $?
}

cvmfs_mount_direct() {
  repositories=$1
  eval $(cvmfs_config showconfig | grep -e ^CVMFS_CONFIG_REPOSITORY=)
  if [ "x$CVMFS_CONFIG_REPOSITORY" != "x" ]; then
    echo "Show config mount $CVMFS_CONFIG_REPOSITORY" >> "/tmp/cvmfs-integration.log"
    _cvmfs_mount_manually "$CVMFS_CONFIG_REPOSITORY"
  fi
  for i in $(echo "$repositories" | tr "," " "); do
    echo "manual mount $i" >> "/tmp/cvmfs-integration.log"
    _cvmfs_mount_manually "$i" || return 102
  done
}

cvmfs_mount() {
  if [ x$PARROT_ENABLED = "xTRUE" ]; then
    return 0
  fi

  repositories=$1
  shift 1

  sudo sh -c "echo \"CVMFS_REPOSITORIES=$repositories\" > /etc/cvmfs/default.local" || return 100
  sudo sh -c "echo \"CVMFS_HTTP_PROXY=\\\"${CVMFS_TEST_PROXY}\\\"\" >> /etc/cvmfs/default.local" || return 100
  sudo sh -c 'echo "CVMFS_TIMEOUT=20" >> /etc/cvmfs/default.local' || return 100
  sudo sh -c 'echo "CVMFS_TIMEOUT_DIRECT=20" >> /etc/cvmfs/default.local' || return 100
  sudo sh -c "echo CVMFS_SYSLOG_FACILITY=$CVMFS_TEST_SYSLOG_FACILITY >> /etc/cvmfs/default.local" || return 100
  sudo sh -c "echo CVMFS_USE_MACFUSE_KEXT=$CVMFS_TEST_USE_MACFUSE_KEXT >> /etc/cvmfs/default.local" || return 100
  sudo sh -c "echo CVMFS_REPOSITORY_TAG=TESTING > /etc/cvmfs/config.d/cvmfs-config.cern.ch.local" || return 100
  if running_on_osx ; then
    sudo sh -c 'echo "CVMFS_MAGIC_XATTRS_VISIBILITY=ALWAYS" >> /etc/cvmfs/default.local' || return 100
  fi

  # add additional parameters
  local proberunner
  while [ $# -gt 0 ]; do
    local param="$1"
    if [ "$param" = "CVMFS_SUID=yes" ]; then
      proberunner="sudo"
    fi
    sudo sh -c "echo \"$param\" >> /etc/cvmfs/default.local" || return 100
    shift 1
  done

  if [ "x$CVMFS_TEST_DEBUGLOG" != "x" ]; then
    sudo sh -c "echo 'CVMFS_DEBUGLOG=${CVMFS_TEST_DEBUGLOG}' >> /etc/cvmfs/default.local" || return 100
  fi

  if [ "x$CVMFS_TEST_QUOTED_PARAM_KEY_VAL" != "x" ]; then
    sudo sh -c "echo '${CVMFS_TEST_QUOTED_PARAM_KEY_VAL}' >> /etc/cvmfs/default.local" || return 100
  fi

  if running_on_osx; then
    cvmfs_mount_direct $repositories
  fi
  if [ "x$cvmfs_test_autofs_on_startup" = "xfalse" ]; then
    cvmfs_mount_direct $repositories
  fi

  $proberunner cvmfs_config probe > /dev/null 2>&1 || return 101

  return 0
}


_cvmfs_mount_manually() {
  local repository=$1
  cvmfs_config status | grep -q "/cvmfs/$repository" && return 0
  mkdir -p "/cvmfs/$repository" 2>/dev/null || sudo mkdir -p "/cvmfs/$repository"
  sudo mount -t cvmfs "$repository" "/cvmfs/$repository" >/dev/null 2>&1
  if running_on_osx; then 
    sleep 3
  fi
}

cvmfs_umount() {
  if [ x$PARROT_ENABLED = "xTRUE" ]; then
    return 0
  fi

  local repositories=$1
  local times=5
  local result=-1

  for r in $(echo $repositories | tr , " "); do
    while (( result != 0 && times > 0  )); do
      if [ "$sys_arch" = "Darwin" ] && ! is_macfuse_enabled; then
        sudo umount -f /cvmfs/$r 2>/dev/null
      else
        sudo umount /cvmfs/$r > /dev/null 2>&1
      fi
      result=$?
      times=$(( $times - 1 ))
      if [ $result -ne 0 ] && [ $times -gt 0 ]; then
        echo "Failed to umount. Trying again..."
        sleep 1
      fi
    done

    if [ $result -ne 0 ]; then
      lsof /cvmfs/$r
      return 200
    fi

    local timeout=5
    while show_mounted | grep -q " /cvmfs/$r "; do
      if [ $timeout -eq 0 ]; then
        return 101
      fi
      timeout=$(($timeout-1))
      sleep 1
    done

    times=5
    result=-1
  done

  return 0
}


cvmfs_remove_breadcrumb() {
  local repo="$1"
  if running_with_external_cache $repo; then
    local pid_cache=$(sudo cvmfs_talk -i $repo pid cachemgr)
    echo "--- sending SIGUSR2 to cache manager at PID $pid_cache"
    sudo kill -12 $pid_cache
    sleep 2
  else
    echo "--- removing $(get_cvmfs_cachedir ${repo})/cvmfschecksum.${repo}"
    sudo rm -f "$(get_cvmfs_cachedir ${repo})/cvmfschecksum.${repo}" || return 1
  fi
}


has_selinux() {
  which sestatus   > /dev/null 2>&1 && \
  which getenforce > /dev/null 2>&1 && \
  getenforce | grep -qi "enforc" || return 1
}


get_cvmfs_cachedir() {
  local repository=$1

  local cache_dir=$(cvmfs_config showconfig $repository | grep CVMFS_WORKSPACE | awk '{print $1}' | cut -d= -f2)
  if [ "x$cache_dir" = "x" ]; then
    cache_dir=$(cvmfs_config showconfig $repository | grep CVMFS_CACHE_DIR | awk '{print $1}' | cut -d= -f2)
  fi
  if [ "x$cache_dir" = "x" ]; then
    echo "Failed to figure out cache directory"
    exit 1
  fi
  echo $cache_dir
}


CVMFS_MEMORY_WARNING_FLAG=0
CVMFS_TIME_WARNING_FLAG=0
CVMFS_GENERAL_WARNING_FLAG=0
CVMFS_MEMORY_WARNING=254
CVMFS_TIME_WARNING=253
CVMFS_GENERAL_WARNING=252
CVMFS_TEST_RETVAL_TIMEOUT=251
mangle_test_retval() {
  local prior_retval=$1

  # if the test case failed... report the retval right away
  if [ $prior_retval -ne 0 ]; then
    return $prior_retval
  fi

  # check if the test case produced memory warnings and report them
  if [ $CVMFS_MEMORY_WARNING_FLAG -ne 0 ]; then
    return $CVMFS_MEMORY_WARNING
  fi

  # check if the test case produced timeout warnings
  if [ $CVMFS_TIME_WARNING_FLAG -ne 0 ]; then
    return $CVMFS_TIME_WARNING
  fi

  # check if the test case produced any other warnings
  if [ $CVMFS_GENERAL_WARNING_FLAG -ne 0 ]; then
    return $CVMFS_GENERAL_WARNING
  fi

  # return 0 if all is good
  return 0
}

reset_test_warning_flags() {
  CVMFS_MEMORY_WARNING_FLAG=0
  CVMFS_TIME_WARNING_FLAG=0
  CVMFS_GENERAL_WARNING_FLAG=0
}


check_time() {
  local start_time=$1
  local end_time=$2
  local limit=$3

  local diff_time=$(($end_time-$start_time))

  if [ $diff_time -gt $limit ]; then
    echo "Time limit exceeded" >&2
    echo "Limit was $limit but it took $diff_time seconds"
    CVMFS_TIME_WARNING_FLAG=1
    return 100
  fi

  return 0
}


check_memory() {
  local instance=$1
  local limit=$2

  local pid=$(get_xattr pid /cvmfs/$instance)            || return 100
  local rss="$(sudo cat /proc/$pid/status | grep VmRSS)" || return 101
  local rss_kb=$(echo $rss | awk '{print $2}')

  if [ $rss_kb -gt $limit ]; then
    local elements=$(sudo cvmfs_talk -i $instance internal affairs | grep '^inode_tracker.no_reference' | cut -d\| -f2)

    echo "Memory limit of $limit kB was exceeded by $instance which used $rss_kb kB"
    echo "We've had $elements items in the inode tracker"
    echo "Output of cat /proc/$pid/status:"
    sudo cat /proc/$pid/status 2>&1
    CVMFS_MEMORY_WARNING_FLAG=1

    return 102
  fi

  return 0
}


min() {
  [ $1 -lt $2 ] && echo $1 || echo $2
}


destroy_repo() {
  local repo=$1
  shift 1
  local additional_flags="$@"

  if [ x"$CVMFS_TEST_S3_CONFIG" != x"" ] && [ x"$CVMFS_TEST_S3_STORAGE" != x"" ]; then
    echo "Wiping repository S3 storage ${CVMFS_TEST_S3_STORAGE}/${repo}"
    sudo rm -rf "${CVMFS_TEST_S3_STORAGE}/${repo}"
  fi

  load_repo_config $repo
  if [ x"$TEST_CVMFS_RECEIVER_UPSTREAM_STORAGE" != x"" ]; then
    # It is possible to have publisher and gateway on the same machine for testing purposes.
    # However, `cvmfs_server rmfs` then incorrectly does not delete the repository backend storage.
    # Let's do it here instead.
    echo "Wiping local repository storage in gateway setup /srv/cvmfs/${repo}"
    sudo rm -rf /srv/cvmfs/${repo}
  fi

  sudo cvmfs_server rmfs -f $additional_flags $repo || return 100
}


has_repo() {
  local repo=$1
  cvmfs_server list 2>/dev/null | grep "^$repo " | grep -q -F "$repo"
}


create_repo() {
  local repo=$1
  local uid=$2
  local debug_log=$3
  shift $(min $# 3)

  echo "Shutting down autofs for the cvmfs mounts"
  autofs_switch off || return 100

  if has_repo $repo; then
    echo "Repository $repo is already present... removing it"
    destroy_repo $repo || return 101
  fi

  # mkfs will no longer overwrite a manifest in an existing bucket, need to remove it manually
  if [ -f "${CVMFS_TEST_S3_STORAGE}/$repo/.cvmfspublished" ]; then
    sudo rm -f "${CVMFS_TEST_S3_STORAGE}/$repo/.cvmfspublished"
  fi


  # Note: We need to do a non-graceful (brutal?) restart of Apache on Fedora 26+ to avoid
  #       a timeout issue
  if [ x"$(which apachectl)" != x"" ]; then
    sudo apachectl restart
  fi

  echo -n "Creating new repository $repo (with extra options "$@") ..."
  local s3_config=""
  local stratum0=""
  local unionfs=""
  if [ x"$CVMFS_TEST_S3_CONFIG" != x"" ]; then
    echo "  S3 Config: $CVMFS_TEST_S3_CONFIG"
    s3_config=" -s $CVMFS_TEST_S3_CONFIG"
  fi
  if [ x"$CVMFS_TEST_HTTP_BASE" != x"" ]; then
    echo "  Stratum0: $CVMFS_TEST_HTTP_BASE"
    stratum0=" -w $CVMFS_TEST_HTTP_BASE"
  fi
  if [ x"$CVMFS_TEST_UNIONFS" != x"" ]; then
    echo "  UnionFS: $CVMFS_TEST_UNIONFS"
    unionfs=" -f $CVMFS_TEST_UNIONFS"
  fi
  sudo -E cvmfs_server mkfs -o $uid -m                \
                            -a ${CVMFS_TEST_HASHALGO} \
                            $s3_config                \
                            $stratum0                 \
                            $unionfs                  \
                            "$@" $repo || return 102

  local client_conf="/etc/cvmfs/repositories.d/${repo}/client.conf"

  if [ x$debug_log != x -a x$debug_log != xNO ]; then
    echo "CVMFS_DEBUGLOG=$debug_log" | sudo tee -a $client_conf
    sudo sed -i.bak -e "s/^\(cvmfs2#$repo.*\)\(allow_other.*\)/\1debug,\2/" /etc/fstab
  fi
}


create_empty_repo() {
  local repo=$1
  local uid=$2
  local debug_log=$3
  shift $(min $# 3)

  create_repo $repo $uid $debug_log "$@" || return 101

  if [ -f /cvmfs/$repo/new_repository ]; then
    sudo cvmfs_server transaction $repo || return 102
    rm -f /cvmfs/$repo/new_repository
    sudo cvmfs_server publish $repo || return 103
  fi
}


# creates a cvmfs repository and fills it with some dummy data
#
# @param repo_name   the name of the repository to create
# @param uid         the user id of the new repository's owner
create_filled_repo() {
  local repo=$1
  local uid=$2
  local debug_log=$3

  create_empty_repo $repo $uid $debug_log || return 101

  sudo cvmfs_server transaction $repo || return 102

  pushdir /cvmfs/$repo

  echo "meaningless file content" > file
  echo "more clever file content" > clever
  ln file hardlinkToFile
  ln -s clever symlinkToClever

  mkdir -p foo/bar/baz
  mkdir -p bar/foo/baz
  touch foo/.cvmfscatalog

  # Put some meaningful stuff into files
  echo "Vom Eise befreit sind Strom und Bäche"         > foo/bar/verse1
  echo "Durch des Frühlings holden, belebenden Blick," > foo/bar/verse2
  echo "Im Tale grünet Hoffnungsglück;"                > foo/bar/verse3
  echo "Der alte Winter, in seiner Schwäche,"          > foo/bar/verse4
  echo "Zog sich in rauhe Berge zurück."               > foo/bar/verse5

  popdir

  sudo cvmfs_server publish $repo || return 103
}


# generates a huge dummy repository content with 500.000 directory entries
# in a somewhat representative directory structure
#
# @param  repo_dir   where to put the dummy dirents
make_huge_repo() {
  local repo_dir=$1
  ${TEST_ROOT}/common/mock_services/make_repo.py                \
    --max-dir-depth        7                                    \
    --num-subdirs          5                                    \
    --num-files-per-dir    5                                    \
    --min-file-size        0                                    \
    --max-file-size     5120                                    \
    $repo_dir
}


import_repo() {
  local repo=$1
  local uid=$2
  shift 2
  local extra_options="$*"

  echo "Importing repository $repo..."
  local unionfs=""
  if [ x"$CVMFS_TEST_UNIONFS" != x"" ]; then
    echo "  UnionFS: $CVMFS_TEST_UNIONFS"
    unionfs=" -f $CVMFS_TEST_UNIONFS"
  fi
  sudo -E cvmfs_server import -o $uid $unionfs $extra_options $repo || return 102
}


get_stratum1_name() {
  local repo_name="$1"
  echo "$repo_name.$(date +%s%N | md5sum | head -c6).replica"
}


# creates a repository replica, as is done on stratum 1s
# @param replica_name  the name of the new replica
# @param replica_owner the owner of the new replica
# @param stratum0_url  the url for the source repository
# @param stratum0_pub  the public key the repository
create_stratum1() {
  local replica_name=$1
  local replica_owner=$2
  local stratum0_url=$3
  local stratum0_pub=$4
  shift 4
  local additional_parameters="$*"
  local s3_config=""
  if [ x"$CVMFS_TEST_S3_CONFIG" != x"" ]; then
    local stratum1_url="$CVMFS_TEST_HTTP_BASE"
    [ ! -z "$stratum1_url" ] || die "\$CVMFS_TEST_HTTP_BASE needs to be set for S3 tests to work!"
    echo "  S3 Config: $CVMFS_TEST_S3_CONFIG"
    echo "  Stratum1:  $stratum1_url"
    s3_config=" -s $CVMFS_TEST_S3_CONFIG -w $stratum1_url -a"
  fi
  sudo -E cvmfs_server add-replica -o $replica_owner      \
                                   -n $replica_name       \
                                   $s3_config             \
                                   $additional_parameters \
                                   $stratum0_url          \
                                   $stratum0_pub
}

# creates a repository replica as a pass-through to a stratum 0
# @param replica_name  the name of the new replica
# @param replica_owner the owner of the new replica
# @param stratum0_url  the url for the source repository
create_passthrough_stratum1() {
  local replica_name=$1
  local replica_owner=$2
  local stratum0_url=$3
  shift 3
  local additional_parameters="$*"
  local s3_config=""
  if [ x"$CVMFS_TEST_S3_CONFIG" != x"" ]; then
    local stratum1_url="$CVMFS_TEST_HTTP_BASE"
    [ ! -z "$stratum1_url" ] || die "\$CVMFS_TEST_HTTP_BASE needs to be set for S3 tests to work!"
    echo "  S3 Config: $CVMFS_TEST_S3_CONFIG"
    echo "  Stratum1:  $stratum1_url"
    s3_config=" -s $CVMFS_TEST_S3_CONFIG -w $stratum1_url -a"
  fi
  sudo -E cvmfs_server add-replica -P \
                                   -o $replica_owner      \
                                   -n $replica_name       \
                                   $s3_config             \
                                   $additional_parameters \
                                   $stratum0_url
}


# creates a big file at a given location with a given size
# @param path  the location of the new file
# @param size  the desired size in megabytes
create_big_file() {
  local path=$1
  local size=$2

  dd if=/dev/zero of=$path bs=1024k count=$size
}


# generate a custom style recursive listing with only the following infos:
# - file name
# - linkcount
# - file mode
# - file size
# - parent directory
# - symlink destination (when applicable)
#
# @param directory          the directory to be listed
# @param show_linkcount     yes be default, otherwise linkcount is left out
# @return                   a custom directory listing
create_listing() {
  local directory=$1
  local show_linkcount=$2
  local lst

  lst=$(ls --almost-all --recursive -l --file-type --time-style=+ $directory | \
  awk '
  {
    # skip the total file count, ls prints at the end of each directory listing
    if(substr($1, 0, 5) == "total") next;

    # length of base directory path
    base_dir_length = length(base_dir)

    # truncate base path from the path printed before each new listing
    if(substr($0, 0, base_dir_length) == base_dir) {
      print substr($0, base_dir_length + 1, length($0)-1)
      next;
    }

    # first character
    first = substr($0, 0, 1);

    # print file meta information
    # $1    => file mode (rwx bits, ...)
    # $2    => linkcount
    # $3,$4 => owner, group (might also be provided from outside)
    # $5    => file size (skipped for directories)
    # $6    => file name
    # $7,$8 => symlink destination (only for symlinks :o) )

    if (first != "-" && first != "d" && first != "l")
    {
      printf "\n"
      next;
    }

    # print out file information
    printf $1                     " ";
    if (show_linkcount == "yes")
    {
      printf $2                   " ";
    }
    printf $3                     " ";
    printf $4                     " ";
    printf $6;

    if(first == "l") printf " " $7 " " $8;
    if(first != "d") printf " " $5;
    printf "\n"
  }' base_dir=$directory show_linkcount=$show_linkcount)

  echo -e "$lst"
}

# compares the file contents and file meta data of two directories
# Note: function creates the files 'listingFile1' and 'listingFile2' in `pwd`
#       `pwd` should NOT be part of the comparison!!
#
# @param dir1    the directory to probe
# @param dir2    the ground truth directory
# @param repo    the underlying repository name (optional - to detect overlayfs)
# @return        != 0 to indicate inequality, 0 means success
compare_directories() {
  local dir1=$1
  local dir2=$2
  local repo_name=$3
  local show_linkcount="yes"

  if [ ! -z $repo_name ] && uses_overlayfs $repo_name; then
    echo "NOTE: $repo_name is based on OverlayFS - ignoring link counts"
    show_linkcount="no"
  fi

  listing1=$(create_listing $dir1 $show_linkcount)
  listing2=$(create_listing $dir2 $show_linkcount)

  echo "check if directory structure and file meta data fits"
  local listingFile1="listing_$(basename $dir1)"
  local listingFile2="listing_$(basename $dir2)"
  echo -e "$listing1" > $listingFile1
  echo -e "$listing2" > $listingFile2
  diff -uN $listingFile1 $listingFile2 2>&1 || return 101

  echo "check if the file contents in both directories are the same"
  diff -ruN $dir1 $dir2 2>&1 || return 102

  return 0
}


# run a binary in the background and return it's PID through stdout
#
# @param logfile  a path to a logfile for outputs of the started program
# @return         the process ID of the created background service
run_background_service() {
  local logfile=$1
  shift 1

  local srv_pid

  # shell for the win!
  # This spawns a shell executing the passed command string (sudo sh -c) which
  # will print it's PID (echo $$) into a fifo and execute the user defined
  # binary redirecting its outputs into the logfile (exec $@ >> $logfile 2>&1).
  # Furthermore it disconnects the new process from the executing shell (nohup),
  # satisfies the I/O requirements of nohup (2>&1 < /dev/null) and sends it to
  # the background (&)
  #   Credit: Dario Berzano helped with that monster!
  local fifo="pid_fifo"
  mkfifo $fifo || return 1
  nohup sh -c "echo \$\$ > $fifo; exec $@ >> $logfile 2>&1" 2>&1 < /dev/null &
  srv_pid=$(cat $fifo)
  rm -f $fifo

  # check if the background process is running
  # TODO(rmeusel): Fix this race. If the background service is a short living
  #                process it might have successfully terminated at this point.
  if ! sudo kill -0 $srv_pid > /dev/null 2>&1; then
    return 4
  fi

  # print the PID of the background process and return successfully
  echo $srv_pid
  return 0
}


# open a port for incoming connections. It will accept the connection but stay
# silent on it
#
# Note: The user is responsible for killing the created server process after
#       usage
#
# @param protocol  either UDP or TCP
# @param port      the desired port number to be opened
# @param logfile   a path to the logfile where connection logs should be written
# @return          the process ID of the created server script
open_silent_port() {
  local protocol=$1
  local port=$2
  local logfile=$3
  local cmd
  local mock

  mock="${TEST_ROOT}/common/mock_services/silent_socket.py $protocol $port"
  echo "Calling $mock" >> $logfile

  # Do similar fifo-trick as in run_background_service() to get the PID of the
  # actual silent_socket.py process rather than just the `sudo` around it.
  local fifo="pid_silent_port_fifo"
  mkfifo $fifo || return 1
  cmd="sudo sh -c \"echo \\\$\\\$ > $fifo; exec $mock\""
  run_background_service $logfile "$cmd" > /dev/null 2>&1 || return $?
  cat $fifo # return the PID of the `python silent_socket.py`
}


# spawns a mocked HTTP server on a given port and document root location
# Note: The user is responsible to kill the server process after usage
#
# @param document_root  absolute path to be served via HTTP
# @param port           http port to be used
# @param logfile        (optional) where to log the HTTP server's status info
# @return               0 and PID of HTTP server in stdout, or >0 on error
open_http_server() {
  local document_root="$1"
  local port="$2"
  local logfile="${3:-/dev/null}"

  local http_server="$CVMFS_SYS_PYTHON ${TEST_ROOT}/common/mock_services/http_server.py"
  local http_pid=
  local http_sentinel="http://localhost:${port}/http_sentinel"

  # spawning the server
  http_pid=$(run_background_service $logfile "$http_server -p $port -r $document_root")
  touch ${document_root}/$(basename $http_sentinel)

  # waiting for the server to come up
  local timeout=15
  while ! curl -Is ${http_sentinel} | head -n1 | grep "200 OK" >> $logfile 2>&1 && \
        [ $timeout -gt 0 ]; do
    sleep 1
    if ! kill -0 $http_pid > /dev/null 2>&1;then
      return 1
    fi
    timeout=$(( $timeout - 1 ))
  done

  [ $timeout -gt 0 ] || return 2
  echo "$http_pid"
}

# spawns a mocked HTTP server on given port which acts as a metalink
# server that redirects to given linkurls referring to other HTTP servers.
# Note: The user is responsible to kill the server process after usage
#
# @param args     quoted args: optional -r to reverse pri, followed by linkurls
# @param port     http port to be used
# @param logfile  (optional) where to log the HTTP server's status info
# @return         0 and PID of HTTP server in stdout, or >0 on error
open_metalink_server() {
  local args="$1"
  local port="$2"
  local logfile="${3:-/dev/null}"

  local http_server="python3 ${TEST_ROOT}/common/mock_services/metalink_server.py"
  local http_pid=
  local url="http://localhost:${port}"

  # spawning the server
  http_pid=$(run_background_service $logfile "$http_server -p $port $args")

  # waiting for the server to come up
  local timeout=15
  while ! curl -Is $url | head -n1 | grep "307 Temporary Redirect" >> $logfile 2>&1 && \
        [ $timeout -gt 0 ]; do
    sleep 1
    if ! kill -0 $http_pid > /dev/null 2>&1;then
      return 1
    fi
    timeout=$(( $timeout - 1 ))
  done

  [ $timeout -gt 0 ] || return 2
  echo "$http_pid"
}

# checks if a nested catalog is part of the current catalog configuration
# of the repository
# @param catalog_path  the catalog root path to be checked
# @param repo_name     the repository to be checked
check_catalog_presence() {
  local catalog_path=$1
  local repo_name=$2

  cvmfs_server list-catalogs -x $repo_name | grep -x $catalog_path > /dev/null 2>&1
  return $?
}

# counts the number of present catalogs in the repository
# @param repo_name  the name of the repository to investigate
# @return           the number of found catalogs
get_catalog_count() {
  local repo_name=$1

  echo $(cvmfs_server list-catalogs -x $repo_name | wc -l)
}

# read one specific manifest field of the specified repository
# @param field_tag  the manifest field to be retrieved (one character)
# @param (stdin)    the manifest to be investigated
# return            the content of the specified manifest field
get_manifest_field_from_manifest() {
  local field_tag="$1"
  grep --text -e "^$field_tag" | head -n1 | sed "s/^$field_tag\(.*\)$/\1/"
}

# read one specific manifest field of the specified repository
# @param repo_name  the repository to be investigated
# @param field_tag  the manifest field to be retrieved (one character)
# return            the content of the specified manifest field
get_manifest_field() {
  local repo_name=$1
  local field_tag=$2
  load_repo_config $repo_name
  local manifest_url="$(get_repo_url $repo_name)/.cvmfspublished"
  curl -s "$manifest_url" | get_manifest_field_from_manifest "$field_tag"
}

# retrieves the hash of the current root catalog of the given
# repository by investigating the manifest
# @param repo_name  the name of the repository
# @return           the current root catalog hash
get_current_root_catalog() {
  local repo_name=$1
  local manifest_url="$(get_repo_url $repo_name)/.cvmfspublished"
  get_manifest_field $repo_name "C"
}

# downloads and decompresses the root catalog of a given repository for deeper
# inspection
# @param repo_name   the repository to download the root catalog from
# @param object_hash the object to be downloaded and decompressed
# @return            prints the file name of the extracted catalog to stdout
get_and_decompress_root_catalog() {
  local repo_name=$1
  local catalog_hash="$(get_current_root_catalog $repo_name)C"
  download_and_decompress_object $repo_name $catalog_hash
}



# uses `cvmfs_server check` to check the integrity of the catalog structure
# additionally it might check the backend storage integrity as well when
# provided with -i
# @param repo    the repository to be checked
check_repository() {
  local repo=$1
  shift 1

  cvmfs_server check $@ $repo || return 100
  return 0
}

check_repo_integrity() {
  local repo=$1
  for f in `find /cvmfs/${repo} -type f`; do
    cat $f >/dev/null
    if [ $? -ne 0 ]; then
      echo "Integrity check: corrupted file ${f}"
      return 1
    fi
  done
  return 0
}

# wrapper function to start a new repository update transaction
# @param repo    the repository you want to start the transaction for
start_transaction() {
  local repo=$1
  shift

  cvmfs_server transaction $@ $repo || return 100
  return 0
}

# wrapper function to abort a transaction
# @param repo    the repository you want to abort a transaction on
abort_transaction() {
  local repo=$1

  cvmfs_server abort -f $repo || return 100
  return 0
}

# wrapper function to rollback a repository to a specified tag
#
# @param repo   the repository you want to rollback
# @param tag    the name of the tag to be rollbacked to
rollback_repo() {
  local repo=$1
  local tag=$2

  cvmfs_server rollback -t $tag -f $repo || return 100
  return 0
}

# wrapper function to publish an repository after its contents were updated.
# @param repo    the repository name to start the transaction in
publish_repo() {
  local repo=$1
  shift 1

  # enable the debug mode?
  # in debug mode we redirect output directly to the interactive shell,
  # overriding any redirections to logfiles or whatever... We want to hack!
  case $CVMFS_TEST_SRVDEBUG in
    fail)
      cvmfs_server publish -dv "$@" $repo > /dev/tty 2>&1 || return 100
    ;;
    startup)
      cvmfs_server publish -Dv "$@" $repo > /dev/tty 2>&1 || return 100
    ;;
    *)
      cvmfs_server publish -v "$@" $repo || return 100
    ;;
  esac

  return 0
}


# sources the repository server configuration file for the given repo name
# @param name  the name of the repository to load the configs for
load_repo_config() {
  local name=$1
  local conf_dir="/etc/cvmfs/repositories.d/${name}"
  [ ! -f "${conf_dir}/server.conf" ] || . ${conf_dir}/server.conf
  [ ! -f "${conf_dir}/client.conf" ] || . ${conf_dir}/client.conf
}


get_millisecond_epoch() {
  if running_on_osx; then
    echo $(( $(date +%s) * 1000 ))
  else
    echo $(( $(date +%s%N) / 1000000 ))
  fi
}


get_iso8601_timestamp() {
  date -u +'%Y-%m-%dT%H:%M:%SZ'
}


milliseconds_to_seconds() {
  local milliseconds="$1"

  # add a zero padding if necessary
  while [ $(echo -n "$milliseconds" | wc -c) -lt 4 ]; do
    milliseconds="0$milliseconds"
  done

  # insert a decimal point to make seconds
  digits=$(echo -n "$milliseconds" | wc -c)
  echo -n "$milliseconds" | head -c $(( $digits - 3 ))
  echo -n "."
  echo -n "$milliseconds" | tail -c 3
}


milliseconds_to_human_readable() {
  local milliseconds="$1"
  local scratch=$milliseconds

  local seconds=1000
  local minutes=$(( $seconds * 60 ))
  local hours=$(( $minutes * 60))
  local days=$(( $hours * 24 ))

  local o_days=$(( $scratch / $days ))
  if [ $o_days -gt 0 ]; then
    echo -n "$o_days days "
    scratch=$(( $scratch % $days ))
  fi

  local o_hours=$(( $scratch / $hours ))
  if [ $o_days -gt 0 ] || [ $o_hours -gt 0 ]; then
    echo -n "$o_hours hours "
    scratch=$(( $scratch % $hours ))
  fi

  local o_minutes=$(( $scratch / $minutes ))
  if [ $o_days -gt 0 ] || [ $o_hours -gt 0 ] || [ $o_minutes -gt 0 ]; then
    echo -n "$o_minutes minutes "
    scratch=$(( $scratch % $minutes ))
  fi

  echo "$(milliseconds_to_seconds $scratch) seconds"
}


# runs the given command and measures the execution time in milliseconds
stop_watch() {
  begin=$(get_millisecond_epoch)
  cmd="$@"
  $cmd
  res=$?
  end=$(get_millisecond_epoch)
  echo $((end - begin))
  return $res
}


# wrapper around pushd (that is not available in dash)
# if pushd is not available, a normal `cd` is used
# Note: We do not reimplement the whole functionality of pushd, with our work-
#       around it is only possible to have ONE directory-level stored
pushdir() {
  type pushd > /dev/null 2>&1
  if [ $? -eq 0 ]; then
    pushd $1
    return $?
  else
    cd $1
    return $?
  fi
}


# wrapper around popd (that is not available in dash)
# if popd is not available, a normal `cd -` is used to go back to the last dir
# Note: We do not reimplement the whole functionality of popd, with our work-
#       around it is only possible to have ONE directory-level stored
popdir() {
  type popd > /dev/null 2>&1
  if [ $? -eq 0 ]; then
    popd $1
    return $?
  else
    cd -
    return $?
  fi
}


# retrieves the apache version string "2.x.xx"
get_apache_version() {
  ${APACHE_BIN} -v | head -n1 | \
    sed 's/^Server version: Apache\/\([0-9]\+\.[0-9]\+\.[0-9]\+\).*$/\1/'
}


# figure out apache config file mode
#
# @return   apache config mode (stdout) (see globals below)
APACHE_CONF_MODE_CONFD=1     # *.conf goes to ${APACHE_CONF}/conf.d
APACHE_CONF_MODE_CONFAVAIL=2 # *.conf goes to ${APACHE_CONF}/conf-available
get_apache_conf_mode() {
  [ -d /etc/${APACHE_CONF}/conf-available ] && echo $APACHE_CONF_MODE_CONFAVAIL \
                                            || echo $APACHE_CONF_MODE_CONFD
}


# find location of apache configuration files
#
# @return   the location of apache configuration files (stdout)
get_apache_conf_path() {
  local res_path="/etc/${APACHE_CONF}"
  if [ x"$(get_apache_conf_mode)" = x"$APACHE_CONF_MODE_CONFAVAIL" ]; then
    echo "${res_path}/conf-available"
  elif [ -d "${res_path}/modules.d" ]; then
    echo "${res_path}/modules.d"
  else
    echo "${res_path}/conf.d"
  fi
}


# returns the apache configuration string for 'allow from all'
# Note: this is necessary, since apache 2.4.x formulates that different
#
# @return   a configuration snippet to allow s'th from all hosts (stdout)
get_compatible_apache_allow_from_all_config() {
  local minor_apache_version=$(version_minor "$(get_apache_version)")
  if [ $minor_apache_version -ge 4 ]; then
    echo "Require all granted"
  else
    local nl='
'
    echo "Order allow,deny${nl}    Allow from all"
  fi
}


# writes apache configuration file
# This figures out where to put the apache configuration file depending
# on the running apache version
# Note: Configuration file content is expected to come through stdin
#
# @param   file_name  the name of the apache config file (no path!)
# @return             0 on success
create_apache_config_file() {
  local file_name=$1
  local conf_path
  conf_path="$(get_apache_conf_path)"

  # create (or append) the conf file
  cat - | sudo tee ${conf_path}/${file_name} > /dev/null || return 1

  # the new apache requires the enable the config afterwards
  if [ x"$(get_apache_conf_mode)" = x"$APACHE_CONF_MODE_CONFAVAIL" ]; then
    sudo a2enconf $file_name > /dev/null || return 2
  fi

  return 0
}


# removes apache config files dependent on the apache version in place
# Note: As of apache 2.4.x `a2disconf` needs to be called before removal
#
# @param   file_name  the name of the conf file to be removed (no path!)
# @return  0 on successful removal
remove_apache_config_file() {
  local file_name=$1
  local conf_path
  conf_path="$(get_apache_conf_path)/${file_name}"

  # disable configuration on newer apache versions
  if [ x"$(get_apache_conf_mode)" = x"$APACHE_CONF_MODE_CONFAVAIL" ]; then
    sudo a2disconf $file_name > /dev/null 2>&1 || return 1
  fi

  # remove configuration file
  sudo rm -f $conf_path
}


get_hash_path() {
  local object_hash="$1"
  local object_path="$(echo -n $object_hash | sed -e 's/^\(..\)\(.*\)$/\1\/\2/')"
  echo "${object_path}"
}

get_local_repo_storage() {
  local repo_name="$1"
  echo "/srv/cvmfs/${repo_name}"
}

get_local_repo_object() {
  local repo_name="$1"
  local object_hash="$2"
  local repo_storage="$(get_local_repo_storage $repo_name)"
  local object_path="$(get_hash_path $object_hash)"
  echo "${repo_storage}/data/${object_path}"
}

get_local_repo_url() {
  local repo_name="$1"
  echo "http://localhost/cvmfs/${repo_name}"
}

get_s3_repo_url() {
  local repo_name="$1"
  local s3_stratum0_url="$CVMFS_TEST_HTTP_BASE"
  [ x"$s3_stratum0_url" != x"" ] || die "\$CVMFS_TEST_HTTP_BASE needs to be set for S3 tests to work!"
  [ $(echo -n "$s3_stratum0_url" | tail -c1) = "/" ] || s3_stratum0_url="${s3_stratum0_url}/"
  echo "${s3_stratum0_url}${repo_name}"
}

get_repo_url() {
  local repo_name="$1"
  if [ x"$CVMFS_TEST_S3_CONFIG" != x"" ]; then
    get_s3_repo_url "$repo_name"
  else
    get_local_repo_url "$repo_name"
  fi
}

get_object_url() {
  local repo_name="$1"
  local object_hash="$2"
  local repo_url="$(get_repo_url $repo_name)"
  local object_path="$(get_hash_path $object_hash)"
  echo "${repo_url}/data/${object_path}"
}

get_apache_config_filename() {
  local repo_name="$1"
  echo "${repo_name}.conf"
}

get_backend_spooler() {
  if [ -n "${TEST_CVMFS_RECEIVER_UPSTREAM_STORAGE}" ] ; then
    # A gateway test is running: access the backend storage directly
    echo "${TEST_CVMFS_RECEIVER_UPSTREAM_STORAGE}"
  else
    echo "${CVMFS_UPSTREAM_STORAGE}"
  fi
}


cleanup_legacy_repo_leftovers() {
  local legacy_repo_name="$1"
  local legacy_repo_storage="$(get_local_repo_storage $legacy_repo_name)"
  local apache_config_file="$(get_apache_config_filename $legacy_repo_name)"

  echo -n "cleanup (if necessary)... "
  has_repo $legacy_repo_name && die "fail"
  if [ -d $legacy_repo_storage ]; then
    sudo rm -fR $legacy_repo_storage
  fi
  if [ -f /etc/cvmfs/keys/${legacy_repo_name}.crt ] || \
     [ -f /etc/cvmfs/keys/${legacy_repo_name}.pub ] || \
     [ -f /etc/cvmfs/keys/${legacy_repo_name}.key ] || \
     [ -f /etc/cvmfs/keys/${legacy_repo_name}.masterkey ]; then
    sudo rm -f /etc/cvmfs/keys/${legacy_repo_name}.*
  fi
  remove_apache_config_file $apache_config_file
  echo "done"
}


# resigns a freshly planted legacy repository (see plant_legacy_repo())
# Note: This expects a matching keychain in /etc/keys/{$repo_name}
# @param repo_name  the FQRN of the legacy repository to be resigned
_resign_legacy_repository() {
  local repo_name="$1"
  local repo_storage="$(get_local_repo_storage $repo_name)"
  local legacy_repo_storage="${repo_storage}"
  [ -d ${repo_storage}/pub ] && legacy_repo_storage="${legacy_repo_storage}/pub/catalogs" # for 2.0.x style repositories
  local tmp_dir="$(pwd)/tmp"
  local upstream="local,${legacy_repo_storage}/data/txn,${legacy_repo_storage}"

  mkdir -p $tmp_dir || return 1

  # recreate whitelist
  local whitelist=${tmp_dir}/whitelist.${repo_name}
  echo `date -u "+%Y%m%d%H%M%S"` > ${whitelist}.unsigned
  echo "E`date -u --date='next month' "+%Y%m%d%H%M%S"`" >> ${whitelist}.unsigned
  echo "N$repo_name" >> ${whitelist}.unsigned
  openssl x509 -in /etc/cvmfs/keys/${repo_name}.crt -outform der | \
    cvmfs_publish hash -a sha1 -f >> ${whitelist}.unsigned || return 2
  local hash;
  hash=`cat ${whitelist}.unsigned | cvmfs_publish hash -a sha1`
  echo "--" >> ${whitelist}.unsigned
  echo $hash >> ${whitelist}.unsigned
  echo -n $hash > ${whitelist}.hash
  openssl rsautl -inkey /etc/cvmfs/keys/${repo_name}.masterkey -sign -in ${whitelist}.hash -out ${whitelist}.signature || return 3
  cat ${whitelist}.unsigned ${whitelist}.signature | sudo tee ${legacy_repo_storage}/.cvmfswhitelist > /dev/null       || return 4
  rm -f ${whitelist}.unsigned ${whitelist}.signature ${whitelist}.hash                                                 || return 5
}


# extracts a tarball into the root directory!
# Note: it expects the tarball to define the full destination path.
# @param tarball  the path to the tarball to be planted
# @param verbose  print the verbose output of tar
plant_tarball() {
  local tarball="$1"
  local verbose="$2"
  local olddir="$(pwd)"
  local tar_params="xzf"

  if [ x"$verbose" != x"" ]; then
    tar_params="${tar_params}v"
  fi

  cd /                          || return 101
  sudo tar $tar_params $tarball || return 102
  cd $olddir                    || return 103
}


# extracts a legacy repository contained in a tarball.
# Note: This expects the tarball to define the full destination path. Furthermore
#       it expects a repo keychain to be planted in /etc/keys/${legacy_repo_name}
# @param tarball           path to the tarball containing the legacy repo
# @param legacy_repo_name  FQRN of the repository to be planted
# @param repo_owner        user ID that is supposed to own the repository
plant_legacy_repository_revision() {
  local tarball="$1"
  local legacy_repo_name="$2"
  local repo_owner="$3"
  local verbose="$4"
  local legacy_repo_storage="$(get_local_repo_storage $legacy_repo_name)"

  plant_tarball "$tarball" "$verbose" || return 104

  # figure out if we just planted a CVMFS 2.0.x repository or something newer
  local is_cvmfs_20=0
  [ -d ${legacy_repo_storage}/pub ] && is_cvmfs_20=1

  # do a couple of sanity checks and adjustments
  if [ $is_cvmfs_20 -eq 1 ]; then
    [ -d ${legacy_repo_storage}/pub ]                                    || return 105
    [ -f ${legacy_repo_storage}/pub/catalogs/.cvmfscatalog ]             || return 106
    [ -f ${legacy_repo_storage}/pub/catalogs/.cvmfswhitelist ]           || return 107
    sudo touch ${legacy_repo_storage}/pub/catalogs/.cvmfs_master_replica || return 108
  else
    [ -d ${legacy_repo_storage}/data ]                      || return 105
    [ -f ${legacy_repo_storage}/.cvmfspublished ]           || return 106
    [ -f ${legacy_repo_storage}/.cvmfswhitelist ]           || return 107
    sudo touch ${legacy_repo_storage}/.cvmfs_master_replica || return 108
  fi

  _resign_legacy_repository "$legacy_repo_name" || return 109
}



# creates a path into the CAS of CVMFS server
#
# @param content_hash   the content hash to be checked
make_path() {
  local content_hash="$1"
  local characters=$(echo -n $content_hash | wc -c)
  local head_cnt=2
  local tail_cnt=$(( $characters - $head_cnt ))
  echo "data/$(echo -n $content_hash | head -c $head_cnt)/$(echo -n $content_hash | tail -c $tail_cnt)"
}


peek_backend_raw() {
  local repo_name="$1"
  local remote_path="$2"

  load_repo_config $repo_name
  cvmfs_swissknife peek -d $remote_path -r $(get_backend_spooler)
}


# peeks in the upstream storage to find a given content hash
#
# @param repository     the repository name to be queried
# @param content_hash   the content has to be checked for availability
# @return 0             on successfully found object
peek_backend() {
  local repo_name="$1"
  local content_hash="$2"

  peek_backend_raw $repo_name $(make_path $content_hash)
}


# downloads a random file from the backend storage of a CVMFS repository
#
# @param repository   the repository name to uploaded into
# @param remote_path  the relative remote path to be uploaded into
# @param local_path   the destination path to be downloaded into
download_from_backend() {
  local repo_name="$1"
  local remote_path="$2"
  local local_path="$3"

  load_repo_config $repo_name
  curl -o $local_path -s "$(get_repo_url $repo_name)/${remote_path}"
}


# removes a random file from the backend storage of a CVMFS repository
#
# @param repository   the repository name to uploaded into
# @param remote_path  the relative remote path to be uploaded into
delete_from_backend() {
  local repo_name="$1"
  local remote_path="$2"

  load_repo_config $repo_name
  cvmfs_swissknife remove -o $remote_path -r $(get_backend_spooler)
}

# removes the file given by its hash from the backend storage of a CVMFS repository
#
# @param repository    the repository name to uploaded into
# @param content_hash  hash of the file
delete_hash_from_backend() {
  local repo_name="$1"
  local content_hash="$2"

  load_repo_config $repo_name
  cvmfs_swissknife remove -o $(make_path $content_hash) -r $(get_backend_spooler)
}


# uploads a random file into the backend storage of a CVMFS repository
#
# @param repository   the repository name to uploaded into
# @param file_path    the file to be uploaded
# @param remote_path  the relative remote path to be uploaded into
upload_into_backend() {
  local repo_name="$1"
  local file_path="$2"
  local remote_path="$3"

  load_repo_config $repo_name

  local spooler=$(get_backend_spooler)
  if [ x"$(echo "$spooler" | cut -f1 -d',')" = x"S3" ]; then
    delete_from_backend $repo_name $remote_path # S3 does a lazy upload
  fi
  cvmfs_swissknife upload -i $file_path              \
                          -o $remote_path            \
                          -r $spooler                \
                          -a $CVMFS_HASH_ALGORITHM
}


# downloads and decompresses a backend object from a given repository
# @param repo_name   the repository to download the root catalog from
# @param object_hash the object to be downloaded and decompressed
# @return            prints the file name of the extracted catalog to stdout
download_and_decompress_object() {
  local repo_name="$1"
  local object_hash="$2"
  local object_url="$(get_object_url $repo_name $object_hash)"
  local destination="${object_hash}.uncompressed"

  curl --silent                 \
       --show-error $object_url \
    | cvmfs_swissknife zpipe -d > $destination || return 1

  echo $destination
}


# reads the upstream config part of a CernVM-FS server configuration file
# This is the third comma-separated field in the upstream string
read_upstream_config() {
  local upstream_string="$1"
  echo "$upstream_string" | cut -f3 -d','
}


# switches on/off the garbage collection for a given repository
#
# @param name  the name of the repository to be changed
# @return      0 return code on success
toggle_gc() {
  local name=$1
  local cfg_file="/etc/cvmfs/repositories.d/${name}/server.conf"
  local tmp_file="$(mktemp)"

  local value="$(cat $cfg_file | grep 'CVMFS_GARBAGE_COLLECTION' | sed -e 's/.*=\([a-z]\+\)/\1/')"
  local new_value="true"
  [ x"$value" = x"true" ] && new_value="false"

  cat $cfg_file | sed -e "s/\(CVMFS_GARBAGE_COLLECTION\)=[a-z]\+/\1=$new_value/" > $tmp_file
  sudo cp -f $tmp_file $cfg_file || return 1
  rm -f $tmp_file                || return 2
  return 0
}


get_object_from_manifest() {
  local name=$1
  local type=$2

  local url="$(get_repo_url $name)"
  hash=$(cvmfs_swissknife info -r $url -R | grep "^$type" | tr -d $type)
  echo "${hash}${type}"
}


set_auto_tag_timespan() {
  local name=$1
  local timespan="$2"

  local cfg_file="/etc/cvmfs/repositories.d/${name}/server.conf"
  local tmp_file="$(mktemp)"

  cat "$cfg_file" > "$tmp_file" || return 1
  cat >> $tmp_file << EOF
CVMFS_AUTO_TAG_TIMESPAN="$timespan"
EOF
  sudo cp $tmp_file $cfg_file || return 2
  rm -f $tmp_file
  return 0
}


# disables the automatic garbage collection of a given repository configuration
#
# @param name  the name of the repository to reconfigure
disable_auto_garbage_collection() {
  local name="$1"
  local cfg_file="/etc/cvmfs/repositories.d/${name}/server.conf"

  if cat $cfg_file | grep -q 'CVMFS_AUTO_GC'; then
    local tmp_cfg="$(mktemp)"
    cat $cfg_file | sed -e 's/^\(CVMFS_AUTO_GC\)=.*$/\1=false/' > $tmp_cfg || return 1
    sudo mv -f $tmp_cfg $cfg_file || return 2
    sudo chmod 0644 $cfg_file     || return 3
  fi
}


# get the current unix timestamp
get_timestamp() {
  date --date='2 seconds ago' +@%s
}

# display a timestamp in human readable form
display_timestamp() {
  local timestamp=$1
  date --date="$timestamp"
}

# this copies all top-level files from source to destination ignoring symlinks
# Note: Using the content of /bin or /usr/bin as integration test guinea pig was
#       never a good idea. This is pretty much technical debt for this poor
#       decision.
_cp_bindir() {
  local source_directory="$1"
  local destination_directory="$2"

  for f in $(find ${source_directory}/ -maxdepth 1 -type f); do
    sudo cp $f $destination_directory || return 1
  done
  sudo chmod -R a+rw $destination_directory || return 2

  for f in $(find $destination_directory -type f); do
    sudo chmod 0444 $f || return 3
  done
}

# copy all content of /bin/* into a given directory (for server testing purposes)
#
# @param destination_directory
cp_bin() {
  local destination_directory="$1"
  _cp_bindir "/bin" "$destination_directory"
}

# copy all content of /usr/bin/* into a given directory (for server testing purposes)
#
# @param destination_directory
cp_usrbin() {
  local destination_directory="$1"
  _cp_bindir "/usr/bin" "$destination_directory"
}


# prints a benchmark log message
benchmark_log() {
  local log_message=$1
  echo "===========    ${log_message}"
}


# waits until cvmfs is mounted
wait_fqrn_mount() {
  if [ x$PARROT_ENABLED = "xTRUE" ]; then
    return 0
  fi

  local waits=0
  while ! mount | grep -q "/cvmfs/${FQRN}"; do
    waits=$(( waits + 1 ))
    if [ "$waits" -ge 120 ]; then
      die "${FQRN} could not be mounted"
    fi
    sleep 1
  done
}


# runs the command once, mounting and unmounting the repository
single_execution() {
  cvmfs2 -f -o config=cvmfs.conf,disable_watchdog,simple_options_parsing $FQRN /cvmfs/$FQRN 2>&1 &
  wait_fqrn_mount
  ( cvmfs_run_benchmark )
  local code=$?
  cvmfs_umount $FQRN
  return $code
}


# creates the default cvmfs.conf file for normal testing
#
# @param destination_file  absolute path of the directory
create_default_cvmfs_config() {
  local destination_directory="$1"
  local config_file=$destination_directory/cvmfs.conf
  local test_folder=$( cd $(dirname $0) ; pwd -P )

  cat > $config_file << EOF
CVMFS_CACHE_BASE=${CVMFS_OPT_CACHEDIR}
CVMFS_RELOAD_SOCKETS=${CVMFS_OPT_CACHEDIR}
CVMFS_SERVER_URL=http://cvmfs-stratum-one.cern.ch/cvmfs/@fqrn@
CVMFS_SHARED_CACHE=no
CVMFS_HTTP_PROXY=${CVMFS_TEST_PROXY}
CVMFS_AUTO_UPDATE=false
CMS_LOCAL_SITE=${test_folder}/benchmarks/004-cms
CVMFS_QUOTA_LIMIT=-1
CVMFS_KEYS_DIR=/etc/cvmfs/keys/cern.ch
EOF

}

clean_benchmark_cache() {
  local cachedir="${CVMFS_OPT_CACHEDIR}/${FQRN}"
  if [ -d "$cachedir" ] && [ "x$cachedir" != x ]; then
  list=$(ls $cachedir | grep -P "^[0-9a-f]{2}$")
    for dir in $list; do
      rm -f "$cachedir/$dir"/*
    done
  fi
}

# sets the valgrind options depending on the test type
#
# @param test_type  concrete test to execute
set_valgrind_options() {
  local option="$1"
  local CVMFS_OPT_CALLGRIND="--tool=callgrind --dump-before=fuse_session_loop_mt --dump-after=fuse_session_loop_mt --dsymutil=yes --log-file=${FQRN}.callgrind.log -v"
  local CVMFS_OPT_MEMCHECK="--tool=memcheck --xml=yes --xml-file=${FQRN}.memcheck.xml --leak-check=yes --log-file=${FQRN}.memcheck.log -v"
  OPTIONS=""
  case $option in
    callgrind)
      OPTIONS=$CVMFS_OPT_CALLGRIND
    ;;
    memcheck)
      OPTIONS=$CVMFS_OPT_MEMCHECK
    ;;
  esac
}


# creates the environment to run the benchmark
#
# @param workspace  absolute path to the directory where the benchmark will put the files in
setup_benchmark_environment() {
  local workspace="$1"
  local me=$(whoami)
  mkdir -p "$CVMFS_OPT_OUTPUT_DIR/$FQRN"
  mkdir -p $workspace/cache

  # make sure the user can execute /bin/fusermount, write into /dev/fuse and write into /cvmfs/<repo>
  sudo chmod 666 /dev/fuse
  sudo chmod 4755 /bin/fusermount
  sudo mkdir -p /cvmfs/$FQRN
  sudo chown $me /cvmfs/$FQRN

  # mtab trick
  if [ ! -h /etc/mtab ]; then
    sudo sh -c "mv /etc/mtab /etc/mtab.save"
    sudo sh -c "ln -s /proc/mounts /etc/mtab"
    CVMFS_OPT_MTAB_MODIFIED="true"
  fi

  # check that the cache directory exists, is empty and is correctly labeled for SElinux
  if [ ! -d "$CVMFS_OPT_CACHEDIR" ]; then
    mkdir -p "$CVMFS_OPT_CACHEDIR"
    has_selinux
    local selinux=$?
    if [ "$selinux" -eq 0 ]; then
      chcon -Rv --type=cvmfs_cache_t $CVMFS_OPT_CACHEDIR
    fi
  else
    clean_benchmark_cache
  fi

  # create a cvmfs.conf file or use a custom one
  if [ -z "$CVMFS_OPT_CONFIG_FILE" ]; then
    create_default_cvmfs_config $workspace
  else
    cp -f "$CVMFS_OPT_CONFIG_FILE" $workspace/cvmfs.conf
  fi

  return 0
}


# creates a system.info file file with information about the environment
create_info_file() {
  cat > system.info << EOF
- Operating system:
$(uname -a)


- CernVM-FS version:
$(cvmfs2 --version)


- cvmfs2 executable md5:
$(md5sum $(which cvmfs2))


- CernVM-FS talk:
$(cvmfs_talk -p ${CVMFS_OPT_CACHEDIR}/${FQRN}/cvmfs_io.${FQRN} -i ${FQRN} internal affairs)

EOF

}


# creates a statistics file
create_statistics_file() {
  local iteration=$1
  local total_time=$2
  local memory_median=$3
  local memory_max=$4
  local statistics_file="${FQRN}_${iteration}.data"
  benchmark_log "creating the statistics file ${statistics_file} calling cvmfs_talk"
  cat >$statistics_file << EOF
# repo=$FQRN
# iterations=$CVMFS_OPT_ITERATIONS
# warm_cache=$CVMFS_OPT_WARM_CACHE

EOF

  benchmark_log "Executing cvmfs_talk"
  cvmfs_talk -p "$CVMFS_OPT_CACHEDIR"/$FQRN/cvmfs_io.$FQRN internal affairs | grep \| | tail -n +2 >> $statistics_file
  echo "global.sz_total_time|$total_time|Total execution time (seconds)" >> $statistics_file # of only one iteration without valgrind!
  echo "global.sz_memory_median|$memory_median|Median memory consumption (KB)" >> $statistics_file
  echo "global.sz_memory_max|$memory_max|Maximum memory consumption (KB)" >> $statistics_file
}


# collects data during the run
statistics_daemon() {
  local dump_file=$1
  local PID=$2
  touch $dump_file
  while [ -f /proc/${PID}/smaps ]; do
    # memory in KB
    echo 0 $(sudo awk '/Rss/ {print "+", $2}' /proc/${PID}/smaps) | bc >> $dump_file
    sleep 2
  done
}


# collects the median of a file
collect_median() {
  local file=$1
  local num_lines=$(cat $file | wc -l)
  local median_pos=$(( num_lines / 2 ))
  echo $(sort $file | sed "${median_pos}q;d")
}


# collects the maximum of a file
collect_maximum() {
  echo $(sort $1 | tail -n 1)
}


# executes the statistics loop
execute_statistics_loop() {
  local code=0
  local iteration=1
  local PID=0
  local processes=()
  local memory_file="memory.stat"

  while (( iteration <= $CVMFS_OPT_ITERATIONS && code == 0 )); do
    local start_time=$(date +%s)
    # cleaning the cache if necessary before executing the actual benchmark
    if [ x"$CVMFS_OPT_WARM_CACHE" != x"yes" ]; then
      clean_benchmark_cache
    fi

    cvmfs2 -f -o config=cvmfs.conf,disable_watchdog,simple_options_parsing $FQRN /cvmfs/$FQRN 2>&1 &
    PID=$!
    wait_fqrn_mount
    statistics_daemon $memory_file $PID >/dev/null 2>&1 & # it kills himself when there is no cvmfs process

    if [ x"$CVMFS_OPT_HOT_CACHE" = x"yes" ]; then
      benchmark_log "Starting early execution to fill the cache"
      ( cvmfs_run_benchmark ) > /dev/null 2>&1
      local code=$?
      if [ $code -ne 0 ]; then
        die "Failed in the hot cache execution"
      fi
    fi

    for i in $(seq 1 1 $CVMFS_OPT_PARALLEL_RUNS); do
      ( cvmfs_run_benchmark ) > /tmp/$FQRN.log.$i 2>&1 &
      processes[$i]=$!
    done

    for pid in ${processes[@]}; do
      wait $pid
      local retval=$?
      if [ $retval -ne 0 ]; then
        code=$retval
        echo "One thread failed in $FQRN. Check /tmp/$FQRN.log"
      fi
    done

    local end_time=$(date +%s)
    local total_time=$(( end_time - start_time )) # in seconds

    local memory_median=$(collect_median $memory_file)
    local memory_max=$(collect_maximum $memory_file)
    create_statistics_file $iteration $total_time $memory_median $memory_max
    cvmfs_umount $FQRN
    wait $PID
    iteration=$(( iteration + 1 ))
  done

  if [ "$code" -ne 0 ]; then
    benchmark_log "Problem executing the tests. Aborting"
    benchmark_log $(lsof /cvmfs/${FQRN})
  fi

  return $code
}


# executes the main valgrind's loop
execute_benchmark_loop() {
  local iteration=1
  local code=0
  local count=$CVMFS_OPT_ITERATIONS

  if [ "x$PARROT_ENABLED" = "xTRUE" ]; then
    benchmark_log "Valgrind is not available with Parrot"
    exit 100
  fi

  valgrind $OPTIONS cvmfs2 -f -o config=cvmfs.conf,disable_watchdog,simple_options_parsing $FQRN /cvmfs/$FQRN &
  PID=$!
  wait_fqrn_mount

  while (( count > 0 && code == 0 )); do
    # cleaning the cache if necessary before executing the actual benchmark
    if [ x"$CVMFS_OPT_WARM_CACHE" != x"yes" ]; then
      clean_benchmark_cache
    fi
    benchmark_log "Starting iteration ${iteration}"
    ( cvmfs_run_benchmark )
    code=$?
    count=$(( count - 1 ))
    iteration=$(( iteration + 1 ))
  done

  create_info_file

  if [ "$code" -ne 0 ]; then
    benchmark_log "Problem executing the tests. Aborting"
    benchmark_log $(lsof /cvmfs/${FQRN})
    exit 100
  fi

  benchmark_log "Iterations finished. Unmounting cvmfs"
  cvmfs_umount $FQRN
  wait $PID
}


# runs a benchmark $CVMFS_OPT_ITERATIONS times
run_benchmark() {
  local statistics_file="$FQRN.data"

  # use or not warm cache
  if [ x"$CVMFS_OPT_WARM_CACHE" = x"yes" ]; then
    benchmark_log "Starting initial execution"
    single_execution
    local code=$?
    if [ $code -ne 0 ]; then
      die "Failed in the warm cache execution"
    fi
  else
    benchmark_log "Using cold cache"
  fi

  # run the statistics collection mode
  if [ x"$CVMFS_OPT_TALK_STATISTICS" = x"yes" ]; then
    benchmark_log "Executing the statistics collection"
    execute_statistics_loop
    local code=$?
    if [ $code -ne 0 ]; then
      die "Failed in the statistics collection"
    fi
  fi

  if [ x"$CVMFS_OPT_VALGRIND" = x"yes" ]; then
    for tool in $CVMFS_OPT_TEST_TYPE; do
      set_valgrind_options $tool
      benchmark_log "Starting loading the profiler and mounting in /cvmfs/${FQRN}"
      benchmark_log "Running valgrind with the following parameters: $OPTIONS"
      benchmark_log "Launching the job $CVMFS_OPT_ITERATIONS times"
      execute_benchmark_loop
      local code=$?
      if [ $code -ne 0 ]; then
        die "Failed in Valgrind"
      fi
    done
  fi
}


# collects the files and pastes them into $CVMFS_OPT_OUTPUT_DIR/$FQRN
collect_benchmark_results() {
  local benchmark_output_dir="$CVMFS_OPT_OUTPUT_DIR/$FQRN"

  # undo the mtab trick
  if [ x"$CVMFS_OPT_MTAB_MODIFIED" = x"true" ]; then
    sudo mv /etc/mtab.save /etc/mtab
  fi

  # collect the results
  if [ -d "$benchmark_output_dir" ] && [ "x$benchmark_output_dir" != x ]; then
    rm -rf "$benchmark_output_dir"/*
  fi
  cp  -rf $workdir/system.info $workdir/$FQRN.*.log $workdir/callgrind.out.* $workdir/$FQRN.memcheck.xml $workdir/cvmfs.conf $workdir/*.data $benchmark_output_dir > /dev/null 2>&1
  benchmark_log "Test results copied into $benchmark_output_dir" >> $logfile
}


#
#  XUnit XML generation helper functions
#

xunit_preamble() {
  local xml_file="$1"
  local scratchdir="$2"

  local num_tests="$(cat ${scratchdir}/num_tests)"
  local num_fails="$(cat ${scratchdir}/num_failures)"
  local num_skips="$(cat ${scratchdir}/num_skipped)"
  local t_start="$(cat ${scratchdir}/starttime)"
  local t_elapsed="$(cat ${scratchdir}/elapsed)"

  cat > $xml_file << EOF
<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="$num_tests" failures="$num_fails" disabled="$num_skips" errors="0" timestamp="$t_start" time="$(milliseconds_to_seconds $t_elapsed)" name="CVMFS Test Runner">
  <testsuite hostname="$CVMFS_PLATFORM_NAME" name="$CVMFS_TEST_SUITE_NAME" timestamp="$CVMFS_TIMESTAMP" tests="$num_tests" failures="$num_fails" disabled="$num_skips" errors="0" time="$(milliseconds_to_seconds $t_elapsed)">
EOF
}

xunit_testcase() {
  local xml_file="$1"
  local test_scratchdir="$2"
  local logfile="$3"

  local test_name="$(cat ${test_scratchdir}/name)"
  local test_number="$(cat ${test_scratchdir}/number)"

  # check if the test was skipped
  if [ -f ${test_scratchdir}/skipped ]; then
    cat >> $xml_file << EOF
    <testcase name="$test_name" status="notrun" time="0.000" classname="$CVMFS_TEST_CLASS_NAME" />
EOF
    return 0
  fi

  local t_elapsed=$(cat ${test_scratchdir}/elapsed)

  # check if the test was a success
  if [ -f ${test_scratchdir}/success ]; then
    cat >> $xml_file << EOF
    <testcase name="$test_name" status="run" time="$(milliseconds_to_seconds $t_elapsed)" classname="$CVMFS_TEST_CLASS_NAME" />
EOF
    return 0
  fi

  local log_begin=$(cat ${test_scratchdir}/log_begin)
  local log_end=$(cat ${test_scratchdir}/log_end)
  local log_length=$(( $log_end - $log_begin ))

  # check if the test has failed
  if [ -f ${test_scratchdir}/failure ]; then
    local retval="$(cat ${test_scratchdir}/retval)"
    cat >> $xml_file << EOF
    <testcase name="$test_name" status="run" time="$(milliseconds_to_seconds $t_elapsed)" classname="$CVMFS_TEST_CLASS_NAME">
      <failure message="Failed with Retval $retval" type="">
      <![CDATA[$(head -n $log_end $logfile | tail -n $log_length | sed 's/[[:cntrl:]]//g')]]>
      </failure>
    </testcase>
EOF
    return 0
  fi
}

xunit_epilogue() {
  local xml_file="$1"
  cat >> $xml_file << EOF
  </testsuite>
</testsuites>
EOF
}

export_xunit_xml() {
  local xml_file="$1"
  local scratchdir="$2"
  local logfile="$3"

  xunit_preamble "$xml_file" "$scratchdir"
  for testcase in $(find $scratchdir -mindepth 1 -maxdepth 1 -type d); do
    xunit_testcase "$xml_file" "$testcase" "$logfile"
  done
  xunit_epilogue "$xml_file"
}


get_cache_directory() {
  local mountpoint="$1"
  echo "${mountpoint}c"
}

__do_local_mount() {
  local as_root="$1"
  local mountpoint="$2"
  local repo_name="$3"
  local repo_url="$4"
  local config_file="$5"   # config file containing CVMFS client options,
                           # if used the options listed below are not added
  local extra_option="$6"  # CVMFS client options as str
  local mount_options="$7" # comma separated mount options

  local cache="$(get_cache_directory $mountpoint)"
  local config="$(mktemp ${mountpoint}.conf.XXXXX)"
  local output="$(mktemp ${mountpoint}.log.XXXXX)"

  mkdir -p $mountpoint $cache
  if [ -z $config_file ]; then
    cat > $config << EOF
CVMFS_MOUNT_DIR=/cvmfs
CVMFS_CACHE_BASE=$cache
CVMFS_RELOAD_SOCKETS=$cache
CVMFS_SERVER_URL=$repo_url
CVMFS_HTTP_PROXY=DIRECT
CVMFS_PUBLIC_KEY=/etc/cvmfs/keys/${repo_name}.pub
CVMFS_KCACHE_TIMEOUT=15
CVMFS_DEBUGLOG=$output
CVMFS_USYSLOG=${mountpoint}.log
EOF
  else
    cat $config_file > $config
  fi
  printf "$extra_option\n" >> $config
  echo "  *** local mount config:"
  cat "$config"

  if [ ! -z $mount_options ]; then
    mount_options="${mount_options},"
  fi

  if [ x"$as_root" != x"" ]; then
    sudo cvmfs2 -d -o allow_other,${mount_options}config=$config $repo_name $mountpoint >> $output 2>&1
  else
    cvmfs2 -d -o ${mount_options}config=$config $repo_name $mountpoint >> $output 2>&1
  fi
}

do_local_mount() {
  __do_local_mount "" "$@"
}

do_local_mount_as_root() {
  __do_local_mount "sudo" "$@"
}

remove_local_mount() {
  local mountpoint="$1"
  local cache="$(get_cache_directory $mountpoint)"

  sudo umount $mountpoint
  rm -fR $cache $mountpoint
}

get_internal_value() {
  local repo=$1
  local key=$2

  echo $(sudo cvmfs_talk -i $repo internal affairs | grep "^$key" | cut -d\| -f2)
}

purge_disk_cache() {
  if running_on_osx; then
    sudo purge
  else
    sudo sh -c "echo 3 > /proc/sys/vm/drop_caches"
  fi
}

try_automount() {
  local repo=$1
  local retval
  if running_on_osx; then
    if mount | grep "$repo"; then
      retval=0
    else
      sudo mount -t cvmfs $repo /cvmfs/$repo >/dev/null 2>&1
      retval=$?
    fi
    sleep 3
    return $retval
  else
    ls /cvmfs/${repo}
  fi
}

# figures out where syslog is found on this system and prints it to stdout
# If $CVMFS_TEST_SYSLOG_TARGET is set, additionally to the system's syslog the
# path `cat $CVMFS_TEST_SYSLOG_TARGET` is sent to stdout
# Note: This function also `cat`s all file paths that are additionally passed to
#       it - appending the default syslog
#
# @param optional  a list of files that should be appended to stdout
cat_syslog() {
  sync
  sleep 1
  if running_on_osx; then
    log show --style syslog --info --last 4h
  else
    local syslog_file=""
    if [ -f "/var/log/syslog" ]; then
      syslog_file="/var/log/syslog"
    elif [ -f "/var/log/messages" ]; then
      syslog_file="/var/log/messages"
    fi

    local cat_retval=0
    if [ x"$syslog_file" != x"" ]; then
      sudo cat $syslog_file || cat_retval=$?
    elif which journalctl > /dev/null 2>&1; then
      sudo journalctl || cat_retval=$?
    else
      echo "default syslog not found on this system"
      cat_retval=1
    fi
  fi

  if [ x"$CVMFS_TEST_SYSLOG_TARGET" != x"" ]; then
    sudo cat $CVMFS_TEST_SYSLOG_TARGET || cat_retval=$?
  fi

  if [ $# -gt 0 ]; then
    sudo cat $@ || cat_retval=$?
  fi

  return $cat_retval
}


# look for the `cvmfs_preload` binary both installed in $PATH and
# pre-built in /tmp. If both lookups fail, try to build it from
# the provided source tree
#
# @param cvmfs_source_dir  absolute path to a CernVM-FS source tree
# @return                  location of `cvmfs_preload` or retval 1
find_or_build_cvmfs_preload() {
  local cvmfs_source_dir="$1"
  local cvmfs_preload_build_dir="/tmp/cvmfs_test_preload_build"
  local cvmfs_preload_build_log="${cvmfs_preload_build_dir}/build.log"
  local cvmfs_preload_path=

  # check if cvmfs_preload is readily available (i.e. installed in $PATH)
  if which cvmfs_preload > /dev/null 2>&1; then
    which cvmfs_preload
    return 0
  fi

  # check if cvmfs_preload has been built before
  if [ -x ${cvmfs_preload_build_dir}/cvmfs/cvmfs_preload ]; then
    echo "${cvmfs_preload_build_dir}/cvmfs/cvmfs_preload"
    return 0
  fi

  # build cvmfs_preload from source $cvmfs_source_dir
  rm -fR   $cvmfs_preload_build_dir > /dev/null 2>&1 || return 1
  mkdir -p $cvmfs_preload_build_dir > /dev/null 2>&1 || return 2
  pushdir  $cvmfs_preload_build_dir > /dev/null 2>&1 || return 3

  cmake -DBUILD_PRELOADER=yes      \
        -DBUILD_SERVER=no          \
        -DBUILD_SERVER_DEBUG=no    \
        -DBUILD_CVMFS=no           \
        -DBUILD_UNITTESTS=no       \
        -DBUILD_UNITTESTS_DEBUG=no \
        -DBUILD_LIBCVMFS=no        \
        -DBUILD_LIBCVMFS_CACHE=no  \
        -DINSTALL_MOUNT_SCRIPTS=no \
        -DBUILD_DOCUMENTATION=no   \
        -DBUILD_RECEIVER=no        \
        $cvmfs_source_dir > $cvmfs_preload_build_log 2>&1 || return 4
  make -j $(nproc)        > $cvmfs_preload_build_log 2>&1 || return 5

  popdir > /dev/null 2>&1

  # check (again) if cvmfs_preload has been built properly and return
  if [ -x ${cvmfs_preload_build_dir}/cvmfs/cvmfs_preload ]; then
    echo "${cvmfs_preload_build_dir}/cvmfs/cvmfs_preload"
    return 0
  fi

  return 1
}


# Sets up the CVMFS repository gateway application
set_up_repository_gateway() {
    echo "Setting up repository gateway"

    gateway_script=/usr/libexec/cvmfs-gateway/scripts/run_cvmfs_gateway.sh
    if [ -f /usr/lib/cvmfs-gateway/scripts/run_cvmfs_gateway.sh ]; then
      gateway_script=/usr/lib/cvmfs-gateway/scripts/run_cvmfs_gateway.sh
    fi

    echo "  Stopping repository gateway"
    sudo $gateway_script stop

    echo "  Cleaning up the repository gateway db"
    sudo rm -rf /var/lib/cvmfs-gateway/*

    echo "  Cleaning up the debug log"
    sudo mkdir -p /var/log/cvmfs_receiver
    sudo rm -f /var/log/cvmfs_receiver/debug.log

    echo "  Writing configuration files"
    sudo bash -c 'cat <<EOF > /etc/cvmfs/gateway/repo.json
{
    "repos" : [
        {
            "domain" : "test.repo.org",
            "keys" : ["key1"]
        }
    ],
    "keys" : [
        {
            "type" : "file",
            "file_name" : "/etc/cvmfs/keys/test.repo.org.gw",
            "repo_subpath" : "/"
        }
    ]
}
EOF'

    echo "  Creating test repo"
    create_repo test.repo.org `whoami`

    echo "  Writing API key file"
    sudo bash -c 'cat <<EOF > /etc/cvmfs/keys/test.repo.org.gw
plain_text key1 secret1
EOF'

    echo "  Modifying test repo configuration"
    local repo_server_config=/etc/cvmfs/repositories.d/test.repo.org/server.conf
    local original_upstream=$(grep CVMFS_UPSTREAM_STORAGE $repo_server_config | cut -d'=' -f2-)
    sudo bash -c "echo CVMFS_UPSTREAM_STORAGE=gw,/srv/cvmfs/test.repo.org/txn,http://localhost:4929/api/v1 >> $repo_server_config"
    sudo bash -c "echo TEST_CVMFS_RECEIVER_UPSTREAM_STORAGE=$original_upstream >> $repo_server_config"
    sudo sed -i -e "s/CVMFS_ROOT_HASH=.*//" /var/spool/cvmfs/test.repo.org/client.local

    echo "  Starting repository gateway"
    sudo $gateway_script start
    # Let the service boot up
    sleep 1
    $gateway_script status
    $gateway_script status
    $gateway_script status
}

restart_repository_gateway() {
    echo "Restarting repository gateway"

    gateway_script=/usr/libexec/cvmfs-gateway/scripts/run_cvmfs_gateway.sh
    if [ -f /usr/lib/cvmfs-gateway/scripts/run_cvmfs_gateway.sh ]; then
      gateway_script=/usr/lib/cvmfs-gateway/scripts/run_cvmfs_gateway.sh
    fi

    sudo $gateway_script restart
    while ! $gateway_script status; do
      sleep 1
    done
}

grep_receiver_debuglog() {
  local needle="$1"

  sudo grep "$needle" /var/log/cvmfs_receiver/debug.log
}

lockfile() {
  if [ ! -x lockfile ]; then
    gcc -Wall -o lockfile ${TEST_ROOT}/common/lockfile.c || return 10
  fi
  ./lockfile $1
}
