.travis.yml

# vim ft=yaml
# travis-ci.org definition for DataLad build
language: python

# Do full clone (~10 extra seconds) to fetch all tags.
# Otherwise we might be missing the tags for maint PRs
# whenever those maint releases were not yet merged into master.
# TODO: specify TRAVIS_BRANCH if ever would become possible
#(support request was sent to Travis)
git:
  depth: false

services:
  - docker

python:
  - 3.7

cache:
  - apt

env:
  global:
    # will be used in the matrix, where neither other variable is used
    - BOTO_CONFIG=/tmp/nowhere
    - DATALAD_TESTS_SSH=1
    - DATALAD_LOG_ENV=GIT_SSH_COMMAND
    - TESTS_TO_PERFORM=datalad
    # Should be an array, travis breaks on it, define/adjust in the "before_install"
    #- PYTEST_OPTS=( -v )
    # Note, that there's "turtle" as well, which is always excluded from
    # running on Travis.
    - PYTEST_SELECTION="integration or usecase or slow or network"
    - PYTEST_SELECTION_OP="not "   # so it would be "not (integration or usecase)"
    - DATALAD_DATASETS_TOPURL=https://datasets-tests.datalad.org
    # How/which git-annex we install.  conda's build would be the fastest, but it must not
    # get ahead in PATH to not shadow travis' python
    - _DL_ANNEX_INSTALL_SCENARIO="miniconda=py37_23.1.0-1 --python-match minor --batch git-annex=8.20201007 -m conda"

matrix:
  include:
  # Additional custom ones
  - python: 3.8
    # Single run for Python 3.7
    env:
    # Run all tests in a single whoop here
    # We cannot have empty -A selector, so the one which always will be fulfilled
    - PYTEST_SELECTION=
    - PYTEST_SELECTION_OP=not
    # To test https://github.com/datalad/datalad/pull/4342 fix.
    # From our testing in that PR seems to have no effect, but kept around since should not hurt.
    - LC_ALL=ru_RU.UTF-8
  - python: 3.9
    dist: bionic # Xenial doesn't have 3.8 pre-installed (only 3.8-dev).
    env:
    - PYTEST_SELECTION=
    - PYTEST_SELECTION_OP=not
  # Two matrix runs for "recent python and git-annex with the recent supported by git annex
  # new version of repo" and various extra options/features enabled for git-annex
  - python: '3.11'
    dist: focal
    env:
      - PYTEST_SELECTION=
      - PYTEST_SELECTION_OP=not
      - DATALAD_REPO_VERSION=10
      - _DL_ANNEX_INSTALL_SCENARIO="miniconda --channel conda-forge --python-match minor --batch git-annex -m conda"
      - DATALAD_TESTS_GITCONFIG="\n[annex]\n stalldetection = 1KB/120s\n"
  - python: '3.11'
    dist: focal
    env:
      - PYTEST_SELECTION=
      - PYTEST_SELECTION_OP=
      - DATALAD_REPO_VERSION=10
      - _DL_ANNEX_INSTALL_SCENARIO="miniconda --channel conda-forge --python-match minor --batch git-annex -m conda"
      - DATALAD_TESTS_GITCONFIG="\n[annex]\n stalldetection = 1KB/120s\n"
  - if: type = cron
    python: 3.7
    # Single run for Python 3.7
    env:
    # Run all tests in a single whoop here
    # We cannot have empty -A selector, so the one which always will be fulfilled
    - PYTEST_SELECTION=
    - PYTEST_SELECTION_OP=not
  - python: 3.7
    # Split runs for v6 since a single one is too long now
    env:
    - PYTEST_SELECTION_OP=not
    - DATALAD_SSH_MULTIPLEX__CONNECTIONS=0
    - DATALAD_RUNTIME_PATHSPEC__FROM__FILE=always
    - _DL_ANNEX_INSTALL_SCENARIO="miniconda=py37_23.1.0-1 --python-match minor --batch git-annex=10.20220525 -m conda"
  - python: 3.7
    env:
    - PYTEST_SELECTION_OP=""
    - DATALAD_SSH_MULTIPLEX__CONNECTIONS=0
    - DATALAD_RUNTIME_PATHSPEC__FROM__FILE=always
    - _DL_ANNEX_INSTALL_SCENARIO="miniconda=py37_23.1.0-1 --python-match minor --batch git-annex=8.20210310 -m conda"
    # To test https://github.com/datalad/datalad/pull/4342 fix in case of no "not" for pytest.
    # From our testing in that PR seems to have no effect, but kept around since should not hurt.
    - LANG=bg_BG.UTF-8
  - python: 3.7
   # Run slow etc tests under a single tricky scenario
    env:
    - _DL_TMPDIR="/var/tmp/sym link"
    - PYTEST_SELECTION_OP=""
    # And the leading - in filenames for the most challenge
    - DATALAD_TESTS_OBSCURE_PREFIX=-
    - DATALAD_LOG_TRACEBACK=collide  # just a smoke test for now
  - python: 3.7
    # A run loaded with various customizations to smoke test those functionalities
    # apparently moving symlink outside has different effects on abspath
    # see  https://github.com/datalad/datalad/issues/878
    env:
    # eventually: - _DL_TMPDIR="/var/tmp/sym ссылка"
    - _DL_TMPDIR="/var/tmp/sym link"
    # and obscure the names more a bit
    - DATALAD_TESTS_OBSCURE_PREFIX=-
    # By default no logs will be output. This one is to test with log output at INFO level
    - DATALAD_LOG_LEVEL=INFO
    - DATALAD_LOG_TRACEBACK=1  # just a smoke test for now
    - DATALAD_LOG_VMEM=1
    - DATALAD_RUNTIME_MAX__BATCHED=2
    - DATALAD_RUNTIME_MAX__INACTIVE__AGE=10
  - python: 3.7
    # By default no logs will be output. This one is to test with low level but dumped to /dev/null
    env:
    - DATALAD_LOG_LEVEL=2
    - DATALAD_LOG_TARGET=/dev/null
    - DATALAD_TESTS_PROTOCOLREMOTE=1
    - DATALAD_TESTS_DATALADREMOTE=1
    - DATALAD_LOG_CWD=1
    - DATALAD_LOG_OUTPUTS=1
    - DATALAD_LOG_ENV=1
    - DATALAD_LOG_STDIN=1
    - DATALAD_TESTS_UI_BACKEND=console
    - DATALAD_TESTS_OBSCURE_PREFIX=-
    - DATALAD_SEED=1
    - GIT_AUTHOR_DATE="Thu, 07 Apr 2005 22:13:13 +0200"
    - GIT_AUTHOR_NAME=blah
    - GIT_AUTHOR_EMAIL=committer@example.com
    - GIT_COMMITTER_DATE="Thu, 07 Apr 2005 22:13:13 +0200"
    - GIT_COMMITTER_NAME=blah
    - GIT_COMMITTER_EMAIL=committer@example.com
  - python: 3.7
    # Test some under NFS mount  (only selected sub-set)
    env:
    # do not run SSH-based tests due to stall(s)
    # https://github.com/datalad/datalad/pull/4172
    - DATALAD_TESTS_SSH=0
    - _DL_TMPDIR="/tmp/nfsmount"
    - TESTS_TO_PERFORM="datalad.tests datalad.support"
    #
  # The ones to run only on weekends against master.
  # They will not contribute to coverage etc, but might lead to failed status
  #
  # run with minimal supported git-annex version as defined in AnnexRepo.GIT_ANNEX_MIN_VERSION
  # TODO: ATM we do not have that minimal version as a Debian package in
  # snapshots!
  - if: type = cron
    python: 3.7
    env:
    - _DL_ANNEX_INSTALL_SCENARIO="miniconda=py37_23.1.0-1 --python-match minor --batch git-annex=8.20200309 -m conda"
  # Run with git's master branch rather the default one on the system.
  - if: type = cron
    python: 3.7
    env:
    - DATALAD_USE_DEFAULT_GIT=1
    - _DL_UPSTREAM_GIT=1
  # Run with our reported minimum Git version.
  - if: type = cron
    python: 3.7
    env:
    - DATALAD_USE_DEFAULT_GIT=1
    - _DL_MIN_GIT=1
    - PATH="$PWD/git-src/bin-wrappers/:$PATH"
  - if: type = cron
    python: 3.7
    env:
    # to test operation under root since also would consider FS "crippled" due to
    # ability to rewrite R/O files
    - PYTEST_WRAPPER="sudo -E"
    # no key authentication for root:
    - DATALAD_TESTS_SSH=0
  - if: type = cron
    python: 3.7
    env:
    - DATALAD_TESTS_NONETWORK=1
    # must operate nicely with those env variables set
    - http_proxy=
    - https_proxy=
  - if: type = cron
    python: 3.7
    # Test under NFS mount  (full, only in master)
    env:
    - _DL_TMPDIR="/tmp/nfsmount"

  allow_failures:
  # Test under NFS mount  (full, only in master)
  - if: type = cron
    python: 3.7
    env:
    - _DL_TMPDIR="/tmp/nfsmount"

# Causes complete laptop or travis instance crash atm, but survives in a docker
# need to figure it out (looks like some PID explosion)
#  - python: 3.7
#    # we would need to migrate to boto3 to test it fully, but SSH should work
#    env:
#    - DATALAD_TESTS_SSH=1
#    - UNSET_S3_SECRETS=1

before_install:
  - PYTEST_OPTS=( -v )
  # If we requested to run only not slow (typically <10sec) tests, fail if a test
  # takes 3x more than that - it needs to get @slow or @turtle annotation
  - if echo "$PYTEST_SELECTION_OP($PYTEST_SELECTION)" | grep -q "^not.*slow"; then
      PYTEST_OPTS=( "${PYTEST_OPTS[@]}" --doctest-modules --durations=0 --durations-min=5 --fail-slow 60 );
      export DATALAD_TESTS_SETUP_TESTREPOS=1;
    fi
  # Show git describe output to ensure that we did fetch all the tags etc
  - git describe
  # Just in case we need to check if nfs is there etc
  - sudo lsmod
  # The ultimate one-liner setup for NeuroDebian repository
  - bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh)
  - travis_retry sudo apt-get update -qq
  - travis_retry sudo apt-get install eatmydata  # to speedup some installations
  - tools/ci/prep-travis-forssh.sh
  - tools/ci/debians_disable_outdated_ssl_cert
  # Install various basic dependencies
  - travis_retry sudo eatmydata apt-get install zip pandoc p7zip-full
  # needed for tests of patool compression fall-back solution
  - travis_retry sudo eatmydata apt-get install xz-utils
  - travis_retry sudo eatmydata apt-get install shunit2
  # Configure _DL_TMPDIR before trying install git-annex -- script might use it
  - if [[ "${_DL_TMPDIR:-}" =~ .*/sym\ link ]]; then echo "Symlinking $_DL_TMPDIR"; ln -s /tmp "$_DL_TMPDIR"; fi
  - if [[ "${_DL_TMPDIR:-}" =~ .*/d\ i\ r ]]; then echo "mkdir $_DL_TMPDIR"; mkdir -p "$_DL_TMPDIR"; fi
  - if [[ "${_DL_TMPDIR:-}" =~ .*/nfsmount ]]; then echo "mkdir $_DL_TMPDIR"; mkdir -p "$_DL_TMPDIR" "${_DL_TMPDIR}_"; echo "/tmp/nfsmount_ localhost(rw)" | sudo bash -c 'cat - > /etc/exports'; sudo apt-get install -y nfs-kernel-server; sudo exportfs -a; sudo mount -t nfs localhost:/tmp/nfsmount_ /tmp/nfsmount; fi
  # Maybe build install custom git.
  - if [ ! -z "${_DL_UPSTREAM_GIT:-}" ]; then source tools/ci/install-upstream-git.sh; fi
  - if [ ! -z "${_DL_MIN_GIT:-}" ]; then tools/ci/install-minimum-git.sh; fi
  # Install git-annex
  - pip install datalad-installer
  - eval datalad-installer --sudo ok -E new.env ${_DL_ANNEX_INSTALL_SCENARIO}
  - source new.env && cat new.env >> ~/.bashrc
  - pip install --upgrade pip

install:
  - git config --global user.email "test@travis.land"
  - git config --global user.name "Travis Almighty"
  # we are pip sudo install  below and versioneer needs to run git. recent git needs to
  # be made certain it is safe to do
  - sudo git config --global --add safe.directory $PWD
  - cd ..; pip install -q codecov; cd -
  - pip install -r requirements-devel.txt
  # So we could test under sudo -E with PATH pointing to installed location
  - sudo sed -i -e 's/^Defaults.*secure_path.*$//' /etc/sudoers
  # git-annex workaround. TODO: remove - should not be needed
  - if [[ "${_DL_TMPDIR:-}" =~ .*/nfsmount ]]; then sudo git config --system annex.pidlock true ; fi

script:
  # Now it should be safe to point TMPDIR to a "tricky" setup just for the purpose of testing
  - if [ -n "${_DL_TMPDIR:-}" ]; then export TMPDIR="${_DL_TMPDIR}"; fi
  # Test installation for user
  - sudo pip install --user .
  # Report WTF information using system wide installed version
  - datalad wtf
  - mkdir -p __testhome__
  - cd __testhome__
  # Run tests
  #  Note: adding --log-cli-level=INFO would result in DATALAD_LOG_TARGET=/dev/null being not
  #  in effect, dumping too many logs.
  - set -x; http_proxy=
    PATH=$PWD/../tools/coverage-bin:$PATH
    $PYTEST_WRAPPER python
      -m pytest "${PYTEST_OPTS[@]}"
      -c ../tox.ini
      -n 2
      -m "${PYTEST_SELECTION:+$PYTEST_SELECTION_OP($PYTEST_SELECTION) and }not(turtle)"
      --doctest-modules
      --cov=datalad
      --cov-report=
      --pyargs
      $TESTS_TO_PERFORM
  - cd ..

after_success:
  # cron jobs test more and then PRs will be falling behind since they would not
  # trigger some codepaths.  So submit coverage only from non-cron jobs, but report for all
  - cd __testhome__;
  - python -m coverage combine -a /tmp/.coverage-entrypoints-*;
  - python -m coverage report;
  - if [ ! ${TRAVIS_EVENT_TYPE} = "cron" ]; then
      codecov;
    fi

# makes it only more difficult to comprehend the failing output.  Enable only when necessary
# for a particular debugging
#after_failure:
#  - if [ ! -z "$DATALAD_TESTS_NONETWORK" ]; then sudo route add -net 0.0.0.0 netmask 0.0.0.0 dev lo; fi
#  - DATALAD_LOG_LEVEL=DEBUG $PYTEST_WRAPPER `which pytest` -s -v --doctest-modules --cov datalad --log-cli-level=DEBUG
#  - if [ ! -z "$DATALAD_TESTS_NONETWORK" ]; then sudo route del -net 0.0.0.0 netmask 0.0.0.0 dev lo; fi