.gitlab-ci.yml

stages:
  - stage_test
  - stage_build_python
  - stage_build_docker
  - stage_build_documentation
  - stage_push_python
  - stage_push_docker
  - stage_push_documentation
  - stage_deploy

# Docker

job_docker_build:
  before_script:
    - set -a && . ./config.env && set +a
    - docker login --username "$CI_REGISTRY_USER" --password "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
    - apk update
    - apk add cairo gcc libffi-dev libjpeg-turbo-dev make musl-dev python3 python3-dev zlib-dev
    - python3 -m venv venv
    - . venv/bin/activate
    - pip install pip==$PYTHON_PIP_VERSION
    - pip install wheel
    - cat requirements/base.txt | grep -v 'django-auth-ldap'| pip install -r /dev/stdin
  cache:
    paths:
      - ~/.cache/pip/
      - /var/cache/apk
  image: docker:20-dind
  only:
    - releases/all_major
    - releases/all_minor
    - releases/docker_major
    - releases/docker_minor
    - tests/all
    - tests/docker
  script:
    - mkdir -p /etc/docker
    - if [ "${REGISTRY_MIRRORS}" ]; then echo "{\"registry-mirrors\":\[\"${REGISTRY_MIRRORS}\"\]}" > /etc/docker/daemon.json; fi
    - DOCKER_VERSION=`cat docker/rootfs/version`
    - DOCKER_IMAGE_MAYAN_NAME=$CI_REGISTRY_IMAGE
    - make docker-build
    - docker tag "mayanedms/mayanedms:$DOCKER_VERSION" "$CI_REGISTRY_IMAGE:$DOCKER_VERSION"
    - SERVICE_POSTGRES_IP=$(cat /etc/hosts | awk '{if ($2 == "postgres") print $1;}')
    - docker run --rm --add-host="postgres:$SERVICE_POSTGRES_IP" -e MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql_psycopg2','NAME':'mayan','USER':'mayan','PASSWORD':'mayanuserpass','HOST':'postgres'}}" "$CI_REGISTRY_IMAGE:$DOCKER_VERSION" run_tests
    - docker login --username "$CI_REGISTRY_USER" --password "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
    - docker push "$CI_REGISTRY_IMAGE:$DOCKER_VERSION"
    - docker logout $CI_REGISTRY
  services:
    - docker:20-dind
    - postgres:12.11-alpine
  stage: stage_build_docker
  tags:
    - postgres
  variables:
    DOCKER_DRIVER: overlay2
    POSTGRES_DB: "mayan"
    POSTGRES_PASSWORD: "mayanuserpass"
    POSTGRES_USER: "mayan"

job_docker_push_major:
  before_script:
    - apk add --no-cache python3
    - docker login --username "$CI_REGISTRY_USER" --password "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
  image: docker:20-dind
  only:
    - releases/all_major
    - releases/docker_major
  script:
    - PYTHON_BIN=/usr/bin/python3
    - DOCKER_VERSION=`cat docker/rootfs/version`
    - docker pull "$CI_REGISTRY_IMAGE:$DOCKER_VERSION"
    - docker tag "$CI_REGISTRY_IMAGE:$DOCKER_VERSION" registry-1.docker.io/mayanedms/mayanedms:v"$DOCKER_VERSION"
    - docker tag "$CI_REGISTRY_IMAGE:$DOCKER_VERSION" registry-1.docker.io/mayanedms/mayanedms:latest
    - VERSION_SCRIPT_BIN=contrib/scripts/version.py
    - MAJOR_VERSION=$($PYTHON_BIN $VERSION_SCRIPT_BIN $DOCKER_VERSION major);
    - docker tag "$CI_REGISTRY_IMAGE:$DOCKER_VERSION" registry-1.docker.io/mayanedms/mayanedms:s"$MAJOR_VERSION"
    - MINOR_VERSION=$($PYTHON_BIN $VERSION_SCRIPT_BIN $DOCKER_VERSION minor);
    - docker tag "$CI_REGISTRY_IMAGE:$DOCKER_VERSION" registry-1.docker.io/mayanedms/mayanedms:s"$MINOR_VERSION"
    - docker logout $CI_REGISTRY
    - docker login --username "$DOCKER_HUB_USERNAME" --password "$DOCKER_HUB_PASSWORD" https://registry-1.docker.io
    - docker push registry-1.docker.io/mayanedms/mayanedms:v"$DOCKER_VERSION"
    - docker push registry-1.docker.io/mayanedms/mayanedms:latest
    - docker push registry-1.docker.io/mayanedms/mayanedms:s"$MAJOR_VERSION"
    - docker push registry-1.docker.io/mayanedms/mayanedms:s"$MINOR_VERSION"
  services:
    - docker:20-dind
  stage: stage_push_docker
  variables:
    DOCKER_DRIVER: overlay2

job_docker_push_minor:
  before_script:
    - apk add --no-cache python3
    - docker login --username "$CI_REGISTRY_USER" --password "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
  image: docker:20-dind
  only:
    - releases/all_minor
    - releases/docker_minor
  script:
    - PYTHON_BIN=/usr/bin/python3
    - DOCKER_VERSION=`cat docker/rootfs/version`
    - docker pull "$CI_REGISTRY_IMAGE:$DOCKER_VERSION"
    - docker tag "$CI_REGISTRY_IMAGE:$DOCKER_VERSION" registry-1.docker.io/mayanedms/mayanedms:v"$DOCKER_VERSION"
    - docker tag "$CI_REGISTRY_IMAGE:$DOCKER_VERSION" registry-1.docker.io/mayanedms/mayanedms:latest
    - VERSION_SCRIPT_BIN=contrib/scripts/version.py
    - MINOR_VERSION=$($PYTHON_BIN $VERSION_SCRIPT_BIN $DOCKER_VERSION minor);
    - docker tag "$CI_REGISTRY_IMAGE:$DOCKER_VERSION" registry-1.docker.io/mayanedms/mayanedms:s"$MINOR_VERSION"
    - docker logout $CI_REGISTRY
    - docker login --username "$DOCKER_HUB_USERNAME" --password "$DOCKER_HUB_PASSWORD" https://registry-1.docker.io
    - docker push registry-1.docker.io/mayanedms/mayanedms:v"$DOCKER_VERSION"
    - docker push registry-1.docker.io/mayanedms/mayanedms:latest
    - docker push registry-1.docker.io/mayanedms/mayanedms:s"$MINOR_VERSION"
  services:
    - docker:20-dind
  stage: stage_push_docker
  variables:
    DOCKER_DRIVER: overlay2

job_docker_nightly:
  before_script:
    - set -a && . ./config.env && set +a
    - docker login --username "$CI_REGISTRY_USER" --password "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
    - apk update
    - apk add cairo gcc libffi-dev libjpeg-turbo-dev make musl-dev python3 python3-dev zlib-dev
    - python3 -m venv venv
    - . venv/bin/activate
    - pip install pip==$PYTHON_PIP_VERSION
    - pip install wheel
    - cat requirements/base.txt | grep -v 'django-auth-ldap'| pip install -r /dev/stdin
  cache:
    paths:
      - ~/.cache/pip/
      - /var/cache/apk
  image: docker:20-dind
  only:
    - releases/nightly
    - releases/staging
  script:
    - mkdir -p /etc/docker
    - if [ "${REGISTRY_MIRRORS}" ]; then echo "{\"registry-mirrors\":\[\"${REGISTRY_MIRRORS}\"\]}" > /etc/docker/daemon.json; fi
    - DOCKER_VERSION=`cat docker/rootfs/version`
    - DOCKER_IMAGE_MAYAN_NAME=$CI_REGISTRY_IMAGE
    - make docker-build
    - docker tag "mayanedms/mayanedms:$DOCKER_VERSION" "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG"
    - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG"
    - docker logout $CI_REGISTRY
  services:
    - docker:20-dind
  stage: stage_build_docker
  variables:
    DOCKER_DRIVER: overlay2

job_docker_testing:
  before_script:
    - set -a && . ./config.env && set +a
    - docker login --username "$CI_REGISTRY_USER" --password "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
    - apk update
    - apk add cairo gcc libffi-dev libjpeg-turbo-dev make musl-dev python3 python3-dev zlib-dev
    - python3 -m venv venv
    - . venv/bin/activate
    - pip install pip==$PYTHON_PIP_VERSION
    - pip install wheel
    - cat requirements/base.txt | grep -v 'django-auth-ldap'| pip install -r /dev/stdin
  cache:
    paths:
      - ~/.cache/pip/
      - /var/cache/apk
  image: docker:20-dind
  only:
    - releases/testing
    - releases/testing_docker
  script:
    - mkdir -p /etc/docker
    - if [ "${REGISTRY_MIRRORS}" ]; then echo "{\"registry-mirrors\":\[\"${REGISTRY_MIRRORS}\"\]}" > /etc/docker/daemon.json; fi
    - DOCKER_VERSION=`cat docker/rootfs/version`
    - DOCKER_IMAGE_MAYAN_NAME=$CI_REGISTRY_IMAGE
    - make docker-build
    - docker tag "mayanedms/mayanedms:$DOCKER_VERSION" "$CI_REGISTRY_IMAGE:$DOCKER_VERSION"
    - docker push "$CI_REGISTRY_IMAGE:$DOCKER_VERSION"
    - docker logout $CI_REGISTRY
  services:
    - docker:20-dind
  stage: stage_build_docker
  variables:
    DOCKER_DRIVER: overlay2

# Documentation

job_documentation_build:
  artifacts:
    expire_in: 2 hr
    paths:
    - docs
  before_script:
    - apt-get update
    - if [ "${APT_PROXY}" ]; then echo "Acquire::http { Proxy \"http://${APT_PROXY}\"; };" > /etc/apt/apt.conf.d/01proxy; fi
    - apt-get install --yes enchant-2 gcc make libjpeg-dev libpng-dev python3-venv python3-dev zlib1g-dev
    - python3 -m venv venv
    - . venv/bin/activate
  image: debian:11.4-slim
  only:
    - releases/all_major
    - releases/documentation
    - tests/all
    - tests/documentation
  script:
    - pip install --requirement requirements/documentation.txt
    - pip install --requirement requirements/documentation_override.txt
    - cd docs
    - make html
  stage: stage_build_documentation

job_documentation_push:
  before_script:
    - mkdir --parents ~/.ssh
    - chmod 700 ~/.ssh
    - echo "$DOCUMENTATION_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
    - chmod 644 ~/.ssh/known_hosts
    - 'which ssh-agent || ( apt-get update --yes && apt-get install --yes openssh-client rsync )'
    - eval $(ssh-agent -s)
    - echo "$DOCUMENTATION_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
  image: debian:11.4-slim
  only:
    - releases/all_major
    - releases/documentation
  script:
    - ssh $DOCUMENTATION_SSH_USERNAME@$DOCUMENTATION_HOSTNAME "rm -Rf $DOCUMENTATION_SERVER_PATH/*"
    - rsync -avzP --progress -e ssh docs/_build/html/* $DOCUMENTATION_SSH_USERNAME@$DOCUMENTATION_HOSTNAME:$DOCUMENTATION_SERVER_PATH
  stage: stage_push_documentation

# Python

job_python_build:
  artifacts:
    expire_in: 2 hr
    paths:
    - dist
  before_script:
    - set -a && . ./config.env && set +a
    - apt-get update
    - if [ "${APT_PROXY}" ]; then echo "Acquire::http { Proxy \"http://${APT_PROXY}\"; };" > /etc/apt/apt.conf.d/01proxy; fi
    - apt-get install --yes make python3-venv python3-dev
    - python3 -m venv venv
    - . venv/bin/activate
    - pip install pip==$PYTHON_PIP_VERSION
  image: debian:11.4-slim
  only:
    - releases/all_major
    - releases/all_minor
    - releases/python_major
    - releases/python_minor
    - releases/testing
    - releases/testing_python
  script:
    - pip install --requirement requirements/build.txt
    - make python-wheel
  stage: stage_build_python

job_python_push_pypi:
  before_script:
    - set -a && . ./config.env && set +a
    - apt-get update
    - if [ "${APT_PROXY}" ]; then echo "Acquire::http { Proxy \"http://${APT_PROXY}\"; };" > /etc/apt/apt.conf.d/01proxy; fi
    - apt-get install --yes make python3-venv python3-dev
    - python3 -m venv venv
    - . venv/bin/activate
    - pip install pip==$PYTHON_PIP_VERSION
  image: debian:11.4-slim
  only:
    - releases/all_major
    - releases/all_minor
    - releases/python_major
    - releases/python_minor
    - releases/testing
    - releases/testing_python
  script:
    - pip install --requirement requirements/build.txt
    - echo "$PYPIRC" > ~/.pypirc
    - twine upload --verbose dist/* -r pypi
  stage: stage_push_python

# Test upgrade from the latest public version to the local version

.job_base_upgrade_test: &job_base_upgrade_test
  before_script:
    - set -a && . ./config.env && set +a
    - if [ "${APT_PROXY}" ]; then echo "Acquire::http { Proxy \"http://${APT_PROXY}\"; };" > /etc/apt/apt.conf.d/01proxy; fi
    - apt-get update
    - apt-get install --yes curl exiftool gcc ghostscript gnupg1 libldap2-dev libjpeg-dev libmagic1 libpng-dev libsasl2-dev libtiff-dev make poppler-utils python3-venv python3-dev tesseract-ocr
    - python3 -m venv venv
    - . venv/bin/activate
    - pip install pip==$PYTHON_PIP_VERSION
    - DOCKER_VERSION=`cat docker/rootfs/version`
    - pip install "mayan-edms<${DOCKER_VERSION}"
  cache:
    paths:
      - ~/.cache/pip/
      - /var/cache/apt/archives/
  image: debian:11.4-slim
  only:
    - releases/all_major
    - releases/all_minor
    - releases/docker_major
    - releases/docker_minor
    - releases/python_major
    - releases/python_minor
    - releases/staging
    - tests/all
    - tests/python
    - tests/upgrade
  stage: stage_test

job_postgres_upgrade_test:
  <<: *job_base_upgrade_test
  script:
    - apt-get install --yes libpq-dev
    - pip install psycopg2==$PYTHON_PSYCOPG2_VERSION_PREVIOUS
    - export MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'postgres'}}"
    - mayan-edms.py initialsetup --no-dependencies
    - echo "from mayan.apps.documents.models import DocumentType;from mayan.apps.documents.tests.literals import TEST_FILE_SMALL_PATH;DocumentType.objects.first().new_document(file_object=open(file=TEST_FILE_SMALL_PATH, mode='rb'))" | mayan-edms.py shell
    - pip install --requirement requirements/build.txt --requirement requirements/testing-base.txt
    - pip install psycopg2==$PYTHON_PSYCOPG2_VERSION
    - make python-wheel
    - pip install dist/mayan_edms-*.whl
    - mayan-edms.py common_perform_upgrade --no-dependencies
  services:
    - postgres:12.11-alpine
  tags:
    - postgres
  variables:
    POSTGRES_DB: "mayan"
    POSTGRES_PASSWORD: "mayanuserpass"
    POSTGRES_USER: "mayan"

job_sqlite_upgrade_test:
  <<: *job_base_upgrade_test
  script:
    - mayan-edms.py initialsetup --no-dependencies
    - echo "from mayan.apps.documents.models import DocumentType;from mayan.apps.documents.tests.literals import TEST_FILE_SMALL_PATH;DocumentType.objects.first().new_document(file_object=open(file=TEST_FILE_SMALL_PATH, mode='rb'))" | mayan-edms.py shell
    - pip install --requirement requirements/build.txt --requirement requirements/testing-base.txt
    - make python-wheel
    - pip install dist/mayan_edms-*.whl
    - mayan-edms.py common_perform_upgrade --no-dependencies

# Run the test suit

.job_base_test: &job_base_test
  before_script:
    - set -a && . ./config.env && set +a
    - if [ "${APT_PROXY}" ]; then echo "Acquire::http { Proxy \"http://${APT_PROXY}\"; };" > /etc/apt/apt.conf.d/01proxy; fi
    - apt-get update
    - apt-get install --yes --no-install-recommends curl exiftool file gcc ghostscript gnupg1 graphviz libfuse2 libldap2-dev libjpeg-dev libmagic1 libpng-dev libreoffice-calc-nogui libreoffice-draw-nogui libreoffice-impress-nogui libreoffice-math-nogui libreoffice-writer-nogui libsasl2-dev libtiff-dev make poppler-utils python3-venv python3-dev sane-utils tesseract-ocr tesseract-ocr-deu
    - python3 -m venv venv
    - . venv/bin/activate
    - pip install pip==$PYTHON_PIP_VERSION
    - pip install --requirement requirements.txt --requirement requirements/testing-base.txt --requirement requirements/build.txt
  cache:
    paths:
      - ~/.cache/pip/
      - /var/cache/apt/archives/
  image: debian:11.4-slim
  only:
    - releases/all_major
    - releases/all_minor
    - releases/docker_major
    - releases/docker_minor
    - releases/python_major
    - releases/python_minor
    - releases/staging
    - tests/all
    - tests/base
    - tests/python
  stage: stage_test

job_postgres_test:
  <<: *job_base_test
  script:
    - apt-get install --yes libpq-dev
    - . venv/bin/activate
    - pip install psycopg2==$PYTHON_PSYCOPG2_VERSION
    - export MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'postgres'}}"
    - make test-all SETTINGS=mayan.settings.testing.gitlab-ci
    - make test-all-migrations SETTINGS=mayan.settings.testing.gitlab-ci
  services:
    - postgres:12.11-alpine
  tags:
    - postgres
  variables:
    POSTGRES_DB: "mayan"
    POSTGRES_PASSWORD: "mayanuserpass"
    POSTGRES_USER: "mayan"

job_sqlite_test:
  <<: *job_base_test
  script:
    - . venv/bin/activate
    - make test-all SETTINGS=mayan.settings.testing.gitlab-ci
    - make test-all-migrations SETTINGS=mayan.settings.testing.gitlab-ci

# Deploy

job_demo_deploy:
  before_script:
    - mkdir --parents ~/.ssh
    - echo "${DEMO_SSH_KNOWN_HOST}" > ~/.ssh/known_hosts
    - chmod 644 ~/.ssh/known_hosts
    - echo "${DEMO_PRIVATE_KEY}" > ~/.ssh/id_rsa
    - chmod 600 ~/.ssh/id_rsa
  environment:
    name: demo
  script:
    - scp contrib/scripts/install/docker.sh root@$DEMO_IP_ADDRESS:~
    - ssh root@$DEMO_IP_ADDRESS "DELETE_VOLUMES=true ./docker.sh"
  stage: stage_deploy
  when: manual

job_staging_deploy:
  before_script:
    - mkdir --parents ~/.ssh
    - echo "${STAGING_SSH_KNOWN_HOST}" > ~/.ssh/known_hosts
    - chmod 644 ~/.ssh/known_hosts
    - echo "${STAGING_PRIVATE_KEY}" > ~/.ssh/id_rsa
    - chmod 600 ~/.ssh/id_rsa
  environment:
    name: staging
  script:
    - scp contrib/scripts/install/docker.sh root@$STAGING_IP_ADDRESS:~
    - ssh root@$STAGING_IP_ADDRESS "DOCKER_MAYAN_IMAGE=registry.gitlab.com/mayan-edms/mayan-edms:staging ./docker.sh"
  stage: stage_deploy
  when: manual