From 8102fd64437ec447b139500d959dca0d1b9d73ca Mon Sep 17 00:00:00 2001 From: Jiri Kucera Date: Wed, 17 Jul 2019 11:18:50 +0200 Subject: [PATCH] Refactor Travis CI setup This commit introduces the refactored Travis CI setup and tox.ini to fullfil the need to have common CI setup across all roles. The current proposal and discussion can be found at: https://github.com/linux-system-roles/template/issues/4 --- .travis.yml | 36 +++++++- .travis/before_install.sh | 17 ++++ molecule_requirements.txt | 2 + tox.ini | 189 ++++++++++++++++++++++++-------------- 4 files changed, 171 insertions(+), 73 deletions(-) create mode 100755 .travis/before_install.sh create mode 100644 molecule_requirements.txt diff --git a/.travis.yml b/.travis.yml index 8e76d66..7f7bcc5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,27 +1,55 @@ --- dist: xenial language: python +env: + global: + - LSR_NEED_PYLINTERS="/^linux-system-roles\/(network|selinux|storage)$/" matrix: include: - python: 2.6 dist: trusty + # Use LSRENV to save the unique value (here the desired Python + # interpreter version) to distinguish between Travis jobs. This can be + # usefull if some role wish to perform a specific action depending on the + # job environment: + env: LSRENV=2.6 + # Skip the jobs where only Python linters are run if the role does not + # contain any Python code: + if: repo =~ env(LSR_NEED_PYLINTERS) - python: 2.7 + env: LSRENV=2.7 + if: repo =~ env(LSR_NEED_PYLINTERS) - python: 3.5 - env: aptpkgs=python3-selinux + env: LSRENV=3.5 + # Under 3.5, we run Molecule - python: 3.6 + env: LSRENV=3.6 + if: repo =~ env(LSR_NEED_PYLINTERS) - python: 3.7 + env: LSRENV=3.7 + if: repo =~ env(LSR_NEED_PYLINTERS) - python: 3.7-dev + env: LSRENV=3.7-dev + if: repo =~ env(LSR_NEED_PYLINTERS) - python: 3.8-dev - # - python: nightly + env: LSRENV=3.8-dev + if: repo =~ env(LSR_NEED_PYLINTERS) + #- python: nightly + # env: LSRENV=nightly + # if: repo =~ env(LSR_NEED_PYLINTERS) services: - docker before_install: - - if [ -n "${aptpkgs}" ]; then sudo apt-get install -y python3-selinux; fi + # In .travis/before_install.sh can be commands that install packages specific + # for the concrette role, or set specific environment variables. To decide + # what to set or install in which environment, the value of LSRENV should be + # helpfull: + - if [ -f .travis/before_install.sh ]; then .travis/before_install.sh; fi install: - - pip install tox tox-travis + - pip install molecule docker tox tox-travis script: - tox diff --git a/.travis/before_install.sh b/.travis/before_install.sh new file mode 100755 index 0000000..23de95b --- /dev/null +++ b/.travis/before_install.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +if [ x${LSRENV} = x2.7 ]; then + LSR_PYLINT_DIRS='library/network_connections.py module_utils/network_lsr tests/unit/test_network_connections.py' +fi + +if [ x${LSRENV} = x3.5 ]; then + LSR_MOLECULE_DEPS='-rmolecule_requirements.txt' + + sudo apt-get install -y python3-selinux +fi + +if [ x${LSRENV} = x3.6 ]; then + LSR_TEXTRA_DEPS='PyYAML' + LSR_TEXTRA_DIR='tests' + LSR_TEXTRA_CMD='./ensure_non_running_provider.py' +fi diff --git a/molecule_requirements.txt b/molecule_requirements.txt new file mode 100644 index 0000000..20a040b --- /dev/null +++ b/molecule_requirements.txt @@ -0,0 +1,2 @@ +jmespath +git+https://github.com/tyll/selinux-pypi-shim@fulllocation diff --git a/tox.ini b/tox.ini index 604c295..8bb7840 100644 --- a/tox.ini +++ b/tox.ini @@ -1,23 +1,15 @@ [tox] -envlist = black, flake8, pylint, py{26,27,36,37}, ensure_non_running_provider skipsdist = true -skip_missing_interpreters = True +skip_missing_interpreters = true [testenv] basepython = python3 +# List common dependencies for Python interpreters here: deps = py{26,27,36,37,38}: pytest-cov py{27,36,37,38}: pytest>=3.5.1 py{26,27}: mock py26: pytest - molecule_{lint,syntax,test}: docker - molecule_{lint,syntax,test}: jmespath - molecule_{lint,syntax,test}: molecule - # The selinux pypi shim does not work with Ubuntu (as used by Travis), yet. - # Therefore use a fork with Ubuntu support. This can be changed once the - # update is available on PyPi. - # molecule_{lint,syntax,test}: selinux - molecule_{lint,syntax,test}: git+https://github.com/tyll/selinux-pypi-shim@fulllocation [base] passenv = * @@ -29,25 +21,28 @@ covtarget = {toxinidir}/library --cov {toxinidir}/module_utils pytesttarget = . [testenv:black] -deps = black - -commands = black --check --diff --include "^[^.].*\.py$" . +deps = + black +commands = + black --check --diff --include "^[^.].*\.py$" --exclude "/(\.[^.].*|tests/roles)/" . [testenv:py26] -install_command = pip install {opts} {packages} -list_dependencies_command = pip freeze +install_command = + pip install {opts} {packages} +list_dependencies_command = + pip freeze basepython = python2.6 passenv = {[base]passenv} setenv = {[base]setenv} changedir = {[base]changedir} commands = - pytest \ - --durations=5 \ - --cov={[base]covtarget} \ - --cov-report=html:htmlcov-py26 --cov-report=term \ - {posargs} \ - {[base]pytesttarget} + pytest --durations=5 \ + --cov={[base]covtarget} \ + --cov-report=html:htmlcov-py26 \ + --cov-report=term \ + {posargs} \ + {[base]pytesttarget} [testenv:py27] basepython = python2.7 @@ -56,12 +51,12 @@ setenv = {[base]setenv} changedir = {[base]changedir} commands = - pytest \ - --durations=5 \ - --cov={[base]covtarget} \ - --cov-report=html:htmlcov-py27 --cov-report=term \ - {posargs} \ - {[base]pytesttarget} + pytest --durations=5 \ + --cov={[base]covtarget} \ + --cov-report=html:htmlcov-py27 \ + --cov-report=term \ + {posargs} \ + {[base]pytesttarget} [testenv:py36] basepython = python3.6 @@ -70,12 +65,12 @@ setenv = {[base]setenv} changedir = {[base]changedir} commands = - pytest \ - --durations=5 \ - --cov={[base]covtarget} \ - --cov-report=html:htmlcov-py36 --cov-report=term \ - {posargs} \ - {[base]pytesttarget} + pytest --durations=5 \ + --cov={[base]covtarget} \ + --cov-report=html:htmlcov-py36 \ + --cov-report=term \ + {posargs} \ + {[base]pytesttarget} [testenv:py37] basepython = python3.7 @@ -84,27 +79,33 @@ setenv = {[base]setenv} changedir = {[base]changedir} commands = - pytest \ - --durations=5 \ - --cov={[base]covtarget} \ - --cov-report=html:htmlcov-py37 --cov-report=term \ - {posargs} \ - {[base]pytesttarget} + pytest --durations=5 \ + --cov={[base]covtarget} \ + --cov-report=html:htmlcov-py37 + --cov-report=term \ + {posargs} \ + {[base]pytesttarget} [testenv:py38] +basepython = python3.8 passenv = {[base]passenv} setenv = {[base]setenv} changedir = {[base]changedir} -basepython = python3.8 commands = - pytest \ - --durations=5 \ - --cov={[base]covtarget} \ - --cov-report=html:htmlcov-py38 --cov-report=term \ - {posargs} \ - {[base]pytesttarget} + pytest --durations=5 \ + --cov={[base]covtarget} \ + --cov-report=html:htmlcov-py38 \ + --cov-report=term \ + {posargs} \ + {[base]pytesttarget} +# Custom role settings may set LSR_PYLINT_DIRS environment variable if there is +# a reason to have other than default directories (library/ module_utils/ and +# tests/). For example, network role should have this set like +# +# LSR_PYLINT_DIRS="library/network_connections.py module_utils/network_lsr tests/unit/test_network_connections.py" +# [testenv:pylint] basepython = python2.7 setenv = @@ -113,21 +114,17 @@ deps = pylint>=1.8.4 ansible commands = - pylint \ - --errors-only \ - {posargs} \ - library/network_connections.py \ - module_utils/network_lsr \ - tests/unit/test_network_connections.py + pylint --errors-only \ + {posargs} \ + {env:LSR_PYLINT_DIRS:library module_utils tests} [testenv:flake8] basepython = python2.7 deps = flake8>=3.5 whitelist_externals = flake8 -commands= - flake8 --statistics {posargs} \ - . +commands = + flake8 --statistics {posargs} . [testenv:coveralls] basepython = python2.7 @@ -138,29 +135,83 @@ changedir = {[base]changedir} commands = coveralls -[testenv:ensure_non_running_provider] +# Here we provide a way how a role can add its custom command to be run. Such +# extra command is run at the end of each testenv run and is driven by +# environment variables. Involved environment variables are: +# +# LSR_TEXTRA_DEPS +# - contains dependency needed by commands to run smoothly; if more than +# one dependency is needed, use external file together with '-r' option +# (see PEP 508) +# +# LSR_TEXTRA_DIR +# - directory to which to cd +# +# LSR_TEXTRA_CMD +# - custom command to be run +# +# Example: `network` system role need to run `./tests/ensure_non_running_provider.py` +# to check for the existence of `*_provider.yml` playbooks. The script +# is run in Python 3.6. +# +# To make this possible, we add to `.travis/before_install.sh` a snippet: +# +# if [ x${LSRENV} = x3.6 ]; then +# LSR_TEXTRA_DEPS='PyYAML' +# LSR_TEXTRA_DIR='tests' +# LSR_TEXTRA_CMD='./ensure_non_running_provider.py' +# fi +# +[testenv:extra] +passenv = * deps = - PyYAML -changedir = {toxinidir}/tests -commands = {toxinidir}/tests/ensure_non_running_provider.py + {env:LSR_TEXTRA_DEPS} +changedir = {toxinidir}/{env:LSR_TEXTRA_DIR:.} +commands = + {env:LSR_TEXTRA_CMD:python --version} + +# LSR_MOLECULE_DEPS may contain aditional Molecule dependencies. For example, +# in `network` system role, LSR_MOLECULE_DEPS can be set as +# +# LSR_MOLECULE_DEPS='-rmolecule_requirements.txt' +# +# where `molecule_requirements.txt` contains two lines: +# +# jmespath +# git+https://github.com/tyll/selinux-pypi-shim@fulllocation +# +[molecule] +deps = + docker + molecule + {env:LSR_MOLECULE_DEPS} [testenv:molecule_lint] +deps = + {[molecule]deps} commands_pre = molecule --version ansible --version -commands = molecule {posargs} lint +commands = + molecule {posargs} lint [testenv:molecule_syntax] -commands = molecule {posargs} syntax +deps = + {[molecule]deps} +commands = + molecule {posargs} syntax [testenv:molecule_test] -commands = molecule {posargs} test +deps = + {[molecule]deps} +commands = + molecule {posargs} test [pytest] addopts = -rxs [flake8] -show_source = True +show_source = true max-line-length = 88 ignore = E402,W503 @@ -173,9 +224,9 @@ max-line-length = 88 [travis] python = - 2.6: py26 - 2.7: py27,coveralls,flake8,pylint - 3.5: molecule_lint,molecule_syntax,molecule_test - 3.6: py36,black,ensure_non_running_provider - 3.7: py37 - 3.8: py38 + 2.6: py26,extra + 2.7: py27,coveralls,flake8,pylint,extra + 3.5: molecule_lint,molecule_syntax,molecule_test,extra + 3.6: py36,black,extra + 3.7: py37,extra + 3.8: py38,extra