make CI work with tox-lsr

Use tox-lsr instead of the scripts in the `.travis/` directory and the
asssociated config files.  Yes, `.travis/custom.sh` is still there,
but once you are familiar with the way that tox-lsr works, you can
remove it and use your own custom tox.ini action.  The goal of this
commit was to make as few changes as possible, to keep feature parity
with the old `.travis/` script way of running CI.  Also note that
linting has moved into tox, so we needed to disable molecule linting.
The new version of ansible-lint adds a lot of checks, so these are
disabled for now, and should be fixed later.  Also, shellcheck is
turned off until the network team can address the issues.

The `.travis/custom.sh` script is not run in the CI environment, so
this was changed slightly to remove the check for `$TRAVIS`, and it is
no longer run as a tox testenv in CI.

Signed-off-by: Rich Megginson <rmeggins@redhat.com>
This commit is contained in:
Rich Megginson 2020-11-04 10:24:31 -07:00 committed by Gris Ge
parent 302e682136
commit 4f9f7a4c94
19 changed files with 69 additions and 1153 deletions

View file

@ -1,4 +1,12 @@
---
skip_list:
- '602'
- '106'
- '106' # Role name does not match ^[a-z][a-z0-9_]+$ pattern
- '206' # Variables should have spaces before and after: {{ var_name }}
- '208' # File permissions unset or incorrect
- '301' # Commands should not change things if nothing needs doing
- '303' # Using command rather than module
- '305' # Use shell only when shell functionality is required
- '403' # Package installs should not use latest
- '502' # All tasks should be named
- '601' # Don't compare to literal True/False
- '602' # Don't compare to empty string

View file

@ -2,6 +2,12 @@
name: tox
on: # yamllint disable-line rule:truthy
- pull_request
env:
TOX_LSR: "git+https://github.com/linux-system-roles/tox-lsr@1.0.0"
LSR_ANSIBLES: 'ansible==2.7.* ansible==2.8.* ansible==2.9.*'
LSR_MSCENARIOS: default
# LSR_EXTRA_PACKAGES: "libdbus-1-dev libgirepository1.0-dev python3-dev"
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
jobs:
python:
runs-on: ubuntu-latest
@ -15,60 +21,57 @@ jobs:
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.pyver }}
- name: Install platform dependencies
run: |
set -euxo pipefail
.travis/preinstall
- name: Install pip, tox, python dependencies
- name: Install platform dependencies, python, tox, tox-lsr
run: |
set -euxo pipefail
python -m pip install --upgrade pip
pip install tox
sudo apt-get update
sudo apt-get install git
pip install "$TOX_LSR"
lsr_ci_preinstall
- name: Run tox tests
run: |
set -euxo pipefail
toxpyver=$(echo "${{ matrix.pyver }}" | tr -d .)
toxenvs="py${toxpyver}"
case "$toxpyver" in
27) toxenvs="${toxenvs},coveralls,flake8,pylint,custom" ;;
36) toxenvs="${toxenvs},coveralls,black,yamllint,custom,collection" ;;
37) toxenvs="${toxenvs},coveralls,custom" ;;
38) toxenvs="${toxenvs},coveralls,custom" ;;
27) toxenvs="${toxenvs},coveralls,flake8,pylint" ;;
36) toxenvs="${toxenvs},coveralls,black,yamllint,collection" ;;
37) toxenvs="${toxenvs},coveralls" ;;
38) toxenvs="${toxenvs},coveralls" ;;
esac
TOXENV="$toxenvs" .travis/runtox
env:
LSR_ANSIBLES: 'ansible==2.7.* ansible==2.8.* ansible==2.9.*'
LSR_MSCENARIOS: default
TOXENV="$toxenvs" lsr_ci_runtox
python-26:
runs-on: ubuntu-16.04
env:
PY26URL: https://storage.googleapis.com/travis-ci-language-archives/python/binaries/ubuntu/14.04/x86_64/python-2.6.tar.bz2
PY26TARFILE: "python-2.6.tar.bz2"
PY26VENVPATH: "/home/travis/virtualenv/python2.6"
VIRTUAL_ENV_DISABLE_PROMPT: "true"
steps:
- name: checkout PR
uses: actions/checkout@v2
- name: Install python, dependencies
run: |
set -euo pipefail
curl -sSf --retry 5 -o python-2.6.tar.bz2 ${PY26URL}
sudo tar xjf python-2.6.tar.bz2 --directory /
curl -sSf --retry 5 -o "$PY26TARFILE" "$PY26URL"
sudo tar xjf "$PY26TARFILE" --directory /
myuid=$(id -u)
mygid=$(id -g)
sudo chown -R $myuid:$mygid /home/travis/virtualenv
source /home/travis/virtualenv/python2.6/bin/activate
sudo chown -H -R $myuid:$mygid "$PY26VENVPATH"
source "$PY26VENVPATH/bin/activate"
set -x
python --version
pip --version
.travis/preinstall
sudo pip install 'tox<3' 'virtualenv==15.*' 'pluggy==0.5.*'
sudo pip list
env:
PY26URL: https://storage.googleapis.com/travis-ci-language-archives/python/binaries/ubuntu/14.04/x86_64/python-2.6.tar.bz2
VIRTUAL_ENV_DISABLE_PROMPT: "true"
sudo apt-get update
sudo apt-get install git
pip install 'tox<3' 'virtualenv==15.*' 'pluggy==0.5.*' \
"$TOX_LSR"
lsr_ci_preinstall
pip list
- name: Run tox tests
run: |
set -euo pipefail
source /home/travis/virtualenv/python2.6/bin/activate
source "$PY26VENVPATH/bin/activate"
set -x
tox -e py26,coveralls,custom
env:
LSR_ANSIBLES: 'ansible==2.7.* ansible==2.8.* ansible==2.9.*'
LSR_MSCENARIOS: default
VIRTUAL_ENV_DISABLE_PROMPT: "true"
tox -e py26,coveralls26

View file

@ -1,58 +0,0 @@
# SPDX-License-Identifier: MIT
#
# Use this file to specify custom configuration for a project. Generally, this
# involves the modification of the content of LSR_* environment variables, see
#
# * .travis/preinstall:
#
# - LSR_EXTRA_PACKAGES
#
# * .travis/runtox:
#
# - LSR_ANSIBLES
# - LSR_MSCENARIOS
#
# * .travis/runcoveralls.sh:
#
# - LSR_PUBLISH_COVERAGE
# - LSR_TESTSDIR
# - function lsr_runcoveralls_hook
#
# Environment variables that not start with LSR_* but have influence on CI
# process:
#
# * .travis/runpylint.sh:
#
# - RUN_PYLINT_INCLUDE
# - RUN_PYLINT_EXCLUDE
# - RUN_PYLINT_DISABLED
# - RUN_PYLINT_SETUP_MODULE_UTILS
#
# * .travis/runblack.sh:
#
# - RUN_BLACK_INCLUDE
# - RUN_BLACK_EXCLUDE
# - RUN_BLACK_DISABLED
# - RUN_BLACK_EXTRA_ARGS
#
# * .travis/runflake8.sh:
#
# - RUN_FLAKE8_DISABLED
# - RUN_FLAKE8_EXTRA_ARGS
type -f lsr_check_python_version > /dev/null 2>&1 || . ${SCRIPTDIR}/utils.sh
if lsr_check_python_version python -lt 3.0
then
PYTHON2_EXCLUDES="tests/ensure_provider_tests.py,scripts/print_all_options.py,tests/network/ensure_provider_tests.py"
FLAKE8_DEFAULT_EXCLUDES=".svn,CVS,.bzr,.hg,.git,__pycache__,.tox,.eggs,*.egg"
RUN_PYLINT_EXCLUDE='^(\..*|ensure_provider_tests\.py|print_all_options\.py)$'
RUN_FLAKE8_EXTRA_ARGS="--exclude ${FLAKE8_DEFAULT_EXCLUDES},${PYTHON2_EXCLUDES}"
fi
#
# * .travis/runsyspycmd.sh:
#
# - function lsr_runsyspycmd_hook
#
# * .travis/runpytest.sh:
#
# - RUN_PYTEST_SETUP_MODULE_UTILS
LSR_PUBLISH_COVERAGE=normal

View file

@ -3,13 +3,9 @@
set -e
ME=$(basename $0)
SCRIPTDIR=$(readlink -f $(dirname $0))
. ${SCRIPTDIR}/utils.sh
. ${SCRIPTDIR}/config.sh
. "$LSR_SCRIPTDIR/utils.sh"
# Write your custom commands here that should be run when `tox -e custom`:
if [[ -z "${TRAVIS}" ]] || lsr_check_python_version python -eq '3.6'; then
(set -x; cd ${TOPDIR}/tests; ${ENVPYTHON} ./ensure_provider_tests.py)
if lsr_check_python_version python -eq '3.6'; then
(set -x; cd "${TOPDIR}/tests"; python ./ensure_provider_tests.py)
fi

View file

@ -1,171 +0,0 @@
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: MIT
#
# Copyright (c) 2019-2020 Red Hat, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
"""
Probe directory tree for python files and pass them to pylint.
Usage: python custom_pylint.py ARGUMENTS
Run pylint with ARGUMENTS followed by the list of python files contained in the
working directory and its subdirectories. As a python file is recognized a
file that match INCPAT. Files and directories that match EXPAT are skipped.
Symbolic links are also skipped. It is assumed that files to be pylinted are
specified only with INCPAT and EXPAT.
There are several cases when argument from ARGUMENTS is not passed to pylint
but it is handled by run_pylint.py instead:
1. if -h or --help is contained in ARGUMENTS, this help screen is printed to
the standard output and run_pylint.py exits with 0;
2. if --include followed by a PATTERN is contained in ARGUMENTS, the PATTERN
is used instead of INCPAT to recognize whether the file is a python file
or not;
3. if --exclude followed by a PATTERN is contained in ARGUMENTS, the PATTERN
is used instead of EXPAT to recognize whether the file or directory should
be skipped.
Exclusion takes a priority over inclusion, i.e. if a file or directory can be
both included and excluded, it is excluded.
The default value of INCPAT is .*\\.py[iw]?$. For EXPAT, it is ^\\..*.
Environment variables:
RUN_PYLINT_INCLUDE
overrides default value of INCPAT;
RUN_PYLINT_EXCLUDE
overrides default value of EXPAT;
RUN_PYLINT_DISABLED
if set to an arbitrary non-empty value, pylint will be not executed
"""
import os
import re
import sys
from colorama import Fore
from pylint.lint import Run
def blue(s):
"""
Return string `s` colorized to blue.
"""
return "%s%s%s" % (Fore.BLUE, s, Fore.RESET)
def print_line(s):
"""
Write `s` followed by the line feed character to the standard output.
"""
sys.stdout.write("%s\n" % s)
def probe_args():
"""
Analyze the command line arguments and return a tuple containing a list of
pylint arguments, pattern string to recognize files to be included, and
pattern string to recognize files and directories to be skipped.
Default values of pattern strings are taken from RUN_PYLINT_INCLUDE and
RUN_PYLINT_EXCLUDE environment variables. In the case they are not defined,
.*\\.py[iw]?$ and ^\\..* are used, respectively.
"""
args = []
include_pattern = os.getenv("RUN_PYLINT_INCLUDE", r".*\.py[iw]?$")
exclude_pattern = os.getenv("RUN_PYLINT_EXCLUDE", r"^\..*")
i, nargs = 1, len(sys.argv)
while i < nargs:
arg = sys.argv[i]
if arg == "--include":
i += 1
assert i < nargs, "--include: missing PATTERN"
include_pattern = sys.argv[i]
elif arg == "--exclude":
i += 1
assert i < nargs, "--exclude: missing PATTERN"
exclude_pattern = sys.argv[i]
else:
args.append(arg)
i += 1
return args, include_pattern, exclude_pattern
def probe_dir(path, include_re, exclude_re):
"""
Recursively go through directory structure starting at `path`, collect
files that match `include_re`, skip files and directories that are either
symbolic links or match `exclude_re`. Return the list of collected files.
"""
files = []
for direntry in os.listdir(path):
fullpath = os.path.join(path, direntry)
if os.path.islink(fullpath) or exclude_re.match(direntry):
continue
elif os.path.isdir(fullpath):
files.extend(probe_dir(fullpath, include_re, exclude_re))
elif os.path.isfile(fullpath) and include_re.match(direntry):
files.append(fullpath)
return files
def show_files(files):
"""
Print `files` to the standard output, one item per line, in a blue color.
"""
print_line(blue("%s: files to be checked:" % sys.argv[0]))
for f in files:
print_line(blue(" %s" % f))
def main():
"""
Script entry point. Return exit code.
"""
args, include_pattern, exclude_pattern = probe_args()
if "-h" in args or "--help" in args:
print_line(__doc__)
return 0
if os.getenv("RUN_PYLINT_DISABLED", "") != "":
return 0
files = probe_dir(
os.getcwd(), re.compile(include_pattern), re.compile(exclude_pattern)
)
if not files:
return 0
show_files(files)
args.extend(files)
sys.argv[0] = "pylint"
return Run(args, None, False).linter.msg_status
if __name__ == "__main__":
sys.exit(main())

View file

@ -1,35 +0,0 @@
#!/bin/bash
# SPDX-License-Identifier: MIT
# Install package specified by user in LSR_EXTRA_PACKAGES. Executed by Travis
# during before_install phase.
#
# LSR_EXTRA_PACKAGES, set by user in .travis/config.sh, is a space separated
# list of packages to be installed on the Travis build environment (Ubuntu).
set -e
SCRIPTDIR=$(readlink -f $(dirname $0))
. ${SCRIPTDIR}/utils.sh
# Add python3-selinux package (needed by Molecule on selinux enabled systems,
# because Molecule is using `copy` and `file` Ansible modules to setup the
# container).
if lsr_venv_python_matches_system_python; then
LSR_EXTRA_PACKAGES='python3-selinux'
fi
# extra packages needed with python 2.6 and ansible 2.6
if lsr_check_python_version python -lt 2.7 ; then
LSR_EXTRA_PACKAGES="$LSR_EXTRA_PACKAGES libffi-dev libssl-dev"
fi
. ${SCRIPTDIR}/config.sh
# Install extra dependencies.
if [[ "${LSR_EXTRA_PACKAGES}" ]]; then
set -x
sudo apt-get update
sudo apt-get install -y ${LSR_EXTRA_PACKAGES}
fi

View file

@ -1,68 +0,0 @@
#!/bin/bash
# SPDX-License-Identifier: MIT
# A shell wrapper around black (Python formatter). The purpose of this wrapper
# is to get a user the opportunity to control black from config.sh via setting
# environment variables.
# The given command line arguments are passed to black.
# Environment variables:
#
# RUN_BLACK_INCLUDE
# a regular expression specifying files to be included; can be overridden
# from command line by --include;
#
# RUN_BLACK_EXCLUDE
# a regular expression specifying files to be excluded; can be overridden
# from command line by --exclude;
#
# RUN_BLACK_DISABLED
# if set to an arbitrary non-empty value, black will be not executed
#
# RUN_BLACK_EXTRA_ARGS
# extra cmd line args to pass to black
set -e
ME=$(basename $0)
SCRIPTDIR=$(readlink -f $(dirname $0))
. ${SCRIPTDIR}/utils.sh
. ${SCRIPTDIR}/config.sh
if [[ "${RUN_BLACK_DISABLED}" ]]; then
lsr_info "${ME}: black is disabled. Skipping."
exit 0
fi
DEFAULT_INCLUDE='^[^.].*\.py$'
DEFAULT_EXCLUDE='/(\.[^.].*|tests/roles)/'
INCLUDE_ARG=""
EXCLUDE_ARG=""
OTHER_ARGS=()
while [[ $# -gt 0 ]]; do
case "$1" in
--include)
shift
INCLUDE_ARG="$1"
;;
--exclude)
shift
EXCLUDE_ARG="$1"
;;
*)
OTHER_ARGS+=( "$1" )
;;
esac
shift
done
set -x
python -m black \
--include "${INCLUDE_ARG:-${RUN_BLACK_INCLUDE:-${DEFAULT_INCLUDE}}}" \
--exclude "${EXCLUDE_ARG:-${RUN_BLACK_EXCLUDE:-${DEFAULT_EXCLUDE}}}" \
${RUN_BLACK_EXTRA_ARGS:-} \
"${OTHER_ARGS[@]}"

View file

@ -1,50 +0,0 @@
#!/bin/bash
# SPDX-License-Identifier: MIT
set -e
#uncomment if you use $ME - otherwise set in utils.sh
#ME=$(basename "$0")
SCRIPTDIR=$(readlink -f "$(dirname "$0")")
. "${SCRIPTDIR}/utils.sh"
. "${SCRIPTDIR}/config.sh"
# Collection commands that are run when `tox -e collection`:
role=$(basename "${TOPDIR}")
toxworkdir=${1:-"${TOPDIR}"/.tox}
STABLE_TAG=${2:-"master"}
cd "${toxworkdir}"
toxworkdir=$(pwd)
testlist="yamllint,py38"
extra="black flake8 shellcheck"
cd "${SCRIPTDIR}"
for item in ${extra}; do
for f in *"${item}"*.sh; do
if [ -f "${f}" ]; then
testlist+=",${item}"
break
fi
done
done
cd "${toxworkdir}"
automaintenancerepo=https://raw.githubusercontent.com/linux-system-roles/auto-maintenance/
curl -L -o lsr_role2collection.py "${automaintenancerepo}${STABLE_TAG}"/lsr_role2collection.py
python lsr_role2collection.py --src-path "${TOPDIR}/.." --dest-path "${toxworkdir}" --role "${role}" > "${toxworkdir}"/collection.out 2>&1
yamllint="${toxworkdir}"/ansible_collections/fedora/system_roles/.yamllint_defaults.yml
sed -i -e 's/\( *\)\(document-start: disable\)/\1\2\n\1line-length:\n\1\1level: warning/' "${yamllint}"
cd ansible_collections/fedora/system_roles
tox -e "${testlist}" 2>&1 | tee "${toxworkdir}"/collection.tox.out || :
rm -rf "${toxworkdir}"/auto-maintenance "${toxworkdir}"/ansible_collections
cd "${TOPDIR}"
res=$( grep "^ERROR: .*failed" "${toxworkdir}"/collection.tox.out || : )
if [ "$res" != "" ]; then
lsr_error "${ME}: tox in the converted collection format failed."
exit 1
fi

View file

@ -1,95 +0,0 @@
#!/bin/bash
# SPDX-License-Identifier: MIT
# Reports coverage results using coveralls. The aim of this script is to
# provide a unified way to reporting coverage results across all linux system
# roles projects.
# The given command line arguments are passed to coveralls.
# Environment variables:
#
# LSR_PUBLISH_COVERAGE
# If the variable is unset or empty (the default), no coverage is published.
# Other valid values for the variable are:
# strict - the reporting is performed in strict mode, so situations
# like missing data to be reported are treated as errors
# debug - coveralls is run in debug mode (see coveralls debug --help)
# normal - coverage results will be reported normally
# LSR_TESTSDIR
# a path to directory where tests and tests artifacts are located; if unset
# or empty, this variable is set to ${TOPDIR}/tests; this path should
# already exists and be populated with tests artifacts before the script
# starts performing actions on it
set -e
ME=$(basename $0)
SCRIPTDIR=$(readlink -f $(dirname $0))
. ${SCRIPTDIR}/utils.sh
. ${SCRIPTDIR}/config.sh
# Publish the results only if it is desired.
if [[ -z "${LSR_PUBLISH_COVERAGE}" ]]; then
lsr_info "${ME}: Publishing coverage report is not enabled. Skipping."
exit 0
fi
case "${LSR_PUBLISH_COVERAGE}" in
strict) : ;;
debug) : ;;
normal) : ;;
*) lsr_error Error: \"${LSR_PUBLISH_COVERAGE}\" is not a valid option ;;
esac
LSR_TESTSDIR=${LSR_TESTSDIR:-${TOPDIR}/tests}
# Ensure we are in $LSR_TESTSDIR. It is supposed that if a user wants to submit
# tests results, $LSR_TESTSDIR always exists.
cd ${LSR_TESTSDIR}
# For simplicity, we suppose that coverage core data file has name .coverage
# and it is situated in $LSR_TESTSDIR. Similarly for .coveragerc.
COVERAGEFILE='.coverage'
COVERAGERCFILE='.coveragerc'
# In case there is no $COVERAGEFILE, there is nothing to report. If we are
# running in strict mode, treat this situation as error.
if [[ ! -s ${COVERAGEFILE} ]]; then
NO_COVERAGEFILE_MSG="${COVERAGEFILE} is missing or empty"
if [[ "${LSR_PUBLISH_COVERAGE}" == "strict" ]]; then
lsr_error "${ME} (strict mode): ${NO_COVERAGEFILE_MSG}!"
fi
lsr_info "${ME}: ${NO_COVERAGEFILE_MSG}, nothing to publish."
exit 0
fi
# Create $COVERAGERCFILE file with a [paths] section. From the official docs:
#
# The first value must be an actual file path on the machine where the
# reporting will happen, so that source code can be found. The other values
# can be file patterns to match against the paths of collected data, or they
# can be absolute or relative file paths on the current machine.
#
# So in our $COVERAGERCFILE file we make both locations to point to the
# project's top directory.
cat > ${COVERAGERCFILE} <<EOF
[paths]
source =
..
$(readlink -f ..)
EOF
# Rename $COVERAGEFILE to ${COVERAGEFILE}.merge. With this trick, coverage
# combine applies configuration in $COVERAGERCFILE also to $COVERAGEFILE.
mv ${COVERAGEFILE} ${COVERAGEFILE}.merge
python -m coverage combine --append
MAYBE_DEBUG=""
if [[ "${LSR_PUBLISH_COVERAGE}" == "debug" ]]; then
MAYBE_DEBUG=debug
fi
set -x
coveralls ${MAYBE_DEBUG} "$@"

View file

@ -1,31 +0,0 @@
#!/bin/bash
# SPDX-License-Identifier: MIT
# A shell wrapper around flake8. The purpose of this wrapper is to get to user
# an opportunity to disable running flake8 via config.sh.
# The given command line arguments are passed to flake8.
# Environment variables:
#
# RUN_FLAKE8_DISABLED
# if set to an arbitrary non-empty value, flake8 will be not executed
#
# RUN_FLAKE8_EXTRA_ARGS
# any extra command line arguments to provide e.g. --ignore=some,errs
set -e
ME=$(basename $0)
SCRIPTDIR=$(readlink -f $(dirname $0))
. ${SCRIPTDIR}/utils.sh
. ${SCRIPTDIR}/config.sh
if [[ "${RUN_FLAKE8_DISABLED}" ]]; then
lsr_info "${ME}: flake8 is disabled. Skipping."
exit 0
fi
set -x
python -m flake8 ${RUN_FLAKE8_EXTRA_ARGS:-} "$@"

View file

@ -1,38 +0,0 @@
#!/bin/bash
# SPDX-License-Identifier: MIT
# A shell wrapper around custom_pylint.py. The purpose of this wrapper is to
# set environment variables defined in config.sh before custom_pylint.py
# invocation, so user can control what should be pylinted via config.sh.
# Note: Prior this change, RUN_PYLINT_* environment variables set in config.sh
# take effect only when running inside Travis (because of runtox script
# which set them by including config.sh). Now, they take effect also when
# running tox locally.
# The given command line arguments are passed to custom_pylint.py.
# Environment variables:
#
# RUN_PYLINT_SETUP_MODULE_UTILS
# if set to an arbitrary non-empty value, the environment will be
# configured so that linting of the module_utils/ code will be run
# correctly
set -e
ME=$(basename $0)
SCRIPTDIR=$(readlink -f $(dirname $0))
. ${SCRIPTDIR}/config.sh
if [[ "${RUN_PYLINT_SETUP_MODULE_UTILS}" ]]; then
. ${SCRIPTDIR}/utils.sh
lsr_setup_module_utils
fi
export RUN_PYLINT_DISABLED
export RUN_PYLINT_EXCLUDE
export RUN_PYLINT_INCLUDE
set -x
python ${SCRIPTDIR}/custom_pylint.py "$@"

View file

@ -1,66 +0,0 @@
#!/bin/bash
# SPDX-License-Identifier: MIT
# A shell wrapper around pytest. As pytest is sensitive on a content of project
# directory: if a project contains no unit tests, pytest fails. Similarly if
# there is nothing to be analyzed with coverage, running pytest with --cov*
# arguments may lead to problems. For this reasons, this shell wrapper skips
# running pytest if there are no unit tests and filters out --cov*/--no-cov*
# arguments if there is nothing to be analyzed with coverage.
# The given command line arguments are passed to pytest.
# Environment variables:
#
# RUN_PYTEST_SETUP_MODULE_UTILS
# if set to an arbitrary non-empty value, the environment will be
# configured so that tests of the module_utils/ code will be run
# correctly
set -e
ME=$(basename $0)
SCRIPTDIR=$(readlink -f $(dirname $0))
. ${SCRIPTDIR}/utils.sh
. ${SCRIPTDIR}/config.sh
if [[ ! -d ${TOPDIR}/tests/unit ]]; then
lsr_info "${ME}: No unit tests found. Skipping."
exit 0
fi
if [[ "${RUN_PYTEST_SETUP_MODULE_UTILS}" ]]; then
lsr_setup_module_utils
fi
PYTEST_OPTS=()
PYTEST_OPTS_NOCOV=()
USE_COV=no
# Filter out coverage options if there is nothing to be analyzed with coverage.
while [[ $# -gt 0 ]]; do
case "$1" in
--cov=*)
if [[ "${1:6}" && -d "${1:6}" ]]; then
USE_COV=yes
PYTEST_OPTS+=( "$1" )
fi
;;
--cov*|--no-cov*)
PYTEST_OPTS+=( "$1" )
;;
*)
PYTEST_OPTS+=( "$1" )
PYTEST_OPTS_NOCOV+=( "$1" )
;;
esac
shift
done
if [[ "${USE_COV}" == "no" ]]; then
PYTEST_OPTS=( "${PYTEST_OPTS_NOCOV[@]}" )
fi
set -x
python -m pytest "${PYTEST_OPTS[@]}"

View file

@ -1,29 +0,0 @@
#!/bin/bash
# SPDX-License-Identifier: MIT
# Execute command in environment python only if environment python has access
# to system python libraries, especially C bindings. The script is run with
# these arguments:
#
# $1 - command runnable in Python (should be present in $PATH)
# ${@:2} - arguments passed to $1
set -e
ME=$(basename $0)
SCRIPTDIR=$(readlink -f $(dirname $0))
. ${SCRIPTDIR}/utils.sh
. ${SCRIPTDIR}/config.sh
if ! lsr_venv_python_matches_system_python ; then
lsr_info "${ME}: ${1:-<missing command>}:" \
"Environment Python has no access to system Python libraries. Skipping."
exit 0
fi
COMMAND=$(command -v $1)
shift
set -x
python ${COMMAND} "$@"

View file

@ -1,63 +0,0 @@
#!/bin/bash
# SPDX-License-Identifier: MIT
# Run tox. Additionally, if LSR_MSCENARIOS is defined, run `tox -e molecule`
# for every scenario from LSR_MSCENARIOS and for every Ansible version from
# LSR_ANSIBLES.
#
# LSR_MSCENARIOS is a space separated list of molecule scenarios.
# LSR_ANSIBLES is a space separated list of Ansible package names with versions
# in pip format, i.e 'ansible ansible==2.6 ansible==2.7 ansible=2.8'.
#
# LSR_MSCENARIOS and LSR_ANSIBLES can be set in .travis/config.sh or as
# environment variables.
ME=$(basename $0)
SCRIPTDIR=$(readlink -f $(dirname $0))
BANNERSIZE=90
# LSR_MOLECULE_DOCKER_VERSION is the version string to pass
# to pip install - this is usually a comparison operator
# followed by a version number e.g. '<4.3' but in reality
# may be anything that pip allows to follow a package name
# in an install specification
if [ -z "${LSR_MOLECULE_DOCKER_VERSION:-}" ]; then
if type docker > /dev/null 2>&1; then
DOCKER_SERVER_API_VERSION=$(docker version --format '{{.Server.APIVersion}}')
case "$DOCKER_SERVER_API_VERSION" in
1.3[0-8]|1.[0-2]*) LSR_MOLECULE_DOCKER_VERSION='<4.3';;
*) LSR_MOLECULE_DOCKER_VERSION="";;
esac
else
LSR_MOLECULE_DOCKER_VERSION=""
fi
fi
export LSR_MOLECULE_DOCKER_VERSION
. ${SCRIPTDIR}/utils.sh
. ${SCRIPTDIR}/config.sh
lsr_banner "tox" ${BANNERSIZE}
(set -x; tox); error_code=$?
# Exit prematurely if the environment is not suitable for running
# Molecule tests.
if ! lsr_venv_python_matches_system_python; then
exit $error_code
fi
for ansible_dependency in ${LSR_ANSIBLES}; do
for molecule_scenario in ${LSR_MSCENARIOS}; do
lsr_banner \
"[${ansible_dependency}] tox -e molecule -- -s ${molecule_scenario}" \
${BANNERSIZE}
(
set -x
LSR_ANSIBLE_DEP="${ansible_dependency}" \
LSR_MSCENARIO=${molecule_scenario} \
tox -e molecule
) || error_code=$?
done
done
exit $error_code

View file

@ -1,199 +0,0 @@
# SPDX-License-Identifier: MIT
#
# Auxiliary functions and variables.
#
# Usage: . utils.sh
# All variables prefixed with __lsr_ are internal.
# Code that prints the version of Python interpreter to standard output; it
# should be compatible across Python 2 and Python 3, the version is printed as
# "{major}.{minor}".
__lsr_get_python_version_py='
import sys
sys.stdout.write("%s.%s\n" % sys.version_info[:2])
'
# Colors for lsr_banner, lsr_info, and lsr_error.
__lsr_color_reset='\e[0m'
__lsr_color_red='\e[31m'
__lsr_color_blue='\e[34m'
__lsr_color_yellow='\e[1;33m'
##
# lsr_banner $1 [$2]
#
# $1 - banner text
# $2 - number of columns to occupy (default: 79)
#
# Print banner (in yellow) with $1 to stdout.
function lsr_banner() {
local maxlen=${2:-79}
local fillchar='_'
local text=" ${1} "
local fillsize=$(( ${maxlen} - ${#text} ))
local line1
local line2
if [[ ${fillsize} -lt 0 ]]; then
maxlen=${#text}
fillsize=0
fi
line1=$(printf "%*s" ${maxlen} "" | tr " " "${fillchar}")
if [[ $(( ${fillsize} % 2 )) -eq 1 ]]; then
text="${text} "
fi
line2=$(printf "%*s" $(( ${fillsize} / 2 )) "" | tr " " "${fillchar}")
line2="${line2}${text}${line2}"
echo -e "${__lsr_color_yellow}${line1}${__lsr_color_reset}"
echo -e "${__lsr_color_yellow}${line2}${__lsr_color_reset}"
}
##
# lsr_info ARGS
#
# Print ARGS (in blue) to stdout.
function lsr_info() {
echo -e "${__lsr_color_blue}$*${__lsr_color_reset}"
}
##
# lsr_error ARGS
#
# Print ARGS (in red) to stderr and exit with exit code 1.
function lsr_error() {
echo -e "${__lsr_color_red}$*${__lsr_color_reset}" >&2
exit 1
}
##
# lsr_get_python_version $1
#
# $1 - command or full path to Python interpreter
#
# If $1 is installed, return its version.
function lsr_get_python_version() {
if command -v $1 >/dev/null 2>&1; then
$1 -c "${__lsr_get_python_version_py}"
fi
}
##
# lsr_compare_versions $1 $2 $3
#
# $1 - version string (see notes below)
# $2 - binary operation (see TEST(1))
# $3 - version string (see notes below)
#
# Exit with 0 if `$1 $2 $3`.
#
# Notes:
# A version string is of the format [:digit:] "." [:digit:].
function lsr_compare_versions() {
if [[ $# -lt 3 ]]; then
return 1
fi
test ${1//./} $2 ${3//./}
}
##
# lsr_check_python_version $1 $2 $3
#
# $1 - command or full path to Python interpreter
# $2 - binary operation (see TEST(1))
# $3 - version string in [:digit:] "." [:digit:] format.
#
# Exit with 0 if `version($1) $2 $3`.
function lsr_check_python_version() {
if [[ $# -lt 3 ]]; then
return 1
fi
lsr_compare_versions $(lsr_get_python_version $1) $2 $3
}
##
# lsr_compare_pythons $1 $2 $3
#
# $1 - command or full path to Python interpreter
# $2 - binary operation (see TEST(1))
# $3 - command or full path to Python interpreter
#
# Exit with 0 if `version($1) $2 version($3)`.
function lsr_compare_pythons() {
if [[ $# -lt 3 ]]; then
return 1
fi
lsr_compare_versions \
$(lsr_get_python_version $1) $2 $(lsr_get_python_version $3)
}
##
# lsr_get_system_python
#
# Return the system python, or /usr/bin/python3 if nothing
# else can be found in a standard location.
function lsr_get_system_python() {
local syspython=$(command -pv python3)
if [[ -z "$syspython" ]]; then
syspython=$(command -pv python)
fi
if [[ -z "$syspython" ]]; then
syspython=$(command -pv python2)
fi
if [[ -z "$syspython" ]]; then
lsr_error Could not determine system python path
fi
echo $syspython
}
##
# lsr_venv_python_matches_system_python [$1] [$2]
#
# $1 - command or full path to venv Python interpreter (default: python)
# $2 - command or full path to the system Python interpreter
# (default: system python as determined by lsr_get_system_python())
#
# Exit with 0 if virtual environment Python version matches the system Python
# version.
function lsr_venv_python_matches_system_python() {
local syspython="${2:-$(lsr_get_system_python)}"
lsr_compare_pythons ${1:-python} -eq $syspython
}
##
# lsr_setup_module_utils [$1] [$2]
#
# $1 - path to the ansible/module_utils/ directory in the venv
# assumes ansible has been installed in the venv
# defaults to env var $SRC_MODULE_UTILS_DIR
# $2 - path to the local module_utils/ directory for the role
# defaults to env var $DEST_MODULE_UTILS_DIR
#
# Exit with 0 if virtual environment Python version matches the system Python
# version.
function lsr_setup_module_utils() {
local srcdir=${1:-$SRC_MODULE_UTILS_DIR}
local destdir=${2:-$DEST_MODULE_UTILS_DIR}
if [ -n "$srcdir" -a -d "$srcdir" -a -n "$destdir" -a -d "$destdir" ]; then
bash $TOPDIR/tests/setup_module_utils.sh "$srcdir" "$destdir"
fi
}
# set TOPDIR
ME=${ME:-$(basename $0)}
SCRIPTDIR=${SCRIPTDIR:-$(readlink -f $(dirname $0))}
TOPDIR=$(readlink -f ${SCRIPTDIR}/..)
# Local Variables:
# mode: Shell-script
# sh-basic-offset: 2
# End:

View file

@ -1,6 +1,6 @@
# SPDX-License-Identifier: MIT
---
extends: .yamllint_defaults.yml
extends: yamllint_defaults.yml
# possible customizations over the base yamllint config
# skip the yaml files in the /tests/ directory
# NOTE: If you want to customize `ignore` you'll have to
@ -15,3 +15,5 @@ extends: .yamllint_defaults.yml
# have to copy all of the rules from the base config
# rules:
# line-length: disable
rules:
truthy: disable

View file

@ -1,14 +0,0 @@
# SPDX-License-Identifier: MIT
---
ignore: |
/.tox/
extends: default
rules:
braces:
max-spaces-inside: 1
level: error
brackets:
max-spaces-inside: 1
level: error
truthy: disable
document-start: disable

View file

@ -6,8 +6,7 @@ driver:
name: docker
lint:
name: yamllint
options:
config-file: .yamllint.yml
enabled: False
platforms:
- name: centos-6
image: docker.io/linuxsystemroles/centos-6
@ -22,6 +21,7 @@ provisioner:
log: true
lint:
name: ansible-lint
enabled: False
playbooks:
converge: ../../tests/tests_default.yml
scenario:
@ -37,3 +37,4 @@ verifier:
name: testinfra
lint:
name: flake8
enabled: False

211
tox.ini
View file

@ -1,199 +1,22 @@
# SPDX-License-Identifier: MIT
[tox]
envlist =
black, pylint, flake8, yamllint
py{26,27,36,37,38},
collection, custom
skipsdist = true
skip_missing_interpreters = true
[lsr_config]
lsr_enable = true
[lsr_yamllint]
configfile = .yamllint.yml
configbasename = .yamllint.yml
[lsr_ansible-lint]
configfile = .ansible-lint
[testenv]
passenv = *
basepython = python3
changedir = {toxinidir}/tests
# List common dependencies for Python interpreters here:
setenv =
PYTHONPATH = {toxinidir}/library:{toxinidir}/module_utils
LC_ALL = C.UTF-8
SRC_MODULE_UTILS_DIR = {envsitepackagesdir}/ansible/module_utils
DEST_MODULE_UTILS_DIR = {toxinidir}/module_utils
deps =
py{26,27,36,37,38}: pytest-cov
py{27,36,37,38}: pytest>=3.5.1
py26: pytest
py{26,27,36,37,38}: -rpytest_extra_requirements.txt
whitelist_externals =
bash
commands =
bash {toxinidir}/.travis/runpytest.sh \
--durations=5 \
--cov={toxinidir}/library --cov={toxinidir}/module_utils \
--cov-report=html:htmlcov-{envname} \
--cov-report=term \
{posargs} \
unit
RUN_PYLINT_EXCLUDE = ^(\..*|ensure_provider_tests\.py|print_all_options\.py)$
RUN_PYTEST_SETUP_MODULE_UTILS = true
RUN_PYLINT_SETUP_MODULE_UTILS = true
RUN_PYTEST_EXTRA_ARGS = -v
RUN_FLAKE8_EXTRA_ARGS = --exclude tests/ensure_provider_tests.py,scripts/print_all_options.py,tests/network/ensure_provider_tests.py,.svn,CVS,.bzr,.hg,.git,__pycache__,.tox,.eggs,*.egg
LSR_PUBLISH_COVERAGE = normal
[base]
[testenv:py26]
envdir = {toxworkdir}/env-{env:TRAVIS_PYTHON_VERSION:2.6}
install_command =
pip install {opts} {packages}
list_dependencies_command =
pip freeze
basepython = python2.6
[testenv:py27]
envdir = {toxworkdir}/env-{env:TRAVIS_PYTHON_VERSION:2.7}
basepython = python2.7
[testenv:py36]
envdir = {toxworkdir}/env-{env:TRAVIS_PYTHON_VERSION:3.6}
basepython = python3.6
[testenv:py37]
envdir = {toxworkdir}/env-{env:TRAVIS_PYTHON_VERSION:3.7}
basepython = python3.7
[testenv:py38]
envdir = {toxworkdir}/env-{env:TRAVIS_PYTHON_VERSION:3.8}
basepython = python3.8
[testenv:black]
envdir = {toxworkdir}/env-{env:TRAVIS_PYTHON_VERSION:3.6}
basepython = python3.6
passenv = RUN_BLACK_*
changedir = {toxinidir}
deps =
black
commands =
bash {toxinidir}/.travis/runblack.sh --check --diff .
[testenv:pylint]
envdir = {toxworkdir}/env-{env:TRAVIS_PYTHON_VERSION:2.7}
basepython = python2.7
passenv = RUN_PYLINT_*
changedir = {toxinidir}
deps =
ansible
colorama
pylint>=1.8.4
-rpylint_extra_requirements.txt
commands =
bash {toxinidir}/.travis/runpylint.sh --errors-only {posargs}
[testenv:flake8]
envdir = {toxworkdir}/env-{env:TRAVIS_PYTHON_VERSION:2.7}
basepython = python2.7
passenv = RUN_FLAKE8_*
changedir = {toxinidir}
deps =
flake8>=3.5
commands =
bash {toxinidir}/.travis/runflake8.sh {posargs} .
[testenv:yamllint]
changedir = {toxinidir}
deps = yamllint
commands = yamllint .
[testenv:coveralls]
envdir = {toxworkdir}/env-{env:TRAVIS_PYTHON_VERSION:coveralls}
passenv = TRAVIS TRAVIS_*
deps =
coveralls
commands =
bash {toxinidir}/.travis/runcoveralls.sh {posargs}
[molecule_common]
changedir = {toxinidir}
deps =
{env:LSR_ANSIBLE_DEP:ansible}
docker{env:LSR_MOLECULE_DOCKER_VERSION:}
molecule<3
selinux
-rmolecule_extra_requirements.txt
runsyspycmd = {toxinidir}/.travis/runsyspycmd.sh
[testenv:molecule_version]
changedir = {[molecule_common]changedir}
envdir = {toxworkdir}/env-{env:TRAVIS_PYTHON_VERSION:molecule}
deps =
{[molecule_common]deps}
commands =
bash {[molecule_common]runsyspycmd} molecule --version
bash {[molecule_common]runsyspycmd} ansible --version
[testenv:molecule_lint]
changedir = {[molecule_common]changedir}
envdir = {toxworkdir}/env-{env:TRAVIS_PYTHON_VERSION:molecule}
deps =
{[molecule_common]deps}
commands =
bash {[molecule_common]runsyspycmd} \
molecule lint -s {env:LSR_MSCENARIO:default} {posargs}
[testenv:molecule_syntax]
changedir = {[molecule_common]changedir}
envdir = {toxworkdir}/env-{env:TRAVIS_PYTHON_VERSION:molecule}
deps =
{[molecule_common]deps}
commands =
bash {[molecule_common]runsyspycmd} \
molecule syntax -s {env:LSR_MSCENARIO:default} {posargs}
[testenv:molecule_test]
changedir = {[molecule_common]changedir}
envdir = {toxworkdir}/env-{env:TRAVIS_PYTHON_VERSION:molecule}
deps =
{[molecule_common]deps}
commands =
bash {[molecule_common]runsyspycmd} \
molecule test -s {env:LSR_MSCENARIO:default} {posargs}
[testenv:molecule]
changedir = {[molecule_common]changedir}
envdir = {toxworkdir}/env-{env:TRAVIS_PYTHON_VERSION:molecule}
deps =
{[molecule_common]deps}
commands =
{[testenv:molecule_version]commands}
{[testenv:molecule_lint]commands}
{[testenv:molecule_syntax]commands}
{[testenv:molecule_test]commands}
[testenv:collection]
changedir = {toxinidir}
ansible_python_interpreter = /usr/bin/python3
deps =
ruamel.yaml
ansible
jmespath
commands =
bash {toxinidir}/.travis/runcollection.sh {toxworkdir}
[testenv:custom]
changedir = {toxinidir}
envdir = {toxworkdir}/env-{env:TRAVIS_PYTHON_VERSION:custom}
deps =
-rcustom_requirements.txt
commands =
bash {toxinidir}/.travis/custom.sh
[pytest]
addopts = -rxs
[flake8]
show_source = true
max-line-length = 88
ignore = E402,W503
exclude = .venv,.tox
statistics = true
#verbose = 3
[pylint]
max-line-length = 88
disable = wrong-import-position
[pycodestyle]
max-line-length = 88
[testenv:shellcheck]
commands = bash -c 'echo shellcheck is currently not enabled - please fix this'