From 23937b6360b11e6c45cd4fddb4a9fa4ec87b3115 Mon Sep 17 00:00:00 2001 From: Rich Megginson Date: Tue, 31 Mar 2020 09:27:10 -0600 Subject: [PATCH] use python instead of envpython Many of the tests and tasks invoked from `tox` would try to use the path to the `python` executable in the tox virtualenv, and in some cases, would try to follow the symlink if the `python` command were a symlink, in some cases, following the symlink to the system python, and trying to use the absolute path. This would fail in some cases because when using the system python with the absolute path, the python modules installed in the tox venv could not be found. So instead, just use the `python` command from the venv with no path. also enable travis 3.8-dev testing, since this fix allows that platform to work correctly. --- .travis.yml | 1 + .travis/custom.sh | 7 +------ .travis/runblack.sh | 10 ++-------- .travis/runflake8.sh | 10 ++-------- .travis/runpylint.sh | 10 ++-------- .travis/runpytest.sh | 10 ++-------- .travis/runsyspycmd.sh | 15 ++++----------- .travis/utils.sh | 25 +++++++++++++++++++++++-- tox.ini | 30 ++++++++++++++---------------- 9 files changed, 51 insertions(+), 67 deletions(-) diff --git a/.travis.yml b/.travis.yml index f15b372..836533c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ matrix: - python: 3.6 - python: 3.7 - python: 3.8 + - python: 3.8-dev services: - docker diff --git a/.travis/custom.sh b/.travis/custom.sh index 3c9cb78..91eabbe 100755 --- a/.travis/custom.sh +++ b/.travis/custom.sh @@ -16,12 +16,7 @@ TOPDIR=$(readlink -f ${SCRIPTDIR}/..) . ${SCRIPTDIR}/utils.sh . ${SCRIPTDIR}/config.sh -# Sanitize arguments (see https://github.com/tox-dev/tox/issues/1463): -ENVPYTHON=$(readlink -f $1) -SYSPYTHON=$(readlink -f $2) -shift 2 - # Write your custom commands here that should be run when `tox -e custom`: -if [[ -z "${TRAVIS}" ]] || lsr_check_python_version ${ENVPYTHON} -eq '3.6'; then +if [[ -z "${TRAVIS}" ]] || lsr_check_python_version python -eq '3.6'; then (set -x; cd ${TOPDIR}/tests; ${ENVPYTHON} ./ensure_non_running_provider.py) fi diff --git a/.travis/runblack.sh b/.travis/runblack.sh index 2b82832..3fadc98 100755 --- a/.travis/runblack.sh +++ b/.travis/runblack.sh @@ -5,8 +5,7 @@ # is to get a user the opportunity to control black from config.sh via setting # environment variables. -# The first script argument is a path to Python interpreter, the rest of -# arguments are passed to black. +# The given command line arguments are passed to black. # Environment variables: # @@ -34,11 +33,6 @@ if [[ "${RUN_BLACK_DISABLED}" ]]; then exit 0 fi -# Sanitize path in case if running within tox (see -# https://github.com/tox-dev/tox/issues/1463): -ENVPYTHON=$(readlink -f $1) -shift - DEFAULT_INCLUDE='^[^.].*\.py$' DEFAULT_EXCLUDE='/(\.[^.].*|tests/roles)/' @@ -64,7 +58,7 @@ while [[ $# -gt 0 ]]; do done set -x -${ENVPYTHON} -m black \ +python -m black \ --include "${INCLUDE_ARG:-${RUN_BLACK_INCLUDE:-${DEFAULT_INCLUDE}}}" \ --exclude "${EXCLUDE_ARG:-${RUN_BLACK_EXCLUDE:-${DEFAULT_EXCLUDE}}}" \ "${OTHER_ARGS[@]}" diff --git a/.travis/runflake8.sh b/.travis/runflake8.sh index 3cecdca..c6c366f 100755 --- a/.travis/runflake8.sh +++ b/.travis/runflake8.sh @@ -4,8 +4,7 @@ # A shell wrapper around flake8. The purpose of this wrapper is to get to user # an opportunity to disable running flake8 via config.sh. -# The first script argument is a path to Python interpreter, the rest of -# arguments are passed to flake8. +# The given command line arguments are passed to flake8. # Environment variables: # @@ -25,10 +24,5 @@ if [[ "${RUN_FLAKE8_DISABLED}" ]]; then exit 0 fi -# Sanitize path in case if running within tox (see -# https://github.com/tox-dev/tox/issues/1463): -ENVPYTHON=$(readlink -f $1) -shift - set -x -${ENVPYTHON} -m flake8 "$@" +python -m flake8 "$@" diff --git a/.travis/runpylint.sh b/.travis/runpylint.sh index 619d85c..8ca82b9 100755 --- a/.travis/runpylint.sh +++ b/.travis/runpylint.sh @@ -10,8 +10,7 @@ # which set them by including config.sh). Now, they take effect also when # running tox locally. -# First argument to the script is a path to environment python, the rest of -# arguments are passed to custom_pylint.py. +# The given command line arguments are passed to custom_pylint.py. set -e @@ -20,10 +19,5 @@ SCRIPTDIR=$(readlink -f $(dirname $0)) . ${SCRIPTDIR}/config.sh -# Sanitize path in case if running within tox (see -# https://github.com/tox-dev/tox/issues/1463): -ENVPYTHON=$(readlink -f $1) -shift - set -x -${ENVPYTHON} ${SCRIPTDIR}/custom_pylint.py "$@" +python ${SCRIPTDIR}/custom_pylint.py "$@" diff --git a/.travis/runpytest.sh b/.travis/runpytest.sh index b54d4eb..30266dd 100755 --- a/.travis/runpytest.sh +++ b/.travis/runpytest.sh @@ -8,8 +8,7 @@ # running pytest if there are no unit tests and filters out --cov*/--no-cov* # arguments if there is nothing to be analyzed with coverage. -# First argument to the script is a path to environment python, the rest of -# arguments are passed to pytest. +# The given command line arguments are passed to pytest. set -e @@ -24,11 +23,6 @@ if [[ ! -d ${TOPDIR}/tests/unit ]]; then exit 0 fi -# Sanitize path in case if running within tox (see -# https://github.com/tox-dev/tox/issues/1463): -ENVPYTHON=$(readlink -f $1) -shift - PYTEST_OPTS=() PYTEST_OPTS_NOCOV=() USE_COV=no @@ -58,4 +52,4 @@ if [[ "${USE_COV}" == "no" ]]; then fi set -x -${ENVPYTHON} -m pytest "${PYTEST_OPTS[@]}" +python -m pytest "${PYTEST_OPTS[@]}" diff --git a/.travis/runsyspycmd.sh b/.travis/runsyspycmd.sh index 13e0344..e08c893 100755 --- a/.travis/runsyspycmd.sh +++ b/.travis/runsyspycmd.sh @@ -5,10 +5,8 @@ # to system python libraries, especially C bindings. The script is run with # these arguments: # -# $1 - path to environment python -# $2 - path to system python -# $3 - command runnable in Python (should be present in $PATH) -# ${@:4} - arguments passed to $3 +# $1 - command runnable in Python (should be present in $PATH) +# ${@:2} - arguments passed to $1 set -e @@ -19,12 +17,7 @@ TOPDIR=$(readlink -f ${SCRIPTDIR}/..) . ${SCRIPTDIR}/utils.sh . ${SCRIPTDIR}/config.sh -# Sanitize arguments (see https://github.com/tox-dev/tox/issues/1463): -ENVPYTHON=$(readlink -f $1) -SYSPYTHON=$(readlink -f $2) -shift 2 - -if ! lsr_venv_python_matches_system_python ${ENVPYTHON} ${SYSPYTHON}; then +if ! lsr_venv_python_matches_system_python ; then lsr_info "${ME}: ${1:-}:" \ "Environment Python has no access to system Python libraries. Skipping." exit 0 @@ -34,4 +27,4 @@ COMMAND=$(command -v $1) shift set -x -${ENVPYTHON} ${COMMAND} "$@" +python ${COMMAND} "$@" diff --git a/.travis/utils.sh b/.travis/utils.sh index 397dedf..5b8607a 100644 --- a/.travis/utils.sh +++ b/.travis/utils.sh @@ -135,15 +135,36 @@ function lsr_compare_pythons() { $(lsr_get_python_version $1) $2 $(lsr_get_python_version $3) } +## +# lsr_get_system_python +# +# Return the system python, or /usr/bin/python3 if nothing +# else can be found in a standard location. +function lsr_get_system_python() { + local syspython=$(command -pv python3) + if [[ -z "$syspython" ]]; then + syspython=$(command -pv python) + fi + if [[ -z "$syspython" ]]; then + syspython=$(command -pv python2) + fi + if [[ -z "$syspython" ]]; then + lsr_error Could not determine system python path + fi + echo $syspython +} + ## # lsr_venv_python_matches_system_python [$1] [$2] # # $1 - command or full path to venv Python interpreter (default: python) # $2 - command or full path to the system Python interpreter -# (default: /usr/bin/python3) +# (default: system python as determined by lsr_get_system_python()) # # Exit with 0 if virtual environment Python version matches the system Python # version. function lsr_venv_python_matches_system_python() { - lsr_compare_pythons ${1:-python} -eq ${2:-/usr/bin/python3} + local syspython="${2:-$(lsr_get_system_python)}" + + lsr_compare_pythons ${1:-python} -eq $syspython } diff --git a/tox.ini b/tox.ini index 24b509e..5432fcf 100644 --- a/tox.ini +++ b/tox.ini @@ -43,7 +43,7 @@ changedir = {[base]changedir} whitelist_externals = {[base]whitelist_externals} commands = - bash {[base]runpytest} {envpython} \ + bash {[base]runpytest} \ --durations=5 \ {[base]covtargets} \ --cov-report=html:htmlcov-py26 \ @@ -61,7 +61,7 @@ changedir = {[base]changedir} whitelist_externals = {[base]whitelist_externals} commands = - bash {[base]runpytest} {envpython} \ + bash {[base]runpytest} \ --durations=5 \ {[base]covtargets} \ --cov-report=html:htmlcov-py27 \ @@ -79,7 +79,7 @@ changedir = {[base]changedir} whitelist_externals = {[base]whitelist_externals} commands = - bash {[base]runpytest} {envpython} \ + bash {[base]runpytest} \ --durations=5 \ {[base]covtargets} \ --cov-report=html:htmlcov-py36 \ @@ -97,7 +97,7 @@ changedir = {[base]changedir} whitelist_externals = {[base]whitelist_externals} commands = - bash {[base]runpytest} {envpython} \ + bash {[base]runpytest} \ --durations=5 \ {[base]covtargets} \ --cov-report=html:htmlcov-py37 \ @@ -115,7 +115,7 @@ changedir = {[base]changedir} whitelist_externals = {[base]whitelist_externals} commands = - bash {[base]runpytest} {envpython} \ + bash {[base]runpytest} \ --durations=5 \ {[base]covtargets} \ --cov-report=html:htmlcov-py38 \ @@ -132,7 +132,7 @@ deps = whitelist_externals = {[base]whitelist_externals} commands = - bash {toxinidir}/.travis/runblack.sh {envpython} --check --diff . + bash {toxinidir}/.travis/runblack.sh --check --diff . [testenv:pylint] envdir = {toxworkdir}/env-{env:TRAVIS_PYTHON_VERSION:2.7} @@ -148,7 +148,7 @@ deps = whitelist_externals = {[base]whitelist_externals} commands = - bash {toxinidir}/.travis/runpylint.sh {envpython} --errors-only {posargs} + bash {toxinidir}/.travis/runpylint.sh --errors-only {posargs} [testenv:flake8] envdir = {toxworkdir}/env-{env:TRAVIS_PYTHON_VERSION:2.7} @@ -159,7 +159,7 @@ deps = whitelist_externals = {[base]whitelist_externals} commands = - bash {toxinidir}/.travis/runflake8.sh {envpython} --statistics {posargs} . + bash {toxinidir}/.travis/runflake8.sh --exclude=.venv,.tox --statistics {posargs} . [testenv:yamllint] deps = yamllint @@ -192,10 +192,8 @@ deps = whitelist_externals = {[base]whitelist_externals} commands = - bash {[molecule_common]runsyspycmd} {envpython} {[base]system_python} \ - molecule --version - bash {[molecule_common]runsyspycmd} {envpython} {[base]system_python} \ - ansible --version + bash {[molecule_common]runsyspycmd} molecule --version + bash {[molecule_common]runsyspycmd} ansible --version [testenv:molecule_lint] envdir = {toxworkdir}/env-{env:TRAVIS_PYTHON_VERSION:molecule} @@ -204,7 +202,7 @@ deps = whitelist_externals = {[base]whitelist_externals} commands = - bash {[molecule_common]runsyspycmd} {envpython} {[base]system_python} \ + bash {[molecule_common]runsyspycmd} \ molecule lint -s {env:LSR_MSCENARIO:default} {posargs} [testenv:molecule_syntax] @@ -214,7 +212,7 @@ deps = whitelist_externals = {[base]whitelist_externals} commands = - bash {[molecule_common]runsyspycmd} {envpython} {[base]system_python} \ + bash {[molecule_common]runsyspycmd} \ molecule syntax -s {env:LSR_MSCENARIO:default} {posargs} [testenv:molecule_test] @@ -224,7 +222,7 @@ deps = whitelist_externals = {[base]whitelist_externals} commands = - bash {[molecule_common]runsyspycmd} {envpython} {[base]system_python} \ + bash {[molecule_common]runsyspycmd} \ molecule test -s {env:LSR_MSCENARIO:default} {posargs} [testenv:molecule] @@ -247,7 +245,7 @@ passenv = * whitelist_externals = {[base]whitelist_externals} commands = - bash {toxinidir}/.travis/custom.sh {envpython} {[base]system_python} + bash {toxinidir}/.travis/custom.sh [pytest] addopts = -rxs