From 910f39966525848e4f26a65e2bb4db51452dd96c Mon Sep 17 00:00:00 2001 From: Till Maas Date: Fri, 18 May 2018 23:00:59 +0200 Subject: [PATCH] Add helper scripts to get coverage data --- .gitignore | 7 ++- tests/ansible_module_network_connections.py | 1 + tests/covstats | 16 +++++ tests/get-coverage.sh | 68 +++++++++++++++++++++ tests/get-coverage.yml | 66 ++++++++++++++++++++ tests/get-total-coverage.sh | 34 +++++++++++ tests/merge-coverage.sh | 35 +++++++++++ 7 files changed, 225 insertions(+), 2 deletions(-) create mode 120000 tests/ansible_module_network_connections.py create mode 100755 tests/covstats create mode 100755 tests/get-coverage.sh create mode 100644 tests/get-coverage.yml create mode 100755 tests/get-total-coverage.sh create mode 100755 tests/merge-coverage.sh diff --git a/.gitignore b/.gitignore index b751a3d..086289b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,10 @@ -*.pyc /.image-cache/ +*.pyc /.pytest_cache /tests/.coverage -/tests/htmlcov-py*/ +/tests/htmlcov* /tests/__pycache__/ +/tests/remote-coveragedata-* +/tests/tmp_merge_coveragerc +/tests/total-*coveragedata /.tox diff --git a/tests/ansible_module_network_connections.py b/tests/ansible_module_network_connections.py new file mode 120000 index 0000000..b30a744 --- /dev/null +++ b/tests/ansible_module_network_connections.py @@ -0,0 +1 @@ +roles/linux-system-roles.network/library/network_connections.py \ No newline at end of file diff --git a/tests/covstats b/tests/covstats new file mode 100755 index 0000000..310327d --- /dev/null +++ b/tests/covstats @@ -0,0 +1,16 @@ +#! /bin/bash +# SPDX-License-Identifier: BSD-3-Clause + +if [ "$#" -lt 1 ] +then + echo "USAGE: ${0} coverage_data_file..." + echo "Show Statistics for each coverage data file" + exit 1 +fi + +for coverage_file in "${@}" +do + echo "coverage statistics for: ${coverage_file}:" + COVERAGE_FILE="${coverage_file}" coverage report + echo +done diff --git a/tests/get-coverage.sh b/tests/get-coverage.sh new file mode 100755 index 0000000..7b6cd21 --- /dev/null +++ b/tests/get-coverage.sh @@ -0,0 +1,68 @@ +#! /bin/bash +# SPDX-License-Identifier: BSD-3-Clause + +if [ -n "${DEBUG}" ] +then + set -x +fi +set -e + +if [ "$#" -lt 2 ] +then + echo "USAGE: ${0} host playbook" + echo "Get coverage info from host for playbook" + exit 1 +fi + +host="${1}" +shift +playbook="${1}" + +coverage_data="remote-coveragedata-${host}-${playbook%.yml}" +coverage="/root/.local/bin/coverage" + +echo "Getting coverage for ${playbook} on ${host}" >&2 + +call_ansible() { + local module="${1}" + shift + local args="${1}" + shift + ansible -m "${module}" -i "${host}", -a "${args}" all "${@}" +} + +remote_coverage_dir="$(mktemp -d /tmp/remote_coverage-XXXXXX)" +trap "rm -rf '${remote_coverage_dir}'" EXIT +ansible-playbook -i "${host}", get-coverage.yml -e "test_playbook=${playbook} destdir=${remote_coverage_dir}" + +#COVERAGE_FILE=remote-coverage coverage combine remote-coverage/tests_*/*/root/.coverage +./merge-coverage.sh coverage "${coverage_data}"-tmp $(find "${remote_coverage_dir}" -type f | tr , _) + +# When https://github.com/nedbat/coveragepy/pull/49 is merged, this can be simplified: +if false +then +cat > tmp_merge_coveragerc < tmp_merge_coveragerc <> tmp_merge_coveragerc +done +fi + +COVERAGE_FILE="${coverage_data}" coverage combine --rcfile tmp_merge_coveragerc "${coverage_data}"-tmp +rm tmp_merge_coveragerc + +COVERAGE_FILE="${coverage_data}" coverage report ||: +COVERAGE_FILE="${coverage_data}" coverage html --directory "htmlcov-${coverage_data}" ||: + +echo "Coverage collected in: ${coverage_data}" diff --git a/tests/get-coverage.yml b/tests/get-coverage.yml new file mode 100644 index 0000000..4845c62 --- /dev/null +++ b/tests/get-coverage.yml @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: BSD-3-Clause +--- +# This expects the variable test_playbook to be set from the outside +- name: Prepare for coverage extraction + hosts: all + tasks: + # Use set_fact to set variables to make them available in all plays + # 'vars:' Would only set variables for the current play + - name: set facts + set_fact: + coverage_module: network_connections + coverage: /root/.local/bin/coverage + destdir: "remote_coverage/{{ test_playbook }}" + + # This uses variables from the other set_fact task, therefore it needs to + # be its own task + - name: set more facts + set_fact: + coverage_file: ansible-coverage-{{ coverage_module }}-{{ test_playbook|replace('.yml', '') }} + + - name: debug info + debug: + msg: Getting coverage for '{{ coverage_module }}' with '{{ test_playbook }}' + + # combine data in case old data is left there + - command: "{{ coverage }} combine" + environment: + COVERAGE_FILE: "{{ coverage_file }}" + ignore_errors: yes + + - name: remove old data + file: + state: absent + path: "{{ coverage_file }}" + + - name: remove old data + shell: rm -f .coverage.* + + - name: copy coveragerc + copy: + content: "[run]\ndisable_warnings = no-data-collected\n" + dest: .coveragerc + + - name: install latest pip + pip: + name: coverage + extra_args: --user --upgrade + +- import_playbook: "{{ test_playbook }}" + vars: + ansible_python_interpreter: "{{ coverage }} run -p --include *ansible_module_{{ coverage_module }}.py" + +- name: Gather coverage data + hosts: all + tasks: + - shell: "{{ coverage }} combine .coverage.*" + environment: + COVERAGE_FILE: "{{ coverage_file }}" + +- name: Get coverage data + hosts: all + tasks: + - fetch: + src: "{{ coverage_file }}" + dest: "{{ destdir }}" + flat: no diff --git a/tests/get-total-coverage.sh b/tests/get-total-coverage.sh new file mode 100755 index 0000000..c3dacfe --- /dev/null +++ b/tests/get-total-coverage.sh @@ -0,0 +1,34 @@ +#! /bin/bash +# SPDX-License-Identifier: BSD-3-Clause + +set -e +coverage_data=total-coveragedata +testhost="${1}" + +if [ "$#" -lt 1 ] +then + echo "USAGE: ${0} host" + echo "Get local and all remote coverage data for host" + exit 1 +fi + +rm -f remote-coveragedata* "${coveragedata}" + + +# collect pytest coverage +tox -e py26,py27,py36,py37 -- --cov-append + +for test_playbook in tests_*.yml +do + ./get-coverage.sh "${testhost}" "${test_playbook}" +done + +./merge-coverage.sh coverage "total-remote-coveragedata" remote-coveragedata-* +./covstats .coverage remote-coveragedata-* "total-remote-coveragedata" + +./merge-coverage.sh coverage "${coverage_data}" .coverage remote-coveragedata-* +echo "Total coverage:" +COVERAGE_FILE="${coverage_data}" coverage report ||: +COVERAGE_FILE="${coverage_data}" coverage html --directory "htmlcov-${coverage_data}" ||: +echo "Open HTML report with:" +echo "xdg-open htmlcov-${coverage_data}/index.html" diff --git a/tests/merge-coverage.sh b/tests/merge-coverage.sh new file mode 100755 index 0000000..a33e94d --- /dev/null +++ b/tests/merge-coverage.sh @@ -0,0 +1,35 @@ +#! /bin/bash +# SPDX-License-Identifier: BSD-3-Clause + +if [ -n "${DEBUG}" ] +then + set -x +fi +set -e + +if [ "$#" -lt 3 ] +then + echo "USAGE: ${0} path_to_coverage_binary output_file input_files..." + echo "Merges all input_files into output file without removing input_files" + exit 1 +fi + +# path to coverage binary +coverage="${1}" +shift + +# read by coverage binary +export COVERAGE_FILE="${1}" +shift + +tempdir="$(mktemp -d /tmp/coverage_merge-XXXXXX)" +trap "rm -rf '${tempdir}'" EXIT + +cp --backup=numbered -- "${@}" "${tempdir}" +# FIXME: Would not work if coverage files are not hidden but they are by +# default +shopt -s dotglob +"${coverage}" combine "${tempdir}/"* + +echo "Merged data into ${COVERAGE_FILE}" +./covstats "${COVERAGE_FILE}"