Add helper scripts to get coverage data

.gitignore

@@ -1,7 +1,10 @@
-*.pyc
 /.image-cache/
+*.pyc
 /.pytest_cache
 /tests/.coverage
-/tests/htmlcov-py*/
+/tests/htmlcov*
 /tests/__pycache__/
+/tests/remote-coveragedata-*
+/tests/tmp_merge_coveragerc
+/tests/total-*coveragedata
 /.tox

tests/ansible_module_network_connections.py

@@ -0,0 +1 @@
+roles/linux-system-roles.network/library/network_connections.py

tests/covstats

@@ -0,0 +1,16 @@
+#! /bin/bash
+# SPDX-License-Identifier: BSD-3-Clause
+
+if [ "$#" -lt 1 ]
+then
+    echo "USAGE: ${0} coverage_data_file..."
+    echo "Show Statistics for each coverage data file"
+    exit 1
+fi
+
+for coverage_file in "${@}"
+do
+    echo "coverage statistics for: ${coverage_file}:"
+    COVERAGE_FILE="${coverage_file}" coverage report
+    echo
+done

tests/get-coverage.sh

@@ -0,0 +1,68 @@
+#! /bin/bash
+# SPDX-License-Identifier: BSD-3-Clause
+
+if [ -n "${DEBUG}" ]
+then
+    set -x
+fi
+set -e
+
+if [ "$#" -lt 2 ]
+then
+    echo "USAGE: ${0} host playbook"
+    echo "Get coverage info from host for playbook"
+    exit 1
+fi
+
+host="${1}"
+shift
+playbook="${1}"
+
+coverage_data="remote-coveragedata-${host}-${playbook%.yml}"
+coverage="/root/.local/bin/coverage"
+
+echo "Getting coverage for ${playbook} on ${host}" >&2
+
+call_ansible() {
+    local module="${1}"
+    shift
+    local args="${1}"
+    shift
+    ansible -m "${module}" -i "${host}", -a "${args}" all "${@}"
+}
+
+remote_coverage_dir="$(mktemp -d /tmp/remote_coverage-XXXXXX)"
+trap "rm -rf '${remote_coverage_dir}'" EXIT
+ansible-playbook -i "${host}", get-coverage.yml -e "test_playbook=${playbook} destdir=${remote_coverage_dir}"
+
+#COVERAGE_FILE=remote-coverage coverage combine remote-coverage/tests_*/*/root/.coverage
+./merge-coverage.sh coverage "${coverage_data}"-tmp $(find "${remote_coverage_dir}" -type f | tr , _)
+
+# When https://github.com/nedbat/coveragepy/pull/49 is merged, this can be simplified:
+if false
+then
+cat > tmp_merge_coveragerc <<EOF
+[paths]
+source =
+    .
+    /tmp/ansible_*/
+EOF
+else
+cat > tmp_merge_coveragerc <<EOF
+[paths]
+source =
+    .
+EOF
+for file in $(COVERAGE_FILE="${coverage_data}"-tmp coverage report | grep -o "/tmp/ansible_[^/]*" | sort -u)
+do
+    echo "    ${file}" >> tmp_merge_coveragerc
+done
+fi
+
+COVERAGE_FILE="${coverage_data}" coverage combine --rcfile tmp_merge_coveragerc "${coverage_data}"-tmp
+rm tmp_merge_coveragerc
+
+COVERAGE_FILE="${coverage_data}" coverage report ||:
+COVERAGE_FILE="${coverage_data}" coverage html --directory "htmlcov-${coverage_data}" ||:
+
+echo "Coverage collected in: ${coverage_data}"

tests/get-coverage.yml

@@ -0,0 +1,66 @@
+# SPDX-License-Identifier: BSD-3-Clause
+---
+# This expects the variable test_playbook to be set from the outside
+- name: Prepare for coverage extraction
+  hosts: all
+  tasks:
+    # Use set_fact to set variables to make them available in all plays
+    # 'vars:' Would only set variables for the current play
+    - name: set facts
+      set_fact:
+        coverage_module: network_connections
+        coverage: /root/.local/bin/coverage
+        destdir: "remote_coverage/{{ test_playbook }}"
+
+    # This uses variables from the other set_fact task, therefore it needs to
+    # be its own task
+    - name: set more facts
+      set_fact:
+        coverage_file: ansible-coverage-{{ coverage_module }}-{{ test_playbook|replace('.yml', '') }}
+
+    - name: debug info
+      debug:
+        msg: Getting coverage for '{{ coverage_module }}' with '{{ test_playbook }}'
+
+    # combine data in case old data is left there
+    - command: "{{ coverage }} combine"
+      environment:
+        COVERAGE_FILE: "{{ coverage_file }}"
+      ignore_errors: yes
+
+    - name: remove old data
+      file:
+        state: absent
+        path: "{{ coverage_file }}"
+
+    - name: remove old data
+      shell: rm -f .coverage.*
+
+    - name: copy coveragerc
+      copy:
+        content: "[run]\ndisable_warnings = no-data-collected\n"
+        dest: .coveragerc
+
+    - name: install latest pip
+      pip:
+        name: coverage
+        extra_args: --user --upgrade
+
+- import_playbook: "{{ test_playbook }}"
+  vars:
+    ansible_python_interpreter: "{{ coverage }} run -p --include *ansible_module_{{ coverage_module }}.py"
+
+- name: Gather coverage data
+  hosts: all
+  tasks:
+    - shell: "{{ coverage }} combine .coverage.*"
+      environment:
+        COVERAGE_FILE: "{{ coverage_file }}"
+
+- name: Get coverage data
+  hosts: all
+  tasks:
+    - fetch:
+        src: "{{ coverage_file }}"
+        dest: "{{ destdir }}"
+        flat: no

tests/get-total-coverage.sh

@@ -0,0 +1,34 @@
+#! /bin/bash
+# SPDX-License-Identifier: BSD-3-Clause
+
+set -e
+coverage_data=total-coveragedata
+testhost="${1}"
+
+if [ "$#" -lt 1 ]
+then
+    echo "USAGE: ${0} host"
+    echo "Get local and all remote coverage data for host"
+    exit 1
+fi
+
+rm -f remote-coveragedata* "${coveragedata}"
+
+
+# collect pytest coverage
+tox -e py26,py27,py36,py37 -- --cov-append
+
+for test_playbook in tests_*.yml
+do
+    ./get-coverage.sh "${testhost}" "${test_playbook}"
+done
+
+./merge-coverage.sh coverage "total-remote-coveragedata" remote-coveragedata-*
+./covstats .coverage remote-coveragedata-* "total-remote-coveragedata"
+
+./merge-coverage.sh coverage "${coverage_data}" .coverage remote-coveragedata-*
+echo "Total coverage:"
+COVERAGE_FILE="${coverage_data}" coverage report ||:
+COVERAGE_FILE="${coverage_data}" coverage html --directory "htmlcov-${coverage_data}" ||:
+echo "Open HTML report with:"
+echo "xdg-open htmlcov-${coverage_data}/index.html"

tests/merge-coverage.sh

@@ -0,0 +1,35 @@
+#! /bin/bash
+# SPDX-License-Identifier: BSD-3-Clause
+
+if [ -n "${DEBUG}" ]
+then
+    set -x
+fi
+set -e
+
+if [ "$#" -lt 3 ]
+then
+    echo "USAGE: ${0} path_to_coverage_binary output_file input_files..."
+    echo "Merges all input_files into output file without removing input_files"
+    exit 1
+fi
+
+# path to coverage binary
+coverage="${1}"
+shift
+
+# read by coverage binary
+export COVERAGE_FILE="${1}"
+shift
+
+tempdir="$(mktemp -d /tmp/coverage_merge-XXXXXX)"
+trap "rm -rf '${tempdir}'" EXIT
+
+cp --backup=numbered -- "${@}" "${tempdir}"
+# FIXME: Would not work if coverage files are not hidden but they are by
+# default
+shopt -s dotglob
+"${coverage}" combine "${tempdir}/"*
+
+echo "Merged data into ${COVERAGE_FILE}"
+./covstats "${COVERAGE_FILE}"