Merge pull request #50 from tyll/more_integration_tests

More integration tests

.gitignore

@@ -1,7 +1,9 @@
 *.pyc
-/.image-cache/
 /.pytest_cache
 /tests/.coverage
-/tests/htmlcov-py*/
+/tests/htmlcov*
 /tests/__pycache__/
+/tests/remote-coveragedata-*
+/tests/tmp_merge_coveragerc
+/tests/total-*coveragedata
 /.tox

defaults/main.yml

@@ -2,8 +2,13 @@
 network_provider:     "{{ network_provider_default }}"
 network_connections:  []
 
+# The python-gobject-base packag depends on the python version and
+# distribution:
+# - python-gobject-base on RHEL7 (no python2-gobject-base :-/)
+# - python-gobject-base or python2-gobject-base on Fedora 27
+# - python3-gobject-base on Fedora 28+
 network_service_name_default_nm: NetworkManager
-network_packages_default_nm: [ ethtool, NetworkManager ]
+network_packages_default_nm: [ ethtool, NetworkManager, "python{{ ansible_python['version']['major'] | replace('2', '') }}-gobject-base"]
 
 network_service_name_default_initscripts: network
 network_packages_default_initscripts: [ ethtool ]

examples/down-profile.yml

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: BSD-3-Clause
+---
+- name: Set {{ profile }} down
+  hosts: all
+  vars:
+    network_connections:
+      - name: "{{ profile }}"
+        state: down
+  roles:
+    - linux-system-roles.network

examples/remove-profile.yml

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: BSD-3-Clause
+---
+- name: Remove {{ profile }}
+  hosts: all
+  vars:
+    network_connections:
+      - name: "{{ profile }}"
+        state: absent
+  roles:
+    - linux-system-roles.network

tests/ansible_module_network_connections.py

@@ -0,0 +1 @@
+roles/linux-system-roles.network/library/network_connections.py

tests/covstats

@@ -0,0 +1,16 @@
+#! /bin/bash
+# SPDX-License-Identifier: BSD-3-Clause
+
+if [ "$#" -lt 1 ]
+then
+    echo "USAGE: ${0} coverage_data_file..."
+    echo "Show Statistics for each coverage data file"
+    exit 1
+fi
+
+for coverage_file in "${@}"
+do
+    echo "coverage statistics for: ${coverage_file}:"
+    COVERAGE_FILE="${coverage_file}" coverage report
+    echo
+done

tests/get-coverage.sh

@@ -0,0 +1,68 @@
+#! /bin/bash
+# SPDX-License-Identifier: BSD-3-Clause
+
+if [ -n "${DEBUG}" ]
+then
+    set -x
+fi
+set -e
+
+if [ "$#" -lt 2 ]
+then
+    echo "USAGE: ${0} host playbook"
+    echo "Get coverage info from host for playbook"
+    exit 1
+fi
+
+host="${1}"
+shift
+playbook="${1}"
+
+coverage_data="remote-coveragedata-${host}-${playbook%.yml}"
+coverage="/root/.local/bin/coverage"
+
+echo "Getting coverage for ${playbook} on ${host}" >&2
+
+call_ansible() {
+    local module="${1}"
+    shift
+    local args="${1}"
+    shift
+    ansible -m "${module}" -i "${host}", -a "${args}" all "${@}"
+}
+
+remote_coverage_dir="$(mktemp -d /tmp/remote_coverage-XXXXXX)"
+trap "rm -rf '${remote_coverage_dir}'" EXIT
+ansible-playbook -i "${host}", get-coverage.yml -e "test_playbook=${playbook} destdir=${remote_coverage_dir}"
+
+#COVERAGE_FILE=remote-coverage coverage combine remote-coverage/tests_*/*/root/.coverage
+./merge-coverage.sh coverage "${coverage_data}"-tmp $(find "${remote_coverage_dir}" -type f | tr , _)
+
+# When https://github.com/nedbat/coveragepy/pull/49 is merged, this can be simplified:
+if false
+then
+cat > tmp_merge_coveragerc <<EOF
+[paths]
+source =
+    .
+    /tmp/ansible_*/
+EOF
+else
+cat > tmp_merge_coveragerc <<EOF
+[paths]
+source =
+    .
+EOF
+for file in $(COVERAGE_FILE="${coverage_data}"-tmp coverage report | grep -o "/tmp/ansible_[^/]*" | sort -u)
+do
+    echo "    ${file}" >> tmp_merge_coveragerc
+done
+fi
+
+COVERAGE_FILE="${coverage_data}" coverage combine --rcfile tmp_merge_coveragerc "${coverage_data}"-tmp
+rm tmp_merge_coveragerc
+
+COVERAGE_FILE="${coverage_data}" coverage report ||:
+COVERAGE_FILE="${coverage_data}" coverage html --directory "htmlcov-${coverage_data}" ||:
+
+echo "Coverage collected in: ${coverage_data}"

tests/get-coverage.yml

@@ -0,0 +1,66 @@
+# SPDX-License-Identifier: BSD-3-Clause
+---
+# This expects the variable test_playbook to be set from the outside
+- name: Prepare for coverage extraction
+  hosts: all
+  tasks:
+    # Use set_fact to set variables to make them available in all plays
+    # 'vars:' Would only set variables for the current play
+    - name: set facts
+      set_fact:
+        coverage_module: network_connections
+        coverage: /root/.local/bin/coverage
+        destdir: "remote_coverage/{{ test_playbook }}"
+
+    # This uses variables from the other set_fact task, therefore it needs to
+    # be its own task
+    - name: set more facts
+      set_fact:
+        coverage_file: ansible-coverage-{{ coverage_module }}-{{ test_playbook|replace('.yml', '') }}
+
+    - name: debug info
+      debug:
+        msg: Getting coverage for '{{ coverage_module }}' with '{{ test_playbook }}'
+
+    # combine data in case old data is left there
+    - command: "{{ coverage }} combine"
+      environment:
+        COVERAGE_FILE: "{{ coverage_file }}"
+      ignore_errors: yes
+
+    - name: remove old data
+      file:
+        state: absent
+        path: "{{ coverage_file }}"
+
+    - name: remove old data
+      shell: rm -f .coverage.*
+
+    - name: copy coveragerc
+      copy:
+        content: "[run]\ndisable_warnings = no-data-collected\n"
+        dest: .coveragerc
+
+    - name: install latest pip
+      pip:
+        name: coverage
+        extra_args: --user --upgrade
+
+- import_playbook: "{{ test_playbook }}"
+  vars:
+    ansible_python_interpreter: "{{ coverage }} run -p --include *ansible_module_{{ coverage_module }}.py"
+
+- name: Gather coverage data
+  hosts: all
+  tasks:
+    - shell: "{{ coverage }} combine .coverage.*"
+      environment:
+        COVERAGE_FILE: "{{ coverage_file }}"
+
+- name: Get coverage data
+  hosts: all
+  tasks:
+    - fetch:
+        src: "{{ coverage_file }}"
+        dest: "{{ destdir }}"
+        flat: no

tests/get-total-coverage.sh

@@ -0,0 +1,34 @@
+#! /bin/bash
+# SPDX-License-Identifier: BSD-3-Clause
+
+set -e
+coverage_data=total-coveragedata
+testhost="${1}"
+
+if [ "$#" -lt 1 ]
+then
+    echo "USAGE: ${0} host"
+    echo "Get local and all remote coverage data for host"
+    exit 1
+fi
+
+rm -f remote-coveragedata* "${coveragedata}"
+
+
+# collect pytest coverage
+tox -e py26,py27,py36,py37 -- --cov-append
+
+for test_playbook in tests_*.yml
+do
+    ./get-coverage.sh "${testhost}" "${test_playbook}"
+done
+
+./merge-coverage.sh coverage "total-remote-coveragedata" remote-coveragedata-*
+./covstats .coverage remote-coveragedata-* "total-remote-coveragedata"
+
+./merge-coverage.sh coverage "${coverage_data}" .coverage remote-coveragedata-*
+echo "Total coverage:"
+COVERAGE_FILE="${coverage_data}" coverage report ||:
+COVERAGE_FILE="${coverage_data}" coverage html --directory "htmlcov-${coverage_data}" ||:
+echo "Open HTML report with:"
+echo "xdg-open htmlcov-${coverage_data}/index.html"

tests/merge-coverage.sh

@@ -0,0 +1,35 @@
+#! /bin/bash
+# SPDX-License-Identifier: BSD-3-Clause
+
+if [ -n "${DEBUG}" ]
+then
+    set -x
+fi
+set -e
+
+if [ "$#" -lt 3 ]
+then
+    echo "USAGE: ${0} path_to_coverage_binary output_file input_files..."
+    echo "Merges all input_files into output file without removing input_files"
+    exit 1
+fi
+
+# path to coverage binary
+coverage="${1}"
+shift
+
+# read by coverage binary
+export COVERAGE_FILE="${1}"
+shift
+
+tempdir="$(mktemp -d /tmp/coverage_merge-XXXXXX)"
+trap "rm -rf '${tempdir}'" EXIT
+
+cp --backup=numbered -- "${@}" "${tempdir}"
+# FIXME: Would not work if coverage files are not hidden but they are by
+# default
+shopt -s dotglob
+"${coverage}" combine "${tempdir}/"*
+
+echo "Merged data into ${COVERAGE_FILE}"
+./covstats "${COVERAGE_FILE}"

tests/run-tasks.yml

@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: BSD-3-Clause
+---
+- name: Run the tasklist {{ task }}
+  hosts: all
+  tasks:
+    - include_tasks: "{{ task }}"

tests/tasks/assert-device_absent.yml

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: BSD-3-Clause
+---
+- setup:
+- assert:
+    name: "assert that {{ interface }} is absent"
+    that: "{{ not interface in ansible_interfaces }}"
+    msg: "{{ interface }} is in ansible_interfaces: {{ ansible_interfaces }}"

tests/tasks/assert-device_present.yml

@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: BSD-3-Clause
+---
+# need to run setup again when this is just a task to make sure ansible_interfaces is correct
+- setup:
+- assert:
+    name: "assert that {{ interface }} is present"
+    that: "{{ interface in ansible_interfaces }}"
+    msg: "{{ interface }} is not in ansible_interfaces: {{ ansible_interfaces }}"

tests/tasks/create-and-remove-interface.yml

@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: BSD-3-Clause
+---
+- include_tasks: show-interfaces.yml
+- include_tasks: manage-test-interface.yml
+  vars:
+    state: absent
+- include_tasks: show-interfaces.yml
+- include_tasks: assert-device_absent.yml
+
+- include_tasks: manage-test-interface.yml
+  vars:
+    state: present
+- include_tasks: show-interfaces.yml
+- include_tasks: assert-device_present.yml
+
+- include_tasks: manage-test-interface.yml
+  vars:
+    state: absent
+- include_tasks: show-interfaces.yml
+- include_tasks: assert-device_absent.yml

tests/tasks/make-profile-absent.yml

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: BSD-3-Clause
+---
+- name: Remove {{ profile }}
+  hosts: all
+  vars:
+    network_connections:
+      - name: "{{ profile }}"
+        state: absent
+  roles:
+    - linux-system-roles.network

tests/tasks/make-profile-down.yml

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: BSD-3-Clause
+---
+- name: Set {{ profile }} down
+  hosts: all
+  vars:
+    network_connections:
+      - name: "{{ profile }}"
+        state: down
+  roles:
+    - linux-system-roles.network

tests/tasks/manage-test-interface.yml

@@ -0,0 +1,39 @@
+# SPDX-License-Identifier: BSD-3-Clause
+---
+- fail:
+    msg: "state needs to be present or absent, not '{{ state }}'"
+  when: state not in ["present", "absent"]
+
+- fail:
+    msg: "type needs to be dummy, tap or veth, not '{{ type }}'"
+  when: type not in ["dummy", "tap", "veth"]
+
+# run setup again to make sure ansible_interfaces is accurate
+- setup:
+
+  # veth
+- name: Create veth interface {{ interface }}
+  shell: ip link add {{ interface }} type veth peer name peer{{ interface }}
+  when: "type == 'veth' and state == 'present' and interface not in ansible_interfaces"
+
+- name: Delete veth interface {{ interface }}
+  shell: ip link del {{ interface }} type veth
+  when: "type == 'veth' and state == 'absent' and interface in ansible_interfaces"
+
+  # dummy
+- name: Create dummy interface {{ interface }}
+  shell: ip link add "{{ interface }}" type dummy
+  when: "type == 'dummy' and state == 'present' and interface not in ansible_interfaces"
+
+- name: Delete dummy interface {{ interface }}
+  shell: ip link del "{{ interface }}" type dummy
+  when: "type == 'dummy' and state == 'absent' and interface in ansible_interfaces"
+
+  # tap
+- name: Create tap interface {{ interface }}
+  shell: ip tuntap add dev {{ interface }} mode tap
+  when: "type == 'tap' and state == 'present' and interface not in ansible_interfaces"
+
+- name: Delete tap interface {{ interface }}
+  shell: ip tuntap del dev {{ interface }} mode tap
+  when: "type == 'tap' and state == 'absent' and interface in ansible_interfaces"

tests/tasks/show-interfaces.yml

@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: BSD-3-Clause
+---
+- debug:
+    msg: "ansible_interfaces: {{ ansible_interfaces }}"

tests/tests_ethernet.yml

@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: BSD-3-Clause
+---
+- name: Test configuring ethernet devices
+  hosts: all
+  vars:
+    type: veth
+    interface: lsr27
+
+  tasks:
+    - name: "set type={{ type }} and interface={{ interface }}"
+      set_fact:
+        type: "{{ type }}"
+        interface: "{{ interface }}"
+    - include_tasks: tasks/show-interfaces.yml
+    - include_tasks: tasks/manage-test-interface.yml state=present
+    - include_tasks: tasks/assert-device_present.yml
+
+- name: Test static interface up
+  hosts: all
+  vars:
+    network_connections:
+      - name: "{{ interface }}"
+        interface_name: "{{ interface }}"
+        state: up
+        type: ethernet
+        autoconnect: yes
+        ip:
+          address: 192.0.27.1/24
+  roles:
+    - linux-system-roles.network
+# FIXME: assert profile present
+# FIXME: assert profile/device up + IP address
+# FIXME: state down does not work currently
+#- import_playbook: down-profile.yml
+#  vars:
+#    profile: "{{ interface }}"
+## FIXME: assert profile/device down
+#- import_playbook: remove-profile.yml
+#  vars:
+#    profile: "{{ interface }}"
+# FIXME: assert profile away
+- name: Remove interfaces
+  hosts: all
+  tasks:
+    - include_tasks: tasks/manage-test-interface.yml state=absent
+    - include_tasks: tasks/assert-device_absent.yml

tests/tests_helpers-and-asserts.yml

@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: BSD-3-Clause
+---
+- name: Check that creating and removing test devices and assertions work
+  hosts: all
+  tasks:
+    - name: test veth interface management
+      include_tasks: tasks/create-and-remove-interface.yml
+      vars:
+        type: veth
+        interface: veth1298
+
+    - name: test veth interface management
+      include_tasks: tasks/create-and-remove-interface.yml
+      vars:
+        type: dummy
+        interface: dummy1298
+
+# FIXME: when: does not seem to work with include_tasks, therefore this cannot be safely tested for now
+#    - name: test tap interfaces
+#      include_tasks: tasks/create-and-remove-interface.yml type=tap interface=tap1298
+#      when: ansible_distribution_major_version > 6
+#      # ip tuntap does not exist on RHEL6
+#      # FIXME: Maybe use some other tool to manage devices, openvpn can do this,
+#      # but it is in EPEL