test: Add DNS, routes and network connectivity checks during cleanup

In order to guarantee each test is cleaned up properly in the end, it
is important to add a post-test check to each test checking that:

- Routes and DNS are restored.
- Network connectivity to certain hosts are preserved.

Signed-off-by: Wen Liang <liangwen12year@gmail.com>

tests/ensure_provider_tests.py

@@ -83,7 +83,7 @@ ibution_major_version | int < 9",
     "playbooks/tests_provider.yml": {
         MINIMUM_VERSION: "'1.20.0'",
         "comment": "# NetworKmanager 1.20.0 added support for forgetting profiles",
-        EXTRA_RUN_CONDITION: "ansible_distribution != 'RedHat' or\n      ansible_distr\
+        EXTRA_RUN_CONDITION: "ansible_distribution not in ['RedHat', 'CentOS'] or\n      ansible_distr\
 ibution_major_version | int < 9",
     },
     "playbooks/tests_eth_pci_address_match.yml": {

tests/playbooks/tests_802_1x.yml

@@ -143,3 +143,5 @@
             - name: Update ca trust
               command: update-ca-trust
               changed_when: false
+            - name: Verify network state restored to default
+              include_tasks: tasks/check_network_dns.yml

tests/playbooks/tests_auto_gateway.yml

@@ -150,3 +150,5 @@
       include_tasks: tasks/manage_test_interface.yml
       vars:
         state: absent
+    - name: Verify network state restored to default
+      include_tasks: tasks/check_network_dns.yml

tests/playbooks/tests_bond.yml

@@ -123,3 +123,5 @@
               when:
                 - network_provider == "initscripts"
               changed_when: false
+            - name: Verify network state restored to default
+              include_tasks: tasks/check_network_dns.yml

tests/playbooks/tests_bond_cloned_mac.yml

@@ -135,3 +135,5 @@
               when:
                 - network_provider == "initscripts"
               changed_when: false
+            - name: Verify network state restored to default
+              include_tasks: tasks/check_network_dns.yml

tests/playbooks/tests_bond_deprecated.yml

@@ -123,3 +123,5 @@
               when:
                 - network_provider == "initscripts"
               changed_when: false
+            - name: Verify network state restored to default
+              include_tasks: tasks/check_network_dns.yml

tests/playbooks/tests_bond_options.yml

@@ -89,3 +89,4 @@
             lsr_cleanup:
               - tasks/cleanup_bond_profile+device.yml
               - tasks/remove_test_interfaces_with_dhcp.yml
+              - tasks/check_network_dns.yml

tests/playbooks/tests_bond_removal.yml

@@ -257,3 +257,5 @@
               when:
                 - network_provider == "initscripts"
               changed_when: false
+            - name: Verify network state restored to default
+              include_tasks: tasks/check_network_dns.yml

tests/playbooks/tests_bridge.yml

@@ -60,3 +60,9 @@
   import_playbook: run_tasks.yml
   vars:
     task: tasks/assert_device_absent.yml
+
+- name: Verify that cleanup restored state to default
+  hosts: all
+  tasks:
+    - name: Verify network state restored to default
+      include_tasks: tasks/check_network_dns.yml

tests/playbooks/tests_bridge_cloned_mac.yml

@@ -60,3 +60,9 @@
   import_playbook: remove_profile.yml
   vars:
     profile: "{{ interface }}"
+
+- name: Verify that cleanup restored state to default
+  hosts: all
+  tasks:
+    - name: Verify network state restored to default
+      include_tasks: tasks/check_network_dns.yml

tests/playbooks/tests_checkpoint_cleanup.yml

@@ -101,3 +101,5 @@
             - name: Delete the device '{{ interface }}'
               command: ip link del "{{ interface }}"
               changed_when: false
+            - name: Verify network state restored to default
+              include_tasks: tasks/check_network_dns.yml

tests/playbooks/tests_dummy.yml

@@ -32,3 +32,4 @@
               - tasks/assert_device_present.yml
             lsr_cleanup:
               - tasks/cleanup_profile+device.yml
+              - tasks/check_network_dns.yml

tests/playbooks/tests_eth_dns_support.yml

@@ -138,4 +138,6 @@
         profile: "{{ interface }}"
     - name: Include the task 'assert_device_absent.yml'
       include_tasks: tasks/assert_device_absent.yml
+    - name: Verify network state restored to default
+      include_tasks: tasks/check_network_dns.yml
 ...

tests/playbooks/tests_eth_pci_address_match.yml

@@ -79,3 +79,5 @@
                     persistent_state: absent
                     state: down
               failed_when: false
+            - name: Verify network state restored to default
+              include_tasks: tasks/check_network_dns.yml

tests/playbooks/tests_ethernet.yml

@@ -74,3 +74,8 @@
         profile: "{{ interface }}"
     - name: Include the task 'assert_device_absent.yml'
       include_tasks: tasks/assert_device_absent.yml
+- name: Verify that cleanup restored state to default
+  hosts: all
+  tasks:
+    - name: Verify network state restored to default
+      include_tasks: tasks/check_network_dns.yml

tests/playbooks/tests_ethtool_coalesce.yml

@@ -174,3 +174,5 @@
               include_tasks: tasks/manage_test_interface.yml
               vars:
                 state: absent
+            - name: Verify network state restored to default
+              include_tasks: tasks/check_network_dns.yml

tests/playbooks/tests_ethtool_features.yml

@@ -224,3 +224,5 @@
               include_tasks: tasks/manage_test_interface.yml
               vars:
                 state: absent
+            - name: Verify network state restored to default
+              include_tasks: tasks/check_network_dns.yml

tests/playbooks/tests_ethtool_ring.yml

@@ -215,3 +215,5 @@
               include_tasks: tasks/manage_test_interface.yml
               vars:
                 state: absent
+            - name: Verify network state restored to default
+              include_tasks: tasks/check_network_dns.yml

tests/playbooks/tests_ignore_auto_dns.yml

@@ -79,4 +79,6 @@
               include_tasks: tasks/assert_profile_absent.yml
               vars:
                 profile: "{{ interface }}"
+            - name: Verify network state restored to default
+              include_tasks: tasks/check_network_dns.yml
 ...

tests/playbooks/tests_infiniband.yml

@@ -89,3 +89,5 @@
               command: ip link del {{ interface }}.000a
               failed_when: false
               changed_when: false
+            - name: Verify network state restored to default
+              include_tasks: tasks/check_network_dns.yml

tests/playbooks/tests_ipv6.yml

@@ -112,3 +112,5 @@
           when:
             - ansible_distribution_major_version != '6'
           changed_when: false
+        - name: Verify network state restored to default
+          include_tasks: tasks/check_network_dns.yml

tests/playbooks/tests_ipv6_disabled.yml

@@ -93,4 +93,6 @@
         profile: "{{ interface }}"
     - name: Include the task 'assert_device_absent.yml'
       include_tasks: tasks/assert_device_absent.yml
+    - name: Verify network state restored to default
+      include_tasks: tasks/check_network_dns.yml
 ...

tests/playbooks/tests_ipv6_dns_search.yml

@@ -161,3 +161,5 @@
                     persistent_state: absent
                     state: down
               failed_when: false
+            - name: Verify network state restored to default
+              include_tasks: tasks/check_network_dns.yml

tests/playbooks/tests_network_state.yml

@@ -185,4 +185,6 @@
       include_tasks: tasks/delete_interface.yml
     - name: Include the task 'assert_device_absent.yml'
       include_tasks: tasks/assert_device_absent.yml
+    - name: Verify network state restored to default
+      include_tasks: tasks/check_network_dns.yml
 ...

tests/playbooks/tests_provider.yml

@@ -38,3 +38,4 @@
               - tasks/assert_profile_present.yml
             lsr_cleanup:
               - tasks/cleanup_profile+device.yml
+              - tasks/check_network_dns.yml

tests/playbooks/tests_reapply.yml

@@ -79,3 +79,5 @@
             - name: Delete the device '{{ interface }}'
               command: ip link del "{{ interface }}"
               changed_when: false
+            - name: Verify network state restored to default
+              include_tasks: tasks/check_network_dns.yml

tests/playbooks/tests_route_device.yml

@@ -185,4 +185,6 @@
               loop:
                 - "{{ interface0 }}"
                 - "{{ interface1 }}"
+            - name: Verify network state restored to default
+              include_tasks: tasks/check_network_dns.yml
 ...

tests/playbooks/tests_route_table.yml

@@ -152,4 +152,6 @@
         profile: "{{ interface }}"
     - name: Include the task 'assert_device_absent.yml'
       include_tasks: tasks/assert_device_absent.yml
+    - name: Verify network state restored to default
+      include_tasks: tasks/check_network_dns.yml
 ...

tests/playbooks/tests_route_type.yml

@@ -191,4 +191,6 @@
         profile: "{{ interface }}"
     - name: Include the task 'assert_device_absent.yml'
       include_tasks: tasks/assert_device_absent.yml
+    - name: Verify network state restored to default
+      include_tasks: tasks/check_network_dns.yml
 ...

tests/playbooks/tests_routing_rules.yml

@@ -273,4 +273,6 @@
         profile: "{{ interface }}"
     - name: Include the task 'assert_device_absent.yml'
       include_tasks: tasks/assert_device_absent.yml
+    - name: Verify network state restored to default
+      include_tasks: tasks/check_network_dns.yml
 ...

tests/playbooks/tests_states.yml

@@ -145,3 +145,4 @@
                 condition: "{{ network_provider == 'nm' }}"
             lsr_cleanup:
               - tasks/cleanup_profile+device.yml
+              - tasks/check_network_dns.yml

tests/playbooks/tests_switch_provider.yml

@@ -61,3 +61,5 @@
       include_tasks: tasks/delete_interface.yml
     - name: Include the task 'assert_device_absent.yml'
       include_tasks: tasks/assert_device_absent.yml
+    - name: Verify network state restored to default
+      include_tasks: tasks/check_network_dns.yml

tests/playbooks/tests_team.yml

@@ -32,3 +32,4 @@
               - tasks/assert_device_present.yml
             lsr_cleanup:
               - tasks/cleanup_profile+device.yml
+              - tasks/check_network_dns.yml

tests/playbooks/tests_team_plugin_installation.yml

@@ -42,4 +42,7 @@
         that:
           - "'NetworkManager-team' in ansible_facts.packages"
         msg: "NetworkManager-team is not installed after team configuration"
+
+    - name: Verify network state restored to default
+      include_tasks: tasks/check_network_dns.yml
 ...

tests/playbooks/tests_vlan_mtu.yml

@@ -74,3 +74,5 @@
       include_tasks: tasks/manage_test_interface.yml
       vars:
         state: absent
+    - name: Verify network state restored to default
+      include_tasks: tasks/check_network_dns.yml

tests/playbooks/tests_wireless.yml

@@ -95,3 +95,5 @@
               failed_when: false
             - name: Include the task 'cleanup_mock_wifi.yml'
               include_tasks: tasks/cleanup_mock_wifi.yml
+            - name: Verify network state restored to default
+              include_tasks: tasks/check_network_dns.yml

tests/playbooks/tests_wireless_plugin_installation.yml

@@ -43,3 +43,6 @@
         that:
           - "'NetworkManager-wifi' in ansible_facts.packages"
         msg: "NetworkManager-wifi is not installed after wireless configured"
+
+    - name: Verify network state restored to default
+      include_tasks: tasks/check_network_dns.yml

tests/playbooks/tests_wireless_wpa3_owe.yml

@@ -60,3 +60,5 @@
               failed_when: false
             - name: Include the task 'cleanup_mock_wifi.yml'
               include_tasks: tasks/cleanup_mock_wifi.yml
+            - name: Verify network state restored to default
+              include_tasks: tasks/check_network_dns.yml

tests/playbooks/tests_wireless_wpa3_sae.yml

@@ -61,3 +61,5 @@
               failed_when: false
             - name: Include the task 'cleanup_mock_wifi.yml'
               include_tasks: tasks/cleanup_mock_wifi.yml
+            - name: Verify network state restored to default
+              include_tasks: tasks/check_network_dns.yml

tests/tasks/check_network_dns.yml

@@ -0,0 +1,39 @@
+# SPDX-License-Identifier: BSD-3-Clause
+---
+# This is typically used after cleanup, to verify that cleanup
+# worked and left networking in the default state - sort of like
+# a post-condition, or pre-condition for subsequent tests
+- name: Check routes and DNS
+  shell: |
+    set -euo pipefail
+    echo IP
+    ip a
+    echo IP ROUTE
+    ip route
+    echo IP -6 ROUTE
+    ip -6 route
+    echo RESOLV
+    if [ -f /etc/resolv.conf ]; then
+      cat /etc/resolv.conf
+    else
+      echo NO /etc/resolv.conf
+      ls -alrtF /etc/resolv.* || :
+    fi
+  changed_when: false
+
+- name: Verify DNS and network connectivity
+  shell: |
+    set -euo pipefail
+    echo CHECK DNS AND CONNECTIVITY
+    for host in mirrors.fedoraproject.org mirrors.centos.org; do
+      if ! getent hosts "$host"; then
+        echo FAILED to lookup host "$host"
+        exit 1
+      fi
+      if ! curl -o /dev/null https://"$host"; then
+        echo FAILED to contact host "$host"
+        exit 1
+      fi
+    done
+  when: ansible_facts["distribution"] == "CentOS"
+  changed_when: false

tests/tests_unit.yml

@@ -176,6 +176,9 @@
             state: absent
             path: "{{ _rundir.path }}"
 
+        - name: Verify network state restored to default
+          include_tasks: tasks/check_network_dns.yml
+
     - name: Ensure that at least one python unit test ran
       fail:
         msg: Tests did not run with python2 or python3