fix ensure_provider_tests.py; fix generated tests; add check to CI

Fix ensure_provider_tests.py so that the tests are correctly generated.
Because the generated tests have not been checked in a while, there was
some drift between the actual tests and what should have been generated,
so the new generated tests have also been included.
When `ensure_provider_tests.py` detects differences, it will now show
the diffs using a unified diff format.
Added a new tox testenv `ensure_provider_tests` - use
`tox -e ensure_provider_tests` to run - to pass arguments, use
`tox -e ensure_provider_tests -- generate`
Added `ensure_provider_tests` as one of the tox testenvs to run for
github tox CI.

Signed-off-by: Rich Megginson <rmeggins@redhat.com>

.github/workflows/tox.yml

@@ -38,7 +38,7 @@ jobs:
           toxenvs="py${toxpyver}"
           case "$toxpyver" in
           27) toxenvs="${toxenvs},coveralls,flake8,pylint" ;;
-          36) toxenvs="${toxenvs},coveralls,black,yamllint,ansible-lint,collection,ansible-test" ;;
+          36) toxenvs="${toxenvs},coveralls,ensure_provider_tests,black,yamllint,ansible-lint,collection,ansible-test" ;;
           37) toxenvs="${toxenvs},coveralls" ;;
           38) toxenvs="${toxenvs},coveralls" ;;
           esac

.sanity-ansible-ignore-2.11.txt

@@ -59,6 +59,8 @@ tests/network/ensure_provider_tests.py metaclass-boilerplate!skip
 tests/network/ensure_provider_tests.py shebang!skip
 tests/network/get_coverage.sh shebang!skip
 tests/network/get_total_coverage.sh shebang!skip
+tests/network/git-pre-commit.sh shebang!skip
+tests/network/git-post-commit.sh shebang!skip
 tests/network/integration/conftest.py future-import-boilerplate!skip
 tests/network/integration/conftest.py metaclass-boilerplate!skip
 tests/network/integration/test_ethernet.py future-import-boilerplate!skip

.sanity-ansible-ignore-2.9.txt

@@ -37,6 +37,8 @@ tests/network/ensure_provider_tests.py metaclass-boilerplate!skip
 tests/network/ensure_provider_tests.py shebang!skip
 tests/network/get_coverage.sh shebang!skip
 tests/network/get_total_coverage.sh shebang!skip
+tests/network/git-pre-commit.sh shebang!skip
+tests/network/git-post-commit.sh shebang!skip
 tests/network/integration/conftest.py future-import-boilerplate!skip
 tests/network/integration/conftest.py metaclass-boilerplate!skip
 tests/network/integration/test_ethernet.py future-import-boilerplate!skip

.travis/custom.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-# SPDX-License-Identifier: MIT
-
-set -e
-
-# https://github.com/koalaman/shellcheck/wiki/SC1091
-# shellcheck disable=SC1091
-. "$LSR_SCRIPTDIR/utils.sh"
-
-# Write your custom commands here that should be run when `tox -e custom`:
-if lsr_check_python_version python -eq '3.6'; then
-    (set -x; cd "${TOPDIR}/tests"; python ./ensure_provider_tests.py)
-fi

tests/ensure_provider_tests.py

@@ -4,6 +4,7 @@
 """
 # vim: fileencoding=utf8
 
+import difflib
 import glob
 import os
 import sys
@@ -40,6 +41,7 @@ RUN_PLAYBOOK_WITH_NM = """# SPDX-License-Identifier: BSD-3-Clause
 - hosts: all
   name: Run playbook '{test_playbook}' with nm as provider
   tasks:
+    - include_tasks: tasks/el_repo_setup.yml
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm
@@ -118,8 +120,13 @@ NM_CONDITIONAL_TESTS = {
         MINIMUM_VERSION: "'1.20.0'",
         "comment": "# NetworkManager 1.20.0 introduced ethtool settings support",
     },
+    "playbooks/tests_ethtool_ring.yml": {
+        MINIMUM_VERSION: "'1.25.2'",
+        "comment": "# NetworkManager 1.25.2 introduced ethtool ring support",
+    },
 }
 
+
 IGNORE = [
     # checked by tests_regression_nm.yml
     "playbooks/tests_checkpoint_cleanup.yml",
@@ -207,6 +214,15 @@ def check_playbook(generate, testfile, test_playbook, nominal_data):
             testdata = ifile.read()
             if testdata != nominal_data:
                 print(f"ERROR: Playbook does not match nominal value: {testfile}")
+                sys.stdout.writelines(
+                    difflib.unified_diff(
+                        nominal_data.splitlines(keepends=True),
+                        testdata.splitlines(keepends=True),
+                        fromfile=f"{testfile}.expected",
+                        tofile=f"{testfile}.actual",
+                    )
+                )
+
                 returncode = 1
 
     return is_missing, returncode

tests/git-post-commit.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# SPDX-License-Identifier: BSD-3-Clause
+
+set -euo pipefail
+
+# Run script ensure_provider_tests.py and append test files generated by it to the commit.
+
+# This script is intended to be used as git post-commit hook.
+# Make sure file is executable and copy it to <your repo>/.git/hooks/post-commit
+# This script has to be used together with pre-commit to work properly.
+
+GITPATH=$(git rev-parse --show-toplevel)
+
+if [ -e "$GITPATH/.commit" ]; then
+    rm "$GITPATH/.commit"
+
+    cd "$GITPATH/tests"
+    python ./ensure_provider_tests.py generate
+    cd "$GITPATH"
+    git add tests/tests_*_nm.yml tests/tests_*_initscripts.yml
+    git commit --amend -C HEAD --no-verify
+fi

tests/git-pre-commit.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+# SPDX-License-Identifier: BSD-3-Clause
+
+set -euo pipefail
+
+# This script is intended to be used as git pre-commit hook.
+# Make sure file is executable and copy it into <your repo>/.git/hooks/pre-commit
+# This script has to be used together with post-commit to work properly.
+
+GITPATH=$(git rev-parse --show-toplevel)
+
+touch "$GITPATH/.commit"
+

tests/tests_auto_gateway_nm.yml

@@ -5,6 +5,7 @@
 - hosts: all
   name: Run playbook 'playbooks/tests_auto_gateway.yml' with nm as provider
   tasks:
+    - include_tasks: tasks/el_repo_setup.yml
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm

tests/tests_bond_deprecated_nm.yml

@@ -5,6 +5,7 @@
 - hosts: all
   name: Run playbook 'playbooks/tests_bond_deprecated.yml' with nm as provider
   tasks:
+    - include_tasks: tasks/el_repo_setup.yml
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm

tests/tests_dummy_nm.yml

@@ -5,6 +5,7 @@
 - hosts: all
   name: Run playbook 'playbooks/tests_dummy.yml' with nm as provider
   tasks:
+    - include_tasks: tasks/el_repo_setup.yml
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm

tests/tests_eth_dns_support_nm.yml

@@ -5,6 +5,7 @@
 - hosts: all
   name: Run playbook 'playbooks/tests_eth_dns_support.yml' with nm as provider
   tasks:
+    - include_tasks: tasks/el_repo_setup.yml
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm

tests/tests_eth_pci_address_match_nm.yml

@@ -5,6 +5,7 @@
 - hosts: all
   name: Run playbook 'playbooks/tests_eth_pci_address_match.yml' with nm as provider
   tasks:
+    - include_tasks: tasks/el_repo_setup.yml
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm

tests/tests_ethernet_nm.yml

@@ -6,7 +6,6 @@
   name: Run playbook 'playbooks/tests_ethernet.yml' with nm as provider
   tasks:
     - include_tasks: tasks/el_repo_setup.yml
-
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm

tests/tests_ethtool_coalesce_nm.yml

@@ -5,6 +5,7 @@
 - hosts: all
   name: Run playbook 'playbooks/tests_ethtool_coalesce.yml' with nm as provider
   tasks:
+    - include_tasks: tasks/el_repo_setup.yml
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm

tests/tests_ethtool_features_nm.yml

@@ -5,6 +5,7 @@
 - hosts: all
   name: Run playbook 'playbooks/tests_ethtool_features.yml' with nm as provider
   tasks:
+    - include_tasks: tasks/el_repo_setup.yml
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm

tests/tests_ethtool_ring_nm.yml

@@ -5,6 +5,7 @@
 - hosts: all
   name: Run playbook 'playbooks/tests_ethtool_ring.yml' with nm as provider
   tasks:
+    - include_tasks: tasks/el_repo_setup.yml
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm
@@ -30,6 +31,7 @@
 
 # The test requires or should run with NetworkManager, therefore it cannot run
 # on RHEL/CentOS 6
+# NetworkManager 1.25.2 introduced ethtool ring support
 - import_playbook: playbooks/tests_ethtool_ring.yml
   when:
     - ansible_distribution_major_version != '6'

tests/tests_ipv6_disabled_nm.yml

@@ -5,6 +5,7 @@
 - hosts: all
   name: Run playbook 'playbooks/tests_ipv6_disabled.yml' with nm as provider
   tasks:
+    - include_tasks: tasks/el_repo_setup.yml
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm

tests/tests_ipv6_nm.yml

@@ -5,6 +5,7 @@
 - hosts: all
   name: Run playbook 'playbooks/tests_ipv6.yml' with nm as provider
   tasks:
+    - include_tasks: tasks/el_repo_setup.yml
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm

tests/tests_provider_nm.yml

@@ -5,6 +5,7 @@
 - hosts: all
   name: Run playbook 'playbooks/tests_provider.yml' with nm as provider
   tasks:
+    - include_tasks: tasks/el_repo_setup.yml
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm

tests/tests_reapply_nm.yml

@@ -6,7 +6,6 @@
   name: Run playbook 'playbooks/tests_reapply.yml' with nm as provider
   tasks:
     - include_tasks: tasks/el_repo_setup.yml
-
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm

tests/tests_states_nm.yml

@@ -6,7 +6,6 @@
   name: Run playbook 'playbooks/tests_states.yml' with nm as provider
   tasks:
     - include_tasks: tasks/el_repo_setup.yml
-
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm

tests/tests_team_nm.yml

@@ -6,7 +6,6 @@
   name: Run playbook 'playbooks/tests_team.yml' with nm as provider
   tasks:
     - include_tasks: tasks/el_repo_setup.yml
-
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm

tests/tests_team_plugin_installation_nm.yml

@@ -5,6 +5,7 @@
 - hosts: all
   name: Run playbook 'playbooks/tests_team_plugin_installation.yml' with nm as provider
   tasks:
+    - include_tasks: tasks/el_repo_setup.yml
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm

tests/tests_vlan_mtu_nm.yml

@@ -6,7 +6,6 @@
   name: Run playbook 'playbooks/tests_vlan_mtu.yml' with nm as provider
   tasks:
     - include_tasks: tasks/el_repo_setup.yml
-
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm

tests/tests_wireless_nm.yml

@@ -6,7 +6,6 @@
   name: Run playbook 'playbooks/tests_wireless.yml' with nm as provider
   tasks:
     - include_tasks: tasks/el_repo_setup.yml
-
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm

tests/tests_wireless_plugin_installation_nm.yml

@@ -5,6 +5,7 @@
 - hosts: all
   name: Run playbook 'playbooks/tests_wireless_plugin_installation.yml' with nm as provider
   tasks:
+    - include_tasks: tasks/el_repo_setup.yml
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm

tests/tests_wireless_wpa3_owe_nm.yml

@@ -5,6 +5,7 @@
 - hosts: all
   name: Run playbook 'playbooks/tests_wireless_wpa3_owe.yml' with nm as provider
   tasks:
+    - include_tasks: tasks/el_repo_setup.yml
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm

tests/tests_wireless_wpa3_sae_nm.yml

@@ -5,6 +5,7 @@
 - hosts: all
   name: Run playbook 'playbooks/tests_wireless_wpa3_sae.yml' with nm as provider
   tasks:
+    - include_tasks: tasks/el_repo_setup.yml
     - name: Set network provider to 'nm'
       set_fact:
         network_provider: nm
@@ -18,4 +19,4 @@
 - import_playbook: playbooks/tests_wireless_wpa3_sae.yml
   when:
     - ansible_distribution_major_version != '6'
-    - ansible_distribution_major_version != '7'
+    - ansible_distribution_major_version != '7' and ansible_distribution != 'RedHat'

tox.ini

@@ -17,3 +17,9 @@ setenv =
     RUN_PYTEST_EXTRA_ARGS = -v
     RUN_FLAKE8_EXTRA_ARGS = --exclude tests/ensure_provider_tests.py,scripts/print_all_options.py,tests/network/ensure_provider_tests.py,.svn,CVS,.bzr,.hg,.git,__pycache__,.tox,.eggs,*.egg
     LSR_PUBLISH_COVERAGE = normal
+
+[testenv:ensure_provider_tests]
+basepython = python3
+changedir = tests
+commands =
+  python ./ensure_provider_tests.py {posargs}