[pytest] improvements for pytest infrastructure #1038:

[pytest] improvements for pytest infrastructure

.gitignore

@@ -1,2 +1,4 @@
 # Compiled Python files
-ansible/plugins/filter/*.pyc
+*.pyc
+__pycache__/
+.pytest_cache/

ansible/roles/test/tasks/lldp.yml

@@ -1,61 +1,4 @@
-# Gather minigraph facts
-- name: Gathering minigraph facts about the device
-  minigraph_facts:
-    host: "{{ inventory_hostname }}"
-
-- name: Print neighbors in minigraph
-  debug: msg="{{ minigraph_neighbors }}"
-
-- name: find minigraph lldp neighbor
-  set_fact:
-    minigraph_lldp_nei:  "{{ minigraph_lldp_nei|default({}) | combine({ item.key : item.value}) }}"
-  when: "'server' not in item.value['name'] | lower"
-  with_dict: minigraph_neighbors
-
-- name: Gather information from LLDP
-  lldp:
+- name: run test
+  include: roles/test/tasks/pytest_runner.yml
   vars:
-    ansible_shell_type: docker
-    ansible_python_interpreter: docker exec -i lldp python
-
-- name: Print LLDP information
-  debug: msg="{{ lldp }}"
-
-- name: Verify LLDP information is available on most interfaces
-  assert: { that: "{{ lldp|length }} > {{ minigraph_lldp_nei|length * 0.8 }}"}
-
-- name: Compare the LLDP neighbor name with minigraph neigbhor name (exclude the management port)
-  assert: { that: "'{{ lldp[item]['chassis']['name'] }}' == '{{ minigraph_lldp_nei[item]['name'] }}'" }
-  with_items: "{{ lldp.keys() }}"
-  when: item != "eth0"
-
-- name: Compare the LLDP neighbor interface with minigraph neigbhor interface (exclude the management port)
-  assert: { that: "'{{ lldp[item]['port']['ifname'] }}' == '{{ minigraph_neighbors[item]['port'] }}'" }
-  with_items: "{{ lldp.keys() }}"
-  when: item != "eth0"
-
-- block:
-  - name: Obtain the system description of the DUT chassis
-    shell: "docker exec -i lldp lldpcli show chassis | grep \"SysDescr:\" | sed -e 's/^\\s*SysDescr:\\s*//g'"
-    register: result
-
-  - name: Store system description of the DUT chassis as a fact
-    set_fact:
-      dut_system_description: "{{ result.stdout }}"
-
-###TODO: fix this lldp_neighbor validation, this part is not running 
-- name: Iterate through each LLDP neighbor and verify the information received by neighbor is correct
-  add_host:
-    name: "{{ lldp[item]['chassis']['mgmt-ip'] }}"
-    groups: "lldp_neighbors,eos"
-    neighbor_interface: "{{ lldp[item]['port']['ifname'] }}"
-    dut_interface: "{{ item }}"
-    hname: "{{ lldp[item]['chassis']['mgmt-ip'] }}"
-    dut_chassis_id: "0x{{ ansible_eth0['macaddress'] | replace(':', '') }}"
-    dut_hostname: "{{ inventory_hostname }}"
-    dut_port_alias: "{{ minigraph_ports[item]['alias'] }}"
-    dut_port_description: "{{ minigraph_neighbors[item]['name'] }}:{{ minigraph_neighbors[item]['port'] }}"
-    dut_system_description: "{{ dut_system_description }}"
-  with_items: "{{ lldp.keys() }}"
-  when: item != "eth0"
-
+    test_node: test_lldp.py

ansible/roles/test/tasks/pytest_runner.yml

@@ -0,0 +1,63 @@
+- name: print a warning
+  debug:
+    msg:
+      - "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+      - "!!!!!! Ansible playbook for running {{ testcase_name }} is now deprecated !!!!!!"
+      - "!!!!!! This playbook is just a wrapper to run py.test in sonic-mgmt/tests !!!!!!"
+      - "!!!!!!!!!!!!!!!!!!!!!!!!!! Consider using py.test !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+      - "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+
+- fail:
+    msg: Test node is not defined
+  when: test_node is not defined
+
+- fail:
+    msg: Testbed is not defined
+  when: testbed_name is not defined
+
+- name: set py.test command variable
+  set_fact:
+    pytest_cmd: 'py.test {{ test_node }}'
+
+- name: append filter expression if needed
+  set_fact:
+    pytest_cmd: '{{ pytest_cmd }} -k "{{ test_filter }}"'
+  when: test_filter is defined
+
+- name: append mark if needed
+  set_fact:
+    pytest_cmd: '{{ pytest_cmd }} -m {{ test_mark }}'
+  when: test_mark is defined
+
+- name: append testbed name
+  set_fact:
+    pytest_cmd: '{{ pytest_cmd }} --testbed={{ testbed_name }}'
+
+- name: append inventory file
+  set_fact:
+    pytest_cmd: '{{ pytest_cmd }} --inventory=../ansible/{{ inventory_file }}'
+
+- name: append testbed file
+  set_fact:
+    pytest_cmd: '{{ pytest_cmd }} --testbed_file=../ansible/{{ testbed_file }}'
+
+- name: append host pattern
+  set_fact:
+    pytest_cmd: '{{ pytest_cmd }} --host-pattern={{ testbed_name }}'
+
+- name: append verbosity flag
+  set_fact:
+    pytest_cmd: '{{ pytest_cmd }} -v'
+
+- debug: var=pytest_cmd
+
+- name: run py.test
+  connection: local
+  shell: '{{ pytest_cmd }}'
+  args:
+    chdir: ../tests/
+  environment:
+    ANSIBLE_LIBRARY: ../ansible/library/
+  register: out
+
+- debug: msg='{{ out.stdout_lines }}'

tests/ansible_fixtures.py

@@ -0,0 +1,31 @@
+""" This module provides few pytest-ansible fixtures overridden """
+
+import pytest
+
+# Here we override ansible_adhoc fixture from pytest-ansible plugin to overcome
+# scope limitation issue; since we want to be able to use ansible_adhoc in module/class scope
+# fixtures we have to override the scope here in global conftest.py
+# Let's have it with module scope for now, so if something really breaks next test module run will have
+# this fixture reevaluated
+@pytest.fixture(scope='module')
+def ansible_adhoc(request):
+    """Return an inventory initialization method."""
+    plugin = request.config.pluginmanager.getplugin("ansible")
+
+    def init_host_mgr(**kwargs):
+        return plugin.initialize(request.config, request, **kwargs)
+    return init_host_mgr
+
+
+# Same as for ansible_adhoc, let's have localhost fixture with session scope
+# as it feels that during session run the localhost object should persist unchanged.
+# Also, we have autouse=True here to force pytest to evaluate localhost fixture to overcome
+# some hidden dependency between localhost and ansible_adhoc (even with default scope) (FIXME)
+@pytest.fixture(scope='session', autouse=True)
+def localhost(request):
+    """Return a host manager representing localhost."""
+    # NOTE: Do not use ansible_adhoc as a dependent fixture since that will assert specific command-line parameters have
+    # been supplied.  In the case of localhost, the parameters are provided as kwargs below.
+    plugin = request.config.pluginmanager.getplugin("ansible")
+    return plugin.initialize(request.config, request, inventory='localhost,', connection='local',
+                             host_pattern='localhost').localhost

tests/ansible_host.py

@@ -16,9 +16,10 @@ def __init__(self, msg, results=None):
     def __str__(self):
         return "{}\nAnsible Results => {}".format(self.message, dump_ansible_results(self.results))
 
-class ansible_host():
-
-    def __init__(self, ansible_adhoc, hostname, is_local = False):
+class AnsibleHost(object):
+    """ wrapper for ansible host object """
+
+    def __init__(self, ansible_adhoc, hostname, is_local=False):
         if is_local:
             self.host = ansible_adhoc(inventory='localhost', connection='local')[hostname]
         else:
@@ -28,11 +29,11 @@ def __init__(self, ansible_adhoc, hostname, is_local = False):
     def __getattr__(self, item):
         self.module_name = item
         self.module = getattr(self.host, item)
- 
+
         return self._run
 
     def _run(self, *module_args, **complex_args):
-   
+
         module_ignore_errors = complex_args.pop('module_ignore_errors', False)
 
         res = self.module(*module_args, **complex_args)[self.hostname]

tests/conftest.py

@@ -1,14 +1,22 @@
 import pytest
 import csv
+import yaml
 import ipaddr as ipaddress
 
-class TestbedInfo():
-    '''
+from ansible_host import AnsibleHost
+
+
+pytest_plugins = ('ptf_fixtures', 'ansible_fixtures')
+
+
+class TestbedInfo(object):
+    """
     Parse the CSV file used to describe whole testbed info
     Please refer to the example of the CSV file format
     CSV file first line is title
     The topology name in title is using uniq-name | conf-name
-    '''
+    """
+
     def __init__(self, testbed_file):
         self.testbed_filename = testbed_file
         self.testbed_topo = {}
@@ -33,16 +41,49 @@ def __init__(self, testbed_file):
                 if name:
                     self.testbed_topo[name] = tb_prop
 
+
 def pytest_addoption(parser):
     parser.addoption("--testbed", action="store", default=None, help="testbed name")
     parser.addoption("--testbed_file", action="store", default=None, help="testbed file name")
 
+
 @pytest.fixture(scope="session")
 def testbed(request):
+    """
+    Create and return testbed information
+    """
     tbname = request.config.getoption("--testbed")
     tbfile = request.config.getoption("--testbed_file")
-    if tbname == None or tbfile == None:
+    if tbname is None or tbfile is None:
         raise ValueError("testbed and testbed_file are required!")
 
     tbinfo = TestbedInfo(tbfile)
     return tbinfo.testbed_topo[tbname]
+
+
+@pytest.fixture(scope="module")
+def duthost(ansible_adhoc, testbed):
+    """
+    Shortcut fixture for getting DUT host
+    """
+
+    hostname = testbed['dut']
+    return AnsibleHost(ansible_adhoc, hostname)
+
+
+@pytest.fixture(scope="module")
+def ptfhost(ansible_adhoc, testbed):
+    """
+    Shortcut fixture for getting PTF host
+    """
+
+    hostname = testbed['ptf']
+    return AnsibleHost(ansible_adhoc, hostname)
+
+
+@pytest.fixture(scope='session')
+def eos():
+    """ read and yield eos configuration """
+    with open('eos/eos.yml') as stream:
+        eos = yaml.safe_load(stream)
+        return eos

tests/eos

@@ -0,0 +1 @@
+../ansible/group_vars/eos/