Add default link parameter support for ethtool module

Testing Done: tested master and 2.5_br images with testifupdown2 suite and hand tested

This patch creates a json defaults file upon bootup
(which can be overridden by customer configs in /etc)
which the ethtool module in ifupdown2 will consult
when "link-x" configs are removed in order to restore
them to the initial settings used by the switch.
(cherry picked from commit 8388664f5a5a85f2a813cafbf40ac92d7b86f4bf)

Conflicts:
	packages/cl-utilities/dist-packages/cumulus/portconfig.py
	packages/cl-utilities/usrlib/update-ports
	tests/tests/smoke/testifupdown2.py

ifupdown2/addons/ethtool.py

@@ -3,98 +3,197 @@
 # Copyright 2014 Cumulus Networks, Inc. All rights reserved.
 # Author: Roopa Prabhu, roopa@cumulusnetworks.com
 #
+import json
+import ifupdown.policymanager as policymanager
 
 try:
     from ipaddr import IPNetwork
     from sets import Set
     from ifupdown.iface import *
+    from ifupdownaddons.utilsbase import *
     from ifupdownaddons.modulebase import moduleBase
     from ifupdownaddons.iproute2 import iproute2
 except ImportError, e:
     raise ImportError (str(e) + "- required module not found")
 
-class ethtool(moduleBase):
+class ethtool(moduleBase,utilsBase):
     """  ifupdown2 addon module to configure ethtool attributes """
 
     _modinfo = {'mhelp' : 'ethtool configuration module for interfaces',
                 'attrs': {
                       'link-speed' :
                             {'help' : 'set link speed',
-                             'example' : ['link-speed 1000']},
+                             'example' : ['link-speed 1000'],
+                             'default' : 'varies by platform and port'},
                       'link-duplex' :
                             {'help': 'set link duplex',
                              'example' : ['link-duplex full'],
                              'validvals' : ['half', 'full'],
-                             'default' : 'half'},
+                             'default' : 'full'},
                       'link-autoneg' :
                             {'help': 'set autonegotiation',
                              'example' : ['link-autoneg on'],
                              'validvals' : ['on', 'off'],
-                             'default' : 'off'}}}
+                             'default' : 'varies by platform and port'}}}
 
     def __init__(self, *args, **kargs):
         moduleBase.__init__(self, *args, **kargs)
         self.ipcmd = None
 
-    def _post_up(self, ifaceobj):
+    def _post_up(self, ifaceobj, operation='post_up'):
+        """
+        _post_up and _pre_down will reset the layer 2 attributes to default policy
+        settings.
+        """
         if not self.ipcmd.link_exists(ifaceobj.name):
             return
         cmd = ''
-        attrval = ifaceobj.get_attr_value_first('link-speed')
+        for attr in ['speed', 'duplex', 'autoneg']:
-        if attrval:
+            # attribute existed before but we must reset to default
-            cmd += ' speed %s' %attrval
+            config_val = ifaceobj.get_attr_value_first('link-%s'%attr)
-        attrval = ifaceobj.get_attr_value_first('link-duplex')
+            default_val = policymanager.policymanager_api.get_iface_default(
-        if attrval:
+                module_name='ethtool',
-            cmd += ' duplex %s' %attrval
+                ifname=ifaceobj.name,
-        attrval = ifaceobj.get_attr_value_first('link-autoneg')
+                attr='link-%s'%attr)
-        if attrval:
+
-            cmd += ' autoneg %s' %attrval
+            # check running values
+            running_val = None
+            if attr == 'autoneg':
+                # we can only get autoneg from ethtool
+                output = self.exec_commandl(['ethtool', ifaceobj.name])
+                running_val = self.get_autoneg(ethtool_output=output)
+            else:
+                running_val = self.read_file_oneline('/sys/class/net/%s/%s' % \
+                                                     (ifaceobj.name, attr))
+            if config_val and config_val == running_val:
+                # running value is what is configured, do nothing
+                continue
+            if not config_val and default_val and default_val == running_val:
+                # nothing configured but the default is running
+                continue
+            # if we got this far, we need to change it
+            if config_val and (config_val != running_val):
+                # if the configured value is not set, set it
+                cmd += ' %s %s' % (attr, config_val)
+            elif default_val and (default_val != running_val):
+                # or if it has a default not equal to running value, set it
+                cmd += ' %s %s' % (attr, default_val)
+            else:
+                # no value set nor default, leave it alone
+                pass
         if cmd:
+            self.logger.debug('ethtool %s: iface %s cmd is %s' % \
+                              (operation, ifaceobj.name, cmd))
             try:
+                # we should only be calling ethtool if there
+                # is a speed set or we can find a default speed
+                # because we should only be calling ethtool on swp ports
                 cmd = 'ethtool -s %s %s' %(ifaceobj.name, cmd)
                 self.exec_command(cmd)
             except Exception, e:
                 ifaceobj.status = ifaceStatus.ERROR
                 self.log_warn('%s: %s' %(ifaceobj.name, str(e)))
+        else:
+            pass
+
+    def _pre_down(self, ifaceobj):
+        pass #self._post_up(ifaceobj,operation="_pre_down")
 
     def _query_check(self, ifaceobj, ifaceobjcurr):
         """
-        Advertised auto-negotiation: No
+        _query_check() needs to compare the configured (or running)
-        Speed: 1000Mb/s
+        attribute with the running attribute.
-        Duplex: Full"""
+
-        ethtool_attrs = self.dict_key_subset(ifaceobj.config,
+        If there is nothing configured, we compare the default attribute with
-                                          self.get_mod_attrs())
+        the running attribute and FAIL if they are different.
-        if not ethtool_attrs:
+        This is because a reboot will lose their running attribute
+        (the default will get set).
+        """
+        for attr in ['speed', 'duplex', 'autoneg']:
+            # autoneg comes from ethtool whereas speed and duplex from /sys/class
+            if attr == 'autoneg':
+                output = self.exec_commandl(['ethtool', ifaceobj.name])
+                running_attr = self.get_autoneg(ethtool_output=output)
+            else:
+                running_attr = self.read_file_oneline('/sys/class/net/%s/%s' % \
+                                                      (ifaceobj.name, attr))
+
+            configured = ifaceobj.get_attr_value_first('link-%s'%attr)
+            default = policymanager.policymanager_api.get_iface_default(
+                module_name='ethtool',
+                ifname=ifaceobj.name,
+                attr='link-%s'%attr)
+
+            # there is a case where there is no running config or
+            # (there is no default and it is not configured).
+            # In this case, we do nothing (e.g. eth0 has only a
+            # default duplex, lo has nothing)
+            if (not running_attr or (not configured and not default)):
+                continue
+
+            # we make sure we can get a running value first
+            if (running_attr and configured and running_attr == configured):
+                # PASS since running is what is configured 
+                ifaceobjcurr.update_config_with_status('link-%s'%attr,
+                                                       running_attr, 0)
+            elif (running_attr and configured and running_attr != configured):
+                # We show a FAIL since it is not the configured or default
+                ifaceobjcurr.update_config_with_status('link-%s'%attr,
+                                                       running_attr, 1)
+            elif (running_attr and default and running_attr == default):
+                # PASS since running is default
+                ifaceobjcurr.update_config_with_status('link-%s'%attr,
+                                                       running_attr, 0)
+            elif (default or configured):
+                # We show a FAIL since it is not the configured or default
+                ifaceobjcurr.update_config_with_status('link-%s'%attr,
+                                                       running_attr, 1)
         return
+
+    def get_autoneg(self,ethtool_output=None):
+        """
+        get_autoneg simply calls the ethtool command and parses out
+        the autoneg value.
+        """
+        ethtool_attrs = ethtool_output.split()
+        if ('Auto-negotiation:' in ethtool_attrs):
+            return(ethtool_attrs[ethtool_attrs.index('Auto-negotiation:')+1])
+        else:
+            return(None)
+
+    def _query_running(self, ifaceobj, ifaceobj_getfunc=None):
+        """
+        _query_running looks at the speed and duplex from /sys/class
+        and retreives autoneg from ethtool.  We do not report autoneg
+        if speed is not available because this usually means the link is
+        down and the autoneg value is not reliable when the link is down.
+        """
+        # do not bother showing swp ifaces that are not up for the speed
+        # duplex and autoneg are not reliable.
+        if not self.ipcmd.is_link_up(ifaceobj.name):
+            return
+        for attr in ['speed', 'duplex', 'autoneg']:
+            # autoneg comes from ethtool whereas speed and duplex from /sys/class
+            running_attr = None
             try:
-            speed = ifaceobj.get_attr_value_first('link-speed')
+                if attr == 'autoneg':
-            if speed:
+                    output=self.exec_commandl(['ethtool', ifaceobj.name])
-                running_speed = self.read_file_oneline(
+                    running_attr = self.get_autoneg(ethtool_output=output)
-                                    '/sys/class/net/%s/speed' %ifaceobj.name)
-                if running_speed and running_speed != speed:
-                     ifaceobjcurr.update_config_with_status('link-speed',
-                             running_speed, 1)
                 else:
-                     ifaceobjcurr.update_config_with_status('link-speed',
+                    running_attr = self.read_file_oneline('/sys/class/net/%s/%s' % \
-                             running_speed, 0)
+                                                          (ifaceobj.name, attr))
-            duplex = ifaceobj.get_attr_value_first('link-duplex')
+            except:
-            if duplex:
+                # for nonexistent interfaces, we get an error (rc = 256 or 19200)
-                running_duplex = self.read_file_oneline(
-                                    '/sys/class/net/%s/duplex' %ifaceobj.name)
-                if running_duplex and running_duplex != duplex:
-                     ifaceobjcurr.update_config_with_status('link-duplex',
-                             running_duplex, 1)
-                else:
-                     ifaceobjcurr.update_config_with_status('link-duplex',
-                             running_duplex, 0)
-        except Exception:
                 pass
+
+            # show it
+            if (running_attr):
+                ifaceobj.update_config('link-%s'%attr, running_attr)
+
         return
 
-    def _query_running(self, ifaceobjrunning):
+    _run_ops = {'pre-down' : _pre_down,
-        return
+                'post-up' : _post_up,
-
-    _run_ops = {'post-up' : _post_up,
                 'query-checkcurr' : _query_check,
                 'query-running' : _query_running }
 
@@ -127,6 +226,14 @@ class ethtool(moduleBase):
         if not op_handler:
             return
         self._init_command_handlers()
+
+        # check to make sure we are only checking/setting interfaces with
+        # no lower interfaces.   No bridges, no vlans, loopbacks.
+        if ifaceobj.lowerifaces != None or \
+           self.ipcmd.link_isloopback(ifaceobj.name) or \
+           self.ipcmd.is_vlan_device_by_name(ifaceobj.name):
+            return
+
         if operation == 'query-checkcurr':
             op_handler(self, ifaceobj, query_ifaceobj)
         else:

ifupdown2/config/addons.conf

@@ -14,6 +14,7 @@ post-up,ethtool
 post-up,usercmds
 post-up,clagd
 pre-down,usercmds
+pre-down,ethtool
 down,dhcp
 down,addressvirtual
 down,address

ifupdown2/ifupdown/ifupdownmain.py

@@ -1117,6 +1117,7 @@ class ifupdownMain(ifupdownBase):
             self.ALL = True
             self.WITH_DEPENDS = True
         if new_ifaceobjdict:
+            # and now, ifaceobjdict is back to current config
             self.ifaceobjdict = new_ifaceobjdict
             self.dependency_graph = new_dependency_graph
 
@@ -1231,6 +1232,7 @@ class ifupdownMain(ifupdownBase):
         if auto:
             self.ALL = True
             self.WITH_DEPENDS = True
+        # and now, we are back to the current config in ifaceobjdict
         self.ifaceobjdict = new_ifaceobjdict
         self.dependency_graph = new_dependency_graph
         ifacenames = self.ifaceobjdict.keys()

ifupdown2/setup.py

@@ -40,6 +40,8 @@ setup(name='ifupdown2',
                       'addons/ethtool.py', 'addons/loopback.py',
                       'addons/addressvirtual.py', 'addons/vxlan.py',
                       'addons/bridgevlan.py']),
-                  ('/var/lib/ifupdownaddons/', ['config/addons.conf'])
+                  ('/var/lib/ifupdownaddons/', ['config/addons.conf']),
+                  ('/var/lib/ifupdownaddons/policy.d/', []),
+                  ('/etc/network/ifupdown2/policy.d/', [])
                   ]
       )

packages/ifupdown2/ifupdown/policymanager.py

@@ -0,0 +1,175 @@
+#!/usr/bin/python
+#
+# Copyright 2015 Cumulus Networks, Inc. All rights reserved.
+#
+#
+'''
+The PolicyManager should be subclassed by addon modules
+to read a JSON policy config file that is later used to
+set defaults:
+
+Initialize: This module defines a list of config file location based
+          on module.  There are defined in the __init__():  All the
+          addon modules need to do is import the policymanager module.
+
+          import ifupdown.policymanager as policymanager
+
+
+Provides: an API to retrieve link attributes based on addon module name,
+          interface name, and attribute.
+
+        The ifupdown.policymanager module provides a global object policymanager_api
+        that can be called like so:
+
+        speed_default = policymanager.policymanager_api.get_default(
+            module_name='ethtool',
+            ifname=ifaceobj.name,
+            attr='link-speed'
+            )
+'''
+
+import json
+import logging
+import glob
+
+class policymanager():
+    def __init__(self):
+        # we should check for these files in order
+        # so that customers can override the /var/lib file settings
+        self.logger = logging.getLogger('ifupdown.' +
+                            self.__class__.__name__)
+
+        # we grab the json files from a known location and make sure that
+        # the defaults_policy is checked first
+        user_files = glob.glob('/etc/network/ifupdown2/policy.d/*.json')
+        # grab the default module files
+        default_files = glob.glob('/var/lib/ifupdownaddons/policy.d/*.json')
+        # keep an array of defaults indexed by module name
+        self.system_policy_array = {}
+        for filename in default_files:
+            system_array  = {}
+            try:
+                fd = open(filename,'r')
+                system_array = json.load(fd)
+                self.logger.debug('reading %s system policy defaults config' \
+                                  % filename)
+            except Exception, e:
+                self.logger.debug('could not read %s system policy defaults config' \
+                                  % filename)
+                self.logger.debug('    exception is %s' % str(e))
+            for module in system_array.keys():
+                if self.system_policy_array.has_key(module):
+                    self.logger.debug('warning: overwriting system module %s from file %s' \
+                                      % (module,filename))
+                self.system_policy_array[module] = system_array[module]
+
+        # take care of user defined policy defaults
+        self.user_policy_array = {}
+        for filename in user_files:
+            user_array  = {}
+            try:
+                fd = open(filename,'r')
+                user_array = json.load(fd)
+                self.logger.debug('reading %s policy user defaults config' \
+                                  % filename)
+            except Exception, e:
+                self.logger.debug('could not read %s user policy defaults config' \
+                                  % filename)
+                self.logger.debug('    exception is %s' % str(e))
+            # customer added module attributes
+            for module in user_array.keys():
+                if self.system_policy_array.has_key(module):
+                    # warn user that we are overriding the system module setting
+                    self.logger.debug('warning: overwriting system with user module %s from file %s' \
+                                      % (module,filename))
+                self.user_policy_array[module] = user_array[module]
+        return
+
+    def get_iface_default(self,module_name=None,ifname=None,attr=None):
+        '''
+        get_iface_default: Addon modules must use one of two types of access methods to
+        the default configs.   In this method, we expect the default to be
+        either in
+            [module]['iface_defaults'][ifname][attr] or
+            [module]['defaults'][attr]
+        We first check the user_policy_array and return that value. But if
+        the user did not specify an override, we use the system_policy_array.
+        '''
+        # make sure we have an index
+        if (not ifname or not attr or not module_name):
+            return None
+
+        val = None
+        # users can specify defaults to override the systemwide settings
+        # look for user specific interface attribute iface_defaults first
+        try:
+            # looks for user specified value
+            val = self.user_policy_array[module_name]['iface_defaults'][ifname][attr]
+            return val
+        except:
+            pass
+        try:
+            # failing that, there may be a user default for all intefaces
+            val = self.user_policy_array[module_name]['defaults'][attr]
+            return val
+        except:
+            pass
+        try:
+            # failing that, look for  system setting for the interface
+            val = self.system_policy_array[module_name]['iface_defaults'][ifname][attr]
+            return val
+        except:
+            pass
+        try:
+            # failing that, look for  system setting for all interfaces
+            val = self.system_policy_array[module_name]['defaults'][attr]
+            return val
+        except:
+            pass
+
+        # could not find any system or user default so return Non
+        return val
+
+    def get_attr_default(self,module_name=None,attr=None):
+        '''
+        get_attr_default: Addon modules must use one of two types of access methods to
+        the default configs.   In this method, we expect the default to be in
+
+        [module][attr] 
+
+        We first check the user_policy_array and return that value. But if
+        the user did not specify an override, we use the system_policy_array.
+        '''
+        if (not attr or not module_name):
+            return None
+        # users can specify defaults to override the systemwide settings
+        # look for user specific interface attribute iface_defaults first
+        val = None
+        if self.user_policy_array.get(module_name):
+            val = self.user_policy_array[module_name].get(attr)
+
+        if not val:
+            if self.system_policy_array.get(module_name):
+                val = self.system_policy_array[module_name].get(attr)
+
+        return val
+
+    def get_module_default(self,module_name=None):
+        '''
+        get_module_default: Addon modules can also access the entire config
+        This method returns indexed by "system" and "user": these are the
+        system-wide and user-defined policy arrays for a specific module.
+        '''
+        if not module_name:
+            return None
+        if self.system_policy_array.get(module_name) and \
+           self.user_policy_array.get(module_name):
+            mod_array = {"system":self.system_policy_array[module_name],
+                         "user":self.user_policy_array[module_name]}
+        else:
+            # the module must not have these defined, return None
+            mod_array = None
+
+        return mod_array
+
+policymanager_api = policymanager()