SRIOV NIC offload feature discovery

Adding functionality to libvirt that will allow it
query the ethtool interface for the availability
of certain NIC HW offload features

Here is an example of the feature XML definition:

<device>
<name>net_eth4_90_e2_ba_5e_a5_45</name>
  <path>/sys/devices/pci0000:00/0000:00:03.0/0000:08:00.1/net/eth4</path>
  <parent>pci_0000_08_00_1</parent>
  <capability type='net'>
    <interface>eth4</interface>
    <address>90:e2:ba:5e:a5:45</address>
    <link speed='10000' state='up'/>
    <feature name='rx'/>
    <feature name='tx'/>
    <feature name='sg'/>
    <feature name='tso'/>
    <feature name='gso'/>
    <feature name='gro'/>
    <feature name='rxvlan'/>
    <feature name='txvlan'/>
    <feature name='rxhash'/>
    <capability type='80203'/>
  </capability>
</device>
Signed-off-by: NJán Tomko <jtomko@redhat.com>

docs/formatnode.html.in

@@ -183,6 +183,24 @@
                 link. So far, the whole element is just for output,
                 not setting.
               </dd>
+              <dt><code>feature</code></dt>
+              <dd>If present, the hw offloads supported by this network
+                interface. Possible features are:
+                <dl>
+                    <dt><code>rx</code></dt><dd>rx-checksumming</dd>
+                    <dt><code>tx</code></dt><dd>tx-checksumming</dd>
+                    <dt><code>sg</code></dt><dd>scatter-gather</dd>
+                    <dt><code>tso</code></dt><dd>tcp-segmentation-offload</dd>
+                    <dt><code>ufo</code></dt><dd>udp-fragmentation-offload</dd>
+                    <dt><code>gso</code></dt><dd>generic-segmentation-offload</dd>
+                    <dt><code>gro</code></dt><dd>generic-receive-offload</dd>
+                    <dt><code>lro</code></dt><dd>large-receive-offload</dd>
+                    <dt><code>rxvlan</code></dt><dd>rx-vlan-offload</dd>
+                    <dt><code>txvlan</code></dt><dd>tx-vlan-offload</dd>
+                    <dt><code>ntuple</code></dt><dd>ntuple-filters</dd>
+                    <dt><code>rxhash</code></dt><dd>receive-hashing</dd>
+                </dl>
+              </dd>
               <dt><code>capability</code></dt>
               <dd>A network protocol exposed by the device, where the
                 attribute <code>type</code> can be "80203" for IEEE

docs/schemas/nodedev.rng

@@ -274,11 +274,25 @@
     </optional>
     <ref name="link-speed-state"/>
 
+    <zeroOrMore>
+      <element name='feature'>
+        <attribute name='name'>
+          <ref name='netfeaturename'/>
+        </attribute>
+      </element>
+    </zeroOrMore>
+
     <zeroOrMore>
       <ref name='subcapnet'/>
     </zeroOrMore>
   </define>
 
+  <define name='netfeaturename'>
+    <data type='string'>
+      <param name='pattern'>[a-zA-Z\-_]+</param>
+    </data>
+  </define>
+
   <define name='subcapnet'>
     <element name='capability'>
       <choice>

src/conf/device_conf.c

@@ -39,6 +39,20 @@ VIR_ENUM_IMPL(virInterfaceState,
               "down", "lowerlayerdown",
               "testing", "dormant", "up")
 
+VIR_ENUM_IMPL(virNetDevFeature,
+              VIR_NET_DEV_FEAT_LAST,
+              "rx",
+              "tx",
+              "sg",
+              "tso",
+              "gso",
+              "gro",
+              "lro",
+              "rxvlan",
+              "txvlan",
+              "ntuple",
+              "rxhash")
+
 int virDevicePCIAddressIsValid(virDevicePCIAddressPtr addr)
 {
     /* PCI bus has 32 slots and 8 functions per slot */

src/conf/device_conf.h

@@ -62,6 +62,23 @@ struct _virInterfaceLink {
     unsigned int speed;      /* link speed in Mbits per second */
 };
 
+typedef enum {
+    VIR_NET_DEV_FEAT_GRXCSUM,
+    VIR_NET_DEV_FEAT_GTXCSUM,
+    VIR_NET_DEV_FEAT_GSG,
+    VIR_NET_DEV_FEAT_GTSO,
+    VIR_NET_DEV_FEAT_GGSO,
+    VIR_NET_DEV_FEAT_GGRO,
+    VIR_NET_DEV_FEAT_LRO,
+    VIR_NET_DEV_FEAT_RXVLAN,
+    VIR_NET_DEV_FEAT_TXVLAN,
+    VIR_NET_DEV_FEAT_NTUPLE,
+    VIR_NET_DEV_FEAT_RXHASH,
+    VIR_NET_DEV_FEAT_LAST
+} virNetDevFeature;
+
+VIR_ENUM_DECL(virNetDevFeature)
+
 int virDevicePCIAddressIsValid(virDevicePCIAddressPtr addr);
 
 int virDevicePCIAddressParseXML(xmlNodePtr node,

src/conf/node_device_conf.c

@@ -437,6 +437,16 @@ char *virNodeDeviceDefFormat(const virNodeDeviceDef *def)
                 virBufferEscapeString(&buf, "<address>%s</address>\n",
                                   data->net.address);
             virInterfaceLinkFormat(&buf, &data->net.lnk);
+            if (data->net.features) {
+                for (i = 0; i < VIR_NET_DEV_FEAT_LAST; i++) {
+                    bool b;
+                    ignore_value(virBitmapGetBit(data->net.features, i, &b));
+                    if (b) {
+                        virBufferAsprintf(&buf, "<feature name='%s'/>\n",
+                                          virNetDevFeatureTypeToString(i));
+                    }
+                }
+            }
             if (data->net.subtype != VIR_NODE_DEV_CAP_NET_LAST) {
                 const char *subtyp =
                     virNodeDevNetCapTypeToString(data->net.subtype);
@@ -927,8 +937,10 @@ virNodeDevCapNetParseXML(xmlXPathContextPtr ctxt,
                          union _virNodeDevCapData *data)
 {
     xmlNodePtr orignode, lnk;
-    int ret = -1;
+    size_t i = -1;
+    int ret = -1, n = -1;
     char *tmp;
+    xmlNodePtr *nodes = NULL;
 
     orignode = ctxt->node;
     ctxt->node = node;
@@ -943,6 +955,31 @@ virNodeDevCapNetParseXML(xmlXPathContextPtr ctxt,
 
     data->net.address = virXPathString("string(./address[1])", ctxt);
 
+    if ((n = virXPathNodeSet("./feature", ctxt, &nodes)) < 0)
+        goto out;
+
+    if (n > 0) {
+        if (!(data->net.features = virBitmapNew(VIR_NET_DEV_FEAT_LAST)))
+            goto out;
+    }
+
+    for (i = 0; i < n; i++) {
+        int val;
+        if (!(tmp = virXMLPropString(nodes[i], "name"))) {
+            virReportError(VIR_ERR_XML_ERROR, "%s",
+                           _("missing network device feature name"));
+            goto out;
+        }
+
+        if ((val = virNetDevFeatureTypeFromString(tmp)) < 0) {
+            virReportError(VIR_ERR_XML_ERROR,
+                           _("unknown network device feature '%s'"),
+                           tmp);
+            goto out;
+        }
+        ignore_value(virBitmapSetBit(data->net.features, val));
+    }
+
     data->net.subtype = VIR_NODE_DEV_CAP_NET_LAST;
 
     tmp = virXPathString("string(./capability/@type)", ctxt);
@@ -1679,6 +1716,8 @@ void virNodeDevCapsDefFree(virNodeDevCapsDefPtr caps)
     case VIR_NODE_DEV_CAP_NET:
         VIR_FREE(data->net.ifname);
         VIR_FREE(data->net.address);
+        virBitmapFree(data->net.features);
+        data->net.features = NULL;
         break;
     case VIR_NODE_DEV_CAP_SCSI_HOST:
         VIR_FREE(data->scsi_host.wwnn);

src/conf/node_device_conf.h

@@ -26,6 +26,7 @@
 # define __VIR_NODE_DEVICE_CONF_H__
 
 # include "internal.h"
+# include "virbitmap.h"
 # include "virutil.h"
 # include "virthread.h"
 # include "virpci.h"
@@ -141,6 +142,7 @@ struct _virNodeDevCapsDef {
             char *ifname;
             virInterfaceLink lnk;
             virNodeDevNetCapType subtype;  /* LAST -> no subtype */
+            virBitmapPtr features; /* enum virNetDevFeature */
         } net;
         struct {
             unsigned int host;

src/libvirt_private.syms

@@ -1669,6 +1669,7 @@ virNetDevAddRoute;
 virNetDevClearIPAddress;
 virNetDevDelMulti;
 virNetDevExists;
+virNetDevGetFeatures;
 virNetDevGetIndex;
 virNetDevGetIPv4Address;
 virNetDevGetLinkInfo;

src/node_device/node_device_udev.c

@@ -719,6 +719,9 @@ static int udevProcessNetworkInterface(struct udev_device *device,
     if (virNetDevGetLinkInfo(data->net.ifname, &data->net.lnk) < 0)
         goto out;
 
+    if (virNetDevGetFeatures(data->net.ifname, &data->net.features) < 0)
+        goto out;
+
     ret = 0;
 
  out:

src/util/virnetdev.c

@@ -2728,3 +2728,131 @@ int virNetDevGetRxFilter(const char *ifname,
     *filter = fil;
     return ret;
 }
+
+#if defined(SIOCETHTOOL) && defined(HAVE_STRUCT_IFREQ)
+
+/**
+ * virNetDevFeatureAvailable
+ * This function checks for the availability of a network device feature
+ *
+ * @ifname: name of the interface
+ * @cmd: reference to an ethtool command structure
+ *
+ * Returns 0 on success, -1 on failure.
+ */
+static int
+virNetDevFeatureAvailable(const char *ifname, struct ethtool_value *cmd)
+{
+    int ret = -1;
+    int sock = -1;
+    virIfreq ifr;
+
+    sock = socket(AF_LOCAL, SOCK_DGRAM, 0);
+    if (sock < 0) {
+        virReportSystemError(errno, "%s", _("Cannot open control socket"));
+        goto cleanup;
+    }
+
+    strcpy(ifr.ifr_name, ifname);
+    ifr.ifr_data = (void*) cmd;
+
+    if (ioctl(sock, SIOCETHTOOL, &ifr) != 0) {
+        switch (errno) {
+            case EPERM:
+                VIR_DEBUG("ethtool ioctl: permission denied");
+                break;
+            case EINVAL:
+                VIR_DEBUG("ethtool ioctl: invalid request");
+                break;
+            case EOPNOTSUPP:
+                VIR_DEBUG("ethtool ioctl: request not supported");
+                break;
+            default:
+                virReportSystemError(errno, "%s", _("ethtool ioctl error"));
+                goto cleanup;
+        }
+    }
+
+    ret = cmd->data > 0 ? 1: 0;
+ cleanup:
+    if (sock)
+        VIR_FORCE_CLOSE(sock);
+
+    return ret;
+}
+
+
+/**
+ * virNetDevGetFeatures:
+ * This function gets the nic offloads features available for ifname
+ *
+ * @ifname: name of the interface
+ * @features: network device feature structures
+ * @nfeatures: number of features available
+ *
+ * Returns 0 on success, -1 on failure.
+ */
+int
+virNetDevGetFeatures(const char *ifname,
+                     virBitmapPtr *out)
+{
+    int ret = -1;
+    size_t i = -1;
+    size_t j = -1;
+    struct ethtool_value cmd = { 0 };
+
+    struct elem{
+        const int cmd;
+        const virNetDevFeature feat;
+    };
+    /* legacy ethtool getters */
+    struct elem cmds[] = {
+        {ETHTOOL_GRXCSUM, VIR_NET_DEV_FEAT_GRXCSUM},
+        {ETHTOOL_GTXCSUM, VIR_NET_DEV_FEAT_GTXCSUM},
+        {ETHTOOL_GSG, VIR_NET_DEV_FEAT_GSG},
+        {ETHTOOL_GTSO, VIR_NET_DEV_FEAT_GTSO},
+        {ETHTOOL_GGSO, VIR_NET_DEV_FEAT_GGSO},
+        {ETHTOOL_GGRO, VIR_NET_DEV_FEAT_GGRO},
+    };
+    /* ethtool masks */
+    struct elem flags[] = {
+        {ETH_FLAG_LRO, VIR_NET_DEV_FEAT_LRO},
+        {ETH_FLAG_RXVLAN, VIR_NET_DEV_FEAT_RXVLAN},
+        {ETH_FLAG_TXVLAN, VIR_NET_DEV_FEAT_TXVLAN},
+        {ETH_FLAG_NTUPLE, VIR_NET_DEV_FEAT_NTUPLE},
+        {ETH_FLAG_RXHASH, VIR_NET_DEV_FEAT_RXHASH},
+    };
+
+    if (!(*out = virBitmapNew(VIR_NET_DEV_FEAT_LAST)))
+        goto cleanup;
+
+    for (i = 0; i < ARRAY_CARDINALITY(cmds); i++) {
+        cmd.cmd = cmds[i].cmd;
+        if (virNetDevFeatureAvailable(ifname, &cmd))
+            ignore_value(virBitmapSetBit(*out, cmds[i].feat));
+    }
+
+    cmd.cmd = ETHTOOL_GFLAGS;
+    if (virNetDevFeatureAvailable(ifname, &cmd)) {
+        for (j = 0; j < ARRAY_CARDINALITY(flags); j++) {
+            if (cmd.data & flags[j].cmd)
+                ignore_value(virBitmapSetBit(*out, flags[j].feat));
+        }
+    }
+
+    ret = 0;
+ cleanup:
+
+    return ret;
+
+}
+#else
+int
+virNetDevGetFeatures(const char *ifname ATTRIBUTE_UNUSED,
+                     virBitmapPtr *out ATTRIBUTE_UNUSED)
+{
+    VIR_DEBUG("Getting network device features on %s is not implemented on this platform",
+              ifname);
+    return 0;
+}
+#endif

src/util/virnetdev.h

@@ -25,12 +25,14 @@
 
 # include <net/if.h>
 
+# include "virbitmap.h"
 # include "virsocketaddr.h"
 # include "virnetlink.h"
 # include "virmacaddr.h"
 # include "virpci.h"
 # include "device_conf.h"
 
+# include <linux/ethtool.h>
 # ifdef HAVE_STRUCT_IFREQ
 typedef struct ifreq virIfreq;
 # else
@@ -182,6 +184,10 @@ int virNetDevGetVirtualFunctionInfo(const char *vfname, char **pfname,
                                     int *vf)
     ATTRIBUTE_NONNULL(1);
 
+int virNetDevGetFeatures(const char *ifname,
+                         virBitmapPtr *out)
+    ATTRIBUTE_NONNULL(1) ATTRIBUTE_RETURN_CHECK;
+
 int virNetDevGetLinkInfo(const char *ifname,
                          virInterfaceLinkPtr lnk)
     ATTRIBUTE_NONNULL(1);

tests/nodedevschemadata/net_00_13_02_b9_f9_d3.xml

@@ -4,6 +4,15 @@
   <capability type='net'>
     <interface>eth0</interface>
     <address>00:13:02:b9:f9:d3</address>
+    <feature name='rx'/>
+    <feature name='tx'/>
+    <feature name='sg'/>
+    <feature name='tso'/>
+    <feature name='gso'/>
+    <feature name='gro'/>
+    <feature name='rxvlan'/>
+    <feature name='txvlan'/>
+    <feature name='rxhash'/>
     <capability type='80211'/>
   </capability>
 </device>

tests/nodedevschemadata/net_00_15_58_2f_e9_55.xml

@@ -4,6 +4,15 @@
   <capability type='net'>
     <interface>eth1</interface>
     <address>00:15:58:2f:e9:55</address>
+    <feature name='rx'/>
+    <feature name='tx'/>
+    <feature name='sg'/>
+    <feature name='tso'/>
+    <feature name='gso'/>
+    <feature name='gro'/>
+    <feature name='rxvlan'/>
+    <feature name='txvlan'/>
+    <feature name='rxhash'/>
     <capability type='80203'/>
   </capability>
 </device>