qemu: Implement ./hugepages/page/[@size, @unit, @nodeset]

Signed-off-by: NMichal Privoznik <mprivozn@redhat.com>

src/qemu/qemu_capabilities.c

@@ -263,6 +263,7 @@ VIR_ENUM_IMPL(virQEMUCaps, QEMU_CAPS_LAST,
 
               "memory-backend-ram", /* 170 */
               "numa",
+              "memory-backend-file",
     );
 
 
@@ -1481,6 +1482,7 @@ struct virQEMUCapsStringFlags virQEMUCapsObjectTypes[] = {
     { "pvpanic", QEMU_CAPS_DEVICE_PANIC },
     { "usb-kbd", QEMU_CAPS_DEVICE_USB_KBD },
     { "memory-backend-ram", QEMU_CAPS_OBJECT_MEMORY_RAM },
+    { "memory-backend-file", QEMU_CAPS_OBJECT_MEMORY_FILE },
 };
 
 static struct virQEMUCapsStringFlags virQEMUCapsObjectPropsVirtioBlk[] = {

src/qemu/qemu_capabilities.h

@@ -211,6 +211,7 @@ typedef enum {
     QEMU_CAPS_CHANGE_BACKING_FILE = 169, /* change name of backing file in metadata */
     QEMU_CAPS_OBJECT_MEMORY_RAM  = 170, /* -object memory-backend-ram */
     QEMU_CAPS_NUMA               = 171, /* newer -numa handling with disjoint cpu ranges */
+    QEMU_CAPS_OBJECT_MEMORY_FILE = 172, /* -object memory-backend-file */
 
     QEMU_CAPS_LAST,                   /* this must always be the last item */
 } virQEMUCapsFlags;

src/qemu/qemu_command.c

@@ -6432,24 +6432,36 @@ qemuBuildSmpArgStr(const virDomainDef *def,
 }
 
 static int
-qemuBuildNumaArgStr(const virDomainDef *def,
+qemuBuildNumaArgStr(virQEMUDriverConfigPtr cfg,
+                    const virDomainDef *def,
                     virCommandPtr cmd,
                     virQEMUCapsPtr qemuCaps)
 {
-    size_t i;
+    size_t i, j;
     virBuffer buf = VIR_BUFFER_INITIALIZER;
+    virDomainHugePagePtr master_hugepage = NULL;
     char *cpumask = NULL, *tmpmask = NULL, *next = NULL;
     char *nodemask = NULL;
+    char *mem_path = NULL;
     int ret = -1;
 
     if (virDomainNumatuneHasPerNodeBinding(def->numatune) &&
-        !virQEMUCapsGet(qemuCaps, QEMU_CAPS_OBJECT_MEMORY_RAM)) {
+        !(virQEMUCapsGet(qemuCaps, QEMU_CAPS_OBJECT_MEMORY_RAM) ||
+          virQEMUCapsGet(qemuCaps, QEMU_CAPS_OBJECT_MEMORY_FILE))) {
         virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                        _("Per-node memory binding is not supported "
                          "with this QEMU"));
         goto cleanup;
     }
 
+    if (def->mem.nhugepages && def->mem.hugepages[0].size &&
+        !virQEMUCapsGet(qemuCaps, QEMU_CAPS_OBJECT_MEMORY_FILE)) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                       _("huge pages per NUMA node are not "
+                         "supported with this QEMU"));
+        goto cleanup;
+    }
+
     for (i = 0; i < def->cpu->ncells; i++) {
         int cellmem = VIR_DIV_UP(def->cpu->cells[i].mem, 1024);
         def->cpu->cells[i].mem = cellmem * 1024;
@@ -6468,15 +6480,74 @@ qemuBuildNumaArgStr(const virDomainDef *def,
             goto cleanup;
         }
 
-        if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_OBJECT_MEMORY_RAM)) {
+        if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_OBJECT_MEMORY_RAM) ||
+            virQEMUCapsGet(qemuCaps, QEMU_CAPS_OBJECT_MEMORY_FILE)) {
             virDomainNumatuneMemMode mode;
+            virDomainHugePagePtr hugepage = NULL;
             const char *policy = NULL;
 
             mode = virDomainNumatuneGetMode(def->numatune, i);
             policy = qemuNumaPolicyTypeToString(mode);
 
-            virBufferAsprintf(&buf, "memory-backend-ram,size=%dM,id=ram-node%zu",
-                              cellmem, i);
+            /* Find the huge page size we want to use */
+            for (j = 0; j < def->mem.nhugepages; j++) {
+                bool thisHugepage = false;
+
+                hugepage = &def->mem.hugepages[j];
+
+                if (!hugepage->nodemask) {
+                    master_hugepage = hugepage;
+                    continue;
+                }
+
+                if (virBitmapGetBit(hugepage->nodemask, i, &thisHugepage) < 0) {
+                    /* Ignore this error. It's not an error after all. Well,
+                     * the nodemask for this <page/> can contain lower NUMA
+                     * nodes than we are querying in here. */
+                    continue;
+                }
+
+                if (thisHugepage) {
+                    /* Hooray, we've found the page size */
+                    break;
+                }
+            }
+
+            if (j == def->mem.nhugepages) {
+                /* We have not found specific huge page to be used with this
+                 * NUMA node. Use the generic setting then (<page/> without any
+                 * @nodemask) if possible. */
+                hugepage = master_hugepage;
+            }
+
+            if (hugepage) {
+                /* Now lets see, if the huge page we want to use is even mounted
+                 * and ready to use */
+
+                for (j = 0; j < cfg->nhugetlbfs; j++) {
+                    if (cfg->hugetlbfs[j].size == hugepage->size)
+                        break;
+                }
+
+                if (j == cfg->nhugetlbfs) {
+                    virReportError(VIR_ERR_INTERNAL_ERROR,
+                                   _("Unable to find any usable hugetlbfs mount for %llu KiB"),
+                                   hugepage->size);
+                    goto cleanup;
+                }
+
+                VIR_FREE(mem_path);
+                if (!(mem_path = qemuGetHugepagePath(&cfg->hugetlbfs[j])))
+                    goto cleanup;
+
+                virBufferAsprintf(&buf,
+                                  "memory-backend-file,prealloc=yes,mem-path=%s",
+                                  mem_path);
+            } else {
+                virBufferAddLit(&buf, "memory-backend-ram");
+            }
+
+            virBufferAsprintf(&buf, ",size=%dM,id=ram-node%zu", cellmem, i);
 
             if (virDomainNumatuneMaybeFormatNodeset(def->numatune, NULL,
                                                     &nodemask, i) < 0)
@@ -6515,7 +6586,8 @@ qemuBuildNumaArgStr(const virDomainDef *def,
             virBufferAdd(&buf, tmpmask, -1);
         }
 
-        if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_OBJECT_MEMORY_RAM)) {
+        if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_OBJECT_MEMORY_RAM) ||
+            virQEMUCapsGet(qemuCaps, QEMU_CAPS_OBJECT_MEMORY_FILE)) {
             virBufferAsprintf(&buf, ",memdev=ram-node%zu", i);
         } else {
             virBufferAsprintf(&buf, ",mem=%d", cellmem);
@@ -6528,6 +6600,7 @@ qemuBuildNumaArgStr(const virDomainDef *def,
  cleanup:
     VIR_FREE(cpumask);
     VIR_FREE(nodemask);
+    VIR_FREE(mem_path);
     virBufferFreeAndReset(&buf);
     return ret;
 }
@@ -7383,7 +7456,7 @@ qemuBuildCommandLine(virConnectPtr conn,
     virCommandAddArg(cmd, "-m");
     def->mem.max_balloon = VIR_DIV_UP(def->mem.max_balloon, 1024) * 1024;
     virCommandAddArgFormat(cmd, "%llu", def->mem.max_balloon / 1024);
-    if (def->mem.nhugepages) {
+    if (def->mem.nhugepages && !def->mem.hugepages[0].size) {
         char *mem_path;
 
         if (!cfg->nhugetlbfs) {
@@ -7427,7 +7500,7 @@ qemuBuildCommandLine(virConnectPtr conn,
     VIR_FREE(smp);
 
     if (def->cpu && def->cpu->ncells)
-        if (qemuBuildNumaArgStr(def, cmd, qemuCaps) < 0)
+        if (qemuBuildNumaArgStr(cfg, def, cmd, qemuCaps) < 0)
             goto error;
 
     if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_UUID))

tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages.args

+LC_ALL=C PATH=/bin HOME=/home/test USER=test LOGNAME=test QEMU_AUDIO_DRV=none \
+/usr/bin/qemu -S -M pc -m 4096 -smp 4 \
+-object memory-backend-file,prealloc=yes,mem-path=/dev/hugepages1G/libvirt/qemu,\
+size=1024M,id=ram-node0,host-nodes=0-3,policy=bind \
+-numa node,nodeid=0,cpus=0,memdev=ram-node0 \
+-object memory-backend-file,prealloc=yes,mem-path=/dev/hugepages2M/libvirt/qemu,\
+size=1024M,id=ram-node1,host-nodes=0-3,policy=bind \
+-numa node,nodeid=1,cpus=1,memdev=ram-node1 \
+-object memory-backend-file,prealloc=yes,mem-path=/dev/hugepages1G/libvirt/qemu,\
+size=1024M,id=ram-node2,host-nodes=0-3,policy=bind \
+-numa node,nodeid=2,cpus=2,memdev=ram-node2 \
+-object memory-backend-file,prealloc=yes,mem-path=/dev/hugepages1G/libvirt/qemu,\
+size=1024M,id=ram-node3,host-nodes=3,policy=bind \
+-numa node,nodeid=3,cpus=3,memdev=ram-node3 \
+-nographic -monitor unix:/tmp/test-monitor,server,nowait -no-acpi -boot c -usb \
+-hda /dev/HostVG/QEMUGuest1 -net none -serial none -parallel none

tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages2.args

+LC_ALL=C PATH=/bin HOME=/home/test USER=test LOGNAME=test QEMU_AUDIO_DRV=none \
+/usr/bin/qemu -S -M pc -m 1024 -smp 2 \
+-object memory-backend-file,prealloc=yes,\
+mem-path=/dev/hugepages2M/libvirt/qemu,size=256M,id=ram-node0 \
+-numa node,nodeid=0,cpus=0,memdev=ram-node0 \
+-object memory-backend-file,prealloc=yes,\
+mem-path=/dev/hugepages2M/libvirt/qemu,size=768M,id=ram-node1 \
+-numa node,nodeid=1,cpus=1,memdev=ram-node1 \
+-nographic -monitor unix:/tmp/test-monitor,server,nowait -no-acpi -boot c \
+-usb -hda /dev/HostVG/QEMUGuest1 -net none -serial none -parallel none

tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages2.xml

+<domain type='qemu'>
+  <name>SomeDummyHugepagesGuest</name>
+  <uuid>ef1bdff4-27f3-4e85-a807-5fb4d58463cc</uuid>
+  <memory unit='KiB'>1048576</memory>
+  <currentMemory unit='KiB'>1048576</currentMemory>
+  <memoryBacking>
+    <hugepages>
+      <page size='2048' unit='KiB'/>
+    </hugepages>
+  </memoryBacking>
+  <vcpu placement='static'>2</vcpu>
+  <os>
+    <type arch='i686' machine='pc'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <cpu>
+    <numa>
+      <cell id='0' cpus='0' memory='262144'/>
+      <cell id='1' cpus='1' memory='786432'/>
+    </numa>
+  </cpu>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>destroy</on_crash>
+  <devices>
+    <emulator>/usr/bin/qemu</emulator>
+    <disk type='block' device='disk'>
+      <source dev='/dev/HostVG/QEMUGuest1'/>
+      <target dev='hda' bus='ide'/>
+      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
+    </disk>
+    <controller type='usb' index='0'/>
+    <controller type='ide' index='0'/>
+    <controller type='pci' index='0' model='pci-root'/>
+    <memballoon model='virtio'/>
+  </devices>
+</domain>

tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages3.args

+LC_ALL=C PATH=/bin HOME=/home/test USER=test LOGNAME=test QEMU_AUDIO_DRV=none \
+/usr/bin/qemu -S -M pc -m 1024 -smp 2 \
+-object memory-backend-ram,size=256M,id=ram-node0 \
+-numa node,nodeid=0,cpus=0,memdev=ram-node0 \
+-object memory-backend-file,prealloc=yes,\
+mem-path=/dev/hugepages1G/libvirt/qemu,size=768M,id=ram-node1 \
+-numa node,nodeid=1,cpus=1,memdev=ram-node1 \
+-nographic -monitor unix:/tmp/test-monitor,server,nowait -no-acpi -boot c -usb \
+-hda /dev/HostVG/QEMUGuest1 -net none -serial none -parallel none

tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages3.xml

+<domain type='qemu'>
+  <name>SomeDummyHugepagesGuest</name>
+  <uuid>ef1bdff4-27f3-4e85-a807-5fb4d58463cc</uuid>
+  <memory unit='KiB'>1048576</memory>
+  <currentMemory unit='KiB'>1048576</currentMemory>
+  <memoryBacking>
+    <hugepages>
+      <page size='1048576' unit='KiB' nodeset='1'/>
+    </hugepages>
+  </memoryBacking>
+  <vcpu placement='static'>2</vcpu>
+  <os>
+    <type arch='i686' machine='pc'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <cpu>
+    <numa>
+      <cell id='0' cpus='0' memory='262144'/>
+      <cell id='1' cpus='1' memory='786432'/>
+    </numa>
+  </cpu>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>destroy</on_crash>
+  <devices>
+    <emulator>/usr/bin/qemu</emulator>
+    <disk type='block' device='disk'>
+      <source dev='/dev/HostVG/QEMUGuest1'/>
+      <target dev='hda' bus='ide'/>
+      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
+    </disk>
+    <controller type='usb' index='0'/>
+    <controller type='ide' index='0'/>
+    <controller type='pci' index='0' model='pci-root'/>
+    <memballoon model='virtio'/>
+  </devices>
+</domain>

tests/qemuxml2argvdata/qemuxml2argv-hugepages.args

 LC_ALL=C PATH=/bin HOME=/home/test USER=test LOGNAME=test QEMU_AUDIO_DRV=none \
 /usr/bin/qemu -S -M \
-pc -m 214 -mem-prealloc -mem-path /dev/hugepages/libvirt/qemu -smp 1 \
+pc -m 214 -mem-prealloc -mem-path /dev/hugepages2M/libvirt/qemu -smp 1 \
 -nographic -monitor unix:/tmp/test-monitor,server,nowait -no-acpi -boot c -usb -hda \
 /dev/HostVG/QEMUGuest1 -net none -serial none -parallel none

tests/qemuxml2argvtest.c

@@ -525,13 +525,15 @@ mymain(void)
     if (VIR_STRDUP_QUIET(driver.config->stateDir, "/nowhere") < 0)
         return EXIT_FAILURE;
     VIR_FREE(driver.config->hugetlbfs);
-    if (VIR_ALLOC_N(driver.config->hugetlbfs, 1) < 0)
+    if (VIR_ALLOC_N(driver.config->hugetlbfs, 2) < 0)
         return EXIT_FAILURE;
-    driver.config->nhugetlbfs = 1;
-    if (VIR_STRDUP(driver.config->hugetlbfs[0].mnt_dir, "/dev/hugepages") < 0)
+    driver.config->nhugetlbfs = 2;
+    if (VIR_STRDUP(driver.config->hugetlbfs[0].mnt_dir, "/dev/hugepages2M") < 0 ||
+        VIR_STRDUP(driver.config->hugetlbfs[1].mnt_dir, "/dev/hugepages1G") < 0)
         return EXIT_FAILURE;
     driver.config->hugetlbfs[0].size = 2048;
     driver.config->hugetlbfs[0].deflt = true;
+    driver.config->hugetlbfs[1].size = 1048576;
     driver.config->spiceTLS = 1;
     if (VIR_STRDUP_QUIET(driver.config->spicePassword, "123456") < 0)
         return EXIT_FAILURE;
@@ -665,6 +667,12 @@ mymain(void)
     DO_TEST("hyperv-off", NONE);
 
     DO_TEST("hugepages", QEMU_CAPS_MEM_PATH);
+    DO_TEST("hugepages-pages", QEMU_CAPS_MEM_PATH, QEMU_CAPS_OBJECT_MEMORY_RAM,
+            QEMU_CAPS_OBJECT_MEMORY_FILE);
+    DO_TEST("hugepages-pages2", QEMU_CAPS_MEM_PATH, QEMU_CAPS_OBJECT_MEMORY_RAM,
+            QEMU_CAPS_OBJECT_MEMORY_FILE);
+    DO_TEST("hugepages-pages3", QEMU_CAPS_MEM_PATH, QEMU_CAPS_OBJECT_MEMORY_RAM,
+            QEMU_CAPS_OBJECT_MEMORY_FILE);
     DO_TEST("nosharepages", QEMU_CAPS_MACHINE_OPT, QEMU_CAPS_MEM_MERGE);
     DO_TEST("disk-cdrom", NONE);
     DO_TEST("disk-cdrom-network-http", QEMU_CAPS_KVM, QEMU_CAPS_DEVICE,

tests/qemuxml2xmltest.c

@@ -198,6 +198,8 @@ mymain(void)
 
     DO_TEST("hugepages");
     DO_TEST("hugepages-pages");
+    DO_TEST("hugepages-pages2");
+    DO_TEST("hugepages-pages3");
     DO_TEST("nosharepages");
     DO_TEST("disk-aio");
     DO_TEST("disk-cdrom");