proxmox-mirrors/qemu-server
1
0
Fork 0
mirror of https://git.proxmox.com/git/qemu-server synced 2025-08-12 00:24:26 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

add hugepages option

Browse Source 
vm configuration
----------------
hugepages: (any|2|1024)

any: we'll try to allocate 1GB hugepage if possible, if not we use 2MB hugepage
2: we want to use 2MB hugepage
1024: we want to use 1GB hugepage. (memory need to be multiple of 1GB in this case)

optionnal host configuration for 1GB hugepages
----------------------------------------------
1GB hugepages can be allocated at boot if user want it.
hugepages need to be contiguous, so sometime it's not possible to reserve them on the fly

/etc/default/grub : GRUB_CMDLINE_LINUX_DEFAULT="quiet hugepagesz=1G hugepages=x"

Signed-off-by: Alexandre Derumier <aderumier@odiso.com>
Acked-by: Wolfgang Bumiller <w.bumiller@proxmox.com>
This commit is contained in:
Alexandre Derumier 2016-06-15 05:04:02 +02:00 committed by Wolfgang Bumiller
parent 062669f9a0
commit 7023f3ea16
2 changed files with 357 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
PVE/QemuServer.pm
View File

@ -321,6 +321,12 @@ EODESC
description => "Enable/disable NUMA.", description => "Enable/disable NUMA.",
default => 0, default => 0,
}, },
hugepages => {
optional => 1,
type => 'string',
description => "Enable/disable hugepages memory.",
enum => [qw(any 2 1024)],
},
vcpus => { vcpus => {
optional => 1, optional => 1,
type => 'integer', type => 'integer',
@ -4354,19 +4360,48 @@ sub vm_start {
my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits} my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
: $defaults->{cpuunits}; : $defaults->{cpuunits};
eval { my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
my %properties = (
Slice => 'qemu.slice', my %properties = (
KillMode => 'none', Slice => 'qemu.slice',
CPUShares => $cpuunits KillMode => 'none',
); CPUShares => $cpuunits
if (my $cpulimit = $conf->{cpulimit}) { );
$properties{CPUQuota} = int($cpulimit * 100);
} if (my $cpulimit = $conf->{cpulimit}) {
$properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick $properties{CPUQuota} = int($cpulimit * 100);
PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties); }
run_command($cmd, timeout => $statefile ? undef : 30, umask => 0077); $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
};
if ($conf->{hugepages}) {
my $code = sub {
my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
PVE::QemuServer::Memory::hugepages_mount();
PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
eval {
PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
run_command($cmd, %run_params);
};
if (my $err = $@) {
PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
die $err;
}
PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
};
eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
} else {
eval {
PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
run_command($cmd, %run_params);
};
}
if (my $err = $@) { if (my $err = $@) {
# deactivate volumes if start fails # deactivate volumes if start fails

329
PVE/QemuServer/Memory.pm
View File

@ -3,6 +3,7 @@ package PVE::QemuServer::Memory;
use strict; use strict;
use warnings; use warnings;
use PVE::QemuServer; use PVE::QemuServer;
use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
use PVE::Exception qw(raise raise_param_exc); use PVE::Exception qw(raise raise_param_exc);
my $MAX_NUMA = 8; my $MAX_NUMA = 8;
@ -13,8 +14,17 @@ sub foreach_dimm{
my ($conf, $vmid, $memory, $sockets, $func) = @_; my ($conf, $vmid, $memory, $sockets, $func) = @_;
my $dimm_id = 0; my $dimm_id = 0;
my $current_size = 1024; my $current_size = 0;
my $dimm_size = 512; my $dimm_size = 0;
if($conf->{hugepages} && $conf->{hugepages} == 1024) {
$current_size = 1024 * $sockets;
$dimm_size = 1024;
} else {
$current_size = 1024;
$dimm_size = 512;
}
return if $current_size == $memory; return if $current_size == $memory;
for (my $j = 0; $j < 8; $j++) { for (my $j = 0; $j < 8; $j++) {
@ -34,8 +44,17 @@ sub foreach_reverse_dimm {
my ($conf, $vmid, $memory, $sockets, $func) = @_; my ($conf, $vmid, $memory, $sockets, $func) = @_;
my $dimm_id = 253; my $dimm_id = 253;
my $current_size = 4177920; my $current_size = 0;
my $dimm_size = 65536; my $dimm_size = 0;
if($conf->{hugepages} && $conf->{hugepages} == 1024) {
$current_size = 8355840;
$dimm_size = 131072;
} else {
$current_size = 4177920;
$dimm_size = 65536;
}
return if $current_size == $memory; return if $current_size == $memory;
for (my $j = 0; $j < 8; $j++) { for (my $j = 0; $j < 8; $j++) {
@ -56,20 +75,19 @@ sub qemu_memory_hotplug {
return $value if !PVE::QemuServer::check_running($vmid); return $value if !PVE::QemuServer::check_running($vmid);
my $sockets = 1;
$sockets = $conf->{sockets} if $conf->{sockets};
my $memory = $conf->{memory} || $defaults->{memory}; my $memory = $conf->{memory} || $defaults->{memory};
$value = $defaults->{memory} if !$value; $value = $defaults->{memory} if !$value;
return $value if $value == $memory; return $value if $value == $memory;
my $static_memory = $STATICMEM; my $static_memory = $STATICMEM;
my $dimm_memory = $memory - $static_memory; $static_memory = $static_memory * $sockets if ($conf->{hugepages} && $conf->{hugepages} == 1024);
die "memory can't be lower than $static_memory MB" if $value < $static_memory; die "memory can't be lower than $static_memory MB" if $value < $static_memory;
die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM; die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
my $sockets = 1;
$sockets = $conf->{sockets} if $conf->{sockets};
if($value > $memory) { if($value > $memory) {
foreach_dimm($conf, $vmid, $value, $sockets, sub { foreach_dimm($conf, $vmid, $value, $sockets, sub {
@ -77,7 +95,31 @@ sub qemu_memory_hotplug {
return if $current_size <= $conf->{memory}; return if $current_size <= $conf->{memory};
eval { PVE::QemuServer::vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) }; if ($conf->{hugepages}) {
my $hugepages_size = hugepages_size($conf, $dimm_size);
my $path = hugepages_mount_path($hugepages_size);
my $hugepages_topology->{$hugepages_size}->{$numanode} = hugepages_nr($dimm_size, $hugepages_size);
my $code = sub {
my $hugepages_host_topology = hugepages_host_topology();
hugepages_allocate($hugepages_topology, $hugepages_host_topology);
eval { PVE::QemuServer::vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-file", id => "mem-$name", props => {
size => int($dimm_size*1024*1024), 'mem-path' => $path, share => JSON::true, prealloc => JSON::true } ); };
if (my $err = $@) {
hugepages_reset($hugepages_host_topology);
die $err;
}
hugepages_pre_deallocate($hugepages_topology);
};
eval { hugepages_update_locked($code); };
} else {
eval { PVE::QemuServer::vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
}
if (my $err = $@) { if (my $err = $@) {
eval { PVE::QemuServer::qemu_objectdel($vmid, "mem-$name"); }; eval { PVE::QemuServer::qemu_objectdel($vmid, "mem-$name"); };
die $err; die $err;
@ -142,14 +184,17 @@ sub config {
my $memory = $conf->{memory} || $defaults->{memory}; my $memory = $conf->{memory} || $defaults->{memory};
my $static_memory = 0; my $static_memory = 0;
my $dimm_memory = 0;
if ($hotplug_features->{memory}) { if ($hotplug_features->{memory}) {
die "NUMA need to be enabled for memory hotplug\n" if !$conf->{numa}; die "NUMA needs to be enabled for memory hotplug\n" if !$conf->{numa};
die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM; die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
my $sockets = 1;
$sockets = $conf->{sockets} if $conf->{sockets};
$static_memory = $STATICMEM; $static_memory = $STATICMEM;
$static_memory = $static_memory * $sockets if ($conf->{hugepages} && $conf->{hugepages} == 1024);
die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory); die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
$dimm_memory = $memory - $static_memory;
push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M"; push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
} else { } else {
@ -158,6 +203,8 @@ sub config {
push @$cmd, '-m', $static_memory; push @$cmd, '-m', $static_memory;
} }
die "numa needs to be enabled to use hugepages" if $conf->{hugepages} && !$conf->{numa};
if ($conf->{numa}) { if ($conf->{numa}) {
my $numa_totalmemory = undef; my $numa_totalmemory = undef;
@ -169,7 +216,8 @@ sub config {
die "missing NUMA node$i memory value\n" if !$numa->{memory}; die "missing NUMA node$i memory value\n" if !$numa->{memory};
my $numa_memory = $numa->{memory}; my $numa_memory = $numa->{memory};
$numa_totalmemory += $numa_memory; $numa_totalmemory += $numa_memory;
my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
my $mem_object = print_mem_object($conf, "ram-node$i", $numa_memory);
# cpus # cpus
my $cpulists = $numa->{cpus}; my $cpulists = $numa->{cpus};
@ -190,17 +238,19 @@ sub config {
$hostnodes .= "-$end" if defined($end); $hostnodes .= "-$end" if defined($end);
$end //= $start; $end //= $start;
for (my $i = $start; $i <= $end; ++$i ) { for (my $i = $start; $i <= $end; ++$i ) {
die "host NUMA node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/"; die "host NUMA node$i doesn't exist\n" if ! -d "/sys/devices/system/node/node$i/";
} }
} }
# policy # policy
my $policy = $numa->{policy}; my $policy = $numa->{policy};
die "you need to define a policy for hostnode $hostnodes\n" if !$policy; die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
$numa_object .= ",host-nodes=$hostnodes,policy=$policy"; $mem_object .= ",host-nodes=$hostnodes,policy=$policy";
} else {
die "numa hostnodes need to be defined to use hugepages" if $conf->{hugepages};
} }
push @$cmd, '-object', $numa_object; push @$cmd, '-object', $mem_object;
push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i"; push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
} }
@ -210,16 +260,19 @@ sub config {
#if no custom tology, we split memory and cores across numa nodes #if no custom tology, we split memory and cores across numa nodes
if(!$numa_totalmemory) { if(!$numa_totalmemory) {
my $numa_memory = ($static_memory / $sockets) . "M"; my $numa_memory = ($static_memory / $sockets);
for (my $i = 0; $i < $sockets; $i++) { for (my $i = 0; $i < $sockets; $i++) {
die "host NUMA node$i doesn't exist\n" if ! -d "/sys/devices/system/node/node$i/";
my $cpustart = ($cores * $i); my $cpustart = ($cores * $i);
my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1; my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
my $cpus = $cpustart; my $cpus = $cpustart;
$cpus .= "-$cpuend" if $cpuend; $cpus .= "-$cpuend" if $cpuend;
push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i"; my $mem_object = print_mem_object($conf, "ram-node$i", $numa_memory);
push @$cmd, '-object', $mem_object;
push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i"; push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
} }
} }
@ -228,7 +281,10 @@ sub config {
if ($hotplug_features->{memory}) { if ($hotplug_features->{memory}) {
foreach_dimm($conf, $vmid, $memory, $sockets, sub { foreach_dimm($conf, $vmid, $memory, $sockets, sub {
my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_; my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
my $mem_object = print_mem_object($conf, "mem-$name", $dimm_size);
push @$cmd, "-object" , $mem_object;
push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode"; push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
#if dimm_memory is not aligned to dimm map #if dimm_memory is not aligned to dimm map
@ -240,6 +296,239 @@ sub config {
} }
} }
sub print_mem_object {
my ($conf, $id, $size) = @_;
if ($conf->{hugepages}) {
my $hugepages_size = hugepages_size($conf, $size);
my $path = hugepages_mount_path($hugepages_size);
return "memory-backend-file,id=$id,size=${size}M,mem-path=$path,share=on,prealloc=yes";
} else {
return "memory-backend-ram,id=$id,size=${size}M";
}
}
sub hugepages_mount {
my $mountdata = PVE::ProcFSTools::parse_proc_mounts();
foreach my $size (qw(2048 1048576)) {
return if (! -d "/sys/kernel/mm/hugepages/hugepages-${size}kB");
my $path = "/run/hugepages/kvm/${size}kB";
my $found = grep {
$_->[2] =~ /^hugetlbfs/ &&
$_->[1] eq $path
} @$mountdata;
if (!$found) {
File::Path::make_path($path) if (!-d $path);
my $cmd = ['/bin/mount', '-t', 'hugetlbfs', '-o', "pagesize=${size}k", 'hugetlbfs', $path];
run_command($cmd, errmsg => "hugepage mount error");
}
}
}
sub hugepages_mount_path {
my ($size) = @_;
$size = $size * 1024;
return "/run/hugepages/kvm/${size}kB";
}
sub hugepages_nr {
my ($size, $hugepages_size) = @_;
return $size / $hugepages_size;
}
sub hugepages_size {
my ($conf, $size) = @_;
die "hugepages option is not enabled" if !$conf->{hugepages};
if ($conf->{hugepages} eq 'any') {
#try to use 1GB if available && memory size is matching
if (-d "/sys/kernel/mm/hugepages/hugepages-1048576kB" && ($size % 1024 == 0)) {
return 1024;
} else {
return 2;
}
} else {
my $hugepagesize = $conf->{hugepages} * 1024 . "kB";
if (! -d "/sys/kernel/mm/hugepages/hugepages-$hugepagesize") {
die "your system doesn't support hugepages of $hugepagesize";
}
die "Memory size $size is not a multiple of the requested hugepages size $hugepagesize" if ($size % $conf->{hugepages}) != 0;
return $conf->{hugepages};
}
}
sub hugepages_topology {
my ($conf) = @_;
my $hugepages_topology = {};
return if !$conf->{numa};
my $defaults = PVE::QemuServer::load_defaults();
my $memory = $conf->{memory} || $defaults->{memory};
my $static_memory = 0;
my $sockets = 1;
$sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
$sockets = $conf->{sockets} if $conf->{sockets};
my $numa_custom_topology = undef;
my $hotplug_features = PVE::QemuServer::parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
if ($hotplug_features->{memory}) {
$static_memory = $STATICMEM;
$static_memory = $static_memory * $sockets if ($conf->{hugepages} && $conf->{hugepages} == 1024);
} else {
$static_memory = $memory;
}
#custom numa topology
for (my $i = 0; $i < $MAX_NUMA; $i++) {
next if !$conf->{"numa$i"};
my $numa = PVE::QemuServer::parse_numa($conf->{"numa$i"});
next if !$numa;
$numa_custom_topology = 1;
my $numa_memory = $numa->{memory};
my $hugepages_size = hugepages_size($conf, $numa_memory);
$hugepages_topology->{$hugepages_size}->{$i} += hugepages_nr($numa_memory, $hugepages_size);
}
#if no custom numa tology, we split memory and cores across numa nodes
if(!$numa_custom_topology) {
my $numa_memory = ($static_memory / $sockets);
for (my $i = 0; $i < $sockets; $i++) {
my $hugepages_size = hugepages_size($conf, $numa_memory);
$hugepages_topology->{$hugepages_size}->{$i} += hugepages_nr($numa_memory, $hugepages_size);
}
}
if ($hotplug_features->{memory}) {
foreach_dimm($conf, undef, $memory, $sockets, sub {
my ($conf, undef, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
my $hugepages_size = hugepages_size($conf, $dimm_size);
$hugepages_topology->{$hugepages_size}->{$numanode} += hugepages_nr($dimm_size, $hugepages_size);
});
}
return $hugepages_topology;
}
sub hugepages_host_topology {
#read host hugepages
my $hugepages_host_topology = {};
dir_glob_foreach("/sys/devices/system/node/", 'node(\d+)', sub {
my ($nodepath, $numanode) = @_;
dir_glob_foreach("/sys/devices/system/node/$nodepath/hugepages/", 'hugepages\-(\d+)kB', sub {
my ($hugepages_path, $hugepages_size) = @_;
$hugepages_size = $hugepages_size / 1024;
my $hugepages_nr = PVE::Tools::file_read_firstline("/sys/devices/system/node/$nodepath/hugepages/$hugepages_path/nr_hugepages");
$hugepages_host_topology->{$hugepages_size}->{$numanode} = $hugepages_nr;
});
});
return $hugepages_host_topology;
}
sub hugepages_allocate {
my ($hugepages_topology, $hugepages_host_topology) = @_;
#allocate new hupages if needed
foreach my $size (sort keys %$hugepages_topology) {
my $nodes = $hugepages_topology->{$size};
foreach my $numanode (keys %$nodes) {
my $hugepages_size = $size * 1024;
my $hugepages_requested = $hugepages_topology->{$size}->{$numanode};
my $path = "/sys/devices/system/node/node${numanode}/hugepages/hugepages-${hugepages_size}kB/";
my $hugepages_free = PVE::Tools::file_read_firstline($path."free_hugepages");
my $hugepages_nr = PVE::Tools::file_read_firstline($path."nr_hugepages");
if ($hugepages_requested > $hugepages_free) {
my $hugepages_needed = $hugepages_requested - $hugepages_free;
PVE::ProcFSTools::write_proc_entry($path."nr_hugepages", $hugepages_nr + $hugepages_needed);
#verify that is correctly allocated
$hugepages_free = PVE::Tools::file_read_firstline($path."free_hugepages");
if ($hugepages_free < $hugepages_requested) {
#rollback to initial host config
hugepages_reset($hugepages_host_topology);
die "hugepage allocation failed";
}
}
}
}
}
sub hugepages_pre_deallocate {
my ($hugepages_topology) = @_;
foreach my $size (sort keys %$hugepages_topology) {
my $hugepages_size = $size * 1024;
my $path = "/sys/kernel/mm/hugepages/hugepages-${hugepages_size}kB/";
my $hugepages_nr = PVE::Tools::file_read_firstline($path."nr_hugepages");
PVE::ProcFSTools::write_proc_entry($path."nr_hugepages", 0);
}
}
sub hugepages_reset {
my ($hugepages_topology) = @_;
foreach my $size (sort keys %$hugepages_topology) {
my $nodes = $hugepages_topology->{$size};
foreach my $numanode (keys %$nodes) {
my $hugepages_nr = $hugepages_topology->{$size}->{$numanode};
my $hugepages_size = $size * 1024;
my $path = "/sys/devices/system/node/node${numanode}/hugepages/hugepages-${hugepages_size}kB/";
PVE::ProcFSTools::write_proc_entry($path."nr_hugepages", $hugepages_nr);
}
}
}
sub hugepages_update_locked {
my ($code, @param) = @_;
my $timeout = 60; #could be long if a lot of hugepages need to be alocated
my $lock_filename = "/var/lock/hugepages.lck";
my $res = lock_file($lock_filename, $timeout, $code, @param);
die $@ if $@;
return $res;
}
1; 1;