proxmox-mirrors/qemu-server
1
0
Fork 0
mirror of https://git.proxmox.com/git/qemu-server synced 2025-07-09 13:58:22 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

migrate: refactor remote VM/tunnel start

Browse Source 
no semantic changes intended, except for:
- no longer passing the main migration UNIX socket to SSH twice for
forwarding
- dropping the 'unix:' prefix in start_remote_tunnel's timeout error message

Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
This commit is contained in:
Fabian Grünbichler 2022-11-17 14:33:43 +01:00 committed by Thomas Lamprecht
parent 347dc13650
commit 05b2a4ae9c
2 changed files with 116 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

159
PVE/QemuMigrate.pm
View File

@ -43,19 +43,24 @@ sub fork_tunnel {
return PVE::Tunnel::fork_ssh_tunnel($self->{rem_ssh}, $cmd, $ssh_forward_info, $log); return PVE::Tunnel::fork_ssh_tunnel($self->{rem_ssh}, $cmd, $ssh_forward_info, $log);
} }
# tunnel_info:
# proto: unix (secure) or tcp (insecure/legacy compat)
# addr: IP or UNIX socket path
# port: optional TCP port
# unix_sockets: additional UNIX socket paths to forward
sub start_remote_tunnel { sub start_remote_tunnel {
my ($self, $raddr, $rport, $ruri, $unix_socket_info) = @_; my ($self, $tunnel_info) = @_;
my $nodename = PVE::INotify::nodename(); my $nodename = PVE::INotify::nodename();
my $migration_type = $self->{opts}->{migration_type}; my $migration_type = $self->{opts}->{migration_type};
if ($migration_type eq 'secure') { if ($migration_type eq 'secure') {
if ($ruri =~ /^unix:/) { if ($tunnel_info->{proto} eq 'unix') {
my $ssh_forward_info = ["$raddr:$raddr"]; my $ssh_forward_info = [];
$unix_socket_info->{$raddr} = 1;
my $unix_sockets = [ keys %$unix_socket_info ]; my $unix_sockets = [ keys %{$tunnel_info->{unix_sockets}} ];
push @$unix_sockets, $tunnel_info->{addr};
for my $sock (@$unix_sockets) { for my $sock (@$unix_sockets) {
push @$ssh_forward_info, "$sock:$sock"; push @$ssh_forward_info, "$sock:$sock";
unlink $sock; unlink $sock;
@ -82,23 +87,23 @@ sub start_remote_tunnel {
if ($unix_socket_try > 100) { if ($unix_socket_try > 100) {
$self->{errors} = 1; $self->{errors} = 1;
PVE::Tunnel::finish_tunnel($self->{tunnel}); PVE::Tunnel::finish_tunnel($self->{tunnel});
die "Timeout, migration socket $ruri did not get ready"; die "Timeout, migration socket $tunnel_info->{addr} did not get ready";
} }
$self->{tunnel}->{unix_sockets} = $unix_sockets if (@$unix_sockets); $self->{tunnel}->{unix_sockets} = $unix_sockets if (@$unix_sockets);
} elsif ($ruri =~ /^tcp:/) { } elsif ($tunnel_info->{proto} eq 'tcp') {
my $ssh_forward_info = []; my $ssh_forward_info = [];
if ($raddr eq "localhost") { if ($tunnel_info->{addr} eq "localhost") {
# for backwards compatibility with older qemu-server versions # for backwards compatibility with older qemu-server versions
my $pfamily = PVE::Tools::get_host_address_family($nodename); my $pfamily = PVE::Tools::get_host_address_family($nodename);
my $lport = PVE::Tools::next_migrate_port($pfamily); my $lport = PVE::Tools::next_migrate_port($pfamily);
push @$ssh_forward_info, "$lport:localhost:$rport"; push @$ssh_forward_info, "$lport:localhost:$tunnel_info->{port}";
} }
$self->{tunnel} = $self->fork_tunnel($ssh_forward_info); $self->{tunnel} = $self->fork_tunnel($ssh_forward_info);
} else { } else {
die "unsupported protocol in migration URI: $ruri\n"; die "unsupported protocol in migration URI: $tunnel_info->{proto}\n";
} }
} else { } else {
#fork tunnel for insecure migration, to send faster commands like resume #fork tunnel for insecure migration, to send faster commands like resume
@ -663,52 +668,45 @@ sub phase1_cleanup {
} }
} }
sub phase2 { sub phase2_start_local_cluster {
my ($self, $vmid) = @_; my ($self, $vmid, $params) = @_;
my $conf = $self->{vmconf}; my $conf = $self->{vmconf};
my $local_volumes = $self->{local_volumes}; my $local_volumes = $self->{local_volumes};
my @online_local_volumes = $self->filter_local_volumes('online'); my @online_local_volumes = $self->filter_local_volumes('online');
$self->{storage_migration} = 1 if scalar(@online_local_volumes); $self->{storage_migration} = 1 if scalar(@online_local_volumes);
my $start = $params->{start_params};
my $migrate = $params->{migrate_opts};
$self->log('info', "starting VM $vmid on remote node '$self->{node}'"); $self->log('info', "starting VM $vmid on remote node '$self->{node}'");
my $raddr; my $tunnel_info = {};
my $rport;
my $ruri; # the whole migration dst. URI (protocol:address[:port])
my $nodename = PVE::INotify::nodename();
## start on remote node ## start on remote node
my $cmd = [@{$self->{rem_ssh}}]; my $cmd = [@{$self->{rem_ssh}}];
my $spice_ticket; push @$cmd, 'qm', 'start', $vmid;
if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) {
my $res = mon_cmd($vmid, 'query-spice'); if ($start->{skiplock}) {
$spice_ticket = $res->{ticket}; push @$cmd, '--skiplock';
} }
push @$cmd , 'qm', 'start', $vmid, '--skiplock', '--migratedfrom', $nodename; push @$cmd, '--migratedfrom', $migrate->{migratedfrom};
my $migration_type = $self->{opts}->{migration_type}; push @$cmd, '--migration_type', $migrate->{type};
push @$cmd, '--migration_type', $migration_type; push @$cmd, '--migration_network', $migrate->{network}
if $migrate->{network};
push @$cmd, '--migration_network', $self->{opts}->{migration_network} push @$cmd, '--stateuri', $start->{statefile};
if $self->{opts}->{migration_network};
if ($migration_type eq 'insecure') { if ($start->{forcemachine}) {
push @$cmd, '--stateuri', 'tcp'; push @$cmd, '--machine', $start->{forcemachine};
} else {
push @$cmd, '--stateuri', 'unix';
} }
if ($self->{forcemachine}) { if ($start->{forcecpu}) {
push @$cmd, '--machine', $self->{forcemachine}; push @$cmd, '--force-cpu', $start->{forcecpu};
}
if ($self->{forcecpu}) {
push @$cmd, '--force-cpu', $self->{forcecpu};
} }
if ($self->{storage_migration}) { if ($self->{storage_migration}) {
@ -716,10 +714,7 @@ sub phase2 {
} }
my $spice_port; my $spice_port;
my $unix_socket_info = {}; my $input = "nbd_protocol_version: $migrate->{nbd_proto_version}\n";
# version > 0 for unix socket support
my $nbd_protocol_version = 1;
my $input = "nbd_protocol_version: $nbd_protocol_version\n";
my @offline_local_volumes = $self->filter_local_volumes('offline'); my @offline_local_volumes = $self->filter_local_volumes('offline');
for my $volid (@offline_local_volumes) { for my $volid (@offline_local_volumes) {
@ -737,7 +732,7 @@ sub phase2 {
} }
} }
$input .= "spice_ticket: $spice_ticket\n" if $spice_ticket; $input .= "spice_ticket: $migrate->{spice_ticket}\n" if $migrate->{spice_ticket};
my @online_replicated_volumes = $self->filter_local_volumes('online', 1); my @online_replicated_volumes = $self->filter_local_volumes('online', 1);
foreach my $volid (@online_replicated_volumes) { foreach my $volid (@online_replicated_volumes) {
@ -767,20 +762,20 @@ sub phase2 {
my $exitcode = PVE::Tools::run_command($cmd, input => $input, outfunc => sub { my $exitcode = PVE::Tools::run_command($cmd, input => $input, outfunc => sub {
my $line = shift; my $line = shift;
if ($line =~ m/^migration listens on tcp:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+)$/) { if ($line =~ m/^migration listens on (tcp):(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+)$/) {
$raddr = $1; $tunnel_info->{addr} = $2;
$rport = int($2); $tunnel_info->{port} = int($3);
$ruri = "tcp:$raddr:$rport"; $tunnel_info->{proto} = $1;
} }
elsif ($line =~ m!^migration listens on unix:(/run/qemu-server/(\d+)\.migrate)$!) { elsif ($line =~ m!^migration listens on (unix):(/run/qemu-server/(\d+)\.migrate)$!) {
$raddr = $1; $tunnel_info->{addr} = $2;
die "Destination UNIX sockets VMID does not match source VMID" if $vmid ne $2; die "Destination UNIX sockets VMID does not match source VMID" if $vmid ne $3;
$ruri = "unix:$raddr"; $tunnel_info->{proto} = $1;
} }
elsif ($line =~ m/^migration listens on port (\d+)$/) { elsif ($line =~ m/^migration listens on port (\d+)$/) {
$raddr = "localhost"; $tunnel_info->{addr} = "localhost";
$rport = int($1); $tunnel_info->{port} = int($1);
$ruri = "tcp:$raddr:$rport"; $tunnel_info->{proto} = "tcp";
} }
elsif ($line =~ m/^spice listens on port (\d+)$/) { elsif ($line =~ m/^spice listens on port (\d+)$/) {
$spice_port = int($1); $spice_port = int($1);
@ -801,7 +796,7 @@ sub phase2 {
$targetdrive =~ s/drive-//g; $targetdrive =~ s/drive-//g;
$handle_storage_migration_listens->($targetdrive, $drivestr, $nbd_uri); $handle_storage_migration_listens->($targetdrive, $drivestr, $nbd_uri);
$unix_socket_info->{$nbd_unix_addr} = 1; $tunnel_info->{unix_sockets}->{$nbd_unix_addr} = 1;
} elsif ($line =~ m/^re-using replicated volume: (\S+) - (.*)$/) { } elsif ($line =~ m/^re-using replicated volume: (\S+) - (.*)$/) {
my $drive = $1; my $drive = $1;
my $volid = $2; my $volid = $2;
@ -816,19 +811,65 @@ sub phase2 {
die "remote command failed with exit code $exitcode\n" if $exitcode; die "remote command failed with exit code $exitcode\n" if $exitcode;
die "unable to detect remote migration address\n" if !$raddr; die "unable to detect remote migration address\n" if !$tunnel_info->{addr} || !$tunnel_info->{proto};
if (scalar(keys %$target_replicated_volumes) != scalar(@online_replicated_volumes)) { if (scalar(keys %$target_replicated_volumes) != scalar(@online_replicated_volumes)) {
die "number of replicated disks on source and target node do not match - target node too old?\n" die "number of replicated disks on source and target node do not match - target node too old?\n"
} }
return ($tunnel_info, $spice_port);
}
sub phase2 {
my ($self, $vmid) = @_;
my $conf = $self->{vmconf};
# version > 0 for unix socket support
my $nbd_protocol_version = 1;
my $spice_ticket;
if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) {
my $res = mon_cmd($vmid, 'query-spice');
$spice_ticket = $res->{ticket};
}
my $migration_type = $self->{opts}->{migration_type};
my $state_uri = $migration_type eq 'insecure' ? 'tcp' : 'unix';
my $params = {
start_params => {
statefile => $state_uri,
forcemachine => $self->{forcemachine},
forcecpu => $self->{forcecpu},
skiplock => 1,
},
migrate_opts => {
spice_ticket => $spice_ticket,
type => $migration_type,
network => $self->{opts}->{migration_network},
storagemap => $self->{opts}->{storagemap},
migratedfrom => PVE::INotify::nodename(),
nbd_proto_version => $nbd_protocol_version,
nbd => $self->{nbd},
},
};
my ($tunnel_info, $spice_port) = $self->phase2_start_local_cluster($vmid, $params);
$self->log('info', "start remote tunnel"); $self->log('info', "start remote tunnel");
$self->start_remote_tunnel($raddr, $rport, $ruri, $unix_socket_info); $self->start_remote_tunnel($tunnel_info);
my $migrate_uri = "$tunnel_info->{proto}:$tunnel_info->{addr}";
$migrate_uri .= ":$tunnel_info->{port}"
if defined($tunnel_info->{port});
if ($self->{storage_migration}) { if ($self->{storage_migration}) {
$self->{storage_migration_jobs} = {}; $self->{storage_migration_jobs} = {};
$self->log('info', "starting storage migration"); $self->log('info', "starting storage migration");
my @online_local_volumes = $self->filter_local_volumes('online');
die "The number of local disks does not match between the source and the destination.\n" die "The number of local disks does not match between the source and the destination.\n"
if (scalar(keys %{$self->{target_drive}}) != scalar(@online_local_volumes)); if (scalar(keys %{$self->{target_drive}}) != scalar(@online_local_volumes));
foreach my $drive (keys %{$self->{target_drive}}){ foreach my $drive (keys %{$self->{target_drive}}){
@ -838,7 +879,7 @@ sub phase2 {
my $source_drive = PVE::QemuServer::parse_drive($drive, $conf->{$drive}); my $source_drive = PVE::QemuServer::parse_drive($drive, $conf->{$drive});
my $source_volid = $source_drive->{file}; my $source_volid = $source_drive->{file};
my $bwlimit = $local_volumes->{$source_volid}->{bwlimit}; my $bwlimit = $self->{local_volumes}->{$source_volid}->{bwlimit};
my $bitmap = $target->{bitmap}; my $bitmap = $target->{bitmap};
$self->log('info', "$drive: start migration to $nbd_uri"); $self->log('info', "$drive: start migration to $nbd_uri");
@ -846,7 +887,7 @@ sub phase2 {
} }
} }
$self->log('info', "starting online/live migration on $ruri"); $self->log('info', "starting online/live migration on $migrate_uri");
$self->{livemigration} = 1; $self->{livemigration} = 1;
# load_defaults # load_defaults
@ -923,12 +964,12 @@ sub phase2 {
my $start = time(); my $start = time();
$self->log('info', "start migrate command to $ruri"); $self->log('info', "start migrate command to $migrate_uri");
eval { eval {
mon_cmd($vmid, "migrate", uri => $ruri); mon_cmd($vmid, "migrate", uri => $migrate_uri);
}; };
my $merr = $@; my $merr = $@;
$self->log('info', "migrate uri => $ruri failed: $merr") if $merr; $self->log('info', "migrate uri => $migrate_uri failed: $merr") if $merr;
my $last_mem_transferred = 0; my $last_mem_transferred = 0;
my $usleep = 1000000; my $usleep = 1000000;

34
PVE/QemuServer.pm
View File

@ -5735,10 +5735,10 @@ sub vm_start_nolock {
return $migration_ip; return $migration_ip;
}; };
my $migrate_uri;
if ($statefile) { if ($statefile) {
if ($statefile eq 'tcp') { if ($statefile eq 'tcp') {
my $localip = "localhost"; my $migrate = $res->{migrate} = { proto => 'tcp' };
$migrate->{addr} = "localhost";
my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg'); my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
my $nodename = nodename(); my $nodename = nodename();
@ -5751,26 +5751,26 @@ sub vm_start_nolock {
} }
if ($migration_type eq 'insecure') { if ($migration_type eq 'insecure') {
$localip = $get_migration_ip->($nodename); $migrate->{addr} = $get_migration_ip->($nodename);
$localip = "[$localip]" if Net::IP::ip_is_ipv6($localip); $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
} }
my $pfamily = PVE::Tools::get_host_address_family($nodename); my $pfamily = PVE::Tools::get_host_address_family($nodename);
my $migrate_port = PVE::Tools::next_migrate_port($pfamily); $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
$migrate_uri = "tcp:${localip}:${migrate_port}"; $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
push @$cmd, '-incoming', $migrate_uri; push @$cmd, '-incoming', $migrate->{uri};
push @$cmd, '-S'; push @$cmd, '-S';
} elsif ($statefile eq 'unix') { } elsif ($statefile eq 'unix') {
# should be default for secure migrations as a ssh TCP forward # should be default for secure migrations as a ssh TCP forward
# tunnel is not deterministic reliable ready and fails regurarly # tunnel is not deterministic reliable ready and fails regurarly
# to set up in time, so use UNIX socket forwards # to set up in time, so use UNIX socket forwards
my $socket_addr = "/run/qemu-server/$vmid.migrate"; my $migrate = $res->{migrate} = { proto => 'unix' };
unlink $socket_addr; $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
unlink $migrate->{addr};
$migrate_uri = "unix:$socket_addr"; $migrate->{uri} = "unix:$migrate->{addr}";
push @$cmd, '-incoming', $migrate->{uri};
push @$cmd, '-incoming', $migrate_uri;
push @$cmd, '-S'; push @$cmd, '-S';
} elsif (-e $statefile) { } elsif (-e $statefile) {
@ -5925,10 +5925,9 @@ sub vm_start_nolock {
eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) }; eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
warn $@ if $@; warn $@ if $@;
print "migration listens on $migrate_uri\n" if $migrate_uri; if (defined($res->{migrate})) {
$res->{migrate_uri} = $migrate_uri; print "migration listens on $res->{migrate}->{uri}\n";
} elsif ($statefile) {
if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix') {
eval { mon_cmd($vmid, "cont"); }; eval { mon_cmd($vmid, "cont"); };
warn $@ if $@; warn $@ if $@;
} }
@ -5943,6 +5942,7 @@ sub vm_start_nolock {
my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate"; my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } ); mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
$migrate_storage_uri = "nbd:unix:$socket_path"; $migrate_storage_uri = "nbd:unix:$socket_path";
$res->{migrate}->{unix_sockets} = [$socket_path];
} else { } else {
my $nodename = nodename(); my $nodename = nodename();
my $localip = $get_migration_ip->($nodename); my $localip = $get_migration_ip->($nodename);
@ -5960,8 +5960,6 @@ sub vm_start_nolock {
$migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}"; $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
} }
$res->{migrate_storage_uri} = $migrate_storage_uri;
foreach my $opt (sort keys %$nbd) { foreach my $opt (sort keys %$nbd) {
my $drivestr = $nbd->{$opt}->{drivestr}; my $drivestr = $nbd->{$opt}->{drivestr};
my $volid = $nbd->{$opt}->{volid}; my $volid = $nbd->{$opt}->{volid};