proxmox-mirrors/qemu-server
1
0
Fork 0
mirror of https://git.proxmox.com/git/qemu-server synced 2025-08-06 15:47:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

migrate: add remote migration handling

Browse Source 
remote migration uses a websocket connection to a task worker running on
the target node instead of commands via SSH to control the migration.
this websocket tunnel is started earlier than the SSH tunnel, and allows
adding UNIX-socket forwarding over additional websocket connections
on-demand.

the main differences to regular intra-cluster migration are:
- source VM config and disks are only removed upon request via --delete
- shared storages are treated like local storages, since we can't
assume they are shared across clusters (with potentical to extend this
by marking storages as shared)
- NBD migrated disks are explicitly pre-allocated on the target node via
tunnel command before starting the target VM instance
- in addition to storages, network bridges and the VMID itself is
transformed via a user defined mapping
- all commands and migration data streams are sent via a WS tunnel proxy
- pending changes and snapshots are discarded on the target side (for
  the time being)

Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
This commit is contained in:
Fabian Grünbichler 2022-11-17 14:33:44 +01:00 committed by Thomas Lamprecht
parent 05b2a4ae9c
commit eef93bc590
4 changed files with 367 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
PVE/API2/Qemu.pm
View File

@ -5252,7 +5252,7 @@ __PACKAGE__->register_method({
# bump/reset for breaking changes # bump/reset for breaking changes
# bump/bump for opt-in changes # bump/bump for opt-in changes
return { return {
api => 2, api => $PVE::QemuMigrate::WS_TUNNEL_VERSION,
age => 0, age => 0,
}; };
}, },

437
PVE/QemuMigrate.pm
View File

@ -5,11 +5,10 @@ use warnings;
use IO::File; use IO::File;
use IPC::Open2; use IPC::Open2;
use POSIX qw( WNOHANG );
use Time::HiRes qw( usleep ); use Time::HiRes qw( usleep );
use PVE::Format qw(render_bytes);
use PVE::Cluster; use PVE::Cluster;
use PVE::Format qw(render_bytes);
use PVE::GuestHelpers qw(safe_boolean_ne safe_string_ne); use PVE::GuestHelpers qw(safe_boolean_ne safe_string_ne);
use PVE::INotify; use PVE::INotify;
use PVE::RPCEnvironment; use PVE::RPCEnvironment;
@ -17,6 +16,7 @@ use PVE::Replication;
use PVE::ReplicationConfig; use PVE::ReplicationConfig;
use PVE::ReplicationState; use PVE::ReplicationState;
use PVE::Storage; use PVE::Storage;
use PVE::StorageTunnel;
use PVE::Tools; use PVE::Tools;
use PVE::Tunnel; use PVE::Tunnel;
@ -31,6 +31,9 @@ use PVE::QemuServer;
use PVE::AbstractMigrate; use PVE::AbstractMigrate;
use base qw(PVE::AbstractMigrate); use base qw(PVE::AbstractMigrate);
# compared against remote end's minimum version
our $WS_TUNNEL_VERSION = 2;
sub fork_tunnel { sub fork_tunnel {
my ($self, $ssh_forward_info) = @_; my ($self, $ssh_forward_info) = @_;
@ -43,6 +46,35 @@ sub fork_tunnel {
return PVE::Tunnel::fork_ssh_tunnel($self->{rem_ssh}, $cmd, $ssh_forward_info, $log); return PVE::Tunnel::fork_ssh_tunnel($self->{rem_ssh}, $cmd, $ssh_forward_info, $log);
} }
sub fork_websocket_tunnel {
my ($self, $storages, $bridges) = @_;
my $remote = $self->{opts}->{remote};
my $conn = $remote->{conn};
my $log = sub {
my ($level, $msg) = @_;
$self->log($level, $msg);
};
my $websocket_url = "https://$conn->{host}:$conn->{port}/api2/json/nodes/$self->{node}/qemu/$remote->{vmid}/mtunnelwebsocket";
my $url = "/nodes/$self->{node}/qemu/$remote->{vmid}/mtunnel";
my $tunnel_params = {
url => $websocket_url,
};
my $storage_list = join(',', keys %$storages);
my $bridge_list = join(',', keys %$bridges);
my $req_params = {
storages => $storage_list,
bridges => $bridge_list,
};
return PVE::Tunnel::fork_websocket_tunnel($conn, $url, $req_params, $tunnel_params, $log);
}
# tunnel_info: # tunnel_info:
# proto: unix (secure) or tcp (insecure/legacy compat) # proto: unix (secure) or tcp (insecure/legacy compat)
# addr: IP or UNIX socket path # addr: IP or UNIX socket path
@ -188,23 +220,34 @@ sub prepare {
} }
my $vollist = PVE::QemuServer::get_vm_volumes($conf); my $vollist = PVE::QemuServer::get_vm_volumes($conf);
my $storages = {};
foreach my $volid (@$vollist) { foreach my $volid (@$vollist) {
my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1); my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
# check if storage is available on both nodes # check if storage is available on source node
my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid); my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
my $targetsid = $sid; my $targetsid = $sid;
# NOTE: we currently ignore shared source storages in mappings so skip here too for now # NOTE: local ignores shared mappings, remote maps them
if (!$scfg->{shared}) { if (!$scfg->{shared} || $self->{opts}->{remote}) {
$targetsid = PVE::JSONSchema::map_id($self->{opts}->{storagemap}, $sid); $targetsid = PVE::JSONSchema::map_id($self->{opts}->{storagemap}, $sid);
} }
my $target_scfg = PVE::Storage::storage_check_enabled($storecfg, $targetsid, $self->{node}); $storages->{$targetsid} = 1;
my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
die "$volid: content type '$vtype' is not available on storage '$targetsid'\n" if (!$self->{opts}->{remote}) {
if !$target_scfg->{content}->{$vtype}; # check if storage is available on target node
my $target_scfg = PVE::Storage::storage_check_enabled(
$storecfg,
$targetsid,
$self->{node},
);
my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
die "$volid: content type '$vtype' is not available on storage '$targetsid'\n"
if !$target_scfg->{content}->{$vtype};
}
if ($scfg->{shared}) { if ($scfg->{shared}) {
# PVE::Storage::activate_storage checks this for non-shared storages # PVE::Storage::activate_storage checks this for non-shared storages
@ -214,10 +257,27 @@ sub prepare {
} }
} }
# test ssh connection if ($self->{opts}->{remote}) {
my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ]; # test & establish websocket connection
eval { $self->cmd_quiet($cmd); }; my $bridges = map_bridges($conf, $self->{opts}->{bridgemap}, 1);
die "Can't connect to destination address using public key\n" if $@; my $tunnel = $self->fork_websocket_tunnel($storages, $bridges);
my $min_version = $tunnel->{version} - $tunnel->{age};
$self->log('info', "local WS tunnel version: $WS_TUNNEL_VERSION");
$self->log('info', "remote WS tunnel version: $tunnel->{version}");
$self->log('info', "minimum required WS tunnel version: $min_version");
die "Remote tunnel endpoint not compatible, upgrade required\n"
if $WS_TUNNEL_VERSION < $min_version;
die "Remote tunnel endpoint too old, upgrade required\n"
if $WS_TUNNEL_VERSION > $tunnel->{version};
print "websocket tunnel started\n";
$self->{tunnel} = $tunnel;
} else {
# test ssh connection
my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
eval { $self->cmd_quiet($cmd); };
die "Can't connect to destination address using public key\n" if $@;
}
return $running; return $running;
} }
@ -255,7 +315,7 @@ sub scan_local_volumes {
my @sids = PVE::Storage::storage_ids($storecfg); my @sids = PVE::Storage::storage_ids($storecfg);
foreach my $storeid (@sids) { foreach my $storeid (@sids) {
my $scfg = PVE::Storage::storage_config($storecfg, $storeid); my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
next if $scfg->{shared}; next if $scfg->{shared} && !$self->{opts}->{remote};
next if !PVE::Storage::storage_check_enabled($storecfg, $storeid, undef, 1); next if !PVE::Storage::storage_check_enabled($storecfg, $storeid, undef, 1);
# get list from PVE::Storage (for unused volumes) # get list from PVE::Storage (for unused volumes)
@ -264,21 +324,20 @@ sub scan_local_volumes {
next if @{$dl->{$storeid}} == 0; next if @{$dl->{$storeid}} == 0;
my $targetsid = PVE::JSONSchema::map_id($self->{opts}->{storagemap}, $storeid); my $targetsid = PVE::JSONSchema::map_id($self->{opts}->{storagemap}, $storeid);
# check if storage is available on target node if (!$self->{opts}->{remote}) {
my $target_scfg = PVE::Storage::storage_check_enabled( # check if storage is available on target node
$storecfg, my $target_scfg = PVE::Storage::storage_check_enabled(
$targetsid, $storecfg,
$self->{node}, $targetsid,
); $self->{node},
);
die "content type 'images' is not available on storage '$targetsid'\n" die "content type 'images' is not available on storage '$targetsid'\n"
if !$target_scfg->{content}->{images}; if !$target_scfg->{content}->{images};
my $bwlimit = PVE::Storage::get_bandwidth_limit( }
'migration',
[$targetsid, $storeid], my $bwlimit = $self->get_bwlimit($storeid, $targetsid);
$self->{opts}->{bwlimit},
);
PVE::Storage::foreach_volid($dl, sub { PVE::Storage::foreach_volid($dl, sub {
my ($volid, $sid, $volinfo) = @_; my ($volid, $sid, $volinfo) = @_;
@ -332,14 +391,17 @@ sub scan_local_volumes {
my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid); my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
my $targetsid = $sid; my $targetsid = $sid;
# NOTE: we currently ignore shared source storages in mappings so skip here too for now # NOTE: local ignores shared mappings, remote maps them
if (!$scfg->{shared}) { if (!$scfg->{shared} || $self->{opts}->{remote}) {
$targetsid = PVE::JSONSchema::map_id($self->{opts}->{storagemap}, $sid); $targetsid = PVE::JSONSchema::map_id($self->{opts}->{storagemap}, $sid);
} }
PVE::Storage::storage_check_enabled($storecfg, $targetsid, $self->{node}); # check target storage on target node if intra-cluster migration
if (!$self->{opts}->{remote}) {
PVE::Storage::storage_check_enabled($storecfg, $targetsid, $self->{node});
return if $scfg->{shared}; return if $scfg->{shared};
}
$local_volumes->{$volid}->{ref} = $attr->{referenced_in_config} ? 'config' : 'snapshot'; $local_volumes->{$volid}->{ref} = $attr->{referenced_in_config} ? 'config' : 'snapshot';
$local_volumes->{$volid}->{ref} = 'storage' if $attr->{is_unused}; $local_volumes->{$volid}->{ref} = 'storage' if $attr->{is_unused};
@ -372,6 +434,8 @@ sub scan_local_volumes {
# exceptions: 'zfspool' or 'qcow2' files (on directory storage) # exceptions: 'zfspool' or 'qcow2' files (on directory storage)
die "online storage migration not possible if snapshot exists\n" if $self->{running}; die "online storage migration not possible if snapshot exists\n" if $self->{running};
die "remote migration with snapshots not supported yet\n" if $self->{opts}->{remote};
if (!($scfg->{type} eq 'zfspool' if (!($scfg->{type} eq 'zfspool'
|| ($scfg->{type} eq 'btrfs' && $local_volumes->{$volid}->{format} eq 'raw') || ($scfg->{type} eq 'btrfs' && $local_volumes->{$volid}->{format} eq 'raw')
|| $local_volumes->{$volid}->{format} eq 'qcow2' || $local_volumes->{$volid}->{format} eq 'qcow2'
@ -428,6 +492,9 @@ sub scan_local_volumes {
my $migratable = $scfg->{type} =~ /^(?:dir|btrfs|zfspool|lvmthin|lvm)$/; my $migratable = $scfg->{type} =~ /^(?:dir|btrfs|zfspool|lvmthin|lvm)$/;
# TODO: what is this even here for?
$migratable = 1 if $self->{opts}->{remote};
die "can't migrate '$volid' - storage type '$scfg->{type}' not supported\n" die "can't migrate '$volid' - storage type '$scfg->{type}' not supported\n"
if !$migratable; if !$migratable;
@ -462,6 +529,10 @@ sub handle_replication {
my $local_volumes = $self->{local_volumes}; my $local_volumes = $self->{local_volumes};
return if !$self->{replication_jobcfg}; return if !$self->{replication_jobcfg};
die "can't migrate VM with replicated volumes to remote cluster/node\n"
if $self->{opts}->{remote};
if ($self->{running}) { if ($self->{running}) {
my $version = PVE::QemuServer::kvm_user_version(); my $version = PVE::QemuServer::kvm_user_version();
@ -561,24 +632,51 @@ sub sync_offline_local_volumes {
$self->log('info', "copying local disk images") if scalar(@volids); $self->log('info', "copying local disk images") if scalar(@volids);
foreach my $volid (@volids) { foreach my $volid (@volids) {
my $targetsid = $local_volumes->{$volid}->{targetsid}; my $new_volid;
my $bwlimit = $local_volumes->{$volid}->{bwlimit};
$bwlimit = $bwlimit * 1024 if defined($bwlimit); # storage_migrate uses bps
my $storage_migrate_opts = { my $opts = $self->{opts};
'ratelimit_bps' => $bwlimit, if ($opts->{remote}) {
'insecure' => $opts->{migration_type} eq 'insecure', my $log = sub {
'with_snapshots' => $local_volumes->{$volid}->{snapshots}, my ($level, $msg) = @_;
'allow_rename' => !$local_volumes->{$volid}->{is_vmstate}, $self->log($level, $msg);
}; };
my $logfunc = sub { $self->log('info', $_[0]); }; $new_volid = PVE::StorageTunnel::storage_migrate(
my $new_volid = eval { $self->{tunnel},
PVE::Storage::storage_migrate($storecfg, $volid, $self->{ssh_info}, $storecfg,
$targetsid, $storage_migrate_opts, $logfunc); $volid,
}; $self->{vmid},
if (my $err = $@) { $opts->{remote}->{vmid},
die "storage migration for '$volid' to storage '$targetsid' failed - $err\n"; $local_volumes->{$volid},
$log,
);
} else {
my $targetsid = $local_volumes->{$volid}->{targetsid};
my $bwlimit = $local_volumes->{$volid}->{bwlimit};
$bwlimit = $bwlimit * 1024 if defined($bwlimit); # storage_migrate uses bps
my $storage_migrate_opts = {
'ratelimit_bps' => $bwlimit,
'insecure' => $opts->{migration_type} eq 'insecure',
'with_snapshots' => $local_volumes->{$volid}->{snapshots},
'allow_rename' => !$local_volumes->{$volid}->{is_vmstate},
};
my $logfunc = sub { $self->log('info', $_[0]); };
$new_volid = eval {
PVE::Storage::storage_migrate(
$storecfg,
$volid,
$self->{ssh_info},
$targetsid,
$storage_migrate_opts,
$logfunc,
);
};
if (my $err = $@) {
die "storage migration for '$volid' to storage '$targetsid' failed - $err\n";
}
} }
$self->{volume_map}->{$volid} = $new_volid; $self->{volume_map}->{$volid} = $new_volid;
@ -594,6 +692,12 @@ sub sync_offline_local_volumes {
sub cleanup_remotedisks { sub cleanup_remotedisks {
my ($self) = @_; my ($self) = @_;
if ($self->{opts}->{remote}) {
PVE::Tunnel::finish_tunnel($self->{tunnel}, 1);
delete $self->{tunnel};
return;
}
my $local_volumes = $self->{local_volumes}; my $local_volumes = $self->{local_volumes};
foreach my $volid (values %{$self->{volume_map}}) { foreach my $volid (values %{$self->{volume_map}}) {
@ -643,8 +747,100 @@ sub phase1 {
$self->handle_replication($vmid); $self->handle_replication($vmid);
$self->sync_offline_local_volumes(); $self->sync_offline_local_volumes();
$self->phase1_remote($vmid) if $self->{opts}->{remote};
}; };
sub map_bridges {
my ($conf, $map, $scan_only) = @_;
my $bridges = {};
foreach my $opt (keys %$conf) {
next if $opt !~ m/^net\d+$/;
next if !$conf->{$opt};
my $d = PVE::QemuServer::parse_net($conf->{$opt});
next if !$d || !$d->{bridge};
my $target_bridge = PVE::JSONSchema::map_id($map, $d->{bridge});
$bridges->{$target_bridge}->{$opt} = $d->{bridge};
next if $scan_only;
$d->{bridge} = $target_bridge;
$conf->{$opt} = PVE::QemuServer::print_net($d);
}
return $bridges;
}
sub phase1_remote {
my ($self, $vmid) = @_;
my $remote_conf = PVE::QemuConfig->load_config($vmid);
PVE::QemuConfig->update_volume_ids($remote_conf, $self->{volume_map});
my $bridges = map_bridges($remote_conf, $self->{opts}->{bridgemap});
for my $target (keys $bridges->%*) {
for my $nic (keys $bridges->{$target}->%*) {
$self->log('info', "mapped: $nic from $bridges->{$target}->{$nic} to $target");
}
}
my @online_local_volumes = $self->filter_local_volumes('online');
my $storage_map = $self->{opts}->{storagemap};
$self->{nbd} = {};
PVE::QemuConfig->foreach_volume($remote_conf, sub {
my ($ds, $drive) = @_;
# TODO eject CDROM?
return if PVE::QemuServer::drive_is_cdrom($drive);
my $volid = $drive->{file};
return if !$volid;
return if !grep { $_ eq $volid} @online_local_volumes;
my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
my $scfg = PVE::Storage::storage_config($self->{storecfg}, $storeid);
my $source_format = PVE::QemuServer::qemu_img_format($scfg, $volname);
# set by target cluster
my $oldvolid = delete $drive->{file};
delete $drive->{format};
my $targetsid = PVE::JSONSchema::map_id($storage_map, $storeid);
my $params = {
format => $source_format,
storage => $targetsid,
drive => $drive,
};
$self->log('info', "Allocating volume for drive '$ds' on remote storage '$targetsid'..");
my $res = PVE::Tunnel::write_tunnel($self->{tunnel}, 600, 'disk', $params);
$self->log('info', "volume '$oldvolid' is '$res->{volid}' on the target\n");
$remote_conf->{$ds} = $res->{drivestr};
$self->{nbd}->{$ds} = $res;
});
my $conf_str = PVE::QemuServer::write_vm_config("remote", $remote_conf);
# TODO expose in PVE::Firewall?
my $vm_fw_conf_path = "/etc/pve/firewall/$vmid.fw";
my $fw_conf_str;
$fw_conf_str = PVE::Tools::file_get_contents($vm_fw_conf_path)
if -e $vm_fw_conf_path;
my $params = {
conf => $conf_str,
'firewall-config' => $fw_conf_str,
};
PVE::Tunnel::write_tunnel($self->{tunnel}, 10, 'config', $params);
}
sub phase1_cleanup { sub phase1_cleanup {
my ($self, $vmid, $err) = @_; my ($self, $vmid, $err) = @_;
@ -675,7 +871,6 @@ sub phase2_start_local_cluster {
my $local_volumes = $self->{local_volumes}; my $local_volumes = $self->{local_volumes};
my @online_local_volumes = $self->filter_local_volumes('online'); my @online_local_volumes = $self->filter_local_volumes('online');
$self->{storage_migration} = 1 if scalar(@online_local_volumes);
my $start = $params->{start_params}; my $start = $params->{start_params};
my $migrate = $params->{migrate_opts}; my $migrate = $params->{migrate_opts};
@ -820,10 +1015,37 @@ sub phase2_start_local_cluster {
return ($tunnel_info, $spice_port); return ($tunnel_info, $spice_port);
} }
sub phase2_start_remote_cluster {
my ($self, $vmid, $params) = @_;
die "insecure migration to remote cluster not implemented\n"
if $params->{migrate_opts}->{type} ne 'websocket';
my $remote_vmid = $self->{opts}->{remote}->{vmid};
# like regular start but with some overhead accounted for
my $timeout = PVE::QemuServer::Helpers::config_aware_timeout($self->{vmconf}) + 10;
my $res = PVE::Tunnel::write_tunnel($self->{tunnel}, $timeout, "start", $params);
foreach my $drive (keys %{$res->{drives}}) {
$self->{stopnbd} = 1;
$self->{target_drive}->{$drive}->{drivestr} = $res->{drives}->{$drive}->{drivestr};
my $nbd_uri = $res->{drives}->{$drive}->{nbd_uri};
die "unexpected NBD uri for '$drive': $nbd_uri\n"
if $nbd_uri !~ s!/run/qemu-server/$remote_vmid\_!/run/qemu-server/$vmid\_!;
$self->{target_drive}->{$drive}->{nbd_uri} = $nbd_uri;
}
return ($res->{migrate}, $res->{spice_port});
}
sub phase2 { sub phase2 {
my ($self, $vmid) = @_; my ($self, $vmid) = @_;
my $conf = $self->{vmconf}; my $conf = $self->{vmconf};
my $local_volumes = $self->{local_volumes};
# version > 0 for unix socket support # version > 0 for unix socket support
my $nbd_protocol_version = 1; my $nbd_protocol_version = 1;
@ -855,10 +1077,39 @@ sub phase2 {
}, },
}; };
my ($tunnel_info, $spice_port) = $self->phase2_start_local_cluster($vmid, $params); my ($tunnel_info, $spice_port);
$self->log('info', "start remote tunnel"); my @online_local_volumes = $self->filter_local_volumes('online');
$self->start_remote_tunnel($tunnel_info); $self->{storage_migration} = 1 if scalar(@online_local_volumes);
if (my $remote = $self->{opts}->{remote}) {
my $remote_vmid = $remote->{vmid};
$params->{migrate_opts}->{remote_node} = $self->{node};
($tunnel_info, $spice_port) = $self->phase2_start_remote_cluster($vmid, $params);
die "only UNIX sockets are supported for remote migration\n"
if $tunnel_info->{proto} ne 'unix';
my $remote_socket = $tunnel_info->{addr};
my $local_socket = $remote_socket;
$local_socket =~ s/$remote_vmid/$vmid/g;
$tunnel_info->{addr} = $local_socket;
$self->log('info', "Setting up tunnel for '$local_socket'");
PVE::Tunnel::forward_unix_socket($self->{tunnel}, $local_socket, $remote_socket);
foreach my $remote_socket (@{$tunnel_info->{unix_sockets}}) {
my $local_socket = $remote_socket;
$local_socket =~ s/$remote_vmid/$vmid/g;
next if $self->{tunnel}->{forwarded}->{$local_socket};
$self->log('info', "Setting up tunnel for '$local_socket'");
PVE::Tunnel::forward_unix_socket($self->{tunnel}, $local_socket, $remote_socket);
}
} else {
($tunnel_info, $spice_port) = $self->phase2_start_local_cluster($vmid, $params);
$self->log('info', "start remote tunnel");
$self->start_remote_tunnel($tunnel_info);
}
my $migrate_uri = "$tunnel_info->{proto}:$tunnel_info->{addr}"; my $migrate_uri = "$tunnel_info->{proto}:$tunnel_info->{addr}";
$migrate_uri .= ":$tunnel_info->{port}" $migrate_uri .= ":$tunnel_info->{port}"
@ -868,8 +1119,6 @@ sub phase2 {
$self->{storage_migration_jobs} = {}; $self->{storage_migration_jobs} = {};
$self->log('info', "starting storage migration"); $self->log('info', "starting storage migration");
my @online_local_volumes = $self->filter_local_volumes('online');
die "The number of local disks does not match between the source and the destination.\n" die "The number of local disks does not match between the source and the destination.\n"
if (scalar(keys %{$self->{target_drive}}) != scalar(@online_local_volumes)); if (scalar(keys %{$self->{target_drive}}) != scalar(@online_local_volumes));
foreach my $drive (keys %{$self->{target_drive}}){ foreach my $drive (keys %{$self->{target_drive}}){
@ -901,7 +1150,8 @@ sub phase2 {
# migrate speed can be set via bwlimit (datacenter.cfg and API) and via the # migrate speed can be set via bwlimit (datacenter.cfg and API) and via the
# migrate_speed parameter in qm.conf - take the lower of the two. # migrate_speed parameter in qm.conf - take the lower of the two.
my $bwlimit = PVE::Storage::get_bandwidth_limit('migration', undef, $self->{opts}->{bwlimit}) // 0; my $bwlimit = $self->get_bwlimit();
my $migrate_speed = $conf->{migrate_speed} // 0; my $migrate_speed = $conf->{migrate_speed} // 0;
$migrate_speed *= 1024; # migrate_speed is in MB/s, bwlimit in KB/s $migrate_speed *= 1024; # migrate_speed is in MB/s, bwlimit in KB/s
@ -942,7 +1192,7 @@ sub phase2 {
}; };
$self->log('info', "migrate-set-parameters error: $@") if $@; $self->log('info', "migrate-set-parameters error: $@") if $@;
if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) { if (PVE::QemuServer::vga_conf_has_spice($conf->{vga}) && !$self->{opts}->{remote}) {
my $rpcenv = PVE::RPCEnvironment::get(); my $rpcenv = PVE::RPCEnvironment::get();
my $authuser = $rpcenv->get_user(); my $authuser = $rpcenv->get_user();
@ -1155,11 +1405,15 @@ sub phase2_cleanup {
my $nodename = PVE::INotify::nodename(); my $nodename = PVE::INotify::nodename();
my $cmd = [@{$self->{rem_ssh}}, 'qm', 'stop', $vmid, '--skiplock', '--migratedfrom', $nodename]; if ($self->{tunnel} && $self->{tunnel}->{version} >= 2) {
eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) }; PVE::Tunnel::write_tunnel($self->{tunnel}, 10, 'stop');
if (my $err = $@) { } else {
$self->log('err', $err); my $cmd = [@{$self->{rem_ssh}}, 'qm', 'stop', $vmid, '--skiplock', '--migratedfrom', $nodename];
$self->{errors} = 1; eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
if (my $err = $@) {
$self->log('err', $err);
$self->{errors} = 1;
}
} }
# cleanup after stopping, otherwise disks might be in-use by target VM! # cleanup after stopping, otherwise disks might be in-use by target VM!
@ -1192,7 +1446,7 @@ sub phase3_cleanup {
my $tunnel = $self->{tunnel}; my $tunnel = $self->{tunnel};
if ($self->{volume_map}) { if ($self->{volume_map} && !$self->{opts}->{remote}) {
my $target_drives = $self->{target_drive}; my $target_drives = $self->{target_drive};
# FIXME: for NBD storage migration we now only update the volid, and # FIXME: for NBD storage migration we now only update the volid, and
@ -1208,20 +1462,26 @@ sub phase3_cleanup {
} }
# transfer replication state before move config # transfer replication state before move config
$self->transfer_replication_state() if $self->{is_replicated}; if (!$self->{opts}->{remote}) {
PVE::QemuConfig->move_config_to_node($vmid, $self->{node}); $self->transfer_replication_state() if $self->{is_replicated};
$self->switch_replication_job_target() if $self->{is_replicated}; PVE::QemuConfig->move_config_to_node($vmid, $self->{node});
$self->switch_replication_job_target() if $self->{is_replicated};
}
if ($self->{livemigration}) { if ($self->{livemigration}) {
if ($self->{stopnbd}) { if ($self->{stopnbd}) {
$self->log('info', "stopping NBD storage migration server on target."); $self->log('info', "stopping NBD storage migration server on target.");
# stop nbd server on remote vm - requirement for resume since 2.9 # stop nbd server on remote vm - requirement for resume since 2.9
my $cmd = [@{$self->{rem_ssh}}, 'qm', 'nbdstop', $vmid]; if ($tunnel && $tunnel->{version} && $tunnel->{version} >= 2) {
PVE::Tunnel::write_tunnel($tunnel, 30, 'nbdstop');
} else {
my $cmd = [@{$self->{rem_ssh}}, 'qm', 'nbdstop', $vmid];
eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) }; eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
if (my $err = $@) { if (my $err = $@) {
$self->log('err', $err); $self->log('err', $err);
$self->{errors} = 1; $self->{errors} = 1;
}
} }
} }
@ -1231,8 +1491,9 @@ sub phase3_cleanup {
if (!$self->{vm_was_paused}) { if (!$self->{vm_was_paused}) {
# config moved and nbd server stopped - now we can resume vm on target # config moved and nbd server stopped - now we can resume vm on target
if ($tunnel && $tunnel->{version} && $tunnel->{version} >= 1) { if ($tunnel && $tunnel->{version} && $tunnel->{version} >= 1) {
my $cmd = $tunnel->{version} == 1 ? "resume $vmid" : "resume";
eval { eval {
PVE::Tunnel::write_tunnel($tunnel, 30, "resume $vmid"); PVE::Tunnel::write_tunnel($tunnel, 30, $cmd);
}; };
if (my $err = $@) { if (my $err = $@) {
$self->log('err', $err); $self->log('err', $err);
@ -1259,11 +1520,15 @@ sub phase3_cleanup {
) { ) {
if (!$self->{vm_was_paused}) { if (!$self->{vm_was_paused}) {
$self->log('info', "issuing guest fstrim"); $self->log('info', "issuing guest fstrim");
my $cmd = [@{$self->{rem_ssh}}, 'qm', 'guest', 'cmd', $vmid, 'fstrim']; if ($self->{opts}->{remote}) {
eval { PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) }; PVE::Tunnel::write_tunnel($self->{tunnel}, 600, 'fstrim');
if (my $err = $@) { } else {
$self->log('err', "fstrim failed - $err"); my $cmd = [@{$self->{rem_ssh}}, 'qm', 'guest', 'cmd', $vmid, 'fstrim'];
$self->{errors} = 1; eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
if (my $err = $@) {
$self->log('err', "fstrim failed - $err");
$self->{errors} = 1;
}
} }
} else { } else {
$self->log('info', "skipping guest fstrim, because VM is paused"); $self->log('info', "skipping guest fstrim, because VM is paused");
@ -1272,12 +1537,14 @@ sub phase3_cleanup {
} }
# close tunnel on successful migration, on error phase2_cleanup closed it # close tunnel on successful migration, on error phase2_cleanup closed it
if ($tunnel) { if ($tunnel && $tunnel->{version} == 1) {
eval { PVE::Tunnel::finish_tunnel($tunnel); }; eval { PVE::Tunnel::finish_tunnel($tunnel); };
if (my $err = $@) { if (my $err = $@) {
$self->log('err', $err); $self->log('err', $err);
$self->{errors} = 1; $self->{errors} = 1;
} }
$tunnel = undef;
delete $self->{tunnel};
} }
eval { eval {
@ -1315,6 +1582,9 @@ sub phase3_cleanup {
# destroy local copies # destroy local copies
foreach my $volid (@not_replicated_volumes) { foreach my $volid (@not_replicated_volumes) {
# remote is cleaned up below
next if $self->{opts}->{remote};
eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); }; eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
if (my $err = $@) { if (my $err = $@) {
$self->log('err', "removing local copy of '$volid' failed - $err"); $self->log('err', "removing local copy of '$volid' failed - $err");
@ -1324,8 +1594,19 @@ sub phase3_cleanup {
} }
# clear migrate lock # clear migrate lock
my $cmd = [ @{$self->{rem_ssh}}, 'qm', 'unlock', $vmid ]; if ($tunnel && $tunnel->{version} >= 2) {
$self->cmd_logerr($cmd, errmsg => "failed to clear migrate lock"); PVE::Tunnel::write_tunnel($tunnel, 10, "unlock");
PVE::Tunnel::finish_tunnel($tunnel);
} else {
my $cmd = [ @{$self->{rem_ssh}}, 'qm', 'unlock', $vmid ];
$self->cmd_logerr($cmd, errmsg => "failed to clear migrate lock");
}
if ($self->{opts}->{remote} && $self->{opts}->{delete}) {
eval { PVE::QemuServer::destroy_vm($self->{storecfg}, $vmid, 1, undef, 0) };
warn "Failed to remove source VM - $@\n" if $@;
}
} }
sub final_cleanup { sub final_cleanup {

7
PVE/QemuServer.pm
View File

@ -5688,7 +5688,10 @@ sub vm_start_nolock {
my $defaults = load_defaults(); my $defaults = load_defaults();
# set environment variable useful inside network script # set environment variable useful inside network script
$ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom; # for remote migration the config is available on the target node!
if (!$migrate_opts->{remote_node}) {
$ENV{PVE_MIGRATED_FROM} = $migratedfrom;
}
PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1); PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
@ -5938,7 +5941,7 @@ sub vm_start_nolock {
my $migrate_storage_uri; my $migrate_storage_uri;
# nbd_protocol_version > 0 for unix socket support # nbd_protocol_version > 0 for unix socket support
if ($nbd_protocol_version > 0 && $migration_type eq 'secure') { if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate"; my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } ); mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
$migrate_storage_uri = "nbd:unix:$socket_path"; $migrate_storage_uri = "nbd:unix:$socket_path";

3
debian/control vendored
View File

@ -37,11 +37,12 @@ Depends: dbus,
libpve-cluster-perl, libpve-cluster-perl,
libpve-common-perl (>= 7.2-5), libpve-common-perl (>= 7.2-5),
libpve-guest-common-perl (>= 4.2-2), libpve-guest-common-perl (>= 4.2-2),
libpve-storage-perl (>= 6.3-8), libpve-storage-perl (>= 7.2-10),
libterm-readline-gnu-perl, libterm-readline-gnu-perl,
libuuid-perl, libuuid-perl,
libxml-libxml-perl, libxml-libxml-perl,
perl (>= 5.10.0-19), perl (>= 5.10.0-19),
proxmox-websocket-tunnel,
pve-cluster, pve-cluster,
pve-edk2-firmware (>= 3.20210831-1), pve-edk2-firmware (>= 3.20210831-1),
pve-firewall, pve-firewall,