From 20a519a05a8a135caa4569c3bf5f6a53c4de5f3e Mon Sep 17 00:00:00 2001 From: Juan Quintela Date: Thu, 20 Apr 2017 14:48:46 +0200 Subject: [PATCH 1/5] migration: Create savevm.h for functions exported from savevm.c This removes last trace of migration functions from sysemu/sysemu.h. Signed-off-by: Juan Quintela Reviewed-by: Laurent Vivier --- include/sysemu/sysemu.h | 47 ---------------------------------------- migration/colo.c | 1 + migration/migration.c | 1 + migration/postcopy-ram.c | 1 + migration/savevm.c | 29 +++++++++++++++++++++---- migration/savevm.h | 41 +++++++++++++++++++++++++++++++++++ 6 files changed, 69 insertions(+), 51 deletions(-) create mode 100644 migration/savevm.h diff --git a/include/sysemu/sysemu.h b/include/sysemu/sysemu.h index 69046ebf1b..723c8dcb1a 100644 --- a/include/sysemu/sysemu.h +++ b/include/sysemu/sysemu.h @@ -97,53 +97,6 @@ int load_vmstate(const char *name, Error **errp); void qemu_announce_self(void); -/* Subcommands for QEMU_VM_COMMAND */ -enum qemu_vm_cmd { - MIG_CMD_INVALID = 0, /* Must be 0 */ - MIG_CMD_OPEN_RETURN_PATH, /* Tell the dest to open the Return path */ - MIG_CMD_PING, /* Request a PONG on the RP */ - - MIG_CMD_POSTCOPY_ADVISE, /* Prior to any page transfers, just - warn we might want to do PC */ - MIG_CMD_POSTCOPY_LISTEN, /* Start listening for incoming - pages as it's running. */ - MIG_CMD_POSTCOPY_RUN, /* Start execution */ - - MIG_CMD_POSTCOPY_RAM_DISCARD, /* A list of pages to discard that - were previously sent during - precopy but are dirty. */ - MIG_CMD_PACKAGED, /* Send a wrapped stream within this stream */ - MIG_CMD_MAX -}; - -#define MAX_VM_CMD_PACKAGED_SIZE (1ul << 24) - -bool qemu_savevm_state_blocked(Error **errp); -void qemu_savevm_state_begin(QEMUFile *f); -void qemu_savevm_state_header(QEMUFile *f); -int qemu_savevm_state_iterate(QEMUFile *f, bool postcopy); -void qemu_savevm_state_cleanup(void); -void qemu_savevm_state_complete_postcopy(QEMUFile *f); -void qemu_savevm_state_complete_precopy(QEMUFile *f, bool iterable_only); -void qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size, - uint64_t *res_non_postcopiable, - uint64_t *res_postcopiable); -void qemu_savevm_command_send(QEMUFile *f, enum qemu_vm_cmd command, - uint16_t len, uint8_t *data); -void qemu_savevm_send_ping(QEMUFile *f, uint32_t value); -void qemu_savevm_send_open_return_path(QEMUFile *f); -int qemu_savevm_send_packaged(QEMUFile *f, const uint8_t *buf, size_t len); -void qemu_savevm_send_postcopy_advise(QEMUFile *f); -void qemu_savevm_send_postcopy_listen(QEMUFile *f); -void qemu_savevm_send_postcopy_run(QEMUFile *f); - -void qemu_savevm_send_postcopy_ram_discard(QEMUFile *f, const char *name, - uint16_t len, - uint64_t *start_list, - uint64_t *length_list); - -int qemu_loadvm_state(QEMUFile *f); - extern int autostart; typedef enum { diff --git a/migration/colo.c b/migration/colo.c index 8c13a3c3f1..3dd1390573 100644 --- a/migration/colo.c +++ b/migration/colo.c @@ -16,6 +16,7 @@ #include "qemu-file-channel.h" #include "migration/migration.h" #include "migration/qemu-file.h" +#include "savevm.h" #include "migration/colo.h" #include "migration/block.h" #include "io/channel-buffer.h" diff --git a/migration/migration.c b/migration/migration.c index ad29e53400..7087d1abbb 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -19,6 +19,7 @@ #include "qemu/main-loop.h" #include "migration/blocker.h" #include "migration/migration.h" +#include "savevm.h" #include "qemu-file-channel.h" #include "migration/qemu-file.h" #include "migration/vmstate.h" diff --git a/migration/postcopy-ram.c b/migration/postcopy-ram.c index a37620dac6..3f9ae1bff2 100644 --- a/migration/postcopy-ram.c +++ b/migration/postcopy-ram.c @@ -22,6 +22,7 @@ #include "exec/target_page.h" #include "migration/migration.h" #include "migration/qemu-file.h" +#include "savevm.h" #include "postcopy-ram.h" #include "sysemu/sysemu.h" #include "sysemu/balloon.h" diff --git a/migration/savevm.c b/migration/savevm.c index a4532b6b58..a2d4f9c53c 100644 --- a/migration/savevm.c +++ b/migration/savevm.c @@ -36,6 +36,7 @@ #include "qemu/timer.h" #include "migration/migration.h" #include "qemu-file-channel.h" +#include "savevm.h" #include "postcopy-ram.h" #include "qapi/qmp/qerror.h" #include "qemu/error-report.h" @@ -63,6 +64,26 @@ const unsigned int postcopy_ram_discard_version = 0; static bool skip_section_footers; +/* Subcommands for QEMU_VM_COMMAND */ +enum qemu_vm_cmd { + MIG_CMD_INVALID = 0, /* Must be 0 */ + MIG_CMD_OPEN_RETURN_PATH, /* Tell the dest to open the Return path */ + MIG_CMD_PING, /* Request a PONG on the RP */ + + MIG_CMD_POSTCOPY_ADVISE, /* Prior to any page transfers, just + warn we might want to do PC */ + MIG_CMD_POSTCOPY_LISTEN, /* Start listening for incoming + pages as it's running. */ + MIG_CMD_POSTCOPY_RUN, /* Start execution */ + + MIG_CMD_POSTCOPY_RAM_DISCARD, /* A list of pages to discard that + were previously sent during + precopy but are dirty. */ + MIG_CMD_PACKAGED, /* Send a wrapped stream within this stream */ + MIG_CMD_MAX +}; + +#define MAX_VM_CMD_PACKAGED_SIZE (1ul << 24) static struct mig_cmd_args { ssize_t len; /* -1 = variable */ const char *name; @@ -807,10 +828,10 @@ static void save_section_footer(QEMUFile *f, SaveStateEntry *se) * @len: Length of associated data * @data: Data associated with command. */ -void qemu_savevm_command_send(QEMUFile *f, - enum qemu_vm_cmd command, - uint16_t len, - uint8_t *data) +static void qemu_savevm_command_send(QEMUFile *f, + enum qemu_vm_cmd command, + uint16_t len, + uint8_t *data) { trace_savevm_command_send(command, len); qemu_put_byte(f, QEMU_VM_COMMAND); diff --git a/migration/savevm.h b/migration/savevm.h new file mode 100644 index 0000000000..eb4487771a --- /dev/null +++ b/migration/savevm.h @@ -0,0 +1,41 @@ +/* + * QEMU save vm functions + * + * Copyright (c) 2003-2008 Fabrice Bellard + * Copyright (c) 2009-2017 Red Hat Inc + * + * Authors: + * Juan Quintela + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#ifndef MIGRATION_SAVEVM_H +#define MIGRATION_SAVEVM_H + +bool qemu_savevm_state_blocked(Error **errp); +void qemu_savevm_state_begin(QEMUFile *f); +void qemu_savevm_state_header(QEMUFile *f); +int qemu_savevm_state_iterate(QEMUFile *f, bool postcopy); +void qemu_savevm_state_cleanup(void); +void qemu_savevm_state_complete_postcopy(QEMUFile *f); +void qemu_savevm_state_complete_precopy(QEMUFile *f, bool iterable_only); +void qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size, + uint64_t *res_non_postcopiable, + uint64_t *res_postcopiable); +void qemu_savevm_send_ping(QEMUFile *f, uint32_t value); +void qemu_savevm_send_open_return_path(QEMUFile *f); +int qemu_savevm_send_packaged(QEMUFile *f, const uint8_t *buf, size_t len); +void qemu_savevm_send_postcopy_advise(QEMUFile *f); +void qemu_savevm_send_postcopy_listen(QEMUFile *f); +void qemu_savevm_send_postcopy_run(QEMUFile *f); + +void qemu_savevm_send_postcopy_ram_discard(QEMUFile *f, const char *name, + uint16_t len, + uint64_t *start_list, + uint64_t *length_list); + +int qemu_loadvm_state(QEMUFile *f); + +#endif From 9884db2814fbf1eb2ed99e02dadf58534d3ecc25 Mon Sep 17 00:00:00 2001 From: Felipe Franciosi Date: Wed, 24 May 2017 17:10:00 +0100 Subject: [PATCH 2/5] migration: keep bytes_xfer_prev init'd to zero The first time migration_bitmap_sync() is called, bytes_xfer_prev is set to ram_state.bytes_transferred which is, at this point, zero. The next time migration_bitmap_sync() is called, an iteration has happened and bytes_xfer_prev is set to 'x' bytes. Most likely, more than one second has passed, so the auto converge logic will be triggered and bytes_xfer_now will also be set to 'x' bytes. This condition is currently masked by dirty_rate_high_cnt, which will wait for a few iterations before throttling. It would otherwise always assume zero bytes have been copied and therefore throttle the guest (possibly) prematurely. Given bytes_xfer_prev is only used by the auto convergence logic, it makes sense to only set its value after a check has been made against bytes_xfer_now. Signed-off-by: Felipe Franciosi Reviewed-by: Peter Xu Reviewed-by: Juan Quintela Signed-off-by: Juan Quintela --- migration/ram.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/migration/ram.c b/migration/ram.c index c07a9c08d9..36bf720c27 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -673,10 +673,6 @@ static void migration_bitmap_sync(RAMState *rs) rs->bitmap_sync_count++; - if (!rs->bytes_xfer_prev) { - rs->bytes_xfer_prev = ram_bytes_transferred(); - } - if (!rs->time_last_bitmap_sync) { rs->time_last_bitmap_sync = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); } From d693c6f10ff46c661b055288abae11deb6181a61 Mon Sep 17 00:00:00 2001 From: Felipe Franciosi Date: Wed, 24 May 2017 17:10:01 +0100 Subject: [PATCH 3/5] migration: set dirty_pages_rate before autoconverge logic Currently, a "period" in the RAM migration logic is at least a second long and accounts for what happened since the last period (or the beginning of the migration). The dirty_pages_rate counter is calculated at the end this logic. If the auto convergence capability is enabled from the start of the migration, it won't be able to use this counter the first time around. This calculates dirty_pages_rate as soon as a period is deemed over, which allows for it to be used immediately. Signed-off-by: Felipe Franciosi Reviewed-by: Peter Xu Reviewed-by: Juan Quintela Signed-off-by: Juan Quintela --- migration/ram.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/migration/ram.c b/migration/ram.c index 36bf720c27..495ecbe492 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -694,6 +694,10 @@ static void migration_bitmap_sync(RAMState *rs) /* more than 1 second = 1000 millisecons */ if (end_time > rs->time_last_bitmap_sync + 1000) { + /* calculate period counters */ + rs->dirty_pages_rate = rs->num_dirty_pages_period * 1000 + / (end_time - rs->time_last_bitmap_sync); + if (migrate_auto_converge()) { /* The following detection logic can be refined later. For now: Check to see if the dirtied bytes is 50% more than the approx. @@ -702,15 +706,14 @@ static void migration_bitmap_sync(RAMState *rs) throttling */ bytes_xfer_now = ram_bytes_transferred(); - if (rs->dirty_pages_rate && - (rs->num_dirty_pages_period * TARGET_PAGE_SIZE > + if ((rs->num_dirty_pages_period * TARGET_PAGE_SIZE > (bytes_xfer_now - rs->bytes_xfer_prev) / 2) && - (rs->dirty_rate_high_cnt++ >= 2)) { + (rs->dirty_rate_high_cnt++ >= 2)) { trace_migration_throttle(); rs->dirty_rate_high_cnt = 0; mig_throttle_guest_down(); - } - rs->bytes_xfer_prev = bytes_xfer_now; + } + rs->bytes_xfer_prev = bytes_xfer_now; } if (migrate_use_xbzrle()) { @@ -723,8 +726,8 @@ static void migration_bitmap_sync(RAMState *rs) rs->iterations_prev = rs->iterations; rs->xbzrle_cache_miss_prev = rs->xbzrle_cache_miss; } - rs->dirty_pages_rate = rs->num_dirty_pages_period * 1000 - / (end_time - rs->time_last_bitmap_sync); + + /* reset period counters */ rs->time_last_bitmap_sync = end_time; rs->num_dirty_pages_period = 0; } From d2a4d85a8a786e1e59fc631c34dd474752810463 Mon Sep 17 00:00:00 2001 From: Felipe Franciosi Date: Wed, 24 May 2017 17:10:02 +0100 Subject: [PATCH 4/5] migration: set bytes_xfer_* outside of autoconverge logic The bytes_xfer_now/prev counters are only used by the auto convergence logic. However, they are used alongside the dirty_pages_rate counter, which is calculated (and required) outside of this logic. The problem with this approach is that if the auto convergence capability is changed while a migration is ongoing, the relationship of the counters will be broken. This moves the management of bytes_xfer_now/prev counters outside of the auto convergence logic to address this issue. Signed-off-by: Felipe Franciosi Reviewed-by: Peter Xu Reviewed-by: Juan Quintela Signed-off-by: Juan Quintela --- migration/ram.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/migration/ram.c b/migration/ram.c index 495ecbe492..1a3d9e6d87 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -697,6 +697,7 @@ static void migration_bitmap_sync(RAMState *rs) /* calculate period counters */ rs->dirty_pages_rate = rs->num_dirty_pages_period * 1000 / (end_time - rs->time_last_bitmap_sync); + bytes_xfer_now = ram_bytes_transferred(); if (migrate_auto_converge()) { /* The following detection logic can be refined later. For now: @@ -704,7 +705,6 @@ static void migration_bitmap_sync(RAMState *rs) amount of bytes that just got transferred since the last time we were in this routine. If that happens twice, start or increase throttling */ - bytes_xfer_now = ram_bytes_transferred(); if ((rs->num_dirty_pages_period * TARGET_PAGE_SIZE > (bytes_xfer_now - rs->bytes_xfer_prev) / 2) && @@ -713,7 +713,6 @@ static void migration_bitmap_sync(RAMState *rs) rs->dirty_rate_high_cnt = 0; mig_throttle_guest_down(); } - rs->bytes_xfer_prev = bytes_xfer_now; } if (migrate_use_xbzrle()) { @@ -730,6 +729,7 @@ static void migration_bitmap_sync(RAMState *rs) /* reset period counters */ rs->time_last_bitmap_sync = end_time; rs->num_dirty_pages_period = 0; + rs->bytes_xfer_prev = bytes_xfer_now; } if (migrate_use_events()) { qapi_event_send_migration_pass(rs->bitmap_sync_count, NULL); From b4a3c64b16856a018869bfd4a9ed3b2a74554541 Mon Sep 17 00:00:00 2001 From: Felipe Franciosi Date: Wed, 24 May 2017 17:10:03 +0100 Subject: [PATCH 5/5] migration: use dirty_rate_high_cnt more aggressively The commit message from 070afca25 suggests that dirty_rate_high_cnt should be used more aggressively to start throttling after two iterations instead of four. The code, however, only changes the auto convergence behaviour to throttle after three iterations. This makes the behaviour more aggressive by kicking off throttling after two iterations as originally intended. Signed-off-by: Felipe Franciosi Reviewed-by: Peter Xu Reviewed-by: Juan Quintela Signed-off-by: Juan Quintela --- migration/ram.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migration/ram.c b/migration/ram.c index 1a3d9e6d87..26e03a5dfa 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -708,7 +708,7 @@ static void migration_bitmap_sync(RAMState *rs) if ((rs->num_dirty_pages_period * TARGET_PAGE_SIZE > (bytes_xfer_now - rs->bytes_xfer_prev) / 2) && - (rs->dirty_rate_high_cnt++ >= 2)) { + (++rs->dirty_rate_high_cnt >= 2)) { trace_migration_throttle(); rs->dirty_rate_high_cnt = 0; mig_throttle_guest_down();