From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Stefan Reiter Date: Mon, 29 Jun 2020 11:06:03 +0200 Subject: [PATCH] PVE-Backup: Add dirty-bitmap tracking for incremental backups Uses QEMU's existing MIRROR_SYNC_MODE_BITMAP and a dirty-bitmap on top of all backed-up drives. This will only execute the data-write callback for any changed chunks, the PBS rust code will reuse chunks from the previous index for everything it doesn't receive if reuse_index is true. On error or cancellation, remove all dirty bitmaps to ensure consistency. Add PBS/incremental specific information to query backup info QMP and HMP commands. Only supported for PBS backups. Signed-off-by: Stefan Reiter Signed-off-by: Dietmar Maurer Signed-off-by: Thomas Lamprecht --- block/monitor/block-hmp-cmds.c | 1 + monitor/hmp-cmds.c | 45 ++++++++++++---- proxmox-backup-client.c | 3 +- proxmox-backup-client.h | 1 + pve-backup.c | 95 ++++++++++++++++++++++++++++++---- qapi/block-core.json | 12 ++++- 6 files changed, 134 insertions(+), 23 deletions(-) diff --git a/block/monitor/block-hmp-cmds.c b/block/monitor/block-hmp-cmds.c index 9ba7c774a2..056d14deee 100644 --- a/block/monitor/block-hmp-cmds.c +++ b/block/monitor/block-hmp-cmds.c @@ -1039,6 +1039,7 @@ void hmp_backup(Monitor *mon, const QDict *qdict) false, NULL, // PBS fingerprint false, NULL, // PBS backup-id false, 0, // PBS backup-time + false, false, // PBS incremental true, dir ? BACKUP_FORMAT_DIR : BACKUP_FORMAT_VMA, false, NULL, false, NULL, !!devlist, devlist, qdict_haskey(qdict, "speed"), speed, &error); diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c index 182e79c943..604026bb37 100644 --- a/monitor/hmp-cmds.c +++ b/monitor/hmp-cmds.c @@ -221,19 +221,42 @@ void hmp_info_backup(Monitor *mon, const QDict *qdict) monitor_printf(mon, "End time: %s", ctime(&info->end_time)); } - int per = (info->has_total && info->total && - info->has_transferred && info->transferred) ? - (info->transferred * 100)/info->total : 0; - int zero_per = (info->has_total && info->total && - info->has_zero_bytes && info->zero_bytes) ? - (info->zero_bytes * 100)/info->total : 0; monitor_printf(mon, "Backup file: %s\n", info->backup_file); monitor_printf(mon, "Backup uuid: %s\n", info->uuid); - monitor_printf(mon, "Total size: %zd\n", info->total); - monitor_printf(mon, "Transferred bytes: %zd (%d%%)\n", - info->transferred, per); - monitor_printf(mon, "Zero bytes: %zd (%d%%)\n", - info->zero_bytes, zero_per); + + if (!(info->has_total && info->total)) { + // this should not happen normally + monitor_printf(mon, "Total size: %d\n", 0); + } else { + bool incremental = false; + size_t total_or_dirty = info->total; + if (info->has_transferred) { + if (info->has_dirty && info->dirty) { + if (info->dirty < info->total) { + total_or_dirty = info->dirty; + incremental = true; + } + } + } + + int per = (info->transferred * 100)/total_or_dirty; + + monitor_printf(mon, "Backup mode: %s\n", incremental ? "incremental" : "full"); + + int zero_per = (info->has_zero_bytes && info->zero_bytes) ? + (info->zero_bytes * 100)/info->total : 0; + monitor_printf(mon, "Total size: %zd\n", info->total); + monitor_printf(mon, "Transferred bytes: %zd (%d%%)\n", + info->transferred, per); + monitor_printf(mon, "Zero bytes: %zd (%d%%)\n", + info->zero_bytes, zero_per); + + if (info->has_reused) { + int reused_per = (info->reused * 100)/total_or_dirty; + monitor_printf(mon, "Reused bytes: %zd (%d%%)\n", + info->reused, reused_per); + } + } } qapi_free_BackupStatus(info); diff --git a/proxmox-backup-client.c b/proxmox-backup-client.c index a8f6653a81..4ce7bc0b5e 100644 --- a/proxmox-backup-client.c +++ b/proxmox-backup-client.c @@ -89,6 +89,7 @@ proxmox_backup_co_register_image( ProxmoxBackupHandle *pbs, const char *device_name, uint64_t size, + bool incremental, Error **errp) { Coroutine *co = qemu_coroutine_self(); @@ -98,7 +99,7 @@ proxmox_backup_co_register_image( int pbs_res = -1; proxmox_backup_register_image_async( - pbs, device_name, size ,proxmox_backup_schedule_wake, &waker, &pbs_res, &pbs_err); + pbs, device_name, size, incremental, proxmox_backup_schedule_wake, &waker, &pbs_res, &pbs_err); qemu_coroutine_yield(); if (pbs_res < 0) { if (errp) error_setg(errp, "backup register image failed: %s", pbs_err ? pbs_err : "unknown error"); diff --git a/proxmox-backup-client.h b/proxmox-backup-client.h index 1dda8b7d8f..8cbf645b2c 100644 --- a/proxmox-backup-client.h +++ b/proxmox-backup-client.h @@ -32,6 +32,7 @@ proxmox_backup_co_register_image( ProxmoxBackupHandle *pbs, const char *device_name, uint64_t size, + bool incremental, Error **errp); diff --git a/pve-backup.c b/pve-backup.c index d40f3f2fd6..d50f03a050 100644 --- a/pve-backup.c +++ b/pve-backup.c @@ -28,6 +28,8 @@ * */ +const char *PBS_BITMAP_NAME = "pbs-incremental-dirty-bitmap"; + static struct PVEBackupState { struct { // Everithing accessed from qmp_backup_query command is protected using lock @@ -39,7 +41,9 @@ static struct PVEBackupState { uuid_t uuid; char uuid_str[37]; size_t total; + size_t dirty; size_t transferred; + size_t reused; size_t zero_bytes; } stat; int64_t speed; @@ -66,6 +70,7 @@ typedef struct PVEBackupDevInfo { uint8_t dev_id; bool completed; char targetfile[PATH_MAX]; + BdrvDirtyBitmap *bitmap; BlockDriverState *target; } PVEBackupDevInfo; @@ -105,11 +110,12 @@ static bool pvebackup_error_or_canceled(void) return error_or_canceled; } -static void pvebackup_add_transfered_bytes(size_t transferred, size_t zero_bytes) +static void pvebackup_add_transfered_bytes(size_t transferred, size_t zero_bytes, size_t reused) { qemu_mutex_lock(&backup_state.stat.lock); backup_state.stat.zero_bytes += zero_bytes; backup_state.stat.transferred += transferred; + backup_state.stat.reused += reused; qemu_mutex_unlock(&backup_state.stat.lock); } @@ -148,7 +154,8 @@ pvebackup_co_dump_pbs_cb( pvebackup_propagate_error(local_err); return pbs_res; } else { - pvebackup_add_transfered_bytes(size, !buf ? size : 0); + size_t reused = (pbs_res == 0) ? size : 0; + pvebackup_add_transfered_bytes(size, !buf ? size : 0, reused); } return size; @@ -208,11 +215,11 @@ pvebackup_co_dump_vma_cb( } else { if (remaining >= VMA_CLUSTER_SIZE) { assert(ret == VMA_CLUSTER_SIZE); - pvebackup_add_transfered_bytes(VMA_CLUSTER_SIZE, zero_bytes); + pvebackup_add_transfered_bytes(VMA_CLUSTER_SIZE, zero_bytes, 0); remaining -= VMA_CLUSTER_SIZE; } else { assert(ret == remaining); - pvebackup_add_transfered_bytes(remaining, zero_bytes); + pvebackup_add_transfered_bytes(remaining, zero_bytes, 0); remaining = 0; } } @@ -248,6 +255,18 @@ static void coroutine_fn pvebackup_co_cleanup(void *unused) if (local_err != NULL) { pvebackup_propagate_error(local_err); } + } else { + // on error or cancel we cannot ensure synchronization of dirty + // bitmaps with backup server, so remove all and do full backup next + GList *l = backup_state.di_list; + while (l) { + PVEBackupDevInfo *di = (PVEBackupDevInfo *)l->data; + l = g_list_next(l); + + if (di->bitmap) { + bdrv_release_dirty_bitmap(di->bitmap); + } + } } proxmox_backup_disconnect(backup_state.pbs); @@ -303,6 +322,12 @@ static void pvebackup_complete_cb(void *opaque, int ret) // remove self from job queue backup_state.di_list = g_list_remove(backup_state.di_list, di); + if (di->bitmap && ret < 0) { + // on error or cancel we cannot ensure synchronization of dirty + // bitmaps with backup server, so remove all and do full backup next + bdrv_release_dirty_bitmap(di->bitmap); + } + g_free(di); qemu_mutex_unlock(&backup_state.backup_mutex); @@ -470,12 +495,18 @@ static bool create_backup_jobs(void) { assert(di->target != NULL); + MirrorSyncMode sync_mode = MIRROR_SYNC_MODE_FULL; + BitmapSyncMode bitmap_mode = BITMAP_SYNC_MODE_NEVER; + if (di->bitmap) { + sync_mode = MIRROR_SYNC_MODE_BITMAP; + bitmap_mode = BITMAP_SYNC_MODE_ON_SUCCESS; + } AioContext *aio_context = bdrv_get_aio_context(di->bs); aio_context_acquire(aio_context); BlockJob *job = backup_job_create( - NULL, di->bs, di->target, backup_state.speed, MIRROR_SYNC_MODE_FULL, NULL, - BITMAP_SYNC_MODE_NEVER, false, NULL, BLOCKDEV_ON_ERROR_REPORT, BLOCKDEV_ON_ERROR_REPORT, + NULL, di->bs, di->target, backup_state.speed, sync_mode, di->bitmap, + bitmap_mode, false, NULL, BLOCKDEV_ON_ERROR_REPORT, BLOCKDEV_ON_ERROR_REPORT, JOB_DEFAULT, pvebackup_complete_cb, di, 1, NULL, &local_err); aio_context_release(aio_context); @@ -526,6 +557,8 @@ typedef struct QmpBackupTask { const char *fingerprint; bool has_fingerprint; int64_t backup_time; + bool has_incremental; + bool incremental; bool has_format; BackupFormat format; bool has_config_file; @@ -617,6 +650,7 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) } size_t total = 0; + size_t dirty = 0; l = di_list; while (l) { @@ -654,6 +688,8 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) int dump_cb_block_size = PROXMOX_BACKUP_DEFAULT_CHUNK_SIZE; // Hardcoded (4M) firewall_name = "fw.conf"; + bool incremental = task->has_incremental && task->incremental; + char *pbs_err = NULL; pbs = proxmox_backup_new( task->backup_file, @@ -673,7 +709,8 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) goto err; } - if (proxmox_backup_co_connect(pbs, task->errp) < 0) + int connect_result = proxmox_backup_co_connect(pbs, task->errp); + if (connect_result < 0) goto err; /* register all devices */ @@ -684,9 +721,32 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) const char *devname = bdrv_get_device_name(di->bs); - int dev_id = proxmox_backup_co_register_image(pbs, devname, di->size, task->errp); - if (dev_id < 0) + BdrvDirtyBitmap *bitmap = bdrv_find_dirty_bitmap(di->bs, PBS_BITMAP_NAME); + + bool use_incremental = false; + if (incremental) { + if (bitmap == NULL) { + bitmap = bdrv_create_dirty_bitmap(di->bs, dump_cb_block_size, PBS_BITMAP_NAME, task->errp); + if (!bitmap) { + goto err; + } + /* mark entire bitmap as dirty to make full backup first */ + bdrv_set_dirty_bitmap(bitmap, 0, di->size); + dirty += di->size; + } else { + use_incremental = true; + dirty += bdrv_get_dirty_count(bitmap); + } + di->bitmap = bitmap; + } else if (bitmap != NULL) { + dirty += di->size; + bdrv_release_dirty_bitmap(bitmap); + } + + int dev_id = proxmox_backup_co_register_image(pbs, devname, di->size, use_incremental, task->errp); + if (dev_id < 0) { goto err; + } if (!(di->target = bdrv_backup_dump_create(dump_cb_block_size, di->size, pvebackup_co_dump_pbs_cb, di, task->errp))) { goto err; @@ -695,6 +755,8 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) di->dev_id = dev_id; } } else if (format == BACKUP_FORMAT_VMA) { + dirty = total; + vmaw = vma_writer_create(task->backup_file, uuid, &local_err); if (!vmaw) { if (local_err) { @@ -722,6 +784,8 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) } } } else if (format == BACKUP_FORMAT_DIR) { + dirty = total; + if (mkdir(task->backup_file, 0640) != 0) { error_setg_errno(task->errp, errno, "can't create directory '%s'\n", task->backup_file); @@ -794,8 +858,10 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) char *uuid_str = g_strdup(backup_state.stat.uuid_str); backup_state.stat.total = total; + backup_state.stat.dirty = dirty; backup_state.stat.transferred = 0; backup_state.stat.zero_bytes = 0; + backup_state.stat.reused = dirty >= total ? 0 : total - dirty; qemu_mutex_unlock(&backup_state.stat.lock); @@ -819,6 +885,10 @@ err: PVEBackupDevInfo *di = (PVEBackupDevInfo *)l->data; l = g_list_next(l); + if (di->bitmap) { + bdrv_release_dirty_bitmap(di->bitmap); + } + if (di->target) { bdrv_unref(di->target); } @@ -860,6 +930,7 @@ UuidInfo *qmp_backup( bool has_fingerprint, const char *fingerprint, bool has_backup_id, const char *backup_id, bool has_backup_time, int64_t backup_time, + bool has_incremental, bool incremental, bool has_format, BackupFormat format, bool has_config_file, const char *config_file, bool has_firewall_file, const char *firewall_file, @@ -878,6 +949,8 @@ UuidInfo *qmp_backup( .backup_id = backup_id, .has_backup_time = has_backup_time, .backup_time = backup_time, + .has_incremental = has_incremental, + .incremental = incremental, .has_format = has_format, .format = format, .has_config_file = has_config_file, @@ -946,10 +1019,14 @@ BackupStatus *qmp_query_backup(Error **errp) info->has_total = true; info->total = backup_state.stat.total; + info->has_dirty = true; + info->dirty = backup_state.stat.dirty; info->has_zero_bytes = true; info->zero_bytes = backup_state.stat.zero_bytes; info->has_transferred = true; info->transferred = backup_state.stat.transferred; + info->has_reused = true; + info->reused = backup_state.stat.reused; qemu_mutex_unlock(&backup_state.stat.lock); diff --git a/qapi/block-core.json b/qapi/block-core.json index 9054db608c..aadd5329b3 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -758,8 +758,13 @@ # # @total: total amount of bytes involved in the backup process # +# @dirty: with incremental mode, this is the amount of bytes involved +# in the backup process which are marked dirty. +# # @transferred: amount of bytes already backed up. # +# @reused: amount of bytes reused due to deduplication. +# # @zero-bytes: amount of 'zero' bytes detected. # # @start-time: time (epoch) when backup job started. @@ -772,8 +777,8 @@ # ## { 'struct': 'BackupStatus', - 'data': {'*status': 'str', '*errmsg': 'str', '*total': 'int', - '*transferred': 'int', '*zero-bytes': 'int', + 'data': {'*status': 'str', '*errmsg': 'str', '*total': 'int', '*dirty': 'int', + '*transferred': 'int', '*zero-bytes': 'int', '*reused': 'int', '*start-time': 'int', '*end-time': 'int', '*backup-file': 'str', '*uuid': 'str' } } @@ -816,6 +821,8 @@ # # @backup-time: backup timestamp (Unix epoch, required for format 'pbs') # +# @incremental: sync incremental changes since last job (optional for format 'pbs') +# # Returns: the uuid of the backup job # ## @@ -826,6 +833,7 @@ '*fingerprint': 'str', '*backup-id': 'str', '*backup-time': 'int', + '*incremental': 'bool', '*format': 'BackupFormat', '*config-file': 'str', '*firewall-file': 'str',