From abc94cda91c919b63d77b6bbf052906bd6d5979b Mon Sep 17 00:00:00 2001 From: Thomas Lamprecht Date: Mon, 29 Jun 2020 14:17:13 +0200 Subject: [PATCH] import 14.2.10 upstream release Signed-off-by: Thomas Lamprecht --- ceph/CMakeLists.txt | 5 +- ceph/PendingReleaseNotes | 61 +- ceph/alpine/APKBUILD | 8 +- ceph/alpine/APKBUILD.in | 2 +- ceph/ceph.spec | 77 +- ceph/ceph.spec.in | 71 +- ceph/changelog.upstream | 10 +- ceph/cmake/modules/BuildFIO.cmake | 2 +- ceph/cmake/modules/CheckCxxAtomic.cmake | 25 +- ceph/cmake/modules/FindFUSE.cmake | 58 + ceph/cmake/modules/FindRDKafka.cmake | 33 + ceph/cmake/modules/FindSanitizers.cmake | 36 +- ceph/cmake/modules/Findfuse.cmake | 28 - ceph/debian/control | 5 +- ceph/debian/libcephfs-dev.install | 2 +- ceph/do_cmake.sh | 54 +- ceph/do_freebsd.sh | 1 + ceph/doc/cephfs/administration.rst | 3 + ceph/doc/cephfs/createfs.rst | 2 + ceph/doc/cephfs/file-layouts.rst | 4 +- ceph/doc/cephfs/fs-volumes.rst | 163 +- ceph/doc/cephfs/mds-config-ref.rst | 16 +- ceph/doc/cephfs/scrub.rst | 14 +- ceph/doc/man/8/rados.rst | 4 + ceph/doc/mgr/dashboard.rst | 19 +- ceph/doc/mgr/prometheus.rst | 10 + ceph/doc/mgr/telemetry.rst | 49 +- .../rados/configuration/mon-config-ref.rst | 8 + ceph/doc/rados/operations/health-checks.rst | 30 +- .../doc/rados/operations/placement-groups.rst | 51 +- ceph/doc/radosgw/bucketpolicy.rst | 2 +- ceph/doc/radosgw/frontends.rst | 9 + ceph/doc/radosgw/keystone.rst | 7 + ceph/doc/radosgw/multitenancy.rst | 7 + ceph/doc/radosgw/notifications.rst | 66 +- ceph/doc/radosgw/pubsub-module.rst | 118 +- .../radosgw/s3-notification-compatibility.rst | 12 +- ceph/doc/radosgw/s3/bucketops.rst | 33 +- ceph/doc/releases/nautilus.rst | 5 +- ceph/install-deps.sh | 26 +- .../grafana/dashboards/ceph-cluster.json | 1 + .../grafana/dashboards/host-details.json | 14 +- .../grafana/dashboards/hosts-overview.json | 10 +- .../dashboards/osd-device-details.json | 14 +- .../grafana/dashboards/pool-detail.json | 7 +- .../grafana/dashboards/radosgw-overview.json | 4 +- .../grafana/dashboards/rbd-details.json | 409 + .../prometheus/alerts/ceph_default_alerts.yml | 25 +- ceph/qa/.teuthology_branch | 1 + .../qa/cephfs/overrides/whitelist_health.yaml | 1 + .../tasks/cfuse_workunit_suites_ffsb.yaml | 1 + ceph/qa/distros/all/centos_7.8.yaml | 2 + ceph/qa/distros/all/rhel_7.8.yaml | 2 + .../supported-all-distro/centos_latest.yaml | 2 +- .../distros/supported-all-distro/rhel_7.yaml | 2 +- .../centos_latest.yaml | 2 +- .../supported-random-distro$/rhel_latest.yaml | 2 +- ceph/qa/distros/supported/centos_latest.yaml | 2 +- ceph/qa/distros/supported/rhel_latest.yaml | 2 +- ...tore-comp.yaml => bluestore-comp-lz4.yaml} | 1 + .../qa/objectstore/bluestore-comp-snappy.yaml | 24 + ceph/qa/objectstore/bluestore-comp-zlib.yaml | 24 + ceph/qa/objectstore/bluestore-comp-zstd.yaml | 24 + ceph/qa/overrides/short_pg_log.yaml | 1 + ceph/qa/run-standalone.sh | 6 +- .../erasure-code/test-erasure-eio.sh | 8 +- .../qa/standalone/mon/mon-last-epoch-clean.sh | 307 + .../qa/standalone/osd/ec-error-rollforward.sh | 14 +- ceph/qa/standalone/osd/osd-backfill-space.sh | 31 +- ceph/qa/standalone/osd/osd-backfill-stats.sh | 7 + .../standalone/osd/osd-bluefs-volume-ops.sh | 24 +- ceph/qa/standalone/osd/osd-dup.sh | 2 +- ceph/qa/standalone/osd/osd-recovery-stats.sh | 4 +- ceph/qa/standalone/osd/osd-rep-recov-eio.sh | 4 +- ceph/qa/standalone/osd/repro_long_log.sh | 2 +- ceph/qa/standalone/scrub/osd-scrub-repair.sh | 16 +- ceph/qa/standalone/scrub/osd-scrub-snaps.sh | 8 +- .../special/ceph_objectstore_tool.py | 8 +- .../tasks/cephfs_scrub_tests.yaml | 1 + .../tasks/volume-client/py/2.yaml | 2 - .../tasks/volume-client/py/3.yaml | 2 - .../client_trim_caps/tasks/trim-i22073.yaml | 1 + .../old_client/tasks/2-upgrade.yaml | 2 + .../upgraded_client/tasks/2-upgrade.yaml | 2 + .../fs/upgrade/snaps/tasks/2-upgrade.yaml | 1 + ceph/qa/suites/fs/verify/centos_latest.yaml | 1 + .../suites/fs/verify/validater/valgrind.yaml | 4 +- .../tasks/kclient_workunit_suites_ffsb.yaml | 1 + .../kclient_workunit_suites_ffsb.yaml | 1 + .../multimds/basic/0-supported-random-distro$ | 1 + .../thrash/0-supported-random-distro$ | 1 + .../suites/multimds/verify/centos_latest.yaml | 1 + .../workloads/cosbench_64K_write.yaml | 1 + .../workloads/fio_4K_rand_write.yaml | 1 + .../workloads/radosbench_4K_write.yaml | 1 + .../rados/mgr/tasks/module_selftest.yaml | 2 +- ceph/qa/suites/rados/monthrash/ceph.yaml | 1 + .../monthrash/workloads/rados_api_tests.yaml | 1 + .../multimon/tasks/mon_clock_with_skews.yaml | 1 + .../rados/multimon/tasks/mon_recovery.yaml | 2 + .../workloads/cosbench_64K_read_write.yaml | 1 + .../perf/workloads/cosbench_64K_write.yaml | 1 + .../perf/workloads/fio_4K_rand_read.yaml | 1 + .../rados/perf/workloads/fio_4K_rand_rw.yaml | 1 + .../perf/workloads/fio_4M_rand_read.yaml | 1 + .../rados/perf/workloads/fio_4M_rand_rw.yaml | 1 + .../perf/workloads/fio_4M_rand_write.yaml | 1 + .../workloads/radosbench_4K_rand_read.yaml | 1 + .../workloads/radosbench_4K_seq_read.yaml | 1 + .../workloads/radosbench_4M_rand_read.yaml | 1 + .../workloads/radosbench_4M_seq_read.yaml | 1 + .../perf/workloads/radosbench_4M_write.yaml | 1 + .../rados/perf/workloads/sample_fio.yaml | 1 + .../perf/workloads/sample_radosbench.yaml | 1 + .../objectstore/bluestore-comp-lz4.yaml | 1 + .../objectstore/bluestore-comp-snappy.yaml | 1 + .../objectstore/bluestore-comp.yaml | 1 - .../rados/singleton/all/ec-lost-unfound.yaml | 1 + .../singleton/all/lost-unfound-delete.yaml | 2 + .../rados/singleton/all/lost-unfound.yaml | 2 + ...mon-memory-target-compliance.yaml.disabled | 2 +- .../rados/singleton/all/osd-recovery.yaml | 2 + .../rados/singleton/all/pg-autoscaler.yaml | 2 +- .../singleton/all/recovery-preemption.yaml | 1 + .../singleton/all/thrash-backfill-full.yaml | 1 + .../rados/singleton/all/thrash-eio.yaml | 1 + .../workloads/radosbench.yaml | 8 + .../radosbench-high-concurrency.yaml | 49 + .../rados/verify/tasks/rados_api_tests.yaml | 3 + .../rados/verify/validater/valgrind.yaml | 9 +- .../rbd_python_api_tests_old_format.yaml | 1 + .../qa/suites/rbd/openstack/base/install.yaml | 7 - .../rbd/openstack/clusters/fixed-2.yaml | 9 - ceph/qa/suites/rbd/openstack/objectstore | 1 - .../rbd/openstack/supported-random-distro$ | 1 - .../workloads/devstack-tempest-gate.yaml | 51 - .../objectstore/bluestore-comp-snappy.yaml | 1 + .../objectstore/bluestore-comp.yaml | 1 - .../rgw/multisite/tasks/test_multi.yaml | 12 +- ceph/qa/suites/rgw/multisite/valgrind.yaml | 14 +- ceph/qa/suites/rgw/tempest/distro/.qa | 1 - .../tempest/distro/supported-random-distro$ | 1 - .../suites/rgw/tempest/distro/valgrind.yaml | 5 - .../suites/rgw/tempest/tasks/rgw_tempest.yaml | 5 +- ceph/qa/suites/rgw/tempest/ubuntu_latest.yaml | 1 + .../verify/striping$/stripe-equals-chunk.yaml | 7 + .../striping$/stripe-greater-than-chunk.yaml | 7 + .../qa/suites/rgw/verify/tasks/0-install.yaml | 8 +- ceph/qa/suites/rgw/verify/tasks/swift.yaml | 3 + .../suites/rgw/verify/validater/valgrind.yaml | 20 +- .../suites/smoke/basic/tasks/mon_thrash.yaml | 1 + .../smoke/basic/tasks/rados_api_tests.yaml | 1 + .../suites/smoke/basic/tasks/rados_bench.yaml | 1 + .../smoke/basic/tasks/rados_cache_snaps.yaml | 1 + .../smoke/basic/tasks/rados_ec_snaps.yaml | 1 + ceph/qa/suites/smoke/basic/tasks/rbd_fsx.yaml | 1 + .../nautilus-client-x}/.qa | 0 .../nautilus-client-x/rbd}/% | 0 .../nautilus-client-x/rbd}/.qa | 0 .../nautilus-client-x/rbd/0-cluster}/+ | 0 .../nautilus-client-x/rbd/0-cluster}/.qa | 0 .../rbd/0-cluster}/openstack.yaml | 2 +- .../rbd/0-cluster/start.yaml | 21 + .../nautilus-client-x/rbd/1-install}/.qa | 0 .../rbd/1-install/nautilus-client-x.yaml | 11 + .../nautilus-client-x/rbd/2-features}/.qa | 0 .../rbd/2-features/defaults.yaml | 6 + .../rbd/2-features/layering.yaml} | 1 + .../nautilus-client-x/rbd/3-workload/.qa | 1 + .../3-workload/rbd_notification_tests.yaml | 33 + .../nautilus-client-x/rbd/supported/.qa | 1 + .../rbd/supported/ubuntu_18.04.yaml | 1 + .../parallel/1-ceph-install/luminous.yaml | 1 + .../stress-split/1-ceph-install/luminous.yaml | 1 + .../mimic-x-singleton/1-install/mimic.yaml | 3 + .../mimic-x/parallel/0-cluster/start.yaml | 1 + .../parallel/1-ceph-install/mimic.yaml | 1 + .../stress-split/1-ceph-install/mimic.yaml | 1 + .../point-to-point-upgrade.yaml | 1 + .../0-cluster/start.yaml | 1 + ceph/qa/tasks/admin_socket.py | 35 +- ceph/qa/tasks/autotest.py | 16 +- ceph/qa/tasks/blktrace.py | 4 +- ceph/qa/tasks/cbt.py | 9 +- ceph/qa/tasks/ceph.conf.template | 2 +- ceph/qa/tasks/ceph.py | 274 +- ceph/qa/tasks/ceph_client.py | 2 +- ceph/qa/tasks/ceph_deploy.py | 53 +- ceph/qa/tasks/ceph_fuse.py | 3 +- ceph/qa/tasks/ceph_manager.py | 135 +- ceph/qa/tasks/ceph_objectstore_tool.py | 171 +- ceph/qa/tasks/ceph_test_case.py | 2 +- ceph/qa/tasks/cephfs/cephfs_test_case.py | 19 +- ceph/qa/tasks/cephfs/filesystem.py | 100 +- ceph/qa/tasks/cephfs/fuse_mount.py | 54 +- ceph/qa/tasks/cephfs/kernel_mount.py | 23 +- ceph/qa/tasks/cephfs/mount.py | 94 +- ceph/qa/tasks/cephfs/test_admin.py | 46 + ceph/qa/tasks/cephfs/test_cap_flush.py | 6 +- ceph/qa/tasks/cephfs/test_cephfs_shell.py | 7 +- ceph/qa/tasks/cephfs/test_client_limits.py | 10 +- ceph/qa/tasks/cephfs/test_client_recovery.py | 18 +- ceph/qa/tasks/cephfs/test_data_scan.py | 34 +- ceph/qa/tasks/cephfs/test_dump_tree.py | 2 +- ceph/qa/tasks/cephfs/test_exports.py | 14 +- ceph/qa/tasks/cephfs/test_failover.py | 11 +- ceph/qa/tasks/cephfs/test_forward_scrub.py | 12 +- ceph/qa/tasks/cephfs/test_full.py | 117 +- .../qa/tasks/cephfs/test_journal_migration.py | 13 +- ceph/qa/tasks/cephfs/test_misc.py | 12 +- ceph/qa/tasks/cephfs/test_openfiletable.py | 4 +- ceph/qa/tasks/cephfs/test_pool_perm.py | 4 +- ceph/qa/tasks/cephfs/test_quota.py | 2 +- ceph/qa/tasks/cephfs/test_recovery_pool.py | 8 +- ceph/qa/tasks/cephfs/test_scrub.py | 9 +- ceph/qa/tasks/cephfs/test_scrub_checks.py | 71 +- ceph/qa/tasks/cephfs/test_sessionmap.py | 27 +- ceph/qa/tasks/cephfs/test_snapshots.py | 89 +- ceph/qa/tasks/cephfs/test_strays.py | 46 +- ceph/qa/tasks/cephfs/test_volume_client.py | 32 +- ceph/qa/tasks/cephfs/test_volumes.py | 380 +- ceph/qa/tasks/cephfs_test_runner.py | 2 +- ceph/qa/tasks/check_counter.py | 3 +- ceph/qa/tasks/cram.py | 18 +- ceph/qa/tasks/create_verify_lfn_objects.py | 2 +- ceph/qa/tasks/devstack.py | 39 +- ceph/qa/tasks/die_on_err.py | 6 +- ceph/qa/tasks/divergent_priors.py | 6 +- ceph/qa/tasks/divergent_priors2.py | 24 +- ceph/qa/tasks/dnsmasq.py | 12 +- ceph/qa/tasks/dump_stuck.py | 5 +- ceph/qa/tasks/ec_lost_unfound.py | 10 +- ceph/qa/tasks/exec_on_cleanup.py | 5 +- ceph/qa/tasks/filestore_idempotent.py | 4 +- ceph/qa/tasks/fs.py | 7 +- ceph/qa/tasks/kclient.py | 4 +- ceph/qa/tasks/keystone.py | 94 +- ceph/qa/tasks/lost_unfound.py | 6 +- ceph/qa/tasks/manypools.py | 38 +- ceph/qa/tasks/mds_creation_failure.py | 7 +- ceph/qa/tasks/mds_thrash.py | 26 +- ceph/qa/tasks/mgr/dashboard/helper.py | 2 +- ceph/qa/tasks/mgr/dashboard/test_cephfs.py | 2 + ceph/qa/tasks/mgr/dashboard/test_ganesha.py | 5 +- ceph/qa/tasks/mgr/dashboard/test_host.py | 2 +- .../tasks/mgr/dashboard/test_perf_counters.py | 2 +- ceph/qa/tasks/mgr/dashboard/test_pool.py | 4 +- ceph/qa/tasks/mgr/dashboard/test_rbd.py | 4 +- .../tasks/mgr/dashboard/test_rbd_mirroring.py | 6 +- ceph/qa/tasks/mgr/dashboard/test_rgw.py | 14 +- ceph/qa/tasks/mgr/dashboard/test_settings.py | 1 - ceph/qa/tasks/mgr/mgr_test_case.py | 6 +- ceph/qa/tasks/mgr/test_crash.py | 15 +- ceph/qa/tasks/mgr/test_dashboard.py | 6 +- ceph/qa/tasks/mgr/test_failover.py | 2 +- ceph/qa/tasks/mgr/test_insights.py | 4 +- ceph/qa/tasks/mgr/test_module_selftest.py | 8 +- ceph/qa/tasks/mgr/test_orchestrator_cli.py | 3 +- ceph/qa/tasks/mgr/test_progress.py | 2 +- ceph/qa/tasks/mgr/test_prometheus.py | 5 +- ceph/qa/tasks/mgr/test_ssh_orchestrator.py | 5 +- ceph/qa/tasks/mon_clock_skew_check.py | 7 +- ceph/qa/tasks/mon_recovery.py | 6 +- ceph/qa/tasks/mon_thrash.py | 8 +- ceph/qa/tasks/multibench.py | 3 +- ceph/qa/tasks/netem.py | 30 +- ceph/qa/tasks/object_source_down.py | 6 +- ceph/qa/tasks/omapbench.py | 8 +- ceph/qa/tasks/openssl_keys.py | 5 +- ceph/qa/tasks/osd_backfill.py | 4 +- ceph/qa/tasks/osd_failsafe_enospc.py | 27 +- ceph/qa/tasks/osd_max_pg_per_osd.py | 6 +- ceph/qa/tasks/osd_recovery.py | 6 +- ceph/qa/tasks/peer.py | 8 +- ceph/qa/tasks/peering_speed_test.py | 2 +- ceph/qa/tasks/populate_rbd_pool.py | 2 +- ceph/qa/tasks/qemu.py | 44 +- ceph/qa/tasks/rados.py | 18 +- ceph/qa/tasks/radosbench.py | 17 +- ceph/qa/tasks/radosbenchsweep.py | 12 +- ceph/qa/tasks/radosgw_admin.py | 78 +- ceph/qa/tasks/radosgw_admin_rest.py | 7 +- ceph/qa/tasks/ragweed.py | 39 +- ceph/qa/tasks/rbd.py | 15 +- ceph/qa/tasks/rbd_fio.py | 16 +- ceph/qa/tasks/rbd_fsx.py | 4 +- ceph/qa/tasks/rbd_mirror.py | 2 +- ceph/qa/tasks/rbd_mirror_thrash.py | 2 - ceph/qa/tasks/rebuild_mondb.py | 14 +- ceph/qa/tasks/reg11184.py | 29 +- ceph/qa/tasks/rep_lost_unfound_delete.py | 9 +- ceph/qa/tasks/repair_test.py | 2 +- ceph/qa/tasks/resolve_stuck_peering.py | 8 +- ceph/qa/tasks/restart.py | 6 +- ceph/qa/tasks/rgw.py | 22 +- ceph/qa/tasks/rgw_logsocket.py | 12 +- ceph/qa/tasks/rgw_multisite.py | 15 +- ceph/qa/tasks/rgw_multisite_tests.py | 3 +- ceph/qa/tasks/s3a_hadoop.py | 11 +- ceph/qa/tasks/s3readwrite.py | 21 +- ceph/qa/tasks/s3roundtrip.py | 18 +- ceph/qa/tasks/s3tests.py | 43 +- ceph/qa/tasks/samba.py | 8 +- ceph/qa/tasks/scrub.py | 6 +- ceph/qa/tasks/scrub_test.py | 22 +- ceph/qa/tasks/swift.py | 21 +- ceph/qa/tasks/systemd.py | 65 +- ceph/qa/tasks/tempest.py | 40 +- ceph/qa/tasks/tests/test_devstack.py | 2 +- ceph/qa/tasks/tests/test_radosgw_admin.py | 8 +- ceph/qa/tasks/tgt.py | 2 +- ceph/qa/tasks/thrashosds-health.yaml | 2 +- ceph/qa/tasks/thrashosds.py | 4 +- ceph/qa/tasks/tox.py | 5 +- ceph/qa/tasks/util/rados.py | 2 +- ceph/qa/tasks/util/rgw.py | 9 +- ceph/qa/tasks/util/test/test_rados.py | 2 +- ceph/qa/tasks/util/workunit.py | 6 +- ceph/qa/tasks/vstart_runner.py | 108 +- ceph/qa/tasks/watch_notify_same_primary.py | 18 +- ceph/qa/tasks/watch_notify_stress.py | 9 +- ceph/qa/tasks/workunit.py | 18 +- ceph/qa/tox.ini | 11 +- ceph/qa/valgrind.supp | 4 +- .../workunits/fs/multiclient_sync_read_eof.py | 2 - ceph/qa/workunits/mon/caps.py | 38 +- ceph/qa/workunits/mon/pg_autoscaler.sh | 37 +- ceph/qa/workunits/mon/ping.py | 12 +- ceph/qa/workunits/mon/pool_ops.sh | 38 + .../rados/test_envlibrados_for_rocksdb.sh | 6 +- .../rados/test_large_omap_detection.py | 2 +- ceph/qa/workunits/rbd/rbd-nbd.sh | 15 +- ceph/qa/workunits/rbd/rbd_mirror_bootstrap.sh | 8 +- ceph/qa/workunits/rbd/rbd_mirror_helpers.sh | 25 +- ceph/qa/workunits/rbd/run_devstack_tempest.sh | 130 - ceph/qa/workunits/rest/test_mgr_rest_api.py | 2 +- ceph/qa/workunits/restart/test-backtraces.py | 10 +- ceph/selinux/ceph.te | 9 + ceph/src/.git_version | 4 +- ceph/src/CMakeLists.txt | 5 +- ceph/src/bash_completion/ceph | 2 +- ceph/src/ceph-volume/ceph_volume/api/lvm.py | 41 +- .../ceph_volume/devices/lvm/batch.py | 16 +- .../ceph_volume/devices/lvm/common.py | 3 + .../ceph_volume/devices/lvm/prepare.py | 5 + .../ceph_volume/devices/raw/list.py | 2 +- .../ceph_volume/devices/simple/scan.py | 16 +- .../ceph_volume/tests/api/test_lvm.py | 62 +- .../batch/playbooks/setup_mixed_type.yml | 16 + .../tests/functional/batch/playbooks/test.yml | 9 + .../batch/playbooks/test_explicit.yml | 9 +- .../functional/batch/playbooks/test_zap.yml | 3 + .../ceph-volume/ceph_volume/util/device.py | 7 +- .../ceph-volume/ceph_volume/util/prepare.py | 10 + ceph/src/ceph.in | 3 + ceph/src/ceph_fuse.cc | 11 + ceph/src/ceph_mon.cc | 19 + ceph/src/client/Client.cc | 145 +- ceph/src/client/Client.h | 35 +- ceph/src/client/Delegation.h | 6 +- ceph/src/client/SyntheticClient.cc | 2 +- ceph/src/client/fuse_ll.cc | 94 +- ceph/src/common/blkdev.cc | 150 +- ceph/src/common/blkdev.h | 13 + ceph/src/common/buffer.cc | 3 +- ceph/src/common/ceph_argparse.cc | 57 +- ceph/src/common/ceph_json.h | 35 +- ceph/src/common/ceph_mutex.h | 11 + ceph/src/common/ceph_time.h | 14 +- ceph/src/common/legacy_config_opts.h | 7 +- ceph/src/common/options.cc | 56 +- ceph/src/common/shared_mutex_debug.h | 11 +- ceph/src/compressor/lz4/LZ4Compressor.h | 12 + ceph/src/global/global_init.cc | 3 + ceph/src/include/ceph_fuse.h | 32 + .../cephfs/{ceph_statx.h => ceph_ll_client.h} | 73 +- ceph/src/include/cephfs/libcephfs.h | 34 +- ceph/src/include/config-h.in.cmake | 3 + ceph/src/kv/RocksDBStore.cc | 27 +- ceph/src/kv/RocksDBStore.h | 12 +- ceph/src/libcephfs.cc | 8 +- ceph/src/librbd/internal.cc | 2 +- ceph/src/mds/Beacon.cc | 2 +- ceph/src/mds/Beacon.h | 2 +- ceph/src/mds/CInode.cc | 4 +- ceph/src/mds/CInode.h | 6 + ceph/src/mds/FSMap.cc | 11 +- ceph/src/mds/Locker.cc | 5 +- ceph/src/mds/MDCache.cc | 5 +- ceph/src/mds/MDSContext.cc | 30 +- ceph/src/mds/MDSContext.h | 5 +- ceph/src/mds/MDSDaemon.cc | 140 +- ceph/src/mds/MDSDaemon.h | 1 - ceph/src/mds/MDSMap.cc | 63 +- ceph/src/mds/MDSMap.h | 43 +- ceph/src/mds/MDSRank.cc | 46 +- ceph/src/mds/MDSRank.h | 11 + ceph/src/mds/Migrator.cc | 37 +- ceph/src/mds/PurgeQueue.cc | 50 +- ceph/src/mds/ScrubStack.cc | 72 +- ceph/src/mds/ScrubStack.h | 32 +- ceph/src/mds/Server.cc | 24 +- ceph/src/mds/Server.h | 1 + ceph/src/mds/SessionMap.cc | 7 +- ceph/src/mds/SessionMap.h | 2 +- ceph/src/mds/journal.cc | 2 +- ceph/src/mds/mdstypes.h | 18 +- ceph/src/messages/MMgrReport.h | 11 +- ceph/src/mgr/ActivePyModules.cc | 52 +- ceph/src/mgr/ClusterState.cc | 1 + ceph/src/mgr/ClusterState.h | 18 +- ceph/src/mgr/DaemonServer.cc | 208 +- ceph/src/mgr/DaemonServer.h | 3 + ceph/src/mgr/Mgr.cc | 27 +- ceph/src/mgr/MgrClient.cc | 23 +- ceph/src/mgr/MgrClient.h | 8 + ceph/src/mgr/PyModule.cc | 3 + ceph/src/mgr/ServiceMap.cc | 14 +- ceph/src/mgr/ServiceMap.h | 50 +- ceph/src/mon/FSCommands.cc | 6 +- ceph/src/mon/Monitor.cc | 69 +- ceph/src/mon/MonitorDBStore.h | 117 +- ceph/src/mon/OSDMonitor.cc | 89 +- ceph/src/mon/OSDMonitor.h | 5 +- ceph/src/mon/PGMap.cc | 105 +- ceph/src/mon/PGMap.h | 159 +- ceph/src/mrgw.sh | 25 +- ceph/src/msg/async/AsyncMessenger.cc | 2 +- ceph/src/msg/async/ProtocolV1.cc | 26 +- ceph/src/msg/async/ProtocolV1.h | 1 + ceph/src/msg/async/ProtocolV2.cc | 52 +- ceph/src/msg/async/ProtocolV2.h | 1 + .../msg/async/rdma/RDMAConnectedSocketImpl.cc | 93 +- ceph/src/msg/async/rdma/RDMAStack.h | 18 +- ceph/src/objsync/boto_del.py | 6 +- ceph/src/os/CMakeLists.txt | 3 +- ceph/src/os/FuseStore.cc | 91 +- .../src/os/bluestore/BitmapFreelistManager.cc | 4 +- ceph/src/os/bluestore/BlueFS.cc | 237 +- ceph/src/os/bluestore/BlueFS.h | 94 +- ceph/src/os/bluestore/BlueRocksEnv.cc | 2 +- ceph/src/os/bluestore/BlueRocksEnv.h | 2 + ceph/src/os/bluestore/BlueStore.cc | 298 +- ceph/src/os/bluestore/BlueStore.h | 196 +- ceph/src/os/bluestore/bluefs_types.cc | 4 +- ceph/src/os/bluestore/bluefs_types.h | 7 +- ceph/src/os/bluestore/bluestore_tool.cc | 5 +- ceph/src/os/bluestore/bluestore_types.h | 5 +- ceph/src/osd/OSD.cc | 153 +- ceph/src/osd/OSD.h | 23 +- ceph/src/osd/OSDMap.cc | 20 + ceph/src/osd/PG.cc | 9 +- ceph/src/osd/PG.h | 7 +- ceph/src/osd/PrimaryLogPG.cc | 5 +- ceph/src/osd/PrimaryLogPG.h | 3 + ceph/src/osd/osd_types.cc | 47 +- ceph/src/osd/osd_types.h | 1 + ceph/src/osdc/Filer.cc | 11 +- ceph/src/osdc/Filer.h | 2 +- ceph/src/osdc/Journaler.cc | 7 + ceph/src/powerdns/pdns-backend-rgw.py | 7 +- ceph/src/pybind/ceph_argparse.py | 3 +- ceph/src/pybind/cephfs/cephfs.pyx | 315 +- ceph/src/pybind/mgr/alerts/module.py | 2 +- ceph/src/pybind/mgr/ansible/tox.ini | 4 +- ceph/src/pybind/mgr/balancer/module.py | 33 +- ceph/src/pybind/mgr/crash/module.py | 2 +- ceph/src/pybind/mgr/dashboard/.pylintrc | 1 - ceph/src/pybind/mgr/dashboard/constraints.txt | 10 + .../mgr/dashboard/controllers/__init__.py | 1 + .../pybind/mgr/dashboard/controllers/docs.py | 7 +- .../pybind/mgr/dashboard/controllers/home.py | 71 +- .../pybind/mgr/dashboard/controllers/iscsi.py | 70 +- .../pybind/mgr/dashboard/controllers/osd.py | 8 +- .../pybind/mgr/dashboard/controllers/pool.py | 11 +- .../mgr/dashboard/controllers/prometheus.py | 7 +- .../pybind/mgr/dashboard/controllers/rbd.py | 6 +- .../pybind/mgr/dashboard/controllers/rgw.py | 13 +- .../mgr/dashboard/frontend/angular.json | 12 +- .../dist/en-US/2.6046dc4f6856988661b4.js | 1 - .../dist/en-US/2.94553c8b5ce53156481e.js | 1 + .../dist/en-US/6.3278f0ea1d1f9a95c993.js | 1 - .../dist/en-US/6.cb5ced2eabd8bbeda2f4.js | 1 + .../dist/en-US/7.6e6e759c25a84becb12a.js | 1 - .../dist/en-US/7.da876ba85e66dec4f9dc.js | 1 + ...23ab13bae.js => 8.6c97130806c828c3c176.js} | 2 +- .../dist/en-US/9.244bd0ad1ded58335301.js | 1 + .../dist/en-US/9.31fa7aacebe2ae24d566.js | 1 - .../dashboard/frontend/dist/en-US/index.html | 2 +- ...a6a3ff.js => main.3eca3295c9e90ad231a3.js} | 2 +- .../en-US/runtime.5b0efe7fef218b4f1732.js | 1 - .../en-US/runtime.8e2dc9ef618a9b9db481.js | 1 + .../frontend/dist/en-US/swagger-ui-bundle.js | 134 + .../frontend/dist/en-US/swagger-ui.css | 4 + .../frontend/e2e/block/iscsi.e2e-spec.ts | 11 +- .../frontend/e2e/cluster/alerts.e2e-spec.ts | 24 - .../frontend/e2e/cluster/alerts.po.ts | 7 - .../e2e/cluster/configuration.e2e-spec.ts | 11 +- .../e2e/cluster/crush-map.e2e-spec.ts | 11 +- .../frontend/e2e/cluster/hosts.e2e-spec.ts | 32 +- .../frontend/e2e/cluster/logs.e2e-spec.ts | 32 +- .../frontend/e2e/cluster/monitors.e2e-spec.ts | 11 +- .../frontend/e2e/cluster/osds.e2e-spec.ts | 32 +- .../frontend/e2e/pools/pools.e2e-spec.ts | 32 +- .../mgr/dashboard/frontend/package-lock.json | 266 +- .../mgr/dashboard/frontend/package.json | 37 +- .../frontend/src/app/app-routing.module.ts | 34 +- .../iscsi-tabs/iscsi-tabs.component.html | 4 +- ...i-target-discovery-modal.component.spec.ts | 20 +- .../iscsi-target-discovery-modal.component.ts | 4 +- .../iscsi-target-form.component.spec.ts | 29 +- .../iscsi-target-form.component.ts | 4 +- .../block/rbd-list/rbd-list.component.html | 1 + .../block/rbd-list/rbd-list.component.spec.ts | 2 +- .../rbd-snapshot-list.component.spec.ts | 2 +- .../rbd-trash-list.component.html | 1 + .../rbd-trash-list.component.spec.ts | 31 +- .../rbd-trash-list.component.ts | 2 + .../rbd-trash-purge-modal.component.html | 4 +- .../cephfs-chart.component.spec.ts | 4 +- .../cephfs-chart/cephfs-chart.component.ts | 5 +- .../cephfs-detail.component.html | 2 +- .../src/app/ceph/cluster/cluster.module.ts | 12 +- .../cluster/crushmap/crushmap.component.html | 2 +- .../osd-flags-modal.component.ts | 6 + .../osd/osd-list/osd-list.component.ts | 2 +- .../active-alert-list.component.html} | 2 - .../active-alert-list.component.scss} | 0 .../active-alert-list.component.spec.ts} | 27 +- .../active-alert-list.component.ts} | 13 +- .../monitoring-list.component.html | 35 + .../monitoring-list.component.scss} | 0 .../monitoring-list.component.spec.ts | 46 + .../monitoring-list.component.ts | 70 + .../prometheus-tabs.component.html | 13 - .../prometheus-tabs.component.spec.ts | 47 - .../prometheus-tabs.component.ts | 19 - .../rules-list/rules-list.component.html | 8 + .../rules-list/rules-list.component.scss | 0 .../rules-list/rules-list.component.spec.ts | 29 + .../rules-list/rules-list.component.ts | 44 + .../silence-form.component.spec.ts | 61 +- .../silence-form/silence-form.component.ts | 13 +- .../silence-list/silence-list.component.html | 2 - .../silence-list.component.spec.ts | 3 +- .../silence-list/silence-list.component.ts | 5 +- .../pool-form/pool-form.component.spec.ts | 4 +- .../pool/pool-list/pool-list.component.ts | 6 +- .../rgw-user-details.component.html | 10 +- .../rgw-user-details.component.spec.ts | 53 + .../rgw-user-details.component.ts | 38 +- .../rgw-user-form.component.html | 33 +- .../rgw-user-form.component.spec.ts | 91 + .../rgw-user-form/rgw-user-form.component.ts | 56 +- .../rgw-user-list/rgw-user-list.component.ts | 7 +- .../auth/user-tabs/user-tabs.component.html | 4 +- .../breadcrumbs/breadcrumbs.component.html | 3 +- .../navigation/navigation.component.html | 17 +- .../navigation/navigation.component.ts | 14 +- .../app/shared/api/prometheus.service.spec.ts | 83 +- .../src/app/shared/api/prometheus.service.ts | 27 +- .../src/app/shared/api/rgw-user.service.ts | 4 +- .../table-actions.component.html | 6 +- .../table-key-value.component.spec.ts | 10 +- .../table-key-value.component.ts | 8 +- .../datatable/table/table.component.html | 8 +- .../datatable/table/table.component.spec.ts | 182 +- .../shared/datatable/table/table.component.ts | 23 + .../src/app/shared/enum/cell-template.enum.ts | 11 +- .../src/app/shared/models/cd-table-action.ts | 2 + .../src/app/shared/models/cd-table-column.ts | 1 + .../src/app/shared/models/pool-form-info.ts | 2 + .../app/shared/models/prometheus-alerts.ts | 7 + .../app/shared/pipes/duration.pipe.spec.ts | 24 + .../src/app/shared/pipes/duration.pipe.ts | 47 + .../src/app/shared/pipes/map.pipe.spec.ts | 25 + .../frontend/src/app/shared/pipes/map.pipe.ts | 15 + .../src/app/shared/pipes/pipes.module.ts | 13 +- .../services/prometheus-alert.service.spec.ts | 78 +- .../services/prometheus-alert.service.ts | 29 +- .../frontend/src/testing/unit-test-helper.ts | 20 + ceph/src/pybind/mgr/dashboard/module.py | 3 +- .../src/pybind/mgr/dashboard/requirements.txt | 5 +- .../mgr/dashboard/run-backend-api-tests.sh | 49 +- .../mgr/dashboard/run-frontend-unittests.sh | 15 +- .../mgr/dashboard/services/access_control.py | 6 +- .../mgr/dashboard/services/ceph_service.py | 58 +- ceph/src/pybind/mgr/dashboard/services/rbd.py | 7 +- .../mgr/dashboard/services/rgw_client.py | 74 +- ceph/src/pybind/mgr/dashboard/services/sso.py | 12 +- .../mgr/dashboard/services/tcmu_service.py | 9 +- ceph/src/pybind/mgr/dashboard/settings.py | 3 +- .../pybind/mgr/dashboard/tests/__init__.py | 13 + .../mgr/dashboard/tests/test_grafana.py | 24 +- .../pybind/mgr/dashboard/tests/test_home.py | 39 +- .../pybind/mgr/dashboard/tests/test_iscsi.py | 58 +- .../pybind/mgr/dashboard/tests/test_rgw.py | 30 + .../mgr/dashboard/tests/test_rgw_client.py | 92 +- ceph/src/pybind/mgr/dashboard/tools.py | 17 +- ceph/src/pybind/mgr/dashboard/tox.ini | 6 +- .../agent/metrics/ceph_cluster.py | 2 +- .../pybind/mgr/diskprediction_cloud/module.py | 4 +- .../pybind/mgr/diskprediction_local/module.py | 4 +- ceph/src/pybind/mgr/insights/module.py | 4 +- ceph/src/pybind/mgr/insights/tox.ini | 4 +- ceph/src/pybind/mgr/k8sevents/module.py | 2 +- ceph/src/pybind/mgr/mgr_module.py | 3 +- ceph/src/pybind/mgr/mgr_util.py | 122 + ceph/src/pybind/mgr/orchestrator_cli/tox.ini | 4 +- ceph/src/pybind/mgr/pg_autoscaler/module.py | 116 +- ceph/src/pybind/mgr/progress/module.py | 13 +- ceph/src/pybind/mgr/prometheus/module.py | 37 +- ceph/src/pybind/mgr/restful/module.py | 12 +- ceph/src/pybind/mgr/selftest/module.py | 4 + ceph/src/pybind/mgr/telegraf/module.py | 10 +- ceph/src/pybind/mgr/telemetry/module.py | 58 +- .../src/pybind/mgr/volumes/fs/async_cloner.py | 125 +- ceph/src/pybind/mgr/volumes/fs/async_job.py | 31 +- ceph/src/pybind/mgr/volumes/fs/fs_util.py | 7 +- .../mgr/volumes/fs/operations/clone_index.py | 16 +- .../pybind/mgr/volumes/fs/operations/group.py | 10 +- .../pybind/mgr/volumes/fs/operations/index.py | 2 + .../pybind/mgr/volumes/fs/operations/op_sm.py | 24 +- .../mgr/volumes/fs/operations/template.py | 2 + .../fs/operations/versions/__init__.py | 5 +- .../operations/versions/metadata_manager.py | 10 +- .../fs/operations/versions/subvolume_base.py | 51 +- .../fs/operations/versions/subvolume_v1.py | 4 +- .../mgr/volumes/fs/operations/volume.py | 18 +- ceph/src/pybind/mgr/volumes/fs/purge_queue.py | 3 +- ceph/src/pybind/mgr/volumes/fs/volume.py | 55 +- ceph/src/pybind/mgr/volumes/module.py | 35 +- ceph/src/pybind/rados/rados.pyx | 4 +- ceph/src/pybind/rbd/rbd.pyx | 139 +- ceph/src/rbd_fuse/CMakeLists.txt | 2 +- ceph/src/rbd_fuse/rbd-fuse.cc | 57 +- ceph/src/rgw/CMakeLists.txt | 32 +- ceph/src/rgw/rgw_admin.cc | 29 +- ceph/src/rgw/rgw_amqp.cc | 17 +- ceph/src/rgw/rgw_asio_frontend.cc | 13 +- ceph/src/rgw/rgw_auth.cc | 83 +- ceph/src/rgw/rgw_auth.h | 48 +- ceph/src/rgw/rgw_auth_registry.h | 18 +- ceph/src/rgw/rgw_auth_s3.cc | 10 +- ceph/src/rgw/rgw_auth_s3.h | 21 +- ceph/src/rgw/rgw_bucket.cc | 64 +- ceph/src/rgw/rgw_bucket.h | 7 + ceph/src/rgw/rgw_common.h | 16 +- ceph/src/rgw/rgw_cors.cc | 11 +- ceph/src/rgw/rgw_data_sync.cc | 37 +- ceph/src/rgw/rgw_dmclock_sync_scheduler.cc | 5 +- ceph/src/rgw/rgw_frontend.h | 9 +- ceph/src/rgw/rgw_http_client.cc | 15 +- ceph/src/rgw/rgw_http_client.h | 6 +- ceph/src/rgw/rgw_kafka.cc | 719 + ceph/src/rgw/rgw_kafka.h | 81 + ceph/src/rgw/rgw_lc.cc | 179 +- ceph/src/rgw/rgw_lc.h | 10 +- ceph/src/rgw/rgw_loadgen.cc | 4 +- ceph/src/rgw/rgw_main.cc | 18 +- ceph/src/rgw/rgw_metadata.cc | 15 +- ceph/src/rgw/rgw_multi.cc | 5 + ceph/src/rgw/rgw_notify.cc | 21 +- ceph/src/rgw/rgw_notify.h | 3 + ceph/src/rgw/rgw_op.cc | 99 +- ceph/src/rgw/rgw_pubsub.cc | 38 +- ceph/src/rgw/rgw_pubsub.h | 84 +- ceph/src/rgw/rgw_pubsub_push.cc | 301 +- ceph/src/rgw/rgw_putobj_processor.cc | 13 +- ceph/src/rgw/rgw_rados.cc | 271 +- ceph/src/rgw/rgw_rados.h | 19 +- ceph/src/rgw/rgw_reshard.cc | 23 +- ceph/src/rgw/rgw_rest.cc | 14 +- ceph/src/rgw/rgw_rest_client.cc | 6 +- ceph/src/rgw/rgw_rest_pubsub.cc | 65 +- ceph/src/rgw/rgw_rest_pubsub_common.cc | 59 +- ceph/src/rgw/rgw_rest_pubsub_common.h | 6 + ceph/src/rgw/rgw_rest_s3.cc | 39 +- ceph/src/rgw/rgw_rest_s3.h | 35 +- ceph/src/rgw/rgw_rest_sts.cc | 21 +- ceph/src/rgw/rgw_sts.cc | 6 +- ceph/src/rgw/rgw_sts.h | 1 + ceph/src/rgw/rgw_swift_auth.h | 30 +- ceph/src/rgw/rgw_sync_module_pubsub.cc | 31 +- ceph/src/rgw/rgw_sync_module_pubsub_rest.cc | 10 +- ceph/src/rgw/rgw_tag.cc | 1 + ceph/src/rgw/rgw_tag.h | 10 +- ceph/src/rgw/rgw_url.cc | 48 + ceph/src/rgw/rgw_url.h | 12 + ceph/src/rgw/rgw_xml_enc.cc | 25 +- ceph/src/rgw/rgw_zone.cc | 2 +- ceph/src/rgw/rgw_zone.h | 2 +- ceph/src/test/CMakeLists.txt | 3 + ceph/src/test/bufferlist.cc | 22 + ceph/src/test/ceph_argparse.cc | 37 +- ceph/src/test/compressor/CMakeLists.txt | 2 +- .../test/compressor/osdmaps/osdmap.2982809 | Bin 0 -> 612069 bytes .../test/compressor/osdmaps/osdmap.2982809.h | 38257 ++++++++++++++++ ceph/src/test/compressor/test_compression.cc | 39 + ceph/src/test/fs/CMakeLists.txt | 6 +- ceph/src/test/fs/test_ino_release_cb.cc | 77 + ceph/src/test/libcephfs/test.cc | 37 + ceph/src/test/librados/misc.cc | 40 + ceph/src/test/librados/watch_notify.cc | 20 +- ceph/src/test/librados/watch_notify_cxx.cc | 20 +- ceph/src/test/objectstore/store_test.cc | 114 + ceph/src/test/objectstore/test_bluefs.cc | 10 +- .../test/objectstore/test_bluestore_types.cc | 92 + ceph/src/test/pybind/test_ceph_argparse.py | 77 +- ceph/src/test/pybind/test_cephfs.py | 193 +- ceph/src/test/pybind/test_rbd.py | 11 +- .../test/rbd_mirror/test_mock_PoolReplayer.cc | 23 +- ceph/src/test/rgw/CMakeLists.txt | 12 + ceph/src/test/rgw/kafka_stub.cc | 68 + ceph/src/test/rgw/rgw_multi/conn.py | 14 + ceph/src/test/rgw/rgw_multi/multisite.py | 15 +- ceph/src/test/rgw/rgw_multi/tests.py | 37 +- ceph/src/test/rgw/rgw_multi/tests_es.py | 15 +- ceph/src/test/rgw/rgw_multi/tests_ps.py | 1041 +- ceph/src/test/rgw/rgw_multi/zone_cloud.py | 8 +- ceph/src/test/rgw/rgw_multi/zone_es.py | 2 +- ceph/src/test/rgw/rgw_multi/zone_ps.py | 132 +- ceph/src/test/rgw/test_multi.py | 34 +- ceph/src/test/rgw/test_rgw_iam_policy.cc | 75 + ceph/src/test/rgw/test_rgw_url.cc | 90 + ceph/src/tools/CMakeLists.txt | 2 +- ceph/src/tools/ceph-dencoder/CMakeLists.txt | 4 + ceph/src/tools/cephfs/JournalTool.cc | 19 +- ceph/src/tools/rados/rados.cc | 4 +- ceph/src/tools/rbd_mirror/InstanceReplayer.cc | 7 + ceph/src/tools/rbd_mirror/InstanceReplayer.h | 5 +- ceph/src/tools/rbd_mirror/LeaderWatcher.cc | 16 +- ceph/src/tools/rbd_mirror/LeaderWatcher.h | 3 + ceph/src/tools/rbd_mirror/PoolReplayer.cc | 28 +- ceph/src/vstart.sh | 4 +- ceph/sudoers.d/ceph-osd-smartctl | 2 +- 736 files changed, 53248 insertions(+), 4923 deletions(-) create mode 100644 ceph/cmake/modules/FindFUSE.cmake create mode 100644 ceph/cmake/modules/FindRDKafka.cmake delete mode 100644 ceph/cmake/modules/Findfuse.cmake create mode 100644 ceph/monitoring/grafana/dashboards/rbd-details.json create mode 100644 ceph/qa/.teuthology_branch create mode 100644 ceph/qa/distros/all/centos_7.8.yaml create mode 100644 ceph/qa/distros/all/rhel_7.8.yaml rename ceph/qa/objectstore/{bluestore-comp.yaml => bluestore-comp-lz4.yaml} (94%) create mode 100644 ceph/qa/objectstore/bluestore-comp-snappy.yaml create mode 100644 ceph/qa/objectstore/bluestore-comp-zlib.yaml create mode 100644 ceph/qa/objectstore/bluestore-comp-zstd.yaml create mode 100755 ceph/qa/standalone/mon/mon-last-epoch-clean.sh delete mode 100644 ceph/qa/suites/fs/basic_functional/tasks/volume-client/py/2.yaml delete mode 100644 ceph/qa/suites/fs/basic_functional/tasks/volume-client/py/3.yaml create mode 120000 ceph/qa/suites/fs/verify/centos_latest.yaml create mode 120000 ceph/qa/suites/multimds/basic/0-supported-random-distro$ create mode 120000 ceph/qa/suites/multimds/thrash/0-supported-random-distro$ create mode 120000 ceph/qa/suites/multimds/verify/centos_latest.yaml create mode 120000 ceph/qa/suites/rados/singleton-bluestore/objectstore/bluestore-comp-lz4.yaml create mode 120000 ceph/qa/suites/rados/singleton-bluestore/objectstore/bluestore-comp-snappy.yaml delete mode 120000 ceph/qa/suites/rados/singleton-bluestore/objectstore/bluestore-comp.yaml create mode 100644 ceph/qa/suites/rados/thrash/workloads/radosbench-high-concurrency.yaml delete mode 100644 ceph/qa/suites/rbd/openstack/base/install.yaml delete mode 100644 ceph/qa/suites/rbd/openstack/clusters/fixed-2.yaml delete mode 120000 ceph/qa/suites/rbd/openstack/objectstore delete mode 120000 ceph/qa/suites/rbd/openstack/supported-random-distro$ delete mode 100644 ceph/qa/suites/rbd/openstack/workloads/devstack-tempest-gate.yaml create mode 120000 ceph/qa/suites/rbd/singleton-bluestore/objectstore/bluestore-comp-snappy.yaml delete mode 120000 ceph/qa/suites/rbd/singleton-bluestore/objectstore/bluestore-comp.yaml delete mode 120000 ceph/qa/suites/rgw/tempest/distro/.qa delete mode 120000 ceph/qa/suites/rgw/tempest/distro/supported-random-distro$ delete mode 100644 ceph/qa/suites/rgw/tempest/distro/valgrind.yaml create mode 120000 ceph/qa/suites/rgw/tempest/ubuntu_latest.yaml create mode 100644 ceph/qa/suites/rgw/verify/striping$/stripe-equals-chunk.yaml create mode 100644 ceph/qa/suites/rgw/verify/striping$/stripe-greater-than-chunk.yaml rename ceph/qa/suites/{rbd/openstack => upgrade/client-upgrade-nautilus-octopus/nautilus-client-x}/.qa (100%) rename ceph/qa/suites/{rbd/openstack => upgrade/client-upgrade-nautilus-octopus/nautilus-client-x/rbd}/% (100%) rename ceph/qa/suites/{rbd/openstack/base => upgrade/client-upgrade-nautilus-octopus/nautilus-client-x/rbd}/.qa (100%) rename ceph/qa/suites/{rbd/openstack/clusters => upgrade/client-upgrade-nautilus-octopus/nautilus-client-x/rbd/0-cluster}/+ (100%) rename ceph/qa/suites/{rbd/openstack/clusters => upgrade/client-upgrade-nautilus-octopus/nautilus-client-x/rbd/0-cluster}/.qa (100%) rename ceph/qa/suites/{rbd/openstack/clusters => upgrade/client-upgrade-nautilus-octopus/nautilus-client-x/rbd/0-cluster}/openstack.yaml (82%) create mode 100644 ceph/qa/suites/upgrade/client-upgrade-nautilus-octopus/nautilus-client-x/rbd/0-cluster/start.yaml rename ceph/qa/suites/{rbd/openstack/features => upgrade/client-upgrade-nautilus-octopus/nautilus-client-x/rbd/1-install}/.qa (100%) create mode 100644 ceph/qa/suites/upgrade/client-upgrade-nautilus-octopus/nautilus-client-x/rbd/1-install/nautilus-client-x.yaml rename ceph/qa/suites/{rbd/openstack/workloads => upgrade/client-upgrade-nautilus-octopus/nautilus-client-x/rbd/2-features}/.qa (100%) create mode 100644 ceph/qa/suites/upgrade/client-upgrade-nautilus-octopus/nautilus-client-x/rbd/2-features/defaults.yaml rename ceph/qa/suites/{rbd/openstack/features/minimum.yaml => upgrade/client-upgrade-nautilus-octopus/nautilus-client-x/rbd/2-features/layering.yaml} (98%) create mode 120000 ceph/qa/suites/upgrade/client-upgrade-nautilus-octopus/nautilus-client-x/rbd/3-workload/.qa create mode 100644 ceph/qa/suites/upgrade/client-upgrade-nautilus-octopus/nautilus-client-x/rbd/3-workload/rbd_notification_tests.yaml create mode 120000 ceph/qa/suites/upgrade/client-upgrade-nautilus-octopus/nautilus-client-x/rbd/supported/.qa create mode 120000 ceph/qa/suites/upgrade/client-upgrade-nautilus-octopus/nautilus-client-x/rbd/supported/ubuntu_18.04.yaml delete mode 100755 ceph/qa/workunits/rbd/run_devstack_tempest.sh create mode 100644 ceph/src/include/ceph_fuse.h rename ceph/src/include/cephfs/{ceph_statx.h => ceph_ll_client.h} (61%) create mode 100644 ceph/src/pybind/mgr/dashboard/constraints.txt delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/2.6046dc4f6856988661b4.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/2.94553c8b5ce53156481e.js delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/6.3278f0ea1d1f9a95c993.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/6.cb5ced2eabd8bbeda2f4.js delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/7.6e6e759c25a84becb12a.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/7.da876ba85e66dec4f9dc.js rename ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/{8.a14771e53c123ab13bae.js => 8.6c97130806c828c3c176.js} (50%) create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/9.244bd0ad1ded58335301.js delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/9.31fa7aacebe2ae24d566.js rename ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/{main.197539d29cc499a6a3ff.js => main.3eca3295c9e90ad231a3.js} (50%) delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/runtime.5b0efe7fef218b4f1732.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/runtime.8e2dc9ef618a9b9db481.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/swagger-ui-bundle.js create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/dist/en-US/swagger-ui.css delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/cluster/alerts.e2e-spec.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/e2e/cluster/alerts.po.ts rename ceph/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/{alert-list/alert-list.component.html => active-alert-list/active-alert-list.component.html} (96%) rename ceph/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/{alert-list/alert-list.component.scss => active-alert-list/active-alert-list.component.scss} (100%) rename ceph/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/{alert-list/alert-list.component.spec.ts => active-alert-list/active-alert-list.component.spec.ts} (83%) rename ceph/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/{alert-list/alert-list.component.ts => active-alert-list/active-alert-list.component.ts} (90%) create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/monitoring-list/monitoring-list.component.html rename ceph/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/{prometheus-tabs/prometheus-tabs.component.scss => monitoring-list/monitoring-list.component.scss} (100%) create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/monitoring-list/monitoring-list.component.spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/monitoring-list/monitoring-list.component.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-tabs/prometheus-tabs.component.html delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-tabs/prometheus-tabs.component.spec.ts delete mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-tabs/prometheus-tabs.component.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/rules-list/rules-list.component.html create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/rules-list/rules-list.component.scss create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/rules-list/rules-list.component.spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/rules-list/rules-list.component.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/duration.pipe.spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/duration.pipe.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/map.pipe.spec.ts create mode 100644 ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/pipes/map.pipe.ts create mode 100644 ceph/src/rgw/rgw_kafka.cc create mode 100644 ceph/src/rgw/rgw_kafka.h create mode 100644 ceph/src/rgw/rgw_url.cc create mode 100644 ceph/src/rgw/rgw_url.h create mode 100644 ceph/src/test/compressor/osdmaps/osdmap.2982809 create mode 100644 ceph/src/test/compressor/osdmaps/osdmap.2982809.h create mode 100644 ceph/src/test/fs/test_ino_release_cb.cc create mode 100644 ceph/src/test/rgw/kafka_stub.cc create mode 100644 ceph/src/test/rgw/test_rgw_url.cc diff --git a/ceph/CMakeLists.txt b/ceph/CMakeLists.txt index 7642c4089..ae684bea2 100644 --- a/ceph/CMakeLists.txt +++ b/ceph/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.5.1) project(ceph CXX C ASM) -set(VERSION 14.2.9) +set(VERSION 14.2.10) if(POLICY CMP0028) cmake_policy(SET CMP0028 NEW) @@ -253,7 +253,7 @@ endif() option(WITH_FUSE "Fuse is here" ON) if(WITH_FUSE) - find_package(fuse) + find_package(FUSE) set(HAVE_LIBFUSE ${FUSE_FOUND}) endif() @@ -445,6 +445,7 @@ option(WITH_RADOSGW_FCGI_FRONTEND "Rados Gateway's FCGI frontend is enabled" OFF option(WITH_RADOSGW_BEAST_FRONTEND "Rados Gateway's Beast frontend is enabled" ON) option(WITH_RADOSGW_BEAST_OPENSSL "Rados Gateway's Beast frontend uses OpenSSL" ON) option(WITH_RADOSGW_AMQP_ENDPOINT "Rados Gateway's pubsub support for AMQP push endpoint" ON) +option(WITH_RADOSGW_KAFKA_ENDPOINT "Rados Gateway's pubsub support for Kafka push endpoint" ON) if(WITH_RADOSGW) find_package(EXPAT REQUIRED) diff --git a/ceph/PendingReleaseNotes b/ceph/PendingReleaseNotes index 527b3444b..87109b088 100644 --- a/ceph/PendingReleaseNotes +++ b/ceph/PendingReleaseNotes @@ -1,45 +1,38 @@ -14.2.8 +14.2.9 ------ -* The following OSD memory config options related to bluestore cache autotuning can now - be configured during runtime: +* Bucket notifications now support Kafka endpoints. This requires librdkafka of + version 0.9.2 and up. Note that Ubuntu 16.04.6 LTS (Xenial Xerus) has an older + version of librdkafka, and would require an update to the library. - - osd_memory_base (default: 768 MB) - - osd_memory_cache_min (default: 128 MB) - - osd_memory_expected_fragmentation (default: 0.15) - - osd_memory_target (default: 4 GB) +* The pool parameter ``target_size_ratio``, used by the pg autoscaler, + has changed meaning. It is now normalized across pools, rather than + specifying an absolute ratio. For details, see :ref:`pg-autoscaler`. + If you have set target size ratios on any pools, you may want to set + these pools to autoscale ``warn`` mode to avoid data movement during + the upgrade:: - The above options can be set with:: + ceph osd pool set pg_autoscale_mode warn - ceph config set global