import ceph pacific 16.2.11

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>

ceph/.github/CODEOWNERS

@@ -49,3 +49,88 @@ COPYING*                                        @ceph/doc-writers
 /doc/                                           @ceph/doc-writers
 README*                                         @ceph/doc-writers
 *.rst                                           @ceph/doc-writers
+
+# core
+/doc/man/8/ceph-authtool.rst                    @ceph/core
+/doc/man/8/ceph-conf.rst                        @ceph/core
+/doc/man/8/ceph-create-keys.rst                 @ceph/core
+/doc/man/8/ceph-kvstore-tool.rst                @ceph/core
+/doc/man/8/ceph-mon.rst                         @ceph/core
+/doc/man/8/ceph-objectstore-tool.rst            @ceph/core
+/doc/man/8/ceph-osd.rst                         @ceph/core
+/doc/man/8/ceph.rst                             @ceph/core
+/doc/man/8/crushtool.rst                        @ceph/core
+/doc/man/8/monmaptool.rst                       @ceph/core
+/doc/man/8/rados.rst                            @ceph/core
+/doc/rados                                      @ceph/core
+/qa/standalone                                  @ceph/core
+/qa/suites/rados                                @ceph/core
+/qa/workunits/erasure-code                      @ceph/core
+/qa/workunits/mgr                               @ceph/core
+/qa/workunits/mon                               @ceph/core
+/qa/workunits/objectstore                       @ceph/core
+/qa/workunits/rados                             @ceph/core
+/src/ceph.in                                    @ceph/core
+/src/ceph_osd.cc                                @ceph/core
+/src/ceph_mon.cc                                @ceph/core
+/src/blk                                        @ceph/core
+/src/crush                                      @ceph/core
+/src/erasure-code                               @ceph/core
+/src/kv                                         @ceph/core
+/src/librados                                   @ceph/core
+/src/mgr                                        @ceph/core
+/src/mon                                        @ceph/core
+/src/msg                                        @ceph/core
+/src/os                                         @ceph/core
+/src/osd                                        @ceph/core
+/src/tools/rados                                @ceph/core
+/src/test/osd                                   @ceph/core
+
+# rbd
+/doc/dev/rbd*                                   @ceph/rbd
+/doc/man/8/ceph-rbdnamer.rst                    @ceph/rbd
+/doc/man/8/rbd*                                 @ceph/rbd
+/doc/rbd                                        @ceph/rbd
+/doc/start/quick-rbd.rst                        @ceph/rbd
+/qa/rbd                                         @ceph/rbd
+/qa/run_xfstests*                               @ceph/rbd
+/qa/suites/krbd                                 @ceph/rbd
+/qa/suites/rbd                                  @ceph/rbd
+/qa/tasks/ceph_iscsi_client.py                  @ceph/rbd
+/qa/tasks/metadata.yaml                         @ceph/rbd
+/qa/tasks/qemu.py                               @ceph/rbd
+/qa/tasks/rbd*                                  @ceph/rbd
+/qa/tasks/userdata*                             @ceph/rbd
+/qa/workunits/cls/test_cls_journal.sh           @ceph/rbd
+/qa/workunits/cls/test_cls_lock.sh              @ceph/rbd
+/qa/workunits/cls/test_cls_rbd.sh               @ceph/rbd
+/qa/workunits/rbd                               @ceph/rbd
+/src/ceph-rbdnamer                              @ceph/rbd
+/src/cls/journal                                @ceph/rbd
+/src/cls/lock                                   @ceph/rbd
+/src/cls/rbd                                    @ceph/rbd
+/src/common/options/rbd*                        @ceph/rbd
+/src/etc-rbdmap                                 @ceph/rbd
+/src/include/krbd.h                             @ceph/rbd
+/src/include/rbd*                               @ceph/rbd
+/src/journal                                    @ceph/rbd
+/src/krbd.cc                                    @ceph/rbd
+/src/librbd                                     @ceph/rbd
+/src/ocf                                        @ceph/rbd
+/src/pybind/mgr/rbd_support                     @ceph/rbd
+/src/pybind/rbd                                 @ceph/rbd
+/src/rbd*                                       @ceph/rbd
+/src/test/cli/rbd                               @ceph/rbd
+/src/test/cli-integration/rbd                   @ceph/rbd
+/src/test/cls_journal                           @ceph/rbd
+/src/test/cls_lock                              @ceph/rbd
+/src/test/cls_rbd                               @ceph/rbd
+/src/test/journal                               @ceph/rbd
+/src/test/librbd                                @ceph/rbd
+/src/test/pybind/test_rbd.py                    @ceph/rbd
+/src/test/rbd*                                  @ceph/rbd
+/src/test/run-rbd*                              @ceph/rbd
+/src/test/test_rbd*                             @ceph/rbd
+/src/tools/rbd*                                 @ceph/rbd
+/systemd/rbdmap.service.in                      @ceph/rbd
+/udev/50-rbd.rules                              @ceph/rbd

ceph/.github/workflows/pr-triage.yml

@@ -12,13 +12,13 @@ jobs:
         with:
           sync-labels: ''
           repo-token: "${{ secrets.GITHUB_TOKEN }}"
-      - name: Assign milestone based on target brach name
-        uses: iyu/actions-milestone@dbf7e5348844c9ddc6b803a5721b85fa70fe3bb9
-        with:
-          configuration-path: .github/milestone.yml
-          repo-token: "${{ secrets.GITHUB_TOKEN }}"
       - name: Assign to Dashboard project
         uses: srggrs/assign-one-project-github-action@65a8ddab497df42ef268001e67bbf976f8fd39e1
         if: contains(github.event.pull_request.labels.*.name, 'dashboard')
         with:
           project: https://github.com/ceph/ceph/projects/6          
+      - name: Assign milestone based on target brach name
+        uses: iyu/actions-milestone@dbf7e5348844c9ddc6b803a5721b85fa70fe3bb9
+        with:
+          configuration-path: .github/milestone.yml
+          repo-token: "${{ secrets.GITHUB_TOKEN }}"

ceph/.readthedocs.yml

@@ -5,9 +5,13 @@
 version: 2
 formats: []
 build:
-  image: latest
+  os: ubuntu-22.04
+  tools:
+    python: "3.8"
+  apt_packages:
+    - ditaa
+    - graphviz
 python:
-  version: 3
   install:
     - requirements: admin/doc-requirements.txt
     - requirements: admin/doc-read-the-docs.txt

ceph/CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.10.2)
 # remove cmake/modules/FindPython* once 3.12 is required
 
 project(ceph
-  VERSION 16.2.10
+  VERSION 16.2.11
   LANGUAGES CXX C ASM)
 
 foreach(policy
@@ -36,7 +36,15 @@ if(WIN32)
   # the targeted Windows version. The availability of certain functions and
   # structures will depend on it.
   set(WIN32_WINNT "0x0A00" CACHE STRING "Targeted Windows version.")
-  add_definitions(-D_WIN32_WINNT=${WIN32_WINNT})
+  # In order to avoid known winpthread issues, we're using the boost
+  # shared mutex implementation.
+  # https://github.com/msys2/MINGW-packages/issues/3319
+  add_definitions(
+    -D_WIN32_WINNT=${WIN32_WINNT}
+    -DBOOST_THREAD_PROVIDES_GENERIC_SHARED_MUTEX_ON_WIN
+    -DBOOST_THREAD_V2_SHARED_MUTEX
+  )
+  set(Boost_THREADAPI "win32")
 endif()
 
 if(MINGW)

ceph/PendingReleaseNotes

@@ -32,9 +32,33 @@
   in certain recovery scenarios, e.g., monitor database lost and rebuilt, and
   the restored file system is expected to have the same ID as before.
 
+>=16.2.11
+--------
+
+* Cephfs: The 'AT_NO_ATTR_SYNC' macro is deprecated, please use the standard
+  'AT_STATX_DONT_SYNC' macro. The 'AT_NO_ATTR_SYNC' macro will be removed in
+  the future.
+* Trimming of PGLog dups is now controlled by the size instead of the version.
+  This fixes the PGLog inflation issue that was happening when the on-line
+  (in OSD) trimming got jammed after a PG split operation. Also, a new off-line
+  mechanism has been added: `ceph-objectstore-tool` got `trim-pg-log-dups` op
+  that targets situations where OSD is unable to boot due to those inflated dups.
+  If that is the case, in OSD logs the "You can be hit by THE DUPS BUG" warning
+  will be visible.
+  Relevant tracker: https://tracker.ceph.com/issues/53729
+* RBD: `rbd device unmap` command gained `--namespace` option.  Support for
+  namespaces was added to RBD in Nautilus 14.2.0 and it has been possible to
+  map and unmap images in namespaces using the `image-spec` syntax since then
+  but the corresponding option available in most other commands was missing.
+
 >=16.2.8
 --------
 
+* RGW: The behavior for Multipart Upload was modified so that only 
+  CompleteMultipartUpload notification is sent at the end of the multipart upload. 
+  The POST notification at the beginning of the upload, and PUT notifications that 
+  were sent on each part are not sent anymore.
+
 * MON/MGR: Pools can now be created with `--bulk` flag. Any pools created with `bulk`
   will use a profile of the `pg_autoscaler` that provides more performance from the start.
   However, any pools created without the `--bulk` flag will remain using it's old behavior

ceph/ceph.spec

@@ -29,7 +29,11 @@
 %else
 %bcond_without tcmalloc
 %endif
+%if 0%{?rhel} >= 9
+%bcond_without system_pmdk
+%else
 %bcond_with system_pmdk
+%endif
 %if 0%{?fedora} || 0%{?rhel}
 %bcond_without selinux
 %ifarch x86_64 ppc64le
@@ -120,11 +124,18 @@
 # disable dwz which compresses the debuginfo
 %global _find_debuginfo_dwz_opts %{nil}
 
+%if 0%{with seastar}
+# disable -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1, as gcc-toolset-{9,10}-annobin
+# do not provide gcc-annobin.so anymore, despite that they provide annobin.so. but
+# redhat-rpm-config still passes -fplugin=gcc-annobin to the compiler.
+%undefine _annotated_build
+%endif
+
 #################################################################################
 # main package definition
 #################################################################################
 Name:		ceph
-Version:	16.2.10
+Version:	16.2.11
 Release:	0%{?dist}
 %if 0%{?fedora} || 0%{?rhel}
 Epoch:		2
@@ -140,7 +151,7 @@ License:	LGPL-2.1 and LGPL-3.0 and CC-BY-SA-3.0 and GPL-2.0 and BSL-1.0 and BSD-
 Group:		System/Filesystems
 %endif
 URL:		http://ceph.com/
-Source0:	%{?_remote_tarball_prefix}ceph-16.2.10.tar.bz2
+Source0:	%{?_remote_tarball_prefix}ceph-16.2.11.tar.bz2
 %if 0%{?suse_version}
 # _insert_obs_source_lines_here
 ExclusiveArch:  x86_64 aarch64 ppc64le s390x
@@ -229,7 +240,6 @@ BuildRequires:  %{luarocks_package_name}
 BuildRequires:  jq
 BuildRequires:	libuuid-devel
 BuildRequires:	python%{python3_pkgversion}-bcrypt
-BuildRequires:	python%{python3_pkgversion}-nose
 BuildRequires:	python%{python3_pkgversion}-pecan
 BuildRequires:	python%{python3_pkgversion}-requests
 BuildRequires:	python%{python3_pkgversion}-dateutil
@@ -304,6 +314,7 @@ BuildRequires:  rdma-core-devel
 BuildRequires:	liblz4-devel >= 1.7
 # for prometheus-alerts
 BuildRequires:  golang-github-prometheus-prometheus
+BuildRequires:	jsonnet
 %endif
 %if 0%{?fedora} || 0%{?rhel}
 Requires:	systemd
@@ -345,6 +356,7 @@ BuildRequires:	python%{python3_pkgversion}-pyOpenSSL
 %endif
 %if 0%{?suse_version}
 BuildRequires:	golang-github-prometheus-prometheus
+BuildRequires:	jsonnet
 BuildRequires:	libxmlsec1-1
 BuildRequires:	libxmlsec1-nss1
 BuildRequires:	libxmlsec1-openssl1
@@ -548,6 +560,7 @@ Group:          System/Filesystems
 Requires:       ceph-mgr = %{_epoch_prefix}%{version}-%{release}
 Requires:       ceph-grafana-dashboards = %{_epoch_prefix}%{version}-%{release}
 Requires:       ceph-prometheus-alerts = %{_epoch_prefix}%{version}-%{release}
+Requires:       python%{python3_pkgversion}-setuptools
 %if 0%{?fedora} || 0%{?rhel}
 Requires:       python%{python3_pkgversion}-cherrypy
 Requires:       python%{python3_pkgversion}-jwt
@@ -597,6 +610,7 @@ Requires:       python%{python3_pkgversion}-pecan
 Requires:       python%{python3_pkgversion}-pyOpenSSL
 Requires:       python%{python3_pkgversion}-requests
 Requires:       python%{python3_pkgversion}-dateutil
+Requires:       python%{python3_pkgversion}-setuptools
 %if 0%{?fedora} || 0%{?rhel} >= 8
 Requires:       python%{python3_pkgversion}-cherrypy
 Requires:       python%{python3_pkgversion}-pyyaml
@@ -1194,12 +1208,14 @@ This package provides Ceph default alerts for Prometheus.
 # common
 #################################################################################
 %prep
-%autosetup -p1 -n ceph-16.2.10
+%autosetup -p1 -n ceph-16.2.11
 
 %build
-# LTO can be enabled as soon as the following GCC bug is fixed:
+# Disable lto on systems that do not support symver attribute
-# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=48200
+# See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=48200 for details
+%if ( 0%{?rhel} && 0%{?rhel} < 9 ) || ( 0%{?suse_version} && 0%{?suse_version} <= 1500 )
 %define _lto_cflags %{nil}
+%endif
 
 %if 0%{with seastar} && 0%{?rhel}
 . /opt/rh/gcc-toolset-9/enable
@@ -1433,6 +1449,9 @@ install -m 644 -D monitoring/ceph-mixin/prometheus_alerts.yml %{buildroot}/etc/p
 
 %clean
 rm -rf %{buildroot}
+# built binaries are no longer necessary at this point,
+# but are consuming ~17GB of disk in the build environment
+rm -rf build
 
 #################################################################################
 # files and systemd scriptlets
@@ -1528,8 +1547,7 @@ exit 0
 
 %if ! 0%{?suse_version}
 %postun -n cephadm
-userdel -r cephadm || true
+[ $1 -ne 0 ] || userdel cephadm || :
-exit 0
 %endif
 
 %files -n cephadm
@@ -1566,6 +1584,8 @@ exit 0
 %{_bindir}/rbd-replay-prep
 %endif
 %{_bindir}/ceph-post-file
+%dir %{_libdir}/ceph/denc
+%{_libdir}/ceph/denc/denc-mod-*.so
 %{_tmpfilesdir}/ceph-common.conf
 %{_mandir}/man8/ceph-authtool.8*
 %{_mandir}/man8/ceph-conf.8*

ceph/ceph.spec.in

@@ -29,7 +29,11 @@
 %else
 %bcond_without tcmalloc
 %endif
+%if 0%{?rhel} >= 9
+%bcond_without system_pmdk
+%else
 %bcond_with system_pmdk
+%endif
 %if 0%{?fedora} || 0%{?rhel}
 %bcond_without selinux
 %ifarch x86_64 ppc64le
@@ -120,6 +124,13 @@
 # disable dwz which compresses the debuginfo
 %global _find_debuginfo_dwz_opts %{nil}
 
+%if 0%{with seastar}
+# disable -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1, as gcc-toolset-{9,10}-annobin
+# do not provide gcc-annobin.so anymore, despite that they provide annobin.so. but
+# redhat-rpm-config still passes -fplugin=gcc-annobin to the compiler.
+%undefine _annotated_build
+%endif
+
 #################################################################################
 # main package definition
 #################################################################################
@@ -229,7 +240,6 @@ BuildRequires:  %{luarocks_package_name}
 BuildRequires:  jq
 BuildRequires:	libuuid-devel
 BuildRequires:	python%{python3_pkgversion}-bcrypt
-BuildRequires:	python%{python3_pkgversion}-nose
 BuildRequires:	python%{python3_pkgversion}-pecan
 BuildRequires:	python%{python3_pkgversion}-requests
 BuildRequires:	python%{python3_pkgversion}-dateutil
@@ -304,6 +314,7 @@ BuildRequires:  rdma-core-devel
 BuildRequires:	liblz4-devel >= 1.7
 # for prometheus-alerts
 BuildRequires:  golang-github-prometheus-prometheus
+BuildRequires:	jsonnet
 %endif
 %if 0%{?fedora} || 0%{?rhel}
 Requires:	systemd
@@ -345,6 +356,7 @@ BuildRequires:	python%{python3_pkgversion}-pyOpenSSL
 %endif
 %if 0%{?suse_version}
 BuildRequires:	golang-github-prometheus-prometheus
+BuildRequires:	jsonnet
 BuildRequires:	libxmlsec1-1
 BuildRequires:	libxmlsec1-nss1
 BuildRequires:	libxmlsec1-openssl1
@@ -548,6 +560,7 @@ Group:          System/Filesystems
 Requires:       ceph-mgr = %{_epoch_prefix}%{version}-%{release}
 Requires:       ceph-grafana-dashboards = %{_epoch_prefix}%{version}-%{release}
 Requires:       ceph-prometheus-alerts = %{_epoch_prefix}%{version}-%{release}
+Requires:       python%{python3_pkgversion}-setuptools
 %if 0%{?fedora} || 0%{?rhel}
 Requires:       python%{python3_pkgversion}-cherrypy
 Requires:       python%{python3_pkgversion}-jwt
@@ -597,6 +610,7 @@ Requires:       python%{python3_pkgversion}-pecan
 Requires:       python%{python3_pkgversion}-pyOpenSSL
 Requires:       python%{python3_pkgversion}-requests
 Requires:       python%{python3_pkgversion}-dateutil
+Requires:       python%{python3_pkgversion}-setuptools
 %if 0%{?fedora} || 0%{?rhel} >= 8
 Requires:       python%{python3_pkgversion}-cherrypy
 Requires:       python%{python3_pkgversion}-pyyaml
@@ -1197,9 +1211,11 @@ This package provides Ceph default alerts for Prometheus.
 %autosetup -p1 -n @TARBALL_BASENAME@
 
 %build
-# LTO can be enabled as soon as the following GCC bug is fixed:
+# Disable lto on systems that do not support symver attribute
-# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=48200
+# See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=48200 for details
+%if ( 0%{?rhel} && 0%{?rhel} < 9 ) || ( 0%{?suse_version} && 0%{?suse_version} <= 1500 )
 %define _lto_cflags %{nil}
+%endif
 
 %if 0%{with seastar} && 0%{?rhel}
 . /opt/rh/gcc-toolset-9/enable
@@ -1433,6 +1449,9 @@ install -m 644 -D monitoring/ceph-mixin/prometheus_alerts.yml %{buildroot}/etc/p
 
 %clean
 rm -rf %{buildroot}
+# built binaries are no longer necessary at this point,
+# but are consuming ~17GB of disk in the build environment
+rm -rf build
 
 #################################################################################
 # files and systemd scriptlets
@@ -1528,8 +1547,7 @@ exit 0
 
 %if ! 0%{?suse_version}
 %postun -n cephadm
-userdel -r cephadm || true
+[ $1 -ne 0 ] || userdel cephadm || :
-exit 0
 %endif
 
 %files -n cephadm
@@ -1566,6 +1584,8 @@ exit 0
 %{_bindir}/rbd-replay-prep
 %endif
 %{_bindir}/ceph-post-file
+%dir %{_libdir}/ceph/denc
+%{_libdir}/ceph/denc/denc-mod-*.so
 %{_tmpfilesdir}/ceph-common.conf
 %{_mandir}/man8/ceph-authtool.8*
 %{_mandir}/man8/ceph-conf.8*

ceph/changelog.upstream

@@ -1,7 +1,13 @@
-ceph (16.2.10-1focal) focal; urgency=medium
+ceph (16.2.11-1focal) focal; urgency=medium
 
 
- -- Jenkins Build Slave User <jenkins-build@adami06.front.sepia.ceph.com>  Thu, 21 Jul 2022 17:38:01 +0000
+ -- Jenkins Build Slave User <jenkins-build@braggi16.front.sepia.ceph.com>  Tue, 24 Jan 2023 21:28:06 +0000
+
+ceph (16.2.11-1) stable; urgency=medium
+
+  * New upstream release
+
+ -- Ceph Release Team <ceph-maintainers@ceph.io>  Tue, 24 Jan 2023 20:43:11 +0000
 
 ceph (16.2.10-1) stable; urgency=medium
 

ceph/cmake/modules/BuildBoost.cmake

@@ -11,6 +11,9 @@
 #  Boost_USE_MULTITHREADED : boolean (default: OFF)
 #  BOOST_J: integer (defanult 1)
 
+# CMAKE_CURRENT_FUNCTION_LIST_DIR is introduced by cmake 3.17, but ubuntu comes with 3.16
+set(_build_boost_list_dir "${CMAKE_CURRENT_LIST_DIR}")
+
 function(check_boost_version source_dir expected_version)
   set(version_hpp "${source_dir}/boost/version.hpp")
   if(NOT EXISTS ${version_hpp})
@@ -70,7 +73,7 @@ function(do_build_boost version)
     if(c MATCHES "^python([0-9])\$")
       set(with_python_version "${CMAKE_MATCH_1}")
       list(APPEND boost_with_libs "python")
-    elseif(c MATCHES "^python([0-9])\\.?([0-9])\$")
+    elseif(c MATCHES "^python([0-9])\\.?([0-9]+)\$")
       set(with_python_version "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}")
       list(APPEND boost_with_libs "python")
     else()
@@ -167,10 +170,12 @@ function(do_build_boost version)
       URL_HASH SHA256=${boost_sha256}
       DOWNLOAD_NO_PROGRESS 1)
   endif()
+  find_program(PATCH_EXECUTABLE patch)
   # build all components in a single shot
   include(ExternalProject)
   ExternalProject_Add(Boost
     ${source_dir}
+    PATCH_COMMAND ${PATCH_EXECUTABLE} -p3 -i ${_build_boost_list_dir}/boost-python-use-public-api-for-filename.patch
     CONFIGURE_COMMAND CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} ${configure_command}
     BUILD_COMMAND CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} ${build_command}
     BUILD_IN_SOURCE 1

ceph/cmake/modules/BuildFIO.cmake

@@ -9,14 +9,15 @@ function(build_fio)
   include(FindMake)
   find_make("MAKE_EXECUTABLE" "make_cmd")
 
+  set(source_dir ${CMAKE_BINARY_DIR}/src/fio)
+  file(MAKE_DIRECTORY ${source_dir})
   ExternalProject_Add(fio_ext
-    DOWNLOAD_DIR ${CMAKE_BINARY_DIR}/src/
     UPDATE_COMMAND "" # this disables rebuild on each run
-    GIT_REPOSITORY "https://github.com/axboe/fio.git"
+    GIT_REPOSITORY "https://github.com/ceph/fio.git"
     GIT_CONFIG advice.detachedHead=false
     GIT_SHALLOW 1
-    GIT_TAG "fio-3.15"
+    GIT_TAG "fio-3.27-cxx"
-    SOURCE_DIR ${CMAKE_BINARY_DIR}/src/fio
+    SOURCE_DIR ${source_dir}
     BUILD_IN_SOURCE 1
     CONFIGURE_COMMAND <SOURCE_DIR>/configure
     BUILD_COMMAND ${make_cmd} fio EXTFLAGS=-Wno-format-truncation ${FIO_EXTLIBS}
@@ -25,5 +26,6 @@ function(build_fio)
   add_library(fio INTERFACE IMPORTED)
   add_dependencies(fio fio_ext)
   set_target_properties(fio PROPERTIES
-    INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_BINARY_DIR}/src/fio)
+    INTERFACE_INCLUDE_DIRECTORIES ${source_dir}
+    INTERFACE_COMPILE_OPTIONS "-include;${source_dir}/config-host.h;$<$<COMPILE_LANGUAGE:C>:-std=gnu99>$<$<COMPILE_LANGUAGE:CXX>:-std=gnu++17>")
 endfunction()

ceph/cmake/modules/Buildpmem.cmake

@@ -21,6 +21,7 @@ function(build_pmem)
     set(PMDK_LIB_DIR "nondebug")
   endif()
 
+  set(pmdk_cflags "-Wno-error -fno-lto")
   include(ExternalProject)
   ExternalProject_Add(pmdk_ext
       ${source_dir_args}
@@ -29,7 +30,7 @@ function(build_pmem)
       # build system tests statically linking to librbd (which uses
       # libpmemobj) will not link (because we don't build the ndctl
       # static library here).
-      BUILD_COMMAND ${make_cmd} CC=${CMAKE_C_COMPILER} NDCTL_ENABLE=n BUILD_EXAMPLES=n BUILD_BENCHMARKS=n DOC=n
+      BUILD_COMMAND ${make_cmd} CC=${CMAKE_C_COMPILER} "EXTRA_CFLAGS=${pmdk_cflags}" NDCTL_ENABLE=n BUILD_EXAMPLES=n BUILD_BENCHMARKS=n DOC=n
       BUILD_IN_SOURCE 1
       BUILD_BYPRODUCTS "<SOURCE_DIR>/src/${PMDK_LIB_DIR}/libpmem.a" "<SOURCE_DIR>/src/${PMDK_LIB_DIR}/libpmemobj.a"
       INSTALL_COMMAND "")

ceph/cmake/modules/CephChecks.cmake

@@ -144,6 +144,34 @@ else(NOT CMAKE_CROSSCOMPILING)
   message(STATUS "Assuming unaligned access is supported")
 endif(NOT CMAKE_CROSSCOMPILING)
 
+set(version_script_source "v1 { }; v2 { } v1;")
+file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/version_script.txt "${version_script_source}")
+cmake_push_check_state(RESET)
+set(CMAKE_REQUIRED_FLAGS "-Werror -Wl,--version-script=${CMAKE_CURRENT_BINARY_DIR}/version_script.txt")
+check_c_source_compiles("
+__attribute__((__symver__ (\"func@v1\"))) void func_v1() {};
+__attribute__((__symver__ (\"func@v2\"))) void func_v2() {};
+
+int main() {}"
+  HAVE_ATTR_SYMVER)
+  if(NOT HAVE_ATTR_SYMVER)
+    if(CMAKE_CXX_FLAGS MATCHES "-flto" AND NOT CMAKE_CXX_FLAGS MATCHES "-flto-partition=none")
+      # https://tracker.ceph.com/issues/40060
+      message(FATAL_ERROR "please pass -flto-partition=none as part of CXXFLAGS")
+    endif()
+  endif()
+set(CMAKE_REQUIRED_FLAGS -Wl,--version-script=${CMAKE_CURRENT_BINARY_DIR}/version_script.txt)
+check_c_source_compiles("
+void func_v1() {}
+__asm__(\".symver func_v1, func@v1\");
+void func_v2() {}
+__asm__(\".symver func_v2, func@v2\");
+
+int main() {}"
+  HAVE_ASM_SYMVER)
+file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/version_script.txt)
+cmake_pop_check_state()
+
 # should use LINK_OPTIONS instead of LINK_LIBRARIES, if we can use cmake v3.14+
 try_compile(HAVE_LINK_VERSION_SCRIPT
   ${CMAKE_CURRENT_BINARY_DIR}

ceph/cmake/modules/Distutils.cmake

@@ -65,14 +65,13 @@ function(distutils_add_cython_module target name src)
   # This little bit of magic wipes out __Pyx_check_single_interpreter()
   # Note: this is reproduced in distutils_install_cython_module
   list(APPEND cflags -D'void0=dead_function\(void\)')
-  list(APPEND cflags -D'__Pyx_check_single_interpreter\(ARG\)=ARG \#\# 0')
+  list(APPEND cflags -D'__Pyx_check_single_interpreter\(ARG\)=ARG\#\#0')
   set(PY_CC ${compiler_launcher} ${CMAKE_C_COMPILER} ${c_compiler_arg1} ${cflags})
   set(PY_CXX ${compiler_launcher} ${CMAKE_CXX_COMPILER} ${cxx_compiler_arg1})
   set(PY_LDSHARED ${link_launcher} ${CMAKE_C_COMPILER} ${c_compiler_arg1} "-shared")
 
-  set(suffix_var "EXT_SUFFIX")
   execute_process(COMMAND "${Python3_EXECUTABLE}" -c
-    "from distutils import sysconfig; print(sysconfig.get_config_var('${suffix_var}'))"
+    "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))"
     RESULT_VARIABLE result
     OUTPUT_VARIABLE ext_suffix
     ERROR_VARIABLE error
@@ -113,7 +112,7 @@ function(distutils_install_cython_module name)
     set(ENV{LDSHARED} \"${PY_LDSHARED}\")
     set(ENV{CPPFLAGS} \"-iquote${CMAKE_SOURCE_DIR}/src/include
                         -D'void0=dead_function\(void\)' \
-                        -D'__Pyx_check_single_interpreter\(ARG\)=ARG \#\# 0'\")
+                        -D'__Pyx_check_single_interpreter\(ARG\)=ARG\#\#0'\")
     set(ENV{LDFLAGS} \"-L${CMAKE_LIBRARY_OUTPUT_DIRECTORY}\")
     set(ENV{CYTHON_BUILD_DIR} \"${CMAKE_CURRENT_BINARY_DIR}\")
     set(ENV{CEPH_LIBDIR} \"${CMAKE_LIBRARY_OUTPUT_DIRECTORY}\")

ceph/cmake/modules/boost-python-use-public-api-for-filename.patch

@@ -0,0 +1,38 @@
+From d9f06052e28873037db7f98629bce72182a42410 Mon Sep 17 00:00:00 2001
+From: Pat Riehecky <riehecky@fnal.gov>
+Date: Mon, 29 Jun 2020 10:51:58 -0500
+Subject: [PATCH] Convert Python 3.1+ to use public C API for filenames
+---
+ src/exec.cpp | 16 ++++++++++++----
+ 1 file changed, 12 insertions(+), 4 deletions(-)
+diff --git a/src/exec.cpp b/src/exec.cpp
+index 171c6f4189..b2eabe59f6 100644
+--- a/src/boost/libs/python/src/exec.cpp
++++ b/src/boost/libs/python/src/exec.cpp
+@@ -104,14 +104,22 @@ object BOOST_PYTHON_DECL exec_file(char const *filename, object global, object l
+   if (local.is_none()) local = global;
+   // should be 'char const *' but older python versions don't use 'const' yet.
+   char *f = const_cast<char *>(filename);
+-  // Let python open the file to avoid potential binary incompatibilities.
+-#if PY_VERSION_HEX >= 0x03040000
+-  FILE *fs = _Py_fopen(f, "r");
++#if PY_VERSION_HEX >= 0x03010000
++  // Let python manage any UTF bits to avoid potential incompatibilities.
++  PyObject *fo = Py_BuildValue("s", f);
++  PyObject *fb = Py_None;
++  PyUnicode_FSConverter(fo, &fb);
++  f = PyBytes_AsString(fb);
++  FILE *fs = fopen(f, "r");
++  Py_DECREF(fo);
++  Py_DECREF(fb);
+ #elif PY_VERSION_HEX >= 0x03000000
++  // Let python open the file to avoid potential binary incompatibilities.
+   PyObject *fo = Py_BuildValue("s", f);
+-  FILE *fs = _Py_fopen(fo, "r");
++  FILE *fs = _Py_fopen(fo, "r"); // Private CPython API
+   Py_DECREF(fo);
+ #else
++  // Let python open the file to avoid potential binary incompatibilities.
+   PyObject *pyfile = PyFile_FromString(f, const_cast<char*>("r"));
+   if (!pyfile) throw std::invalid_argument(std::string(f) + " : no such file");
+   python::handle<> file(pyfile);

ceph/debian/ceph-common.install

@@ -23,6 +23,7 @@ usr/bin/rbd-replay*
 usr/bin/ceph-post-file
 usr/sbin/mount.ceph sbin
 usr/lib/ceph/compressor/*
+usr/lib/ceph/denc/*
 usr/lib/ceph/crypto/* [amd64]
 usr/share/man/man8/ceph-authtool.8
 usr/share/man/man8/ceph-conf.8

ceph/debian/control

@@ -24,6 +24,7 @@ Build-Depends: automake,
                g++ (>= 7),
                javahelper,
                jq <pkg.ceph.check>,
+               jsonnet <pkg.ceph.check>,
                junit4,
                libaio-dev,
                libbabeltrace-ctf-dev,
@@ -37,7 +38,7 @@ Build-Depends: automake,
                libcurl4-openssl-dev,
 # Jaeger       libevent-dev,
                libexpat1-dev,
-# Make-Check   libffi-dev [!amd64],
+               libffi-dev [!amd64] <pkg.ceph.check>,
                libfuse-dev,
                libgoogle-perftools-dev [i386 amd64 arm64],
 # Crimson      libgnutls28-dev,
@@ -68,44 +69,44 @@ Build-Depends: automake,
                librabbitmq-dev,
                librdkafka-dev,
                luarocks,
-# Make-Check   libxmlsec1,
+               libxmlsec1 <pkg.ceph.check>,
-# Make-Check   libxmlsec1-nss,
+               libxmlsec1-nss <pkg.ceph.check>,
-# Make-Check   libxmlsec1-openssl,
+               libxmlsec1-openssl <pkg.ceph.check>,
-# Make-Check   libxmlsec1-dev,
+               libxmlsec1-dev <pkg.ceph.check>,
 # Crimson      libyaml-cpp-dev,
 # Jaeger       nlohmann-json-dev | nlohmann-json3-dev,
                parted,
                patch,
                pkg-config,
-# Make-Check   prometheus,
+               prometheus <pkg.ceph.check>,
 # Crimson      protobuf-compiler,
                python3-all-dev,
                python3-cherrypy3,
-# Make-Check   python3-jwt,
+               python3-jwt <pkg.ceph.check>,
-# Make-Check   python3-nose,
+               python3-pecan <pkg.ceph.check>,
-# Make-Check   python3-pecan,
+               python3-bcrypt <pkg.ceph.check>,
-# Make-Check   python3-bcrypt,
+               tox <pkg.ceph.check>,
-# Make-Check   tox,
+               python3-coverage <pkg.ceph.check>,
-# Make-Check   python3-coverage,
+               python3-dateutil <pkg.ceph.check>,
-# Make-Check   python3-dateutil,
+               python3-pkg-resources <pkg.ceph.check>,
-# Make-Check   python3-openssl,
+               python3-openssl <pkg.ceph.check>,
-# Make-Check   python3-prettytable,
+               python3-prettytable <pkg.ceph.check>,
-# Make-Check   python3-requests,
+               python3-requests <pkg.ceph.check>,
-# Make-Check   python3-scipy,
+               python3-scipy <pkg.ceph.check>,
                python3-setuptools,
                python3-sphinx,
-# Make-Check   python3-werkzeug,
+               python3-werkzeug <pkg.ceph.check>,
                python3-setuptools,
                python3-venv,
 # Crimson      ragel,
-# Make-Check   socat,
+               socat <pkg.ceph.check>,
 # Crimson      systemtap-sdt-dev,
-# Make-Check   uuid-dev,
+               uuid-dev <pkg.ceph.check>,
                uuid-runtime,
                valgrind,
                xfslibs-dev,
-# Make-Check   xfsprogs,
+               xfsprogs <pkg.ceph.check>,
-# Make-Check   xmlstarlet,
+               xmlstarlet <pkg.ceph.check>,
                nasm [amd64],
                zlib1g-dev,
 # Jaeger       Built-Using:   libyaml-cpp-dev (>= 0.6),
@@ -234,6 +235,8 @@ Depends: ceph-base (= ${binary:Version}),
          python3-pecan,
          python3-requests,
          python3-werkzeug,
+         libsqlite3-mod-ceph (= ${binary:Version}),
+         librados2 (= ${binary:Version}),
          ${misc:Depends},
          ${python:Depends},
          ${shlibs:Depends},
@@ -258,6 +261,7 @@ Depends: ceph-mgr (= ${binary:Version}),
          python3-cherrypy3,
          python3-jwt,
          python3-bcrypt,
+         python3-pkg-resources,
          python3-werkzeug,
          python3-routes,
          ${misc:Depends},
@@ -735,7 +739,8 @@ Description: RADOS distributed object store client C++ library (development file
 Package: libsqlite3-mod-ceph
 Architecture: any
 Section: libs
-Depends: ${misc:Depends},
+Depends: librados2 (= ${binary:Version}),
+         ${misc:Depends},
          ${shlibs:Depends},
 Description: SQLite3 VFS for Ceph
  A SQLite3 VFS for storing and manipulating databases stored on Ceph's RADOS
@@ -1249,3 +1254,4 @@ Description: prometheus alerts for the ceph dashboard
  .
  This package contains alerts used for prometheus to interact with the
  Ceph Dashboard.
+

ceph/debian/python3-ceph-argparse.install

@@ -1,2 +1,4 @@
-usr/lib/python3*/dist-packages/ceph_argparse.py
+#! /usr/bin/dh-exec
-usr/lib/python3*/dist-packages/ceph_daemon.py
+
+usr/lib/python3*/*-packages/ceph_argparse.py /usr/lib/python3/dist-packages/
+usr/lib/python3*/*-packages/ceph_daemon.py /usr/lib/python3/dist-packages/

ceph/debian/python3-cephfs.install

@@ -1,3 +1,5 @@
-usr/lib/python3*/dist-packages/ceph_volume_client.py
+#! /usr/bin/dh-exec
+
+usr/lib/python3*/*-packages/ceph_volume_client.py /usr/lib/python3/dist-packages/
 usr/lib/python3*/dist-packages/cephfs-*.egg-info
 usr/lib/python3*/dist-packages/cephfs.cpython*.so

ceph/do_cmake.sh

@@ -36,6 +36,13 @@ if [ -r /etc/os-release ]; then
           ARGS+=" -DWITH_RADOSGW_AMQP_ENDPOINT=OFF"
           ARGS+=" -DWITH_RADOSGW_KAFKA_ENDPOINT=OFF"
           ;;
+      ubuntu)
+          MAJOR_VER=$(echo "$VERSION_ID" | sed -e 's/\..*$//')
+          if [ "$MAJOR_VER" -ge "22" ] ; then
+              PYBUILD="3.10"
+          fi
+          ;;
+
   esac
 elif [ "$(uname)" == FreeBSD ] ; then
   PYBUILD="3"

ceph/doc/_static/css/custom.css

@@ -1,3 +1,23 @@
+dt { 
+    scroll-margin-top: 3em; 
+}
+
+h2 {
+    scroll-margin-top: 4em; 
+}
+
+h3 {
+    scroll-margin-top: 4em; 
+}
+
+section {
+    scroll-margin-top: 4em; 
+}
+
+span {
+    scroll-margin-top: 2em; 
+}
+
 ul.simple > li > ul > li:last-child {
     margin-block-end : 1em;
 }

ceph/doc/architecture.rst

@@ -13,6 +13,7 @@ replicate and redistribute data dynamically.
 
 .. image:: images/stack.png
 
+.. _arch-ceph-storage-cluster:
 
 The Ceph Storage Cluster
 ========================
@@ -59,7 +60,7 @@ service interfaces built on top of ``librados``.
 Storing Data
 ------------
 
-The Ceph Storage Cluster receives data from :term:`Ceph Clients`--whether it
+The Ceph Storage Cluster receives data from :term:`Ceph Client`\s--whether it
 comes through a :term:`Ceph Block Device`, :term:`Ceph Object Storage`, the
 :term:`Ceph File System` or a custom implementation you create using
 ``librados``-- which is stored as RADOS objects. Each object is stored on an
@@ -80,7 +81,7 @@ stored in a monolithic database-like fashion.
 Ceph OSD Daemons store data as objects in a flat namespace (e.g., no
 hierarchy of directories). An object has an identifier, binary data, and
 metadata consisting of a set of name/value pairs. The semantics are completely
-up to :term:`Ceph Clients`. For example, CephFS uses metadata to store file
+up to :term:`Ceph Client`\s. For example, CephFS uses metadata to store file
 attributes such as the file owner, created date, last modified date, and so
 forth.
 
@@ -135,6 +136,8 @@ Placement of Replicated Data`_.
 
 .. index:: architecture; cluster map
 
+.. _architecture_cluster_map:
+
 Cluster Map
 ~~~~~~~~~~~
 
@@ -581,7 +584,7 @@ objects.
 Peering and Sets
 ~~~~~~~~~~~~~~~~
 
-In previous sections, we noted that Ceph OSD Daemons check each others
+In previous sections, we noted that Ceph OSD Daemons check each other's
 heartbeats and report back to the Ceph Monitor. Another thing Ceph OSD daemons
 do is called 'peering', which is the process of bringing all of the OSDs that
 store a Placement Group (PG) into agreement about the state of all of the
@@ -1619,13 +1622,13 @@ instance for high availability.
 
 
 
-.. _RADOS - A Scalable, Reliable Storage Service for Petabyte-scale Storage Clusters: https://ceph.com/wp-content/uploads/2016/08/weil-rados-pdsw07.pdf
+.. _RADOS - A Scalable, Reliable Storage Service for Petabyte-scale Storage Clusters: https://ceph.io/assets/pdfs/weil-rados-pdsw07.pdf
 .. _Paxos: https://en.wikipedia.org/wiki/Paxos_(computer_science)
 .. _Monitor Config Reference: ../rados/configuration/mon-config-ref
 .. _Monitoring OSDs and PGs: ../rados/operations/monitoring-osd-pg
 .. _Heartbeats: ../rados/configuration/mon-osd-interaction
 .. _Monitoring OSDs: ../rados/operations/monitoring-osd-pg/#monitoring-osds
-.. _CRUSH - Controlled, Scalable, Decentralized Placement of Replicated Data: https://ceph.com/wp-content/uploads/2016/08/weil-crush-sc06.pdf
+.. _CRUSH - Controlled, Scalable, Decentralized Placement of Replicated Data: https://ceph.io/assets/pdfs/weil-crush-sc06.pdf
 .. _Data Scrubbing: ../rados/configuration/osd-config-ref#scrubbing
 .. _Report Peering Failure: ../rados/configuration/mon-osd-interaction#osds-report-peering-failure
 .. _Troubleshooting Peering Failure: ../rados/troubleshooting/troubleshooting-pg#placement-group-down-peering-failure

ceph/doc/ceph-volume/lvm/prepare.rst

@@ -2,25 +2,22 @@
 
 ``prepare``
 ===========
-This subcommand allows a :term:`filestore` or :term:`bluestore` setup. It is
+Before you run ``ceph-volume lvm prepare``, we recommend that you provision a
-recommended to pre-provision a logical volume before using it with
+logical volume. Then you can run ``prepare`` on that logical volume. 
-``ceph-volume lvm``.
 
-Logical volumes are not altered except for adding extra metadata.
+``prepare`` adds metadata to logical volumes but does not alter them in any
+other way. 
 
-.. note:: This is part of a two step process to deploy an OSD. If looking for
+.. note:: This is part of a two-step process to deploy an OSD. If you prefer 
-          a single-call way, please see :ref:`ceph-volume-lvm-create`
+   to deploy an OSD by using only one command, see :ref:`ceph-volume-lvm-create`.
 
-To help identify volumes, the process of preparing a volume (or volumes) to
+``prepare`` uses :term:`LVM tags` to assign several pieces of metadata to a
-work with Ceph, the tool will assign a few pieces of metadata information using
+logical volume. Volumes tagged in this way are easier to identify and easier to
-:term:`LVM tags`.
+use with Ceph. :term:`LVM tags` identify logical volumes by the role that they
-
+play in the Ceph cluster (for example: BlueStore data or BlueStore WAL+DB).
-:term:`LVM tags` makes volumes easy to discover later, and help identify them as
-part of a Ceph system, and what role they have (journal, filestore, bluestore,
-etc...)
-
-Although :term:`bluestore` is the default, the back end can be specified with:
 
+:term:`BlueStore<bluestore>` is the default backend. Ceph permits changing
+the backend, which can be done by using the following flags and arguments:
 
 * :ref:`--filestore <ceph-volume-lvm-prepare_filestore>`
 * :ref:`--bluestore <ceph-volume-lvm-prepare_bluestore>`
@@ -29,50 +26,58 @@ Although :term:`bluestore` is the default, the back end can be specified with:
 
 ``bluestore``
 -------------
-The :term:`bluestore` objectstore is the default for new OSDs. It offers a bit
+:term:`Bluestore<bluestore>` is the default backend for new OSDs. It
-more flexibility for devices compared to :term:`filestore`.
+offers more flexibility for devices than :term:`filestore` does.  Bluestore
-Bluestore supports the following configurations:
+supports the following configurations:
 
-* A block device, a block.wal, and a block.db device
+* a block device, a block.wal device, and a block.db device
-* A block device and a block.wal device
+* a block device and a block.wal device
-* A block device and a block.db device
+* a block device and a block.db device
-* A single block device
+* a single block device
 
-The bluestore subcommand accepts physical block devices, partitions on
+The ``bluestore`` subcommand accepts physical block devices, partitions on physical
-physical block devices or logical volumes as arguments for the various device parameters
+block devices, or logical volumes as arguments for the various device
-If a physical device is provided, a logical volume will be created. A volume group will
+parameters. If a physical block device is provided, a logical volume will be
-either be created or reused it its name begins with ``ceph``.
+created. If the provided volume group's name begins with `ceph`, it will be
-This allows a simpler approach at using LVM but at the cost of flexibility:
+created if it does not yet exist and it will be clobbered and reused if it
-there are no options or configurations to change how the LV is created.
+already exists. This allows for a simpler approach to using LVM but at the
+cost of flexibility: no option or configuration can be used to change how the
+logical volume is created.
 
 The ``block`` is specified with the ``--data`` flag, and in its simplest use
-case it looks like::
+case it looks like:
+
+.. prompt:: bash #
 
     ceph-volume lvm prepare --bluestore --data vg/lv
 
-A raw device can be specified in the same way::
+A raw device can be specified in the same way:
+
+.. prompt:: bash #
 
     ceph-volume lvm prepare --bluestore --data /path/to/device
 
-For enabling :ref:`encryption <ceph-volume-lvm-encryption>`, the ``--dmcrypt`` flag is required::
+For enabling :ref:`encryption <ceph-volume-lvm-encryption>`, the ``--dmcrypt`` flag is required:
+
+.. prompt:: bash #
 
     ceph-volume lvm prepare --bluestore --dmcrypt --data vg/lv
 
-If a ``block.db`` or a ``block.wal`` is needed (they are optional for
+If a ``block.db`` device or a ``block.wal`` device is needed, it can be
-bluestore) they can be specified with ``--block.db`` and ``--block.wal``
+specified with ``--block.db`` or ``--block.wal``. These can be physical
-accordingly. These can be a physical device, a partition  or
+devices, partitions, or logical volumes. ``block.db`` and ``block.wal`` are
-a logical volume.
+optional for bluestore.
 
-For both ``block.db`` and ``block.wal`` partitions aren't made logical volumes
+For both ``block.db`` and ``block.wal``, partitions can be used as-is, and 
-because they can be used as-is.
+therefore are not made into logical volumes.
 
-While creating the OSD directory, the process will use a ``tmpfs`` mount to
+While creating the OSD directory, the process uses a ``tmpfs`` mount to hold
-place all the files needed for the OSD. These files are initially created by
+the files needed for the OSD. These files are created by ``ceph-osd --mkfs``
-``ceph-osd --mkfs`` and are fully ephemeral.
+and are ephemeral.
 
-A symlink is always created for the ``block`` device, and optionally for
+A symlink is created for the ``block`` device, and is optional for ``block.db``
-``block.db`` and ``block.wal``. For a cluster with a default name, and an OSD
+and ``block.wal``. For a cluster with a default name and an OSD ID of 0, the
-id of 0, the directory could look like::
+directory looks like this::
 
     # ls -l /var/lib/ceph/osd/ceph-0
     lrwxrwxrwx. 1 ceph ceph 93 Oct 20 13:05 block -> /dev/ceph-be2b6fbd-bcf2-4c51-b35d-a35a162a02f0/osd-block-25cf0a05-2bc6-44ef-9137-79d65bd7ad62
@@ -85,11 +90,11 @@ id of 0, the directory could look like::
     -rw-------. 1 ceph ceph 10 Oct 20 13:05 type
     -rw-------. 1 ceph ceph  2 Oct 20 13:05 whoami
 
-In the above case, a device was used for ``block`` so ``ceph-volume`` create
+In the above case, a device was used for ``block``, so ``ceph-volume`` created
-a volume group and a logical volume using the following convention:
+a volume group and a logical volume using the following conventions:
 
-* volume group name: ``ceph-{cluster fsid}`` or if the vg exists already
+* volume group name: ``ceph-{cluster fsid}`` (or if the volume group already
-  ``ceph-{random uuid}``
+  exists: ``ceph-{random uuid}``)
 
 * logical volume name: ``osd-block-{osd_fsid}``
 
@@ -98,78 +103,100 @@ a volume group and a logical volume using the following convention:
 
 ``filestore``
 -------------
-This is the OSD backend that allows preparation of logical volumes for
+``Filestore<filestore>`` is the OSD backend that prepares logical volumes for a
-a :term:`filestore` objectstore OSD.
+:term:`filestore`-backed object-store OSD.
 
-It can use a logical volume for the OSD data and a physical device, a partition
-or logical volume for the journal. A physical device will have a logical volume
-created on it. A volume group will either be created or reused it its name begins
-with ``ceph``.  No special preparation is needed for these volumes other than
-following the minimum size requirements for data and journal.
 
-The CLI call looks like this of a basic standalone filestore OSD::
+``Filestore<filestore>`` uses a logical volume to store OSD data and it uses
+physical devices, partitions, or logical volumes to store the journal.  If a
+physical device is used to create a filestore backend, a logical volume will be
+created on that physical device. If the provided volume group's name begins
+with `ceph`, it will be created if it does not yet exist and it will be
+clobbered and reused if it already exists. No special preparation is needed for
+these volumes, but be sure to meet the minimum size requirements for OSD data and
+for the journal.
+
+Use the following command to create a basic filestore OSD:
+
+.. prompt:: bash #
 
    ceph-volume lvm prepare --filestore --data <data block device>
 
-To deploy file store with an external journal::
+Use this command to deploy filestore with an external journal:
+
+.. prompt:: bash #
 
    ceph-volume lvm prepare --filestore --data <data block device> --journal <journal block device>
 
-For enabling :ref:`encryption <ceph-volume-lvm-encryption>`, the ``--dmcrypt`` flag is required::
+Use this command to enable :ref:`encryption <ceph-volume-lvm-encryption>`, and note that the ``--dmcrypt`` flag is required:
+
+.. prompt:: bash #
 
    ceph-volume lvm prepare --filestore --dmcrypt --data <data block device> --journal <journal block device>
 
-Both the journal and data block device can take three forms:
+The data block device and the journal can each take one of three forms: 
 
 * a physical block device
 * a partition on a physical block device
 * a logical volume
 
-When using logical volumes the value *must* be of the format
+If you use a logical volume to deploy filestore, the value that you pass in the
-``volume_group/logical_volume``. Since logical volume names
+command *must* be of the format ``volume_group/logical_volume_name``. Since logical
-are not enforced for uniqueness, this prevents accidentally 
+volume names are not enforced for uniqueness, using this format is an important 
-choosing the wrong volume.
+safeguard against accidentally choosing the wrong volume (and clobbering its data).
 
-When using a partition, it *must* contain a ``PARTUUID``, that can be 
+If you use a partition to deploy filestore, the partition *must* contain a
-discovered by ``blkid``. THis ensure it can later be identified correctly
+``PARTUUID`` that can be discovered by ``blkid``. This ensures that the
-regardless of the device name (or path).
+partition can be identified correctly regardless of the device's name (or path).
 
-For example: passing a logical volume for data and a partition ``/dev/sdc1`` for
+For example, to use a logical volume for OSD data and a partition
-the journal::
+(``/dev/sdc1``) for the journal, run a command of this form:
 
-    ceph-volume lvm prepare --filestore --data volume_group/lv_name --journal /dev/sdc1
+.. prompt:: bash #
 
-Passing a bare device for data and a logical volume ias the journal::
+   ceph-volume lvm prepare --filestore --data volume_group/logical_volume_name --journal /dev/sdc1
+
+Or, to use a bare device for data and a logical volume for the journal:
+
+.. prompt:: bash #
 
    ceph-volume lvm prepare --filestore --data /dev/sdc --journal volume_group/journal_lv
 
-A generated uuid is used to ask the cluster for a new OSD. These two pieces are
+A generated UUID is used when asking the cluster for a new OSD. These two
-crucial for identifying an OSD and will later be used throughout the
+pieces of information (the OSD ID and the OSD UUID) are necessary for
-:ref:`ceph-volume-lvm-activate` process.
+identifying a given OSD and will later be used throughout the
+:ref:`activation<ceph-volume-lvm-activate>` process.
 
 The OSD data directory is created using the following convention::
 
     /var/lib/ceph/osd/<cluster name>-<osd id>
 
-At this point the data volume is mounted at this location, and the journal
+To link the journal volume to the mounted data volume, use this command:
-volume is linked::
+
+.. prompt:: bash #
 
    ln -s /path/to/journal /var/lib/ceph/osd/<cluster_name>-<osd-id>/journal
 
-The monmap is fetched using the bootstrap key from the OSD::
+To fetch the monmap by using the bootstrap key from the OSD, use this command:
 
-      /usr/bin/ceph --cluster ceph --name client.bootstrap-osd
+.. prompt:: bash #
-      --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring
-      mon getmap -o /var/lib/ceph/osd/<cluster name>-<osd id>/activate.monmap
 
-``ceph-osd`` will be called to populate the OSD directory, that is already
+   /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring
-mounted, re-using all the pieces of information from the initial steps::
+   /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o
+   /var/lib/ceph/osd/<cluster name>-<osd id>/activate.monmap
+
+To populate the OSD directory (which has already been mounted), use this ``ceph-osd`` command:  
+.. prompt:: bash #
+
+   ceph-osd --cluster ceph --mkfs --mkkey -i <osd id> \ --monmap
+   /var/lib/ceph/osd/<cluster name>-<osd id>/activate.monmap --osd-data \
+   /var/lib/ceph/osd/<cluster name>-<osd id> --osd-journal
+   /var/lib/ceph/osd/<cluster name>-<osd id>/journal \ --osd-uuid <osd uuid>
+   --keyring /var/lib/ceph/osd/<cluster name>-<osd id>/keyring \ --setuser ceph
+   --setgroup ceph
+
+All of the information from the previous steps is used in the above command.      
 
-      ceph-osd --cluster ceph --mkfs --mkkey -i <osd id> \
-      --monmap /var/lib/ceph/osd/<cluster name>-<osd id>/activate.monmap --osd-data \
-      /var/lib/ceph/osd/<cluster name>-<osd id> --osd-journal /var/lib/ceph/osd/<cluster name>-<osd id>/journal \
-      --osd-uuid <osd uuid> --keyring /var/lib/ceph/osd/<cluster name>-<osd id>/keyring \
-      --setuser ceph --setgroup ceph
 
 
 .. _ceph-volume-lvm-partitions:

ceph/doc/cephadm/adoption.rst

@@ -113,15 +113,15 @@ Adoption process
       ssh-copy-id -f -i ~/ceph.pub root@<host>
 
    .. note::
-     It is also possible to import an existing ssh key. See
+     It is also possible to import an existing SSH key. See
-     :ref:`ssh errors <cephadm-ssh-errors>` in the troubleshooting
+     :ref:`SSH errors <cephadm-ssh-errors>` in the troubleshooting
      document for instructions that describe how to import existing
-     ssh keys.
+     SSH keys.
 
    .. note::
-     It is also possible to have cephadm use a non-root user to ssh
+     It is also possible to have cephadm use a non-root user to SSH 
      into cluster hosts. This user needs to have passwordless sudo access.
-     Use ``ceph cephadm set-user <user>`` and copy the ssh key to that user.
+     Use ``ceph cephadm set-user <user>`` and copy the SSH key to that user.
      See :ref:`cephadm-ssh-user`
 
 #. Tell cephadm which hosts to manage:

ceph/doc/cephadm/compatibility.rst

@@ -8,11 +8,10 @@ Compatibility and Stability
 Compatibility with Podman Versions
 ----------------------------------
 
-Podman and Ceph have different end-of-life strategies that
+Podman and Ceph have different end-of-life strategies. This means that care
-might make it challenging to find compatible Podman and Ceph 
+must be taken in finding a version of Podman that is compatible with Ceph.
-versions
 
-Those versions are expected to work:
+These versions are expected to work:
 
 
 +-----------+---------------------------------------+
@@ -28,7 +27,13 @@ Those versions are expected to work:
 +-----------+-------+-------+-------+-------+-------+
 
 .. warning:: 
-  Only podman versions that are 2.0.0 and higher work with Ceph Pacific, with the exception of podman version 2.2.1, which does not work with Ceph Pacific. kubic stable is known to work with Ceph Pacific, but it must be run with a newer kernel.
+
+   To use Podman with Ceph Pacific, you must use **a version of Podman that
+   is 2.0.0 or higher**. However, **Podman version 2.2.1 does not work with
+   Ceph Pacific**.
+   
+   "Kubic stable" is known to work with Ceph Pacific, but it must be run
+   with a newer kernel.
 
 
 .. _cephadm-stability:
@@ -36,19 +41,18 @@ Those versions are expected to work:
 Stability
 ---------
 
-Cephadm is actively in development. Please be aware that some
+Cephadm is under development. Some functionality is incomplete. Be aware
-functionality is still rough around the edges. Especially the 
+that some of the components of Ceph may not work perfectly with cephadm.
-following components are working with cephadm, but the
+These include:
-documentation is not as complete as we would like, and there may be some
-changes in the near future:
 
 - RGW
 
-Cephadm support for the following features is still under development and may see breaking
+Cephadm support remains under development for the following features:
-changes in future releases:
 
 - Ingress
 - Cephadm exporter daemon
 - cephfs-mirror
 
-In case you encounter issues, see also :ref:`cephadm-pause`.
+If a cephadm command fails or a service stops running properly, see
+:ref:`cephadm-pause` for instructions on how to pause the Ceph cluster's
+background activity and how to disable cephadm.

ceph/doc/cephadm/host-management.rst

@@ -4,17 +4,26 @@
 Host Management
 ===============
 
-To list hosts associated with the cluster:
+Listing Hosts
+=============
+
+Run a command of this form to list hosts associated with the cluster:
 
 .. prompt:: bash #
 
    ceph orch host ls [--format yaml] [--host-pattern <name>] [--label <label>] [--host-status <status>]
 
-where the optional arguments "host-pattern", "label" and "host-status" are used for filtering.
+In commands of this form, the arguments "host-pattern", "label" and
-"host-pattern" is a regex that will match against hostnames and will only return matching hosts
+"host-status" are optional and are used for filtering. 
-"label" will only return hosts with the given label
+
-"host-status" will only return hosts with the given status (currently "offline" or "maintenance")
+- "host-pattern" is a regex that matches against hostnames and returns only
-Any combination of these filtering flags is valid. You may filter against name, label and/or status simultaneously
+  matching hosts.
+- "label" returns only hosts with the specified label.
+- "host-status" returns only hosts with the specified status (currently
+  "offline" or "maintenance").
+- Any combination of these filtering flags is valid. It is possible to filter
+  against name, label and status simultaneously, or to filter against any
+  proper subset of name, label and status.
 
 .. _cephadm-adding-hosts:    
     
@@ -70,31 +79,35 @@ To add each new host to the cluster, perform two steps:
 Removing Hosts
 ==============
 
-A host can safely be removed from a the cluster once all daemons are removed from it.
+A host can safely be removed from the cluster after all daemons are removed
+from it.
 
-To drain all daemons from a host do the following:
+To drain all daemons from a host, run a command of the following form:
 
 .. prompt:: bash #
 
    ceph orch host drain *<host>*
 
-The '_no_schedule' label will be applied to the host. See :ref:`cephadm-special-host-labels`
+The ``_no_schedule`` label will be applied to the host. See
+:ref:`cephadm-special-host-labels`.
 
-All osds on the host will be scheduled to be removed. You can check osd removal progress with the following:
+All OSDs on the host will be scheduled to be removed. You can check the progress of the OSD removal operation with the following command:
 
 .. prompt:: bash #
 
    ceph orch osd rm status
 
-see :ref:`cephadm-osd-removal` for more details about osd removal
+See :ref:`cephadm-osd-removal` for more details about OSD removal.
 
-You can check if there are no deamons left on the host with the following:
+Use the following command to determine whether any daemons are still on the
+host:
 
 .. prompt:: bash #
 
    ceph orch ps <host> 
 
-Once all daemons are removed you can remove the host with the following:
+After all daemons have been removed from the host, remove the host from the
+cluster by running the following command: 
 
 .. prompt:: bash #
 
@@ -103,14 +116,16 @@ Once all daemons are removed you can remove the host with the following:
 Offline host removal
 --------------------
 
-If a host is offline and can not be recovered it can still be removed from the cluster with the following:
+Even if a host is offline and can not be recovered, it can be removed from the
+cluster by running a command of the following form:
 
 .. prompt:: bash #
 
    ceph orch host rm <host> --offline --force
 
-This can potentially cause data loss as osds will be forcefully purged from the cluster by calling ``osd purge-actual`` for each osd.
+.. warning:: This can potentially cause data loss. This command forcefully
-Service specs that still contain this host should be manually updated.
+   purges OSDs from the cluster by calling ``osd purge-actual`` for each OSD.
+   Any service specs that still contain this host should be manually updated.
 
 .. _orchestrator-host-labels:
 
@@ -122,16 +137,22 @@ are free form and have no particular meaning by itself and each host
 can have multiple labels. They can be used to specify placement
 of daemons. See :ref:`orch-placement-by-labels`
 
-Labels can be added when adding a host with the ``--labels`` flag::
+Labels can be added when adding a host with the ``--labels`` flag:
+
+.. prompt:: bash #
 
    ceph orch host add my_hostname --labels=my_label1
    ceph orch host add my_hostname --labels=my_label1,my_label2
 
-To add a label a existing host, run::
+To add a label a existing host, run:
+
+.. prompt:: bash #
 
    ceph orch host label add my_hostname my_label
 
-To remove a label, run::
+To remove a label, run:
+
+.. prompt:: bash #
 
    ceph orch host label rm my_hostname my_label
 
@@ -166,7 +187,9 @@ The following host labels have a special meaning to cephadm.  All start with ``_
 Maintenance Mode
 ================
 
-Place a host in and out of maintenance mode (stops all Ceph daemons on host)::
+Place a host in and out of maintenance mode (stops all Ceph daemons on host):
+
+.. prompt:: bash #
 
    ceph orch host maintenance enter <hostname> [--force]
    ceph orch host maintenance exit <hostname>
@@ -175,6 +198,28 @@ Where the force flag when entering maintenance allows the user to bypass warning
 
 See also :ref:`cephadm-fqdn`
 
+Rescanning Host Devices
+=======================
+
+Some servers and external enclosures may not register device removal or insertion with the
+kernel. In these scenarios, you'll need to perform a host rescan. A rescan is typically
+non-disruptive, and can be performed with the following CLI command:
+
+.. prompt:: bash #
+
+   ceph orch host rescan <hostname> [--with-summary]
+
+The ``with-summary`` flag provides a breakdown of the number of HBAs found and scanned, together
+with any that failed:
+
+.. prompt:: bash [ceph:root@rh9-ceph1/]#
+
+   ceph orch host rescan rh9-ceph1 --with-summary
+   
+::
+
+   Ok. 2 adapters detected: 2 rescanned, 0 skipped, 0 failed (0.32s)
+
 Creating many hosts at once
 ===========================
 
@@ -241,24 +286,34 @@ connect to remote hosts.  When the cluster is bootstrapped, this SSH
 key is generated automatically and no additional configuration
 is necessary.
 
-A *new* SSH key can be generated with::
+A *new* SSH key can be generated with:
+
+.. prompt:: bash #
 
    ceph cephadm generate-key
 
-The public portion of the SSH key can be retrieved with::
+The public portion of the SSH key can be retrieved with:
+
+.. prompt:: bash #
 
    ceph cephadm get-pub-key
 
-The currently stored SSH key can be deleted with::
+The currently stored SSH key can be deleted with:
+
+.. prompt:: bash #
 
    ceph cephadm clear-key
 
-You can make use of an existing key by directly importing it with::
+You can make use of an existing key by directly importing it with:
+
+.. prompt:: bash #
 
    ceph config-key set mgr/cephadm/ssh_identity_key -i <key>
    ceph config-key set mgr/cephadm/ssh_identity_pub -i <pub>
 
-You will then need to restart the mgr daemon to reload the configuration with::
+You will then need to restart the mgr daemon to reload the configuration with:
+
+.. prompt:: bash #
 
    ceph mgr fail
 
@@ -272,11 +327,13 @@ that has enough privileges to download container images, start containers
 and execute commands without prompting for a password. If you do not want
 to use the "root" user (default option in cephadm), you must provide
 cephadm the name of the user that is going to be used to perform all the
-cephadm operations. Use the command::
+cephadm operations. Use the command:
+
+.. prompt:: bash #
 
    ceph cephadm set-user <user>
 
-Prior to running this the cluster ssh key needs to be added to this users
+Prior to running this the cluster SSH key needs to be added to this users
 authorized_keys file and non-root users must have passwordless sudo access.
 
 
@@ -295,15 +352,21 @@ something like this::
 There are two ways to customize this configuration for your environment:
 
 #. Import a customized configuration file that will be stored
-   by the monitor with::
+   by the monitor with:
+
+   .. prompt:: bash #
 
       ceph cephadm set-ssh-config -i <ssh_config_file>
 
-   To remove a customized SSH config and revert back to the default behavior::
+   To remove a customized SSH config and revert back to the default behavior:
+
+   .. prompt:: bash #
 
       ceph cephadm clear-ssh-config
 
-#. You can configure a file location for the SSH configuration file with::
+#. You can configure a file location for the SSH configuration file with:
+
+   .. prompt:: bash #
 
       ceph config set mgr mgr/cephadm/ssh_config_file <path>
 
@@ -370,4 +433,4 @@ requires the bare host name when adding a host to the cluster:
 
 ..
   TODO: This chapter needs to provide way for users to configure
-  Grafana in the dashboard, as this is right no very hard to do.
+  Grafana in the dashboard, as this is right now very hard to do.

ceph/doc/cephadm/index.rst

@@ -4,21 +4,36 @@
 Cephadm
 =======
 
-``cephadm`` deploys and manages a Ceph cluster. It does this by connecting the
+``cephadm`` is a utility that is used to manage a Ceph cluster. 
-manager daemon to hosts via SSH. The manager daemon is able to add, remove, and
-update Ceph containers. ``cephadm`` does not rely on external configuration
-tools such as Ansible, Rook, and Salt.
 
-``cephadm`` manages the full lifecycle of a Ceph cluster. This lifecycle
+Here is a list of some of the things that ``cephadm`` can do:
-starts with the bootstrapping process, when ``cephadm`` creates a tiny
-Ceph cluster on a single node. This cluster consists of one monitor and
-one manager. ``cephadm`` then uses the orchestration interface ("day 2"
-commands) to expand the cluster, adding all hosts and provisioning all
-Ceph daemons and services. Management of this lifecycle can be performed
-either via the Ceph command-line interface (CLI) or via the dashboard (GUI).
 
-``cephadm`` is new in Ceph release v15.2.0 (Octopus) and does not support older
+- ``cephadm`` can add a Ceph container to the cluster.
-versions of Ceph.
+- ``cephadm`` can remove a Ceph container from the cluster.
+- ``cephadm`` can update Ceph containers.
+
+``cephadm`` does not rely on external configuration tools like Ansible, Rook,
+or Salt. However, those external configuration tools can be used to automate
+operations not performed by cephadm itself. To learn more about these external
+configuration tools, visit their pages:
+
+ * https://github.com/ceph/cephadm-ansible
+ * https://rook.io/docs/rook/v1.10/Getting-Started/intro/
+ * https://github.com/ceph/ceph-salt
+
+``cephadm`` manages the full lifecycle of a Ceph cluster. This lifecycle starts
+with the bootstrapping process, when ``cephadm`` creates a tiny Ceph cluster on
+a single node. This cluster consists of one monitor and one manager.
+``cephadm`` then uses the orchestration interface to expand the cluster, adding
+hosts and provisioning Ceph daemons and services. Management of this lifecycle
+can be performed either via the Ceph command-line interface (CLI) or via the
+dashboard (GUI).
+
+To use ``cephadm`` to get started with Ceph, follow the instructions in
+:ref:`cephadm_deploying_new_cluster`.
+
+``cephadm`` was introduced in Ceph release v15.2.0 (Octopus) and does not
+support older versions of Ceph.
 
 .. toctree::
     :maxdepth: 2

ceph/doc/cephadm/install.rst

@@ -1,3 +1,5 @@
+.. _cephadm_deploying_new_cluster:
+
 ============================
 Deploying a new Ceph cluster
 ============================
@@ -8,7 +10,6 @@ then deploying the needed services.
 
 .. highlight:: console
 
-
 .. _cephadm-host-requirements:
 
 Requirements
@@ -35,17 +36,12 @@ Ceph.
 Install cephadm
 ===============
 
-The ``cephadm`` command can 
-
-#. bootstrap a new cluster
-#. launch a containerized shell with a working Ceph CLI
-#. aid in debugging containerized Ceph daemons
-
 There are two ways to install ``cephadm``:
 
 #. a :ref:`curl-based installation<cephadm_install_curl>` method
 #. :ref:`distribution-specific installation methods<cephadm_install_distros>`
 
+
 .. _cephadm_install_curl:
 
 curl-based installation
@@ -214,8 +210,8 @@ available options.
       EOF
       $ ./cephadm bootstrap --config initial-ceph.conf ...
 
-* The ``--ssh-user *<user>*`` option makes it possible to choose which ssh
+* The ``--ssh-user *<user>*`` option makes it possible to choose which SSH 
-  user cephadm will use to connect to hosts. The associated ssh key will be
+  user cephadm will use to connect to hosts. The associated SSH key will be
   added to ``/home/*<user>*/.ssh/authorized_keys``. The user that you 
   designate with this option must have passwordless sudo access.
 
@@ -366,38 +362,78 @@ Different deployment scenarios
 Single host
 -----------
 
-To configure a Ceph cluster to run on a single host, use the ``--single-host-defaults`` flag when bootstrapping. For use cases of this, see :ref:`one-node-cluster`.
+To configure a Ceph cluster to run on a single host, use the
+``--single-host-defaults`` flag when bootstrapping. For use cases of this, see
+:ref:`one-node-cluster`.
 
 The ``--single-host-defaults`` flag sets the following configuration options::
 
-  global/osd_crush_choose_leaf_type = 0
+  global/osd_crush_chooseleaf_type = 0
   global/osd_pool_default_size = 2
   mgr/mgr_standby_modules = False
 
-For more information on these options, see :ref:`one-node-cluster` and ``mgr_standby_modules`` in :ref:`mgr-administrator-guide`.
+For more information on these options, see :ref:`one-node-cluster` and
+``mgr_standby_modules`` in :ref:`mgr-administrator-guide`.
+
+.. _cephadm-airgap:
 
 Deployment in an isolated environment
 -------------------------------------
 
-You can install Cephadm in an isolated environment by using a custom container registry. You can either configure Podman or Docker to use an insecure registry, or make the registry secure. Ensure your container image is inside the registry and that you have access to all hosts you wish to add to the cluster.
+You might need to install cephadm in an environment that is not connected
+directly to the internet (such an environment is also called an "isolated
+environment"). This can be done if a custom container registry is used. Either
+of two kinds of custom container registry can be used in this scenario: (1) a
+Podman-based or Docker-based insecure registry, or (2) a secure registry.
 
-Run a local container registry:
+The practice of installing software on systems that are not connected directly
+to the internet is called "airgapping" and registries that are not connected
+directly to the internet are referred to as "airgapped".
 
-.. prompt:: bash #
+Make sure that your container image is inside the registry. Make sure that you
+have access to all hosts that you plan to add to the cluster.
+
+#. Run a local container registry:
+
+   .. prompt:: bash #
 
       podman run --privileged -d --name registry -p 5000:5000 -v /var/lib/registry:/var/lib/registry --restart=always registry:2
 
-If you are using an insecure registry, configure Podman or Docker with the hostname and port where the registry is running.
+#. If you are using an insecure registry, configure Podman or Docker with the
+   hostname and port where the registry is running.
 
-.. note:: For every host which accesses the local insecure registry, you will need to repeat this step on the host.
+   .. note:: You must repeat this step for every host that accesses the local
+             insecure registry.
 
-Next, push your container image to your local registry.
+#. Push your container image to your local registry. Here are some acceptable
+   kinds of container images: 
 
-Then run bootstrap using the ``--image`` flag with your container image. For example:
+   * Ceph container image. See :ref:`containers`.
+   * Prometheus container image
+   * Node exporter container image
+   * Grafana container image
+   * Alertmanager container image
 
-.. prompt:: bash #
+#. Create a temporary configuration file to store the names of the monitoring
+   images. (See :ref:`cephadm_monitoring-images`):
+
+   .. prompt:: bash $
+
+      cat <<EOF > initial-ceph.conf
+
+   ::
+
+      [mgr]
+      mgr/cephadm/container_image_prometheus *<hostname>*:5000/prometheus
+      mgr/cephadm/container_image_node_exporter *<hostname>*:5000/node_exporter
+      mgr/cephadm/container_image_grafana *<hostname>*:5000/grafana
+      mgr/cephadm/container_image_alertmanager *<hostname>*:5000/alertmanger
+
+#. Run bootstrap using the ``--image`` flag and pass the name of your
+   container image as the argument of the image flag. For example:
+
+   .. prompt:: bash #
 
       cephadm --image *<hostname>*:5000/ceph/ceph bootstrap --mon-ip *<mon-ip>*
 
-
 .. _cluster network: ../rados/configuration/network-config-ref#cluster-network

ceph/doc/cephadm/services/index.rst

@@ -86,7 +86,20 @@ Service Specification
 =====================
 
 A *Service Specification* is a data structure that is used to specify the
-deployment of services.  Here is an example of a service specification in YAML:
+deployment of services. In addition to parameters such as `placement` or
+`networks`, the user can set initial values of service configuration parameters
+by means of the `config` section. For each param/value configuration pair,
+cephadm calls the following command to set its value:
+
+   .. prompt:: bash #
+
+    ceph config set <service-name> <param> <value>
+
+cephadm raises health warnings in case invalid configuration parameters are
+found in the spec (`CEPHADM_INVALID_CONFIG_OPTION`) or if any error while
+trying to apply the new configuration option(s) (`CEPHADM_FAILED_SET_OPTION`).
+
+Here is an example of a service specification in YAML:
 
 .. code-block:: yaml
 
@@ -97,6 +110,10 @@ deployment of services.  Here is an example of a service specification in YAML:
         - host1
         - host2
         - host3
+    config:
+      param_1: val_1
+      ...
+      param_N: val_N
     unmanaged: false
     networks:
     - 192.169.142.0/24
@@ -414,7 +431,7 @@ Cephadm supports the deployment of multiple daemons on the same host:
     service_type: rgw
     placement:
       label: rgw
-      count-per-host: 2
+      count_per_host: 2
 
 The main reason for deploying multiple daemons per host is an additional
 performance benefit for running multiple RGW and MDS daemons on the same host.
@@ -505,6 +522,28 @@ a spec like
 
 which would cause each mon daemon to be deployed with `--cpus=2`.
 
+Mounting Files with Extra Container Arguments
+---------------------------------------------
+
+A common use case for extra container arguments is to mount additional
+files within the container. However, some intuitive formats for doing
+so can cause deployment to fail (see https://tracker.ceph.com/issues/57338).
+The recommended syntax for mounting a file with extra container arguments is:
+
+.. code-block:: yaml
+
+    extra_container_args:
+      - "-v"
+      - "/absolute/file/path/on/host:/absolute/file/path/in/container"
+
+For example:
+
+.. code-block:: yaml
+
+    extra_container_args:
+      - "-v"
+      - "/opt/ceph_cert/host.cert:/etc/grafana/certs/cert_file:ro"
+
 .. _orch-rm:
 
 Removing a Service

ceph/doc/cephadm/services/monitoring.rst

@@ -103,6 +103,8 @@ example spec file:
     spec:
       port: 4200
 
+.. _cephadm_monitoring-images:
+
 Using custom images
 ~~~~~~~~~~~~~~~~~~~
 
@@ -161,6 +163,8 @@ For example, if you had changed the prometheus image
 
           ceph config rm mgr mgr/cephadm/container_image_prometheus
 
+See also :ref:`cephadm-airgap`.
+
 .. _cephadm-overwrite-jinja2-templates:
 
 Using custom configuration files
@@ -195,6 +199,7 @@ set``:
 - ``services/grafana/ceph-dashboard.yml``
 - ``services/grafana/grafana.ini``
 - ``services/prometheus/prometheus.yml``
+- ``services/prometheus/alerting/custom_alerts.yml``
 
 You can look up the file templates that are currently used by cephadm in
 ``src/pybind/mgr/cephadm/templates``:
@@ -240,6 +245,15 @@ Example
   # reconfig the prometheus service
   ceph orch reconfig prometheus
 
+.. code-block:: bash
+
+  # set additional custom alerting rules for Prometheus
+  ceph config-key set mgr/cephadm/services/prometheus/alerting/custom_alerts.yml \
+    -i $PWD/custom_alerts.yml
+
+  # Note that custom alerting rules are not parsed by Jinja and hence escaping
+  # will not be an issue.
+
 Deploying monitoring without cephadm
 ------------------------------------
 
@@ -282,6 +296,32 @@ Due to performance reasons, monitoring of RBD images is disabled by default. For
 :ref:`prometheus-rbd-io-statistics`. If disabled, the overview and details dashboards will stay empty in Grafana
 and the metrics will not be visible in Prometheus.
 
+Setting up Prometheus
+-----------------------
+
+Setting Prometheus Retention Time
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Cephadm provides the option to set the Prometheus TDSB retention time using
+a ``retention_time`` field in the Prometheus service spec. The value defaults
+to 15 days (15d). If you would like a different value, such as 1 year (1y) you
+can apply a service spec similar to:
+
+.. code-block:: yaml
+
+    service_type: prometheus
+    placement:
+      count: 1
+    spec:
+      retention_time: "1y"
+
+.. note::
+
+  If you already had Prometheus daemon(s) deployed before and are updating an
+  existent spec as opposed to doing a fresh Prometheus deployment, you must also
+  tell cephadm to redeploy the Prometheus daemon(s) to put this change into effect.
+  This can be done with a ``ceph orch redeploy prometheus`` command.
+
 Setting up Grafana
 ------------------
 

ceph/doc/cephadm/services/osd.rst

@@ -138,6 +138,12 @@ There are a few ways to create new OSDs:
 
     ceph orch daemon add osd host1:/dev/sdb
 
+  Advanced OSD creation from specific devices on a specific host:
+
+  .. prompt:: bash #
+
+    ceph orch daemon add osd host1:data_devices=/dev/sda,/dev/sdb,db_devices=/dev/sdc,osds_per_device=2
+
 * You can use :ref:`drivegroups` to categorize device(s) based on their
   properties. This might be useful in forming a clearer picture of which
   devices are available to consume. Properties include device type (SSD or
@@ -239,6 +245,18 @@ Expected output::
 
 OSDs that are not safe to destroy will be rejected.
 
+.. note::
+    After removing OSDs, if the drives the OSDs were deployed on once again
+    become available, cephadm may automatically try to deploy more OSDs
+    on these drives if they match an existing drivegroup spec. If you deployed
+    the OSDs you are removing with a spec and don't want any new OSDs deployed on
+    the drives after removal, it's best to modify the drivegroup spec before removal.
+    Either set ``unmanaged: true`` to stop it from picking up new drives at all,
+    or modify it in some way that it no longer matches the drives used for the
+    OSDs you wish to remove. Then re-apply the spec. For more info on drivegroup
+    specs see :ref:`drivegroups`. For more info on the declarative nature of
+    cephadm in reference to deploying OSDs, see :ref:`cephadm-osd-declarative`
+
 Monitoring OSD State
 --------------------
 
@@ -283,6 +301,7 @@ Expected output::
 
 This resets the initial state of the OSD and takes it off the removal queue.
 
+.. _cephadm-replacing-an-osd:
 
 Replacing an OSD
 ----------------

ceph/doc/cephadm/services/rgw.rst

@@ -65,14 +65,14 @@ example spec file:
 .. code-block:: yaml
 
     service_type: rgw
-    service_name: foo
+    service_id: foo
     placement:
       label: rgw
-      count-per-host: 2
+      count_per_host: 2
     networks:
     - 192.169.142.0/24
     spec:
-      port: 8000
+      rgw_frontend_port: 8080
 
 
 Multisite zones
@@ -224,6 +224,33 @@ It is a yaml format file with the following properties:
         ...
         -----END PRIVATE KEY-----
 
+.. code-block:: yaml
+
+    service_type: ingress
+    service_id: rgw.something    # adjust to match your existing RGW service
+    placement:
+      hosts:
+        - host1
+        - host2
+        - host3
+    spec:
+      backend_service: rgw.something      # adjust to match your existing RGW service
+      virtual_ips_list:
+      - <string>/<string>                 # ex: 192.168.20.1/24
+      - <string>/<string>                 # ex: 192.168.20.2/24
+      - <string>/<string>                 # ex: 192.168.20.3/24
+      frontend_port: <integer>            # ex: 8080
+      monitor_port: <integer>             # ex: 1967, used by haproxy for load balancer status
+      virtual_interface_networks: [ ... ] # optional: list of CIDR networks
+      ssl_cert: |                         # optional: SSL certificate and key
+        -----BEGIN CERTIFICATE-----
+        ...
+        -----END CERTIFICATE-----
+        -----BEGIN PRIVATE KEY-----
+        ...
+        -----END PRIVATE KEY-----
+
+
 where the properties of this service specification are:
 
 * ``service_type``
@@ -237,6 +264,10 @@ where the properties of this service specification are:
     to match the nodes where RGW is deployed.
 * ``virtual_ip``
     The virtual IP (and network) in CIDR format where the ingress service will be available.
+* ``virtual_ips_list``
+    The virtual IP address in CIDR format where the ingress service will be available.
+    Each virtual IP address will be primary on one node running the ingress service. The number
+    of virtual IP addresses must be less than or equal to the number of ingress nodes.
 * ``virtual_interface_networks``
     A list of networks to identify which ethernet interface to use for the virtual IP.
 * ``frontend_port``

ceph/doc/cephadm/troubleshooting.rst

@@ -179,7 +179,7 @@ container execution command.
 
 .. _cephadm-ssh-errors:
 
-ssh errors
+SSH errors
 ----------
 
 Error message::
@@ -208,7 +208,7 @@ Things users can do:
 
      [root@mon1 ~]# cat ~/cephadm_private_key | cephadm shell -- ceph cephadm set-ssk-key -i -
 
-2. Ensure that the ssh config is correct::
+2. Ensure that the SSH config is correct::
 
      [root@mon1 ~]# cephadm shell -- ceph cephadm get-ssh-config > config
 

ceph/doc/cephadm/upgrade.rst

@@ -96,6 +96,12 @@ You can stop the upgrade process at any time by running the following command:
 
   ceph orch upgrade stop
 
+Post upgrade actions
+====================
+
+In case the new version is based on ``cephadm``, once done with the upgrade the user
+has to update the ``cephadm`` package (or ceph-common package in case the user
+doesn't use ``cephadm shell``) to a version compatible with the new version.
 
 Potential problems
 ==================
@@ -165,3 +171,100 @@ you need. For example, the following command upgrades to a development build:
   ceph orch upgrade start --image quay.io/ceph-ci/ceph:recent-git-branch-name
 
 For more information about available container images, see :ref:`containers`.
+
+Staggered Upgrade
+=================
+
+Some users may prefer to upgrade components in phases rather than all at once.
+The upgrade command, starting in 16.2.11 and 17.2.1 allows parameters
+to limit which daemons are upgraded by a single upgrade command. The options in
+include ``daemon_types``, ``services``, ``hosts`` and ``limit``. ``daemon_types``
+takes a comma-separated list of daemon types and will only upgrade daemons of those
+types. ``services`` is mutually exclusive with ``daemon_types``, only takes services
+of one type at a time (e.g. can't provide an OSD and RGW service at the same time), and
+will only upgrade daemons belonging to those services. ``hosts`` can be combined
+with ``daemon_types`` or ``services`` or provided on its own. The ``hosts`` parameter
+follows the same format as the command line options for :ref:`orchestrator-cli-placement-spec`.
+``limit`` takes an integer > 0 and provides a numerical limit on the number of
+daemons cephadm will upgrade. ``limit`` can be combined with any of the other
+parameters. For example, if you specify to upgrade daemons of type osd on host
+Host1 with ``limit`` set to 3, cephadm will upgrade (up to) 3 osd daemons on
+Host1.
+
+Example: specifying daemon types and hosts:
+
+.. prompt:: bash #
+
+  ceph orch upgrade start --image <image-name> --daemon-types mgr,mon --hosts host1,host2
+
+Example: specifying services and using limit:
+
+.. prompt:: bash #
+
+  ceph orch upgrade start --image <image-name> --services rgw.example1,rgw.example2 --limit 2
+
+.. note::
+
+   Cephadm strictly enforces an order to the upgrade of daemons that is still present
+   in staggered upgrade scenarios. The current upgrade ordering is
+   ``mgr -> mon -> crash -> osd -> mds -> rgw -> rbd-mirror -> cephfs-mirror -> iscsi -> nfs``.
+   If you specify parameters that would upgrade daemons out of order, the upgrade
+   command will block and note which daemons will be missed if you proceed.
+
+.. note::
+
+  Upgrade commands with limiting parameters will validate the options before beginning the
+  upgrade, which may require pulling the new container image. Do not be surprised
+  if the upgrade start command takes a while to return when limiting parameters are provided.
+
+.. note::
+
+   In staggered upgrade scenarios (when a limiting parameter is provided) monitoring
+   stack daemons including Prometheus and node-exporter are refreshed after the Manager
+   daemons have been upgraded. Do not be surprised if Manager upgrades thus take longer
+   than expected. Note that the versions of monitoring stack daemons may not change between
+   Ceph releases, in which case they are only redeployed.
+
+Upgrading to a version that supports staggered upgrade from one that doesn't
+----------------------------------------------------------------------------
+
+While upgrading from a version that already supports staggered upgrades the process
+simply requires providing the necessary arguments. However, if you wish to upgrade
+to a version that supports staggered upgrade from one that does not, there is a
+workaround. It requires first manually upgrading the Manager daemons and then passing
+the limiting parameters as usual.
+
+.. warning::
+  Make sure you have multiple running mgr daemons before attempting this procedure.
+
+To start with, determine which Manager is your active one and which are standby. This
+can be done in a variety of ways such as looking at the ``ceph -s`` output. Then,
+manually upgrade each standby mgr daemon with:
+
+.. prompt:: bash #
+
+  ceph orch daemon redeploy mgr.example1.abcdef --image <new-image-name>
+
+.. note::
+
+   If you are on a very early version of cephadm (early Octopus) the ``orch daemon redeploy``
+   command may not have the ``--image`` flag. In that case, you must manually set the
+   Manager container image ``ceph config set mgr container_image <new-image-name>`` and then
+   redeploy the Manager ``ceph orch daemon redeploy mgr.example1.abcdef``
+
+At this point, a Manager fail over should allow us to have the active Manager be one
+running the new version.
+
+.. prompt:: bash #
+
+  ceph mgr fail
+
+Verify the active Manager is now one running the new version. To complete the Manager
+upgrading:
+
+.. prompt:: bash #
+
+  ceph orch upgrade start --image <new-image-name> --daemon-types mgr
+
+You should now have all your Manager daemons on the new version and be able to
+specify the limiting parameters for the rest of the upgrade.

ceph/doc/cephfs/add-remove-mds.rst

@@ -1,3 +1,12 @@
+.. _cephfs_add_remote_mds:
+
+.. note::
+   It is highly recommended to use :doc:`/cephadm/index` or another Ceph
+   orchestrator for setting up the ceph cluster. Use this approach only if you
+   are setting up the ceph cluster manually. If one still intends to use the
+   manual way for deploying MDS daemons, :doc:`/cephadm/services/mds/` can
+   also be used.
+
 ============================
  Deploying Metadata Servers
 ============================
@@ -62,7 +71,7 @@ means limiting its cache size.
 Adding an MDS
 =============
 
-#. Create an mds data point ``/var/lib/ceph/mds/ceph-${id}``. The daemon only uses this directory to store its keyring.
+#. Create an mds directory ``/var/lib/ceph/mds/ceph-${id}``. The daemon only uses this directory to store its keyring.
 
 #. Create the authentication key, if you use CephX: ::
 

ceph/doc/cephfs/ceph-dokan.rst

@@ -1,3 +1,5 @@
+.. _ceph-dokan:
+
 =======================
 Mount CephFS on Windows
 =======================

ceph/doc/cephfs/cephfs-shell.rst

@@ -64,7 +64,7 @@ Copy a file/directory to Ceph File System from Local File System.
 
 Usage : 
     
-        put [options] <source_path> [target_path]
+        put [options] <source_path> <target_path>
 
 * source_path - local file/directory path to be copied to cephfs.
     * if `.` copies all the file/directories in the local working directory.
@@ -84,7 +84,7 @@ Copy a file from Ceph File System to Local File System.
 
 Usage : 
 
-    get [options] <source_path> [target_path]
+    get [options] <source_path> <target_path>
 
 * source_path - remote file/directory path which is to be copied to local file system.
     * if `.` copies all the file/directories in the remote working directory.

ceph/doc/cephfs/cephfs-top.rst

@@ -1,3 +1,5 @@
+.. _cephfs-top:
+
 ==================
 CephFS Top Utility
 ==================
@@ -7,7 +9,7 @@ in realtime. `cephfs-top` is a curses based python script which makes use of `st
 plugin in Ceph Manager to fetch (and display) metrics.
 
 Manager Plugin
---------------
+==============
 
 Ceph Filesystem clients periodically forward various metrics to Ceph Metadata Servers (MDS)
 which in turn get forwarded to Ceph Manager by MDS rank zero. Each active MDS forward its
@@ -27,9 +29,8 @@ metrics are for a particular MDS rank (e.g., number of subtrees handled by an MD
 Once enabled, Ceph Filesystem metrics can be fetched via::
 
   $ ceph fs perf stats
-  {"version": 1, "global_counters": ["cap_hit", "read_latency", "write_latency", "metadata_latency", "dentry_lease"], "counters": [], "client_metadata": {"client.614146": {"IP": "10.1.1.100", "hostname"  : "ceph-host1", "root": "/", "mount_point": "/mnt/cephfs", "valid_metrics": ["cap_hit", "read_latency", "write_latency", "metadata_latency", "dentry_lease"]}}, "global_metrics": {"client.614146": [[0,  0], [0, 0], [0, 0], [0, 0], [0, 0]]}, "metrics": {"delayed_ranks": [], "mds.0": {"client.614146": []}}}
 
-Details of the JSON command output are as follows:
+The output format is JSON and contains fields as follows:
 
 - `version`: Version of stats output
 - `global_counters`: List of global performance metrics
@@ -52,7 +53,7 @@ To fetch metrics only for a subset of active MDSs (e.g., MDS rank 1 and 2)::
   $ ceph fs perf stats --mds_rank=1,2
 
 `cephfs-top`
-------------
+============
 
 `cephfs-top` utility relies on `stats` plugin to fetch performance metrics and display in
 `top(1)` like format. `cephfs-top` is available as part of `cephfs-top` package.
@@ -62,6 +63,9 @@ By default, `cephfs-top` uses `client.fstop` user to connect to a Ceph cluster::
   $ ceph auth get-or-create client.fstop mon 'allow r' mds 'allow r' osd 'allow r' mgr 'allow r'
   $ cephfs-top
 
+Command-Line Options
+--------------------
+
 To use a non-default user (other than `client.fstop`) use::
 
   $ cephfs-top --id <name>
@@ -76,8 +80,18 @@ By default, `cephfs-top` connects to cluster name `ceph`. To use a non-default c
 
 Interval should be greater or equal to 0.5 second. Fractional seconds are honoured.
 
-Sample screenshot running `cephfs-top` with 2 clients:
+Interactive Commands
+--------------------
+
+1. m : Filesystem selection
+      Displays a menu of filesystems for selection.
+
+2. q : Quit
+      Exit the utility if you are at the home screen (All Filesystem Info),
+      otherwise escape back to the home screen.
+
+The metrics display can be scrolled using the Arrow Keys, PgUp/PgDn, Home/End and mouse.
+
+Sample screenshot running `cephfs-top` with 2 filesystems:
 
 .. image:: cephfs-top.png
-
-.. note:: As of now, `cephfs-top` does not reliably work with multiple Ceph Filesystems.

ceph/doc/cephfs/fs-volumes.rst

@@ -80,18 +80,63 @@ List volumes using::
 
     $ ceph fs volume ls
 
+Fetch the information of a CephFS volume using::
+
+    $ ceph fs volume info vol_name [--human_readable]
+
+The ``--human_readable`` flag shows used and available pool capacities in KB/MB/GB.
+
+The output format is JSON and contains fields as follows:
+
+* pools: Attributes of data and metadata pools
+        * avail: The amount of free space available in bytes
+        * used: The amount of storage consumed in bytes
+        * name: Name of the pool
+* mon_addrs: List of monitor addresses
+* used_size: Current used size of the CephFS volume in bytes
+* pending_subvolume_deletions: Number of subvolumes pending deletion
+
+Sample output of volume info command::
+
+  $ ceph fs volume info vol_name
+  {
+      "mon_addrs": [
+          "192.168.1.7:40977"
+      ],
+      "pending_subvolume_deletions": 0,
+      "pools": {
+          "data": [
+              {
+                  "avail": 106288709632,
+                  "name": "cephfs.vol_name.data",
+                  "used": 4096
+              }
+          ],
+          "metadata": [
+              {
+                  "avail": 106288709632,
+                  "name": "cephfs.vol_name.meta",
+                  "used": 155648
+              }
+          ]
+      },
+      "used_size": 0
+  }
+
 FS Subvolume groups
 -------------------
 
 Create a subvolume group using::
 
-    $ ceph fs subvolumegroup create <vol_name> <group_name> [--pool_layout <data_pool_name>] [--uid <uid>] [--gid <gid>] [--mode <octal_mode>]
+    $ ceph fs subvolumegroup create <vol_name> <group_name> [--size <size_in_bytes>] [--pool_layout <data_pool_name>] [--uid <uid>] [--gid <gid>] [--mode <octal_mode>]
 
 The command succeeds even if the subvolume group already exists.
 
 When creating a subvolume group you can specify its data pool layout (see
-:doc:`/cephfs/file-layouts`), uid, gid, and file mode in octal numerals. By default, the
+:doc:`/cephfs/file-layouts`), uid, gid, file mode in octal numerals and
-subvolume group is created with an octal file mode '755', uid '0', gid '0' and data pool
+size in bytes. The size of the subvolume group is specified by setting
+a quota on it (see :doc:`/cephfs/quota`). By default, the subvolume group
+is created with an octal file mode '755', uid '0', gid '0' and the data pool
 layout of its parent directory.
 
 
@@ -114,6 +159,47 @@ List subvolume groups using::
 .. note:: Subvolume group snapshot feature is no longer supported in mainline CephFS (existing group
           snapshots can still be listed and deleted)
 
+Fetch the metadata of a subvolume group using::
+
+    $ ceph fs subvolumegroup info <vol_name> <group_name>
+
+The output format is JSON and contains fields as follows.
+
+* atime: access time of subvolume group path in the format "YYYY-MM-DD HH:MM:SS"
+* mtime: modification time of subvolume group path in the format "YYYY-MM-DD HH:MM:SS"
+* ctime: change time of subvolume group path in the format "YYYY-MM-DD HH:MM:SS"
+* uid: uid of subvolume group path
+* gid: gid of subvolume group path
+* mode: mode of subvolume group path
+* mon_addrs: list of monitor addresses
+* bytes_pcent: quota used in percentage if quota is set, else displays "undefined"
+* bytes_quota: quota size in bytes if quota is set, else displays "infinite"
+* bytes_used: current used size of the subvolume group in bytes
+* created_at: time of creation of subvolume group in the format "YYYY-MM-DD HH:MM:SS"
+* data_pool: data pool the subvolume group belongs to
+
+Check the presence of any subvolume group using::
+
+    $ ceph fs subvolumegroup exist <vol_name>
+
+The strings returned by the 'exist' command:
+
+* "subvolumegroup exists": if any subvolumegroup is present
+* "no subvolumegroup exists": if no subvolumegroup is present
+
+.. note:: It checks for the presence of custom groups and not the default one. To validate the emptiness of the volume, subvolumegroup existence check alone is not sufficient. The subvolume existence also needs to be checked as there might be subvolumes in the default group.
+
+Resize a subvolume group using::
+
+    $ ceph fs subvolumegroup resize <vol_name> <group_name> <new_size> [--no_shrink]
+
+The command resizes the subvolume group quota using the size specified by 'new_size'.
+The '--no_shrink' flag prevents the subvolume group to shrink below the current used
+size of the subvolume group.
+
+The subvolume group can be resized to an unlimited size by passing 'inf' or 'infinite'
+as the new_size.
+
 Remove a snapshot of a subvolume group using::
 
     $ ceph fs subvolumegroup snapshot rm <vol_name> <group_name> <snap_name> [--force]
@@ -195,7 +281,7 @@ Fetch the absolute path of a subvolume using::
 
     $ ceph fs subvolume getpath <vol_name> <subvol_name> [--group_name <subvol_group_name>]
 
-Fetch the metadata of a subvolume using::
+Fetch the information of a subvolume using::
 
     $ ceph fs subvolume info <vol_name> <subvol_name> [--group_name <subvol_group_name>]
 
@@ -243,6 +329,40 @@ List subvolumes using::
 
 .. note:: subvolumes that are removed but have snapshots retained, are also listed.
 
+Check the presence of any subvolume using::
+
+    $ ceph fs subvolume exist <vol_name> [--group_name <subvol_group_name>]
+
+The strings returned by the 'exist' command:
+
+* "subvolume exists": if any subvolume of given group_name is present
+* "no subvolume exists": if no subvolume of given group_name is present
+
+Set custom metadata on the subvolume as a key-value pair using::
+
+    $ ceph fs subvolume metadata set <vol_name> <subvol_name> <key_name> <value> [--group_name <subvol_group_name>]
+
+.. note:: If the key_name already exists then the old value will get replaced by the new value.
+
+.. note:: key_name and value should be a string of ASCII characters (as specified in python's string.printable). key_name is case-insensitive and always stored in lower case.
+
+.. note:: Custom metadata on a subvolume is not preserved when snapshotting the subvolume, and hence, is also not preserved when cloning the subvolume snapshot.
+
+Get custom metadata set on the subvolume using the metadata key::
+
+    $ ceph fs subvolume metadata get <vol_name> <subvol_name> <key_name> [--group_name <subvol_group_name>]
+
+List custom metadata (key-value pairs) set on the subvolume using::
+
+    $ ceph fs subvolume metadata ls <vol_name> <subvol_name> [--group_name <subvol_group_name>]
+
+Remove custom metadata set on the subvolume using the metadata key::
+
+    $ ceph fs subvolume metadata rm <vol_name> <subvol_name> <key_name> [--group_name <subvol_group_name>] [--force]
+
+Using the '--force' flag allows the command to succeed that would otherwise
+fail if the metadata key did not exist.
+
 Create a snapshot of a subvolume using::
 
     $ ceph fs subvolume snapshot create <vol_name> <subvol_name> <snap_name> [--group_name <subvol_group_name>]
@@ -261,16 +381,73 @@ List snapshots of a subvolume using::
 
     $ ceph fs subvolume snapshot ls <vol_name> <subvol_name> [--group_name <subvol_group_name>]
 
-Fetch the metadata of a snapshot using::
+Fetch the information of a snapshot using::
 
     $ ceph fs subvolume snapshot info <vol_name> <subvol_name> <snap_name> [--group_name <subvol_group_name>]
 
-The output format is json and contains fields as follows.
+The output format is JSON and contains fields as follows.
 
 * created_at: time of creation of snapshot in the format "YYYY-MM-DD HH:MM:SS:ffffff"
 * data_pool: data pool the snapshot belongs to
 * has_pending_clones: "yes" if snapshot clone is in progress otherwise "no"
-* size: snapshot size in bytes
+* pending_clones: list of in progress or pending clones and their target group if exist otherwise this field is not shown
+* orphan_clones_count: count of orphan clones if snapshot has orphan clones otherwise this field is not shown
+
+Sample output when snapshot clones are in progress or pending state::
+
+  $ ceph fs subvolume snapshot info cephfs subvol snap
+  {
+      "created_at": "2022-06-14 13:54:58.618769",
+      "data_pool": "cephfs.cephfs.data",
+      "has_pending_clones": "yes",
+      "pending_clones": [
+          {
+              "name": "clone_1",
+              "target_group": "target_subvol_group"
+          },
+          {
+              "name": "clone_2"
+          },
+          {
+              "name": "clone_3",
+              "target_group": "target_subvol_group"
+          }
+      ]
+  }
+
+Sample output when no snapshot clone is in progress or pending state::
+
+  $ ceph fs subvolume snapshot info cephfs subvol snap
+  {
+      "created_at": "2022-06-14 13:54:58.618769",
+      "data_pool": "cephfs.cephfs.data",
+      "has_pending_clones": "no"
+  }
+
+Set custom metadata on the snapshot as a key-value pair using::
+
+    $ ceph fs subvolume snapshot metadata set <vol_name> <subvol_name> <snap_name> <key_name> <value> [--group_name <subvol_group_name>]
+
+.. note:: If the key_name already exists then the old value will get replaced by the new value.
+
+.. note:: The key_name and value should be a string of ASCII characters (as specified in python's string.printable). The key_name is case-insensitive and always stored in lower case.
+
+.. note:: Custom metadata on a snapshots is not preserved when snapshotting the subvolume, and hence, is also not preserved when cloning the subvolume snapshot.
+
+Get custom metadata set on the snapshot using the metadata key::
+
+    $ ceph fs subvolume snapshot metadata get <vol_name> <subvol_name> <snap_name> <key_name> [--group_name <subvol_group_name>]
+
+List custom metadata (key-value pairs) set on the snapshot using::
+
+    $ ceph fs subvolume snapshot metadata ls <vol_name> <subvol_name> <snap_name> [--group_name <subvol_group_name>]
+
+Remove custom metadata set on the snapshot using the metadata key::
+
+    $ ceph fs subvolume snapshot metadata rm <vol_name> <subvol_name> <snap_name> <key_name> [--group_name <subvol_group_name>] [--force]
+
+Using the '--force' flag allows the command to succeed that would otherwise
+fail if the metadata key did not exist.
 
 Cloning Snapshots
 -----------------
@@ -323,8 +500,14 @@ A clone can be in one of the following states:
 #. `in-progress` : Clone operation is in progress
 #. `complete`    : Clone operation has successfully finished
 #. `failed`      : Clone operation has failed
+#. `canceled`    : Clone operation is cancelled by user
 
-Sample output from an `in-progress` clone operation::
+The reason for a clone failure is shown as below:
+
+#. `errno`     : error number
+#. `error_msg` : failure error string
+
+Sample output of an `in-progress` clone operation::
 
   $ ceph fs subvolume snapshot clone cephfs subvol1 snap1 clone1
   $ ceph fs clone status cephfs clone1
@@ -339,6 +522,28 @@ Sample output from an `in-progress` clone operation::
     }
   }
 
+.. note:: The `failure` section will be shown only if the clone is in failed or cancelled state
+
+Sample output of a `failed` clone operation::
+
+  $ ceph fs subvolume snapshot clone cephfs subvol1 snap1 clone1
+  $ ceph fs clone status cephfs clone1
+  {
+    "status": {
+      "state": "failed",
+      "source": {
+        "volume": "cephfs",
+        "subvolume": "subvol1",
+        "snapshot": "snap1"
+        "size": "104857600"
+      },
+      "failure": {
+        "errno": "122",
+        "errstr": "Disk quota exceeded"
+      }
+    }
+  }
+
 (NOTE: since `subvol1` is in default group, `source` section in `clone status` does not include group name)
 
 .. note:: Cloned subvolumes are accessible only after the clone operation has successfully completed.

ceph/doc/cephfs/mds-state-diagram.dot

@@ -57,6 +57,7 @@ D0 -> N3 [color=red,penwidth=2.0];
 // terminal (but not "in")
 node [shape=polygon,sides=6,color=black,peripheries=1];
 D1 [label="down:damaged"]
+S2 -> D1 [color=black,penwidth=2.0];
 N3 -> D1 [color=black,penwidth=2.0];
 N4 -> D1 [color=black,penwidth=2.0];
 N5 -> D1 [color=black,penwidth=2.0];
@@ -69,5 +70,6 @@ D1 -> D0 [color=red,penwidth=2.0]
 node [shape=polygon,sides=6,color=purple,peripheries=1];
 D3 [label="down:stopped"]
 S3 -> D3 [color=purple,penwidth=2.0];
+N6 -> D3 [color=purple,penwidth=2.0];
 
 }

ceph/doc/cephfs/posix.rst

@@ -31,7 +31,7 @@ POSIX semantics for various reasons:
   writes are not coherently propagated to other clients' caches.  That
   is, if a page is cached on host A, and then updated on host B, host
   A's page is not coherently invalidated.  (Shared writable mmap
-  appears to be quite rare--we have yet to here any complaints about this
+  appears to be quite rare--we have yet to hear any complaints about this
   behavior, and implementing cache coherency properly is complex.)
 - CephFS clients present a hidden ``.snap`` directory that is used to
   access, create, delete, and rename snapshots.  Although the virtual
@@ -62,17 +62,15 @@ as client data may not even be flushed to the server until the file is
 closed (and more generally writes will be significantly more
 time-shifted than CephFS, leading to less predictable results).
 
-However, all of there are very close to POSIX, and most of the time
+Regardless, these are all similar enough to POSIX, and applications still work
-applications don't notice too much.  Many other storage systems (e.g.,
+most of the time. Many other storage systems (e.g., HDFS) claim to be
-HDFS) claim to be "POSIX-like" but diverge significantly from the
+"POSIX-like" but diverge significantly from the standard by dropping support
-standard by dropping support for things like in-place file
+for things like in-place file modifications, truncate, or directory renames.
-modifications, truncate, or directory renames.
-
 
 Bottom line
 -----------
 
-CephFS relaxes more than local Linux kernel file systems (e.g., writes
+CephFS relaxes more than local Linux kernel file systems (for example, writes
 spanning object boundaries may be torn).  It relaxes strictly less
 than NFS when it comes to multiclient consistency, and generally less
 than NFS when it comes to write atomicity.

ceph/doc/cephfs/quota.rst

@@ -41,6 +41,17 @@ Limitations
    the directory the client is restricted too (e.g., ``/home/user``)
    or something nested beneath it.
 
+   Kernel clients need access to the parent of the directory inode on
+   which quotas are configured in order to enforce them. If quota is
+   configured on a directory path (e.g., ``/home/volumes/group``), the
+   kclient needs to have access to the parent (e.g., ``/home/volumes``).
+
+   An example command to create such an user is as below::
+
+     $ ceph auth get-or-create client.guest mds 'allow r path=/home/volumes, allow rw path=/home/volumes/group' mgr 'allow rw' osd 'allow rw tag cephfs metadata=*' mon 'allow r'
+
+   See also: https://tracker.ceph.com/issues/55090
+
 #. *Snapshot file data which has since been deleted or changed does not count
    towards the quota.* See also: http://tracker.ceph.com/issues/24284
 

ceph/doc/cephfs/snap-schedule.rst

@@ -1,3 +1,5 @@
+.. _snap-schedule:
+
 ==========================
 Snapshot Scheduling Module
 ==========================
@@ -47,10 +49,9 @@ The following time periods are recognized: `h(our), d(ay), w(eek), m(onth),
 y(ear)` and `n`. The latter is a special modifier where e.g. `10n` means keep
 the last 10 snapshots regardless of timing,
 
-All subcommands take optional `fs` and `subvol` arguments to specify paths in
+All subcommands take optional `fs` argument to specify paths in
 multi-fs setups and :doc:`/cephfs/fs-volumes` managed setups. If not
-passed `fs` defaults to the first file system listed in the fs_map, `subvolume`
+passed `fs` defaults to the first file system listed in the fs_map.
-defaults to nothing.
 When using :doc:`/cephfs/fs-volumes` the argument `fs` is equivalent to a
 `volume`.
 
@@ -64,16 +65,21 @@ When no subcommand is supplied a synopsis is printed::
 
   #> ceph fs snap-schedule
   no valid command found; 8 closest matches:
-  fs snap-schedule status [<path>] [<subvol>] [<fs>] [<format>]
+  fs snap-schedule status [<path>] [<fs>] [<format>]
-  fs snap-schedule list <path> [<subvol>] [--recursive] [<fs>] [<format>]
+  fs snap-schedule list <path> [--recursive] [<fs>] [<format>]
-  fs snap-schedule add <path> <snap_schedule> [<start>] [<fs>] [<subvol>]
+  fs snap-schedule add <path> <snap_schedule> [<start>] [<fs>]
-  fs snap-schedule remove <path> [<repeat>] [<start>] [<subvol>] [<fs>]
+  fs snap-schedule remove <path> [<repeat>] [<start>] [<fs>]
-  fs snap-schedule retention add <path> <retention_spec_or_period> [<retention_count>] [<fs>] [<subvol>]
+  fs snap-schedule retention add <path> <retention_spec_or_period> [<retention_count>] [<fs>]
-  fs snap-schedule retention remove <path> <retention_spec_or_period> [<retention_count>] [<fs>] [<subvol>]
+  fs snap-schedule retention remove <path> <retention_spec_or_period> [<retention_count>] [<fs>]
-  fs snap-schedule activate <path> [<repeat>] [<start>] [<subvol>] [<fs>]
+  fs snap-schedule activate <path> [<repeat>] [<start>] [<fs>]
-  fs snap-schedule deactivate <path> [<repeat>] [<start>] [<subvol>] [<fs>]
+  fs snap-schedule deactivate <path> [<repeat>] [<start>] [<fs>]
   Error EINVAL: invalid command
 
+Note:
+^^^^^
+A `subvolume` argument is no longer accepted by the commands.
+
+
 Inspect snapshot schedules
 --------------------------
 

ceph/doc/conf.py

@@ -109,6 +109,9 @@ extensions = [
 
 ditaa = shutil.which("ditaa")
 if ditaa is not None:
+    # in case we don't have binfmt_misc enabled or jar is not registered
+    ditaa_args = ['-jar', ditaa]
+    ditaa = 'java'
     extensions += ['sphinxcontrib.ditaa']
 else:
     extensions += ['plantweb.directive']

ceph/doc/dev/cephadm/host-maintenance.rst

@@ -32,8 +32,8 @@ The ceph orch command will be extended to support maintenance.
 
 .. code-block::
 
-    ceph orch host enter-maintenance <host> [ --check ]
+    ceph orch host maintenance enter <host> [ --force ]
-    ceph orch host exit-maintenance <host> 
+    ceph orch host maintenance exit <host>
 
 .. note:: In addition, the host's status should be updated to reflect whether it
    is in maintenance or not. 

ceph/doc/dev/continuous-integration.rst

@@ -0,0 +1,285 @@
+Continuous Integration Architecture
+===================================
+
+In Ceph, we rely on multiple CI pipelines in our development. Most of these pipelines
+are centered around Jenkins. And their configurations are generated using `Jenkins Job Builder`_.
+
+.. _Jenkins Job Builder: https://docs.openstack.org/infra/jenkins-job-builder/
+
+Let's take the ``make check`` performed by Jenkins as an example.
+
+ceph-pull-requests
+------------------
+
+``ceph-pull-requests`` is a jenkins job which gets triggered by a GitHub pull
+request or a trigger phrase like::
+
+    jenkins test make check
+
+There are multiple parties involved in this jenkins job:
+
+.. graphviz::
+
+   digraph  {
+     rankdir="LR";
+     github [
+      label="<git> git_repo | <webhooks> webhooks | <api> api";
+      shape=record;
+      href="https://github.com/ceph/ceph";
+     ];
+     subgraph cluster_lab {
+       label="Sepia Lab";
+       href="https://wiki.sepia.ceph.com/doku.php";
+       shape=circle;
+       apt_mirror [
+         href="http://apt-mirror.front.sepia.ceph.com";
+       ];
+       shaman [
+         href="https://shaman.ceph.com";
+       ];
+       chacra [
+         peripheries=3;
+         href="https://chacra.ceph.com";
+       ];
+       subgraph cluster_jenkins {
+         label="jenkins";
+         href="https://jenkins.ceph.com";
+         jenkins_controller [ label = "controller" ];
+         jenkins_agents [ label = "agents", peripheries=3 ];
+       };
+     };
+     {
+       rank=same;
+       package_repos [ peripheries=3 ];
+       pypi;
+       npm;
+     }
+     github:webhooks -> jenkins_controller [ label = "notify", color = "crimson" ];
+     jenkins_controller -> jenkins_agents [ label = "schedule jobs" ];
+     jenkins_agents -> github:git [ label = "git pull" ];
+     jenkins_agents -> shaman [ label = "query for chacra repo URL" ];
+     jenkins_agents -> chacra [ label = "pull build dependencies" ];
+     jenkins_agents -> package_repos [ label = "pull build dependencies" ];
+     jenkins_agents -> pypi [ label = "pull Python packages" ];
+     jenkins_agents -> npm [ label = "pull JavaScript packages" ];
+     jenkins_agents -> apt_mirror [ label = "pull build dependencies" ];
+     jenkins_agents -> github:api [ label = "update", color = "crimson" ];
+   }
+
+Where
+
+Sepia Lab
+   `Sepia Lab`_ is a test lab used by the Ceph project. This lab offers
+   the storage and computing resources required by our CI infra.
+
+Jenkins agents
+   are a set of machines which perform the CI jobs. In this case, they
+
+   #. pull the git repo from GitHub and
+   #. rebase the pull request against the latest master
+   #. set necessary environment variables
+   #. run ``run-make-check.sh``
+
+Chacra
+   is a server offering RESTful API allowing the clients to store and
+   retrieve binary packages. It also creates the repo for uploaded
+   packages automatically. Once a certain repo is created on chacra, the
+   configured shaman server is updated as well, then we can query shaman
+   for the corresponding repo address. Chacra not only hosts Ceph packages,
+   it also hosts quite a few other packages like various build dependencies.
+
+Shaman
+   is a server offering RESTful API allowing the clients to query the
+   information of repos hosted by chacra nodes. Shaman is also known
+   for its `Web UI`_. But please note, shaman does not build the
+   packages, it just offers information on the builds.
+
+As the following shows, `chacra`_ manages multiple projects whose metadata
+are stored in a database. These metadata are exposed via Shaman as a web
+service. `chacractl`_ is a utility to interact with the `chacra`_ service.
+
+.. graphviz::
+
+   digraph  {
+     libboost [
+       shape=cylinder;
+     ];
+     libzbd [
+       shape=cylinder;
+     ];
+     other_repos [
+       label="...";
+       shape=cylinder;
+     ];
+     postgresql [
+       shape=cylinder;
+       style=filled;
+     ]
+     shaman -> postgresql;
+     chacra -> postgresql;
+     chacractl -> chacra;
+     chacra -> libboost;
+     chacra -> libzbd;
+     chacra -> other_repos;
+   }
+
+.. _Sepia Lab: https://wiki.sepia.ceph.com/doku.php
+.. _Web UI: https://shaman.ceph.com
+
+build dependencies
+------------------
+
+Just like lots of other software projects, Ceph has both build-time and
+run-time dependencies. Most of time, we are inclined to use the packages
+prebuilt by the distro. But there are cases where
+
+- the necessary dependencies are either missing in the distro, or
+- their versions are too old, or
+- they are packaged without some important feature enabled.
+- we want to ensure that the version of a certain runtime dependency is
+  identical to the one we tested in our lab.
+
+No matter what the reason is, we either need to build them from source, or
+to package them as binary packages instead of using the ones shipped by the
+distro. Quite a few build-time dependencies are included as git submodules,
+but in order to avoid rebuilding these dependencies repeatedly, we pre-built
+some of them and uploaded them to our own repos. So, when performing
+``make check``, the building hosts in our CI just pull them from our internal
+repos hosting these packages instead of building them.
+
+So far, following packages are prebuilt for ubuntu focal, and then uploaded to
+`chacra`_:
+
+libboost
+    packages `boost`_. The packages' names are changed from ``libboost-*`` to
+    ``ceph-libboost-*``, and they are instead installed into ``/opt/ceph``, so
+    they don't interfere with the official ``libboost`` packages shipped by
+    distro. Its build scripts are hosted at https://github.com/ceph/ceph-boost.
+    See https://github.com/ceph/ceph-boost/commit/2a8ae02932b2a1fd6a68072da8ca0df2b99b805c
+    for an example of how to bump the version number. The commands used to
+    build 1.79 on a vanilla Ubuntu Focal OS are below.
+
+    .. prompt:: bash $
+
+       sudo apt install debhelper dctrl-tools chrpath libbz2-dev libicu-dev bison \
+         flex docbook-to-man help2man xsltproc doxygen dh-python python3-all-dev graphviz
+       wget http://download.ceph.com/qa/boost_1_79_0.tar.bz2
+       git clone https://github.com/ceph/ceph-boost
+       tar xjf boost_1_79_0.tar.bz2
+       cp -ra ceph-boost/debian boost_1_79_0/
+       pushd boost_1_79_0
+       export DEB_BUILD_OPTIONS='parallel=6 nodoc'
+       dpkg-buildpackage -us -uc -b
+       popd
+       BOOST_SHA=$(git ls-remote https://github.com/ceph/ceph-boost main | awk '{ print $1 }')
+       ls *.deb | chacractl binary create \
+         libboost/master/$BOOST_SHA/ubuntu/focal/amd64/flavors/default
+
+libzbd
+    packages `libzbd`_ . The upstream libzbd includes debian packaging already.
+
+libpmem
+    packages `pmdk`_ . Please note, ``ndctl`` is one of the build dependencies of
+    pmdk, for an updated debian packaging, please see
+    https://github.com/ceph/ceph-ndctl .
+
+.. note::
+
+   please ensure that the package version and the release number of the
+   packaging are properly updated when updating/upgrading the packaging,
+   otherwise it would be difficult to tell which version of the package
+   is installed. We check the package version before trying to upgrade
+   it in ``install-deps.sh``.
+
+.. _boost: https://www.boost.org
+.. _libzbd: https://github.com/westerndigitalcorporation/libzbd
+.. _pmdk: https://github.com/pmem/pmdk
+
+But in addition to these libraries, ``ceph-mgr-dashboard``'s frontend uses lots of
+JavaScript packages. Quite a few of them are not packaged by distros. Not to
+mention the trouble of testing different combination of versions of these
+packages. So we decided to include these JavaScript packages in our dist tarball
+using ``make-dist``.
+
+Also, because our downstream might not want to use the prepackaged binaries when
+redistributing the precompiled Ceph packages, we also need to include these
+libraries in our dist tarball. They are
+
+- boost
+- liburing
+- pmdk
+
+``make-dist`` is a script used by our CI pipeline to create dist tarball so the
+tarball can be used to build the Ceph packages in a clean room environment. When
+we need to upgrade these third party libraries, we should
+
+- update the CMake script
+- rebuild the prebuilt packages and
+- update this script to reflect the change.
+
+Uploading Dependencies
+----------------------
+
+To ensure that prebuilt packages are available by the jenkins agents, we need to
+upload them to either ``apt-mirror.front.sepia.ceph.com`` or `chacra`_. To upload
+packages to the former would require the help of our lab administrator, so if we
+want to maintain the package repositories on regular basis, a better choice would be
+to manage them using `chacractl`_. `chacra`_ represents packages repositories using
+a resource hierarchy, like::
+
+  <project>/<branch>/<ref>/<distro>/<distro-version>/<arch>
+
+In which:
+
+project
+    in general, it is used for denoting a set of related packages. For instance,
+    ``libboost``.
+
+branch
+    branch of project. This mirrors the concept of a Git repo.
+
+ref
+    a unique id of a given version of a set packages. This id is used to reference
+    the set packages under the ``<project>/<branch>``. It is a good practice to
+    version the packaging recipes, like the ``debian`` directory for building DEB
+    packages and the ``spec`` for building RPM packages, and use the SHA1 of the
+    packaging recipe for the ``ref``. But you could also use a random string for
+    ``ref``, like the tag name of the built source tree.
+
+distro
+    the distro name for which the packages are built. Currently, following distros are
+    supported:
+
+    - centos
+    - debian
+    - fedora
+    - rhel
+    - ubuntu
+
+distro-version
+    the version of the distro. For instance, if a package is built on ubuntu focal,
+    the ``distro-version`` should be ``20.04``.
+
+arch
+    the architecture of the packages. It could be:
+
+    - arm64
+    - amd64
+    - noarch
+
+So, for example, we can upload the prebuilt boost packages to chacra like
+
+.. prompt:: bash $
+
+   ls *.deb | chacractl binary create \
+     libboost/master/099c0fd56b4a54457e288a2eff8fffdc0d416f7a/ubuntu/focal/amd64/flavors/default
+
+.. _chacra: https://github.com/ceph/chacra
+.. _chacractl: https://github.com/ceph/chacractl
+
+Update ``install-deps.sh``
+--------------------------
+
+We also need to update ``install-deps.sh`` to point the built script to the new
+repo. Please refer to the `script <https://github.com/ceph/ceph/blob/master/install-deps.sh>`_,
+for more details.

ceph/doc/dev/deduplication.rst

@@ -1,151 +0,0 @@
-===============
- Deduplication
-===============
-
-
-Introduction
-============
-
-Applying data deduplication on an existing software stack is not easy 
-due to additional metadata management and original data processing 
-procedure. 
-
-In a typical deduplication system, the input source as a data
-object is split into multiple chunks by a chunking algorithm.
-The deduplication system then compares each chunk with
-the existing data chunks, stored in the storage previously.
-To this end, a fingerprint index that stores the hash value
-of each chunk is employed by the deduplication system
-in order to easily find the existing chunks by comparing
-hash value rather than searching all contents that reside in
-the underlying storage.
-
-There are many challenges in order to implement deduplication on top
-of Ceph. Among them, two issues are essential for deduplication.
-First is managing scalability of fingerprint index; Second is
-it is complex to ensure compatibility between newly applied
-deduplication metadata and existing metadata.
-
-Key Idea
-========
-1. Content hashing (Double hashing): Each client can find an object data 
-for an object ID using CRUSH. With CRUSH, a client knows object's location
-in Base tier. 
-By hashing object's content at Base tier, a new OID (chunk ID) is generated.
-Chunk tier stores in the new OID that has a partial content of original object.
-
- Client 1 -> OID=1 -> HASH(1's content)=K -> OID=K -> 
- CRUSH(K) -> chunk's location
-
-
-2. Self-contained object: The external metadata design
-makes difficult for integration with storage feature support
-since existing storage features cannot recognize the
-additional external data structures. If we can design data
-deduplication system without any external component, the
-original storage features can be reused.
-
-More details in https://ieeexplore.ieee.org/document/8416369
-
-Design
-======
-
-.. ditaa::
-
-           +-------------+
-           | Ceph Client |
-           +------+------+
-                  ^
-     Tiering is   |  
-    Transparent   |               Metadata
-        to Ceph   |           +---------------+
-     Client Ops   |           |               |   
-                  |    +----->+   Base Pool   |
-                  |    |      |               |
-                  |    |      +-----+---+-----+
-                  |    |            |   ^ 
-                  v    v            |   |   Dedup metadata in Base Pool
-           +------+----+--+         |   |   (Dedup metadata contains chunk offsets
-           |   Objecter   |         |   |    and fingerprints)
-           +-----------+--+         |   |
-                       ^            |   |   Data in Chunk Pool
-                       |            v   |
-                       |      +-----+---+-----+
-                       |      |               |
-                       +----->|  Chunk Pool   |
-                              |               |
-                              +---------------+
-                                    Data
-
-
-Pool-based object management:
-We define two pools.
-The metadata pool stores metadata objects and the chunk pool stores
-chunk objects. Since these two pools are divided based on
-the purpose and usage, each pool can be managed more
-efficiently according to its different characteristics. Base
-pool and the chunk pool can separately select a redundancy
-scheme between replication and erasure coding depending on
-its usage and each pool can be placed in a different storage
-location depending on the required performance.
-
-Regarding how to use, please see ``osd_internals/manifest.rst``
-
-Usage Patterns
-==============
-
-The different Ceph interface layers present potentially different oportunities
-and costs for deduplication and tiering in general.
-
-RadosGW
--------
-
-S3 big data workloads seem like a good opportunity for deduplication.  These
-objects tend to be write once, read mostly objects which don't see partial
-overwrites.  As such, it makes sense to fingerprint and dedup up front.
-
-Unlike cephfs and rbd, radosgw has a system for storing
-explicit metadata in the head object of a logical s3 object for
-locating the remaining pieces.  As such, radosgw could use the
-refcounting machinery (``osd_internals/refcount.rst``) directly without
-needing direct support from rados for manifests.
-
-RBD/Cephfs
-----------
-
-RBD and CephFS both use deterministic naming schemes to partition
-block devices/file data over rados objects.  As such, the redirection
-metadata would need to be included as part of rados, presumably
-transparently.
-
-Moreover, unlike radosgw, rbd/cephfs rados objects can see overwrites.
-For those objects, we don't really want to perform dedup, and we don't
-want to pay a write latency penalty in the hot path to do so anyway.
-As such, performing tiering and dedup on cold objects in the background
-is likely to be preferred.
-   
-One important wrinkle, however, is that both rbd and cephfs workloads
-often feature usage of snapshots.  This means that the rados manifest
-support needs robust support for snapshots.
-
-RADOS Machinery
-===============
-
-For more information on rados redirect/chunk/dedup support, see ``osd_internals/manifest.rst``.
-For more information on rados refcount support, see ``osd_internals/refcount.rst``.
-
-Status and Future Work
-======================
-
-At the moment, there exists some preliminary support for manifest
-objects within the OSD as well as a dedup tool.
-
-RadosGW data warehouse workloads probably represent the largest
-opportunity for this feature, so the first priority is probably to add
-direct support for fingerprinting and redirects into the refcount pool
-to radosgw.
-
-Aside from radosgw, completing work on manifest object support in the
-OSD particularly as it relates to snapshots would be the next step for
-rbd and cephfs workloads.
-

ceph/doc/dev/delayed-delete.rst

@@ -2,11 +2,12 @@
  CephFS delayed deletion
 =========================
 
-When you delete a file, the data is not immediately removed. Each
+The deletion of a file does not immediately remove its data. Each of the file's
-object in the file needs to be removed independently, and sending
+underlying objects must be removed independently. If these objects were removed 
-``size_of_file / stripe_size * replication_count`` messages would slow
+immediately, the client would have to send ``size_of_file / stripe_size *
-the client down too much, and use a too much of the clients
+replication_count`` messages. This would consume significant bandwith and would
-bandwidth. Additionally, snapshots may mean some objects should not be
+slow the client unacceptably. If snapshots exist, their existence can prevent
-deleted.
+the deletion of objects associated with them.
 
-Instead, the file is marked as deleted on the MDS, and deleted lazily.
+In these cases, such files are (1) marked as deleted on the MDS and (2) deleted
+lazily.

ceph/doc/dev/developer_guide/basic-workflow.rst

@@ -1,3 +1,5 @@
+.. _basic workflow dev guide:
+
 Basic Workflow
 ==============
 
@@ -8,7 +10,7 @@ The following chart illustrates the basic Ceph development workflow:
             Upstream Code                       Your Local Environment
 
            /----------\        git clone           /-------------\
-           |   Ceph   | -------------------------> | ceph/master |
+           |   Ceph   | -------------------------> | ceph/main   |
            \----------/                            \-------------/
                 ^                                    |
                 |                                    | git branch fix_1
@@ -29,50 +31,79 @@ The following chart illustrates the basic Ceph development workflow:
            \--------------/
 
 This page assumes that you are a new contributor with an idea for a bugfix or
-enhancement, but do not know how to proceed. Watch the `Getting Started with
+an enhancement, but you do not know how to proceed. Watch the `Getting Started
-Ceph Development <https://www.youtube.com/watch?v=t5UIehZ1oLs>`_ video for a
+with Ceph Development <https://www.youtube.com/watch?v=t5UIehZ1oLs>`_ video for
-practical summary of this workflow.
+a practical summary of this workflow.
 
 Updating the tracker
 --------------------
 
-Before you start, you should know the :ref:`issue-tracker` (Redmine) number
+Find the :ref:`issue-tracker` (Redmine) number of the bug you intend to fix. If
-of the bug you intend to fix. If there is no tracker issue, now is the time to
+no tracker issue exists, create one. There is only one case in which you do not
-create one for code changes.  Straightforward documentation cleanup does
+have to create a Redmine tracker issue: the case of minor documentation changes.
-not necessarily require a corresponding tracker issue. However, an issue
-(ticket) should be created if one is adding new documentation chapters or
-files, or for other substantial changes.
 
-The tracker ticket serves to explain the issue (bug) to your fellow Ceph
+Simple documentation cleanup does not require a corresponding tracker issue.
-developers and keep them informed as you make progress toward resolution.  To
+Major documenatation changes do require a tracker issue. Major documentation
-this end, please provide a descriptive title and write appropriate information
+changes include adding new documentation chapters or files, and making 
-and details into the description.  When composing the ticket's title, consider "If I
+substantial changes to the structure or content of the documentation.
-want to search for this ticket two years from now, what keywords will I search
-for?"
 
-If you have sufficient tracker permissions, assign the bug to yourself by
+A (Redmine) tracker ticket explains the issue (bug) to other Ceph developers to
-setting the ``Assignee`` field.  If your tracker permissions have not been
+keep them informed as the bug nears resolution. Provide a useful, clear title
-elevated, simply add a comment with a short message like "I am working on this
+and include detailed information in the description. When composing the title
-issue".
+of the ticket, ask yourself "If I need to search for this ticket two years from
+now, which keywords am I likely to search for?" Then include those keywords in
+the title.
 
-Forking and Cloning the Ceph Repository
+If your tracker permissions are elevated, assign the bug to yourself by setting
----------------------------------------
+the ``Assignee`` field. If your tracker permissions have not been elevated,
+just add a comment with a short message that says "I am working on this issue".
 
-This section, and the ones that follow, correspond to nodes in the above chart.
+Ceph Workflow Overview
+----------------------
 
-The upstream code is found at https://github.com/ceph/ceph.git, which is known
+Three repositories are involved in the Ceph workflow. They are:
-as the "upstream repo", or simply "upstream". As the chart shows, we will make
-a local copy of this repository, modify it, test our modifications, then submit
-the modifications for review and merging.
 
-A local copy of the upstream code is made by
+1. The upstream repository (ceph/ceph)
+2. Your fork of the upstream repository (your_github_id/ceph)
+3. Your local working copy of the repository (on your workstation)
 
-1. Forking the upstream repo on GitHub, and
+The procedure for making changes to the Ceph repository is as follows:
-2. Cloning your fork to make a local working copy
 
+#. Configure your local environment
 
-Forking The Ceph Repository
+   #. :ref:`Create a fork<forking>` of the "upstream Ceph"
-^^^^^^^^^^^^^^^^^^^^^^^^^^^
+      repository.
+
+   #. :ref:`Clone the fork<cloning>` to your local filesystem.
+
+#. Fix the bug
+
+   #. :ref:`Synchronize local main with upstream main<synchronizing>`.
+         
+   #. :ref:`Create a bugfix branch<bugfix_branch>` in your local working copy.
+         
+   #. :ref:`Make alterations to the local working copy of the repository in your
+      local filesystem<fixing_bug_locally>`.
+   
+   #. :ref:`Push the changes in your local working copy to your fork<push_changes>`.
+
+#. Create a Pull Request to push the change upstream
+
+   #. Create a Pull Request that asks for your changes to be added into the
+      "upstream Ceph" repository.
+
+Preparing Your Local Working Copy of the Ceph Repository 
+--------------------------------------------------------
+
+The procedures in this section, "Preparing Your Local Working Copy of the Ceph
+Repository", must be followed only when you are first setting up your local
+environment. If this is your first time working with the Ceph project, then
+these commands are necessary and are the first commands that you should run.
+
+.. _forking:
+
+Creating a Fork of the Ceph Repository
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 See the `GitHub documentation
 <https://help.github.com/articles/fork-a-repo/#platform-linux>`_ for
@@ -80,148 +111,234 @@ detailed instructions on forking. In short, if your GitHub username is
 "mygithubaccount", your fork of the upstream repo will appear at
 ``https://github.com/mygithubaccount/ceph``. 
 
+.. _cloning:
+
 Cloning Your Fork  
 ^^^^^^^^^^^^^^^^^
 
-Once you have created your fork, clone it by running:
+After you have created your fork, clone it by running the following command:
 
 .. prompt:: bash $
 
    git clone https://github.com/mygithubaccount/ceph
 
-You must fork the Ceph repository before you clone it.  Without forking, you cannot 
+You must fork the Ceph repository before you clone it.  If you fail to fork,
-open a `GitHub pull request
+you cannot open a `GitHub pull request
 <https://docs.github.com/en/free-pro-team@latest/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request>`_.
 
 For more information on using GitHub, refer to `GitHub Help
 <https://help.github.com/>`_.
 
 Configuring Your Local Environment
-----------------------------------
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-In the local environment created in the previous step, you now have a copy of
+The commands in this section configure your local git environment so that it
-the ``master`` branch in ``remotes/origin/master``. This fork
+generates "Signed-off-by:" tags. These commands also set up your local
-(https://github.com/mygithubaccount/ceph.git) is frozen in time and the
+environment so that it can stay synchronized with the upstream repository.
-upstream repo (https://github.com/ceph/ceph.git, typically abbreviated to
-``ceph/ceph.git``) is updated frequently by other contributors. This means that
-you must sync your fork periodically. Failure to synchronize your fork may
-result in your commits and pull requests failing to merge because they refer to
-file contents that have changed since you last synchronized your fork.
 
-Configure your local git environment with your name and email address.  
+These commands are necessary only during the initial setup of your local
+working copy. Another way to say that is "These commands are necessary
+only the first time that you are working with the Ceph repository. They are,
+however, unavoidable, and if you fail to run them then you will not be able
+to work on the Ceph repository.".
 
-.. prompt:: bash $
+1. Configure your local git environment with your name and email address.  
+
+   .. note::
+      These commands will work only from within the ``ceph/`` directory
+      that was created when you cloned your fork.
+
+   .. prompt:: bash $
 
       git config user.name "FIRST_NAME LAST_NAME"
       git config user.email "MY_NAME@example.com"
 
-Add the upstream repo as a "remote" and fetch it:
+2. Add the upstream repo as a "remote" and fetch it:
 
-.. prompt:: bash $
+   .. prompt:: bash $
 
       git remote add ceph https://github.com/ceph/ceph.git
       git fetch ceph
 
-Fetching is a process that downloads all objects (commits, branches) that have
+   These commands fetch all the branches and commits from ``ceph/ceph.git`` to
-been added since the last sync. These commands download all the branches from
+   the local git repo as ``remotes/ceph/$BRANCH_NAME`` and can be referenced as
-``ceph/ceph.git`` to the local git repo as ``remotes/ceph/$BRANCH_NAME`` and
+   ``ceph/$BRANCH_NAME`` in local git commands.
-can be referenced as ``ceph/$BRANCH_NAME`` in local git commands.
 
+Fixing the Bug
+--------------
 
-Resetting Local Master to Upstream Master
+.. _synchronizing:
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Your local ``master`` branch can be reset to the upstream Ceph ``master``
+Synchronizing Local Main with Upstream Main
-branch by running the following commands:
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In your local working copy, there is a copy of the ``main`` branch in
+``remotes/origin/main``. This is called "local main". This copy of the
+main branch (https://github.com/your_github_id/ceph.git) is "frozen in time"
+at the moment that you cloned it, but the upstream repo
+(https://github.com/ceph/ceph.git, typically abbreviated to ``ceph/ceph.git``)
+that it was forked from is not frozen in time: the upstream repo is still being
+updated by other contributors. 
+
+Because upstream main is continually receiving updates from other
+contributors, your fork will drift farther and farther from the state of the
+upstream repo when you cloned it.
+
+Keep your fork's ``main`` branch synchronized with upstream main to reduce drift
+between your fork's main branch and the upstream main branch.
+
+Here are the commands for keeping your fork synchronized with the
+upstream repository:
 
 .. prompt:: bash $
 
    git fetch ceph
-   git checkout master
+   git checkout main 
-   git reset --hard ceph/master
+   git reset --hard ceph/main
-   git push -u origin master
+   git push -u origin main
 
-This procedure should be followed often, in order to keep your local ``master``
+Follow this procedure often to keep your local ``main`` in sync with upstream
-in sync with upstream ``master``.
+``main``.
+
+If the command ``git status`` returns a line that reads "Untracked files", see
+:ref:`the procedure on updating submodules <update-submodules>`.
+
+.. _bugfix_branch:
 
 Creating a Bugfix branch
-------------------------
+^^^^^^^^^^^^^^^^^^^^^^^^
 
 Create a branch for your bugfix:
 
 .. prompt:: bash $
 
-   git checkout master
+   git checkout main 
    git checkout -b fix_1
    git push -u origin fix_1
 
-This creates a local branch called ``fix_1`` in our GitHub fork. At this point,
+The first command (git checkout main) makes sure that the bugfix branch
-the ``fix_1`` branch is identical to the ``master`` branch, but not for long!
+"fix_1" is created from the most recent state of the main branch of the
-You are now ready to modify the code.  Be careful to always run `git checkout
+upstream repository. 
-master` first, otherwise you may find commits from an unrelated branch mixed
-with your new work.
 
-Fixing the bug locally
+The second command (git checkout -b fix_1) creates a "bugfix branch" called
-----------------------
+"fix_1" in your local working copy of the repository. The changes that you make
+in order to fix the bug will be commited to this branch.
 
-In the `Ceph issue tracker <https://tracker.ceph.com>`_, change the status of
+The third command (git push -u origin fix_1) pushes the bugfix branch from
-the tracker issue to "In progress".  This communicates to other Ceph
+your local working repository to your fork of the upstream repository.
-contributors that you have begun working on a fix, which helps to avoid
-duplication of effort.  If you don't have permission to change that field, your
-previous comment that you are working on the issue is sufficient.
 
-Your fix may be very simple and require only minimal testing. But that's not
+.. _fixing_bug_locally:
-likely. It is more likely that the process of fixing your bug will be iterative
-and will involve trial and error, as well as skill. An explanation of how to
-fix bugs is beyond the scope of this document. Instead, we focus on the
-mechanics of the process in the context of the Ceph project.
 
-For a detailed discussion of the tools available for validating bugfixes,
+Fixing the bug in the local working copy
-see the chapters on testing.
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-For now, let us assume that you have finished work on the bugfix, that you have
+#. **Updating the tracker**
-tested the bugfix , and that you believe that it works. Commit the changes to
-your local branch using the ``--signoff`` option (here represented as the `s`
-portion of the `-as` flag): 
 
-.. prompt:: bash $
+   In the `Ceph issue tracker <https://tracker.ceph.com>`_, change the status
+   of the tracker issue to "In progress".  This communicates to other Ceph
+   contributors that you have begun working on a fix, which helps to avoid
+   duplication of effort. If you don't have permission to change that field,
+   just comment that you are working on the issue. 
+
+#. **Fixing the bug itself**
+
+   This guide cannot tell you how to fix the bug that you have chosen to fix.
+   This guide assumes that you know what required improvement, and that you
+   know what to do to provide that improvement.
+
+   It might be that your fix is simple and requires only minimal testing. But
+   that's unlikely. It is more likely that the process of fixing your bug will
+   be iterative and will involve trial, error, skill, and patience. 
+
+   For a detailed discussion of the tools available for validating bugfixes,
+   see the chapters on testing.
+
+Pushing the Fix to Your Fork
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   
+You have finished work on the bugfix. You have tested the bugfix, and you
+believe that it works. 
+   
+#. Commit the changes to your local working copy.
+
+   Commit the changes to the `fix_1` branch of your local working copy by using
+   the ``--signoff`` option (here represented as the `s` portion of the `-as`
+   flag): 
+
+   .. prompt:: bash $
 
       git commit -as
 
-Push the changes to your fork:
+   .. _push_changes:
 
-.. prompt:: bash $
+#. Push the changes to your fork:
+
+   Push the changes from the `fix_1` branch of your local working copy to the
+   `fix_1` branch of your fork of the upstream repository:
+
+   .. prompt:: bash $
 
       git push origin fix_1
    
+   .. note::
+
+      In the command ``git push origin fix_1``, ``origin`` is the name of your
+      fork of the upstream Ceph repository, and can be thought of as a nickname
+      for ``git@github.com:username/ceph.git``, where ``username`` is your
+      GitHub username.
+
+      It is possible that ``origin`` is not the name of your fork. Discover the
+      name of your fork by running ``git remote -v``, as shown here:
+
+      .. code-block:: bash
+
+         $ git remote -v
+         ceph	https://github.com/ceph/ceph.git (fetch)
+         ceph	https://github.com/ceph/ceph.git (push)
+         origin	git@github.com:username/ceph.git (fetch)
+         origin	git@github.com:username/ceph.git (push)
+
+      The line::
+     
+         origin git@github.com:username/ceph.git (fetch) 
+      
+      and the line:: 
+        
+         origin git@github.com:username/ceph.git (push) 
+         
+      provide the information that "origin" is the name of your fork of the
+      Ceph repository.
+
+
 Opening a GitHub pull request
 -----------------------------
 
-The next step is to open a GitHub pull request (PR). This makes your bugfix
+After you have pushed the bugfix to your fork, open a GitHub pull request
-visible to the community of Ceph contributors.  They will review it and may
+(PR). This makes your bugfix visible to the community of Ceph contributors.
-perform additional testing and / or request changes.
+They will review it. They may perform additional testing on your bugfix, and
+they might request changes to the bugfix.
 
-This is the point where you "go public" with your modifications.  Be prepared
+Be prepared to receive suggestions and constructive criticism in the form of
-to receive suggestions and constructive criticism in the form of comments
+comments within the PR. 
-within the PR. Don't worry!  The Ceph project is a friendly place!
 
-If you are uncertain how to create and manage pull requests, you may read
+If you don't know how to create and manage pull requests, read `this GitHub
-`this GitHub pull request tutorial`_.
+pull request tutorial`_.
 
 .. _`this GitHub pull request tutorial`:
    https://help.github.com/articles/using-pull-requests/
 
-For ideas on what constitutes a "good" pull request, see
+To learn what constitutes a "good" pull request, see
 the `Git Commit Good Practice`_ article at the `OpenStack Project Wiki`_.
 
 .. _`Git Commit Good Practice`: https://wiki.openstack.org/wiki/GitCommitMessages
 .. _`OpenStack Project Wiki`: https://wiki.openstack.org/wiki/Main_Page
 
-and our own `Submitting Patches <https://github.com/ceph/ceph/blob/master/SubmittingPatches.rst>`_ document.
+See also our own `Submitting Patches
+<https://github.com/ceph/ceph/blob/main/SubmittingPatches.rst>`_ document.
 
-Once your pull request (PR) is opened, update the :ref:`issue-tracker` by
+After your pull request (PR) has been opened, update the :ref:`issue-tracker`
-adding a comment directing other contributors to your PR. The comment can be
+by adding a comment directing other contributors to your PR. The comment can be
-as simple as::
+as simple as this::
 
     *PR*: https://github.com/ceph/ceph/pull/$NUMBER_OF_YOUR_PULL_REQUEST
 
@@ -230,8 +347,8 @@ Understanding Automated PR validation
 
 When you create or update your PR, the Ceph project's `Continuous Integration
 (CI) <https://en.wikipedia.org/wiki/Continuous_integration>`_ infrastructure
-automatically tests it. At the time of this writing (September 2020), the
+automatically tests it. At the time of this writing (May 2022), the automated
-automated CI testing included five tests: 
+CI testing included many tests. These five are among them:
 
 #. a test to check that the commits are properly signed (see :ref:`submitting-patches`): 
 #. a test to check that the documentation builds
@@ -239,15 +356,15 @@ automated CI testing included five tests:
 #. a test to check that the API is in order
 #. a :ref:`make check<make-check>` test
 
-Additional tests may be performed depending on which files your PR modifies.
+Additional tests may be run depending on which files your PR modifies.
 
-The :ref:`make check<make-check>` test builds the PR and runs it through a battery of
+The :ref:`make check<make-check>` test builds the PR and runs it through a
-tests. These tests run on servers operated by the Ceph Continuous
+battery of tests. These tests run on servers that are operated by the Ceph
-Integration (CI) team. When the tests complete, the result will be shown
+Continuous Integration (CI) team. When the tests have completed their run, the
-on GitHub in the pull request itself.
+result is shown on GitHub in the pull request itself.
 
-You should test your modifications before you open a PR.
+Test your modifications before you open a PR.  Refer to the chapters
-Refer to the chapters on testing for details.
+on testing for details.
 
 Notes on PR make check test
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -255,26 +372,28 @@ Notes on PR make check test
 The GitHub :ref:`make check<make-check>` test is driven by a Jenkins instance.
 
 Jenkins merges your PR branch into the latest version of the base branch before
-starting tests. This means that you don't have to rebase the PR to pick up any fixes.
+it starts any tests. This means that you don't have to rebase the PR in order
+to pick up any fixes.
 
 You can trigger PR tests at any time by adding a comment to the PR - the
-comment should contain the string "test this please". Since a human subscribed
+comment should contain the string "test this please". Since a human who is
-to the PR might interpret that as a request for him or her to test the PR, we
+subscribed to the PR might interpret that as a request for him or her to test
-recommend that you address Jenkins directly. For example, write "jenkins retest
+the PR, you must address Jenkins directly. For example, write "jenkins retest
-this please".  For efficiency a single re-test can also be requested with
+this please". If you need to run only one of the tests, you can request it with
-e.g. "jenkins test signed".  For reference, a list of these requests is
+a command like "jenkins test signed". A list of these requests is automatically
-automatically added to the end of each new PR's description.
+added to the end of each new PR's description, so check there to find the
+single test you need.
 
 If there is a build failure and you aren't sure what caused it, check the
-:ref:`make check<make-check>` log. To access it, click on the "details" (next
+:ref:`make check<make-check>` log. To access the make check log, click the
-to the :ref:`make check<make-check>` test in the PR) link to enter the Jenkins web
+"details" (next to the :ref:`make check<make-check>` test in the PR) link to
-GUI. Then click on "Console Output" (on the left).
+enter the Jenkins web GUI. Then click "Console Output" (on the left).
 
-Jenkins is configured to search logs for strings known to have been associated
+Jenkins is configured to search logs for strings that are known to have been
-with :ref:`make check<make-check>` failures in the past. However, there is no
+associated with :ref:`make check<make-check>` failures in the past. However,
-guarantee that these known strings are associated with any given
+there is no guarantee that these known strings are associated with any given
-:ref:`make check<make-check>` failure. You'll have to read through the log to determine the
+:ref:`make check<make-check>` failure. You'll have to read through the log to
-cause of your specific failure.
+determine the cause of your specific failure.
 
 Integration tests AKA ceph-qa-suite
 -----------------------------------
@@ -284,7 +403,7 @@ see how it behaves on real clusters running on physical or virtual
 hardware. Tests designed for this purpose live in the `ceph/qa
 sub-directory`_ and are run via the `teuthology framework`_.
 
-.. _`ceph/qa sub-directory`: https://github.com/ceph/ceph/tree/master/qa/
+.. _`ceph/qa sub-directory`: https://github.com/ceph/ceph/tree/main/qa/
 .. _`teuthology repository`: https://github.com/ceph/teuthology
 .. _`teuthology framework`: https://github.com/ceph/teuthology
 
@@ -329,7 +448,7 @@ will need to force push your branch with:
    git push --force origin fix_1
 
 Why do we take these extra steps instead of simply adding additional commits
-the the PR?  It is best practice for a PR to consist of a single commit; this
+the PR?  It is best practice for a PR to consist of a single commit; this
 makes for clean history, eases peer review of your changes, and facilitates
 merges.  In rare circumstances it also makes it easier to cleanly revert
 changes.
@@ -388,7 +507,7 @@ Another method of generating merge commits involves using Patrick Donnelly's
 **/ceph/src/script/ptl-tool.py**.  Merge commits that have been generated by
 the **ptl-tool** have the following form::
 
-     Merge PR #36257 into master
+     Merge PR #36257 into main 
      * refs/pull/36257/head:
              client: move client_lock to _unmount()
              client: add timer_lock support

ceph/doc/dev/developer_guide/essentials.rst

@@ -139,8 +139,46 @@ using `Internet Relay Chat`_.
 
 .. _`Internet Relay Chat`: http://www.irchelp.org/
 
-See ``https://ceph.com/irc/`` for how to set up your IRC
+The Ceph community gathers in the #ceph channel of the Open and Free Technology
-client and a list of channels.
+Community (OFTC) IRC network.
+
+Created in 1988, Internet Relay Chat (IRC) is a relay-based, real-time chat
+protocol. It is mainly designed for group (many-to-many) communication in
+discussion forums called channels, but also allows one-to-one communication via
+private message. On IRC you can talk to many other members using Ceph, on
+topics ranging from idle chit-chat to support questions. Though a channel might
+have many people in it at any one time, they might not always be at their
+keyboard; so if no-one responds, just wait around and someone will hopefully
+answer soon enough.
+
+Registration
+^^^^^^^^^^^^
+
+If you intend to use the IRC service on a continued basis, you are advised to
+register an account. Registering gives you a unique IRC identity and allows you
+to access channels where unregistered users have been locked out for technical
+reasons.
+
+See ``the official OFTC (Open and Free Technology Community) documentation's
+registration instructions
+<https://www.oftc.net/Services/#register-your-account>`` to learn how to
+register your IRC account.
+
+Channels
+~~~~~~~~
+
+To connect to the OFTC IRC network, download an IRC client and configure it to
+connect to ``irc.oftc.net``. Then join one or more of the channels. Discussions
+inside #ceph are logged and archives are available online.
+
+Here are the real-time discussion channels for the Ceph community:
+
+  -  #ceph
+  -  #ceph-devel
+  -  #cephfs
+  -  #ceph-dashboard
+  -  #ceph-orchestrators
+  -  #sepia
 
 .. _submitting-patches:
 
@@ -152,7 +190,7 @@ file `CONTRIBUTING.rst`_ in the top-level directory of the source-code
 tree. There may be some overlap between this guide and that file.
 
 .. _`CONTRIBUTING.rst`:
-  https://github.com/ceph/ceph/blob/master/CONTRIBUTING.rst
+  https://github.com/ceph/ceph/blob/main/CONTRIBUTING.rst
 
 All newcomers are encouraged to read that file carefully.
 
@@ -252,7 +290,7 @@ See :ref:`kubernetes-dev`
 Backporting
 -----------
 
-All bugfixes should be merged to the ``master`` branch before being
+All bugfixes should be merged to the ``main`` branch before being
 backported. To flag a bugfix for backporting, make sure it has a
 `tracker issue`_ associated with it and set the ``Backport`` field to a
 comma-separated list of previous releases (e.g. "hammer,jewel") that you think
@@ -263,6 +301,36 @@ The rest (including the actual backporting) will be taken care of by the
 .. _`tracker issue`: http://tracker.ceph.com/
 .. _`Stable Releases and Backports`: http://tracker.ceph.com/projects/ceph-releases/wiki
 
+Dependabot
+----------
+
+Dependabot is a GitHub bot that scans the dependencies in the repositories for
+security vulnerabilities (CVEs). If a fix is available for a discovered CVE,
+Dependabot creates a pull request to update the dependency.
+
+Dependabot also indicates the compatibility score of the upgrade. This score is
+based on the number of CI failures that occur in other GitHub repositories
+where the fix was applied. 
+
+With some configuration, Dependabot can perform non-security updates (for
+example, it can upgrade to the latest minor version or patch version).
+
+Dependabot supports `several languages and package managers
+<https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/about-dependabot-version-updates#supported-repositories-and-ecosystems>`_.
+As of July 2022, the Ceph project receives alerts only from pip (based on the
+`requirements.txt` files) and npm (`package*.json`). It is possible to extend
+these alerts to git submodules, Golang, and Java. As of July 2022, there is no
+support for C++ package managers such as vcpkg, conan, C++20 modules.
+
+Many of the dependencies discovered by Dependabot will best be updated
+elsewhere than the Ceph Github repository (distribution packages, for example,
+will be a better place to update some of the dependencies). Nonetheless, the
+list of new and existing vulnerabilities generated by Dependabot will be
+useful.
+
+`Here is an example of a Dependabot pull request.
+<https://github.com/ceph/ceph/pull/46998>`_
+
 Guidance for use of cluster log
 -------------------------------
 

ceph/doc/dev/developer_guide/merging.rst

@@ -18,10 +18,10 @@ What ?
 Where ?
 ^^^^^^^
 
-Features are merged to the *master* branch. Bug fixes should be merged to the
+Features are merged to the *main* branch. Bug fixes should be merged to the
 corresponding named branch (e.g. *nautilus* for 14.0.z, *pacific* for 16.0.z,
 etc.). However, this is not mandatory - bug fixes and documentation
-enhancements can be merged to the *master* branch as well, since the *master*
+enhancements can be merged to the *main* branch as well, since the *main*
 branch is itself occasionally merged to the named branch during the development
 releases phase. In either case, if a bug fix is important it can also be
 flagged for backport to one or more previous stable releases.
@@ -32,16 +32,16 @@ When ?
 After each stable release, candidate branches for previous releases enter
 phase 2 (see below).  For example: the *jewel* named branch was created when
 the *infernalis* release candidates entered phase 2. From this point on,
-*master* was no longer associated with *infernalis*. After he named branch of
+*main* was no longer associated with *infernalis*. After he named branch of
-the next stable release is created, *master* will be occasionally merged into
+the next stable release is created, *main* will be occasionally merged into
 it.
 
 Branch merges
 ^^^^^^^^^^^^^
 
-* The latest stable release branch is merged periodically into master.
+* The latest stable release branch is merged periodically into main.
-* The master branch is merged periodically into the branch of the stable release.
+* The main branch is merged periodically into the branch of the stable release.
-* The master is merged into the stable release branch
+* The main is merged into the stable release branch
   immediately after each development (x.0.z) release.
 
 Stable release candidates (i.e. x.1.z) phase 1
@@ -56,12 +56,12 @@ Where ?
 ^^^^^^^
 
 The stable release branch (e.g. *jewel* for 10.0.z, *luminous*
-for 12.0.z, etc.) or *master*.  Bug fixes should be merged to the named
+for 12.0.z, etc.) or *main*.  Bug fixes should be merged to the named
 branch corresponding to the stable release candidate (e.g. *jewel* for
-10.1.z) or to *master*. During this phase, all commits to *master* will be
+10.1.z) or to *main*. During this phase, all commits to *main* will be
 merged to the named branch, and vice versa. In other words, it makes
 no difference whether a commit is merged to the named branch or to
-*master* - it will make it into the next release candidate either way.
+*main* - it will make it into the next release candidate either way.
 
 When ?
 ^^^^^^
@@ -72,9 +72,9 @@ x.1.0 tag is set in the release branch.
 Branch merges
 ^^^^^^^^^^^^^
 
-* The stable release branch is merged periodically into *master*.
+* The stable release branch is merged periodically into *main*.
-* The *master* branch is merged periodically into the stable release branch.
+* The *main* branch is merged periodically into the stable release branch.
-* The *master* branch is merged into the stable release branch
+* The *main* branch is merged into the stable release branch
   immediately after each x.1.z release candidate.
 
 Stable release candidates (i.e. x.1.z) phase 2
@@ -90,7 +90,7 @@ Where ?
 
 The stable release branch (e.g. *mimic* for 13.0.z, *octopus* for 15.0.z
 ,etc.). During this phase, all commits to the named branch will be merged into
-*master*. Cherry-picking to the named branch during release candidate phase 2
+*main*. Cherry-picking to the named branch during release candidate phase 2
 is performed manually since the official backporting process begins only when
 the release is pronounced "stable".
 
@@ -102,7 +102,7 @@ After Sage Weil announces that it is time for phase 2 to happen.
 Branch merges
 ^^^^^^^^^^^^^
 
-* The stable release branch is occasionally merged into master.
+* The stable release branch is occasionally merged into main.
 
 Stable releases (i.e. x.2.z)
 ----------------------------
@@ -112,8 +112,8 @@ What ?
 
 * Bug fixes
 * Features are sometime accepted
-* Commits should be cherry-picked from *master* when possible
+* Commits should be cherry-picked from *main* when possible
-* Commits that are not cherry-picked from *master* must pertain to a bug unique to
+* Commits that are not cherry-picked from *main* must pertain to a bug unique to
   the stable release
 * See also the `backport HOWTO`_ document
 

ceph/doc/dev/developer_guide/tests-unit-tests.rst

@@ -166,5 +166,12 @@ Unit test caveats
    explicitly linked against something else. This enables tools such as
    **valgrind** to be used in the tests.
 
+#. Google Test unit testing library hides the client output from the shell.
+   In order to debug the client after setting the desired debug level
+   (e.g ``ceph config set client debug_rbd 20``), the debug log file can
+   be found at ``build/out/client.admin.<pid>.log``.
+   This can also be handy when examining teuthology failed unit test
+   jobs, the job's debug level can be set at the relevant yaml file.
+
 .. _make check:
 .. _teuthology framework: https://github.com/ceph/teuthology

ceph/doc/dev/documenting.rst

@@ -86,6 +86,10 @@ separate file, like this::
 
   .. graphviz:: myfile.dot
 
+See the `Dot User's Manual <https://www.graphviz.org/pdf/dotguide.pdf>`_ by
+Emden R. Gansner, Eleftherios Koutsofios, and Stephen North for examples of
+digraphs. This is especially useful if this is your first time encountering
+GraphViz.
 
 Ditaa
 -----

ceph/doc/dev/object-store.rst

@@ -4,10 +4,6 @@
 
 .. graphviz::
 
-  /*
-   * Rough outline of object store module dependencies
-   */
-
    digraph object_store {
     size="7,7";
     node [color=lightblue2, style=filled, fontname="Serif"];
@@ -68,3 +64,4 @@
 
 
 .. todo:: write more here
+

ceph/doc/dev/osd_internals/erasure_coding.rst

@@ -6,47 +6,52 @@ Glossary
 --------
 
 *chunk* 
-   when the encoding function is called, it returns chunks of the same
+   When the encoding function is called, it returns chunks of the same
-   size. Data chunks which can be concatenated to reconstruct the original
+   size as each other. There are two kinds of chunks: (1) *data
-   object and coding chunks which can be used to rebuild a lost chunk.
+   chunks*, which can be concatenated to reconstruct the original
+   object, and (2) *coding chunks*, which can be used to rebuild a
+   lost chunk.
 
 *chunk rank*
-   the index of a chunk when returned by the encoding function. The
+   The index of a chunk, as determined by the encoding function. The
-   rank of the first chunk is 0, the rank of the second chunk is 1
+   rank of the first chunk is 0, the rank of the second chunk is 1,
-   etc.
+   and so on.
-
-*stripe* 
-   when an object is too large to be encoded with a single call,
-   each set of chunks created by a call to the encoding function is
-   called a stripe.
-
-*shard|strip*
-   an ordered sequence of chunks of the same rank from the same
-   object.  For a given placement group, each OSD contains shards of
-   the same rank. When dealing with objects that are encoded with a
-   single operation, *chunk* is sometime used instead of *shard*
-   because the shard is made of a single chunk. The *chunks* in a
-   *shard* are ordered according to the rank of the stripe they belong
-   to.
 
 *K*
-   the number of data *chunks*, i.e. the number of *chunks* in which the
+   The number of data chunks into which an object is divided. For
-   original object is divided. For instance if *K* = 2 a 10KB object
+   example, if *K* = 2, then a 10KB object is divided into two objects
-   will be divided into *K* objects of 5KB each.
+   of 5KB each.
 
 *M* 
-   the number of coding *chunks*, i.e. the number of additional *chunks*
+   The number of coding chunks computed by the encoding function. *M*
-   computed by the encoding functions. If there are 2 coding *chunks*, 
+   is equal to the number of OSDs that can be missing from the cluster
-   it means 2 OSDs can be out without losing data.
+   without the cluster suffering data loss. For example, if there are
+   two coding chunks, then two OSDs can be missing without data loss.
 
 *N*
-   the number of data *chunks* plus the number of coding *chunks*, 
+   The number of data chunks plus the number of coding chunks: that
-   i.e. *K+M*.
+   is, *K* + *M*.
 
 *rate*
-   the proportion of the *chunks* that contains useful information, i.e. *K/N*.
+   The proportion of the total chunks containing useful information:
-   For instance, for *K* = 9 and *M* = 3 (i.e. *K+M* = *N* = 12) the rate is 
+   that is, *K* divided by *N*. For example, suppose that *K* = 9 and
-   *K* = 9 / *N* = 12 = 0.75, i.e. 75% of the chunks contain useful information.
+   *M* = 3. This would mean that *N* = 12 (because *K* + *M* = 9 + 3).
+   Therefore, the *rate* (*K* / *N*) would be 9 / 12 = 0.75. In other
+   words, 75% of the chunks would contain useful information.
+
+*shard* (also called *strip*)
+   An ordered sequence of chunks of the same rank from the same object. For a
+   given placement group, each OSD contains shards of the same rank. In the
+   special case in which an object is encoded with only one call to the
+   encoding function, the term *chunk* may be used instead of *shard* because
+   the shard is made of a single chunk. The chunks in a shard are ordered
+   according to the rank of the stripe (see *stripe* below) they belong to.
+
+
+*stripe* 
+   If an object is so large that encoding it requires more than one
+   call to the encoding function, each of these calls creates a set of
+   chunks called a *stripe*.
 
 The definitions are illustrated as follows (PG stands for placement group):
 ::
@@ -71,8 +76,8 @@ The definitions are illustrated as follows (PG stands for placement group):
        |         ...             | |         ...             |
        +-------------------------+ +-------------------------+
 
-Table of content
+Table of contents
-----------------
+-----------------
 
 .. toctree::
    :maxdepth: 1

ceph/doc/dev/release-process.rst

@@ -2,172 +2,224 @@
   Ceph Release Process
 ======================
 
-1. Build environment
+Prerequisites
-====================
+=============
 
-There are multiple build environments, debian based packages are built via pbuilder for multiple distributions.  The build hosts are listed in the ``deb_hosts`` file, and the list of distributions are in ``deb_dist``.  All distributions are build on each of the build hosts.  Currently there is 1 64 bit and 1 32 bit build host.
+Signing Machine
+---------------
+The signing machine is a virtual machine in the `Sepia lab
+<https://wiki.sepia.ceph.com/doku.php?id=start>`_. SSH access to the signing
+machine is limited to the usual Infrastructure Admins along with a few other
+component leads (e.g., nfs-ganesha, ceph-iscsi).
 
-The RPM based packages are built natively, so one distribution per build host.  The list of hosts is found in ``rpm_hosts``.
+The ``ubuntu`` user on the machine has some `build scripts <https://github.com/ceph/ceph-build/tree/main/scripts>`_ that help with pulling, pushing, and signing packages.
 
-Prior to building, it's necessary to update the pbuilder seed tarballs::
+The GPG signing key permanently lives on a `Nitrokey Pro <https://shop.nitrokey.com/shop/product/nkpr2-nitrokey-pro-2-3>`_ and is passed through to the VM via RHV. This helps to ensure that the key cannot be exported or leave the datacenter in any way.
 
-    ./update_all_pbuilders.sh
+New Major Releases
+------------------
+For each new major (alphabetical) release, you must create one ``ceph-release`` RPM for each RPM repo (e.g., one for el8 and one for el9). `chacra <https://github.com/ceph/chacra>`_ is a python service we use to store DEB and RPM repos. The chacra repos are configured to include this ceph-release RPM, but it must be built separately. You must make sure that chacra is properly configured to include this RPM for each particular release.
 
-2. Setup keyring for signing packages
+1. Update chacra so it is aware of the new Ceph release.  See `this PR <https://github.com/ceph/chacra/pull/219>`_ for an example.
-=====================================
+2. Redeploy chacra (e.g., ``ansible-playbook chacra.ceph.com.yml``)
+3. Run https://jenkins.ceph.com/view/all/job/ceph-release-rpm/
 
-::
+Summarized build process
+========================
 
-    export GNUPGHOME=<path to keyring dir>
+1. QE finishes testing and finds a stopping point.  That commit is pushed to the ``$release-release`` branch in ceph.git (e.g., ``quincy-release``).  This allows work to continue in the working ``$release`` branch without having to freeze it during the release process.
+2. The Ceph Council approves and notifies the "Build Lead".
+3. The "Build Lead" starts the `Jenkins multijob <https://jenkins.ceph.com/view/all/job/ceph>`_, which triggers all builds.
+4. Packages are pushed to chacra.ceph.com.
+5. Packages are pulled from chacra.ceph.com to the Signer VM.
+6. Packages are signed.
+7. Packages are pushed to download.ceph.com.
+8. Release containers are built and pushed to quay.io.
 
-    # verify it's accessible
+Hotfix Release Process Deviation
-    gpg --list-keys
+--------------------------------
 
-The release key should be present::
+A hotfix release has a couple differences.
 
-  pub   4096R/17ED316D 2012-05-20
+1. Check out the most recent tag. For example, if we're releasing a hotfix on top of 17.2.3, ``git checkout -f -B quincy-release origin/v17.2.3``
-  uid   Ceph Release Key <sage@newdream.net>
+2. ``git cherry-pick -x`` the necessary hotfix commits
+3. ``git push -f origin quincy-release``
+4. Notify the "Build Lead" to start the build.
+5. The "Build Lead" should set ``RELEASE_TYPE=HOTFIX`` instead of ``STABLE``.
 
+Security Release Process Deviation
+----------------------------------
 
-3. Set up build area
+A security/CVE release is similar to a hotfix release with two differences:
-====================
 
-Clone the ceph and ceph-build source trees::
+    1. The fix should be pushed to the `ceph-private <https://github.com/ceph/ceph-private>`_ repo instead of ceph.git (requires GitHub Admin Role).
+    2. The tags (e.g., v17.2.4) must be manually pushed to ceph.git by the "Build Lead."
 
-    git clone http://github.com/ceph/ceph.git
+1. Check out the most recent tag. For example, if we're releasing a security fix on top of 17.2.3, ``git checkout -f -B quincy-release origin/v17.2.3``
-    git clone http://github.com/ceph/ceph-build.git
+2. ``git cherry-pick -x`` the necessary security fix commits
+3. ``git remote add security git@github.com:ceph/ceph-private.git``
+4. ``git push -f security quincy-release``
+5. Notify the "Build Lead" to start the build.
+6. The "Build Lead" should set ``RELEASE_TYPE=SECURITY`` instead of ``STABLE``.
+7. Finally, the `ceph-tag <https://github.com/ceph/ceph-build/blob/main/ansible/roles/ceph-release/tasks/push.yml>`_ steps need to be manually run by the "Build Lead" as close to the Announcement time as possible::
 
-In the ceph source directory, checkout next branch (for point releases use the {codename} branch)::
+    # Example using quincy pretending 17.2.4 is the security release version
+    # Add the ceph-releases repo (also requires GitHub Admin Role). The `ceph-setup <https://jenkins.ceph.com/job/ceph-setup>`_ job will have already created and pushed the tag to ceph-releases.git.
+    git remote add releases git@github.com:ceph/ceph-releases.git
+    git fetch --all
+    # Check out the version commit
+    git checkout -f -B quincy-release releases/quincy-release
+    git push -f origin quincy-release
+    git push origin v17.2.4
+    # Now create a Pull Request of quincy-release targeting quincy to merge the version commit and security fixes back into the quincy branch
 
-    git checkout next
+1. Preparing the release branch
+===============================
 
-Checkout the submodules::
+Once QE has determined a stopping point in the working (e.g., ``quincy``) branch, that commit should be pushed to the corresponding ``quincy-release`` branch.
 
-    git submodule update --force --init --recursive
+Notify the "Build Lead" that the release branch is ready.
 
-4.  Update Build version numbers
+2. Starting the build
-================================
-
-Substitute the ceph release number where indicated below by the string ``0.xx``.
-
-Edit configure.ac and update the version number. Example diff::
-
-	-AC_INIT([ceph], [0.54], [ceph-devel@vger.kernel.org])
-	+AC_INIT([ceph], [0.55], [ceph-devel@vger.kernel.org])
- 
-Update the version number in the debian change log::
-
-	DEBEMAIL user@host dch -v 0.xx-1
-
-Commit the changes::
-
-	git commit -a
-
-Tag the release::
-
-	../ceph-build/tag-release v0.xx
-
-
-5. Create Makefiles
-===================
-
-The actual configure options used to build packages are in the
-``ceph.spec.in`` and ``debian/rules`` files.  At this point we just
-need to create a Makefile.::
-
-	./do_autogen.sh
-
-
-6. Run the release scripts
-==========================
-
-This creates tarballs and copies them, with other needed files to
-the build hosts listed in deb_hosts and rpm_hosts, runs a local build
-script, then rsyncs the results back to the specified release directory.::
-
-	../ceph-build/do_release.sh /tmp/release
-
-
-7. Create RPM Repo
-==================
-
-Copy the rpms to the destination repo::
-
-       mkdir /tmp/rpm-repo
-       ../ceph-build/push_to_rpm_repo.sh /tmp/release /tmp/rpm-repo 0.xx
-
-Next add any additional rpms to the repo that are needed such as leveldb.
-See RPM Backports section
-
-Finally, sign the rpms and build the repo indexes::
-
-  ../ceph-build/sign_and_index_rpm_repo.sh /tmp/release /tmp/rpm-repo 0.xx
-
-
-8. Create Debian repo
 =====================
 
-The key-id used below is the id of the ceph release key from step 2::
+We'll use a stable/regular 15.2.17 release of Octopus as an example throughout this document.
 
-	mkdir /tmp/debian-repo
+1. Browse to https://jenkins.ceph.com/view/all/job/ceph/build?delay=0sec
-	../ceph-build/gen_reprepro_conf.sh /tmp/debian-repo key-id
+2. Log in with GitHub OAuth
-	../ceph-build/push_to_deb_repo.sh /tmp/release /tmp/debian-repo 0.xx main
+3. Set the parameters as necessary::
 
+    BRANCH=octopus
+    TAG=checked
+    VERSION=15.2.17
+    RELEASE_TYPE=STABLE
+    ARCHS=x86_64 arm64
 
-Next add any addition debian packages that are needed such as leveldb.
+4. Use https://docs.ceph.com/en/latest/start/os-recommendations/?highlight=debian#platforms to determine the ``DISTROS`` parameter.  For example,
-See the Debian Backports section below.
 
-Debian packages are signed when added to the repo, so no further action is
+    +-------------------+-------------------------------------------+
-needed.
+    | Release           | Distro Codemap                            |
+    +===================+===========================================+
+    | octopus (15.X.X)  | ``focal bionic centos7 centos8 buster``   |
+    +-------------------+-------------------------------------------+
+    | pacific (16.X.X)  | ``focal bionic centos8 buster bullseye``  |
+    +-------------------+-------------------------------------------+
+    | quincy (17.X.X)   | ``focal centos8 centos9 bullseye``        |
+    +-------------------+-------------------------------------------+
 
+5. Click ``Build``.
 
-9.  Push repos to ceph.org
+3. Release Notes
-==========================
+================
 
-For a development release::
+Packages take hours to build. Use those hours to create the Release Notes and Announcements:
 
-	rcp ceph-0.xx.tar.bz2 ceph-0.xx.tar.gz \
+1. ceph.git Release Notes (e.g., `v15.2.17's ceph.git (docs.ceph.com) PR <https://github.com/ceph/ceph/pull/47198>`_)
-	     ceph_site@ceph.com:ceph.com/downloads/.
+2. ceph.io Release Notes (e.g., `v15.2.17's ceph.io.git (www.ceph.io) PR <https://github.com/ceph/ceph.io/pull/427>`_)
-	rsync -av /tmp/rpm-repo/0.xx/ ceph_site@ceph.com:ceph.com/rpm-testing
+3. E-mail announcement
-	rsync -av /tmp/debian-repo/ ceph_site@ceph.com:ceph.com/debian-testing
 
-For a stable release, replace {CODENAME} with the release codename (e.g., ``argonaut`` or ``bobtail``)::
+See `the Ceph Tracker wiki page that explains how to write the release notes <https://tracker.ceph.com/projects/ceph-releases/wiki/HOWTO_write_the_release_notes>`_. 
 
-        rcp ceph-0.xx.tar.bz2 \
+4. Signing and Publishing the Build
-            ceph_site@ceph.com:ceph.com/downloads/ceph-0.xx.tar.bz2
+===================================
-        rcp ceph-0.xx.tar.gz  \
-            ceph_site@ceph.com:ceph.com/downloads/ceph-0.xx.tar.gz
-        rsync -av /tmp/rpm-repo/0.xx/ ceph_site@ceph.com:ceph.com/rpm-{CODENAME}
-        rsync -auv /tmp/debian-repo/ ceph_site@ceph.com:ceph.com/debian-{CODENAME}
 
-10. Update Git
+#. Obtain the sha1 of the version commit from the `build job <https://jenkins.ceph.com/view/all/job/ceph>`_ or the ``sha1`` file created by the `ceph-setup <https://jenkins.ceph.com/job/ceph-setup/>`_ job.
-==============
 
-Point release
+#. Download the packages from chacra.ceph.com to the signing virtual machine. These packages get downloaded to ``/opt/repos`` where the `Sepia Lab Long Running (Ceph) Cluster <https://wiki.sepia.ceph.com/doku.php?id=services:longrunningcluster>`_ is mounted.
--------------
 
-For point releases just push the version number update to the
+   .. prompt:: bash $
-branch and the new tag::
 
-    git push origin {codename}
+      ssh ubuntu@signer.front.sepia.ceph.com
-    git push origin v0.xx
+      sync-pull ceph [pacific|quincy|etc] <sha1>
  
-Development and Stable releases
+   Example::
--------------------------------
     
-For a development release, update tags for ``ceph.git``::
+      $ sync-pull ceph octopus 8a82819d84cf884bd39c17e3236e0632ac146dc4
+      sync for: ceph octopus
+      ********************************************
+      Found the most packages (332) in ubuntu/bionic.
+      No JSON object could be decoded
+      No JSON object could be decoded
+      ubuntu@chacra.ceph.com:/opt/repos/ceph/octopus/8a82819d84cf884bd39c17e3236e0632ac146dc4/ubuntu/bionic/flavors/default/* /opt/repos/ceph/octopus-15.2.17/debian/jessie/
+      --------------------------------------------
+      receiving incremental file list
+      db/
+       db/checksums.db
+              180.22K 100%    2.23MB/s    0:00:00 (xfr#1, to-chk=463/467)
+      db/contents.cache.db
+              507.90K 100%    1.95MB/s    0:00:00 (xfr#2, to-chk=462/467)
+      db/packages.db
       
-    git push origin v0.xx
+      etc...
-    git push origin HEAD:last
-    git checkout master
-    git merge next
-    git push origin master
-    git push origin HEAD:next
 
-Similarly, for a development release, for both ``teuthology.git`` and ``ceph-qa-suite.git``::
+#. Sign the DEBs:
 
-    git checkout master
+   .. prompt:: bash
-    git reset --hard origin/master
+
-    git branch -f last origin/next
+      merfi gpg /opt/repos/ceph/octopus-15.2.17/debian
-    git push -f origin last
+
-    git push -f origin master:next
+   Example::
+
+      $ merfi gpg /opt/repos/ceph/octopus-15.2.17/debian
+      --> Starting path collection, looking for files to sign
+      --> 18 matching paths found
+      --> will sign with the following commands:
+      --> gpg --batch --yes --armor --detach-sig --output Release.gpg Release
+      --> gpg --batch --yes --clearsign --output InRelease Release
+      --> signing: /opt/repos/ceph/octopus-15.2.17/debian/jessie/dists/bionic/Release
+      --> Running command: gpg --batch --yes --armor --detach-sig --output Release.gpg Release
+      --> Running command: gpg --batch --yes --clearsign --output InRelease Release
+      --> signing: /opt/repos/ceph/octopus-15.2.17/debian/jessie/dists/focal/Release
+      --> Running command: gpg --batch --yes --armor --detach-sig --output Release.gpg Release
+      --> Running command: gpg --batch --yes --clearsign --output InRelease Release
+      
+      etc...
+
+#. Sign the RPMs:
+
+   .. prompt:: bash
+
+      sign-rpms octopus
+
+   Example::
+   
+      $ sign-rpms octopus
+      Checking packages in: /opt/repos/ceph/octopus-15.2.17/centos/7
+      signing:  /opt/repos/ceph/octopus-15.2.17/centos/7/SRPMS/ceph-release-1-1.el7.src.rpm
+      /opt/repos/ceph/octopus-15.2.17/centos/7/SRPMS/ceph-release-1-1.el7.src.rpm:
+      signing:  /opt/repos/ceph/octopus-15.2.17/centos/7/SRPMS/ceph-15.2.17-0.el7.src.rpm
+      /opt/repos/ceph/octopus-15.2.17/centos/7/SRPMS/ceph-15.2.17-0.el7.src.rpm:
+      signing:  /opt/repos/ceph/octopus-15.2.17/centos/7/noarch/ceph-mgr-modules-core-15.2.17-0.el7.noarch.rpm
+   
+      etc...
+
+5. Publish the packages to download.ceph.com:
+
+   .. prompt:: bash $
+
+      sync-push octopus
+
+5. Build Containers
+===================
+
+Start the following two jobs:
+
+#. https://2.jenkins.ceph.com/job/ceph-container-build-ceph-base-push-imgs/
+#. https://2.jenkins.ceph.com/job/ceph-container-build-ceph-base-push-imgs-arm64/
+
+6. Announce the Release
+=======================
+
+Version Commit PR
+-----------------
+
+The `ceph-tag Jenkins job <https://jenkins.ceph.com/job/ceph-tag>`_ creates a Pull Request in ceph.git that targets the release branch.
+
+If this was a regular release (not a hotfix release or a security release), the only commit in that Pull Request should be the version commit.  For example, see `v15.2.17's version commit PR <https://github.com/ceph/ceph/pull/47520>`_. 
+
+Request a review and then merge the Pull Request.
+
+Announcing
+----------
+
+Publish the Release Notes on ceph.io before announcing the release by email, because the e-mail announcement references the ceph.io blog post.

ceph/doc/glossary.rst

@@ -2,198 +2,355 @@
  Ceph Glossary
 ===============
 
-Ceph is growing rapidly. As firms deploy Ceph, the technical terms such as
-"RADOS", "RBD," "RGW" and so forth require corresponding marketing terms
-that explain what each component does. The terms in this glossary are
-intended to complement the existing technical terminology.
-
-Sometimes more than one term applies to a definition. Generally, the first
-term reflects a term consistent with Ceph's marketing, and secondary terms
-reflect either technical terms or legacy ways of referring to Ceph systems.
-
-
 .. glossary::
 
-	Ceph Project
+	:ref:`BlueStore<rados_config_storage_devices_bluestore>`
-		The aggregate term for the people, software, mission and infrastructure
+                OSD BlueStore is a storage back end used by OSD daemons, and
-		of Ceph.
+                was designed specifically for use with Ceph. BlueStore was
-
+                introduced in the Ceph Kraken release. In the Ceph Luminous
-	cephx
+                release, BlueStore became Ceph's default storage back end,
-		The Ceph authentication protocol. Cephx operates like Kerberos, but it
+                supplanting FileStore. Unlike :term:`filestore`, BlueStore
-		has no single point of failure.
+                stores objects directly on Ceph block devices without any file
+                system interface. Since Luminous (12.2), BlueStore has been
+                Ceph's default and recommended storage back end.
 
 	Ceph
+                Ceph is a distributed network storage and file system with
+                distributed metadata management and POSIX semantics.
+
+	Ceph Block Device
+                A software instrument that orchestrates the storage of
+                block-based data in Ceph. Ceph Block Device (also called "RBD",
+                or "RADOS block device") splits block-based application data
+                into "chunks". RADOS stores these chunks as objects. Ceph Block
+                Device orchestrates the storage of those objects across the
+                storage cluster. See also :term:`RBD`.
+
+	Ceph Block Storage
+                One of the three kinds of storage supported by Ceph (the other
+                two are object storage and file storage). Ceph Block Storage is
+                the block storage "product", which refers to block-storage
+                related services and capabilities when used in conjunction with
+                the collection of (1) ``librbd`` (a python module that provides
+                file-like access to :term:`RBD` images), (2) a hypervisor such
+                as QEMU or Xen, and (3) a hypervisor abstraction layer such as
+                ``libvirt``.
+
+	Ceph Client
+                Any of the Ceph components that can access a Ceph Storage
+                Cluster. This includes the Ceph Object Gateway, the Ceph Block
+                Device, the Ceph File System, and their corresponding
+                libraries. It also includes kernel modules, and FUSEs
+                (Filesystems in USERspace).
+
+	Ceph Client Libraries
+                The collection of libraries that can be used to interact with
+                components of the Ceph Cluster.
+
+	Ceph Cluster Map
+                See :term:`Cluster Map`
+
+	Ceph Dashboard
+                :ref:`The Ceph Dashboard<mgr-dashboard>` is a built-in
+                web-based Ceph management and monitoring application through
+                which you can inspect and administer various resources within
+                the cluster. It is implemented as a :ref:`ceph-manager-daemon`
+                module.
+
+	Ceph File System
+                See :term:`CephFS`
+
+	:ref:`CephFS<ceph-file-system>`
+                The **Ceph F**\ile **S**\ystem, or CephFS, is a
+                POSIX-compliant file system built on top of Ceph’s distributed
+                object store, RADOS.  See :ref:`CephFS Architecture
+                <arch-cephfs>` for more details.
+
+	Ceph Interim Release
+                See :term:`Releases`.
+
+	Ceph Kernel Modules
+                The collection of kernel modules that can be used to interact
+                with the Ceph Cluster (for example: ``ceph.ko``, ``rbd.ko``).
+
+	:ref:`Ceph Manager<ceph-manager-daemon>`
+                The Ceph manager daemon (ceph-mgr) is a daemon that runs
+                alongside monitor daemons to provide monitoring and interfacing
+                to external monitoring and management systems. Since the
+                Luminous release (12.x), no Ceph cluster functions properly
+                unless it contains a running ceph-mgr daemon. 
+
+	Ceph Manager Dashboard
+                See :term:`Ceph Dashboard`.
+
+	Ceph Metadata Server
+                See :term:`MDS`.
+
+	Ceph Monitor
+                A daemon that maintains a map of the state of the cluster. This
+                "cluster state" includes the monitor map, the manager map, the
+                OSD map, and the CRUSH map. A Ceph cluster must contain a
+                minimum of three running monitors in order to be both redundant
+                and highly-available. Ceph monitors and the nodes on which they
+                run are often referred to as "mon"s. See :ref:`Monitor Config
+                Reference <monitor-config-reference>`.
+
+	Ceph Node
+               A Ceph node is a unit of the Ceph Cluster that communicates with
+               other nodes in the Ceph Cluster in order to replicate and
+               redistribute data. All of the nodes together are called the
+               :term:`Ceph Storage Cluster`. Ceph nodes include :term:`OSD`\s,
+               :term:`Ceph Monitor`\s, :term:`Ceph Manager`\s, and
+               :term:`MDS`\es. The term "node" is usually equivalent to "host"
+               in the Ceph documentation. If you have a running Ceph Cluster,
+               you can list all of the nodes in it by running the command
+               ``ceph node ls all``.
+                
+	:ref:`Ceph Object Gateway<object-gateway>`
+                An object storage interface built on top of librados. Ceph
+                Object Gateway provides a RESTful gateway between applications
+                and Ceph storage clusters.
+
+	Ceph Object Storage
+                See :term:`Ceph Object Store`.
+
+	Ceph Object Store
+                A Ceph Object Store consists of a :term:`Ceph Storage Cluster`
+                and a :term:`Ceph Object Gateway` (RGW).
+
+	:ref:`Ceph OSD<rados_configuration_storage-devices_ceph_osd>`
+                Ceph **O**\bject **S**\torage **D**\aemon. The Ceph OSD
+                software, which interacts with logical disks (:term:`OSD`).
+                Around 2013, there was an attempt by "research and industry"
+                (Sage's own words) to insist on using the term "OSD" to mean
+                only "Object Storage Device", but the Ceph community has always
+                persisted in using the term to mean "Object Storage Daemon" and
+                no less an authority than Sage Weil himself confirms in
+                November of 2022 that "Daemon is more accurate for how Ceph is
+                built" (private correspondence between Zac Dover and Sage Weil,
+                07 Nov 2022). 
+
+	Ceph OSD Daemon
+                See :term:`Ceph OSD`.
+
+	Ceph OSD Daemons
+                See :term:`Ceph OSD`.
+
 	Ceph Platform
                 All Ceph software, which includes any piece of code hosted at
                 `https://github.com/ceph`_.
 
-	Ceph System
+	Ceph Point Release
+                See :term:`Releases`.
+
+	Ceph Project
+                The aggregate term for the people, software, mission and
+                infrastructure of Ceph.
+
+	Ceph Release
+                See :term:`Releases`.
+
+	Ceph Release Candidate
+                See :term:`Releases`.
+
+	Ceph Stable Release
+                See :term:`Releases`.
+
 	Ceph Stack
 		A collection of two or more components of Ceph.
 
-	Ceph Node
+	:ref:`Ceph Storage Cluster<arch-ceph-storage-cluster>`
-	Node
+                The collection of :term:`Ceph Monitor`\s, :term:`Ceph
-	Host
+                Manager`\s, :term:`Ceph Metadata Server`\s, and :term:`OSD`\s
-		Any single machine or server in a Ceph System.
+                that work together to store and replicate data for use by
+                applications, Ceph Users, and :term:`Ceph Client`\s. Ceph
+                Storage Clusters receive data from :term:`Ceph Client`\s.
 
-	Ceph Storage Cluster
+	cephx
-	Ceph Object Store
+                The Ceph authentication protocol. Cephx operates like Kerberos,
-	RADOS
+                but it has no single point of failure.
-	RADOS Cluster
-	Reliable Autonomic Distributed Object Store
-		The core set of storage software which stores the user's data (MON+OSD).
-
-	Ceph Cluster Map
-	Cluster Map
-		The set of maps comprising the monitor map, OSD map, PG map, MDS map and
-		CRUSH map. See `Cluster Map`_ for details.
-
-	Ceph Object Storage
-		The object storage "product", service or capabilities, which consists
-		essentially of a Ceph Storage Cluster and a Ceph Object Gateway.
-
-	Ceph Object Gateway
-	RADOS Gateway
-	RGW
-		The S3/Swift gateway component of Ceph.
-
-	Ceph Block Device
-	RBD
-		The block storage component of Ceph.
-
-	Ceph Block Storage
-		The block storage "product," service or capabilities when used in
-		conjunction with ``librbd``, a hypervisor such as QEMU or Xen, and a
-		hypervisor abstraction layer such as ``libvirt``.
-
-	Ceph File System
-	CephFS
-	Ceph FS
-		The POSIX filesystem components of Ceph. Refer
-		:ref:`CephFS Architecture <arch-cephfs>` and :ref:`ceph-file-system` for
-		more details.
 
 	Cloud Platforms
 	Cloud Stacks
-		Third party cloud provisioning platforms such as OpenStack, CloudStack,
+                Third party cloud provisioning platforms such as OpenStack,
-		OpenNebula, ProxMox, etc.
+                CloudStack, OpenNebula, and Proxmox VE.
 
-	Object Storage Device
+	Cluster Map
-	OSD
+                The set of maps consisting of the monitor map, OSD map, PG map,
-		A physical or logical storage unit (*e.g.*, LUN).
+                MDS map, and CRUSH map, which together report the state of the
-		Sometimes, Ceph users use the
+                Ceph cluster. See :ref:`the "Cluster Map" section of the
-		term "OSD" to refer to :term:`Ceph OSD Daemon`, though the
+                Architecture document<architecture_cluster_map>` for details.
-		proper term is "Ceph OSD".
 
-	Ceph OSD Daemon
+	CRUSH
-	Ceph OSD Daemons
+                Controlled Replication Under Scalable Hashing. It is the
-	Ceph OSD
+                algorithm Ceph uses to compute object storage locations.
-		The Ceph OSD software, which interacts with a logical
-		disk (:term:`OSD`). Sometimes, Ceph users use the
-		term "OSD" to refer to "Ceph OSD Daemon", though the
-		proper term is "Ceph OSD".
 
-	OSD id
+	CRUSH rule
-		The integer that defines an OSD. It is generated by the monitors as part
+                The CRUSH data placement rule that applies to a particular
-		of the creation of a new OSD.
+                pool(s).
 
-	OSD fsid
+        DAS
-		This is a unique identifier used to further improve the uniqueness of an
+                **D**\irect-\ **A**\ttached **S**\torage. Storage that is
-		OSD and it is found in the OSD path in a file called ``osd_fsid``. This
+                attached directly to the computer accessing it, without passing
-		``fsid`` term is used interchangeably with ``uuid``
+                through a network.  Contrast with NAS and SAN.
 
-	OSD uuid
+	:ref:`Dashboard<mgr-dashboard>`
-		Just like the OSD fsid, this is the OSD unique identifier and is used
+                A built-in web-based Ceph management and monitoring application
-		interchangeably with ``fsid``
+                to administer various aspects and objects of the cluster. The
+                dashboard is implemented as a Ceph Manager module. See
+                :ref:`mgr-dashboard` for more details.
 
-	bluestore
+	Dashboard Module
-		OSD BlueStore is a new back end for OSD daemons (kraken and newer
+                Another name for :term:`Dashboard`.
-		versions). Unlike :term:`filestore` it stores objects directly on the
-		Ceph block devices without any file system interface.
 
+	Dashboard Plugin
 	filestore
-		A back end for OSD daemons, where a Journal is needed and files are
+                A back end for OSD daemons, where a Journal is needed and files
-		written to the filesystem.
+                are written to the filesystem.
+
+        FQDN
+                **F**\ully **Q**\ualified **D**\omain **N**\ame. A domain name
+                that is applied to a node in a network and that specifies the
+                node's exact location in the tree hierarchy of the DNS.
+
+                In the context of Ceph cluster administration, FQDNs are often
+                applied to hosts. In this documentation, the term "FQDN" is
+                used mostly to distinguish between FQDNs and relatively simpler
+                hostnames, which do not specify the exact location of the host
+                in the tree hierarchy of the DNS but merely name the host.
+
+	Host
+                Any single machine or server in a Ceph Cluster. See :term:`Ceph
+                Node`.
+
+	LVM tags
+                Extensible metadata for LVM volumes and groups. It is used to
+                store Ceph-specific information about devices and its
+                relationship with OSDs.
+
+	:ref:`MDS<cephfs_add_remote_mds>`
+                The Ceph **M**\eta\ **D**\ata **S**\erver daemon. Also referred
+                to as "ceph-mds". The Ceph metadata server daemon must be
+                running in any Ceph cluster that runs the CephFS file system.
+                The MDS stores all filesystem metadata. 
+
+	MGR
+                The Ceph manager software, which collects all the state from
+                the whole cluster in one place.
 
-	Ceph Monitor
 	MON
 		The Ceph monitor software.
 
-	Ceph Manager
+	Node
-	MGR
+                See :term:`Ceph Node`.
-		The Ceph manager software, which collects all the state from the whole
-		cluster in one place.
 
-	Ceph Manager Dashboard
+	Object Storage Device
-	Ceph Dashboard
+                See :term:`OSD`.
-	Dashboard Module
-	Dashboard Plugin
-	Dashboard
-		A built-in web-based Ceph management and monitoring application to
-		administer various aspects and objects of the cluster. The dashboard is
-		implemented as a Ceph Manager module. See :ref:`mgr-dashboard` for more
-		details.
 
-	Ceph Metadata Server
+	OSD
-	MDS
+                Probably :term:`Ceph OSD`, but not necessarily. Sometimes
-		The Ceph metadata software.
+                (especially in older correspondence, and especially in
+                documentation that is not written specifically for Ceph), "OSD"
+                means "**O**\bject **S**\torage **D**\evice", which refers to a
+                physical or logical storage unit (for example: LUN). The Ceph
+                community has always used the term "OSD" to refer to
+                :term:`Ceph OSD Daemon` despite an industry push in the
+                mid-2010s to insist that "OSD" should refer to "Object Storage
+                Device", so it is important to know which meaning is intended. 
 
-	Ceph Clients
+	OSD fsid
-	Ceph Client
+                This is a unique identifier used to identify an OSD. It is
-		The collection of Ceph components which can access a Ceph Storage
+                found in the OSD path in a file called ``osd_fsid``. The
-		Cluster. These include the Ceph Object Gateway, the Ceph Block Device,
+                term ``fsid`` is used interchangeably with ``uuid``
-		the Ceph File System, and their corresponding libraries, kernel modules,
-		and FUSEs.
 
-	Ceph Kernel Modules
+	OSD id
-		The collection of kernel modules which can be used to interact with the
+                The integer that defines an OSD. It is generated by the
-		Ceph System (e.g., ``ceph.ko``, ``rbd.ko``).
+                monitors during the creation of each OSD.
 
-	Ceph Client Libraries
+	OSD uuid
-		The collection of libraries that can be used to interact with components
+                This is the unique identifier of an OSD. This term is used
-		of the Ceph System.
+                interchangeably with ``fsid``
+
+	:ref:`Pool<rados_pools>`
+		A pool is a logical partition used to store objects.
+
+	Pools
+                See :term:`pool`.
+
+	RADOS
+                **R**\eliable **A**\utonomic **D**\istributed **O**\bject
+                **S**\tore. RADOS is the object store that provides a scalable
+                service for variably-sized objects. The RADOS object store is
+                the core component of a Ceph cluster.  `This blog post from
+                2009
+                <https://ceph.io/en/news/blog/2009/the-rados-distributed-object-store/>`_
+                provides a beginner's introduction to RADOS. Readers interested
+                in a deeper understanding of RADOS are directed to `RADOS: A
+                Scalable, Reliable Storage Service for Petabyte-scale Storage
+                Clusters <https://ceph.io/assets/pdfs/weil-rados-pdsw07.pdf>`_.
+
+	RADOS Cluster
+                A proper subset of the Ceph Cluster consisting of
+                :term:`OSD`\s, :term:`Ceph Monitor`\s, and :term:`Ceph
+                Manager`\s.
+                
+	RADOS Gateway
+                See :term:`RGW`.
+
+	RBD
+                The block storage component of Ceph. Also called "RADOS Block
+                Device" or :term:`Ceph Block Device`.
+
+        Releases
+
+	        Ceph Interim Release
+                        A version of Ceph that has not yet been put through
+                        quality assurance testing. May contain new features.
+
+                Ceph Point Release
+                        Any ad hoc release that includes only bug fixes and
+                        security fixes.
 
                 Ceph Release
                         Any distinct numbered version of Ceph.
 
-	Ceph Point Release
-		Any ad-hoc release that includes only bug or security fixes.
-
-	Ceph Interim Release
-		Versions of Ceph that have not yet been put through quality assurance
-		testing, but may contain new features.
-
                 Ceph Release Candidate
-		A major version of Ceph that has undergone initial quality assurance
+                        A major version of Ceph that has undergone initial
-		testing and is ready for beta testers.
+                        quality assurance testing and is ready for beta
+                        testers.
 
                 Ceph Stable Release
-		A major version of Ceph where all features from the preceding interim
+                        A major version of Ceph where all features from the
-		releases have been put through quality assurance testing successfully.
+                        preceding interim releases have been put through
+                        quality assurance testing successfully.
+
+	Reliable Autonomic Distributed Object Store
+                The core set of storage software which stores the user's data
+                (MON+OSD). See also :term:`RADOS`.
+
+	:ref:`RGW<object-gateway>`
+                **R**\ADOS **G**\ate **W**\ay.
+
+                The component of Ceph that provides a gateway to both the
+                Amazon S3 RESTful API and the OpenStack Swift API. Also called
+                "RADOS Gateway" and "Ceph Object Gateway".
+
+        secrets
+                Secrets are credentials used to perform digital authentication
+                whenever privileged users must access systems that require
+                authentication. Secrets can be passwords, API keys, tokens, SSH
+                keys, private certificates, or encryption keys.
+
+        SDS
+                Software-defined storage.
+
+	systemd oneshot
+                A systemd ``type`` where a command is defined in ``ExecStart``
+                which will exit upon completion (it is not intended to
+                daemonize)
 
-	Ceph Test Framework
 	Teuthology
 		The collection of software that performs scripted tests on Ceph.
 
-	CRUSH
-		Controlled Replication Under Scalable Hashing. It is the algorithm
-		Ceph uses to compute object storage locations.
-
-	CRUSH rule
-		The CRUSH data placement rule that applies to a particular pool(s).
-
-	Pool
-	Pools
-		Pools are logical partitions for storing objects.
-
-	systemd oneshot
-		A systemd ``type`` where a command is defined in ``ExecStart`` which will
-		exit upon completion (it is not intended to daemonize)
-
-	LVM tags
-		Extensible metadata for LVM volumes and groups. It is used to store
-		Ceph-specific information about devices and its relationship with
-		OSDs.
-
 .. _https://github.com/ceph: https://github.com/ceph
 .. _Cluster Map: ../architecture#cluster-map   

ceph/doc/index.rst

@@ -5,10 +5,18 @@
 Ceph uniquely delivers **object, block, and file storage in one unified
 system**.
 
+.. warning:: 
+
+   :ref:`If this is your first time using Ceph, read the "Basic Workflow" 
+   page in the Ceph Developer Guide to learn how to contribute to the 
+   Ceph project. (Click anywhere in this paragraph to read the "Basic 
+   Workflow" page of the Ceph Developer Guide.) <basic workflow dev guide>`.
+
 .. raw:: html
 
       <style type="text/css">div.body h3{margin:5px 0px 0px 0px;}</style>
-	<table cellpadding="10"><colgroup><col width="33%"><col width="33%"><col width="33%"></colgroup><tbody valign="top"><tr><td><h3>Ceph Object Store</h3>
+      <table cellpadding="10"><colgroup><col width="33%"><col width="33%">
+      <col width="33%"></colgroup><tbody valign="top"><tr><td><h3>Ceph Object Store</h3>
 
 - RESTful Interface
 - S3- and Swift-compliant APIs
@@ -107,6 +115,7 @@ about Ceph, see our `Architecture`_ section.
    governance
    foundation
    ceph-volume/index
-   releases/general
+   Ceph Releases (general) <https://docs.ceph.com/en/latest/releases/general/>
-   releases/index
+   Ceph Releases (index) <https://docs.ceph.com/en/latest/releases/>
+   security/index
    Glossary <glossary>

ceph/doc/install/clone-source.rst

@@ -2,33 +2,37 @@
  Cloning the Ceph Source Code Repository
 =========================================
 
-You may clone a Ceph branch of the Ceph source code by going to `github Ceph
+To clone a Ceph branch of the Ceph source code, go to `github Ceph
-Repository`_,  selecting a branch (``master`` by default), and clicking the
+Repository`_, select a branch (``main`` by default), and click the **Download
-**Download ZIP** button.
+ZIP** button.
 
 .. _github Ceph Repository: https://github.com/ceph/ceph
 
+To clone the entire git repository, :ref:`install <install-git>` and configure
+``git``.
 
-To clone the entire git repository, install and configure ``git``.
+.. _install-git:
-
 
 Install Git
 ===========
 
-To install ``git`` on Debian/Ubuntu, execute::
+To install ``git`` on Debian/Ubuntu, run the following command:
+
+.. prompt:: bash $
 
    sudo apt-get install git
 
 
-To install ``git`` on CentOS/RHEL, execute::
+To install ``git`` on CentOS/RHEL, run the following command:
+
+.. prompt:: bash $
 
    sudo yum install git
 
 
-You must also have a ``github`` account. If you do not have a
+You must have a ``github`` account. If you do not have a ``github``
-``github`` account, go to `github.com`_ and register.
+account, go to `github.com`_ and register.  Follow the directions for setting
-Follow the directions for setting up git at 
+up git at `Set Up Git`_.
-`Set Up Git`_.
 
 .. _github.com: https://github.com
 .. _Set Up Git: https://help.github.com/linux-set-up-git
@@ -37,26 +41,31 @@ Follow the directions for setting up git at
 Add SSH Keys (Optional)
 =======================
 
-If you intend to commit code to Ceph or to clone using SSH
+To commit code to Ceph or to clone the respository by using SSH
 (``git@github.com:ceph/ceph.git``), you must generate SSH keys for github. 
 
-.. tip:: If you only intend to clone the repository, you may 
+.. tip:: If you want only to clone the repository, you can 
    use ``git clone --recursive https://github.com/ceph/ceph.git`` 
    without generating SSH keys.
 
-To generate SSH keys for ``github``, execute::
+To generate SSH keys for ``github``, run the following command:
+
+.. prompt:: bash $
 
    ssh-keygen
 
-Get the key to add to your ``github`` account (the following example
+To print the SSH key that you just generated and that you will add to your
-assumes you used the default file path)::
+``github`` account, use the ``cat`` command. (The following example assumes you
+used the default file path.):
+
+.. prompt:: bash $
 
    cat .ssh/id_rsa.pub
 
 Copy the public key.
 
-Go to your ``github`` account, click on "Account Settings" (i.e., the
+Go to your ``github`` account, click "Account Settings" (represented by the
-'tools' icon); then, click "SSH Keys" on the left side navbar.
+'tools' icon), and click "SSH Keys" on the left side navbar.
 
 Click "Add SSH key" in the "SSH Keys" list, enter a name for the key, paste the
 key you generated, and press the "Add key" button.
@@ -65,37 +74,122 @@ key you generated, and press the "Add key" button.
 Clone the Source
 ================
 
-To clone the Ceph source code repository, execute::
+To clone the Ceph source code repository, run the following command:
+
+.. prompt:: bash $
 
    git clone --recursive https://github.com/ceph/ceph.git
 
-Once ``git clone`` executes, you should have a full copy of the Ceph 
+After ``git clone`` has run, you should have a full copy of the Ceph 
 repository.
 
-.. tip:: Make sure you maintain the latest copies of the submodules
+.. tip:: Make sure you maintain the latest copies of the submodules included in
-   included in the repository. Running ``git status`` will tell you if
+   the repository. Running ``git status`` will tell you whether the submodules
-   the submodules are out of date.
+   are out of date. See :ref:`update-submodules` for more information.
 
-::
+
+.. prompt:: bash $
 
    cd ceph
    git status
 
-If your submodules are out of date, run::
+.. _update-submodules:
+
+Updating Submodules
+-------------------
+
+#. Determine whether your submodules are out of date:
+
+   .. prompt:: bash $
+
+      git status
+
+   A. If your submodules are up to date 
+         If your submodules are up to date, the following console output will
+         appear: 
+
+         ::
+   
+           On branch main
+           Your branch is up to date with 'origin/main'.
+           
+           nothing to commit, working tree clean
+   
+         If you see this console output, then your submodules are up to date.
+         You do not need this procedure.
+
+
+   B. If your submodules are not up to date 
+         If your submodules are not up to date, you will see a message that
+         includes a list of "untracked files". The example here shows such a
+         list, which was generated from a real situation in which the
+         submodules were no longer current. Your list of files will not be the
+         same as this list of files, but this list is provided as an example.
+         If in your case any untracked files are listed, then you should
+         continue to the next step of this procedure.
+
+         ::
+
+            On branch main
+            Your branch is up to date with 'origin/main'.
+            
+            Untracked files:
+              (use "git add <file>..." to include in what will be committed)
+            src/pybind/cephfs/build/
+            src/pybind/cephfs/cephfs.c
+            src/pybind/cephfs/cephfs.egg-info/
+            src/pybind/rados/build/
+            src/pybind/rados/rados.c
+            src/pybind/rados/rados.egg-info/
+            src/pybind/rbd/build/
+            src/pybind/rbd/rbd.c
+            src/pybind/rbd/rbd.egg-info/
+            src/pybind/rgw/build/
+            src/pybind/rgw/rgw.c
+            src/pybind/rgw/rgw.egg-info/
+            
+            nothing added to commit but untracked files present (use "git add" to track)
+
+#. If your submodules are out of date, run the following commands:
+
+   .. prompt:: bash $
 
       git submodule update --force --init --recursive
+      git clean -fdx
+      git submodule foreach git clean -fdx
+
+   If you still have problems with a submodule directory, use ``rm -rf
+   [directory name]`` to remove the directory. Then run ``git submodule update
+   --init --recursive`` again.
+
+#. Run ``git status`` again:
+
+   .. prompt:: bash $
+
+      git status
+   
+   Your submodules are up to date if you see the following message:
+
+   ::
+
+     On branch main
+     Your branch is up to date with 'origin/main'.
+     
+     nothing to commit, working tree clean
 
 Choose a Branch
 ===============
 
 Once you clone the source code and submodules, your Ceph repository 
-will be on the ``master`` branch by default, which is the unstable 
+will be on the ``main`` branch by default, which is the unstable 
 development branch. You may choose other branches too.
 
-- ``master``: The unstable development branch.
+- ``main``: The unstable development branch.
-- ``stable``: The bugfix branch.
+- ``stable-release-name``: The name of the stable, `Active Releases`_. e.g. ``Pacific``
 - ``next``: The release candidate branch.
 
 ::
 
-	git checkout master
+	git checkout main
+
+.. _Active Releases: https://docs.ceph.com/en/latest/releases/#active-releases

ceph/doc/install/containers.rst

@@ -19,7 +19,7 @@ Ceph Container Images
 Official Releases
 -----------------
 
-Ceph Container images are available from both Quay and Docker Hub::
+Ceph Container images are available from Quay:
 
   https://quay.io/repository/ceph/ceph
   https://hub.docker.com/r/ceph

ceph/doc/install/get-packages.rst

@@ -120,7 +120,7 @@ For RPMs::
 
   https://download.ceph.com/rpm-{version}
 
-The major releases of Ceph are summarized at: :ref:`ceph-releases-general`
+The major releases of Ceph are summarized at: `Releases`_
 
 .. tip:: For non-US users: There might be a mirror close to you where
          to download Ceph from. For more information see: `Ceph Mirrors`_.
@@ -396,6 +396,7 @@ line to get the short codename.
 
 
 
+.. _Releases: https://docs.ceph.com/en/latest/releases/
 .. _the testing Debian repository: https://download.ceph.com/debian-testing/dists
 .. _the shaman page: https://shaman.ceph.com
 .. _Ceph Mirrors: ../mirrors

ceph/doc/man/8/ceph-osd.rst

@@ -1,5 +1,7 @@
 :orphan:
 
+.. _ceph_osd-daemon:
+
 ========================================
  ceph-osd -- ceph object storage daemon
 ========================================

ceph/doc/man/8/ceph-rbdnamer.rst

@@ -16,15 +16,10 @@ Synopsis
 Description
 ===========
 
-**ceph-rbdnamer** prints the pool and image name for the given RBD devices
+**ceph-rbdnamer** prints the pool, namespace, image and snapshot names
-to stdout. It is used by `udev` (using a rule like the one below) to
+for a given RBD device to stdout. It is used by `udev` device manager
-set up a device symlink.
+to set up RBD device symlinks. The appropriate `udev` rules are
-
+provided in a file named `50-rbd.rules`.
-
-::
-
-        KERNEL=="rbd[0-9]*", PROGRAM="/usr/bin/ceph-rbdnamer %n", SYMLINK+="rbd/%c{1}/%c{2}"
-
 
 Availability
 ============

ceph/doc/man/8/cephfs-top.rst

@@ -43,18 +43,6 @@ Descriptions of fields
 
    cap hit rate
 
-.. describe:: rlat
-
-   read latency
-
-.. describe:: wlat
-
-   write latency
-
-.. describe:: mlat
-
-   metadata latency
-
 .. describe:: dlease
 
    dentry lease rate
@@ -95,6 +83,29 @@ Descriptions of fields
 
    speed of write IOs compared with the last refresh
 
+.. describe:: rlatavg
+
+   average read latency
+
+.. describe:: rlatsd
+
+   standard deviation (variance) for read latency
+
+.. describe:: wlatavg
+
+   average write latency
+
+.. describe:: wlatsd
+
+   standard deviation (variance) for write latency
+
+.. describe:: mlatavg
+
+   average metadata latency
+
+.. describe:: mlatsd
+
+   standard deviation (variance) for metadata latency
 
 Availability
 ============

ceph/doc/man/8/crushtool.rst

@@ -108,6 +108,16 @@ pools; it only runs simulations by mapping values in the range
    shows that value **24** is mapped to devices **[11,6]** by rule
    **1**.
 
+   One of the following is required when using the ``--show-mappings`` option:
+   
+        (a) ``--num-rep`` 
+        (b) both ``--min-rep`` and ``--max-rep``
+
+   ``--num-rep`` stands for "number of replicas, indicates the number of
+   replicas in a pool, and is used to specify an exact number of replicas (for
+   example ``--num-rep 5``). ``--min-rep`` and ``--max-rep`` are used together
+   to specify a range of replicas (for example, ``--min-rep 1 --max-rep 10``).
+
 .. option:: --show-bad-mappings
 
    Displays which value failed to be mapped to the required number of

ceph/doc/man/8/rbd.rst

@@ -825,7 +825,8 @@ Per mapping (block device) `rbd device map` options:
 * alloc_size - Minimum allocation unit of the underlying OSD object store
   backend (since 5.1, default is 64K bytes).  This is used to round off and
   drop discards that are too small.  For bluestore, the recommended setting is
-  bluestore_min_alloc_size (typically 64K for hard disk drives and 16K for
+  bluestore_min_alloc_size (currently set to 4K for all types of drives,
+  previously used to be set to 64K for hard disk drives and 16K for
   solid-state drives).  For filestore with filestore_punch_hole = false, the
   recommended setting is image object size (typically 4M).
 

ceph/doc/mgr/dashboard_plugins/debug.inc.rst

@@ -4,13 +4,30 @@ Debug
 ^^^^^
 
 This plugin allows to customize the behaviour of the dashboard according to the
-debug mode. It can be enabled, disabled or checked with the following command::
+debug mode. It can be enabled, disabled or checked with the following command:
+
+.. prompt:: bash $
+
+   ceph dashboard debug status
+
+::
 
-  $ ceph dashboard debug status
   Debug: 'disabled'
-  $ ceph dashboard debug enable
+
+.. prompt:: bash $
+
+   ceph dashboard debug enable
+
+::
+
   Debug: 'enabled'
-  $ ceph dashboard debug disable
+
+.. prompt:: bash $
+
+   ceph dashboard debug disable
+
+::
+
   Debug: 'disabled'
 
 By default, it's disabled. This is the recommended setting for production

ceph/doc/mgr/dashboard_plugins/feature_toggles.inc.rst

@@ -25,9 +25,14 @@ The list of features that can be enabled/disabled is:
 
 By default all features come enabled.
 
-To retrieve a list of features and their current statuses::
+To retrieve a list of features and their current statuses:
+
+.. prompt:: bash $
+
+   ceph dashboard feature status
+
+::
 
-  $ ceph dashboard feature status
   Feature 'cephfs': 'enabled'
   Feature 'iscsi': 'enabled'
   Feature 'mirroring': 'enabled'
@@ -35,9 +40,14 @@ To retrieve a list of features and their current statuses::
   Feature 'rgw': 'enabled'
   Feature 'nfs': 'enabled'
 
-To enable or disable the status of a single or multiple features::
+To enable or disable the status of a single or multiple features:
+
+.. prompt:: bash $
+
+   ceph dashboard feature disable iscsi mirroring
+
+:: 
 
-  $ ceph dashboard feature disable iscsi mirroring
   Feature 'iscsi': disabled
   Feature 'mirroring': disabled
 

ceph/doc/mgr/dashboard_plugins/motd.inc.rst

@@ -12,17 +12,23 @@ syntax to specify the expiration time: `Ns|m|h|d|w` for seconds, minutes,
 hours, days and weeks. If the MOTD should expire after 2 hours, use `2h`
 or `5w` for 5 weeks. Use `0` to configure a MOTD that does not expire.
 
-To configure a MOTD, run the following command::
+To configure a MOTD, run the following command:
 
-  $ ceph dashboard motd set <severity:info|warning|danger> <expires> <message>
+.. prompt:: bash $
 
-To show the configured MOTD::
+   ceph dashboard motd set <severity:info|warning|danger> <expires> <message>
 
-  $ ceph dashboard motd get
+To show the configured MOTD:
 
-To clear the configured MOTD run::
+.. prompt:: bash $
 
-  $ ceph dashboard motd clear
+   ceph dashboard motd get
+
+To clear the configured MOTD run:
+
+.. prompt:: bash $
+
+   ceph dashboard motd clear
 
 A MOTD with a `info` or `warning` severity can be closed by the user. The
 `info` MOTD is not displayed anymore until the local storage cookies are

ceph/doc/mgr/orchestrator.rst

@@ -5,14 +5,18 @@
 Orchestrator CLI
 ================
 
-This module provides a command line interface (CLI) to orchestrator
+This module provides a command line interface (CLI) for orchestrator modules.
-modules (``ceph-mgr`` modules which interface with external orchestration services).
+Orchestrator modules are ``ceph-mgr`` plugins that interface with external
+orchestration services.
 
-As the orchestrator CLI unifies multiple external orchestrators, a common nomenclature
+Definition of Terms
-for the orchestrator module is needed.
+===================
+
+The orchestrator CLI unifies multiple external orchestrators, so we need a
+common nomenclature for the orchestrator module: 
 
 +--------------------------------------+---------------------------------------+
-| *host*                               | hostname (not DNS name) of the        |
+| *host*                               | hostname (not the DNS name) of the    |
 |                                      | physical host. Not the podname,       |
 |                                      | container name, or hostname inside    |
 |                                      | the container.                        |
@@ -20,7 +24,7 @@ for the orchestrator module is needed.
 | *service type*                       | The type of the service. e.g., nfs,   |
 |                                      | mds, osd, mon, rgw, mgr, iscsi        |
 +--------------------------------------+---------------------------------------+
-| *service*                            | A logical service, Typically          |
+| *service*                            | A logical service. Typically          |
 |                                      | comprised of multiple service         |
 |                                      | instances on multiple hosts for HA    |
 |                                      |                                       |
@@ -34,29 +38,28 @@ for the orchestrator module is needed.
 |                                      | like LIO or knfsd or whatever)        |
 |                                      |                                       |
 |                                      | This identifier should                |
-|                                      | uniquely identify the instance        |
+|                                      | uniquely identify the instance.       |
 +--------------------------------------+---------------------------------------+
 
-The relation between the names is the following:
+Here is how the names relate: 
-
-* A *service* has a specific *service type*
-* A *daemon* is a physical instance of a *service type*
 
+* A *service* has a specific *service type*.
+* A *daemon* is a physical instance of a *service type*.
 
 .. note::
 
-    Orchestrator modules may only implement a subset of the commands listed below.
+    Orchestrator modules might implement only a subset of the commands listed
-    Also, the implementation of the commands may differ between modules.
+    below. The implementation of the commands may differ between modules.
 
 Status
 ======
 
-::
+.. prompt:: bash $
 
    ceph orch status [--detail]
 
-Show current orchestrator mode and high-level status (whether the orchestrator
+This command shows the current orchestrator mode and its high-level status
-plugin is available and operational)
+(whether the orchestrator plugin is available and operational).
 
 
 ..
@@ -92,15 +95,20 @@ plugin is available and operational)
 Stateless services (MDS/RGW/NFS/rbd-mirror/iSCSI)
 =================================================
 
-(Please note: The orchestrator will not configure the services. Please look into the corresponding
+.. note::
-documentation for service configuration details.)
 
-The ``name`` parameter is an identifier of the group of instances:
+   The orchestrator will not configure the services. See the relevant
+   documentation for details about how to configure particular services. 
 
-* a CephFS file system for a group of MDS daemons,
+The ``name`` parameter identifies the kind of the group of instances. The
-* a zone name for a group of RGWs
+following short list explains the meaning of the ``name`` parameter:
 
-Creating/growing/shrinking/removing services::
+* A CephFS file system identifies a group of MDS daemons.
+* A zone name identifies a group of RGWs.
+
+Creating/growing/shrinking/removing services:
+
+.. prompt:: bash $
 
     ceph orch apply mds <fs_name> [--placement=<placement>] [--dry-run]
     ceph orch apply rgw <name> [--realm=<realm>] [--zone=<zone>] [--port=<port>] [--ssl] [--placement=<placement>] [--dry-run]
@@ -111,33 +119,73 @@ where ``placement`` is a :ref:`orchestrator-cli-placement-spec`.
 
 e.g., ``ceph orch apply mds myfs --placement="3 host1 host2 host3"``
 
-Service Commands::
+Service Commands:
+
+.. prompt:: bash $
 
     ceph orch <start|stop|restart|redeploy|reconfig> <service_name>
 
+.. note:: These commands apply only to cephadm containerized daemons.
+
+Options
+=======
+
+.. option:: start
+
+   Start the daemon on the corresponding host.
+
+.. option:: stop
+
+   Stop the daemon on the corresponding host.
+
+.. option:: restart
+
+   Restart the daemon on the corresponding host.
+
+.. option:: redeploy
+
+   Redeploy the ceph daemon on the corresponding host. This will recreate the daemon directory
+   structure under ``/var/lib/ceph/<fsid>/<daemon-name>`` (if it doesn't exist), refresh its
+   configuration files, regenerate its unit-files and restarts the systemd daemon.
+
+.. option:: reconfig
+
+   Reconfigure the daemon on the corresponding host. This will refresh configuration files then restart the daemon.
+
+   .. note:: this command assumes the daemon directory ``/var/lib/ceph/<fsid>/<daemon-name>`` already exists.
 
 
 Configuring the Orchestrator CLI
 ================================
 
-To enable the orchestrator, select the orchestrator module to use
+Enable the orchestrator by using the ``set backend`` command to select the orchestrator module that will be used:
-with the ``set backend`` command::
+
+.. prompt:: bash $
 
     ceph orch set backend <module>
 
-For example, to enable the Rook orchestrator module and use it with the CLI::
+Example - Configuring the Orchestrator CLI
+------------------------------------------
+
+For example, to enable the Rook orchestrator module and use it with the CLI:
+
+.. prompt:: bash $
 
     ceph mgr module enable rook
     ceph orch set backend rook
 
-Check the backend is properly configured::
+Confirm that the backend is properly configured:
+
+.. prompt:: bash $
 
     ceph orch status
 
 Disable the Orchestrator
 ------------------------
 
-To disable the orchestrator, use the empty string ``""``::
+To disable the orchestrator, use the empty string ``""``:
+
+.. prompt:: bash $
 
     ceph orch set backend ""
     ceph mgr module disable rook

ceph/doc/rados/api/index.rst

@@ -4,7 +4,7 @@
 
 The :term:`Ceph Storage Cluster` has a messaging layer protocol that enables
 clients to interact with a :term:`Ceph Monitor` and a :term:`Ceph OSD Daemon`.
-``librados`` provides this functionality to :term:`Ceph Clients` in the form of
+``librados`` provides this functionality to :term:`Ceph Client`\s in the form of
 a library.  All Ceph Clients either use ``librados`` or the same functionality
 encapsulated in ``librados`` to interact with the object store.  For example,
 ``librbd`` and ``libcephfs`` leverage this functionality. You may use

ceph/doc/rados/api/librados-intro.rst

@@ -43,17 +43,23 @@ Getting librados for C/C++
 --------------------------
 
 To install ``librados`` development support files for C/C++ on Debian/Ubuntu
-distributions, execute the following::
+distributions, execute the following:
+
+.. prompt:: bash $
 
    sudo apt-get install librados-dev
 
 To install ``librados`` development support files for C/C++ on RHEL/CentOS
-distributions, execute the following::
+distributions, execute the following:
+
+.. prompt:: bash $
 
    sudo yum install librados2-devel
 
 Once you install ``librados`` for developers, you can find the required 
-headers for C/C++ under ``/usr/include/rados``. ::
+headers for C/C++ under ``/usr/include/rados``:
+
+.. prompt:: bash $
 
    ls /usr/include/rados
 
@@ -68,15 +74,26 @@ and the ``librados2-devel`` package for RHEL/CentOS will install the
 directly too.
 
 To install ``librados`` development support files for Python on Debian/Ubuntu
-distributions, execute the following::
+distributions, execute the following:
 
-	sudo apt-get install python-rados
+.. prompt:: bash $
+
+   sudo apt-get install python3-rados
 
 To install ``librados`` development support files for Python on RHEL/CentOS
-distributions, execute the following::
+distributions, execute the following:
+
+.. prompt:: bash $
 
    sudo yum install python-rados
 
+To install ``librados`` development support files for Python on SLE/openSUSE
+distributions, execute the following:
+
+.. prompt:: bash $
+
+   sudo zypper install python3-rados
+
 You can find the module under ``/usr/share/pyshared`` on Debian systems,
 or under ``/usr/lib/python*/site-packages`` on CentOS/RHEL systems.
 
@@ -86,21 +103,29 @@ Getting librados for Java
 
 To install ``librados`` for Java, you need to execute the following procedure:
 
-#. Install ``jna.jar``. For Debian/Ubuntu, execute:: 
+#. Install ``jna.jar``. For Debian/Ubuntu, execute:
+
+   .. prompt:: bash $
 
       sudo apt-get install libjna-java
 
-   For CentOS/RHEL, execute::
+   For CentOS/RHEL, execute:
+
+   .. prompt:: bash $
 
       sudo yum install jna
 
    The JAR files are located in ``/usr/share/java``.
 
-#. Clone the ``rados-java`` repository::
+#. Clone the ``rados-java`` repository:
+
+   .. prompt:: bash $
 
       git clone --recursive https://github.com/ceph/rados-java.git
 
-#. Build the ``rados-java`` repository:: 
+#. Build the ``rados-java`` repository:
+
+   .. prompt:: bash $
 
       cd rados-java
       ant
@@ -108,13 +133,17 @@ To install ``librados`` for Java, you need to execute the following procedure:
    The JAR file is located under ``rados-java/target``.
 
 #. Copy the JAR for RADOS to a common location (e.g., ``/usr/share/java``) and 
-   ensure that it and the JNA JAR are in your JVM's classpath. For example::
+   ensure that it and the JNA JAR are in your JVM's classpath. For example:
+
+   .. prompt:: bash $
 
       sudo cp target/rados-0.1.3.jar /usr/share/java/rados-0.1.3.jar
       sudo ln -s /usr/share/java/jna-3.2.7.jar /usr/lib/jvm/default-java/jre/lib/ext/jna-3.2.7.jar  
       sudo ln -s /usr/share/java/rados-0.1.3.jar  /usr/lib/jvm/default-java/jre/lib/ext/rados-0.1.3.jar
 
-To build the documentation, execute the following::
+To build the documentation, execute the following:
+
+.. prompt:: bash $
 
    ant docs
 
@@ -124,19 +153,27 @@ Getting librados for PHP
 
 To install the ``librados`` extension for PHP, you need to execute the following procedure:
 
-#. Install php-dev. For Debian/Ubuntu, execute::
+#. Install php-dev. For Debian/Ubuntu, execute:
+
+   .. prompt:: bash $
 
       sudo apt-get install php5-dev build-essential
 
-   For CentOS/RHEL, execute::
+   For CentOS/RHEL, execute:
+
+   .. prompt:: bash $
 
       sudo yum install php-devel
 
-#. Clone the ``phprados`` repository::
+#. Clone the ``phprados`` repository:
+
+   .. prompt:: bash $
 
       git clone https://github.com/ceph/phprados.git
 
-#. Build ``phprados``::
+#. Build ``phprados``:
+
+   .. prompt:: bash $
 
       cd phprados
       phpize
@@ -144,7 +181,7 @@ To install the ``librados`` extension for PHP, you need to execute the following
       make
       sudo make install
 
-#. Enable ``phprados`` in php.ini by adding::
+#. Enable ``phprados`` by adding the following line to ``php.ini``::
    
       extension=rados.so
 
@@ -321,7 +358,9 @@ it and connecting to the cluster might look something like this:
 
 	}
 
-Compile your client and link to ``librados`` using ``-lrados``. For example:: 
+Compile your client and link to ``librados`` using ``-lrados``. For example:
+
+.. prompt:: bash $
 
    gcc ceph-client.c -lrados -o ceph-client
 
@@ -399,7 +438,9 @@ you to initialize a ``librados::Rados`` cluster handle object:
 	
 
 Compile the source; then, link ``librados`` using ``-lrados``. 
-For example::
+For example:
+
+.. prompt:: bash $
 
    g++ -g -c ceph-client.cc -o ceph-client.o
    g++ -g ceph-client.o -lrados -o ceph-client
@@ -436,7 +477,9 @@ into exceptions.
 		print "Connected to the cluster."
 
 
-Execute the example to verify that it connects to your cluster. ::
+Execute the example to verify that it connects to your cluster:
+
+.. prompt:: bash $
 
    python ceph-client.py
 
@@ -478,7 +521,9 @@ binding converts C++-based errors into exceptions.
 
 Compile the source; then, run it. If you have copied the JAR to
 ``/usr/share/java`` and sym linked from your ``ext`` directory, you won't need
-to specify the classpath. For example::
+to specify the classpath. For example:
+
+.. prompt:: bash $
 
    javac CephClient.java
    java CephClient
@@ -502,7 +547,9 @@ With the RADOS extension enabled in PHP you can start creating a new cluster han
 	}
 
 
-Save this as rados.php and run the code::
+Save this as rados.php and run the code:
+
+.. prompt:: bash $
 
    php rados.php
 

ceph/doc/rados/api/python.rst

@@ -68,7 +68,7 @@ Your Python client also requires a client keyring. For this example, we use the
 ``client.admin`` key by default. If you would like to specify the keyring when
 creating the cluster handle, you may use the ``conf`` argument. Alternatively,
 you may specify the keyring path in your Ceph configuration file. For example,
-you may add something like the following line to you Ceph configuration file::
+you may add something like the following line to your Ceph configuration file::
 
 	keyring = /path/to/ceph.client.admin.keyring
 

ceph/doc/rados/configuration/auth-config-ref.rst

@@ -195,7 +195,9 @@ specify a ``keyring`` entry in your Ceph configuration file.
 We recommend copying the Ceph Storage Cluster's keyring file to nodes where you
 will run administrative commands, because it contains the ``client.admin`` key.
 
-To perform this step manually, execute the following::
+To perform this step manually, execute the following:
+
+.. prompt:: bash $
 
    sudo scp {user}@{ceph-cluster-host}:/etc/ceph/ceph.client.admin.keyring /etc/ceph/ceph.client.admin.keyring
 

ceph/doc/rados/configuration/bluestore-config-ref.rst

@@ -42,11 +42,15 @@ it will fit).  This means that if a DB device is specified but an explicit
 WAL device is not, the WAL will be implicitly colocated with the DB on the faster
 device.
 
-A single-device (colocated) BlueStore OSD can be provisioned with::
+A single-device (colocated) BlueStore OSD can be provisioned with:
+
+.. prompt:: bash $
 
    ceph-volume lvm prepare --bluestore --data <device>
 
-To specify a WAL device and/or DB device, ::
+To specify a WAL device and/or DB device:
+   
+.. prompt:: bash $
 
    ceph-volume lvm prepare --bluestore --data <device> --block.wal <wal-device> --block.db <db-device>
 
@@ -64,15 +68,19 @@ the deployment strategy:
 **block (data) only**
 ^^^^^^^^^^^^^^^^^^^^^
 If all devices are the same type, for example all rotational drives, and
-there are no fast devices to use for metadata, it makes sense to specifiy the
+there are no fast devices to use for metadata, it makes sense to specify the
 block device only and to not separate ``block.db`` or ``block.wal``. The
-:ref:`ceph-volume-lvm` command for a single ``/dev/sda`` device looks like::
+:ref:`ceph-volume-lvm` command for a single ``/dev/sda`` device looks like:
+
+.. prompt:: bash $
 
    ceph-volume lvm create --bluestore --data /dev/sda
 
 If logical volumes have already been created for each device, (a single LV
 using 100% of the device), then the :ref:`ceph-volume-lvm` call for an LV named
-``ceph-vg/block-lv`` would look like::
+``ceph-vg/block-lv`` would look like:
+
+.. prompt:: bash $
 
    ceph-volume lvm create --bluestore --data ceph-vg/block-lv
 
@@ -88,35 +96,43 @@ You must create these volume groups and logical volumes manually as
 the ``ceph-volume`` tool is currently not able to do so automatically.
 
 For the below example, let us assume four rotational (``sda``, ``sdb``, ``sdc``, and ``sdd``)
-and one (fast) solid state drive (``sdx``). First create the volume groups::
+and one (fast) solid state drive (``sdx``). First create the volume groups:
 
-    $ vgcreate ceph-block-0 /dev/sda
+.. prompt:: bash $
-    $ vgcreate ceph-block-1 /dev/sdb
-    $ vgcreate ceph-block-2 /dev/sdc
-    $ vgcreate ceph-block-3 /dev/sdd
 
-Now create the logical volumes for ``block``::
+   vgcreate ceph-block-0 /dev/sda
+   vgcreate ceph-block-1 /dev/sdb
+   vgcreate ceph-block-2 /dev/sdc
+   vgcreate ceph-block-3 /dev/sdd
 
-    $ lvcreate -l 100%FREE -n block-0 ceph-block-0
+Now create the logical volumes for ``block``:
-    $ lvcreate -l 100%FREE -n block-1 ceph-block-1
+
-    $ lvcreate -l 100%FREE -n block-2 ceph-block-2
+.. prompt:: bash $
-    $ lvcreate -l 100%FREE -n block-3 ceph-block-3
+
+   lvcreate -l 100%FREE -n block-0 ceph-block-0
+   lvcreate -l 100%FREE -n block-1 ceph-block-1
+   lvcreate -l 100%FREE -n block-2 ceph-block-2
+   lvcreate -l 100%FREE -n block-3 ceph-block-3
 
 We are creating 4 OSDs for the four slow spinning devices, so assuming a 200GB
-SSD in ``/dev/sdx`` we will create 4 logical volumes, each of 50GB::
+SSD in ``/dev/sdx`` we will create 4 logical volumes, each of 50GB:
 
-    $ vgcreate ceph-db-0 /dev/sdx
+.. prompt:: bash $
-    $ lvcreate -L 50GB -n db-0 ceph-db-0
-    $ lvcreate -L 50GB -n db-1 ceph-db-0
-    $ lvcreate -L 50GB -n db-2 ceph-db-0
-    $ lvcreate -L 50GB -n db-3 ceph-db-0
 
-Finally, create the 4 OSDs with ``ceph-volume``::
+   vgcreate ceph-db-0 /dev/sdx
+   lvcreate -L 50GB -n db-0 ceph-db-0
+   lvcreate -L 50GB -n db-1 ceph-db-0
+   lvcreate -L 50GB -n db-2 ceph-db-0
+   lvcreate -L 50GB -n db-3 ceph-db-0
 
-    $ ceph-volume lvm create --bluestore --data ceph-block-0/block-0 --block.db ceph-db-0/db-0
+Finally, create the 4 OSDs with ``ceph-volume``:
-    $ ceph-volume lvm create --bluestore --data ceph-block-1/block-1 --block.db ceph-db-0/db-1
+
-    $ ceph-volume lvm create --bluestore --data ceph-block-2/block-2 --block.db ceph-db-0/db-2
+.. prompt:: bash $
-    $ ceph-volume lvm create --bluestore --data ceph-block-3/block-3 --block.db ceph-db-0/db-3
+
+   ceph-volume lvm create --bluestore --data ceph-block-0/block-0 --block.db ceph-db-0/db-0
+   ceph-volume lvm create --bluestore --data ceph-block-1/block-1 --block.db ceph-db-0/db-1
+   ceph-volume lvm create --bluestore --data ceph-block-2/block-2 --block.db ceph-db-0/db-2
+   ceph-volume lvm create --bluestore --data ceph-block-3/block-3 --block.db ceph-db-0/db-3
 
 These operations should end up creating four OSDs, with ``block`` on the slower
 rotational drives with a 50 GB logical volume (DB) for each on the solid state
@@ -139,7 +155,7 @@ In older releases, internal level sizes mean that the DB can fully utilize only
 specific partition / LV sizes that correspond to sums of L0, L0+L1, L1+L2,
 etc. sizes, which with default settings means roughly 3 GB, 30 GB, 300 GB, and
 so forth.  Most deployments will not substantially benefit from sizing to
-accomodate L3 and higher, though DB compaction can be facilitated by doubling
+accommodate L3 and higher, though DB compaction can be facilitated by doubling
 these figures to 6GB, 60GB, and 600GB.
 
 Improvements in releases beginning with Nautilus 14.2.12 and Octopus 15.2.6
@@ -167,93 +183,6 @@ of priorities.  If priority information is not available, the
 ``bluestore_cache_meta_ratio`` and ``bluestore_cache_kv_ratio`` options are
 used as fallbacks.
 
-``bluestore_cache_autotune``
-
-:Description: Automatically tune the space ratios assigned to various BlueStore
-              caches while respecting minimum values.
-:Type: Boolean
-:Required: Yes
-:Default: ``True``
-
-``osd_memory_target``
-
-:Description: When TCMalloc is available and cache autotuning is enabled, try to
-              keep this many bytes mapped in memory. Note: This may not exactly
-              match the RSS memory usage of the process.  While the total amount
-              of heap memory mapped by the process should usually be close
-              to this target, there is no guarantee that the kernel will actually
-              reclaim  memory that has been unmapped.  During initial development,
-              it was found that some kernels result in the OSD's RSS memory
-              exceeding the mapped memory by up to 20%.  It is hypothesised
-              however, that the kernel generally may be more aggressive about
-              reclaiming unmapped memory when there is a high amount of memory
-              pressure.  Your mileage may vary.
-:Type: Unsigned Integer
-:Required: Yes
-:Default: ``4294967296``
-
-``bluestore_cache_autotune_chunk_size``
-
-:Description: The chunk size in bytes to allocate to caches when cache autotune
-              is enabled.  When the autotuner assigns memory to various caches,
-              it will allocate memory in chunks.  This is done to avoid
-              evictions when there are minor fluctuations in the heap size or
-              autotuned cache ratios.
-:Type: Unsigned Integer
-:Required: No
-:Default: ``33554432``
-
-``bluestore_cache_autotune_interval``
-
-:Description: The number of seconds to wait between rebalances when cache autotune
-              is enabled.  This setting changes how quickly the allocation ratios of
-              various caches are recomputed.  Note:  Setting this interval too small
-              can result in high CPU usage and lower performance.
-:Type: Float
-:Required: No
-:Default: ``5``
-
-``osd_memory_base``
-
-:Description: When TCMalloc and cache autotuning are enabled, estimate the minimum
-              amount of memory in bytes the OSD will need.  This is used to help
-              the autotuner estimate the expected aggregate memory consumption of
-              the caches.
-:Type: Unsigned Integer
-:Required: No
-:Default: ``805306368``
-
-``osd_memory_expected_fragmentation``
-
-:Description: When TCMalloc and cache autotuning is enabled, estimate the
-              percentage of memory fragmentation.  This is used to help the
-              autotuner estimate the expected aggregate memory consumption
-              of the caches.
-:Type: Float
-:Required: No
-:Default: ``0.15``
-
-``osd_memory_cache_min``
-
-:Description: When TCMalloc and cache autotuning are enabled, set the minimum
-              amount of memory used for caches. Note: Setting this value too
-              low can result in significant cache thrashing.
-:Type: Unsigned Integer
-:Required: No
-:Default: ``134217728``
-
-``osd_memory_cache_resize_interval``
-
-:Description: When TCMalloc and cache autotuning are enabled, wait this many
-              seconds between resizing caches.  This setting changes the total
-              amount of memory available for BlueStore to use for caching.  Note
-              that setting this interval too small can result in memory allocator
-              thrashing and lower performance.
-:Type: Float
-:Required: No
-:Default: ``1``
-
-
 Manual Cache Sizing
 ===================
 
@@ -286,53 +215,6 @@ device) as well as the meta and kv ratios.
 The data fraction can be calculated by
 ``<effective_cache_size> * (1 - bluestore_cache_meta_ratio - bluestore_cache_kv_ratio)``
 
-``bluestore_cache_size``
-
-:Description: The amount of memory BlueStore will use for its cache.  If zero,
-              ``bluestore_cache_size_hdd`` or ``bluestore_cache_size_ssd`` will
-              be used instead.
-:Type: Unsigned Integer
-:Required: Yes
-:Default: ``0``
-
-``bluestore_cache_size_hdd``
-
-:Description: The default amount of memory BlueStore will use for its cache when
-              backed by an HDD.
-:Type: Unsigned Integer
-:Required: Yes
-:Default: ``1 * 1024 * 1024 * 1024`` (1 GB)
-
-``bluestore_cache_size_ssd``
-
-:Description: The default amount of memory BlueStore will use for its cache when
-              backed by an SSD.
-:Type: Unsigned Integer
-:Required: Yes
-:Default: ``3 * 1024 * 1024 * 1024`` (3 GB)
-
-``bluestore_cache_meta_ratio``
-
-:Description: The ratio of cache devoted to metadata.
-:Type: Floating point
-:Required: Yes
-:Default: ``.4``
-
-``bluestore_cache_kv_ratio``
-
-:Description: The ratio of cache devoted to key/value data (RocksDB).
-:Type: Floating point
-:Required: Yes
-:Default: ``.4``
-
-``bluestore_cache_kv_max``
-
-:Description: The maximum amount of cache devoted to key/value data (RocksDB).
-:Type: Unsigned Integer
-:Required: Yes
-:Default: ``512 * 1024*1024`` (512 MB)
-
-
 Checksums
 =========
 
@@ -358,19 +240,12 @@ The smaller checksum values can be used by selecting `crc32c_16` or
 `crc32c_8` as the checksum algorithm.
 
 The *checksum algorithm* can be set either via a per-pool
-``csum_type`` property or the global config option.  For example, ::
+``csum_type`` property or the global config option.  For example:
+
+.. prompt:: bash $
 
    ceph osd pool set <pool-name> csum_type <algorithm>
 
-``bluestore_csum_type``
-
-:Description: The default checksum algorithm to use.
-:Type: String
-:Required: Yes
-:Valid Settings: ``none``, ``crc32c``, ``crc32c_16``, ``crc32c_8``, ``xxhash32``, ``xxhash64``
-:Default: ``crc32c``
-
-
 Inline Compression
 ==================
 
@@ -401,7 +276,9 @@ must be 70% of the size of the original (or smaller).
 The *compression mode*, *compression algorithm*, *compression required
 ratio*, *min blob size*, and *max blob size* can be set either via a
 per-pool property or a global config option.  Pool properties can be
-set with::
+set with:
+
+.. prompt:: bash $
 
    ceph osd pool set <pool-name> compression_algorithm <algorithm>
    ceph osd pool set <pool-name> compression_mode <mode>
@@ -409,99 +286,37 @@ set with::
    ceph osd pool set <pool-name> compression_min_blob_size <size>
    ceph osd pool set <pool-name> compression_max_blob_size <size>
 
-``bluestore_compression_algorithm``
+.. _bluestore-rocksdb-sharding:
 
-:Description: The default compressor to use (if any) if the per-pool property
+RocksDB Sharding
-              ``compression_algorithm`` is not set. Note that ``zstd`` is *not*
+================
-              recommended for BlueStore due to high CPU overhead when
-              compressing small amounts of data.
-:Type: String
-:Required: No
-:Valid Settings: ``lz4``, ``snappy``, ``zlib``, ``zstd``
-:Default: ``snappy``
 
-``bluestore_compression_mode``
+Internally BlueStore uses multiple types of key-value data,
+stored in RocksDB.  Each data type in BlueStore is assigned a
+unique prefix. Until Pacific all key-value data was stored in
+single RocksDB column family: 'default'.  Since Pacific,
+BlueStore can divide this data into multiple RocksDB column
+families. When keys have similar access frequency, modification
+frequency and lifetime, BlueStore benefits from better caching
+and more precise compaction. This improves performance, and also
+requires less disk space during compaction, since each column
+family is smaller and can compact independent of others.
 
-:Description: The default policy for using compression if the per-pool property
+OSDs deployed in Pacific or later use RocksDB sharding by default.
-              ``compression_mode`` is not set. ``none`` means never use
+If Ceph is upgraded to Pacific from a previous version, sharding is off.
-              compression. ``passive`` means use compression when
-              :c:func:`clients hint <rados_set_alloc_hint>` that data is
-              compressible.  ``aggressive`` means use compression unless
-              clients hint that data is not compressible.  ``force`` means use
-              compression under all circumstances even if the clients hint that
-              the data is not compressible.
-:Type: String
-:Required: No
-:Valid Settings: ``none``, ``passive``, ``aggressive``, ``force``
-:Default: ``none``
 
-``bluestore_compression_required_ratio``
+To enable sharding and apply the Pacific defaults, stop an OSD and run
 
-:Description: The ratio of the size of the data chunk after
+    .. prompt:: bash #
-              compression relative to the original size must be at
-              least this small in order to store the compressed
-              version.
 
-:Type: Floating point
+      ceph-bluestore-tool \
-:Required: No
+        --path <data path> \
-:Default: .875
+        --sharding="m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L P" \
+        reshard
 
-``bluestore_compression_min_blob_size``
 
-:Description: Chunks smaller than this are never compressed.
+Throttling
-              The per-pool property ``compression_min_blob_size`` overrides
+==========
-              this setting.
-
-:Type: Unsigned Integer
-:Required: No
-:Default: 0
-
-``bluestore_compression_min_blob_size_hdd``
-
-:Description: Default value of ``bluestore compression min blob size``
-              for rotational media.
-
-:Type: Unsigned Integer
-:Required: No
-:Default: 128K
-
-``bluestore_compression_min_blob_size_ssd``
-
-:Description: Default value of ``bluestore compression min blob size``
-              for non-rotational (solid state) media.
-
-:Type: Unsigned Integer
-:Required: No
-:Default: 8K
-
-``bluestore_compression_max_blob_size``
-
-:Description: Chunks larger than this value are broken into smaller blobs of at most
-              ``bluestore_compression_max_blob_size`` bytes before being compressed.
-              The per-pool property ``compression_max_blob_size`` overrides
-              this setting.
-
-:Type: Unsigned Integer
-:Required: No
-:Default: 0
-
-``bluestore_compression_max_blob_size_hdd``
-
-:Description: Default value of ``bluestore compression max blob size``
-              for rotational media.
-
-:Type: Unsigned Integer
-:Required: No
-:Default: 512K
-
-``bluestore_compression_max_blob_size_ssd``
-
-:Description: Default value of ``bluestore compression max blob size``
-              for non-rotational (SSD, NVMe) media.
-
-:Type: Unsigned Integer
-:Required: No
-:Default: 64K
 
 SPDK Usage
 ==================
@@ -512,29 +327,38 @@ Refer to `SPDK document`__ for more details.
 .. __: http://www.spdk.io/doc/getting_started.html#getting_started_examples
 
 SPDK offers a script to configure the device automatically. Users can run the
-script as root::
+script as root:
 
-  $ sudo src/spdk/scripts/setup.sh
+.. prompt:: bash $
+
+   sudo src/spdk/scripts/setup.sh
 
 You will need to specify the subject NVMe device's device selector with
 the "spdk:" prefix for ``bluestore_block_path``.
 
-For example, you can find the device selector of an Intel PCIe SSD with::
+For example, you can find the device selector of an Intel PCIe SSD with:
 
-  $ lspci -mm -n -D -d 8086:0953
+.. prompt:: bash $
+
+   lspci -mm -n -D -d 8086:0953
 
 The device selector always has the form of ``DDDD:BB:DD.FF`` or ``DDDD.BB.DD.FF``.
 
 and then set::
 
-  bluestore_block_path = spdk:0000:01:00.0
+  bluestore_block_path = "spdk:trtype:PCIe traddr:0000:01:00.0"
 
 Where ``0000:01:00.0`` is the device selector found in the output of ``lspci``
 command above.
 
+You may also specify a remote NVMeoF target over the TCP transport as in the
+following example::
+
+  bluestore_block_path = "spdk:trtype:TCP traddr:10.67.110.197 trsvcid:4420 subnqn:nqn.2019-02.io.spdk:cnode1"
+
 To run multiple SPDK instances per node, you must specify the
 amount of dpdk memory in MB that each instance will use, to make sure each
-instance uses its own dpdk memory
+instance uses its own DPDK memory.
 
 In most cases, a single device can be used for data, DB, and WAL.  We describe
 this strategy as *colocating* these components. Be sure to enter the below
@@ -547,3 +371,112 @@ settings to ensure that all IOs are issued through SPDK.::
 
 Otherwise, the current implementation will populate the SPDK map files with
 kernel file system symbols and will use the kernel driver to issue DB/WAL IO.
+
+Minimum Allocation Size
+========================
+
+There is a configured minimum amount of storage that BlueStore will allocate on
+an OSD.  In practice, this is the least amount of capacity that a RADOS object
+can consume.  The value of `bluestore_min_alloc_size` is derived from the
+value of `bluestore_min_alloc_size_hdd` or `bluestore_min_alloc_size_ssd`
+depending on the OSD's ``rotational`` attribute.  This means that when an OSD
+is created on an HDD, BlueStore will be initialized with the current value
+of `bluestore_min_alloc_size_hdd`, and SSD OSDs (including NVMe devices)
+with the value of `bluestore_min_alloc_size_ssd`.
+
+Through the Mimic release, the default values were 64KB and 16KB for rotational
+(HDD) and non-rotational (SSD) media respectively.  Octopus changed the default
+for SSD (non-rotational) media to 4KB, and Pacific changed the default for HDD
+(rotational) media to 4KB as well.
+
+These changes were driven by space amplification experienced by Ceph RADOS
+GateWay (RGW) deployments that host large numbers of small files
+(S3/Swift objects).
+
+For example, when an RGW client stores a 1KB S3 object, it is written to a
+single RADOS object.  With the default `min_alloc_size` value, 4KB of
+underlying drive space is allocated.  This means that roughly
+(4KB - 1KB) == 3KB is allocated but never used, which corresponds to 300%
+overhead or 25% efficiency. Similarly, a 5KB user object will be stored
+as one 4KB and one 1KB RADOS object, again stranding 4KB of device capcity,
+though in this case the overhead is a much smaller percentage.  Think of this
+in terms of the remainder from a modulus operation. The overhead *percentage*
+thus decreases rapidly as user object size increases.
+
+An easily missed additional subtlety is that this
+takes place for *each* replica.  So when using the default three copies of
+data (3R), a 1KB S3 object actually consumes roughly 9KB of storage device
+capacity.  If erasure coding (EC) is used instead of replication, the
+amplification may be even higher: for a ``k=4,m=2`` pool, our 1KB S3 object
+will allocate (6 * 4KB) = 24KB of device capacity.
+
+When an RGW bucket pool contains many relatively large user objects, the effect
+of this phenomenon is often negligible, but should be considered for deployments
+that expect a signficiant fraction of relatively small objects.
+
+The 4KB default value aligns well with conventional HDD and SSD devices.  Some
+new coarse-IU (Indirection Unit) QLC SSDs however perform and wear best
+when `bluestore_min_alloc_size_ssd`
+is set at OSD creation to match the device's IU:. 8KB, 16KB, or even 64KB.
+These novel storage drives allow one to achieve read performance competitive
+with conventional TLC SSDs and write performance faster than HDDs, with
+high density and lower cost than TLC SSDs.
+
+Note that when creating OSDs on these devices, one must carefully apply the
+non-default value only to appropriate devices, and not to conventional SSD and
+HDD devices.  This may be done through careful ordering of OSD creation, custom
+OSD device classes, and especially by the use of central configuration _masks_.
+
+Quincy and later releases add
+the `bluestore_use_optimal_io_size_for_min_alloc_size`
+option that enables automatic discovery of the appropriate value as each OSD is
+created.  Note that the use of ``bcache``, ``OpenCAS``, ``dmcrypt``,
+``ATA over Ethernet``, `iSCSI`, or other device layering / abstraction
+technologies may confound the determination of appropriate values. OSDs
+deployed on top of VMware storage have been reported to also
+sometimes report a ``rotational`` attribute that does not match the underlying
+hardware.
+
+We suggest inspecting such OSDs at startup via logs and admin sockets to ensure that
+behavior is appropriate.  Note that this also may not work as desired with
+older kernels.  You can check for this by examining the presence and value
+of ``/sys/block/<drive>/queue/optimal_io_size``.
+
+You may also inspect a given OSD:
+
+.. prompt:: bash #
+
+   ceph osd metadata osd.1701 | grep rotational
+
+This space amplification may manifest as an unusually high ratio of raw to
+stored data reported by ``ceph df``.  ``ceph osd df`` may also report
+anomalously high ``%USE`` / ``VAR`` values when
+compared to other, ostensibly identical OSDs.  A pool using OSDs with
+mismatched ``min_alloc_size`` values may experience unexpected balancer
+behavior as well.
+
+Note that this BlueStore attribute takes effect *only* at OSD creation; if
+changed later, a given OSD's behavior will not change unless / until it is
+destroyed and redeployed with the appropriate option value(s).  Upgrading
+to a later Ceph release will *not* change the value used by OSDs deployed
+under older releases or with other settings.
+
+DSA (Data Streaming Accelerator Usage)
+======================================
+
+If you want to use the DML library to drive DSA device for offloading
+read/write operations on Persist memory in Bluestore. You need to install
+`DML`_ and `idxd-config`_ library in your machine with SPR (Sapphire Rapids) CPU.
+
+.. _DML: https://github.com/intel/DML
+.. _idxd-config: https://github.com/intel/idxd-config
+
+After installing the DML software, you need to configure the shared
+work queues (WQs) with the following WQ configuration example via accel-config tool:
+
+.. prompt:: bash $
+
+   accel-config config-wq --group-id=1 --mode=shared --wq-size=16 --threshold=15 --type=user --name="MyApp1" --priority=10 --block-on-fault=1 dsa0/wq0.1
+   accel-config config-engine dsa0/engine0.1 --group-id=1
+   accel-config enable-device dsa0
+   accel-config enable-wq dsa0/wq0.1

ceph/doc/rados/configuration/ceph-conf.rst

@@ -484,17 +484,26 @@ The following CLI commands are used to configure the cluster:
 Help
 ====
 
-You can get help for a particular option with::
+You can get help for a particular option with:
+
+.. prompt:: bash $
 
    ceph config help <option>
 
-Note that this will use the configuration schema that is compiled into the running monitors.  If you have a mixed-version cluster (e.g., during an upgrade), you might also want to query the option schema from a specific running daemon::
+Note that this will use the configuration schema that is compiled into the running monitors.  If you have a mixed-version cluster (e.g., during an upgrade), you might also want to query the option schema from a specific running daemon:
+
+.. prompt:: bash $
 
    ceph daemon <name> config help [option]
 
-For example,::
+For example:
+
+.. prompt:: bash $
+
+   ceph config help log_file
+
+:: 
 
-  $ ceph config help log_file
   log_file - path to log file
     (std::string, basic)
     Default (non-daemon):
@@ -502,9 +511,14 @@ For example,::
     Can update at runtime: false
     See also: [log_to_stderr,err_to_stderr,log_to_syslog,err_to_syslog]
 
-or::
+or:
+
+.. prompt:: bash $
+
+   ceph config help log_file -f json-pretty
+
+::
 
-  $ ceph config help log_file -f json-pretty
   {
       "name": "log_file",
       "type": "std::string",
@@ -541,7 +555,9 @@ increasing/decreasing logging output, enabling/disabling debug
 settings, and even for runtime optimization.
 
 Generally speaking, configuration options can be updated in the usual
-way via the ``ceph config set`` command.  For example, do enable the debug log level on a specific OSD,::
+way via the ``ceph config set`` command.  For example, do enable the debug log level on a specific OSD:
+
+.. prompt:: bash $
 
    ceph config set osd.123 debug_ms 20
 
@@ -559,26 +575,36 @@ the daemon or process restarts.
 
 Override values can be set in two ways:
 
-#. From any host, we can send a message to a daemon over the network with::
+#. From any host, we can send a message to a daemon over the network with:
+   
+   .. prompt:: bash $
 
       ceph tell <name> config set <option> <value>
 
-   For example,::
+   For example:
+   
+   .. prompt:: bash $
 
       ceph tell osd.123 config set debug_osd 20
 
    The `tell` command can also accept a wildcard for the daemon
    identifier.  For example, to adjust the debug level on all OSD
-   daemons,::
+   daemons:
+   
+   .. prompt:: bash $
 
       ceph tell osd.* config set debug_osd 20
 
 #. From the host the process is running on, we can connect directly to
-   the process via a socket in ``/var/run/ceph`` with::
+   the process via a socket in ``/var/run/ceph`` with:
+
+   .. prompt:: bash $
 
       ceph daemon <name> config set <option> <value>
 
-   For example,::
+   For example:
+   
+   .. prompt:: bash $
 
       ceph daemon osd.4 config set debug_osd 20
 
@@ -589,27 +615,39 @@ values will be shown with a source of ``override``.
 Viewing runtime settings
 ========================
 
-You can see the current options set for a running daemon with the ``ceph config show`` command.  For example,::
+You can see the current options set for a running daemon with the ``ceph config show`` command.  For example:
+
+.. prompt:: bash $
 
    ceph config show osd.0
 
-will show you the (non-default) options for that daemon.  You can also look at a specific option with::
+will show you the (non-default) options for that daemon.  You can also look at a specific option with:
+
+.. prompt:: bash $
 
    ceph config show osd.0 debug_osd
 
-or view all options (even those with default values) with::
+or view all options (even those with default values) with:
+
+.. prompt:: bash $
 
    ceph config show-with-defaults osd.0
 
-You can also observe settings for a running daemon by connecting to it from the local host via the admin socket.  For example,::
+You can also observe settings for a running daemon by connecting to it from the local host via the admin socket.  For example:
+
+.. prompt:: bash $
 
    ceph daemon osd.0 config show
 
-will dump all current settings,::
+will dump all current settings:
+
+.. prompt:: bash $
 
    ceph daemon osd.0 config diff
 
-will show only non-default settings (as well as where the value came from: a config file, the monitor, an override, etc.), and::
+will show only non-default settings (as well as where the value came from: a config file, the monitor, an override, etc.), and:
+
+.. prompt:: bash $
 
    ceph daemon osd.0 config get debug_osd
 

ceph/doc/rados/configuration/index.rst

@@ -2,12 +2,13 @@
  Configuration
 ===============
 
-Each Ceph process, daemon, or utility draws its configuration from
+Each Ceph process, daemon, or utility draws its configuration from several
-several sources on startup, include a local configuration, the
+sources on startup. Such sources can include (1) a local configuration, (2) the
-monitors, the command line, or environment variables.  Configuration
+monitors, (3) the command line, and (4) environment variables.
-options may be set globally such that they apply to all daemons, to
+
-all daemons or services of a particular type, or only to a specific
+Configuration options can be set globally so that they apply (1) to all
-daemon, process, or client.
+daemons, (2) to all daemons or services of a particular type, or (3) to only a
+specific daemon, process, or client.
 
 .. raw:: html
 

ceph/doc/rados/configuration/mon-config-ref.rst

@@ -1,3 +1,5 @@
+.. _monitor-config-reference:
+
 ==========================
  Monitor Config Reference
 ==========================

ceph/doc/rados/configuration/msgr2.rst

@@ -163,14 +163,19 @@ By default, ``ms_bind_msgr2`` is true starting with Nautilus 14.2.z.
 However, until the monitors start using v2, only limited services will
 start advertising v2 addresses.
 
-For most users, the monitors are binding to the default legacy port ``6789`` for the v1 protocol.  When this is the case, enabling v2 is as simple as::
+For most users, the monitors are binding to the default legacy port ``6789``
+for the v1 protocol.  When this is the case, enabling v2 is as simple as:
+
+.. prompt:: bash $
 
    ceph mon enable-msgr2
 
 If the monitors are bound to non-standard ports, you will need to
 specify an additional port for v2 explicitly.  For example, if your
 monitor ``mon.a`` binds to ``1.2.3.4:1111``, and you want to add v2 on
-port ``1112``,::
+port ``1112``:
+
+.. prompt:: bash $
 
    ceph mon set-addrs a [v2:1.2.3.4:1112,v1:1.2.3.4:1111]
 

ceph/doc/rados/configuration/network-config-ref.rst

@@ -60,6 +60,8 @@ By default, daemons `bind`_ to ports within the ``6800:7300`` range. You may
 configure this range at your discretion. Before configuring your IP tables,
 check the default ``iptables`` configuration.
 
+.. prompt:: bash $
+
    sudo iptables -L
 
 Some Linux distributions include rules that reject all inbound requests
@@ -80,7 +82,9 @@ default. Additionally, Ceph Monitors always operate on the public
 network. When you add the rule using the example below, make sure you
 replace ``{iface}`` with the public network interface (e.g., ``eth0``,
 ``eth1``, etc.), ``{ip-address}`` with the IP address of the public
-network and ``{netmask}`` with the netmask for the public network. ::
+network and ``{netmask}`` with the netmask for the public network. :
+
+.. prompt:: bash $
 
    sudo iptables -A INPUT -i {iface} -p tcp -s {ip-address}/{netmask} --dport 6789 -j ACCEPT
 
@@ -98,7 +102,9 @@ you replace ``{iface}`` with the public network interface (e.g., ``eth0``,
 ``eth1``, etc.), ``{ip-address}`` with the IP address of the public network
 and ``{netmask}`` with the netmask of the public network.
 
-For example:: 
+For example:
+
+.. prompt:: bash $
 
    sudo iptables -A INPUT -i {iface} -m multiport -p tcp -s {ip-address}/{netmask} --dports 6800:7300 -j ACCEPT
 
@@ -139,7 +145,9 @@ the public network and other Ceph OSD Daemons will connect using the cluster
 network. When you add the rule using the example below, make sure you replace
 ``{iface}`` with the network interface (e.g., ``eth0``, ``eth1``, etc.),
 ``{ip-address}`` with the IP address and ``{netmask}`` with the netmask of the
-public or cluster network. For example:: 
+public or cluster network. For example:
+
+.. prompt:: bash $
 
    sudo iptables -A INPUT -i {iface}  -m multiport -p tcp -s {ip-address}/{netmask} --dports 6800:7300 -j ACCEPT
 

ceph/doc/rados/configuration/storage-devices.rst

@@ -4,37 +4,50 @@
 
 There are two Ceph daemons that store data on devices:
 
-* **Ceph OSDs** (or Object Storage Daemons) are where most of the
+.. _rados_configuration_storage-devices_ceph_osd:
-  data is stored in Ceph.  Generally speaking, each OSD is backed by
+
-  a single storage device, like a traditional hard disk (HDD) or
+* **Ceph OSDs** (Object Storage Daemons) store most of the data
-  solid state disk (SSD).  OSDs can also be backed by a combination
+  in Ceph. Usually each OSD is backed by a single storage device.
-  of devices, like a HDD for most data and an SSD (or partition of an
+  This can be a traditional hard disk (HDD) or a solid state disk
+  (SSD). OSDs can also be backed by a combination of devices: for
+  example, a HDD for most data and an SSD (or partition of an
   SSD) for some metadata. The number of OSDs in a cluster is
-  generally a function of how much data will be stored, how big each
+  usually a function of the amount of data to be stored, the size
-  storage device will be, and the level and type of redundancy
+  of each storage device, and the level and type of redundancy
-  (replication or erasure coding).
+  specified (replication or erasure coding).
-* **Ceph Monitor** daemons manage critical cluster state like cluster
+* **Ceph Monitor** daemons manage critical cluster state. This
-  membership and authentication information.  For smaller clusters a
+  includes cluster membership and authentication information.
-  few gigabytes is all that is needed, although for larger clusters
+  Small clusters require only a few gigabytes of storage to hold
-  the monitor database can reach tens or possibly hundreds of
+  the monitor database. In large clusters, however, the monitor
+  database can reach sizes of tens of gigabytes to hundreds of
   gigabytes.  
+* **Ceph Manager** daemons run alongside monitor daemons, providing
+  additional monitoring and providing interfaces to external
+  monitoring and management systems.
 
 
-OSD Backends
+OSD Back Ends
-============
+=============
 
-There are two ways that OSDs can manage the data they store.  Starting
+There are two ways that OSDs manage the data they store.  As of the Luminous
-with the Luminous 12.2.z release, the new default (and recommended) backend is
+12.2.z release, the default (and recommended) back end is *BlueStore*.  Prior
-*BlueStore*.  Prior to Luminous, the default (and only option) was
+to the Luminous release, the default (and only) back end was *Filestore*.
-*Filestore*.
+
+.. _rados_config_storage_devices_bluestore:
 
 BlueStore
 ---------
 
+<<<<<<< HEAD
 BlueStore is a special-purpose storage backend designed specifically
 for managing data on disk for Ceph OSD workloads.  It is motivated by
 experience supporting and managing OSDs using FileStore over the
 last ten years.  Key BlueStore features include:
+=======
+BlueStore is a special-purpose storage back end designed specifically for
+managing data on disk for Ceph OSD workloads.  BlueStore's design is based on
+a decade of experience of supporting and managing Filestore OSDs. 
+>>>>>>> 28abc6a9a59 (doc/rados: s/backend/back end/)
 
 * Direct management of storage devices.  BlueStore consumes raw block
   devices or partitions.  This avoids any intervening layers of

ceph/doc/rados/operations/add-or-rm-mons.rst

@@ -95,7 +95,9 @@ without the ``mon.`` prefix (i.e., ``{mon-id}`` should be the ``a``
 on ``mon.a``).
 
 #. Create the default directory on the machine that will host your 
-   new monitor. :: 
+   new monitor:
+
+   .. prompt:: bash $
 
 	ssh {new-mon-host}
 	sudo mkdir /var/lib/ceph/mon/ceph-{mon-id}
@@ -103,26 +105,34 @@ on ``mon.a``).
 #. Create a temporary directory ``{tmp}`` to keep the files needed during 
    this process. This directory should be different from the monitor's default 
    directory created in the previous step, and can be removed after all the 
-   steps are executed. :: 
+   steps are executed:
+
+   .. prompt:: bash $
 
 	mkdir {tmp}
 
 #. Retrieve the keyring for your monitors, where ``{tmp}`` is the path to 
    the retrieved keyring, and ``{key-filename}`` is the name of the file 
-   containing the retrieved monitor key. :: 
+   containing the retrieved monitor key:
+
+   .. prompt:: bash $
 
       ceph auth get mon. -o {tmp}/{key-filename}
 
 #. Retrieve the monitor map, where ``{tmp}`` is the path to 
    the retrieved monitor map, and ``{map-filename}`` is the name of the file 
-   containing the retrieved monitor map. :: 
+   containing the retrieved monitor map:
+
+   .. prompt:: bash $
 
       ceph mon getmap -o {tmp}/{map-filename}
 
 #. Prepare the monitor's data directory created in the first step. You must 
    specify the path to the monitor map so that you can retrieve the 
    information about a quorum of monitors and their ``fsid``. You must also 
-   specify a path to the monitor keyring:: 
+   specify a path to the monitor keyring:
+   
+   .. prompt:: bash $
 
       sudo ceph-mon -i {mon-id} --mkfs --monmap {tmp}/{map-filename} --keyring {tmp}/{key-filename}
 	
@@ -130,7 +140,9 @@ on ``mon.a``).
 #. Start the new monitor and it will automatically join the cluster.
    The daemon needs to know which address to bind to, via either the
    ``--public-addr {ip}`` or ``--public-network {network}`` argument.
-   For example::
+   For example:
+   
+   .. prompt:: bash $
 
       ceph-mon -i {mon-id} --public-addr {ip:port}
 
@@ -154,11 +166,15 @@ procedure results in only two monitor daemons, you may add or remove another
 monitor until you have a number of ``ceph-mon`` daemons that can achieve a 
 quorum.
 
-#. Stop the monitor. ::
+#. Stop the monitor:
+
+   .. prompt:: bash $
 
       service ceph -a stop mon.{mon-id}
 	
-#. Remove the monitor from the cluster. ::
+#. Remove the monitor from the cluster:
+
+   .. prompt:: bash $
 
       ceph mon remove {mon-id}
 	
@@ -174,37 +190,60 @@ cluster, for example a cluster where the monitors cannot form a
 quorum.
 
 
-#. Stop all ``ceph-mon`` daemons on all monitor hosts. ::
+#. Stop all ``ceph-mon`` daemons on all monitor hosts:
+
+   .. prompt:: bash $
 
       ssh {mon-host}
       systemctl stop ceph-mon.target
-	# and repeat for all mons
 
-#. Identify a surviving monitor and log in to that host. :: 
+   Repeat for all monitor hosts.
+
+#. Identify a surviving monitor and log in to that host:
+
+   .. prompt:: bash $
 
       ssh {mon-host}
 
-#. Extract a copy of the monmap file.  ::
+#. Extract a copy of the monmap file:
+
+   .. prompt:: bash $
 
       ceph-mon -i {mon-id} --extract-monmap {map-path}
-        # in most cases, that's
+
+   In most cases, this command will be:
+
+   .. prompt:: bash $
+
       ceph-mon -i `hostname` --extract-monmap /tmp/monmap
 
 #. Remove the non-surviving or problematic monitors.  For example, if
    you have three monitors, ``mon.a``, ``mon.b``, and ``mon.c``, where
-   only ``mon.a`` will survive, follow the example below::
+   only ``mon.a`` will survive, follow the example below:
+
+   .. prompt:: bash $
 
       monmaptool {map-path} --rm {mon-id}
-	# for example,
+
+   For example,
+
+   .. prompt:: bash $
+
       monmaptool /tmp/monmap --rm b
       monmaptool /tmp/monmap --rm c
 	
 #. Inject the surviving map with the removed monitors into the
    surviving monitor(s).  For example, to inject a map into monitor
-   ``mon.a``, follow the example below::
+   ``mon.a``, follow the example below:
+
+   .. prompt:: bash $
 
       ceph-mon -i {mon-id} --inject-monmap {map-path}
-	# for example,
+
+   For example:
+
+   .. prompt:: bash $
+
       ceph-mon -i a --inject-monmap /tmp/monmap
 
 #. Start only the surviving monitors.
@@ -316,13 +355,19 @@ networks  are unable to communicate.  Use the following procedure:
 
 #. Retrieve the monitor map, where ``{tmp}`` is the path to 
    the retrieved monitor map, and ``{filename}`` is the name of the file 
-   containing the retrieved monitor map. :: 
+   containing the retrieved monitor map:
+
+   .. prompt:: bash $
 
       ceph mon getmap -o {tmp}/{filename}
 
-#. The following example demonstrates the contents of the monmap. ::
+#. The following example demonstrates the contents of the monmap:
 
-	$ monmaptool --print {tmp}/{filename}
+   .. prompt:: bash $
+
+      monmaptool --print {tmp}/{filename}
+
+   ::	
 
 	monmaptool: monmap file {tmp}/{filename}
 	epoch 1
@@ -333,9 +378,14 @@ networks  are unable to communicate.  Use the following procedure:
 	1: 10.0.0.2:6789/0 mon.b
 	2: 10.0.0.3:6789/0 mon.c
 
-#. Remove the existing monitors. ::
+#. Remove the existing monitors:
 
-	$ monmaptool --rm a --rm b --rm c {tmp}/{filename}
+   .. prompt:: bash $
+
+      monmaptool --rm a --rm b --rm c {tmp}/{filename}
+	
+
+   ::
 
 	monmaptool: monmap file {tmp}/{filename}
 	monmaptool: removing a
@@ -343,16 +393,25 @@ networks  are unable to communicate.  Use the following procedure:
 	monmaptool: removing c
 	monmaptool: writing epoch 1 to {tmp}/{filename} (0 monitors)
 
-#. Add the new monitor locations. ::
+#. Add the new monitor locations:
 
-	$ monmaptool --add a 10.1.0.1:6789 --add b 10.1.0.2:6789 --add c 10.1.0.3:6789 {tmp}/{filename}
+   .. prompt:: bash $
+
+      monmaptool --add a 10.1.0.1:6789 --add b 10.1.0.2:6789 --add c 10.1.0.3:6789 {tmp}/{filename}
+
+
+   ::
 	
       monmaptool: monmap file {tmp}/{filename}
       monmaptool: writing epoch 1 to {tmp}/{filename} (3 monitors)
 
-#. Check new contents. ::
+#. Check new contents:
 
-	$ monmaptool --print {tmp}/{filename}
+   .. prompt:: bash $
+
+       monmaptool --print {tmp}/{filename}
+	
+   ::
 
 	monmaptool: monmap file {tmp}/{filename}
 	epoch 1
@@ -370,7 +429,9 @@ monitors, and inject the modified monmap into each new monitor.
 #. First, make sure to stop all your monitors.  Injection must be done while 
    the daemon is not running.
 
-#. Inject the monmap. ::
+#. Inject the monmap: 
+
+   .. prompt:: bash $
 
       ceph-mon -i {mon-id} --inject-monmap {tmp}/{filename}
 

ceph/doc/rados/operations/add-or-rm-osds.rst

@@ -71,7 +71,9 @@ weight).
 
 #. Create the OSD. If no UUID is given, it will be set automatically when the
    OSD starts up. The following command will output the OSD number, which you
-   will need for subsequent steps. ::
+   will need for subsequent steps:
+
+   .. prompt:: bash $
 
       ceph osd create [{uuid} [{id}]]
 
@@ -84,21 +86,25 @@ weight).
       clusters are large. If {id} is not specified, the smallest available is
       used.
 
-#. Create the default directory on your new OSD. ::
+#. Create the default directory on your new OSD:
+
+   .. prompt:: bash $
 
       ssh {new-osd-host}
       sudo mkdir /var/lib/ceph/osd/ceph-{osd-number}
 
-
 #. If the OSD is for a drive other than the OS drive, prepare it
-   for use with Ceph, and mount it to the directory you just created::
+   for use with Ceph, and mount it to the directory you just created:
+
+   .. prompt:: bash $
 
       ssh {new-osd-host}
       sudo mkfs -t {fstype} /dev/{drive}
       sudo mount -o user_xattr /dev/{hdd} /var/lib/ceph/osd/ceph-{osd-number}
 
+#. Initialize the OSD data directory:
 
-#. Initialize the OSD data directory. ::
+   .. prompt:: bash $
 
       ssh {new-osd-host}
       ceph-osd -i {osd-num} --mkfs --mkkey
@@ -107,11 +113,12 @@ weight).
 
 #. Register the OSD authentication key. The value of ``ceph`` for
    ``ceph-{osd-num}`` in the path is the ``$cluster-$id``.  If your
-   cluster name differs from ``ceph``, use your cluster name instead.::
+   cluster name differs from ``ceph``, use your cluster name instead:
+
+   .. prompt:: bash $
 
       ceph auth add osd.{osd-num} osd 'allow *' mon 'allow rwx' -i /var/lib/ceph/osd/ceph-{osd-num}/keyring
 
-
 #. Add the OSD to the CRUSH map so that the OSD can begin receiving data. The
    ``ceph osd crush add`` command allows you to add OSDs to the CRUSH hierarchy
    wherever you wish. If you specify at least one bucket, the command
@@ -120,7 +127,9 @@ weight).
    you specify only the root bucket, the command will attach the OSD directly
    to the root, but CRUSH rules expect OSDs to be inside of hosts.
 
-   Execute the following::
+   Execute the following:
+
+   .. prompt:: bash $
 
       ceph osd crush add {id-or-name} {weight}  [{bucket-type}={bucket-name} ...]
 
@@ -135,34 +144,49 @@ weight).
 Replacing an OSD
 ----------------
 
+.. note:: If the instructions in this section do not work for you, try the
+   instructions in the cephadm documentation: :ref:`cephadm-replacing-an-osd`.
+
 When disks fail, or if an administrator wants to reprovision OSDs with a new
 backend, for instance, for switching from FileStore to BlueStore, OSDs need to
 be replaced. Unlike `Removing the OSD`_, replaced OSD's id and CRUSH map entry
 need to be keep intact after the OSD is destroyed for replacement.
 
-#. Make sure it is safe to destroy the OSD::
+#. Make sure it is safe to destroy the OSD:
+
+   .. prompt:: bash $
 
       while ! ceph osd safe-to-destroy osd.{id} ; do sleep 10 ; done
 
-#. Destroy the OSD first::
+#. Destroy the OSD first:
+
+   .. prompt:: bash $
 
       ceph osd destroy {id} --yes-i-really-mean-it
 
 #. Zap a disk for the new OSD, if the disk was used before for other purposes.
-   It's not necessary for a new disk::
+   It's not necessary for a new disk:
+
+   .. prompt:: bash $
 
       ceph-volume lvm zap /dev/sdX
 
-#. Prepare the disk for replacement by using the previously destroyed OSD id::
+#. Prepare the disk for replacement by using the previously destroyed OSD id:
+
+   .. prompt:: bash $
 
       ceph-volume lvm prepare --osd-id {id} --data /dev/sdX
 
-#. And activate the OSD::
+#. And activate the OSD:
+
+   .. prompt:: bash $
 
       ceph-volume lvm activate {id} {fsid}
 
 Alternatively, instead of preparing and activating, the device can be recreated
-in one call, like::
+in one call, like:
+
+   .. prompt:: bash $
 
       ceph-volume lvm create --osd-id {id} --data /dev/sdX
 
@@ -174,13 +198,9 @@ After you add an OSD to Ceph, the OSD is in your configuration. However,
 it is not yet running. The OSD is ``down`` and ``in``. You must start
 your new OSD before it can begin receiving data. You may use
 ``service ceph`` from your admin host or start the OSD from its host
-machine.
+machine:
 
-For Ubuntu Trusty use Upstart. ::
+   .. prompt:: bash $
-
-	sudo start ceph-osd id={osd-num}
-
-For all other distros use systemd. ::
 
       sudo systemctl start ceph-osd@{osd-num}
 
@@ -193,7 +213,9 @@ Observe the Data Migration
 
 Once you have added your new OSD to the CRUSH map, Ceph  will begin rebalancing
 the server by migrating placement groups to your new OSD. You can observe this
-process with  the `ceph`_ tool. ::
+process with  the `ceph`_ tool. :
+
+   .. prompt:: bash $
 
       ceph -w
 
@@ -201,7 +223,6 @@ You should see the placement group states change from ``active+clean`` to
 ``active, some degraded objects``, and finally ``active+clean`` when migration
 completes. (Control-c to exit.)
 
-
 .. _Add/Move an OSD: ../crush-map#addosd
 .. _ceph: ../monitoring
 
@@ -228,7 +249,9 @@ Take the OSD out of the Cluster
 
 Before you remove an OSD, it is usually ``up`` and ``in``.  You need to take it
 out of the cluster so that Ceph can begin rebalancing and copying its data to
-other OSDs. ::
+other OSDs. :
+
+   .. prompt:: bash $
 
       ceph osd out {osd-num}
 
@@ -238,7 +261,9 @@ Observe the Data Migration
 
 Once you have taken your OSD ``out`` of the cluster, Ceph  will begin
 rebalancing the cluster by migrating placement groups out of the OSD you
-removed. You can observe  this process with  the `ceph`_ tool. ::
+removed. You can observe  this process with  the `ceph`_ tool. :
+
+   .. prompt:: bash $
 
       ceph -w
 
@@ -252,12 +277,16 @@ completes. (Control-c to exit.)
    ``active+remapped`` state. If you are in this case, you should mark
    the OSD ``in`` with:
 
-       ``ceph osd in {osd-num}``
+   .. prompt:: bash $
+
+      ceph osd in {osd-num}
 
    to come back to the initial state and then, instead of marking ``out``
    the OSD, set its weight to 0 with:
 
-       ``ceph osd crush reweight osd.{osd-num} 0``
+   .. prompt:: bash $
+
+      ceph osd crush reweight osd.{osd-num} 0
 
    After that, you can observe the data migration which should come to its
    end. The difference between marking ``out`` the OSD and reweighting it
@@ -273,7 +302,9 @@ Stopping the OSD
 
 After you take an OSD out of the cluster, it may still be running.
 That is, the OSD may be ``up`` and ``out``. You must stop
-your OSD before you remove it from the configuration. ::
+your OSD before you remove it from the configuration: 
+
+   .. prompt:: bash $
 
       ssh {osd-host}
       sudo systemctl stop ceph-osd@{osd-num}
@@ -292,50 +323,64 @@ OSD for each drive by repeating this procedure.
 #. Let the cluster forget the OSD first. This step removes the OSD from the CRUSH
    map, removes its authentication key. And it is removed from the OSD map as
    well. Please note the :ref:`purge subcommand <ceph-admin-osd>` is introduced in Luminous, for older
-   versions, please see below ::
+   versions, please see below:
+
+   .. prompt:: bash $
 
       ceph osd purge {id} --yes-i-really-mean-it
 
 #. Navigate to the host where you keep the master copy of the cluster's
-   ``ceph.conf`` file. ::
+   ``ceph.conf`` file:
+
+   .. prompt:: bash $
 
       ssh {admin-host}
       cd /etc/ceph
       vim ceph.conf
 
-#. Remove the OSD entry from your ``ceph.conf`` file (if it exists). ::
+#. Remove the OSD entry from your ``ceph.conf`` file (if it exists)::
 
 	[osd.1]
 		host = {hostname}
 
-#. From the host where you keep the master copy of the cluster's ``ceph.conf`` file,
+#. From the host where you keep the master copy of the cluster's ``ceph.conf``
-   copy the updated ``ceph.conf`` file to the ``/etc/ceph`` directory of other
+   file, copy the updated ``ceph.conf`` file to the ``/etc/ceph`` directory of
-   hosts in your cluster.
+   other hosts in your cluster.
 
-If your Ceph cluster is older than Luminous, instead of using ``ceph osd purge``,
+If your Ceph cluster is older than Luminous, instead of using ``ceph osd
-you need to perform this step manually:
+purge``, you need to perform this step manually:
 
 
 #. Remove the OSD from the CRUSH map so that it no longer receives data. You may
    also decompile the CRUSH map, remove the OSD from the device list, remove the
    device as an item in the host bucket or remove the host  bucket (if it's in the
    CRUSH map and you intend to remove the host), recompile the map and set it.
-   See `Remove an OSD`_ for details. ::
+   See `Remove an OSD`_ for details:
+
+   .. prompt:: bash $
 
       ceph osd crush remove {name}
 
-#. Remove the OSD authentication key. ::
+#. Remove the OSD authentication key:
+
+   .. prompt:: bash $
 
       ceph auth del osd.{osd-num}
 
-   The value of ``ceph`` for ``ceph-{osd-num}`` in the path is the ``$cluster-$id``.
+   The value of ``ceph`` for ``ceph-{osd-num}`` in the path is the
-   If your cluster name differs from ``ceph``, use your cluster name instead.
+   ``$cluster-$id``.  If your cluster name differs from ``ceph``, use your
+   cluster name instead.
 
-#. Remove the OSD. ::
+#. Remove the OSD:
+
+   .. prompt:: bash $
 
       ceph osd rm {osd-num}
-	#for example
+
+   for example:
+
+   .. prompt:: bash $
+
       ceph osd rm 1
 
-
 .. _Remove an OSD: ../crush-map#removeosd

ceph/doc/rados/operations/balancer.rst

@@ -1,4 +1,3 @@
-
 .. _balancer:
 
 Balancer
@@ -11,7 +10,9 @@ supervised fashion.
 Status
 ------
 
-The current status of the balancer can be checked at any time with::
+The current status of the balancer can be checked at any time with:
+
+   .. prompt:: bash $
 
       ceph balancer status
 
@@ -21,7 +22,9 @@ Automatic balancing
 
 The automatic balancing feature is enabled by default in ``upmap``
 mode. Please refer to :ref:`upmap` for more details. The balancer can be
-turned off with::
+turned off with:
+
+   .. prompt:: bash $
 
       ceph balancer off
 
@@ -40,35 +43,49 @@ healed itself).
 When the cluster is healthy, the balancer will throttle its changes
 such that the percentage of PGs that are misplaced (i.e., that need to
 be moved) is below a threshold of (by default) 5%.  The
-``target_max_misplaced_ratio`` threshold can be adjusted with::
+``target_max_misplaced_ratio`` threshold can be adjusted with:
+
+   .. prompt:: bash $
 
       ceph config set mgr target_max_misplaced_ratio .07   # 7%
 
-Set the number of seconds to sleep in between runs of the automatic balancer::
+Set the number of seconds to sleep in between runs of the automatic balancer:
+
+   .. prompt:: bash $
 
       ceph config set mgr mgr/balancer/sleep_interval 60
 
-Set the time of day to begin automatic balancing in HHMM format::
+Set the time of day to begin automatic balancing in HHMM format:
+
+   .. prompt:: bash $
 
       ceph config set mgr mgr/balancer/begin_time 0000
 
-Set the time of day to finish automatic balancing in HHMM format::
+Set the time of day to finish automatic balancing in HHMM format:
 
-  ceph config set mgr mgr/balancer/end_time 2400
+   .. prompt:: bash $
+
+      ceph config set mgr mgr/balancer/end_time 2359
 
 Restrict automatic balancing to this day of the week or later. 
-Uses the same conventions as crontab, 0 or 7 is Sunday, 1 is Monday, and so on::
+Uses the same conventions as crontab, 0 is Sunday, 1 is Monday, and so on:
+
+   .. prompt:: bash $
 
       ceph config set mgr mgr/balancer/begin_weekday 0
 
 Restrict automatic balancing to this day of the week or earlier. 
-Uses the same conventions as crontab, 0 or 7 is Sunday, 1 is Monday, and so on::
+Uses the same conventions as crontab, 0 is Sunday, 1 is Monday, and so on:
 
-  ceph config set mgr mgr/balancer/end_weekday 7
+   .. prompt:: bash $
+
+      ceph config set mgr mgr/balancer/end_weekday 6
 
 Pool IDs to which the automatic balancing will be limited. 
 The default for this is an empty string, meaning all pools will be balanced. 
-The numeric pool IDs can be gotten with the :command:`ceph osd pool ls detail` command::
+The numeric pool IDs can be gotten with the :command:`ceph osd pool ls detail` command:
+
+   .. prompt:: bash $
 
       ceph config set mgr mgr/balancer/pool_ids 1,2,3
 
@@ -112,7 +129,9 @@ There are currently two supported balancer modes:
 
    Note that using upmap requires that all clients be Luminous or newer.
 
-The default mode is ``upmap``.  The mode can be adjusted with::
+The default mode is ``upmap``.  The mode can be adjusted with:
+
+   .. prompt:: bash $
 
       ceph balancer mode crush-compat
 
@@ -125,43 +144,63 @@ The balancer operation is broken into a few distinct phases:
 #. evaluating the quality of the data distribution, either for the current PG distribution, or the PG distribution that would result after executing a *plan*
 #. executing the *plan*
 
-To evaluate and score the current distribution::
+To evaluate and score the current distribution:
+
+   .. prompt:: bash $
 
       ceph balancer eval
 
-You can also evaluate the distribution for a single pool with::
+You can also evaluate the distribution for a single pool with:
+
+   .. prompt:: bash $
 
       ceph balancer eval <pool-name>
 
-Greater detail for the evaluation can be seen with::
+Greater detail for the evaluation can be seen with:
+
+   .. prompt:: bash $
 
       ceph balancer eval-verbose ...
   
-The balancer can generate a plan, using the currently configured mode, with::
+The balancer can generate a plan, using the currently configured mode, with:
+
+   .. prompt:: bash $
 
       ceph balancer optimize <plan-name>
 
-The name is provided by the user and can be any useful identifying string.  The contents of a plan can be seen with::
+The name is provided by the user and can be any useful identifying string.  The contents of a plan can be seen with:
+
+   .. prompt:: bash $
 
       ceph balancer show <plan-name>
 
-All plans can be shown with::
+All plans can be shown with:
+
+   .. prompt:: bash $
 
       ceph balancer ls
 
-Old plans can be discarded with::
+Old plans can be discarded with:
+
+   .. prompt:: bash $
 
       ceph balancer rm <plan-name>
 
-Currently recorded plans are shown as part of the status command::
+Currently recorded plans are shown as part of the status command:
+
+   .. prompt:: bash $
 
       ceph balancer status
 
-The quality of the distribution that would result after executing a plan can be calculated with::
+The quality of the distribution that would result after executing a plan can be calculated with:
+
+   .. prompt:: bash $
 
       ceph balancer eval <plan-name>
 
-Assuming the plan is expected to improve the distribution (i.e., it has a lower score than the current cluster state), the user can execute that plan with::
+Assuming the plan is expected to improve the distribution (i.e., it has a lower score than the current cluster state), the user can execute that plan with:
+
+   .. prompt:: bash $
 
       ceph balancer execute <plan-name>
 

ceph/doc/rados/operations/bluestore-migration.rst

@@ -41,48 +41,68 @@ more data migration than should be necessary, so it is not optimal.
      ID=<osd-id-number>
      DEVICE=<disk-device>
 
-   You can tell whether a given OSD is FileStore or BlueStore with::
+   You can tell whether a given OSD is FileStore or BlueStore with:
+
+   .. prompt:: bash $
 
       ceph osd metadata $ID | grep osd_objectstore
 
-   You can get a current count of filestore vs bluestore with::
+   You can get a current count of filestore vs bluestore with:
+
+   .. prompt:: bash $
 
       ceph osd count-metadata osd_objectstore
 
-#. Mark the filestore OSD out::
+#. Mark the filestore OSD out:
+
+   .. prompt:: bash $
 
       ceph osd out $ID
 
-#. Wait for the data to migrate off the OSD in question::
+#. Wait for the data to migrate off the OSD in question:
+
+   .. prompt:: bash $
 
       while ! ceph osd safe-to-destroy $ID ; do sleep 60 ; done
 
-#. Stop the OSD::
+#. Stop the OSD:
+
+   .. prompt:: bash $
 
       systemctl kill ceph-osd@$ID
 
-#. Make note of which device this OSD is using::
+#. Make note of which device this OSD is using:
+
+   .. prompt:: bash $
 
       mount | grep /var/lib/ceph/osd/ceph-$ID
 
-#. Unmount the OSD::
+#. Unmount the OSD:
+
+   .. prompt:: bash $
 
       umount /var/lib/ceph/osd/ceph-$ID
 
 #. Destroy the OSD data. Be *EXTREMELY CAREFUL* as this will destroy
    the contents of the device; be certain the data on the device is
-   not needed (i.e., that the cluster is healthy) before proceeding. ::
+   not needed (i.e., that the cluster is healthy) before proceeding:
+
+   .. prompt:: bash $
 
       ceph-volume lvm zap $DEVICE
 
 #. Tell the cluster the OSD has been destroyed (and a new OSD can be
-   reprovisioned with the same ID)::
+   reprovisioned with the same ID):
+
+   .. prompt:: bash $
 
       ceph osd destroy $ID --yes-i-really-mean-it
 
 #. Reprovision a BlueStore OSD in its place with the same OSD ID.
    This requires you do identify which device to wipe based on what you saw
-   mounted above. BE CAREFUL! ::
+   mounted above. BE CAREFUL! :
+
+   .. prompt:: bash $
 
       ceph-volume lvm create --bluestore --data $DEVICE --osd-id $ID
 
@@ -127,7 +147,9 @@ doesn't strictly matter). ::
 
   NEWHOST=<empty-host-name>
 
-Add the host to the CRUSH hierarchy, but do not attach it to the root::
+Add the host to the CRUSH hierarchy, but do not attach it to the root:
+
+.. prompt:: bash $
 
    ceph osd crush add-bucket $NEWHOST host
 
@@ -142,14 +164,22 @@ space on that host so that all of its data can be migrated off,
 then you can instead do::
 
   OLDHOST=<existing-cluster-host-to-offload>
+
+.. prompt:: bash $ 
+   
    ceph osd crush unlink $OLDHOST default
 
 where "default" is the immediate ancestor in the CRUSH map. (For
 smaller clusters with unmodified configurations this will normally
 be "default", but it might also be a rack name.)  You should now
-see the host at the top of the OSD tree output with no parent::
+see the host at the top of the OSD tree output with no parent:
+
+.. prompt:: bash $
+
+   bin/ceph osd tree
+
+::
 
-  $ bin/ceph osd tree
   ID CLASS WEIGHT  TYPE NAME     STATUS REWEIGHT PRI-AFF
   -5             0 host oldhost
   10   ssd 1.00000     osd.10        up  1.00000 1.00000
@@ -172,11 +202,15 @@ Migration process
 If you're using a new host, start at step #1.  For an existing host,
 jump to step #5 below.
 
-#. Provision new BlueStore OSDs for all devices::
+#. Provision new BlueStore OSDs for all devices:
+
+   .. prompt:: bash $
 
       ceph-volume lvm create --bluestore --data /dev/$DEVICE
 
-#. Verify OSDs join the cluster with::
+#. Verify OSDs join the cluster with:
+
+   .. prompt:: bash $
 
       ceph osd tree
 
@@ -198,11 +232,15 @@ jump to step #5 below.
       2   ssd 1.00000         osd.2     up  1.00000 1.00000
      ...
 
-#. Identify the first target host to convert ::
+#. Identify the first target host to convert :
+
+   .. prompt:: bash $
 
       OLDHOST=<existing-cluster-host-to-convert>
 
-#. Swap the new host into the old host's position in the cluster::
+#. Swap the new host into the old host's position in the cluster:
+
+   .. prompt:: bash $
 
       ceph osd crush swap-bucket $NEWHOST $OLDHOST
 
@@ -212,24 +250,32 @@ jump to step #5 below.
    other nodes in the cluster, but as long as the hosts are similarly
    sized this will be a relatively small amount of data.
 
-#. Wait for data migration to complete::
+#. Wait for data migration to complete:
+
+   .. prompt:: bash $
 
       while ! ceph osd safe-to-destroy $(ceph osd ls-tree $OLDHOST); do sleep 60 ; done
 
-#. Stop all old OSDs on the now-empty ``$OLDHOST``::
+#. Stop all old OSDs on the now-empty ``$OLDHOST``:
+
+   .. prompt:: bash $
 
       ssh $OLDHOST
      systemctl kill ceph-osd.target
      umount /var/lib/ceph/osd/ceph-*
 
-#. Destroy and purge the old OSDs::
+#. Destroy and purge the old OSDs:
+
+   .. prompt:: bash $
 
       for osd in `ceph osd ls-tree $OLDHOST`; do
          ceph osd purge $osd --yes-i-really-mean-it
      done
 
 #. Wipe the old OSD devices. This requires you do identify which
-   devices are to be wiped manually (BE CAREFUL!). For each device,::
+   devices are to be wiped manually (BE CAREFUL!). For each device:
+
+   .. prompt:: bash $
 
       ceph-volume lvm zap $DEVICE
 

ceph/doc/rados/operations/cache-tiering.rst

@@ -45,16 +45,22 @@ and the backing storage tier automatically. However, admins have the ability to
 configure how this migration takes place by setting the ``cache-mode``. There are
 two main scenarios:
 
-- **writeback** mode: When admins configure tiers with ``writeback`` mode, Ceph
+- **writeback** mode: If the base tier and the cache tier are configured in
-  clients write data to the cache tier and receive an ACK from the cache tier.
+  ``writeback`` mode, Ceph clients receive an ACK from the base tier every time
-  In time, the data written to the cache tier migrates to the storage tier
+  they write data to it. Then the cache tiering agent determines whether
-  and gets flushed from the cache tier. Conceptually, the cache tier is 
+  ``osd_tier_default_cache_min_write_recency_for_promote`` has been set. If it
-  overlaid "in front" of the backing storage tier. When a Ceph client needs 
+  has been set and the data has been written more than a specified number of
-  data that resides in the storage tier, the cache tiering agent migrates the
+  times per interval, the data is promoted to the cache tier.
-  data to the cache tier on read, then it is sent to the Ceph client. 
+
-  Thereafter, the Ceph client can perform I/O using the cache tier, until the 
+  When Ceph clients need access to data stored in the base tier, the cache
-  data becomes inactive. This is ideal for mutable data (e.g., photo/video 
+  tiering agent reads the data from the base tier and returns it to the client.
-  editing, transactional data, etc.).
+  While data is being read from the base tier, the cache tiering agent consults
+  the value of ``osd_tier_default_cache_min_read_recency_for_promote`` and
+  decides whether to promote that data from the base tier to the cache tier.
+  When data has been promoted from the base tier to the cache tier, the Ceph
+  client is able to perform I/O operations on it using the cache tier. This is
+  well-suited for mutable data (for example, photo/video editing, transactional
+  data).
 
 - **readproxy** mode: This mode will use any objects that already
   exist in the cache tier, but if an object is not present in the
@@ -199,30 +205,42 @@ Creating a Cache Tier
 =====================
 
 Setting up a cache tier involves associating a backing storage pool with
-a cache pool ::
+a cache pool:
+
+.. prompt:: bash $
 
    ceph osd tier add {storagepool} {cachepool}
 
-For example ::
+For example:
+
+.. prompt:: bash $
 
    ceph osd tier add cold-storage hot-storage
 
-To set the cache mode, execute the following::
+To set the cache mode, execute the following:
+
+.. prompt:: bash $
 
    ceph osd tier cache-mode {cachepool} {cache-mode}
 
-For example:: 
+For example:
+
+.. prompt:: bash $
 
    ceph osd tier cache-mode hot-storage writeback
 
 The cache tiers overlay the backing storage tier, so they require one
 additional step: you must direct all client traffic from the storage pool to 
 the cache pool. To direct client traffic directly to the cache pool, execute 
-the following:: 
+the following:
+
+.. prompt:: bash $
 
    ceph osd tier set-overlay {storagepool} {cachepool}
 
-For example:: 
+For example:
+
+.. prompt:: bash $
 
    ceph osd tier set-overlay cold-storage hot-storage
 
@@ -231,7 +249,9 @@ Configuring a Cache Tier
 ========================
 
 Cache tiers have several configuration options. You may set
-cache tier configuration options with the following usage:: 
+cache tier configuration options with the following usage:
+
+.. prompt:: bash $
 
    ceph osd pool set {cachepool} {key} {value}
    
@@ -241,16 +261,22 @@ See `Pools - Set Pool Values`_ for details.
 Target Size and Type
 --------------------
 
-Ceph's production cache tiers use a `Bloom Filter`_ for the ``hit_set_type``::
+Ceph's production cache tiers use a `Bloom Filter`_ for the ``hit_set_type``:
+
+.. prompt:: bash $
 
    ceph osd pool set {cachepool} hit_set_type bloom
 
-For example::
+For example:
+
+.. prompt:: bash $
 
    ceph osd pool set hot-storage hit_set_type bloom
 
 The ``hit_set_count`` and ``hit_set_period`` define how many such HitSets to
-store, and how much time each HitSet should cover. ::
+store, and how much time each HitSet should cover:
+
+.. prompt:: bash $
 
    ceph osd pool set {cachepool} hit_set_count 12
    ceph osd pool set {cachepool} hit_set_period 14400
@@ -273,7 +299,9 @@ number of archive HitSets are checked. The object is promoted if the object is
 found in any of the most recent ``min_read_recency_for_promote`` HitSets.
 
 A similar parameter can be set for the write operation, which is
-``min_write_recency_for_promote``. ::
+``min_write_recency_for_promote``:
+
+.. prompt:: bash $
 
    ceph osd pool set {cachepool} min_read_recency_for_promote 2
    ceph osd pool set {cachepool} min_write_recency_for_promote 2
@@ -303,20 +331,27 @@ Absolute Sizing
 
 The cache tiering agent can flush or evict objects based upon the total number
 of bytes or the total number of objects. To specify a maximum number of bytes,
-execute the following::
+execute the following:
+
+.. prompt:: bash $
 
    ceph osd pool set {cachepool} target_max_bytes {#bytes}
 
-For example, to flush or evict at 1 TB, execute the following::
+For example, to flush or evict at 1 TB, execute the following:
+
+.. prompt:: bash $
 
    ceph osd pool set hot-storage target_max_bytes 1099511627776
 
+To specify the maximum number of objects, execute the following:
 
-To specify the maximum number of objects, execute the following::
+.. prompt:: bash $
 
    ceph osd pool set {cachepool} target_max_objects {#objects}
 
-For example, to flush or evict at 1M objects, execute the following::
+For example, to flush or evict at 1M objects, execute the following:
+
+.. prompt:: bash $
 
    ceph osd pool set hot-storage target_max_objects 1000000
 
@@ -335,34 +370,46 @@ The cache tiering agent can flush or evict objects relative to the size of the
 cache pool(specified by ``target_max_bytes`` / ``target_max_objects`` in
 `Absolute sizing`_).  When the cache pool consists of a certain percentage of
 modified (or dirty) objects, the cache tiering agent will flush them to the
-storage pool. To set the ``cache_target_dirty_ratio``, execute the following::
+storage pool. To set the ``cache_target_dirty_ratio``, execute the following:
+
+.. prompt:: bash $
 
    ceph osd pool set {cachepool} cache_target_dirty_ratio {0.0..1.0}
 
 For example, setting the value to ``0.4`` will begin flushing modified
-(dirty) objects when they reach 40% of the cache pool's capacity:: 
+(dirty) objects when they reach 40% of the cache pool's capacity:
+
+.. prompt:: bash $
 
    ceph osd pool set hot-storage cache_target_dirty_ratio 0.4
 
 When the dirty objects reaches a certain percentage of its capacity, flush dirty
-objects with a higher speed. To set the ``cache_target_dirty_high_ratio``::
+objects with a higher speed. To set the ``cache_target_dirty_high_ratio``:
+
+.. prompt:: bash $
 
    ceph osd pool set {cachepool} cache_target_dirty_high_ratio {0.0..1.0}
 
-For example, setting the value to ``0.6`` will begin aggressively flush dirty objects
+For example, setting the value to ``0.6`` will begin aggressively flush dirty
-when they reach 60% of the cache pool's capacity. obviously, we'd better set the value
+objects when they reach 60% of the cache pool's capacity. obviously, we'd
-between dirty_ratio and full_ratio::
+better set the value between dirty_ratio and full_ratio:
+
+.. prompt:: bash $
 
    ceph osd pool set hot-storage cache_target_dirty_high_ratio 0.6
 
 When the cache pool reaches a certain percentage of its capacity, the cache
 tiering agent will evict objects to maintain free capacity. To set the 
-``cache_target_full_ratio``, execute the following:: 
+``cache_target_full_ratio``, execute the following:
+
+.. prompt:: bash $
 
    ceph osd pool set {cachepool} cache_target_full_ratio {0.0..1.0}
 
 For example, setting the value to ``0.8`` will begin flushing unmodified
-(clean) objects when they reach 80% of the cache pool's capacity:: 
+(clean) objects when they reach 80% of the cache pool's capacity:
+
+.. prompt:: bash $
 
    ceph osd pool set hot-storage cache_target_full_ratio 0.8
 
@@ -371,21 +418,29 @@ Cache Age
 ---------
 
 You can specify the minimum age of an object before the cache tiering agent 
-flushes a recently modified (or dirty) object to the backing storage pool::
+flushes a recently modified (or dirty) object to the backing storage pool:
+
+.. prompt:: bash $
 
    ceph osd pool set {cachepool} cache_min_flush_age {#seconds}
 
-For example, to flush modified (or dirty) objects after 10 minutes, execute 
+For example, to flush modified (or dirty) objects after 10 minutes, execute the
-the following:: 
+following:
+
+.. prompt:: bash $
 
    ceph osd pool set hot-storage cache_min_flush_age 600
 
-You can specify the minimum age of an object before it will be evicted from
+You can specify the minimum age of an object before it will be evicted from the
-the cache tier::
+cache tier:
+
+.. prompt:: bash $
 
    ceph osd pool {cache-tier} cache_min_evict_age {#seconds}
 
-For example, to evict objects after 30 minutes, execute the following:: 
+For example, to evict objects after 30 minutes, execute the following:
+
+.. prompt:: bash $
 
    ceph osd pool set hot-storage cache_min_evict_age 1800
 
@@ -403,24 +458,31 @@ Removing a Read-Only Cache
 Since a read-only cache does not have modified data, you can disable
 and remove it without losing any recent changes to objects in the cache. 
 
-#. Change the cache-mode to ``none`` to disable it. :: 
+#. Change the cache-mode to ``none`` to disable it.:
+
+   .. prompt:: bash 
 
       ceph osd tier cache-mode {cachepool} none
 
-   For example:: 
+   For example:
+
+   .. prompt:: bash $
 
       ceph osd tier cache-mode hot-storage none
 
-#. Remove the cache pool from the backing pool. ::
+#. Remove the cache pool from the backing pool.:
+
+   .. prompt:: bash $
 
       ceph osd tier remove {storagepool} {cachepool}
 
-   For example::
+   For example:
+
+   .. prompt:: bash $
 
       ceph osd tier remove cold-storage hot-storage
 
 
-
 Removing a Writeback Cache
 --------------------------
 
@@ -430,39 +492,55 @@ disable and remove it.
 
 
 #. Change the cache mode to ``proxy`` so that new and modified objects will 
-   flush to the backing storage pool. ::
+   flush to the backing storage pool.:
+
+   .. prompt:: bash $
 
       ceph osd tier cache-mode {cachepool} proxy
 
-   For example:: 
+   For example: 
+
+   .. prompt:: bash $
 
       ceph osd tier cache-mode hot-storage proxy
 
 
-#. Ensure that the cache pool has been flushed. This may take a few minutes::
+#. Ensure that the cache pool has been flushed. This may take a few minutes:
+
+   .. prompt:: bash $
 
       rados -p {cachepool} ls
 
    If the cache pool still has objects, you can flush them manually. 
-   For example::
+   For example:
+
+   .. prompt:: bash $
 
       rados -p {cachepool} cache-flush-evict-all
 
 
-#. Remove the overlay so that clients will not direct traffic to the cache. ::
+#. Remove the overlay so that clients will not direct traffic to the cache.:
+
+   .. prompt:: bash $
 
       ceph osd tier remove-overlay {storagetier}
 
-   For example::
+   For example:
+
+   .. prompt:: bash $
 
       ceph osd tier remove-overlay cold-storage
 
 
-#. Finally, remove the cache tier pool from the backing storage pool. ::
+#. Finally, remove the cache tier pool from the backing storage pool.:
+
+   .. prompt:: bash $
 
       ceph osd tier remove {storagepool} {cachepool} 
 
-   For example::
+   For example:
+
+   .. prompt:: bash $
 
       ceph osd tier remove cold-storage hot-storage
 

ceph/doc/rados/operations/change-mon-elections.rst

@@ -30,18 +30,24 @@ This mode lets you mark monitors as disallowd, in which case they will
 participate in the quorum and serve clients, but cannot be elected leader. You
 may wish to use this if you have some monitors which are known to be far away
 from clients.
-You can disallow a leader by running ::
+You can disallow a leader by running:
 
-  $ ceph mon add disallowed_leader {name}
+.. prompt:: bash $
+
+   ceph mon add disallowed_leader {name}
 
 You can remove a monitor from the disallowed list, and allow it to become
-a leader again, by running ::
+a leader again, by running:
 
-  $ ceph mon rm disallowed_leader {name}
+.. prompt:: bash $
 
-The list of disallowed_leaders is included when you run ::
+   ceph mon rm disallowed_leader {name}
 
-  $ ceph mon dump
+The list of disallowed_leaders is included when you run:
+
+.. prompt:: bash $
+
+   ceph mon dump
 
 The connectivity Mode
 =====================
@@ -58,7 +64,9 @@ Examining connectivity scores
 =============================
 The monitors maintain connection scores even if they aren't in
 the connectivity election mode. You can examine the scores a monitor
-has by running ::
+has by running:
+
+.. prompt:: bash $
 
    ceph daemon mon.{name} connection scores dump
 
@@ -68,7 +76,9 @@ whether it returned its latest ping within the timeout).
 
 While this would be an unexpected occurrence, if for some reason you experience
 problems and troubleshooting makes you think your scores have become invalid,
-you can forget history and reset them by running ::
+you can forget history and reset them by running:
+
+.. prompt:: bash $
 
    ceph daemon mon.{name} connection scores reset
 

ceph/doc/rados/operations/control.rst

@@ -8,11 +8,15 @@
 Monitor Commands
 ================
 
-Monitor commands are issued using the ``ceph`` utility::
+Monitor commands are issued using the ``ceph`` utility:
+
+.. prompt:: bash $
 
    ceph [-m monhost] {command}
 
-The command is usually (though not always) of the form::
+The command is usually (though not always) of the form:
+
+.. prompt:: bash $
 
    ceph {subsystem} {command}
 
@@ -20,24 +24,32 @@ The command is usually (though not always) of the form::
 System Commands
 ===============
 
-Execute the following to display the current cluster status.  ::
+Execute the following to display the current cluster status.  :
+
+.. prompt:: bash $
 
    ceph -s
    ceph status
 
 Execute the following to display a running summary of cluster status
-and major events. ::
+and major events. :
+
+.. prompt:: bash $
 
    ceph -w
 
 Execute the following to show the monitor quorum, including which monitors are
-participating and which one is the leader. ::
+participating and which one is the leader. :
+
+.. prompt:: bash $
 
    ceph mon stat
    ceph quorum_status
 
 Execute the following to query the status of a single monitor, including whether
-or not it is in the quorum. ::
+or not it is in the quorum. :
+
+.. prompt:: bash $
 
    ceph tell mon.[id] mon_status
 
@@ -47,11 +59,15 @@ where the value of ``[id]`` can be determined, e.g., from ``ceph -s``.
 Authentication Subsystem
 ========================
 
-To add a keyring for an OSD, execute the following::
+To add a keyring for an OSD, execute the following:
+
+.. prompt:: bash $
 
    ceph auth add {osd} {--in-file|-i} {path-to-osd-keyring}
 
-To list the cluster's keys and their capabilities, execute the following::
+To list the cluster's keys and their capabilities, execute the following:
+
+.. prompt:: bash $
 
    ceph auth ls
 
@@ -59,7 +75,9 @@ To list the cluster's keys and their capabilities, execute the following::
 Placement Group Subsystem
 =========================
 
-To display the statistics for all placement groups (PGs), execute the following:: 
+To display the statistics for all placement groups (PGs), execute the following: 
+
+.. prompt:: bash $
 
    ceph pg dump [--format {format}]
 
@@ -70,7 +88,9 @@ less variable from release to release.  The ``jq`` utility can be invaluable whe
 data from JSON output.
 
 To display the statistics for all placement groups stuck in a specified state, 
-execute the following:: 
+execute the following: 
+
+.. prompt:: bash $
 
    ceph pg dump_stuck inactive|unclean|stale|undersized|degraded [--format {format}] [-t|--threshold {seconds}]
 
@@ -90,7 +110,9 @@ reported to the monitor cluster in a while (configured by
 ``mon_osd_report_timeout``).
 
 Delete "lost" objects or revert them to their prior state, either a previous version
-or delete them if they were just created. ::
+or delete them if they were just created. :
+
+.. prompt:: bash $
 
    ceph pg {pgid} mark_unfound_lost revert|delete
 
@@ -100,102 +122,146 @@ or delete them if they were just created. ::
 OSD Subsystem
 =============
 
-Query OSD subsystem status. ::
+Query OSD subsystem status. :
+
+.. prompt:: bash $
 
    ceph osd stat
 
 Write a copy of the most recent OSD map to a file. See
-:ref:`osdmaptool <osdmaptool>`. ::
+:ref:`osdmaptool <osdmaptool>`. :
+
+.. prompt:: bash $
 
    ceph osd getmap -o file
 
 Write a copy of the crush map from the most recent OSD map to
-file. ::
+file. :
+
+.. prompt:: bash $
 
    ceph osd getcrushmap -o file
 
-The foregoing is functionally equivalent to ::
+The foregoing is functionally equivalent to :
+
+.. prompt:: bash $
 
    ceph osd getmap -o /tmp/osdmap
    osdmaptool /tmp/osdmap --export-crush file
 
 Dump the OSD map. Valid formats for ``-f`` are ``plain``, ``json``, ``json-pretty``,
 ``xml``, and ``xml-pretty``. If no ``--format`` option is given, the OSD map is 
-dumped as plain text.  As above, JSON format is best for tools, scripting, and other automation. ::
+dumped as plain text.  As above, JSON format is best for tools, scripting, and other automation. :
+
+.. prompt:: bash $
 
    ceph osd dump [--format {format}]
 
 Dump the OSD map as a tree with one line per OSD containing weight
-and state. ::
+and state. :
+
+.. prompt:: bash $
 
    ceph osd tree [--format {format}]
 
-Find out where a specific object is or would be stored in the system::
+Find out where a specific object is or would be stored in the system:
+
+.. prompt:: bash $
 
    ceph osd map <pool-name> <object-name>
 
 Add or move a new item (OSD) with the given id/name/weight at the specified
-location. ::
+location. :
+
+.. prompt:: bash $
 
    ceph osd crush set {id} {weight} [{loc1} [{loc2} ...]]
 
-Remove an existing item (OSD) from the CRUSH map. ::
+Remove an existing item (OSD) from the CRUSH map. :
+
+.. prompt:: bash $
 
    ceph osd crush remove {name}
 
-Remove an existing bucket from the CRUSH map. ::
+Remove an existing bucket from the CRUSH map. :
+
+.. prompt:: bash $
 
    ceph osd crush remove {bucket-name}
 
-Move an existing bucket from one position in the hierarchy to another.  ::
+Move an existing bucket from one position in the hierarchy to another.  :
+
+.. prompt:: bash $
 
    ceph osd crush move {id} {loc1} [{loc2} ...]
 
-Set the weight of the item given by ``{name}`` to ``{weight}``. ::
+Set the weight of the item given by ``{name}`` to ``{weight}``. :
+
+.. prompt:: bash $
 
    ceph osd crush reweight {name} {weight}
 
-Mark an OSD as ``lost``. This may result in permanent data loss. Use with caution. ::
+Mark an OSD as ``lost``. This may result in permanent data loss. Use with caution. :
+
+.. prompt:: bash $
 
    ceph osd lost {id} [--yes-i-really-mean-it]
 
 Create a new OSD. If no UUID is given, it will be set automatically when the OSD
-starts up. ::
+starts up. :
+
+.. prompt:: bash $
 
    ceph osd create [{uuid}]
 
-Remove the given OSD(s). ::
+Remove the given OSD(s). :
+
+.. prompt:: bash $
 
    ceph osd rm [{id}...]
 
-Query the current ``max_osd`` parameter in the OSD map. ::
+Query the current ``max_osd`` parameter in the OSD map. :
+
+.. prompt:: bash $
 
    ceph osd getmaxosd
 
-Import the given crush map. ::
+Import the given crush map. :
+
+.. prompt:: bash $
 
    ceph osd setcrushmap -i file
 
 Set the ``max_osd`` parameter in the OSD map. This defaults to 10000 now so
-most admins will never need to adjust this. ::
+most admins will never need to adjust this. :
+
+.. prompt:: bash $
 
    ceph osd setmaxosd
 
-Mark OSD ``{osd-num}`` down. ::
+Mark OSD ``{osd-num}`` down. :
+
+.. prompt:: bash $
 
    ceph osd down {osd-num}
 
-Mark OSD ``{osd-num}`` out of the distribution (i.e. allocated no data). ::
+Mark OSD ``{osd-num}`` out of the distribution (i.e. allocated no data). :
+
+.. prompt:: bash $
 
    ceph osd out {osd-num}
 
-Mark ``{osd-num}`` in the distribution (i.e. allocated data). ::
+Mark ``{osd-num}`` in the distribution (i.e. allocated data). :
+
+.. prompt:: bash $
 
    ceph osd in {osd-num}
 
 Set or clear the pause flags in the OSD map. If set, no IO requests
 will be sent to any OSD. Clearing the flags via unpause results in
-resending pending requests. ::
+resending pending requests. :
+
+.. prompt:: bash $
 
    ceph osd pause
    ceph osd unpause
@@ -209,7 +275,9 @@ otherwise live on this drive. It does not change weights assigned
 to the buckets above the OSD in the crush map, and is a corrective
 measure in case the normal CRUSH distribution is not working out quite
 right. For instance, if one of your OSDs is at 90% and the others are
-at 50%, you could reduce this weight to compensate. ::
+at 50%, you could reduce this weight to compensate. :
+
+.. prompt:: bash $
 
    ceph osd reweight {osd-num} {weight}
 
@@ -219,7 +287,9 @@ default to 1.00000 and are relative only to each other; they not absolute.
 It is crucial to distinguish them from CRUSH weights, which reflect the
 absolute capacity of a bucket in TiB.  By default this command adjusts
 override weight on OSDs which have + or - 20% of the average utilization,
-but if you include a ``threshold`` that percentage will be used instead. ::
+but if you include a ``threshold`` that percentage will be used instead. :
+
+.. prompt:: bash $
 
    ceph osd reweight-by-utilization [threshold [max_change [max_osds]]] [--no-increasing]
 
@@ -230,7 +300,9 @@ parameters can speed leveling of OSD utilization, at the potential cost of
 greater impact on client operations due to more data moving at once.
 
 To determine which and how many PGs and OSDs will be affected by a given invocation
-you can test before executing. ::
+you can test before executing. :
+
+.. prompt:: bash $
 
    ceph osd test-reweight-by-utilization [threshold [max_change max_osds]] [--no-increasing]
 
@@ -243,31 +315,44 @@ Deployments utilizing Nautilus (or later revisions of Luminous and Mimic)
 that have no pre-Luminous cients may instead wish to instead enable the
 `balancer`` module for ``ceph-mgr``.
 
-Add/remove an IP address to/from the blocklist. When adding an address,
+Add/remove an IP address or CIDR range to/from the blocklist.
+When adding to the blocklist,
 you can specify how long it should be blocklisted in seconds; otherwise,
 it will default to 1 hour. A blocklisted address is prevented from
-connecting to any OSD. Blocklisting is most often used to prevent a
+connecting to any OSD. If you blocklist an IP or range containing an OSD, be aware
-lagging metadata server from making bad changes to data on the OSDs.
+that OSD will also be prevented from performing operations on its peers where it
+acts as a client. (This includes tiering and copy-from functionality.)
+
+If you want to blocklist a range (in CIDR format), you may do so by
+including the ``range`` keyword.
 
 These commands are mostly only useful for failure testing, as
 blocklists are normally maintained automatically and shouldn't need
-manual intervention. ::
+manual intervention. :
 
-	ceph osd blocklist add ADDRESS[:source_port] [TIME]
+.. prompt:: bash $
-	ceph osd blocklist rm ADDRESS[:source_port]
 
-Creates/deletes a snapshot of a pool. ::
+   ceph osd blocklist ["range"] add ADDRESS[:source_port][/netmask_bits] [TIME]
+   ceph osd blocklist ["range"] rm ADDRESS[:source_port][/netmask_bits]
+
+Creates/deletes a snapshot of a pool. :
+
+.. prompt:: bash $
 
    ceph osd pool mksnap {pool-name} {snap-name}
    ceph osd pool rmsnap {pool-name} {snap-name}
 
-Creates/deletes/renames a storage pool. ::
+Creates/deletes/renames a storage pool. :
+
+.. prompt:: bash $
 
    ceph osd pool create {pool-name} [pg_num [pgp_num]]
    ceph osd pool delete {pool-name} [{pool-name} --yes-i-really-really-mean-it]
    ceph osd pool rename {old-name} {new-name}
 
-Changes a pool setting. :: 
+Changes a pool setting. : 
+
+.. prompt:: bash $
 
    ceph osd pool set {pool-name} {field} {value}
 
@@ -278,7 +363,9 @@ Valid fields are:
 	* ``pgp_num``: Effective number when calculating pg placement.
 	* ``crush_rule``: rule number for mapping placement.
 
-Get the value of a pool setting. ::
+Get the value of a pool setting. :
+
+.. prompt:: bash $
 
    ceph osd pool get {pool-name} {field}
 
@@ -288,11 +375,15 @@ Valid fields are:
 	* ``pgp_num``: Effective number of placement groups when calculating placement.
 
 
-Sends a scrub command to OSD ``{osd-num}``. To send the command to all OSDs, use ``*``. ::
+Sends a scrub command to OSD ``{osd-num}``. To send the command to all OSDs, use ``*``. :
+
+.. prompt:: bash $
 
    ceph osd scrub {osd-num}
 
-Sends a repair command to OSD.N. To send the command to all OSDs, use ``*``. ::
+Sends a repair command to OSD.N. To send the command to all OSDs, use ``*``. :
+
+.. prompt:: bash $
 
    ceph osd repair N
 
@@ -301,34 +392,48 @@ in write requests of ``BYTES_PER_WRITE`` each. By default, the test
 writes 1 GB in total in 4-MB increments.
 The benchmark is non-destructive and will not overwrite existing live
 OSD data, but might temporarily affect the performance of clients
-concurrently accessing the OSD. ::
+concurrently accessing the OSD. :
+
+.. prompt:: bash $
 
    ceph tell osd.N bench [TOTAL_DATA_BYTES] [BYTES_PER_WRITE]
 
-To clear an OSD's caches between benchmark runs, use the 'cache drop' command ::
+To clear an OSD's caches between benchmark runs, use the 'cache drop' command :
+
+.. prompt:: bash $
 
    ceph tell osd.N cache drop
 
-To get the cache statistics of an OSD, use the 'cache status' command ::
+To get the cache statistics of an OSD, use the 'cache status' command :
+
+.. prompt:: bash $
 
    ceph tell osd.N cache status
 
 MDS Subsystem
 =============
 
-Change configuration parameters on a running mds. ::
+Change configuration parameters on a running mds. :
+
+.. prompt:: bash $
 
    ceph tell mds.{mds-id} config set {setting} {value}
 
-Example::
+Example:
+
+.. prompt:: bash $
 
    ceph tell mds.0 config set debug_ms 1
 
-Enables debug messages. ::
+Enables debug messages. :
+
+.. prompt:: bash $
 
    ceph mds stat
 
-Displays the status of all metadata servers. ::
+Displays the status of all metadata servers. :
+
+.. prompt:: bash $
 
    ceph mds fail 0
 
@@ -340,16 +445,22 @@ Marks the active MDS as failed, triggering failover to a standby if present.
 Mon Subsystem
 =============
 
-Show monitor stats::
+Show monitor stats:
+
+.. prompt:: bash $
 
    ceph mon stat
 
+::
+
 	e2: 3 mons at {a=127.0.0.1:40000/0,b=127.0.0.1:40001/0,c=127.0.0.1:40002/0}, election epoch 6, quorum 0,1,2 a,b,c
 
 
 The ``quorum`` list at the end lists monitor nodes that are part of the current quorum.
 
-This is also available more directly::
+This is also available more directly:
+
+.. prompt:: bash $
 
    ceph quorum_status -f json-pretty
 	
@@ -405,7 +516,9 @@ This is also available more directly::
 
 The above will block until a quorum is reached.
 
-For a status of just a single monitor::
+For a status of just a single monitor:
+
+.. prompt:: bash $
 
    ceph tell mon.[name] mon_status
 	
@@ -469,10 +582,14 @@ output::
 	    }
 	}
 
-A dump of the monitor state::
+A dump of the monitor state:
+
+       .. prompt:: bash $
 
 	ceph mon dump
 
+  ::
+
 	dumped monmap epoch 2
 	epoch 2
 	fsid ba807e74-b64f-4b72-b43f-597dfe60ddbc

ceph/doc/rados/operations/crush-map-edits.rst

@@ -35,7 +35,9 @@ Pool Values`_.
 Get a CRUSH Map
 ---------------
 
-To get the CRUSH map for your cluster, execute the following::
+To get the CRUSH map for your cluster, execute the following:
+
+.. prompt:: bash $
 
 	ceph osd getcrushmap -o {compiled-crushmap-filename}
 
@@ -48,7 +50,9 @@ edit it.
 Decompile a CRUSH Map
 ---------------------
 
-To decompile a CRUSH map, execute the following::
+To decompile a CRUSH map, execute the following:
+
+.. prompt:: bash $
 
 	crushtool -d {compiled-crushmap-filename} -o {decompiled-crushmap-filename}
 
@@ -57,7 +61,9 @@ To decompile a CRUSH map, execute the following::
 Recompile a CRUSH Map
 ---------------------
 
-To compile a CRUSH map, execute the following::
+To compile a CRUSH map, execute the following:
+
+.. prompt:: bash $
 
 	crushtool -c {decompiled-crushmap-filename} -o {compiled-crushmap-filename}
 
@@ -66,7 +72,9 @@ To compile a CRUSH map, execute the following::
 Set the CRUSH Map
 -----------------
 
-To set the CRUSH map for your cluster, execute the following::
+To set the CRUSH map for your cluster, execute the following:
+
+.. prompt:: bash $
 
 	ceph osd setcrushmap -i {compiled-crushmap-filename}
 
@@ -118,14 +126,22 @@ Devices may also have a *device class* associated with them (e.g.,
 ``hdd`` or ``ssd``), allowing them to be conveniently targeted by a
 crush rule.
 
+.. prompt:: bash #
+
+	devices
+
 ::
 
-	# devices
 	device {num} {osd.name} [class {class}]
 
-For example::
+For example:
+
+.. prompt:: bash #
+
+	devices
+
+::
 
-	# devices
 	device 0 osd.0 class ssd
 	device 1 osd.1 class hdd
 	device 2 osd.2
@@ -136,10 +152,6 @@ is normally a single storage device, a pair of devices (for example,
 one for data and one for a journal or metadata), or in some cases a
 small RAID device.
 
-
-
-
-
 CRUSH Map Bucket Types
 ----------------------
 
@@ -157,9 +169,9 @@ media.
 To add a bucket type to the CRUSH map, create a new line under your list of
 bucket types. Enter ``type`` followed by a unique numeric ID and a bucket name.
 By convention, there is one leaf bucket and it is ``type 0``;  however, you may
-give it any name you like (e.g., osd, disk, drive, storage, etc.)::
+give it any name you like (e.g., osd, disk, drive, storage)::
 
-	#types
+	# types
 	type {num} {bucket-name}
 
 For example::
@@ -199,8 +211,8 @@ distribution units, pods, rows, rooms, and data centers. With the exception of
 the leaf nodes representing OSDs, the rest of the hierarchy is arbitrary, and
 you may define it according to your own needs.
 
-We recommend adapting your CRUSH map to your firms's hardware naming conventions
+We recommend adapting your CRUSH map to your firm's hardware naming conventions
-and using instances names that reflect the physical hardware. Your naming
+and using instance names that reflect the physical hardware. Your naming
 practice can make it easier to administer the cluster and troubleshoot
 problems when an OSD and/or other hardware malfunctions and the administrator
 need access to physical hardware.
@@ -655,27 +667,36 @@ There are three types of transformations possible:
    single bucket.  For example, in the previous example, we want the
    ``ssd`` bucket to be mapped to the ``default`` bucket.
 
-The final command to convert the map comprised of the above fragments would be something like::
+The final command to convert the map comprising the above fragments would be something like:
 
-  $ ceph osd getcrushmap -o original
+.. prompt:: bash $
-  $ crushtool -i original --reclassify \
+
+  ceph osd getcrushmap -o original
+  crushtool -i original --reclassify \
     --set-subtree-class default hdd \
     --reclassify-root default hdd \
     --reclassify-bucket %-ssd ssd default \
     --reclassify-bucket ssd ssd default \
     -o adjusted
 
-In order to ensure that the conversion is correct, there is a ``--compare`` command that will test a large sample of inputs to the CRUSH map and ensure that the same result comes back out.  These inputs are controlled by the same options that apply to the ``--test`` command.  For the above example,::
+In order to ensure that the conversion is correct, there is a ``--compare`` command that will test a large sample of inputs against the CRUSH map and check that the same result is output. These inputs are controlled by the same options that apply to the ``--test`` command.  For the above example,:
+
+.. prompt:: bash $
+
+   crushtool -i original --compare adjusted
+
+::
 
-  $ crushtool -i original --compare adjusted
   rule 0 had 0/10240 mismatched mappings (0)
   rule 1 had 0/10240 mismatched mappings (0)
   maps appear equivalent
 
-If there were difference, you'd see what ratio of inputs are remapped
+If there were differences, the ratio of remapped inputs would be reported in
-in the parentheses.
+the parentheses.
 
-If you are satisfied with the adjusted map, you can apply it to the cluster with something like::
+When you are satisfied with the adjusted map, apply it to the cluster with a command of the form:
+
+.. prompt:: bash $
 
    ceph osd setcrushmap -i adjusted
 
@@ -686,7 +707,9 @@ If you can ensure that all clients are running recent code, you can
 adjust the tunables by extracting the CRUSH map, modifying the values,
 and reinjecting it into the cluster.
 
-* Extract the latest CRUSH map::
+* Extract the latest CRUSH map:
+
+  .. prompt:: bash $
 
 	ceph osd getcrushmap -o /tmp/crush
 
@@ -694,11 +717,15 @@ and reinjecting it into the cluster.
   for both large and small clusters we tested with.  You will need to
   additionally specify the ``--enable-unsafe-tunables`` argument to
   ``crushtool`` for this to work.  Please use this option with
-  extreme care.::
+  extreme care.:
+
+  .. prompt:: bash $
 
      crushtool -i /tmp/crush --set-choose-local-tries 0 --set-choose-local-fallback-tries 0 --set-choose-total-tries 50 -o /tmp/crush.new
 
-* Reinject modified map::
+* Reinject modified map:
+
+  .. prompt:: bash $
 
      ceph osd setcrushmap -i /tmp/crush.new
 
@@ -706,7 +733,9 @@ Legacy values
 -------------
 
 For reference, the legacy values for the CRUSH tunables can be set
-with::
+with:
+
+.. prompt:: bash $
 
    crushtool -i /tmp/crush --set-choose-local-tries 2 --set-choose-local-fallback-tries 5 --set-choose-total-tries 19 --set-chooseleaf-descend-once 0 --set-chooseleaf-vary-r 0 -o /tmp/crush.legacy
 
@@ -715,4 +744,4 @@ Further, as noted above, be careful running old versions of the
 ``ceph-osd`` daemon after reverting to legacy values as the feature
 bit is not perfectly enforced.
 
-.. _CRUSH - Controlled, Scalable, Decentralized Placement of Replicated Data: https://ceph.com/wp-content/uploads/2016/08/weil-crush-sc06.pdf
+.. _CRUSH - Controlled, Scalable, Decentralized Placement of Replicated Data: https://ceph.io/assets/pdfs/weil-crush-sc06.pdf

ceph/doc/rados/operations/crush-map.rst

@@ -184,7 +184,9 @@ will be the total of all devices contained beneath it.  Normally
 weights are in units of terabytes (TB).
 
 You can get a simple view the of CRUSH hierarchy for your cluster,
-including weights, with::
+including weights, with:
+
+.. prompt:: bash $
 
    ceph osd tree
 
@@ -208,11 +210,15 @@ erasure coded), the *failure domain*, and optionally a *device class*.
 In rare cases rules must be written by hand by manually editing the
 CRUSH map.
 
-You can see what rules are defined for your cluster with::
+You can see what rules are defined for your cluster with:
+
+.. prompt:: bash $
 
    ceph osd crush rule ls
 
-You can view the contents of the rules with::
+You can view the contents of the rules with:
+
+.. prompt:: bash $
 
    ceph osd crush rule dump
 
@@ -224,23 +230,31 @@ default, OSDs automatically set their class at startup to
 `hdd`, `ssd`, or `nvme` based on the type of device they are backed
 by.
 
-The device class for one or more OSDs can be explicitly set with::
+The device class for one or more OSDs can be explicitly set with:
+
+.. prompt:: bash $
 
    ceph osd crush set-device-class <class> <osd-name> [...]
 
 Once a device class is set, it cannot be changed to another class
-until the old class is unset with::
+until the old class is unset with:
+
+.. prompt:: bash $
 
    ceph osd crush rm-device-class <osd-name> [...]
 
 This allows administrators to set device classes without the class
 being changed on OSD restart or by some other script.
 
-A placement rule that targets a specific device class can be created with::
+A placement rule that targets a specific device class can be created with:
+
+.. prompt:: bash $
 
    ceph osd crush rule create-replicated <rule-name> <root> <failure-domain> <class>
 
-A pool can then be changed to use the new rule with::
+A pool can then be changed to use the new rule with:
+
+.. prompt:: bash $
 
    ceph osd pool set <pool-name> crush_rule <rule-name>
 
@@ -249,7 +263,9 @@ for each device class in use that contains only devices of that class.
 CRUSH rules can then distribute data over the shadow hierarchy.
 This approach is fully backward compatible with
 old Ceph clients.  You can view the CRUSH hierarchy with shadow items
-with::
+with:
+
+.. prompt:: bash $
 
    ceph osd crush tree --show-shadow
 
@@ -295,7 +311,9 @@ There are two types of weight sets supported:
 
 When weight sets are in use, the weights associated with each node in
 the hierarchy is visible as a separate column (labeled either
-``(compat)`` or the pool name) from the command::
+``(compat)`` or the pool name) from the command:
+
+.. prompt:: bash $
 
    ceph osd tree
 
@@ -320,7 +338,9 @@ Add/Move an OSD
 .. note: OSDs are normally automatically added to the CRUSH map when
          the OSD is created.  This command is rarely needed.
 
-To add or move an OSD in the CRUSH map of a running cluster::
+To add or move an OSD in the CRUSH map of a running cluster:
+
+.. prompt:: bash $
 
    ceph osd crush set {name} {weight} root={root} [{bucket-type}={bucket-name} ...]
 
@@ -359,7 +379,9 @@ Where:
 
 
 The following example adds ``osd.0`` to the hierarchy, or moves the
-OSD from a previous location. ::
+OSD from a previous location:
+
+.. prompt:: bash $
 
    ceph osd crush set osd.0 1.0 root=default datacenter=dc1 room=room1 row=foo rack=bar host=foo-bar-1
 
@@ -372,7 +394,9 @@ Adjust OSD weight
          is rarely needed.
 
 To adjust an OSD's CRUSH weight in the CRUSH map of a running cluster, execute
-the following::
+the following:
+
+.. prompt:: bash $
 
    ceph osd crush reweight {name} {weight}
 
@@ -403,7 +427,9 @@ Remove an OSD
    ``ceph osd purge`` command.  This command is rarely needed.
 
 To remove an OSD from the CRUSH map of a running cluster, execute the
-following::
+following:
+
+.. prompt:: bash $
 
    ceph osd crush remove {name}
 
@@ -431,7 +457,9 @@ Add a Bucket
    ``default`` or other root as described below.
 
 To add a bucket in the CRUSH map of a running cluster, execute the
-``ceph osd crush add-bucket`` command::
+``ceph osd crush add-bucket`` command:
+
+.. prompt:: bash $
 
    ceph osd crush add-bucket {bucket-name} {bucket-type}
 
@@ -453,7 +481,9 @@ Where:
 :Example: ``rack``
 
 
-The following example adds the ``rack12`` bucket to the hierarchy::
+The following example adds the ``rack12`` bucket to the hierarchy:
+
+.. prompt:: bash $
 
    ceph osd crush add-bucket rack12 rack
 
@@ -461,7 +491,9 @@ Move a Bucket
 -------------
 
 To move a bucket to a different location or position in the CRUSH map
-hierarchy, execute the following::
+hierarchy, execute the following:
+
+.. prompt:: bash $
 
    ceph osd crush move {bucket-name} {bucket-type}={bucket-name}, [...]
 
@@ -484,7 +516,9 @@ Where:
 Remove a Bucket
 ---------------
 
-To remove a bucket from the CRUSH hierarchy, execute the following::
+To remove a bucket from the CRUSH hierarchy, execute the following:
+
+.. prompt:: bash $
 
    ceph osd crush remove {bucket-name}
 
@@ -499,7 +533,9 @@ Where:
 :Required: Yes
 :Example: ``rack12``
 
-The following example removes the ``rack12`` bucket from the hierarchy::
+The following example removes the ``rack12`` bucket from the hierarchy:
+
+.. prompt:: bash $
 
    ceph osd crush remove rack12
 
@@ -509,22 +545,30 @@ Creating a compat weight set
 .. note: This step is normally done automatically by the ``balancer``
    module when enabled.
 
-To create a *compat* weight set::
+To create a *compat* weight set:
+
+.. prompt:: bash $
 
    ceph osd crush weight-set create-compat
 
-Weights for the compat weight set can be adjusted with::
+Weights for the compat weight set can be adjusted with:
+
+.. prompt:: bash $
 
    ceph osd crush weight-set reweight-compat {name} {weight}
 
-The compat weight set can be destroyed with::
+The compat weight set can be destroyed with:
+
+.. prompt:: bash $
 
    ceph osd crush weight-set rm-compat
 
 Creating per-pool weight sets
 -----------------------------
 
-To create a weight set for a specific pool,::
+To create a weight set for a specific pool:
+
+.. prompt:: bash $
 
    ceph osd crush weight-set create {pool-name} {mode}
 
@@ -553,15 +597,21 @@ Where:
 :Required: Yes
 :Example: ``flat``
 
-To adjust the weight of an item in a weight set::
+To adjust the weight of an item in a weight set:
+
+.. prompt:: bash $
 
    ceph osd crush weight-set reweight {pool-name} {item-name} {weight [...]}
 
-To list existing weight sets,::
+To list existing weight sets:
+
+.. prompt:: bash $
 
    ceph osd crush weight-set ls
 
-To remove a weight set,::
+To remove a weight set:
+
+.. prompt:: bash $
 
    ceph osd crush weight-set rm {pool-name}
 
@@ -588,7 +638,9 @@ classify themselves as either ``hdd`` or ``ssd``, depending on the
 underlying type of device being used.  These classes can also be
 customized.
 
-To create a replicated rule,::
+To create a replicated rule:
+
+.. prompt:: bash $
 
    ceph osd crush rule create-replicated {name} {root} {failure-domain-type} [{class}]
 
@@ -635,11 +687,15 @@ you must include this information in the *erasure code profile*.  A CRUSH
 rule will then be created from that either explicitly or automatically when
 the profile is used to create a pool.
 
-The erasure code profiles can be listed with::
+The erasure code profiles can be listed with:
+
+.. prompt:: bash $
 
    ceph osd erasure-code-profile ls
 
-An existing profile can be viewed with::
+An existing profile can be viewed with:
+
+.. prompt:: bash $
 
    ceph osd erasure-code-profile get {profile-name}
 
@@ -659,7 +715,9 @@ The erasure code profile properties of interest are:
  * **crush-device-class**: the device class on which to place data [default: none, meaning all devices are used].
  * **k** and **m** (and, for the ``lrc`` plugin, **l**): these determine the number of erasure code shards, affecting the resulting CRUSH rule.
 
-Once a profile is defined, you can create a CRUSH rule with::
+Once a profile is defined, you can create a CRUSH rule with:
+
+.. prompt:: bash $
 
    ceph osd crush rule create-erasure {name} {profile-name}
 
@@ -671,7 +729,9 @@ Once a profile is defined, you can create a CRUSH rule with::
 Deleting rules
 --------------
 
-Rules that are not in use by pools can be deleted with::
+Rules that are not in use by pools can be deleted with:
+
+.. prompt:: bash $
 
    ceph osd crush rule rm {rule-name}
 
@@ -882,14 +942,18 @@ To make this warning go away, you have two options:
    result in some data movement (possibly as much as 10%).  This is the
    preferred route, but should be taken with care on a production cluster
    where the data movement may affect performance.  You can enable optimal
-   tunables with::
+   tunables with:
+
+   .. prompt:: bash $
 
       ceph osd crush tunables optimal
 
    If things go poorly (e.g., too much load) and not very much
    progress has been made, or there is a client compatibility problem
    (old kernel CephFS or RBD clients, or pre-Bobtail ``librados``
-   clients), you can switch back with::
+   clients), you can switch back with:
+
+   .. prompt:: bash $
 
       ceph osd crush tunables legacy
 
@@ -899,7 +963,9 @@ To make this warning go away, you have two options:
       mon warn on legacy crush tunables = false
 
    For the change to take effect, you will need to restart the monitors, or
-   apply the option to running monitors with::
+   apply the option to running monitors with:
+
+   .. prompt:: bash $
 
       ceph tell mon.\* config set mon_warn_on_legacy_crush_tunables false
 
@@ -936,7 +1002,7 @@ sets known as *profiles*.  As of the Octopus release these are:
  * ``firefly``: the values supported by the firefly release
  * ``hammer``: the values supported by the hammer release
  * ``jewel``: the values supported by the jewel release
- * ``optimal``: the best (ie optimal) values of the current version of Ceph
+ * ``optimal``: the best (i.e. optimal) values of the current version of Ceph
  * ``default``: the default values of a new cluster installed from
    scratch. These values, which depend on the current version of Ceph,
    are hardcoded and are generally a mix of optimal and legacy values.
@@ -944,7 +1010,9 @@ sets known as *profiles*.  As of the Octopus release these are:
    LTS release, or the most recent release for which we generally expect
    most users to have up-to-date clients for.
 
-You can apply a profile to a running cluster with the command::
+You can apply a profile to a running cluster with the command:
+
+.. prompt:: bash $
 
    ceph osd crush tunables {PROFILE}
 
@@ -953,8 +1021,7 @@ release notes and documentation carefully before changing the profile on a
 running cluster, and consider throttling recovery/backfill parameters to
 limit the impact of a bolus of backfill.
 
-
+.. _CRUSH - Controlled, Scalable, Decentralized Placement of Replicated Data: https://ceph.io/assets/pdfs/weil-crush-sc06.pdf
-.. _CRUSH - Controlled, Scalable, Decentralized Placement of Replicated Data: https://ceph.com/wp-content/uploads/2016/08/weil-crush-sc06.pdf
 
 
 Primary Affinity
@@ -987,19 +1054,20 @@ interface bandwidth and CPU cycles more evenly.
 By default, all ceph OSDs have primary affinity of ``1``, which indicates that
 any OSD may act as a primary with equal probability.
 
-You can reduce a Ceph OSD's primary affinity so that CRUSH is less likely to choose
+You can reduce a Ceph OSD's primary affinity so that CRUSH is less likely to
-the OSD as primary in a PG's acting set.::
+choose the OSD as primary in a PG's acting set.:
+
+.. prompt:: bash $
 
    ceph osd primary-affinity <osd-id> <weight>
 
-You may set an OSD's primary affinity to a real number in the range
+You may set an OSD's primary affinity to a real number in the range ``[0-1]``,
-``[0-1]``, where ``0`` indicates that the OSD may **NOT** be used as a primary
+where ``0`` indicates that the OSD may **NOT** be used as a primary and ``1``
-and ``1`` indicates that an OSD may be used as a primary.  When the weight is
+indicates that an OSD may be used as a primary.  When the weight is between
-between these extremes, it is less likely that
+these extremes, it is less likely that CRUSH will select that OSD as a primary.
-CRUSH will select that OSD as a primary.  The process for
+The process for selecting the lead OSD is more nuanced than a simple
-selecting the lead OSD is more nuanced than a simple probability based on
+probability based on relative affinity values, but measurable results can be
-relative affinity values, but measurable results can be achieved even with
+achieved even with first-order approximations of desirable values.
-first-order approximations of desirable values.
 
 Custom CRUSH Rules
 ------------------
@@ -1052,7 +1120,6 @@ must not contain the same servers::
         }
 
 
-
 Note also that on failure of an SSD, requests to a PG will be served temporarily
 from a (slower) HDD OSD until the PG's data has been replicated onto the replacement
 primary SSD OSD.

ceph/doc/rados/operations/devices.rst

@@ -1,4 +1,3 @@
-
 .. _devices:
 
 Device Management
@@ -11,17 +10,23 @@ provide tools to predict and/or automatically respond to hardware failure.
 Device tracking
 ---------------
 
-You can query which storage devices are in use with::
+You can query which storage devices are in use with:
+
+.. prompt:: bash $
 
    ceph device ls
 
-You can also list devices by daemon or by host::
+You can also list devices by daemon or by host:
+
+.. prompt:: bash $
 
    ceph device ls-by-daemon <daemon>
    ceph device ls-by-host <host>
 
 For any individual device, you can query information about its
-location and how it is being consumed with::
+location and how it is being consumed with:
+
+.. prompt:: bash $
 
    ceph device info <devid>
 
@@ -34,7 +39,9 @@ failed disks easy and less error-prone.  Use the following command::
   device light on|off <devid> [ident|fault] [--force]
 
 The ``<devid>`` parameter is the device identification. You can obtain this
-information using the following command::
+information using the following command:
+
+.. prompt:: bash $
 
    ceph device ls
 
@@ -43,7 +50,9 @@ By default, the `identification` light is used.
 
 .. note::
    This command needs the Cephadm or the Rook `orchestrator <https://docs.ceph.com/docs/master/mgr/orchestrator/#orchestrator-cli-module>`_ module enabled.
-   The orchestrator module enabled is shown by executing the following command::
+   The orchestrator module enabled is shown by executing the following command:
+
+   .. prompt:: bash $
 
       ceph orch status
 
@@ -77,11 +86,15 @@ or unrecoverable read errors.  Other device types like SAS and NVMe
 implement a similar set of metrics (via slightly different standards).
 All of these can be collected by Ceph via the ``smartctl`` tool.
 
-You can enable or disable health monitoring with::
+You can enable or disable health monitoring with:
+
+.. prompt:: bash $
 
    ceph device monitoring on
 
-or::
+or:
+
+.. prompt:: bash $
 
    ceph device monitoring off
 
@@ -89,26 +102,36 @@ or::
 Scraping
 --------
 
-If monitoring is enabled, metrics will automatically be scraped at regular intervals.  That interval can be configured with::
+If monitoring is enabled, metrics will automatically be scraped at regular intervals.  That interval can be configured with:
+
+.. prompt:: bash $
 
    ceph config set mgr mgr/devicehealth/scrape_frequency <seconds>
 
 The default is to scrape once every 24 hours.
 
-You can manually trigger a scrape of all devices with::
+You can manually trigger a scrape of all devices with:
+   
+.. prompt:: bash $
 
    ceph device scrape-health-metrics
 
-A single device can be scraped with::
+A single device can be scraped with:
+
+.. prompt:: bash $
 
    ceph device scrape-health-metrics <device-id>
 
-Or a single daemon's devices can be scraped with::
+Or a single daemon's devices can be scraped with:
+
+.. prompt:: bash $
 
    ceph device scrape-daemon-health-metrics <who>
 
 The stored health metrics for a device can be retrieved (optionally
-for a specific timestamp) with::
+for a specific timestamp) with:
+
+.. prompt:: bash $
 
    ceph device get-health-metrics <devid> [sample-timestamp]
 
@@ -121,27 +144,37 @@ health metrics it collects.  There are three modes:
 * *none*: disable device failure prediction.
 * *local*: use a pre-trained prediction model from the ceph-mgr daemon
 
-The prediction mode can be configured with::
+The prediction mode can be configured with:
+
+.. prompt:: bash $
 
    ceph config set global device_failure_prediction_mode <mode>
 
 Prediction normally runs in the background on a periodic basis, so it
 may take some time before life expectancy values are populated.  You
-can see the life expectancy of all devices in output from::
+can see the life expectancy of all devices in output from:
+
+.. prompt:: bash $
 
    ceph device ls
 
-You can also query the metadata for a specific device with::
+You can also query the metadata for a specific device with:
+
+.. prompt:: bash $
 
    ceph device info <devid>
 
-You can explicitly force prediction of a device's life expectancy with::
+You can explicitly force prediction of a device's life expectancy with:
+
+.. prompt:: bash $
 
    ceph device predict-life-expectancy <devid>
 
 If you are not using Ceph's internal device failure prediction but
 have some external source of information about device failures, you
-can inform Ceph of a device's life expectancy with::
+can inform Ceph of a device's life expectancy with:
+
+.. prompt:: bash $
 
    ceph device set-life-expectancy <devid> <from> [<to>]
 
@@ -156,7 +189,9 @@ The ``mgr/devicehealth/warn_threshold`` controls how soon an expected
 device failure must be before we generate a health warning.
 
 The stored life expectancy of all devices can be checked, and any
-appropriate health alerts generated, with::
+appropriate health alerts generated, with:
+
+.. prompt:: bash $
 
    ceph device check-health
 

ceph/doc/rados/operations/erasure-code-clay.rst

@@ -38,19 +38,23 @@ to achieve recovery from an OSD failure.
 Erasure-code profile examples
 =============================
 
-An example configuration that can be used to observe reduced bandwidth usage::
+An example configuration that can be used to observe reduced bandwidth usage:
 
-        $ ceph osd erasure-code-profile set CLAYprofile \
+.. prompt:: bash $
+
+   ceph osd erasure-code-profile set CLAYprofile \
       plugin=clay \
       k=4 m=2 d=5 \
       crush-failure-domain=host
-        $ ceph osd pool create claypool erasure CLAYprofile
+   ceph osd pool create claypool erasure CLAYprofile
 
 
 Creating a clay profile
 =======================
 
-To create a new clay code profile::
+To create a new clay code profile:
+
+.. prompt:: bash $
 
    ceph osd erasure-code-profile set {name} \
         plugin=clay \
@@ -60,6 +64,7 @@ To create a new clay code profile::
         [scalar_mds={plugin-name}] \
         [technique={technique-name}] \
         [crush-failure-domain={bucket-type}] \
+        [crush-device-class={device-class}] \
         [directory={directory}] \
         [--force]
 

ceph/doc/rados/operations/erasure-code-isa.rst

@@ -9,7 +9,9 @@ library.
 Create an isa profile
 =====================
 
-To create a new *isa* erasure code profile::
+To create a new *isa* erasure code profile:
+
+.. prompt:: bash $
 
    ceph osd erasure-code-profile set {name} \
      plugin=isa \

ceph/doc/rados/operations/erasure-code-jerasure.rst

@@ -13,7 +13,9 @@ understanding of the parameters.
 Create a jerasure profile
 =========================
 
-To create a new *jerasure* erasure code profile::
+To create a new *jerasure* erasure code profile:
+ 
+.. prompt:: bash $
 
    ceph osd erasure-code-profile set {name} \
      plugin=jerasure \