ceph: add CephFS create and list API

Allow to create a new CephFS instance and allow to list them.

As deletion requires coordination between the active MDS and all
standby MDS next in line this needs a bit more work. One could mark
the MDS cluster down and stop the active, that should work but as
destroying is quite a sensible operation, in production not often
needed I deemed it better to document this only, and leaving API
endpoints for this to the future.

For index/list I slightly transform the result of an RADOS `fs ls`
monitor command, this would allow relative easy display of a CephFS
and it's backing metadata and data pools in a GUI.

While for now it's not enabled by default and marked as experimental,
this API is designed to host multiple CephFS instances - we may not
need this at all, but I did not want to limit us early. And anybody
liking to experiment can use it after the respective ceph.conf
settings.

When encountering errors try to rollback. As we verified at the
beginning that we did not reused pools, destroy the ones which we
created.

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
Co-authored-by: Alwin Antreich <a.antreich@proxmox.com>

PVE/API2/Ceph.pm

@@ -548,6 +548,7 @@ use PVE::RPCEnvironment;
 use PVE::Storage;
 use PVE::Tools qw(run_command file_get_contents file_set_contents);
 
+use PVE::API2::Ceph::FS;
 use PVE::API2::Ceph::MDS;
 use PVE::API2::Storage::Config;
 
@@ -565,6 +566,11 @@ __PACKAGE__->register_method ({
     path => 'mds',
 });
 
+__PACKAGE__->register_method ({
+    subclass => "PVE::API2::Ceph::FS",
+    path => 'fs',
+});
+
 __PACKAGE__->register_method ({
     name => 'index',
     path => '',
@@ -596,6 +602,7 @@ __PACKAGE__->register_method ({
 	    { name => 'mon' },
 	    { name => 'osd' },
 	    { name => 'pools' },
+	    { name => 'fs' },
 	    { name => 'mds' },
 	    { name => 'stop' },
 	    { name => 'start' },

PVE/API2/Ceph/FS.pm

@@ -0,0 +1,207 @@
+package PVE::API2::Ceph::FS;
+
+use strict;
+use warnings;
+
+use PVE::CephTools;
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::RADOS;
+use PVE::RESTHandler;
+use PVE::RPCEnvironment;
+
+use PVE::API2::Storage::Config;
+
+use base qw(PVE::RESTHandler);
+
+__PACKAGE__->register_method ({
+    name => 'index',
+    path => '',
+    method => 'GET',
+    description => "Directory index.",
+    permissions => {
+	check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
+    },
+    protected => 1,
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+	},
+    },
+    returns => {
+	type => 'array',
+	items => {
+	    type => "object",
+	    properties => {
+		name => {
+		    description => "The ceph filesystem name.",
+		    type => 'string',
+		},
+		metadata_pool => {
+		    description => "The name of the metadata pool.",
+		    type => 'string',
+		},
+		data_pool => {
+		    description => "The name of the data pool.",
+		    type => 'string',
+		},
+	    },
+	},
+	links => [ { rel => 'child', href => "{name}" } ],
+    },
+    code => sub {
+	my ($param) = @_;
+
+	my $rados = PVE::RADOS->new();
+
+	my $cephfs_list = $rados->mon_command({ prefix => "fs ls" });
+	# we get something like:
+	#{
+	#   'metadata_pool_id' => 2,
+	#   'data_pool_ids' => [ 1 ],
+	#   'metadata_pool' => 'cephfs_metadata',
+	#   'data_pools' => [ 'cephfs_data' ],
+	#   'name' => 'cephfs',
+	#}
+
+	my $res = [
+	    map {{
+		name => $_->{name},
+		metadata_pool => $_->{metadata_pool},
+		data_pool => $_->{data_pools}->[0],
+	    }} @$cephfs_list
+	];
+
+	return $res;
+    }
+});
+
+__PACKAGE__->register_method ({
+    name => 'createfs',
+    path => '{name}',
+    method => 'POST',
+    description => "Create a Ceph filesystem",
+    proxyto => 'node',
+    protected => 1,
+    permissions => {
+	check => ['perm', '/', [ 'Sys.Modify' ]],
+    },
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+	    name => {
+		description => "The ceph filesystem name.",
+		type => 'string',
+		default => 'cephfs',
+		optional => 1,
+	    },
+	    pg_num => {
+		description => "Number of placement groups for the backing data pool. The metadata pool will use a quarter of this.",
+		type => 'integer',
+		default => 64,
+		optional => 1,
+		minimum => 8,
+		maximum => 32768,
+	    },
+	    add_storage => {
+		description => "Configure the created CephFS as storage for this cluster.",
+		type => 'boolean',
+		optional => 1,
+		default => 0,
+	    },
+	},
+    },
+    returns => { type => 'string' },
+    code => sub {
+	my ($param) = @_;
+
+	PVE::CephTools::check_ceph_inited();
+
+	my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
+	die "Ceph is not fully configured - missing '$pve_ckeyring_path'\n"
+	    if ! -f $pve_ckeyring_path;
+
+	my $fs_name = $param->{name} // 'cephfs';
+	my $pg_num = $param->{pg_num} // 64;
+
+	my $pool_data = "${fs_name}_data";
+	my $pool_metadata = "${fs_name}_metadata";
+
+	my $rados = PVE::RADOS->new();
+	my $ls_pools = PVE::CephTools::ls_pools();
+	my $existing_pools = { map { $_->{poolname} => 1 } @$ls_pools };
+
+	die "ceph pools '$pool_data' and/or '$pool_metadata' already exist\n"
+	    if $existing_pools->{$pool_data} || $existing_pools->{$pool_metadata};
+
+	my $running_mds = PVE::CephTools::get_cluster_mds_state($rados);
+	die "no running Metadata Server (MDS) found!\n" if !scalar(keys %$running_mds);
+
+	my $worker = sub {
+	    $rados = PVE::RADOS->new();
+
+	    my $pool_param = {
+		application => 'cephfs',
+		pg_num => $pg_num,
+	    };
+
+	    my @created_pools = ();
+	    eval {
+		print "creating data pool '$pool_data'...\n";
+		PVE::CephTools::create_pool($pool_data, $pool_param, $rados);
+		push @created_pools, $pool_data;
+
+		print "creating metadata pool '$pool_metadata'...\n";
+		$pool_param->{pg_num} = $pg_num >= 32 ? $pg_num / 4 : 8;
+		PVE::CephTools::create_pool($pool_metadata, $pool_param, $rados);
+		push @created_pools, $pool_metadata;
+
+		print "configuring new CephFS '$fs_name'\n";
+		$rados->mon_command({
+		    prefix => "fs new",
+		    fs_name => $fs_name,
+		    metadata => $pool_metadata,
+		    data => $pool_data,
+		    format => 'plain',
+		});
+	    };
+	    if (my $err = $@) {
+		$@ = undef;
+
+		if (@created_pools > 0) {
+		    warn "Encountered error after creating at least one pool\n";
+		    # our old connection is very likely broken now, recreate
+		    $rados = PVE::RADOS->new();
+		    foreach my $pool (@created_pools) {
+			warn "cleaning up left over pool '$pool'\n";
+			eval { PVE::CephTools::destroy_pool($pool, $rados) };
+			warn "$@\n" if $@;
+		    }
+		}
+
+		die "$err\n";
+	    }
+
+	    if ($param->{add_storage}) {
+		my $err;
+		eval {
+		    PVE::API2::Storage::Config->create({
+			type => 'cephfs',
+			storage => $fs_name,
+			content => 'backup,iso,vztmpl',
+		    })
+		};
+		die "adding storage for CephFS '$fs_name' failed, check log ".
+		    "and add manually!\n$@\n" if $@;
+	    }
+	};
+
+	my $rpcenv = PVE::RPCEnvironment::get();
+	my $user = $rpcenv->get_user();
+
+	return $rpcenv->fork_worker('cephfscreate', $fs_name,  $user, $worker);
+    }
+});
+
+1;

PVE/API2/Ceph/Makefile

@@ -1,6 +1,7 @@
 include ../../../defines.mk
 
 PERLSOURCE= 			\
+	FS.pm			\
 	MDS.pm
 
 all:

PVE/CLI/pveceph.pm

@@ -19,6 +19,7 @@ use PVE::Tools qw(run_command);
 use PVE::JSONSchema qw(get_standard_option);
 use PVE::CephTools;
 use PVE::API2::Ceph;
+use PVE::API2::Ceph::FS;
 use PVE::API2::Ceph::MDS;
 
 use PVE::CLIHandler;
@@ -170,6 +171,7 @@ our $cmddef = {
     }],
     createpool => [ 'PVE::API2::Ceph', 'createpool', ['name'], { node => $nodename }],
     destroypool => [ 'PVE::API2::Ceph', 'destroypool', ['name'], { node => $nodename } ],
+    createfs => [ 'PVE::API2::Ceph::FS', 'createfs', [], { node => $nodename }],
     createosd => [ 'PVE::API2::CephOSD', 'createosd', ['dev'], { node => $nodename }, $upid_exit],
     destroyosd => [ 'PVE::API2::CephOSD', 'destroyosd', ['osdid'], { node => $nodename }, $upid_exit],
     createmon => [ 'PVE::API2::Ceph', 'createmon', [], { node => $nodename }, $upid_exit],

PVE/CephTools.pm

@@ -240,6 +240,18 @@ sub create_pool {
 
 }
 
+sub ls_pools {
+    my ($pool, $rados) = @_;
+
+    if (!defined($rados)) {
+	$rados = PVE::RADOS->new();
+    }
+
+    my $res = $rados->mon_command({ prefix => "osd lspools" });
+
+    return $res;
+}
+
 sub destroy_pool {
     my ($pool, $rados) = @_;