pve-manager/PVE/Ceph/Services.pm

package PVE::Ceph::Services;

use strict;
use warnings;

use PVE::Ceph::Tools;
use PVE::Tools qw(run_command);
use PVE::RADOS;

use File::Path;

sub ceph_service_cmd {
    my ($action, $service) = @_;

    my $pve_ceph_cfgpath = PVE::Ceph::Tools::get_config('pve_ceph_cfgpath');
    if (PVE::Ceph::Tools::systemd_managed()) {

	if ($service && $service =~ m/^(mon|osd|mds|mgr|radosgw)(\.([A-Za-z0-9\-]{1,32}))?$/) {
	    $service = defined($3) ? "ceph-$1\@$3" : "ceph-$1.target";
	} else {
	    $service = "ceph.target";
	}

	PVE::Tools::run_command(['/bin/systemctl', $action, $service]);

    } else {
	# ceph daemons does not call 'setsid', so we do that ourself
	# (fork_worker send KILL to whole process group)
	PVE::Tools::run_command(['setsid', 'service', 'ceph', '-c', $pve_ceph_cfgpath, $action, $service]);
    }
}

# MDS

sub list_local_mds_ids {
    my $mds_list = [];
    my $ceph_mds_data_dir = PVE::Ceph::Tools::get_config('ceph_mds_data_dir');
    my $ccname = PVE::Ceph::Tools::get_config('ccname');

    PVE::Tools::dir_glob_foreach($ceph_mds_data_dir, qr/$ccname-(\S+)/, sub {
	my (undef, $mds_id) = @_;
	push @$mds_list, $mds_id;
    });

    return $mds_list;
}

sub get_cluster_mds_state {
    my ($rados) = @_;

    my $mds_state = {};

    if (!defined($rados)) {
	$rados = PVE::RADOS->new();
    }

    my $add_state = sub {
	my ($mds) = @_;

	my $state = {};
	$state->{addr} = $mds->{addr};
	$state->{rank} = $mds->{rank};
	$state->{standby_replay} = $mds->{standby_replay} ? 1 : 0;
	$state->{state} = $mds->{state};

	$mds_state->{$mds->{name}} = $state;
    };

    my $mds_dump = $rados->mon_command({ prefix => 'mds stat' });
    my $fsmap = $mds_dump->{fsmap};


    foreach my $mds (@{$fsmap->{standbys}}) {
	$add_state->($mds);
    }

    my $fs_info = $fsmap->{filesystems}->[0];
    my $active_mds = $fs_info->{mdsmap}->{info};

    # normally there's only one active MDS, but we can have multiple active for
    # different ranks (e.g., different cephs path hierarchy). So just add all.
    foreach my $mds (values %$active_mds) {
	$add_state->($mds);
    }

    return $mds_state;
}

sub is_any_mds_active {
    my ($rados) = @_;

    if (!defined($rados)) {
	$rados = PVE::RADOS->new();
    }

    my $mds_dump = $rados->mon_command({ prefix => 'mds stat' });
    my $fs = $mds_dump->{fsmap}->{filesystems};

    if (!($fs && scalar(@$fs) > 0)) {
	return undef;
    }
    my $active_mds = $fs->[0]->{mdsmap}->{info};

    for my $mds (values %$active_mds) {
	return 1 if $mds->{state} eq 'up:active';
    }

    return 0;
}

sub create_mds {
    my ($id, $rados) = @_;

    # `ceph fs status` fails with numeric only ID.
    die "ID: $id, numeric only IDs are not supported\n"
	if $id =~ /^\d+$/;

    if (!defined($rados)) {
	$rados = PVE::RADOS->new();
    }

    my $ccname = PVE::Ceph::Tools::get_config('ccname');
    my $service_dir = "/var/lib/ceph/mds/$ccname-$id";
    my $service_keyring = "$service_dir/keyring";
    my $service_name = "mds.$id";

    die "ceph MDS directory '$service_dir' already exists\n"
	if -d $service_dir;

    print "creating MDS directory '$service_dir'\n";
    eval { File::Path::mkpath($service_dir) };
    my $err = $@;
    die "creation MDS directory '$service_dir' failed\n" if $err;

    # http://docs.ceph.com/docs/luminous/install/manual-deployment/#adding-mds
    my $priv = [
	mon => 'allow profile mds',
	osd => 'allow rwx',
	mds => 'allow *',
    ];

    print "creating keys for '$service_name'\n";
    my $output = $rados->mon_command({
	prefix => 'auth get-or-create',
	entity => $service_name,
	caps => $priv,
	format => 'plain',
    });

    PVE::Tools::file_set_contents($service_keyring, $output);

    print "setting ceph as owner for service directory\n";
    run_command(["chown", 'ceph:ceph', '-R', $service_dir]);

    print "enabling service 'ceph-mds\@$id.service'\n";
    ceph_service_cmd('enable', $service_name);
    print "starting service 'ceph-mds\@$id.service'\n";
    ceph_service_cmd('start', $service_name);

    return undef;
};

sub destroy_mds {
    my ($id, $rados) = @_;

    if (!defined($rados)) {
	$rados = PVE::RADOS->new();
    }

    my $ccname = PVE::Ceph::Tools::get_config('ccname');

    my $service_name = "mds.$id";
    my $service_dir = "/var/lib/ceph/mds/$ccname-$id";

    print "disabling service 'ceph-mds\@$id.service'\n";
    ceph_service_cmd('disable', $service_name);
    print "stopping service 'ceph-mds\@$id.service'\n";
    ceph_service_cmd('stop', $service_name);

    if (-d $service_dir) {
	print "removing ceph-mds directory '$service_dir'\n";
	File::Path::remove_tree($service_dir);
    } else {
	warn "cannot cleanup MDS $id directory, '$service_dir' not found\n"
    }

    print "removing ceph auth for '$service_name'\n";
    $rados->mon_command({
	    prefix => 'auth del',
	    entity => $service_name,
	    format => 'plain'
	});

    return undef;
};

# MGR

sub create_mgr {
    my ($id, $rados) = @_;

    my $clustername = PVE::Ceph::Tools::get_config('ccname');
    my $mgrdir = "/var/lib/ceph/mgr/$clustername-$id";
    my $mgrkeyring = "$mgrdir/keyring";
    my $mgrname = "mgr.$id";

    die "ceph manager directory '$mgrdir' already exists\n"
	if -d $mgrdir;

    print "creating manager directory '$mgrdir'\n";
    mkdir $mgrdir;
    print "creating keys for '$mgrname'\n";
    my $output = $rados->mon_command({ prefix => 'auth get-or-create',
				       entity => $mgrname,
				       caps => [
					   mon => 'allow profile mgr',
					   osd => 'allow *',
					   mds => 'allow *',
				       ],
				       format => 'plain'});
    PVE::Tools::file_set_contents($mgrkeyring, $output);

    print "setting owner for directory\n";
    run_command(["chown", 'ceph:ceph', '-R', $mgrdir]);

    print "enabling service 'ceph-mgr\@$id.service'\n";
    ceph_service_cmd('enable', $mgrname);
    print "starting service 'ceph-mgr\@$id.service'\n";
    ceph_service_cmd('start', $mgrname);
}

sub destroy_mgr {
    my ($mgrid) = @_;

    my $clustername = PVE::Ceph::Tools::get_config('ccname');
    my $mgrname = "mgr.$mgrid";
    my $mgrdir = "/var/lib/ceph/mgr/$clustername-$mgrid";

    die "ceph manager directory '$mgrdir' not found\n"
	if ! -d $mgrdir;

    print "disabling service 'ceph-mgr\@$mgrid.service'\n";
    ceph_service_cmd('disable', $mgrname);
    print "stopping service 'ceph-mgr\@$mgrid.service'\n";
    ceph_service_cmd('stop', $mgrname);

    print "removing manager directory '$mgrdir'\n";
    File::Path::remove_tree($mgrdir);
}

1;