package PVE::API2::Ceph::MON; use strict; use warnings; use Net::IP; use File::Path; use PVE::Ceph::Tools; use PVE::Ceph::Services; use PVE::Cluster qw(cfs_read_file cfs_write_file); use PVE::JSONSchema qw(get_standard_option); use PVE::Network; use PVE::RADOS; use PVE::RESTHandler; use PVE::RPCEnvironment; use PVE::Tools qw(run_command file_set_contents); use base qw(PVE::RESTHandler); my $find_mon_ip = sub { my ($cfg, $rados, $node, $overwrite_ip) = @_; my $pubnet; if ($rados) { $pubnet = $rados->mon_command({ prefix => "config get" , who => "mon.", key => "public_network", format => 'plain' }); # if not defined in the db, the result is empty, it is also always # followed by a newline ($pubnet) = $pubnet =~ m/^(\S+)$/; } $pubnet //= $cfg->{global}->{public_network}; if (!$pubnet) { return $overwrite_ip // PVE::Cluster::remote_node_ip($node); } my $allowed_ips = PVE::Network::get_local_ip_from_cidr($pubnet); die "No active IP found for the requested ceph public network '$pubnet' on node '$node'\n" if scalar(@$allowed_ips) < 1; if (!$overwrite_ip) { if (scalar(@$allowed_ips) == 1) { return $allowed_ips->[0]; } die "Multiple IPs for ceph public network '$pubnet' detected on $node:\n". join("\n", @$allowed_ips) ."\nuse 'mon-address' to specify one of them.\n"; } else { if (grep { $_ eq $overwrite_ip } @$allowed_ips) { return $overwrite_ip; } die "Monitor IP '$overwrite_ip' not in ceph public network '$pubnet'\n" if !PVE::Network::is_ip_in_cidr($overwrite_ip, $pubnet); die "Specified monitor IP '$overwrite_ip' not configured or up on $node!\n"; } }; my $assert_mon_prerequisites = sub { my ($cfg, $monhash, $monid, $monip) = @_; my $ip_regex = '(?:^|[^\d])' . # start or nondigit "\Q$monip\E" . # the ip not interpreted as regex '(?:[^\d]|$)'; # end or nondigit if (($cfg->{global}->{mon_host} && $cfg->{global}->{mon_host} =~ m/$ip_regex/) || grep { $_->{addr} && $_->{addr} =~ m/$ip_regex/ } values %$monhash) { die "monitor address '$monip' already in use\n"; } if (defined($monhash->{$monid})) { die "monitor '$monid' already exists\n"; } }; __PACKAGE__->register_method ({ name => 'listmon', path => '', method => 'GET', description => "Get Ceph monitor list.", proxyto => 'node', protected => 1, permissions => { check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), }, }, returns => { type => 'array', items => { type => "object", properties => { name => { type => 'string' }, addr => { type => 'string', optional => 1 }, host => { type => 'string', optional => 1 }, }, }, links => [ { rel => 'child', href => "{name}" } ], }, code => sub { my ($param) = @_; PVE::Ceph::Tools::check_ceph_inited(); my $res = []; my $cfg = cfs_read_file('ceph.conf'); my $rados = eval { PVE::RADOS->new() }; warn $@ if $@; my $monhash = PVE::Ceph::Services::get_services_info("mon", $cfg, $rados); if ($rados) { my $monstat = $rados->mon_command({ prefix => 'mon_status' }); my $mons = $monstat->{monmap}->{mons}; foreach my $d (@$mons) { next if !defined($d->{name}); $monhash->{$d->{name}}->{rank} = $d->{rank}; $monhash->{$d->{name}}->{addr} = $d->{addr}; $monhash->{$d->{name}}->{state} = 'running'; if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) { $monhash->{$d->{name}}->{quorum} = 1; } } } else { # we cannot check the status if we do not have a RADOS # object, so set the state to unknown foreach my $monid (sort keys %$monhash) { $monhash->{$monid}->{state} = 'unknown'; } } return PVE::RESTHandler::hash_to_array($monhash, 'name'); }}); __PACKAGE__->register_method ({ name => 'createmon', path => '{monid}', method => 'POST', description => "Create Ceph Monitor and Manager", proxyto => 'node', protected => 1, permissions => { check => ['perm', '/', [ 'Sys.Modify' ]], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), monid => { type => 'string', optional => 1, pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?', description => "The ID for the monitor, when omitted the same as the nodename", }, 'mon-address' => { description => 'Overwrites autodetected monitor IP address. ' . 'Must be in the public network of ceph.', type => 'string', format => 'ip', optional => 1, }, }, }, returns => { type => 'string' }, code => sub { my ($param) = @_; PVE::Ceph::Tools::check_ceph_installed('ceph_mon'); PVE::Ceph::Tools::check_ceph_inited(); PVE::Ceph::Tools::setup_pve_symlinks(); my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $cfg = cfs_read_file('ceph.conf'); my $rados = eval { PVE::RADOS->new() }; # try a rados connection, fails for first monitor my $monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados); if (!defined($rados) && (scalar(keys %$monhash) || $cfg->{global}->{mon_host})) { die "Could not connect to ceph cluster despite configured monitors\n"; } my $monid = $param->{monid} // $param->{node}; my $monsection = "mon.$monid"; my $ip = $find_mon_ip->($cfg, $rados, $param->{node}, $param->{'mon-address'}); $assert_mon_prerequisites->($cfg, $monhash, $monid, $ip); my $worker = sub { my $upid = shift; my $client_keyring = PVE::Ceph::Tools::get_or_create_admin_keyring(); my $mon_keyring = PVE::Ceph::Tools::get_config('pve_mon_key_path'); if (! -f $mon_keyring) { run_command("ceph-authtool --create-keyring $mon_keyring ". " --gen-key -n mon. --cap mon 'allow *'"); run_command("ceph-authtool $mon_keyring --import-keyring $client_keyring"); } my $ccname = PVE::Ceph::Tools::get_config('ccname'); my $mondir = "/var/lib/ceph/mon/$ccname-$monid"; -d $mondir && die "monitor filesystem '$mondir' already exist\n"; my $monmap = "/tmp/monmap"; eval { mkdir $mondir; run_command("chown ceph:ceph $mondir"); if (defined($rados)) { # we can only have a RADOS object if we have a monitor my $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout')); my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' }); file_set_contents($monmap, $mapdata); } else { # we need to create a monmap for the first monitor my $monaddr = $ip; if (Net::IP::ip_is_ipv6($ip)) { $monaddr = "[$ip]"; $cfg->{global}->{ms_bind_ipv6} = 'true'; } run_command("monmaptool --create --clobber --addv $monid '[v2:$monaddr:3300,v1:$monaddr:6789]' --print $monmap"); } run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $mon_keyring --public-addr $ip"); run_command("chown ceph:ceph -R $mondir"); }; my $err = $@; unlink $monmap; if ($err) { File::Path::remove_tree($mondir); die $err; } # update ceph.conf my $monhost = $cfg->{global}->{mon_host} // ""; $monhost .= " $ip"; $cfg->{global}->{mon_host} = $monhost; cfs_write_file('ceph.conf', $cfg); PVE::Ceph::Services::ceph_service_cmd('start', $monsection); eval { PVE::Ceph::Services::ceph_service_cmd('enable', $monsection) }; warn "Enable ceph-mon\@${monid}.service failed, do manually: $@\n" if $@; PVE::Ceph::Services::broadcast_ceph_services(); }; return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker); }}); __PACKAGE__->register_method ({ name => 'destroymon', path => '{monid}', method => 'DELETE', description => "Destroy Ceph Monitor and Manager.", proxyto => 'node', protected => 1, permissions => { check => ['perm', '/', [ 'Sys.Modify' ]], }, parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), monid => { description => 'Monitor ID', type => 'string', pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?', }, }, }, returns => { type => 'string' }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); PVE::Ceph::Tools::check_ceph_inited(); my $cfg = cfs_read_file('ceph.conf'); my $monid = $param->{monid}; my $monsection = "mon.$monid"; my $rados = PVE::RADOS->new(); my $monstat = $rados->mon_command({ prefix => 'mon_status' }); my $monlist = $monstat->{monmap}->{mons}; die "no such monitor id '$monid'\n" if !defined($cfg->{$monsection}); my $ccname = PVE::Ceph::Tools::get_config('ccname'); my $mondir = "/var/lib/ceph/mon/$ccname-$monid"; -d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n"; die "can't remove last monitor\n" if scalar(@$monlist) <= 1; my $worker = sub { my $upid = shift; # reopen with longer timeout $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout')); $rados->mon_command({ prefix => "mon remove", name => $monid, format => 'plain' }); eval { PVE::Ceph::Services::ceph_service_cmd('stop', $monsection); }; warn $@ if $@; delete $cfg->{$monsection}; cfs_write_file('ceph.conf', $cfg); File::Path::remove_tree($mondir); eval { PVE::Ceph::Services::ceph_service_cmd('disable', $monsection) }; warn $@ if $@; PVE::Ceph::Services::broadcast_ceph_services(); }; return $rpcenv->fork_worker('cephdestroymon', $monsection, $authuser, $worker); }}); 1;