From 39ab3ea6e6bbbde16dffecdb8c2a56142eca4a61 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 13 Nov 2013 06:18:41 +0100 Subject: [PATCH 01/27] import pveceph script --- bin/Makefile | 5 + bin/pveceph | 757 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 762 insertions(+) create mode 100755 bin/pveceph diff --git a/bin/Makefile b/bin/Makefile index 1b0a3bc8..dc1f8603 100644 --- a/bin/Makefile +++ b/bin/Makefile @@ -3,6 +3,7 @@ include ../defines.mk SUBDIRS = init.d cron ocf test SCRIPTS = \ + pveceph \ vzdump \ vzrestore \ pvestatd \ @@ -20,6 +21,7 @@ SCRIPTS = \ pveperf MANS = \ + pveceph.1 \ pvectl.1 \ vzdump.1 \ vzrestore.1 \ @@ -44,6 +46,9 @@ all: ${MANS} pvemailforward pvectl.1.pod: pvectl perl -I.. ./pvectl printmanpod >$@ +pveceph.1.pod: pveceph + perl -I.. ./pveceph printmanpod >$@ + vzdump.1.pod: vzdump perl -I.. -T ./vzdump printmanpod >$@ diff --git a/bin/pveceph b/bin/pveceph new file mode 100755 index 00000000..cf08a92a --- /dev/null +++ b/bin/pveceph @@ -0,0 +1,757 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use Getopt::Long; +use Fcntl ':flock'; +use File::Path; +use IO::File; +use JSON; +use Data::Dumper; + +use PVE::SafeSyslog; +use PVE::Cluster; +use PVE::INotify; +use PVE::RPCEnvironment; +use PVE::Storage; +use PVE::Tools qw(run_command); +use PVE::JSONSchema qw(get_standard_option); + +use PVE::CLIHandler; + +use base qw(PVE::CLIHandler); + +$ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin'; + +initlog ('pveceph'); + +die "please run as root\n" if $> != 0; + +PVE::INotify::inotify_init(); + +my $rpcenv = PVE::RPCEnvironment->init('cli'); + +$rpcenv->init_request(); +$rpcenv->set_language($ENV{LANG}); +$rpcenv->set_user('root@pam'); + +my $nodename = PVE::INotify::nodename(); + +my $ccname = 'ceph'; # ceph cluster name +my $ceph_cfgdir = "/etc/ceph"; +my $pve_ceph_cfgpath = "/etc/pve/$ccname.conf"; +my $ceph_cfgpath = "$ceph_cfgdir/$ccname.conf"; +my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring"; +my $ceph_mon_key_path = "$ceph_cfgdir/$ccname.mon.keyring"; +my $pve_ckeyring_path = "/etc/pve/priv/$ccname.keyring"; +my $ceph_ckeyring_path = "$ceph_cfgdir/$ccname.client.admin.keyring"; + +my $ceph_bootstrap_osd_keyring = "/var/lib/ceph/bootstrap-osd/$ccname.keyring"; +my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring"; + +my $ceph_bin = "/usr/bin/ceph"; + +my $check_ceph_installed = sub { + my ($noerr) = @_; + + if (! -x $ceph_bin) { + die "ceph binaries not installed\n" if !$noerr; + return undef; + } + + return 1; +}; + +my $check_ceph_inited = sub { + my ($noerr) = @_; + + return undef if !&$check_ceph_installed($noerr); + + if (! -f $pve_ceph_cfgpath) { + die "pveceph configuration not initialized\n" if !$noerr; + return undef; + } + + return 1; +}; + +my $force_symlink = sub { + my ($old, $new) = @_; + + return if (-l $new) && (readlink($new) eq $old); + + unlink $new; + symlink($old, $new) || + die "unable to create symlink '$new' - $!\n"; +}; + +my $parse_ceph_config = sub { + my ($filename) = @_; + + my $cfg = {}; + + return $cfg if ! -f $filename; + + my $fh = IO::File->new($filename, "r") || + die "unable to open '$filename' - $!\n"; + + my $section; + + while (defined(my $line = <$fh>)) { + $line =~ s/[;#].*$//; + $line =~ s/^\s+//; + $line =~ s/\s+$//; + next if !$line; + + $section = $1 if $line =~ m/^\[(\S+)\]$/; + if (!$section) { + warn "no section - skip: $line\n"; + next; + } + + if ($line =~ m/^(.*\S)\s*=\s*(\S.*)$/) { + $cfg->{$section}->{$1} = $2; + } + + } + + return $cfg; +}; + +my $run_ceph_cmd = sub { + my ($cmd, %params) = @_; + + my $timeout = 3; + + my $oldalarm; + eval { + local $SIG{ALRM} = sub { die "timeout\n" }; + $oldalarm = alarm($timeout); + # Note: --connect-timeout does not work with current version + # '--connect-timeout', $timeout, + + run_command(['ceph', '-c', $ceph_cfgpath, @$cmd], %params); + alarm(0); + }; + my $err = $@; + + alarm($oldalarm) if $oldalarm; + + die $err if $err; + +}; + +my $ceph_mon_status = sub { + my ($quiet) = @_; + + my $json = ''; + my $parser = sub { + my $line = shift; + $json .= $line; + }; + + my $errfunc = sub { + my $line = shift; + print "$line\n" if !$quiet; + }; + + &$run_ceph_cmd(['mon_status'], outfunc => $parser, errfunc => $errfunc); + + my $res = decode_json($json); + + return $res; +}; + +my $ceph_osd_status = sub { + my ($quiet) = @_; + + my $json = ''; + my $parser = sub { + my $line = shift; + $json .= $line; + }; + + my $errfunc = sub { + my $line = shift; + print "$line\n" if !$quiet; + }; + + &$run_ceph_cmd(['osd', 'dump', '--format', 'json'], + outfunc => $parser, errfunc => $errfunc); + + my $res = decode_json($json); + + return $res; +}; + +my $write_ceph_config = sub { + my ($cfg) = @_; + + my $out = ''; + foreach my $section (keys %$cfg) { + $out .= "[$section]\n"; + foreach my $key (sort keys %{$cfg->{$section}}) { + $out .= "\t $key = $cfg->{$section}->{$key}\n"; + } + $out .= "\n"; + } + + PVE::Tools::file_set_contents($pve_ceph_cfgpath, $out); +}; + +my $setup_pve_symlinks = sub { + # fail if we find a real file instead of a link + if (-f $ceph_cfgpath) { + my $lnk = readlink($ceph_cfgpath); + die "file '$ceph_cfgpath' already exists\n" + if !$lnk || $lnk ne $pve_ceph_cfgpath; + } + + # now assume we are allowed to setup/overwrite content + &$force_symlink($pve_ceph_cfgpath, $ceph_cfgpath); + &$force_symlink($pve_mon_key_path, $ceph_mon_key_path); + &$force_symlink($pve_ckeyring_path, $ceph_ckeyring_path); +}; + +my $ceph_service_cmd = sub { + run_command(['service', 'ceph', '-c', $ceph_cfgpath, @_]); +}; + +__PACKAGE__->register_method ({ + name => 'init', + path => 'init', + method => 'POST', + description => "Create initial ceph configuration.", + parameters => { + additionalProperties => 0, + properties => { + size => { + description => 'Number of replicas per object', + type => 'interger', + default => 2, + optional => 1, + minimum => 1, + maximum => 3, + }, + pg_bits => { + description => "Placement group bits, used to specify the default number of placement groups (Note: 'osd pool default pg num' does not work for deafult pools)", + type => 'interger', + default => 9, + optional => 1, + minimum => 6, + maximum => 14, + }, + }, + }, + returns => { type => 'null' }, + code => sub { + my ($param) = @_; + + &$check_ceph_installed(); + + -f $pve_ceph_cfgpath && + die "configuration file '$pve_ceph_cfgpath' already exists.\n"; + + my $pg_bits = $param->{pg_bits} || 9; + my $size = $param->{size} || 2; + + my $global = { + 'auth supported' => 'cephx', + 'auth cluster required' => 'cephx', + 'auth service required' => 'cephx', + 'auth client required' => 'cephx', + 'filestore xattr use omap' => 'true', + 'osd journal size' => '1024', + 'osd pool default size' => $size, + 'osd pool default min size' => 1, + 'osd pg bits' => $pg_bits, + 'osd pgp bits' => $pg_bits, + }; + + # this does not work for default pools + #'osd pool default pg num' => $pg_num, + #'osd pool default pgp num' => $pg_num, + + &$write_ceph_config({global => $global}); + + &$setup_pve_symlinks(); + + return undef; + }}); + +__PACKAGE__->register_method ({ + name => 'createmon', + path => 'createmon', + method => 'POST', + description => "Create Ceph Monitor", + parameters => { + additionalProperties => 0, + properties => { + }, + }, + returns => { type => 'null' }, + code => sub { + my ($param) = @_; + + &$check_ceph_inited(); + + &$setup_pve_symlinks(); + + if (! -f $pve_ckeyring_path) { + run_command("ceph-authtool $pve_ckeyring_path --create-keyring " . + "--gen-key -n client.admin"); + } + + if (! -f $pve_mon_key_path) { + run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp"); + run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " . + "--cap mds 'allow *' " . + "--cap osd 'allow *' " . + "--cap mon 'allow *'"); + run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'"); + run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path"); + } + + + my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); + + my $moncount = 0; + + my $monaddrhash = {}; + + foreach my $section (keys %$cfg) { + next if $section eq 'global'; + my $d = $cfg->{$section}; + if ($section =~ m/^mon\./) { + $moncount++; + if ($d->{'mon addr'}) { + $monaddrhash->{$d->{'mon addr'}} = $section; + } + } + } + + my $monid; + for (my $i = 0; $i < 7; $i++) { + if (!$cfg->{"mon.$i"}) { + $monid = $i; + last; + } + } + die "unable to find usable monitor id\n" if !defined($monid); + + my $monsection = "mon.$monid"; + my $monaddr = PVE::Cluster::remote_node_ip($nodename) . ":6789"; + my $monname = $nodename; + + die "monitor '$monsection' already exists\n" if $cfg->{$monsection}; + die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n" + if $monaddrhash->{$monaddr}; + + + my $mondir = "/var/lib/ceph/mon/$ccname-$monid"; + -d $mondir && die "monitor filesystem '$mondir' already exist\n"; + + my $monmap = "/tmp/monmap"; + + eval { + mkdir $mondir; + + if ($moncount > 0) { + my $monstat = &$ceph_mon_status(); # online test + &$run_ceph_cmd(['mon', 'getmap', '-o', $monmap]); + } else { + run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap"); + } + + run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path"); + }; + my $err = $@; + unlink $monmap; + if ($err) { + File::Path::remove_tree($mondir); + die $err; + } + + $cfg->{$monsection} = { + 'host' => $monname, + 'mon addr' => $monaddr, + }; + + &$write_ceph_config($cfg); + + &$ceph_service_cmd('start', $monsection); + + return undef; + + }}); + +__PACKAGE__->register_method ({ + name => 'destroymon', + path => 'destroymon', + method => 'POST', + description => "Destroy Ceph monitor.", + parameters => { + additionalProperties => 0, + properties => { + monid => { + destription => 'Monitor ID', + type => 'integer', + }, + }, + }, + returns => { type => 'null' }, + code => sub { + my ($param) = @_; + + &$check_ceph_inited(); + + my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); + + my $monid = $param->{monid}; + my $monsection = "mon.$monid"; + + my $monstat = &$ceph_mon_status(); + my $monlist = $monstat->{monmap}->{mons}; + + die "no such monitor id '$monid'\n" + if !defined($cfg->{$monsection}); + + + my $mondir = "/var/lib/ceph/mon/$ccname-$monid"; + -d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n"; + + die "can't remove last monitor\n" if scalar(@$monlist) <= 1; + + &$run_ceph_cmd(['mon', 'remove', $monid]); + + eval { &$ceph_service_cmd('stop', $monsection); }; + warn $@ if $@; + + delete $cfg->{$monsection}; + &$write_ceph_config($cfg); + File::Path::remove_tree($mondir); + + return undef; + }}); + +__PACKAGE__->register_method ({ + name => 'purge', + path => 'purge', + method => 'POST', + description => "Destroy ceph related data and configuration files.", + parameters => { + additionalProperties => 0, + properties => { + }, + }, + returns => { type => 'null' }, + code => sub { + my ($param) = @_; + + my $monstat; + + eval { $monstat = &$ceph_mon_status(1); }; + my $err = $@; + + die "detected running ceph services- unable to purge data\n" + if !$err; + + # fixme: this is dangerous - should we really support this function? + + unlink $ceph_cfgpath; + unlink $ceph_mon_key_path; + unlink $ceph_ckeyring_path; + + unlink $pve_ceph_cfgpath; + unlink $pve_ckeyring_path; + unlink $pve_mon_key_path; + + unlink $ceph_bootstrap_osd_keyring; + unlink $ceph_bootstrap_mds_keyring; + + system("rm -rf /var/lib/ceph/mon/ceph-*"); + + # remove osd? + + return undef; + }}); + +__PACKAGE__->register_method ({ + name => 'install', + path => 'install', + method => 'POST', + description => "Install ceph related packages.", + parameters => { + additionalProperties => 0, + properties => { + }, + }, + returns => { type => 'null' }, + code => sub { + my ($param) = @_; + + my $cephver = 'dumpling'; + + local $ENV{DEBIAN_FRONTEND} = 'noninteractive'; + + my $keyurl = "https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc"; + + print "download and import ceph reqpository keys\n"; + system("wget -q -O- '$keyurl'| apt-key add - 2>&1 >/dev/null") == 0 || + die "unable to download ceph release key\n"; + + + my $source = "deb http://ceph.com/debian-$cephver wheezy main\n"; + + PVE::Tools::file_set_contents("/etc/apt/sources.list.d/ceph.list", $source); + + print "update available package list\n"; + eval { run_command(['apt-get', '-q', 'update'], outfunc => sub {}, errfunc => sub {}); }; + + run_command(['apt-get', '-q', '--assume-yes', '--no-install-recommends', + '-o', 'Dpkg::Options::=--force-confnew', + 'install', '--', + 'ceph', 'ceph-common', 'gdisk']); + + return undef; + }}); + +__PACKAGE__->register_method ({ + name => 'stop', + path => 'stop', + method => 'POST', + description => "Stop ceph services.", + parameters => { + additionalProperties => 0, + properties => { + }, + }, + returns => { type => 'null' }, + code => sub { + my ($param) = @_; + + &$check_ceph_inited(); + + my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); + scalar(keys %$cfg) || die "no configuration\n"; + + &$ceph_service_cmd('stop'); + + return undef; + }}); + +__PACKAGE__->register_method ({ + name => 'start', + path => 'start', + method => 'POST', + description => "Start ceph services.", + parameters => { + additionalProperties => 0, + properties => { + }, + }, + returns => { type => 'null' }, + code => sub { + my ($param) = @_; + + &$check_ceph_inited(); + + my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); + scalar(keys %$cfg) || die "no configuration\n"; + + &$ceph_service_cmd('start'); + + return undef; + }}); + +__PACKAGE__->register_method ({ + name => 'status', + path => 'status', + method => 'GET', + description => "Get ceph status.", + parameters => { + additionalProperties => 0, + properties => { + }, + }, + returns => { type => 'object' }, + code => sub { + my ($param) = @_; + + my $res = { status => 'unknown' }; + + eval { + if (!&$check_ceph_installed(1)) { + $res->{status} = 'notinstalled'; + } + if (! -f $ceph_cfgpath) { + $res->{status} = 'notconfigured'; + return; + } else { + $res->{status} = 'configured'; + } + + my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); + $res->{config} = $cfg; + + eval { + my $monstat = &$ceph_mon_status(1); + $res->{monstat} = $monstat; + }; + warn $@ if $@; + + eval { + my $osdstat = &$ceph_osd_status(1); + $res->{osdstat} = $osdstat; + }; + warn $@ if $@; + + }; + warn $@ if $@; + + return $res; + }}); + +__PACKAGE__->register_method ({ + name => 'createosd', + path => 'createosd', + method => 'POST', + description => "Create OSD", + parameters => { + additionalProperties => 0, + properties => { + dev => { + description => "Block device name.", + type => 'string', + } + }, + }, + returns => { type => 'null' }, + code => sub { + my ($param) = @_; + + &$check_ceph_inited(); + + die "not fully configured - missing '$pve_ckeyring_path'\n" + if ! -f $pve_ckeyring_path; + + &$setup_pve_symlinks(); + + print "create OSD on $param->{dev}\n"; + + -b $param->{dev} || die "no such block device '$param->{dev}'\n"; + + my $monstat = &$ceph_mon_status(1); + die "unable to get fsid\n" if !$monstat->{monmap} || !$monstat->{monmap}->{fsid}; + my $fsid = $monstat->{monmap}->{fsid}; + + if (! -f $ceph_bootstrap_osd_keyring) { + &$run_ceph_cmd(['auth', 'get', 'client.bootstrap-osd', '-o', $ceph_bootstrap_osd_keyring]); + }; + + run_command(['ceph-disk', 'prepare', '--zap-disk', '--fs-type', 'xfs', + '--cluster', $ccname, '--cluster-uuid', $fsid, + '--', $param->{dev}]); + + return undef; + }}); + +__PACKAGE__->register_method ({ + name => 'destroyosd', + path => 'destroyosd', + method => 'POST', + description => "Destroy OSD", + parameters => { + additionalProperties => 0, + properties => { + osdid => { + destription => 'OSD ID', + type => 'integer', + }, + }, + }, + returns => { type => 'null' }, + code => sub { + my ($param) = @_; + + &$check_ceph_inited(); + + my $osdid = $param->{osdid}; + + print "destroy OSD $param->{osdid}\n"; + + # fixme: not sure what we should do here + + my $stat = &$ceph_osd_status(); + + my $osdlist = $stat->{osds} || []; + + my $osdstat; + foreach my $d (@$osdlist) { + if ($d->{osd} == $osdid) { + $osdstat = $d; + last; + } + } + die "no such OSD '$osdid'\n" if !$osdstat; + + die "osd is in use (in == 1)\n" if $osdstat->{in}; + #&$run_ceph_cmd(['osd', 'out', $osdid]); + + die "osd is still runnung (up == 1)\n" if $osdstat->{up}; + + my $osdsection = "osd.$osdid"; + + eval { &$ceph_service_cmd('stop', $osdsection); }; + warn $@ if $@; + + print "Remove $osdsection from the CRUSH map\n"; + &$run_ceph_cmd(['osd', 'crush', 'remove', $osdid]); + + print "Remove the $osdsection authentication key.\n"; + &$run_ceph_cmd(['auth', 'del', $osdsection]); + + print "Remove OSD $osdsection\n"; + &$run_ceph_cmd(['osd', 'rm', $osdid]); + + return undef; + }}); + +my $cmddef = { + init => [ __PACKAGE__, 'init', [] ], + createosd => [ __PACKAGE__, 'createosd', ['dev'] ], + destroyosd => [ __PACKAGE__, 'destroyosd', ['osdid'] ], + createmon => [ __PACKAGE__, 'createmon', [] ], + destroymon => [ __PACKAGE__, 'destroymon', ['monid'] ], + start => [ __PACKAGE__, 'start', [] ], + stop => [ __PACKAGE__, 'stop', [] ], + install => [ __PACKAGE__, 'install', [] ], + purge => [ __PACKAGE__, 'purge', [] ], + status => [ __PACKAGE__, 'status', [], undef, sub { + my $res = shift; + my $json = JSON->new->allow_nonref; + print $json->pretty->encode($res) . "\n"; + }], +}; + +my $cmd = shift; + +PVE::CLIHandler::handle_cmd($cmddef, "pveceph", $cmd, \@ARGV, undef, $0); + +exit 0; + +__END__ + +=head1 NAME + +pveceph - tool to manage ceph services on pve nodes + +=head1 SYNOPSIS + +=include synopsis + +=head1 DESCRIPTION + +Tool to manage ceph services on pve nodes. + +=include pve_copyright From 38db610ae668ba67ac3515a2d2e7443ceec8799f Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 13 Nov 2013 06:59:07 +0100 Subject: [PATCH 02/27] move code to Ceph.pm, create API for ceph --- PVE/API2/Ceph.pm | 702 ++++++++++++++++++++++++++++++++++++++++++++++ PVE/API2/Makefile | 1 + PVE/API2/Nodes.pm | 7 + bin/pveceph | 635 +---------------------------------------- 4 files changed, 722 insertions(+), 623 deletions(-) create mode 100644 PVE/API2/Ceph.pm diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm new file mode 100644 index 00000000..a16caac1 --- /dev/null +++ b/PVE/API2/Ceph.pm @@ -0,0 +1,702 @@ +package PVE::API2::Ceph; + +use strict; +use warnings; +use File::Basename; +use File::Path; +use POSIX qw (LONG_MAX); + +use PVE::SafeSyslog; +use PVE::Tools qw(extract_param run_command); +use PVE::Exception qw(raise raise_param_exc); +use PVE::INotify; +use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file); +use PVE::AccessControl; +use PVE::Storage; +use PVE::RESTHandler; +use PVE::RPCEnvironment; +use PVE::JSONSchema qw(get_standard_option); +use JSON; + +use base qw(PVE::RESTHandler); + +use Data::Dumper; # fixme: remove + +my $ccname = 'ceph'; # ceph cluster name +my $ceph_cfgdir = "/etc/ceph"; +my $pve_ceph_cfgpath = "/etc/pve/$ccname.conf"; +my $ceph_cfgpath = "$ceph_cfgdir/$ccname.conf"; +my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring"; +my $ceph_mon_key_path = "$ceph_cfgdir/$ccname.mon.keyring"; +my $pve_ckeyring_path = "/etc/pve/priv/$ccname.keyring"; +my $ceph_ckeyring_path = "$ceph_cfgdir/$ccname.client.admin.keyring"; + +my $ceph_bootstrap_osd_keyring = "/var/lib/ceph/bootstrap-osd/$ccname.keyring"; +my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring"; + +my $ceph_bin = "/usr/bin/ceph"; + +sub purge_all_ceph_files { + # fixme: this is very dangerous - should we really support this function? + + unlink $ceph_cfgpath; + unlink $ceph_mon_key_path; + unlink $ceph_ckeyring_path; + + unlink $pve_ceph_cfgpath; + unlink $pve_ckeyring_path; + unlink $pve_mon_key_path; + + unlink $ceph_bootstrap_osd_keyring; + unlink $ceph_bootstrap_mds_keyring; + + system("rm -rf /var/lib/ceph/mon/ceph-*"); + + # remove osd? + +} + +my $check_ceph_installed = sub { + my ($noerr) = @_; + + if (! -x $ceph_bin) { + die "ceph binaries not installed\n" if !$noerr; + return undef; + } + + return 1; +}; + +my $check_ceph_inited = sub { + my ($noerr) = @_; + + return undef if !&$check_ceph_installed($noerr); + + if (! -f $pve_ceph_cfgpath) { + die "pveceph configuration not initialized\n" if !$noerr; + return undef; + } + + return 1; +}; + +my $force_symlink = sub { + my ($old, $new) = @_; + + return if (-l $new) && (readlink($new) eq $old); + + unlink $new; + symlink($old, $new) || + die "unable to create symlink '$new' - $!\n"; +}; + +my $parse_ceph_config = sub { + my ($filename) = @_; + + my $cfg = {}; + + return $cfg if ! -f $filename; + + my $fh = IO::File->new($filename, "r") || + die "unable to open '$filename' - $!\n"; + + my $section; + + while (defined(my $line = <$fh>)) { + $line =~ s/[;#].*$//; + $line =~ s/^\s+//; + $line =~ s/\s+$//; + next if !$line; + + $section = $1 if $line =~ m/^\[(\S+)\]$/; + if (!$section) { + warn "no section - skip: $line\n"; + next; + } + + if ($line =~ m/^(.*\S)\s*=\s*(\S.*)$/) { + $cfg->{$section}->{$1} = $2; + } + + } + + return $cfg; +}; + +my $run_ceph_cmd = sub { + my ($cmd, %params) = @_; + + my $timeout = 3; + + my $oldalarm; + eval { + local $SIG{ALRM} = sub { die "timeout\n" }; + $oldalarm = alarm($timeout); + # Note: --connect-timeout does not work with current version + # '--connect-timeout', $timeout, + + run_command(['ceph', '-c', $ceph_cfgpath, @$cmd], %params); + alarm(0); + }; + my $err = $@; + + alarm($oldalarm) if $oldalarm; + + die $err if $err; + +}; + +sub ceph_mon_status { + my ($quiet) = @_; + + my $json = ''; + my $parser = sub { + my $line = shift; + $json .= $line; + }; + + my $errfunc = sub { + my $line = shift; + print "$line\n" if !$quiet; + }; + + &$run_ceph_cmd(['mon_status'], outfunc => $parser, errfunc => $errfunc); + + my $res = decode_json($json); + + return $res; +} + +my $ceph_osd_status = sub { + my ($quiet) = @_; + + my $json = ''; + my $parser = sub { + my $line = shift; + $json .= $line; + }; + + my $errfunc = sub { + my $line = shift; + print "$line\n" if !$quiet; + }; + + &$run_ceph_cmd(['osd', 'dump', '--format', 'json'], + outfunc => $parser, errfunc => $errfunc); + + my $res = decode_json($json); + + return $res; +}; + +my $write_ceph_config = sub { + my ($cfg) = @_; + + my $out = ''; + foreach my $section (keys %$cfg) { + $out .= "[$section]\n"; + foreach my $key (sort keys %{$cfg->{$section}}) { + $out .= "\t $key = $cfg->{$section}->{$key}\n"; + } + $out .= "\n"; + } + + PVE::Tools::file_set_contents($pve_ceph_cfgpath, $out); +}; + +my $setup_pve_symlinks = sub { + # fail if we find a real file instead of a link + if (-f $ceph_cfgpath) { + my $lnk = readlink($ceph_cfgpath); + die "file '$ceph_cfgpath' already exists\n" + if !$lnk || $lnk ne $pve_ceph_cfgpath; + } + + # now assume we are allowed to setup/overwrite content + &$force_symlink($pve_ceph_cfgpath, $ceph_cfgpath); + &$force_symlink($pve_mon_key_path, $ceph_mon_key_path); + &$force_symlink($pve_ckeyring_path, $ceph_ckeyring_path); +}; + +my $ceph_service_cmd = sub { + run_command(['service', 'ceph', '-c', $ceph_cfgpath, @_]); +}; + +__PACKAGE__->register_method ({ + name => 'index', + path => '', + method => 'GET', + description => "Directory index.", + permissions => { user => 'all' }, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + }, + }, + returns => { + type => 'array', + items => { + type => "object", + properties => {}, + }, + links => [ { rel => 'child', href => "{name}" } ], + }, + code => sub { + my ($param) = @_; + + my $result = [ + { name => 'init' }, + { name => 'createmon' }, + { name => 'destroymon' }, + { name => 'createosd' }, + { name => 'destroyosd' }, + { name => 'stop' }, + { name => 'start' }, + { name => 'status' }, + ]; + + return $result; + }}); + +__PACKAGE__->register_method ({ + name => 'init', + path => 'init', + method => 'POST', + description => "Create initial ceph configuration.", + proxyto => 'node', + protected => 1, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + size => { + description => 'Number of replicas per object', + type => 'integer', + default => 2, + optional => 1, + minimum => 1, + maximum => 3, + }, + pg_bits => { + description => "Placement group bits, used to specify the default number of placement groups (Note: 'osd pool default pg num' does not work for deafult pools)", + type => 'integer', + default => 9, + optional => 1, + minimum => 6, + maximum => 14, + }, + }, + }, + returns => { type => 'null' }, + code => sub { + my ($param) = @_; + + &$check_ceph_installed(); + + -f $pve_ceph_cfgpath && + die "configuration file '$pve_ceph_cfgpath' already exists.\n"; + + my $pg_bits = $param->{pg_bits} || 9; + my $size = $param->{size} || 2; + + my $global = { + 'auth supported' => 'cephx', + 'auth cluster required' => 'cephx', + 'auth service required' => 'cephx', + 'auth client required' => 'cephx', + 'filestore xattr use omap' => 'true', + 'osd journal size' => '1024', + 'osd pool default size' => $size, + 'osd pool default min size' => 1, + 'osd pg bits' => $pg_bits, + 'osd pgp bits' => $pg_bits, + }; + + # this does not work for default pools + #'osd pool default pg num' => $pg_num, + #'osd pool default pgp num' => $pg_num, + + &$write_ceph_config({global => $global}); + + &$setup_pve_symlinks(); + + return undef; + }}); + +__PACKAGE__->register_method ({ + name => 'createmon', + path => 'createmon', + method => 'POST', + description => "Create Ceph Monitor", + proxyto => 'node', + protected => 1, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + }, + }, + returns => { type => 'null' }, + code => sub { + my ($param) = @_; + + &$check_ceph_inited(); + + &$setup_pve_symlinks(); + + if (! -f $pve_ckeyring_path) { + run_command("ceph-authtool $pve_ckeyring_path --create-keyring " . + "--gen-key -n client.admin"); + } + + if (! -f $pve_mon_key_path) { + run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp"); + run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " . + "--cap mds 'allow *' " . + "--cap osd 'allow *' " . + "--cap mon 'allow *'"); + run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'"); + run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path"); + } + + + my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); + + my $moncount = 0; + + my $monaddrhash = {}; + + foreach my $section (keys %$cfg) { + next if $section eq 'global'; + my $d = $cfg->{$section}; + if ($section =~ m/^mon\./) { + $moncount++; + if ($d->{'mon addr'}) { + $monaddrhash->{$d->{'mon addr'}} = $section; + } + } + } + + my $monid; + for (my $i = 0; $i < 7; $i++) { + if (!$cfg->{"mon.$i"}) { + $monid = $i; + last; + } + } + die "unable to find usable monitor id\n" if !defined($monid); + + my $monsection = "mon.$monid"; + my $monaddr = PVE::Cluster::remote_node_ip($param->{node}) . ":6789"; + my $monname = $param->{node}; + + die "monitor '$monsection' already exists\n" if $cfg->{$monsection}; + die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n" + if $monaddrhash->{$monaddr}; + + + my $mondir = "/var/lib/ceph/mon/$ccname-$monid"; + -d $mondir && die "monitor filesystem '$mondir' already exist\n"; + + my $monmap = "/tmp/monmap"; + + eval { + mkdir $mondir; + + if ($moncount > 0) { + my $monstat = ceph_mon_status(); # online test + &$run_ceph_cmd(['mon', 'getmap', '-o', $monmap]); + } else { + run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap"); + } + + run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path"); + }; + my $err = $@; + unlink $monmap; + if ($err) { + File::Path::remove_tree($mondir); + die $err; + } + + $cfg->{$monsection} = { + 'host' => $monname, + 'mon addr' => $monaddr, + }; + + &$write_ceph_config($cfg); + + &$ceph_service_cmd('start', $monsection); + + return undef; + + }}); + +__PACKAGE__->register_method ({ + name => 'destroymon', + path => 'destroymon', + method => 'POST', + description => "Destroy Ceph monitor.", + proxyto => 'node', + protected => 1, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + monid => { + description => 'Monitor ID', + type => 'integer', + }, + }, + }, + returns => { type => 'null' }, + code => sub { + my ($param) = @_; + + &$check_ceph_inited(); + + my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); + + my $monid = $param->{monid}; + my $monsection = "mon.$monid"; + + my $monstat = ceph_mon_status(); + my $monlist = $monstat->{monmap}->{mons}; + + die "no such monitor id '$monid'\n" + if !defined($cfg->{$monsection}); + + + my $mondir = "/var/lib/ceph/mon/$ccname-$monid"; + -d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n"; + + die "can't remove last monitor\n" if scalar(@$monlist) <= 1; + + &$run_ceph_cmd(['mon', 'remove', $monid]); + + eval { &$ceph_service_cmd('stop', $monsection); }; + warn $@ if $@; + + delete $cfg->{$monsection}; + &$write_ceph_config($cfg); + File::Path::remove_tree($mondir); + + return undef; + }}); + +__PACKAGE__->register_method ({ + name => 'stop', + path => 'stop', + method => 'POST', + description => "Stop ceph services.", + proxyto => 'node', + protected => 1, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + }, + }, + returns => { type => 'null' }, + code => sub { + my ($param) = @_; + + &$check_ceph_inited(); + + my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); + scalar(keys %$cfg) || die "no configuration\n"; + + &$ceph_service_cmd('stop'); + + return undef; + }}); + +__PACKAGE__->register_method ({ + name => 'start', + path => 'start', + method => 'POST', + description => "Start ceph services.", + proxyto => 'node', + protected => 1, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + }, + }, + returns => { type => 'null' }, + code => sub { + my ($param) = @_; + + &$check_ceph_inited(); + + my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); + scalar(keys %$cfg) || die "no configuration\n"; + + &$ceph_service_cmd('start'); + + return undef; + }}); + +__PACKAGE__->register_method ({ + name => 'status', + path => 'status', + method => 'GET', + description => "Get ceph status.", + proxyto => 'node', + protected => 1, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + }, + }, + returns => { type => 'object' }, + code => sub { + my ($param) = @_; + + my $res = { status => 'unknown' }; + + eval { + if (!&$check_ceph_installed(1)) { + $res->{status} = 'notinstalled'; + } + if (! -f $ceph_cfgpath) { + $res->{status} = 'notconfigured'; + return; + } else { + $res->{status} = 'configured'; + } + + my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); + $res->{config} = $cfg; + + eval { + my $monstat = ceph_mon_status(1); + $res->{monstat} = $monstat; + }; + warn $@ if $@; + + eval { + my $osdstat = &$ceph_osd_status(1); + $res->{osdstat} = $osdstat; + }; + warn $@ if $@; + + }; + warn $@ if $@; + + return $res; + }}); + +__PACKAGE__->register_method ({ + name => 'createosd', + path => 'createosd', + method => 'POST', + description => "Create OSD", + proxyto => 'node', + protected => 1, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + dev => { + description => "Block device name.", + type => 'string', + } + }, + }, + returns => { type => 'null' }, + code => sub { + my ($param) = @_; + + &$check_ceph_inited(); + + die "not fully configured - missing '$pve_ckeyring_path'\n" + if ! -f $pve_ckeyring_path; + + &$setup_pve_symlinks(); + + print "create OSD on $param->{dev}\n"; + + -b $param->{dev} || die "no such block device '$param->{dev}'\n"; + + my $monstat = ceph_mon_status(1); + die "unable to get fsid\n" if !$monstat->{monmap} || !$monstat->{monmap}->{fsid}; + my $fsid = $monstat->{monmap}->{fsid}; + + if (! -f $ceph_bootstrap_osd_keyring) { + &$run_ceph_cmd(['auth', 'get', 'client.bootstrap-osd', '-o', $ceph_bootstrap_osd_keyring]); + }; + + run_command(['ceph-disk', 'prepare', '--zap-disk', '--fs-type', 'xfs', + '--cluster', $ccname, '--cluster-uuid', $fsid, + '--', $param->{dev}]); + + return undef; + }}); + +__PACKAGE__->register_method ({ + name => 'destroyosd', + path => 'destroyosd', + method => 'POST', + description => "Destroy OSD", + proxyto => 'node', + protected => 1, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + osdid => { + description => 'OSD ID', + type => 'integer', + }, + }, + }, + returns => { type => 'null' }, + code => sub { + my ($param) = @_; + + &$check_ceph_inited(); + + my $osdid = $param->{osdid}; + + print "destroy OSD $param->{osdid}\n"; + + # fixme: not sure what we should do here + + my $stat = &$ceph_osd_status(); + + my $osdlist = $stat->{osds} || []; + + my $osdstat; + foreach my $d (@$osdlist) { + if ($d->{osd} == $osdid) { + $osdstat = $d; + last; + } + } + die "no such OSD '$osdid'\n" if !$osdstat; + + die "osd is in use (in == 1)\n" if $osdstat->{in}; + #&$run_ceph_cmd(['osd', 'out', $osdid]); + + die "osd is still runnung (up == 1)\n" if $osdstat->{up}; + + my $osdsection = "osd.$osdid"; + + eval { &$ceph_service_cmd('stop', $osdsection); }; + warn $@ if $@; + + print "Remove $osdsection from the CRUSH map\n"; + &$run_ceph_cmd(['osd', 'crush', 'remove', $osdid]); + + print "Remove the $osdsection authentication key.\n"; + &$run_ceph_cmd(['auth', 'del', $osdsection]); + + print "Remove OSD $osdsection\n"; + &$run_ceph_cmd(['osd', 'rm', $osdid]); + + return undef; + }}); diff --git a/PVE/API2/Makefile b/PVE/API2/Makefile index bd979182..dfc2d387 100644 --- a/PVE/API2/Makefile +++ b/PVE/API2/Makefile @@ -1,6 +1,7 @@ include ../../defines.mk PERLSOURCE = \ + Ceph.pm \ APT.pm \ Subscription.pm \ VZDump.pm \ diff --git a/PVE/API2/Nodes.pm b/PVE/API2/Nodes.pm index 4cfcfd25..1f8f8ace 100644 --- a/PVE/API2/Nodes.pm +++ b/PVE/API2/Nodes.pm @@ -30,6 +30,7 @@ use PVE::API2::Qemu; use PVE::API2::OpenVZ; use PVE::API2::VZDump; use PVE::API2::APT; +use PVE::API2::Ceph; use JSON; use base qw(PVE::RESTHandler); @@ -39,6 +40,11 @@ __PACKAGE__->register_method ({ path => 'qemu', }); +__PACKAGE__->register_method ({ + subclass => "PVE::API2::Ceph", + path => 'ceph', +}); + __PACKAGE__->register_method ({ subclass => "PVE::API2::OpenVZ", path => 'openvz', @@ -108,6 +114,7 @@ __PACKAGE__->register_method ({ my ($param) = @_; my $result = [ + { name => 'ceph' }, { name => 'apt' }, { name => 'version' }, { name => 'syslog' }, diff --git a/bin/pveceph b/bin/pveceph index cf08a92a..d138ef95 100755 --- a/bin/pveceph +++ b/bin/pveceph @@ -16,6 +16,7 @@ use PVE::RPCEnvironment; use PVE::Storage; use PVE::Tools qw(run_command); use PVE::JSONSchema qw(get_standard_option); +use PVE::API2::Ceph; use PVE::CLIHandler; @@ -37,403 +38,6 @@ $rpcenv->set_user('root@pam'); my $nodename = PVE::INotify::nodename(); -my $ccname = 'ceph'; # ceph cluster name -my $ceph_cfgdir = "/etc/ceph"; -my $pve_ceph_cfgpath = "/etc/pve/$ccname.conf"; -my $ceph_cfgpath = "$ceph_cfgdir/$ccname.conf"; -my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring"; -my $ceph_mon_key_path = "$ceph_cfgdir/$ccname.mon.keyring"; -my $pve_ckeyring_path = "/etc/pve/priv/$ccname.keyring"; -my $ceph_ckeyring_path = "$ceph_cfgdir/$ccname.client.admin.keyring"; - -my $ceph_bootstrap_osd_keyring = "/var/lib/ceph/bootstrap-osd/$ccname.keyring"; -my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring"; - -my $ceph_bin = "/usr/bin/ceph"; - -my $check_ceph_installed = sub { - my ($noerr) = @_; - - if (! -x $ceph_bin) { - die "ceph binaries not installed\n" if !$noerr; - return undef; - } - - return 1; -}; - -my $check_ceph_inited = sub { - my ($noerr) = @_; - - return undef if !&$check_ceph_installed($noerr); - - if (! -f $pve_ceph_cfgpath) { - die "pveceph configuration not initialized\n" if !$noerr; - return undef; - } - - return 1; -}; - -my $force_symlink = sub { - my ($old, $new) = @_; - - return if (-l $new) && (readlink($new) eq $old); - - unlink $new; - symlink($old, $new) || - die "unable to create symlink '$new' - $!\n"; -}; - -my $parse_ceph_config = sub { - my ($filename) = @_; - - my $cfg = {}; - - return $cfg if ! -f $filename; - - my $fh = IO::File->new($filename, "r") || - die "unable to open '$filename' - $!\n"; - - my $section; - - while (defined(my $line = <$fh>)) { - $line =~ s/[;#].*$//; - $line =~ s/^\s+//; - $line =~ s/\s+$//; - next if !$line; - - $section = $1 if $line =~ m/^\[(\S+)\]$/; - if (!$section) { - warn "no section - skip: $line\n"; - next; - } - - if ($line =~ m/^(.*\S)\s*=\s*(\S.*)$/) { - $cfg->{$section}->{$1} = $2; - } - - } - - return $cfg; -}; - -my $run_ceph_cmd = sub { - my ($cmd, %params) = @_; - - my $timeout = 3; - - my $oldalarm; - eval { - local $SIG{ALRM} = sub { die "timeout\n" }; - $oldalarm = alarm($timeout); - # Note: --connect-timeout does not work with current version - # '--connect-timeout', $timeout, - - run_command(['ceph', '-c', $ceph_cfgpath, @$cmd], %params); - alarm(0); - }; - my $err = $@; - - alarm($oldalarm) if $oldalarm; - - die $err if $err; - -}; - -my $ceph_mon_status = sub { - my ($quiet) = @_; - - my $json = ''; - my $parser = sub { - my $line = shift; - $json .= $line; - }; - - my $errfunc = sub { - my $line = shift; - print "$line\n" if !$quiet; - }; - - &$run_ceph_cmd(['mon_status'], outfunc => $parser, errfunc => $errfunc); - - my $res = decode_json($json); - - return $res; -}; - -my $ceph_osd_status = sub { - my ($quiet) = @_; - - my $json = ''; - my $parser = sub { - my $line = shift; - $json .= $line; - }; - - my $errfunc = sub { - my $line = shift; - print "$line\n" if !$quiet; - }; - - &$run_ceph_cmd(['osd', 'dump', '--format', 'json'], - outfunc => $parser, errfunc => $errfunc); - - my $res = decode_json($json); - - return $res; -}; - -my $write_ceph_config = sub { - my ($cfg) = @_; - - my $out = ''; - foreach my $section (keys %$cfg) { - $out .= "[$section]\n"; - foreach my $key (sort keys %{$cfg->{$section}}) { - $out .= "\t $key = $cfg->{$section}->{$key}\n"; - } - $out .= "\n"; - } - - PVE::Tools::file_set_contents($pve_ceph_cfgpath, $out); -}; - -my $setup_pve_symlinks = sub { - # fail if we find a real file instead of a link - if (-f $ceph_cfgpath) { - my $lnk = readlink($ceph_cfgpath); - die "file '$ceph_cfgpath' already exists\n" - if !$lnk || $lnk ne $pve_ceph_cfgpath; - } - - # now assume we are allowed to setup/overwrite content - &$force_symlink($pve_ceph_cfgpath, $ceph_cfgpath); - &$force_symlink($pve_mon_key_path, $ceph_mon_key_path); - &$force_symlink($pve_ckeyring_path, $ceph_ckeyring_path); -}; - -my $ceph_service_cmd = sub { - run_command(['service', 'ceph', '-c', $ceph_cfgpath, @_]); -}; - -__PACKAGE__->register_method ({ - name => 'init', - path => 'init', - method => 'POST', - description => "Create initial ceph configuration.", - parameters => { - additionalProperties => 0, - properties => { - size => { - description => 'Number of replicas per object', - type => 'interger', - default => 2, - optional => 1, - minimum => 1, - maximum => 3, - }, - pg_bits => { - description => "Placement group bits, used to specify the default number of placement groups (Note: 'osd pool default pg num' does not work for deafult pools)", - type => 'interger', - default => 9, - optional => 1, - minimum => 6, - maximum => 14, - }, - }, - }, - returns => { type => 'null' }, - code => sub { - my ($param) = @_; - - &$check_ceph_installed(); - - -f $pve_ceph_cfgpath && - die "configuration file '$pve_ceph_cfgpath' already exists.\n"; - - my $pg_bits = $param->{pg_bits} || 9; - my $size = $param->{size} || 2; - - my $global = { - 'auth supported' => 'cephx', - 'auth cluster required' => 'cephx', - 'auth service required' => 'cephx', - 'auth client required' => 'cephx', - 'filestore xattr use omap' => 'true', - 'osd journal size' => '1024', - 'osd pool default size' => $size, - 'osd pool default min size' => 1, - 'osd pg bits' => $pg_bits, - 'osd pgp bits' => $pg_bits, - }; - - # this does not work for default pools - #'osd pool default pg num' => $pg_num, - #'osd pool default pgp num' => $pg_num, - - &$write_ceph_config({global => $global}); - - &$setup_pve_symlinks(); - - return undef; - }}); - -__PACKAGE__->register_method ({ - name => 'createmon', - path => 'createmon', - method => 'POST', - description => "Create Ceph Monitor", - parameters => { - additionalProperties => 0, - properties => { - }, - }, - returns => { type => 'null' }, - code => sub { - my ($param) = @_; - - &$check_ceph_inited(); - - &$setup_pve_symlinks(); - - if (! -f $pve_ckeyring_path) { - run_command("ceph-authtool $pve_ckeyring_path --create-keyring " . - "--gen-key -n client.admin"); - } - - if (! -f $pve_mon_key_path) { - run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp"); - run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " . - "--cap mds 'allow *' " . - "--cap osd 'allow *' " . - "--cap mon 'allow *'"); - run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'"); - run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path"); - } - - - my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); - - my $moncount = 0; - - my $monaddrhash = {}; - - foreach my $section (keys %$cfg) { - next if $section eq 'global'; - my $d = $cfg->{$section}; - if ($section =~ m/^mon\./) { - $moncount++; - if ($d->{'mon addr'}) { - $monaddrhash->{$d->{'mon addr'}} = $section; - } - } - } - - my $monid; - for (my $i = 0; $i < 7; $i++) { - if (!$cfg->{"mon.$i"}) { - $monid = $i; - last; - } - } - die "unable to find usable monitor id\n" if !defined($monid); - - my $monsection = "mon.$monid"; - my $monaddr = PVE::Cluster::remote_node_ip($nodename) . ":6789"; - my $monname = $nodename; - - die "monitor '$monsection' already exists\n" if $cfg->{$monsection}; - die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n" - if $monaddrhash->{$monaddr}; - - - my $mondir = "/var/lib/ceph/mon/$ccname-$monid"; - -d $mondir && die "monitor filesystem '$mondir' already exist\n"; - - my $monmap = "/tmp/monmap"; - - eval { - mkdir $mondir; - - if ($moncount > 0) { - my $monstat = &$ceph_mon_status(); # online test - &$run_ceph_cmd(['mon', 'getmap', '-o', $monmap]); - } else { - run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap"); - } - - run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path"); - }; - my $err = $@; - unlink $monmap; - if ($err) { - File::Path::remove_tree($mondir); - die $err; - } - - $cfg->{$monsection} = { - 'host' => $monname, - 'mon addr' => $monaddr, - }; - - &$write_ceph_config($cfg); - - &$ceph_service_cmd('start', $monsection); - - return undef; - - }}); - -__PACKAGE__->register_method ({ - name => 'destroymon', - path => 'destroymon', - method => 'POST', - description => "Destroy Ceph monitor.", - parameters => { - additionalProperties => 0, - properties => { - monid => { - destription => 'Monitor ID', - type => 'integer', - }, - }, - }, - returns => { type => 'null' }, - code => sub { - my ($param) = @_; - - &$check_ceph_inited(); - - my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); - - my $monid = $param->{monid}; - my $monsection = "mon.$monid"; - - my $monstat = &$ceph_mon_status(); - my $monlist = $monstat->{monmap}->{mons}; - - die "no such monitor id '$monid'\n" - if !defined($cfg->{$monsection}); - - - my $mondir = "/var/lib/ceph/mon/$ccname-$monid"; - -d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n"; - - die "can't remove last monitor\n" if scalar(@$monlist) <= 1; - - &$run_ceph_cmd(['mon', 'remove', $monid]); - - eval { &$ceph_service_cmd('stop', $monsection); }; - warn $@ if $@; - - delete $cfg->{$monsection}; - &$write_ceph_config($cfg); - File::Path::remove_tree($mondir); - - return undef; - }}); - __PACKAGE__->register_method ({ name => 'purge', path => 'purge', @@ -450,28 +54,14 @@ __PACKAGE__->register_method ({ my $monstat; - eval { $monstat = &$ceph_mon_status(1); }; + eval { $monstat = PVE::API2::Ceph::ceph_mon_status(1); }; my $err = $@; die "detected running ceph services- unable to purge data\n" if !$err; # fixme: this is dangerous - should we really support this function? - - unlink $ceph_cfgpath; - unlink $ceph_mon_key_path; - unlink $ceph_ckeyring_path; - - unlink $pve_ceph_cfgpath; - unlink $pve_ckeyring_path; - unlink $pve_mon_key_path; - - unlink $ceph_bootstrap_osd_keyring; - unlink $ceph_bootstrap_mds_keyring; - - system("rm -rf /var/lib/ceph/mon/ceph-*"); - - # remove osd? + PVE::API2::Ceph::purge_all_ceph_files(); return undef; }}); @@ -516,218 +106,17 @@ __PACKAGE__->register_method ({ return undef; }}); -__PACKAGE__->register_method ({ - name => 'stop', - path => 'stop', - method => 'POST', - description => "Stop ceph services.", - parameters => { - additionalProperties => 0, - properties => { - }, - }, - returns => { type => 'null' }, - code => sub { - my ($param) = @_; - - &$check_ceph_inited(); - - my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); - scalar(keys %$cfg) || die "no configuration\n"; - - &$ceph_service_cmd('stop'); - - return undef; - }}); - -__PACKAGE__->register_method ({ - name => 'start', - path => 'start', - method => 'POST', - description => "Start ceph services.", - parameters => { - additionalProperties => 0, - properties => { - }, - }, - returns => { type => 'null' }, - code => sub { - my ($param) = @_; - - &$check_ceph_inited(); - - my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); - scalar(keys %$cfg) || die "no configuration\n"; - - &$ceph_service_cmd('start'); - - return undef; - }}); - -__PACKAGE__->register_method ({ - name => 'status', - path => 'status', - method => 'GET', - description => "Get ceph status.", - parameters => { - additionalProperties => 0, - properties => { - }, - }, - returns => { type => 'object' }, - code => sub { - my ($param) = @_; - - my $res = { status => 'unknown' }; - - eval { - if (!&$check_ceph_installed(1)) { - $res->{status} = 'notinstalled'; - } - if (! -f $ceph_cfgpath) { - $res->{status} = 'notconfigured'; - return; - } else { - $res->{status} = 'configured'; - } - - my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); - $res->{config} = $cfg; - - eval { - my $monstat = &$ceph_mon_status(1); - $res->{monstat} = $monstat; - }; - warn $@ if $@; - - eval { - my $osdstat = &$ceph_osd_status(1); - $res->{osdstat} = $osdstat; - }; - warn $@ if $@; - - }; - warn $@ if $@; - - return $res; - }}); - -__PACKAGE__->register_method ({ - name => 'createosd', - path => 'createosd', - method => 'POST', - description => "Create OSD", - parameters => { - additionalProperties => 0, - properties => { - dev => { - description => "Block device name.", - type => 'string', - } - }, - }, - returns => { type => 'null' }, - code => sub { - my ($param) = @_; - - &$check_ceph_inited(); - - die "not fully configured - missing '$pve_ckeyring_path'\n" - if ! -f $pve_ckeyring_path; - - &$setup_pve_symlinks(); - - print "create OSD on $param->{dev}\n"; - - -b $param->{dev} || die "no such block device '$param->{dev}'\n"; - - my $monstat = &$ceph_mon_status(1); - die "unable to get fsid\n" if !$monstat->{monmap} || !$monstat->{monmap}->{fsid}; - my $fsid = $monstat->{monmap}->{fsid}; - - if (! -f $ceph_bootstrap_osd_keyring) { - &$run_ceph_cmd(['auth', 'get', 'client.bootstrap-osd', '-o', $ceph_bootstrap_osd_keyring]); - }; - - run_command(['ceph-disk', 'prepare', '--zap-disk', '--fs-type', 'xfs', - '--cluster', $ccname, '--cluster-uuid', $fsid, - '--', $param->{dev}]); - - return undef; - }}); - -__PACKAGE__->register_method ({ - name => 'destroyosd', - path => 'destroyosd', - method => 'POST', - description => "Destroy OSD", - parameters => { - additionalProperties => 0, - properties => { - osdid => { - destription => 'OSD ID', - type => 'integer', - }, - }, - }, - returns => { type => 'null' }, - code => sub { - my ($param) = @_; - - &$check_ceph_inited(); - - my $osdid = $param->{osdid}; - - print "destroy OSD $param->{osdid}\n"; - - # fixme: not sure what we should do here - - my $stat = &$ceph_osd_status(); - - my $osdlist = $stat->{osds} || []; - - my $osdstat; - foreach my $d (@$osdlist) { - if ($d->{osd} == $osdid) { - $osdstat = $d; - last; - } - } - die "no such OSD '$osdid'\n" if !$osdstat; - - die "osd is in use (in == 1)\n" if $osdstat->{in}; - #&$run_ceph_cmd(['osd', 'out', $osdid]); - - die "osd is still runnung (up == 1)\n" if $osdstat->{up}; - - my $osdsection = "osd.$osdid"; - - eval { &$ceph_service_cmd('stop', $osdsection); }; - warn $@ if $@; - - print "Remove $osdsection from the CRUSH map\n"; - &$run_ceph_cmd(['osd', 'crush', 'remove', $osdid]); - - print "Remove the $osdsection authentication key.\n"; - &$run_ceph_cmd(['auth', 'del', $osdsection]); - - print "Remove OSD $osdsection\n"; - &$run_ceph_cmd(['osd', 'rm', $osdid]); - - return undef; - }}); - my $cmddef = { - init => [ __PACKAGE__, 'init', [] ], - createosd => [ __PACKAGE__, 'createosd', ['dev'] ], - destroyosd => [ __PACKAGE__, 'destroyosd', ['osdid'] ], - createmon => [ __PACKAGE__, 'createmon', [] ], - destroymon => [ __PACKAGE__, 'destroymon', ['monid'] ], - start => [ __PACKAGE__, 'start', [] ], - stop => [ __PACKAGE__, 'stop', [] ], - install => [ __PACKAGE__, 'install', [] ], + init => [ 'PVE::API2::Ceph', 'init', [] ], + createosd => [ 'PVE::API2::Ceph', 'createosd', ['dev'] ], + destroyosd => [ 'PVE::API2::Ceph', 'destroyosd', ['osdid'] ], + createmon => [ 'PVE::API2::Ceph', 'createmon', [] ], + destroymon => [ 'PVE::API2::Ceph', 'destroymon', ['monid'] ], + start => [ 'PVE::API2::Ceph', 'start', [] ], + stop => [ 'PVE::API2::Ceph', 'stop', [] ], + install => [ __PACKAGE__, 'install', [] ], purge => [ __PACKAGE__, 'purge', [] ], - status => [ __PACKAGE__, 'status', [], undef, sub { + status => [ 'PVE::API2::Ceph', 'status', [], undef, sub { my $res = shift; my $json = JSON->new->allow_nonref; print $json->pretty->encode($res) . "\n"; From f016ec966cea15035c85581796c8b4c6bedd0cc7 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 13 Nov 2013 07:07:19 +0100 Subject: [PATCH 03/27] pass required node name parameter --- bin/pveceph | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/bin/pveceph b/bin/pveceph index d138ef95..f6dc5489 100755 --- a/bin/pveceph +++ b/bin/pveceph @@ -107,16 +107,16 @@ __PACKAGE__->register_method ({ }}); my $cmddef = { - init => [ 'PVE::API2::Ceph', 'init', [] ], - createosd => [ 'PVE::API2::Ceph', 'createosd', ['dev'] ], - destroyosd => [ 'PVE::API2::Ceph', 'destroyosd', ['osdid'] ], - createmon => [ 'PVE::API2::Ceph', 'createmon', [] ], - destroymon => [ 'PVE::API2::Ceph', 'destroymon', ['monid'] ], - start => [ 'PVE::API2::Ceph', 'start', [] ], - stop => [ 'PVE::API2::Ceph', 'stop', [] ], + init => [ 'PVE::API2::Ceph', 'init', [], { node => $nodename } ], + createosd => [ 'PVE::API2::Ceph', 'createosd', ['dev'], { node => $nodename } ], + destroyosd => [ 'PVE::API2::Ceph', 'destroyosd', ['osdid'], { node => $nodename } ], + createmon => [ 'PVE::API2::Ceph', 'createmon', [], { node => $nodename } ], + destroymon => [ 'PVE::API2::Ceph', 'destroymon', ['monid'], { node => $nodename } ], + start => [ 'PVE::API2::Ceph', 'start', [], { node => $nodename } ], + stop => [ 'PVE::API2::Ceph', 'stop', [], { node => $nodename } ], install => [ __PACKAGE__, 'install', [] ], purge => [ __PACKAGE__, 'purge', [] ], - status => [ 'PVE::API2::Ceph', 'status', [], undef, sub { + status => [ 'PVE::API2::Ceph', 'status', [], { node => $nodename }, sub { my $res = shift; my $json = JSON->new->allow_nonref; print $json->pretty->encode($res) . "\n"; From 9662c1952a5c1d2bbb38eb877ed6638d5ac372f5 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 13 Nov 2013 11:01:24 +0100 Subject: [PATCH 04/27] start ceph management GUI --- PVE/API2/Ceph.pm | 90 +++++++---------- www/manager/Makefile | 1 + www/manager/node/Ceph.js | 198 +++++++++++++++++++++++++++++++++++++ www/manager/node/Config.js | 6 ++ 4 files changed, 239 insertions(+), 56 deletions(-) create mode 100644 www/manager/node/Ceph.js diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm index a16caac1..07f4f246 100644 --- a/PVE/API2/Ceph.pm +++ b/PVE/API2/Ceph.pm @@ -80,6 +80,19 @@ my $check_ceph_inited = sub { return 1; }; +my $check_ceph_enabled = sub { + my ($noerr) = @_; + + return undef if !&$check_ceph_inited($noerr); + + if (! -f $ceph_cfgpath) { + die "pveceph configuration not enabled\n" if !$noerr; + return undef; + } + + return 1; +}; + my $force_symlink = sub { my ($old, $new) = @_; @@ -143,13 +156,15 @@ my $run_ceph_cmd = sub { alarm($oldalarm) if $oldalarm; die $err if $err; - }; -sub ceph_mon_status { - my ($quiet) = @_; +my $run_ceph_cmd_json = sub { + my ($cmd, %opts) = @_; my $json = ''; + + my $quiet = delete $opts{quiet}; + my $parser = sub { my $line = shift; $json .= $line; @@ -160,28 +175,7 @@ sub ceph_mon_status { print "$line\n" if !$quiet; }; - &$run_ceph_cmd(['mon_status'], outfunc => $parser, errfunc => $errfunc); - - my $res = decode_json($json); - - return $res; -} - -my $ceph_osd_status = sub { - my ($quiet) = @_; - - my $json = ''; - my $parser = sub { - my $line = shift; - $json .= $line; - }; - - my $errfunc = sub { - my $line = shift; - print "$line\n" if !$quiet; - }; - - &$run_ceph_cmd(['osd', 'dump', '--format', 'json'], + &$run_ceph_cmd([@$cmd, '--format', 'json'], outfunc => $parser, errfunc => $errfunc); my $res = decode_json($json); @@ -189,6 +183,19 @@ my $ceph_osd_status = sub { return $res; }; +sub ceph_mon_status { + my ($quiet) = @_; + + return &$run_ceph_cmd_json(['mon_status'], quiet => $quiet); + +} + +my $ceph_osd_status = sub { + my ($quiet) = @_; + + return &$run_ceph_cmd_json(['osd', 'dump'], quiet => $quiet); +}; + my $write_ceph_config = sub { my ($cfg) = @_; @@ -556,38 +563,9 @@ __PACKAGE__->register_method ({ code => sub { my ($param) = @_; - my $res = { status => 'unknown' }; + &$check_ceph_enabled(); - eval { - if (!&$check_ceph_installed(1)) { - $res->{status} = 'notinstalled'; - } - if (! -f $ceph_cfgpath) { - $res->{status} = 'notconfigured'; - return; - } else { - $res->{status} = 'configured'; - } - - my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); - $res->{config} = $cfg; - - eval { - my $monstat = ceph_mon_status(1); - $res->{monstat} = $monstat; - }; - warn $@ if $@; - - eval { - my $osdstat = &$ceph_osd_status(1); - $res->{osdstat} = $osdstat; - }; - warn $@ if $@; - - }; - warn $@ if $@; - - return $res; + return &$run_ceph_cmd_json(['status'], quiet => 1); }}); __PACKAGE__->register_method ({ diff --git a/www/manager/Makefile b/www/manager/Makefile index 46c8a328..4c73b8a6 100644 --- a/www/manager/Makefile +++ b/www/manager/Makefile @@ -85,6 +85,7 @@ JSSRC= \ node/Tasks.js \ node/Subscription.js \ node/APT.js \ + node/Ceph.js \ node/Config.js \ qemu/StatusView.js \ window/Migrate.js \ diff --git a/www/manager/node/Ceph.js b/www/manager/node/Ceph.js new file mode 100644 index 00000000..31959a5b --- /dev/null +++ b/www/manager/node/Ceph.js @@ -0,0 +1,198 @@ +Ext.define('PVE.node.CephStatus', { + extend: 'PVE.grid.ObjectGrid', + alias: 'widget.pveNodeCephStatus', + + initComponent: function() { + var me = this; + + var nodename = me.pveSelNode.data.node; + if (!nodename) { + throw "no node name specified"; + } + + var renderquorum = function(value) { + if (!value) { + return ''; + } + var txt = ''; + + Ext.Array.each(value, function(name) { + txt += name + ' '; + }); + + return txt; + }; + + var rendermonmap = function(d) { + if (!d) { + return ''; + } + + var txt = 'e' + d.epoch + ': ' + d.mons.length + " mons at "; + + Ext.Array.each(d.mons, function(d) { + txt += d.name + '=' + d.addr + ','; + }); + + return txt; + }; + + var renderosdmap = function(value) { + if (!value || !value.osdmap) { + return ''; + } + + var d = value.osdmap; + + var txt = 'e' + d.epoch + ': '; + + txt += d.num_osds + ' osds: ' + d.num_up_osds + ' up, ' + + d.num_in_osds + " in"; + + return txt; + }; + + var renderhealth = function(value) { + if (!value || !value.overall_status) { + return ''; + } + + var txt = value.overall_status; + Ext.Array.each(value.summary, function(d) { + txt += " " + d.summary + ';'; + }); + + return txt; + }; + + var renderpgmap = function(d) { + if (!d) { + return ''; + } + + var txt = 'v' + d.version + ': '; + + txt += d.num_pgs + " pgs:"; + + Ext.Array.each(d.pgs_by_state, function(s) { + txt += " " + s.count + " " + s.state_name; + }); + txt += '; '; + + txt += d.data_bytes + " bytes data, "; + txt += d.bytes_used + " bytes used, "; + txt += d.bytes_avail + " bytes avail"; + + return txt; + }; + + Ext.applyIf(me, { + url: "/api2/json/nodes/" + nodename + "/ceph/status", + cwidth1: 150, + interval: 3000, + rows: { + health: { + header: 'health', + renderer: renderhealth, + required: true + }, + fsid: { + header: 'cluster', + required: true + }, + monmap: { + header: 'monmap', + renderer: rendermonmap, + required: true + }, + quorum_names: { + header: 'quorum', + renderer: renderquorum, + required: true + }, + osdmap: { + header: 'osdmap', + renderer: renderosdmap, + required: true + }, + pgmap: { + header: 'pgmap', + renderer: renderpgmap, + required: true + } + } + }); + + me.callParent(); + + me.on('show', me.rstore.startUpdate); + me.on('hide', me.rstore.stopUpdate); + me.on('destroy', me.rstore.stopUpdate); + } +}); + +Ext.define('PVE.node.Ceph', { + extend: 'Ext.tab.Panel', + alias: 'widget.pveNodeCeph', + + initComponent: function() { + var me = this; + + var nodename = me.pveSelNode.data.node; + if (!nodename) { + throw "no node name specified"; + } + + Ext.apply(me, { + plain: true, + tabPosition: 'bottom', + defaults: { + border: false, + pveSelNode: me.pveSelNode + }, + items: [ + { + xtype: 'pveNodeCephStatus', + title: 'Status', + itemId: 'status' + }, + { + title: 'Config', + itemId: 'config', + html: "ABCD" + }, + { + title: 'Monitor', + itemId: 'test2', + html: "ABCD" + }, + { + title: 'OSD', + itemId: 'test3', + html: "ABCD" + }, + { + title: 'Pool', + itemId: 'test4', + html: "ABCD" + }, + { + title: 'Crush', + itemId: 'test5', + html: "ABCD" + } + ], + listeners: { + afterrender: function(tp) { + var first = tp.items.get(0); + if (first) { + first.fireEvent('show', first); + } + } + } + }); + + me.callParent(); + + } +}); \ No newline at end of file diff --git a/www/manager/node/Config.js b/www/manager/node/Config.js index d80c725d..fb18c2d4 100644 --- a/www/manager/node/Config.js +++ b/www/manager/node/Config.js @@ -154,6 +154,12 @@ Ext.define('PVE.node.Config', { xtype: 'pveNodeAPT', nodename: nodename }]); + me.items.push([{ + title: 'Ceph', + itemId: 'ceph', + xtype: 'pveNodeCeph', + nodename: nodename + }]); } me.callParent(); From 2f692121ce9a5e67c82b608c9f7aff7e8c9085b1 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 13 Nov 2013 12:16:20 +0100 Subject: [PATCH 05/27] display crush map --- PVE/API2/Ceph.pm | 46 ++++++++++++++++++++++++++++++------ www/manager/node/Ceph.js | 51 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 88 insertions(+), 9 deletions(-) diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm index 07f4f246..3d84142d 100644 --- a/PVE/API2/Ceph.pm +++ b/PVE/API2/Ceph.pm @@ -158,16 +158,16 @@ my $run_ceph_cmd = sub { die $err if $err; }; -my $run_ceph_cmd_json = sub { +my $run_ceph_cmd_text = sub { my ($cmd, %opts) = @_; - my $json = ''; + my $out = ''; my $quiet = delete $opts{quiet}; my $parser = sub { my $line = shift; - $json .= $line; + $out .= "$line\n"; }; my $errfunc = sub { @@ -175,12 +175,17 @@ my $run_ceph_cmd_json = sub { print "$line\n" if !$quiet; }; - &$run_ceph_cmd([@$cmd, '--format', 'json'], - outfunc => $parser, errfunc => $errfunc); + &$run_ceph_cmd($cmd, outfunc => $parser, errfunc => $errfunc); - my $res = decode_json($json); + return $out; +}; - return $res; +my $run_ceph_cmd_json = sub { + my ($cmd, %opts) = @_; + + my $json = &$run_ceph_cmd_text([@$cmd, '--format', 'json'], %opts); + + return decode_json($json); }; sub ceph_mon_status { @@ -261,6 +266,7 @@ __PACKAGE__->register_method ({ { name => 'stop' }, { name => 'start' }, { name => 'status' }, + { name => 'crush' }, ]; return $result; @@ -678,3 +684,29 @@ __PACKAGE__->register_method ({ return undef; }}); + +__PACKAGE__->register_method ({ + name => 'crush', + path => 'crush', + method => 'GET', + description => "Get OSD crush map", + proxyto => 'node', + protected => 1, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + }, + }, + returns => { type => 'string' }, + code => sub { + my ($param) = @_; + + &$check_ceph_inited(); + + my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1); + + return $txt; + }}); + + diff --git a/www/manager/node/Ceph.js b/www/manager/node/Ceph.js index 31959a5b..bb134034 100644 --- a/www/manager/node/Ceph.js +++ b/www/manager/node/Ceph.js @@ -1,3 +1,50 @@ +Ext.define('PVE.node.CephCrushMap', { + extend: 'Ext.panel.Panel', + alias: 'widget.pveNodeCephCrushMap', + + load: function() { + var me = this; + + PVE.Utils.API2Request({ + url: me.url, + waitMsgTarget: me, + failure: function(response, opts) { + me.update(gettext('Error') + " " + response.htmlStatus); + }, + success: function(response, opts) { + var data = response.result.data; + me.update(data); + } + }); + }, + + initComponent: function() { + var me = this; + + var nodename = me.pveSelNode.data.node; + if (!nodename) { + throw "no node name specified"; + } + + Ext.apply(me, { + url: '/api2/extjs/nodes/' + nodename + '/ceph/crush', + style: 'padding-left:10px', + bodyStyle: 'white-space:pre', + bodyPadding: 5, + autoScroll: true, + listeners: { + show: function() { + me.load(); + } + } + }); + + me.callParent(); + + me.load(); + } +}); + Ext.define('PVE.node.CephStatus', { extend: 'PVE.grid.ObjectGrid', alias: 'widget.pveNodeCephStatus', @@ -178,8 +225,8 @@ Ext.define('PVE.node.Ceph', { }, { title: 'Crush', - itemId: 'test5', - html: "ABCD" + xtype: 'pveNodeCephCrushMap', + itemId: 'crushmap' } ], listeners: { From 721214ac372890495dab7adf1d4141979ff58ee2 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 14 Nov 2013 06:09:46 +0100 Subject: [PATCH 06/27] correctly encode html --- www/manager/node/Ceph.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/manager/node/Ceph.js b/www/manager/node/Ceph.js index bb134034..2cbb38b7 100644 --- a/www/manager/node/Ceph.js +++ b/www/manager/node/Ceph.js @@ -13,7 +13,7 @@ Ext.define('PVE.node.CephCrushMap', { }, success: function(response, opts) { var data = response.result.data; - me.update(data); + me.update(Ext.htmlEncode(data)); } }); }, From 51f558d25b67ac4413c1e940dec7e7a7f8e96594 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 14 Nov 2013 06:20:59 +0100 Subject: [PATCH 07/27] improve quorum renderer --- www/manager/node/Ceph.js | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/www/manager/node/Ceph.js b/www/manager/node/Ceph.js index 2cbb38b7..59a20a59 100644 --- a/www/manager/node/Ceph.js +++ b/www/manager/node/Ceph.js @@ -58,16 +58,11 @@ Ext.define('PVE.node.CephStatus', { } var renderquorum = function(value) { - if (!value) { - return ''; + if (!value || value.length < 0) { + return 'No'; } - var txt = ''; - Ext.Array.each(value, function(name) { - txt += name + ' '; - }); - - return txt; + return 'Yes {' + value.join(' ') + '}'; }; var rendermonmap = function(d) { @@ -143,6 +138,11 @@ Ext.define('PVE.node.CephStatus', { renderer: renderhealth, required: true }, + quorum_names: { + header: 'quorum', + renderer: renderquorum, + required: true + }, fsid: { header: 'cluster', required: true @@ -152,11 +152,6 @@ Ext.define('PVE.node.CephStatus', { renderer: rendermonmap, required: true }, - quorum_names: { - header: 'quorum', - renderer: renderquorum, - required: true - }, osdmap: { header: 'osdmap', renderer: renderosdmap, From 68e0c4bd41a192027010c5a8fd6f7c4bdcd33abc Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 14 Nov 2013 10:15:36 +0100 Subject: [PATCH 08/27] add monitor GUI --- PVE/API2/Ceph.pm | 139 ++++++++++++++++++++++++++++++-- bin/pveceph | 12 ++- www/manager/node/Ceph.js | 170 +++++++++++++++++++++++++++++++++++++-- 3 files changed, 308 insertions(+), 13 deletions(-) diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm index 3d84142d..5ede3ff9 100644 --- a/PVE/API2/Ceph.pm +++ b/PVE/API2/Ceph.pm @@ -261,17 +261,104 @@ __PACKAGE__->register_method ({ { name => 'init' }, { name => 'createmon' }, { name => 'destroymon' }, + { name => 'listmon' }, { name => 'createosd' }, { name => 'destroyosd' }, { name => 'stop' }, { name => 'start' }, { name => 'status' }, { name => 'crush' }, + { name => 'config' }, ]; return $result; }}); +__PACKAGE__->register_method ({ + name => 'config', + path => 'config', + method => 'GET', + description => "Get Ceph configuration.", + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + }, + }, + returns => { type => 'string' }, + code => sub { + my ($param) = @_; + + &$check_ceph_inited(); + + return PVE::Tools::file_get_contents($pve_ceph_cfgpath); + + }}); + +__PACKAGE__->register_method ({ + name => 'listmon', + path => 'listmon', + method => 'GET', + description => "Get Ceph monitor list.", + proxyto => 'node', + protected => 1, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + }, + }, + returns => { + type => 'array', + items => { + type => "object", + properties => { + name => { type => 'string' }, + addr => { type => 'string' }, + }, + }, + }, + code => sub { + my ($param) = @_; + + &$check_ceph_inited(); + + my $res = []; + + my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); + + my $monhash = {}; + foreach my $section (keys %$cfg) { + my $d = $cfg->{$section}; + if ($section =~ m/^mon\.(\S+)$/) { + my $monid = $1; + if ($d->{'mon addr'} && $d->{'host'}) { + $monhash->{$monid} = { + addr => $d->{'mon addr'}, + host => $d->{'host'}, + name => $monid, + } + } + } + } + + eval { + my $monstat = ceph_mon_status(); + my $mons = $monstat->{monmap}->{mons}; + foreach my $d (@$mons) { + next if !defined($d->{name}); + $monhash->{$d->{name}}->{rank} = $d->{rank}; + $monhash->{$d->{name}}->{addr} = $d->{addr}; + if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) { + $monhash->{$d->{name}}->{quorum} = 1; + } + } + }; + warn $@ if $@; + + return PVE::RESTHandler::hash_to_array($monhash, 'name'); + }}); + __PACKAGE__->register_method ({ name => 'init', path => 'init', @@ -509,20 +596,40 @@ __PACKAGE__->register_method ({ additionalProperties => 0, properties => { node => get_standard_option('pve-node'), + service => { + description => 'Ceph service name.', + type => 'string', + optional => 1, + pattern => '(mon|mds|osd)\.[A-Za-z0-9]{1,32}', + }, }, }, - returns => { type => 'null' }, + returns => { type => 'string' }, code => sub { my ($param) = @_; + my $rpcenv = PVE::RPCEnvironment::get(); + + my $authuser = $rpcenv->get_user(); + &$check_ceph_inited(); my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); scalar(keys %$cfg) || die "no configuration\n"; - &$ceph_service_cmd('stop'); + my $worker = sub { + my $upid = shift; - return undef; + my $cmd = ['stop']; + if ($param->{service}) { + push @$cmd, $param->{service}; + } + + &$ceph_service_cmd(@$cmd); + }; + + return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph', + $authuser, $worker); }}); __PACKAGE__->register_method ({ @@ -536,20 +643,40 @@ __PACKAGE__->register_method ({ additionalProperties => 0, properties => { node => get_standard_option('pve-node'), + service => { + description => 'Ceph service name.', + type => 'string', + optional => 1, + pattern => '(mon|mds|osd)\.[A-Za-z0-9]{1,32}', + }, }, }, - returns => { type => 'null' }, + returns => { type => 'string' }, code => sub { my ($param) = @_; + my $rpcenv = PVE::RPCEnvironment::get(); + + my $authuser = $rpcenv->get_user(); + &$check_ceph_inited(); my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); scalar(keys %$cfg) || die "no configuration\n"; - &$ceph_service_cmd('start'); + my $worker = sub { + my $upid = shift; - return undef; + my $cmd = ['start']; + if ($param->{service}) { + push @$cmd, $param->{service}; + } + + &$ceph_service_cmd(@$cmd); + }; + + return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph', + $authuser, $worker); }}); __PACKAGE__->register_method ({ diff --git a/bin/pveceph b/bin/pveceph index f6dc5489..21b0aeca 100755 --- a/bin/pveceph +++ b/bin/pveceph @@ -36,6 +36,12 @@ $rpcenv->init_request(); $rpcenv->set_language($ENV{LANG}); $rpcenv->set_user('root@pam'); +my $upid_exit = sub { + my $upid = shift; + my $status = PVE::Tools::upid_read_status($upid); + exit($status eq 'OK' ? 0 : -1); +}; + my $nodename = PVE::INotify::nodename(); __PACKAGE__->register_method ({ @@ -112,8 +118,10 @@ my $cmddef = { destroyosd => [ 'PVE::API2::Ceph', 'destroyosd', ['osdid'], { node => $nodename } ], createmon => [ 'PVE::API2::Ceph', 'createmon', [], { node => $nodename } ], destroymon => [ 'PVE::API2::Ceph', 'destroymon', ['monid'], { node => $nodename } ], - start => [ 'PVE::API2::Ceph', 'start', [], { node => $nodename } ], - stop => [ 'PVE::API2::Ceph', 'stop', [], { node => $nodename } ], + start => [ 'PVE::API2::Ceph', 'start', ['service'], { node => $nodename }, + $upid_exit], + stop => [ 'PVE::API2::Ceph', 'stop', ['service'], { node => $nodename }, + $upid_exit], install => [ __PACKAGE__, 'install', [] ], purge => [ __PACKAGE__, 'purge', [] ], status => [ 'PVE::API2::Ceph', 'status', [], { node => $nodename }, sub { diff --git a/www/manager/node/Ceph.js b/www/manager/node/Ceph.js index 59a20a59..c4d77ca2 100644 --- a/www/manager/node/Ceph.js +++ b/www/manager/node/Ceph.js @@ -1,3 +1,163 @@ +Ext.define('PVE.node.CephMonList', { + extend: 'Ext.grid.GridPanel', + alias: 'widget.pveNodeCephMonList', + + + initComponent: function() { + var me = this; + + var nodename = me.pveSelNode.data.node; + if (!nodename) { + throw "no node name specified"; + } + + var sm = Ext.create('Ext.selection.RowModel', {}); + + var rstore = Ext.create('PVE.data.UpdateStore', { + interval: 3000, + storeid: 'ceph-mon-list', + model: 'ceph-mon-list', + proxy: { + type: 'pve', + url: "/api2/json/nodes/" + nodename + "/ceph/listmon" + } + }); + + var store = Ext.create('PVE.data.DiffStore', { rstore: rstore }); + + PVE.Utils.monStoreErrors(me, rstore); + + var service_cmd = function(cmd) { + var rec = sm.getSelection()[0]; + PVE.Utils.API2Request({ + url: "/nodes/" + nodename + "/ceph/" + cmd, + method: 'POST', + params: { service: "mon." + rec.data.name }, + failure: function(response, opts) { + Ext.Msg.alert(gettext('Error'), response.htmlStatus); + } + }); + }; + + var start_btn = new PVE.button.Button({ + text: gettext('Start'), + selModel: sm, + disabled: true, + handler: function(){ + service_cmd("start"); + } + }); + + var stop_btn = new PVE.button.Button({ + text: gettext('Stop'), + selModel: sm, + disabled: true, + handler: function(){ + service_cmd("stop"); + } + }); + + Ext.apply(me, { + store: store, + selModel: sm, + stateful: false, + tbar: [ start_btn, stop_btn ], + columns: [ + { + header: gettext('Name'), + width: 50, + sortable: true, + renderer: function(v) { return "mon." + v; }, + dataIndex: 'name' + }, + { + header: gettext('Host'), + width: 100, + sortable: true, + renderer: function(v) { + return v ? v : 'unknown'; + }, + dataIndex: 'host' + }, + { + header: gettext('Quorum'), + width: 50, + sortable: false, + renderer: PVE.Utils.format_boolean, + dataIndex: 'quorum' + }, + { + header: gettext('Address'), + flex: 1, + sortable: true, + dataIndex: 'addr' + } + ], + listeners: { + show: rstore.startUpdate, + hide: rstore.stopUpdate, + destroy: rstore.stopUpdate + } + }); + + me.callParent(); + } +}, function() { + + Ext.define('ceph-mon-list', { + extend: 'Ext.data.Model', + fields: [ 'addr', 'name', 'rank', 'host', 'quorum' ], + idProperty: 'name' + }); +}); + +Ext.define('PVE.node.CephConfig', { + extend: 'Ext.panel.Panel', + alias: 'widget.pveNodeCephConfig', + + load: function() { + var me = this; + + PVE.Utils.API2Request({ + url: me.url, + waitMsgTarget: me, + failure: function(response, opts) { + me.update(gettext('Error') + " " + response.htmlStatus); + }, + success: function(response, opts) { + var data = response.result.data; + me.update(Ext.htmlEncode(data)); + } + }); + }, + + initComponent: function() { + var me = this; + + var nodename = me.pveSelNode.data.node; + if (!nodename) { + throw "no node name specified"; + } + + Ext.apply(me, { + url: '/api2/extjs/nodes/' + nodename + '/ceph/config', +// style: 'padding-left:10px', + bodyStyle: 'white-space:pre', + bodyPadding: 5, + autoScroll: true, + listeners: { + show: function() { + me.load(); + } + } + }); + + me.callParent(); + + me.load(); + } +}); + Ext.define('PVE.node.CephCrushMap', { extend: 'Ext.panel.Panel', alias: 'widget.pveNodeCephCrushMap', @@ -28,7 +188,7 @@ Ext.define('PVE.node.CephCrushMap', { Ext.apply(me, { url: '/api2/extjs/nodes/' + nodename + '/ceph/crush', - style: 'padding-left:10px', +// style: 'padding-left:10px', bodyStyle: 'white-space:pre', bodyPadding: 5, autoScroll: true, @@ -199,14 +359,14 @@ Ext.define('PVE.node.Ceph', { itemId: 'status' }, { + xtype: 'pveNodeCephConfig', title: 'Config', - itemId: 'config', - html: "ABCD" + itemId: 'config' }, { + xtype: 'pveNodeCephMonList', title: 'Monitor', - itemId: 'test2', - html: "ABCD" + itemId: 'monlist' }, { title: 'OSD', From 39e1ad708665c6e9d7e1625f1c5feb06c3db142f Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 14 Nov 2013 10:37:39 +0100 Subject: [PATCH 09/27] improve ceph API paths --- PVE/API2/Ceph.pm | 22 ++++++++++------------ www/manager/node/Ceph.js | 2 +- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm index 5ede3ff9..a649a822 100644 --- a/PVE/API2/Ceph.pm +++ b/PVE/API2/Ceph.pm @@ -259,11 +259,8 @@ __PACKAGE__->register_method ({ my $result = [ { name => 'init' }, - { name => 'createmon' }, - { name => 'destroymon' }, - { name => 'listmon' }, - { name => 'createosd' }, - { name => 'destroyosd' }, + { name => 'mon' }, + { name => 'osd' }, { name => 'stop' }, { name => 'start' }, { name => 'status' }, @@ -297,7 +294,7 @@ __PACKAGE__->register_method ({ __PACKAGE__->register_method ({ name => 'listmon', - path => 'listmon', + path => 'mon', method => 'GET', description => "Get Ceph monitor list.", proxyto => 'node', @@ -317,6 +314,7 @@ __PACKAGE__->register_method ({ addr => { type => 'string' }, }, }, + links => [ { rel => 'child', href => "{name}" } ], }, code => sub { my ($param) = @_; @@ -426,7 +424,7 @@ __PACKAGE__->register_method ({ __PACKAGE__->register_method ({ name => 'createmon', - path => 'createmon', + path => 'mon', method => 'POST', description => "Create Ceph Monitor", proxyto => 'node', @@ -535,8 +533,8 @@ __PACKAGE__->register_method ({ __PACKAGE__->register_method ({ name => 'destroymon', - path => 'destroymon', - method => 'POST', + path => 'mon/{monid}', + method => 'DELETE', description => "Destroy Ceph monitor.", proxyto => 'node', protected => 1, @@ -703,7 +701,7 @@ __PACKAGE__->register_method ({ __PACKAGE__->register_method ({ name => 'createosd', - path => 'createosd', + path => 'osd', method => 'POST', description => "Create OSD", proxyto => 'node', @@ -750,8 +748,8 @@ __PACKAGE__->register_method ({ __PACKAGE__->register_method ({ name => 'destroyosd', - path => 'destroyosd', - method => 'POST', + path => 'osd/{osdid}', + method => 'DELETE', description => "Destroy OSD", proxyto => 'node', protected => 1, diff --git a/www/manager/node/Ceph.js b/www/manager/node/Ceph.js index c4d77ca2..0114339c 100644 --- a/www/manager/node/Ceph.js +++ b/www/manager/node/Ceph.js @@ -19,7 +19,7 @@ Ext.define('PVE.node.CephMonList', { model: 'ceph-mon-list', proxy: { type: 'pve', - url: "/api2/json/nodes/" + nodename + "/ceph/listmon" + url: "/api2/json/nodes/" + nodename + "/ceph/mon" } }); From 22facb10812bc71a16fb3a0bb9be216b8ec9a417 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 14 Nov 2013 13:05:33 +0100 Subject: [PATCH 10/27] implement add/remove monitor --- www/manager/node/Ceph.js | 93 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 87 insertions(+), 6 deletions(-) diff --git a/www/manager/node/Ceph.js b/www/manager/node/Ceph.js index 0114339c..bcc27217 100644 --- a/www/manager/node/Ceph.js +++ b/www/manager/node/Ceph.js @@ -1,3 +1,49 @@ +Ext.define('PVE.CephCreateMon', { + extend: 'PVE.window.Edit', + alias: ['widget.pveCephCreateMon'], + + create: true, + + subject: 'Ceph Monitor', + + setNode: function(nodename) { + var me = this; + + me.nodename = nodename; + me.url = "/nodes/" + nodename + "/ceph/mon"; + }, + + initComponent : function() { + var me = this; + + if (!me.nodename) { + throw "no node name specified"; + } + + me.setNode(me.nodename); + + Ext.applyIf(me, { + method: 'POST', + items: [ + { + xtype: 'PVE.form.NodeSelector', + submitValue: false, + fieldLabel: gettext('Comment'), + selectCurNode: true, + allowBlank: false, + listeners: { + change: function(f, value) { + me.setNode(value); + } + } + } + ] + }); + + me.callParent(); + } +}); + Ext.define('PVE.node.CephMonList', { extend: 'Ext.grid.GridPanel', alias: 'widget.pveNodeCephMonList', @@ -29,8 +75,12 @@ Ext.define('PVE.node.CephMonList', { var service_cmd = function(cmd) { var rec = sm.getSelection()[0]; + if (!rec.data.host) { + Ext.Msg.alert(gettext('Error'), "entry has no host"); + return; + } PVE.Utils.API2Request({ - url: "/nodes/" + nodename + "/ceph/" + cmd, + url: "/nodes/" + rec.data.host + "/ceph/" + cmd, method: 'POST', params: { service: "mon." + rec.data.name }, failure: function(response, opts) { @@ -57,11 +107,44 @@ Ext.define('PVE.node.CephMonList', { } }); + var add_btn = new Ext.Button({ + text: gettext('Create'), + handler: function(){ + var win = Ext.create('PVE.CephCreateMon', { + nodename: nodename + }); + win.show(); + } + }); + + var remove_btn = new PVE.button.Button({ + text: gettext('Remove'), + selModel: sm, + disabled: true, + handler: function() { + var rec = sm.getSelection()[0]; + + if (!rec.data.host) { + Ext.Msg.alert(gettext('Error'), "entry has no host"); + return; + } + + PVE.Utils.API2Request({ + url: "/nodes/" + rec.data.host + "/ceph/mon/" + + rec.data.name, + method: 'DELETE', + failure: function(response, opts) { + Ext.Msg.alert(gettext('Error'), response.htmlStatus); + } + }); + } + }); + Ext.apply(me, { store: store, selModel: sm, stateful: false, - tbar: [ start_btn, stop_btn ], + tbar: [ start_btn, stop_btn, add_btn, remove_btn ], columns: [ { header: gettext('Name'), @@ -140,8 +223,7 @@ Ext.define('PVE.node.CephConfig', { } Ext.apply(me, { - url: '/api2/extjs/nodes/' + nodename + '/ceph/config', -// style: 'padding-left:10px', + url: '/nodes/' + nodename + '/ceph/config', bodyStyle: 'white-space:pre', bodyPadding: 5, autoScroll: true, @@ -187,8 +269,7 @@ Ext.define('PVE.node.CephCrushMap', { } Ext.apply(me, { - url: '/api2/extjs/nodes/' + nodename + '/ceph/crush', -// style: 'padding-left:10px', + url: '/nodes/' + nodename + '/ceph/crush', bodyStyle: 'white-space:pre', bodyPadding: 5, autoScroll: true, From 570278fab02a118b118f08974e1e82ad210e4170 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Fri, 15 Nov 2013 08:02:35 +0100 Subject: [PATCH 11/27] add ceph log view --- PVE/API2/Ceph.pm | 58 ++++++++++++++++++++++++++++++++++++++++ www/manager/node/Ceph.js | 6 +++++ 2 files changed, 64 insertions(+) diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm index a649a822..17e77f24 100644 --- a/PVE/API2/Ceph.pm +++ b/PVE/API2/Ceph.pm @@ -266,6 +266,7 @@ __PACKAGE__->register_method ({ { name => 'status' }, { name => 'crush' }, { name => 'config' }, + { name => 'log' }, ]; return $result; @@ -834,4 +835,61 @@ __PACKAGE__->register_method ({ return $txt; }}); +__PACKAGE__->register_method({ + name => 'log', + path => 'log', + method => 'GET', + description => "Read ceph log", + proxyto => 'node', + permissions => { + check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]], + }, + protected => 1, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + start => { + type => 'integer', + minimum => 0, + optional => 1, + }, + limit => { + type => 'integer', + minimum => 0, + optional => 1, + }, + }, + }, + returns => { + type => 'array', + items => { + type => "object", + properties => { + n => { + description=> "Line number", + type=> 'integer', + }, + t => { + description=> "Line text", + type => 'string', + } + } + } + }, + code => sub { + my ($param) = @_; + + my $rpcenv = PVE::RPCEnvironment::get(); + my $user = $rpcenv->get_user(); + my $node = $param->{node}; + + my $logfile = "/var/log/ceph/ceph.log"; + my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit}); + + $rpcenv->set_result_attrib('total', $count); + + return $lines; + }}); + diff --git a/www/manager/node/Ceph.js b/www/manager/node/Ceph.js index bcc27217..0180c75d 100644 --- a/www/manager/node/Ceph.js +++ b/www/manager/node/Ceph.js @@ -463,6 +463,12 @@ Ext.define('PVE.node.Ceph', { title: 'Crush', xtype: 'pveNodeCephCrushMap', itemId: 'crushmap' + }, + { + title: 'Log', + itemId: 'log', + xtype: 'pveLogView', + url: "/api2/extjs/nodes/" + nodename + "/ceph/log" } ], listeners: { From e9ea4677598bd04a1ffbced5feccdf9d8e3062e0 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Fri, 15 Nov 2013 08:15:39 +0100 Subject: [PATCH 12/27] isert section when writing ceph.conf --- PVE/API2/Ceph.pm | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm index 17e77f24..684c97c3 100644 --- a/PVE/API2/Ceph.pm +++ b/PVE/API2/Ceph.pm @@ -205,13 +205,25 @@ my $write_ceph_config = sub { my ($cfg) = @_; my $out = ''; - foreach my $section (keys %$cfg) { - $out .= "[$section]\n"; - foreach my $key (sort keys %{$cfg->{$section}}) { - $out .= "\t $key = $cfg->{$section}->{$key}\n"; + + my $cond_write_sec = sub { + my $re = shift; + + foreach my $section (keys %$cfg) { + next if $section !~ m/^$re$/; + $out .= "[$section]\n"; + foreach my $key (sort keys %{$cfg->{$section}}) { + $out .= "\t $key = $cfg->{$section}->{$key}\n"; + } + $out .= "\n"; } - $out .= "\n"; - } + }; + + &$cond_write_sec('global'); + &$cond_write_sec('mon'); + &$cond_write_sec('osd'); + &$cond_write_sec('mon\..*'); + &$cond_write_sec('osd\..*'); PVE::Tools::file_set_contents($pve_ceph_cfgpath, $out); }; From 13f4d7623ddd439a1cf608931e5e6b54a39f2534 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Fri, 15 Nov 2013 12:24:14 +0100 Subject: [PATCH 13/27] allow to create OSD on unused disks --- PVE/API2/Ceph.pm | 162 ++++++++++++++++++++++++++++++++++++++- www/manager/node/Ceph.js | 128 ++++++++++++++++++++++++++++++- 2 files changed, 287 insertions(+), 3 deletions(-) diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm index 684c97c3..b26937bc 100644 --- a/PVE/API2/Ceph.pm +++ b/PVE/API2/Ceph.pm @@ -5,9 +5,11 @@ use warnings; use File::Basename; use File::Path; use POSIX qw (LONG_MAX); +use Cwd qw(abs_path); +use IO::Dir; use PVE::SafeSyslog; -use PVE::Tools qw(extract_param run_command); +use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach); use PVE::Exception qw(raise raise_param_exc); use PVE::INotify; use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file); @@ -246,6 +248,112 @@ my $ceph_service_cmd = sub { run_command(['service', 'ceph', '-c', $ceph_cfgpath, @_]); }; + +sub list_disks { + my $disklist = {}; + + my $fd = IO::File->new("/proc/mounts", "r") || + die "unable to open /proc/mounts - $!\n"; + + my $mounted = {}; + + while (defined(my $line = <$fd>)) { + my ($dev, $path, $fstype) = split(/\s+/, $line); + next if !($dev && $path && $fstype); + next if $dev !~ m|^/dev/|; + my $real_dev = abs_path($dev); + $mounted->{$real_dev} = $path; + } + close($fd); + + my $dev_is_mounted = sub { + my ($dev) = @_; + return $mounted->{$dev}; + }; + + my $dir_is_epmty = sub { + my ($dir) = @_; + + my $dh = IO::Dir->new ($dir); + return 1 if !$dh; + + while (defined(my $tmp = $dh->read)) { + next if $tmp eq '.' || $tmp eq '..'; + $dh->close; + return 0; + } + $dh->close; + return 1; + }; + + dir_glob_foreach('/sys/block', '.*', sub { + my ($dev) = @_; + + return if $dev eq '.'; + return if $dev eq '..'; + + return if $dev =~ m|^ram\d+$|; # skip ram devices + return if $dev =~ m|^loop\d+$|; # skip loop devices + return if $dev =~ m|^md\d+$|; # skip md devices + return if $dev =~ m|^dm-.*$|; # skip dm related things + return if $dev =~ m|^fd\d+$|; # skip Floppy + return if $dev =~ m|^sr\d+$|; # skip CDs + + my $devdir = "/sys/block/$dev/device"; + return if ! -d $devdir; + + my $size = file_read_firstline("/sys/block/$dev/size"); + return if !$size; + + $size = $size * 512; + + my $info = `udevadm info --path /sys/block/$dev --query all`; + return if !$info; + + return if $info !~ m/^E: DEVTYPE=disk$/m; + return if $info =~ m/^E: ID_CDROM/m; + + my $serial = 'unknown'; + if ($info =~ m/^E: ID_SERIAL_SHORT=(\S+)$/m) { + $serial = $1; + } + + my $vendor = file_read_firstline("$devdir/vendor") || 'unknown'; + my $model = file_read_firstline("$devdir/model") || 'unknown'; + + my $used = &$dir_is_epmty("/sys/block/$dev/holders") ? 0 : 1; + + $used = 1 if &$dev_is_mounted("/dev/$dev"); + + $disklist->{$dev} = { + vendor => $vendor, + model => $model, + size => $size, + serial => $serial, + }; + + my $osdid = -1; + + dir_glob_foreach("/sys/block/$dev", "$dev.+", sub { + my ($part) = @_; + if (!&$dir_is_epmty("/sys/block/$dev/$part/holders")) { + $used = 1; + } + if (my $mp = &$dev_is_mounted("/dev/$part")) { + $used = 1; + if ($mp =~ m|^/var/lib/ceph/osd/ceph-(\d+)$|) { + $osdid = $1; + } + } + }); + + $disklist->{$dev}->{used} = $used; + $disklist->{$dev}->{osdid} = $osdid; + }); + + return $disklist; +} + __PACKAGE__->register_method ({ name => 'index', path => '', @@ -279,11 +387,51 @@ __PACKAGE__->register_method ({ { name => 'crush' }, { name => 'config' }, { name => 'log' }, + { name => 'disks' }, ]; return $result; }}); +__PACKAGE__->register_method ({ + name => 'disks', + path => 'disks', + method => 'GET', + description => "List local disks.", + proxyto => 'node', + protected => 1, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + }, + }, + returns => { + type => 'array', + items => { + type => "object", + properties => { + dev => { type => 'string' }, + used => { type => 'boolean' }, + size => { type => 'integer' }, + osdid => { type => 'integer' }, + vendor => { type => 'string', optional => 1 }, + model => { type => 'string', optional => 1 }, + serial => { type => 'string', optional => 1 }, + }, + }, + # links => [ { rel => 'child', href => "{}" } ], + }, + code => sub { + my ($param) = @_; + + &$check_ceph_inited(); + + my $res = list_disks(); + + return PVE::RESTHandler::hash_to_array($res, 'dev'); + }}); + __PACKAGE__->register_method ({ name => 'config', path => 'config', @@ -744,6 +892,18 @@ __PACKAGE__->register_method ({ -b $param->{dev} || die "no such block device '$param->{dev}'\n"; + my $disklist = list_disks(); + + my $devname = $param->{dev}; + $devname =~ s|/dev/||; + + my $diskinfo = $disklist->{$devname}; + die "unable to get device info for '$devname'\n" + if !$diskinfo; + + die "device '$param->{dev}' is in use\n" + if $diskinfo->{used}; + my $monstat = ceph_mon_status(1); die "unable to get fsid\n" if !$monstat->{monmap} || !$monstat->{monmap}->{fsid}; my $fsid = $monstat->{monmap}->{fsid}; diff --git a/www/manager/node/Ceph.js b/www/manager/node/Ceph.js index 0180c75d..c4fe5846 100644 --- a/www/manager/node/Ceph.js +++ b/www/manager/node/Ceph.js @@ -1,3 +1,122 @@ +Ext.define('PVE.node.CephDiskList', { + extend: 'Ext.grid.GridPanel', + alias: 'widget.pveNodeCephDiskList', + + + initComponent: function() { + var me = this; + + var nodename = me.pveSelNode.data.node; + if (!nodename) { + throw "no node name specified"; + } + + var sm = Ext.create('Ext.selection.RowModel', {}); + + var rstore = Ext.create('PVE.data.UpdateStore', { + interval: 3000, + storeid: 'ceph-disk-list', + model: 'ceph-disk-list', + proxy: { + type: 'pve', + url: "/api2/json/nodes/" + nodename + "/ceph/disks" + } + }); + + var store = Ext.create('PVE.data.DiffStore', { rstore: rstore }); + + PVE.Utils.monStoreErrors(me, rstore); + + var create_btn = new PVE.button.Button({ + text: gettext('Create') + ': OSD', + selModel: sm, + disabled: true, + handler: function() { + var rec = sm.getSelection()[0]; + + console.log("CREATEOSD " + rec.data.dev); + + PVE.Utils.API2Request({ + url: "/nodes/" + nodename + "/ceph/osd", + method: 'POST', + params: { dev: "/dev/" + rec.data.dev }, + failure: function(response, opts) { + Ext.Msg.alert(gettext('Error'), response.htmlStatus); + } + }); + } + }); + + Ext.apply(me, { + store: store, + selModel: sm, + stateful: false, + tbar: [ create_btn ], + columns: [ + { + header: gettext('Device'), + width: 100, + sortable: true, + dataIndex: 'dev' + }, + { + header: gettext('used'), + width: 50, + sortable: false, + renderer: function(v, metaData, rec) { + if (rec && (rec.data.osdid >= 0)) { + return "osd." + rec.data.osdid; + } + return PVE.Utils.format_boolean(v); + }, + dataIndex: 'used' + }, + { + header: gettext('Size'), + width: 100, + sortable: false, + renderer: PVE.Utils.format_size, + dataIndex: 'size' + }, + { + header: gettext('Vendor'), + width: 100, + sortable: true, + dataIndex: 'vendor' + }, + { + header: gettext('Model'), + width: 200, + sortable: true, + dataIndex: 'model' + }, + { + header: gettext('Serial'), + flex: 1, + sortable: true, + dataIndex: 'serial' + } + ], + listeners: { + show: rstore.startUpdate, + hide: rstore.stopUpdate, + destroy: rstore.stopUpdate + } + }); + + me.callParent(); + } +}, function() { + + Ext.define('ceph-disk-list', { + extend: 'Ext.data.Model', + fields: [ 'dev', 'used', { name: 'size', type: 'number'}, + {name: 'osdid', type: 'number'}, + 'vendor', 'model', 'serial'], + idProperty: 'dev' + }); +}); + Ext.define('PVE.CephCreateMon', { extend: 'PVE.window.Edit', alias: ['widget.pveCephCreateMon'], @@ -107,7 +226,7 @@ Ext.define('PVE.node.CephMonList', { } }); - var add_btn = new Ext.Button({ + var create_btn = new Ext.Button({ text: gettext('Create'), handler: function(){ var win = Ext.create('PVE.CephCreateMon', { @@ -144,7 +263,7 @@ Ext.define('PVE.node.CephMonList', { store: store, selModel: sm, stateful: false, - tbar: [ start_btn, stop_btn, add_btn, remove_btn ], + tbar: [ start_btn, stop_btn, create_btn, remove_btn ], columns: [ { header: gettext('Name'), @@ -449,6 +568,11 @@ Ext.define('PVE.node.Ceph', { title: 'Monitor', itemId: 'monlist' }, + { + xtype: 'pveNodeCephDiskList', + title: 'Disks', + itemId: 'disklist' + }, { title: 'OSD', itemId: 'test3', From dd7e2a94f279145f6e661e3c9354af0111589e3e Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Tue, 19 Nov 2013 12:13:53 +0100 Subject: [PATCH 14/27] add OSD tree --- PVE/API2/Ceph.pm | 76 +++++++++++++++++ www/manager/node/Ceph.js | 170 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 242 insertions(+), 4 deletions(-) diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm index b26937bc..ae32f73a 100644 --- a/PVE/API2/Ceph.pm +++ b/PVE/API2/Ceph.pm @@ -860,6 +860,82 @@ __PACKAGE__->register_method ({ return &$run_ceph_cmd_json(['status'], quiet => 1); }}); +__PACKAGE__->register_method ({ + name => 'listosd', + path => 'osd', + method => 'GET', + description => "Get Ceph osd list/tree.", + proxyto => 'node', + protected => 1, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + }, + }, + returns => { + type => "object", + }, + code => sub { + my ($param) = @_; + + &$check_ceph_inited(); + + my $res = &$run_ceph_cmd_json(['osd', 'tree'], quiet => 1); + + die "no tree nodes found\n" if !($res && $res->{nodes}); + + my $nodes = {}; + my $newnodes = {}; + foreach my $e (@{$res->{nodes}}) { + $nodes->{$e->{id}} = $e; + + my $new = { + id => $e->{id}, + name => $e->{name}, + type => $e->{type} + }; + + foreach my $opt (qw(status crush_weight reweight)) { + $new->{$opt} = $e->{$opt} if defined($e->{$opt}); + } + + $newnodes->{$e->{id}} = $new; + } + + foreach my $e (@{$res->{nodes}}) { + my $new = $newnodes->{$e->{id}}; + if ($e->{children} && scalar(@{$e->{children}})) { + $new->{children} = []; + $new->{leaf} = 0; + foreach my $cid (@{$e->{children}}) { + $nodes->{$cid}->{parent} = $e->{id}; + if ($nodes->{$cid}->{type} eq 'osd' && + $e->{type} eq 'host') { + $newnodes->{$cid}->{host} = $e->{name}; + } + push @{$new->{children}}, $newnodes->{$cid}; + } + } else { + $new->{leaf} = ($e->{id} >= 0) ? 1 : 0; + } + } + + my $rootnode; + foreach my $e (@{$res->{nodes}}) { + if (!$nodes->{$e->{id}}->{parent}) { + $rootnode = $newnodes->{$e->{id}}; + last; + } + } + + die "no root node\n" if !$rootnode; + + my $data = { root => $rootnode }; + + return $data; + }}); + __PACKAGE__->register_method ({ name => 'createosd', path => 'osd', diff --git a/www/manager/node/Ceph.js b/www/manager/node/Ceph.js index c4fe5846..0b5fd3d0 100644 --- a/www/manager/node/Ceph.js +++ b/www/manager/node/Ceph.js @@ -1,3 +1,167 @@ +Ext.define('PVE.node.CephOsdTree', { + extend: 'Ext.tree.Panel', + alias: 'widget.pveNodeCephOsdTree', + + initComponent: function() { + var me = this; + + var nodename = me.pveSelNode.data.node; + if (!nodename) { + throw "no node name specified"; + } + + var sm = Ext.create('Ext.selection.TreeModel', {}); + + var service_cmd = function(cmd) { + var rec = sm.getSelection()[0]; + if (!(rec && rec.data.name && rec.data.host)) { + return; + } + PVE.Utils.API2Request({ + url: "/nodes/" + rec.data.host + "/ceph/" + cmd, + params: { service: rec.data.name }, + waitMsgTarget: me, + method: 'POST', + failure: function(response, opts) { + Ext.Msg.alert(gettext('Error'), response.htmlStatus); + } + }); + } + + var start_btn = new Ext.Button({ + text: gettext('Start'), + disabled: true, + handler: function(){ service_cmd('start'); } + }); + + var stop_btn = new Ext.Button({ + text: gettext('Stop'), + disabled: true, + handler: function(){ service_cmd('stop'); } + }); + + var remove_btn = new Ext.Button({ + text: gettext('Remove'), + disabled: true, + handler: function(){ + var rec = sm.getSelection()[0]; + if (!(rec && (rec.data.id >= 0) && rec.data.host)) { + return; + } + PVE.Utils.API2Request({ + url: "/nodes/" + rec.data.host + "/ceph/osd/" + rec.data.id, + waitMsgTarget: me, + method: 'DELETE', + failure: function(response, opts) { + Ext.Msg.alert(gettext('Error'), response.htmlStatus); + } + }); + } + }); + + var set_button_status = function() { + var rec = sm.getSelection()[0]; + + if (!rec) { + start_btn.setDisabled(true); + stop_btn.setDisabled(true); + remove_btn.setDisabled(true); + return; + } + + var isOsd = (rec.data.host && (rec.data.type === 'osd') && (rec.data.id >= 0)); + + start_btn.setDisabled(!(isOsd && (rec.data.status !== 'up'))); + stop_btn.setDisabled(!(isOsd && (rec.data.status !== 'down'))); + remove_btn.setDisabled(!(isOsd && (rec.data.status === 'down'))); + }; + + sm.on('selectionchange', set_button_status); + + var reload = function() { + PVE.Utils.API2Request({ + url: "/nodes/" + nodename + "/ceph/osd", + waitMsgTarget: me, + method: 'GET', + failure: function(response, opts) { + PVE.Utils.setErrorMask(me, response.htmlStatus); + }, + success: function(response, opts) { + sm.deselectAll(); + me.setRootNode(response.result.data.root); + me.expandAll(); + set_button_status(); + } + }); + }; + + var reload_btn = new Ext.Button({ + text: gettext('Reload'), + handler: reload + }); + + Ext.apply(me, { + tbar: [ reload_btn, start_btn, stop_btn, remove_btn ], + rootVisible: false, + fields: ['name', 'type', 'status', 'host', + { type: 'integre', name: 'id' }, + { type: 'number', name: 'reweight' }, + { type: 'number', name: 'crush_weight' }], + stateful: false, + selModel: sm, + columns: [ + { + xtype: 'treecolumn', + text: 'Name', + dataIndex: 'name', + width: 200 + }, + { + text: 'ID', + dataIndex: 'id', + align: 'right', + width: 60 + }, + { + text: 'weight', + dataIndex: 'crush_weight', + align: 'right', + width: 60 + }, + { + text: 'Type', + dataIndex: 'type', + align: 'right', + width: 100 + }, + { + text: 'Status', + dataIndex: 'status', + align: 'right', + width: 100 + }, + { + text: 'reweight', + dataIndex: 'reweight', + align: 'right', + width: 60 + } + ], + listeners: { + show: function() { + console.log("RELOAD"); + reload(); + } + } + }); + + me.callParent(); + + reload(); + } +}); + + Ext.define('PVE.node.CephDiskList', { extend: 'Ext.grid.GridPanel', alias: 'widget.pveNodeCephDiskList', @@ -34,8 +198,6 @@ Ext.define('PVE.node.CephDiskList', { handler: function() { var rec = sm.getSelection()[0]; - console.log("CREATEOSD " + rec.data.dev); - PVE.Utils.API2Request({ url: "/nodes/" + nodename + "/ceph/osd", method: 'POST', @@ -574,9 +736,9 @@ Ext.define('PVE.node.Ceph', { itemId: 'disklist' }, { + xtype: 'pveNodeCephOsdTree', title: 'OSD', - itemId: 'test3', - html: "ABCD" + itemId: 'osdtree' }, { title: 'Pool', From b0537f7be94af3960d78de16b2834ba011cc8dd2 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Tue, 19 Nov 2013 13:25:36 +0100 Subject: [PATCH 15/27] add API for ceph pools --- PVE/API2/Ceph.pm | 114 +++++++++++++++++++++++++++++++++++++++++++++++ bin/pveceph | 10 +++++ 2 files changed, 124 insertions(+) diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm index ae32f73a..27e7f9a2 100644 --- a/PVE/API2/Ceph.pm +++ b/PVE/API2/Ceph.pm @@ -381,6 +381,7 @@ __PACKAGE__->register_method ({ { name => 'init' }, { name => 'mon' }, { name => 'osd' }, + { name => 'pools' }, { name => 'stop' }, { name => 'start' }, { name => 'status' }, @@ -860,6 +861,119 @@ __PACKAGE__->register_method ({ return &$run_ceph_cmd_json(['status'], quiet => 1); }}); +__PACKAGE__->register_method ({ + name => 'lspools', + path => 'pools', + method => 'GET', + description => "List all pools.", + proxyto => 'node', + protected => 1, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + }, + }, + returns => { + type => 'array', + items => { + type => "object", + properties => { + pool => { type => 'integer' }, + pool_name => { type => 'string' }, + size => { type => 'integer' }, + }, + }, + links => [ { rel => 'child', href => "{pool_name}" } ], + }, + code => sub { + my ($param) = @_; + + &$check_ceph_inited(); + + my $res = &$run_ceph_cmd_json(['osd', 'dump'], quiet => 1); + + my $data = []; + foreach my $e (@{$res->{pools}}) { + my $d = {}; + foreach my $attr (qw(pool pool_name size min_size pg_num crush_ruleset)) { + $d->{$attr} = $e->{$attr} if defined($e->{$attr}); + } + push @$data, $d; + } + + return $data; + }}); + +__PACKAGE__->register_method ({ + name => 'createpool', + path => 'pools', + method => 'POST', + description => "Create POOL", + proxyto => 'node', + protected => 1, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + name => { + description => "The name of the pool. It must be unique.", + type => 'string', + }, + pg_num => { + description => "Number of placement groups.", + type => 'integer', + default => 512, + optional => 1, + minimum => 8, + maximum => 32768, + }, + }, + }, + returns => { type => 'null' }, + code => sub { + my ($param) = @_; + + &$check_ceph_inited(); + + die "not fully configured - missing '$pve_ckeyring_path'\n" + if ! -f $pve_ckeyring_path; + + my $pg_num = $param->{pg_num} || 512; + + &$run_ceph_cmd(['osd', 'pool', 'create', $param->{name}, $pg_num]); + + return undef; + }}); + +__PACKAGE__->register_method ({ + name => 'destroypool', + path => 'pools/{name}', + method => 'DELETE', + description => "Destroy pool", + proxyto => 'node', + protected => 1, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + name => { + description => "The name of the pool. It must be unique.", + type => 'string', + }, + }, + }, + returns => { type => 'null' }, + code => sub { + my ($param) = @_; + + &$check_ceph_inited(); + + &$run_ceph_cmd(['osd', 'pool', 'delete', $param->{name}, $param->{name}, '--yes-i-really-really-mean-it']); + + return undef; + }}); + __PACKAGE__->register_method ({ name => 'listosd', path => 'osd', diff --git a/bin/pveceph b/bin/pveceph index 21b0aeca..f9844604 100755 --- a/bin/pveceph +++ b/bin/pveceph @@ -114,6 +114,16 @@ __PACKAGE__->register_method ({ my $cmddef = { init => [ 'PVE::API2::Ceph', 'init', [], { node => $nodename } ], + lspools => [ 'PVE::API2::Ceph', 'lspools', [], { node => $nodename }, sub { + my $res = shift; + + printf("%-20s %10s %10s\n", "Name", "size", "pg_num"); + foreach my $p (sort {$a->{pool_name} cmp $b->{pool_name}} @$res) { + printf("%-20s %10d %10d\n", $p->{pool_name}, $p->{size}, $p->{pg_num}); + } + }], + createpool => [ 'PVE::API2::Ceph', 'createpool', ['name'], { node => $nodename }], + destroypool => [ 'PVE::API2::Ceph', 'destroypool', ['name'], { node => $nodename } ], createosd => [ 'PVE::API2::Ceph', 'createosd', ['dev'], { node => $nodename } ], destroyosd => [ 'PVE::API2::Ceph', 'destroyosd', ['osdid'], { node => $nodename } ], createmon => [ 'PVE::API2::Ceph', 'createmon', [], { node => $nodename } ], From c7881bf602cff149bb9d6253b4c5b8b675f276eb Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 20 Nov 2013 07:53:43 +0100 Subject: [PATCH 16/27] add ceph pool GUI --- PVE/API2/Ceph.pm | 11 +++ www/manager/node/Ceph.js | 183 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 190 insertions(+), 4 deletions(-) diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm index 27e7f9a2..f5ba562b 100644 --- a/PVE/API2/Ceph.pm +++ b/PVE/API2/Ceph.pm @@ -920,6 +920,14 @@ __PACKAGE__->register_method ({ description => "The name of the pool. It must be unique.", type => 'string', }, + size => { + description => 'Number of replicas per object', + type => 'integer', + default => 2, + optional => 1, + minimum => 1, + maximum => 3, + }, pg_num => { description => "Number of placement groups.", type => 'integer', @@ -940,9 +948,12 @@ __PACKAGE__->register_method ({ if ! -f $pve_ckeyring_path; my $pg_num = $param->{pg_num} || 512; + my $size = $param->{size} || 2; &$run_ceph_cmd(['osd', 'pool', 'create', $param->{name}, $pg_num]); + &$run_ceph_cmd(['osd', 'pool', 'set', $param->{name}, 'size', $size]); + return undef; }}); diff --git a/www/manager/node/Ceph.js b/www/manager/node/Ceph.js index 0b5fd3d0..44e769b4 100644 --- a/www/manager/node/Ceph.js +++ b/www/manager/node/Ceph.js @@ -1,3 +1,179 @@ +Ext.define('PVE.CephCreatePool', { + extend: 'PVE.window.Edit', + alias: ['widget.pveCephCreatePool'], + + create: true, + + subject: 'Ceph Pool', + + initComponent : function() { + var me = this; + + if (!me.nodename) { + throw "no node name specified"; + } + + Ext.applyIf(me, { + url: "/nodes/" + me.nodename + "/ceph/pools", + method: 'POST', + items: [ + { + xtype: 'textfield', + fieldLabel: gettext('Name'), + name: 'name', + allowBlank: false + }, + { + xtype: 'numberfield', + fieldLabel: gettext('Size'), + name: 'size', + value: 2, + minValue: 1, + maxValue: 3, + allowBlank: false + }, + { + xtype: 'numberfield', + fieldLabel: 'pg_num', + name: 'pg_num', + value: 512, + minValue: 8, + maxValue: 32768, + allowBlank: false + } + ] + }); + + me.callParent(); + } +}); + +Ext.define('PVE.node.CephPoolList', { + extend: 'Ext.grid.GridPanel', + alias: 'widget.pveNodeCephPoolList', + + initComponent: function() { + var me = this; + + var nodename = me.pveSelNode.data.node; + if (!nodename) { + throw "no node name specified"; + } + + var sm = Ext.create('Ext.selection.RowModel', {}); + + var rstore = Ext.create('PVE.data.UpdateStore', { + interval: 3000, + storeid: 'ceph-pool-list', + model: 'ceph-pool-list', + proxy: { + type: 'pve', + url: "/api2/json/nodes/" + nodename + "/ceph/pools" + } + }); + + var store = Ext.create('PVE.data.DiffStore', { rstore: rstore }); + + PVE.Utils.monStoreErrors(me, rstore); + + var create_btn = new Ext.Button({ + text: gettext('Create'), + handler: function() { + var win = Ext.create('PVE.CephCreatePool', { + nodename: nodename + }); + win.show(); + } + }); + + var remove_btn = new PVE.button.Button({ + text: gettext('Remove'), + selModel: sm, + disabled: true, + confirmMsg: function(rec) { + var msg = Ext.String.format(gettext('Are you sure you want to remove entry {0}'), + "'" + rec.data.pool_name + "'"); + msg += " " + gettext('This will permanently erase all image data.'); + + return msg; + }, + handler: function() { + var rec = sm.getSelection()[0]; + + if (!rec.data.pool_name) { + return; + } + + PVE.Utils.API2Request({ + url: "/nodes/" + nodename + "/ceph/pools/" + + rec.data.pool_name, + method: 'DELETE', + failure: function(response, opts) { + Ext.Msg.alert(gettext('Error'), response.htmlStatus); + } + }); + } + }); + + Ext.apply(me, { + store: store, + selModel: sm, + stateful: false, + tbar: [ create_btn, remove_btn ], + columns: [ + { + header: gettext('Name'), + width: 100, + sortable: true, + dataIndex: 'pool_name' + }, + { + header: gettext('Size') + '/min', + width: 50, + sortable: false, + renderer: function(v, meta, rec) { + return v + '/' + rec.data.min_size; + }, + dataIndex: 'size' + }, + { + header: 'pg_num', + width: 100, + sortable: false, + dataIndex: 'pg_num' + }, + { + header: 'ruleset', + width: 50, + sortable: false, + dataIndex: 'crush_ruleset' + } + ], + listeners: { + show: rstore.startUpdate, + hide: rstore.stopUpdate, + destroy: rstore.stopUpdate + } + }); + + me.callParent(); + } +}, function() { + + Ext.define('ceph-pool-list', { + extend: 'Ext.data.Model', + fields: [ 'pool_name', + { name: 'pool', type: 'integer'}, + { name: 'size', type: 'integer'}, + { name: 'min_size', type: 'integer'}, + { name: 'pg_num', type: 'integer'}, + { name: 'crush_ruleset', type: 'integer'} + ], + idProperty: 'pool_name' + }); +}); + + Ext.define('PVE.node.CephOsdTree', { extend: 'Ext.tree.Panel', alias: 'widget.pveNodeCephOsdTree', @@ -149,7 +325,6 @@ Ext.define('PVE.node.CephOsdTree', { ], listeners: { show: function() { - console.log("RELOAD"); reload(); } } @@ -741,9 +916,9 @@ Ext.define('PVE.node.Ceph', { itemId: 'osdtree' }, { - title: 'Pool', - itemId: 'test4', - html: "ABCD" + xtype: 'pveNodeCephPoolList', + title: 'Pools', + itemId: 'pools' }, { title: 'Crush', From 64af631078dcdd12b2b67c994f7599da09f52d7a Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 20 Nov 2013 09:21:34 +0100 Subject: [PATCH 17/27] correct handle history state for ceph sub-config --- www/manager/node/Ceph.js | 42 ++++++++++++++++++++++++++++++++ www/manager/node/Config.js | 1 + www/manager/panel/ConfigPanel.js | 17 +++++++++---- 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/www/manager/node/Ceph.js b/www/manager/node/Ceph.js index 44e769b4..505a4745 100644 --- a/www/manager/node/Ceph.js +++ b/www/manager/node/Ceph.js @@ -882,6 +882,21 @@ Ext.define('PVE.node.Ceph', { throw "no node name specified"; } + if (!me.phstateid) { + throw "no parent history state specified"; + } + + var sp = Ext.state.Manager.getProvider(); + var state = sp.get(me.phstateid); + var hsregex = /^ceph-(\S+)$/; + + if (state && state.value) { + var res = hsregex.exec(state.value); + if (res && res[1]) { + me.activeTab = res[1]; + } + } + Ext.apply(me, { plain: true, tabPosition: 'bottom', @@ -938,11 +953,38 @@ Ext.define('PVE.node.Ceph', { if (first) { first.fireEvent('show', first); } + }, + tabchange: function(tp, newcard, oldcard) { + var first = tp.items.get(0); + var ntab; + + // Note: '' is alias for first tab. + if (newcard.itemId === first.itemId) { + ntab = 'ceph'; + } else { + ntab = 'ceph-' + newcard.itemId; + } + + var state = { value: ntab }; + sp.set(me.phstateid, state); } } }); me.callParent(); + var statechange = function(sp, key, state) { + if ((key === me.phstateid) && state) { + var first = me.items.get(0); + var atab = me.getActiveTab().itemId; + var res = hsregex.exec(state.value); + var ntab = (res && res[1]) ? res[1] : first.itemId; + if (ntab && (atab != ntab)) { + me.setActiveTab(ntab); + } + } + }; + + me.mon(sp, 'statechange', statechange); } }); \ No newline at end of file diff --git a/www/manager/node/Config.js b/www/manager/node/Config.js index fb18c2d4..564cf269 100644 --- a/www/manager/node/Config.js +++ b/www/manager/node/Config.js @@ -158,6 +158,7 @@ Ext.define('PVE.node.Config', { title: 'Ceph', itemId: 'ceph', xtype: 'pveNodeCeph', + phstateid: me.hstateid, nodename: nodename }]); } diff --git a/www/manager/panel/ConfigPanel.js b/www/manager/panel/ConfigPanel.js index d7c3f3b0..36ecad3f 100644 --- a/www/manager/panel/ConfigPanel.js +++ b/www/manager/panel/ConfigPanel.js @@ -11,10 +11,15 @@ Ext.define('PVE.panel.Config', { var activeTab; + var hsregex = /^([^\-\s]+)(-\S+)?$/; + if (stateid) { var state = sp.get(stateid); if (state && state.value) { - activeTab = state.value; + var res = hsregex.exec(state.value); + if (res && res[1]) { + activeTab = res[1]; + } } } @@ -70,13 +75,14 @@ Ext.define('PVE.panel.Config', { }, tabchange: function(tp, newcard, oldcard) { var ntab = newcard.itemId; + // Note: '' is alias for first tab. // First tab can be 'search' or something else if (newcard.itemId === items[0].itemId) { ntab = ''; } var state = { value: ntab }; - if (stateid) { + if (stateid && !newcard.phstateid) { sp.set(stateid, state); } } @@ -91,10 +97,11 @@ Ext.define('PVE.panel.Config', { me.callParent(); var statechange = function(sp, key, state) { - if (stateid && key === stateid) { + if (stateid && (key === stateid) && state) { var atab = tab.getActiveTab().itemId; - var ntab = state.value || items[0].itemId; - if (state && ntab && (atab != ntab)) { + var res = hsregex.exec(state.value); + var ntab = (res && res[1]) ? res[1] : items[0].itemId; + if (ntab && (atab != ntab)) { tab.setActiveTab(ntab); } } From f3224487a38d4d6367c5fa422c765dcec688ea59 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 20 Nov 2013 10:28:41 +0100 Subject: [PATCH 18/27] avoid useles symlinks to ceph keyrings --- PVE/API2/Ceph.pm | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm index f5ba562b..a2aa5371 100644 --- a/PVE/API2/Ceph.pm +++ b/PVE/API2/Ceph.pm @@ -29,9 +29,7 @@ my $ceph_cfgdir = "/etc/ceph"; my $pve_ceph_cfgpath = "/etc/pve/$ccname.conf"; my $ceph_cfgpath = "$ceph_cfgdir/$ccname.conf"; my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring"; -my $ceph_mon_key_path = "$ceph_cfgdir/$ccname.mon.keyring"; -my $pve_ckeyring_path = "/etc/pve/priv/$ccname.keyring"; -my $ceph_ckeyring_path = "$ceph_cfgdir/$ccname.client.admin.keyring"; +my $pve_ckeyring_path = "/etc/pve/priv/$ccname.client.admin.keyring"; my $ceph_bootstrap_osd_keyring = "/var/lib/ceph/bootstrap-osd/$ccname.keyring"; my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring"; @@ -42,8 +40,6 @@ sub purge_all_ceph_files { # fixme: this is very dangerous - should we really support this function? unlink $ceph_cfgpath; - unlink $ceph_mon_key_path; - unlink $ceph_ckeyring_path; unlink $pve_ceph_cfgpath; unlink $pve_ckeyring_path; @@ -95,16 +91,6 @@ my $check_ceph_enabled = sub { return 1; }; -my $force_symlink = sub { - my ($old, $new) = @_; - - return if (-l $new) && (readlink($new) eq $old); - - unlink $new; - symlink($old, $new) || - die "unable to create symlink '$new' - $!\n"; -}; - my $parse_ceph_config = sub { my ($filename) = @_; @@ -150,7 +136,7 @@ my $run_ceph_cmd = sub { # Note: --connect-timeout does not work with current version # '--connect-timeout', $timeout, - run_command(['ceph', '-c', $ceph_cfgpath, @$cmd], %params); + run_command(['ceph', '-c', $pve_ceph_cfgpath, @$cmd], %params); alarm(0); }; my $err = $@; @@ -236,16 +222,14 @@ my $setup_pve_symlinks = sub { my $lnk = readlink($ceph_cfgpath); die "file '$ceph_cfgpath' already exists\n" if !$lnk || $lnk ne $pve_ceph_cfgpath; + } else { + symlink($pve_ceph_cfgpath, $ceph_cfgpath) || + die "unable to create symlink '$ceph_cfgpath' - $!\n"; } - - # now assume we are allowed to setup/overwrite content - &$force_symlink($pve_ceph_cfgpath, $ceph_cfgpath); - &$force_symlink($pve_mon_key_path, $ceph_mon_key_path); - &$force_symlink($pve_ckeyring_path, $ceph_ckeyring_path); }; my $ceph_service_cmd = sub { - run_command(['service', 'ceph', '-c', $ceph_cfgpath, @_]); + run_command(['service', 'ceph', '-c', $pve_ceph_cfgpath, @_]); }; @@ -565,6 +549,7 @@ __PACKAGE__->register_method ({ 'auth cluster required' => 'cephx', 'auth service required' => 'cephx', 'auth client required' => 'cephx', + 'keyring' => '/etc/pve/priv/$cluster.$name.keyring', 'filestore xattr use omap' => 'true', 'osd journal size' => '1024', 'osd pool default size' => $size, From eac465a5747b604c94f6ca9fd48ad8272ec4b07b Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 20 Nov 2013 10:48:22 +0100 Subject: [PATCH 19/27] generate and save ceph fsid add dependency on libuuid-perl --- PVE/API2/Ceph.pm | 8 ++++++++ debian/control.in | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm index a2aa5371..5d733104 100644 --- a/PVE/API2/Ceph.pm +++ b/PVE/API2/Ceph.pm @@ -7,6 +7,7 @@ use File::Path; use POSIX qw (LONG_MAX); use Cwd qw(abs_path); use IO::Dir; +use UUID; use PVE::SafeSyslog; use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach); @@ -544,7 +545,14 @@ __PACKAGE__->register_method ({ my $pg_bits = $param->{pg_bits} || 9; my $size = $param->{size} || 2; + my $fsid; + my $uuid; + + UUID::generate($uuid); + UUID::unparse($uuid, $fsid); + my $global = { + 'fsid' => $fsid, 'auth supported' => 'cephx', 'auth cluster required' => 'cephx', 'auth service required' => 'cephx', diff --git a/debian/control.in b/debian/control.in index a70c3191..fa99715d 100644 --- a/debian/control.in +++ b/debian/control.in @@ -3,7 +3,7 @@ Version: @VERSION@-@PACKAGERELEASE@ Section: admin Priority: optional Architecture: amd64 -Depends: perl (>= 5.10.0-19), libtimedate-perl, libauthen-pam-perl, libintl-perl, rsync, libjson-perl, liblockfile-simple-perl, vncterm, qemu-server (>= 1.1-1), libwww-perl (>= 6.04-1), libnet-http-perl (>= 6.06-1), libhttp-daemon-perl, wget, libnet-dns-perl, vlan, ifenslave-2.6 (>= 1.1.0-10), liblinux-inotify2-perl, debconf (>= 0.5) | debconf-2.0, netcat-traditional, pve-cluster (>= 1.0-29), libpve-common-perl, libpve-storage-perl, libterm-readline-gnu-perl, libpve-access-control (>= 3.0-2), libio-socket-ssl-perl, libfilesys-df-perl, libfile-readbackwards-perl, libfile-sync-perl, redhat-cluster-pve, resource-agents-pve, fence-agents-pve, cstream, postfix | mail-transport-agent, libxml-parser-perl, lzop, dtach, libanyevent-perl, liburi-perl, logrotate, libanyevent-http-perl, apt-transport-https, libapt-pkg-perl, libcrypt-ssleay-perl, liblwp-protocol-https-perl +Depends: perl (>= 5.10.0-19), libtimedate-perl, libauthen-pam-perl, libintl-perl, rsync, libjson-perl, liblockfile-simple-perl, vncterm, qemu-server (>= 1.1-1), libwww-perl (>= 6.04-1), libnet-http-perl (>= 6.06-1), libhttp-daemon-perl, wget, libnet-dns-perl, vlan, ifenslave-2.6 (>= 1.1.0-10), liblinux-inotify2-perl, debconf (>= 0.5) | debconf-2.0, netcat-traditional, pve-cluster (>= 1.0-29), libpve-common-perl, libpve-storage-perl, libterm-readline-gnu-perl, libpve-access-control (>= 3.0-2), libio-socket-ssl-perl, libfilesys-df-perl, libfile-readbackwards-perl, libfile-sync-perl, redhat-cluster-pve, resource-agents-pve, fence-agents-pve, cstream, postfix | mail-transport-agent, libxml-parser-perl, lzop, dtach, libanyevent-perl, liburi-perl, logrotate, libanyevent-http-perl, apt-transport-https, libapt-pkg-perl, libcrypt-ssleay-perl, liblwp-protocol-https-perl, libuuid-perl Conflicts: netcat-openbsd, vzdump Replaces: vzdump Provides: vzdump From a08988dc4841d161286ada76c4b6d214f0d35b8f Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 20 Nov 2013 11:28:23 +0100 Subject: [PATCH 20/27] allow to call ceph init multiple times --- PVE/API2/Ceph.pm | 66 +++++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm index 5d733104..b2655ef7 100644 --- a/PVE/API2/Ceph.pm +++ b/PVE/API2/Ceph.pm @@ -508,7 +508,7 @@ __PACKAGE__->register_method ({ name => 'init', path => 'init', method => 'POST', - description => "Create initial ceph configuration.", + description => "Create initial ceph default configuration and setup symlinks.", proxyto => 'node', protected => 1, parameters => { @@ -526,7 +526,7 @@ __PACKAGE__->register_method ({ pg_bits => { description => "Placement group bits, used to specify the default number of placement groups (Note: 'osd pool default pg num' does not work for deafult pools)", type => 'integer', - default => 9, + default => 6, optional => 1, minimum => 6, maximum => 14, @@ -539,38 +539,45 @@ __PACKAGE__->register_method ({ &$check_ceph_installed(); - -f $pve_ceph_cfgpath && - die "configuration file '$pve_ceph_cfgpath' already exists.\n"; + # simply load old config if it already exists + my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); - my $pg_bits = $param->{pg_bits} || 9; - my $size = $param->{size} || 2; + my $keyring = '/etc/pve/priv/$cluster.$name.keyring'; - my $fsid; - my $uuid; + if (!$cfg->{global}) { - UUID::generate($uuid); - UUID::unparse($uuid, $fsid); + my $fsid; + my $uuid; - my $global = { - 'fsid' => $fsid, - 'auth supported' => 'cephx', - 'auth cluster required' => 'cephx', - 'auth service required' => 'cephx', - 'auth client required' => 'cephx', - 'keyring' => '/etc/pve/priv/$cluster.$name.keyring', - 'filestore xattr use omap' => 'true', - 'osd journal size' => '1024', - 'osd pool default size' => $size, - 'osd pool default min size' => 1, - 'osd pg bits' => $pg_bits, - 'osd pgp bits' => $pg_bits, - }; + UUID::generate($uuid); + UUID::unparse($uuid, $fsid); - # this does not work for default pools - #'osd pool default pg num' => $pg_num, - #'osd pool default pgp num' => $pg_num, + $cfg->{global} = { + 'fsid' => $fsid, + 'auth supported' => 'cephx', + 'auth cluster required' => 'cephx', + 'auth service required' => 'cephx', + 'auth client required' => 'cephx', + 'filestore xattr use omap' => 'true', + 'osd journal size' => '1024', + 'osd pool default min size' => 1, + }; - &$write_ceph_config({global => $global}); + # this does not work for default pools + #'osd pool default pg num' => $pg_num, + #'osd pool default pgp num' => $pg_num, + } + + $cfg->{global}->{keyring} = $keyring; + + $cfg->{global}->{'osd pool default size'} = $param->{size} if $param->{size}; + + if ($param->{pg_bits}) { + $cfg->{global}->{'osd pg bits'} = $param->{pg_bits}; + $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits}; + } + + &$write_ceph_config($cfg); &$setup_pve_symlinks(); @@ -1077,9 +1084,6 @@ __PACKAGE__->register_method ({ &$check_ceph_inited(); - die "not fully configured - missing '$pve_ckeyring_path'\n" - if ! -f $pve_ckeyring_path; - &$setup_pve_symlinks(); print "create OSD on $param->{dev}\n"; From 0b1efc01d5474707ecad5b98090eb39ea3e39623 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 21 Nov 2013 07:34:30 +0100 Subject: [PATCH 21/27] fix mds permissions in client.admin key --- PVE/API2/Ceph.pm | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm index b2655ef7..046c2d0f 100644 --- a/PVE/API2/Ceph.pm +++ b/PVE/API2/Ceph.pm @@ -613,14 +613,13 @@ __PACKAGE__->register_method ({ if (! -f $pve_mon_key_path) { run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp"); run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " . - "--cap mds 'allow *' " . + "--cap mds 'allow' " . "--cap osd 'allow *' " . "--cap mon 'allow *'"); run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'"); run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path"); } - my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); my $moncount = 0; From 8214fc6f212098f7ea26ab344cc9137189b359e6 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 21 Nov 2013 08:00:30 +0100 Subject: [PATCH 22/27] depend on hdparm, because ceph-osd uses that --- debian/control.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control.in b/debian/control.in index fa99715d..a13d7acd 100644 --- a/debian/control.in +++ b/debian/control.in @@ -3,7 +3,7 @@ Version: @VERSION@-@PACKAGERELEASE@ Section: admin Priority: optional Architecture: amd64 -Depends: perl (>= 5.10.0-19), libtimedate-perl, libauthen-pam-perl, libintl-perl, rsync, libjson-perl, liblockfile-simple-perl, vncterm, qemu-server (>= 1.1-1), libwww-perl (>= 6.04-1), libnet-http-perl (>= 6.06-1), libhttp-daemon-perl, wget, libnet-dns-perl, vlan, ifenslave-2.6 (>= 1.1.0-10), liblinux-inotify2-perl, debconf (>= 0.5) | debconf-2.0, netcat-traditional, pve-cluster (>= 1.0-29), libpve-common-perl, libpve-storage-perl, libterm-readline-gnu-perl, libpve-access-control (>= 3.0-2), libio-socket-ssl-perl, libfilesys-df-perl, libfile-readbackwards-perl, libfile-sync-perl, redhat-cluster-pve, resource-agents-pve, fence-agents-pve, cstream, postfix | mail-transport-agent, libxml-parser-perl, lzop, dtach, libanyevent-perl, liburi-perl, logrotate, libanyevent-http-perl, apt-transport-https, libapt-pkg-perl, libcrypt-ssleay-perl, liblwp-protocol-https-perl, libuuid-perl +Depends: perl (>= 5.10.0-19), libtimedate-perl, libauthen-pam-perl, libintl-perl, rsync, libjson-perl, liblockfile-simple-perl, vncterm, qemu-server (>= 1.1-1), libwww-perl (>= 6.04-1), libnet-http-perl (>= 6.06-1), libhttp-daemon-perl, wget, libnet-dns-perl, vlan, ifenslave-2.6 (>= 1.1.0-10), liblinux-inotify2-perl, debconf (>= 0.5) | debconf-2.0, netcat-traditional, pve-cluster (>= 1.0-29), libpve-common-perl, libpve-storage-perl, libterm-readline-gnu-perl, libpve-access-control (>= 3.0-2), libio-socket-ssl-perl, libfilesys-df-perl, libfile-readbackwards-perl, libfile-sync-perl, redhat-cluster-pve, resource-agents-pve, fence-agents-pve, cstream, postfix | mail-transport-agent, libxml-parser-perl, lzop, dtach, libanyevent-perl, liburi-perl, logrotate, libanyevent-http-perl, apt-transport-https, libapt-pkg-perl, libcrypt-ssleay-perl, liblwp-protocol-https-perl, libuuid-perl, hdparm Conflicts: netcat-openbsd, vzdump Replaces: vzdump Provides: vzdump From 88a6e29ad19a5c5c4111d03bc6040f16d2a7ee7c Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 21 Nov 2013 09:11:02 +0100 Subject: [PATCH 23/27] set correct keyring location for osd --- PVE/API2/Ceph.pm | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm index 046c2d0f..b01663e3 100644 --- a/PVE/API2/Ceph.pm +++ b/PVE/API2/Ceph.pm @@ -542,8 +542,6 @@ __PACKAGE__->register_method ({ # simply load old config if it already exists my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); - my $keyring = '/etc/pve/priv/$cluster.$name.keyring'; - if (!$cfg->{global}) { my $fsid; @@ -568,7 +566,8 @@ __PACKAGE__->register_method ({ #'osd pool default pgp num' => $pg_num, } - $cfg->{global}->{keyring} = $keyring; + $cfg->{global}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring'; + $cfg->{osd}->{keyring} = "/var/lib/ceph/osd/ceph-$id/keyring"; $cfg->{global}->{'osd pool default size'} = $param->{size} if $param->{size}; From 52d7be4197d2bed1457e30203fbfbde0ce8e16e1 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Fri, 22 Nov 2013 08:41:37 +0100 Subject: [PATCH 24/27] fix osd keyring, use tasks --- PVE/API2/Ceph.pm | 176 +++++++++++++++++++++++---------------- bin/pveceph | 14 ++-- www/manager/Utils.js | 4 + www/manager/node/Ceph.js | 3 +- 4 files changed, 115 insertions(+), 82 deletions(-) diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm index b01663e3..a3dc322c 100644 --- a/PVE/API2/Ceph.pm +++ b/PVE/API2/Ceph.pm @@ -567,7 +567,7 @@ __PACKAGE__->register_method ({ } $cfg->{global}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring'; - $cfg->{osd}->{keyring} = "/var/lib/ceph/osd/ceph-$id/keyring"; + $cfg->{osd}->{keyring} = '/var/lib/ceph/osd/ceph-$id/keyring'; $cfg->{global}->{'osd pool default size'} = $param->{size} if $param->{size}; @@ -596,7 +596,7 @@ __PACKAGE__->register_method ({ node => get_standard_option('pve-node'), }, }, - returns => { type => 'null' }, + returns => { type => 'string' }, code => sub { my ($param) = @_; @@ -604,20 +604,9 @@ __PACKAGE__->register_method ({ &$setup_pve_symlinks(); - if (! -f $pve_ckeyring_path) { - run_command("ceph-authtool $pve_ckeyring_path --create-keyring " . - "--gen-key -n client.admin"); - } + my $rpcenv = PVE::RPCEnvironment::get(); - if (! -f $pve_mon_key_path) { - run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp"); - run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " . - "--cap mds 'allow' " . - "--cap osd 'allow *' " . - "--cap mon 'allow *'"); - run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'"); - run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path"); - } + my $authuser = $rpcenv->get_user(); my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); @@ -653,42 +642,59 @@ __PACKAGE__->register_method ({ die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n" if $monaddrhash->{$monaddr}; + my $worker = sub { + my $upid = shift; - my $mondir = "/var/lib/ceph/mon/$ccname-$monid"; - -d $mondir && die "monitor filesystem '$mondir' already exist\n"; - - my $monmap = "/tmp/monmap"; - - eval { - mkdir $mondir; - - if ($moncount > 0) { - my $monstat = ceph_mon_status(); # online test - &$run_ceph_cmd(['mon', 'getmap', '-o', $monmap]); - } else { - run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap"); + if (! -f $pve_ckeyring_path) { + run_command("ceph-authtool $pve_ckeyring_path --create-keyring " . + "--gen-key -n client.admin"); } - run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path"); - }; - my $err = $@; - unlink $monmap; - if ($err) { - File::Path::remove_tree($mondir); - die $err; - } + if (! -f $pve_mon_key_path) { + run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp"); + run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " . + "--cap mds 'allow' " . + "--cap osd 'allow *' " . + "--cap mon 'allow *'"); + run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'"); + run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path"); + } - $cfg->{$monsection} = { - 'host' => $monname, - 'mon addr' => $monaddr, + my $mondir = "/var/lib/ceph/mon/$ccname-$monid"; + -d $mondir && die "monitor filesystem '$mondir' already exist\n"; + + my $monmap = "/tmp/monmap"; + + eval { + mkdir $mondir; + + if ($moncount > 0) { + my $monstat = ceph_mon_status(); # online test + &$run_ceph_cmd(['mon', 'getmap', '-o', $monmap]); + } else { + run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap"); + } + + run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path"); + }; + my $err = $@; + unlink $monmap; + if ($err) { + File::Path::remove_tree($mondir); + die $err; + } + + $cfg->{$monsection} = { + 'host' => $monname, + 'mon addr' => $monaddr, + }; + + &$write_ceph_config($cfg); + + &$ceph_service_cmd('start', $monsection); }; - &$write_ceph_config($cfg); - - &$ceph_service_cmd('start', $monsection); - - return undef; - + return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker); }}); __PACKAGE__->register_method ({ @@ -708,10 +714,14 @@ __PACKAGE__->register_method ({ }, }, }, - returns => { type => 'null' }, + returns => { type => 'string' }, code => sub { my ($param) = @_; + my $rpcenv = PVE::RPCEnvironment::get(); + + my $authuser = $rpcenv->get_user(); + &$check_ceph_inited(); my $cfg = &$parse_ceph_config($pve_ceph_cfgpath); @@ -731,16 +741,20 @@ __PACKAGE__->register_method ({ die "can't remove last monitor\n" if scalar(@$monlist) <= 1; - &$run_ceph_cmd(['mon', 'remove', $monid]); + my $worker = sub { + my $upid = shift; - eval { &$ceph_service_cmd('stop', $monsection); }; - warn $@ if $@; + &$run_ceph_cmd(['mon', 'remove', $monid]); - delete $cfg->{$monsection}; - &$write_ceph_config($cfg); - File::Path::remove_tree($mondir); + eval { &$ceph_service_cmd('stop', $monsection); }; + warn $@ if $@; - return undef; + delete $cfg->{$monsection}; + &$write_ceph_config($cfg); + File::Path::remove_tree($mondir); + }; + + return $rpcenv->fork_worker('cephdestroymon', $monsection, $authuser, $worker); }}); __PACKAGE__->register_method ({ @@ -1076,16 +1090,18 @@ __PACKAGE__->register_method ({ } }, }, - returns => { type => 'null' }, + returns => { type => 'string' }, code => sub { my ($param) = @_; + my $rpcenv = PVE::RPCEnvironment::get(); + + my $authuser = $rpcenv->get_user(); + &$check_ceph_inited(); &$setup_pve_symlinks(); - print "create OSD on $param->{dev}\n"; - -b $param->{dev} || die "no such block device '$param->{dev}'\n"; my $disklist = list_disks(); @@ -1108,11 +1124,17 @@ __PACKAGE__->register_method ({ &$run_ceph_cmd(['auth', 'get', 'client.bootstrap-osd', '-o', $ceph_bootstrap_osd_keyring]); }; - run_command(['ceph-disk', 'prepare', '--zap-disk', '--fs-type', 'xfs', - '--cluster', $ccname, '--cluster-uuid', $fsid, - '--', $param->{dev}]); + my $worker = sub { + my $upid = shift; - return undef; + print "create OSD on $param->{dev}\n"; + + run_command(['ceph-disk', 'prepare', '--zap-disk', '--fs-type', 'xfs', + '--cluster', $ccname, '--cluster-uuid', $fsid, + '--', $param->{dev}]); + }; + + return $rpcenv->fork_worker('cephcreateods', $param->{dev}, $authuser, $worker); }}); __PACKAGE__->register_method ({ @@ -1132,17 +1154,19 @@ __PACKAGE__->register_method ({ }, }, }, - returns => { type => 'null' }, + returns => { type => 'string' }, code => sub { my ($param) = @_; + my $rpcenv = PVE::RPCEnvironment::get(); + + my $authuser = $rpcenv->get_user(); + &$check_ceph_inited(); my $osdid = $param->{osdid}; - print "destroy OSD $param->{osdid}\n"; - - # fixme: not sure what we should do here + # fixme: not 100% sure what we should do here my $stat = &$ceph_osd_status(); @@ -1164,19 +1188,25 @@ __PACKAGE__->register_method ({ my $osdsection = "osd.$osdid"; - eval { &$ceph_service_cmd('stop', $osdsection); }; - warn $@ if $@; + my $worker = sub { + my $upid = shift; - print "Remove $osdsection from the CRUSH map\n"; - &$run_ceph_cmd(['osd', 'crush', 'remove', $osdid]); + print "destroy OSD $param->{osdid}\n"; - print "Remove the $osdsection authentication key.\n"; - &$run_ceph_cmd(['auth', 'del', $osdsection]); + eval { &$ceph_service_cmd('stop', $osdsection); }; + warn $@ if $@; - print "Remove OSD $osdsection\n"; - &$run_ceph_cmd(['osd', 'rm', $osdid]); + print "Remove $osdsection from the CRUSH map\n"; + &$run_ceph_cmd(['osd', 'crush', 'remove', $osdid]); - return undef; + print "Remove the $osdsection authentication key.\n"; + &$run_ceph_cmd(['auth', 'del', $osdsection]); + + print "Remove OSD $osdsection\n"; + &$run_ceph_cmd(['osd', 'rm', $osdid]); + }; + + return $rpcenv->fork_worker('cephdestroyods', $osdsection, $authuser, $worker); }}); __PACKAGE__->register_method ({ diff --git a/bin/pveceph b/bin/pveceph index f9844604..96ca63c9 100755 --- a/bin/pveceph +++ b/bin/pveceph @@ -124,14 +124,12 @@ my $cmddef = { }], createpool => [ 'PVE::API2::Ceph', 'createpool', ['name'], { node => $nodename }], destroypool => [ 'PVE::API2::Ceph', 'destroypool', ['name'], { node => $nodename } ], - createosd => [ 'PVE::API2::Ceph', 'createosd', ['dev'], { node => $nodename } ], - destroyosd => [ 'PVE::API2::Ceph', 'destroyosd', ['osdid'], { node => $nodename } ], - createmon => [ 'PVE::API2::Ceph', 'createmon', [], { node => $nodename } ], - destroymon => [ 'PVE::API2::Ceph', 'destroymon', ['monid'], { node => $nodename } ], - start => [ 'PVE::API2::Ceph', 'start', ['service'], { node => $nodename }, - $upid_exit], - stop => [ 'PVE::API2::Ceph', 'stop', ['service'], { node => $nodename }, - $upid_exit], + createosd => [ 'PVE::API2::Ceph', 'createosd', ['dev'], { node => $nodename }, $upid_exit], + destroyosd => [ 'PVE::API2::Ceph', 'destroyosd', ['osdid'], { node => $nodename }, $upid_exit], + createmon => [ 'PVE::API2::Ceph', 'createmon', [], { node => $nodename }, $upid_exit], + destroymon => [ 'PVE::API2::Ceph', 'destroymon', ['monid'], { node => $nodename }, $upid_exit], + start => [ 'PVE::API2::Ceph', 'start', ['service'], { node => $nodename }, $upid_exit], + stop => [ 'PVE::API2::Ceph', 'stop', ['service'], { node => $nodename }, $upid_exit], install => [ __PACKAGE__, 'install', [] ], purge => [ __PACKAGE__, 'purge', [] ], status => [ 'PVE::API2::Ceph', 'status', [], { node => $nodename }, sub { diff --git a/www/manager/Utils.js b/www/manager/Utils.js index 9cd9959d..9d06efbe 100644 --- a/www/manager/Utils.js +++ b/www/manager/Utils.js @@ -536,6 +536,10 @@ Ext.define('PVE.Utils', { statics: { srvstop: ['SRV', gettext('Stop') ], srvrestart: ['SRV', gettext('Restart') ], srvreload: ['SRV', gettext('Reload') ], + cephcreatemon: ['Ceph Monitor', gettext('Create') ], + cephdestroymon: ['Ceph Monitor', gettext('Destroy') ], + cephcreateosd: ['Ceph OSD', gettext('Create') ], + cephdestroyosd: ['Ceph OSD', gettext('Destroy') ], imgcopy: ['', gettext('Copy data') ], imgdel: ['', gettext('Erase data') ], download: ['', gettext('Download') ], diff --git a/www/manager/node/Ceph.js b/www/manager/node/Ceph.js index 505a4745..861f4a67 100644 --- a/www/manager/node/Ceph.js +++ b/www/manager/node/Ceph.js @@ -484,7 +484,7 @@ Ext.define('PVE.CephCreateMon', { { xtype: 'PVE.form.NodeSelector', submitValue: false, - fieldLabel: gettext('Comment'), + fieldLabel: gettext('Host'), selectCurNode: true, allowBlank: false, listeners: { @@ -797,6 +797,7 @@ Ext.define('PVE.node.CephStatus', { } var txt = value.overall_status; + Ext.Array.each(value.summary, function(d) { txt += " " + d.summary + ';'; }); From b1389cd438236d9bbf749d48580af84dfa5eded3 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Fri, 22 Nov 2013 09:04:18 +0100 Subject: [PATCH 25/27] use newer ceph emperor packages --- bin/pveceph | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/pveceph b/bin/pveceph index 96ca63c9..58ec71d6 100755 --- a/bin/pveceph +++ b/bin/pveceph @@ -86,7 +86,7 @@ __PACKAGE__->register_method ({ code => sub { my ($param) = @_; - my $cephver = 'dumpling'; + my $cephver = 'emperor'; local $ENV{DEBIAN_FRONTEND} = 'noninteractive'; From c56a583f4e5511c1f2f7fa9925d2f44058057167 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Fri, 22 Nov 2013 09:17:42 +0100 Subject: [PATCH 26/27] use new --connect-timeout for ceph commands --- PVE/API2/Ceph.pm | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm index a3dc322c..fb37c443 100644 --- a/PVE/API2/Ceph.pm +++ b/PVE/API2/Ceph.pm @@ -127,24 +127,12 @@ my $parse_ceph_config = sub { my $run_ceph_cmd = sub { my ($cmd, %params) = @_; + + my $timeout = 5; - my $timeout = 3; - - my $oldalarm; - eval { - local $SIG{ALRM} = sub { die "timeout\n" }; - $oldalarm = alarm($timeout); - # Note: --connect-timeout does not work with current version - # '--connect-timeout', $timeout, - - run_command(['ceph', '-c', $pve_ceph_cfgpath, @$cmd], %params); - alarm(0); - }; - my $err = $@; - - alarm($oldalarm) if $oldalarm; - - die $err if $err; + run_command(['ceph', '-c', $pve_ceph_cfgpath, + '--connect-timeout', $timeout, + @$cmd], %params); }; my $run_ceph_cmd_text = sub { From 1fe6c0844389c879e1006182bdf2173e51bb289e Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Fri, 22 Nov 2013 11:06:50 +0100 Subject: [PATCH 27/27] use format_size to display ceph storage sizes --- www/manager/node/Ceph.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/www/manager/node/Ceph.js b/www/manager/node/Ceph.js index 861f4a67..108dab89 100644 --- a/www/manager/node/Ceph.js +++ b/www/manager/node/Ceph.js @@ -819,9 +819,9 @@ Ext.define('PVE.node.CephStatus', { }); txt += '; '; - txt += d.data_bytes + " bytes data, "; - txt += d.bytes_used + " bytes used, "; - txt += d.bytes_avail + " bytes avail"; + txt += PVE.Utils.format_size(d.data_bytes) + " data, "; + txt += PVE.Utils.format_size(d.bytes_used) + " used, "; + txt += PVE.Utils.format_size(d.bytes_avail) + " avail"; return txt; };