import pveceph script

bin/Makefile

@@ -3,6 +3,7 @@ include ../defines.mk
 SUBDIRS = init.d cron ocf test
 
 SCRIPTS =  			\
+	pveceph			\
 	vzdump			\
 	vzrestore		\
 	pvestatd		\
@@ -20,6 +21,7 @@ SCRIPTS =  			\
 	pveperf
 
 MANS = 				\
+	pveceph.1		\
 	pvectl.1		\
 	vzdump.1		\
 	vzrestore.1		\
@@ -44,6 +46,9 @@ all: ${MANS} pvemailforward
 pvectl.1.pod: pvectl
 	perl -I.. ./pvectl printmanpod >$@
 
+pveceph.1.pod: pveceph
+	perl -I.. ./pveceph printmanpod >$@
+
 vzdump.1.pod: vzdump
 	perl -I.. -T ./vzdump printmanpod >$@
 

bin/pveceph

+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use Getopt::Long;
+use Fcntl ':flock';
+use File::Path;
+use IO::File;
+use JSON;
+use Data::Dumper;
+
+use PVE::SafeSyslog;
+use PVE::Cluster;
+use PVE::INotify;
+use PVE::RPCEnvironment;
+use PVE::Storage;
+use PVE::Tools qw(run_command);
+use PVE::JSONSchema qw(get_standard_option);
+
+use PVE::CLIHandler;
+
+use base qw(PVE::CLIHandler);
+
+$ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin';
+
+initlog ('pveceph');
+
+die "please run as root\n" if $> != 0;
+
+PVE::INotify::inotify_init();
+
+my $rpcenv = PVE::RPCEnvironment->init('cli');
+
+$rpcenv->init_request();
+$rpcenv->set_language($ENV{LANG});
+$rpcenv->set_user('root@pam');
+
+my $nodename = PVE::INotify::nodename();
+
+my $ccname = 'ceph'; # ceph cluster name
+my $ceph_cfgdir = "/etc/ceph";
+my $pve_ceph_cfgpath = "/etc/pve/$ccname.conf";
+my $ceph_cfgpath = "$ceph_cfgdir/$ccname.conf";
+my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring";
+my $ceph_mon_key_path = "$ceph_cfgdir/$ccname.mon.keyring";
+my $pve_ckeyring_path = "/etc/pve/priv/$ccname.keyring";
+my $ceph_ckeyring_path = "$ceph_cfgdir/$ccname.client.admin.keyring";
+
+my $ceph_bootstrap_osd_keyring = "/var/lib/ceph/bootstrap-osd/$ccname.keyring";
+my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring";
+
+my $ceph_bin = "/usr/bin/ceph";
+
+my $check_ceph_installed = sub {
+    my ($noerr) = @_;
+
+    if (! -x $ceph_bin) {
+	die "ceph binaries not installed\n" if !$noerr;
+	return undef;
+    }
+
+    return 1;
+};
+
+my $check_ceph_inited = sub {
+    my ($noerr) = @_;
+
+    return undef if !&$check_ceph_installed($noerr);
+
+    if (! -f $pve_ceph_cfgpath) {
+	die "pveceph configuration not initialized\n" if !$noerr;
+	return undef;
+    }
+
+    return 1;
+};
+
+my $force_symlink = sub {
+    my ($old, $new) = @_;
+
+    return if (-l $new) && (readlink($new) eq $old);
+	
+    unlink $new;
+    symlink($old, $new) ||
+	die "unable to create symlink '$new' - $!\n";
+};
+
+my $parse_ceph_config = sub {
+    my ($filename) = @_;
+
+    my $cfg = {};
+
+    return $cfg if ! -f $filename;
+
+    my $fh = IO::File->new($filename, "r") ||
+	die "unable to open '$filename' - $!\n";
+
+    my $section;
+
+    while (defined(my $line = <$fh>)) {
+	$line =~ s/[;#].*$//;
+	$line =~ s/^\s+//;
+	$line =~ s/\s+$//;
+	next if !$line;
+
+	$section = $1 if $line =~ m/^\[(\S+)\]$/;
+	if (!$section) {
+	    warn "no section - skip: $line\n";
+	    next;
+	}
+
+	if ($line =~ m/^(.*\S)\s*=\s*(\S.*)$/) {
+	    $cfg->{$section}->{$1} = $2;
+	}
+
+    }
+
+    return $cfg;
+};
+
+my $run_ceph_cmd = sub {
+    my ($cmd, %params) = @_;
+
+    my $timeout = 3;
+
+    my $oldalarm;
+    eval {
+	local $SIG{ALRM} = sub { die "timeout\n" };
+	$oldalarm = alarm($timeout); 
+	# Note: --connect-timeout does not work with current version
+	# '--connect-timeout', $timeout,
+
+	run_command(['ceph', '-c', $ceph_cfgpath, @$cmd], %params);
+	alarm(0);
+    };
+    my $err = $@;
+
+    alarm($oldalarm) if $oldalarm;
+
+    die $err if $err;
+
+};
+
+my $ceph_mon_status = sub {
+    my ($quiet) = @_;
+
+    my $json = '';
+    my $parser = sub {
+	my $line = shift;
+	$json .= $line;
+    };
+
+    my $errfunc = sub {
+	my $line = shift;
+	print "$line\n" if !$quiet;
+    };
+
+    &$run_ceph_cmd(['mon_status'], outfunc => $parser, errfunc => $errfunc);
+
+    my $res = decode_json($json);
+
+    return $res;
+};
+
+my $ceph_osd_status = sub {
+    my ($quiet) = @_;
+
+    my $json = '';
+    my $parser = sub {
+	my $line = shift;
+	$json .= $line;
+    };
+
+    my $errfunc = sub {
+	my $line = shift;
+	print "$line\n" if !$quiet;
+    };
+
+    &$run_ceph_cmd(['osd', 'dump', '--format', 'json'], 
+		   outfunc => $parser, errfunc => $errfunc);
+
+    my $res = decode_json($json);
+
+    return $res;
+};
+
+my $write_ceph_config = sub {
+    my ($cfg) = @_;
+
+    my $out = '';
+    foreach my $section (keys %$cfg) {
+	$out .= "[$section]\n";
+	foreach my $key (sort keys %{$cfg->{$section}}) {
+	    $out .= "\t $key = $cfg->{$section}->{$key}\n";
+	}
+	$out .= "\n";
+    }
+
+    PVE::Tools::file_set_contents($pve_ceph_cfgpath, $out);
+};
+
+my $setup_pve_symlinks = sub {
+    # fail if we find a real file instead of a link
+    if (-f $ceph_cfgpath) {
+	my $lnk = readlink($ceph_cfgpath);
+	die "file '$ceph_cfgpath' already exists\n"
+	    if !$lnk || $lnk ne $pve_ceph_cfgpath;
+    }
+
+    # now assume we are allowed to setup/overwrite content 
+    &$force_symlink($pve_ceph_cfgpath, $ceph_cfgpath);
+    &$force_symlink($pve_mon_key_path, $ceph_mon_key_path);
+    &$force_symlink($pve_ckeyring_path, $ceph_ckeyring_path);
+};
+
+my $ceph_service_cmd = sub {
+    run_command(['service', 'ceph', '-c', $ceph_cfgpath, @_]);
+};
+
+__PACKAGE__->register_method ({
+    name => 'init',
+    path => 'init',
+    method => 'POST',
+    description => "Create initial ceph configuration.",
+    parameters => {
+    	additionalProperties => 0,
+	properties => {
+	    size => {
+		description => 'Number of replicas per object',
+		type => 'interger',
+		default => 2,
+		optional => 1,
+		minimum => 1,
+		maximum => 3,
+	    },
+	    pg_bits => {
+		description => "Placement group bits, used to specify the default number of placement groups (Note: 'osd pool default pg num' does not work for deafult pools)",
+		type => 'interger',
+		default => 9,
+		optional => 1,
+		minimum => 6,
+		maximum => 14,
+	    },
+	},
+    },
+    returns => { type => 'null' },
+    code => sub {
+	my ($param) = @_;
+
+	&$check_ceph_installed();
+
+	-f $pve_ceph_cfgpath &&
+	    die "configuration file '$pve_ceph_cfgpath' already exists.\n";
+
+	my $pg_bits = $param->{pg_bits} || 9;
+	my $size = $param->{size} || 2;
+
+	my $global = {
+	    'auth supported' => 'cephx',
+	    'auth cluster required' => 'cephx',
+	    'auth service required' => 'cephx',
+	    'auth client required' => 'cephx',
+	    'filestore xattr use omap' => 'true',
+	    'osd journal size' => '1024',
+	    'osd pool default size' => $size,
+	    'osd pool default min size' => 1,
+	    'osd pg bits' => $pg_bits,
+	    'osd pgp bits' => $pg_bits,
+	};
+
+	# this does not work for default pools 
+	#'osd pool default pg num' => $pg_num,
+	#'osd pool default pgp num' => $pg_num, 
+
+	&$write_ceph_config({global => $global});
+
+	&$setup_pve_symlinks();
+
+	return undef;
+    }});
+
+__PACKAGE__->register_method ({
+    name => 'createmon',
+    path => 'createmon',
+    method => 'POST',
+    description => "Create Ceph Monitor",
+    parameters => {
+    	additionalProperties => 0,
+	properties => {
+	},
+    },
+    returns => { type => 'null' },
+    code => sub {
+	my ($param) = @_;
+
+	&$check_ceph_inited();
+
+	&$setup_pve_symlinks();
+
+	if (! -f $pve_ckeyring_path) {
+	    run_command("ceph-authtool $pve_ckeyring_path --create-keyring " .
+			"--gen-key -n client.admin");
+	}
+
+	if (! -f $pve_mon_key_path) {
+	    run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp");
+	    run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " .
+			"--cap mds 'allow *' " .
+			"--cap osd 'allow *' " .
+			"--cap mon 'allow *'");
+	    run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'");
+	    run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path");
+	}
+
+
+	my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
+
+	my $moncount = 0;
+
+	my $monaddrhash = {}; 
+
+	foreach my $section (keys %$cfg) {
+	    next if $section eq 'global';
+	    my $d = $cfg->{$section};
+	    if ($section =~ m/^mon\./) {
+		$moncount++;
+		if ($d->{'mon addr'}) {
+		    $monaddrhash->{$d->{'mon addr'}} = $section;
+		}
+	    }
+	}
+
+	my $monid;
+	for (my $i = 0; $i < 7; $i++) {
+	    if (!$cfg->{"mon.$i"}) {
+		$monid = $i;
+		last;
+	    }
+	}
+	die "unable to find usable monitor id\n" if !defined($monid);
+
+	my $monsection = "mon.$monid";	
+	my $monaddr = PVE::Cluster::remote_node_ip($nodename) . ":6789";
+	my $monname = $nodename;
+
+	die "monitor '$monsection' already exists\n" if $cfg->{$monsection};
+	die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n" 
+	    if $monaddrhash->{$monaddr};
+
+
+	my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
+	-d $mondir && die "monitor filesystem '$mondir' already exist\n";
+ 
+	my $monmap = "/tmp/monmap";
+	
+	eval {
+	    mkdir $mondir;
+
+	    if ($moncount > 0) {
+		my $monstat = &$ceph_mon_status(); # online test
+		&$run_ceph_cmd(['mon', 'getmap', '-o', $monmap]);
+	    } else {
+		run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap");
+	    }
+
+	    run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path");
+	};
+	my $err = $@;
+	unlink $monmap;
+	if ($err) {
+	    File::Path::remove_tree($mondir);
+	    die $err;
+	}
+
+	$cfg->{$monsection} = {
+	    'host' => $monname,
+	    'mon addr' => $monaddr,
+	};
+
+	&$write_ceph_config($cfg);
+
+	&$ceph_service_cmd('start', $monsection);
+
+	return undef;
+
+    }});
+
+__PACKAGE__->register_method ({
+    name => 'destroymon',
+    path => 'destroymon',
+    method => 'POST',
+    description => "Destroy Ceph monitor.",
+    parameters => {
+    	additionalProperties => 0,
+	properties => {
+	    monid => {
+		destription => 'Monitor ID',
+		type => 'integer',
+	    },
+	},
+    },
+    returns => { type => 'null' },
+    code => sub {
+	my ($param) = @_;
+
+	&$check_ceph_inited();
+
+	my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
+       
+	my $monid = $param->{monid};
+	my $monsection = "mon.$monid";	
+
+	my $monstat = &$ceph_mon_status();
+	my $monlist = $monstat->{monmap}->{mons};
+
+	die "no such monitor id '$monid'\n" 
+	    if !defined($cfg->{$monsection});
+
+
+	my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
+	-d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n";
+
+	die "can't remove last monitor\n" if scalar(@$monlist) <= 1;
+
+	&$run_ceph_cmd(['mon', 'remove', $monid]);
+
+	eval { &$ceph_service_cmd('stop', $monsection); };
+	warn $@ if $@;
+
+	delete $cfg->{$monsection};
+	&$write_ceph_config($cfg);
+	File::Path::remove_tree($mondir);
+
+	return undef;
+    }});
+
+__PACKAGE__->register_method ({
+    name => 'purge',
+    path => 'purge',
+    method => 'POST',
+    description => "Destroy ceph related data and configuration files.",
+    parameters => {
+    	additionalProperties => 0,
+	properties => {
+	},
+    },
+    returns => { type => 'null' },
+    code => sub {
+	my ($param) = @_;
+
+	my $monstat;
+
+	eval { $monstat	= &$ceph_mon_status(1); };
+	my $err = $@;
+
+	die "detected running ceph services- unable to purge data\n" 
+	    if !$err;
+
+	# fixme: this is dangerous - should we really support this function?
+
+	unlink $ceph_cfgpath;
+	unlink $ceph_mon_key_path;
+	unlink $ceph_ckeyring_path;
+
+	unlink $pve_ceph_cfgpath;
+	unlink $pve_ckeyring_path;
+	unlink $pve_mon_key_path;
+
+	unlink $ceph_bootstrap_osd_keyring;
+	unlink $ceph_bootstrap_mds_keyring;
+
+	system("rm -rf /var/lib/ceph/mon/ceph-*");
+
+	# remove osd?
+
+	return undef;
+    }});
+
+__PACKAGE__->register_method ({
+    name => 'install',
+    path => 'install',
+    method => 'POST',
+    description => "Install ceph related packages.",
+    parameters => {
+    	additionalProperties => 0,
+	properties => {
+	},
+    },
+    returns => { type => 'null' },
+    code => sub {
+	my ($param) = @_;
+
+	my $cephver = 'dumpling';
+
+	local $ENV{DEBIAN_FRONTEND} = 'noninteractive';
+
+	my $keyurl = "https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc";
+
+	print "download and import ceph reqpository keys\n";
+	system("wget -q -O- '$keyurl'| apt-key add - 2>&1 >/dev/null") == 0 ||
+	    die "unable to download ceph release key\n";
+
+
+	my $source = "deb http://ceph.com/debian-$cephver wheezy main\n";
+
+	PVE::Tools::file_set_contents("/etc/apt/sources.list.d/ceph.list", $source);
+
+	print "update available package list\n";
+	eval { run_command(['apt-get', '-q', 'update'], outfunc => sub {}, errfunc => sub {}); };
+
+	run_command(['apt-get', '-q', '--assume-yes', '--no-install-recommends',
+		     '-o', 'Dpkg::Options::=--force-confnew',
+		     'install', '--',
+		     'ceph', 'ceph-common', 'gdisk']);
+
+	return undef;
+    }});
+
+__PACKAGE__->register_method ({
+    name => 'stop',
+    path => 'stop',
+    method => 'POST',
+    description => "Stop ceph services.",
+    parameters => {
+    	additionalProperties => 0,
+	properties => {
+	},
+    },
+    returns => { type => 'null' },
+    code => sub {
+	my ($param) = @_;
+
+	&$check_ceph_inited();
+
+	my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
+	scalar(keys %$cfg) || die "no configuration\n";
+
+	&$ceph_service_cmd('stop');
+
+	return undef;
+    }});
+
+__PACKAGE__->register_method ({
+    name => 'start',
+    path => 'start',
+    method => 'POST',
+    description => "Start ceph services.",
+    parameters => {
+    	additionalProperties => 0,
+	properties => {
+	},
+    },
+    returns => { type => 'null' },
+    code => sub {
+	my ($param) = @_;
+
+	&$check_ceph_inited();
+
+	my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
+	scalar(keys %$cfg) || die "no configuration\n";
+
+	&$ceph_service_cmd('start');
+
+	return undef;
+    }});
+
+__PACKAGE__->register_method ({
+    name => 'status',
+    path => 'status',
+    method => 'GET',
+    description => "Get ceph status.",
+    parameters => {
+    	additionalProperties => 0,
+	properties => {
+	},
+    },
+    returns => { type => 'object' },
+    code => sub {
+	my ($param) = @_;
+
+	my $res = { status => 'unknown' };
+
+	eval {
+	    if (!&$check_ceph_installed(1)) {
+		$res->{status} = 'notinstalled';
+	    }
+	    if (! -f $ceph_cfgpath) {
+		$res->{status} = 'notconfigured';		
+		return;
+	    } else {
+		$res->{status} = 'configured';
+	    }
+
+	    my $cfg = &$parse_ceph_config($pve_ceph_cfgpath);
+	    $res->{config} = $cfg;
+
+	    eval { 
+		my $monstat = &$ceph_mon_status(1);
+		$res->{monstat} = $monstat;
+	    };
+	    warn $@ if $@;
+
+	    eval {
+		my $osdstat =  &$ceph_osd_status(1);
+		$res->{osdstat} = $osdstat;
+	    };
+	    warn $@ if $@;
+
+	};
+	warn $@ if $@;
+
+	return $res;
+    }});
+
+__PACKAGE__->register_method ({
+    name => 'createosd',
+    path => 'createosd',
+    method => 'POST',
+    description => "Create OSD",
+    parameters => {
+    	additionalProperties => 0,
+	properties => {
+	    dev => {
+		description => "Block device name.",
+		type => 'string',
+	    }
+	},
+    },
+    returns => { type => 'null' },
+    code => sub {
+	my ($param) = @_;
+
+	&$check_ceph_inited();
+
+	die "not fully configured - missing '$pve_ckeyring_path'\n" 
+	    if ! -f $pve_ckeyring_path;
+
+	&$setup_pve_symlinks();
+
+	print "create OSD on $param->{dev}\n";
+
+	-b  $param->{dev} || die "no such block device '$param->{dev}'\n";
+
+	my $monstat = &$ceph_mon_status(1);
+	die "unable to get fsid\n" if !$monstat->{monmap} || !$monstat->{monmap}->{fsid};
+	my $fsid = $monstat->{monmap}->{fsid};
+
+	if (! -f $ceph_bootstrap_osd_keyring) {
+	    &$run_ceph_cmd(['auth', 'get', 'client.bootstrap-osd', '-o', $ceph_bootstrap_osd_keyring]);
+	};
+
+	run_command(['ceph-disk', 'prepare', '--zap-disk', '--fs-type', 'xfs',
+		     '--cluster', $ccname, '--cluster-uuid', $fsid,
+		     '--', $param->{dev}]);
+
+	return undef;
+    }});
+
+__PACKAGE__->register_method ({
+    name => 'destroyosd',
+    path => 'destroyosd',
+    method => 'POST',
+    description => "Destroy OSD",
+    parameters => {
+    	additionalProperties => 0,
+	properties => {
+	    osdid => {
+		destription => 'OSD ID',
+		type => 'integer',
+	    },
+	},
+    },
+    returns => { type => 'null' },
+    code => sub {
+	my ($param) = @_;
+
+	&$check_ceph_inited();
+
+	my $osdid = $param->{osdid};
+
+	print "destroy OSD $param->{osdid}\n";
+
+	# fixme: not sure what we should do here
+ 
+	my $stat = &$ceph_osd_status();
+
+	my $osdlist = $stat->{osds} || [];
+
+	my $osdstat;
+	foreach my $d (@$osdlist) {
+	    if ($d->{osd} == $osdid) {
+		$osdstat = $d;
+		last;
+	    }
+	}
+	die "no such OSD '$osdid'\n" if !$osdstat;
+
+	die "osd is in use (in == 1)\n" if $osdstat->{in};
+	#&$run_ceph_cmd(['osd', 'out', $osdid]);
+
+	die "osd is still runnung (up == 1)\n" if $osdstat->{up};
+
+	my $osdsection = "osd.$osdid";	
+
+	eval { &$ceph_service_cmd('stop', $osdsection); };
+	warn $@ if $@;
+
+	print "Remove $osdsection from the CRUSH map\n";
+	&$run_ceph_cmd(['osd', 'crush', 'remove', $osdid]);
+
+	print "Remove the $osdsection authentication key.\n";
+	&$run_ceph_cmd(['auth', 'del', $osdsection]);
+
+	print "Remove OSD $osdsection\n";
+	&$run_ceph_cmd(['osd', 'rm', $osdid]);
+
+	return undef;
+    }});
+
+my $cmddef = {
+    init => [  __PACKAGE__, 'init', [] ],
+    createosd => [  __PACKAGE__, 'createosd', ['dev'] ],
+    destroyosd => [  __PACKAGE__, 'destroyosd', ['osdid'] ],
+    createmon => [  __PACKAGE__, 'createmon', [] ],
+    destroymon => [  __PACKAGE__, 'destroymon', ['monid'] ],
+    start => [  __PACKAGE__, 'start', [] ],
+    stop => [  __PACKAGE__, 'stop', [] ],
+    install => [  __PACKAGE__, 'install', [] ],
+    purge => [  __PACKAGE__, 'purge', [] ],
+    status => [  __PACKAGE__, 'status', [], undef, sub {
+	my $res = shift;
+	my $json = JSON->new->allow_nonref;
+	print $json->pretty->encode($res) . "\n";
+    }],
+};
+
+my $cmd = shift;
+
+PVE::CLIHandler::handle_cmd($cmddef, "pveceph", $cmd, \@ARGV, undef, $0);
+
+exit 0;
+
+__END__
+
+=head1 NAME
+
+pveceph - tool to manage ceph services on pve nodes
+
+=head1 SYNOPSIS
+
+=include synopsis
+
+=head1 DESCRIPTION
+
+Tool to manage ceph services on pve nodes.
+
+=include pve_copyright