Commit 120afef4 authored by Dietmar Maurer's avatar Dietmar Maurer

ceph: allow to specify separate journal disks

parent c93b26a7
......@@ -38,6 +38,8 @@ my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring";
my $ceph_bin = "/usr/bin/ceph";
my $pve_osd_default_journal_size = 1024*5;
sub purge_all_ceph_files {
# fixme: this is very dangerous - should we really support this function?
......@@ -222,7 +224,6 @@ my $ceph_service_cmd = sub {
run_command(['service', 'ceph', '-c', $pve_ceph_cfgpath, @_]);
};
sub list_disks {
my $disklist = {};
......@@ -260,6 +261,15 @@ sub list_disks {
return 1;
};
my $journal_uuid = '45b0969e-9b03-4f30-b4c6-b4b80ceff106';
my $journalhash = {};
dir_glob_foreach('/dev/disk/by-parttypeuuid', "$journal_uuid\..+", sub {
my ($entry) = @_;
my $real_dev = abs_path("/dev/disk/by-parttypeuuid/$entry");
$journalhash->{$real_dev} = 1;
});
dir_glob_foreach('/sys/block', '.*', sub {
my ($dev) = @_;
......@@ -292,42 +302,90 @@ sub list_disks {
$serial = $1;
}
my $gpt = 0;
if ($info =~ m/^E: ID_PART_TABLE_TYPE=gpt$/m) {
$gpt = 1;
}
# detect SSD (fixme - currently only works for ATA disks)
my $rpm = 7200; # default guess
if ($info =~ m/^E: ID_ATA_ROTATION_RATE_RPM=(\d+)$/m) {
$rpm = $1;
}
my $vendor = file_read_firstline("$devdir/vendor") || 'unknown';
my $model = file_read_firstline("$devdir/model") || 'unknown';
my $used = &$dir_is_epmty("/sys/block/$dev/holders") ? 0 : 1;
my $used;
$used = 'LVM' if !&$dir_is_epmty("/sys/block/$dev/holders");
$used = 1 if &$dev_is_mounted("/dev/$dev");
$used = 'mounted' if &$dev_is_mounted("/dev/$dev");
$disklist->{$dev} = {
vendor => $vendor,
model => $model,
size => $size,
serial => $serial,
gpt => $gpt,
rmp => $rpm,
};
my $osdid = -1;
my $journal_count = 0;
dir_glob_foreach("/sys/block/$dev", "$dev.+", sub {
my ($part) = @_;
if (!&$dir_is_epmty("/sys/block/$dev/$part/holders")) {
$used = 1;
}
if (my $mp = &$dev_is_mounted("/dev/$part")) {
$used = 1;
$used = 'mounted' if !$used;
if ($mp =~ m|^/var/lib/ceph/osd/ceph-(\d+)$|) {
$osdid = $1;
}
}
}
if (!&$dir_is_epmty("/sys/block/$dev/$part/holders")) {
$used = 'LVM' if !$used;
}
$used = 'partitions' if !$used;
$journal_count++ if $journalhash->{"/dev/$part"};
});
$disklist->{$dev}->{used} = $used;
$disklist->{$dev}->{used} = $used if $used;
$disklist->{$dev}->{osdid} = $osdid;
});
$disklist->{$dev}->{journals} = $journal_count;
});
return $disklist;
}
my $lookup_diskinfo = sub {
my ($disklist, $disk) = @_;
my $real_dev = abs_path($disk);
$real_dev =~ s|/dev/||;
my $diskinfo = $disklist->{$real_dev};
die "disk '$disk' not found in disk list\n" if !$diskinfo;
return wantarray ? ($diskinfo, $real_dev) : $diskinfo;
};
my $count_journal_disks = sub {
my ($disklist, $disk) = @_;
my $count = 0;
my ($diskinfo, $real_dev) = &$lookup_diskinfo($disklist, $disk);
die "journal disk '$disk' does not contain a GUID partition table\n"
if !$diskinfo->{gpt};
$count = $diskinfo->{journals} if $diskinfo->{journals};
return $count;
};
__PACKAGE__->register_method ({
name => 'index',
path => '',
......@@ -379,6 +437,12 @@ __PACKAGE__->register_method ({
additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
type => {
description => "Only list specific types of disks.",
type => 'string',
enum => ['unused', 'journal_disks'],
optional => 1,
},
},
},
returns => {
......@@ -387,7 +451,8 @@ __PACKAGE__->register_method ({
type => "object",
properties => {
dev => { type => 'string' },
used => { type => 'boolean' },
used => { type => 'string', optional => 1 },
gpt => { type => 'boolean' },
size => { type => 'integer' },
osdid => { type => 'integer' },
vendor => { type => 'string', optional => 1 },
......@@ -402,9 +467,27 @@ __PACKAGE__->register_method ({
&$check_ceph_inited();
my $res = list_disks();
my $disks = list_disks();
return PVE::RESTHandler::hash_to_array($res, 'dev');
my $res = [];
foreach my $dev (keys %$disks) {
my $d = $disks->{$dev};
if ($param->{type}) {
if ($param->{type} eq 'journal_disks') {
next if $d->{osdid} >= 0;
next if !$d->{gpt};
} elsif ($param->{type} eq 'unused') {
next if $d->{used};
} else {
die "internal error"; # should not happen
}
}
$d->{dev} = "/dev/$dev";
push @$res, $d;
}
return $res;
}});
__PACKAGE__->register_method ({
......@@ -552,7 +635,7 @@ __PACKAGE__->register_method ({
'auth service required' => 'cephx',
'auth client required' => 'cephx',
'filestore xattr use omap' => 'true',
'osd journal size' => '1024',
'osd journal size' => $pve_osd_default_journal_size,
'osd pool default min size' => 1,
};
......@@ -565,7 +648,7 @@ __PACKAGE__->register_method ({
$cfg->{osd}->{keyring} = '/var/lib/ceph/osd/ceph-$id/keyring';
$cfg->{global}->{'osd pool default size'} = $param->{size} if $param->{size};
if ($param->{pg_bits}) {
$cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
$cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
......@@ -1136,6 +1219,11 @@ __PACKAGE__->register_method ({
description => "Block device name.",
type => 'string',
},
journal_dev => {
description => "Block device name for journal.",
optional => 1,
type => 'string',
},
fstype => {
description => "File system type.",
type => 'string',
......@@ -1157,6 +1245,13 @@ __PACKAGE__->register_method ({
&$setup_pve_symlinks();
my $journal_dev;
if ($param->{journal_dev} && ($param->{journal_dev} ne $param->{dev})) {
-b $param->{journal_dev} || die "no such block device '$param->{journal_dev}'\n";
$journal_dev = $param->{journal_dev};
}
-b $param->{dev} || die "no such block device '$param->{dev}'\n";
my $disklist = list_disks();
......@@ -1186,9 +1281,17 @@ __PACKAGE__->register_method ({
print "create OSD on $param->{dev} ($fstype)\n";
run_command(['ceph-disk', 'prepare', '--zap-disk', '--fs-type', $fstype,
'--cluster', $ccname, '--cluster-uuid', $fsid,
'--', $param->{dev}]);
my $cmd = ['ceph-disk', 'prepare', '--zap-disk', '--fs-type', $fstype,
'--cluster', $ccname, '--cluster-uuid', $fsid ];
if ($journal_dev) {
print "using device '$journal_dev' for journal\n";
push @$cmd, '--journal-dev', $param->{dev}, $journal_dev;
} else {
push @$cmd, $param->{dev};
}
run_command($cmd);
};
return $rpcenv->fork_worker('cephcreateosd', $devname, $authuser, $worker);
......@@ -1209,6 +1312,12 @@ __PACKAGE__->register_method ({
description => 'OSD ID',
type => 'integer',
},
cleanup => {
description => "If set, we remove partition table entries.",
type => 'boolean',
optional => 1,
default => 0,
},
},
},
returns => { type => 'string' },
......@@ -1262,11 +1371,57 @@ __PACKAGE__->register_method ({
print "Remove OSD $osdsection\n";
&$run_ceph_cmd(['osd', 'rm', $osdid]);
# try to unmount fro standard mount point
# try to unmount from standard mount point
my $mountpoint = "/var/lib/ceph/osd/ceph-$osdid";
my $remove_partition = sub {
my ($disklist, $part) = @_;
return if !$part || (! -b $part );
foreach my $real_dev (keys %$disklist) {
my $diskinfo = $disklist->{$real_dev};
next if !$diskinfo->{gpt};
if ($part =~ m|^/dev/${real_dev}(\d+)$|) {
my $partnum = $1;
print "remove partition $part (disk '/dev/${real_dev}', partnum $partnum)\n";
eval { run_command(['/sbin/sgdisk', '-d', $partnum, "/dev/${real_dev}"]); };
warn $@ if $@;
last;
}
}
};
my $journal_part;
my $data_part;
if ($param->{cleanup}) {
my $jpath = "$mountpoint/journal";
$journal_part = abs_path($jpath);
if (my $fd = IO::File->new("/proc/mounts", "r")) {
while (defined(my $line = <$fd>)) {
my ($dev, $path, $fstype) = split(/\s+/, $line);
next if !($dev && $path && $fstype);
next if $dev !~ m|^/dev/|;
if ($path eq $mountpoint) {
$data_part = abs_path($dev);
last;
}
}
close($fd);
}
}
print "Unmount OSD $osdsection from $mountpoint\n";
eval { run_command(['umount', $mountpoint]); };
warn $@ if $@;
if (my $err = $@) {
warn $err;
} elsif ($param->{cleanup}) {
my $disklist = list_disks();
&$remove_partition($disklist, $journal_part);
&$remove_partition($disklist, $data_part);
}
};
return $rpcenv->fork_worker('cephdestroyosd', $osdsection, $authuser, $worker);
......
......@@ -191,6 +191,91 @@ Ext.define('PVE.node.CephPoolList', {
});
});
Ext.define('PVE.CephCreateOsd', {
extend: 'PVE.window.Edit',
alias: ['widget.pveCephCreateOsd'],
subject: 'Ceph OSD',
initComponent : function() {
/*jslint confusion: true */
var me = this;
if (!me.nodename) {
throw "no node name specified";
}
me.create = true;
Ext.applyIf(me, {
url: "/nodes/" + me.nodename + "/ceph/osd",
method: 'POST',
items: [
{
xtype: 'pveCephDiskSelector',
name: 'dev',
nodename: me.nodename,
value: me.dev,
diskType: 'unused',
fieldLabel: gettext('Disk'),
allowBlank: false
},
{
xtype: 'pveCephDiskSelector',
name: 'journal_dev',
nodename: me.nodename,
diskType: 'journal_disks',
fieldLabel: gettext('Journal Disk'),
value: '',
autoSelect: false,
allowBlank: true,
emptyText: 'use OSD disk'
}
]
});
me.callParent();
}
});
Ext.define('PVE.CephRemoveOsd', {
extend: 'PVE.window.Edit',
alias: ['widget.pveCephRemoveOsd'],
isRemove: true,
initComponent : function() {
/*jslint confusion: true */
var me = this;
if (!me.nodename) {
throw "no node name specified";
}
if (me.osdid === undefined || me.osdid < 0) {
throw "no osdid specified";
}
me.create = true;
me.title = gettext('Remove') + ': ' + 'Ceph OSD osd.' + me.osdid;
Ext.applyIf(me, {
url: "/nodes/" + me.nodename + "/ceph/osd/" + me.osdid,
method: 'DELETE',
items: [
{
xtype: 'pvecheckbox',
name: 'cleanup',
checked: true,
labelWidth: 130,
fieldLabel: gettext('Remove Partitions')
}
]
});
me.callParent();
}
});
Ext.define('PVE.node.CephOsdTree', {
extend: 'Ext.tree.Panel',
......@@ -243,14 +328,12 @@ Ext.define('PVE.node.CephOsdTree', {
if (!(rec && (rec.data.id >= 0) && rec.data.host)) {
return;
}
PVE.Utils.API2Request({
url: "/nodes/" + rec.data.host + "/ceph/osd/" + rec.data.id,
waitMsgTarget: me,
method: 'DELETE',
failure: function(response, opts) {
Ext.Msg.alert(gettext('Error'), response.htmlStatus);
}
var win = Ext.create('PVE.CephRemoveOsd', {
nodename: rec.data.host,
osdid: rec.data.id
});
win.show();
}
});
......@@ -360,7 +443,6 @@ Ext.define('PVE.node.CephDiskList', {
extend: 'Ext.grid.GridPanel',
alias: ['widget.pveNodeCephDiskList'],
initComponent: function() {
/*jslint confusion: true */
var me = this;
......@@ -390,17 +472,17 @@ Ext.define('PVE.node.CephDiskList', {
text: gettext('Create') + ': OSD',
selModel: sm,
disabled: true,
enableFn: function(rec) {
return !rec.data.used;
},
handler: function() {
var rec = sm.getSelection()[0];
PVE.Utils.API2Request({
url: "/nodes/" + nodename + "/ceph/osd",
method: 'POST',
params: { dev: "/dev/" + rec.data.dev },
failure: function(response, opts) {
Ext.Msg.alert(gettext('Error'), response.htmlStatus);
}
var win = Ext.create('PVE.CephCreateOsd', {
nodename: nodename,
dev: rec.data.dev
});
win.show();
}
});
......@@ -424,7 +506,7 @@ Ext.define('PVE.node.CephDiskList', {
if (rec && (rec.data.osdid >= 0)) {
return "osd." + rec.data.osdid;
}
return PVE.Utils.format_boolean(v);
return v || PVE.Utils.noText;
},
dataIndex: 'used'
},
......@@ -474,6 +556,71 @@ Ext.define('PVE.node.CephDiskList', {
});
});
Ext.define('PVE.form.CephDiskSelector', {
extend: 'PVE.form.ComboGrid',
alias: ['widget.pveCephDiskSelector'],
diskType: 'journal_disks',
initComponent: function() {
var me = this;
var nodename = me.nodename;
if (!nodename) {
throw "no node name specified";
}
var store = Ext.create('Ext.data.Store', {
filterOnLoad: true,
model: 'ceph-disk-list',
proxy: {
type: 'pve',
url: "/api2/json/nodes/" + nodename + "/ceph/disks",
extraParams: { type: me.diskType }
},
sorters: [
{
property : 'dev',
direction: 'ASC'
}
]
});
Ext.apply(me, {
store: store,
valueField: 'dev',
displayField: 'dev',
listConfig: {
columns: [
{
header: gettext('Device'),
width: 80,
sortable: true,
dataIndex: 'dev'
},
{
header: gettext('Size'),
width: 60,
sortable: false,
renderer: PVE.Utils.format_size,
dataIndex: 'size'
},
{
header: gettext('Serial'),
flex: 1,
sortable: true,
dataIndex: 'serial'
}
]
}
});
me.callParent();
store.load();
}
});
Ext.define('PVE.CephCreateMon', {
extend: 'PVE.window.Edit',
alias: ['widget.pveCephCreateMon'],
......@@ -497,8 +644,9 @@ Ext.define('PVE.CephCreateMon', {
me.setNode(me.nodename);
me.create = true;
Ext.applyIf(me, {
create: true,
method: 'POST',
items: [
{
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment