Skip to content

HOWTO: Ceph Alces

Jump to bottom
Steve Norledge edited this page Mar 13, 2016 · 32 revisions

Alces Reference Ceph install (manual)

IDM - no support for wildcard domains so no bucket.monitor1.x.x

#Openstack controller

openstack service create --name swift object-store
openstack endpoint create --publicurl https://radosgw.kelvin.compute.estate/swift/v1 --internalurl https://radosgw.kelvin.compute.estate/swift/v1 --adminurl https://radosgw.kelvin.compute.estate/swift/v1 swift

mkdir -p /var/ceph/nss
openssl x509 -in /etc/keystone/ssl/certs/ca.pem -pubkey | \
        certutil -d /var/ceph/nss -A -n ca -t "TCu,Cu,Tuw"
openssl x509 -in /etc/keystone/ssl/certs/signing_cert.pem -pubkey | \
        certutil -A -d /var/ceph/nss -n signing_cert -t "P,P,P"

ssh key login

  • Create a cephmaster root key on a monitor and copy to all members of the ceph cluster:
ssh-keygen
ssh-copy-id <all ceph members>

scp .ssh/id_rsa* <all ceph monitors>:/root/.ssh/.

##All Ceph:

useradd ceph -M -N -u 505
echo "ceph ALL = (root) NOPASSWD:ALL" > /etc/sudoers.d/ceph
chmod 0440 /etc/sudoers.d/ceph
sed -i -e "s/^Defaults *requiretty$/#Defaults    requiretty/g" /etc/sudoers

PRV_INT=bond0.2
BUILD_INT=bond0
DMZ_INT=

systemctl enable firewalld
systemctl start firewalld
echo 'ZONE=prv' >> /etc/sysconfig/network-scripts/ifcfg-$PRV_INT
echo 'ZONE=build' >> /etc/sysconfig/network-scripts/ifcfg-$BUILD_INT
if ! [ -z "$DMZ_INT" ]; then
  echo 'ZONE=dmz' >>  /etc/sysconfig/network-scripts/ifcfg-$DMZ_INT
  firewall-cmd --add-interface $DMZ_INT --zone dmz --permanent
  firewall-cmd --add-interface $DMZ_INT --zone dmz
fi
firewall-cmd --new-zone prv --permanent
firewall-cmd --new-zone build --permanent
firewall-cmd --add-interface $PRV_INT --zone prv --permanent
firewall-cmd --add-interface $BUILD_INT --zone build --permanent
firewall-cmd --add-interface $PRV_INT --zone prv
firewall-cmd --add-interface $BUILD_INT --zone build
firewall-cmd --add-service ssh --zone prv --permanent
firewall-cmd --add-service ssh --zone build --permanent
firewall-cmd --reload

(required for now (march15))# echo 'check_obsoletes=1' >> /etc/yum/pluginconf.d/priorities.conf
yum -y install snappy leveldb gdisk python-argparse gperftools-libs ceph

##MONITOR[1-x]:

firewall-cmd --zone=prv --add-port=6789/tcp --permanent
firewall-cmd --reload
FSID=`uuidgen`

COPY_WITH_MODS /etc/ceph/ceph.conf
ceph-authtool --create-keyring /etc/ceph/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow'
ceph-authtool /etc/ceph/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring

#COPY KEYS TO OTHER MONITORS
#Reboot monitors to enable firewall

#CREATE MONITOR MAP
FSID=`cat /etc/ceph/ceph.conf | grep fsid | awk ' { print $3 } '`
monmaptool --create --add `hostname -s` `hostname -i` --fsid $FSID /etc/ceph/monmap
#For all the other monitors
monmaptool --add <hostname> <ip> --fsid $FSID /etc/ceph/monmap

#CREATE MONITOR FS
mkdir /var/lib/ceph/mon/ceph-`hostname -s`
ceph-mon --mkfs -i `hostname -s` --monmap /etc/ceph/monmap --keyring /etc/ceph/ceph.mon.keyring
touch /var/lib/ceph/mon/ceph-`hostname -s`/done
touch /var/lib/ceph/mon/ceph-`hostname -s`/sysvinit
/etc/init.d/ceph start mon.`hostname -s`
chkconfig ceph on


#Copy config to each block client
scp /etc/ceph/ceph.conf block1:/etc/ceph/.

##GW[1-x]:

yum -y install httpd mod_fastcgi
COPY_WITH_MODS /etc/httpd/conf/httpd.conf (ServerName)
systemctl restart httpd
yum -y install ceph-radosgw ceph radosgw-agent
yum -y install mod_ssl openssl
COPY_WITH_MODS /etc/httpd/conf.d/rgw.conf

mkdir /etc/certs
cp chain/CA.crt > /etc/certs/CA.crt
cp crt.pem > /etc/certs/apache_crt.pem
cp key.pem > /etc/certs/apache_key.pem
chmod 600 /etc/certs/*

systemctl restart httpd
systemctl start ceph-radosgw

systemctl enable httpd
chkconfig ceph-radosgw on

firewall-cmd --add-service http --zone external --permanent
firewall-cmd --add-service https --zone external --permanent
firewall-cmd --reload

radosgw-admin user create --uid="admin" --display-name="Admin Test User"


#On the primary monitor...
ceph-authtool --create-keyring /etc/ceph/ceph.client.radosgw.keyring
chmod +r /etc/ceph/ceph.client.radosgw.keyring
ceph-authtool /etc/ceph/ceph.client.radosgw.keyring -n client.radosgw.gateway --gen-key
ceph-authtool -n client.radosgw.gateway --cap osd 'allow rwx' --cap mon 'allow rwx' /etc/ceph/ceph.client.radosgw.keyring
ceph -k /etc/ceph/ceph.client.admin.keyring auth add client.radosgw.gateway -i /etc/ceph/ceph.client.radosgw.keyring

copy /etc/ceph/ceph.client.radosgw.keyring to the gateway machine

sudo chown apache:apache /var/run/ceph
sudo chown apache:apache /var/log/radosgw/client.radosgw.gateway.log

cat << EOF > /vaw/www/html/s3gw.fcgi
#!/bin/sh
exec /usr/bin/radosgw -c /etc/ceph/ceph.conf -n client.radosgw.gateway
EOF
chmod 755 /var/www/html/s3gw.fcgi

copy in /etc/httpd/conf.d/fastcgi.conf

##BLOCK[1-x]:

First - setup the OSD volumes:

Warning - these instructions will wipe your data disks

# Assume your OSD volumes are called /dev/sdb and /dev/sdc
# Prepare larger disks to use with parted
parted /dev/sdb mklabel gpt
parted /dev/sdc mklabel gpt
parted -a optimal /dev/sdb mkpart primary 0% 100%
parted -a optimal /dev/sdc mkpart primary 0% 100%
parted -a optimal /dev/sdb  set 1 lvm on
parted -a optimal /dev/sdc  set 1 lvm on
# Create your physical volumes
pvcreate -f /dev/sdb1
pvcreate -f /dev/sdc1
# Create your volume group
vgcreate -c n storageVG /dev/sdb1 /dev/sdc1
# Create the logical volumes
lvcreate -l 100%PVS -n osd1 storageVG /dev/sdb1
lvcreate -l 100%PVS -n osd2 storageVG /dev/sdc1
# Format the filesystem (need -f parameter if they've previously been formatted)
mkfs.xfs /dev/mapper/storageVG-osd1
mkfs.xfs /dev/mapper/storageVG-osd2
# Make mount-mounts and mount
mkdir /mnt/osd1 /mnt/osd2
echo "/dev/mapper/storageVG-osd1 /mnt/osd1      xfs     noatime,allocsize=1g,nobarrier,logbufs=8,inode64,wsync 0 0" >> /etc/fstab
echo "/dev/mapper/storageVG-osd2 /mnt/osd2      xfs     noatime,allocsize=1g,nobarrier,logbufs=8,inode64,wsync 0 0" >> /etc/fstab
mount /mnt/osd1
mount /mnt/osd2

Now - prepare the server:

firewall-cmd --zone=prv --add-port=6800-7100/tcp --permanent
firewall-cmd --reload

#pre formatted fstab mountpoint, mounting fs up correctly
for EXISTING_MP in /mnt/osd1 /mnt/osd2 /mnt/osd3 ; do
echo $EXISTING_MP
OSDUUID=`uuidgen`
OSDID=`ceph osd create $OSDUUID`
mkdir /var/lib/ceph/osd/ceph-$OSDID
umount $EXISTING_MP
sed -i -e "s|$EXISTING_MP|/var/lib/ceph/osd/ceph-$OSDID|g" /etc/fstab
mount /var/lib/ceph/osd/ceph-$OSDID
if ( mount | grep -q /var/lib/ceph/osd/ceph-$OSDID ); then
  ceph-osd -i $OSDID --mkfs --mkkey --osd-uuid $OSDUUID
  ceph auth add osd.$OSDID osd 'allow *' mon 'allow profile osd' -i /var/lib/ceph/osd/ceph-$OSDID/keyring
  ceph osd crush add-bucket `hostname -s` host
  ceph osd crush move `hostname -s` root=default
  ceph osd crush add osd.$OSDID 1.0 host=`hostname -s`

  touch /var/lib/ceph/osd/ceph-$OSDID/sysvinit
  /etc/init.d/ceph start osd.$OSDID
fi
done
chkconfig ceph on

##Starting Ceph filesystem for the first time

# Increase the number of placement groups to suit the number of OSDs and pools you have
# Less than 5 OSDs set pg_num to 128
# Between 5 and 10 OSDs set pg_num to 512
# Between 10 and 50 OSDs set pg_num to 4096
# N.B. Default max is 300 PGs per OSD to cover ALL POOLS, so if you have 32 OSDs, your PGs should be
# one pool = 1024
# ten pools = 640
# N.B. Set pgp_num to be equal to pg_num for current Ceph releases
# Example below sets number of placement groups (PGs) and placement groups for placement (PGPs) to 512
ceph osd pool set rbd pg_num 512
ceph osd pool set rbd pgp_num 512

# Give Ceph time to create and pair the placement groups before using the filesystem
# Once ceph -s is healthy, run a quick benchmark - e.g. here on the default rbd pool
rados --pool rbd bench 60 write

##Configuring a Crush map

# By default, Crush map assigns equal weight to each OSD
# You may want to configure some OSDs to have different properties to other
# e.g. A set of 10 primary OSDs which are always replicated to a set of 10 secondary OSDs
# N.B. OSDs don't need to be the same type (e.g. SSD primary and SATA secondary)

# The following commands retrieve the current Crush map, and decompile it for editing
ceph osd getcrushmap -o current_crush.compiled
crushtool -d current_crush.compiled -o current_crush.base

# Make edits to the Crush map as appropriate
# Run these commands to recompile and set the Crush map (compiler will check syntax - do not ignore warnings!)
crushtool -c new_crush.base -o new_crush.compiled 
ceph osd setcrushmap -i new_crush.compiled

##Recovering from lost OSDs

If an OSD fails because of a disk failure, then Ceph should automatically re-replicate data if it can to work around the missing devices. If you don't have enough capacity to generate new replicas (i.e. you already had a over-capacity warning flagged before a single OSD failed), then the filesystem may pause and wait for the OSDs to come back.

To replace a missing OSD, do the following (using osd.32 in the example below):

  • Disable the OSD in Ceph, if it hasn't already been disabled due to its failure (run this command on any Ceph server):
ceph osd out osd.32

  • Clean up the Ceph OSD server. This includes stopping the Ceph OSD daemon if it's still running, replacing the disk, rebooting the server, recreating the underlying XFS filesystem and mounting it back up, including any journals you were using. Make sure the Ceph OSD daemon for the disk to be replaced is NOT running.

  • Disable and permanently remove the OSD from Ceph (run these commands on any Ceph server):

ceph osd crush remove osd.32
ceph auth del osd.32
ceph osd rm 32
  • Create the new OSD; run these commands on the Ceph OSD server itself as root:
ceph osd create
ceph-osd -i 32 --mkfs --mkkey
ceph auth add osd.32 osd 'allow *' mon 'allow profile osd' -i /var/lib/ceph/osd/ceph-32/keyring
ceph osd crush add osd.32 1.0 host=`hostname -s`
touch /var/lib/ceph/osd/ceph-32/sysvinit
/etc/init.d/ceph start osd.32
  • Use ceph -s on any Ceph server to view recovery I/O. Use ceph osd tree to check that the new OSD is part of the filesystem again and marked as up.

RBD Cinder

  • Create a pool for use by cinder, the exact parameters will be dependents on your ceph layout, for example
ceph osd pool create cinder 128

N.B. RBD cannot store data on an erasure coded pool by default; use a replicated cache tier in front of the EC pool if you want block devices to use EC.

# add cinder repos to symphony-storage
yum install ceph-common

ceph auth get-or-create client.cinder mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=cinder'

ceph auth get-or-create client.cinder and copy to /etc/ceph/ceph.client.cinder.keyring on symphony-stg
chown cinder:cinder /etc/ceph/ceph.client.glance.keyring

copy ceph.conf the cinder.keyring and the admin.keyring to symphony-stg
edit /etc/ceph/ceph.conf and set the keyring to cinder.keyrind

edit /etc/cinder/cinder.conf
openstack volume type create RBD
openstack volume type set --property volume_backend_name=RBD RBD

RBD GLANCE

  • Create a pool for use by glance, the exact parameters will be dependents on your ceph layout, for example
ceph osd pool create glance 16

N.B. RBD cannot store data on an erasure coded pool by default; use a replicated cache tier in front of the EC pool if you want block devices to use EC.

ceph auth get-or-create client.glance mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=glance'

ceph auth get-or-create client.glance and copy to /etc/ceph/ceph.client.glance.keyring on symphony-stg
chown glance:glance /etc/ceph/ceph.client.glance.keyring

/etc/glance/glance-api.conf
under [glance_store]
default_store=rbd
stores=glance.store.filesystem.Store,
         glance.store.http.Store,
         glance.store.rbd.Store
rbd_store_ceph_conf=/etc/ceph/ceph.conf
rbd_store_user=glance
rbd_store_pool=glance

service openstack-glance-api restart

RBD NOVA

# add cinder repos to symphony-storage
yum install ceph-common
copy ceph.conf and keyring to nova host

cat << EOF > /tmp/secret.xml
<secret ephemeral='no' private='no'>
  <usage type='ceph'>
    <name>client.cinder secret</name>
  </usage>
</secret>
EOF
virsh secret-define --file /tmp/secret.xml

# keep resulting UUID, set in cinder and nova configs

virsh secret-set-value --secret $UUID --base64 $CEPHCINDERKEY
virsh secret-dumpxml $UUID > /tmp/secret.xml

all other novas:
virsh secret-define /tmp/secret.xml
virsh secret-set-value --secret $UUID --base64 $CEPHCINDERKEY

edit nova.conf, add uuid, add uuid to cinder.conf on stg
sed -i -e 's/^#images_rbd_pool=.*$/images_rbd_pool=cinder/g' /etc/nova/nova.conf
sed -i -e 's/^#rbd_user=.*$/rbd_user=cinder/g' /etc/nova/nova.conf
sed -i -e "s/^#rbd_secret_uuid=.*$/rbd_secret_uuid=<UUID>/g" /etc/nova/nova.conf

Copyright (c) 2008-2015 Alces Software Ltd

Clone this wiki locally