Add Ceph playbook
parent
1f66096a25
commit
e6837e4af3
|
|
@ -0,0 +1,4 @@
|
|||
# все *.keyring нужно взять свои, от ceph-deploy
|
||||
[client.bootstrap-mds]
|
||||
key = ...
|
||||
caps mon = "allow profile bootstrap-mds"
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
[client.bootstrap-mgr]
|
||||
key = ...
|
||||
caps mon = "allow profile bootstrap-mgr"
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
[client.bootstrap-osd]
|
||||
key = ...
|
||||
caps mon = "allow profile bootstrap-osd"
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
[client.bootstrap-rgw]
|
||||
key = ...
|
||||
caps mon = "allow profile bootstrap-rgw"
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
[client.admin]
|
||||
key = ...
|
||||
caps mds = "allow *"
|
||||
caps mgr = "allow *"
|
||||
caps mon = "allow *"
|
||||
caps osd = "allow *"
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
[client.libvirt]
|
||||
key = ...
|
||||
caps mon = "profile rbd"
|
||||
caps osd = "profile rbd pool=rpool"
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
[global]
|
||||
# сюда нужно записать uuid сгенерированный ceph-deploy
|
||||
fsid = ...
|
||||
public network = 192.168.5.0/24
|
||||
mon_initial_members = 192.168.5.11 192.168.5.12 192.168.5.13
|
||||
mon_host = 192.168.5.11 192.168.5.12 192.168.5.13
|
||||
|
||||
rbd cache = false
|
||||
#rbd cache max dirty age = 5
|
||||
debug ms = 0/0
|
||||
|
||||
rbd_default_data_pool = ecpool
|
||||
|
||||
auth_cluster_required = cephx
|
||||
auth_service_required = cephx
|
||||
auth_client_required = cephx
|
||||
|
||||
#ms_async_op_threads=1
|
||||
#ms_async_max_op_threads=1
|
||||
|
||||
# отключим на хрен подписи
|
||||
cephx_require_signatures = false
|
||||
cephx_cluster_require_signatures = false
|
||||
cephx_sign_messages = false
|
||||
|
||||
ms_bind_msgr2 = false
|
||||
|
||||
#ms_crc_data = false
|
||||
#ms_crc_header = false
|
||||
|
||||
debug osd = 0/0
|
||||
debug bluefs = 0/0
|
||||
debug perfcounter = 0/0
|
||||
#debug rbd = 0/0
|
||||
#debug rbd = 20/20
|
||||
#log file = /var/log/one/rbd.log
|
||||
#log file = /root/rbd-fio.log
|
||||
#admin socket = /var/run/one/$cluster-$type.$id.$pid.$cctid.asok
|
||||
debug rocksdb = 0/0
|
||||
debug bluestore = 0/0
|
||||
debug tp = 0/0
|
||||
debug objecter = 0/0
|
||||
debug client = 0/0
|
||||
debug crush = 0/0
|
||||
|
||||
[osd]
|
||||
bluestore_prefer_deferred_size = 0
|
||||
bluestore_prefer_deferred_size_hdd = 0
|
||||
bluestore_prefer_deferred_size_ssd = 0
|
||||
bluestore_min_alloc_size = 4096
|
||||
bluestore_min_alloc_size_ssd = 4096
|
||||
bluestore_max_blob_size = 4194304
|
||||
bluestore_max_blob_size_ssd = 4194304
|
||||
osd_op_num_threads_per_shard = 8
|
||||
osd_op_num_shards = 2
|
||||
#osd_op_num_threads_per_shard = 1
|
||||
#osd_op_num_shards = 1
|
||||
bluestore_throttle_cost_per_io = 4000
|
||||
bluestore_sync_submit_transaction = true
|
||||
bluestore_compression_mode = passive
|
||||
bluestore_compression_min_blob_size_ssd = 131072
|
||||
bluestore_compression_max_blob_size_ssd = 4194304
|
||||
bdev_enable_discard = true
|
||||
bdev_async_discard = true
|
||||
|
||||
# https://github.com/ceph/ceph/pull/26909, можно юзать с 14.2.4
|
||||
bluefs_preextend_wal_files = true
|
||||
|
||||
#rocksdb_perf = true
|
||||
#rocksdb_collect_compaction_stats = true
|
||||
#rocksdb_collect_extended_stats = true
|
||||
#rocksdb_collect_memory_stats = true
|
||||
|
||||
bluestore_rocksdb_options = compression=kNoCompression,max_write_buffer_number=32,min_write_buffer_number_to_merge=8,recycle_log_file_num=32,write_buffer_size=33554432,writable_file_max_buffer_size=0,compaction_readahead_size=2097152
|
||||
|
||||
[mon]
|
||||
mon allow pool delete = true
|
||||
mgr initial modules = dashboard
|
||||
ms_bind_msgr2 = true
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
[mon.]
|
||||
key = ...
|
||||
caps mon = allow *
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
#!/bin/bash
|
||||
# Install & configure Ceph (mon+mgr+osds)
|
||||
|
||||
# -e = stop on exception, -x = debug, -a = export all variables
|
||||
set -e -x -a
|
||||
|
||||
# Include config
|
||||
. all_vars
|
||||
|
||||
### Check host variables
|
||||
if [ -z "$play_host" -o -z "$node_name" -o -z "$int_ip" ]; then
|
||||
echo "play_host/node_name/int_ip not specified"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
### Configure network
|
||||
. ./network.sh
|
||||
|
||||
# Setup passwordless self-ssh for root
|
||||
ssh root@$play_host <<EOF
|
||||
set -e -x
|
||||
|
||||
if [ ! -f /root/.ssh/id_rsa.pub ]; then
|
||||
ssh-keygen -t rsa -f /root/.ssh/id_rsa -q -P ""
|
||||
fi
|
||||
|
||||
> tmp$$
|
||||
cat /root/.ssh/known_hosts >> tmp$$; true
|
||||
ssh-keyscan localhost >> tmp$$
|
||||
ssh-keyscan $int_ip >> tmp$$
|
||||
sort tmp$$ | uniq > /root/.ssh/known_hosts
|
||||
rm tmp$$
|
||||
|
||||
> tmp$$
|
||||
cat /root/.ssh/authorized_keys >> tmp$$; true
|
||||
cat /root/.ssh/id_rsa.pub >> tmp$$
|
||||
sort tmp$$ | uniq > /root/.ssh/authorized_keys
|
||||
rm tmp$$
|
||||
EOF
|
||||
|
||||
### Set time sync
|
||||
scp ./etc/systemd/timesyncd.conf root@$play_host:/etc/systemd/
|
||||
ssh root@$play_host 'systemctl enable systemd-timesyncd && systemctl restart systemd-timesyncd'
|
||||
|
||||
### Install packages, deploy mon, mgr and osds
|
||||
scp -r ceph-deploy root@$play_host:~/
|
||||
|
||||
ssh root@$play_host <<EOF
|
||||
set -e -x
|
||||
|
||||
DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" \
|
||||
-o Dpkg::Options::="--force-confold" install -y \
|
||||
ceph ceph-mds ceph-deploy jq
|
||||
|
||||
cd ~/ceph-deploy
|
||||
chmod 600 \$(find ~/ceph-deploy -type f)
|
||||
ceph-deploy mon add $int_ip
|
||||
ceph-deploy mgr create $node_name
|
||||
|
||||
# Don't redeploy anything, just activate existing OSDs, then create new ones
|
||||
ceph-volume lvm activate --all
|
||||
|
||||
# Select available drives larger than 1.5 TB
|
||||
DRIVES=\$(ceph-volume inventory --format json | jq -r '.[] | select(.available == true and .sys_api.size >= 1500000000000) | .sys_api.path')
|
||||
|
||||
for DEV in \$DRIVES; do
|
||||
SIZE=\$(blockdev --getsz \$DEV)
|
||||
# Reserve 32 GB partition on each drive for emergency (value is in 512b sectors)
|
||||
RESERVED_SIZE=67108864
|
||||
OSD_SIZE=\$((SIZE-RESERVED_SIZE-2048))
|
||||
RESERVED_START=\$((OSD_SIZE+2048))
|
||||
PREFIX=\$(perl -e "\\\$a = '\$DEV'; \\\$a =~ s/(\d)\\\$/\\\$1p/; print \\\$a;")
|
||||
echo "PREFIX=\$PREFIX"
|
||||
sfdisk \$DEV <<EOD
|
||||
label: gpt
|
||||
\${PREFIX}1 : start=2048, size=\$OSD_SIZE, type=0FC63DAF-8483-4772-8E79-3D69D8477DE4
|
||||
\${PREFIX}2 : start=\$RESERVED_START, type=0FC63DAF-8483-4772-8E79-3D69D8477DE4
|
||||
EOD
|
||||
cp ~/ceph-deploy/ceph.bootstrap-osd.keyring /var/lib/ceph/bootstrap-osd/ceph.keyring
|
||||
chmod 600 /var/lib/ceph/bootstrap-osd/ceph.keyring
|
||||
ceph-volume lvm prepare --bluestore --data \${PREFIX}1
|
||||
done
|
||||
|
||||
ceph-volume lvm activate --all
|
||||
EOF
|
||||
|
|
@ -1,6 +1,9 @@
|
|||
#deb http://http.debian.net/debian/ sid main contrib non-free
|
||||
#deb-src http://http.debian.net/debian/ sid main contrib non-free
|
||||
|
||||
deb http://http.debian.net/debian/ stretch main contrib non-free
|
||||
deb-src http://http.debian.net/debian/ stretch main contrib non-free
|
||||
|
||||
deb http://http.debian.net/debian/ buster main contrib non-free
|
||||
deb-src http://http.debian.net/debian/ buster main contrib non-free
|
||||
|
||||
|
|
@ -12,3 +15,7 @@ deb http://http.debian.net/debian/ buster-updates main
|
|||
deb-src http://http.debian.net/debian/ buster-updates main
|
||||
|
||||
#deb http://hwraid.le-vert.net/debian stretch main
|
||||
|
||||
# Ceph is needed for both OpenNebula nodes and Ceph nodes
|
||||
deb http://download.ceph.com/debian-nautilus/ bionic main
|
||||
deb-src http://download.ceph.com/debian-nautilus/ bionic main
|
||||
|
|
|
|||
|
|
@ -1,2 +0,0 @@
|
|||
deb http://download.ceph.com/debian-nautilus/ bionic main
|
||||
deb-src http://download.ceph.com/debian-nautilus/ bionic main
|
||||
|
|
@ -22,6 +22,10 @@ echo $node_name > /etc/hostname
|
|||
|
||||
hostname `cat /etc/hostname`
|
||||
|
||||
grep -q -P '127.0.1.1\s+$node_name' /etc/hosts || (echo "127.0.1.1 $node_name" >> /etc/hosts)
|
||||
|
||||
wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add -
|
||||
|
||||
apt-get update
|
||||
|
||||
# gpg and friends for apt-key to work correctly
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e -x -a
|
||||
|
||||
# Run once
|
||||
if [ -z "$eth10g" -o -z "$eth1g" ]; then
|
||||
|
||||
### Check host variables
|
||||
if [ -z "$play_host" -o -z "$node_name" -o -z "$int_ip" ]; then
|
||||
echo "play_host/node_name/int_ip not specified"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
### Find 1G and 10G network interfaces (10G is used for keepalived and galera)
|
||||
eth10g=
|
||||
eth1g=
|
||||
export $(ssh root@$play_host '
|
||||
for i in /sys/class/net/*; do
|
||||
ip link set ${i##/sys/class/net/} up
|
||||
if [ x`cat /sys/class/net/enp4s0/carrier 2>/dev/null` == "x1" ]; then
|
||||
if [ x`cat $i/speed 2>/dev/null` == "x10000" ]; then
|
||||
echo eth10g=${i##/sys/class/net/}
|
||||
elif [ x`cat $i/speed 2>/dev/null` == "x1000" ]; then
|
||||
echo eth1g=${i##/sys/class/net/}
|
||||
fi
|
||||
fi
|
||||
done')
|
||||
|
||||
if [ -z "$eth10g" ]; then
|
||||
echo "10GbE network not found on $play_host"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
### Configure network
|
||||
envsubst < ./etc/rc.local.env | ssh root@$play_host 'cat > /etc/rc.local'
|
||||
envsubst < ./etc/network/interfaces.env | ssh root@$play_host 'cat > /etc/network/interfaces.new'
|
||||
ssh root@$play_host <<EOF
|
||||
set -e
|
||||
|
||||
DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" \
|
||||
-o Dpkg::Options::="--force-confold" install -y \
|
||||
bridge-utils
|
||||
|
||||
chmod 755 /etc/rc.local
|
||||
/etc/rc.local
|
||||
systemctl enable rc-local
|
||||
if ! cmp -s /etc/network/interfaces /etc/network/interfaces.new; then
|
||||
nmcli dev disconnect $eth1g; true
|
||||
mv /etc/network/interfaces.new /etc/network/interfaces
|
||||
ifup br0
|
||||
service networking restart
|
||||
fi
|
||||
EOF
|
||||
|
||||
# -z eth10g / eth1g
|
||||
fi
|
||||
|
|
@ -15,46 +15,8 @@ if [ -z "$play_host" -o -z "$node_name" -o -z "$int_ip" ]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
### Find 1G and 10G network interfaces (10G is used for keepalived and galera)
|
||||
eth10g=
|
||||
eth1g=
|
||||
export $(ssh root@$play_host '
|
||||
for i in /sys/class/net/*; do
|
||||
ip link set ${i##/sys/class/net/} up
|
||||
if [ x`cat /sys/class/net/enp4s0/carrier 2>/dev/null` == "x1" ]; then
|
||||
if [ x`cat $i/speed 2>/dev/null` == "x10000" ]; then
|
||||
echo eth10g=${i##/sys/class/net/}
|
||||
elif [ x`cat $i/speed 2>/dev/null` == "x1000" ]; then
|
||||
echo eth1g=${i##/sys/class/net/}
|
||||
fi
|
||||
fi
|
||||
done')
|
||||
|
||||
if [ -z "$eth10g" ]; then
|
||||
echo "10GbE network not found on $play_host"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
### Configure network
|
||||
envsubst < ./etc/rc.local.env | ssh root@$play_host 'cat > /etc/rc.local'
|
||||
envsubst < ./etc/network/interfaces.env | ssh root@$play_host 'cat > /etc/network/interfaces.new'
|
||||
ssh root@$play_host <<EOF
|
||||
set -e
|
||||
|
||||
DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" \
|
||||
-o Dpkg::Options::="--force-confold" install -y \
|
||||
bridge-utils
|
||||
|
||||
chmod 755 /etc/rc.local
|
||||
/etc/rc.local
|
||||
systemctl enable rc-local
|
||||
if ! cmp -s /etc/network/interfaces /etc/network/interfaces.new; then
|
||||
nmcli dev disconnect $eth1g; true
|
||||
mv /etc/network/interfaces.new /etc/network/interfaces
|
||||
ifup br0
|
||||
service networking restart
|
||||
fi
|
||||
EOF
|
||||
. ./network.sh
|
||||
|
||||
### Install packages
|
||||
scp ./etc/apt/sources.list.d/opennebula.list root@$play_host:/etc/apt/sources.list.d/opennebula.list
|
||||
|
|
@ -70,7 +32,7 @@ apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74C
|
|||
DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" \
|
||||
-o Dpkg::Options::="--force-confold" install -y \
|
||||
lsb-release keepalived sudo qemu-kvm qemu-block-extra mariadb-server netcat-openbsd \
|
||||
opennebula opennebula-sunstone opennebula-gate opennebula-flow opennebula-node opennebula-tools
|
||||
opennebula opennebula-sunstone opennebula-gate opennebula-flow opennebula-node opennebula-tools ceph
|
||||
|
||||
/usr/share/one/install_gems --yes
|
||||
EOF
|
||||
|
|
@ -177,7 +139,7 @@ ssh root@$play_host <<EOF
|
|||
set -e -x
|
||||
|
||||
if [ ! -f /var/lib/one/.ssh/id_rsa.pub ]; then
|
||||
su - oneadmin -c 'ssh-keygen -t rsa -f /root/.ssh/id_rsa -q -P ""'
|
||||
su - oneadmin -c 'ssh-keygen -t rsa -f /var/lib/one/.ssh/id_rsa -q -P ""'
|
||||
fi
|
||||
|
||||
if [ ! -f /var/lib/one/.ssh/known_hosts ]; then
|
||||
|
|
@ -222,3 +184,27 @@ if ! grep -q clone /var/lib/one/remotes/tm/ceph/cpds; then
|
|||
patch -p0 < /root/opennebula-ceph-cpds-clone.diff
|
||||
fi
|
||||
EOF
|
||||
|
||||
# Copy ceph configs
|
||||
scp ./ceph-deploy/ceph.conf root@$play_host:/etc/ceph/
|
||||
scp ./ceph-deploy/ceph.client.admin.keyring root@$play_host:/etc/ceph/
|
||||
scp ./ceph-deploy/ceph.client.libvirt.keyring root@$play_host:/etc/ceph/
|
||||
ssh root@$play_host 'chmod 600 /etc/ceph/ceph.client.admin.keyring'
|
||||
|
||||
# Add Ceph secret
|
||||
ssh root@$play_host <<EOF
|
||||
set -e -x
|
||||
|
||||
if [ ! -f /etc/libvirt/secrets/$libvirt_secret_uuid.base64 ]; then
|
||||
cat > secret.xml <<EOS
|
||||
<secret ephemeral='no' private='no'>
|
||||
<uuid>$libvirt_secret_uuid</uuid>
|
||||
<usage type='ceph'>
|
||||
<name>client.libvirt secret</name>
|
||||
</usage>
|
||||
</secret>
|
||||
EOS
|
||||
KEY=\$(ceph auth get-key client.libvirt)
|
||||
virsh -c qemu:///system secret-define secret.xml; virsh -c qemu:///system secret-set-value --secret $libvirt_secret_uuid --base64 \$KEY
|
||||
fi
|
||||
EOF
|
||||
|
|
|
|||
Loading…
Reference in New Issue