ceph-opennebula-playbooks/opennebula.sh

#!/bin/bash
# Install & configure OpenNebula with MariaDB Galera Cluster as HA
# This script is idempotent like an Ansible playbook!
# I.e. run it as many times as you want to, it won't hurt!

# -e = stop on exception, -x = debug, -a = export all variables
set -e -x -a

# Include config
. ./load-config.sh

### Check host variables
if [ -z "$play_host" -o -z "$node_name" -o -z "$int_ip" ]; then
    echo "play_host/node_name/int_ip not specified"
    exit 1
fi

### Configure network
. ./network.sh

### Install packages
scp ./etc/apt/sources.list.d/opennebula.list root@$play_host:/etc/apt/sources.list.d/opennebula.list
scp ./etc/apt/sources.list.d/mariadb.list root@$play_host:/etc/apt/sources.list.d/mariadb.list

ssh root@$play_host <<EOF
set -e -x

wget -q -O - https://downloads.opennebula.org/repo/repo.key | apt-key add -

apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8

apt-get update || true

DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" \
    -o Dpkg::Options::="--force-confold" install -y \
    lsb-release sudo mariadb-server netcat-openbsd \
    opennebula opennebula-sunstone opennebula-gate opennebula-flow opennebula-node opennebula-tools ceph ruby-bundler

# Keepalived from buster crashes with libc6 from sid
DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" \
    -o Dpkg::Options::="--force-confold" install -y -t sid \
    qemu qemu-kvm qemu-system-common qemu-system-data qemu-system-x86 qemu-utils qemu-block-extra keepalived

/usr/share/one/install_gems --yes
EOF

### Setup or join MariaDB Galera Cluster

# Create a user for Galera (if not yet)
ssh root@$play_host <<EOF
set -e -x

if ! (echo 'SELECT 1' | mysql --host=$play_host -u sst_user --password=$galera_password); then
    service mysql restart
    mysql <<EOM
        GRANT USAGE ON *.* to sst_user@'%' IDENTIFIED BY '$galera_password';
        GRANT ALL PRIVILEGES on *.* to sst_user@'%';
        FLUSH PRIVILEGES;
EOM
fi

if [ -f /etc/mysql/my.cnf ]; then
    rm /etc/mysql/my.cnf
fi
if [ ! -h /etc/mysql/my.cnf ]; then
    ln -fs /etc/mysql/mariadb.cnf /etc/mysql/my.cnf
fi
EOF

# Copy configs
scp ./etc/mysql/mariadb.conf.d/50-client.cnf root@$play_host:/etc/mysql/mariadb.conf.d/50-client.cnf

# Put all hosts except this one in wsrep_cluster_address
galera_hosts=
for ip in $int_ips; do
    galera_hosts=$galera_hosts,$ip
done
galera_hosts=${galera_hosts##,}
envsubst < ./etc/mysql/mariadb.conf.d/50-server.cnf.env | \
    ssh root@$play_host 'cat > /etc/mysql/mariadb.conf.d/50-server.cnf'

if [ "$init_db" -eq 1 ]; then
    # Create a new cluster
    ssh root@$play_host <<EOF
        set -e -x
        systemctl stop mysql
        systemctl start mysql --wsrep-new-cluster
        mysql <<EOM
            CREATE DATABASE opennebula;
            GRANT ALL PRIVILEGES ON opennebula.* TO 'oneadmin' IDENTIFIED BY '$opennebula_db_password';
            FLUSH PRIVILEGES;
EOM
EOF
else
    # Or just join it
    ssh root@$play_host 'systemctl restart mysql'
fi

### Setup OpenNebula oned and sunstone
ssh root@$play_host <<EOF
echo oneadmin:$oneadmin_password > /var/lib/one/.one/one_auth
echo serveradmin:$serveradmin_password > /var/lib/one/.one/ec2_auth
echo serveradmin:$serveradmin_password > /var/lib/one/.one/occi_auth
echo serveradmin:$serveradmin_password > /var/lib/one/.one/oneflow_auth
echo serveradmin:$serveradmin_password > /var/lib/one/.one/onegate_auth
echo serveradmin:$serveradmin_password > /var/lib/one/.one/sunstone_auth
echo $one_key > /var/lib/one/.one/one_key
chown oneadmin:oneadmin /var/lib/one/.one/*
chmod 600 /var/lib/one/.one/*
EOF

ssh root@$play_host <<EOF
set -e -x

perl -i -pe 's!^DB\s*=.*!DB = [ backend = "mysql", server = "localhost", port = 0, user = "oneadmin", passwd = "$opennebula_db_password", db_name = "opennebula" ]!' /etc/one/oned.conf

perl -i -pe 's!^LIVE_RESCHEDS\s*=.*!LIVE_RESCHEDS = 1!' /etc/one/sched.conf

while ! echo SELECT 1 | mysql; do
    echo Waiting for MySQL...
done

systemctl disable opennebula
systemctl disable opennebula-sunstone
systemctl stop opennebula
systemctl stop opennebula-sunstone
EOF

### Setup keepalived
scp etc/one/one-cluster.sh root@$play_host:/etc/one/
envsubst < ./etc/keepalived/keepalived.conf.env | \
    ssh root@$play_host 'cat > /etc/keepalived/keepalived.conf'
ssh root@$play_host 'chmod 755 /etc/one/one-cluster.sh && systemctl restart keepalived'

# Setup onedns
envsubst < ./etc/systemd/system/onedns.service.env | \
    ssh root@$play_host 'cat > /etc/systemd/system/onedns.service'
scp etc/sysctl.conf root@$play_host:/etc/
ssh root@$play_host <<EOF
set -e -x
[ -e onedns ] || git clone https://github.com/vitalif/onedns
cd onedns
DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" \
    -o Dpkg::Options::="--force-confold" install -y \
    python-setuptools
python setup.py install
sysctl --load=/etc/sysctl.conf
systemctl enable onedns
systemctl restart onedns
EOF

# Setup passwordless ssh for `oneadmin` (authorized_keys and known_hosts)
key_hosts=$play_host ./opennebula-keys.sh

# Add a host to OpenNebula and set reserved memory to 16G
ssh root@$play_host <<EOF
set -e -x
service libvirtd restart
host_name=
export \$(onehost show -x $play_host | /var/lib/one/remotes/datastore/xpath.rb \
    "concat('host_name=', /HOST/NAME/text(), ' host_mem=', /HOST/TEMPLATE/RESERVED_MEM/text())" | tr '\0' '\n')
if [ -z "\$host_name" ]; then
    onehost create $play_host -i kvm -v kvm
fi
if [ "\$host_mem" != "16777216" ]; then
    echo 'RESERVED_MEM="16777216"' > tmp$$
    onehost update -a $play_host tmp$$
    rm tmp$$
fi
EOF

# Apply the patch
scp opennebula-ceph-cpds-clone.diff root@$play_host:~/
ssh root@$play_host <<EOF
set -e -x
if ! grep -q clone /var/lib/one/remotes/tm/ceph/cpds; then
    cd /
    patch -p0 < /root/opennebula-ceph-cpds-clone.diff
fi
EOF

# Copy ceph configs
cat ./ceph-deploy/ceph.conf | perl -pe "s/MON_IPS/$int_ips/" | ssh root@$play_host 'cat > /etc/ceph/ceph.conf'
scp ./ceph-deploy/ceph.client.admin.keyring root@$play_host:/etc/ceph/
scp ./ceph-deploy/ceph.client.libvirt.keyring root@$play_host:/etc/ceph/
ssh root@$play_host 'chmod 600 /etc/ceph/ceph.client.admin.keyring'

# Add Ceph secret
ssh root@$play_host <<EOF
set -e -x

if [ ! -f /etc/libvirt/secrets/$libvirt_secret_uuid.base64 ]; then
    cat > secret.xml <<EOS
<secret ephemeral='no' private='no'>
    <uuid>$libvirt_secret_uuid</uuid>
    <usage type='ceph'>
        <name>client.libvirt secret</name>
    </usage>
</secret>
EOS
    KEY=\$(ceph auth get-key client.libvirt)
    virsh -c qemu:///system secret-define secret.xml; virsh -c qemu:///system secret-set-value --secret $libvirt_secret_uuid --base64 \$KEY
fi
EOF