ceph-opennebula-playbooks/opennebula.sh

#!/bin/bash
# Install & configure OpenNebula with MariaDB Galera Cluster as HA
# This script is idempotent like an Ansible playbook!
# I.e. run it as many times as you want to, it won't hurt!

# -e = stop on exception, -x = debug, -a = export all variables
set -e -x -a

# Include config
. all_vars

### Check host variables
if [ -z "$play_host" -o -z "$node_name" -o -z "$int_ip" ]; then
    echo "play_host/node_name/int_ip not specified"
    exit 1
fi

### Find 1G and 10G network interfaces (10G is used for keepalived and galera)
eth10g=
eth1g=
export $(ssh root@$play_host '
    for i in /sys/class/net/*; do
        ip link set ${i##/sys/class/net/} up
        if [ x`cat /sys/class/net/enp4s0/carrier 2>/dev/null` == "x1" ]; then
            if [ x`cat $i/speed 2>/dev/null` == "x10000" ]; then
                echo eth10g=${i##/sys/class/net/}
            elif [ x`cat $i/speed 2>/dev/null` == "x1000" ]; then
                echo eth1g=${i##/sys/class/net/}
            fi
        fi
    done')

if [ -z "$eth10g" ]; then
    echo "10GbE network not found on $play_host"
    exit 1
fi

### Configure network
envsubst < ./etc/rc.local.env | ssh root@$play_host 'cat > /etc/rc.local'
envsubst < ./etc/network/interfaces.env | ssh root@$play_host 'cat > /etc/network/interfaces.new'
ssh root@$play_host <<EOF
set -e

DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" \
    -o Dpkg::Options::="--force-confold" install -y \
    bridge-utils

chmod 755 /etc/rc.local
/etc/rc.local
systemctl enable rc-local
if ! cmp -s /etc/network/interfaces /etc/network/interfaces.new; then
    nmcli dev disconnect $eth1g; true
    mv /etc/network/interfaces.new /etc/network/interfaces
    ifup br0
    service networking restart
fi
EOF

### Install packages
scp ./etc/apt/sources.list.d/opennebula.list root@$play_host:/etc/apt/sources.list.d/opennebula.list
scp ./etc/apt/sources.list.d/mariadb.list root@$play_host:/etc/apt/sources.list.d/mariadb.list

ssh root@$play_host <<EOF
set -e

wget -q -O - https://downloads.opennebula.org/repo/repo.key | apt-key add -

apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8

DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" \
    -o Dpkg::Options::="--force-confold" install -y \
    lsb-release keepalived sudo qemu-kvm qemu-block-extra mariadb-server netcat-openbsd \
    opennebula opennebula-sunstone opennebula-gate opennebula-flow opennebula-node opennebula-tools

/usr/share/one/install_gems --yes
EOF

### Setup keepalived
envsubst < ./etc/keepalived/keepalived.conf.env | \
    ssh root@$play_host 'cat > /etc/keepalived/keepalived.conf'
ssh root@$play_host 'systemctl restart keepalived'

### Setup or join MariaDB Galera Cluster
scp ./etc/mysql/mariadb.conf.d/50-client.cnf root@$play_host:/etc/mysql/mariadb.conf.d/50-client.cnf

# Put all hosts except this one in wsrep_cluster_address
galera_hosts=
for host in $opennebula_hosts; do
    if [ "$host" != "$play_host" ]; then
        galera_hosts=$galera_hosts,$host
    fi
done
galera_hosts=${galera_hosts##,}
envsubst < ./etc/mysql/mariadb.conf.d/50-server.cnf.env | \
    ssh root@$play_host 'cat > /etc/mysql/mariadb.conf.d/50-server.cnf'

# Create a user for Galera (if not yet)
ssh root@$play_host <<EOF
set -e -x

if ! (echo 'SELECT 1' | mysql --host=$play_host -u sst_user --password=$galera_password); then
    service mysql restart
    mysql <<EOM
        GRANT USAGE ON *.* to sst_user@'%' IDENTIFIED BY '$galera_password';
        GRANT ALL PRIVILEGES on *.* to sst_user@'%';
        FLUSH PRIVILEGES;
EOM
fi

[ -h /etc/mysql/my.cnf ] || rm /etc/mysql/my.cnf && ln -fs /etc/mysql/mariadb.cnf /etc/mysql/my.cnf
EOF

if [ "$init_db" -eq 1 ]; then
    # Create a new cluster
    ssh root@$play_host <<EOF
        set -e -x
        systemctl stop mysql
        systemctl start mysql --wsrep-new-cluster
        mysql <<EOM
            CREATE DATABASE opennebula;
            GRANT ALL PRIVILEGES ON opennebula.* TO 'oneadmin' IDENTIFIED BY '$opennebula_db_password';
            FLUSH PRIVILEGES;
EOM
EOF
else
    # Or just join it
    ssh root@$play_host 'systemctl restart mysql'
fi

### Setup OpenNebula oned and sunstone
ssh root@$play_host <<EOF
echo oneadmin:$oneadmin_password > /var/lib/one/.one/one_auth
echo serveradmin:$serveradmin_password > /var/lib/one/.one/ec2_auth
echo serveradmin:$serveradmin_password > /var/lib/one/.one/occi_auth
echo serveradmin:$serveradmin_password > /var/lib/one/.one/oneflow_auth
echo serveradmin:$serveradmin_password > /var/lib/one/.one/onegate_auth
echo serveradmin:$serveradmin_password > /var/lib/one/.one/sunstone_auth
echo $one_key > /var/lib/one/.one/one_key
chown oneadmin:oneadmin /var/lib/one/.one/*
chmod 600 /var/lib/one/.one/*
EOF

ssh root@$play_host <<EOF
set -e -x

perl -i -pe 's!^DB\s*=.*!DB = [ backend = "mysql", server = "localhost", port = 0, user = "oneadmin", passwd = "$opennebula_db_password", db_name = "opennebula" ]!' /etc/one/oned.conf

while ! echo SELECT 1 | mysql; do
    echo Waiting for MySQL...
done

systemctl enable opennebula
systemctl enable opennebula-sunstone
systemctl restart opennebula
systemctl restart opennebula-sunstone
EOF

# Setup onedns
envsubst < ./etc/systemd/system/onedns.service.env | \
    ssh root@$play_host 'cat > /etc/systemd/system/onedns.service'
scp etc/sysctl.conf root@$play_host:/etc/
ssh root@$play_host <<EOF
set -e -x
[ -e onedns ] || git clone https://github.com/vitalif/onedns
cd onedns
DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" \
    -o Dpkg::Options::="--force-confold" install -y \
    python-setuptools
python setup.py install
sysctl --load=/etc/sysctl.conf
systemctl enable onedns
systemctl restart onedns
EOF

# Setup passwordless ssh for `oneadmin` (authorized_keys and known_hosts)
ssh root@$play_host <<EOF
set -e -x

if [ ! -f /var/lib/one/.ssh/id_rsa.pub ]; then
    su - oneadmin -c 'ssh-keygen -t rsa -f /root/.ssh/id_rsa -q -P ""'
fi

if [ ! -f /var/lib/one/.ssh/known_hosts ]; then
    su - oneadmin -c 'ssh-keyscan localhost >> /var/lib/one/.ssh/known_hosts'
    for host in $opennebula_hosts; do
        su - oneadmin -c "ssh-keyscan \$host >> /var/lib/one/.ssh/known_hosts"
    done
fi
EOF

> tmp$$
for host in $opennebula_hosts; do
    ssh root@$host 'cat /var/lib/one/.ssh/id_rsa.pub' >> tmp$$
done
ssh root@$play_host 'cat /var/lib/one/.ssh/authorized_keys; true' >> tmp$$
cat tmp$$ | sort | uniq | ssh root@$play_host 'cat > /var/lib/one/.ssh/authorized_keys'
rm tmp$$

# Add a host to OpenNebula and set reserved memory to 16G
ssh root@$play_host <<EOF
set -e -x
service libvirtd restart
host_name=
export \$(onehost show -x $play_host | /var/lib/one/remotes/datastore/xpath.rb \
    "concat('host_name=', /HOST/NAME/text(), ' host_mem=', /HOST/TEMPLATE/RESERVED_MEM/text())" | tr '\0' '\n')
if [ -z "\$host_name" ]; then
    onehost create $play_host -i kvm -v kvm
fi
if [ "\$host_mem" != "16777216" ]; then
    echo 'RESERVED_MEM="16777216"' > tmp$$
    onehost update -a $play_host tmp$$
    rm tmp$$
fi
EOF

# Apply the patch
scp opennebula-ceph-cpds-clone.diff root@$play_host:~/
ssh root@$play_host <<EOF
set -e -x
if ! grep -q clone /var/lib/one/remotes/tm/ceph/cpds; then
    cd /
    patch -p0 < /root/opennebula-ceph-cpds-clone.diff
fi
EOF