Bash "playbooks" for OpenNebula

README.md

@@ -0,0 +1,7 @@
+"Плейбуки" на баше для конфигурации Ceph + OpenNebula
+
+Почему на баше?
+
+Потому что с ансиблом бесит yaml программирование!
+
+Логика та же: можно запускать многократно.

etc/apt/apt.conf

@@ -0,0 +1,2 @@
+APT::Install-Suggests false;
+APT::Install-Recommends false;

etc/apt/sources.list

@@ -0,0 +1,14 @@
+#deb http://http.debian.net/debian/ sid main contrib non-free
+#deb-src http://http.debian.net/debian/ sid main contrib non-free
+
+deb http://http.debian.net/debian/ buster main contrib non-free
+deb-src http://http.debian.net/debian/ buster main contrib non-free
+
+deb http://security.debian.org/debian-security buster/updates main
+deb-src http://security.debian.org/debian-security buster/updates main
+
+# buster-updates, previously known as 'volatile'
+deb http://http.debian.net/debian/ buster-updates main
+deb-src http://http.debian.net/debian/ buster-updates main
+
+#deb http://hwraid.le-vert.net/debian stretch main

etc/apt/sources.list.d/ceph.list

@@ -0,0 +1,2 @@
+deb http://download.ceph.com/debian-nautilus/ bionic main
+deb-src http://download.ceph.com/debian-nautilus/ bionic main

etc/apt/sources.list.d/mariadb.list

@@ -0,0 +1,2 @@
+deb [arch=amd64] http://mirror.mephi.ru/mariadb/repo/10.3/debian buster main
+deb-src http://mirror.mephi.ru/mariadb/repo/10.3/debian buster main

etc/apt/sources.list.d/opennebula.list

@@ -0,0 +1 @@
+deb https://downloads.opennebula.org/repo/5.8/Debian/9 stable opennebula

etc/keepalived/keepalived.conf.env

@@ -0,0 +1,29 @@
+! Configuration File for keepalived
+
+global_defs {
+    notification_email {
+        filippov@custis.ru
+    }
+    notification_email_from filippov@custis.ru
+    smtp_server localhost
+    smtp_connect_timeout 30
+    router_id $keepalived_router_id
+}
+
+vrrp_instance VI_1 {
+    state BACKUP
+    interface br0
+    garp_master_delay 10
+    smtp_alert
+    virtual_router_id 51
+    lvs_sync_daemon_interface $eth10g
+    priority 100
+    advert_int 1
+    authentication {
+        auth_type PASS
+        auth_pass $keepalived_password
+    }
+    virtual_ipaddress {
+        $keepalived_virtual_ip
+    }
+}

etc/locale.gen

@@ -0,0 +1,2 @@
+en_US.UTF-8 UTF-8
+ru_RU.UTF-8 UTF-8

etc/mysql/mariadb.conf.d/50-client.cnf

@@ -0,0 +1,3 @@
+[client]
+default-character-set = utf8mb4
+socket = /var/run/mysqld/mysqld.sock

etc/mysql/mariadb.conf.d/50-server.cnf.env

@@ -0,0 +1,35 @@
+[mysqld]
+user=mysql
+pid-file=/var/run/mysqld/mysqld.pid
+socket=/var/run/mysqld/mysqld.sock
+port=3306
+basedir=/usr
+datadir=/var/lib/mysql
+tmpdir=/tmp
+lc-messages-dir=/usr/share/mysql
+skip-external-locking
+
+character-set-server=utf8mb4
+collation-server=utf8mb4_general_ci
+init-connect='SET NAMES utf8'
+binlog_format=ROW
+default-storage-engine=innodb
+innodb_autoinc_lock_mode=2
+innodb_locks_unsafe_for_binlog=1
+query_cache_size=0
+query_cache_type=0
+bind-address=0.0.0.0
+innodb_log_file_size=100M
+innodb_file_per_table
+innodb_flush_log_at_trx_commit=2
+transaction-isolation=READ-COMMITTED
+innodb_default_row_format=dynamic
+
+wsrep_on=ON
+wsrep_provider=/usr/lib/galera/libgalera_smm.so
+wsrep_cluster_address="gcomm://$galera_hosts"
+wsrep_cluster_name='$galera_cluster_name'
+wsrep_node_address='$int_ip'
+wsrep_node_name='$node_name'
+wsrep_sst_method=rsync
+wsrep_sst_auth=sst_user:$galera_password

etc/network/interfaces.env

@@ -0,0 +1,18 @@
+source /etc/network/interfaces.d/*
+
+auto lo
+iface lo inet loopback
+
+#allow-hotplug $eth1g
+iface $eth1g inet manual
+
+auto br0
+iface br0 inet dhcp
+    bridge_ports $eth1g
+    # Fucking ifupdown 0.8.35 uses DUID for DHCP4 O_o
+    post-up ip addr add $play_host dev br0
+
+auto $eth10g
+iface $eth10g inet static
+    address $int_ip/24
+    mtu 9000

etc/rc.local.env

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+ethtool -K $eth10g gro off gso off tso off lro off sg off
+ip l set $eth10g mtu 9000
+exit 0

etc/sysctl.conf

@@ -0,0 +1,2 @@
+net.ipv4.ip_nonlocal_bind=1
+vm.swappiness=20

etc/systemd/timesyncd.conf

@@ -0,0 +1,7 @@
+[Time]
+NTP=10.200.20.11
+#NTP=
+#FallbackNTP=0.debian.pool.ntp.org 1.debian.pool.ntp.org 2.debian.pool.ntp.org 3.debian.pool.ntp.org
+#RootDistanceMaxSec=5
+#PollIntervalMinSec=32
+#PollIntervalMaxSec=2048

general.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# -e = stop on exception, -x = debug, -a = export all variables
+set -e -x -a
+
+# Include config
+. all_vars
+
+### Check host variables
+if [ -z "$play_host" -o -z "$node_name" -o -z "$int_ip" ]; then
+    echo "play_host/node_name/int_ip not specified"
+    exit 1
+fi
+
+scp ./etc/apt/apt.conf root@$play_host:/etc/apt/apt.conf
+scp ./etc/apt/sources.list root@$play_host:/etc/apt/sources.list
+scp ./etc/locale.gen root@$play_host:/etc/locale.gen
+ssh root@$play_host <<EOF
+set -e -x
+
+echo $node_name > /etc/hostname
+
+hostname `cat /etc/hostname`
+
+apt-get update
+
+# gpg and friends for apt-key to work correctly
+DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" \
+    -o Dpkg::Options::="--force-confold" install -y \
+    mc wget less locales telnet atop sysstat tmux \
+    gpg gpg-agent dirmngr apt-transport-https \
+    prometheus-node-exporter
+
+echo Europe/Moscow > /etc/timezone
+ln -fs /usr/share/zoneinfo/Europe/Moscow /etc/timezone
+
+if ! grep -q '^PermitRootLogin' /etc/ssh/sshd_config; then
+    echo PermitRootLogin without-password >> /etc/ssh/sshd_config
+    service ssh restart
+elif ! grep -q '^PermitRootLogin without-password' /etc/ssh/sshd_config; then
+    perl -i -pe 's/^PermitRootLogin.*/PermitRootLogin without-password/' /etc/ssh/sshd_config
+    service ssh restart
+fi
+EOF

opennebula.sh

@@ -0,0 +1,180 @@
+#!/bin/bash
+# Install & configure OpenNebula with MariaDB Galera Cluster as HA
+# This script is idempotent like an Ansible playbook!
+# I.e. run it as many times as you want to, it won't hurt!
+
+# -e = stop on exception, -x = debug, -a = export all variables
+set -e -x -a
+
+# Include config
+. all_vars
+
+### Check host variables
+if [ -z "$play_host" -o -z "$node_name" -o -z "$int_ip" ]; then
+    echo "play_host/node_name/int_ip not specified"
+    exit 1
+fi
+
+### Find 1G and 10G network interfaces (10G is used for keepalived and galera)
+eth10g=
+eth1g=
+export $(ssh root@$play_host '
+    for i in /sys/class/net/*; do
+        ip link set ${i##/sys/class/net/} up
+        if [ x`cat /sys/class/net/enp4s0/carrier 2>/dev/null` == "x1" ]; then
+            if [ x`cat $i/speed 2>/dev/null` == "x10000" ]; then
+                echo eth10g=${i##/sys/class/net/}
+            elif [ x`cat $i/speed 2>/dev/null` == "x1000" ]; then
+                echo eth1g=${i##/sys/class/net/}
+            fi
+        fi
+    done')
+
+if [ -z "$eth10g" ]; then
+    echo "10GbE network not found on $play_host"
+    exit 1
+fi
+
+### Configure network
+envsubst < ./etc/rc.local.env | ssh root@$play_host 'cat > /etc/rc.local'
+envsubst < ./etc/network/interfaces.env | ssh root@$play_host 'cat > /etc/network/interfaces.new'
+ssh root@$play_host <<EOF
+set -e
+
+DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" \
+    -o Dpkg::Options::="--force-confold" install -y \
+    bridge-utils
+
+chmod 755 /etc/rc.local
+/etc/rc.local
+systemctl enable rc-local
+if ! cmp -s /etc/network/interfaces /etc/network/interfaces.new; then
+    nmcli dev disconnect $eth1g; true
+    mv /etc/network/interfaces.new /etc/network/interfaces
+    ifup br0
+    service networking restart
+fi
+EOF
+
+### Install packages
+scp ./etc/apt/sources.list.d/opennebula.list root@$play_host:/etc/apt/sources.list.d/opennebula.list
+scp ./etc/apt/sources.list.d/mariadb.list root@$play_host:/etc/apt/sources.list.d/mariadb.list
+
+ssh root@$play_host <<EOF
+set -e
+
+wget -q -O - https://downloads.opennebula.org/repo/repo.key | apt-key add -
+
+apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8
+
+DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" \
+    -o Dpkg::Options::="--force-confold" install -y \
+    lsb-release keepalived sudo qemu-kvm qemu-block-extra mariadb-server netcat-openbsd \
+    opennebula opennebula-sunstone opennebula-gate opennebula-flow opennebula-node opennebula-tools
+
+/usr/share/one/install_gems --yes
+EOF
+
+### Setup keepalived
+envsubst < ./etc/keepalived/keepalived.conf.env | \
+    ssh root@$play_host 'cat > /etc/keepalived/keepalived.conf'
+ssh root@$play_host 'systemctl restart keepalived'
+
+### Setup or join MariaDB Galera Cluster
+scp ./etc/mysql/mariadb.conf.d/50-client.cnf root@$play_host:/etc/mysql/mariadb.conf.d/50-client.cnf
+
+# Put all hosts except this one in wsrep_cluster_address
+galera_hosts=
+for host in $opennebula_hosts; do
+    if [ "$host" != "$play_host" ]; then
+        galera_hosts=$galera_hosts,$host
+    fi
+done
+galera_hosts=${galera_hosts##,}
+envsubst < ./etc/mysql/mariadb.conf.d/50-server.cnf.env | \
+    ssh root@$play_host 'cat > /etc/mysql/mariadb.conf.d/50-server.cnf'
+
+# Create a user for Galera (if not yet)
+ssh root@$play_host <<EOF
+set -e -x
+
+if ! (echo 'SELECT 1' | mysql --host=$play_host -u sst_user --password=$galera_password); then
+    service mysql restart
+    mysql <<EOM
+        GRANT USAGE ON *.* to sst_user@'%' IDENTIFIED BY '$galera_password';
+        GRANT ALL PRIVILEGES on *.* to sst_user@'%';
+        FLUSH PRIVILEGES;
+EOM
+fi
+
+[ -h /etc/mysql/my.cnf ] || rm /etc/mysql/my.cnf && ln -fs /etc/mysql/mariadb.cnf /etc/mysql/my.cnf
+EOF
+
+if [ "$init_db" -eq 1 ]; then
+    # Create a new cluster
+    ssh root@$play_host <<EOF
+        set -e -x
+        systemctl stop mysql
+        systemctl start mysql --wsrep-new-cluster
+        mysql <<EOM
+            CREATE DATABASE opennebula;
+            GRANT ALL PRIVILEGES ON opennebula.* TO 'oneadmin' IDENTIFIED BY '$opennebula_db_password';
+            FLUSH PRIVILEGES;
+EOM
+EOF
+else
+    # Or just join it
+    ssh root@$play_host 'systemctl restart mysql'
+fi
+
+### Setup OpenNebula oned and sunstone
+ssh root@$play_host <<EOF
+echo oneadmin:$oneadmin_password > /var/lib/one/.one/one_auth
+echo serveradmin:$serveradmin_password > /var/lib/one/.one/ec2_auth
+echo serveradmin:$serveradmin_password > /var/lib/one/.one/occi_auth
+echo serveradmin:$serveradmin_password > /var/lib/one/.one/oneflow_auth
+echo serveradmin:$serveradmin_password > /var/lib/one/.one/onegate_auth
+echo serveradmin:$serveradmin_password > /var/lib/one/.one/sunstone_auth
+echo $one_key > /var/lib/one/.one/one_key
+chown oneadmin:oneadmin /var/lib/one/.one/*
+chmod 600 /var/lib/one/.one/*
+EOF
+
+ssh root@$play_host <<EOF
+set -e -x
+
+perl -i -pe 's!^DB\s*=.*!DB = [ backend = "mysql", server = "localhost", port = 0, user = "oneadmin", passwd = "$opennebula_db_password", db_name = "opennebula" ]!' /etc/one/oned.conf
+
+while ! echo SELECT 1 | mysql; do
+    echo Waiting for MySQL...
+done
+
+systemctl enable opennebula
+systemctl enable opennebula-sunstone
+systemctl restart opennebula
+systemctl restart opennebula-sunstone
+
+if [ ! -f /var/lib/one/.ssh/id_rsa.pub ]; then
+    su - oneadmin -c ssh-keygen
+    su - oneadmin -c 'ssh-keyscan localhost >> /var/lib/one/.ssh/known_hosts'
+    for host in $opennebula_hosts; do
+        su - oneadmin -c "ssh-keyscan $host >> /var/lib/one/.ssh/known_hosts"
+    done
+fi
+EOF
+
+# Setup passwordless ssh for `oneadmin`
+> tmp$$
+for host in $opennebula_hosts; do
+    ssh root@$host 'cat /var/lib/one/.ssh/id_rsa.pub' >> tmp$$
+done
+ssh root@$play_host 'cat /var/lib/one/.ssh/authorized_keys; true' >> tmp$$
+cat tmp$$ | sort | uniq | ssh root@$play_host 'cat > /var/lib/one/.ssh/authorized_keys'
+rm tmp$$
+
+# Add a host to OpenNebula
+ssh root@$play_host <<EOF
+set -e -x
+service libvirtd restart
+onehost list | grep $play_host || onehost create $play_host -i kvm -v kvm
+EOF

run.sh

@@ -0,0 +1,2 @@
+play_host=172.31.1.9 node_name=ripper4 int_ip=192.168.5.14 ./general.sh
+play_host=172.31.1.9 node_name=ripper4 int_ip=192.168.5.14 ./opennebula.sh