Some fixes based on the deployment of another threadripper node :)

all_vars.sample

@@ -11,3 +11,6 @@ oneadmin_password=
 serveradmin_password=
 one_domain=one.custis.ru
 libvirt_secret_uuid=d46404df-387c-4a3e-877e-1c63955f06d8
+gateway=172.31.1.1
+dns=172.31.1.1
+ntp_server=10.200.20.11

ceph.sh

@@ -25,23 +25,19 @@ if [ ! -f /root/.ssh/id_rsa.pub ]; then
 fi
 
 > tmp$$
-cat /root/.ssh/known_hosts >> tmp$$; true
+cat /root/.ssh/known_hosts >> tmp$$ || true
 ssh-keyscan localhost >> tmp$$
 ssh-keyscan $int_ip >> tmp$$
 sort tmp$$ | uniq > /root/.ssh/known_hosts
 rm tmp$$
 
 > tmp$$
-cat /root/.ssh/authorized_keys >> tmp$$; true
+cat /root/.ssh/authorized_keys >> tmp$$ || true
 cat /root/.ssh/id_rsa.pub >> tmp$$
 sort tmp$$ | uniq > /root/.ssh/authorized_keys
 rm tmp$$
 EOF
 
-### Set time sync
-scp ./etc/systemd/timesyncd.conf root@$play_host:/etc/systemd/
-ssh root@$play_host 'systemctl enable systemd-timesyncd && systemctl restart systemd-timesyncd'
-
 ### Install packages, deploy mon, mgr and osds
 scp -r ceph-deploy root@$play_host:~/
 

etc/network/interfaces.env

@@ -6,11 +6,14 @@ iface lo inet loopback
 #allow-hotplug $eth1g
 iface $eth1g inet manual
 
+# Fucking ifupdown 0.8.35 uses DUID for DHCP4 O_o
+# So we'll use static IPs
 auto br0
-iface br0 inet dhcp
+iface br0 inet static
     bridge_ports $eth1g
-    # Fucking ifupdown 0.8.35 uses DUID for DHCP4 O_o
-    post-up ip addr add $play_host dev br0
+    address $play_host/24
+    gateway $gateway
+    dns-nameservers $dns
 
 auto $eth10g
 iface $eth10g inet static

etc/systemd/timesyncd.conf.env

@@ -1,5 +1,5 @@
 [Time]
-NTP=10.200.20.11
+NTP=$ntp_server
 #NTP=
 #FallbackNTP=0.debian.pool.ntp.org 1.debian.pool.ntp.org 2.debian.pool.ntp.org 3.debian.pool.ntp.org
 #RootDistanceMaxSec=5

general.sh

@@ -15,18 +15,21 @@ fi
 scp ./etc/apt/apt.conf root@$play_host:/etc/apt/apt.conf
 scp ./etc/apt/sources.list root@$play_host:/etc/apt/sources.list
 scp ./etc/locale.gen root@$play_host:/etc/locale.gen
+
+# Set time sync
+envsubst < ./etc/systemd/timesyncd.conf.env | \
+    ssh root@$play_host 'cat > /etc/systemd/timesyncd.conf'
+
 ssh root@$play_host <<EOF
 set -e -x
 
 echo $node_name > /etc/hostname
 
-hostname `cat /etc/hostname`
+hostname \`cat /etc/hostname\`
 
-grep -q -P '127.0.1.1\s+$node_name' /etc/hosts || (echo "127.0.1.1 $node_name" >> /etc/hosts)
+systemctl enable systemd-timesyncd && systemctl restart systemd-timesyncd
 
-wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add -
-
-apt-get update
+apt-get update || true
 
 # gpg and friends for apt-key to work correctly
 DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" \
@@ -35,6 +38,12 @@ DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" \
     gpg gpg-agent dirmngr apt-transport-https \
     prometheus-node-exporter
 
+grep -q -P '127.0.1.1\s+$node_name' /etc/hosts || (echo "127.0.1.1 $node_name" >> /etc/hosts)
+
+wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add -
+
+apt-get update || true
+
 echo Europe/Moscow > /etc/timezone
 ln -fs /usr/share/zoneinfo/Europe/Moscow /etc/timezone
 

network.sh

@@ -35,17 +35,17 @@ fi
 envsubst < ./etc/rc.local.env | ssh root@$play_host 'cat > /etc/rc.local'
 envsubst < ./etc/network/interfaces.env | ssh root@$play_host 'cat > /etc/network/interfaces.new'
 ssh root@$play_host <<EOF
-set -e
+set -e -x
 
 DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" \
     -o Dpkg::Options::="--force-confold" install -y \
-    bridge-utils
+    bridge-utils resolvconf
 
 chmod 755 /etc/rc.local
 /etc/rc.local
 systemctl enable rc-local
 if ! cmp -s /etc/network/interfaces /etc/network/interfaces.new; then
-    nmcli dev disconnect $eth1g; true
+    nmcli dev disconnect $eth1g || true
     mv /etc/network/interfaces.new /etc/network/interfaces
     ifup br0
     service networking restart

opennebula.sh

@@ -23,12 +23,14 @@ scp ./etc/apt/sources.list.d/opennebula.list root@$play_host:/etc/apt/sources.li
 scp ./etc/apt/sources.list.d/mariadb.list root@$play_host:/etc/apt/sources.list.d/mariadb.list
 
 ssh root@$play_host <<EOF
-set -e
+set -e -x
 
 wget -q -O - https://downloads.opennebula.org/repo/repo.key | apt-key add -
 
 apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8
 
+apt-get update || true
+
 DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" \
     -o Dpkg::Options::="--force-confold" install -y \
     lsb-release keepalived sudo qemu-kvm qemu-block-extra mariadb-server netcat-openbsd \
@@ -135,6 +137,7 @@ systemctl restart onedns
 EOF
 
 # Setup passwordless ssh for `oneadmin` (authorized_keys and known_hosts)
+# FIXME: Maybe extract to separate script
 ssh root@$play_host <<EOF
 set -e -x
 
@@ -154,7 +157,7 @@ EOF
 for host in $opennebula_hosts; do
     ssh root@$host 'cat /var/lib/one/.ssh/id_rsa.pub' >> tmp$$
 done
-ssh root@$play_host 'cat /var/lib/one/.ssh/authorized_keys; true' >> tmp$$
+ssh root@$play_host 'cat /var/lib/one/.ssh/authorized_keys || true' >> tmp$$
 cat tmp$$ | sort | uniq | ssh root@$play_host 'cat > /var/lib/one/.ssh/authorized_keys'
 rm tmp$$
 

run.sh

@@ -1,5 +1,12 @@
 #!/bin/bash
 
+set -e -x
+
 play_host=172.31.1.9 node_name=ripper4 int_ip=192.168.5.14 ./general.sh
 play_host=172.31.1.9 node_name=ripper4 int_ip=192.168.5.14 ./opennebula.sh
 play_host=172.31.1.9 node_name=ripper4 int_ip=192.168.5.14 ./ceph.sh
+exit
+
+play_host=172.31.1.16 node_name=ripper5 int_ip=192.168.5.15 ./general.sh
+play_host=172.31.1.16 node_name=ripper5 int_ip=192.168.5.15 ./opennebula.sh
+play_host=172.31.1.16 node_name=ripper5 int_ip=192.168.5.15 ./ceph.sh