forked from vitalif/vitastor
A little better systemd unit generator
parent
91a70dfb1b
commit
00e98f64f3
|
@ -0,0 +1,76 @@
|
||||||
|
#!/bin/bash
|
||||||
|
# Very simple systemd unit generator for vitastor-osd services
|
||||||
|
# Not the final solution yet, mostly for tests
|
||||||
|
# Copyright (c) Vitaliy Filippov, 2019+
|
||||||
|
# License: MIT
|
||||||
|
|
||||||
|
# USAGE: ./make-osd.sh /dev/disk/by-partuuid/xxx [ /dev/disk/by-partuuid/yyy]...
|
||||||
|
|
||||||
|
IP_SUBSTR="10.200.1."
|
||||||
|
ETCD_HOSTS="etcd0=http://10.200.1.10:2380,etcd1=http://10.200.1.11:2380,etcd2=http://10.200.1.12:2380"
|
||||||
|
|
||||||
|
set -e -x
|
||||||
|
|
||||||
|
IP=`ip -json a s | jq -r '.[].addr_info[] | select(.local | startswith("'$IP_SUBSTR'")) | .local'`
|
||||||
|
[ "$IP" != "" ] || exit 1
|
||||||
|
ETCD_MON=$(echo $ETCD_HOSTS | perl -pe 's/:2380/:2379/g; s/etcd\d*=//g;')
|
||||||
|
D=`dirname $0`
|
||||||
|
|
||||||
|
# Create OSDs on all passed devices
|
||||||
|
OSD_NUM=1
|
||||||
|
for DEV in $*; do
|
||||||
|
|
||||||
|
# Ugly :) -> node.js rework pending
|
||||||
|
while true; do
|
||||||
|
ST=$(etcdctl --endpoints="$ETCD_MON" get --print-value-only /vitastor/osd/stats/$OSD_NUM)
|
||||||
|
if [ "$ST" = "" ]; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
OSD_NUM=$((OSD_NUM+1))
|
||||||
|
done
|
||||||
|
etcdctl --endpoints="$ETCD_MON" put /vitastor/osd/stats/$OSD_NUM '{}'
|
||||||
|
|
||||||
|
echo Creating OSD $OSD_NUM on $DEV
|
||||||
|
|
||||||
|
OPT=`node $D/simple-offsets.js --device $DEV --format options | tr '\n' ' '`
|
||||||
|
META=`echo $OPT | grep -Po '(?<=data_offset )\d+'`
|
||||||
|
dd if=/dev/zero of=$DEV bs=1048576 count=$(((META+1048575)/1048576)) oflag=direct
|
||||||
|
|
||||||
|
cat >/etc/systemd/system/vitastor-osd$OSD_NUM.service <<EOF
|
||||||
|
[Unit]
|
||||||
|
Description=Vitastor object storage daemon osd.$OSD_NUM
|
||||||
|
After=network-online.target local-fs.target time-sync.target
|
||||||
|
Wants=network-online.target local-fs.target time-sync.target
|
||||||
|
PartOf=vitastor.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
LimitNOFILE=1048576
|
||||||
|
LimitNPROC=1048576
|
||||||
|
LimitMEMLOCK=infinity
|
||||||
|
ExecStart=/usr/bin/vitastor-osd \\
|
||||||
|
--etcd_address $IP:2379/v3 \\
|
||||||
|
--bind_address $IP \\
|
||||||
|
--osd_num $OSD_NUM \\
|
||||||
|
--disable_data_fsync 1 \\
|
||||||
|
--immediate_commit all \\
|
||||||
|
--flusher_count 8 \\
|
||||||
|
--disk_alignment 4096 --journal_block_size 4096 --meta_block_size 4096 \\
|
||||||
|
--journal_no_same_sector_overwrites true \\
|
||||||
|
--journal_sector_buffer_count 1024 \\
|
||||||
|
$OPT
|
||||||
|
WorkingDirectory=/
|
||||||
|
ExecStartPre=+chown vitastor:vitastor $DEV
|
||||||
|
User=vitastor
|
||||||
|
PrivateTmp=false
|
||||||
|
TasksMax=infinity
|
||||||
|
Restart=always
|
||||||
|
StartLimitInterval=0
|
||||||
|
RestartSec=10
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=vitastor.target
|
||||||
|
EOF
|
||||||
|
|
||||||
|
systemctl enable vitastor-osd$OSD_NUM
|
||||||
|
|
||||||
|
done
|
|
@ -1,19 +1,25 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
# Example startup script generator
|
# Very simple systemd unit generator for etcd & vitastor-mon services
|
||||||
# Of course this isn't a production solution yet, this is just for tests
|
# Not the final solution yet, mostly for tests
|
||||||
# Copyright (c) Vitaliy Filippov, 2019+
|
# Copyright (c) Vitaliy Filippov, 2019+
|
||||||
# License: MIT
|
# License: MIT
|
||||||
|
|
||||||
IP=`ip -json a s | jq -r '.[].addr_info[] | select(.broadcast == "10.115.0.255") | .local'`
|
# USAGE: ./make-units.sh
|
||||||
|
|
||||||
|
IP_SUBSTR="10.200.1."
|
||||||
|
ETCD_HOSTS="etcd0=http://10.200.1.10:2380,etcd1=http://10.200.1.11:2380,etcd2=http://10.200.1.12:2380"
|
||||||
|
|
||||||
|
# determine IP
|
||||||
|
IP=`ip -json a s | jq -r '.[].addr_info[] | select(.local | startswith("'$IP_SUBSTR'")) | .local'`
|
||||||
[ "$IP" != "" ] || exit 1
|
[ "$IP" != "" ] || exit 1
|
||||||
|
ETCD_NUM=${ETCD_HOSTS/$IP*/}
|
||||||
|
[ "$ETCD_NUM" != "$ETCD_HOSTS" ] || exit 1
|
||||||
|
ETCD_NUM=$(echo $ETCD_NUM | tr -d -c , | wc -c)
|
||||||
|
|
||||||
BASE=${IP/*./}
|
# etcd
|
||||||
BASE=$((BASE-10))
|
|
||||||
|
|
||||||
useradd etcd
|
useradd etcd
|
||||||
|
|
||||||
mkdir -p /var/lib/etcd$BASE.etcd
|
mkdir -p /var/lib/etcd$ETCD_NUM.etcd
|
||||||
cat >/etc/systemd/system/etcd.service <<EOF
|
cat >/etc/systemd/system/etcd.service <<EOF
|
||||||
[Unit]
|
[Unit]
|
||||||
Description=etcd for vitastor
|
Description=etcd for vitastor
|
||||||
|
@ -22,19 +28,18 @@ Wants=network-online.target local-fs.target time-sync.target
|
||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
Restart=always
|
Restart=always
|
||||||
ExecStart=/usr/local/bin/etcd -name etcd$BASE --data-dir /var/lib/etcd$BASE.etcd \\
|
ExecStart=/usr/local/bin/etcd -name etcd$ETCD_NUM --data-dir /var/lib/etcd$ETCD_NUM.etcd \\
|
||||||
--advertise-client-urls http://$IP:2379 --listen-client-urls http://$IP:2379 \\
|
--advertise-client-urls http://$IP:2379 --listen-client-urls http://$IP:2379 \\
|
||||||
--initial-advertise-peer-urls http://$IP:2380 --listen-peer-urls http://$IP:2380 \\
|
--initial-advertise-peer-urls http://$IP:2380 --listen-peer-urls http://$IP:2380 \\
|
||||||
--initial-cluster-token vitastor-etcd-1 --initial-cluster etcd0=http://10.115.0.10:2380,etcd1=http://10.115.0.11:2380,etcd2=http://10.115.0.12:2380,etcd3=http://10.115.0.13:2380 \\
|
--initial-cluster-token vitastor-etcd-1 --initial-cluster $ETCD_HOSTS \\
|
||||||
--initial-cluster-state new --max-txn-ops=100000 --auto-compaction-retention=10 --auto-compaction-mode=revision
|
--initial-cluster-state new --max-txn-ops=100000 --auto-compaction-retention=10 --auto-compaction-mode=revision
|
||||||
WorkingDirectory=/var/lib/etcd$BASE.etcd
|
WorkingDirectory=/var/lib/etcd$ETCD_NUM.etcd
|
||||||
ExecStartPre=+chown -R etcd /var/lib/etcd$BASE.etcd
|
ExecStartPre=+chown -R etcd /var/lib/etcd$ETCD_NUM.etcd
|
||||||
User=etcd
|
User=etcd
|
||||||
PrivateTmp=false
|
PrivateTmp=false
|
||||||
TasksMax=infinity
|
TasksMax=infinity
|
||||||
Restart=always
|
Restart=always
|
||||||
StartLimitInterval=0
|
StartLimitInterval=0
|
||||||
StartLimitIntervalSec=0
|
|
||||||
RestartSec=10
|
RestartSec=10
|
||||||
|
|
||||||
[Install]
|
[Install]
|
||||||
|
@ -48,9 +53,7 @@ systemctl start etcd
|
||||||
useradd vitastor
|
useradd vitastor
|
||||||
chmod 755 /root
|
chmod 755 /root
|
||||||
|
|
||||||
BASE=${IP/*./}
|
# Vitastor target
|
||||||
BASE=$(((BASE-10)*12))
|
|
||||||
|
|
||||||
cat >/etc/systemd/system/vitastor.target <<EOF
|
cat >/etc/systemd/system/vitastor.target <<EOF
|
||||||
[Unit]
|
[Unit]
|
||||||
Description=vitastor target
|
Description=vitastor target
|
||||||
|
@ -58,116 +61,25 @@ Description=vitastor target
|
||||||
WantedBy=multi-user.target
|
WantedBy=multi-user.target
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
i=1
|
# Monitor unit
|
||||||
for DEV in `ls /dev/disk/by-id/ | grep ata-INTEL_SSDSC2KB`; do
|
ETCD_MON=$(echo $ETCD_HOSTS | perl -pe 's/:2380/:2379/g; s/etcd\d*=//g;')
|
||||||
dd if=/dev/zero of=/dev/disk/by-id/$DEV bs=1048576 count=$(((427814912+1048575)/1048576+2))
|
cat >/etc/systemd/system/vitastor-mon.service <<EOF
|
||||||
dd if=/dev/zero of=/dev/disk/by-id/$DEV bs=1048576 count=$(((427814912+1048575)/1048576+2)) seek=$((1920377991168/1048576))
|
|
||||||
cat >/etc/systemd/system/vitastor-osd$((BASE+i)).service <<EOF
|
|
||||||
[Unit]
|
[Unit]
|
||||||
Description=Vitastor object storage daemon osd.$((BASE+i))
|
Description=Vitastor monitor
|
||||||
After=network-online.target local-fs.target time-sync.target
|
After=network-online.target local-fs.target time-sync.target
|
||||||
Wants=network-online.target local-fs.target time-sync.target
|
Wants=network-online.target local-fs.target time-sync.target
|
||||||
PartOf=vitastor.target
|
|
||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
LimitNOFILE=1048576
|
Restart=always
|
||||||
LimitNPROC=1048576
|
ExecStart=node /usr/lib/vitastor/mon/mon-main.js --etcd_url '$ETCD_MON' --etcd_prefix '/vitastor' --etcd_start_timeout 5
|
||||||
LimitMEMLOCK=infinity
|
WorkingDirectory=/
|
||||||
ExecStart=/root/vitastor/osd \\
|
|
||||||
--etcd_address $IP:2379/v3 \\
|
|
||||||
--bind_address $IP \\
|
|
||||||
--osd_num $((BASE+i)) \\
|
|
||||||
--disable_data_fsync 1 \\
|
|
||||||
--disable_device_lock 1 \\
|
|
||||||
--immediate_commit all \\
|
|
||||||
--flusher_count 8 \\
|
|
||||||
--disk_alignment 4096 --journal_block_size 4096 --meta_block_size 4096 \\
|
|
||||||
--journal_no_same_sector_overwrites true \\
|
|
||||||
--journal_sector_buffer_count 1024 \\
|
|
||||||
--journal_offset 0 \\
|
|
||||||
--meta_offset 16777216 \\
|
|
||||||
--data_offset 427814912 \\
|
|
||||||
--data_size $((1920377991168-427814912)) \\
|
|
||||||
--data_device /dev/disk/by-id/$DEV
|
|
||||||
WorkingDirectory=/root/vitastor
|
|
||||||
ExecStartPre=+chown vitastor:vitastor /dev/disk/by-id/$DEV
|
|
||||||
User=vitastor
|
User=vitastor
|
||||||
PrivateTmp=false
|
PrivateTmp=false
|
||||||
TasksMax=infinity
|
TasksMax=infinity
|
||||||
Restart=always
|
Restart=always
|
||||||
StartLimitInterval=0
|
StartLimitInterval=0
|
||||||
StartLimitIntervalSec=0
|
|
||||||
RestartSec=10
|
RestartSec=10
|
||||||
|
|
||||||
[Install]
|
[Install]
|
||||||
WantedBy=vitastor.target
|
WantedBy=vitastor.target
|
||||||
EOF
|
EOF
|
||||||
systemctl enable vitastor-osd$((BASE+i))
|
|
||||||
i=$((i+1))
|
|
||||||
cat >/etc/systemd/system/vitastor-osd$((BASE+i)).service <<EOF
|
|
||||||
[Unit]
|
|
||||||
Description=Vitastor object storage daemon osd.$((BASE+i))
|
|
||||||
After=network-online.target local-fs.target time-sync.target
|
|
||||||
Wants=network-online.target local-fs.target time-sync.target
|
|
||||||
PartOf=vitastor.target
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
LimitNOFILE=1048576
|
|
||||||
LimitNPROC=1048576
|
|
||||||
LimitMEMLOCK=infinity
|
|
||||||
ExecStart=/root/vitastor/osd \\
|
|
||||||
--etcd_address $IP:2379/v3 \\
|
|
||||||
--bind_address $IP \\
|
|
||||||
--osd_num $((BASE+i)) \\
|
|
||||||
--disable_data_fsync 1 \\
|
|
||||||
--immediate_commit all \\
|
|
||||||
--flusher_count 8 \\
|
|
||||||
--disk_alignment 4096 --journal_block_size 4096 --meta_block_size 4096 \\
|
|
||||||
--journal_no_same_sector_overwrites true \\
|
|
||||||
--journal_sector_buffer_count 1024 \\
|
|
||||||
--journal_offset 1920377991168 \\
|
|
||||||
--meta_offset $((1920377991168+16777216)) \\
|
|
||||||
--data_offset $((1920377991168+427814912)) \\
|
|
||||||
--data_size $((1920377991168-427814912)) \\
|
|
||||||
--data_device /dev/disk/by-id/$DEV
|
|
||||||
WorkingDirectory=/root/vitastor
|
|
||||||
ExecStartPre=+chown vitastor:vitastor /dev/disk/by-id/$DEV
|
|
||||||
User=vitastor
|
|
||||||
PrivateTmp=false
|
|
||||||
TasksMax=infinity
|
|
||||||
Restart=always
|
|
||||||
StartLimitInterval=0
|
|
||||||
StartLimitIntervalSec=0
|
|
||||||
RestartSec=10
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=vitastor.target
|
|
||||||
EOF
|
|
||||||
systemctl enable vitastor-osd$((BASE+i))
|
|
||||||
i=$((i+1))
|
|
||||||
done
|
|
||||||
|
|
||||||
exit
|
|
||||||
|
|
||||||
node mon-main.js --etcd_url 'http://10.115.0.10:2379,http://10.115.0.11:2379,http://10.115.0.12:2379,http://10.115.0.13:2379' --etcd_prefix '/vitastor' --etcd_start_timeout 5
|
|
||||||
|
|
||||||
podman run -d --network host --restart always -v /var/lib/etcd0.etcd:/etcd0.etcd --name etcd quay.io/coreos/etcd:v3.4.13 etcd -name etcd0 \
|
|
||||||
-advertise-client-urls http://10.115.0.10:2379 -listen-client-urls http://10.115.0.10:2379 \
|
|
||||||
-initial-advertise-peer-urls http://10.115.0.10:2380 -listen-peer-urls http://10.115.0.10:2380 \
|
|
||||||
-initial-cluster-token vitastor-etcd-1 -initial-cluster etcd0=http://10.115.0.10:2380,etcd1=http://10.115.0.11:2380,etcd2=http://10.115.0.12:2380,etcd3=http://10.115.0.13:2380 \
|
|
||||||
-initial-cluster-state new --max-txn-ops=100000 --auto-compaction-retention=10 --auto-compaction-mode=revision
|
|
||||||
|
|
||||||
etcdctl --endpoints http://10.115.0.10:2379 put /vitastor/config/global '{"immediate_commit":"all"}'
|
|
||||||
|
|
||||||
etcdctl --endpoints http://10.115.0.10:2379 put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":48,"failure_domain":"host"}}'
|
|
||||||
|
|
||||||
#let pgs = {};
|
|
||||||
#for (let n = 0; n < 48; n++) { let i = n/2 | 0; pgs[1+n] = { osd_set: [ (1+i%12+(i/12 | 0)*24), (1+12+i%12+(i/12 | 0)*24) ], primary: (1+(n%2)*12+i%12+(i/12 | 0)*24) }; };
|
|
||||||
#console.log(JSON.stringify({ items: { 1: pgs } }));
|
|
||||||
#etcdctl --endpoints http://10.115.0.10:2379 put /vitastor/config/pgs ...
|
|
||||||
|
|
||||||
# --disk_alignment 4096 --journal_block_size 4096 --meta_block_size 4096 \\
|
|
||||||
# --data_offset 427814912 \\
|
|
||||||
|
|
||||||
# --disk_alignment 4096 --journal_block_size 512 --meta_block_size 512 \\
|
|
||||||
# --data_offset 433434624 \\
|
|
||||||
|
|
Loading…
Reference in New Issue