provisioning tool for building opinionated architecture
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

691 lines
21 KiB

#!/bin/bash
RED='\e[0;41m\e[1;37m'
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
PURPLE='\033[0;35m'
DARK='\e[100m'
NC='\033[0m' # No Color
TO_BE_DEFINED="TO BE DEFINED"
# BOLD='\033[1m'
# DIM='\e[2m\e[0;90m'
function echo() {
[[ -n ${PREFIX:-} ]] && printf "${DARK}%25.25s${NC} " "${PREFIX}"
builtin echo "$@"
}
function check_normal_user() {
[[ $(id -u) -lt 1000 ]] && echoerr "normal user (>1000) expected, please connect as a normal user then call again!" && exit 100
return 0
}
function sudo_required() {
check_normal_user
command -v sudo &>/dev/null &&
id -G | grep -q sudo && echoerr "command <sudo> not found, please install as so: \`apt install -y sudo\`" && exit 1
if ! sudo -n true &>/dev/null; then
if [[ -n "${1:-}" ]]; then
echowarnn "[sudo] requiring authorized access for: [ $1 ]"
else
echowarnn "[sudo] requiring authorized access for further processing"
fi
fi
sudo -vp ' : '
}
# idempotent cargo install <package1 package2 ...>
function idem_cargo_install() {
for i in "$@"; do
if [ ! -f ~/.cargo/bin/"$i" ]; then
cargo install "$i"
fi
done
}
# display error in red
function echoerr() {
echo -e "${RED}$*${NC}" >&2
}
function echoerrn() {
echo -en "${RED}$*${NC}" >&2
}
# display warn in yellow
function echowarn() {
echo -e "${YELLOW}$*${NC}" >&2
}
function echowarnn() {
echo -en "${YELLOW}$*${NC}" >&2
}
# display error in green
function echoinfo() {
echo -e "${GREEN}$*${NC}" >&2
}
function echoinfon() {
echo -en "${GREEN}$*${NC}" >&2
}
# test whether <ip> is a valid ipv4 address?
function valid_ipv4() {
local ip="$1"
if [[ $ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
IFS='.' read -ra ADDR <<<"$ip"
[[ ${ADDR[0]} -le 255 && ${ADDR[1]} -le 255 && ${ADDR[2]} -le 255 && ${ADDR[3]} -le 255 ]]
return $?
fi
return 1
}
function enable_trace() {
trap 'trap_error $? ${LINENO:-0} ${BASH_LINENO:-0} ${BASH_COMMAND:-empty} $(printf "::%s" ${FUNCNAME[@]})' ERR
}
function disable_trace() {
trap - ERR
}
function disable_all_signals {
trap - ERR
trap - HUP
trap - INT
trap - TERM
}
function prepare_nftables() {
local PREFIX="miaou:nftables"
if ! [[ -f /etc/nftables.rules.d/firewall.table ]]; then
echo "installing nftables ..."
sudo apt install -y nftables
sudo cp -f "$MIAOU_BASEDIR/templates/hardened/nftables.conf" /etc/
sudo mkdir -p /etc/nftables.rules.d
sudo systemctl enable nftables
else
echo "nftables already installed!"
fi
current_target="${TARGET:-not_defined_yet}"
if [[ $current_target == not_defined_yet ]]; then
echo -n "generating new firewall table first time... "
else
echo -n "generating new firewall table according to target=<${current_target}>... "
fi
sudo env target="$current_target" tera -e --env-only --env-key env -t "$MIAOU_BASEDIR/templates/nftables/firewall.table.j2" -o /etc/nftables.rules.d/firewall.table &>/dev/null
sudo systemctl reload nftables
echo "OK"
}
function miaou_init() {
# shellcheck source=/dev/null
[[ -f /opt/miaou-bash/lib/functions.sh ]] && source /opt/miaou-bash/lib/functions.sh
# shellcheck source=/dev/null
. "$MIAOU_BASEDIR/lib/functions.sh"
export MIAOU_CONFIGDIR="$HOME/.config/miaou"
set -Eeuo pipefail
enable_trace
trap 'ctrl_c $? ${LINENO:-0} ${BASH_LINENO:-0} ${BASH_COMMAND:-empty} $(printf "::%s" ${FUNCNAME[@]})' INT
}
function ctrl_c() {
PREFIX="miaou:trap" echoerr "Ctrl + C happened, exiting!!! $*"
exit 125
}
# extract source code error triggered on trap error <error_code> <error_line>
function trap_error() {
ERRORS_COUNT=0
if [[ -f "$MIAOU_CONFIGDIR"/error_count ]]; then
ERRORS_COUNT=$(cat "$MIAOU_CONFIGDIR"/error_count)
else
mkdir -p "$MIAOU_CONFIGDIR"
printf 0 >"$MIAOU_CONFIGDIR"/error_count
fi
ERRORS_COUNT=$((ERRORS_COUNT + 1))
printf '%s' $ERRORS_COUNT >"$MIAOU_CONFIGDIR"/error_count
local PREFIX=""
# local file="${0:-}"
local err=$1 # error status
local line=$2 # LINENO
# local linecallfunc=${3:-}
local command="${4:-}"
local funcstack="${5:-}"
local caller
caller=$(caller | cut -d' ' -f2)
# echo >&2
# if [ "$funcstack" != "::" ]; then
# echo -e "${RED}ERROR <$err>, due to command <$command> at line $line from <$caller>, stack=${funcstack}${NC}" >&2
# else
# echo >&2 "ERROR DETECTED"
# fi
# echo
# echo -e "${PURPLE}$caller:$line ${NC}EXIT ${RED}<$err>${NC}" >&2
# echo -e "${PURPLE}------------------------------------------ ${NC}" >&2
if [[ $ERRORS_COUNT == 1 ]]; then
echo
echo -e "${RED}ERROR <$err>, due to command <$command $funcstack>${NC}" >&2
fi
echo -e "${PURPLE}$ERRORS_COUNT: $caller:$line ${RED}$command $funcstack${NC}" >&2
# echo -e "${PURPLE}----------------------------- ${PURPLE}EXIT CODE ${PURPLE}--------------${PURPLE} $err ${NC}" >&2
# if [[ $line -gt 2 ]]; then
# sed "$((line - 2))q;d" "$caller" >&2
# sed "$((line - 1))q;d" "$caller" >&2
# fi
# echo -ne "${BOLD}" >&2
# sed "${line}q;d" "$caller" >&2
# echo -e "${PURPLE}------------------------------------------ ${NC}" >&2
}
# exist_command(cmd1, ...)
# test all commands exist, else fail
function exist_command() {
for i in "$@"; do
command -v "$i" &>/dev/null || return 50
done
}
# test whether container <ct> is up and running?
function container_running() {
arg1_required "$@"
container_exists "$1" && lxc list "$1" -c ns -f csv | head -n1 | grep -q "$1,RUNNING"
lxc exec "$1" -- bash <<EOF
set -Eeuo pipefail
if [[ ! -f /root/cloud-status.json ]]; then
cloud-init status --wait >/dev/null
fi
EOF
}
# test arg1 required
function arg1_required() {
[[ -z "${1:-}" ]] && echoerr "ERROR: arg#1 expected!" && return 125
return 0
}
# test arg2 required
function arg2_required() {
[[ -z "${2:-}" ]] && echoerr "ERROR: arg#2 expected!" && return 125
return 0
}
# test arg3 required
function arg3_required() {
[[ -z "${3:-}" ]] && echoerr "ERROR: arg#3 expected!" && return 125 || return 0
}
# test whether container <ct> exists yet?
function container_exists() {
arg1_required "$@"
lxc list "$1" -c n -f csv | grep -q "^$1\$"
}
# 3 args expected: <commmand> <delay in s, example: 0.2> <max_attempts>
function wait_for_command {
arg3_required "$@"
command=$1
delay=$2
max_attempt=$3
attempt=0
while ! eval "$command"; do
attempt=$((attempt + 1))
if [[ $attempt -gt $max_attempt ]]; then
echoerr "command <$command> failed after a delay of $(bc <<<"$max_attempt * $delay")s and $max_attempt attempts"
return 1
else
sleep "$delay"
fi
done
echo SUCCESS
}
function wait_for_container_full_initialization {
arg1_required "$@"
wait_for_command "lxc exec $1 -- test -f /root/cloud-status.json" 0.2 40
}
# build debian image with prebuild miaou-bash and various useful settings
# ARG1=release [bullseye, buster]
function build_miaou_image() {
local RELEASE="$1"
local IMAGE_LABEL="$RELEASE-miaou"
local PREFIX="miaou:image"
local DEB_REPOSITORY
DEB_REPOSITORY=$(grep ^deb /etc/apt/sources.list | head -n1 | cut -d ' ' -f2 | cut -d '/' -f3)
if ! lxc image -cl list -f csv | grep -q "$IMAGE_LABEL"; then
echo "building lxc image <$IMAGE_LABEL> ... "
echo "image will reuse same local repository <$DEB_REPOSITORY>"
creation_date=$(date +%s)
sudo /opt/miaou-bash/tools/idem_apt_install debootstrap
cat <<EOF1 | sudo bash
set -euo pipefail
rm -rf /tmp/$IMAGE_LABEL{,-image}
mkdir -p /tmp/$IMAGE_LABEL{,-image}
debootstrap $RELEASE /tmp/$IMAGE_LABEL http://$DEB_REPOSITORY/debian
echo
echo "DEBOOTSTRAP ... OK"
echo
cat <<EOF2 | chroot /tmp/$IMAGE_LABEL
set -euo pipefail
echo "image prepare source.list from $DEB_REPOSITORY"
if [[ "$RELEASE" == "buster" ]]; then
cat <<EOF3 >/etc/apt/sources.list
deb http://$DEB_REPOSITORY/debian $RELEASE main contrib
deb http://$DEB_REPOSITORY/debian $RELEASE-updates main contrib
deb http://$DEB_REPOSITORY/debian-security/ $RELEASE/updates main contrib
EOF3
else
cat <<EOF3 >/etc/apt/sources.list
deb http://$DEB_REPOSITORY/debian $RELEASE main contrib
deb http://$DEB_REPOSITORY/debian $RELEASE-updates main contrib
deb http://$DEB_REPOSITORY/debian-security/ $RELEASE-security main contrib
EOF3
fi
echo APT UPDATE
apt update && apt dist-upgrade -y
apt install -y curl wget file git sudo bash-completion
curl https://git.artcode.re/miaou/miaou-bash/raw/branch/main/install.sh | sudo bash -s -- --host
# TODO: remove line below
# ln -sf /usr/share/zoneinfo/Indian/Reunion /etc/localtime
cat <<EOF3 >/etc/network/interfaces
# This file describes the network interfaces available on your system
# and how to activate them. For more information, see interfaces(5).
# The loopback network interface
auto lo
iface lo inet loopback
auto eth0
iface eth0 inet dhcp
source /etc/network/interfaces.d/*
EOF3
echo "deboostrap ready!"
EOF2
cd /tmp/$IMAGE_LABEL-image
tar -czf rootfs.tar.gz -C /tmp/$IMAGE_LABEL .
cat <<EOF2 >metadata.yaml
architecture: "x86_64"
creation_date: $creation_date
properties:
architecture: "x86_64"
description: "Debian $RELEASE for miaou instances"
os: "debian"
release: "$RELEASE"
EOF2
tar -czf metadata.tar.gz metadata.yaml
EOF1
lxc image import "/tmp/$IMAGE_LABEL-image/metadata.tar.gz" "/tmp/$IMAGE_LABEL-image/rootfs.tar.gz" --alias "$IMAGE_LABEL"
echo "image <$IMAGE_LABEL> successfully built!"
echo DONE
else
echo "image <$IMAGE_LABEL> already built!"
fi
}
# convert array to string according to IFS arg1
# example: join "," "${MY_ARRAY[@]}" => one,two,three
function join() {
local IFS="$1"
shift
builtin echo "$*"
}
# execute remote scripting onto one LXC container <CONTAINER> [COMMANDS, ...]
# may use one command like: `lxc_exec ct1 uname -a`
# or pipe like so: `
# cat <<EOF | lxc_exec ct1
# ls -l
# uname -a
# echo [\$0] [\$1] [\$2] # toto titi tata
# EOF
# `
function lxc_exec() {
arg1_required "$@"
container="$1"
shift
declare -a ARGUMENTS
ARGUMENTS=(toto titi tata) # might be overriden with interesting stuff!
if ((${#} == 0)); then
multiline=""
while read -r line; do
if [[ ! "$line" =~ ^\# ]] && [[ ! "$line" =~ ^[[:space:]]*$ ]]; then
if [[ "$line" =~ .*\;$ ]] || [[ "$line" =~ do$ ]] || [[ "$line" =~ then$ ]] || [[ "$line" =~ else$ ]]; then
multiline+="${line} " # append space in case of ending with either '; do then else'
else
multiline+="${line};" # append ; for multiple commands
fi
fi
done
# echo "DEBUG: multiline = [$multiline]"
# echo DEBUG: lxc exec "$container" -- bash -lc "$multiline" "${ARGUMENTS[@]}"
lxc exec "$container" -- bash -lc "$multiline" "${ARGUMENTS[@]}"
else
lxc exec "$container" -- bash -lc "$*" "${ARGUMENTS[@]}"
fi
}
# check container exist and running
function check_container() {
arg1_required "$@"
local CT="$1"
container_exists "$CT"
container_running "$CT"
}
function launch_container() {
arg1_required "$@"
local ct="$1"
if ! container_exists "$ct"; then
echo "container <$ct> about to be created ..."
local extra_release="${2:-}"
if [[ -n "$extra_release" ]] && ! lxc image info "${extra_release}-miaou" >/dev/null; then
echoerrn "unknown extra_release <${extra_release}-miaou>!\nHINT : please add it into /etc/miaou/defaults.yaml, then re-install miaou!"
exit 128
fi
if [[ -n "$extra_release" ]]; then
echoerrn "FIXME: lxc-miaou-create -o release=bookworm should be implemented ...."
lxc-miaou-create "$ct" "$extra_release"
else
lxc-miaou-create "$ct"
fi
echo "DONE"
fi
if ! container_running "$ct"; then
echowarn "container <$ct> seems to be asleep, starting ..."
lxc start "$ct"
echowarn DONE
fi
}
function load_yaml_from_expanded {
arg1_required "$@"
yaml_key="$1"
yaml_file="$MIAOU_CONFIGDIR/miaou.expanded.yaml"
yaml_value=$(yq ".$yaml_key" "$yaml_file")
if [[ -n "$yaml_value" ]] && [[ "$yaml_value" != "null" ]] && [[ "$yaml_value" != "$TO_BE_DEFINED" ]]; then
PREFIX="" echo "$yaml_value"
else
echoerr "undefined value for key: <$yaml_key> from file: <$yaml_file>"
return 98
fi
}
function check_yaml_defined_value {
yaml_file="$1"
yaml_key="$2"
yaml_value=$(yq ".$yaml_key" "$yaml_file")
if [[ -n "$yaml_value" ]] && [[ "$yaml_value" != "null" ]] && [[ "$yaml_value" != "$TO_BE_DEFINED" ]]; then
return 0
else
echoerr "undefined value for key: <$yaml_key> from file: <$yaml_file>"
return 99
fi
}
# halt unless current user is root
function root_required() {
[[ $(id -u) == 0 ]] || (echoerr "root required" && return 1)
}
# arg#1: environment variable
# read from environment or ask entry before exporting new variable
function env_or_ask {
if [[ -n ${1+x} ]]; then
if printenv "$1" >/dev/null; then
echo "value defined as $(printenv "$1")"
else
printf "Please define %20s: " "$1"
read -r
export "$1=\"$REPLY\"" >/dev/null
fi
else
echoerr "env_or_ask requires one argument: <VARIABLE_NAME>" && exit 5
fi
}
# grab and install related project
function install_miaou_bash() {
local PREFIX="miaou-bash:install"
if [[ ! -d /opt/miaou-bash ]]; then
echo "installing curl wget commands ..."
sudo apt install -y curl wget
echo "installing miaou-bash..."
curl https://git.artcode.re/miaou/miaou-bash/raw/branch/main/install.sh | sudo bash -s -- --host
export PATH=$PATH:/opt/miaou-bash/tools/
echo "OK"
else
echo "addon <miaou-bash> already installed!"
fi
# shellcheck source=/dev/null
source /etc/bash.bashrc
sudo /opt/miaou-bash/tools/idem_apt_install bash-completion
}
function add_toolbox_sudoers {
local PREFIX="toolbox:sudoers"
echo -n "creating sudoers file to allow sudo as command from /TOOLBOX... "
sudo mkdir -p /etc/sudoers.d
if [[ ! -f /etc/sudoers.d/add_TOOLBOX_to_PATH ]]; then
sudo tee /etc/sudoers.d/add_TOOLBOX_to_PATH &>/dev/null <<EOF
Defaults secure_path="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/TOOLBOX"
EOF
PREFIX="" echo "updated!"
else
PREFIX="" echo "already done!"
fi
}
function prepare_toolbox() {
local PREFIX="toolbox:prepare"
sudo mkdir -p /TOOLBOX
if ! command -v cargo &>/dev/null; then
echo -n "installing <cargo> ... "
curl -sSf https://sh.rustup.rs | sh -s -- -y
# shellcheck source=/dev/null
source "$HOME/.cargo/env"
/opt/miaou-bash/tools/append_or_replace "^PATH=\$PATH:\$HOME/\\.cargo/bin" "PATH=\$PATH:\$HOME/.cargo/bin" ~/.bashrc
PREFIX="" echo "OK"
else
echo "command <cargo> already installed!"
fi
echo -n "installing <fd> ... "
if [ ! -f "/TOOLBOX/fd" ]; then
idem_cargo_install fd-find
sudo cp "$HOME"/.cargo/bin/fd /TOOLBOX/fd
PREFIX="" echo "successfully installed!"
else
PREFIX="" echo "already done!"
fi
echo -n "installing <viu> ... "
if [ ! -f "/TOOLBOX/viu" ]; then
idem_cargo_install viu
sudo cp "$HOME"/.cargo/bin/viu /TOOLBOX/
PREFIX="" echo "successfully installed!"
else
PREFIX="" echo "already done!"
fi
echo -n "installing <rg> alias <ripgrep> ... "
if [ ! -f "/TOOLBOX/rg" ]; then
sudo /opt/miaou-bash/tools/idem_apt_install ripgrep
sudo ln /usr/bin/rg /TOOLBOX/
PREFIX="" echo "successfully installed"
else
PREFIX="" echo "already done!"
fi
echo -n "installing <ag> alias <silversearcher-ag> ... "
if [ ! -f "/TOOLBOX/ag" ]; then
sudo /opt/miaou-bash/tools/idem_apt_install silversearcher-ag
sudo ln /usr/bin/ag /TOOLBOX/
PREFIX="" echo "successfully installed"
else
PREFIX="" echo "already done!"
fi
echo -n "installing <bandwhich> ... "
if [ ! -f "/TOOLBOX/bandwhich" ]; then
idem_cargo_install bandwhich
sudo cp "$HOME"/.cargo/bin/bandwhich /TOOLBOX/bandwhich
PREFIX="" echo "successfully installed"
else
PREFIX="" echo "already done!"
fi
echo -n "installing <btm> alias <bottom> ... "
if [ ! -f "/TOOLBOX/btm" ]; then
VERSION=$(wget_semver github ClementTsang/bottom)
cd /tmp
wget "https://github.com/ClementTsang/bottom/releases/download/$VERSION/bottom_x86_64-unknown-linux-musl.tar.gz"
tar -xzvf bottom_x86_64-unknown-linux-musl.tar.gz
sudo cp btm /usr/local/bin/
sudo ln /usr/local/bin/btm /TOOLBOX/
PREFIX="" echo "successfully installed"
else
PREFIX="" echo "already done!"
fi
echo -n "installing <micro> ... "
if [ ! -f "/TOOLBOX/micro" ]; then
cd /tmp || (echoerr "/tmp wrong permission" && exit 101)
curl -q https://getmic.ro | GETMICRO_REGISTER=n sh
sudo mv micro /TOOLBOX/micro
sudo chown root:root /TOOLBOX/micro
PREFIX="" echo "successfully installed"
else
PREFIX="" echo "already done!"
fi
echo -n "installing <ncdu> ... "
if [ ! -f "/TOOLBOX/ncdu" ]; then
sudo /opt/miaou-bash/tools/idem_apt_install ncdu
sudo cp /usr/bin/ncdu /TOOLBOX/ncdu
PREFIX="" echo "successfully installed"
else
PREFIX="" echo "already done!"
fi
echo -n "installing <unzip> ... "
if [ ! -f "/TOOLBOX/unzip" ]; then
sudo /opt/miaou-bash/tools/idem_apt_install unzip
sudo cp /usr/bin/unzip /TOOLBOX/unzip
PREFIX="" echo "successfully installed"
else
PREFIX="" echo "already done!"
fi
echo -n "installing <tree> ... "
if [ ! -f "/TOOLBOX/tree" ]; then
sudo /opt/miaou-bash/tools/idem_apt_install tree
sudo cp /bin/tree /TOOLBOX/tree
PREFIX="" echo "successfully installed"
else
PREFIX="" echo "already done!"
fi
echo -n "installing <duf> ... "
if [ ! -f "/TOOLBOX/duf" ]; then
VERSION=$(/opt/miaou-bash/tools/wget_semver github muesli/duf)
VERSION_WITHOUT_V=${VERSION#v}
wget -O /tmp/duf.deb "https://github.com/muesli/duf/releases/download/${VERSION}/duf_${VERSION_WITHOUT_V}_linux_amd64.deb"
sudo dpkg -i /tmp/duf.deb
sudo cp /bin/duf /TOOLBOX/duf
PREFIX="" echo "successfully installed"
else
PREFIX="" echo "already done!"
fi
echo -n "installing <curl> ... "
if [ ! -f "/TOOLBOX/curl" ]; then
sudo wget -O /TOOLBOX/curl "https://github.com/moparisthebest/static-curl/releases/latest/download/curl-amd64"
sudo chmod +x /TOOLBOX/curl
PREFIX="" echo "successfully installed"
else
PREFIX="" echo "already done!"
fi
echo -n "installing <wget> ... "
if [ ! -f "/TOOLBOX/wget" ]; then
sudo ln -f /usr/bin/wget /TOOLBOX/wget
PREFIX="" echo "successfully installed"
else
PREFIX="" echo "already done!"
fi
}
# install_mandatory_commands
function install_mandatory_commands() {
local PREFIX="mandatory:commands"
echo "installing various mandatory commands"
sudo /opt/miaou-bash/tools/idem_apt_install dnsutils build-essential curl mariadb-client postgresql-client
if ! exist_command tera; then
echo "installing <tera> ..."
local version=v0.2.4
wget -q "https://github.com/chevdor/tera-cli/releases/download/${version}/tera-cli_linux_amd64.deb" -O /tmp/tera-cli_linux_amd64.deb
sudo dpkg -i /tmp/tera-cli_linux_amd64.deb
else
echo "command <tera> already installed!"
fi
if ! exist_command yq; then
local version binary
version='v4.35.2'
binary='yq_linux_amd64'
sudo sh -c "wget https://github.com/mikefarah/yq/releases/download/${version}/${binary}.tar.gz -O - |\
tar -xz ./${binary} && sudo mv ${binary} /usr/bin/yq"
else
echo "command <yq> already installed!"
fi
}
# flatten array, aka remove duplicated elements in array
# return: `mapfile -t OUTPUT_ARRAY < <(sort_array "${INPUT_ARRAY[@]}")`
function flatten_array {
declare -a array=("$@")
IFS=" " read -r -a array <<<"$(tr ' ' '\n' <<<"${array[@]}" | sort -u | tr '\n' ' ')"
printf '%s\n' "${array[@]}"
}