#!/bin/bash usage() { PREFIX="miaou:usage" echo '' exit 0 } yqm() { #read only yq "$1" "$EXPANDED_CONF" } yqmi() { # for update yq "$1" "$EXPANDED_CONF" -i } yqmt() { # tabular yq "$1" "$EXPANDED_CONF" -o t } compute_fqdn_middlepart() { case "$1" in prod) local fqdn_middlepart="." ;; beta) local fqdn_middlepart=".beta." ;; dev) local fqdn_middlepart=".dev." ;; *) echowarn "unknown target <${target}>, please fix with correct value from {prod, beta, dev} and try again..." exit 1 ;; esac builtin echo "$fqdn_middlepart" } # archive_conf(FILE) # save patch in archived folder of current file function archive_conf() { PREFIX="miaou:conf:archive" file="$1" filename=$(basename "$file") mkdir -p "$MIAOU_CONFIGDIR/archived/$filename" previous="$MIAOU_CONFIGDIR/archived/$filename/previous" # shellcheck disable=SC2012 latest_patch=$(ls -1tr "$MIAOU_CONFIGDIR/archived/$filename/" | tail -n1) if [[ -z "$latest_patch" ]]; then echo -n "archiving first file <$file> ..." cp "$file" "$previous" PREFIX="" echoinfo OK elif [[ "$file" -nt "$latest_patch" ]]; then patchname="$MIAOU_CONFIGDIR/archived/$filename/$(date +%F_%T)" if ! diff "$previous" "$file" >"$patchname"; then echo -n "archiving patch <$patchname> ..." cp "$file" "$previous" PREFIX="" echoinfo OK else rm "$patchname" fi fi } function archive_allconf() { mkdir -p "$MIAOU_CONFIGDIR" archive_conf "$CONF" archive_conf "$DEFAULTS" } function check_expand_conf() { PREFIX="miaou:conf:check" if ! "$FORCE" && [ -f "$EXPANDED_CONF" ] && [ "$EXPANDED_CONF" -nt "$CONF" ] && [ "$EXPANDED_CONF" -nt "$DEFAULTS" ]; then echo "already expanded!" return 1 fi } function expand_conf() { PREFIX="miaou:conf" if [[ -f "$EXPANDED_CONF" ]]; then current_target=$(grep -Es "^target:" /etc/miaou/defaults.yaml | cut -d ' ' -f2) previous_target=$(grep -Es "^target:" "$EXPANDED_CONF" | cut -d ' ' -f2) [[ "$current_target" != "$previous_target" ]] && echoerr "TARGET <$previous_target> mismatched <$current_target>" && exit 101 fi # initialize expanded conf by merging default # shellcheck disable=SC2016 yq eval-all '. as $item ireduce ({}; . * $item )' "$CONF" "$DEFAULTS" >"$EXPANDED_CONF" # append unique container unless overridden mapfile -t services_app_only < <(yqmt '.services.[].[] | has("container") | select ( . == false) | [(parent|key)+" " +key]') for i in "${services_app_only[@]}"; do read -r -a item <<<"$i" domain=${item[0]} subdomain=${item[1]} app=$(yqm ".services.\"$domain\".\"$subdomain\".app") container=$(get_container_for_domain_subdomain_app "$domain" "$subdomain" "$app") yqmi ".services.\"$domain\".\"$subdomain\".container=\"$container\"" done # append enabled=true unless overridden mapfile -t services_app_only < <(yqmt '.services.[].[] | has("enabled") | select ( . == false) | [(parent|key)+" " +key] | unique ') # echo "found <${#services_app_only[@]}> enabled services" for i in "${services_app_only[@]}"; do read -r -a item <<<"$i" domain=${item[0]} subdomain=${item[1]} yqmi ".services.\"$domain\".\"$subdomain\".enabled=true" done # compute fqdn target=$(yqm '.target') fqdn_middlepart=$(compute_fqdn_middlepart "$target") # write fqdn_middlepart yqmi ".expanded.fqdn_middlepart = \"$fqdn_middlepart\"" # add monitored.containers section yqmi '.expanded.monitored.containers = ([ .services[] | to_entries | .[] | .value | select (.enabled == true ) | .container ] | unique)' # add monitored.hosts section yqmi '.expanded.monitored.hosts = [( .services[][] | select (.enabled == true ) | {"domain": ( parent | key ), "subdomain": key, "fqdn": key + (parent | parent | parent | .expanded.fqdn_middlepart) + ( parent | key ), "container":.container, "port":.port, "app":.app })]' # add services section if [[ ${#services_app_only[@]} -gt 0 ]]; then yqmi '.expanded.services = [( .services[][] | select (.enabled == true ) | {"domain": ( parent | key ), "subdomain": key, "fqdn": key + (parent | parent | parent | .expanded.fqdn_middlepart) + ( parent | key ), "container":.container, "port":.port, "app":.app, "name": .name // ""})]' else yqmi '.expanded.services = []' fi # add firewall section, bridge_subnet + mail_passthrough if any bridge_subnet=$(lxc network get lxdbr0 ipv4.address) yqmi ".firewall.bridge_subnet = \"$bridge_subnet\"" container_mail_passthrough=$(yqm ".firewall.container_mail_passthrough") } function build_routes() { PREFIX="miaou:routes" mapfile -t fqdns < <(yqm '.expanded.services[].fqdn') echo "found <${#fqdns[@]}> fqdn" raw_dnsmasq='' for i in "${fqdns[@]}"; do raw_dnsmasq+="address=/$i/$DMZ_IP\\n" # append domains to conf echo "re-routing any connection from <$i> to internal container <$DMZ_CONTAINER.lxd>" done builtin echo -e "$raw_dnsmasq" | lxc network set $BRIDGE raw.dnsmasq - } function build_dmz_reverseproxy() { PREFIX="miaou:build:dmz" echo -n "building configuration for nginx ... " mkdir -p "$MIAOU_CONFIGDIR/nginx" tera -t "$MIAOU_BASEDIR/templates/nginx/_default.j2" "$EXPANDED_CONF" -o "$MIAOU_CONFIGDIR/nginx/_default" &>/dev/null tera -t "$MIAOU_BASEDIR/templates/nginx/hosts.j2" "$EXPANDED_CONF" -o "$MIAOU_CONFIGDIR/nginx/hosts" &>/dev/null PREFIX="" echo OK echo -n "pushing configuration to <$DMZ_CONTAINER> ... " for f in "$MIAOU_CONFIGDIR"/nginx/*; do lxc file push --uid=0 --gid=0 "$f" "$DMZ_CONTAINER/etc/nginx/sites-available/" &>/dev/null done PREFIX="" echo OK cat <" echoerr "please review configuration for fqdn: $fqdn" exit 2 fi if ! curl_check_unsecure "https://$fqdn"; then echoerr echoerr "DMZ does not seem to dispatch please review DMZ Nginx proxy" exit 3 elif [[ "$target" != 'dev' ]] && ! curl_check "https://$fqdn"; then PREFIX="" echo echowarn "T=$target missing valid certificate for fqdn please review DMZ certbot" fi done PREFIX="" echo OK # templates for monit echo -n "copying templates for monit ..." mkdir -p "$MIAOU_CONFIGDIR/monit" tera -t "$MIAOU_BASEDIR/templates/monit/containers.j2" "$EXPANDED_CONF" -o "$MIAOU_CONFIGDIR/monit/containers" >/dev/null tera -t "$MIAOU_BASEDIR/templates/monit/hosts.j2" "$EXPANDED_CONF" -o "$MIAOU_CONFIGDIR/monit/hosts" >/dev/null PREFIX="" echo OK } # count_service_for_container(container: string) # returns how many services run inside container according to expanded conf function count_service_for_container() { container_mail_passthrough="$1" count=$(yqm ".expanded.services.[] | select(.container == \"$container_mail_passthrough\") | .fqdn" | wc -l) builtin echo "$count" } function build_nftables() { PREFIX="miaou:nftables:build" mkdir -p "$MIAOU_CONFIGDIR/nftables.rules.d" container_mail_passthrough=$(yqm '.firewall.container_mail_passthrough') if [[ "$container_mail_passthrough" != null ]]; then ip_mail_passthrough=$(lxc list "$container_mail_passthrough" -c4 -f csv | grep eth0 | cut -d ' ' -f1) [[ -z "$ip_mail_passthrough" ]] && echoerr "container <$container_mail_passthrough> passthrough unknown ip!" && exit 55 echo "passthrough=$container_mail_passthrough/$ip_mail_passthrough" count=$(count_service_for_container "$container_mail_passthrough") [[ $count == 0 ]] && echowarn "no service detected => no passthrough, no change!" [[ $count -gt 1 ]] && echoerr "count <$count> services detected on container <$container_mail_passthrough>, please disable some and leave only one service for safety!!!" && exit 56 ip_mail_passthrough=$ip_mail_passthrough tera -e --env-key env -t "$MIAOU_BASEDIR/templates/nftables/lxd.table.j2" "$EXPANDED_CONF" -o "$MIAOU_CONFIGDIR/nftables.rules.d/lxd.table" &>/dev/null else echo "no container passthrough" tera -t "$MIAOU_BASEDIR/templates/nftables/lxd.table.j2" "$EXPANDED_CONF" -o "$MIAOU_CONFIGDIR/nftables.rules.d/lxd.table" &>/dev/null fi if ! diff -q "$MIAOU_CONFIGDIR/nftables.rules.d/lxd.table" /etc/nftables.rules.d/lxd.table; then sudo_required "reloading nftables" echo -n "reloading nftables..." sudo cp "$MIAOU_CONFIGDIR/nftables.rules.d/lxd.table" /etc/nftables.rules.d/lxd.table sudo systemctl reload nftables PREFIX="" echo OK fi } # check whether http server responds 200 OK, required , ie: http://example.com:8001, https://example.com function curl_check() { arg1_required "$@" # echo "curl $1" curl -m $MAX_WAIT -sLI4 "$1" | grep -q "^HTTP.* 200" } # check whether https server responds 200 OK, even unsecured certificate (auto-signed in mode DEV) function curl_check_unsecure() { arg1_required "$@" curl -m $MAX_WAIT -skLI4 "$1" | grep -q "^HTTP.* 200" } function get_dmz_ip() { if ! container_running "$DMZ_CONTAINER"; then echowarn "Container running dmz <$DMZ_CONTAINER> seems down" echoerr "please \`lxc start $DMZ_CONTAINER\` or initialize first!" exit 1 fi dmz_ip=$(host "$DMZ_CONTAINER.lxd" | cut -d ' ' -f4) if ! valid_ipv4 "$dmz_ip"; then echowarn "dmz seems up but no valid ip <$dmz_ip> found!" echoerr "please fix this networking issue, then retry..." exit 1 else builtin echo "$dmz_ip" fi } function fetch_container_of_type() { local type="$1" readarray -t dmzs < <(yqm ".containers.[].[] | select(.==\"$type\") | parent | key") case ${#dmzs[@]} in 0) : ;; 1) builtin echo "${dmzs[0]}" ;; *) for d in "${dmzs[@]}"; do builtin echo "$d" done ;; esac } function get_container_for_domain_subdomain_app() { local domain="$1" local subdomain="$2" local app="$3" readarray -t containers < <(fetch_container_of_type "$app") case ${#containers[@]} in 0) echoerr "no container of type <$app> found amongst containers for $subdomain.$domain\nHINT : Please, either :\n1. define at least one container for recipe <$app>\n2. remove all services related to recipe <$app>" && exit 1 ;; 1) builtin echo "${containers[0]}" ;; *) for d in "${containers[@]}"; do echowarn "container of type $app found in <$d>" done echoerr "multiple containers (${#containers[@]}) provided same app <$app>, therefore container is mandatory alongside $subdomain.$domain" && exit 2 ;; esac } function get_unique_container_dmz() { readarray -t containers < <(fetch_container_of_type "dmz") case ${#containers[@]} in 0) echoerr "no container of type found amongst containers" && exit 1 ;; 1) builtin echo "${containers[0]}" ;; *) for d in "${containers[@]}"; do echowarn "container of type dmz found in <$d>" done echoerr "multiple dmz (${#containers[@]}) are not allowed, please select only one " && exit 2 ;; esac } function prepare_dmz_container() { "$MIAOU_BASEDIR"/recipes/dmz/install.sh "$DMZ_CONTAINER" } function check_resolv_conf() { local bridge_gw resolver bridge_gw=$(lxc network get lxdbr0 ipv4.address | cut -d'/' -f1) resolver=$(grep nameserver /etc/resolv.conf | head -n1 | cut -d ' ' -f2) PREFIX="resolver:check" echo "container resolver is <$resolver>" PREFIX="resolver:check" echo "container bridge is <$bridge_gw>" [[ "$bridge_gw" != "$resolver" ]] && return 21 return 0 } function prepare_containers() { PREFIX="miaou:prepare" readarray -t containers < <(yqmt ".containers.[] | [ key, .[] ] ") for i in "${containers[@]}"; do read -r -a item <<<"$i" container=${item[0]} for ((j = 1; j < ${#item[@]}; j++)); do service="${item[$j]}" recipe_install="$MIAOU_BASEDIR/recipes/$service/install.sh" if [[ -f "$recipe_install" ]]; then echo "install [$service] onto container <$container>" "$recipe_install" "$container" else echoerr "FAILURE, for container <$container>, install recipe [$service] not found!" echoerr "please review configuration, mismatch recipe name maybe?" exit 50 fi done done } function build_services() { PREFIX="miaou:build:services" echo "building services..." readarray -t services < <(yqmt '.expanded.services[] | [ .[] ]') for i in "${services[@]}"; do read -r -a item <<<"$i" fqdn=${item[2]} container=${item[3]} port=${item[4]} app=${item[5]} name=${item[6]:-} recipe="$MIAOU_BASEDIR/recipes/$app/crud.sh" if [[ -f "$recipe" ]]; then echo "read [$app:$name] onto container <$container>" if ! "$recipe" -r --port "$port" --container "$container" --name "$name" --fqdn "$fqdn"; then echoinfo "CREATE RECIPE" "$recipe" -c --port "$port" --container "$container" --name "$name" --fqdn "$fqdn" echoinfo "CREATE RECIPE: OK" fi else echowarn "for container <$container>, crud recipe [$app] not found!" fi done } ### MAIN . "$MIAOU_BASEDIR/lib/init.sh" readonly CONF="/etc/miaou/miaou.yaml" readonly DEFAULTS="/etc/miaou/defaults.yaml" readonly EXPANDED_CONF="$MIAOU_CONFIGDIR/miaou.expanded.yaml" readonly BRIDGE="lxdbr0" readonly MAX_WAIT=3 # timeout in seconds # shellcheck disable=SC2034 declare -a options=("$@") FORCE=false if containsElement options "-f" || containsElement options "--force"; then FORCE=true fi if containsElement options "history"; then echo "TODO: HISTORY" exit 0 fi if containsElement options "config"; then editor /etc/miaou/miaou.yaml if diff -q /etc/miaou/miaou.yaml $HOME/.config/miaou/archived/miaou.yaml/previous; then exit 0 fi fi if check_expand_conf; then archive_allconf expand_conf check_resolv_conf build_nftables prepare_containers DMZ_CONTAINER=$(get_unique_container_dmz) readonly DMZ_CONTAINER build_services DMZ_IP=$(get_dmz_ip) readonly DMZ_IP build_dmz_reverseproxy build_routes build_monit fi monit_show