From 6b736c31b540720bb38e8411b60057013d413fef Mon Sep 17 00:00:00 2001 From: Erich Eckner Date: Tue, 10 Oct 2023 13:34:31 +0200 Subject: rename backup -> hardlinked-backup * avoid file conflict with tar's /usr/bin/backup script --- .gitignore | 2 +- Makefile | 6 +- backup.in | 302 --------------------------------------------------- backup@.service.in | 2 +- hardlinked-backup.in | 302 +++++++++++++++++++++++++++++++++++++++++++++++++++ last-backups.in | 2 +- man.commons.in | 2 +- 7 files changed, 309 insertions(+), 309 deletions(-) delete mode 100755 backup.in create mode 100755 hardlinked-backup.in diff --git a/.gitignore b/.gitignore index 0aa8bb6..4987e86 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,10 @@ -backup backup@.service backup@.timer backup-progress backup-statistics fast-repair forwarddown +hardlinked-backup last-backups remove-old-backups man.commons diff --git a/Makefile b/Makefile index 1e7b867..9a37b27 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ MANDIR = /usr/share/man VERSION = 1.7 all: man.commons \ - backup backup.1 \ + hardlinked-backup hardlinked-backup.1 \ backup@.service backup@.timer \ backup-progress \ backup-statistics backup-statistics.1 \ @@ -55,9 +55,9 @@ all: man.commons \ .PHONY: install dist clean install: all - install -D -m0755 -t $(DESTDIR)$(BINDIR) backup backup-progress backup-statistics forwarddown last-backups remove-old-backups + install -D -m0755 -t $(DESTDIR)$(BINDIR) backup-progress backup-statistics forwarddown hardlinked-backup last-backups remove-old-backups install -D -m0644 -t $(DESTDIR)$(LIBDIR)/systemd/system backup@.service backup@.timer remove-old-backups.service remove-old-backups.timer - install -D -m0644 -t $(DESTDIR)$(MANDIR)/man1 backup.1 backup-statistics.1 forwarddown.1 last-backups.1 remove-old-backups.1 + install -D -m0644 -t $(DESTDIR)$(MANDIR)/man1 backup-statistics.1 forwarddown.1 hardlinked-backup.1 last-backups.1 remove-old-backups.1 install -D -m0644 -t $(DESTDIR)$(ETCDIR) backup.conf install -D -m0755 -T sendmailadvanced.hook $(DESTDIR)$(ETCDIR)/sendmailadvanced.hooks/last-backups diff --git a/backup.in b/backup.in deleted file mode 100755 index f530e08..0000000 --- a/backup.in +++ /dev/null @@ -1,302 +0,0 @@ -#!/bin/bash - -rsyncOptions='-vxcaHAXF' -rsync --help | \ - grep -q -- --may-modify-others && \ - rsyncOptions="${rsyncOptions} --may-modify-others" - -[ -r "#ETCDIR#/backup.conf" ] && \ - . "#ETCDIR#/backup.conf" - -usage() -{ - >&2 echo \ -'Usage: backup /tmp/pidFile /path/to/destination/ user@source:path [proxy_user@ssh_host] -Backup files from remote with rsync, possibly via SSH-tunnel. - -With no arguments, information is expected to be in array $backups in #ETCDIR#/backup.conf with name of executable (e.g. a symlink) as key. -With one argument, information is expected to be in array ${backups[$1]} in #ETCDIR#/backup.conf. - -Options: - /tmp/pidFile location of file to store PID in - /path/to/destination location to store backups in - user@source:path remote data to back up - proxy_user@ssh_host ssh login to proxy node (optional) -#HELPTEXT# #' - [ -z "$1" ] && exit 1 - exit $1 -} - -extract_ssh_host() { - { - printf '%s\n' "$1" - if [ -f ~/.ssh/config ]; then - sed ' - /^Host\s\+'"$1"'$/,/^Host\s/ { - s/^\s*Hostname\s\+// - t - } - d - ' ~/.ssh/config - fi - } \ - | tail -n1 -} - -extract_ssh_ip_protocols() { - getent ahosts "$(extract_ssh_host "$1")" \ - | sed ' - s/^\([0-9]\+\.\)\{3\}[0-9]\+\s\+STREAM\(\s.*\)\?$/4/ - t - s/^[0-9a-f][0-9a-f:]\+\s\+STREAM\(\s.*\)\?$/6/ - t - d - ' \ - | sort -u \ - | if [ -f ~/.ssh/config ]; then - grep -vxF "$( - sed ' - /^Host\s\+'"$1"'$/,/^Host\s/ { - s/^\s*AddressFamily\s\+inet\s*$/6/ - t - s/^\s*AddressFamily\s\+inet6\s*$/4/ - t - } - d - ' ~/.ssh/config - )" - else - cat - fi -} - -connected_ip_versions() { - ip -o addr show scope global \ - | awk '{print $4}' \ - | sed ' - s@^\([0-9]\+\.\)\{3\}[0-9]\+\(/[0-9]\+\)\?$@4@ - t - s@^[0-9a-f][0-9a-f:]\+\(/[0-9]\+\)\?$@6@ - t - d - ' \ - | sort -u -} - -if [ $# -eq 1 ]; then - if [ "$1" == "--help" ]; then - usage 0 - elif [ "$1" == "--version" ]; then - echo '#VERSION#' - exit - fi -fi - -seldom=false -if [ $# -eq 0 ] || [ $# -eq 1 ]; then - if [ $# -eq 0 ]; then - backupID="$(basename $0)" - else - backupID="$1" - fi - [ -z "${backups[${backupID}]}" ] && usage - set /tmp/${backupID}.pid ${backups[${backupID}]} - if printf '%s\n' "${seldomBackups[@]}" | \ - grep -qxF "${backupID}"; then - seldom=true - fi -fi - -Basis="$2" -pidFile="$1" -QuellIP=$(echo "$3" | sed "s|^[a-zA-Z]*://||; s|^[a-zA-Z]*@||; s|:\?/.*$||") - -if [ "$#" -eq 3 ]; then - Quelle="$3" - ipVer=$( - { - extract_ssh_ip_protocols "${QuellIP}" \ - | sort -n - extract_ssh_ip_protocols "${QuellIP}" \ - | grep -xF "$(connected_ip_versions)" \ - | sort -n - } \ - | tail -n1 - ) -elif [ "$#" -eq 4 ]; then - sshHopp="$4" - lokPort=$[$RANDOM/2+8000] - HoppIP="${sshHopp#*@}" - ipVer=$( - { - extract_ssh_ip_protocols "${QuellIP}" - extract_ssh_ip_protocols "${HoppIP}" - } \ - | sort -n \ - | uniq -d \ - | tail -n1 - ) - if [ -z ${ipVer} ]; then - >&2 echo 'not reachable' - exit 11 - fi - if [ ${ipVer} -eq 4 ]; then - localAddress='127.0.0.1' - elif [ ${ipVer} -eq 6 ]; then - localAddress='[::1]' - else - >&2 echo 'ip version of hopp and target must be the same' - exit 1 - fi - rsyncShell="-e ssh -${ipVer} -p${lokPort} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" - tunnelBefehl="ssh -${ipVer} -t -t -L${localAddress}:${lokPort}:${QuellIP}:22 ${sshHopp} while sleep 60; do date; done" - Quelle="$(echo "$3" | sed "s|${QuellIP}|${localAddress}|")" -else - usage -fi - -rsyncOptions="${rsyncOptions}${ipVer}" - -if [ ! -d ${Basis} ]; then - for neededMount in "${neededMounts[@]}"; do - if ! mountpoint -q "${neededMount}"; then - >&2 printf 'Mountpoint %s is not available.\n' "${neededMount}" - exit 11 - fi - done - exit 2 -fi - -neues_Datum="${Basis}/$(date "+%Y_%m_%d")" -neues="${Basis}/aktuell" -linkdests="" -for s in $(ls -1 "${Basis}" | sort -r | grep -vxF -m 20 aktuell ); do - linkdests="${linkdests} --link-dest ${Basis}/${s}" -done - -if [ ! "$(whoami)" == "root" ]; then - echo "I need to be root." - exit 3 -fi - -[ -e "${neues_Datum}" ] && exit 4 -if ${seldom}; then - for date_diff in $(seq ${seldomness}); do - if [ -e "${Basis}/$(date '+%Y_%m_%d' -d@$(($(date '+%s')-24*60*60*date_diff)))" ]; then - exit 4 - fi - done -fi -[ -w "${Basis}" ] || exit 11 -[ -s "${pidFile}" ] && [ -d "/proc/$(cat "${pidFile}")" ] && exit 5 - -echo $$ > "${pidFile}" - -if [ -n "${tunnelBefehl}" ]; then - preConnectHook - ${tunnelBefehl} & - backgroundPid=$! - sleep 4 -fi - -for toExclude in "${excludes[@]}"; do - excludeArgs="${excludeArgs} --exclude ${toExclude}" -done - -mkdir -p "${neues}/wip" -chmod 750 "${neues}"{,/wip} -chown root:root "${neues}"{,/wip} -if [ -z "${rsyncShell}" ]; then - preConnectHook - if [ -z "${Quelle#*@*:/}" ]; then - ssh "-${ipVer}" "${Quelle%:/}" ' - pacman-conf Architecture - find /var/lib/pacman/local -name desc -exec cat {} + - ' \ - | { - read -r arch - sed -n ' - /^%\(NAME\|VERSION\|ARCH\)%/ { - N - s/\n/ / - p - } - ' \ - | sed ' - N - N - s@^%NAME% \(\S\+\)\n%VERSION% \(\S\+\)\n%ARCH% \(\S\+\)$@\1-\2-\3.pkg.tar.zst@ - t - w /dev/stderr - d - ' \ - | while read -r file; do - cache_file='/var/cache/pacman/pkg/'"${file}" - if [ ! -f "${cache_file}" ]; then - case "${arch}" in - 'x86_64') - for url in \ - 'https://mirror.f4st.host/archlinux/pool/packages/'"${file}" \ - 'https://archive.archlinux.org/packages/'"${file:0:1}"'/'"${file%-*}"'/'"${file}"; do - curl -Lo "${cache_file}" "${url}" || continue - tar --zstd -tf "${cache_file}" >/dev/null 2>&1 && break - done - ;; - 'i486'|'i686'|'pentium4') - for url in \ - 'https://mirror.archlinux32.org/pool/'"${file}" \ - 'https://archive.archlinux32.org/packages/'"${file:0:1}"'/'"${file%-*}"'/'"${file}"; do - curl -Lo "${cache_file}" "${url}" || continue - tar --zstd -tf "${cache_file}" >/dev/null 2>&1 && break - done - ;; - *) - for url in \ - 'https://mirror.archlinux.org/'"${arch}"'/'{extra,community,core,alarm,aur}'/'"${file}"; do - curl -Lo "${cache_file}" "${url}" || continue - tar --zstd -tf "${cache_file}" >/dev/null 2>&1 && break - done - ;; - esac - fi - if [ -f "${cache_file}" ]; then - tar -C "${neues:?}/wip/" --zstd -xkf "${cache_file}" - fi - done - } - fi - rsync ${rsyncOptions} \ - ${linkdests} \ - ${excludeArgs} \ - ${Quelle} "${neues}/wip/" - sleep 1 - preConnectHook - rsync ${Quelle} -else - preConnectHook - rsync "${rsyncShell}" \ - ${rsyncOptions} \ - ${linkdests} \ - ${excludeArgs} \ - ${Quelle} "${neues}/wip/" - sleep 1 - preConnectHook - rsync "${rsyncShell}" ${Quelle} -fi -erg=$? - -[ -n "${backgroundPid}" ] && kill "${backgroundPid}" - -if [ ${erg} -eq 0 ] || [ ${erg} -eq 24 ] && ! rmdir "${neues}/wip" 2>/dev/null; then - chmod o-rwx "${neues}/wip" - neueres_Datum="${Basis}/$(date "+%Y_%m_%d")" - if [ ! -e "${neueres_Datum}" ]; then - neues_Datum="${neueres_Datum}" - fi - mv "${neues}/wip" "${neues_Datum}" - rmdir "${neues}" - rm "${pidFile}" -else - rm "${pidFile}" - exit 11 -fi diff --git a/backup@.service.in b/backup@.service.in index 19fef7d..4ce52d5 100644 --- a/backup@.service.in +++ b/backup@.service.in @@ -5,7 +5,7 @@ After=network-online.target local-fs.target [Service] Type=simple -ExecStart=#BINDIR#/backup %I +ExecStart=#BINDIR#/hardlinked-backup %I SuccessExitStatus=4 RestartForceExitStatus=5 11 RestartSec=10 diff --git a/hardlinked-backup.in b/hardlinked-backup.in new file mode 100755 index 0000000..d5c76e2 --- /dev/null +++ b/hardlinked-backup.in @@ -0,0 +1,302 @@ +#!/bin/bash + +rsyncOptions='-vxcaHAXF' +rsync --help | \ + grep -q -- --may-modify-others && \ + rsyncOptions="${rsyncOptions} --may-modify-others" + +[ -r "#ETCDIR#/backup.conf" ] && \ + . "#ETCDIR#/backup.conf" + +usage() +{ + >&2 echo \ +'Usage: hardlinked-backup /tmp/pidFile /path/to/destination/ user@source:path [proxy_user@ssh_host] +Backup files from remote with rsync, possibly via SSH-tunnel. + +With no arguments, information is expected to be in array $backups in #ETCDIR#/backup.conf with name of executable (e.g. a symlink) as key. +With one argument, information is expected to be in array ${backups[$1]} in #ETCDIR#/backup.conf. + +Options: + /tmp/pidFile location of file to store PID in + /path/to/destination location to store backups in + user@source:path remote data to back up + proxy_user@ssh_host ssh login to proxy node (optional) +#HELPTEXT# #' + [ -z "$1" ] && exit 1 + exit $1 +} + +extract_ssh_host() { + { + printf '%s\n' "$1" + if [ -f ~/.ssh/config ]; then + sed ' + /^Host\s\+'"$1"'$/,/^Host\s/ { + s/^\s*Hostname\s\+// + t + } + d + ' ~/.ssh/config + fi + } \ + | tail -n1 +} + +extract_ssh_ip_protocols() { + getent ahosts "$(extract_ssh_host "$1")" \ + | sed ' + s/^\([0-9]\+\.\)\{3\}[0-9]\+\s\+STREAM\(\s.*\)\?$/4/ + t + s/^[0-9a-f][0-9a-f:]\+\s\+STREAM\(\s.*\)\?$/6/ + t + d + ' \ + | sort -u \ + | if [ -f ~/.ssh/config ]; then + grep -vxF "$( + sed ' + /^Host\s\+'"$1"'$/,/^Host\s/ { + s/^\s*AddressFamily\s\+inet\s*$/6/ + t + s/^\s*AddressFamily\s\+inet6\s*$/4/ + t + } + d + ' ~/.ssh/config + )" + else + cat + fi +} + +connected_ip_versions() { + ip -o addr show scope global \ + | awk '{print $4}' \ + | sed ' + s@^\([0-9]\+\.\)\{3\}[0-9]\+\(/[0-9]\+\)\?$@4@ + t + s@^[0-9a-f][0-9a-f:]\+\(/[0-9]\+\)\?$@6@ + t + d + ' \ + | sort -u +} + +if [ $# -eq 1 ]; then + if [ "$1" == "--help" ]; then + usage 0 + elif [ "$1" == "--version" ]; then + echo '#VERSION#' + exit + fi +fi + +seldom=false +if [ $# -eq 0 ] || [ $# -eq 1 ]; then + if [ $# -eq 0 ]; then + backupID="$(basename $0)" + else + backupID="$1" + fi + [ -z "${backups[${backupID}]}" ] && usage + set /tmp/${backupID}.pid ${backups[${backupID}]} + if printf '%s\n' "${seldomBackups[@]}" | \ + grep -qxF "${backupID}"; then + seldom=true + fi +fi + +Basis="$2" +pidFile="$1" +QuellIP=$(echo "$3" | sed "s|^[a-zA-Z]*://||; s|^[a-zA-Z]*@||; s|:\?/.*$||") + +if [ "$#" -eq 3 ]; then + Quelle="$3" + ipVer=$( + { + extract_ssh_ip_protocols "${QuellIP}" \ + | sort -n + extract_ssh_ip_protocols "${QuellIP}" \ + | grep -xF "$(connected_ip_versions)" \ + | sort -n + } \ + | tail -n1 + ) +elif [ "$#" -eq 4 ]; then + sshHopp="$4" + lokPort=$[$RANDOM/2+8000] + HoppIP="${sshHopp#*@}" + ipVer=$( + { + extract_ssh_ip_protocols "${QuellIP}" + extract_ssh_ip_protocols "${HoppIP}" + } \ + | sort -n \ + | uniq -d \ + | tail -n1 + ) + if [ -z ${ipVer} ]; then + >&2 echo 'not reachable' + exit 11 + fi + if [ ${ipVer} -eq 4 ]; then + localAddress='127.0.0.1' + elif [ ${ipVer} -eq 6 ]; then + localAddress='[::1]' + else + >&2 echo 'ip version of hopp and target must be the same' + exit 1 + fi + rsyncShell="-e ssh -${ipVer} -p${lokPort} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" + tunnelBefehl="ssh -${ipVer} -t -t -L${localAddress}:${lokPort}:${QuellIP}:22 ${sshHopp} while sleep 60; do date; done" + Quelle="$(echo "$3" | sed "s|${QuellIP}|${localAddress}|")" +else + usage +fi + +rsyncOptions="${rsyncOptions}${ipVer}" + +if [ ! -d ${Basis} ]; then + for neededMount in "${neededMounts[@]}"; do + if ! mountpoint -q "${neededMount}"; then + >&2 printf 'Mountpoint %s is not available.\n' "${neededMount}" + exit 11 + fi + done + exit 2 +fi + +neues_Datum="${Basis}/$(date "+%Y_%m_%d")" +neues="${Basis}/aktuell" +linkdests="" +for s in $(ls -1 "${Basis}" | sort -r | grep -vxF -m 20 aktuell ); do + linkdests="${linkdests} --link-dest ${Basis}/${s}" +done + +if [ ! "$(whoami)" == "root" ]; then + echo "I need to be root." + exit 3 +fi + +[ -e "${neues_Datum}" ] && exit 4 +if ${seldom}; then + for date_diff in $(seq ${seldomness}); do + if [ -e "${Basis}/$(date '+%Y_%m_%d' -d@$(($(date '+%s')-24*60*60*date_diff)))" ]; then + exit 4 + fi + done +fi +[ -w "${Basis}" ] || exit 11 +[ -s "${pidFile}" ] && [ -d "/proc/$(cat "${pidFile}")" ] && exit 5 + +echo $$ > "${pidFile}" + +if [ -n "${tunnelBefehl}" ]; then + preConnectHook + ${tunnelBefehl} & + backgroundPid=$! + sleep 4 +fi + +for toExclude in "${excludes[@]}"; do + excludeArgs="${excludeArgs} --exclude ${toExclude}" +done + +mkdir -p "${neues}/wip" +chmod 750 "${neues}"{,/wip} +chown root:root "${neues}"{,/wip} +if [ -z "${rsyncShell}" ]; then + preConnectHook + if [ -z "${Quelle#*@*:/}" ]; then + ssh "-${ipVer}" "${Quelle%:/}" ' + pacman-conf Architecture + find /var/lib/pacman/local -name desc -exec cat {} + + ' \ + | { + read -r arch + sed -n ' + /^%\(NAME\|VERSION\|ARCH\)%/ { + N + s/\n/ / + p + } + ' \ + | sed ' + N + N + s@^%NAME% \(\S\+\)\n%VERSION% \(\S\+\)\n%ARCH% \(\S\+\)$@\1-\2-\3.pkg.tar.zst@ + t + w /dev/stderr + d + ' \ + | while read -r file; do + cache_file='/var/cache/pacman/pkg/'"${file}" + if [ ! -f "${cache_file}" ]; then + case "${arch}" in + 'x86_64') + for url in \ + 'https://mirror.f4st.host/archlinux/pool/packages/'"${file}" \ + 'https://archive.archlinux.org/packages/'"${file:0:1}"'/'"${file%-*}"'/'"${file}"; do + curl -Lo "${cache_file}" "${url}" || continue + tar --zstd -tf "${cache_file}" >/dev/null 2>&1 && break + done + ;; + 'i486'|'i686'|'pentium4') + for url in \ + 'https://mirror.archlinux32.org/pool/'"${file}" \ + 'https://archive.archlinux32.org/packages/'"${file:0:1}"'/'"${file%-*}"'/'"${file}"; do + curl -Lo "${cache_file}" "${url}" || continue + tar --zstd -tf "${cache_file}" >/dev/null 2>&1 && break + done + ;; + *) + for url in \ + 'https://mirror.archlinux.org/'"${arch}"'/'{extra,community,core,alarm,aur}'/'"${file}"; do + curl -Lo "${cache_file}" "${url}" || continue + tar --zstd -tf "${cache_file}" >/dev/null 2>&1 && break + done + ;; + esac + fi + if [ -f "${cache_file}" ]; then + tar -C "${neues:?}/wip/" --zstd -xkf "${cache_file}" + fi + done + } + fi + rsync ${rsyncOptions} \ + ${linkdests} \ + ${excludeArgs} \ + ${Quelle} "${neues}/wip/" + sleep 1 + preConnectHook + rsync ${Quelle} +else + preConnectHook + rsync "${rsyncShell}" \ + ${rsyncOptions} \ + ${linkdests} \ + ${excludeArgs} \ + ${Quelle} "${neues}/wip/" + sleep 1 + preConnectHook + rsync "${rsyncShell}" ${Quelle} +fi +erg=$? + +[ -n "${backgroundPid}" ] && kill "${backgroundPid}" + +if [ ${erg} -eq 0 ] || [ ${erg} -eq 24 ] && ! rmdir "${neues}/wip" 2>/dev/null; then + chmod o-rwx "${neues}/wip" + neueres_Datum="${Basis}/$(date "+%Y_%m_%d")" + if [ ! -e "${neueres_Datum}" ]; then + neues_Datum="${neueres_Datum}" + fi + mv "${neues}/wip" "${neues_Datum}" + rmdir "${neues}" + rm "${pidFile}" +else + rm "${pidFile}" + exit 11 +fi diff --git a/last-backups.in b/last-backups.in index 61117d0..a636f46 100755 --- a/last-backups.in +++ b/last-backups.in @@ -71,7 +71,7 @@ do ' ) ) - if [ -f "/tmp/${backupID}.pid" ] && pgrep '(^|/|\s)backup\S*$' | grep -qxF "$(cat "/tmp/${backupID}.pid")"; then + if [ -f "/tmp/${backupID}.pid" ] && pgrep '(^|/|\s)hardlinked-backup\S*$' | grep -qxF "$(cat "/tmp/${backupID}.pid")"; then laeuft='_laeuft' else laeuft='' diff --git a/man.commons.in b/man.commons.in index 4ebb720..d94775f 100644 --- a/man.commons.in +++ b/man.commons.in @@ -3,8 +3,8 @@ [AUTHOR] Erich Eckner . [SEE ALSO] -backup(1) backup-statistics(1) +hardlinked-backup(1) last-backups(1) remove-old-backups(1) [CONFIG] -- cgit v1.2.3-54-g00ecf