summaryrefslogtreecommitdiff
path: root/backupStatistics.in
diff options
context:
space:
mode:
Diffstat (limited to 'backupStatistics.in')
-rw-r--r--backupStatistics.in174
1 files changed, 105 insertions, 69 deletions
diff --git a/backupStatistics.in b/backupStatistics.in
index 160ba0c..9e9608c 100644
--- a/backupStatistics.in
+++ b/backupStatistics.in
@@ -29,27 +29,19 @@ do_stage()
do
echo "${dat}:"
find "${dest}/${dat}" -type f -links -64001 -printf '%i %D-%m-%U-%G %p\n' >> \
- "${cacheDir}/${backupID}.inodes"
+ "${cacheDir}/${backupID}.inodes"
done
;;
2)
if [ "$2" == '##DESCRIBE##' ]
then
- echo 'sort and partition previous lists by $inode'
+ echo 'sort previous lists by $inode'
return 0
fi
tmpDirA="$(mktemp -d)"
tmpDirB="$(mktemp -d "${cacheDir}/tmp.XXXXXX")"
- rm -rf "${cacheDir}/${backupID}.inodes.sorted"
- mkdir "${cacheDir}/${backupID}.inodes.sorted"
- sort -T "${tmpDirA}" -T "${tmpDirB}" -u "${cacheDir}/${backupID}.inodes" | \
- while read -r line
- do
- part="${line:0:4}"
- part="${part%% *}"
- echo "${line}" >> \
- "${cacheDir}/${backupID}.inodes.sorted/part.${part}"
- done
+ sort -T "${tmpDirA}" -T "${tmpDirB}" -u "${cacheDir}/${backupID}.inodes" > \
+ "${cacheDir}/${backupID}.inodes.sorted"
rmdir "${tmpDirA}" "${tmpDirB}"
;;
3)
@@ -58,8 +50,7 @@ do_stage()
echo 'generate lists $inode -> $count, $contentHash'
return 0
fi
- cat "${cacheDir}/${backupID}.inodes.sorted/"part.* | \
- uniq -cm2 | \
+ uniq -cm2 "${cacheDir}/${backupID}.inodes.sorted" | \
parallel \
sha512sum {=s/^ *\([[:digit:]]\+ \)\{2\}[0-9-]\+ //=} \| \
sed '"s|^\([0-9a-f]\{128\}\) .*\$|\1'{=s/^ *\([[:digit:]]\+ [[:digit:]]\+\) \([0-9-]\+\) .*/-\\2 \\1/=}'|"' \
@@ -81,96 +72,141 @@ do_stage()
5)
if [ "$2" == '##DESCRIBE##' ]
then
- echo 'find duplicate hashes'
+ echo 'generate sorted lists of groups of inodes with the same hashes'
return 0
fi
- (
- uniq -m1 --all-repeated=separate "${cacheDir}/${backupID}.content.sorted"
- echo ""
- ) | \
+ index=0
+ tmpDirA="$(mktemp -d)"
+ tmpDirB="$(mktemp -d "${cacheDir}/tmp.XXXXXX")"
+ uniq -m1 --all-repeated=separate "${cacheDir}/${backupID}.content.sorted" | \
sed 's|^\(\S\+ \)\{2\}||' | \
while read s
do
if [ -z "${s}" ]
then
- echo ""
+ index=$[${index}+1]
else
- echo -n "${s} "
+ echo "${s#* } B ${index}"
fi
done | \
- sed 's| $||' > \
+ sort -T "${tmpDirA}" -T "${tmpDirB}" > \
"${cacheDir}/${backupID}.duplicates"
+ rmdir "${tmpDirA}" "${tmpDirB}"
;;
6)
if [ "$2" == '##DESCRIBE##' ]
then
- echo 'remove inodes with duplicate hashes'
+ echo 'find files to inodes of previous lists'
return 0
fi
- if [ -r "${cacheDir}/next.action" ]
+ tmpDirA="$(mktemp -d)"
+ tmpDirB="$(mktemp -d "${cacheDir}/tmp.XXXXXX")"
+
+ unset block
+ unset lastBlock
+ unset firstInode
+ unset lastInode
+
+ sed '
+ s|^\(\S\+\) \S\+ |\1 F |
+ ' "${cacheDir}/${backupID}.inodes.sorted" | \
+ sort -m -T "${tmpDirA}" -T "${tmpDirB}" -- \
+ - "${cacheDir}/${backupID}.duplicates" | \
+ while read -r inode type extra
+ do
+ if [ "${type}" == "B" ]
+ then
+ block="${extra}"
+ elif [ "${lastInode}" == "${inode}" ] && [ -n "${block}" ]
+ then
+ echo "${block} ${inode} ${extra}"
+ else
+ unset block
+ fi
+ lastInode="${inode}"
+ done | \
+ sort -T "${tmpDirA}" -T "${tmpDirB}" -k1n,1 | \
+ while read -r block inode extra
+ do
+ if [ "${lastBlock}" != "${block}" ]
+ then
+ firstInode="${inode}"
+ fi
+ if [ "${lastBlock}" != "${block}" ] || [ "${firstInode}" != "${inode}" ]
+ then
+ echo "${block} ${extra}"
+ fi
+ lastBlock="${block}"
+ done | \
+ uniq -m1 --group=separate > \
+ "${cacheDir}/${backupID}.duplicates.files"
+ rmdir "${tmpDirA}" "${tmpDirB}"
+ ;;
+ 7)
+ if [ "$2" == '##DESCRIBE##' ]
then
- startInode="$(cat "${cacheDir}/next.action")"
+ echo 'relink files with different inodes and same hashes'
+ return 0
+ fi
+ if [ ! -r "${cacheDir}/next.action" ]
+ then
+ cat "${cacheDir}/${backupID}.duplicates.files"
+ elif [ "$(head -n1 "${cacheDir}/next.action")" == "${backupID}" ]
+ then
+ startBlock="$(tail -n1 "${cacheDir}/next.action")"
sed "
:vor;
- / ${startInode}\( \|$\)/{
- s@^\(\S\+ \)\(.* \)\?${startInode}\( \|$\)@\1${startInode}\3@;
- bnach
- };
+ /^${startBlock} /bnach;
d;
bvor;
:nach;
n;
bnach
- " "${cacheDir}/${backupID}.duplicates"
- else
- cat "${cacheDir}/${backupID}.duplicates"
+ " "${cacheDir}/${backupID}.duplicates.files"
fi | \
- while read line
+ while read -r oBlock original
do
- originalInode="${line%% *}"
- original="$(
- grep -m1 "^${originalInode} " "${cacheDir}/${backupID}.inodes.sorted/part.${originalInode:0:4}" | \
- sed 's|^\S\+ ||'
- )"
- for kopieInode in ${line#* }
+ echo "${backupID}" > "${cacheDir}/next.action2"
+ echo "${oBlock}" >> "${cacheDir}/next.action2"
+ mv "${cacheDir}/next.action2" "${cacheDir}/next.action"
+ while read -r kBlock kopie
do
- echo "${kopieInode}" > "${cacheDir}/next.action2"
- mv "${cacheDir}/next.action2" "${cacheDir}/next.action"
- OIFS="${IFS}"
- IFS="$(printf '\n\t')"
- for kopie in $(
- grep "^${kopieInode} " "${cacheDir}/${backupID}.inodes.sorted/part.${kopieInode:0:4}" | \
- sed 's|^\S\+ ||'
- )
- do
- IFS="${OIFS}"
- if ${paranoid}
+ [ -z "${kopie}" ] && break
+ if [ "${kBlock}" != "${oBlock}" ]
+ then
+ >&2 echo "'${kBlock}' != '${oBlock}'"
+ >&2 echo "'${backupID}':"
+ >&2 echo "'${original}'"
+ >&2 echo "'${kopie}'"
+ exit 1
+ fi
+
+ if ${paranoid}
+ then
+ diff "${original}" "${kopie}"
+ fi
+ if [ $(stat -c'%h' "${original}") -ge 65000 ]
+ then
+ echo "rm \"${original}\""
+ echo "ln \"${kopie}\" \"${original}\""
+ if ! ${dummy}
then
- diff "${original}" "${kopie}"
+ rm "${original}"
+ ln "${kopie}" "${original}"
fi
- if [ $(stat -c'%h' "${original}") -ge 65000 ]
+ else
+ echo "rm \"${kopie}\""
+ echo "ln \"${original}\" \"${kopie}\""
+ if ! ${dummy}
then
- echo "rm \"${original}\""
- echo "ln \"${kopie}\" \"${original}\""
- if ! ${dummy}
- then
- rm "${original}"
- ln "${kopie}" "${original}"
- fi
- else
- echo "rm \"${kopie}\""
- echo "ln \"${original}\" \"${kopie}\""
- if ! ${dummy}
- then
- rm "${kopie}"
- ln "${original}" "${kopie}"
- fi
+ rm "${kopie}"
+ ln "${original}" "${kopie}"
fi
- done
+ fi
done
done
if [ -r "${cacheDir}/next.action" ] && \
- grep -q " $(cat "${cacheDir}/next.action")\( \|$\)" "${cacheDir}/${backupID}.duplicates"
+ [ "$(head -n1 "${cacheDir}/next.action")" == "${backupID}" ]
then
rm -f "${cacheDir}/next.action" "${cacheDir}/next.action2"
fi