blob: 6b43c812e9a29a4124e9c947b4dd2a07d0ae7c1a (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
|
#!/bin/bash
base_dir=$(readlink -f "$(dirname "$0")")
if [ $# -ne 3 ]; then
>&2 printf 'usage: %s wiki-depth markov-depth word-count\n' "${0##*/}"
exit 1
fi
depth=$1
shift
if [ ! -s "${base_dir}/.words.${depth}" ]; then
urls=()
for i in $(seq "${depth}"); do
urls=(
$(
"${base_dir}/dive-into-wikipedia.sh" "${urls[@]}" \
| sort -u
)
)
echo $i >&2
done
printf '%s\n' "${urls[@]}" \
| parallel -j0 -n1 curl -x 'socks5://127.0.0.1:9050' -s \
| "${base_dir}/remove-tags.sh" \
| tr -C '[a-zA-ZäöüÄÖÜß]' '\n' \
| grep -vxF '' \
| sort -u \
> "${base_dir}/.words.${depth}"
fi
"${base_dir}/markov" "${base_dir}/.words.${depth}" "$@"
|
