blob: 447cc81ec30f55d55b8bd4e93bb476056f541fd6 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
|
#!/bin/bash
base_dir=$(readlink -f "$(dirname "$0")")
if [ $# -ne 3 ]; then
>&2 printf 'usage: %s wiki-depth markov-depth word-count\n' "${0##*/}"
exit 1
fi
depth=$1
shift
if [ ! -s "${base_dir}/.words.${depth}" ]; then
urls=()
for i in $(seq "${depth}"); do
urls=(
$(
"${base_dir}/dive-into-wikipedia.sh" "${urls[@]}" \
| sort -u
)
)
echo $i >&2
done
printf '%s\n' "${urls[@]}" \
| parallel -j0 -n1 "${base_dir}/remove-tags.sh" \
| sort -u \
| grep -vxF '' \
> "${base_dir}/.words.${depth}"
fi
"${base_dir}/markov" "${base_dir}/.words.${depth}" "$@"
|