concurrent_dl.rc


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

fn rand {
	dd if=/dev/urandom bs=1 count=2 >[2]/dev/null | base16 | tr 'a-f' '0-9' | sed -e 's|^|.|'
}

fn dl {
	echo $2
	curl --max-time 10 -sLo $1 $2 -H @$3
}

#fn index_out {
#	sed -e 's|.*&||' -e 's|^|index_d|'
#}
fn strip_path {
	sed -e 's|.*/||'
}

chunk = 10
# $1 is a function for determining output file names from urls
# $2 is a file with HTTP headers

fn concurrent_dl {
	in=`{cat}
	filter = $1

	if (test -z $filter) {
		filter = strip_path
	}

	i = 0
	limit = $#in
	while (test $i -lt $limit) {
		end=`{echo $i + $chunk | bc}
		list = `{for (j in `{seq $i $end}) {
				echo $in($j)
			}
		}

		pids = ''
		for (url in $list) {
			o = `{echo $url | $filter}

			if (! test -f $o) {
				dl $o $url $2 &
				pids = ($apid $pids)
			}
		}

		if (test -n $"pids) {
			for (pid in $pids) {
				echo waiting on $pid
				wait $pid
			}
		}

		i = $end
	}
}