Initial

1 files changed, 57 insertions, 0 deletions

diff --git a/concurrent_dl.rc b/concurrent_dl.rc
new file mode 100755
index 0000000..d08a56a
--- /dev/null
+++ b/concurrent_dl.rc
@@ -0,0 +1,57 @@
+fn rand {
+	dd if=/dev/urandom bs=1 count=2 >[2]/dev/null | base16 | tr 'a-f' '0-9' | sed -e 's|^|.|'
+}
+
+fn dl {
+	echo $2
+	curl --max-time 10 -sLo $1 $2 -H @$3
+}
+
+#fn index_out {
+#	sed -e 's|.*&||' -e 's|^|index_d|'
+#}
+fn strip_path {
+	sed -e 's|.*/||'
+}
+
+chunk = 10
+# $1 is a function for determining output file names from urls
+# $2 is a file with HTTP headers
+
+fn concurrent_dl {
+	in=`{cat}
+	filter = $1
+
+	if (test -z $filter) {
+		filter = strip_path
+	}
+
+	i = 0
+	limit = $#in
+	while (test $i -lt $limit) {
+		end=`{echo $i + $chunk | bc}
+		list = `{for (j in `{seq $i $end}) {
+				echo $in($j)
+			}
+		}
+
+		pids = ''
+		for (url in $list) {
+			o = `{echo $url | $filter}
+
+			if (! test -f $o) {
+				dl $o $url $2 &
+				pids = ($apid $pids)
+			}
+		}
+
+		if (test -n $"pids) {
+			for (pid in $pids) {
+				echo waiting on $pid
+				wait $pid
+			}
+		}
+
+		i = $end
+	}
+}