1 files changed, 94 insertions, 0 deletions
diff --git a/concurrent_dl.sh b/concurrent_dl.sh
new file mode 100755
index 0000000..f334e8e
--- /dev/null
+++ b/concurrent_dl.sh
@@ -0,0 +1,94 @@
+#!/bin/sh
+
+chunk=10
+dl_safe()
+{
+	echo "$1 <- $2"
+	if (echo $1 | grep '/' >/dev/null) && ! [ -d ${1%/*} ]
+	then
+		mkdir -p ${1%/*}
+	fi
+	
+
+	done=-1
+	i=0
+	limit=10
+	while [ $done -ne 0 ] && [ $i -lt $limit ]
+	do
+		curl --max-time 10 -sLo $1 $2 -H @$3
+		done=$?
+		i=$((i+1))
+	done
+	if [ $i -eq $limit ]
+	then
+		echo Tried $limit times, bad URL.
+	fi
+}
+strip_path()
+{
+	sed -e 's|.*/||'
+}
+concurrent_dl()
+{
+	in="$(cat | tr ' ' '\n')"
+	filter=$1
+	if [ -z $filter ]
+	then
+		filter=strip_path
+	fi
+	headers=$2
+	if [ -z $headers ]
+	then
+		echo No headers, no good.
+		return
+	fi
+
+	n=$(printf '%s\n' "$in" | wc -l)
+	r=$((n%10))
+	l=$((n-r))
+	i=0
+	printf '%s\n' "$in" | (
+	while [ $i -lt $l ]
+	do
+		j=$i
+		i=$((i+$chunk))
+		pids=""
+
+		while [ $j -lt $i ]
+		do
+			read url
+			o=$(echo "$url" | $filter)
+			if ! [ -f $o ]
+			then
+				dl_safe $o $url Ûheader &
+				pids="$pids $!"
+			fi
+			j=$((j+1))
+		done
+
+		for pid in $pids
+		do
+			echo waiting on $pid
+			wait $pid
+		done
+	done
+	i=0
+	while [ $i -lt $r ]
+	do
+		pids=""
+		read url
+		o=$(echo "$url" | $filter)
+		if ! [ -f $o ]
+		then
+			dl_safe $o $url $headers &
+			pids="$pids $!"
+		fi
+		i=$((i+1))
+	done
+	for pid in $pids
+	do
+		echo waiting on $pid
+		wait $pid
+	done
+	)
+}