#!/bin/sh

chunk=10
dl_safe()
{
	echo "$1 <- $2"
	if (echo $1 | grep '/' >/dev/null) && ! [ -d ${1%/*} ]
	then
		mkdir -p ${1%/*}
	fi
	

	done=-1
	i=0
	limit=10
	while [ $done -ne 0 ] && [ $i -lt $limit ]
	do
		curl --max-time 10 -sLo $1 $2 -H @$3
		done=$?
		i=$((i+1))
	done
	if [ $i -eq $limit ]
	then
		echo Tried $limit times, bad URL.
	fi
}
strip_path()
{
	sed -e 's|.*/|page/|'
	if ! [ -d page ]
	then
		mkdir page
	fi
}
concurrent_dl()
{
	in="$(cat | tr ' ' '\n')"
	filter=$1
	if [ -z $filter ]
	then
		filter=strip_path
	fi
	headers=$2
	if [ -z $headers ]
	then
		echo No headers, no good.
		return
	fi

	n=$(printf '%s\n' "$in" | wc -l)
	r=$((n%10))
	l=$((n-r))
	i=0
	printf '%s\n' "$in" | (
	while [ $i -lt $l ]
	do
		j=$i
		i=$((i+$chunk))
		pids=""

		while [ $j -lt $i ]
		do
			read url
			o=$(echo "$url" | $filter)
			if ! [ -f $o ]
			then
				dl_safe $o $url $headers &
				pids="$pids $!"
			fi
			j=$((j+1))
		done

		for pid in $pids
		do
			echo waiting on $pid
			wait $pid
		done
	done
	i=0
	while [ $i -lt $r ]
	do
		pids=""
		read url
		o=$(echo "$url" | $filter)
		if ! [ -f $o ]
		then
			dl_safe $o $url $headers &
			pids="$pids $!"
		fi
		i=$((i+1))
	done
	for pid in $pids
	do
		echo waiting on $pid
		wait $pid
	done
	)
}