diff options
Diffstat (limited to 'concurrent_dl.sh')
| -rwxr-xr-x | concurrent_dl.sh | 32 |
1 files changed, 24 insertions, 8 deletions
diff --git a/concurrent_dl.sh b/concurrent_dl.sh index a824514..b44b3d0 100755 --- a/concurrent_dl.sh +++ b/concurrent_dl.sh @@ -4,9 +4,9 @@ chunk=10 dl_safe() { echo "$1 <- $2" - if (echo $1 | grep '/' >/dev/null) && ! [ -d ${1%/*} ] + if (echo $1 | grep '/' >/dev/null) && ! [ -d "${1%/*}" ] then - mkdir -p ${1%/*} + mkdir -p "${1%/*}" fi @@ -15,7 +15,7 @@ dl_safe() limit=10 while [ $done -ne 0 ] && [ $i -lt $limit ] do - curl --max-time 10 -sLo $1 $2 -H @$3 + curl --compressed --max-time 10 -sLo "$1" "$2" -H "@$3" done=$? i=$((i+1)) done @@ -24,12 +24,28 @@ dl_safe() echo Tried $limit times, bad URL. fi } +if [ -z "$outdir" ] +then + outdir=page +fi +if [ -z "$depth" ] +then + depth=0 +fi strip_path() { - sed -e 's|.*/|page/|' - if ! [ -d page ] + sed -e 's|/$||' -e "s|?.*||" | ( + if [ $depth -eq 1 ] + then + sed -E -e "s|.*/([^/]*/[^/]*)$|\1|" + else + sed -e "s|.*/|$outdir/|" + fi + ) + + if ! [ -d "$outdir" ] then - mkdir page + mkdir -p "$outdir" fi } concurrent_dl() @@ -82,9 +98,9 @@ concurrent_dl() pids="" read url o=$(echo "$url" | $filter) - if ! [ -f $o ] + if ! [ -f "$o" ] then - dl_safe $o $url $headers & + dl_safe "$o" "$url" "$headers" & pids="$pids $!" fi i=$((i+1)) |