diff options
| -rwxr-xr-x | concurrent_dl.sh | 24 |
1 files changed, 20 insertions, 4 deletions
diff --git a/concurrent_dl.sh b/concurrent_dl.sh index a824514..d2c9d36 100755 --- a/concurrent_dl.sh +++ b/concurrent_dl.sh @@ -15,7 +15,7 @@ dl_safe() limit=10 while [ $done -ne 0 ] && [ $i -lt $limit ] do - curl --max-time 10 -sLo $1 $2 -H @$3 + curl --compressed --max-time 10 -sLo $1 $2 -H @$3 done=$? i=$((i+1)) done @@ -24,12 +24,28 @@ dl_safe() echo Tried $limit times, bad URL. fi } +if [ -z "$outdir" ] +then + outdir=page +fi +if [ -z "$depth" ] +then + depth=0 +fi strip_path() { - sed -e 's|.*/|page/|' - if ! [ -d page ] + sed -e 's|/$||' -e "s|?.*||" | ( + if [ $depth -eq 1 ] + then + sed -E -e "s|.*/([^/]*/[^/]*)$|\1|" + else + sed -e "s|.*/|$outdir/|" + fi + ) + + if ! [ -d $outdir ] then - mkdir page + mkdir -p $outdir fi } concurrent_dl() |