natano's soup

#!/bin/bash


start=0
end=1650896
outfolder=./out

baseurl='http://futurezone.orf.at/stories'

get_retcode() {
    curl -s -S -I "$baseurl/$1/" |sed -n -r 's/HTTP\/1\.0\s([0-9]+)\s.*/\1/p'
}

found=0
for i in `seq $start $end`; do
    retcode=`get_retcode $i`
    if test $retcode = 200; then
        curl -s -S -o $outfolder/$i.html $baseurl/$i &
        let found+=1
    fi
    echo -ne "\r$i/$end (found $found articles) " 1>&2
done