downloader
Posted by root Mon, 09 Jun 2008 08:09:00 GMT
#!/usr/bin/perl
use strict;
use warnings;
use WWW::Mechanize;
use LWP::Simple;
use Data::Dumper;
use Time::HiRes qw( usleep tv_interval gettimeofday );
$|++;
my $url = shift || die "no url given!\n";
my $ext = shift || 'mp3'; #die "no file extention given!\n";
my $mech = WWW::Mechanize->new(agent=>"Mozilla/5.0");
my $ua = LWP::UserAgent->new;
my %seen;
$mech->get($url);
$ua->timeout(5);
my $links = $mech->links;
no warnings;
my $c = 0;
for my $link ( @{$links} ) {
my $url = $link->url_abs;
my $res = $ua->head($url);
my $http_res = HTTP::Response->new($res);
my $abs_name = $http_res ->{'_rc'}->{'_previous'}->{'_headers'}->{'location'};
my ($rel_name) = $abs_name =~ /.+\/(.+)$/;
my $local_name = $rel_name;
$seen{$local_name}++ and next;
if( $rel_name =~ /\Q$ext\E$/i ){
$rel_name =~ s/\%\d+/ /g;
$rel_name =~ s/(.{35}).+/$1.../;
print(pack('A45', "[$rel_name]"), " is being downloaded ... ");
my $s_time = q{};
my $e_time = q{};
my $flag = [];
for( 1..5 ){
local $SIG{ALRM} = sub { die("timeout") };
eval {
alarm(10);
$flag = [head($abs_name)];
alarm(0);
};
next if $@ =~ m|timeout|;
last if $@ !~ m|timeout|;
}
if( $flag->[0] ){
$c++;
$s_time = [gettimeofday];
getstore($abs_name, $c."-".$local_name);
$e_time = tv_interval ($s_time, [gettimeofday]);
}
$e_time = $e_time ? $e_time . " seconds" : 'less than 1 milisecond';
print "timeout failure in $e_time seconds :(\n" and next if $@;
print "done in $e_time\n";
}
}
`ruby -e 'Dir["*"].each{|f| File.rename f, f.sub(/^\d+-/, "")}'`;
`ruby -e 'Dir["*"].each{|f| File.rename f, f.gsub(/%20/, "_")}'`;