From 771fcc7e7309dcb054d652812cead6b6bf47be5f Mon Sep 17 00:00:00 2001 From: Ralf Ertzinger Date: Sat, 11 Feb 2012 15:43:41 +0100 Subject: [PATCH] Replace use of LWP::Simple in all grabbers by $self->simple_get() --- videosite/BlipTVGrabber.pm | 11 +++++------ videosite/CollegeHumorGrabber.pm | 3 +-- videosite/DailyMotionGrabber.pm | 3 +-- videosite/DoubleVikingGrabber.pm | 3 +-- videosite/GoogleGrabber.pm | 3 +-- videosite/LiveLeakGrabber.pm | 6 +++--- videosite/MNCastGrabber.pm | 6 +++--- videosite/MetaCafeGrabber.pm | 3 +-- videosite/MyVideoGrabber.pm | 3 +-- videosite/RedTubeGrabber.pm | 13 +++---------- videosite/SevenloadGrabber.pm | 3 +-- videosite/SpikedHumorGrabber.pm | 3 +-- videosite/VeohGrabber.pm | 7 +++---- videosite/VimeoGrabber.pm | 7 +------ videosite/WimpGrabber.pm | 3 +-- videosite/YahooGrabber.pm | 6 +++--- videosite/ZeroPunctuationGrabber.pm | 3 +-- 17 files changed, 31 insertions(+), 55 deletions(-) diff --git a/videosite/BlipTVGrabber.pm b/videosite/BlipTVGrabber.pm index 5a53d2c..86bbffd 100644 --- a/videosite/BlipTVGrabber.pm +++ b/videosite/BlipTVGrabber.pm @@ -8,8 +8,6 @@ package videosite::BlipTVGrabber; use videosite::GrabberBase; @ISA = qw(videosite::GrabberBase); -use LWP::Simple qw(!get); -use LWP::UserAgent; use URI; use URI::QueryParam; use HTML::Parser; @@ -43,7 +41,7 @@ sub _parse { my @text; my $e; my $xml = undef; - my $ua; + my $ua = $self->ua(); $url =~ m|$pattern|; $url = $1; @@ -57,7 +55,7 @@ sub _parse { if ($pattern eq $self->{'PATTERNS'}->[0]) { # blip.tv/file pattern - unless(defined($content = LWP::Simple::get(sprintf('http://blip.tv/file/%s', $2)))) { + unless(defined($content = $self->simple_get(sprintf('http://blip.tv/file/%s', $2), $ua))) { $self->error('Could not download page'); return undef; } @@ -78,7 +76,7 @@ sub _parse { my $r; my $u; - $ua = LWP::UserAgent->new(max_redirect => 0); + $ua->max_redirect(0); $r = $ua->get(sprintf('http://blip.tv/play/%s', $2)); unless(defined($r)) { @@ -110,7 +108,8 @@ sub _parse { } # Download the XML file containing the stream information - unless(defined($content = LWP::Simple::get(sprintf('http://blip.tv/rss/flash/%s', $xml)))) { + $ua->max_redirect(7); + unless(defined($content = $self->simple_get(sprintf('http://blip.tv/rss/flash/%s', $xml), $ua))) { $self->error('Could not download XML metadata'); return undef; } diff --git a/videosite/CollegeHumorGrabber.pm b/videosite/CollegeHumorGrabber.pm index fb03286..48ccc7b 100644 --- a/videosite/CollegeHumorGrabber.pm +++ b/videosite/CollegeHumorGrabber.pm @@ -9,7 +9,6 @@ use videosite::GrabberBase; @ISA = qw(videosite::GrabberBase); use videosite::HTMLHelper; -use LWP::Simple qw(!get); use XML::Simple; use Data::Dumper; @@ -50,7 +49,7 @@ sub _parse { $metadata->{'DLURL'} = undef; # Get the XML file containing the video metadata - unless(defined($content = LWP::Simple::get(sprintf('http://www.collegehumor.com/moogaloop/video/%s', $2)))) { + unless(defined($content = $self->simple_get(sprintf('http://www.collegehumor.com/moogaloop/video/%s', $2)))) { $self->error('Could not download XML metadata'); return undef; } diff --git a/videosite/DailyMotionGrabber.pm b/videosite/DailyMotionGrabber.pm index a98fae6..6544882 100644 --- a/videosite/DailyMotionGrabber.pm +++ b/videosite/DailyMotionGrabber.pm @@ -8,7 +8,6 @@ package videosite::DailyMotionGrabber; use videosite::GrabberBase; @ISA = qw(videosite::GrabberBase); -use LWP::Simple qw(!get); use HTML::Parser; use videosite::JSArrayParser; use Data::Dumper; @@ -50,7 +49,7 @@ sub _parse { $metadata->{'TITLE'} = undef; $metadata->{'DLURL'} = undef; - unless(defined($content = LWP::Simple::get(sprintf('http://www.dailymotion.com/video/%s', $2)))) { + unless(defined($content = $self->simple_get(sprintf('http://www.dailymotion.com/video/%s', $2)))) { $self->error('Could not download %s', $url); return undef; } diff --git a/videosite/DoubleVikingGrabber.pm b/videosite/DoubleVikingGrabber.pm index 322e75e..e79c261 100644 --- a/videosite/DoubleVikingGrabber.pm +++ b/videosite/DoubleVikingGrabber.pm @@ -8,7 +8,6 @@ package videosite::DoubleVikingGrabber; use videosite::GrabberBase; @ISA = qw(videosite::GrabberBase); -use LWP::Simple qw(!get); use HTML::TokeParser; use Data::Dumper; @@ -48,7 +47,7 @@ sub _parse { $metadata->{'TITLE'} = undef; $metadata->{'DLURL'} = undef; - unless(defined($content = LWP::Simple::get(sprintf('http://www.doubleviking.com/videos/page0.html/%s.html', $2)))) { + unless(defined($content = $self->simple_get(sprintf('http://www.doubleviking.com/videos/page0.html/%s.html', $2)))) { $self->error('Could not download %s', $url); return undef; } diff --git a/videosite/GoogleGrabber.pm b/videosite/GoogleGrabber.pm index a33f027..4915603 100644 --- a/videosite/GoogleGrabber.pm +++ b/videosite/GoogleGrabber.pm @@ -8,7 +8,6 @@ package videosite::GoogleGrabber; use videosite::GrabberBase; @ISA = qw(videosite::GrabberBase); -use LWP::Simple qw(!get); use HTML::TokeParser; use Data::Dumper; @@ -48,7 +47,7 @@ sub _parse { $metadata->{'TITLE'} = undef; $metadata->{'DLURL'} = undef; - unless(defined($content = LWP::Simple::get(sprintf('http://video.google.com/videohosted?docid=%s', $2)))) { + unless(defined($content = $self->simple_get(sprintf('http://video.google.com/videohosted?docid=%s', $2)))) { $self->error('Could not download %s', $url); return undef; } diff --git a/videosite/LiveLeakGrabber.pm b/videosite/LiveLeakGrabber.pm index d640e68..27f4c7f 100644 --- a/videosite/LiveLeakGrabber.pm +++ b/videosite/LiveLeakGrabber.pm @@ -8,7 +8,6 @@ package videosite::LiveLeakGrabber; use videosite::GrabberBase; @ISA = qw(videosite::GrabberBase); -use LWP::Simple qw(!get); use HTML::TokeParser; use Data::Dumper; @@ -34,6 +33,7 @@ sub _parse { my $content; my $metadata = {}; my $p; + my $ua = $self->ua(); $url =~ m|$pattern|; $url = $1; @@ -46,7 +46,7 @@ sub _parse { $metadata->{'DLURL'} = undef; # Get the site to extract the title - unless(defined($content = LWP::Simple::get($url))) { + unless(defined($content = $self->simple_get($url, $ua))) { $self->error('Could not download page'); return undef; } @@ -60,7 +60,7 @@ sub _parse { } # Get the file containing the video metadata - unless(defined($content = LWP::Simple::get(sprintf('http://www.liveleak.com/mi?token=%s', $2)))) { + unless(defined($content = $self->simple_get(sprintf('http://www.liveleak.com/mi?token=%s', $2), $ua))) { $self->error('Could not download metadata'); return undef; } diff --git a/videosite/MNCastGrabber.pm b/videosite/MNCastGrabber.pm index 563d9c7..19c1969 100644 --- a/videosite/MNCastGrabber.pm +++ b/videosite/MNCastGrabber.pm @@ -8,7 +8,6 @@ package videosite::MNCastGrabber; use videosite::GrabberBase; @ISA = qw(videosite::GrabberBase); -use LWP::Simple qw(!get); use XML::Simple; use HTML::TokeParser; use Data::Dumper; @@ -38,6 +37,7 @@ sub _parse { my $t; my @accum; my $vid; + my $ua = $self->ua(); $url =~ m|$pattern|; $url = $1; @@ -50,7 +50,7 @@ sub _parse { $metadata->{'DLURL'} = undef; # First, get a webpage containing the video ID - unless(defined($content = LWP::Simple::get(sprintf('http://www.mncast.com/player/index.asp?mnum=%s', $2)))) { + unless(defined($content = $self->simple_get(sprintf('http://www.mncast.com/player/index.asp?mnum=%s', $2), $ua))) { $self->error('Could not download player page'); return undef; } @@ -65,7 +65,7 @@ sub _parse { } # Get the XML file containing the video metadata - unless(defined($content = LWP::Simple::get(sprintf('http://www.mncast.com/_MovieInfo_/_MovieInfoXML_Tag_v2.asp?movieID=%s&loginPNum=-1&player=0', $vid)))) { + unless(defined($content = $self->simple_get(sprintf('http://www.mncast.com/_MovieInfo_/_MovieInfoXML_Tag_v2.asp?movieID=%s&loginPNum=-1&player=0', $vid), $ua))) { $self->error('Could not download XML metadata'); return undef; } diff --git a/videosite/MetaCafeGrabber.pm b/videosite/MetaCafeGrabber.pm index 5f14049..e134bec 100644 --- a/videosite/MetaCafeGrabber.pm +++ b/videosite/MetaCafeGrabber.pm @@ -8,7 +8,6 @@ package videosite::MetaCafeGrabber; use videosite::GrabberBase; @ISA = qw(videosite::GrabberBase); -use LWP::Simple qw(!get); use HTML::Parser; use Data::Dumper; @@ -49,7 +48,7 @@ sub _parse { $metadata->{'TITLE'} = undef; $metadata->{'DLURL'} = undef; - unless(defined($content = LWP::Simple::get(sprintf('http://www.metacafe.com/watch/%s', $2)))) { + unless(defined($content = $self->simple_get(sprintf('http://www.metacafe.com/watch/%s', $2)))) { $self->error('Could not download %s', $url); return undef; } diff --git a/videosite/MyVideoGrabber.pm b/videosite/MyVideoGrabber.pm index be83de2..3028466 100644 --- a/videosite/MyVideoGrabber.pm +++ b/videosite/MyVideoGrabber.pm @@ -8,7 +8,6 @@ package videosite::MyVideoGrabber; use videosite::GrabberBase; @ISA = qw(videosite::GrabberBase); -use LWP::Simple qw(!get); use HTML::Parser; use Data::Dumper; @@ -48,7 +47,7 @@ sub _parse { $metadata->{'TITLE'} = undef; $metadata->{'DLURL'} = undef; - unless(defined($content = LWP::Simple::get(sprintf('http://www.myvideo.de/watch/%s', $2)))) { + unless(defined($content = $self->simple_get(sprintf('http://www.myvideo.de/watch/%s', $2)))) { $self->error('Could not download %s', $url); return undef; } diff --git a/videosite/RedTubeGrabber.pm b/videosite/RedTubeGrabber.pm index d461c45..4e05dfd 100644 --- a/videosite/RedTubeGrabber.pm +++ b/videosite/RedTubeGrabber.pm @@ -11,8 +11,6 @@ package videosite::RedTubeGrabber; use videosite::GrabberBase; @ISA = qw(videosite::GrabberBase); -use LWP::UserAgent; -use HTTP::Cookies; use HTML::TokeParser; use Data::Dumper; @@ -72,8 +70,7 @@ sub _parse { my $self = shift; my $url = shift; my $pattern = shift; - my $jar = HTTP::Cookies->new(); - my $ua = LWP::UserAgent->new('agent' => 'Mozilla/5.0'); + my $ua = $self->ua(); my $content; my $metadata = {}; my $p; @@ -92,17 +89,13 @@ sub _parse { $metadata->{'DLURL'} = undef; # Set the cookies necessary to get the video data - $jar->set_cookie(undef, 'pp', '1', '/', '.redtube.com'); - $ua->cookie_jar($jar); + $ua->cookie_jar->set_cookie(undef, 'pp', '1', '/', '.redtube.com'); - unless(defined($r = $ua->get(sprintf("http://www.redtube.com/%s", $2)))) { + unless(defined($content = $self->simple_get(sprintf("http://www.redtube.com/%s", $2), $ua))) { $self->error('Could not download page'); return undef; } - # Get the site to extract the title - $content = $r->decoded_content(); - $p = HTML::TokeParser->new(\$content); # Look for the title diff --git a/videosite/SevenloadGrabber.pm b/videosite/SevenloadGrabber.pm index 59e113b..ee47b11 100644 --- a/videosite/SevenloadGrabber.pm +++ b/videosite/SevenloadGrabber.pm @@ -8,7 +8,6 @@ package videosite::SevenloadGrabber; use videosite::GrabberBase; @ISA = qw(videosite::GrabberBase); -use LWP::Simple qw(!get); use XML::Simple; use Data::Dumper; @@ -47,7 +46,7 @@ sub _parse { $metadata->{'DLURL'} = undef; # Get the XML file containing the video metadata - unless(defined($content = LWP::Simple::get(sprintf('http://api.sevenload.com/api/player/id/%s', $2)))) { + unless(defined($content = $self->simple_get(sprintf('http://api.sevenload.com/api/player/id/%s', $2)))) { $self->error('Could not download XML metadata'); return undef; } diff --git a/videosite/SpikedHumorGrabber.pm b/videosite/SpikedHumorGrabber.pm index 7e763ca..5d0f6ae 100644 --- a/videosite/SpikedHumorGrabber.pm +++ b/videosite/SpikedHumorGrabber.pm @@ -8,7 +8,6 @@ package videosite::SpikedHumorGrabber; use videosite::GrabberBase; @ISA = qw(videosite::GrabberBase); -use LWP::Simple qw(!get); use XML::Simple; use Data::Dumper; @@ -47,7 +46,7 @@ sub _parse { $metadata->{'DLURL'} = undef; # Get the XML file containing the video metadata - unless(defined($content = LWP::Simple::get(sprintf('http://www.spikedhumor.com/playxml/%s/data.xml', $2)))) { + unless(defined($content = $self->simple_get(sprintf('http://www.spikedhumor.com/playxml/%s/data.xml', $2)))) { $self->error('Could not download XML metadata'); return undef; } diff --git a/videosite/VeohGrabber.pm b/videosite/VeohGrabber.pm index d54e49c..4cac1ea 100644 --- a/videosite/VeohGrabber.pm +++ b/videosite/VeohGrabber.pm @@ -8,8 +8,6 @@ package videosite::VeohGrabber; use videosite::GrabberBase; @ISA = qw(videosite::GrabberBase); -use LWP::Simple qw(!get); -use LWP::UserAgent; use XML::Simple; use Data::Dumper; @@ -35,7 +33,7 @@ sub _parse { my $content; my $metadata = {}; my $p = XML::Simple->new(); - my $ua = LWP::UserAgent->new(max_redirect => 0); + my $ua = $self->ua(); my $t; my $dlurl; my $r; @@ -51,7 +49,7 @@ sub _parse { $metadata->{'DLURL'} = undef; # Get the XML file containing the video metadata - unless(defined($content = LWP::Simple::get(sprintf('http://www.veoh.com/rest/v2/execute.xml?apiKey=5697781E-1C60-663B-FFD8-9B49D2B56D36&method=veoh.search.search&type=video&maxResults=1&permalink=%s&contentRatingId=1&', $2)))) { + unless(defined($content = $self->simple_get(sprintf('http://www.veoh.com/rest/v2/execute.xml?apiKey=5697781E-1C60-663B-FFD8-9B49D2B56D36&method=veoh.search.search&type=video&maxResults=1&permalink=%s&contentRatingId=1&', $2), $ua))) { $self->error('Could not download XML metadata'); return undef; } @@ -75,6 +73,7 @@ sub _parse { # We now have to fetch the dlurl to get the redirect target after it, # because the dlurl itself must be called with the right referer set + $ua->max_redirect(0); $r = $ua->get($dlurl, 'referer' => 'http://www.veoh.com'); unless ($r->is_redirect) { diff --git a/videosite/VimeoGrabber.pm b/videosite/VimeoGrabber.pm index ba76cff..89d3158 100644 --- a/videosite/VimeoGrabber.pm +++ b/videosite/VimeoGrabber.pm @@ -8,7 +8,6 @@ package videosite::VimeoGrabber; use videosite::GrabberBase; @ISA = qw(videosite::GrabberBase); -use LWP::UserAgent; use XML::Simple; use Digest::MD5 qw(md5_hex); use Data::Dumper; @@ -41,7 +40,6 @@ sub _parse { my $dlpath; my $timestamp; my $hash; - my $ua = LWP::UserAgent->new(agent => 'Mozilla'); $url =~ m|$pattern|; $url = $1; @@ -54,14 +52,11 @@ sub _parse { $metadata->{'DLURL'} = undef; # Get the XML file containing the video metadata - $content = $ua->get(sprintf('http://www.vimeo.com/moogaloop/load/clip:%s', $2)); - unless ($content->is_success()) { + unless(defined($content = $self->simple_get(sprintf('http://www.vimeo.com/moogaloop/load/clip:%s', $2)))) { $self->error('Could not download XML metadata'); return undef; } - $content = $content->decoded_content(); - unless(defined($t = $p->XMLin($content, KeepRoot => 1))) { $self->error('Could not parse XML metadata'); return undef; diff --git a/videosite/WimpGrabber.pm b/videosite/WimpGrabber.pm index c4f259f..db95e42 100644 --- a/videosite/WimpGrabber.pm +++ b/videosite/WimpGrabber.pm @@ -8,7 +8,6 @@ package videosite::WimpGrabber; use videosite::GrabberBase; @ISA = qw(videosite::GrabberBase); -use LWP::Simple qw(!get); use MIME::Base64; use HTML::Parser; use Data::Dumper; @@ -50,7 +49,7 @@ sub _parse { $metadata->{'TITLE'} = undef; $metadata->{'DLURL'} = undef; - unless(defined($content = LWP::Simple::get(sprintf('http://www.wimp.com/%s', $2)))) { + unless(defined($content = $self->simple_get(sprintf('http://www.wimp.com/%s', $2)))) { $self->error('Could not download %s', $url); return undef; } diff --git a/videosite/YahooGrabber.pm b/videosite/YahooGrabber.pm index c209351..fc88043 100644 --- a/videosite/YahooGrabber.pm +++ b/videosite/YahooGrabber.pm @@ -8,7 +8,6 @@ package videosite::YahooGrabber; use videosite::GrabberBase; @ISA = qw(videosite::GrabberBase); -use LWP::Simple qw(!get); use XML::Simple; use HTML::Parser; use Data::Dumper; @@ -37,6 +36,7 @@ sub _parse { my $p = XML::Simple->new(); my $t; my @accum; + my $ua = $self->ua(); $url =~ m|$pattern|; $url = $1; @@ -49,7 +49,7 @@ sub _parse { $metadata->{'DLURL'} = undef; # Get the XML file containing the video metadata - unless(defined($content = LWP::Simple::get(sprintf('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=%s', $2)))) { + unless(defined($content = $self->simple_get(sprintf('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=%s', $2), $ua))) { $self->error('Could not download XML metadata'); return undef; } @@ -68,7 +68,7 @@ sub _parse { # The XML does not contain the title of the video, for # reasons possibly known to some jerk at yahoo. # So we'll have to parse the actual HTML, too. - unless(defined($content = LWP::Simple::get($url))) { + unless(defined($content = $self->simple_get($url, $ua))) { $self->error('Could not download HTML'); return undef; } diff --git a/videosite/ZeroPunctuationGrabber.pm b/videosite/ZeroPunctuationGrabber.pm index fbad1fd..0ee5ffb 100644 --- a/videosite/ZeroPunctuationGrabber.pm +++ b/videosite/ZeroPunctuationGrabber.pm @@ -8,7 +8,6 @@ package videosite::ZeroPunctuationGrabber; use videosite::GrabberBase; @ISA = qw(videosite::GrabberBase); -use LWP::Simple qw(!get); use HTML::TokeParser; use Data::Dumper; @@ -46,7 +45,7 @@ sub _parse { $metadata->{'TITLE'} = undef; $metadata->{'DLURL'} = undef; - unless(defined($content = LWP::Simple::get($url))) { + unless(defined($content = $self->simple_get($url))) { $self->error('Could not download %s', $url); return undef; } -- 1.8.3.1