Replace use of LWP::Simple in all grabbers by $self->simple_get()

diff --git a/videosite/BlipTVGrabber.pm b/videosite/BlipTVGrabber.pm
index 5a53d2c..86bbffd 100644 (file)
--- a/videosite/BlipTVGrabber.pm
+++ b/videosite/BlipTVGrabber.pm
@@ -8,8 +8,6 @@ package videosite::BlipTVGrabber;
 use videosite::GrabberBase;
 @ISA = qw(videosite::GrabberBase);
 
-use LWP::Simple qw(!get);
-use LWP::UserAgent;
 use URI;
 use URI::QueryParam;
 use HTML::Parser;
@@ -43,7 +41,7 @@ sub _parse {
     my @text;
     my $e;
     my $xml = undef;
-    my $ua;
+    my $ua = $self->ua();
 
     $url =~ m|$pattern|;
     $url = $1;
@@ -57,7 +55,7 @@ sub _parse {
 
     if ($pattern eq $self->{'PATTERNS'}->[0]) {
         # blip.tv/file pattern
-        unless(defined($content = LWP::Simple::get(sprintf('http://blip.tv/file/%s', $2)))) {
+        unless(defined($content = $self->simple_get(sprintf('http://blip.tv/file/%s', $2), $ua))) {
             $self->error('Could not download page');
             return undef;
         }
@@ -78,7 +76,7 @@ sub _parse {
         my $r;
         my $u;
 
-        $ua = LWP::UserAgent->new(max_redirect => 0);
+        $ua->max_redirect(0);
         $r = $ua->get(sprintf('http://blip.tv/play/%s', $2));
 
         unless(defined($r)) {
@@ -110,7 +108,8 @@ sub _parse {
     }
 
     # Download the XML file containing the stream information
-    unless(defined($content = LWP::Simple::get(sprintf('http://blip.tv/rss/flash/%s', $xml)))) {
+    $ua->max_redirect(7);
+    unless(defined($content = $self->simple_get(sprintf('http://blip.tv/rss/flash/%s', $xml), $ua))) {
         $self->error('Could not download XML metadata');
         return undef;
     }

diff --git a/videosite/CollegeHumorGrabber.pm b/videosite/CollegeHumorGrabber.pm
index fb03286..48ccc7b 100644 (file)
--- a/videosite/CollegeHumorGrabber.pm
+++ b/videosite/CollegeHumorGrabber.pm
@@ -9,7 +9,6 @@ use videosite::GrabberBase;
 @ISA = qw(videosite::GrabberBase);
 
 use videosite::HTMLHelper;
-use LWP::Simple qw(!get);
 use XML::Simple;
 use Data::Dumper;
 
@@ -50,7 +49,7 @@ sub _parse {
     $metadata->{'DLURL'} = undef;
 
     # Get the XML file containing the video metadata
-    unless(defined($content = LWP::Simple::get(sprintf('http://www.collegehumor.com/moogaloop/video/%s', $2)))) {
+    unless(defined($content = $self->simple_get(sprintf('http://www.collegehumor.com/moogaloop/video/%s', $2)))) {
         $self->error('Could not download XML metadata');
         return undef;
     }

diff --git a/videosite/DailyMotionGrabber.pm b/videosite/DailyMotionGrabber.pm
index a98fae6..6544882 100644 (file)
--- a/videosite/DailyMotionGrabber.pm
+++ b/videosite/DailyMotionGrabber.pm
@@ -8,7 +8,6 @@ package videosite::DailyMotionGrabber;
 use videosite::GrabberBase;
 @ISA = qw(videosite::GrabberBase);
 
-use LWP::Simple qw(!get);
 use HTML::Parser;
 use videosite::JSArrayParser;
 use Data::Dumper;
@@ -50,7 +49,7 @@ sub _parse {
     $metadata->{'TITLE'} = undef;
     $metadata->{'DLURL'} = undef;
 
-    unless(defined($content = LWP::Simple::get(sprintf('http://www.dailymotion.com/video/%s', $2)))) {
+    unless(defined($content = $self->simple_get(sprintf('http://www.dailymotion.com/video/%s', $2)))) {
         $self->error('Could not download %s', $url);
         return undef;
     }

diff --git a/videosite/DoubleVikingGrabber.pm b/videosite/DoubleVikingGrabber.pm
index 322e75e..e79c261 100644 (file)
--- a/videosite/DoubleVikingGrabber.pm
+++ b/videosite/DoubleVikingGrabber.pm
@@ -8,7 +8,6 @@ package videosite::DoubleVikingGrabber;
 use videosite::GrabberBase;
 @ISA = qw(videosite::GrabberBase);
 
-use LWP::Simple qw(!get);
 use HTML::TokeParser;
 use Data::Dumper;
 
@@ -48,7 +47,7 @@ sub _parse {
     $metadata->{'TITLE'} = undef;
     $metadata->{'DLURL'} = undef;
 
-    unless(defined($content = LWP::Simple::get(sprintf('http://www.doubleviking.com/videos/page0.html/%s.html', $2)))) {
+    unless(defined($content = $self->simple_get(sprintf('http://www.doubleviking.com/videos/page0.html/%s.html', $2)))) {
         $self->error('Could not download %s', $url);
         return undef;
     }

diff --git a/videosite/GoogleGrabber.pm b/videosite/GoogleGrabber.pm
index a33f027..4915603 100644 (file)
--- a/videosite/GoogleGrabber.pm
+++ b/videosite/GoogleGrabber.pm
@@ -8,7 +8,6 @@ package videosite::GoogleGrabber;
 use videosite::GrabberBase;
 @ISA = qw(videosite::GrabberBase);
 
-use LWP::Simple qw(!get);
 use HTML::TokeParser;
 use Data::Dumper;
 
@@ -48,7 +47,7 @@ sub _parse {
     $metadata->{'TITLE'} = undef;
     $metadata->{'DLURL'} = undef;
 
-    unless(defined($content = LWP::Simple::get(sprintf('http://video.google.com/videohosted?docid=%s', $2)))) {
+    unless(defined($content = $self->simple_get(sprintf('http://video.google.com/videohosted?docid=%s', $2)))) {
         $self->error('Could not download %s', $url);
         return undef;
     }

diff --git a/videosite/LiveLeakGrabber.pm b/videosite/LiveLeakGrabber.pm
index d640e68..27f4c7f 100644 (file)
--- a/videosite/LiveLeakGrabber.pm
+++ b/videosite/LiveLeakGrabber.pm
@@ -8,7 +8,6 @@ package videosite::LiveLeakGrabber;
 use videosite::GrabberBase;
 @ISA = qw(videosite::GrabberBase);
 
-use LWP::Simple qw(!get);
 use HTML::TokeParser;
 use Data::Dumper;
 
@@ -34,6 +33,7 @@ sub _parse {
     my $content;
     my $metadata = {};
     my $p;
+    my $ua = $self->ua();
 
     $url =~ m|$pattern|;
     $url = $1;
@@ -46,7 +46,7 @@ sub _parse {
     $metadata->{'DLURL'} = undef;
 
     # Get the site to extract the title
-    unless(defined($content = LWP::Simple::get($url))) {
+    unless(defined($content = $self->simple_get($url, $ua))) {
         $self->error('Could not download page');
         return undef;
     }
@@ -60,7 +60,7 @@ sub _parse {
     }
 
     # Get the file containing the video metadata
-    unless(defined($content = LWP::Simple::get(sprintf('http://www.liveleak.com/mi?token=%s', $2)))) {
+    unless(defined($content = $self->simple_get(sprintf('http://www.liveleak.com/mi?token=%s', $2), $ua))) {
         $self->error('Could not download metadata');
         return undef;
     }

diff --git a/videosite/MNCastGrabber.pm b/videosite/MNCastGrabber.pm
index 563d9c7..19c1969 100644 (file)
--- a/videosite/MNCastGrabber.pm
+++ b/videosite/MNCastGrabber.pm
@@ -8,7 +8,6 @@ package videosite::MNCastGrabber;
 use videosite::GrabberBase;
 @ISA = qw(videosite::GrabberBase);
 
-use LWP::Simple qw(!get);
 use XML::Simple;
 use HTML::TokeParser;
 use Data::Dumper;
@@ -38,6 +37,7 @@ sub _parse {
     my $t;
     my @accum;
     my $vid;
+    my $ua = $self->ua();
 
     $url =~ m|$pattern|;
     $url = $1;
@@ -50,7 +50,7 @@ sub _parse {
     $metadata->{'DLURL'} = undef;
 
     # First, get a webpage containing the video ID
-    unless(defined($content = LWP::Simple::get(sprintf('http://www.mncast.com/player/index.asp?mnum=%s', $2)))) {
+    unless(defined($content = $self->simple_get(sprintf('http://www.mncast.com/player/index.asp?mnum=%s', $2), $ua))) {
         $self->error('Could not download player page');
         return undef;
     }
@@ -65,7 +65,7 @@ sub _parse {
     }
 
     # Get the XML file containing the video metadata
-    unless(defined($content = LWP::Simple::get(sprintf('http://www.mncast.com/_MovieInfo_/_MovieInfoXML_Tag_v2.asp?movieID=%s&loginPNum=-1&player=0', $vid)))) {
+    unless(defined($content = $self->simple_get(sprintf('http://www.mncast.com/_MovieInfo_/_MovieInfoXML_Tag_v2.asp?movieID=%s&loginPNum=-1&player=0', $vid), $ua))) {
         $self->error('Could not download XML metadata');
         return undef;
     }

diff --git a/videosite/MetaCafeGrabber.pm b/videosite/MetaCafeGrabber.pm
index 5f14049..e134bec 100644 (file)
--- a/videosite/MetaCafeGrabber.pm
+++ b/videosite/MetaCafeGrabber.pm
@@ -8,7 +8,6 @@ package videosite::MetaCafeGrabber;
 use videosite::GrabberBase;
 @ISA = qw(videosite::GrabberBase);
 
-use LWP::Simple qw(!get);
 use HTML::Parser;
 use Data::Dumper;
 
@@ -49,7 +48,7 @@ sub _parse {
     $metadata->{'TITLE'} = undef;
     $metadata->{'DLURL'} = undef;
 
-    unless(defined($content = LWP::Simple::get(sprintf('http://www.metacafe.com/watch/%s', $2)))) {
+    unless(defined($content = $self->simple_get(sprintf('http://www.metacafe.com/watch/%s', $2)))) {
         $self->error('Could not download %s', $url);
         return undef;
     }

diff --git a/videosite/MyVideoGrabber.pm b/videosite/MyVideoGrabber.pm
index be83de2..3028466 100644 (file)
--- a/videosite/MyVideoGrabber.pm
+++ b/videosite/MyVideoGrabber.pm
@@ -8,7 +8,6 @@ package videosite::MyVideoGrabber;
 use videosite::GrabberBase;
 @ISA = qw(videosite::GrabberBase);
 
-use LWP::Simple qw(!get);
 use HTML::Parser;
 use Data::Dumper;
 
@@ -48,7 +47,7 @@ sub _parse {
     $metadata->{'TITLE'} = undef;
     $metadata->{'DLURL'} = undef;
 
-    unless(defined($content = LWP::Simple::get(sprintf('http://www.myvideo.de/watch/%s', $2)))) {
+    unless(defined($content = $self->simple_get(sprintf('http://www.myvideo.de/watch/%s', $2)))) {
         $self->error('Could not download %s', $url);
         return undef;
     }

diff --git a/videosite/RedTubeGrabber.pm b/videosite/RedTubeGrabber.pm
index d461c45..4e05dfd 100644 (file)
--- a/videosite/RedTubeGrabber.pm
+++ b/videosite/RedTubeGrabber.pm
@@ -11,8 +11,6 @@ package videosite::RedTubeGrabber;
 use videosite::GrabberBase;
 @ISA = qw(videosite::GrabberBase);
 
-use LWP::UserAgent;
-use HTTP::Cookies;
 use HTML::TokeParser;
 use Data::Dumper;
 
@@ -72,8 +70,7 @@ sub _parse {
     my $self = shift;
     my $url = shift;
     my $pattern = shift;
-    my $jar = HTTP::Cookies->new();
-    my $ua = LWP::UserAgent->new('agent' => 'Mozilla/5.0');
+    my $ua = $self->ua();
     my $content;
     my $metadata = {};
     my $p;
@@ -92,17 +89,13 @@ sub _parse {
     $metadata->{'DLURL'} = undef;
 
     # Set the cookies necessary to get the video data
-    $jar->set_cookie(undef, 'pp', '1', '/', '.redtube.com');
-    $ua->cookie_jar($jar);
+    $ua->cookie_jar->set_cookie(undef, 'pp', '1', '/', '.redtube.com');
 
-    unless(defined($r = $ua->get(sprintf("http://www.redtube.com/%s", $2)))) {
+    unless(defined($content = $self->simple_get(sprintf("http://www.redtube.com/%s", $2), $ua))) {
         $self->error('Could not download page');
         return undef;
     }
 
-    # Get the site to extract the title
-    $content = $r->decoded_content();
-
     $p = HTML::TokeParser->new(\$content);
 
     # Look for the title

diff --git a/videosite/SevenloadGrabber.pm b/videosite/SevenloadGrabber.pm
index 59e113b..ee47b11 100644 (file)
--- a/videosite/SevenloadGrabber.pm
+++ b/videosite/SevenloadGrabber.pm
@@ -8,7 +8,6 @@ package videosite::SevenloadGrabber;
 use videosite::GrabberBase;
 @ISA = qw(videosite::GrabberBase);
 
-use LWP::Simple qw(!get);
 use XML::Simple;
 use Data::Dumper;
 
@@ -47,7 +46,7 @@ sub _parse {
     $metadata->{'DLURL'} = undef;
 
     # Get the XML file containing the video metadata
-    unless(defined($content = LWP::Simple::get(sprintf('http://api.sevenload.com/api/player/id/%s', $2)))) {
+    unless(defined($content = $self->simple_get(sprintf('http://api.sevenload.com/api/player/id/%s', $2)))) {
         $self->error('Could not download XML metadata');
         return undef;
     }

diff --git a/videosite/SpikedHumorGrabber.pm b/videosite/SpikedHumorGrabber.pm
index 7e763ca..5d0f6ae 100644 (file)
--- a/videosite/SpikedHumorGrabber.pm
+++ b/videosite/SpikedHumorGrabber.pm
@@ -8,7 +8,6 @@ package videosite::SpikedHumorGrabber;
 use videosite::GrabberBase;
 @ISA = qw(videosite::GrabberBase);
 
-use LWP::Simple qw(!get);
 use XML::Simple;
 use Data::Dumper;
 
@@ -47,7 +46,7 @@ sub _parse {
     $metadata->{'DLURL'} = undef;
 
     # Get the XML file containing the video metadata
-    unless(defined($content = LWP::Simple::get(sprintf('http://www.spikedhumor.com/playxml/%s/data.xml', $2)))) {
+    unless(defined($content = $self->simple_get(sprintf('http://www.spikedhumor.com/playxml/%s/data.xml', $2)))) {
         $self->error('Could not download XML metadata');
         return undef;
     }

diff --git a/videosite/VeohGrabber.pm b/videosite/VeohGrabber.pm
index d54e49c..4cac1ea 100644 (file)
--- a/videosite/VeohGrabber.pm
+++ b/videosite/VeohGrabber.pm
@@ -8,8 +8,6 @@ package videosite::VeohGrabber;
 use videosite::GrabberBase;
 @ISA = qw(videosite::GrabberBase);
 
-use LWP::Simple qw(!get);
-use LWP::UserAgent;
 use XML::Simple;
 use Data::Dumper;
 
@@ -35,7 +33,7 @@ sub _parse {
     my $content;
     my $metadata = {};
     my $p = XML::Simple->new();
-    my $ua = LWP::UserAgent->new(max_redirect => 0);
+    my $ua = $self->ua();
     my $t;
     my $dlurl;
     my $r;
@@ -51,7 +49,7 @@ sub _parse {
     $metadata->{'DLURL'} = undef;
 
     # Get the XML file containing the video metadata
-    unless(defined($content = LWP::Simple::get(sprintf('http://www.veoh.com/rest/v2/execute.xml?apiKey=5697781E-1C60-663B-FFD8-9B49D2B56D36&method=veoh.search.search&type=video&maxResults=1&permalink=%s&contentRatingId=1&', $2)))) {
+    unless(defined($content = $self->simple_get(sprintf('http://www.veoh.com/rest/v2/execute.xml?apiKey=5697781E-1C60-663B-FFD8-9B49D2B56D36&method=veoh.search.search&type=video&maxResults=1&permalink=%s&contentRatingId=1&', $2), $ua))) {
         $self->error('Could not download XML metadata');
         return undef;
     }
@@ -75,6 +73,7 @@ sub _parse {
     # We now have to fetch the dlurl to get the redirect target after it,
     # because the dlurl itself must be called with the right referer set
 
+    $ua->max_redirect(0);
     $r = $ua->get($dlurl, 'referer' => 'http://www.veoh.com');
 
     unless ($r->is_redirect) {

diff --git a/videosite/VimeoGrabber.pm b/videosite/VimeoGrabber.pm
index ba76cff..89d3158 100644 (file)
--- a/videosite/VimeoGrabber.pm
+++ b/videosite/VimeoGrabber.pm
@@ -8,7 +8,6 @@ package videosite::VimeoGrabber;
 use videosite::GrabberBase;
 @ISA = qw(videosite::GrabberBase);
 
-use LWP::UserAgent;
 use XML::Simple;
 use Digest::MD5 qw(md5_hex);
 use Data::Dumper;
@@ -41,7 +40,6 @@ sub _parse {
     my $dlpath;
     my $timestamp;
     my $hash;
-    my $ua = LWP::UserAgent->new(agent => 'Mozilla');
 
     $url =~ m|$pattern|;
     $url = $1;
@@ -54,14 +52,11 @@ sub _parse {
     $metadata->{'DLURL'} = undef;
 
     # Get the XML file containing the video metadata
-    $content = $ua->get(sprintf('http://www.vimeo.com/moogaloop/load/clip:%s', $2));
-    unless ($content->is_success()) {
+    unless(defined($content = $self->simple_get(sprintf('http://www.vimeo.com/moogaloop/load/clip:%s', $2)))) {
         $self->error('Could not download XML metadata');
         return undef;
     }
 
-    $content = $content->decoded_content();
-
     unless(defined($t = $p->XMLin($content, KeepRoot => 1))) {
         $self->error('Could not parse XML metadata');
         return undef;

diff --git a/videosite/WimpGrabber.pm b/videosite/WimpGrabber.pm
index c4f259f..db95e42 100644 (file)
--- a/videosite/WimpGrabber.pm
+++ b/videosite/WimpGrabber.pm
@@ -8,7 +8,6 @@ package videosite::WimpGrabber;
 use videosite::GrabberBase;
 @ISA = qw(videosite::GrabberBase);
 
-use LWP::Simple qw(!get);
 use MIME::Base64;
 use HTML::Parser;
 use Data::Dumper;
@@ -50,7 +49,7 @@ sub _parse {
     $metadata->{'TITLE'} = undef;
     $metadata->{'DLURL'} = undef;
 
-    unless(defined($content = LWP::Simple::get(sprintf('http://www.wimp.com/%s', $2)))) {
+    unless(defined($content = $self->simple_get(sprintf('http://www.wimp.com/%s', $2)))) {
         $self->error('Could not download %s', $url);
         return undef;
     }

diff --git a/videosite/YahooGrabber.pm b/videosite/YahooGrabber.pm
index c209351..fc88043 100644 (file)
--- a/videosite/YahooGrabber.pm
+++ b/videosite/YahooGrabber.pm
@@ -8,7 +8,6 @@ package videosite::YahooGrabber;
 use videosite::GrabberBase;
 @ISA = qw(videosite::GrabberBase);
 
-use LWP::Simple qw(!get);
 use XML::Simple;
 use HTML::Parser;
 use Data::Dumper;
@@ -37,6 +36,7 @@ sub _parse {
     my $p = XML::Simple->new();
     my $t;
     my @accum;
+    my $ua = $self->ua();
 
     $url =~ m|$pattern|;
     $url = $1;
@@ -49,7 +49,7 @@ sub _parse {
     $metadata->{'DLURL'} = undef;
 
     # Get the XML file containing the video metadata
-    unless(defined($content = LWP::Simple::get(sprintf('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=%s', $2)))) {
+    unless(defined($content = $self->simple_get(sprintf('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=%s', $2), $ua))) {
         $self->error('Could not download XML metadata');
         return undef;
     }
@@ -68,7 +68,7 @@ sub _parse {
     # The XML does not contain the title of the video, for
     # reasons possibly known to some jerk at yahoo.
     # So we'll have to parse the actual HTML, too.
-    unless(defined($content = LWP::Simple::get($url))) {
+    unless(defined($content = $self->simple_get($url, $ua))) {
         $self->error('Could not download HTML');
         return undef;
     }

diff --git a/videosite/ZeroPunctuationGrabber.pm b/videosite/ZeroPunctuationGrabber.pm
index fbad1fd..0ee5ffb 100644 (file)
--- a/videosite/ZeroPunctuationGrabber.pm
+++ b/videosite/ZeroPunctuationGrabber.pm
@@ -8,7 +8,6 @@ package videosite::ZeroPunctuationGrabber;
 use videosite::GrabberBase;
 @ISA = qw(videosite::GrabberBase);
 
-use LWP::Simple qw(!get);
 use HTML::TokeParser;
 use Data::Dumper;
 
@@ -46,7 +45,7 @@ sub _parse {
     $metadata->{'TITLE'} = undef;
     $metadata->{'DLURL'} = undef;
 
-    unless(defined($content = LWP::Simple::get($url))) {
+    unless(defined($content = $self->simple_get($url))) {
         $self->error('Could not download %s', $url);
         return undef;
     }