Merge branch 'master' of http://10.200.0.3/GIT/videosite

diff --git a/videosite-dl.pl b/videosite-dl.pl
index 8b1b161..257da2d 100755 (executable)
--- a/videosite-dl.pl
+++ b/videosite-dl.pl
@@ -48,6 +48,21 @@ sub ploader {
     return @g;
 }
 
+sub connectors {
+    my $c = {-name => 'environment', -schemas => {}};
+
+    if (exists($ENV{'http_proxy'})) {
+        $c->{-schemas}->{'http'} = $ENV{'http_proxy'}
+    }
+
+    if (exists($ENV{'https_proxy'})) {
+        $c->{-schemas}->{'https'} = $ENV{'https_proxy'}
+    }
+
+    return ( $c );
+}
+
+
 my $hq = 0;
 my $ext = '.flv';
 my $y;
@@ -55,6 +70,10 @@ my $f;
 my $m;
 my @g;
 my $bp;
+my $info = 0;
+my $debug = 0;
+
+GetOptions("i" => \$info, "d" => \$debug);
 
 # This is some dark magic to find out our real base directory,
 # where we hope to find our plugins.
@@ -69,14 +88,29 @@ unless(@g and defined($f)) {
     exit 1;
 }
 
+foreach (@g, $f) {
+    $_->setio(sub { printf(@_); print("\n"); } );
+
+    if ($debug) {
+        $_->setdebug(1);
+        $_->setconn(\&connectors);
+    }
+}
+
 $f->setval('FILEPATTERN', './%3$s' . $ext);
 
 foreach (@ARGV) {
     foreach $y (@g) {
         ($m, undef) = $y->get($_);
         if (defined($m)) {
-            print("Downloading $m->{'TITLE'}\n");
-            $f->get($m);
+            if ($info) {
+                foreach (keys(%{$m})) {
+                    printf("%s: %s\n", $_, defined($m->{$_})?$m->{$_}:'(undef)');
+                }
+            } else {
+                print("Downloading $m->{'TITLE'}\n");
+                $f->get($m);
+            }
         }
     }
 }

diff --git a/videosite/Base.pm b/videosite/Base.pm
index 577eb52..5381911 100644 (file)
--- a/videosite/Base.pm
+++ b/videosite/Base.pm
@@ -10,7 +10,12 @@ use Data::Dumper;
 
 sub new {
     my $class = shift;
-    my $self = {'_DEBUG' => 0, '_OUT' => sub {printf(@_)}};
+    my $self = {'_DEBUG' => 0,
+                '_OUT' => sub {printf(@_)},
+                '_CONNECTORS' => sub { return ({ -name => 'direct',
+                                                 -schemas => {} }) },
+                '_CONNECTOR' => undef,
+               };
     
     bless($self, $class);
 
@@ -174,8 +179,45 @@ sub setdebug {
 
 sub ua {
     my $self = shift;
+    my $ua;
+
+    $ua = LWP::UserAgent->new('agent' => 'Mozilla/5.0', 'cookie_jar' => HTTP::Cookies->new);
+
+    # Remove a currently defined HTTPS proxy. See below for a longer explanation.
+    delete($ENV{'HTTPS_PROXY'});
+
+    if (defined($self->{'_CONNECTOR'})) {
+        my $schemas = $self->{'_CONNECTOR'}->{-schemas};
+        foreach (keys(%{$schemas})) {
+            $self->debug("Adding schema %s with proxy %s", $_, $schemas->{$_});
+            if ($_ eq 'https') {
+                # OK, so here's the gist.
+                #
+                # The usual way of reqesting an HTTPS URL through a proxy is
+                # to connect to the proxy server, issue a CONNECT request to 
+                # create a channel to the web server and start an SSL session over
+                # this channel, so there is an end-to-end connection between
+                # the client and the server.
+                #
+                # Setting a proxy for the https schema in LWP WILL NOT ACCOMPLISH
+                # THIS.
+                #
+                # LWP will connect to the proxy server, and issue a standard GET
+                # request for the target URL, which most proxy servers will refuse
+                # to get.
+                #
+                # The way to use a proxy server is to set some environment variables
+                # and let the underlying Crypt::SSLeay module do the rest.
+                #
+                # This is positively appaling.
+                $ENV{'HTTPS_PROXY'} = $schemas->{$_};
+            } else {
+                $ua->proxy($_, $schemas->{$_});
+            }
+        }
+    }
 
-    return LWP::UserAgent->new('agent' => 'Mozilla/5.0', 'cookie_jar' => HTTP::Cookies->new);
+    return $ua;
 }
 
 sub decode_hexurl {
@@ -187,4 +229,28 @@ sub decode_hexurl {
     return $d;
 }
 
+sub decode_querystring {
+    my $self = shift;
+
+    return { map { split /=/, $_, 2; } split /&/, shift };
+}
+
+sub connectors {
+    my $self = shift;
+    
+    return $self->{'_CONNECTORS'}->();
+}
+
+sub selectconn {
+    my $self = shift;
+
+    $self->{'_CONNECTOR'} = shift;
+}
+
+sub setconn {
+    my $self = shift;
+
+    $self->{'_CONNECTORS'} = shift;
+}
+
 1;

diff --git a/videosite/GrabberBase.pm b/videosite/GrabberBase.pm
index b0bf480..8ff32b4 100644 (file)
--- a/videosite/GrabberBase.pm
+++ b/videosite/GrabberBase.pm
@@ -23,6 +23,7 @@ sub get($$) {
     my $self = shift;
     my $url = shift;
     my $pattern;
+    my $res;
 
     return undef unless $self->_getval('enabled');
 
@@ -30,7 +31,16 @@ sub get($$) {
         $self->debug("Matching %s against %s", $pattern, $url);
         if ($url =~ m|$pattern|) {
             $self->debug("Match");
-            return wantarray?($self->_parse($url, $pattern), $pattern):$self->_parse($url, $pattern);
+            foreach ($self->connectors()) {
+                $self->debug("Using connector %s", $_->{-name});
+                $self->selectconn($_);
+                $res = $self->_parse($url, $pattern);
+                if (defined($res)) {
+                    $res->{'CONNECTOR'} = $_;
+                    last;
+                }
+            }
+            return wantarray?($res, $pattern):$res;
         }
     }
 

diff --git a/videosite/YouTubeGrabber.pm b/videosite/YouTubeGrabber.pm
index ede2dcc..1a331d1 100644 (file)
--- a/videosite/YouTubeGrabber.pm
+++ b/videosite/YouTubeGrabber.pm
@@ -59,7 +59,8 @@ sub new {
                     'h264' => 'high resolution MPEG4 video',
                     'hd' => 'HD720 resolution'}],
             'USERNAME' => ['', 'Username to use for YouTube login'],
-            'PASSWORD' => ['', 'Password to use for YouTube login']};
+            'PASSWORD' => ['', 'Password to use for YouTube login'],
+            'HTTPS' => [1, 'Whether to use HTTPS (if available) to connect to YouTube']};
 
     bless($self, $class);
     $self->_prepare_parameters();
@@ -114,8 +115,8 @@ sub _parse_by_video_info {
     $preflist = $preflist{$quality};
     $self->debug("Quality: %s, preflist: [%s]", $quality, join(", ", @{$preflist}));
 
-    $videourl = sprintf('https://www.youtube.com/get_video_info?video_id=%s&eurl=%s',
-            $id, 'http%3A%2F%2Fwww%2Eyoutube%2Ecom%2F');
+    $videourl = sprintf('%s://www.youtube.com/get_video_info?video_id=%s&eurl=%s',
+            $self->_getval('HTTPS')?'https':'http', $id, 'http%3A%2F%2Fwww%2Eyoutube%2Ecom%2F');
     $self->debug("Video info URL: %s", $videourl);
 
     $r = $ua->get($videourl);
@@ -128,7 +129,7 @@ sub _parse_by_video_info {
     $self->debug('Content from get_video_info: %s', $content);
 
     # Decode content
-    $content = { split /[&=]/, $content };
+    $content = $self->decode_querystring($content);
 
     if ($content->{'status'} ne 'ok') {
         $self->debug("Non OK status code found: %s", $content->{'status'});
@@ -196,7 +197,7 @@ sub _parse_by_scrape {
     $preflist = $preflist{$quality};
     $self->debug("Quality: %s, preflist: [%s]", $quality, join(", ", @{$preflist}));
 
-    $videourl = sprintf('https://www.youtube.com/watch?v=%s', $id);
+    $videourl = sprintf('%s://www.youtube.com/watch?v=%s', $self->_getval('HTTPS')?'https':'http', $id);
 
     unless(defined($r = $ua->get($videourl))) {
         $self->error('Could not download %s', $url);
@@ -476,7 +477,7 @@ sub _decode_url_encoded_fmt_stream_map {
     #
     # @data will be an array of hash references
     
-    @data = map { { map { $self->decode_hexurl($_) } split /[=&]/ } } split /,/, $data;
+    @data = map { { map { $self->decode_hexurl($_) } split /[&=]/  } } split /,/, $data;
     $self->debug("_decode_url_encoded_fmt_stream_map() decoded %s", Dumper(\@data));
 
     # From each array entry, pick the itag and the url values and return that