Merge branch 'master' of ssh://ryoko:22003/home/sun/GIT/videosite

diff --git a/videosite-dl.pl b/videosite-dl.pl
index 257da2d..1b0079c 100755 (executable)
--- a/videosite-dl.pl
+++ b/videosite-dl.pl
@@ -49,14 +49,14 @@ sub ploader {
 }
 
 sub connectors {
-    my $c = {-name => 'environment', -schemas => {}};
+    my $c = {name => 'environment', schemas => {}};
 
     if (exists($ENV{'http_proxy'})) {
-        $c->{-schemas}->{'http'} = $ENV{'http_proxy'}
+        $c->{schemas}->{'http'} = $ENV{'http_proxy'}
     }
 
     if (exists($ENV{'https_proxy'})) {
-        $c->{-schemas}->{'https'} = $ENV{'https_proxy'}
+        $c->{schemas}->{'https'} = $ENV{'https_proxy'}
     }
 
     return ( $c );
@@ -90,10 +90,10 @@ unless(@g and defined($f)) {
 
 foreach (@g, $f) {
     $_->setio(sub { printf(@_); print("\n"); } );
+    $_->setconn(\&connectors);
 
     if ($debug) {
         $_->setdebug(1);
-        $_->setconn(\&connectors);
     }
 }
 

diff --git a/videosite-test.pl b/videosite-test.pl
index a0b71b9..34b6f52 100755 (executable)
--- a/videosite-test.pl
+++ b/videosite-test.pl
@@ -49,14 +49,14 @@ sub ploader {
 }
 
 sub connectors {
-    my $c = {-name => 'environment', -schemas => {}};
+    my $c = {name => 'environment', schemas => {}};
 
     if (exists($ENV{'http_proxy'})) {
-        $c->{-schemas}->{'http'} = $ENV{'http_proxy'}
+        $c->{schemas}->{'http'} = $ENV{'http_proxy'}
     }
 
     if (exists($ENV{'https_proxy'})) {
-        $c->{-schemas}->{'https'} = $ENV{'https_proxy'}
+        $c->{schemas}->{'https'} = $ENV{'https_proxy'}
     }
 
     return ( $c );
@@ -97,6 +97,8 @@ foreach (@g, $f) {
     }
 }
 
+select(STDOUT);
+$| = 1;
 printf("Doing self tests:\n");
 foreach(@g) {
     my $r;

diff --git a/videosite/VeohGrabber.pm b/videosite/VeohGrabber.pm
index 4cac1ea..dda2b5d 100644 (file)
--- a/videosite/VeohGrabber.pm
+++ b/videosite/VeohGrabber.pm
@@ -8,9 +8,8 @@ package videosite::VeohGrabber;
 use videosite::GrabberBase;
 @ISA = qw(videosite::GrabberBase);
 
-use XML::Simple;
+use HTML::TokeParser;
 use Data::Dumper;
-
 use strict;
 
 sub new {
@@ -18,7 +17,9 @@ sub new {
     my $self = $class->SUPER::new();
 
     $self->{'NAME'} = 'veoh';
-    $self->{'PATTERNS'} = ['(http://(?:[-a-zA-Z0-9_.]+\.)*veoh.com/browse/videos/category/[^/]+/watch/([^/]+))'];
+    $self->{_SELFTESTURL} = 'http://www.veoh.com/watch/v18348952fyn2twbe';
+    $self->{_SELFTESTTITLE} = '518_2 kureyon shinchan';
+    $self->{'PATTERNS'} = ['(http://(?:[-a-zA-Z0-9_.]+\.)*veoh.com/+watch/(\w+)\??)'];
 
     bless($self, $class);
     $self->_prepare_parameters();
@@ -32,9 +33,11 @@ sub _parse {
     my $pattern = shift;
     my $content;
     my $metadata = {};
-    my $p = XML::Simple->new();
     my $ua = $self->ua();
+    my $p;
     my $t;
+    my @text;
+    my @accum;
     my $dlurl;
     my $r;
 
@@ -48,41 +51,40 @@ sub _parse {
     $metadata->{'TITLE'} = undef;
     $metadata->{'DLURL'} = undef;
 
-    # Get the XML file containing the video metadata
-    unless(defined($content = $self->simple_get(sprintf('http://www.veoh.com/rest/v2/execute.xml?apiKey=5697781E-1C60-663B-FFD8-9B49D2B56D36&method=veoh.search.search&type=video&maxResults=1&permalink=%s&contentRatingId=1&', $2), $ua))) {
-        $self->error('Could not download XML metadata');
+    unless(defined($content = $self->simple_get(sprintf('http://www.veoh.com/watch/%s', $2), $ua))) {
+        $self->error('Could not download');
         return undef;
     }
 
-    unless(defined($t = $p->XMLin($content, KeepRoot => 1))) {
-        $self->error('Could not parse XML metadata');
-        return undef;
-    }
+    $p = HTML::TokeParser->new(\$content);
 
-    if (exists($t->{'rsp'}->{'videoList'}->{'video'}->{'fullPreviewHashPath'})) {
-        $dlurl = $t->{'rsp'}->{'videoList'}->{'video'}->{'fullPreviewHashPath'};
-    } else {
-        $dlurl = $t->{'rsp'}->{'videoList'}->{'video'}->{'fullPreviewHashLowPath'}
-    }
+    while ($t = $p->get_tag('script')) {
+        if ($t->[0] eq 'script') {
+            my $e = $p->get_text();
+            my $jsp;
 
-    unless(defined($dlurl)) {
-        $self->error('No dlurl found in XML');
-        return undef;
-    }
+            if ($e =~ m|__watch.videoDetailsJSON = '([^\x27]+)|) {
+                $self->debug("Found JSON: %s", $1);
+                $jsp = videosite::JSArrayParser->new();
+                $r = $jsp->parse($1);
 
-    # We now have to fetch the dlurl to get the redirect target after it,
-    # because the dlurl itself must be called with the right referer set
+                unless(defined($r)) {
+                    $self->error("Found information hash, but could not parse");
+                    return undef;
+                }
 
-    $ua->max_redirect(0);
-    $r = $ua->get($dlurl, 'referer' => 'http://www.veoh.com');
+                $self->debug("Parsed JSON: %s", Dumper($r));
 
-    unless ($r->is_redirect) {
-        $self->error('Expected redirect, got %s', $r->code);
-        return undef;
-    }
+                unless(exists($r->{previewUrl})) {
+                    $self->error("previewUrl not found in information hash");
+                    return undef;
+                }
 
-    $metadata->{'DLURL'} = $r->header('Location');
-    $metadata->{'TITLE'} = $t->{'rsp'}->{'videoList'}->{'video'}->{'title'};
+                $metadata->{'DLURL'} = $r->{previewUrl};
+                $metadata->{'TITLE'} = $r->{title};
+            }
+        }
+    }
 
     unless(defined($metadata->{'DLURL'}) && defined($metadata->{'TITLE'})) {
         $self->error('Could not extract download URL and title');

diff --git a/videosite/VimeoGrabber.pm b/videosite/VimeoGrabber.pm
index 89d3158..6b1dd60 100644 (file)
--- a/videosite/VimeoGrabber.pm
+++ b/videosite/VimeoGrabber.pm
@@ -19,6 +19,8 @@ sub new {
     my $self = $class->SUPER::new();
 
     $self->{'NAME'} = 'vimeo';
+    $self->{_SELFTESTURL} = 'http://vimeo.com/35055590';
+    $self->{_SELFTESTTITLE} = 'Hello';
     $self->{'PATTERNS'} = ['(http://(?:[-a-zA-Z0-9_.]+\.)*vimeo.com/(\d+))'];
 
     bless($self, $class);

diff --git a/videosite/YouTubeGrabber.pm b/videosite/YouTubeGrabber.pm
index b0e6964..81c6dc1 100644 (file)
--- a/videosite/YouTubeGrabber.pm
+++ b/videosite/YouTubeGrabber.pm
@@ -123,7 +123,6 @@ sub _parse_by_video_info {
     $metadata->{'SOURCE'} = $self->{'NAME'};
     $metadata->{'TITLE'} = undef;
     $metadata->{'DLURL'} = undef;
-    $metadata->{'COOKIE'} = undef;
 
     $preflist = $preflist{$quality};
     $self->debug("Quality: %s, preflist: [%s]", $quality, join(", ", @{$preflist}));
@@ -204,7 +203,6 @@ sub _parse_by_scrape {
     $metadata->{'SOURCE'} = $self->{'NAME'};
     $metadata->{'TITLE'} = undef;
     $metadata->{'DLURL'} = undef;
-    $metadata->{'COOKIE'} = undef;
 
 
     $preflist = $preflist{$quality};
@@ -221,7 +219,6 @@ sub _parse_by_scrape {
         $self->debug('Video requires age verification');
         my @logindata = $self->__login($videourl, $ua);
         $r = $logindata[0];
-        $metadata->{'COOKIE'} = $logindata[1];
         unless(defined($r)) {
             $self->error('Could not log into YouTube');
             return undef;
@@ -341,8 +338,35 @@ sub _parse_by_scrape {
                     $self->error('fmt_url_map not found in PLAYER_CONFIG');
                     return undef;
                 }
+            } elsif ($e =~ m|yt\.playerConfig\s*=\s*(.+);\n|) {
+                my $args = $1;
+                $self->debug("Found yt.playerConfig: %s", $args);
+
+                $jsp = videosite::JSArrayParser->new();
+                $self->debug("Using %s to parse", ref($jsp));
+                $r = $jsp->parse($args);
+
+                unless(defined($r)) {
+                    $self->error("Found information hash, but could not parse");
+                    return undef;
+                }
+
+                if (exists($r->{'args'}) and exists($r->{'args'}->{'ps'}) and ($r->{'args'}->{'ps'} eq 'live')) {
+                    $self->error("Video URL seems to point to a live stream, cannot save this");
+                    return undef;
+                }
+
+                if (exists($r->{'args'}) and exists($r->{'args'}->{'url_encoded_fmt_stream_map'}) and ($r->{'args'}->{'url_encoded_fmt_stream_map'} ne '')) {
+                    %urls = %{$self->_decode_url_encoded_fmt_stream_map($r->{'args'}->{'url_encoded_fmt_stream_map'}, 0)};
+
+                    $self->debug("Pagetype: 2012 (yt.playerConfig), url_encoded_fmt_stream_map");
+                } else {
+                    $self->error('url_map not found in yt.playerConfig');
+                    return undef;
+                }
             }
 
+
             if (%urls) {
                 $self->__pick_url(\%urls, $preflist, $metadata);
                 last SWF_ARGS;

diff --git a/videosite/ZeroPunctuationGrabber.pm b/videosite/ZeroPunctuationGrabber.pm
index 0ee5ffb..a9538fe 100644 (file)
--- a/videosite/ZeroPunctuationGrabber.pm
+++ b/videosite/ZeroPunctuationGrabber.pm
@@ -10,6 +10,7 @@ use videosite::GrabberBase;
 
 use HTML::TokeParser;
 use Data::Dumper;
+use videosite::JSArrayParser;
 
 use strict;
 
@@ -18,7 +19,9 @@ sub new {
     my $self = $class->SUPER::new();
 
     $self->{'NAME'} = 'zeropunctuation';
-    $self->{'PATTERNS'} = ['(http://www.escapistmagazine.com/articles/view/editorials/zeropunctuation/([-A-Za-z0-9]+))'];
+    $self->{_SELFTESTURL} = 'http://www.escapistmagazine.com/videos/view/zero-punctuation/5346-Amy';
+    $self->{_SELFTESTTITLE} = 'Amy';
+    $self->{'PATTERNS'} = ['(http://www.escapistmagazine.com/videos/view/zero-punctuation/([-A-Za-z0-9]+))'];
 
     bless($self, $class);
     $self->_prepare_parameters();
@@ -34,6 +37,9 @@ sub _parse {
     my $metadata = {};
     my $p;
     my $e;
+    my $j;
+    my $jsp;
+    my $r;
 
     $url =~ m|$pattern|;
     $url = $1;
@@ -52,16 +58,33 @@ sub _parse {
 
     $p = HTML::TokeParser->new(\$content);
 
-    # Look for the title
-    if ($p->get_tag('title')) {
-        $metadata->{'TITLE'} = $p->get_text();
-        $metadata->{'TITLE'} =~ s/^The Escapist : Zero Punctuation: (.*)$/$1/im;
+    while ($e = $p->get_tag('div', 'param')) {
+        if (($e->[0] eq 'param') and exists($e->[1]->{name}) and ($e->[1]->{name} eq 'flashvars')) {
+            my %r = map { $self->decode_hexurl($_) } split(/[&=]/, $e->[1]->{value});
+
+            unless(exists($r{config})) {
+                $self->error("config URL not found in flashvars");
+                return undef;
+            }
+
+            $j = $r{config};
+        } elsif (($e->[0] eq 'div') and exists($e->[1]->{itemprop}) and ($e->[1]->{itemprop} eq 'name')) {
+            $metadata->{'TITLE'} = $p->get_phrase();
+        }
     }
 
-    while ($e = $p->get_tag('script')) {
-        my $c = $p->get_text();
-        if ($c =~ m|var vars = \{file:\x27([^\x27]+)\x27|) {
-            $metadata->{'DLURL'} = 'http://' . $1 . '_high.flv';
+    unless(defined($content = $self->simple_get($j))) {
+        $self->error("Could not download %s", $j);
+        return undef;
+    }
+
+    $jsp = videosite::JSArrayParser->new();
+    $r = $jsp->parse($content);
+
+    foreach (@{$r->{playlist}}) {
+        if ($_->{eventCategory} eq 'Video') {
+            $metadata->{'DLURL'} = $_->{url};
+            last;
         }
     }