Youtube: Add a parser that uses /get_video_info, with a fallback to the old method

diff --git a/videosite/YouTubeGrabber.pm b/videosite/YouTubeGrabber.pm
index 609b673..dc830c6 100644 (file)
--- a/videosite/YouTubeGrabber.pm
+++ b/videosite/YouTubeGrabber.pm
@@ -11,8 +11,6 @@ package videosite::YouTubeGrabber;
 use videosite::GrabberBase;
 @ISA = qw(videosite::GrabberBase);
 
-use LWP::UserAgent;
-use HTTP::Cookies;
 use HTML::TokeParser;
 use HTML::Entities qw(decode_entities);
 use Encode;
@@ -21,6 +19,22 @@ use videosite::JSArrayParser;
 
 use strict;
 
+my %preflist = (
+    'insane' => [38, 37, 22, 35, 18, 34, 6, 5],
+    'hd' => [37, 22, 35, 18, 34, 6, 5, 38],
+    'h264' => [18, 34, 37, 22, 35, 6, 5, 38],
+    'high' => [34, 35, 18, 37, 22, 6, 5, 38],
+    'normal' => [6, 5, 34, 35, 18, 22, 37, 38]);
+my %videoformats = (
+    38 => 'mp4,h264,4k',
+    37 => 'mp4,h264,1080p',
+    35 => 'flv,h264,large',
+    34 => 'flv,h264',
+    22 => 'mp4,h264,720p',
+    18 => 'mp4,h264',
+    5 => 'flv,flv',
+    );
+
 sub new {
     my $class = shift;
     my $self = $class->SUPER::new();
@@ -53,51 +67,141 @@ sub _parse {
     my $self = shift;
     my $url = shift;
     my $pattern = shift;
+    my $id;
+    my $res;
+
+    my $url =~ m|$pattern|;
+    my $url = $1;
+    my $id = $2;
+
+    $self->debug("Matched id %s from pattern %s", $id, $pattern);
+
+    $res = $self->_parse_by_video_info($url, $id);
+    if (defined($res) && ref($res)) {
+        return $res;
+    } else {
+        $res = $self->_parse_by_scrape($url, $id);
+    }
+
+    return $res;
+}
+
+sub _parse_by_video_info {
+    my $self = shift;
+    my $url = shift;
+    my $id = shift;
+    my $quality = $self->_getval('QUALITY');
+    my $metadata;
+    my $videourl;
+    my $ua = $self->{_ua};
+    my $preflist;
+    my $r;
+    my $content;
+    my $urls;
+
+    $metadata->{'URL'} = $url;
+    $metadata->{'ID'} = $id;
+    $metadata->{'TYPE'} = 'video';
+    $metadata->{'SOURCE'} = $self->{'NAME'};
+    $metadata->{'TITLE'} = undef;
+    $metadata->{'DLURL'} = undef;
+    $metadata->{'COOKIE'} = undef;
+
+    $preflist = $preflist{$quality};
+    $self->debug("Quality: %s, preflist: [%s]", $quality, join(", ", @{$preflist}));
+
+    $videourl = sprintf('https://www.youtube.com/get_video_info?video_id=%s&eurl=%s',
+            $id, 'http%3A%2F%2Fwww%2Eyoutube%2Ecom%2F');
+    $self->debug("Video info URL: %s", $videourl);
+
+    $r = $ua->get($videourl);
+    unless($r->is_success()) {
+        $self->debug('Could not download %s: %s', $videourl, $r->code());
+        return undef;
+    }
+
+    $content = $r->content();
+    $self->debug('Content from get_video_info: %s', $content);
+
+    # Decode content
+    $content = { split /[&=]/, $content };
+
+    if ($content->{'status'} ne 'ok') {
+        $self->debug("Non OK status code found: %s", $content->{'status'});
+        return undef;
+    }
+
+    unless(exists($content->{'fmt_url_map'}) and exists($content->{'title'})) {
+        $self->debug("No fmt_url_map or no title found");
+        return undef;
+    }
+
+    # Decode fmt_url_map
+    $urls = $content->{'fmt_url_map'};
+    $urls =~ s/%(..)/chr(hex($1))/ge;
+    $urls = { split /[\|,]/, $urls };
+
+    foreach (keys(%{$urls})) {
+        if (exists($videoformats{$_})) {
+            $self->debug('Found URL for format %s (%s): %s', $_, $videoformats{$_}, $urls->{$_});
+        } else {
+            $self->error('Unknown format %s: %s', $_, $urls->{$_});
+        }
+    }
+
+    foreach (@{$preflist}) {
+        if (exists($urls->{$_})) {
+            $self->debug("Selected URL with quality level %s", $_);
+            $metadata->{'DLURL'} = $urls->{$_};
+            last;
+        }
+    }
+
+    $self->debug('URL found: %s', $metadata->{'DLURL'});
+
+    $metadata->{'TITLE'} = $content->{'title'};
+    $metadata->{'TITLE'} =~ s/\+/ /g;
+    $metadata->{'TITLE'} =~ s/%(..)/chr(hex($1))/ge;
+
+    $self->debug('Title found: %s', $metadata->{'TITLE'});
+
+    unless(defined($metadata->{'DLURL'}) && defined($metadata->{'TITLE'})) {
+        $self->error('Could not determine download URL');
+        return undef;
+    }
+
+    return $metadata;
+}
+
+sub _parse_by_scrape {
+    my $self = shift;
+    my $url = shift;
+    my $id = shift;
     my $content;
     my $metadata = {};
     my $p;
     my $e;
     my $tag;
-    my $jar = HTTP::Cookies->new;
-    my $ua = LWP::UserAgent->new('agent' => 'Mozilla/5.0');
+    my $ua = $self->{_ua};
     my $r;
     my $videourl;
     my $quality = $self->_getval('QUALITY');
-    my %preflist = (
-        'insane' => [38, 37, 22, 35, 18, 34, 6, 5],
-        'hd' => [37, 22, 35, 18, 34, 6, 5, 38],
-        'h264' => [18, 34, 37, 22, 35, 6, 5, 38],
-        'high' => [34, 35, 18, 37, 22, 6, 5, 38],
-        'normal' => [6, 5, 34, 35, 18, 22, 37, 38]);
-    my %videoformats = (
-        38 => 'mp4,h264,4k',
-        37 => 'mp4,h264,1080p',
-        35 => 'flv,h264,large',
-        34 => 'flv,h264',
-        22 => 'mp4,h264,720p',
-        18 => 'mp4,h264',
-        5 => 'flv,flv',
-        );
     my $preflist;
     my $jsp;
 
-    $url =~ m|$pattern|;
-    $url = $1;
-
     $metadata->{'URL'} = $url;
-    $metadata->{'ID'} = $2;
+    $metadata->{'ID'} = $id;
     $metadata->{'TYPE'} = 'video';
     $metadata->{'SOURCE'} = $self->{'NAME'};
     $metadata->{'TITLE'} = undef;
     $metadata->{'DLURL'} = undef;
     $metadata->{'COOKIE'} = undef;
 
-    $self->debug("Matched id %s from pattern %s", $2, $pattern);
 
     $preflist = $preflist{$quality};
     $self->debug("Quality: %s, preflist: [%s]", $quality, join(", ", @{$preflist}));
 
-    $videourl = sprintf('https://www.youtube.com/watch?v=%s', $2);
+    $videourl = sprintf('https://www.youtube.com/watch?v=%s', $id);
 
     unless(defined($r = $ua->get($videourl))) {
         $self->error('Could not download %s', $url);
@@ -106,7 +210,6 @@ sub _parse {
 
     if ($r->base->as_string() =~ m,/verify_age,) {
         $self->debug('Video requires age verification');
-        $ua->cookie_jar($jar);
         my @logindata = $self->__login($videourl, $ua);
         $r = $logindata[0];
         $metadata->{'COOKIE'} = $logindata[1];