Add a new JSON parser, replacing jsarray. This parser

[videosite.git] / videosite / YouTubeGrabber.pm

diff --git a/videosite/YouTubeGrabber.pm b/videosite/YouTubeGrabber.pm
index 83cee59..5d328de 100644 (file)
--- a/videosite/YouTubeGrabber.pm
+++ b/videosite/YouTubeGrabber.pm
@@ -14,6 +14,8 @@ use videosite::GrabberBase;
 use LWP::UserAgent;
 use HTTP::Cookies;
 use HTML::TokeParser;
+use HTML::Entities qw(decode_entities);
+use Encode;
 use Data::Dumper;
 use videosite::JSArrayParser;
 
@@ -24,8 +26,11 @@ sub new {
     my $self = $class->SUPER::new();
 
     $self->{'NAME'} = 'youtube';
-    $self->{'PATTERNS'} = ['(http://(?:[-a-zA-Z0-9_.]+\.)*youtube.(?:com|de|co.uk)/watch\?(?:.+=.+&)*v=([-a-zA-Z0-9_]+))',
-                           '(http://(?:[-a-zA-Z0-9_.]+\.)*youtube.(?:com|de|co.uk)/v/([-a-zA-Z0-9_]+))'];
+    $self->{'PATTERNS'} = ['(http://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/watch\?(?:.+=.+&)*v=([-a-zA-Z0-9_]+))',
+                           '(http://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/watch\#\!v=([-a-zA-Z0-9_]+))',
+                           '(http://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/v/([-a-zA-Z0-9_]+))',
+                           '(http://(?:[-a-zA-Z0-9_.]+\.)*youtu\.be/([-a-zA-Z0-9_]+))',
+                           '(http://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/user/\w+\?.*/([-a-zA-Z0-9_]+))'];
     $self->{'_PARAMS'} = {
             'QUALITY' => ['normal', 'Quality of the video to download.', {
                     'normal' => 'standard resolution flash video',
@@ -73,6 +78,8 @@ sub _parse {
     $metadata->{'TITLE'} = undef;
     $metadata->{'DLURL'} = undef;
 
+    $self->debug("Matched id %s from pattern %s", $2, $pattern);
+
     $preflist = $preflist{$quality};
     $self->debug("Quality: %s, preflist: [%s]", $quality, join(", ", @{$preflist}));
 
@@ -100,13 +107,20 @@ sub _parse {
         if ('meta' eq $tag->[0]) {
             if ('title' eq $tag->[1]->{'name'}) {
                 $metadata->{'TITLE'} = $tag->[1]->{'content'};
+                # Convert HTML entities in the title. This is a bit convoluted.
+                $metadata->{'TITLE'} = encode("utf8",
+                                         decode_entities(
+                                           decode("utf8", $metadata->{'TITLE'})));
+                    
                 $self->debug('Title found: %s', $metadata->{'TITLE'});
             }
         } elsif ('script' eq $tag->[0]) {
+            my %urls;
+
             $e = $p->get_text();
             $self->debug("Found script: %s", $e);
+
             if ($e =~ m|\x27SWF_ARGS\x27:\s+(.+),|) {
-                my %urls;
                 my $args = $1;
 
                 $self->debug("Found SWF_ARGS: %s", $args);
@@ -126,6 +140,8 @@ sub _parse {
 
                     $urls =~ s/%([[:xdigit:]]{2})/chr(hex($1))/ge;
                     %urls = split(/[\|,]/, $urls);
+                    $self->debug("Pagetype: old (SWF_ARGS), fmt_url_map");
+
                 } elsif (exists($r->{'t'}) and ($r->{'t'} ne '')) {
                     my $thash = $r->{'t'};
 
@@ -143,17 +159,28 @@ sub _parse {
                                 $_[0],
                                 $thash);
                         }
+                        $self->debug("Pagetype: 2009 (SWF_ARGS), t with fmt_map");
+
                     } else {
                         $urls{5} = sprintf('http://www.youtube.com/get_video?video_id=%s&t=%s',
                             $metadata->{'ID'},
                             $thash);
+                        $self->debug("Pagetype: 2009 (SWF_ARGS), t without fmt_map");
                     }
                 } else {
                     $self->error('Neither fmt_url_map nor t found in video information hash');
                     return undef;
                 }
-                $self->debug("Found quality levels [%s]", join(", ", keys(%urls)));
+            } elsif ($e =~ m|var swfHTML = .*fmt_url_map=([^\&]+)\&|) {
+                my $urls = $1;
+                $self->debug("Video has fmt_url_map: %s", $urls);
+
+                $urls =~ s/%([[:xdigit:]]{2})/chr(hex($1))/ge;
+                %urls = split(/[\|,]/, $urls);
+                $self->debug("Pagetype: 2010 (swfHTML), fmt_url_map");
+            }
 
+            if (%urls) {
                 foreach (keys(%urls)) {
                     if ($_ == 35) {
                         $self->debug('Found flv,h264,large: %s', $urls{$_});
@@ -184,7 +211,7 @@ sub _parse {
                 last SWF_ARGS;
             }
         } elsif ('div' eq $tag->[0]) {
-            if (exists($tag->[1]->{'class'}) and ('errorBox' eq $tag->[1]->{'class'})) {
+            if (exists($tag->[1]->{'class'}) and ('yt-alert-content' eq $tag->[1]->{'class'})) {
                 $self->error("Could not get video data for youtube %s: %s",
                         $metadata->{'ID'}, $p->get_trimmed_text());
                 return undef;
@@ -235,6 +262,8 @@ sub __login {
         return undef;
     }
 
+    $self->debug("Got a cookie");
+
     $r = $ua->get($videourl);
     if ($r->base->as_string() =~ m,/verify_age,) {
         $self->debug("Looking for session token...");