X-Git-Url: https://git.camperquake.de/gitweb.cgi?a=blobdiff_plain;f=videosite%2FYouTubeGrabber.pm;h=199143e8b3d99e46f09035e45f182b5ce3c6baf9;hb=76a9c69c78eec62f96cacba4b5ca1ac67b030661;hp=f4bac102695f61c02fffc588f8a7486f706ae49e;hpb=bd9a8a86176b546d3336587a8f28f9c25299f5f6;p=videosite.git diff --git a/videosite/YouTubeGrabber.pm b/videosite/YouTubeGrabber.pm index f4bac10..199143e 100644 --- a/videosite/YouTubeGrabber.pm +++ b/videosite/YouTubeGrabber.pm @@ -6,15 +6,18 @@ # download strategy revised using # http://www.kde-apps.org/content/show.php?content=41456 -package YouTubeGrabber; +package videosite::YouTubeGrabber; -use GrabberBase; -@ISA = qw(GrabberBase); +use videosite::GrabberBase; +@ISA = qw(videosite::GrabberBase); use LWP::UserAgent; use HTTP::Cookies; use HTML::TokeParser; +use HTML::Entities qw(decode_entities); +use Encode; use Data::Dumper; +use videosite::JSArrayParser; use strict; @@ -25,7 +28,14 @@ sub new { $self->{'NAME'} = 'youtube'; $self->{'PATTERNS'} = ['(http://(?:[-a-zA-Z0-9_.]+\.)*youtube.(?:com|de|co.uk)/watch\?(?:.+=.+&)*v=([-a-zA-Z0-9_]+))', '(http://(?:[-a-zA-Z0-9_.]+\.)*youtube.(?:com|de|co.uk)/v/([-a-zA-Z0-9_]+))']; - $self->{'_PARAMS'} = {'QUALITY' => ['normal', 'Quality of the video to download. normal = standard resolution flash video, high = higher resolution flash video, h264 = high resolution MPEG4 video'], 'USERNAME' => ['', 'Username to use for YouTube login'], 'PASSWORD' => ['', 'Password to use for YouTube login']}; + $self->{'_PARAMS'} = { + 'QUALITY' => ['normal', 'Quality of the video to download.', { + 'normal' => 'standard resolution flash video', + 'high' => 'higher resolution flash video', + 'h264' => 'high resolution MPEG4 video', + 'hd' => 'HD720 resolution'}], + 'USERNAME' => ['', 'Username to use for YouTube login'], + 'PASSWORD' => ['', 'Password to use for YouTube login']}; bless($self, $class); $self->_prepare_parameters(); @@ -47,7 +57,13 @@ sub _parse { my $r; my $videourl; my $quality = $self->_getval('QUALITY'); - my $append = ''; + my %preflist = ( + 'hd' => [37, 22, 35, 18, 34, 6, 5], + 'h264' => [18, 34, 37, 22, 35, 6, 5], + 'high' => [34, 35, 18, 37, 22, 6, 5], + 'normal' => [6, 5, 34, 35, 18, 22, 37]); + my $preflist; + my $jsp; $url =~ m|$pattern|; $url = $1; @@ -59,13 +75,12 @@ sub _parse { $metadata->{'TITLE'} = undef; $metadata->{'DLURL'} = undef; - if ($quality eq 'high') { - $append = '&fmt=6'; - } elsif ($quality eq 'h264') { - $append = '&fmt=18'; - } + $self->debug("Matched id %s from pattern %s", $2, $pattern); + + $preflist = $preflist{$quality}; + $self->debug("Quality: %s, preflist: [%s]", $quality, join(", ", @{$preflist})); - $videourl = sprintf('http://www.youtube.com/watch?v=%s%s', $2, $append); + $videourl = sprintf('http://www.youtube.com/watch?v=%s', $2); unless(defined($r = $ua->get($videourl))) { $self->error('Could not download %s', $url); @@ -85,22 +100,100 @@ sub _parse { $p = HTML::TokeParser->new(\$content); - while ($tag = $p->get_tag('div', 'meta', 'script')) { + SWF_ARGS: while ($tag = $p->get_tag('div', 'meta', 'script')) { if ('meta' eq $tag->[0]) { if ('title' eq $tag->[1]->{'name'}) { $metadata->{'TITLE'} = $tag->[1]->{'content'}; + # Convert HTML entities in the title. This is a bit convoluted. + $metadata->{'TITLE'} = encode("utf8", + decode_entities( + decode("utf8", $metadata->{'TITLE'}))); + $self->debug('Title found: %s', $metadata->{'TITLE'}); } } elsif ('script' eq $tag->[0]) { $e = $p->get_text(); - if ($e =~ m|/watch_fullscreen\?(.+)\x27|) { - my %args = map { split(/=/, $_, 2); } split(/&/, $1); - $metadata->{'DLURL'} = sprintf('http://www.youtube.com/get_video.php?video_id=%s&t=%s%s', - $metadata->{'ID'}, $args{'t'}, $append); + $self->debug("Found script: %s", $e); + if ($e =~ m|\x27SWF_ARGS\x27:\s+(.+),|) { + my %urls; + my $args = $1; + + $self->debug("Found SWF_ARGS: %s", $args); + $jsp = videosite::JSArrayParser->new(); + $self->debug("Using %s to parse", ref($jsp)); + $r = $jsp->parse($args); + + unless(defined($r)) { + $self->error("Found information hash, but could not parse"); + return undef; + } + + if (exists($r->{'fmt_url_map'}) and ($r->{'fmt_url_map'} ne '')) { + my $urls = $r->{'fmt_url_map'}; + + $self->debug("Video has fmt_url_map: %s", $urls); + + $urls =~ s/%([[:xdigit:]]{2})/chr(hex($1))/ge; + %urls = split(/[\|,]/, $urls); + } elsif (exists($r->{'t'}) and ($r->{'t'} ne '')) { + my $thash = $r->{'t'}; + + if (exists($r->{'fmt_map'}) && ($r->{'fmt_map'} ne '')) { + my $fmt = $r->{'fmt_map'}; + my @fmt; + + $self->debug('Video has fmt_map'); + $fmt =~ s/%([[:xdigit:]]{2})/chr(hex($1))/ge; + @fmt = split(/,/, $fmt); + foreach (@fmt) { + split(/\//); + $urls{$_[0]} = sprintf('http://www.youtube.com/get_video?video_id=%s&fmt=%d&t=%s', + $metadata->{'ID'}, + $_[0], + $thash); + } + } else { + $urls{5} = sprintf('http://www.youtube.com/get_video?video_id=%s&t=%s', + $metadata->{'ID'}, + $thash); + } + } else { + $self->error('Neither fmt_url_map nor t found in video information hash'); + return undef; + } + $self->debug("Found quality levels [%s]", join(", ", keys(%urls))); + + foreach (keys(%urls)) { + if ($_ == 35) { + $self->debug('Found flv,h264,large: %s', $urls{$_}); + } elsif ($_ == 34) { + $self->debug('Found flv,h264: %s', $urls{$_}); + } elsif ($_ == 22) { + $self->debug('Found mp4,h264,720p: %s', $urls{$_}); + } elsif ($_ == 37) { + $self->debug('Found mp4,h264,1080p: %s', $urls{$_}); + } elsif ($_ == 18) { + $self->debug('Found mp4,h264: %s', $urls{$_}); + } elsif ($_ == 5) { + $self->debug('Found flv,flv: %s', $urls{$_}); + } else { + $self->error('Unknown tag %s: %s', $_, $urls{$_}); + } + } + + foreach (@{$preflist}) { + if (exists($urls{$_})) { + $self->debug("Selected URL with quality level %s", $_); + $metadata->{'DLURL'} = $urls{$_}; + last; + } + } + $self->debug('URL found: %s', $metadata->{'DLURL'}); + last SWF_ARGS; } } elsif ('div' eq $tag->[0]) { - if (exists($tag->[1]->{'class'}) and ('errorBox' eq $tag->[1]->{'class'})) { + if (exists($tag->[1]->{'class'}) and ('yt-alert-content' eq $tag->[1]->{'class'})) { $self->error("Could not get video data for youtube %s: %s", $metadata->{'ID'}, $p->get_trimmed_text()); return undef;