X-Git-Url: https://git.camperquake.de/gitweb.cgi?a=blobdiff_plain;f=videosite%2FYouTubeGrabber.pm;h=ad03f6a6366f9860de6377502a5b5abd98808fd1;hb=58d2597498d4162e2a3811978f59be7bedf2f9b6;hp=5930c347afbbd653c9c1ef9d68a309464e2b2492;hpb=671cb9fa217ef151045e7191f175b1437c64f45d;p=videosite.git diff --git a/videosite/YouTubeGrabber.pm b/videosite/YouTubeGrabber.pm index 5930c34..ad03f6a 100644 --- a/videosite/YouTubeGrabber.pm +++ b/videosite/YouTubeGrabber.pm @@ -1,77 +1,383 @@ +# (c) 2007 by Ralf Ertzinger +# licensed under GNU GPL v2 +# +# Grabber for youtube.com/de/... # -# based on trigger.pl by Wouter Coekaerts # download strategy revised using # http://www.kde-apps.org/content/show.php?content=41456 -package YouTubeGrabber; +package videosite::YouTubeGrabber; -use GrabberBase; -@ISA = qw(GrabberBase); +use videosite::GrabberBase; +@ISA = qw(videosite::GrabberBase); -use LWP::Simple qw(!get); -use HTML::Parser; +use HTML::TokeParser; +use HTML::Entities qw(decode_entities); +use Encode; use Data::Dumper; +use videosite::JSArrayParser; use strict; +my %preflist = ( + 'insane' => [38, 37, 137, 22, 136, 35, 135, 18, 134, 34, 6, 5, 43], + 'hd' => [37, 137, 22, 136, 35, 135, 18, 134, 34, 6, 5, 38, 43], + 'h264' => [18, 134, 34, 37, 137, 22, 136, 35, 135, 6, 5, 38, 43], + 'high' => [34, 35, 135, 18, 134, 37, 137, 22, 136, 6, 5, 38, 43], + 'normal' => [6, 5, 34, 35, 135, 18, 134, 22, 136, 37, 137, 38, 43]); +my %audiopreflist = ( + 'insane' => [172,141,171,140,139], + 'hd' => [172,141,171,140,139], + 'h264' => [172,141,171,140,139], + 'high' => [171,140,172,141,139], + 'normal' => [171,140,172,141,139]); +my %videoformats = ( + # Container/Video codec/Audio codec/Resolution + 5 => 'FLV/Sorenson/MP3/240p', + 6 => 'FLV/Sorenson/MP3/270p', + 13 => '3GP/MPEG4-Visual/144p', # 0.5MBit + 17 => '3GP/MPEG4-Visual/144p', # 2MBit + 18 => 'MP4/H264/AAC/360p', # isommp42, Baseline + 22 => 'MP4/H264/AAC/720p', # isommp42, High + 34 => 'FLV/H264/AAC/360p', # Main + 35 => 'FLV/H264/AAC/480p', # Main + 36 => '3GP/MPEG4-Visual/240p', + 37 => 'MP4/H264/AAC/1080p', # High + 38 => 'MP4/H264/AAC/3072p', # High + 43 => 'WebM/VP8/Vorbis/360p', + 44 => 'WebM/VP8/Vorbis/480p', + 45 => 'WebM/VP8/Vorbis/720p', + 46 => 'WebM/VP8/Vorbis/1080p/3D', # effective 540p + 82 => 'MP4/H264/AAC/360p/3D', # isomavc1mp42 + 83 => 'MP4/H264/AAC/240p/3D', # isomavc1mp42 + 84 => 'MP4/H264/AAC/720p/3D', # isomavc1mp42 + 85 => 'MP4/H264/AAC/1080p/3D', # isomavc1mp42, effective 540p + 100 => 'WebM/VP8/Vorbis/360p/3D', + 101 => 'WebM/VP8/Vorbis/480p/3D', + 102 => 'WebM/VP8/Vorbis/720p/3D', + 133 => 'MP4/H264/240p/AdaptiveVideo', + 134 => 'MP4/H264/360p/AdaptiveVideo', + 135 => 'MP4/H264/480p/AdaptiveVideo', + 136 => 'MP4/H264/720p/AdaptiveVideo', + 137 => 'MP4/H264/1080p/AdaptiveVideo', + 139 => 'MP4/AAC/48kbit/AdaptiveAudio', + 140 => 'MP4/AAC/128kbit/AdaptiveAudio', + 141 => 'MP4/AAC/256kbit/AdaptiveAudio', + 160 => 'MP4/H264/144p/AdaptiveVideo', + 171 => 'WebM/Vorbis/128kbit/AdaptiveAudio', + 172 => 'WebM/Vorbis/160kbit/AdaptiveAudio', + ); + sub new { my $class = shift; - my $self = $class->SUPER::new(); - - $self->{'NAME'} = 'youtube'; - $self->{'PATTERNS'} = ['(http://(?:[-a-zA-Z0-9_.]+\.)*youtube.(?:com|de|co.uk)/watch\?(?:.+=.+&)*v=([-a-zA-Z0-9_]+))', - '(http://(?:[-a-zA-Z0-9_.]+\.)*youtube.(?:com|de|co.uk)/v/([-a-zA-Z0-9_]+))']; + my $self = $class->SUPER::new( + NAME => 'youtube', + _SELFTESTURL => 'http://www.youtube.com/watch?v=dMH0bHeiRNg', + _SELFTESTTITLE => 'Evolution of Dance - By Judson Laipply', + PATTERNS => ['(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/watch(?:_popup)?/?\?.*?v=([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/watch\#\!v=([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/v/([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/embed/([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/user/[[:alnum:]]+\?v=([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/(?:user/)?[[:alnum:]]+#p/(?:\w+/)+\d+/([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtu\.be/watch\?v=([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtu\.be/([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/user/\w+\?.*/([-a-zA-Z0-9_]+))'], + _PARAMS => { + QUALITY => ['normal', 'Quality of the video to download.', { + normal => 'standard resolution flash video', + high => 'higher resolution flash video', + h264 => 'high resolution MPEG4 video', + hd => 'HD720 resolution'}], + USERNAME => ['', 'Username to use for YouTube login'], + PASSWORD => ['', 'Password to use for YouTube login'], + HTTPS => [1, 'Whether to use HTTPS (if available) to connect to YouTube'], + ADAPTIVE => [0, 'Whether to use adaptive rate URLs (if available)'], + }, + @_, + ); - bless($self, $class); - $self->_prepare_parameters(); - - return $self; + return bless($self, $class); } sub _parse { my $self = shift; my $url = shift; my $pattern = shift; - my $content; - my $metadata = {}; - my $p = HTML::Parser->new(api_version => 3); - my @accum; - my @text; - my $e; + my $id; + my $res; $url =~ m|$pattern|; $url = $1; + $id = $2; + + $self->debug("Matched id %s from pattern %s", $id, $pattern); + + $res = $self->_parse_by_video_info($url, $id); + if (defined($res)) { + if (ref($res)) { + return $res; + } else { + $self->debug("_parse_by_video_info failed with status noretry"); + return undef; + } + } else { + $self->debug("_parse_by_video_info failed with status retry"); + $res = $self->_parse_by_scrape($url, $id); + } + + return $res; +} + +# +# Try to get video information by using the API call. +# +# Returns hashref on success +# Returns undef on retryable error (try to scrape the website) +# Returns 0 on non-retryable error +# +sub _parse_by_video_info { + my $self = shift; + my $url = shift; + my $id = shift; + my $quality = $self->_getval('QUALITY'); + my $metadata; + my $videourl; + my $ua = $self->ua(); + my $preflist; + my $audiopreflist; + my $r; + my $content; + my $urls; $metadata->{'URL'} = $url; - $metadata->{'ID'} = $2; - $metadata->{'TYPE'} = 'youtube'; + $metadata->{'ID'} = $id; + $metadata->{'TYPE'} = 'video'; + $metadata->{'SOURCE'} = $self->{'NAME'}; $metadata->{'TITLE'} = undef; $metadata->{'DLURL'} = undef; - unless(defined($content = LWP::Simple::get(sprintf('http://youtube.com/watch?v=%s', $2)))) { - $self->error('Could not download %s', $url); + $preflist = $preflist{$quality}; + $audiopreflist = $audiopreflist{$quality}; + $self->debug("Quality: %s, preflist: [%s], audiopreflist: [%s]", + $quality, + join(", ", @{$preflist}), + join(", ", @{$audiopreflist})); + + $videourl = sprintf('%s://www.youtube.com/get_video_info?video_id=%s&eurl=%s', + $self->_getval('HTTPS')?'https':'http', $id, 'http%3A%2F%2Fwww%2Eyoutube%2Ecom%2F'); + $self->debug("Video info URL: %s", $videourl); + + $r = $ua->get($videourl); + unless($r->is_success()) { + $self->debug('Could not download %s: %s', $videourl, $r->code()); + return undef; + } + + $content = $r->content(); + $self->debug('Content from get_video_info: %s', $content); + + # Decode content + $content = $self->decode_querystring($content); + + $self->debug("Decoded get_video_info: %s", Dumper($content)); + + if ($content->{'status'} ne 'ok') { + $self->debug("Non OK status code found: %s", $content->{'status'}); return undef; } - $p->handler(start => \@accum, "tagname, attr"); - $p->handler(text => \@text, "text"); - $p->report_tags(qw(meta script)); - $p->utf8_mode(1); - $p->parse($content); + # Check if this is live content + if (exists($content->{ps}) and ($content->{ps} eq 'live')) { + $self->error("Video URL seems to point to a live stream, cannot save this"); + return 0; + } - # Look for the title in the meta tags - foreach $e (@accum) { - if ('meta' eq $e->[0]) { - if ('title' eq $e->[1]->{'name'}) { - $metadata->{'TITLE'} = $e->[1]->{'content'}; + if (exists($content->{'fmt_url_map'})) { + # Decode fmt_url_map + $urls = $self->decode_hexurl($content->{'fmt_url_map'}); + $urls = { split /[\|,]/, $urls }; + } elsif (exists($content->{'url_encoded_fmt_stream_map'})) { + $urls = $self->_decode_url_encoded_fmt_stream_map($content->{'url_encoded_fmt_stream_map'}, 1); + } else { + $self->debug("No URL data found"); + return undef; + } + + if (exists($content->{'adaptive_fmts'})) { + $self->debug('Video has adaptive_fmts'); + if ($self->_getval('ADAPTIVE')) { + my %af = %{$self->_decode_url_encoded_fmt_stream_map($content->{'adaptive_fmts'}, 1)}; + while (my ($k, $v) = each %af) { + $urls->{$k} = $v; } + } else { + $self->debug('Use of adaptive URLs not enabled'); } } - # Look for the download URL - foreach $e (@text) { - if ($e->[0] =~ m|/watch_fullscreen\?(.*)\&fs|) { - $metadata->{'DLURL'} = 'http://www.youtube.com/get_video.php?' . $1; + unless(exists($content->{'title'})) { + $self->debug("No title found"); + return undef; + } + + $self->__pick_url($urls, $preflist, $audiopreflist, $metadata); + + $metadata->{'TITLE'} = $content->{'title'}; + $metadata->{'TITLE'} =~ s/\+/ /g; + $metadata->{'TITLE'} = $self->decode_hexurl($metadata->{'TITLE'}); + $metadata->{'TITLE'} = decode("utf8", $metadata->{'TITLE'}); + + $self->debug('Title found: %s', $metadata->{'TITLE'}); + + unless(defined($metadata->{'DLURL'}) && defined($metadata->{'TITLE'})) { + $self->error('Could not determine download URL'); + return undef; + } + + return $metadata; +} + +sub _parse_by_scrape { + my $self = shift; + my $url = shift; + my $id = shift; + my $content; + my $metadata = {}; + my $p; + my $e; + my $tag; + my $ua = $self->ua(); + my $r; + my $videourl; + my $quality = $self->_getval('QUALITY'); + my $preflist; + my $jsp; + + $metadata->{'URL'} = $url; + $metadata->{'ID'} = $id; + $metadata->{'TYPE'} = 'video'; + $metadata->{'SOURCE'} = $self->{'NAME'}; + $metadata->{'TITLE'} = undef; + $metadata->{'DLURL'} = undef; + + + $preflist = $preflist{$quality}; + $self->debug("Quality: %s, preflist: [%s]", $quality, join(", ", @{$preflist})); + + $videourl = sprintf('%s://www.youtube.com/watch?v=%s', $self->_getval('HTTPS')?'https':'http', $id); + + unless(defined($r = $ua->get($videourl))) { + $self->error('Could not download %s', $url); + return undef; + } + + if ($r->base->as_string() =~ m,/verify_age,) { + $self->debug('Video requires age verification'); + my @logindata = $self->__login($videourl, $ua); + $r = $logindata[0]; + unless(defined($r)) { + $self->error('Could not log into YouTube'); + return undef; + } + } + $content = $r->content(); + + $p = HTML::TokeParser->new(\$content); + + SWF_ARGS: while ($tag = $p->get_tag('div', 'meta', 'script')) { + if ('meta' eq $tag->[0]) { + if (exists($tag->[1]->{'name'}) and ('title' eq $tag->[1]->{'name'})) { + $metadata->{'TITLE'} = $tag->[1]->{'content'}; + # Convert HTML entities in the title. This is a bit convoluted. + $metadata->{'TITLE'} = decode_entities( + decode("utf8", $metadata->{'TITLE'})); + + $self->debug('Title found: %s', $metadata->{'TITLE'}); + } + } elsif ('script' eq $tag->[0]) { + my %urls; + + $e = $p->get_text(); + $self->debug("Found script: %s", $e); + + if ($e =~ m|ytplayer\.config\s*=\s*(.+);$|) { + my $args = $1; + $self->debug("Found PLAYER_CONFIG: %s", $args); + + $jsp = videosite::JSArrayParser->new(); + $self->debug("Using %s to parse", ref($jsp)); + $r = $jsp->parse($args); + + unless(defined($r)) { + $self->error("Found information hash, but could not parse"); + return undef; + } + + if (exists($r->{'args'}) and exists($r->{'args'}->{'ps'}) and ($r->{'args'}->{'ps'} eq 'live')) { + $self->error("Video URL seems to point to a live stream, cannot save this"); + return undef; + } + + if (exists($r->{'args'}) and exists($r->{'args'}->{'fmt_url_map'}) and ($r->{'args'}->{'fmt_url_map'} ne '')) { + my $urls = $r->{'args'}->{'fmt_url_map'}; + + $self->debug("Video has fmt_url_map: %s", $urls); + + %urls = split(/[\|,]/, $urls); + foreach (keys(%urls)) { + $urls{$_} = $self->decode_hexurl($urls{$_}); + } + $self->debug("Pagetype: 2011 (PLAYER_CONFIG), fmt_url_map"); + } elsif (exists($r->{'args'}) and exists($r->{'args'}->{'url_encoded_fmt_stream_map'}) and ($r->{'args'}->{'url_encoded_fmt_stream_map'} ne '')) { + %urls = %{$self->_decode_url_encoded_fmt_stream_map($r->{'args'}->{'url_encoded_fmt_stream_map'}, 0)}; + + $self->debug("Pagetype: 2011 (PLAYER_CONFIG), url_encoded_fmt_stream_map"); + } else { + $self->error('fmt_url_map not found in PLAYER_CONFIG'); + return undef; + } + } elsif ($e =~ m|yt\.playerConfig\s*=\s*(.+);\n|) { + my $args = $1; + $self->debug("Found yt.playerConfig: %s", $args); + + $jsp = videosite::JSArrayParser->new(); + $self->debug("Using %s to parse", ref($jsp)); + $r = $jsp->parse($args); + + unless(defined($r)) { + $self->error("Found information hash, but could not parse"); + return undef; + } + + if (exists($r->{'args'}) and exists($r->{'args'}->{'ps'}) and ($r->{'args'}->{'ps'} eq 'live')) { + $self->error("Video URL seems to point to a live stream, cannot save this"); + return undef; + } + + if (exists($r->{'args'}) and exists($r->{'args'}->{'url_encoded_fmt_stream_map'}) and ($r->{'args'}->{'url_encoded_fmt_stream_map'} ne '')) { + %urls = %{$self->_decode_url_encoded_fmt_stream_map($r->{'args'}->{'url_encoded_fmt_stream_map'}, 0)}; + + $self->debug("Pagetype: 2012 (yt.playerConfig), url_encoded_fmt_stream_map"); + } else { + $self->error('url_map not found in yt.playerConfig'); + return undef; + } + } + + + if (%urls) { + $self->__pick_url(\%urls, $preflist, undef, $metadata); + last SWF_ARGS; + } + } elsif ('div' eq $tag->[0]) { + if (exists($tag->[1]->{'id'}) and ('watch-player-unavailable-message-container' eq $tag->[1]->{'id'})) { + # Search forward to the next
+ $tag = $p->get_tag('div'); + $self->error("Could not get video data for youtube %s: %s", + $metadata->{'ID'}, $p->get_trimmed_text()); + return undef; + } } } @@ -83,4 +389,205 @@ sub _parse { return $metadata; } +sub __login { + my $self = shift; + my $videourl = shift; + my $ua = shift; + my $user = $self->_getval('USERNAME'); + my $pass = $self->_getval('PASSWORD'); + my $r; + my $p; + my $c; + my $token; + + sub check_cookie { + + my $jar = shift; + my $key = shift; + my $found = undef; + + $jar->scan(sub { $found = 1 if ( $key eq $_[1]) }); + + return $found; + } + + sub get_all_cookies { + + my $jar = shift; + my $key = shift; + my $val = ""; + $jar->scan(sub { $val .= "; " if !( $val eq "" ); $val .= "$_[1]=$_[2]" }); + + return $val; + } + + if (($user eq '') or ($pass eq '')) { + $self->error('No username or password defined for YouTube'); + return undef; + } + + $self->debug('Logging in'); + $r = $ua->get('https://www.google.com/accounts/ServiceLoginAuth?service=youtube'); + unless($r->is_success()) { + $self->debug("Could not get login page (make sure your LWP supports HTTPS!)"); + return undef; + } + $c = $r->decoded_content(); + $p = HTML::TokeParser->new(\$c); + while (my $tag = $p->get_tag('input')) { + $self->debug("%s", Dumper($tag)); + if ($tag->[1]{name} eq 'GALX') { + $token = $tag->[1]{value}; + last; + } + } + $self->debug("GALX = %s", $token); + $r = $ua->post('https://www.google.com/accounts/ServiceLoginAuth?service=youtube', { 'service' => 'youtube', 'Email' => $user, 'Passwd' => $pass, 'GALX' => $token }); + unless($r->is_success()) { + $self->debug("Could not get login page (make sure your LWP supports HTTPS!)"); + return undef; + } + $c = $r -> decoded_content(); + $p = HTML::TokeParser->new(\$c); + while (my $tag = $p->get_tag('script')) { + if($p->get_text() =~ /location\.replace\("(.+)"\)/) { + $token = $1; + $token =~ s/\\x([A-Fa-f0-9]{2})/pack('C', hex($1))/seg; + last; + } + } + $r = $ua->get($token); + unless(check_cookie($ua->cookie_jar, 'LOGIN_INFO')) { + $self->error('Could not log into YouTube'); + return undef; + } + + $self->debug("Got a cookie"); + + $r = $ua->get($videourl); + if ($r->base->as_string() =~ m,/verify_age,) { + $self->debug("Looking for session token..."); + $c = $r->decoded_content(); + $p = HTML::TokeParser->new(\$c); + while (my $tag = $p->get_tag('script')) { + if ($p->get_text() =~ /'XSRF_TOKEN': '(.+)'/) { + $token = $1; + last; + } + } + + unless(defined($token)) { + $self->error("Could not find session token"); + return undef; + } + + $self->debug('Authenticating session...'); + $r = $ua->post($r->base->as_string, { 'next_url' => $r->base->path, 'action_confirm' => 'Confirm Birth Date', 'session_token' => $token }); + } + +# Apparently there is no longer a specific "is_adult" cookie +# or, by the looks of it, anything similar +# +# unless(check_cookie($ua->cookie_jar, 'is_adult')) { +# $self->error('Could not authenticate session'); +# return undef; +# } + + my $cookie = get_all_cookies($ua->cookie_jar); + return ($ua->get($videourl), $cookie); +} + +# Take an encoded url_encoded_fmt_stream_map and return a hash +# matching video IDs to download URLs +sub _decode_url_encoded_fmt_stream_map { + my $self = shift; + my $data = shift; + my $dataencoded = shift; + my @data; + my $h = {}; + + $data = $self->decode_hexurl($data) if $dataencoded; + # This will + # - Split the decoded string into segments (along ,) + # - Interpret each segment as a concatenated key-value list (key and value separated by =, pairs separated by & + # - URL-decode each key and value _again_ + # + # @data will be an array of hash references + + @data = map { { map { $self->decode_hexurl($_) } split /[&=]/ } } split /,/, $data; + $self->debug("_decode_url_encoded_fmt_stream_map() decoded %s", Dumper(\@data)); + + # From each array entry, pick the itag and the url values and return that + # as a hash reference + + + foreach (@data) { + if (exists($_->{'sig'})) { + $h->{$_->{'itag'}} = sprintf('%s&signature=%s', $_->{'url'}, $_->{'sig'}); + } else { + $h->{$_->{'itag'}} = $_->{'url'}; + } + } + + return $h; +} + + + +sub __pick_url { + my $self = shift; + my $urls = shift; + my $preflist = shift; + my $audiopreflist = shift; + my $metadata = shift; + my $picked_format; + + foreach (keys(%{$urls})) { + if (exists($videoformats{$_})) { + $self->debug('Found URL for format %s (%s): %s', $_, $videoformats{$_}, $urls->{$_}); + } else { + $self->error('Unknown format %s: %s', $_, $urls->{$_}); + } + } + + foreach (@{$preflist}) { + if (exists($urls->{$_})) { + $self->debug("Selected URL with quality level %s", $_); + $metadata->{'DLURL'} = $urls->{$_}; + $picked_format = $_; + if (exists($videoformats{$_})) { + $metadata->{'FORMAT'} = $videoformats{$_}; + } else { + $metadata->{'FORMAT'} = 'unknown'; + } + last; + } + } + + if (defined($picked_format)) { + if ($metadata->{'FORMAT'} =~ m/Adaptive/) { + if (defined($audiopreflist)) { + # Pick an audio format as well + foreach (@{$audiopreflist}) { + if (exists($urls->{$_})) { + $self->debug("Selected audio URL with quality level %s", $_); + $metadata->{'DLURL_AUDIO'} = $urls->{$_}; + if (exists($videoformats{$_})) { + $metadata->{'FORMAT_AUDIO'} = $videoformats{$_}; + } else { + $metadata->{'FORMAT_AUDIO'} = 'unknown'; + } + last; + } + } + } else { + $self->error('Need an audio preflist, but none provided'); + } + } + } + + $self->debug('URL found: %s', $metadata->{'DLURL'}); +} + 1; +