X-Git-Url: https://git.camperquake.de/gitweb.cgi?a=blobdiff_plain;f=videosite%2FYouTubeGrabber.pm;h=ad03f6a6366f9860de6377502a5b5abd98808fd1;hb=58d2597498d4162e2a3811978f59be7bedf2f9b6;hp=bf954cc98289cfbb484284e9c9a95b7e2653bd28;hpb=1b803a8856b61ffde5e2c8f9e3f91ea3bcd6d01f;p=videosite.git diff --git a/videosite/YouTubeGrabber.pm b/videosite/YouTubeGrabber.pm index bf954cc..ad03f6a 100644 --- a/videosite/YouTubeGrabber.pm +++ b/videosite/YouTubeGrabber.pm @@ -20,11 +20,17 @@ use videosite::JSArrayParser; use strict; my %preflist = ( - 'insane' => [38, 37, 22, 35, 18, 34, 6, 5, 43], - 'hd' => [37, 22, 35, 18, 34, 6, 5, 38, 43], - 'h264' => [18, 34, 37, 22, 35, 6, 5, 38, 43], - 'high' => [34, 35, 18, 37, 22, 6, 5, 38, 43], - 'normal' => [6, 5, 34, 35, 18, 22, 37, 38, 43]); + 'insane' => [38, 37, 137, 22, 136, 35, 135, 18, 134, 34, 6, 5, 43], + 'hd' => [37, 137, 22, 136, 35, 135, 18, 134, 34, 6, 5, 38, 43], + 'h264' => [18, 134, 34, 37, 137, 22, 136, 35, 135, 6, 5, 38, 43], + 'high' => [34, 35, 135, 18, 134, 37, 137, 22, 136, 6, 5, 38, 43], + 'normal' => [6, 5, 34, 35, 135, 18, 134, 22, 136, 37, 137, 38, 43]); +my %audiopreflist = ( + 'insane' => [172,141,171,140,139], + 'hd' => [172,141,171,140,139], + 'h264' => [172,141,171,140,139], + 'high' => [171,140,172,141,139], + 'normal' => [171,140,172,141,139]); my %videoformats = ( # Container/Video codec/Audio codec/Resolution 5 => 'FLV/Sorenson/MP3/240p', @@ -35,6 +41,7 @@ my %videoformats = ( 22 => 'MP4/H264/AAC/720p', # isommp42, High 34 => 'FLV/H264/AAC/360p', # Main 35 => 'FLV/H264/AAC/480p', # Main + 36 => '3GP/MPEG4-Visual/240p', 37 => 'MP4/H264/AAC/1080p', # High 38 => 'MP4/H264/AAC/3072p', # High 43 => 'WebM/VP8/Vorbis/360p', @@ -48,37 +55,49 @@ my %videoformats = ( 100 => 'WebM/VP8/Vorbis/360p/3D', 101 => 'WebM/VP8/Vorbis/480p/3D', 102 => 'WebM/VP8/Vorbis/720p/3D', + 133 => 'MP4/H264/240p/AdaptiveVideo', + 134 => 'MP4/H264/360p/AdaptiveVideo', + 135 => 'MP4/H264/480p/AdaptiveVideo', + 136 => 'MP4/H264/720p/AdaptiveVideo', + 137 => 'MP4/H264/1080p/AdaptiveVideo', + 139 => 'MP4/AAC/48kbit/AdaptiveAudio', + 140 => 'MP4/AAC/128kbit/AdaptiveAudio', + 141 => 'MP4/AAC/256kbit/AdaptiveAudio', + 160 => 'MP4/H264/144p/AdaptiveVideo', + 171 => 'WebM/Vorbis/128kbit/AdaptiveAudio', + 172 => 'WebM/Vorbis/160kbit/AdaptiveAudio', ); sub new { my $class = shift; - my $self = $class->SUPER::new(); - - $self->{'NAME'} = 'youtube'; - $self->{_SELFTESTURL} = 'http://www.youtube.com/watch?v=dMH0bHeiRNg'; - $self->{_SELFTESTTITLE} = 'Evolution of Dance - By Judson Laipply'; - $self->{'PATTERNS'} = ['(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/watch(?:_popup)?\?(?:.+=.+&|&)*v=([-a-zA-Z0-9_]+))', - '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/watch\#\!v=([-a-zA-Z0-9_]+))', - '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/v/([-a-zA-Z0-9_]+))', - '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/user/[[:alnum:]]+\?v=([-a-zA-Z0-9_]+))', - '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/(?:user/)?[[:alnum:]]+#p/(?:\w+/)+\d+/([-a-zA-Z0-9_]+))', - '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtu\.be/watch\?v=([-a-zA-Z0-9_]+))', - '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtu\.be/([-a-zA-Z0-9_]+))', - '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/user/\w+\?.*/([-a-zA-Z0-9_]+))']; - $self->{'_PARAMS'} = { - 'QUALITY' => ['normal', 'Quality of the video to download.', { - 'normal' => 'standard resolution flash video', - 'high' => 'higher resolution flash video', - 'h264' => 'high resolution MPEG4 video', - 'hd' => 'HD720 resolution'}], - 'USERNAME' => ['', 'Username to use for YouTube login'], - 'PASSWORD' => ['', 'Password to use for YouTube login'], - 'HTTPS' => [1, 'Whether to use HTTPS (if available) to connect to YouTube']}; - - bless($self, $class); - $self->_prepare_parameters(); - - return $self; + my $self = $class->SUPER::new( + NAME => 'youtube', + _SELFTESTURL => 'http://www.youtube.com/watch?v=dMH0bHeiRNg', + _SELFTESTTITLE => 'Evolution of Dance - By Judson Laipply', + PATTERNS => ['(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/watch(?:_popup)?/?\?.*?v=([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/watch\#\!v=([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/v/([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/embed/([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/user/[[:alnum:]]+\?v=([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/(?:user/)?[[:alnum:]]+#p/(?:\w+/)+\d+/([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtu\.be/watch\?v=([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtu\.be/([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/user/\w+\?.*/([-a-zA-Z0-9_]+))'], + _PARAMS => { + QUALITY => ['normal', 'Quality of the video to download.', { + normal => 'standard resolution flash video', + high => 'higher resolution flash video', + h264 => 'high resolution MPEG4 video', + hd => 'HD720 resolution'}], + USERNAME => ['', 'Username to use for YouTube login'], + PASSWORD => ['', 'Password to use for YouTube login'], + HTTPS => [1, 'Whether to use HTTPS (if available) to connect to YouTube'], + ADAPTIVE => [0, 'Whether to use adaptive rate URLs (if available)'], + }, + @_, + ); + + return bless($self, $class); } sub _parse { @@ -95,15 +114,28 @@ sub _parse { $self->debug("Matched id %s from pattern %s", $id, $pattern); $res = $self->_parse_by_video_info($url, $id); - if (defined($res) && ref($res)) { - return $res; + if (defined($res)) { + if (ref($res)) { + return $res; + } else { + $self->debug("_parse_by_video_info failed with status noretry"); + return undef; + } } else { + $self->debug("_parse_by_video_info failed with status retry"); $res = $self->_parse_by_scrape($url, $id); } return $res; } +# +# Try to get video information by using the API call. +# +# Returns hashref on success +# Returns undef on retryable error (try to scrape the website) +# Returns 0 on non-retryable error +# sub _parse_by_video_info { my $self = shift; my $url = shift; @@ -113,6 +145,7 @@ sub _parse_by_video_info { my $videourl; my $ua = $self->ua(); my $preflist; + my $audiopreflist; my $r; my $content; my $urls; @@ -125,7 +158,11 @@ sub _parse_by_video_info { $metadata->{'DLURL'} = undef; $preflist = $preflist{$quality}; - $self->debug("Quality: %s, preflist: [%s]", $quality, join(", ", @{$preflist})); + $audiopreflist = $audiopreflist{$quality}; + $self->debug("Quality: %s, preflist: [%s], audiopreflist: [%s]", + $quality, + join(", ", @{$preflist}), + join(", ", @{$audiopreflist})); $videourl = sprintf('%s://www.youtube.com/get_video_info?video_id=%s&eurl=%s', $self->_getval('HTTPS')?'https':'http', $id, 'http%3A%2F%2Fwww%2Eyoutube%2Ecom%2F'); @@ -143,11 +180,19 @@ sub _parse_by_video_info { # Decode content $content = $self->decode_querystring($content); + $self->debug("Decoded get_video_info: %s", Dumper($content)); + if ($content->{'status'} ne 'ok') { $self->debug("Non OK status code found: %s", $content->{'status'}); return undef; } + # Check if this is live content + if (exists($content->{ps}) and ($content->{ps} eq 'live')) { + $self->error("Video URL seems to point to a live stream, cannot save this"); + return 0; + } + if (exists($content->{'fmt_url_map'})) { # Decode fmt_url_map $urls = $self->decode_hexurl($content->{'fmt_url_map'}); @@ -159,12 +204,24 @@ sub _parse_by_video_info { return undef; } + if (exists($content->{'adaptive_fmts'})) { + $self->debug('Video has adaptive_fmts'); + if ($self->_getval('ADAPTIVE')) { + my %af = %{$self->_decode_url_encoded_fmt_stream_map($content->{'adaptive_fmts'}, 1)}; + while (my ($k, $v) = each %af) { + $urls->{$k} = $v; + } + } else { + $self->debug('Use of adaptive URLs not enabled'); + } + } + unless(exists($content->{'title'})) { $self->debug("No title found"); return undef; } - $self->__pick_url($urls, $preflist, $metadata); + $self->__pick_url($urls, $preflist, $audiopreflist, $metadata); $metadata->{'TITLE'} = $content->{'title'}; $metadata->{'TITLE'} =~ s/\+/ /g; @@ -244,66 +301,7 @@ sub _parse_by_scrape { $e = $p->get_text(); $self->debug("Found script: %s", $e); -# if ($e =~ m|\x27SWF_ARGS\x27:\s+(.+),|) { -# my $args = $1; -# -# $self->debug("Found SWF_ARGS: %s", $args); -# $jsp = videosite::JSArrayParser->new(); -# $self->debug("Using %s to parse", ref($jsp)); -# $r = $jsp->parse($args); -# -# unless(defined($r)) { -# $self->error("Found information hash, but could not parse"); -# return undef; -# } -# -# if (exists($r->{'fmt_url_map'}) and ($r->{'fmt_url_map'} ne '')) { -# my $urls = $r->{'fmt_url_map'}; -# -# $self->debug("Video has fmt_url_map: %s", $urls); -# -# $urls = $self->decode_hexurl($urls); -# %urls = split(/[\|,]/, $urls); -# $self->debug("Pagetype: old (SWF_ARGS), fmt_url_map"); -# -# } elsif (exists($r->{'t'}) and ($r->{'t'} ne '')) { -# my $thash = $r->{'t'}; -# -# if (exists($r->{'fmt_map'}) && ($r->{'fmt_map'} ne '')) { -# my $fmt = $r->{'fmt_map'}; -# my @fmt; -# -# $self->debug('Video has fmt_map'); -# $fmt = $self->decode_hexurl($fmt); -# @fmt = split(/,/, $fmt); -# foreach (@fmt) { -# @_=split(/\//); -# $urls{$_[0]} = sprintf('http://www.youtube.com/get_video?video_id=%s&fmt=%d&t=%s', -# $metadata->{'ID'}, -# $_[0], -# $thash); -# } -# $self->debug("Pagetype: 2009 (SWF_ARGS), t with fmt_map"); -# -# } else { -# $urls{5} = sprintf('http://www.youtube.com/get_video?video_id=%s&t=%s', -# $metadata->{'ID'}, -# $thash); -# $self->debug("Pagetype: 2009 (SWF_ARGS), t without fmt_map"); -# } -# } else { -# $self->error('Neither fmt_url_map nor t found in video information hash'); -# return undef; -# } -# } elsif ($e =~ m|var swfHTML = .*fmt_url_map=([^\&]+)\&|) { -# my $urls = $1; -# $self->debug("Video has fmt_url_map: %s", $urls); -# -# $urls = $self->decode_hexurl($urls); -# %urls = split(/[\|,]/, $urls); -# $self->debug("Pagetype: 2010 (swfHTML), fmt_url_map"); -# } elsif ($e =~ m|\x27PLAYER_CONFIG\x27:\s+(.+)(?:\}\);)?|) { - if ($e =~ m|\x27PLAYER_CONFIG\x27:\s+(.+)(?:\}\);)?|) { + if ($e =~ m|ytplayer\.config\s*=\s*(.+);$|) { my $args = $1; $self->debug("Found PLAYER_CONFIG: %s", $args); @@ -369,7 +367,7 @@ sub _parse_by_scrape { if (%urls) { - $self->__pick_url(\%urls, $preflist, $metadata); + $self->__pick_url(\%urls, $preflist, undef, $metadata); last SWF_ARGS; } } elsif ('div' eq $tag->[0]) { @@ -506,6 +504,7 @@ sub _decode_url_encoded_fmt_stream_map { my $data = shift; my $dataencoded = shift; my @data; + my $h = {}; $data = $self->decode_hexurl($data) if $dataencoded; # This will @@ -521,7 +520,16 @@ sub _decode_url_encoded_fmt_stream_map { # From each array entry, pick the itag and the url values and return that # as a hash reference - return { map { $_->{'itag'}, $_->{'url'} } @data }; + + foreach (@data) { + if (exists($_->{'sig'})) { + $h->{$_->{'itag'}} = sprintf('%s&signature=%s', $_->{'url'}, $_->{'sig'}); + } else { + $h->{$_->{'itag'}} = $_->{'url'}; + } + } + + return $h; } @@ -530,7 +538,9 @@ sub __pick_url { my $self = shift; my $urls = shift; my $preflist = shift; + my $audiopreflist = shift; my $metadata = shift; + my $picked_format; foreach (keys(%{$urls})) { if (exists($videoformats{$_})) { @@ -544,6 +554,7 @@ sub __pick_url { if (exists($urls->{$_})) { $self->debug("Selected URL with quality level %s", $_); $metadata->{'DLURL'} = $urls->{$_}; + $picked_format = $_; if (exists($videoformats{$_})) { $metadata->{'FORMAT'} = $videoformats{$_}; } else { @@ -553,6 +564,28 @@ sub __pick_url { } } + if (defined($picked_format)) { + if ($metadata->{'FORMAT'} =~ m/Adaptive/) { + if (defined($audiopreflist)) { + # Pick an audio format as well + foreach (@{$audiopreflist}) { + if (exists($urls->{$_})) { + $self->debug("Selected audio URL with quality level %s", $_); + $metadata->{'DLURL_AUDIO'} = $urls->{$_}; + if (exists($videoformats{$_})) { + $metadata->{'FORMAT_AUDIO'} = $videoformats{$_}; + } else { + $metadata->{'FORMAT_AUDIO'} = 'unknown'; + } + last; + } + } + } else { + $self->error('Need an audio preflist, but none provided'); + } + } + } + $self->debug('URL found: %s', $metadata->{'DLURL'}); }