X-Git-Url: https://git.camperquake.de/gitweb.cgi?a=blobdiff_plain;f=videosite%2FYouTubeGrabber.pm;h=e611a1f1a80d69b5343f26544ae608657bc9f621;hb=3bcd25114d4e3ba793e2183743d779deef246a75;hp=97327bd3389f4c71df28221f45af1618290ea474;hpb=f2ffeb5224e7b897b89409ecba705ca85ae335ed;p=videosite.git diff --git a/videosite/YouTubeGrabber.pm b/videosite/YouTubeGrabber.pm index 97327bd..e611a1f 100644 --- a/videosite/YouTubeGrabber.pm +++ b/videosite/YouTubeGrabber.pm @@ -26,16 +26,29 @@ my %preflist = ( 'high' => [34, 35, 18, 37, 22, 6, 5, 38, 43], 'normal' => [6, 5, 34, 35, 18, 22, 37, 38, 43]); my %videoformats = ( - 43 => 'webm,360p', - 44 => 'webm,480p', - 45 => 'webm,720p', - 38 => 'mp4,h264,4k', - 37 => 'mp4,h264,1080p', - 35 => 'flv,h264,large', - 34 => 'flv,h264', - 22 => 'mp4,h264,720p', - 18 => 'mp4,h264', - 5 => 'flv,flv', + # Container/Video codec/Audio codec/Resolution + 5 => 'FLV/Sorenson/MP3/240p', + 6 => 'FLV/Sorenson/MP3/270p', + 13 => '3GP/MPEG4-Visual/144p', # 0.5MBit + 17 => '3GP/MPEG4-Visual/144p', # 2MBit + 18 => 'MP4/H264/AAC/360p', # isommp42, Baseline + 22 => 'MP4/H264/AAC/720p', # isommp42, High + 34 => 'FLV/H264/AAC/360p', # Main + 35 => 'FLV/H264/AAC/480p', # Main + 36 => '3GP/MPEG4-Visual/240p', + 37 => 'MP4/H264/AAC/1080p', # High + 38 => 'MP4/H264/AAC/3072p', # High + 43 => 'WebM/VP8/Vorbis/360p', + 44 => 'WebM/VP8/Vorbis/480p', + 45 => 'WebM/VP8/Vorbis/720p', + 46 => 'WebM/VP8/Vorbis/1080p/3D', # effective 540p + 82 => 'MP4/H264/AAC/360p/3D', # isomavc1mp42 + 83 => 'MP4/H264/AAC/240p/3D', # isomavc1mp42 + 84 => 'MP4/H264/AAC/720p/3D', # isomavc1mp42 + 85 => 'MP4/H264/AAC/1080p/3D', # isomavc1mp42, effective 540p + 100 => 'WebM/VP8/Vorbis/360p/3D', + 101 => 'WebM/VP8/Vorbis/480p/3D', + 102 => 'WebM/VP8/Vorbis/720p/3D', ); sub new { @@ -43,9 +56,12 @@ sub new { my $self = $class->SUPER::new(); $self->{'NAME'} = 'youtube'; - $self->{'PATTERNS'} = ['(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/watch(?:_popup)?\?(?:.+=.+&)*v=([-a-zA-Z0-9_]+))', + $self->{_SELFTESTURL} = 'http://www.youtube.com/watch?v=dMH0bHeiRNg'; + $self->{_SELFTESTTITLE} = 'Evolution of Dance - By Judson Laipply'; + $self->{'PATTERNS'} = ['(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/watch(?:_popup)?\?.*?v=([-a-zA-Z0-9_]+))', '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/watch\#\!v=([-a-zA-Z0-9_]+))', '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/v/([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/embed/([-a-zA-Z0-9_]+))', '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/user/[[:alnum:]]+\?v=([-a-zA-Z0-9_]+))', '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/(?:user/)?[[:alnum:]]+#p/(?:\w+/)+\d+/([-a-zA-Z0-9_]+))', '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtu\.be/watch\?v=([-a-zA-Z0-9_]+))', @@ -58,7 +74,8 @@ sub new { 'h264' => 'high resolution MPEG4 video', 'hd' => 'HD720 resolution'}], 'USERNAME' => ['', 'Username to use for YouTube login'], - 'PASSWORD' => ['', 'Password to use for YouTube login']}; + 'PASSWORD' => ['', 'Password to use for YouTube login'], + 'HTTPS' => [1, 'Whether to use HTTPS (if available) to connect to YouTube']}; bless($self, $class); $self->_prepare_parameters(); @@ -108,13 +125,12 @@ sub _parse_by_video_info { $metadata->{'SOURCE'} = $self->{'NAME'}; $metadata->{'TITLE'} = undef; $metadata->{'DLURL'} = undef; - $metadata->{'COOKIE'} = undef; $preflist = $preflist{$quality}; $self->debug("Quality: %s, preflist: [%s]", $quality, join(", ", @{$preflist})); - $videourl = sprintf('https://www.youtube.com/get_video_info?video_id=%s&eurl=%s', - $id, 'http%3A%2F%2Fwww%2Eyoutube%2Ecom%2F'); + $videourl = sprintf('%s://www.youtube.com/get_video_info?video_id=%s&eurl=%s', + $self->_getval('HTTPS')?'https':'http', $id, 'http%3A%2F%2Fwww%2Eyoutube%2Ecom%2F'); $self->debug("Video info URL: %s", $videourl); $r = $ua->get($videourl); @@ -127,7 +143,7 @@ sub _parse_by_video_info { $self->debug('Content from get_video_info: %s', $content); # Decode content - $content = { split /[&=]/, $content }; + $content = $self->decode_querystring($content); if ($content->{'status'} ne 'ok') { $self->debug("Non OK status code found: %s", $content->{'status'}); @@ -189,13 +205,12 @@ sub _parse_by_scrape { $metadata->{'SOURCE'} = $self->{'NAME'}; $metadata->{'TITLE'} = undef; $metadata->{'DLURL'} = undef; - $metadata->{'COOKIE'} = undef; $preflist = $preflist{$quality}; $self->debug("Quality: %s, preflist: [%s]", $quality, join(", ", @{$preflist})); - $videourl = sprintf('https://www.youtube.com/watch?v=%s', $id); + $videourl = sprintf('%s://www.youtube.com/watch?v=%s', $self->_getval('HTTPS')?'https':'http', $id); unless(defined($r = $ua->get($videourl))) { $self->error('Could not download %s', $url); @@ -206,7 +221,6 @@ sub _parse_by_scrape { $self->debug('Video requires age verification'); my @logindata = $self->__login($videourl, $ua); $r = $logindata[0]; - $metadata->{'COOKIE'} = $logindata[1]; unless(defined($r)) { $self->error('Could not log into YouTube'); return undef; @@ -232,10 +246,10 @@ sub _parse_by_scrape { $e = $p->get_text(); $self->debug("Found script: %s", $e); - if ($e =~ m|\x27SWF_ARGS\x27:\s+(.+),|) { + if ($e =~ m|\x27PLAYER_CONFIG\x27:\s+(.+)(?:\}\);)?|) { my $args = $1; + $self->debug("Found PLAYER_CONFIG: %s", $args); - $self->debug("Found SWF_ARGS: %s", $args); $jsp = videosite::JSArrayParser->new(); $self->debug("Using %s to parse", ref($jsp)); $r = $jsp->parse($args); @@ -245,54 +259,32 @@ sub _parse_by_scrape { return undef; } - if (exists($r->{'fmt_url_map'}) and ($r->{'fmt_url_map'} ne '')) { - my $urls = $r->{'fmt_url_map'}; + if (exists($r->{'args'}) and exists($r->{'args'}->{'ps'}) and ($r->{'args'}->{'ps'} eq 'live')) { + $self->error("Video URL seems to point to a live stream, cannot save this"); + return undef; + } + + if (exists($r->{'args'}) and exists($r->{'args'}->{'fmt_url_map'}) and ($r->{'args'}->{'fmt_url_map'} ne '')) { + my $urls = $r->{'args'}->{'fmt_url_map'}; $self->debug("Video has fmt_url_map: %s", $urls); - $urls = $self->decode_hexurl($urls); %urls = split(/[\|,]/, $urls); - $self->debug("Pagetype: old (SWF_ARGS), fmt_url_map"); - - } elsif (exists($r->{'t'}) and ($r->{'t'} ne '')) { - my $thash = $r->{'t'}; - - if (exists($r->{'fmt_map'}) && ($r->{'fmt_map'} ne '')) { - my $fmt = $r->{'fmt_map'}; - my @fmt; - - $self->debug('Video has fmt_map'); - $fmt = $self->decode_hexurl($fmt); - @fmt = split(/,/, $fmt); - foreach (@fmt) { - @_=split(/\//); - $urls{$_[0]} = sprintf('http://www.youtube.com/get_video?video_id=%s&fmt=%d&t=%s', - $metadata->{'ID'}, - $_[0], - $thash); - } - $self->debug("Pagetype: 2009 (SWF_ARGS), t with fmt_map"); - - } else { - $urls{5} = sprintf('http://www.youtube.com/get_video?video_id=%s&t=%s', - $metadata->{'ID'}, - $thash); - $self->debug("Pagetype: 2009 (SWF_ARGS), t without fmt_map"); + foreach (keys(%urls)) { + $urls{$_} = $self->decode_hexurl($urls{$_}); } + $self->debug("Pagetype: 2011 (PLAYER_CONFIG), fmt_url_map"); + } elsif (exists($r->{'args'}) and exists($r->{'args'}->{'url_encoded_fmt_stream_map'}) and ($r->{'args'}->{'url_encoded_fmt_stream_map'} ne '')) { + %urls = %{$self->_decode_url_encoded_fmt_stream_map($r->{'args'}->{'url_encoded_fmt_stream_map'}, 0)}; + + $self->debug("Pagetype: 2011 (PLAYER_CONFIG), url_encoded_fmt_stream_map"); } else { - $self->error('Neither fmt_url_map nor t found in video information hash'); + $self->error('fmt_url_map not found in PLAYER_CONFIG'); return undef; } - } elsif ($e =~ m|var swfHTML = .*fmt_url_map=([^\&]+)\&|) { - my $urls = $1; - $self->debug("Video has fmt_url_map: %s", $urls); - - $urls = $self->decode_hexurl($urls); - %urls = split(/[\|,]/, $urls); - $self->debug("Pagetype: 2010 (swfHTML), fmt_url_map"); - } elsif ($e =~ m|\x27PLAYER_CONFIG\x27:\s+(.+)(?:\}\);)?|) { + } elsif ($e =~ m|yt\.playerConfig\s*=\s*(.+);\n|) { my $args = $1; - $self->debug("Found PLAYER_CONFIG: %s", $args); + $self->debug("Found yt.playerConfig: %s", $args); $jsp = videosite::JSArrayParser->new(); $self->debug("Using %s to parse", ref($jsp)); @@ -308,26 +300,17 @@ sub _parse_by_scrape { return undef; } - if (exists($r->{'args'}) and exists($r->{'args'}->{'fmt_url_map'}) and ($r->{'args'}->{'fmt_url_map'} ne '')) { - my $urls = $r->{'args'}->{'fmt_url_map'}; - - $self->debug("Video has fmt_url_map: %s", $urls); - - %urls = split(/[\|,]/, $urls); - foreach (keys(%urls)) { - $urls{$_} = $self->decode_hexurl($urls{$_}); - } - $self->debug("Pagetype: 2011 (PLAYER_CONFIG), fmt_url_map"); - } elsif (exists($r->{'args'}) and exists($r->{'args'}->{'url_encoded_fmt_stream_map'}) and ($r->{'args'}->{'url_encoded_fmt_stream_map'} ne '')) { + if (exists($r->{'args'}) and exists($r->{'args'}->{'url_encoded_fmt_stream_map'}) and ($r->{'args'}->{'url_encoded_fmt_stream_map'} ne '')) { %urls = %{$self->_decode_url_encoded_fmt_stream_map($r->{'args'}->{'url_encoded_fmt_stream_map'}, 0)}; - $self->debug("Pagetype: 2011 (PLAYER_CONFIG), url_encoded_fmt_stream_map"); + $self->debug("Pagetype: 2012 (yt.playerConfig), url_encoded_fmt_stream_map"); } else { - $self->error('fmt_url_map not found in PLAYER_CONFIG'); + $self->error('url_map not found in yt.playerConfig'); return undef; } } + if (%urls) { $self->__pick_url(\%urls, $preflist, $metadata); last SWF_ARGS; @@ -466,6 +449,7 @@ sub _decode_url_encoded_fmt_stream_map { my $data = shift; my $dataencoded = shift; my @data; + my $h = {}; $data = $self->decode_hexurl($data) if $dataencoded; # This will @@ -475,13 +459,22 @@ sub _decode_url_encoded_fmt_stream_map { # # @data will be an array of hash references - @data = map { { map { $self->decode_hexurl($_) } split /[=&]/ } } split /,/, $data; + @data = map { { map { $self->decode_hexurl($_) } split /[&=]/ } } split /,/, $data; $self->debug("_decode_url_encoded_fmt_stream_map() decoded %s", Dumper(\@data)); # From each array entry, pick the itag and the url values and return that # as a hash reference - return { map { $_->{'itag'}, $_->{'url'} } @data }; + + foreach (@data) { + if (exists($_->{'sig'})) { + $h->{$_->{'itag'}} = sprintf('%s&signature=%s', $_->{'url'}, $_->{'sig'}); + } else { + $h->{$_->{'itag'}} = $_->{'url'}; + } + } + + return $h; } @@ -504,6 +497,11 @@ sub __pick_url { if (exists($urls->{$_})) { $self->debug("Selected URL with quality level %s", $_); $metadata->{'DLURL'} = $urls->{$_}; + if (exists($videoformats{$_})) { + $metadata->{'FORMAT'} = $videoformats{$_}; + } else { + $metadata->{'FORMAT'} = 'unknown'; + } last; } }