X-Git-Url: https://git.camperquake.de/gitweb.cgi?a=blobdiff_plain;f=videosite%2FYouTubeGrabber.pm;h=ebb2f692d0cd01b40f4abd69d6f16a7f9a9dcc75;hb=3ed1591b5189c9fd6474880b50ff9af831df6e16;hp=a7157f53b6ce99ace8e2f31aa8ac6931a85f5e03;hpb=4e48e7f74923ee8168864a65f68305d0dbe8100a;p=videosite.git diff --git a/videosite/YouTubeGrabber.pm b/videosite/YouTubeGrabber.pm index a7157f5..ebb2f69 100644 --- a/videosite/YouTubeGrabber.pm +++ b/videosite/YouTubeGrabber.pm @@ -11,8 +11,6 @@ package videosite::YouTubeGrabber; use videosite::GrabberBase; @ISA = qw(videosite::GrabberBase); -use LWP::UserAgent; -use HTTP::Cookies; use HTML::TokeParser; use HTML::Entities qw(decode_entities); use Encode; @@ -21,16 +19,35 @@ use videosite::JSArrayParser; use strict; +my %preflist = ( + 'insane' => [38, 37, 22, 35, 18, 34, 6, 5], + 'hd' => [37, 22, 35, 18, 34, 6, 5, 38], + 'h264' => [18, 34, 37, 22, 35, 6, 5, 38], + 'high' => [34, 35, 18, 37, 22, 6, 5, 38], + 'normal' => [6, 5, 34, 35, 18, 22, 37, 38]); +my %videoformats = ( + 38 => 'mp4,h264,4k', + 37 => 'mp4,h264,1080p', + 35 => 'flv,h264,large', + 34 => 'flv,h264', + 22 => 'mp4,h264,720p', + 18 => 'mp4,h264', + 5 => 'flv,flv', + ); + sub new { my $class = shift; my $self = $class->SUPER::new(); $self->{'NAME'} = 'youtube'; - $self->{'PATTERNS'} = ['(http://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/watch\?(?:.+=.+&)*v=([-a-zA-Z0-9_]+))', - '(http://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/watch\#\!v=([-a-zA-Z0-9_]+))', - '(http://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/v/([-a-zA-Z0-9_]+))', - '(http://(?:[-a-zA-Z0-9_.]+\.)*youtu\.be/([-a-zA-Z0-9_]+))', - '(http://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/user/\w+\?.*/([-a-zA-Z0-9_]+))']; + $self->{'PATTERNS'} = ['(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/watch(?:_popup)?\?(?:.+=.+&)*v=([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/watch\#\!v=([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/v/([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/user/[[:alnum:]]+\?v=([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/(?:user/)?[[:alnum:]]+#p/(?:\w+/)+\d+/([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtu\.be/watch\?v=([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtu\.be/([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/user/\w+\?.*/([-a-zA-Z0-9_]+))']; $self->{'_PARAMS'} = { 'QUALITY' => ['normal', 'Quality of the video to download.', { 'normal' => 'standard resolution flash video', @@ -50,42 +67,125 @@ sub _parse { my $self = shift; my $url = shift; my $pattern = shift; + my $id; + my $res; + + my $url =~ m|$pattern|; + my $url = $1; + my $id = $2; + + $self->debug("Matched id %s from pattern %s", $id, $pattern); + + $res = $self->_parse_by_video_info($url, $id); + if (defined($res) && ref($res)) { + return $res; + } else { + $res = $self->_parse_by_scrape($url, $id); + } + + return $res; +} + +sub _parse_by_video_info { + my $self = shift; + my $url = shift; + my $id = shift; + my $quality = $self->_getval('QUALITY'); + my $metadata; + my $videourl; + my $ua = $self->ua(); + my $preflist; + my $r; + my $content; + my $urls; + + $metadata->{'URL'} = $url; + $metadata->{'ID'} = $id; + $metadata->{'TYPE'} = 'video'; + $metadata->{'SOURCE'} = $self->{'NAME'}; + $metadata->{'TITLE'} = undef; + $metadata->{'DLURL'} = undef; + $metadata->{'COOKIE'} = undef; + + $preflist = $preflist{$quality}; + $self->debug("Quality: %s, preflist: [%s]", $quality, join(", ", @{$preflist})); + + $videourl = sprintf('https://www.youtube.com/get_video_info?video_id=%s&eurl=%s', + $id, 'http%3A%2F%2Fwww%2Eyoutube%2Ecom%2F'); + $self->debug("Video info URL: %s", $videourl); + + $r = $ua->get($videourl); + unless($r->is_success()) { + $self->debug('Could not download %s: %s', $videourl, $r->code()); + return undef; + } + + $content = $r->content(); + $self->debug('Content from get_video_info: %s', $content); + + # Decode content + $content = { split /[&=]/, $content }; + + if ($content->{'status'} ne 'ok') { + $self->debug("Non OK status code found: %s", $content->{'status'}); + return undef; + } + + unless(exists($content->{'fmt_url_map'}) and exists($content->{'title'})) { + $self->debug("No fmt_url_map or no title found"); + return undef; + } + + # Decode fmt_url_map + $urls = $content->{'fmt_url_map'}; + $urls =~ s/%(..)/chr(hex($1))/ge; + $urls = { split /[\|,]/, $urls }; + + $self->__pick_url($urls, $preflist, $metadata); + + $metadata->{'TITLE'} = $content->{'title'}; + $metadata->{'TITLE'} =~ s/\+/ /g; + $metadata->{'TITLE'} =~ s/%(..)/chr(hex($1))/ge; + + $self->debug('Title found: %s', $metadata->{'TITLE'}); + + unless(defined($metadata->{'DLURL'}) && defined($metadata->{'TITLE'})) { + $self->error('Could not determine download URL'); + return undef; + } + + return $metadata; +} + +sub _parse_by_scrape { + my $self = shift; + my $url = shift; + my $id = shift; my $content; my $metadata = {}; my $p; my $e; my $tag; - my $jar = HTTP::Cookies->new; - my $ua = LWP::UserAgent->new('agent' => 'Mozilla/5.0'); + my $ua = $self->ua(); my $r; my $videourl; my $quality = $self->_getval('QUALITY'); - my %preflist = ( - 'insane' => [38, 37, 22, 35, 18, 34, 6, 5], - 'hd' => [37, 22, 35, 18, 34, 6, 5, 38], - 'h264' => [18, 34, 37, 22, 35, 6, 5, 38], - 'high' => [34, 35, 18, 37, 22, 6, 5, 38], - 'normal' => [6, 5, 34, 35, 18, 22, 37, 38]); my $preflist; my $jsp; - $url =~ m|$pattern|; - $url = $1; - $metadata->{'URL'} = $url; - $metadata->{'ID'} = $2; + $metadata->{'ID'} = $id; $metadata->{'TYPE'} = 'video'; $metadata->{'SOURCE'} = $self->{'NAME'}; $metadata->{'TITLE'} = undef; $metadata->{'DLURL'} = undef; $metadata->{'COOKIE'} = undef; - $self->debug("Matched id %s from pattern %s", $2, $pattern); $preflist = $preflist{$quality}; $self->debug("Quality: %s, preflist: [%s]", $quality, join(", ", @{$preflist})); - $videourl = sprintf('http://www.youtube.com/watch?v=%s', $2); + $videourl = sprintf('https://www.youtube.com/watch?v=%s', $id); unless(defined($r = $ua->get($videourl))) { $self->error('Could not download %s', $url); @@ -94,7 +194,6 @@ sub _parse { if ($r->base->as_string() =~ m,/verify_age,) { $self->debug('Video requires age verification'); - $ua->cookie_jar($jar); my @logindata = $self->__login($videourl, $ua); $r = $logindata[0]; $metadata->{'COOKIE'} = $logindata[1]; @@ -109,7 +208,7 @@ sub _parse { SWF_ARGS: while ($tag = $p->get_tag('div', 'meta', 'script')) { if ('meta' eq $tag->[0]) { - if ('title' eq $tag->[1]->{'name'}) { + if (exists($tag->[1]->{'name'}) and ('title' eq $tag->[1]->{'name'})) { $metadata->{'TITLE'} = $tag->[1]->{'content'}; # Convert HTML entities in the title. This is a bit convoluted. $metadata->{'TITLE'} = encode("utf8", @@ -182,42 +281,50 @@ sub _parse { $urls =~ s/%([[:xdigit:]]{2})/chr(hex($1))/ge; %urls = split(/[\|,]/, $urls); $self->debug("Pagetype: 2010 (swfHTML), fmt_url_map"); - } + } elsif ($e =~ m|\x27PLAYER_CONFIG\x27:\s+(.+)\}\);|) { + my $args = $1; + $self->debug("Found PLAYER_CONFIG: %s", $args); - if (%urls) { - foreach (keys(%urls)) { - if ($_ == 35) { - $self->debug('Found flv,h264,large: %s', $urls{$_}); - } elsif ($_ == 34) { - $self->debug('Found flv,h264: %s', $urls{$_}); - } elsif ($_ == 22) { - $self->debug('Found mp4,h264,720p: %s', $urls{$_}); - } elsif ($_ == 38) { - $self->debug('Found mp4,h264,4k: %s', $urls{$_}); - } elsif ($_ == 37) { - $self->debug('Found mp4,h264,1080p: %s', $urls{$_}); - } elsif ($_ == 18) { - $self->debug('Found mp4,h264: %s', $urls{$_}); - } elsif ($_ == 5) { - $self->debug('Found flv,flv: %s', $urls{$_}); - } else { - $self->error('Unknown tag %s: %s', $_, $urls{$_}); - } + $jsp = videosite::JSArrayParser->new(); + $self->debug("Using %s to parse", ref($jsp)); + $r = $jsp->parse($args); + + unless(defined($r)) { + $self->error("Found information hash, but could not parse"); + return undef; + } + + if (exists($r->{'args'}) and exists($r->{'args'}->{'ps'}) and ($r->{'args'}->{'ps'} eq 'live')) { + $self->error("Video URL seems to point to a live stream, cannot save this"); + return undef; } - foreach (@{$preflist}) { - if (exists($urls{$_})) { - $self->debug("Selected URL with quality level %s", $_); - $metadata->{'DLURL'} = $urls{$_}; - last; + if (exists($r->{'args'}) and exists($r->{'args'}->{'fmt_url_map'}) and ($r->{'args'}->{'fmt_url_map'} ne '')) { + my $urls = $r->{'args'}->{'fmt_url_map'}; + + $self->debug("Video has fmt_url_map: %s", $urls); + + %urls = split(/[\|,]/, $urls); + foreach (keys(%urls)) { + my $u = $urls{$_}; + $u =~ s/%([[:xdigit:]]{2})/chr(hex($1))/ge; + $urls{$_} = $u; } + $self->debug("Pagetype: 2011 (PLAYER_CONFIG), fmt_url_map"); + } else { + $self->error('fmt_url_map not found in PLAYER_CONFIG'); + return undef; } + } - $self->debug('URL found: %s', $metadata->{'DLURL'}); + if (%urls) { + $self->__pick_url(\%urls, $preflist, $metadata); last SWF_ARGS; } } elsif ('div' eq $tag->[0]) { - if (exists($tag->[1]->{'class'}) and ('yt-alert-content' eq $tag->[1]->{'class'})) { + if (exists($tag->[1]->{'id'}) and ('watch-player-unavailable-message-container' eq $tag->[1]->{'id'})) { + # Search forward to the next
+ $tag = $p->get_tag('div'); $self->error("Could not get video data for youtube %s: %s", $metadata->{'ID'}, $p->get_trimmed_text()); return undef; @@ -272,16 +379,25 @@ sub __login { $self->debug('Logging in'); $r = $ua->get('https://www.google.com/accounts/ServiceLoginAuth?service=youtube'); + unless($r->is_success()) { + $self->debug("Could not get login page (make sure your LWP supports HTTPS!)"); + return undef; + } $c = $r->decoded_content(); $p = HTML::TokeParser->new(\$c); while (my $tag = $p->get_tag('input')) { + $self->debug("%s", Dumper($tag)); if ($tag->[1]{name} eq 'GALX') { $token = $tag->[1]{value}; last; } } - $self->debug("GALX = .$token"); + $self->debug("GALX = %s", $token); $r = $ua->post('https://www.google.com/accounts/ServiceLoginAuth?service=youtube', { 'service' => 'youtube', 'Email' => $user, 'Passwd' => $pass, 'GALX' => $token }); + unless($r->is_success()) { + $self->debug("Could not get login page (make sure your LWP supports HTTPS!)"); + return undef; + } $c = $r -> decoded_content(); $p = HTML::TokeParser->new(\$c); while (my $tag = $p->get_tag('script')) { @@ -332,4 +448,29 @@ sub __login { return ($ua->get($videourl), $cookie); } +sub __pick_url { + my $self = shift; + my $urls = shift; + my $preflist = shift; + my $metadata = shift; + + foreach (keys(%{$urls})) { + if (exists($videoformats{$_})) { + $self->debug('Found URL for format %s (%s): %s', $_, $videoformats{$_}, $urls->{$_}); + } else { + $self->error('Unknown format %s: %s', $_, $urls->{$_}); + } + } + + foreach (@{$preflist}) { + if (exists($urls->{$_})) { + $self->debug("Selected URL with quality level %s", $_); + $metadata->{'DLURL'} = $urls->{$_}; + last; + } + } + + $self->debug('URL found: %s', $metadata->{'DLURL'}); +} + 1;