X-Git-Url: https://git.camperquake.de/gitweb.cgi?a=blobdiff_plain;f=videosite%2FYouTubeGrabber.pm;h=2a363da1156c2e72627292245a643e66c98b86c8;hb=93a703d581dfbd6613fa05da16e76cb4a8d88f2c;hp=dc6bba72c9ab1d5d3d58a064414e5d5fe09b50bc;hpb=56e21ccb455187b27d7ec4c8c12aa1bf2f61e22f;p=videosite.git diff --git a/videosite/YouTubeGrabber.pm b/videosite/YouTubeGrabber.pm index dc6bba7..2a363da 100644 --- a/videosite/YouTubeGrabber.pm +++ b/videosite/YouTubeGrabber.pm @@ -6,14 +6,18 @@ # download strategy revised using # http://www.kde-apps.org/content/show.php?content=41456 -package YouTubeGrabber; +package videosite::YouTubeGrabber; -use GrabberBase; -@ISA = qw(GrabberBase); +use videosite::GrabberBase; +@ISA = qw(videosite::GrabberBase); -use LWP::Simple qw(!get); +use LWP::UserAgent; +use HTTP::Cookies; use HTML::TokeParser; +use HTML::Entities qw(decode_entities); +use Encode; use Data::Dumper; +use videosite::JSArrayParser; use strict; @@ -22,8 +26,21 @@ sub new { my $self = $class->SUPER::new(); $self->{'NAME'} = 'youtube'; - $self->{'PATTERNS'} = ['(http://(?:[-a-zA-Z0-9_.]+\.)*youtube.(?:com|de|co.uk)/watch\?(?:.+=.+&)*v=([-a-zA-Z0-9_]+))', - '(http://(?:[-a-zA-Z0-9_.]+\.)*youtube.(?:com|de|co.uk)/v/([-a-zA-Z0-9_]+))']; + $self->{'PATTERNS'} = ['(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/watch\?(?:.+=.+&)*v=([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/watch\#\!v=([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/v/([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/user/[[:alnum:]]+\?v=([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtu\.be/watch\?v=([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtu\.be/([-a-zA-Z0-9_]+))', + '(https?://(?:[-a-zA-Z0-9_.]+\.)*youtube\.(?:com|de|co.uk)/user/\w+\?.*/([-a-zA-Z0-9_]+))']; + $self->{'_PARAMS'} = { + 'QUALITY' => ['normal', 'Quality of the video to download.', { + 'normal' => 'standard resolution flash video', + 'high' => 'higher resolution flash video', + 'h264' => 'high resolution MPEG4 video', + 'hd' => 'HD720 resolution'}], + 'USERNAME' => ['', 'Username to use for YouTube login'], + 'PASSWORD' => ['', 'Password to use for YouTube login']}; bless($self, $class); $self->_prepare_parameters(); @@ -40,6 +57,28 @@ sub _parse { my $p; my $e; my $tag; + my $jar = HTTP::Cookies->new; + my $ua = LWP::UserAgent->new('agent' => 'Mozilla/5.0'); + my $r; + my $videourl; + my $quality = $self->_getval('QUALITY'); + my %preflist = ( + 'insane' => [38, 37, 22, 35, 18, 34, 6, 5], + 'hd' => [37, 22, 35, 18, 34, 6, 5, 38], + 'h264' => [18, 34, 37, 22, 35, 6, 5, 38], + 'high' => [34, 35, 18, 37, 22, 6, 5, 38], + 'normal' => [6, 5, 34, 35, 18, 22, 37, 38]); + my %videoformats = ( + 38 => 'mp4,h264,4k', + 37 => 'mp4,h264,1080p', + 35 => 'flv,h264,large', + 34 => 'flv,h264', + 22 => 'mp4,h264,720p', + 18 => 'mp4,h264', + 5 => 'flv,flv', + ); + my $preflist; + my $jsp; $url =~ m|$pattern|; $url = $1; @@ -50,30 +89,134 @@ sub _parse { $metadata->{'SOURCE'} = $self->{'NAME'}; $metadata->{'TITLE'} = undef; $metadata->{'DLURL'} = undef; + $metadata->{'COOKIE'} = undef; - unless(defined($content = LWP::Simple::get(sprintf('http://youtube.com/watch?v=%s', $2)))) { + $self->debug("Matched id %s from pattern %s", $2, $pattern); + + $preflist = $preflist{$quality}; + $self->debug("Quality: %s, preflist: [%s]", $quality, join(", ", @{$preflist})); + + $videourl = sprintf('https://www.youtube.com/watch?v=%s', $2); + + unless(defined($r = $ua->get($videourl))) { $self->error('Could not download %s', $url); return undef; } + if ($r->base->as_string() =~ m,/verify_age,) { + $self->debug('Video requires age verification'); + $ua->cookie_jar($jar); + my @logindata = $self->__login($videourl, $ua); + $r = $logindata[0]; + $metadata->{'COOKIE'} = $logindata[1]; + unless(defined($r)) { + $self->error('Could not log into YouTube'); + return undef; + } + } + $content = $r->content(); + $p = HTML::TokeParser->new(\$content); - while ($tag = $p->get_tag('div', 'meta', 'script')) { + SWF_ARGS: while ($tag = $p->get_tag('div', 'meta', 'script')) { if ('meta' eq $tag->[0]) { - if ('title' eq $tag->[1]->{'name'}) { + if (exists($tag->[1]->{'name'}) and ('title' eq $tag->[1]->{'name'})) { $metadata->{'TITLE'} = $tag->[1]->{'content'}; + # Convert HTML entities in the title. This is a bit convoluted. + $metadata->{'TITLE'} = encode("utf8", + decode_entities( + decode("utf8", $metadata->{'TITLE'}))); + $self->debug('Title found: %s', $metadata->{'TITLE'}); } } elsif ('script' eq $tag->[0]) { + my %urls; + $e = $p->get_text(); - if ($e =~ m|/watch_fullscreen\?(.+)\x27|) { - my %args = map { split(/=/, $_, 2); } split(/&/, $1); - $metadata->{'DLURL'} = sprintf('http://www.youtube.com/get_video.php?video_id=%s&t=%s', - $metadata->{'ID'}, $args{'t'}); + $self->debug("Found script: %s", $e); + + if ($e =~ m|\x27SWF_ARGS\x27:\s+(.+),|) { + my $args = $1; + + $self->debug("Found SWF_ARGS: %s", $args); + $jsp = videosite::JSArrayParser->new(); + $self->debug("Using %s to parse", ref($jsp)); + $r = $jsp->parse($args); + + unless(defined($r)) { + $self->error("Found information hash, but could not parse"); + return undef; + } + + if (exists($r->{'fmt_url_map'}) and ($r->{'fmt_url_map'} ne '')) { + my $urls = $r->{'fmt_url_map'}; + + $self->debug("Video has fmt_url_map: %s", $urls); + + $urls =~ s/%([[:xdigit:]]{2})/chr(hex($1))/ge; + %urls = split(/[\|,]/, $urls); + $self->debug("Pagetype: old (SWF_ARGS), fmt_url_map"); + + } elsif (exists($r->{'t'}) and ($r->{'t'} ne '')) { + my $thash = $r->{'t'}; + + if (exists($r->{'fmt_map'}) && ($r->{'fmt_map'} ne '')) { + my $fmt = $r->{'fmt_map'}; + my @fmt; + + $self->debug('Video has fmt_map'); + $fmt =~ s/%([[:xdigit:]]{2})/chr(hex($1))/ge; + @fmt = split(/,/, $fmt); + foreach (@fmt) { + @_=split(/\//); + $urls{$_[0]} = sprintf('http://www.youtube.com/get_video?video_id=%s&fmt=%d&t=%s', + $metadata->{'ID'}, + $_[0], + $thash); + } + $self->debug("Pagetype: 2009 (SWF_ARGS), t with fmt_map"); + + } else { + $urls{5} = sprintf('http://www.youtube.com/get_video?video_id=%s&t=%s', + $metadata->{'ID'}, + $thash); + $self->debug("Pagetype: 2009 (SWF_ARGS), t without fmt_map"); + } + } else { + $self->error('Neither fmt_url_map nor t found in video information hash'); + return undef; + } + } elsif ($e =~ m|var swfHTML = .*fmt_url_map=([^\&]+)\&|) { + my $urls = $1; + $self->debug("Video has fmt_url_map: %s", $urls); + + $urls =~ s/%([[:xdigit:]]{2})/chr(hex($1))/ge; + %urls = split(/[\|,]/, $urls); + $self->debug("Pagetype: 2010 (swfHTML), fmt_url_map"); + } + + if (%urls) { + foreach (keys(%urls)) { + if (exists($videoformats{$_})) { + $self->debug('Found URL for format %s (%s): %s', $_, $videoformats{$_}, $urls{$_}); + } else { + $self->error('Unknown format %s: %s', $_, $urls{$_}); + } + } + + foreach (@{$preflist}) { + if (exists($urls{$_})) { + $self->debug("Selected URL with quality level %s", $_); + $metadata->{'DLURL'} = $urls{$_}; + last; + } + } + $self->debug('URL found: %s', $metadata->{'DLURL'}); + last SWF_ARGS; } } elsif ('div' eq $tag->[0]) { - if ('errorBox' eq $tag->[1]->{'class'}) { + if (exists($tag->[1]->{'class'}) and ('yt-alert-content' eq $tag->[1]->{'class'})) { $self->error("Could not get video data for youtube %s: %s", $metadata->{'ID'}, $p->get_trimmed_text()); return undef; @@ -89,4 +232,112 @@ sub _parse { return $metadata; } +sub __login { + my $self = shift; + my $videourl = shift; + my $ua = shift; + my $user = $self->_getval('USERNAME'); + my $pass = $self->_getval('PASSWORD'); + my $r; + my $p; + my $c; + my $token; + + sub check_cookie { + + my $jar = shift; + my $key = shift; + my $found = undef; + + $jar->scan(sub { $found = 1 if ( $key eq $_[1]) }); + + return $found; + } + + sub get_all_cookies { + + my $jar = shift; + my $key = shift; + my $val = ""; + $jar->scan(sub { $val .= "; " if !( $val eq "" ); $val .= "$_[1]=$_[2]" }); + + return $val; + } + + if (($user eq '') or ($pass eq '')) { + $self->error('No username or password defined for YouTube'); + return undef; + } + + $self->debug('Logging in'); + $r = $ua->get('https://www.google.com/accounts/ServiceLoginAuth?service=youtube'); + unless($r->is_success()) { + $self->debug("Could not get login page (make sure your LWP supports HTTPS!)"); + return undef; + } + $c = $r->decoded_content(); + $p = HTML::TokeParser->new(\$c); + while (my $tag = $p->get_tag('input')) { + $self->debug("%s", Dumper($tag)); + if ($tag->[1]{name} eq 'GALX') { + $token = $tag->[1]{value}; + last; + } + } + $self->debug("GALX = %s", $token); + $r = $ua->post('https://www.google.com/accounts/ServiceLoginAuth?service=youtube', { 'service' => 'youtube', 'Email' => $user, 'Passwd' => $pass, 'GALX' => $token }); + unless($r->is_success()) { + $self->debug("Could not get login page (make sure your LWP supports HTTPS!)"); + return undef; + } + $c = $r -> decoded_content(); + $p = HTML::TokeParser->new(\$c); + while (my $tag = $p->get_tag('script')) { + if($p->get_text() =~ /location\.replace\("(.+)"\)/) { + $token = $1; + $token =~ s/\\x([A-Fa-f0-9]{2})/pack('C', hex($1))/seg; + last; + } + } + $r = $ua->get($token); + unless(check_cookie($ua->cookie_jar, 'LOGIN_INFO')) { + $self->error('Could not log into YouTube'); + return undef; + } + + $self->debug("Got a cookie"); + + $r = $ua->get($videourl); + if ($r->base->as_string() =~ m,/verify_age,) { + $self->debug("Looking for session token..."); + $c = $r->decoded_content(); + $p = HTML::TokeParser->new(\$c); + while (my $tag = $p->get_tag('script')) { + if ($p->get_text() =~ /'XSRF_TOKEN': '(.+)'/) { + $token = $1; + last; + } + } + + unless(defined($token)) { + $self->error("Could not find session token"); + return undef; + } + + $self->debug('Authenticating session...'); + $r = $ua->post($r->base->as_string, { 'next_url' => $r->base->path, 'action_confirm' => 'Confirm Birth Date', 'session_token' => $token }); + } + +# Apparently there is no longer a specific "is_adult" cookie +# or, by the looks of it, anything similar +# +# unless(check_cookie($ua->cookie_jar, 'is_adult')) { +# $self->error('Could not authenticate session'); +# return undef; +# } + + my $cookie = get_all_cookies($ua->cookie_jar); + return ($ua->get($videourl), $cookie); +} + 1;