X-Git-Url: https://git.camperquake.de/gitweb.cgi?a=blobdiff_plain;f=videosite%2FYouTubeGrabber.pm;h=174f76d2dd94e7e5ecba3aee927cdafa70eb918b;hb=77303721994d561545a7a6457de9377292f7710f;hp=006ec17b1366cbe568a9b75a3dece8e8afbf2ae3;hpb=89b8c37caad7b96532e77592e82ff1585991bb9a;p=videosite.git diff --git a/videosite/YouTubeGrabber.pm b/videosite/YouTubeGrabber.pm index 006ec17..174f76d 100644 --- a/videosite/YouTubeGrabber.pm +++ b/videosite/YouTubeGrabber.pm @@ -6,13 +6,14 @@ # download strategy revised using # http://www.kde-apps.org/content/show.php?content=41456 -package YouTubeGrabber; +package videosite::YouTubeGrabber; -use GrabberBase; -@ISA = qw(GrabberBase); +use videosite::GrabberBase; +@ISA = qw(videosite::GrabberBase); -use LWP::Simple qw(!get); -use HTML::Parser; +use LWP::UserAgent; +use HTTP::Cookies; +use HTML::TokeParser; use Data::Dumper; use strict; @@ -24,6 +25,14 @@ sub new { $self->{'NAME'} = 'youtube'; $self->{'PATTERNS'} = ['(http://(?:[-a-zA-Z0-9_.]+\.)*youtube.(?:com|de|co.uk)/watch\?(?:.+=.+&)*v=([-a-zA-Z0-9_]+))', '(http://(?:[-a-zA-Z0-9_.]+\.)*youtube.(?:com|de|co.uk)/v/([-a-zA-Z0-9_]+))']; + $self->{'_PARAMS'} = { + 'QUALITY' => ['normal', 'Quality of the video to download.', { + 'normal' => 'standard resolution flash video', + 'high' => 'higher resolution flash video', + 'h264' => 'high resolution MPEG4 video', + 'hd' => 'HD720 resolution'}], + 'USERNAME' => ['', 'Username to use for YouTube login'], + 'PASSWORD' => ['', 'Password to use for YouTube login']}; bless($self, $class); $self->_prepare_parameters(); @@ -37,10 +46,15 @@ sub _parse { my $pattern = shift; my $content; my $metadata = {}; - my $p = HTML::Parser->new(api_version => 3); - my @accum; - my @text; + my $p; my $e; + my $tag; + my $jar = HTTP::Cookies->new; + my $ua = LWP::UserAgent->new('agent' => 'Mozilla/5.0'); + my $r; + my $videourl; + my $quality = $self->_getval('QUALITY'); + my $append = ''; $url =~ m|$pattern|; $url = $1; @@ -52,39 +66,54 @@ sub _parse { $metadata->{'TITLE'} = undef; $metadata->{'DLURL'} = undef; - unless(defined($content = LWP::Simple::get(sprintf('http://youtube.com/watch?v=%s', $2)))) { - $self->error('Could not download %s', $url); - return undef; + if ($quality eq 'high') { + $append = '&fmt=6'; + } elsif ($quality eq 'h264') { + $append = '&fmt=18'; + } elsif ($quality eq 'hd') { + $append = '&fmt=22'; } - if ($content =~ /This video has been removed due to terms of use violation/) { - $self->error('Video has been removed'); + $videourl = sprintf('http://www.youtube.com/watch?v=%s%s', $2, $append); + + unless(defined($r = $ua->get($videourl))) { + $self->error('Could not download %s', $url); return undef; } - $p->handler(start => \@accum, "tagname, attr"); - $p->handler(text => \@text, "text"); - $p->report_tags(qw(meta script)); - $p->utf8_mode(1); - $p->parse($content); - - # Look for the title in the meta tags - foreach $e (@accum) { - if ('meta' eq $e->[0]) { - if ('title' eq $e->[1]->{'name'}) { - $metadata->{'TITLE'} = $e->[1]->{'content'}; - $self->debug('Title found: %s', $metadata->{'TITLE'}); - } + if ($r->base->as_string() =~ m,/verify_age,) { + $self->debug('Video requires age verification'); + $ua->cookie_jar($jar); + $r = $self->__login($videourl, $ua); + unless(defined($r)) { + $self->error('Could not log into YouTube'); + return undef; } } + $content = $r->content(); + + $p = HTML::TokeParser->new(\$content); - # Look for the download URL - foreach $e (@text) { - if ($e->[0] =~ m|/watch_fullscreen\?(.+)\x27|) { - my %args = map { my @a = split(/=/); ($a[0], $a[1]) } split(/&/, $1); - $metadata->{'DLURL'} = sprintf('http://www.youtube.com/get_video.php?video_id=%s&t=%s', - $metadata->{'ID'}, $args{'t'}); + while ($tag = $p->get_tag('div', 'meta', 'script')) { + if ('meta' eq $tag->[0]) { + if ('title' eq $tag->[1]->{'name'}) { + $metadata->{'TITLE'} = $tag->[1]->{'content'}; + $self->debug('Title found: %s', $metadata->{'TITLE'}); + } + } elsif ('script' eq $tag->[0]) { + $e = $p->get_text(); + if ($e =~ m|/watch_fullscreen\?(.+)\x27|) { + my %args = map { split(/=/, $_, 2); } split(/&(?!amp;)/, $1); + $metadata->{'DLURL'} = sprintf('http://www.youtube.com/get_video.php?video_id=%s&t=%s%s', + $metadata->{'ID'}, $args{'t'}, $append); $self->debug('URL found: %s', $metadata->{'DLURL'}); + } + } elsif ('div' eq $tag->[0]) { + if (exists($tag->[1]->{'class'}) and ('errorBox' eq $tag->[1]->{'class'})) { + $self->error("Could not get video data for youtube %s: %s", + $metadata->{'ID'}, $p->get_trimmed_text()); + return undef; + } } } @@ -96,4 +125,68 @@ sub _parse { return $metadata; } +sub __login { + my $self = shift; + my $videourl = shift; + my $ua = shift; + my $user = $self->_getval('USERNAME'); + my $pass = $self->_getval('PASSWORD'); + my $r; + my $p; + my $c; + my $token; + + sub check_cookie { + + my $jar = shift; + my $key = shift; + my $found = undef; + + $jar->scan(sub { $found = 1 if ( $key eq $_[1]) }); + + return $found; + } + + if (($user eq '') or ($pass eq '')) { + $self->error('No username or password defined for YouTube'); + return undef; + } + + $self->debug('Logging in'); + $r = $ua->post('http://www.youtube.com/signup', { 'next' => '/', 'current_form' => 'login', 'action_login' => '1', 'username' => $user, 'password' => $pass }); + + unless(check_cookie($ua->cookie_jar, 'LOGIN_INFO')) { + $self->error('Could not log into YouTube'); + return undef; + } + + $r = $ua->get($videourl); + if ($r->base->as_string() =~ m,/verify_age,) { + $self->debug("Looking for session token..."); + $c = $r->decoded_content(); + $p = HTML::TokeParser->new(\$c); + while (my $tag = $p->get_tag('script')) { + if ($p->get_text() =~ /gXSRF_token = '(.+)'/) { + $token = $1; + last; + } + } + + unless(defined($token)) { + $self->error("Could not find session token"); + return undef; + } + + $self->debug('Authenticating session...'); + $r = $ua->post($r->base->as_string, { 'next_url' => $r->base->path, 'action_confirm' => 'Confirm Birth Date', 'session_token' => $token }); + } + + unless(check_cookie($ua->cookie_jar, 'is_adult')) { + $self->error('Could not authenticate session'); + return undef; + } + + return $ua->get($videourl); +} + 1;