# (c) 2007 by Ralf Ertzinger # licensed under GNU GPL v2 # # Grabber for polit-bash.org package quotesite::PolitBashOrgGrabber; use quotesite::GrabberBase; @ISA = qw(quotesite::GrabberBase); use LWP::Simple qw(!get); use HTML::TokeParser; use Data::Dumper; use Encode; use strict; sub new { my $class = shift; my $self = $class->SUPER::new(); $self->{'NAME'} = 'polit-bash.org'; $self->{'PATTERNS'} = ['(http://(?:[-a-zA-Z0-9_.]+\.)*polit-bash\.org/\?(\d+))']; bless($self, $class); $self->_prepare_parameters(); return $self; } sub _parse { my $self = shift; my $url = shift; my $pattern = shift; my $content; my $metadata = {}; my $p; my $t; my $t2; $url =~ m|$pattern|; $url = $1; $metadata->{'URL'} = $url; $metadata->{'ID'} = $2; $metadata->{'TYPE'} = 'quote'; $metadata->{'SOURCE'} = $self->{'NAME'}; $metadata->{'CONTENT'} = undef; # Get the HTML file containing the quote unless(defined($content = LWP::Simple::get(sprintf('http://polit-bash.org/?%s', $2)))) { $self->error('Could not download quote'); return undef; } $content = decode("iso-8859-1", $content); $p = HTML::TokeParser->new(\$content); OUTER: while ($t = $p->get_tag('p')) { if (exists($t->[1]->{'class'}) && ($t->[1]->{'class'} eq 'quote')) { $self->debug("Found p"); $metadata->{'CONTENT'} = $p->get_text('/p'); last OUTER; } } unless(defined($metadata->{'CONTENT'})) { $self->error('Could not extract quote content'); return undef; } $metadata->{'CONTENT'} = encode("utf-8", $metadata->{'CONTENT'}); return $metadata; } 1;