Youtube: Simplify regex for scraping

[videosite.git] / videosite / Base.pm

diff --git a/videosite/Base.pm b/videosite/Base.pm
index 93ac841..f6108f6 100644 (file)
--- a/videosite/Base.pm
+++ b/videosite/Base.pm
@@ -10,10 +10,13 @@ use Data::Dumper;
 
 sub new {
     my $class = shift;
-    my $self = {'_DEBUG' => 0, '_OUT' => sub {print shift}};
+    my $self = {'_DEBUG' => 0,
+                '_OUT' => sub {printf(@_)},
+                '_CONNECTORS' => sub { return ({ 'name' => 'direct',
+                                                 'schemas' => {} }) },
+                '_CONNECTOR' => undef,
+               };
     
-    $self->{_ua} = LWP::UserAgent->new('agent' => 'Mozilla/5.0', 'cookie_jar' => HTTP::Cookies->new);
-
     bless($self, $class);
 
     $self->_prepare_parameters();
@@ -23,9 +26,11 @@ sub new {
 
 sub error {
     my $self = shift;
-    my $t;
+    my @data = @_;
+
+    $data[0] = "(" . ref($self) . ") " . $data[0];
 
-    $self->{'_OUT'}(@_);
+    $self->{'_OUT'}(@data);
 }
 
 sub debug {
@@ -172,4 +177,103 @@ sub setdebug {
     $self->{'_DEBUG'} = shift;
 }
 
+sub ua {
+    my $self = shift;
+    my $ua;
+
+    $ua = LWP::UserAgent->new(
+            'agent' => 'Mozilla/5.0',
+            'cookie_jar' => HTTP::Cookies->new,
+            'timeout' => 15,
+            );
+
+    # Remove a currently defined HTTPS proxy. See below for a longer explanation.
+    delete($ENV{'HTTPS_PROXY'});
+
+    if (defined($self->{'_CONNECTOR'})) {
+        my $schemas = $self->{'_CONNECTOR'}->{'schemas'};
+        foreach (keys(%{$schemas})) {
+            $self->debug("Adding schema %s with proxy %s", $_, $schemas->{$_});
+            if ($_ eq 'https') {
+                # OK, so here's the gist.
+                #
+                # The usual way of reqesting an HTTPS URL through a proxy is
+                # to connect to the proxy server, issue a CONNECT request to 
+                # create a channel to the web server and start an SSL session over
+                # this channel, so there is an end-to-end connection between
+                # the client and the server.
+                #
+                # Setting a proxy for the https schema in LWP WILL NOT ACCOMPLISH
+                # THIS.
+                #
+                # LWP will connect to the proxy server, and issue a standard GET
+                # request for the target URL, which most proxy servers will refuse
+                # to get.
+                #
+                # The way to use a proxy server is to set some environment variables
+                # and let the underlying Crypt::SSLeay module do the rest.
+                #
+                # This is positively appaling.
+                $ENV{'HTTPS_PROXY'} = $schemas->{$_};
+            } else {
+                $ua->proxy($_, $schemas->{$_});
+            }
+        }
+    }
+
+    $self->{_CACHED_UA} = $ua;
+
+    return $ua;
+}
+
+sub _cached_ua {
+    my $self = shift;
+
+    return $self->{_CACHED_UA};
+}
+
+sub simple_get {
+    my $self = shift;
+    my $url = shift;
+    my $ua = shift || $self->ua();
+    my $r;
+
+    $r = $ua->get($url);
+    return $r->decoded_content() if $r->is_success();
+    return undef;
+}
+
+sub decode_hexurl {
+    my $self = shift;
+    my $d = shift;
+
+    $d =~ s/%([[:xdigit:]]{2})/chr(hex($1))/ge;
+
+    return $d;
+}
+
+sub decode_querystring {
+    my $self = shift;
+
+    return { map { split /=/, $_, 2; } split /&/, shift };
+}
+
+sub connectors {
+    my $self = shift;
+    
+    return $self->{'_CONNECTORS'}->();
+}
+
+sub selectconn {
+    my $self = shift;
+
+    $self->{'_CONNECTOR'} = shift;
+}
+
+sub setconn {
+    my $self = shift;
+
+    $self->{'_CONNECTORS'} = shift;
+}
+
 1;