From fc449da24ace2ba9ec5304afaae579d9fe55e967 Mon Sep 17 00:00:00 2001 From: Ralf Ertzinger Date: Tue, 30 Apr 2013 15:49:09 +0200 Subject: [PATCH] Add libvideosite.pm --- libvideosite.pm | 1047 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1047 insertions(+) create mode 100644 libvideosite.pm diff --git a/libvideosite.pm b/libvideosite.pm new file mode 100644 index 0000000..4920801 --- /dev/null +++ b/libvideosite.pm @@ -0,0 +1,1047 @@ +package libvideosite; +require Exporter; + +use vars qw(@ISA @EXPORT_OK); +use File::Spec; +use Module::Load; +use LWP::UserAgent; +use Data::Dumper; +use File::Basename; +use Cwd qw(realpath); +use strict; + +@ISA = qw(Exporter); +@EXPORT_OK = qw(init register_api check_for_link); + +my @outputstack; +my $outputprefix; +my $debug = 0; +my @grabbers; +my @getters; +my $getter; +my %builtin_config = (); +our $error; + +# +# The default config. These values will be set in the config +# if they do not exist already. +# +my $defaultconfig = { + 'getter' => 'filegetter', + 'mode' => 'download', + 'active-connectors' => 'direct', + 'defined-connectors' => 'direct', + 'connectors' => { + 'direct' => { + 'name' => 'direct', + '_immutable' => '1', + 'schemas' => {}, + } + } +}; + +# +# List of known commands and handlers +# +my $videosite_commands = { + 'save' => sub { + _cmd_save(); + }, + + 'set' => sub { + _cmd_set(@_); + }, + + 'show' => sub { + _cmd_show(@_); + }, + + 'help' => sub { + _cmd_help(@_); + }, + + 'getter' => sub { + _cmd_getter(@_); + }, + + 'enable' => sub { + _cmd_enable(@_); + }, + + 'disable' => sub { + _cmd_disable(@_); + }, + + 'reload' => sub { + init(); + }, + + 'mode' => sub { + _cmd_mode(@_); + }, + + 'connector' => sub { + _cmd_connector(@_); + }, + + 'debug' => sub { + $debug = 1; + foreach (@grabbers, @getters) { + $_->setdebug(1); + } + _io('Enabled debugging'); + }, + + 'nodebug' => sub { + $debug = 0; + foreach (@grabbers, @getters) { + $_->setdebug(0); + } + _io('Disabled debugging'); + }, +}; + +# +# This is a list of default values for the remote API. These +# are used if the values are not registered by the library user. +# +my $remote_api = { + io => sub { print @_, "\n" }, + config_init => \&_builtin_config_init, + config_get => \&_builtin_config_get, + config_set => \&_builtin_config_set, + config_has => \&_builtin_config_has, + config_save => \&_builtin_config_save, + config_del => \&_builtin_config_del, + color => sub { return '' }, + module_path => sub { return dirname(realpath($0)) }, + quote => sub { return $_ }, +}; + +# +# Output a string on the client. +# Works like (s)printf in that it takes a format string and a list of +# values to be replaced. Undefined values will be printed as '(undef)' +# +# All parameters (except for the format string itself) will be quoted +# using the client specific quote function +# +sub _io { + my @text = @_; + my $format; + + # This will define the outputprefix once, so we don't have + # do do this every time. + $outputprefix //= sprintf("%svideosite: %s", + $remote_api->{color}->('magenta'), + $remote_api->{color}->()); + $format = $outputprefix . shift(@text); + + # + # The format string is assumed to be appropriately quoted. + # Quote the rest of the text, replacing undefined strings by (undef) + # + @text = map { defined($_)?$remote_api->{quote}->($_):'(undef)' } @text; + + $outputstack[0]->(sprintf($format, @text)); +} + +# +# Recursively walk through a hash-of-hashes, calling the given function +# for each found leaf with the path to the leaf +# +sub _recursive_hash_walk { + my $hash = shift; + my $callback = shift; + my @path = @_; + + foreach (keys(%{$hash})) { + if (ref($hash->{$_}) eq 'HASH') { + _recursive_hash_walk($hash->{$_}, $callback, @path, $_); + } else { + $callback->([@path, $_], $hash->{$_}); + } + } +} + +# +# Return the color code for the given foreground/background color +# pair. Both can be undef, which means "default" +# +sub _colorpair { + my ($fg, $bg) = @_; + + return $remote_api->{color}->($fg, $bg); +} + +# +# Sets the given config item if it is not set already +# +sub _init_config_item { + my $path = shift; + my $value = shift; + + unless($remote_api->{config_has}->($path)) { + $remote_api->{config_set}->($path, $value); + } +} + +# +# Print a message if debug is enabled +# +sub _debug { + if ($debug) { + _io(@_); + } +} + +# +# Load a list of modules matching a pattern from a given directory. +# +sub _ploader { + + my $dir = shift; + my $pattern = shift; + my $type = shift; + my @list; + my $p; + my $g; + my @g = (); + + opendir(D, $dir) || return (); + @list = grep {/$pattern/ && -f File::Spec->catfile($dir, $_) } readdir(D); + closedir(D); + + foreach $p (@list) { + _debug("Trying to load $p:"); + $p =~ s/\.pm$//; + eval { + load "videosite::$p"; + }; + if ($@) { + _io("Failed to load plugin: $@"); + next; + } + + eval { + $g = "videosite::$p"->new(); + }; + if ($@) { + _io("Failed to instanciate: $@"); + delete($INC{$p}); + next; + } + + _debug("found $g->{'TYPE'} $g->{'NAME'}"); + if ($type eq $g->{'TYPE'}) { + push(@g, $g); + $g->register_api({ + io => \&_io, + connectors => sub { return _connectorlist('active-connectors') }, + config_get => \&_config_get, + config_set => \&_config_set, + config_has => \&_config_has, + }); + $g->setdebug($debug); + } else { + _io('%s has wrong type (got %s, expected %s)', $p, $g->{'TYPE'}, $type); + delete($INC{$p}); + } + } + + _debug("Loaded %d plugins", $#g+1); + + return @g; +} + +# +# Populate the @grabbers and @getters lists from the given +# directory +# +sub _load_modules($) { + + my $path = shift; + + foreach (keys(%INC)) { + if ($INC{$_} =~ m|^$path|) { + _debug("Removing %s from \$INC", $_); + delete($INC{$_}); + } + } + @grabbers = _ploader($path, '.*Grabber\.pm$', 'grabber'); + @getters = _ploader($path, '.*Getter\.pm$', 'getter'); +} + +# +# Wrapper functions for config management to put in +# debugging +# +sub _config_get { + my $path = shift; + my $value; + + $value = $remote_api->{config_get}->($path); + _debug("config: getting %s=%s", join('.', @{$path}), $value); + + return $value; +} + +sub _config_set { + my $path = shift; + my $value = shift; + + _debug("config: setting %s=%s", join('.', @{$path}), $value); + return $remote_api->{config_set}->($path, $value); +} + +sub _config_has { + my $path = shift; + my $b; + + $b = $remote_api->{config_has}->($path); + _debug("config: testing %s (%s)", join('.', @{$path}), $b?'true':'false'); + + return $b; +} + +# +# The _config_list_* are helper functions taking a path to a comma separated +# string. The string is interpreted as a list and the action performed +# on it, storing back the modified version +# + +# +# Add an item to the list, checking for duplicates +# +sub _config_list_add { + my $path = shift; + my $item = shift; + my @c; + + if (_config_has($path)) { + @c = split(/\s*,\s*/, _config_get($path)); + } else { + @c = (); + } + + _debug("Adding %s to list %s", $item, join(".", $path)); + unless(grep { $_ eq $item } @c) { + push(@c, $item); + }; + + _config_set($path, join(',', @c)); +} + +# +# Remove an item from the list +# +sub _config_list_del { + my $path = shift; + my $item = shift; + my @c; + + unless(_config_has($path)) { + return; + } + + _debug("Removing %s from list %s", $item, join('.', $path)); + @c = map { $item ne $_ } split(/\s*,\s*/, _config_get($path)); + + _config_set($path, join('.', @c)); +} + +# +# Return true if the item contains the given list, false otherwise +# +sub _config_list_has { + my $path = shift; + my $item = shift; + + unless(_config_has($path)) { + return 0; + } + + _debug("Checking for %s in list %s", $item, join('.', $path)); + + return grep { $item eq $_ } split(/\s*,\s*/, _config_get($path)); +} + +# +# Replace a list with the given items +# +sub _config_list_set { + my $path = shift; + + _debug("Replacing %s with (%s)", join('.', $path), join(",", @_)); + + _config_set($path, join(',', @_)); +} + +# +# Return the list of currently active connectors, in the configured +# order +# +sub _connectorlist { + my $key = shift; + my @c; + + foreach(split(/,/, _config_get([$key]))) { + push(@c, _unserialize_connector_hash($_)); + } + + return @c; +} + +# +# Convert a connector hash from it's config structure back to a perl +# hash +# +sub _unserialize_connector_hash { + my $name = shift; + my $connector = {}; + + if (_config_has(['connectors', $name, 'name'])) { + $connector->{name} = _config_get(['connectors', $name, 'name']); + $connector->{schemas} = {}; + foreach ('http', 'https') { + if (_config_has(['connectors', $name, 'schemas', $_])) { + $connector->{schemas}->{$_} = _config_get(['connectors', $name, 'schemas', $_]); + } + } + } + + _debug("Returning connector %s: %s", $name, Dumper($connector)); + + return $connector; +} + +# +# Push a new output function on the IO stack. +# +sub _push_output { + unshift(@outputstack, shift); +} + +# +# Pop the topmost output function from the stack, leaving +# at least one function on it. +# +sub _pop_output { + if (scalar(@outputstack) > 0) { + shift(@outputstack); + } +} + +# +# Takes a string and replaces commonly used URL shorteners recursively, +# up to 10 levels deep +# +sub _expand_url_shortener { + my $s = shift; + my $os = ''; + my @urlshortener = ( + 'is\.gd/[[:alnum:]]+', + 'otf\.me/[[:alnum:]]+', + 'hel\.me/[[:alnum:]]+', + '7ax\.de/[[:alnum:]]+', + 'ow\.ly/[[:alnum:]]+', + 'j\.mp/[[:alnum:]]+', + 'bit\.ly/[[:alnum:]]+', + 'tinyurl\.com/[[:alnum:]]+', + 'pop\.is/[[:alnum:]]+', + 'post\.ly/[[:alnum:]]+', + '1\.ly/[[:alnum:]]+', + '2\.ly/[[:alnum:]]+', + 't\.co/[[:alnum:]]+', + 'shar\.es/[[:alnum:]]+', + 'goo\.gl/[[:alnum:]]+', + ); + my $ua = LWP::UserAgent->new(agent => 'Mozilla', max_redirect => 0, timeout => 5); + my $i = 10; + + OUTER: while (($os ne $s) and ($i > 0)) { + study($s); + $os = $s; + $i--; + + foreach my $pattern (@urlshortener) { + my $p = "https?:\/\/" . $pattern; + + _debug("Matching %s against %s", $p, $s); + if ($s =~ m|($p)|) { + my $matched = $1; + my $res; + + _debug("Found %s", $matched); + $res = $ua->head($matched); + if ($res->is_redirect()) { + my $new = $res->headers()->header("Location"); + + _debug("Replacing %s with %s", $matched, $new); + $s =~ s/$matched/$new/; + next OUTER; + } else { + _debug("Error resolving %s", $matched); + } + } + } + } + + if ($i == 0) { + _debug("Loop terminated by counter"); + } + + _debug("Final string: %s", $s); + + return $s; +} + +# +# Save the config to durable storage +# +sub _cmd_save { + $remote_api->{config_save}->(); +} + +# +# Set a configuration element +# +sub _cmd_set { + my $target = shift; + my $key = shift; + my $val = shift; + my $p; + + foreach $p (@getters, @grabbers) { + if ($p->{'NAME'} eq $target) { + $p->setval($key, $val); + return; + } + } + _io('No such module'); +} + + +# +# Enable a given module +# +sub _cmd_enable { + my $target = shift; + my $p; + + foreach $p (@grabbers) { + if ($p->{'NAME'} eq $target) { + $p->enable(); + return; + } + } + _io('No such module'); +} + +# +# Disable given module +# +sub _cmd_disable { + my $target = shift; + my $p; + + foreach $p (@grabbers) { + if ($p->{'NAME'} eq $target) { + $p->disable(); + return; + } + } + _io('No such module'); +} + +# +# Show settings for modules +# +sub _cmd_show { + my $target = shift; + my $p; + my $e; + + if (defined($target)) { + foreach $p (@getters, @grabbers) { + if ($p->{'NAME'} eq $target) { + _io($p->getconfstr()); + return; + } + } + _io('No such module'); + } else { + _io('Loaded grabbers (* denotes enabled modules):'); + foreach $p (@grabbers) { + $e = $p->_getval('enabled'); + _io(' %s%s', $p->{'NAME'}, $e?'*':''); + }; + + _io('Loaded getters:'); + foreach $p (@getters) { + _io(' %s', $p->{'NAME'}); + }; + } +} + +# +# Show help for the commands +# +sub _cmd_help { + my $target = shift; + my $p; + + if (defined($target)) { + foreach $p (@getters, @grabbers) { + if ($p->{'NAME'} eq $target) { + _io($p->gethelpstr()); + return; + } + } + _io('No such module'); + } else { + _io(<<'EOT'); +Supported commands: + save: save the current configuration + help [modulename]: display this help, or module specific help + show [modulename]: show loaded modules, or the current parameters of a module + set modulename parameter value: set a module parameter to a new value + getter [modulename]: display or set the getter to use + enable [modulename]: enable the usage of this module (grabbers only) + disable [modulename]: disable the usage of this module (grabbers only) + reload: reload all modules (this is somewhat experimental) + mode [modename]: display or set the operation mode (download/display) + connector [subcommand]: manage connectors (proxies) + debug: enable debugging messages + nodebug: disable debugging messages +EOT + } +} + +# +# Set the getter to use +# +sub _cmd_getter { + my $target = shift; + my $p; + + if (defined($target)) { + foreach $p (@getters) { + if ($p->{'NAME'} eq $target) { + $getter = $p; + _config_set(['getter'], $target); + _io("Getter changed to %s", $target); + return; + } + } + _io('No such getter'); + } else { + _io('Current getter: %s', _config_get(['getter'])); + } +} + +# +# Show/set the working mode +# +sub _cmd_mode { + my $mode = shift; + + if (defined($mode)) { + $mode = lc($mode); + if (('download' eq $mode) or ('display' eq $mode)) { + _config_set(['mode'], $mode); + _io('Now using %s mode', $mode); + } else { + _io('Invalid mode: %s', $mode); + } + } else { + _io('Current mode: %s', _config_get(['mode'])); + } +} + + +# +# Manage the connectors +# +sub _cmd_connector { + my $subcmd = shift; + my $c; + + unless(defined($subcmd)) { + $subcmd = "help"; + } + + $subcmd = lc($subcmd); + + if ($subcmd eq 'list') { + _io("Defined connectors"); + foreach $c (_connectorlist('defined-connectors')) { + _io($c->{name}); + my $schemas = $c->{schemas}; + if (scalar(keys(%{$schemas})) == 0) { + _io(" No schemas defined"); + } else { + foreach (keys(%{$schemas})) { + _io(' %s: %s', $_, $schemas->{$_}); + } + } + } + + _io(); + _io("Selected connectors: %s", _config_get(['active-connectors'])); + } elsif ($subcmd eq 'add') { + my ($name) = @_; + + unless(defined($name)) { + _io("No name given"); + return; + } + + $name = lc($name); + + if (_config_list_has(['defined-connectors'], $name)) { + _io("Connector already exists"); + return; + } + + _config_set(['connectors', $name, 'name'], $name); + _config_list_add(['defined-connectors'], $name); + } elsif ($subcmd eq 'del') { + my ($name) = @_; + my @dcon; + + unless(defined($name)) { + _io("No name given"); + return; + } + + unless (_config_list_has(['defined-connectors'])) { + _io("Connector does not exist"); + return; + } + + if (_config_has(['connectors', $name, '_immutable'])) { + _io("Connector cannot be removed"); + return; + } + + # Remove from list of active connectors + _config_list_del(['defined-connectors'], $name); + _config_list_del(['active-connectors'], $name); + + _config_del(['connectors', $name, 'name']); + _config_del(['connectors', $name, '_immutable']); + _config_del(['connectors', $name, 'schemas', 'http']); + _config_del(['connectors', $name, 'schemas', 'https']); + + @dcon = split(/,/, _config_get(['active-connectors'])); + + if (scalar(@dcon) == 0) { + _io("List of selected connectors is empty, resetting to direct"); + _config_list_add(['active-connectors', 'direct']); + } + } elsif ($subcmd eq 'addschema') { + my ($conn, $schema, $proxy) = @_; + + unless(defined($conn)) { + _io("No connector name given"); + return; + } + + unless(defined($schema)) { + _io("No schema given"); + return; + } + + unless(defined($proxy)) { + _io("No proxy given"); + return; + } + + $conn = lc($conn); + unless(_config_list_has(['defined-connectors'], $conn)) { + _io("Connector does not exist"); + return; + } + + if (_config_has(['connectors', $conn, '_immutable'])) { + _io("Connector cannot be modified"); + return; + } + + $schema = lc($schema); + _config_set(['connectors', $conn, 'schemas', $schema], $proxy); + } elsif ($subcmd eq 'delschema') { + my ($conn, $schema) = @_; + + unless(defined($conn)) { + _io("No connector name given"); + return; + } + + unless(defined($schema)) { + _io("No schema given"); + return; + } + + $conn = lc($conn); + unless(_config_list_has(['defined-connectors'], $conn)) { + _io("Connector does not exist"); + return; + } + + $schema = lc($schema); + _config_del(['connectors', $conn, 'schemas', $schema]); + } elsif ($subcmd eq 'select') { + my @connlist = map { lc } @_; + + if (scalar(@connlist) == 0) { + _io("No connectors given"); + return; + } + + foreach (@connlist) { + unless(_config_list_has(['defined-connectors'], $_)) { + _io("Connector %s does not exist", $_); + return; + } + } + + _config_list_set(['active-connectors'], @connlist); + } else { + _io("connector [list|add|del|addschema|delschema|help] "); + _io(" help: Show this help"); + _io(" list: List the defined connectors"); + _io(" add : Add a connector with name "); + _io(" del : Delete the connector with name "); + _io(" addschema : Add proxy to connector for the given schema"); + _io(" delschema : Remove the schema from the connector"); + _io(" select [...]: Select the connectors to use"); + } +} + +# +# Return the list of loaded grabbers. +# This is used by the test programs, and is not meant to be +# used in general. +# +sub _grabbers { + return @grabbers; +} + +# +# ============================================== +# Builtin config handling functions +# These are used if the library used does not +# register it's own config_* handlers +# ============================================== +# +sub _builtin_config_init { +} + +sub _builtin_config_get { + return $builtin_config{join(".", @{$_[0]})}; +} + +sub _builtin_config_set { + $builtin_config{join(".", @{$_[0]})} = $_[1]; +} + +sub _builtin_config_has { + return exists($builtin_config{join(".", @{$_[0]})}); +} + +sub _builtin_config_save { +} + +sub _builtin_config_del { + delete($builtin_config{join(".", @{$_[0]})}); +} + +# +# ============================================== +# From this point on publicly callable functions +# ============================================== +# + + +# +# Initialization function for the library +# Actually not the first thing to be called, it expects an API +# has (register_api) to be registered first +# +sub init { + unless($remote_api) { + $error = "No API set"; + return 0; + } + + # Initialize configuration data + $remote_api->{config_init}->(); + + # Check/create default values, if they do not exist + _recursive_hash_walk($defaultconfig, \&_init_config_item); + + # Load modules + _load_modules(File::Spec->catfile($remote_api->{module_path}->(), 'videosite')); + + unless (@grabbers && @getters) { + _io('No grabbers or no getters found, can not proceed.'); + return 0; + } + + # Set the getter + $getter = $getters[0]; + foreach my $p (@getters) { + if (_config_get(['getter']) eq $p->{'NAME'}) { + $getter = $p; + } + } + _debug('Selected %s as getter', $getter->{'NAME'}); + _config_set(['getter'], $getter->{'NAME'}); + + # Done. + _io('initialized successfully'); + return 1; +} + +# +# Register a remote API. This API contains a basic output function (used +# when no window specific function is available), some config functions +# and a color code function. +# +sub register_api { + my $a = shift; + my @config_functions = qw(config_init config_set config_get config_has config_save config_del); + my $c; + my @missing; + + unless(defined($a)) { + die("No API given"); + } + + # + # The config_* handlers are special in that they either all have + # provided by the user, or none. In the latter case builtin + # handlers will be used, but the config will not persist. + # + $c = 0; + foreach (@config_functions) { + if (exists($a->{$_})) { + $c++; + } else { + push(@missing, $_); + } + } + + unless (($c == 0) or ($c == scalar(@config_functions))) { + $error = sprintf("Missing config function: %s", $missing[0]); + return 0; + } + + foreach (keys(%{$a})) { + if (ref($a->{$_}) ne 'CODE') { + $error = sprintf("API handler %s is not a subroutine reference", $_); + } + $remote_api->{$_} = $a->{$_}; + } + + if (exists($a->{_debug})) { + $debug = $a->{_debug}->(); + } + + @outputstack = ($remote_api->{'io'}); + + return 1; +} + +# +# Check a message for useable links +# +sub check_for_link { + my $event = shift; + my $message = $event->{message}; + my $g; + my $m; + my $p; + + + # Look if we should ignore this line + if ($message =~ m,(?:\s|^)/nosave(?:\s|$),) { + return; + } + + _push_output($event->{ewpf}); + $message = _expand_url_shortener($message); + + study($message); + + # Offer the message to all Grabbers in turn + GRABBER: foreach $g (@grabbers) { + ($m, $p) = $g->get($message); + while (defined($m)) { + _debug('Metadata: %s', Dumper($m)); + if (exists($remote_api->{link_callback})) { + $remote_api->{link_callback}->($m); + } + if ('download' eq _config_get(['mode'])) { + _io( + sprintf('%s>>> %sSaving %s%%s%s %s%%s', + $remote_api->{color}->('*red'), + $remote_api->{color}->(), + $remote_api->{color}->('*yellow'), + $remote_api->{color}->(), + $remote_api->{color}->('*green'), + ), + $m->{'SOURCE'}, + $m->{'TITLE'} + ); + unless($getter->get($m)) { + _io(sprintf('%s>>> FAILED', $remote_api->{color}->('*red'))); + } + } elsif ('display' eq _config_get(['mode'])) { + _io( + sprintf('%s>>> %sSaw %s%%s%s %s%%s', + $remote_api->{color}->('*magenta'), + $remote_api->{color}->(), + $remote_api->{color}->('*yellow'), + $remote_api->{color}->(), + $remote_api->{color}->('*green') + ), + $m->{'SOURCE'}, + $m->{'TITLE'} + ); + } else { + _io(sprintf('%s>>> Invalid operation mode', $remote_api->{color}->('*red'))); + } + + # Remove the matched part from the message and try again (there may be + # more!) + $message =~ s/$p//; + study($message); + last GRABBER if ($message =~ /^\s*$/); + + ($m, $p) = $g->get($message); + } + } + + _pop_output(); +} + +# +# Handle a videosite command (/videosite ...) entered in the client +# +sub handle_command { + my $event = shift; + my ($cmd, @params) = split(/\s+/, $event->{message}); + + _push_output($event->{ewpf}); + + if (exists($videosite_commands->{$cmd})) { + $videosite_commands->{$cmd}->(@params); + } + + _pop_output(); +} + +1; -- 1.8.3.1