package libvideosite; require Exporter; use vars qw(@ISA @EXPORT_OK); use File::Spec; use Module::Load; use LWP::UserAgent; use Data::Dumper; use File::Basename; use Cwd qw(realpath); use strict; @ISA = qw(Exporter); @EXPORT_OK = qw(init register_api check_for_link); my @outputstack; my $outputprefix; my $debug = 0; my @grabbers; my @getters; my $getter; my %builtin_config = (); our $error; # # The default config. These values will be set in the config # if they do not exist already. # my $defaultconfig = { 'getter' => 'filegetter', 'mode' => 'download', 'active-connectors' => 'direct', 'defined-connectors' => 'direct', 'connectors' => { 'direct' => { 'name' => 'direct', '_immutable' => '1', 'schemas' => {}, } } }; # # List of known commands and handlers # my $videosite_commands = { 'save' => sub { _cmd_save(); }, 'set' => sub { _cmd_set(@_); }, 'show' => sub { _cmd_show(@_); }, 'help' => sub { _cmd_help(@_); }, 'getter' => sub { _cmd_getter(@_); }, 'enable' => sub { _cmd_enable(@_); }, 'disable' => sub { _cmd_disable(@_); }, 'reload' => sub { init(); }, 'mode' => sub { _cmd_mode(@_); }, 'connector' => sub { _cmd_connector(@_); }, 'debug' => sub { $debug = 1; foreach (@grabbers, @getters) { $_->setdebug(1); } _io('Enabled debugging'); }, 'nodebug' => sub { $debug = 0; foreach (@grabbers, @getters) { $_->setdebug(0); } _io('Disabled debugging'); }, }; # # This is a list of default values for the remote API. These # are used if the values are not registered by the library user. # my $remote_api = { io => sub { print @_, "\n" }, config_init => \&_builtin_config_init, config_get => \&_builtin_config_get, config_set => \&_builtin_config_set, config_has => \&_builtin_config_has, config_save => \&_builtin_config_save, config_del => \&_builtin_config_del, color => sub { return '' }, module_path => sub { return dirname(realpath($0)) }, quote => sub { return $_ }, }; # # Output a string on the client. # Works like (s)printf in that it takes a format string and a list of # values to be replaced. Undefined values will be printed as '(undef)' # # All parameters (except for the format string itself) will be quoted # using the client specific quote function # sub _io { my @text = @_; my $format; @text = ('') unless(@text); # This will define the outputprefix once, so we don't have # do do this every time. $outputprefix = sprintf("%svideosite: %s", $remote_api->{color}->('magenta'), $remote_api->{color}->()) unless(defined($outputprefix)); $format = $outputprefix . shift(@text); # # The format string is assumed to be appropriately quoted. # Quote the rest of the text, replacing undefined strings by (undef) # @text = map { defined($_)?$remote_api->{quote}->($_):'(undef)' } @text; $outputstack[0]->(sprintf($format, @text)); } # # Recursively walk through a hash-of-hashes, calling the given function # for each found leaf with the path to the leaf # sub _recursive_hash_walk { my $hash = shift; my $callback = shift; my @path = @_; foreach (keys(%{$hash})) { if (ref($hash->{$_}) eq 'HASH') { _recursive_hash_walk($hash->{$_}, $callback, @path, $_); } else { $callback->([@path, $_], $hash->{$_}); } } } # # Return the color code for the given foreground/background color # pair. Both can be undef, which means "default" # sub _colorpair { my ($fg, $bg) = @_; return $remote_api->{color}->($fg, $bg); } # # Sets the given config item if it is not set already # sub _init_config_item { my $path = shift; my $value = shift; unless($remote_api->{config_has}->($path)) { $remote_api->{config_set}->($path, $value); } } # # Print a message if debug is enabled # sub _debug { if ($debug) { _io(@_); } } # # Load a list of modules matching a pattern from a given directory. # sub _ploader { my $dir = shift; my $pattern = shift; my $type = shift; my @list; my $p; my $g; my @g = (); opendir(D, $dir) || return (); @list = grep {/$pattern/ && -f File::Spec->catfile($dir, $_) } readdir(D); closedir(D); foreach $p (@list) { _debug("Trying to load $p:"); $p =~ s/\.pm$//; eval { load "videosite::$p"; }; if ($@) { _io("Failed to load plugin: $@"); next; } eval { $g = "videosite::$p"->new(); }; if ($@) { _io("Failed to instanciate: $@"); delete($INC{$p}); next; } _debug("found $g->{'TYPE'} $g->{'NAME'}"); if ($type eq $g->{'TYPE'}) { push(@g, $g); $g->register_api({ io => \&_io, connectors => sub { return _connectorlist('active-connectors') }, config_get => \&_config_get, config_set => \&_config_set, config_has => \&_config_has, }); $g->setdebug($debug); } else { _io('%s has wrong type (got %s, expected %s)', $p, $g->{'TYPE'}, $type); delete($INC{$p}); } } _debug("Loaded %d plugins", $#g+1); return @g; } # # Populate the @grabbers and @getters lists from the given # directory # sub _load_modules($) { my $path = shift; foreach (keys(%INC)) { if ($INC{$_} =~ m|^$path|) { _debug("Removing %s from \$INC", $_); delete($INC{$_}); } } @grabbers = _ploader($path, '.*Grabber\.pm$', 'grabber'); @getters = _ploader($path, '.*Getter\.pm$', 'getter'); } # # Wrapper functions for config management to put in # debugging # sub _config_get { my $path = shift; my $value; $value = $remote_api->{config_get}->($path); _debug("config: getting %s=%s", join('.', @{$path}), $value); return $value; } sub _config_set { my $path = shift; my $value = shift; _debug("config: setting %s=%s", join('.', @{$path}), $value); return $remote_api->{config_set}->($path, $value); } sub _config_has { my $path = shift; my $b; $b = $remote_api->{config_has}->($path); _debug("config: testing %s (%s)", join('.', @{$path}), $b?'true':'false'); return $b; } # # The _config_list_* are helper functions taking a path to a comma separated # string. The string is interpreted as a list and the action performed # on it, storing back the modified version # # # Add an item to the list, checking for duplicates # sub _config_list_add { my $path = shift; my $item = shift; my @c; if (_config_has($path)) { @c = split(/\s*,\s*/, _config_get($path)); } else { @c = (); } _debug("Adding %s to list %s", $item, join(".", $path)); unless(grep { $_ eq $item } @c) { push(@c, $item); }; _config_set($path, join(',', @c)); } # # Remove an item from the list # sub _config_list_del { my $path = shift; my $item = shift; my @c; unless(_config_has($path)) { return; } _debug("Removing %s from list %s", $item, join('.', $path)); @c = map { $item ne $_ } split(/\s*,\s*/, _config_get($path)); _config_set($path, join('.', @c)); } # # Return true if the item contains the given list, false otherwise # sub _config_list_has { my $path = shift; my $item = shift; unless(_config_has($path)) { return 0; } _debug("Checking for %s in list %s", $item, join('.', $path)); return grep { $item eq $_ } split(/\s*,\s*/, _config_get($path)); } # # Replace a list with the given items # sub _config_list_set { my $path = shift; _debug("Replacing %s with (%s)", join('.', $path), join(",", @_)); _config_set($path, join(',', @_)); } # # Return the list of currently active connectors, in the configured # order # sub _connectorlist { my $key = shift; my @c; foreach(split(/,/, _config_get([$key]))) { push(@c, _unserialize_connector_hash($_)); } return @c; } # # Convert a connector hash from it's config structure back to a perl # hash # sub _unserialize_connector_hash { my $name = shift; my $connector = {}; if (_config_has(['connectors', $name, 'name'])) { $connector->{name} = _config_get(['connectors', $name, 'name']); $connector->{schemas} = {}; foreach ('http', 'https') { if (_config_has(['connectors', $name, 'schemas', $_])) { $connector->{schemas}->{$_} = _config_get(['connectors', $name, 'schemas', $_]); } } } _debug("Returning connector %s: %s", $name, Dumper($connector)); return $connector; } # # Push a new output function on the IO stack. # sub _push_output { unshift(@outputstack, shift); } # # Pop the topmost output function from the stack, leaving # at least one function on it. # sub _pop_output { if (scalar(@outputstack) > 0) { shift(@outputstack); } } # # Takes a string and replaces commonly used URL shorteners recursively, # up to 10 levels deep # sub _expand_url_shortener { my $s = shift; my $os = ''; my @urlshortener = ( 'is\.gd/[[:alnum:]]+', 'otf\.me/[[:alnum:]]+', 'hel\.me/[[:alnum:]]+', '7ax\.de/[[:alnum:]]+', 'ow\.ly/[[:alnum:]]+', 'j\.mp/[[:alnum:]]+', 'bit\.ly/[[:alnum:]]+', 'tinyurl\.com/[[:alnum:]]+', 'pop\.is/[[:alnum:]]+', 'post\.ly/[[:alnum:]]+', '1\.ly/[[:alnum:]]+', '2\.ly/[[:alnum:]]+', 't\.co/[[:alnum:]]+', 'shar\.es/[[:alnum:]]+', 'goo\.gl/[[:alnum:]]+', ); my $ua = LWP::UserAgent->new(agent => 'Mozilla', max_redirect => 0, timeout => 5); my $i = 10; OUTER: while (($os ne $s) and ($i > 0)) { study($s); $os = $s; $i--; foreach my $pattern (@urlshortener) { my $p = "https?:\/\/" . $pattern; _debug("Matching %s against %s", $p, $s); if ($s =~ m|($p)|) { my $matched = $1; my $res; _debug("Found %s", $matched); $res = $ua->head($matched); if ($res->is_redirect()) { my $new = $res->headers()->header("Location"); _debug("Replacing %s with %s", $matched, $new); $s =~ s/$matched/$new/; next OUTER; } else { _debug("Error resolving %s", $matched); } } } } if ($i == 0) { _debug("Loop terminated by counter"); } _debug("Final string: %s", $s); return $s; } # # Save the config to durable storage # sub _cmd_save { $remote_api->{config_save}->(); } # # Set a configuration element # sub _cmd_set { my $target = shift; my $key = shift; my $val = shift; my $p; foreach $p (@getters, @grabbers) { if ($p->{'NAME'} eq $target) { $p->setval($key, $val); return; } } _io('No such module'); } # # Enable a given module # sub _cmd_enable { my $target = shift; my $p; foreach $p (@grabbers) { if ($p->{'NAME'} eq $target) { $p->enable(); return; } } _io('No such module'); } # # Disable given module # sub _cmd_disable { my $target = shift; my $p; foreach $p (@grabbers) { if ($p->{'NAME'} eq $target) { $p->disable(); return; } } _io('No such module'); } # # Show settings for modules # sub _cmd_show { my $target = shift; my $p; my $e; if (defined($target)) { foreach $p (@getters, @grabbers) { if ($p->{'NAME'} eq $target) { _io($p->getconfstr()); return; } } _io('No such module'); } else { _io('Loaded grabbers (* denotes enabled modules):'); foreach $p (@grabbers) { $e = $p->_getval('enabled'); _io(' %s%s', $p->{'NAME'}, $e?'*':''); }; _io('Loaded getters:'); foreach $p (@getters) { _io(' %s', $p->{'NAME'}); }; } } # # Show help for the commands # sub _cmd_help { my $target = shift; my $p; if (defined($target)) { foreach $p (@getters, @grabbers) { if ($p->{'NAME'} eq $target) { _io($p->gethelpstr()); return; } } _io('No such module'); } else { _io(<<'EOT'); Supported commands: save: save the current configuration help [modulename]: display this help, or module specific help show [modulename]: show loaded modules, or the current parameters of a module set modulename parameter value: set a module parameter to a new value getter [modulename]: display or set the getter to use enable [modulename]: enable the usage of this module (grabbers only) disable [modulename]: disable the usage of this module (grabbers only) reload: reload all modules (this is somewhat experimental) mode [modename]: display or set the operation mode (download/display) connector [subcommand]: manage connectors (proxies) debug: enable debugging messages nodebug: disable debugging messages EOT } } # # Set the getter to use # sub _cmd_getter { my $target = shift; my $p; if (defined($target)) { foreach $p (@getters) { if ($p->{'NAME'} eq $target) { $getter = $p; _config_set(['getter'], $target); _io("Getter changed to %s", $target); return; } } _io('No such getter'); } else { _io('Current getter: %s', _config_get(['getter'])); } } # # Show/set the working mode # sub _cmd_mode { my $mode = shift; if (defined($mode)) { $mode = lc($mode); if (('download' eq $mode) or ('display' eq $mode)) { _config_set(['mode'], $mode); _io('Now using %s mode', $mode); } else { _io('Invalid mode: %s', $mode); } } else { _io('Current mode: %s', _config_get(['mode'])); } } # # Manage the connectors # sub _cmd_connector { my $subcmd = shift; my $c; unless(defined($subcmd)) { $subcmd = "help"; } $subcmd = lc($subcmd); if ($subcmd eq 'list') { _io("Defined connectors"); foreach $c (_connectorlist('defined-connectors')) { _io($c->{name}); my $schemas = $c->{schemas}; if (scalar(keys(%{$schemas})) == 0) { _io(" No schemas defined"); } else { foreach (keys(%{$schemas})) { _io(' %s: %s', $_, $schemas->{$_}); } } } _io(); _io("Selected connectors: %s", _config_get(['active-connectors'])); } elsif ($subcmd eq 'add') { my ($name) = @_; unless(defined($name)) { _io("No name given"); return; } $name = lc($name); if (_config_list_has(['defined-connectors'], $name)) { _io("Connector already exists"); return; } _config_set(['connectors', $name, 'name'], $name); _config_list_add(['defined-connectors'], $name); } elsif ($subcmd eq 'del') { my ($name) = @_; my @dcon; unless(defined($name)) { _io("No name given"); return; } unless (_config_list_has(['defined-connectors'])) { _io("Connector does not exist"); return; } if (_config_has(['connectors', $name, '_immutable'])) { _io("Connector cannot be removed"); return; } # Remove from list of active connectors _config_list_del(['defined-connectors'], $name); _config_list_del(['active-connectors'], $name); _config_del(['connectors', $name, 'name']); _config_del(['connectors', $name, '_immutable']); _config_del(['connectors', $name, 'schemas', 'http']); _config_del(['connectors', $name, 'schemas', 'https']); @dcon = split(/,/, _config_get(['active-connectors'])); if (scalar(@dcon) == 0) { _io("List of selected connectors is empty, resetting to direct"); _config_list_add(['active-connectors', 'direct']); } } elsif ($subcmd eq 'addschema') { my ($conn, $schema, $proxy) = @_; unless(defined($conn)) { _io("No connector name given"); return; } unless(defined($schema)) { _io("No schema given"); return; } unless(defined($proxy)) { _io("No proxy given"); return; } $conn = lc($conn); unless(_config_list_has(['defined-connectors'], $conn)) { _io("Connector does not exist"); return; } if (_config_has(['connectors', $conn, '_immutable'])) { _io("Connector cannot be modified"); return; } $schema = lc($schema); _config_set(['connectors', $conn, 'schemas', $schema], $proxy); } elsif ($subcmd eq 'delschema') { my ($conn, $schema) = @_; unless(defined($conn)) { _io("No connector name given"); return; } unless(defined($schema)) { _io("No schema given"); return; } $conn = lc($conn); unless(_config_list_has(['defined-connectors'], $conn)) { _io("Connector does not exist"); return; } $schema = lc($schema); _config_del(['connectors', $conn, 'schemas', $schema]); } elsif ($subcmd eq 'select') { my @connlist = map { lc } @_; if (scalar(@connlist) == 0) { _io("No connectors given"); return; } foreach (@connlist) { unless(_config_list_has(['defined-connectors'], $_)) { _io("Connector %s does not exist", $_); return; } } _config_list_set(['active-connectors'], @connlist); } else { _io("connector [list|add|del|addschema|delschema|help] "); _io(" help: Show this help"); _io(" list: List the defined connectors"); _io(" add : Add a connector with name "); _io(" del : Delete the connector with name "); _io(" addschema : Add proxy to connector for the given schema"); _io(" delschema : Remove the schema from the connector"); _io(" select [...]: Select the connectors to use"); } } # # Return the list of loaded grabbers. # This is used by the test programs, and is not meant to be # used in general. # sub _grabbers { return @grabbers; } # # ============================================== # Builtin config handling functions # These are used if the library used does not # register it's own config_* handlers # ============================================== # sub _builtin_config_init { } sub _builtin_config_get { return $builtin_config{join(".", @{$_[0]})}; } sub _builtin_config_set { $builtin_config{join(".", @{$_[0]})} = $_[1]; } sub _builtin_config_has { return exists($builtin_config{join(".", @{$_[0]})}); } sub _builtin_config_save { } sub _builtin_config_del { delete($builtin_config{join(".", @{$_[0]})}); } # # ============================================== # From this point on publicly callable functions # ============================================== # # # Initialization function for the library # Actually not the first thing to be called, it expects an API # has (register_api) to be registered first # sub init { unless($remote_api) { $error = "No API set"; return 0; } # Initialize configuration data $remote_api->{config_init}->(); # Check/create default values, if they do not exist _recursive_hash_walk($defaultconfig, \&_init_config_item); # Load modules _load_modules(File::Spec->catfile($remote_api->{module_path}->(), 'videosite')); unless (@grabbers && @getters) { _io('No grabbers or no getters found, can not proceed.'); return 0; } # Set the getter $getter = $getters[0]; foreach my $p (@getters) { if (_config_get(['getter']) eq $p->{'NAME'}) { $getter = $p; } } _debug('Selected %s as getter', $getter->{'NAME'}); _config_set(['getter'], $getter->{'NAME'}); # Done. _io('initialized successfully'); return 1; } # # Register a remote API. This API contains a basic output function (used # when no window specific function is available), some config functions # and a color code function. # sub register_api { my $a = shift; my @config_functions = qw(config_init config_set config_get config_has config_save config_del); my $c; my @missing; unless(defined($a)) { die("No API given"); } # # The config_* handlers are special in that they either all have # provided by the user, or none. In the latter case builtin # handlers will be used, but the config will not persist. # $c = 0; foreach (@config_functions) { if (exists($a->{$_})) { $c++; } else { push(@missing, $_); } } unless (($c == 0) or ($c == scalar(@config_functions))) { $error = sprintf("Missing config function: %s", $missing[0]); return 0; } foreach (keys(%{$a})) { if (ref($a->{$_}) ne 'CODE') { $error = sprintf("API handler %s is not a subroutine reference", $_); } $remote_api->{$_} = $a->{$_}; } if (exists($a->{_debug})) { $debug = $a->{_debug}->(); } @outputstack = ($remote_api->{'io'}); return 1; } # # Check a message for useable links # sub check_for_link { my $event = shift; my $message = $event->{message}; my $g; my $m; my $p; # Look if we should ignore this line if ($message =~ m,(?:\s|^)/nosave(?:\s|$),) { return; } _push_output($event->{ewpf}); $message = _expand_url_shortener($message); study($message); # Offer the message to all Grabbers in turn GRABBER: foreach $g (@grabbers) { ($m, $p) = $g->get($message); while (defined($m)) { _debug('Metadata: %s', Dumper($m)); if (exists($remote_api->{link_callback})) { $remote_api->{link_callback}->($m); } if ('download' eq _config_get(['mode'])) { _io( sprintf('%s>>> %sSaving %s%%s%s %s%%s', $remote_api->{color}->('*red'), $remote_api->{color}->(), $remote_api->{color}->('*yellow'), $remote_api->{color}->(), $remote_api->{color}->('*green'), ), $m->{'SOURCE'}, $m->{'TITLE'} ); unless($getter->get($m)) { _io(sprintf('%s>>> FAILED', $remote_api->{color}->('*red'))); } } elsif ('display' eq _config_get(['mode'])) { _io( sprintf('%s>>> %sSaw %s%%s%s %s%%s', $remote_api->{color}->('*magenta'), $remote_api->{color}->(), $remote_api->{color}->('*yellow'), $remote_api->{color}->(), $remote_api->{color}->('*green') ), $m->{'SOURCE'}, $m->{'TITLE'} ); } else { _io(sprintf('%s>>> Invalid operation mode', $remote_api->{color}->('*red'))); } # Remove the matched part from the message and try again (there may be # more!) $message =~ s/$p//; study($message); last GRABBER if ($message =~ /^\s*$/); ($m, $p) = $g->get($message); } } _pop_output(); } # # Handle a videosite command (/videosite ...) entered in the client # sub handle_command { my $event = shift; my ($cmd, @params) = split(/\s+/, $event->{message}); _push_output($event->{ewpf}); if (exists($videosite_commands->{$cmd})) { $videosite_commands->{$cmd}->(@params); } _pop_output(); } 1;