1 # library to autodownload flash videos
3 # (c) 2007-2008 by Ralf Ertzinger <ralf@camperquake.de>
4 # licensed under GNU GPL v2
6 # Based on youtube.pl by Christian Garbs <mitch@cgarbs.de>
8 # based on trigger.pl by Wouter Coekaerts <wouter@coekaerts.be>
13 use vars qw(@ISA @EXPORT_OK);
23 @EXPORT_OK = qw(init register_api check_for_link);
28 my %debugwindows = ();
32 my %builtin_config = ();
36 # The default config. These values will be set in the config
37 # if they do not exist already.
40 'getter' => 'filegetter',
42 'active-connectors' => 'direct',
43 'defined-connectors' => 'direct',
51 'config-version' => '2',
55 # This is a list of default values for the remote API. These
56 # are used if the values are not registered by the library user.
59 io => sub { print @_, "\n" },
60 config_init => \&_builtin_config_init,
61 config_get => \&_builtin_config_get,
62 config_set => \&_builtin_config_set,
63 config_has => \&_builtin_config_has,
64 config_save => \&_builtin_config_save,
65 config_del => \&_builtin_config_del,
66 color => sub { return '' },
67 module_path => sub { return dirname(realpath($0)) },
68 quote => sub { return $_ },
73 # List of known commands and handlers
75 my $videosite_commands = {
105 $remote_api->{reload}->();
126 # Output a string on the client.
127 # Works like (s)printf in that it takes a format string and a list of
128 # values to be replaced. Undefined values will be printed as '(undef)'
130 # All parameters (except for the format string itself) will be quoted
131 # using the client specific quote function
137 @text = ('') unless(@text);
139 # This will define the outputprefix once, so we don't have
140 # do do this every time.
141 $outputprefix = sprintf("%svideosite: %s",
142 _colorpair('magenta'),
143 _colorpair()) unless(defined($outputprefix));
144 $format = $outputprefix . shift(@text);
147 # The format string is assumed to be appropriately quoted.
148 # Quote the rest of the text, replacing undefined strings by (undef)
150 @text = map { defined($_)?$remote_api->{quote}->($_):'(undef)' } @text;
152 $outputstack[0]->{ewpf}->(sprintf($format, @text));
156 # Recursively walk through a hash-of-hashes, calling the given function
157 # for each found leaf with the path to the leaf
159 sub _recursive_hash_walk {
161 my $callback = shift;
164 foreach (keys(%{$hash})) {
165 if (ref($hash->{$_}) eq 'HASH') {
166 _recursive_hash_walk($hash->{$_}, $callback, @path, $_);
168 $callback->([@path, $_], $hash->{$_});
174 # Return the color code for the given foreground/background color
175 # pair. Both can be undef, which means "default"
180 $fg = defined($fg)?$fg:'default';
181 $bg = defined($bg)?$bg:'default';
183 return $remote_api->{color}->($fg, $bg);
187 # Sets the given config item if it is not set already
189 sub _init_config_item {
193 unless(_config_has($path)) {
194 _config_set($path, $value);
199 # Print a message if debug is enabled
204 $data[0] = "DEBUG: " . $data[0];
206 # Check for global debug
210 # Check if current window is in the per-window-debug list
211 if (exists($debugwindows{$outputstack[0]->{window}})) {
218 # Load a list of modules matching a pattern from a given directory.
230 opendir(D, $dir) || return ();
231 @list = grep {/$pattern/ && -f File::Spec->catfile($dir, $_) } readdir(D);
235 _debug("Trying to load $p:");
238 load "videosite::$p";
241 _io("Failed to load plugin: $@");
246 $g = "videosite::$p"->new();
249 _io("Failed to instanciate: $@");
254 _debug("found $g->{'TYPE'} $g->{'NAME'}");
255 if ($type eq $g->{'TYPE'}) {
259 io_debug => \&_debug,
260 connectors => sub { return _connectorlist('active-connectors') },
261 config_get => \&_config_get,
262 config_set => \&_config_set,
263 config_has => \&_config_has,
266 _io('%s has wrong type (got %s, expected %s)', $p, $g->{'TYPE'}, $type);
271 _debug("Loaded %d plugins", $#g+1);
277 # Populate the @grabbers and @getters lists from the given
280 sub _load_modules($) {
284 foreach (keys(%INC)) {
285 if ($INC{$_} =~ m|^$path|) {
286 _debug("Removing %s from \$INC", $_);
290 @grabbers = _ploader($path, '.*Grabber\.pm$', 'grabber');
291 @getters = _ploader($path, '.*Getter\.pm$', 'getter');
295 # Wrapper functions for config management to put in
302 $value = $remote_api->{config_get}->($path);
303 _debug("config: getting %s=%s", join('.', @{$path}), $value);
312 _debug("config: setting %s=%s", join('.', @{$path}), $value);
313 return $remote_api->{config_set}->($path, $value);
320 $b = $remote_api->{config_has}->($path);
321 _debug("config: testing %s (%s)", join('.', @{$path}), $b?'true':'false');
329 _debug("config: removing %s", join('.', @{$path}));
330 $remote_api->{config_del}->($path);
334 # The _config_list_* are helper functions taking a path to a comma separated
335 # string. The string is interpreted as a list and the action performed
336 # on it, storing back the modified version
340 # Add an item to the list, checking for duplicates
342 sub _config_list_add {
347 if (_config_has($path)) {
348 @c = split(/\s*,\s*/, _config_get($path));
353 _debug("Adding %s to list %s", $item, join(".", @{$path}));
354 unless(grep { $_ eq $item } @c) {
358 _config_set($path, join(',', @c));
362 # Remove an item from the list
364 sub _config_list_del {
369 unless(_config_has($path)) {
373 _debug("Removing %s from list %s", $item, join('.', @{$path}));
374 @c = grep { $item ne $_ } split(/\s*,\s*/, _config_get($path));
376 _config_set($path, join(',', @c));
380 # Return true if the item contains the given list, false otherwise
382 sub _config_list_has {
386 unless(_config_has($path)) {
390 _debug("Checking for %s in list %s", $item, join('.', @{$path}));
392 return grep { $item eq $_ } split(/\s*,\s*/, _config_get($path));
396 # Replace a list with the given items
398 sub _config_list_set {
401 _debug("Replacing %s with (%s)", join('.', @{$path}), join(",", @_));
403 _config_set($path, join(',', @_));
407 # Return the list of currently active connectors, in the configured
414 foreach(split(/,/, _config_get([$key]))) {
415 push(@c, _unserialize_connector_hash($_));
422 # Convert a connector hash from it's config structure back to a perl
425 sub _unserialize_connector_hash {
429 if (_config_has(['connectors', $name, 'name'])) {
430 $connector->{name} = _config_get(['connectors', $name, 'name']);
431 $connector->{schemas} = {};
432 foreach ('http', 'https') {
433 if (_config_has(['connectors', $name, 'schemas', $_])) {
434 $connector->{schemas}->{$_} = _config_get(['connectors', $name, 'schemas', $_]);
439 _debug("Returning connector %s: %s", $name, Dumper($connector));
445 # Push a new output function on the IO stack.
448 unshift(@outputstack, shift);
452 # Pop the topmost output function from the stack, leaving
453 # at least one function on it.
456 if (scalar(@outputstack) > 0) {
462 # Takes a string and replaces commonly used URL shorteners recursively,
463 # up to 10 levels deep
465 sub _expand_url_shortener {
469 'is\.gd/[[:alnum:]]+',
470 'otf\.me/[[:alnum:]]+',
471 'hel\.me/[[:alnum:]]+',
472 '7ax\.de/[[:alnum:]]+',
473 'ow\.ly/[[:alnum:]]+',
474 'j\.mp/[[:alnum:]]+',
475 'bit\.ly/[[:alnum:]]+',
476 'tinyurl\.com/[[:alnum:]]+',
477 'pop\.is/[[:alnum:]]+',
478 'post\.ly/[[:alnum:]]+',
479 '1\.ly/[[:alnum:]]+',
480 '2\.ly/[[:alnum:]]+',
481 't\.co/[[:alnum:]]+',
482 'shar\.es/[[:alnum:]]+',
483 'goo\.gl/[[:alnum:]]+',
485 my $ua = LWP::UserAgent->new(agent => 'Mozilla', max_redirect => 0, timeout => 5);
488 OUTER: while (($os ne $s) and ($i > 0)) {
493 foreach my $pattern (@urlshortener) {
494 my $p = "https?:\/\/" . $pattern;
496 _debug("Matching %s against %s", $p, $s);
501 _debug("Found %s", $matched);
502 $res = $ua->head($matched);
503 if ($res->is_redirect()) {
504 my $new = $res->headers()->header("Location");
506 _debug("Replacing %s with %s", $matched, $new);
507 $s =~ s/$matched/$new/;
510 _debug("Error resolving %s", $matched);
517 _debug("Loop terminated by counter");
520 _debug("Final string: %s", $s);
526 # Save the config to durable storage
531 if ($remote_api->{config_save}->()) {
534 _io(sprintf("%sConfig save failed%s", _colorpair("*red"), _colorpair()));
539 # Set a configuration element
548 foreach $p (@getters, @grabbers) {
549 if ($p->{'NAME'} eq $target) {
550 $p->setval($key, $val);
554 _io('No such module');
559 # Enable a given module
566 foreach $p (@grabbers) {
567 if ($p->{'NAME'} eq $target) {
572 _io('No such module');
576 # Disable given module
583 foreach $p (@grabbers) {
584 if ($p->{'NAME'} eq $target) {
589 _io('No such module');
593 # Show settings for modules
601 if (defined($target)) {
602 foreach $p (@getters, @grabbers) {
603 if ($p->{'NAME'} eq $target) {
604 _io($p->getconfstr());
608 _io('No such module');
610 _io('Loaded grabbers (* denotes enabled modules):');
611 foreach $p (@grabbers) {
612 $e = $p->_getval('enabled');
613 _io(' %s%s', $p->{'NAME'}, $e?'*':'');
616 _io('Loaded getters:');
617 foreach $p (@getters) {
618 _io(' %s', $p->{'NAME'});
624 # Show help for the commands
631 if (defined($target)) {
632 foreach $p (@getters, @grabbers) {
633 if ($p->{'NAME'} eq $target) {
634 _io($p->gethelpstr());
638 _io('No such module');
642 save: save the current configuration
643 help [modulename]: display this help, or module specific help
644 show [modulename]: show loaded modules, or the current parameters of a module
645 set modulename parameter value: set a module parameter to a new value
646 getter [modulename]: display or set the getter to use
647 enable [modulename]: enable the usage of this module (grabbers only)
648 disable [modulename]: disable the usage of this module (grabbers only)
649 reload: reload all modules (this is somewhat experimental)
650 mode [modename]: display or set the operation mode (download/display)
651 connector [subcommand]: manage connectors (proxies)
652 debug: enable debugging messages
653 nodebug: disable debugging messages
659 # Set the getter to use
666 if (defined($target)) {
667 foreach $p (@getters) {
668 if ($p->{'NAME'} eq $target) {
670 _config_set(['getter'], $target);
671 _io("Getter changed to %s", $target);
675 _io('No such getter');
677 _io('Current getter: %s', _config_get(['getter']));
682 # Show/set the working mode
688 if (defined($mode)) {
690 if (('download' eq $mode) or ('display' eq $mode)) {
691 _config_set(['mode'], $mode);
692 _io('Now using %s mode', $mode);
694 _io('Invalid mode: %s', $mode);
697 _io('Current mode: %s', _config_get(['mode']));
703 # Manage the connectors
710 unless(defined($subcmd)) {
714 $subcmd = lc($subcmd);
716 if ($subcmd eq 'list') {
717 _io("Defined connectors");
718 foreach $c (_connectorlist('defined-connectors')) {
720 my $schemas = $c->{schemas};
721 if (scalar(keys(%{$schemas})) == 0) {
722 _io(" No schemas defined");
724 foreach (keys(%{$schemas})) {
725 _io(' %s: %s', $_, $schemas->{$_});
731 _io("Selected connectors: %s", _config_get(['active-connectors']));
732 } elsif ($subcmd eq 'add') {
735 unless(defined($name)) {
736 _io("No name given");
742 unless($name =~ m|^[a-z]+$|) {
743 _io("%s is not a valid connector name (only letters are allowed)", $name);
747 if (_config_list_has(['defined-connectors'], $name)) {
748 _io("Connector already exists");
752 _config_set(['connectors', $name, 'name'], $name);
753 _config_list_add(['defined-connectors'], $name);
754 } elsif ($subcmd eq 'del') {
758 unless(defined($name)) {
759 _io("No name given");
763 unless (_config_list_has(['defined-connectors'], $name)) {
764 _io("Connector does not exist");
768 if (_config_has(['connectors', $name, '_immutable'])) {
769 _io("Connector cannot be removed");
773 # Remove from list of active connectors
774 _config_list_del(['defined-connectors'], $name);
775 _config_list_del(['active-connectors'], $name);
777 _config_del(['connectors', $name, 'name']);
778 _config_del(['connectors', $name, '_immutable']);
779 _config_del(['connectors', $name, 'schemas', 'http']);
780 _config_del(['connectors', $name, 'schemas', 'https']);
782 @dcon = split(/,/, _config_get(['active-connectors']));
784 if (scalar(@dcon) == 0) {
785 _io("List of selected connectors is empty, resetting to direct");
786 _config_list_add(['active-connectors', 'direct']);
788 } elsif ($subcmd eq 'addschema') {
789 my ($conn, $schema, $proxy) = @_;
791 unless(defined($conn)) {
792 _io("No connector name given");
796 unless(defined($schema)) {
797 _io("No schema given");
801 unless(defined($proxy)) {
802 _io("No proxy given");
807 unless(_config_list_has(['defined-connectors'], $conn)) {
808 _io("Connector does not exist");
812 if (_config_has(['connectors', $conn, '_immutable'])) {
813 _io("Connector cannot be modified");
817 $schema = lc($schema);
818 _config_set(['connectors', $conn, 'schemas', $schema], $proxy);
819 } elsif ($subcmd eq 'delschema') {
820 my ($conn, $schema) = @_;
822 unless(defined($conn)) {
823 _io("No connector name given");
827 unless(defined($schema)) {
828 _io("No schema given");
833 unless(_config_list_has(['defined-connectors'], $conn)) {
834 _io("Connector does not exist");
838 $schema = lc($schema);
839 _config_del(['connectors', $conn, 'schemas', $schema]);
840 } elsif ($subcmd eq 'select') {
841 my @connlist = map { lc } @_;
843 if (scalar(@connlist) == 0) {
844 _io("No connectors given");
848 foreach (@connlist) {
849 unless(_config_list_has(['defined-connectors'], $_)) {
850 _io("Connector %s does not exist", $_);
855 _config_list_set(['active-connectors'], @connlist);
857 _io("connector [list|add|del|addschema|delschema|help] <options>");
858 _io(" help: Show this help");
859 _io(" list: List the defined connectors");
860 _io(" add <name>: Add a connector with name <name>");
861 _io(" del <name>: Delete the connector with name <name>");
862 _io(" addschema <name> <schema> <proxy>: Add proxy to connector for the given schema");
863 _io(" delschema <name> <schema>: Remove the schema from the connector");
864 _io(" select <name> [<name>...]: Select the connectors to use");
870 # Global debug if the keyword "all" is given, or just for the
871 # current window otherwise
877 if (defined($scope) and (lc($scope) eq 'all')) {
878 _io("Global debug enabled");
881 _io("Debug for this window enabled");
882 $debugwindows{$event->{window}} = 1;
885 _io("keys in debugwindows: %s", join(", ", keys(%debugwindows)));
890 # Disable global debug if the keyword "all" is given (this will
891 # also disable all per-window debugs) or just for the current
898 if (defined($scope) and (lc($scope) eq 'all')) {
901 _io("Global debug disabled");
903 delete($debugwindows{$event->{window}});
904 _io("Debug for this window disabled");
907 _io("keys in debugwindows: %s", join(", ", keys(%debugwindows)));
912 # Return the list of loaded grabbers.
913 # This is used by the test programs, and is not meant to be
921 # ==============================================
922 # Builtin config handling functions
923 # These are used if the library used does not
924 # register it's own config_* handlers
925 # ==============================================
927 sub _builtin_config_init {
930 sub _builtin_config_get {
931 return $builtin_config{join(".", @{$_[0]})};
934 sub _builtin_config_set {
935 $builtin_config{join(".", @{$_[0]})} = $_[1];
938 sub _builtin_config_has {
939 return exists($builtin_config{join(".", @{$_[0]})});
942 sub _builtin_config_save {
945 sub _builtin_config_del {
946 delete($builtin_config{join(".", @{$_[0]})});
950 # ==============================================
951 # From this point on publicly callable functions
952 # ==============================================
957 # Initialization function for the library
958 # Actually not the first thing to be called, it expects an API
959 # has (register_api) to be registered first
962 unless($remote_api) {
963 $error = "No API set";
967 # Initialize configuration data
968 $remote_api->{config_init}->();
970 # Check/create default values, if they do not exist
971 _recursive_hash_walk($defaultconfig, \&_init_config_item);
974 _load_modules(File::Spec->catfile($remote_api->{module_path}->(), 'videosite'));
976 unless (@grabbers && @getters) {
977 _io('No grabbers or no getters found, can not proceed.');
982 $getter = $getters[0];
983 foreach my $p (@getters) {
984 if (_config_get(['getter']) eq $p->{'NAME'}) {
988 _debug('Selected %s as getter', $getter->{'NAME'});
989 _config_set(['getter'], $getter->{'NAME'});
992 _io('initialized successfully');
997 # Register a remote API. This API contains a basic output function (used
998 # when no window specific function is available), some config functions
999 # and a color code function.
1003 my @config_functions = qw(config_init config_set config_get config_has config_save config_del);
1007 unless(defined($a)) {
1008 die("No API given");
1012 # The config_* handlers are special in that they either all have
1013 # provided by the user, or none. In the latter case builtin
1014 # handlers will be used, but the config will not persist.
1017 foreach (@config_functions) {
1018 if (exists($a->{$_})) {
1025 unless (($c == 0) or ($c == scalar(@config_functions))) {
1026 $error = sprintf("Missing config function: %s", $missing[0]);
1030 foreach (keys(%{$a})) {
1031 if (ref($a->{$_}) ne 'CODE') {
1032 $error = sprintf("API handler %s is not a subroutine reference", $_);
1034 $remote_api->{$_} = $a->{$_};
1037 if (exists($a->{_debug})) {
1038 $debug = $a->{_debug}->();
1041 @outputstack = ({ewpf => $remote_api->{'io'}, window => ""});
1047 # Check a message for useable links
1049 sub check_for_link {
1051 my $message = $event->{message};
1056 my $mode = _config_get(['mode']);
1060 # If /nosave is present in the message switch to display mode, regardless
1063 if ($message =~ m,(?:\s|^)/nosave(?:\s|$),) {
1067 _push_output($event);
1068 $message = _expand_url_shortener($message);
1072 # Offer the message to all Grabbers in turn
1073 GRABBER: foreach $g (@grabbers) {
1074 ($m, $p) = $g->get($message);
1075 while (defined($m)) {
1076 _debug('Metadata: %s', Dumper($m));
1078 if (exists($remote_api->{link_callback})) {
1079 $skip = $remote_api->{link_callback}->($m);
1082 if ('download' eq $mode) {
1084 sprintf('%s>>> %sSaving %s%%s%s %s%%s',
1087 _colorpair('*yellow'),
1089 _colorpair('*green'),
1094 unless($getter->get($m)) {
1095 _io(sprintf('%s>>> FAILED', _colorpair('*red')));
1097 } elsif ('display' eq $mode) {
1099 sprintf('%s>>> %sSaw %s%%s%s %s%%s',
1100 _colorpair('*magenta'),
1102 _colorpair('*yellow'),
1104 _colorpair('*green')
1110 _io(sprintf('%s>>> Invalid operation mode', _colorpair('*red')));
1114 # Remove the matched part from the message and try again (there may be
1118 last GRABBER if ($message =~ /^\s*$/);
1120 ($m, $p) = $g->get($message);
1128 # Handle a videosite command (/videosite ...) entered in the client
1130 sub handle_command {
1132 my ($cmd, @params) = split(/\s+/, $event->{message});
1134 _push_output($event);
1136 if (exists($videosite_commands->{$cmd})) {
1137 $videosite_commands->{$cmd}->($event, @params);