From 31ad81fae87b027f06540391c563e92d38244f18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 29 Nov 2014 13:23:50 -0500 Subject: [PATCH 01/28] add bookie plugin --- bookie.py | 195 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 bookie.py diff --git a/bookie.py b/bookie.py new file mode 100644 index 0000000..fa864f5 --- /dev/null +++ b/bookie.py @@ -0,0 +1,195 @@ +# coding=utf8 +""" +bookie.py - Willie URL storage into bookie +""" +from __future__ import unicode_literals + +from willie import web, tools +from willie.module import commands, rule, example +from willie.modules.url import get_hostname, url_finder, exclusion_char, title_tag_data, quoted_title, re_dcc +from willie.config import ConfigurationError + +from datetime import datetime +import getpass +import json +try: + import pytz +except: + pytz = None +import re +import sys + +# we match all URLs to override the builtin url.py module +regex = re.compile('.*') + + +def configure(config): + """ + | [url] | example | purpose | + | ---- | ------- | ------- | + """ + if config.option('Configure Bookie?', False): + if not config.has_section('bookie'): + config.add_section('bookie') + config.interactive_add( + 'bookie', + 'api_url', + 'URL of the Bookie API', + 'https://bookie.io/api/v1') + config.interactive_add( + 'bookie', + 'api_user', + 'Username on the Bookie site', + getpass.getuser()) + config.interactive_add( + 'bookie', + 'api_key', + 'API key on the Bookie site', + None) + +def setup(bot): + global url_finder, exclusion_char + + if not bot.config.bookie.api_user or not bot.config.bookie.api_key: + raise ConfigurationError('Bookie module not configured') + + if bot.config.has_option('url', 'exclusion_char'): + exclusion_char = bot.config.url.exclusion_char + + url_finder = re.compile(r'(?u)(%s?(?:http|https|ftp)(?:://\S+))' % + (exclusion_char)) + if not bot.memory.contains('url_callbacks'): + bot.memory['url_callbacks'] = tools.WillieMemory() + bot.memory['url_callbacks'][regex] = bmark + + + +def shutdown(bot): + del bot.memory['url_callbacks'][regex] + +@commands('bmark') +@example('.bmark http://example.com', '[ Example ] - example.com') +def bmark(bot, trigger): + if not trigger.group(2): + if trigger.sender not in bot.memory['last_seen_url']: + return + matched = check_callbacks(bot, trigger, + bot.memory['last_seen_url'][trigger.sender], + True) + if matched: + return + else: + urls = [bot.memory['last_seen_url'][trigger.sender]] + else: + urls = re.findall(url_finder, trigger) + process_urls(bot, trigger, urls) + + +@rule('(?u).*(https?://\S+).*') +def title_auto(bot, trigger): + """Automatically show titles for URLs. For shortened URLs/redirects, find + where the URL redirects to and show the title for that (or call a function + from another module to give more information). + + Unfortunate copy of modules.url.title_auto because I couldn't hook + into it. + + """ + if re.match(bot.config.core.prefix + 'bmark', trigger): + return + + # Avoid fetching known malicious links + if 'safety_cache' in bot.memory and trigger in bot.memory['safety_cache']: + if bot.memory['safety_cache'][trigger]['positives'] > 1: + return + + urls = re.findall(url_finder, trigger) + results = process_urls(bot, trigger, urls) + +def process_urls(bot, trigger, urls): + for url in urls: + if not url.startswith(exclusion_char): + # Magic stuff to account for international domain names + try: + url = willie.web.iri_to_uri(url) + except: + pass + bot.memory['last_seen_url'][trigger.sender] = url + (title, domain, resp) = api_bmark(bot, url) + try: + # assumes that bookie's times are UTC + timestamp = datetime.strptime(json.loads(resp)['bmark']['stored'], '%Y-%m-%d %H:%M:%S') + if pytz: + tz = tools.get_timezone(bot.db, bot.config, + trigger.nick, trigger.sender) + timestamp = tools.format_time(bot.db, bot.config, tz, trigger.nick, + trigger.sender, timestamp) + else: + timestamp += 'Z' + except KeyError: + timestamp = 'no timestamp in %s' % json.loads(resp) + except ValueError as e: + if 'JSON' in str(e): + timestamp = u'cannot parse JSON response: %s' % resp.decode('utf-8', 'ignore') + else: + raise + message = '[ %s ] - %s (%s)' % (title, domain, timestamp) + # Guard against responding to other instances of this bot. + if message != trigger: + bot.say(message) + + +def api_bmark(bot, trigger, found_match=None): + match = trigger or found_match + bytes = web.get(match) + # XXX: needs a patch to the URL module + title = find_title(content=bytes) + api = '%s/%s/bmark?api_key=%s' % ( bot.config.bookie.api_url, + bot.config.bookie.api_user, + bot.config.bookie.api_key ) + if title: + bot.debug('bookie', 'submitting %s with title %s' % (match.encode('utf-8'), + repr(title)), 'warning') + result = web.post(api, {u'url': match, + u'is_private': False, + u'description': title.encode('utf-8')}) + return (title, get_hostname(match), result) + else: + bot.debug('bookie', 'no title found in %s' % match, 'warning') + +def find_title(url=None, content=None): + """Return the title for the given URL. + + Copy of find_title that allows for avoiding duplicate requests.""" + if (not content and not url) or (content and url): + raise ValueError('url *or* content needs to be provided to find_title') + if url: + try: + content, headers = web.get(url, return_headers=True, limit_bytes=max_bytes) + except UnicodeDecodeError: + return # Fail silently when data can't be decoded + assert content + + # Some cleanup that I don't really grok, but was in the original, so + # we'll keep it (with the compiled regexes made global) for now. + content = title_tag_data.sub(r'<\1title>', content) + content = quoted_title.sub('', content) + + start = content.find('') + end = content.find('') + if start == -1 or end == -1: + return + title = web.decode(content[start + 7:end]) + title = title.strip()[:200] + + title = ' '.join(title.split()) # cleanly remove multiple spaces + + # More cryptic regex substitutions. This one looks to be myano's invention. + title = re_dcc.sub('', title) + + return title or None + + +if __name__ == "__main__": + from willie.test_tools import run_example_tests + run_example_tests(__file__) From 6293b4f32edd9ef7a995deb667629ef0b87f75b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 29 Nov 2014 13:35:13 -0500 Subject: [PATCH 02/28] print url in debug --- bookie.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bookie.py b/bookie.py index fa864f5..83ca4a5 100644 --- a/bookie.py +++ b/bookie.py @@ -148,8 +148,9 @@ def api_bmark(bot, trigger, found_match=None): bot.config.bookie.api_user, bot.config.bookie.api_key ) if title: - bot.debug('bookie', 'submitting %s with title %s' % (match.encode('utf-8'), - repr(title)), 'warning') + bot.debug('bookie', 'submitting %s with title %s to %s' % (match.encode('utf-8'), + repr(title), + api), 'warning') result = web.post(api, {u'url': match, u'is_private': False, u'description': title.encode('utf-8')}) From 12d8fe5111e4a0b06dffabe8f55af3c1790c8135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 29 Nov 2014 13:35:31 -0500 Subject: [PATCH 03/28] cosmetic --- bookie.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bookie.py b/bookie.py index 83ca4a5..8cb7021 100644 --- a/bookie.py +++ b/bookie.py @@ -126,19 +126,19 @@ def process_urls(bot, trigger, urls): trigger.sender, timestamp) else: timestamp += 'Z' + status = timestamp except KeyError: - timestamp = 'no timestamp in %s' % json.loads(resp) + status = 'no timestamp in %s' % json.loads(resp) except ValueError as e: if 'JSON' in str(e): - timestamp = u'cannot parse JSON response: %s' % resp.decode('utf-8', 'ignore') + status = u'cannot parse JSON response: %s' % resp.decode('utf-8', 'ignore') else: raise - message = '[ %s ] - %s (%s)' % (title, domain, timestamp) + message = '[ %s ] - %s (%s)' % (title, domain, status) # Guard against responding to other instances of this bot. if message != trigger: bot.say(message) - def api_bmark(bot, trigger, found_match=None): match = trigger or found_match bytes = web.get(match) From 804992dc123bcd36e9e90f896a670b83e4196580 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 29 Nov 2014 13:56:45 -0500 Subject: [PATCH 04/28] parse API failures more clearly --- bookie.py | 59 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 23 deletions(-) diff --git a/bookie.py b/bookie.py index 8cb7021..b9d0eb5 100644 --- a/bookie.py +++ b/bookie.py @@ -22,6 +22,14 @@ # we match all URLs to override the builtin url.py module regex = re.compile('.*') +# an HTML tag. cargo-culted from etymology.py +r_tag = re.compile(r'<[^>]+>') +r_whitespace = re.compile(r'[\t\r\n ]+') + +def text(html): + html = r_tag.sub('', html) + html = r_whitespace.sub(' ', html) + return web.decode(html.strip()) def configure(config): """ @@ -115,25 +123,28 @@ def process_urls(bot, trigger, urls): except: pass bot.memory['last_seen_url'][trigger.sender] = url - (title, domain, resp) = api_bmark(bot, url) - try: - # assumes that bookie's times are UTC - timestamp = datetime.strptime(json.loads(resp)['bmark']['stored'], '%Y-%m-%d %H:%M:%S') - if pytz: - tz = tools.get_timezone(bot.db, bot.config, - trigger.nick, trigger.sender) - timestamp = tools.format_time(bot.db, bot.config, tz, trigger.nick, - trigger.sender, timestamp) - else: - timestamp += 'Z' - status = timestamp - except KeyError: - status = 'no timestamp in %s' % json.loads(resp) - except ValueError as e: - if 'JSON' in str(e): - status = u'cannot parse JSON response: %s' % resp.decode('utf-8', 'ignore') - else: - raise + (title, domain, resp, headers) = api_bmark(bot, url) + if headers['_http_status'] != 200: + status = 'error from bookie API: %s' % text(resp.decode('utf-8', 'ignore')) + else: + try: + # assumes that bookie's times are UTC + timestamp = datetime.strptime(json.loads(resp)['bmark']['stored'], '%Y-%m-%d %H:%M:%S') + if pytz: + tz = tools.get_timezone(bot.db, bot.config, + trigger.nick, trigger.sender) + timestamp = tools.format_time(bot.db, bot.config, tz, trigger.nick, + trigger.sender, timestamp) + else: + timestamp += 'Z' + status = timestamp + except KeyError: + status = 'no timestamp in %s' % json.loads(resp) + except ValueError as e: + if 'JSON' in str(e): + status = u'cannot parse JSON response: %s' % resp.decode('utf-8', 'ignore') + else: + raise message = '[ %s ] - %s (%s)' % (title, domain, status) # Guard against responding to other instances of this bot. if message != trigger: @@ -151,10 +162,12 @@ def api_bmark(bot, trigger, found_match=None): bot.debug('bookie', 'submitting %s with title %s to %s' % (match.encode('utf-8'), repr(title), api), 'warning') - result = web.post(api, {u'url': match, - u'is_private': False, - u'description': title.encode('utf-8')}) - return (title, get_hostname(match), result) + # XXX: requires PR https://github.com/embolalia/willie/pull/670 + (result, headers) = web.post(api, {u'url': match, + u'is_private': False, + u'description': title.encode('utf-8')}, + return_headers=True) + return (title, get_hostname(match), result, headers) else: bot.debug('bookie', 'no title found in %s' % match, 'warning') From 5a54dc6d939025e64a645476b431b9f2889ab52b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 29 Nov 2014 14:28:28 -0500 Subject: [PATCH 05/28] convert to requests to fix submissions --- bookie.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/bookie.py b/bookie.py index b9d0eb5..7e3807e 100644 --- a/bookie.py +++ b/bookie.py @@ -17,6 +17,7 @@ except: pytz = None import re +import requests import sys # we match all URLs to override the builtin url.py module @@ -159,15 +160,15 @@ def api_bmark(bot, trigger, found_match=None): bot.config.bookie.api_user, bot.config.bookie.api_key ) if title: - bot.debug('bookie', 'submitting %s with title %s to %s' % (match.encode('utf-8'), - repr(title), - api), 'warning') - # XXX: requires PR https://github.com/embolalia/willie/pull/670 - (result, headers) = web.post(api, {u'url': match, - u'is_private': False, - u'description': title.encode('utf-8')}, - return_headers=True) - return (title, get_hostname(match), result, headers) + data = {u'url': match, + u'is_private': False, + u'description': title.encode('utf-8')} + bot.debug('bookie', 'submitting %s with title %s to %s with data %s' % (match, + repr(title), + api, data), 'warning') + r = requests.post(api, data) + r.headers['_http_status'] = r.status_code + return (title, get_hostname(match), r.text, r.headers) else: bot.debug('bookie', 'no title found in %s' % match, 'warning') From c52104074ffbe322e2b40737964672eb17cc7725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 29 Nov 2014 14:31:16 -0500 Subject: [PATCH 06/28] Revert "convert to requests to fix submissions" it works with some URLs This reverts commit 5a54dc6d939025e64a645476b431b9f2889ab52b. --- bookie.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/bookie.py b/bookie.py index 7e3807e..b9d0eb5 100644 --- a/bookie.py +++ b/bookie.py @@ -17,7 +17,6 @@ except: pytz = None import re -import requests import sys # we match all URLs to override the builtin url.py module @@ -160,15 +159,15 @@ def api_bmark(bot, trigger, found_match=None): bot.config.bookie.api_user, bot.config.bookie.api_key ) if title: - data = {u'url': match, - u'is_private': False, - u'description': title.encode('utf-8')} - bot.debug('bookie', 'submitting %s with title %s to %s with data %s' % (match, - repr(title), - api, data), 'warning') - r = requests.post(api, data) - r.headers['_http_status'] = r.status_code - return (title, get_hostname(match), r.text, r.headers) + bot.debug('bookie', 'submitting %s with title %s to %s' % (match.encode('utf-8'), + repr(title), + api), 'warning') + # XXX: requires PR https://github.com/embolalia/willie/pull/670 + (result, headers) = web.post(api, {u'url': match, + u'is_private': False, + u'description': title.encode('utf-8')}, + return_headers=True) + return (title, get_hostname(match), result, headers) else: bot.debug('bookie', 'no title found in %s' % match, 'warning') From 2e52f37c1b5ec5296340e9dd8c8261cf5b2b4067 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 29 Nov 2014 14:36:48 -0500 Subject: [PATCH 07/28] make private/public status of bookmarks configurable --- bookie.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bookie.py b/bookie.py index b9d0eb5..7411774 100644 --- a/bookie.py +++ b/bookie.py @@ -54,6 +54,11 @@ def configure(config): 'api_key', 'API key on the Bookie site', None) + config.interactive_add( + 'bookie', + 'private', + 'Mark bookmarks as private', + True) def setup(bot): global url_finder, exclusion_char @@ -164,7 +169,7 @@ def api_bmark(bot, trigger, found_match=None): api), 'warning') # XXX: requires PR https://github.com/embolalia/willie/pull/670 (result, headers) = web.post(api, {u'url': match, - u'is_private': False, + u'is_private': int(bot.config.bookie.private), u'description': title.encode('utf-8')}, return_headers=True) return (title, get_hostname(match), result, headers) From 1ac7102a4b97e17b957148a6c498cec4e2f61515 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 29 Nov 2014 14:28:28 -0500 Subject: [PATCH 08/28] convert to requests to fix submissions Conflicts: bookie.py --- bookie.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/bookie.py b/bookie.py index 7411774..db6d5e0 100644 --- a/bookie.py +++ b/bookie.py @@ -17,6 +17,7 @@ except: pytz = None import re +import requests import sys # we match all URLs to override the builtin url.py module @@ -164,15 +165,15 @@ def api_bmark(bot, trigger, found_match=None): bot.config.bookie.api_user, bot.config.bookie.api_key ) if title: - bot.debug('bookie', 'submitting %s with title %s to %s' % (match.encode('utf-8'), - repr(title), - api), 'warning') - # XXX: requires PR https://github.com/embolalia/willie/pull/670 - (result, headers) = web.post(api, {u'url': match, - u'is_private': int(bot.config.bookie.private), - u'description': title.encode('utf-8')}, - return_headers=True) - return (title, get_hostname(match), result, headers) + data = {u'url': match, + u'is_private': bot.config.bookie.private, + u'description': title.encode('utf-8')} + bot.debug('bookie', 'submitting %s with title %s to %s with data %s' % (match, + repr(title), + api, data), 'warning') + r = requests.post(api, data) + r.headers['_http_status'] = r.status_code + return (title, get_hostname(match), r.text, r.headers) else: bot.debug('bookie', 'no title found in %s' % match, 'warning') From 02d3a1a77856f0229394c0925b2d07c0d4ffa200 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 29 Nov 2014 14:49:43 -0500 Subject: [PATCH 09/28] send an integer as the is_private field, otherwise Bookie errors 500 --- bookie.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookie.py b/bookie.py index db6d5e0..d5c173f 100644 --- a/bookie.py +++ b/bookie.py @@ -166,7 +166,7 @@ def api_bmark(bot, trigger, found_match=None): bot.config.bookie.api_key ) if title: data = {u'url': match, - u'is_private': bot.config.bookie.private, + u'is_private': int(bot.config.bookie.private), u'description': title.encode('utf-8')} bot.debug('bookie', 'submitting %s with title %s to %s with data %s' % (match, repr(title), From fa4647081ec228f9ac2b134c31f628754184ac86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 29 Nov 2014 14:50:04 -0500 Subject: [PATCH 10/28] deal with unconfigured private settings --- bookie.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bookie.py b/bookie.py index d5c173f..7627bfb 100644 --- a/bookie.py +++ b/bookie.py @@ -66,7 +66,11 @@ def setup(bot): if not bot.config.bookie.api_user or not bot.config.bookie.api_key: raise ConfigurationError('Bookie module not configured') - + + # deal with non-configured private setting + if bot.config.bookie.private is None: + bot.config.bookie.private = True + if bot.config.has_option('url', 'exclusion_char'): exclusion_char = bot.config.url.exclusion_char From 453149a4c93fda57a53375db4f8bab4391ece6a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 29 Nov 2014 15:29:04 -0500 Subject: [PATCH 11/28] refactor to prepare supporting per channel configs --- bookie.py | 62 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/bookie.py b/bookie.py index 7627bfb..c560e33 100644 --- a/bookie.py +++ b/bookie.py @@ -20,6 +20,14 @@ import requests import sys +if sys.version_info.major < 3: + import urlparse + urlparse = urlparse.urlparse +else: + import urllibe + urlparse = urllib.parse.urlparse + + # we match all URLs to override the builtin url.py module regex = re.compile('.*') @@ -27,6 +35,12 @@ r_tag = re.compile(r'<[^>]+>') r_whitespace = re.compile(r'[\t\r\n ]+') +api_url = None +api_user = None +api_key = None +api_suffix = '/api/v1/' +private = None + def text(html): html = r_tag.sub('', html) html = r_whitespace.sub(' ', html) @@ -44,32 +58,43 @@ def configure(config): 'bookie', 'api_url', 'URL of the Bookie API', - 'https://bookie.io/api/v1') - config.interactive_add( - 'bookie', - 'api_user', - 'Username on the Bookie site', - getpass.getuser()) - config.interactive_add( - 'bookie', - 'api_key', - 'API key on the Bookie site', - None) + 'https://bookie.io/api/v1/admin/account?api_key=XXXXXX') config.interactive_add( 'bookie', 'private', 'Mark bookmarks as private', True) + if config.option('Would you like to configure individual accounts per channel?', False): + c = 'Enter the API URL as #channel:account' + config.add_list('bookie', 'url_per_channel', c, 'Channel:') + def setup(bot): - global url_finder, exclusion_char + global url_finder, exclusion_char, api_url, api_key, api_user - if not bot.config.bookie.api_user or not bot.config.bookie.api_key: + if bot.config.bookie.api_url: + try: + p = urlparse(bot.config.bookie.api_url) + # https + api_url = p.scheme + '://' + p.netloc + prefix = p.path.split(api_suffix)[0] + if prefix: + api_url += prefix + api_url += api_suffix + # the path element after api_suffix + api_user = p.path.split(api_suffix)[1].split('/')[0] + api_key = p.query.split('=')[1] + except Exception as e: + raise ConfigurationError('Bookie api_url badly formatted: %s' % str(e)) + else: raise ConfigurationError('Bookie module not configured') + private = bot.config.bookie.private # deal with non-configured private setting - if bot.config.bookie.private is None: - bot.config.bookie.private = True + if private is None: + private = True + if (type(private) == str): + private = True if private == 'True' else False if bot.config.has_option('url', 'exclusion_char'): exclusion_char = bot.config.url.exclusion_char @@ -161,16 +186,15 @@ def process_urls(bot, trigger, urls): bot.say(message) def api_bmark(bot, trigger, found_match=None): + global api_url, api_user, api_key match = trigger or found_match bytes = web.get(match) # XXX: needs a patch to the URL module title = find_title(content=bytes) - api = '%s/%s/bmark?api_key=%s' % ( bot.config.bookie.api_url, - bot.config.bookie.api_user, - bot.config.bookie.api_key ) + api = '%s/%s/bmark?api_key=%s' % ( api_url, api_user, api_key ) if title: data = {u'url': match, - u'is_private': int(bot.config.bookie.private), + u'is_private': private, u'description': title.encode('utf-8')} bot.debug('bookie', 'submitting %s with title %s to %s with data %s' % (match, repr(title), From 076d3bb165b0f85f67410b0f7eacec4fd6d1e593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 29 Nov 2014 15:46:43 -0500 Subject: [PATCH 12/28] complete per channel user support --- bookie.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/bookie.py b/bookie.py index c560e33..9ec7a28 100644 --- a/bookie.py +++ b/bookie.py @@ -66,7 +66,7 @@ def configure(config): True) if config.option('Would you like to configure individual accounts per channel?', False): - c = 'Enter the API URL as #channel:account' + c = 'Enter the API URL as #channel:account:key' config.add_list('bookie', 'url_per_channel', c, 'Channel:') def setup(bot): @@ -158,7 +158,7 @@ def process_urls(bot, trigger, urls): except: pass bot.memory['last_seen_url'][trigger.sender] = url - (title, domain, resp, headers) = api_bmark(bot, url) + (title, domain, resp, headers) = api_bmark(bot, trigger, url) if headers['_http_status'] != 200: status = 'error from bookie API: %s' % text(resp.decode('utf-8', 'ignore')) else: @@ -187,11 +187,20 @@ def process_urls(bot, trigger, urls): def api_bmark(bot, trigger, found_match=None): global api_url, api_user, api_key - match = trigger or found_match + match = found_match or trigger bytes = web.get(match) # XXX: needs a patch to the URL module title = find_title(content=bytes) - api = '%s/%s/bmark?api_key=%s' % ( api_url, api_user, api_key ) + user = api_user + key = api_key + if (trigger.sender and not trigger.sender.is_nick() and + bot.config.has_option('bookie', 'url_per_channel')): + res = re.search(trigger.sender + ':(\w+):(\w+)', + bot.config.bookie.url_per_channel) + if res is not None: + user = res.group(1) + key = res.group(2) + api = '%s%s/bmark?api_key=%s' % ( api_url, user, key ) if title: data = {u'url': match, u'is_private': private, From 69e3a130c58d65a2317bf5febf7dad49b9c634c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 29 Nov 2014 15:59:08 -0500 Subject: [PATCH 13/28] cosmetic: clarify some variable names --- bookie.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/bookie.py b/bookie.py index 9ec7a28..0e4e10e 100644 --- a/bookie.py +++ b/bookie.py @@ -187,32 +187,32 @@ def process_urls(bot, trigger, urls): def api_bmark(bot, trigger, found_match=None): global api_url, api_user, api_key - match = found_match or trigger - bytes = web.get(match) + url = found_match or trigger + bytes = web.get(url) # XXX: needs a patch to the URL module title = find_title(content=bytes) user = api_user key = api_key if (trigger.sender and not trigger.sender.is_nick() and bot.config.has_option('bookie', 'url_per_channel')): - res = re.search(trigger.sender + ':(\w+):(\w+)', - bot.config.bookie.url_per_channel) - if res is not None: - user = res.group(1) - key = res.group(2) + match = re.search(trigger.sender + ':(\w+):(\w+)', + bot.config.bookie.url_per_channel) + if match is not None: + user = match.group(1) + key = match.group(2) api = '%s%s/bmark?api_key=%s' % ( api_url, user, key ) if title: - data = {u'url': match, + data = {u'url': url, u'is_private': private, u'description': title.encode('utf-8')} - bot.debug('bookie', 'submitting %s with title %s to %s with data %s' % (match, + bot.debug('bookie', 'submitting %s with title %s to %s with data %s' % (url, repr(title), api, data), 'warning') r = requests.post(api, data) r.headers['_http_status'] = r.status_code - return (title, get_hostname(match), r.text, r.headers) + return (title, get_hostname(url), r.text, r.headers) else: - bot.debug('bookie', 'no title found in %s' % match, 'warning') + bot.debug('bookie', 'no title found in %s' % url, 'warning') def find_title(url=None, content=None): """Return the title for the given URL. From a485aed13625d9b473393c44af4039079f2816a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 29 Nov 2014 16:01:25 -0500 Subject: [PATCH 14/28] add more ideas --- bookie.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/bookie.py b/bookie.py index 0e4e10e..546a1a7 100644 --- a/bookie.py +++ b/bookie.py @@ -1,6 +1,15 @@ # coding=utf8 -""" -bookie.py - Willie URL storage into bookie +"""bookie.py - Willie URL storage into bookie + +Missing: +* add tags, extended descriptions options to .bmark +* parse #tags on the auto url parser + +The above is annoyingly hard with regexes... but i've had good success +with "non-hungry" patterns: + +>>> re.findall(r'(?u)(.*?)(!?(?:http|https|ftp)(?:://\S+))(.*?)', 'cool url: http://example.com and another http://example.org') +[('cool url: ', 'http://example.com', ''), (' and another ', 'http://example.org', '')] """ from __future__ import unicode_literals From 98b1873e2da68d54a6bd77855eb8bb0fa76cc064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 29 Nov 2014 16:17:55 -0500 Subject: [PATCH 15/28] also send the content of the webpage, the whole point of this --- bookie.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bookie.py b/bookie.py index 546a1a7..0e7f199 100644 --- a/bookie.py +++ b/bookie.py @@ -213,7 +213,8 @@ def api_bmark(bot, trigger, found_match=None): if title: data = {u'url': url, u'is_private': private, - u'description': title.encode('utf-8')} + u'description': title.encode('utf-8'), + u'content': bytes} bot.debug('bookie', 'submitting %s with title %s to %s with data %s' % (url, repr(title), api, data), 'warning') From dc8bfb70151252010f32e563d2fd3844ead5bd49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 29 Nov 2014 17:11:56 -0500 Subject: [PATCH 16/28] have per-channel private settings as well --- bookie.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/bookie.py b/bookie.py index 0e7f199..873de4d 100644 --- a/bookie.py +++ b/bookie.py @@ -48,7 +48,7 @@ api_user = None api_key = None api_suffix = '/api/v1/' -private = None +api_private = None def text(html): html = r_tag.sub('', html) @@ -75,11 +75,19 @@ def configure(config): True) if config.option('Would you like to configure individual accounts per channel?', False): - c = 'Enter the API URL as #channel:account:key' + c = 'Enter the API URL as #channel:account:key:private' config.add_list('bookie', 'url_per_channel', c, 'Channel:') +def validate_private(private): + # deal with non-configured private setting + if private is None: + private = True + if (type(private) == str): + private = True if private == 'True' else False + return private + def setup(bot): - global url_finder, exclusion_char, api_url, api_key, api_user + global url_finder, exclusion_char, api_url, api_key, api_user, api_private if bot.config.bookie.api_url: try: @@ -98,13 +106,7 @@ def setup(bot): else: raise ConfigurationError('Bookie module not configured') - private = bot.config.bookie.private - # deal with non-configured private setting - if private is None: - private = True - if (type(private) == str): - private = True if private == 'True' else False - + api_private = validate_private( bot.config.bookie.private) if bot.config.has_option('url', 'exclusion_char'): exclusion_char = bot.config.url.exclusion_char @@ -181,7 +183,7 @@ def process_urls(bot, trigger, urls): trigger.sender, timestamp) else: timestamp += 'Z' - status = timestamp + status = 'posted on ' + timestamp except KeyError: status = 'no timestamp in %s' % json.loads(resp) except ValueError as e: @@ -202,17 +204,19 @@ def api_bmark(bot, trigger, found_match=None): title = find_title(content=bytes) user = api_user key = api_key + private = api_private if (trigger.sender and not trigger.sender.is_nick() and bot.config.has_option('bookie', 'url_per_channel')): - match = re.search(trigger.sender + ':(\w+):(\w+)', + match = re.search(trigger.sender + ':(\w+):(\w+)(?::(\w+))?', bot.config.bookie.url_per_channel) if match is not None: user = match.group(1) key = match.group(2) + private = validate_private(match.group(3)) api = '%s%s/bmark?api_key=%s' % ( api_url, user, key ) if title: data = {u'url': url, - u'is_private': private, + u'is_private': int(private), u'description': title.encode('utf-8'), u'content': bytes} bot.debug('bookie', 'submitting %s with title %s to %s with data %s' % (url, From 478a564f4bbce72c52cd2df7e8e9013df71d6ba2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 29 Nov 2014 17:22:09 -0500 Subject: [PATCH 17/28] abstract the bookmark API away so we can do more things --- bookie.py | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/bookie.py b/bookie.py index 873de4d..bfb3a39 100644 --- a/bookie.py +++ b/bookie.py @@ -196,15 +196,10 @@ def process_urls(bot, trigger, urls): if message != trigger: bot.say(message) -def api_bmark(bot, trigger, found_match=None): +def api(bot, trigger, func, data=None): global api_url, api_user, api_key - url = found_match or trigger - bytes = web.get(url) - # XXX: needs a patch to the URL module - title = find_title(content=bytes) user = api_user key = api_key - private = api_private if (trigger.sender and not trigger.sender.is_nick() and bot.config.has_option('bookie', 'url_per_channel')): match = re.search(trigger.sender + ':(\w+):(\w+)(?::(\w+))?', @@ -212,21 +207,23 @@ def api_bmark(bot, trigger, found_match=None): if match is not None: user = match.group(1) key = match.group(2) - private = validate_private(match.group(3)) + data['is_private'] = int(validate_private(match.group(3))) api = '%s%s/bmark?api_key=%s' % ( api_url, user, key ) - if title: - data = {u'url': url, - u'is_private': int(private), - u'description': title.encode('utf-8'), - u'content': bytes} - bot.debug('bookie', 'submitting %s with title %s to %s with data %s' % (url, - repr(title), - api, data), 'warning') - r = requests.post(api, data) - r.headers['_http_status'] = r.status_code - return (title, get_hostname(url), r.text, r.headers) - else: - bot.debug('bookie', 'no title found in %s' % url, 'warning') + bot.debug('bookie', 'submitting to %s data %s' % (api, api), 'warning') + r = requests.post(api, data) + r.headers['_http_status'] = r.status_code + return (r.text, r.headers) + +def api_bmark(bot, trigger, found_match=None): + url = found_match or trigger + bytes = web.get(url) + # XXX: needs a patch to the URL module + title = find_title(content=bytes) + data = {u'url': url, + u'is_private': int(api_private), + u'description': title.encode('utf-8'), + u'content': bytes} + return [title, get_hostname(url)] + list(api(bot, trigger, 'bmark', data)) def find_title(url=None, content=None): """Return the title for the given URL. From 27a74efd077674535c79d9cec9d2adfbcae2ed0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 29 Nov 2014 17:35:30 -0500 Subject: [PATCH 18/28] add copyright notice --- bookie.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bookie.py b/bookie.py index bfb3a39..29faafb 100644 --- a/bookie.py +++ b/bookie.py @@ -1,5 +1,7 @@ # coding=utf8 """bookie.py - Willie URL storage into bookie +Copyright 2014, Antoine Beaupré +Licensed under the Eiffel Forum License 2. Missing: * add tags, extended descriptions options to .bmark From 1a19c70aac4af1d2ee58d9534e681d0d9ab51a8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 29 Nov 2014 17:35:40 -0500 Subject: [PATCH 19/28] fix debug logging --- bookie.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bookie.py b/bookie.py index 29faafb..e0df965 100644 --- a/bookie.py +++ b/bookie.py @@ -211,9 +211,10 @@ def api(bot, trigger, func, data=None): key = match.group(2) data['is_private'] = int(validate_private(match.group(3))) api = '%s%s/bmark?api_key=%s' % ( api_url, user, key ) - bot.debug('bookie', 'submitting to %s data %s' % (api, api), 'warning') + bot.debug('bookie', 'submitting to %s data %s' % (api, data), 'verbose') r = requests.post(api, data) r.headers['_http_status'] = r.status_code + bot.debug('bookie', 'response: %s (headers: %s, body: %s)' % (r, r.text, r.headers), 'verbose') return (r.text, r.headers) def api_bmark(bot, trigger, found_match=None): From 71da47b98043a84b28205149cb59f5b002f0a6ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 29 Nov 2014 17:35:49 -0500 Subject: [PATCH 20/28] clarify functionalities --- bookie.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/bookie.py b/bookie.py index e0df965..3ead415 100644 --- a/bookie.py +++ b/bookie.py @@ -3,6 +3,10 @@ Copyright 2014, Antoine Beaupré Licensed under the Eiffel Forum License 2. +This will store links found on an IRC channel into a Bookie +instance. It needs to be configured with a username/key to be +functional, per-channel configs are possible. + Missing: * add tags, extended descriptions options to .bmark * parse #tags on the auto url parser @@ -12,6 +16,12 @@ >>> re.findall(r'(?u)(.*?)(!?(?:http|https|ftp)(?:://\S+))(.*?)', 'cool url: http://example.com and another http://example.org') [('cool url: ', 'http://example.com', ''), (' and another ', 'http://example.org', '')] + +Also, this uses only a tiny part of the Bookie API, we could expand +functionalities here significantly: + +https://github.com/bookieio/Bookie/blob/develop/docs/api/user.rst + """ from __future__ import unicode_literals From 1710f35a443a687f78fa7ad33aef9b8671e0bb4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 29 Nov 2014 17:45:16 -0500 Subject: [PATCH 21/28] add comments on tricky sections while i remember them --- bookie.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/bookie.py b/bookie.py index 3ead415..258f8f1 100644 --- a/bookie.py +++ b/bookie.py @@ -63,6 +63,9 @@ api_private = None def text(html): + '''html to text dumb converter + + cargo-culted from etymology.py''' html = r_tag.sub('', html) html = r_whitespace.sub(' ', html) return web.decode(html.strip()) @@ -71,6 +74,9 @@ def configure(config): """ | [url] | example | purpose | | ---- | ------- | ------- | + | api_url | https://bookie.io/api/v1/admin/account?api_key=XXXXXX | template URL for the bookie instance | + | private | True | if bookmarks are private by default | + | url_per_channel | #channel:admin:XXXXXX:True | per-channel configuration | """ if config.option('Configure Bookie?', False): if not config.has_section('bookie'): @@ -91,6 +97,12 @@ def configure(config): config.add_list('bookie', 'url_per_channel', c, 'Channel:') def validate_private(private): + '''convert the private setting to a real bool + + this is necessary because it could be the "true" string... + + we consider every string but lower(true) to be false + ''' # deal with non-configured private setting if private is None: private = True @@ -103,15 +115,20 @@ def setup(bot): if bot.config.bookie.api_url: try: + # say we have "https://example.com/prefix/api/v1/admin/account?api_key=XXXXXX" p = urlparse(bot.config.bookie.api_url) - # https + # "https://example.com" api_url = p.scheme + '://' + p.netloc + # "/prefix" prefix = p.path.split(api_suffix)[0] if prefix: api_url += prefix + # "/api/v1/" api_url += api_suffix # the path element after api_suffix + # that is, "admin" api_user = p.path.split(api_suffix)[1].split('/')[0] + # "XXXXXX" api_key = p.query.split('=')[1] except Exception as e: raise ConfigurationError('Bookie api_url badly formatted: %s' % str(e)) @@ -136,7 +153,9 @@ def shutdown(bot): @commands('bmark') @example('.bmark http://example.com', '[ Example ] - example.com') def bmark(bot, trigger): + # cargo-culted from url.py if not trigger.group(2): + # unsure what this does if trigger.sender not in bot.memory['last_seen_url']: return matched = check_callbacks(bot, trigger, @@ -181,10 +200,13 @@ def process_urls(bot, trigger, urls): except: pass bot.memory['last_seen_url'][trigger.sender] = url + # post the bookmark to the Bookie API (title, domain, resp, headers) = api_bmark(bot, trigger, url) if headers['_http_status'] != 200: status = 'error from bookie API: %s' % text(resp.decode('utf-8', 'ignore')) else: + # try to show the user when the bookmark was posted, + # so they can tell if it's new try: # assumes that bookie's times are UTC timestamp = datetime.strptime(json.loads(resp)['bmark']['stored'], '%Y-%m-%d %H:%M:%S') @@ -197,6 +219,7 @@ def process_urls(bot, trigger, urls): timestamp += 'Z' status = 'posted on ' + timestamp except KeyError: + # the 'stored' field is not in the response? status = 'no timestamp in %s' % json.loads(resp) except ValueError as e: if 'JSON' in str(e): @@ -222,6 +245,8 @@ def api(bot, trigger, func, data=None): data['is_private'] = int(validate_private(match.group(3))) api = '%s%s/bmark?api_key=%s' % ( api_url, user, key ) bot.debug('bookie', 'submitting to %s data %s' % (api, data), 'verbose') + # we use requests instead of web.post because Bookie expects + # JSON-encoded submissions, which web.post doesn't support r = requests.post(api, data) r.headers['_http_status'] = r.status_code bot.debug('bookie', 'response: %s (headers: %s, body: %s)' % (r, r.text, r.headers), 'verbose') From 758bc667f0889695eaae27dfd4ede111bf071889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 29 Nov 2014 17:45:31 -0500 Subject: [PATCH 22/28] be more tolerant of private string values --- bookie.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookie.py b/bookie.py index 258f8f1..f31c003 100644 --- a/bookie.py +++ b/bookie.py @@ -107,7 +107,7 @@ def validate_private(private): if private is None: private = True if (type(private) == str): - private = True if private == 'True' else False + private = True if lower(private) == 'true' else False return private def setup(bot): From 8db9ea112d1f36adfc61aad4e18e4c9cd833cd4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 29 Nov 2014 19:42:48 -0500 Subject: [PATCH 23/28] handle cases where no title is found in page --- bookie.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bookie.py b/bookie.py index f31c003..4806828 100644 --- a/bookie.py +++ b/bookie.py @@ -257,6 +257,8 @@ def api_bmark(bot, trigger, found_match=None): bytes = web.get(url) # XXX: needs a patch to the URL module title = find_title(content=bytes) + if title is None: + title = '[untitled]' data = {u'url': url, u'is_private': int(api_private), u'description': title.encode('utf-8'), From 1d561d1278630d204b5c2eed5039e9588d3b3215 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sun, 30 Nov 2014 16:28:17 -0500 Subject: [PATCH 24/28] fix syntax error in b952ce7 --- bookie.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookie.py b/bookie.py index 4806828..356333f 100644 --- a/bookie.py +++ b/bookie.py @@ -107,7 +107,7 @@ def validate_private(private): if private is None: private = True if (type(private) == str): - private = True if lower(private) == 'true' else False + private = True if private.lower() == 'true' else False return private def setup(bot): From 933bab6013acd432a6ef2967061013c0329edc6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Mon, 1 Dec 2014 23:34:54 -0500 Subject: [PATCH 25/28] don't check_callbacks, that's url.py's job --- bookie.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/bookie.py b/bookie.py index 356333f..827714c 100644 --- a/bookie.py +++ b/bookie.py @@ -155,16 +155,10 @@ def shutdown(bot): def bmark(bot, trigger): # cargo-culted from url.py if not trigger.group(2): - # unsure what this does + # this bookmarks the last URL seen by url.py or this module if trigger.sender not in bot.memory['last_seen_url']: return - matched = check_callbacks(bot, trigger, - bot.memory['last_seen_url'][trigger.sender], - True) - if matched: - return - else: - urls = [bot.memory['last_seen_url'][trigger.sender]] + urls = [bot.memory['last_seen_url'][trigger.sender]] else: urls = re.findall(url_finder, trigger) process_urls(bot, trigger, urls) From 8e7aedd245c61e3d35b7c39d33c589544b71108c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Mon, 1 Dec 2014 23:59:49 -0500 Subject: [PATCH 26/28] parse tags and descriptions in URLs --- bookie.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/bookie.py b/bookie.py index 827714c..dc9d3ea 100644 --- a/bookie.py +++ b/bookie.py @@ -7,16 +7,6 @@ instance. It needs to be configured with a username/key to be functional, per-channel configs are possible. -Missing: -* add tags, extended descriptions options to .bmark -* parse #tags on the auto url parser - -The above is annoyingly hard with regexes... but i've had good success -with "non-hungry" patterns: - ->>> re.findall(r'(?u)(.*?)(!?(?:http|https|ftp)(?:://\S+))(.*?)', 'cool url: http://example.com and another http://example.org') -[('cool url: ', 'http://example.com', ''), (' and another ', 'http://example.org', '')] - Also, this uses only a tiny part of the Bookie API, we could expand functionalities here significantly: @@ -139,7 +129,7 @@ def setup(bot): if bot.config.has_option('url', 'exclusion_char'): exclusion_char = bot.config.url.exclusion_char - url_finder = re.compile(r'(?u)(%s?(?:http|https|ftp)(?:://\S+))' % + url_finder = re.compile(r'(?u)(.*?)\s*(%s?(?:http|https|ftp)(?:://\S+)\s*(.*?))' % (exclusion_char)) if not bot.memory.contains('url_callbacks'): bot.memory['url_callbacks'] = tools.WillieMemory() @@ -151,7 +141,7 @@ def shutdown(bot): del bot.memory['url_callbacks'][regex] @commands('bmark') -@example('.bmark http://example.com', '[ Example ] - example.com') +@example('.bmark #tag description http://example.com', '[ Example ] - example.com') def bmark(bot, trigger): # cargo-culted from url.py if not trigger.group(2): @@ -186,7 +176,7 @@ def title_auto(bot, trigger): results = process_urls(bot, trigger, urls) def process_urls(bot, trigger, urls): - for url in urls: + for pre, url, post in urls: if not url.startswith(exclusion_char): # Magic stuff to account for international domain names try: @@ -195,7 +185,7 @@ def process_urls(bot, trigger, urls): pass bot.memory['last_seen_url'][trigger.sender] = url # post the bookmark to the Bookie API - (title, domain, resp, headers) = api_bmark(bot, trigger, url) + (title, domain, resp, headers) = api_bmark(bot, trigger, url, pre+post) if headers['_http_status'] != 200: status = 'error from bookie API: %s' % text(resp.decode('utf-8', 'ignore')) else: @@ -246,7 +236,7 @@ def api(bot, trigger, func, data=None): bot.debug('bookie', 'response: %s (headers: %s, body: %s)' % (r, r.text, r.headers), 'verbose') return (r.text, r.headers) -def api_bmark(bot, trigger, found_match=None): +def api_bmark(bot, trigger, found_match=None, extra=None): url = found_match or trigger bytes = web.get(url) # XXX: needs a patch to the URL module @@ -257,6 +247,17 @@ def api_bmark(bot, trigger, found_match=None): u'is_private': int(api_private), u'description': title.encode('utf-8'), u'content': bytes} + if extra is not None: + # extract #tags, uniquely + # copied from http://stackoverflow.com/a/6331688/1174784 + tags = {tag.strip("#") for tag in extra.split() if tag.startswith("#")} + if tags: + data['tags'] = ' '.join(tags) + # strip tags from message and see what's left + message = re.sub(r'#\w+', '', extra).strip() + if message <> '': + # something more than hashtags was provided + data['extended'] = extra return [title, get_hostname(url)] + list(api(bot, trigger, 'bmark', data)) def find_title(url=None, content=None): From a4b679004f649cce9502d6727af82a78578749e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Tue, 2 Dec 2014 18:47:44 -0500 Subject: [PATCH 27/28] don't automatically add bookmarks by default without command --- bookie.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/bookie.py b/bookie.py index dc9d3ea..14148e3 100644 --- a/bookie.py +++ b/bookie.py @@ -39,9 +39,6 @@ urlparse = urllib.parse.urlparse -# we match all URLs to override the builtin url.py module -regex = re.compile('.*') - # an HTML tag. cargo-culted from etymology.py r_tag = re.compile(r'<[^>]+>') r_whitespace = re.compile(r'[\t\r\n ]+') @@ -81,6 +78,11 @@ def configure(config): 'private', 'Mark bookmarks as private', True) + config.interactive_add( + 'bookie', + 'auto', + 'Automatically parse bookmarks', + False) if config.option('Would you like to configure individual accounts per channel?', False): c = 'Enter the API URL as #channel:account:key:private' @@ -131,14 +133,15 @@ def setup(bot): url_finder = re.compile(r'(?u)(.*?)\s*(%s?(?:http|https|ftp)(?:://\S+)\s*(.*?))' % (exclusion_char)) - if not bot.memory.contains('url_callbacks'): - bot.memory['url_callbacks'] = tools.WillieMemory() - bot.memory['url_callbacks'][regex] = bmark - + if bot.config.bookie.auto: + if not bot.memory.contains('url_callbacks'): + bot.memory['url_callbacks'] = tools.WillieMemory() + bot.memory['url_callbacks'][re.compile('.*')] = bmark def shutdown(bot): - del bot.memory['url_callbacks'][regex] + if bot.config.bookie.auto: + del bot.memory['url_callbacks'][re.compile('.*')] @commands('bmark') @example('.bmark #tag description http://example.com', '[ Example ] - example.com') From ec0503ce9bf5705910f6e1b7303e00a15396312b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Wed, 3 Dec 2014 10:13:41 -0500 Subject: [PATCH 28/28] explain better what Bookie is and why it's useful --- bookie.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/bookie.py b/bookie.py index 14148e3..6e7af0a 100644 --- a/bookie.py +++ b/bookie.py @@ -7,7 +7,18 @@ instance. It needs to be configured with a username/key to be functional, per-channel configs are possible. -Also, this uses only a tiny part of the Bookie API, we could expand +Bookie is an open-source bookmarking application that is hosted on +http://bookie.io/ and can also be self-hosted. It is similar in +functionality to the http://del.icio.us/ commercial service. + +Bookie can be useful to store a cached copy of links mentionned on +IRC. It will also generate an RSS feed of those links automatically, +and more! The author, for example, turns those RSS feeds into ePUB +e-books that are then transfered on his e-book reader so in effect, +Bookie and this plugin create a way to read links mentionned on IRC on +his ebook reader, offline. + +This plugin uses only a tiny part of the Bookie API, we could expand functionalities here significantly: https://github.com/bookieio/Bookie/blob/develop/docs/api/user.rst