.. _fbf.plugs.extra.overflow:

overflow
~~~~~~~~

.. automodule:: fbf.plugs.extra.overflow
        :show-inheritance:
        :members:
        :undoc-members:

CODE
----

    ::

        # fbf/plugs/common/overflow.py
        #
        #
        
        """ Grabs data for a StackOverflow user. You must enable this plugin first by running .. ;overflow-cfg enable 1 """
        
.. _fbf.plugs.extra.overflow_fbf_imports:

fbf imports
--------------

    ::

        
        from fbf.utils.exception import handle_exception
        from fbf.utils.lazydict import LazyDict
        from fbf.utils.url import geturl2, striphtml, re_url_match
        from fbf.utils.generic import splittxt
        from fbf.utils.timeutils import today
        from fbf.lib.persistconfig import PersistConfig
        from fbf.lib.persiststate import PlugState
        from fbf.lib.fbfimport import import_byfile
        from fbf.lib.datadir import getdatadir
        from fbf.lib.callbacks import callbacks
        from fbf.lib.commands import cmnds
        from fbf.lib.examples import examples
        from fbf.lib.fleet import getfleet
        from fbf.lib.periodical import minutely, periodical
        from fbf.lib.errors import URLNotEnabled
        from fbf.imports import getjson
        
        json = getjson()
        
.. _fbf.plugs.extra.overflow_basic_imports:

basic imports
----------------

    ::

        
        import os
        import logging
        import uuid
        import time
        import io
        import gzip
        import urllib.request, urllib.parse, urllib.error
        
.. _fbf.plugs.extra.overflow_defines_:

defines
----------

    ::

        
        cfg = PersistConfig()
        cfg.define("enable", 0)
        cfg.define("sleep", 5)
        
        state = PlugState()
        state.define("ids", {})
        state.define("seen", [])
        state.define("names", {})
        state.define("watch", [])
        
        teller = 0
        dostop = False
        
.. _fbf.plugs.extra.overflow_plugin_init:

plugin init
--------------

    ::

        
        def init_threaded():
            try:
                if cfg.enable: sync() ; scan()
            except URLNotEnabled: logging.error("URL fetching is not enabled")
        
.. _fbf.plugs.extra.overflow_plugin_shutdown:

plugin shutdown
------------------

    ::

        
        def shutdown():
            periodical.kill()
        
.. _fbf.plugs.extra.overflow_make_sure_plugin_gets_autoloaded_on_start:

make sure plugin gets autoloaded on start
--------------------------------------------

    ::

        
        def dummycb(bot, event): pass
        
        callbacks.add("START", dummycb)
        
        
.. _fbf.plugs.extra.overflow_geturls_function:

geturls function
-------------------

    ::

        
        def geturls(txt):
            result = []
            if "http://" in txt or "https://" in txt:
                for item in re_url_match.findall(txt):
                    logging.debug("web - raw - found url - %s" % item)
                    try: txt = txt.replace(item, '')
                    except ValueError:  logging.error("web - invalid url - %s" % url)
                    i = item
                    if i.endswith('"'): i = i[:-1]
                    if i.endswith('")'): i = i[:-2]
                    result.append(i)
            return (result, striphtml(txt))
        
.. _fbf.plugs.extra.overflow_OverFlowAPI_:

OverFlowAPI
--------------

    ::

        
        class OverFlowAPI(object):
        
            def __init__(self, api_key=None):
                self.api_key = api_key
        
            def api(self, mount, size=30, options={}):
                url = 'http://api.stackoverflow.com/1.0%s/%s?body=true&pagesize=%s' % (mount, urllib.parse.urlencode(options), size)
                if self.api_key is not None:
                    url += '&key=%s' % self.api_key
                content = geturl2(url, timeout=15, bytes=True)
                contentstream = io.BytesIO(content)
                return str(gzip.GzipFile(fileobj=contentstream).read(), "utf-8")
          
            def timeline(self, target, size=30):
                json_data = self.api("/users/%s/timeline" % target, size)
                try: res = json.loads(json_data)['user_timelines']
                except ValueError as ex: logging.error("can't parse %s" % json_data) ; res = None
                return res
        
            def answers(self, target, size=30):
                json_data = self.api("/answers/%s/" % target, size)
                return json.loads(json_data)['answers']
        
            def search(self, title, size=30, tags=["google-app-engine",]):
                json_data = self.api("/search?intitle=%s&tagged=%s" % (title, ";".join(tags)), size)
                return json.loads(json_data)
        
        of = OverFlowAPI()
        
.. _fbf.plugs.extra.overflow_gettimeline_function:

gettimeline function
-----------------------

    ::

        
        def gettimeline(target, nr=30):
            answers = of.timeline(target, nr)
            logging.info("grabbed %s timeline items for %s" % (len(answers), target))
            return answers
        
        def getanswers(target, nr=30):
            answers = of.answers(target, nr)
            logging.info("grabbed %s answer items for %s" % (len(answers), target))
            return answers
        
        def search(target, nr=30):
            answers = of.search(target, nr)
            logging.info("grabbed %s answer items for %s" % (len(answers), target))
            return answers
        
        def sync():
            target = ";".join(state.data.watch)
            if not target: logging.warn("no channels started yet") ; return
            res = gettimeline(target)
            if not res: logging.warn("no result from %s" % id) ; return
            todo = []
            for r in res:
                a = LazyDict(r)
                logging.debug("got %s" % a.tojson())
                if a.creation_date not in state.data.seen: state.data.seen.insert(0, a.creation_date) ; todo.append(a)
                #todo.append(a)
            state.data.seen = state.data.seen[:100]
            state.save()
            logging.info("returned %s items" % len(todo))
            return todo
        
.. _fbf.plugs.extra.overflow_scan_function:

scan function
----------------

    ::

        
        @minutely
        def scan(skip=False):
            global teller, dostop
            if dostop: return
            teller += 1
            try: do = int(cfg.sleep)
            except ValueError: do = 5
            if do < 1: do = 5
            if teller % do != 0: return 
            logging.info("running")
            fleet = getfleet()
            todo = sync()
            if not todo: logging.info("nothing todo") ; return
            for b in todo:
                uid = str(b.user_id)
                if not uid in state.data.ids: logging.warn("we don't follow id %s" % uid) ; continue
                for channel in state.data.ids[uid]:
                    if dostop: return
                    botname, chan = channel
                    bot = fleet.byname(botname)
                    if bot:
                        if b.post_id: url = ("http://stackoverflow.com/questions/%s" % b.post_id) or "no url found"
                        bot.say(chan, "*%s* %s - *%s* - %s - %s - %s (%s)" % (state.data.names[uid].upper(), b.action, b.description, url, time.ctime(b.creation_date), b.detail or "no detail", b.post_type))
                        if b.action == "answered":
                            aa = getanswers(b.post_id)
                            if aa:
                                a = aa[-1]
                                try: body = a['body']
                                except KeyError: continue
                                (urls, c) = geturls(body)
                                if c: bot.say(chan, "> " + c)
                                else: bot.say(chan, "can't find answers")
                                if urls: bot.say(chan, "urls: %s" % " -=- ".join(urls))
                    else: logging.warn("no %s bot in fleet" % botname)
        
.. _fbf.plugs.extra.overflow_overflow-start_command:

overflow-start command
-------------------------

    ::

        
        def handle_overflowstart(bot, event):
            global state
            for bla in event.args:
                try: name, gid = bla.split(":")
                except: name = gid = bla
                state.data.names[gid] = name
                target = [bot.cfg.name, event.channel]
                if gid not in state.data.ids: state.data.ids[gid] = []
                if not gid in state.data.watch or not target in state.data.ids[gid]: state.data.ids[gid].append(target) ; state.data.watch.append(gid)
                else: event.reply("we are already monitoring %s in %s" % (gid, str(target)))
            state.save()
            sync()
            event.done()
        
        cmnds.add("overflow-start", handle_overflowstart, ["OPER", ])
        examples.add("overflow-start", "start monitoring a stackoverflow id into the channel", "overflow-start feedbackoverflow:625681")
        
.. _fbf.plugs.extra.overflow_overflow-stop_command:

overflow-stop command
------------------------

    ::

        
        def handle_overflowstop(bot, event):
            if not event.args: event.missing("<stackoveflow id") ; return
            global state
            id = event.args[0]
            try:
                del state.data.ids[id]
                del state.data.names[id]
                state.save()
                event.done()
            except (KeyError, ValueError): event.reply("we are not monitoring %s in %s" % (id, event.channel))
            
        cmnds.add("overflow-stop", handle_overflowstop, ["OPER", ])
        examples.add("overflow-stop", "stop monitoring a stackoverflow id", "overflow-stop 625680")
        
.. _fbf.plugs.extra.overflow_overflow-list_command:

overflow-list command
------------------------

    ::

        
        def handle_overflowlist(bot, event): event.reply("ids list: ", state.data.ids)
        
        cmnds.add("overflow-list", handle_overflowlist, ['OPER', ])
        
.. _fbf.plugs.extra.overflow_overflow-names_command:

overflow-names command
-------------------------

    ::

        
        def handle_overflownames(bot, event): event.reply("names list: ", state.data.names)
        
        cmnds.add("overflow-names", handle_overflownames, ['OPER', ])
        
.. _fbf.plugs.extra.overflow_overflow-disable_command:

overflow-disable command
---------------------------

    ::

        
        def handle_overflowdisable(bot, event): global dostop ; dostop = True ; event.done()
        
        cmnds.add("overflow-disable", handle_overflowdisable, ['OPER', ])
        
.. _fbf.plugs.extra.overflow_overflow-enable_command:

overflow-enable command
--------------------------

    ::

        
        def handle_overflowenable(bot, event): global dostop ; dostop = False ; event.done()
        
        cmnds.add("overflow-enable", handle_overflowenable, ['OPER', ])
        
.. _fbf.plugs.extra.overflow_overflow-answers_command:

overflow-answers command
---------------------------

    ::

        
        def handle_overflowanswers(bot, event):
            result = []
            for aa in getanswers(event.rest):
                a = LazyDict(aa)
                result.append("%s - %s" % (a.owner['display_name'], striphtml(a.body)))
            event.reply("answers for %s: " % event.rest, result)
            
        cmnds.add("overflow-answers", handle_overflowanswers, ["OPER", "USER"])
        
.. _fbf.plugs.extra.overflow_overflow-search_command:

overflow-search command
--------------------------

    ::

        
        def handle_overflowsearch(bot, event):
            result = []
            res = search(event.rest, 10)
            nritems = 10
            for aa in res['questions'][:nritems]:
                a = LazyDict(aa)
                if not a.accepted_answer_id: continue
                url = "http://stackoverflow.com/questions/%s" % a.accepted_answer_id
                result.append("%s - %s - %s" % (a.title, ";".join(a.tags), url))
            if result: event.reply("results: ", result, dot=" -=- ")
            else: event.reply("no result found")
        
        cmnds.add("overflow-search", handle_overflowsearch, ["OPER", "USER"])
        
        
        """
        
        {"user_id": 818274, 
         "description": "What is the proper way to write to the Google App Engine blobstore as a file in Python 2.5", 
         "comment_id": 11614983, 
         "detail": "Yes, but it hardly explains the out of memory warning. A list of 4000 integers is something like 4000 times 40 bytes, i.e. 160 KB. That may seem a lot but is a drop in the 128 MB bucket...", 
        "creation_date": 1328857684, 
        "post_id": 9219465, 
        "post_type": "answer", 
        "action": "comment", 
        "timeline_type": "comment"}
        
        
        """