Source code for core.service.rss

# core/service/rss.py
#
#

""" rss module. """

__copyright__ = "Copyright 2015, B.H.J Thate"

## IMPORTS

from core.utils.url import get_url, strip_html, need_redirect
from core.utils.lock import locked
from core.utils.time import short_date
from core.errors import error
from core.clock import Repeater
from core.kernel import kernel
from core.thing import Thing
from core.cfg import cfg_rss

import logging
import time

## DEFINES

seen = Thing()
seen.list = []
seen.results = "rss"
seen.prefix = "results"

## display function

def display(*args, **kwargs):
    entry = args[0]
    result = ""
    for key in cfg_rss.keys_list:
        if entry.get(key, None): result += "%s - " % entry[key]
    return result[:-3]

## get function

@locked
def get(url):
    import core.extra.feedparser as fp
    result = []
    logging.info("^ rss %s" % url)
    try:
        resp = get_url("GET", url)
        url = need_redirect(resp)
        if url: resp = get_url("GET", url)
        data = resp.read()
        if data: result = fp.parse(data)
    except Exception as ex: error(url) ; return
    for entry in result["entries"]: yield Thing(entry)

def fetch(*args, **kwargs):
    global seen
    obj = args[0]
    for o in get(obj.rss):
        if o.link in seen.list: continue
        seen.list.append(o.link)
        s = o.slice(cfg_rss.keys_list)
        s.service = "rss"
        s.prefix = "rss"
        s.short = short_date(time.ctime())
        kernel.announce(display(s))

## look function

def rss_fetch(*args, **kwargs):
    for fn in kernel.all("rss"):
        obj = Thing().load(fn)
        if "deleted" in obj and obj.deleted: continue
        if "rss" not in obj: continue
        if "http" not in obj.rss: continue
        kernel.put(fetch, obj)
        time.sleep(0.1)

kernel.register("rss.fetch", rss_fetch)

## 

def write_rss(event):
    o = Thing(event)
    o.prefix = "rss"  
    o.rss = event.parsed.rest
    o.service = "rss"
    path = o.save()  
    event.ok()

kernel.register("write.rss", write_rss)

## INIT

def init(*args, **kwargs):
    global seen
    last_rss = kernel.last("results", "rss")
    if last_rss: seen = last_rss
    cfg = kernel.last("cfg", "rss")
    if not cfg: cfg = cfg_rss ; cfg.save()
    rss = Repeater(rss_fetch, 600)
    kernel.put(rss.start)

def shutdown(*args, **kwargs): seen.sync()