Source code for zbot.plugs.spider

# zbot/plugs/spider.py
#
#


""" 
    spider .. spider websites and make an index into them.

    taken from http://code.activestate.com/recipes/576551-simple-web-crawler/

"""

## IMPORTS

from zbot import TaskRunner, kernel, Object, Spider
from zbot.utils import strip_html, do_url

import logging

## spider command

def do_spider(event):
    try: url = event.args[1]
    except IndexError: logging.warn("spider <url>") ; return
    s = Spider()
    s.crawl(url)

kernel.cmnds.register("spider", do_spider)