Source code for core
# core/__init__.py
#
#
""" basic package for the program. """
__copyright__ = "Copyright 2014 B.H.J Thate"
__version__ = 40
## IMPORTS
from core.utils import show_eggs, get_how, get_clsname, headertxt, get_plugname
from core.utils import txt_parse, list_files, pretty, get_strace, get_name
from core.utils import j, cdir, rtime, error, smooth, short_date
from core.utils import a_time, nr_days
from core.defines import YELLOW, BLUE, GREEN, ENDC
from core.log import datefmt
import collections
import hashlib
import threading
import importlib
import logging
import inspect
import _thread
import pkgutil
import socket
import fcntl
import queue
import types
import errno
import core
import json
import time
import cgi
import sys
import os
import re
## BOOT
homedir = os.path.expanduser("~")
try: hostname = socket.getfqdn()
except: hostname = "localhost"
port = 10102
## BASE
[docs]class Object(dict):
""" basic Object on which the rest of the program is based. """
def __getattribute__(zelf, *args, **kwargs):
name = args[0]
if name == "parsed": return txt_parse(zelf.get_rest())
if name == "url": return zelf.get_url("get")
if name == "url_show": return zelf.get_url("show")
if name == "what": return get_clsname(zelf)
if name == "how" and "func" in zelf: return get_how(zelf.func)
return dict.__getattribute__(zelf, *args, **kwargs)
def __getattr__(zelf, name):
if name in zelf: return zelf[name]
if name == "_ready": zelf["_ready"] = threading.Event()
if name == "_status": zelf["_status"] = Object()
if name == "_state": zelf["_state"] = Object()
if name == "_run": zelf["_run"] = Object()
if name == "_no_save": zelf["_no_save"] = Object()
if name not in zelf: raise AttributeError(name)
return zelf[name]
def __setattr__(zelf, name, value):
if name in zelf and type(zelf[name]) in [types.BuiltinMethodType, types.BuiltinFunctionType, types.MethodType, types.FunctionType]: raise OverloadError(name)
return dict.__setitem__(zelf, name, value)
def __iter__(zelf):
for key in zelf.keys(): yield key
## FLEET
[docs] def announce(zelf, *args, **kwargs):
""" announce to all the running cores. """
for bot in kernel.fleet: bot.announce(args[0])
## STATE
[docs] def define(zelf, *args, **kwargs):
""" set a attribute on this object. """
name = args[0]
value = args[1]
zelf[name] = value
[docs] def register(zelf, *args, **kwargs):
""" register a value in a list on this object. """
o = Object()
o.name = args[0]
o.func = args[1]
if o.name not in zelf: zelf[o.name] = []
zelf[o.name].append(o)
[docs] def remove(zelf, *args, **kwargs):
name = args[0]
if name in zelf: del zelf[name]
zelf[name] = []
## GETTERS
[docs] def get_root(zelf, *args, **kwargs):
""" root directory of the program. """
return j(os.path.expanduser("~"), *args)
[docs] def get_cmnd(zelf, *args, **kwargs):
""" determine the command in the zelf.txt attribute, if present. """
if "txt" in zelf and zelf.txt:
val = zelf.txt.split()[0]
if not val: return
if "cc" in zelf:
if val[0] != zelf.cc: return
else: return val[1:]
return val
[docs] def get_rest(zelf, *args, **kwargs):
""" get the rest of the txt attribute. """
try: cmnd, rest = zelf.txt.split(" ", 1) ; return rest
except ValueError: return ""
[docs] def get_args(zelf, *args, **kwargs):
""" get the arguments of the txt attribute. """
if "txt" in zelf: return zelf.txt.split()
[docs] def get_days(zelf, *args, **kwargs):
""" get the number of days relative to the object's creation time. """
t1 = time.time()
t2 = a_time(zelf.get_timed())
if t2:
time_diff = float(t1 - t2)
return nr_days(time_diff)
[docs] def get_timed(zelf, *args, **kwargs):
""" retrieve the creation time of an object. """
val = ""
if "Date" in zelf: val = short_date(zelf.Date)
elif "date" in zelf: val = short_date(zelf.date)
elif "published" in zelf: val = short_date(zelf.published)
elif "saved" in zelf: val = short_date(zelf.saved)
elif "added" in zelf: val = short_date(zelf.added)
return val
[docs] def get_parsed(zelf, *args, **kwargs):
""" parse the txt attribute. """
rest = zelf.get_rest()
return txt_parse(rest)
[docs] def get_keys(zelf, *args, **kwargs):
""" skip the unwanted keys e.g those that start with a "_". """
for key in zelf.keys():
k = str(key)
if k.startswith("_"): continue
yield key
[docs] def get_obj(zelf, *args, **kwargs):
""" cloned object, with only the proper keys used. """
res = Object()
for key in zelf.get_keys(): res[key] = zelf[key]
return res
[docs] def get_slice(zelf, *args, **kwargs):
""" take a slice of the Object. """
o = Object()
arguments = args[0]
for arg in arguments:
try: o[arg] = zelf[arg]
except KeyError: continue
return o
[docs] def get_path(zelf, *args, **kwargs):
""" timestamp of related filename. """
if "path" in zelf._no_save: return zelf._no_save.path
[docs] def get_filetime(zelf, *args, **kwargs):
""" timestamp of related filename. """
if "path" in zelf._no_save: return zelf._no_save.path.split(os.sep)[-1]
[docs] def get_url(zelf, *args, **kwargs):
""" url of the object's file so that it can be retrieved when API server is running. """
import core.plugs.api as api
fn = os.sep.join(zelf.get_path().split(os.sep)[-2:])
if fn: return "http://%s:%s/%s/%s" % (hostname, port, args[0], fn)
## CHECKERS
[docs] def check_wanted(zelf, *args, **kwargs):
""" whether an object is desired. """
want = args[0]
for key, value in want.items():
if key == "format": continue
if key not in zelf: continue
if value.startswith("-"): continue
if value not in str(zelf[key]): return False
return True
[docs] def check_notwanted(zelf, *args, **kwargs):
""" whether an object is not desired. """
not_want = args[0]
for key, value in not_want.items():
if key == "format": continue
if key not in zelf: continue
if value in zelf[key]: return True
return False
## INPUT
[docs] def load(zelf, *args, **kwargs):
""" load a JSON file into this object. """
if args: path = args[0]
else: path = zelf._no_save["path"]
ondisk = zelf.read(path)
fromdisk = json.loads(ondisk)
zelf._no_save.path = path
zelf.update(fromdisk)
if "data" in zelf: zelf.update(zelf.data)
return zelf
[docs] def read(zelf, *args, **kwargs):
""" read the JSON file from disk. """
path = args[0]
try: f = open(path, "r")
except IOError as ex:
if ex.errno == errno.ENOENT: return "{}"
raise
res = ""
for line in f:
if not line.strip().startswith("#"): res += line
if not res.strip(): return "{}"
f.close()
return res
## PERSISTENCE
[docs] def write(zelf, *args, **kwargs):
""" actual write to disk. """
path = args[0]
logging.warn("write %s" % path)
base, fn = os.path.split(os.path.abspath(os.path.expanduser(path)))
cdir(base)
datafile = open(path + ".tmp", 'w')
fcntl.flock(datafile, fcntl.LOCK_EX | fcntl.LOCK_NB)
datafile.write(zelf.to_json(*args, **kwargs))
datafile.write("\n")
fcntl.flock(datafile, fcntl.LOCK_UN)
datafile.close()
os.rename(path + ".tmp", path)
return zelf
[docs] def to_disk(zelf, *args, **kwargs):
""" create JSON ready to be saved to disk. """
path = args[0]
todisk = Object()
todisk.create_type = zelf.what
todisk.version = __version__
todisk.saved = time.ctime(time.time())
todisk.data = zelf.get_obj()
todisk.signature = todisk.make_signature()
try: result = todisk.to_json(indent=2, ensure_ascii=False, sort_keys=True)
except TypeError: raise NoJSON()
return result
[docs] def save(zelf, *args, **kwargs):
""" save JSON to disk. """
if not args: t = rtime()
else: t = args[0]
root = j(kernel.cfg.workdir, t)
#cdir(root)
zelf.sync(root)
return root
[docs] def sync(zelf, *args, **kwargs):
""" sync JSON to disk. """
try: path = args[0] ; zelf._no_save.path = path
except IndexError: path = zelf._no_save.path
logging.warn("sync %s" % path)
d, fn = os.path.split(path)
cdir(d)
todisk = zelf.to_disk(path, **kwargs)
datafile = open(os.path.abspath(path) + ".tmp", 'w')
fcntl.flock(datafile, fcntl.LOCK_EX | fcntl.LOCK_NB)
datafile.write(headertxt % (__version__, "%s characters" % len(todisk)))
datafile.write(todisk)
datafile.write("\n")
fcntl.flock(datafile, fcntl.LOCK_UN)
datafile.close()
os.rename(path + ".tmp", path)
return zelf
## OUTPUT
[docs] def reply(zelf, *args, **kwargs):
""" send reply to origin. """
if "outer" in zelf: zelf.outer.write(str(args[0]) + "\n") ; zelf.outer.flush() ; return
txt = args[0]
if "channel" in zelf: zelf._target.say(zelf.channel, txt)
else: zelf._target._raw(txt)
[docs] def say(zelf, *args, **kwargs):
""" output text through the _target attribute. """
zelf._target.say(args[0], args[1])
[docs] def ok(zelf, *args, **kwargs):
""" signal ok. """
if "_target" in zelf: zelf.reply("ok %s" % " ".join([str(a) for a in args]))
[docs] def show(zelf):
""" list of key,value pairs. """
return ["%s=%s" % (a, b) for a, b in zelf.items()]
## ITERATORS
[docs] def objects(zelf, *args, **kwargs):
""" list of all object's. """
if "time" in kwargs: desired_time = kwargs["time"]
else: desired_time = ""
objs = []
path = zelf.get_root(kernel.cfg.workdir)
logging.warn("loading %s" % path)
for fnn in list_files(path, **kwargs):
obj = Object().load(fnn, **kwargs)
if args and args[0] not in obj: continue
if desired_time and desired_time not in obj.get_timed(): continue
if "deleted" in obj and obj.deleted: continue
objs.append(obj)
return objs
[docs] def selected(zelf, *args, **kwargs):
""" list of desired objects. """
parsed = args[0]
if kernel.cfg.get("debug", None): logging.warn("> parse %s" % parsed.pretty())
if "exclude" in kwargs: exclude = kwargs["exclude"]
else: exclude = None
if exclude in parsed.args: parsed.args.remove(exclude)
if "time" in parsed.wanted: kwargs["time"] = parsed.wanted.time
objs = []
for obj in zelf.objects(**kwargs):
if not obj.check_wanted(parsed.wanted): continue
if not "full" in kwargs and not obj.selector(parsed.args): continue
if obj.check_notwanted(parsed.not_wanted): continue
objs.append(obj)
return objs
## SELECTOR
[docs] def selector(zelf, *args, **kwargs):
""" see if this objects has the desired attributes. """
wanted = args[0]
if not wanted: return True
go = False
for w in wanted:
if zelf.get(w, ""): go = True ; break
return go
## TIME RELATED
[docs] def first(zelf, *args, **kwargs):
""" return the first object where the key and/or value matches. """
key = args[0]
try: value = args[1]
except IndexError: value = None
result = None
for o in zelf.objects():
if key not in o: continue
if value and o[key] != value: continue
if not result: result = o
if o.get("saved", "") < result.get("saved", ""): result = o
return result
[docs] def last(zelf, *args, **kwargs):
""" return the last object where the key and/or value matches. """
key = args[0]
try: value = args[1]
except IndexError: value = None
result = None
for o in zelf.objects():
if key not in o: continue
if value and value != o[key]: continue
if not result: result = o
if a_time(short_date(o.saved)) > a_time(short_date(result.saved)): result = o
return result
## WAITERS
[docs] def ready(zelf):
""" signal to ready state. """
logging.debug("ready %s %s" % (get_name(zelf), get_strace()))
zelf._ready.set()
[docs] def clear(zelf):
""" clear the ready state. """
logging.debug("clear %s %s" % (get_name(zelf), get_strace()))
zelf._ready.clear()
[docs] def wait(zelf, sec=180.0):
""" wait for ready state. """
logging.debug("wait %s %s" % (get_name(zelf), get_strace()))
try: zelf._ready.wait(sec)
except: pass
## HELPERS
[docs] def to_json(zelf, *args, **kwargs):
""" JSON string representation of this object. """
return json.dumps(zelf.get_obj(*args, **kwargs), default=smooth, *args, **kwargs)
[docs] def to_full(zelf, *args, **kwargs):
""" full JSON dump of this object. """
return json.dumps(zelf, default=smooth, indent=4)
[docs] def make_signature(zelf, sig=None):
""" signature of the data contained in this object. """
if "data" in zelf: return str(hashlib.sha1(bytes(str(zelf.data), "utf-8")).hexdigest())
raise NotSet
[docs] def make_path(zelf, *args, **kwargs):
""" create workdir if necessary. """
path = zelf.get_path(*args, **kwargs) ; cdir(path) ; return path
[docs] def pretty(zelf):
""" nice formatted JSON string of this object. """
return json.dumps(zelf.get_obj(), indent=2, default=pretty, sort_keys=True, ensure_ascii=True)
## BOOT
[docs]def boot(*args, **kwargs):
""" get the program properly initialized. """
global kernel
global workdir
if args: cfg = args[0]
else: cfg = Object()
if "mods" not in cfg: cfg.mods = []
cfg.loglevel = cfg.get("loglevel", "error")
core.log.init(cfg.loglevel)
cfg.workdir = cfg.get("workdir", None) or cfg.get("wantdir", None) or homedir + os.sep + "core.data"
if cfg.get("eggs", None): cfg.workdir = os.path.expanduser(j(os.getcwd(), "data"))
cdir(cfg.workdir)
kernel.cfg.update(cfg)
if kernel.cfg.loglevel != "error":
for line in kernel.cfg.show(): logging.warn(line)
if kernel.cfg.loglevel != "error":
print("")
print("%sDEPENDS%s" % (GREEN, ENDC))
print("")
show_eggs("core")
for mod in kernel.cfg.mods: show_eggs(mod)
print("")
print("%sPLUGINS%s" % (BLUE, ENDC))
print("")
kernel.plugs.load_plugs("core.plugs")
for mod in kernel.cfg.mods: kernel.plugs.load_plugs("%s.plugs" % mod)
kernel.plugs.init(cfg.get("init", ""))
kernel.plugs.wait()
if kernel.cfg.shell and kernel.cfg.loglevel != "error": print("")
kernel.ready()
return kernel
## SHUTDOWN
[docs]def shutdown():
""" close the program in a appropiate manner. """
global kernel
for obj in kernel.services:
if obj == None: continue
try: obj.exit()
except AttributeError: pass
except: error()
kernel.cmnds.exit()
kernel.plugs.exit()
if kernel.cfg.shell: print("") ; logging.warn("exit")
os._exit(0)
## KERNEL
import core.dispatch
import core.plugins
kernel = Object()
kernel.cmnds = core.dispatch.Dispatcher()
kernel.workers = core.dispatch.Dispatcher()
kernel.plugs = core.plugins.Plugins()
kernel.services = Object()
kernel.cfg = Object()
kernel.fleet = []