Source code for zot.object
# zot/object.py
#
#
""" basic package for the program. """
__copyright__ = "Copyright 2015, B.H.J Thate"
## IMPORTS
from zot.utils import get_strace, smooth, pretty, get_plugname, get_name, elapsed_days, get_type
from zot.utils import short_date, j, mj, rtime, a_time, str_day, cdir, nr_days, check_permissions
from zot.utils import headertxt, txt_parse, list_files, error, get_func
from zot.defines import hostname, port
from zot.errors import WrongSignature
from zot import __version__
import threading
import hashlib
import logging
import errno
import fcntl
import types
import json
import time
import os
## BASE
[docs]class Object(dict):
""" basic Object on which the rest of the program is based. """
def __getattr__(zelf, name):
if name in zelf: return zelf[name]
if name == "_ready": zelf._ready = threading.Event()
if name == "timed": return zelf.get_timed()
if name == "parsed": return zelf.get_parsed()
if name == "type": return get_type(zelf)
if name == "url": return zelf.get_url("get")
if name == "url_show": return zelf.get_url("show")
if name not in zelf: raise AttributeError(name)
return zelf[name]
def __contains__(zelf, name):
try: zelf[name] ; return True
except KeyError: return False
def __setattr__(zelf, name, value): return dict.__setitem__(zelf, name, value)
[docs] def announce(zelf, *args, **kwargs):
from zot.runtime import fleet
thrs = []
for bot in fleet: thrs.append(zelf.put(bot.announce, *args, **kwargs))
if thrs: zelf.collect(thrs)
return thrs
## FLEET
[docs] def search(zelf, *args, **kwargs):
l = []
for key in zelf.names():
if args[0] in key: l.append(key)
return l
[docs] def represent(zelf, *args, **kwargs):
return ", ".join([str(getattr(zelf, x, None)) for x in zelf.names() if getattr(zelf, x, None)])
## STATE
[docs] def define(zelf, *args, **kwargs):
""" set a attribute on this object. """
name = args[0]
value = args[1]
zelf[name] = value
[docs] def register(zelf, *args, **kwargs):
""" callback type with corresponding callback function. """
o = Object()
o.cmnd = args[0]
o.func = args[1]
o.plugname = get_plugname()
if o.cmnd not in zelf: zelf[o.cmnd] = []
todo = []
for obj in zelf[o.cmnd]:
if obj.cmnd == o.cmnd and obj.plugname == o.plugname: todo.append(obj)
for obj in todo: zelf[o.cmnd].remove(obj) ; logging.info("# remove %s" % get_name(obj))
zelf[o.cmnd].append(o)
logging.debug("# %s/register %s %s" % (zelf.type, o.cmnd, get_name(o.func)))
[docs] def remove(zelf, *args, **kwargs):
name = args[0]
if name in zelf: del zelf[name]
zelf[name] = []
## GETTERS
[docs] def find(zelf, *args, **kwargs):
rest = args[0]
result = []
for x in set(zelf.search(rest)):
if x.split(".")[-1] != rest: continue
obj = zelf.get(x)
if type(obj) in [list, tuple]: result.extend(obj) ; continue
result.append(obj)
return result
[docs] def check(zelf, *args, **kwargs):
""" determine the command in the zelf.txt attribute, if present. """
from zot.runtime import kernel
if "txt" in zelf and zelf.txt:
splitted = zelf.txt.split()
if not splitted: return
val = splitted[0]
if "cc" in zelf:
if val[0] != zelf.cc: return False
val = val[1:]
try: kernel[val] ; return val
except KeyError: pass
return False
[docs] def rest(zelf, *args, **kwargs):
""" get the rest of the txt attribute. """
try: cmnd, rest = zelf.txt.split(" ", 1) ; return rest
except ValueError: return ""
[docs] def words(zelf, *args, **kwargs):
""" get the arguments of the txt attribute. """
if "txt" in zelf: return zelf.txt.split()
[docs] def days(zelf, *args, **kwargs):
""" get the number of days relative to the object's creation time. """
t1 = time.time()
t2 = a_time(zelf.get_timed())
if t2:
time_diff = float(t1 - t2)
return elapsed_days(time_diff)
[docs] def dated(zelf, *args, **kwargs):
""" retrieve the creation time of an object. """
val = ""
if "Date" in zelf: val = zelf.Date
elif "date" in zelf: val = zelf.date
elif "published" in zelf: val = zelf.published
elif "added" in zelf: val = zelf.added
elif "saved" in zelf: val = zelf.saved
return val
[docs] def get_timed(zelf, *args, **kwargs):
""" retrieve the creation time of an object. """
return short_date(zelf.dated()) or ""
[docs] def get_parsed(zelf, *args, **kwargs):
""" parse the txt attribute. """
if "cc" in zelf: parsed = txt_parse(zelf.txt, zelf.cc)
else: parsed = txt_parse(zelf.txt)
logging.debug("! %s/%s %s" % (zelf.type, "parsed", str(parsed)))
return parsed
[docs] def names(zelf, *args, **kwargs):
""" skip the unwanted keys e.g those that start with a "_". """
for key in zelf.keys():
k = str(key)
if k.startswith("_"): continue
yield key
[docs] def obj(zelf, *args, **kwargs):
""" cloned object, with only the proper keys used. """
res = Object()
for key in zelf.names(): res[key] = zelf[key]
return res
[docs] def slice(zelf, *args, **kwargs):
""" take a slice of the Object. """
o = Object()
arguments = args[0]
for arg in arguments:
try: o[arg] = zelf[arg]
except KeyError: continue
return o
[docs] def filetime(zelf, *args, **kwargs):
""" timestamp of related filename. """
return zelf._path.split(os.sep)[-1]
[docs] def filedate(zelf, *args, **kwargs):
""" timestamp of related filename. """
return zelf._path.split(os.sep)[-2:]
[docs] def get_url(zelf, *args, **kwargs):
""" url of the object's file so that it can be retrieved when API server is running. """
from zot.runtime import cfg
if "workdir" in zelf: root = zelf.workdir
else: root = cfg.workdir
fn = os.path.normpath(zelf._path.split(root)[-1])
if fn: return "http://%s:%s/%s%s" % (hostname, cfg.port, args[0], fn)
[docs] def get_root(zelf, *args, **kwargs):
from zot.runtime import cfg
if "workdir" in zelf: root = zelf.workdir
else: root = cfg.workdir
path = os.path.abspath(root)
check_permissions(path)
return path
[docs] def get_path(zelf, *args, **kwargs):
root = zelf.get_root()
if "prefix" in zelf: root = j(root, zelf.prefix)
return os.path.abspath(root)
## CHECKERS
[docs] def check_wanted(zelf, *args, **kwargs):
""" whether an object is desired. """
want = args[0]
for key, value in want.items():
if key == "format": continue
if key not in zelf: continue
if value.startswith("-"): continue
if value not in str(zelf[key]): return False
return True
[docs] def check_notwanted(zelf, *args, **kwargs):
""" whether an object is not desired. """
not_want = args[0]
for key, value in not_want.items():
if key == "format": continue
if key not in zelf: continue
if value in zelf[key]: return True
return False
## INPUT
[docs] def load(zelf, *args, **kwargs):
""" load a JSON file into this object. """
if args: path = args[0]
else: path = zelf._path
ondisk = zelf.read(path)
fromdisk = json.loads(ondisk)
if "data" in fromdisk: zelf.update(fromdisk["data"])
else: zelf.update(fromdisk)
if "saved" in fromdisk: zelf.saved = fromdisk["saved"]
zelf._path = path
return zelf
[docs] def read(zelf, *args, **kwargs):
""" read the JSON file from disk. """
path = args[0]
try: f = open(path, "r")
except IOError as ex:
if ex.errno == errno.ENOENT: return "{}"
raise
res = ""
for line in f:
if not line.strip().startswith("#"): res += line
if not res.strip(): return "{}"
f.close()
return res
## PERSISTENCE
[docs] def prepare(zelf, *args, **kwargs):
""" create JSON ready to be saved to disk. """
path = args[0]
todisk = Object()
todisk.data = zelf.obj()
todisk.saved_from = get_plugname(2)
todisk.type = zelf.type
todisk.saved = zelf.saved = time.ctime(time.time())
todisk.signature = todisk.data.make_signature()
try: result = todisk.json(indent=2, ensure_ascii=False, sort_keys=True)
except TypeError: raise NoJSON()
return result
[docs] def save(zelf, *args, **kwargs):
""" save JSON to disk. """
if not args: t = rtime()
else: t = args[0]
zelf.sync(j(zelf.get_path(), t))
return t
[docs] def sync(zelf, *args, **kwargs):
""" sync JSON to disk. """
try: path = args[0]
except IndexError:
try: path = zelf._path
except AttributeError: path = zelf._path = j(zelf.get_path(), rtime())
logging.info("# sync %s" % path)
d, fn = os.path.split(path)
cdir(d)
todisk = zelf.prepare(path, **kwargs)
datafile = open(os.path.abspath(path) + ".tmp", 'w')
fcntl.flock(datafile, fcntl.LOCK_EX | fcntl.LOCK_NB)
datafile.write(headertxt % "%s characters" % len(todisk))
datafile.write(todisk)
datafile.write("\n")
fcntl.flock(datafile, fcntl.LOCK_UN)
datafile.close()
os.rename(path + ".tmp", path)
return zelf
## OUTPUT
[docs] def reply(zelf, *args, **kwargs):
""" send reply to origin. """
if "outer" in zelf: zelf.outer.write(str(args[0]) + "\n") ; zelf.outer.flush() ; return
txt = args[0]
if "channel" in zelf: zelf._target.say(zelf.channel, txt)
else: zelf._target.say(zelf.origin, txt)
[docs] def ok(zelf, *args, **kwargs):
""" signal ok. """
if "_target" in zelf: zelf.reply("ok %s" % " ".join([str(a) for a in args]))
[docs] def show(zelf, *args, **kwargs):
""" list of key,value pairs. """
return ["%s=%s" % (a, zelf[a]) for a in sorted(zelf.names())]
[docs] def format(zelf, *args, **kwargs):
if args: nr = args[0]
else: nr = 0
if "skiplist" in kwargs: skiplist = kwargs["skiplist"]
else: skiplist = []
if "wanted" in kwargs: wanted = kwargs["wanted"]
else: wanted = []
txt = "%s %s - %s\n" % (nr, zelf.dated(), zelf.days())
txt += "\n".join(["%s %s: %s" % (nr, key, zelf[key]) for key in sorted(list(zelf.names()) + wanted) if key not in skiplist])
return txt
[docs] def display(zelf, *args, **kwargs):
if "keys" in kwargs: keys = kwargs["keys"]
else: keys = zelf.names()
txt = " ".join([str(getattr(zelf, str(key), "")) for key in keys])
txt = txt.rstrip() + " - %s" % zelf.days()
return txt
[docs] def display_list(zelf, *args, **kwargs):
try: index = zelf.index
except AttributeError: index = None
nr = -1
for obj in args[0]:
nr += 1
if index and nr != index: continue
zelf.reply("%s %s" % (nr, obj.display(*args, **kwargs)))
[docs] def reply_list(zelf, *args, **kwargs):
try: index = zelf.parsed.index
except AttributeError: index = None
nr = -1
for obj in args[0]:
nr += 1
if index and nr != index: continue
zelf.reply(obj.format(nr, *args, **kwargs) + "\n")
## ITERATORS
[docs] def all(zelf, *args, **kwargs):
result = []
if args: key = args[0]
else: key = ""
for o in zelf.objects(full=True):
if "data" in o and key in o.data: result.append(o)
elif key in o: result.append(o)
return result
[docs] def objects(zelf, *args, **kwargs):
""" list of all object's. """
if "time" in kwargs: desired_time = kwargs["time"]
else: desired_time = ""
objs = []
if args: path = args[0]
else: path = zelf.get_path()
nr = 1
skipped = 0
for fnn in list_files(path):
if os.path.isdir(fnn): objs.extend(zelf.objects(fnn)) ; continue
try: obj = Object().load(fnn)
except UnicodeDecodeError as ex: logging.info("%s %s" % (fnn, str(ex))); continue
if not obj: logging.error("noload %s" % fnn) ; continue
if desired_time and desired_time not in obj.timed(): continue
if "full" not in kwargs and "deleted" in obj and obj.deleted: skipped += 1 ; continue
objs.append(obj)
nr += 1
logging.info("# objects %s skipped %s" % (nr, skipped))
return objs
[docs] def full(zelf, *args, **kwargs):
parsed = args[0]
objs = []
result = []
skipped = 0
for obj in zelf.objects(**kwargs):
if obj.check_notwanted(parsed.not_wanted): continue
if "uniq" in parsed.switch:
if obj[parsed.switch["uniq"]] in result: continue
else: result.append(obj[parsed.switch["uniq"]])
objs.append(obj)
return objs
[docs] def selected(zelf, *args, **kwargs):
""" list of desired objects. """
if args: parsed = args[0]
else: parsed = zelf.parsed
if not parsed.args: return []
if "exclude" in kwargs: exclude = kwargs["exclude"]
else: exclude = None
if "exclude" in parsed.args: parsed.args.remove(exclude)
if "time" in parsed.wanted: kwargs["time"] = parsed.wanted.time
objs = []
result = []
nr = -1
for obj in zelf.objects(**kwargs):
nr += 1
if not obj.check_wanted(parsed.wanted): continue
if not obj.selector(parsed.args): continue
if obj.check_notwanted(parsed.not_wanted): continue
if "uniq" in parsed.switch:
if obj[parsed.switch["uniq"]] in result: continue
else: result.append(obj[parsed.switch["uniq"]])
if "index" in parsed and nr == parsed.index: return [obj, ]
objs.append(obj)
return objs
## SELECTOR
[docs] def selector(zelf, *args, **kwargs):
""" see if this objects has the desired attributes. """
wanted = args[0]
if not wanted: return True
go = False
for w in wanted:
if zelf.get(w, ""): go = True ; break
return go
## TIME RELATED
[docs] def first(zelf, *args, **kwargs):
""" return the first object where the key and/or value matches. """
key = args[0]
try: value = args[1]
except IndexError: value = None
result = None
for o in zelf.objects():
if key not in o: continue
if value and o[key] != value: continue
if not result: result = o
if o.get("timed", "") < result.get("timed", ""): result = o
return result
[docs] def last(zelf, *args, **kwargs):
""" return the last object where the key and/or value matches. """
key = args[0]
try: value = args[1]
except IndexError: value = None
result = None
for o in zelf.objects():
if key not in o: continue
if value and value != o[key]: continue
if not result: result = o
if o.timed > result.timed: result = o
logging.info("last %s %s" % (key, value))
return result
## WAITERS
[docs] def ready(zelf):
""" signal to ready state. """
logging.debug("! %s/ready %s %s" % (zelf.type, str(zelf._ready), get_func()))
zelf._ready.set()
[docs] def clear(zelf):
""" clear the ready state. """
logging.debug("! %s/clear %s" % (zelf.type, str(zelf._ready)))
zelf._ready.clear()
[docs] def wait(zelf, sec=180.0):
""" wait for ready state. """
logging.debug("! %s/wait %s %s" % (zelf.type, str(zelf._ready), get_func()))
zelf._ready.wait(sec)
## HELPERS
[docs] def json(zelf, *args, **kwargs):
""" JSON string representation of this object. """
return json.dumps(zelf.obj(*args, **kwargs), default=smooth, *args, **kwargs)
[docs] def to_full(zelf, *args, **kwargs):
""" full JSON dump of this object. """
return json.dumps(zelf, default=smooth, indent=4)
[docs] def make_signature(zelf):
""" signature of the data contained in this object. """
return str(hashlib.sha1(bytes(str(zelf), "utf-8")).hexdigest())
[docs] def pretty(zelf, *args, **kwargs):
""" nice formatted JSON string of this object. """
return json.dumps(zelf.obj(), indent=2, default=pretty, sort_keys=True, ensure_ascii=True)