#!/usr/bin/env python
#
# -*- mode:python; sh-basic-offset:4; indent-tabs-mode:nil; coding:utf-8 -*-
# vim:set tabstop=4 softtabstop=4 expandtab shiftwidth=4 fileencoding=utf-8:
#
# Copyright 2013, Jorge Gallegos <kad@blegh.net>
"""
Overview
--------
The Clusto API Server will work as an alternative to the direct database
access traditional clusto commands and libraries use.
Sometimes your database has restricted access or you want to expose your clusto
information across facilities but don't want to risk opening up your database
port to the outside world or don't have the required know-how to do it in a
secure manner.
Exposing an HTTP(S) endpoint is a more common problem and as such there are
several well-understood solutions. Scalability and security are also points
you have to consider.
The Clusto API Server should thus have the following required features:
* Complete object manipulation: create, delete, insert, and remove objects,
besides displaying them.
* Complete object attribute manipulation: add, delete, update, query, and
displaying attributes
* Resource manipulation: allocate and deallocate objects from resources
* Querying
Custom Headers
--------------
The Clusto API Server comes with the ability to pass certain headers to
multiple operations.
:Clusto-Mode: Determines if an object is ``compact`` or ``expanded``.
Compaction of objects helps speed up response time for multiple object
lookups. ``expanded`` is the default mode if the function returns only one
object, and is ``compact`` by default for all listing functions.
:Clusto-Per-Page: Number of entities to return when pagination is requested.
Defaults to ``50``.
:Clusto-Page: Requests the current page in a list of entities, delimited by
``Clusto-Per-Page``.
:Clusto-Pages: Response only. This header returns the total number of pages
to the requester.
:Clusto-Minify: If set to ``True`` (not case sensitive), clusto will not
give a response that has been pretty-printed.
Configurable Response Headers
-----------------------------
The Clusto API Server comes with the ability to define and configure static
response headers within the ``clusto.conf``.
:Example: To enable CORS, a line in the ``clusto.conf`` would read:
``response_headers = Access-Control-Allow-Origin:*``
API Docs
--------
"""
import bottle
import clusto
from clusto import script_helper
import clustoapi
import functools
import importlib
import inspect
import os
import string
import sys
import util
DOC_SUBSTITUTIONS = {
'get': "curl -X GET -G -s -w '\\nHTTP: %{http_code}\\nContent-type: %{content_type}'",
'get_i': "curl -X GET -G -si",
'post': "curl -X POST -s -w '\\nHTTP: %{http_code}\\nContent-type: %{content_type}'",
'post_i': "curl -X POST -si",
'put': "curl -X PUT -s -w '\\nHTTP: %{http_code}\\nContent-type: %{content_type}'",
'put_i': "curl -X PUT -si",
'delete': "curl -X DELETE -s -w '\\nHTTP: %{http_code}\\nContent-type: %{content_type}'",
'delete_i': "curl -X DELETE -si",
'head': "curl -s -I",
'sample_json_attrs': '[{"key":"group","subkey":"admin","value":"apache"},{"key":"group","subkey":"member","value":"webapp"}]'
}
root_app = bottle.Bottle(autojson=False)
def _get_url(path=False):
"""
Returns the server's normalized URL
"""
(scheme, netloc, qpath, qs, fragment) = bottle.request.urlparts
if path:
return u'%s://%s%s' % (scheme, netloc, qpath)
else:
return u'%s://%s' % (scheme, netloc)
@root_app.get('/favicon.ico')
[docs]def favicon():
"""
Send an HTTP code to clients so they stop asking for favicon. Example:
.. code:: bash
$ ${get} -o /dev/null ${server_url}/favicon.ico
HTTP: 410
Content-type: text/html; charset=UTF-8
"""
return bottle.HTTPResponse('', status=410)
@root_app.route('/', method='OPTIONS')
@root_app.route('/<url:re:.+>', method='OPTIONS')
[docs]def options(**kwargs):
"""
Defined from w3.org:
"The OPTIONS method represents a request for information about the communication
options available on the request/response chain identified by the Request-URI.
This method allows the client to determine the options and/or requirements
associated with a resource, or the capabilities of a server, without implying
a resource action or initiating a resource retrieval."
The clusto-apiserver team plans to roll this out to individual resources once
it has been used a proper amount, but for now we will return OPTIONS with
the minimum amount of required headers (and an empty content) no matter what
resource is requested.
.. code:: bash
$ ${head} -X OPTIONS ${server_url}/
HTTP/1.0 204 No Content
...
Content-Length: 0
$ ${head} -X OPTIONS ${server_url}/return/headers/no/matter/where
HTTP/1.0 204 No Content
...
Content-Length: 0
The same applies to any mounted application:
.. code:: bash
$ ${head} -X OPTIONS ${server_url}/entity/
HTTP/1.0 204 No Content
...
Content-Length: 0
$ ${head} -X OPTIONS ${server_url}/attribute/who/knows/where
HTTP/1.0 204 No Content
...
Content-Length: 0
"""
return bottle.HTTPResponse('', status=204)
@root_app.route('/', method='HEAD')
@root_app.get('/__version__')
[docs]def version():
"""
This shows the current version running, example
.. code:: bash
$ ${get} ${server_url}/__version__
"${server_version}"
HTTP: 200
Content-type: application/json
If you make a HEAD request to the / endpoint, the response is also the version
string, as that's less heavy to build than the regular / page:
.. code:: bash
$ ${head} ${server_url}/
HTTP/1.0 200 OK
...
"""
return clustoapi.util.dumps(clustoapi.__version__)
def _get_mounts_and_modules():
mods = {}
for route in root_app.routes:
mp = route.config.get('mountpoint')
if mp:
target = mp.get('target')
if target and target.config.get('source_module'):
mods[mp['prefix']] = target.config['source_module']
return mods
@root_app.get('/__meta__')
@root_app.get('/')
@root_app.get('/__doc__')
[docs]def build_docs(module=__name__):
"""
This will build documentation for the given module and all its methods.
If python-rest is available, it will attempt to parse it as a restructured
text document. You can get to the docs by going to the __doc__ endpoint on
each mounted application, the main __doc__ endpoint, or on the main endpoint:
.. code:: bash
$ ${get} ${server_url}/__doc__
<?xml version="1.0" encoding="utf-8" ?>
...
HTTP: 200
Content-type: text/html; charset=UTF-8
If you pass the ``Accept`` headers and specify ``text/plain``, you should get
the plain text version back
.. code:: bash
$ ${get} -H 'Accept: text/plain' ${server_url}/__doc__
...
HTTP: 200
Content-type: text/plain
$ diff -q <( curl -s -H 'Accept: text/plain' ${server_url}/__doc__ ) <( curl -s -H 'Accept: text/plain' ${server_url}/ ) && echo 'equal' || echo 'diff'
equal
In the test config, CORS is configured to be returned with every response header.
.. code::bash
$ ${head} ${server_url}
...
Access-Control-Allow-Origin: *
...
Or:
.. code::bash
$ ( ${head} ${server_url} | grep -q "^Access-Control-Allow-Origin:" ) && echo "Header Found" || echo "Not Here"
Header Found
If you try to get a non-configured header, it shouldn't be in the output
.. code::bash
$ ( ${head} ${server_url} | grep "^Spurious-Header:" ) && echo "Header Found" || echo "Not Here"
Not Here
"""
# Get the request path so we can look at the module index
mod = sys.modules[module]
docs = ['\n%s\n%s\n%s\n%s' % (
'=' * len(mod.__name__),
mod.__name__,
'=' * len(mod.__name__),
mod.__doc__ or '')]
if module == __name__:
mods = _get_mounts_and_modules()
if mods:
docs.append('\nMounted Applications\n%s\n' % ('-' * 20, ))
for k, v in mods.items():
docs.append(
'\n * `%s <${server_url}%s/__doc__>`_\n' % (v, k,)
)
docs.append('\nModule methods\n%s\n' % ('-' * 32,))
toc = []
methods = []
for name in dir(mod):
method = getattr(mod, name)
if name != 'main' and not name.startswith('_') and inspect.isfunction(method):
toc.append('\n * `%s()`_' % (name,))
methods.append(
'\n%s()\n%s\n%s' % (
name, '~' * (len(name) + 2), method.__doc__)
)
docs.extend(toc)
docs.extend(methods)
tpl = string.Template('\n'.join(docs))
text = tpl.safe_substitute(
server_url=_get_url(),
server_version=clustoapi.__version__,
**DOC_SUBSTITUTIONS
)
accept = bottle.request.headers.get('accept', 'text/plain')
if accept != 'text/plain':
try:
from docutils import core
return core.publish_string(source=text, writer_name='html')
except ImportError:
bottle.response.content_type = 'text/plain'
return text
else:
bottle.response.content_type = 'text/plain'
return text
@root_app.get('/driverlist')
[docs]def get_driverlist():
"""
Returns clusto.driverlist, a list of drivers and their qualified class paths.
Examples:
.. code:: bash
$ ${get} ${server_url}/driverlist
{
"basicappliance": "clusto.drivers.devices.appliance.basicappliance.BasicAppliance",
"basiccage": "clusto.drivers.locations.datacenters.basiccage.BasicCage",
"basicconsoleserver": "clusto.drivers.devices.consoleservers.basicconsoleserver.BasicConsoleServer",
"basicdatacenter": "clusto.drivers.locations.datacenters.basicdatacenter.BasicDatacenter",
"basicnetworkswitch": "clusto.drivers.devices.networkswitches.basicnetworkswitch.BasicNetworkSwitch",
"basicpowerstrip": "clusto.drivers.devices.powerstrips.basicpowerstrip.BasicPowerStrip",
"basicrack": "clusto.drivers.locations.racks.basicrack.BasicRack",
"basicserver": "clusto.drivers.devices.servers.basicserver.BasicServer",
"basicvirtualserver": "clusto.drivers.devices.servers.basicserver.BasicVirtualServer",
"basiczone": "clusto.drivers.locations.zones.basiczone.BasicZone",
"clustometa": "clusto.drivers.base.clustometa.ClustoMeta",
"device": "clusto.drivers.base.device.Device",
"entity": "clusto.drivers.base.driver.Driver",
"exclusive_pool": "clusto.drivers.categories.pool.ExclusivePool",
"ipmanager": "clusto.drivers.resourcemanagers.ipmanager.IPManager",
"location": "clusto.drivers.base.location.Location",
"pool": "clusto.drivers.categories.pool.Pool",
"resourcemanager": "clusto.drivers.base.resourcemanager.ResourceManager",
"simpleentitynamemanager": "clusto.drivers.resourcemanagers.simplenamemanager.SimpleEntityNameManager",
"simplenamemanager": "clusto.drivers.resourcemanagers.simplenamemanager.SimpleNameManager",
"unique_pool": "clusto.drivers.categories.pool.UniquePool"
}
HTTP: 200
Content-type: application/json
"""
driverlist = {}
for name, driver in clusto.driverlist.items():
# This looks silly, but it is the easiest way to get class paths in a string.
driverlist[name] = "{0}.{1}".format(driver.__module__, driver.__name__)
return util.dumps(driverlist, 200)
@root_app.get('/from-pools')
[docs]def get_from_pools():
"""
One of the main ``clusto`` operations. Parameters:
* Required: at least one ``pool`` parameter
* Optional: one or more ``driver`` parameter to filter out results
* Optional: one or more ``type`` parameter to filter out results
* Optional: a boolean ``children`` parameter to search for children
recursively (True by default)
Examples:
.. code:: bash
$ ${get} ${server_url}/from-pools
"Provide at least one pool to get data from"
HTTP: 412
Content-type: application/json
$ ${get} -H 'Clusto-Page: notanint' -d 'pool=emptypool' ${server_url}/from-pools
"invalid literal for int() with base 10: 'notanint'"
HTTP: 400
Content-type: application/json
$ ${get} -d 'pool=emptypool' ${server_url}/from-pools
[]
HTTP: 200
Content-type: application/json
$ ${get} -d 'pool=singlepool' -d 'pool=multipool' ${server_url}/from-pools
[
"/basicserver/testserver1"
]
HTTP: 200
Content-type: application/json
$ ${get} -H 'Clusto-Mode: expanded' -d 'pool=multipool' ${server_url}/from-pools
[
{
"attrs": [
{
"datatype": "string",
"key": "key1",
"number": null,
"subkey": "subkey1",
"value": "value1"
}
],
"contents": [],
"driver": "basicserver",
"ips": [],
"name": "testserver1",
"parents": [
"/pool/singlepool",
"/pool/multipool"
]
},
{
"attrs": [
{
"datatype": "string",
"key": "key1",
"number": null,
"subkey": "subkey2",
"value": "value2"
}
],
"contents": [],
"driver": "basicserver",
"ips": [],
"name": "testserver2",
"parents": [
"/pool/multipool"
]
}
]
HTTP: 200
Content-type: application/json
$ ${get} -H 'Clusto-Page: 1' -H 'Clusto-Per-Page: 1' -d 'pool=multipool' ${server_url}/from-pools
[
"/basicserver/testserver1"
]
HTTP: 200
Content-type: application/json
$ ${get} -H 'Clusto-Page: 1' -H 'Clusto-Per-Page: 100' -d 'pool=multipool' ${server_url}/from-pools
[
"/basicserver/testserver1",
"/basicserver/testserver2"
]
HTTP: 200
Content-type: application/json
$ ${get} -H 'Clusto-Page: 100' -H 'Clusto-Per-Page: 100' -d 'pool=multipool' ${server_url}/from-pools
[]
HTTP: 200
Content-type: application/json
$ ${get} -H 'Clusto-Minify: True' -d 'pool=multipool' ${server_url}/from-pools
["/basicserver/testserver1", "/basicserver/testserver2"]
HTTP: 200
Content-type: application/json
"""
pools = bottle.request.params.getall('pool')
if not pools:
return util.dumps('Provide at least one pool to get data from', 412)
types = bottle.request.params.getall('type')
drivers = bottle.request.params.getall('driver')
children = bottle.request.params.get('children', default=True, type=bool)
mode = bottle.request.headers.get('Clusto-Mode', default='compact')
headers = {}
try:
# Assignments are moved into the try block because of the int casting.
current = int(bottle.request.headers.get('Clusto-Page', default='0'))
per = int(bottle.request.headers.get('Clusto-Per-Page', default='50'))
ents = clusto.get_from_pools(
pools, clusto_types=types, clusto_drivers=drivers, search_children=children
)
results = []
if current:
ents, total = util.page(list(ents), current=current, per=per)
headers['Clusto-Pages'] = total
headers['Clusto-Per-Page'] = per
headers['Clusto-Page'] = current
for ent in ents:
results.append(util.show(ent, mode))
return util.dumps(results, headers=headers)
except ValueError as ve:
return util.dumps('%s' % (ve,), 400)
except TypeError as te:
return util.dumps('%s' % (te,), 409)
except LookupError as le:
return util.dumps('%s' % (le,), 404)
except Exception as e:
return util.dumps('%s' % (e,), 500)
@root_app.get('/by-name/<name>')
[docs]def get_by_name(name):
"""
One of the main ``clusto`` operations. Parameters:
* Required path parameter: ``name`` - The name you're looking for
* Optional: ``driver`` - If provided, a driver check will be added to
ensure the resulting object is the type you're expecting
.. note:: This function returns expanded objects by default in order
to reduce the amount of required custom headers. Therefore, the header
is not required to receive expanded objects.
Examples:
.. code:: bash
$ ${get} ${server_url}/by-name/nonserver
"Object \"nonserver\" not found (nonserver does not exist.)"
HTTP: 404
Content-type: application/json
$ ${get} -H 'Clusto-Mode: compact' ${server_url}/by-name/testserver1
"/basicserver/testserver1"
HTTP: 200
Content-type: application/json
$ ${get} ${server_url}/by-name/testserver1
{
"attrs": [
{
"datatype": "string",
"key": "key1",
"number": null,
"subkey": "subkey1",
"value": "value1"
}
],
"contents": [],
"driver": "basicserver",
"ips": [],
"name": "testserver1",
"parents": [
"/pool/singlepool",
"/pool/multipool"
]
}
HTTP: 200
Content-type: application/json
$ ${get} -d 'driver=pool' ${server_url}/by-name/testserver1
"The driver for object \"testserver1\" is not \"pool\""
HTTP: 409
Content-type: application/json
$ ${get} -d 'driver=nondriver' ${server_url}/by-name/testserver1
"The driver \"nondriver\" is not a valid driver"
HTTP: 412
Content-type: application/json
"""
driver = bottle.request.params.get('driver', default=None)
obj, status, msg = util.get(name, driver)
if not obj:
return util.dumps(msg, status)
try:
return util.dumps(util.show(obj))
except TypeError as te:
return util.dumps('%s' % (te,), 409)
@root_app.get('/by-names')
[docs]def get_by_names():
"""
One of the main ``clusto`` operations. Parameters:
* Required parameter: At least one ``name`` parameter
Returns ``HTTP: 404`` when all entites requested do not exist and
``HTTP: 206`` when a percent of entities requested do not exist.
Examples:
.. code:: bash
$ ${get} ${server_url}/by-names
"Provide at least one name to get data from"
HTTP: 412
Content-type: application/json
$ ${get} -d 'name=nonserver' ${server_url}/by-names
[
null
]
HTTP: 404
Content-type: application/json
$ ${get} -d 'name=testserver1' -d 'name=nonserver' ${server_url}/by-names
[
"/basicserver/testserver1",
null
]
HTTP: 206
Content-type: application/json
$ ${get} -H 'Clusto-Mode: expanded' -d 'name=testserver1' -d 'name=testserver2' ${server_url}/by-names
[
{
"attrs": [
{
"datatype": "string",
"key": "key1",
"number": null,
"subkey": "subkey1",
"value": "value1"
}
],
"contents": [],
"driver": "basicserver",
"ips": [],
"name": "testserver1",
"parents": [
"/pool/singlepool",
"/pool/multipool"
]
},
{
"attrs": [
{
"datatype": "string",
"key": "key1",
"number": null,
"subkey": "subkey2",
"value": "value2"
}
],
"contents": [],
"driver": "basicserver",
"ips": [],
"name": "testserver2",
"parents": [
"/pool/multipool"
]
}
]
HTTP: 200
Content-type: application/json
$ ${get} -d 'name=nonserver1' -d 'name=nonserver2' ${server_url}/by-names
[
null,
null
]
HTTP: 404
Content-type: application/json
"""
objs = []
names = bottle.request.params.getall('name')
if not names:
return util.dumps('Provide at least one name to get data from', 412)
mode = bottle.request.headers.get('Clusto-Mode', default='compact')
for name in names:
obj, status, msg = util.get(name)
try:
objs.append(util.show(obj, mode) if obj else None)
except TypeError as te:
return util.dumps('%s' % (te,), 409)
return util.dumps(objs, 200 if all(objs) else 206 if any(objs) else 404)
@root_app.get('/by-attr')
[docs]def get_by_attr():
"""
One of the main ``clusto`` operations. Parameters:
* Required: the ``key`` parameter
* Optional: the ``subkey`` parameter
* Optional: the ``value`` parameter
Examples:
.. code:: bash
$ ${get} ${server_url}/by-attr
"Provide a key to use get_by_attr"
HTTP: 412
Content-type: application/json
$ ${get} -d 'key=nonkey' ${server_url}/by-attr
[]
HTTP: 200
Content-type: application/json
$ ${get} -d 'key=key1' -d 'subkey=subkey1' -d 'value=value1' ${server_url}/by-attr
[
"/basicserver/testserver1"
]
HTTP: 200
Content-type: application/json
$ ${get} -H 'Clusto-Mode: expanded' -d 'key=key1' ${server_url}/by-attr
[
{
"attrs": [
{
"datatype": "string",
"key": "key1",
"number": null,
"subkey": "subkey1",
"value": "value1"
}
],
"contents": [],
"driver": "basicserver",
"ips": [],
"name": "testserver1",
"parents": [
"/pool/singlepool",
"/pool/multipool"
]
},
{
"attrs": [
{
"datatype": "string",
"key": "key1",
"number": null,
"subkey": "subkey2",
"value": "value2"
}
],
"contents": [],
"driver": "basicserver",
"ips": [],
"name": "testserver2",
"parents": [
"/pool/multipool"
]
}
]
HTTP: 200
Content-type: application/json
"""
params = ['key', 'subkey', 'value']
kwargs = {}
for param in params:
val = bottle.request.params.get(param)
if val:
kwargs[param] = val
if not kwargs.get('key'):
return util.dumps('Provide a key to use get_by_attr', 412)
mode = bottle.request.headers.get('Clusto-Mode', default='compact')
try:
ents = clusto.get_by_attr(**kwargs)
results = []
for ent in ents:
results.append(util.show(ent, mode))
return util.dumps(results)
except TypeError as te:
return util.dumps('%s' % (te,), 409)
except LookupError as le:
return util.dumps('%s' % (le,), 404)
except Exception as e:
return util.dumps('%s' % (e,), 500)
def _configure(config={}, configfile=None, init_data={}):
"""
Configure the root app
"""
if configfile:
cfg = configfile
else:
cfg = os.environ.get(
'CLUSTOCONFIG',
'/etc/clusto/clusto.conf'
)
cfg = script_helper.load_config(cfg)
clusto.connect(cfg)
# This is an idempotent operation
clusto.init_clusto()
# If init_data is provided, populate it in the clusto database
if init_data:
for name, data in init_data.items():
ent = clusto.get_or_create(
name,
data['driver'],
**data.get('attrs', {})
)
for pool in data.get('member_of', []):
clusto.get_by_name(pool).insert(ent)
for attr in data.get('attr_list', []):
ent.set_attr(**attr)
kwargs = {}
kwargs['host'] = config.get(
'host',
script_helper.get_conf(
cfg, 'apiserver.host', default='127.0.0.1'
),
)
kwargs['port'] = config.get(
'port',
script_helper.get_conf(
cfg, 'apiserver.port', default=9664, datatype=int
),
)
kwargs['server'] = config.get(
'server',
script_helper.get_conf(
cfg, 'apiserver.server', default='wsgiref'
),
)
kwargs['server_kwargs'] = config.get(
'server_kwargs',
script_helper.get_conf(
cfg, 'apiserver.server_kwargs', default={}, datatype=dict
),
)
kwargs['debug'] = config.get(
'debug',
script_helper.get_conf(
cfg, 'apiserver.debug', default=False, datatype=bool
)
)
kwargs['quiet'] = config.get(
'quiet',
script_helper.get_conf(
cfg, 'apiserver.quiet', default=False, datatype=bool
)
)
kwargs['reloader'] = config.get(
'reloader',
script_helper.get_conf(
cfg, 'apiserver.reloader', default=False, datatype=bool
)
)
mount_apps = config.get(
'apps',
script_helper.get_conf(
cfg, 'apiserver.apps', default={}, datatype=dict
)
)
response_headers = config.get(
'response_headers',
script_helper.get_conf(
cfg, 'apiserver.response_headers', default={}, datatype=dict
)
)
root_app.route('/__doc__', 'GET', functools.partial(build_docs))
for mount_point, cls in mount_apps.items():
module = importlib.import_module(cls)
root_app.mount(mount_point, module.app)
# Documentation endpoints
module.app.route('/__doc__', 'GET', functools.partial(build_docs, cls))
module.app.route('/__doc__/', 'GET', functools.partial(build_docs, cls))
# OPTIONS dummy routers
module.app.route('/', 'OPTIONS', functools.partial(options))
module.app.route('/<url:re:.+>', 'OPTIONS', functools.partial(options))
@root_app.hook('before_request')
def enable_response_headers():
for header, value in response_headers.items():
bottle.response.headers[header] = value
return kwargs
[docs]def main():
"""
Main entry point for the clusto-apiserver console program
"""
kwargs = _configure()
kwargs.update(kwargs.pop('server_kwargs'))
root_app.run(**kwargs)
if __name__ == '__main__':
sys.exit(main())