#!/usr/bin/env python
# -*- coding: utf-8 -*-
""" :py:mod:`weblayer.static` provides
:py:class:`MemoryCachedStaticURLGenerator`, an implementation of
:py:class:`~weblayer.interfaces.IStaticURLGenerator`.
The purpose of an :py:class:`~weblayer.interfaces.IStaticURLGenerator` is to
generate static urls for use in templates. It does not serve static files.
:py:class:`MemoryCachedStaticURLGenerator` adapts an
:py:class:`~weblayer.interfaces.IRequest` and requires two
:py:mod:`~weblayer.settings`, ``settings['static_files_path']`` and
``settings['static_url_prefix']`` (which defaults to ``u'/static/'``).
>>> from mock import Mock
>>> from os.path import normpath, join as join_path
>>> static_files_path = normpath('/var/www/static')
>>> request = Mock()
>>> request.host_url = 'http://foo.com'
>>> settings = {}
>>> settings['static_files_path'] = static_files_path
>>> settings['static_url_prefix'] = u'/static/'
>>> MemoryCachedStaticURLGenerator._cache = {}
>>> static = MemoryCachedStaticURLGenerator(request, settings)
>>> static._cache_path = Mock()
When :py:meth:`~MemoryCachedStaticURLGenerator.get_url` is called, it looks
for a file at the ``path`` passed in, relative to
``settings['static_files_path']``. If the file exists, it hashes it
and appends the first few characters of the hash digest to the returned url.
For example, imagine we've hashed and cached ``/var/www/static/foo.js``::
>>> static._cache[join_path(static_files_path, 'foo.js')] = 'abcdefghij'
The static URL returned is::
>>> static.get_url('foo.js')
u'http://foo.com/static/foo.js?v=abcdefg'
Use ``settings['static_host_url']`` to specify the url static files should
be requested on (if this is different from the request url)::
>>> settings['static_host_url'] = 'http://static.foo.com'
>>> static = MemoryCachedStaticURLGenerator(request, settings)
>>> static.get_url('foo.js')
u'http://static.foo.com/static/foo.js?v=abcdefg'
As the :py:class:`MemoryCachedStaticURLGenerator` name suggests, the hash
digests are cached in memory using a static class attribute. This means
that:
* `multiple threads`_ can all access the same cache
* `multiple processes`_ each have to populate their own cache
* each time the application is restarted, the cache is cleared
The last two points mean that applications serving static files may incur
CPU and memory overhead that could be avoided using a dedicated cache (like
`memcached`_ or `redis`_). Production systems thus may want to provide their
own :py:class:`~weblayer.interfaces.IStaticURLGenerator` implementation,
(potentially by subclassing
:py:class:`~weblayer.static.MemoryCachedStaticURLGenerator` and overriding
the :py:meth:`~MemoryCachedStaticURLGenerator._cache_path` method).
.. note::
Alternative implementations must consider invalidating the hash
digests when files change. One benefit of the default
:py:class:`~weblayer.static.MemoryCachedStaticURLGenerator` implementation
is that, as hash digests are invalidated when the application restarts,
deployment setups that watch for changes to the underlying source code and
restart when files change cause the cache to be invalidated.
For example, one way to integrate with `paste.reloader`_ so it reloaded your
application every time a cached file changed would be to use::
paster serve --reload
With::
def watch_cached_static_files():
return MemoryCachedStaticURLGenerator._cache.keys()
paste.reloader.add_file_callback(watch_cached_static_files)
.. _`multiple threads`: http://docs.python.org/library/threading.html
.. _`multiple processes`: http://docs.python.org/library/multiprocessing.html
.. _`memcached`: http://memcached.org/
.. _`redis`: http://redis.io/
.. _`paste.reloader`: http://pythonpaste.org/modules/reloader.html
"""
__all__ = [
'MemoryCachedStaticURLGenerator'
]
import logging
from os.path import join
from zope.component import adapts
from zope.interface import implements
from interfaces import IRequest, ISettings, IStaticURLGenerator
from settings import require_setting
from utils import generate_hash
require_setting('static_files_path')
require_setting('static_url_prefix', default=u'/static/')
[docs]class MemoryCachedStaticURLGenerator(object):
""" Adapter to generate static URLs from a request.
"""
_cache = {}
adapts(IRequest, ISettings)
implements(IStaticURLGenerator)
def __init__(
self,
request,
settings,
join_path_=None,
open_file_=None,
generate_hash_=None
):
""" ``request.host_url`` is available as ``self._host_url``::
>>> from mock import Mock
>>> req = Mock()
>>> req.host_url = 'http://foo.com'
>>> settings = {}
>>> settings['static_files_path'] = '/var/www/static'
>>> settings['static_url_prefix'] = u'/static/'
>>> static = MemoryCachedStaticURLGenerator(
... req,
... settings
... )
>>> static._host_url
'http://foo.com'
Unless ``settings['static_host_url']`` is provided::
>>> settings['static_host_url'] = 'http://static.foo.com'
>>> static = MemoryCachedStaticURLGenerator(
... req,
... settings
... )
>>> static._host_url
'http://static.foo.com'
``settings['static_files_path']`` and
``settings['static_url_prefix']`` are
available as ``self._static_files_path`` and
``self._static_url_prefix``::
>>> static = MemoryCachedStaticURLGenerator(
... req,
... settings
... )
>>> static._static_files_path
'/var/www/static'
>>> static._static_url_prefix
u'/static/'
``join_path_`` defaults to ``join`` and is available as
``self._join_path``::
>>> static = MemoryCachedStaticURLGenerator(req, settings)
>>> static._join_path == join
True
>>> static = MemoryCachedStaticURLGenerator(
... req,
... settings,
... join_path_='join'
... )
>>> static._join_path
'join'
``open_file_`` defaults to ``open`` and is available as
``self._open_file``::
>>> static = MemoryCachedStaticURLGenerator(req, settings)
>>> static._open_file == open
True
>>> static = MemoryCachedStaticURLGenerator(
... req,
... settings,
... open_file_='open'
... )
>>> static._open_file
'open'
``generate_hash_`` defaults to ``generate_hash`` and is available as
``self._generate_hash``::
>>> static = MemoryCachedStaticURLGenerator(req, settings)
>>> static._generate_hash == generate_hash
True
>>> static = MemoryCachedStaticURLGenerator(
... req,
... settings,
... generate_hash_='generate'
... )
>>> static._generate_hash
'generate'
"""
self._host_url = settings.get('static_host_url', request.host_url)
self._static_files_path = settings['static_files_path']
self._static_url_prefix = settings['static_url_prefix']
if join_path_ is None:
self._join_path = join
else:
self._join_path = join_path_
if open_file_ is None:
self._open_file = open
else:
self._open_file = open_file_
if generate_hash_ is None:
self._generate_hash = generate_hash
else:
self._generate_hash = generate_hash_
def _cache_path(self, file_path):
""" Check to see if ``file_path`` exists. I it does, hash it and store
the ``digest`` against the ``file_path`` in ``self._cache``.
>>> from mock import Mock
>>> from StringIO import StringIO
>>> request = Mock()
>>> open_file = Mock()
>>> sock = StringIO()
>>> open_file.return_value = sock
>>> generate_hash = Mock()
>>> generate_hash.return_value = 'digest'
>>> settings = {
... 'static_files_path': '/var/www/static',
... 'static_url_prefix': u'/static/'
... }
>>> static = MemoryCachedStaticURLGenerator(
... request,
... settings,
... open_file_=open_file,
... generate_hash_=generate_hash
... )
>>> static._cache_path('/var/www/static/foo.js')
If ``file_path`` exists::
>>> open_file.assert_called_with('/var/www/static/foo.js')
It's hashed::
>>> generate_hash.assert_called_with(s=sock)
The hash is cached::
>>> static._cache['/var/www/static/foo.js']
'digest'
Unless the file_path can't be opened::
>>> def open_file(file_path):
... raise IOError
...
>>> static = MemoryCachedStaticURLGenerator(
... request,
... settings,
... open_file_=open_file,
... generate_hash_=generate_hash
... )
>>> static._cache_path('/var/www/static/foo.js')
>>> static._cache['/var/www/static/foo.js'] is None
True
"""
try:
sock = self._open_file(file_path)
except IOError:
logging.warning(u'Couldn\'t open static file %s' % file_path)
self._cache[file_path] = None
else:
digest = self._generate_hash(s=sock)
sock.close()
self._cache[file_path] = digest
[docs] def get_url(self, path, snip_digest_at=7):
""" Get a fully expanded url for the given static resource ``path``::
>>> from mock import Mock
>>> request = Mock()
>>> request.host_url = 'http://static.foo.com'
>>> settings = {}
>>> settings['static_files_path'] = '/var/www/static'
>>> settings['static_url_prefix'] = u'/static/'
>>> join_path = Mock()
>>> join_path.return_value = '/var/www/static/foo.js'
>>> MemoryCachedStaticURLGenerator._cache = {}
>>> static = MemoryCachedStaticURLGenerator(
... request,
... settings,
... join_path_=join_path
... )
>>> static._cache_path = Mock()
``path`` is expanded into ``file_path``::
>>> url = static.get_url('foo.js')
>>> join_path.assert_called_with('/var/www/static', 'foo.js')
If ``path`` isn't in ``self._cache``, calls ``self._cache_path()``::
>>> static._cache_path.assert_called_with('/var/www/static/foo.js')
If the digest is ``None``, just joins the host url, prefix and path::
>>> static._cache['/var/www/static/foo.js'] = None
>>> static.get_url('foo.js')
u'http://static.foo.com/static/foo.js'
Else also appends ``'?v='`` plus upto the first ``snip_digest_at``
chars of the digest, which defaults to ``7``::
>>> static._cache['/var/www/static/foo.js'] = 'abcdefghijkl'
>>> static.get_url('foo.js')
u'http://static.foo.com/static/foo.js?v=abcdefg'
>>> static.get_url('foo.js', snip_digest_at=4)
u'http://static.foo.com/static/foo.js?v=abcd'
Cleanup::
>>> MemoryCachedStaticURLGenerator._cache = {}
"""
file_path = self._join_path(self._static_files_path, path)
if not file_path in self._cache:
self._cache_path(file_path)
digest = self._cache.get(file_path)
if digest is None:
return u'%s%s%s' % (
self._host_url,
self._static_url_prefix,
path
)
else:
return u'%s%s%s?v=%s' % (
self._host_url,
self._static_url_prefix,
path,
digest[:snip_digest_at]
)