Source code for anybox.recipe.openerp.base
# coding: utf-8
from os.path import join, basename
import os
import sys
import re
import urllib
import tarfile
import setuptools
import logging
import stat
import imp
import shutil
import ConfigParser
import distutils.core
import pkg_resources
try:
from collections import OrderedDict
except ImportError: # Python < 2.7
from ordereddict import OrderedDict # noqa
from zc.buildout.easy_install import MissingDistribution
from zc.buildout import UserError
from zc.buildout.easy_install import VersionConflict
try: # need to distinguish, because of differing semantics
from zc.buildout.easy_install import IncompatibleConstraintError
except ImportError: # zc.buildout < 1.7
IncompatibleConstraintError = None # 'except None' is very fine
from zc.buildout.easy_install import IncompatibleVersionError
else:
IncompatibleVersionError = None
import zc.recipe.egg
import httplib
import rfc822
from urlparse import urlparse
from . import vcs
from . import utils
from .utils import option_splitlines, option_strip
logger = logging.getLogger(__name__)
[docs]def rfc822_time(h):
"""Parse RFC 2822-formatted http header and return a time int."""
rfc822.mktime_tz(rfc822.parsedate_tz(h))
[docs]class MainSoftware(object):
"""Placeholder to represent the main software instead of an addon location.
Should just have a singleton instance: :data:`main_software`,
whose meaning depends on the concrete recipe class using it.
For example, in :class:`anybox.recipe.openerp.server.ServerRecipe`,
:data:`main_software` represents the OpenObject server or the OpenERP
standard distribution.
"""
def __str__(self):
return 'Main Software'
main_software = MainSoftware()
GP_VCS_EXTEND_DEVELOP = 'vcs-extend-develop'
[docs]class BaseRecipe(object):
"""Base class for other recipes.
It implements notably fetching of the main software part plus addons.
The :attr:`sources` attribute is a ``dict`` storing how to fetch the main
software part and the specified addons, with the following structure:
``local path -> (type, location_spec, options)``, in which:
:local path: is either the :data:`main_software` singleton
(see :class:`MainSoftware`) or a local path to an
addons directory.
:type: can be either
* ``'local'``
* ``'downloadable'``
* one of the supported vcs
:location_spec: is, depending on the type, a tuple specifying how
fetch is to be done:
``url``, or ``(vcs_url, vcs_revision)``
or ``None``
:addons options: are typically used to specify that the addons
directory is actually a subdir of the specified one.
VCS support classes (see
:mod:`anybox.recipe.openerp.vcs`) can implemented their
dedicated options
The :attr:`merges` attribute is a ``dict`` storing how to fetch additional
changes to merge into VCS type sources:
``local path -> [(type, location_spec, options), ... ]``
See :attr:`sources` for the meaning of the various components. Note that
in :attr:`merges`, values are a list of triples instead of only a single
triple as values in :attr:`sources` because there can be multiple merges
on the same local path.
"""
release_dl_url = {'6.0': 'http://nightly.odoo.com/old/openerp-6/',
'6.1': 'http://nightly.odoo.com/old/openerp-6.1/',
'5.0': 'http://nightly.odoo.com/old/openerp-5/',
}
nightly_dl_url = {'6.0': 'http://nightly.odoo.com/6.0/6.0/',
'6.1': 'http://nightly.odoo.com/6.1/nightly/src/',
'7.0': 'http://nightly.odoo.com/7.0/nightly/src/',
'8.0': 'http://nightly.odoo.com/8.0/nightly/src/',
'trunk': 'http://nightly.odoo.com/trunk/nightly/src/',
}
recipe_requirements = () # distribution required for the recipe itself
recipe_requirements_paths = () # a default value is useful in unit tests
requirements = () # requirements for what the recipe installs to run
soft_requirements = () # subset of requirements that's not necessary
addons_paths = ()
# Caching logic for the main OpenERP part (e.g, without addons)
# Can be 'filename' or 'http-head'
main_http_caching = 'filename'
is_git_layout = False
"""True if this is the git layout, as seen from the move to GitHub.
In this layout, the standard addons other than ``base`` are in a ``addons``
directory right next to the ``openerp`` package.
"""
def __init__(self, buildout, name, options):
self.requirements = list(self.requirements)
self.recipe_requirements_path = []
self.buildout, self.name, self.options = buildout, name, options
self.b_options = self.buildout['buildout']
self.buildout_dir = self.b_options['directory']
# GR: would prefer lower() but doing as in 'zc.recipe.egg'
# (later) the standard way for all booleans is to use
# options.query_bool() or get_bool(), but it doesn't lower() at all
self.offline = self.b_options['offline'] == 'true'
self.clean = options.get('clean') == 'true'
clear_locks = options.get('vcs-clear-locks', '').lower()
self.vcs_clear_locks = clear_locks == 'true'
clear_retry = options.get('vcs-clear-retry', '').lower()
self.clear_retry = clear_retry == 'true'
# same as in zc.recipe.eggs
self.extra_paths = [
join(self.buildout_dir, p.strip())
for p in option_splitlines(self.options.get('extra-paths'))
]
self.options['extra-paths'] = os.linesep.join(self.extra_paths)
self.downloads_dir = self.make_absolute(
self.b_options.get('openerp-downloads-directory', 'downloads'))
self.version_wanted = None # from the buildout
self.version_detected = None # string from the openerp setup.py
self.parts = self.buildout['buildout']['parts-directory']
self.openerp_dir = None
self.archive_filename = None
self.archive_path = None # downloaded tar.gz
if options.get('scripts') is None:
options['scripts'] = ''
# a dictionnary of messages to display in case a distribution is
# not installable.
self.missing_deps_instructions = {
'PIL': ("You don't need to require it for OpenERP any more, since "
"the recipe automatically adds a dependency to Pillow. "
"If you really need it for other reasons, installing it "
"system-wide is a good option. "),
}
self.openerp_installed = []
self.etc = self.make_absolute(options.get('etc-directory', 'etc'))
self.bin_dir = self.buildout['buildout']['bin-directory']
self.config_path = join(self.etc, self.name + '.cfg')
for d in self.downloads_dir, self.etc:
if not os.path.exists(d):
logger.info('Created %s/ directory' % basename(d))
os.mkdir(d)
self.sources = OrderedDict()
self.merges = OrderedDict()
self.parse_addons(options)
self.parse_version()
self.parse_revisions(options)
self.parse_merges(options)
[docs] def parse_version(self):
"""Set the main software in :attr:`sources` and related attributes.
"""
self.version_wanted = option_strip(self.options.get('version'))
if self.version_wanted is None:
raise UserError('You must specify the version')
self.preinstall_version_check()
version_split = self.version_wanted.split()
if len(version_split) == 1:
# version can be a simple version name, such as 6.1-1
major_wanted = self.version_wanted[:3]
pattern = self.release_filenames[major_wanted]
if pattern is None:
raise UserError('OpenERP version %r'
'is not supported' % self.version_wanted)
self.archive_filename = pattern % self.version_wanted
self.archive_path = join(self.downloads_dir, self.archive_filename)
base_url = self.options.get(
'base_url', self.release_dl_url[major_wanted])
self.sources[main_software] = (
'downloadable',
'/'.join((base_url.strip('/'), self.archive_filename)), None)
return
# in all other cases, the first token is the type of version
type_spec = version_split[0]
if type_spec in ('local', 'path'):
self.openerp_dir = join(self.buildout_dir, version_split[1])
self.sources[main_software] = ('local', None)
elif type_spec == 'url':
url = version_split[1]
self.archive_filename = urlparse(url).path.split('/')[-1]
self.archive_path = join(self.downloads_dir, self.archive_filename)
self.sources[main_software] = ('downloadable', url, None)
elif type_spec == 'nightly':
if len(version_split) != 3:
raise UserError(
"Unrecognized nightly version specification: "
"%r (expecting series, number) % version_split[1:]")
self.nightly_series, self.version_wanted = version_split[1:]
type_spec = 'downloadable'
if self.version_wanted == 'latest':
self.main_http_caching = 'http-head'
series = self.nightly_series
self.archive_filename = (
self.nightly_filenames[series] % self.version_wanted)
self.archive_path = join(self.downloads_dir, self.archive_filename)
base_url = self.options.get('base_url',
self.nightly_dl_url[series])
self.sources[main_software] = (
'downloadable',
'/'.join((base_url.strip('/'), self.archive_filename)),
None)
else:
# VCS types
type_spec, url, repo_dir, self.version_wanted = version_split[0:4]
options = dict(opt.split('=') for opt in version_split[4:])
self.openerp_dir = join(self.parts, repo_dir)
self.sources[main_software] = (type_spec,
(url, self.version_wanted), options)
[docs] def preinstall_version_check(self):
"""Perform version checks before any attempt to install.
To be subclassed.
"""
[docs] def install_recipe_requirements(self):
"""Install requirements for the recipe to run."""
to_install = self.recipe_requirements
eggs_option = os.linesep.join(to_install)
eggs = zc.recipe.egg.Eggs(self.buildout, '', dict(eggs=eggs_option))
ws = eggs.install()
_, ws = eggs.working_set()
self.recipe_requirements_paths = [ws.by_key[dist].location
for dist in to_install]
sys.path.extend(self.recipe_requirements_paths)
[docs] def merge_requirements(self):
"""Merge eggs option with self.requirements."""
if 'eggs' not in self.options:
self.options['eggs'] = '\n'.join(self.requirements)
else:
self.options['eggs'] += '\n' + '\n'.join(self.requirements)
[docs] def install_requirements(self):
"""Install egg requirements and scripts.
If some distributions are known as soft requirements, will retry
without them
"""
while True:
missing = None
eggs_recipe = zc.recipe.egg.Scripts(self.buildout, '',
self.options)
try:
eggs_recipe.install()
except MissingDistribution as exc:
missing = exc.data[0].project_name
except VersionConflict as exc:
# GR not 100% sure, but this should mean a conflict with an
# already loaded version (don't know what can lead to this
# 'already', have seen it with zc.buildout itself only so far)
# In any case, removing the requirement can't make for a sane
# recovery
raise
except IncompatibleVersionError as exc: # zc.buildout < 1.7
# In exc's attrs, we have the version but not the project name
raise
except IncompatibleConstraintError as exc:
missing = exc.args[2].project_name
except UserError, exc: # happens only for zc.buildout >= 2.0
missing = exc.message.split(os.linesep)[0].split()[-1]
missing = re.split(r'[=<>]', missing)[0]
else:
break
logger.error("Could not find or install %r. "
+ self.missing_deps_instructions.get(missing, '')
+ " Original exception %s.%s says: %s",
missing,
exc.__class__.__module__, exc.__class__.__name__, exc)
if missing not in self.soft_requirements:
raise exc
eggs = set(self.options['eggs'].split(os.linesep))
if missing not in eggs:
logger.error("Soft requirement %r is also an indirect "
"dependency (either of OpenERP/Odoo or of "
"one listed in config file). Can't retry.",
missing)
raise exc
logger.warn("%r is a direct soft requirement, "
"retrying without it", missing)
eggs.discard(missing)
self.options['eggs'] = os.linesep.join(eggs)
self.eggs_reqs, self.eggs_ws = eggs_recipe.working_set()
self.ws = self.eggs_ws
[docs] def apply_version_dependent_decisions(self):
"""Store some booleans depending on detected version.
To be refined by subclasses.
"""
pass
@property
[docs] def major_version(self):
detected = self.version_detected
if detected is None:
return None
return utils.major_version(detected)
[docs] def read_release(self):
"""Try and read the release.py file directly.
Should be used only if reading setup.py failed, which happens
with OpenERP 5.0
"""
with open(join(self.openerp_dir, 'bin', 'release.py'), 'rb') as f:
mod = imp.load_module('release', f, 'release.py',
('.py', 'r', imp.PY_SOURCE))
self.version_detected = mod.version
[docs] def read_openerp_setup(self):
"""Ugly method to extract requirements & version from ugly setup.py.
Primarily designed for 6.0, but works with 6.1 as well.
"""
old_setup = setuptools.setup
old_distutils_setup = distutils.core.setup # 5.0 directly imports this
def new_setup(*args, **kw):
self.requirements.extend(kw.get('install_requires', ()))
self.version_detected = kw['version']
setuptools.setup = new_setup
distutils.core.setup = new_setup
sys.path.insert(0, '.')
with open(join(self.openerp_dir, 'setup.py'), 'rb') as f:
saved_argv = sys.argv
sys.argv = ['setup.py', 'develop']
try:
imp.load_module('setup', f, 'setup.py',
('.py', 'r', imp.PY_SOURCE))
except SystemExit as exception:
msg = exception.message
if not isinstance(msg, int) and 'dsextras' in msg:
raise EnvironmentError(
'Please first install PyGObject and PyGTK !')
else:
try:
self.read_release()
except Exception as exc:
raise EnvironmentError(
'Problem while reading OpenERP release.py: '
+ exc.message)
except ImportError, exception:
if 'babel' in exception.message:
raise EnvironmentError(
'OpenERP setup.py has an unwanted import Babel.\n'
'=> First install Babel on your system or '
'virtualenv :(\n'
'(sudo aptitude install python-babel, '
'or pip install babel)')
else:
raise exception
except Exception, exception:
raise EnvironmentError('Problem while reading OpenERP '
'setup.py: ' + exception.message)
finally:
sys.argv = saved_argv
sys.path.pop(0)
setuptools.setup = old_setup
distutils.core.setup = old_distutils_setup
self.apply_version_dependent_decisions()
[docs] def make_absolute(self, path):
"""Make a path absolute if needed.
If not already absolute, it is interpreted as relative to the
buildout directory."""
if os.path.isabs(path):
return path
return join(self.buildout_dir, path)
[docs] def sandboxed_tar_extract(self, sandbox, tarfile, first=None):
"""Extract those members that are below the tarfile path 'sandbox'.
The tarfile module official doc warns against attacks with .. in tar.
The option to start with a first member is useful for this case, since
the recipe consumes a first member in the tar file to get the openerp
main directory in parts.
It is taken for granted that this first member has already been
checked.
"""
if first is not None:
tarfile.extract(first)
for tinfo in tarfile:
if tinfo.name.startswith(sandbox + '/'):
tarfile.extract(tinfo)
else:
logger.warn('Tarball member %r is outside of %r. Ignored.',
tinfo, sandbox)
def _produce_setup_without_pil(self, src_directory):
"""Create a copy of setup.py without PIL and return a path to it."""
new_setup_path = join(src_directory, 'setup.nopil.py')
with open(join(src_directory, 'setup.py')) as inp:
setup_str = inp.read()
with open(new_setup_path, 'w') as out:
out.write(setup_str.replace("'PIL',", ''))
return new_setup_path
[docs] def develop(self, src_directory, setup_has_pil=False):
"""Develop the specified source distribution.
Any call to ``zc.recipe.eggs`` will use that developped version.
:meth:`develop` launches a subprocess, to which we need to forward
the paths to requirements via PYTHONPATH.
:param setup_has_pil: if ``True``, an altered version of setup that
does not require PIL is produced to perform the
develop, so that installation can be done with
``Pillow`` instead. Recent enough versions of
OpenERP/Odoo are directly based on Pillow.
:returns: project name of the distribution that's been "developed"
This is useful for OpenERP/Odoo itself, whose project name
changed within the 8.0 stable branch.
"""
logger.debug("Developing %r", src_directory)
develop_dir = self.b_options['develop-eggs-directory']
pythonpath_bak = os.getenv('PYTHONPATH')
os.putenv('PYTHONPATH', ':'.join(self.recipe_requirements_paths))
if setup_has_pil:
setup = self._produce_setup_without_pil(src_directory)
else:
setup = src_directory
try:
egg_link = zc.buildout.easy_install.develop(setup, develop_dir)
finally:
if setup_has_pil:
os.unlink(setup)
suffix = '.egg-link'
if pythonpath_bak is None:
os.unsetenv('PYTHONPATH')
else:
os.putenv('PYTHONPATH', pythonpath_bak)
if not egg_link.endswith(suffix):
raise RuntimeError(
"Development of OpenERP/Odoo distribution "
"produced an unexpected egg link: %r" % egg_link)
return os.path.basename(egg_link)[:-len(suffix)]
[docs] def parse_addons(self, options):
"""Parse the addons options into :attr:`sources`.
See :class:`BaseRecipe` for the structure of :attr:`sources`.
"""
for line in option_splitlines(options.get('addons')):
split = line.split()
if not split:
return
try:
loc_type = split[0]
spec_len = 2 if loc_type == 'local' else 4
options = dict(opt.split('=') for opt in split[spec_len:])
if loc_type == 'local':
addons_dir = split[1]
location_spec = None
else: # vcs
repo_url, addons_dir, repo_rev = split[1:4]
location_spec = (repo_url, repo_rev)
except:
raise UserError("Could not parse addons line: %r. "
"Please check format " % line)
addons_dir = addons_dir.rstrip('/') # trailing / can be harmful
group = options.get('group')
if group:
split = os.path.split(addons_dir)
addons_dir = os.path.join(split[0], group, split[1])
self.sources[addons_dir] = (loc_type, location_spec, options)
[docs] def parse_merges(self, options):
"""Parse the merge options into :attr:`merges`.
See :class:`BaseRecipe` for the structure of :attr:`merges`.
"""
for line in option_splitlines(options.get('merges')):
split = line.split()
if not split:
return
loc_type = split[0]
if loc_type not in ('bzr', 'git'):
raise UserError("Only merges of type 'bzr' and 'git' are "
"currently supported.")
options = dict(opt.split('=') for opt in split[4:])
if loc_type == 'bzr':
options['bzr-init'] = 'merge'
else:
options['merge'] = True
repo_url, local_dir, repo_rev = split[1:4]
location_spec = (repo_url, repo_rev)
local_dir = local_dir.rstrip('/') # trailing / can be harmful
self.merges.setdefault(local_dir, []).append(
(loc_type, location_spec, options))
[docs] def parse_revisions(self, options):
"""Parse revisions options and update :attr:`sources`.
It is assumed that :attr:`sources` has already been populated, and
notably has a :data:`main_software` entry.
This allows for easy fixing of revisions in an extension buildout
See :class:`BaseRecipe` for the structure of :attr:`sources`.
"""
for line in option_splitlines(options.get('revisions')):
split = line.split()
if len(split) > 2:
raise UserError("Invalid revisions line: %r" % line)
# addon or main software
if len(split) == 2:
local_path = split[0]
else:
local_path = main_software
revision = split[-1]
source = self.sources.get(local_path)
if source is None: # considered harmless for now
logger.warn("Ignoring attempt to fix revision on unknown "
"source %r. You may have a leftover to clean",
local_path)
continue
if source[0] in ('downloadable', 'local'):
raise UserError("In revision line %r : can't fix a revision "
"for non-vcs source" % line)
logger.info("%s will be on revision %r", local_path, revision)
self.sources[local_path] = ((source[0], (source[1][0], revision))
+ source[2:])
[docs] def retrieve_addons(self):
"""Peform all lookup and downloads specified in :attr:`sources`.
See :class:`BaseRecipe` for the structure of :attr:`sources`.
"""
self.addons_paths = []
for local_dir, source_spec in self.sources.items():
if local_dir is main_software:
continue
loc_type, loc_spec, addons_options = source_spec
local_dir = self.make_absolute(local_dir)
options = dict(offline=self.offline,
clear_locks=self.vcs_clear_locks,
clean=self.clean)
if loc_type == 'git':
options['depth'] = self.options.get('git-depth')
options.update(addons_options)
group = addons_options.get('group')
group_dir = None
if group:
if loc_type == 'local':
raise UserError(
"Automatic grouping of addons is not supported for "
"local addons such as %r, because the recipe "
"considers that write operations in a local "
"directory is "
"outside of its reponsibilities (in other words, "
"it's better if "
"you create yourself the intermediate directory." % (
local_dir, ))
group_dir = os.path.dirname(local_dir)
if not os.path.exists(group_dir):
os.makedirs(group_dir)
if loc_type != 'local':
for k, v in self.options.items():
if k.startswith(loc_type + '-'):
options[k] = v
repo_url, repo_rev = loc_spec
vcs.get_update(loc_type, local_dir, repo_url, repo_rev,
clear_retry=self.clear_retry,
**options)
elif self.clean:
utils.clean_object_files(local_dir)
subdir = addons_options.get('subdir')
if group_dir:
addons_dir = group_dir
else:
addons_dir = local_dir
if subdir:
addons_dir = join(addons_dir, subdir)
manifest = os.path.join(addons_dir, '__openerp__.py')
manifest_pre_v6 = os.path.join(addons_dir, '__terp__.py')
if os.path.isfile(manifest) or os.path.isfile(manifest_pre_v6):
raise UserError("Standalone addons such as %r "
"are now supported by means "
"of the explicit 'group' option. Please "
"update your buildout configuration. " % (
addons_dir))
if addons_dir not in self.addons_paths:
self.addons_paths.append(addons_dir)
[docs] def revert_sources(self):
"""Revert all sources to the revisions specified in :attr:`sources`.
"""
for target, desc in self.sources.iteritems():
if desc[0] in ('local', 'downloadable'):
continue
vcs_type, vcs_spec, options = desc
local_dir = self.openerp_dir if target is main_software else target
local_dir = self.make_absolute(local_dir)
repo = vcs.repo(vcs_type, local_dir, vcs_spec[0], **options)
try:
repo.revert(vcs_spec[1])
except NotImplementedError:
logger.warn("vcs-revert: not implemented for %s "
"repository at %s", vcs_type, local_dir)
else:
logger.info("Reverted %s repository at %s",
vcs_type, local_dir)
[docs] def retrieve_merges(self):
"""Peform all VCS merges specified in :attr:`merges`.
"""
if self.options.get('vcs-revert', '').strip().lower() == 'on-merge':
logger.info("Reverting all sources before merge")
self.revert_sources()
for local_dir, source_specs in self.merges.items():
for source_spec in source_specs:
loc_type, loc_spec, merge_options = source_spec
local_dir = self.make_absolute(local_dir)
options = dict(offline=self.offline,
clear_locks=self.vcs_clear_locks)
options.update(merge_options)
for k, v in self.options.items():
if k.startswith(loc_type + '-'):
options[k] = v
repo_url, repo_rev = loc_spec
vcs.get_update(loc_type, local_dir, repo_url, repo_rev,
clear_retry=self.clear_retry,
**options)
[docs] def main_download(self):
"""HTTP download for main part of the software to self.archive_path.
"""
if self.offline:
raise IOError("%s not found, and offline "
"mode requested" % self.archive_path)
url = self.sources[main_software][1]
logger.info("Downloading %s ..." % url)
try:
msg = urllib.urlretrieve(url, self.archive_path)
if msg[1].type == 'text/html':
os.unlink(self.archive_path)
raise LookupError(
'Wanted version %r not found on server (tried %s)' % (
self.version_wanted, url))
except (tarfile.TarError, IOError):
# GR: ContentTooShortError subclasses IOError
os.unlink(self.archive_path)
raise IOError('The archive does not seem valid: ' +
repr(self.archive_path))
[docs] def is_stale_http_head(self):
"""Tell if the download is stale by doing a HEAD request.
Assumes the correct date had been written upon download.
This is the same system as in GNU Wget 1.12. It works even if
the server does not implement conditional responses such as 304
"""
archivestat = os.stat(self.archive_path)
length, modified = archivestat.st_size, archivestat.st_mtime
url = self.sources[main_software][1]
logger.info("Checking if %s if fresh wrt %s",
self.archive_path, url)
parsed = urlparse(url)
if parsed.scheme == 'https':
cnx_cls = httplib.HTTPSConnection
else:
cnx_cls = httplib.HTTPConnection
try:
cnx = cnx_cls(parsed.netloc)
cnx.request('HEAD', parsed.path) # TODO query ? fragment ?
res = cnx.getresponse()
except IOError:
return True
if res.status != 200:
return True
if int(res.getheader('Content-Length')) != length:
return True
head_modified = res.getheader('Last-Modified')
logger.debug("Last-modified from HEAD request: %s", head_modified)
if rfc822_time(head_modified) > modified:
return True
logger.info("No need to re-download %s", self.archive_path)
[docs] def retrieve_main_software(self):
"""Lookup or fetch the main software.
See :class:`MainSoftware` and :class:`BaseRecipe` for explanations.
"""
source = self.sources[main_software]
type_spec = source[0]
logger.info('Selected install type: %s', type_spec)
if type_spec == 'local':
logger.info('Local directory chosen, nothing to do')
if self.clean:
utils.clean_object_files(self.openerp_dir)
elif type_spec == 'downloadable':
# download if needed
if ((self.archive_path and not os.path.exists(self.archive_path))
or (self.main_http_caching == 'http-head'
and self.is_stale_http_head())):
self.main_download()
logger.info(u'Inspecting %s ...' % self.archive_path)
tar = tarfile.open(self.archive_path)
first = tar.next()
# Everything that follows assumes all tarball members
# are inside a directory with an expected name such
# as openerp-6.1-1
assert(first.isdir())
extracted_name = first.name.split('/')[0]
self.openerp_dir = join(self.parts, extracted_name)
# protection against malicious tarballs
assert(not os.path.isabs(extracted_name))
assert(self.openerp_dir.startswith(self.parts))
logger.info("Cleaning existing %s", self.openerp_dir)
if os.path.exists(self.openerp_dir):
shutil.rmtree(self.openerp_dir)
logger.info(u'Extracting %s ...' % self.archive_path)
self.sandboxed_tar_extract(extracted_name, tar, first=first)
tar.close()
else:
url, rev = source[1]
options = dict((k, v) for k, v in self.options.iteritems()
if k.startswith(type_spec + '-'))
if type_spec == 'git':
options['depth'] = options.pop('git-depth', None)
options.update(source[2])
if self.clean:
options['clean'] = True
vcs.get_update(type_spec, self.openerp_dir, url, rev,
offline=self.offline,
clear_retry=self.clear_retry, **options)
def _register_extra_paths(self):
"""Add openerp paths into the extra-paths (used in scripts' sys.path).
This is useful up to the 6.0 series only, because in later version,
the 'openerp' directory is a proper distribution that we develop, with
the effect of putting it on the path automatically.
"""
extra = self.extra_paths
if self.major_version < (6, 1):
extra.extend((join(self.openerp_dir, 'bin'),
join(self.openerp_dir, 'bin', 'addons')))
self.options['extra-paths'] = os.linesep.join(extra)
[docs] def install(self):
os.chdir(self.parts)
freeze_to = self.options.get('freeze-to')
extract_downloads_to = self.options.get('extract-downloads-to')
if ((freeze_to is not None or extract_downloads_to is not None)
and not self.offline):
raise UserError("To freeze a part, you must run offline "
"so that there's no modification from what "
"you just tested. Please rerun with -o.")
if extract_downloads_to is not None and freeze_to is None:
freeze_to = os.path.join(extract_downloads_to,
'extracted_from.cfg')
self.retrieve_main_software()
self.retrieve_addons()
self.retrieve_merges()
self.install_recipe_requirements()
os.chdir(self.openerp_dir) # GR probably not needed any more
self.read_openerp_setup()
if (self.sources[main_software][0] == 'downloadable'
and self.version_wanted == 'latest'):
self.nightly_version = self.version_detected.split('-', 1)[1]
logger.warn("Detected 'nightly latest version', you may want to "
"fix it in your config file for replayability: \n "
"version = " + self.dump_nightly_latest_version())
self.finalize_addons_paths()
self._register_extra_paths()
if self.version_detected is None:
raise EnvironmentError('Version of OpenERP could not be detected')
self.merge_requirements()
self.install_requirements()
self._install_startup_scripts()
# create the config file
if os.path.exists(self.config_path):
os.remove(self.config_path)
logger.info('Creating config file: %s',
os.path.relpath(self.config_path, self.buildout_dir))
self._create_default_config()
# modify the config file according to recipe options
config = ConfigParser.RawConfigParser()
config.read(self.config_path)
for recipe_option in self.options:
if '.' not in recipe_option:
continue
section, option = recipe_option.split('.', 1)
if not config.has_section(section):
config.add_section(section)
config.set(section, option, self.options[recipe_option])
with open(self.config_path, 'wb') as configfile:
config.write(configfile)
if extract_downloads_to:
self.extract_downloads_to(extract_downloads_to)
if freeze_to:
self.freeze_to(freeze_to)
return self.openerp_installed
[docs] def dump_nightly_latest_version(self):
"""After download/analysis of 'nightly latest', give equivalent spec.
"""
return ' '.join(('nightly', self.nightly_series, self.nightly_version))
[docs] def freeze_to(self, out_config_path):
"""Create an extension buildout freezing current revisions & versions.
"""
logger.info("Freezing part %r to config file %r", self.name,
out_config_path)
out_conf = ConfigParser.ConfigParser()
frozen = getattr(self.buildout, '_openerp_recipe_frozen', None)
if frozen is None:
frozen = self.buildout._openerp_recipe_frozen = set()
if out_config_path in frozen:
# read configuration started by other recipe
out_conf.read(self.make_absolute(out_config_path))
else:
self._prepare_frozen_buildout(out_conf)
self._freeze_egg_versions(out_conf, 'versions')
out_conf.add_section(self.name)
addons_option = []
self.local_modifications = []
for local_path, source in self.sources.items():
source_type = source[0]
if source_type == 'local':
continue
if local_path is main_software:
if source_type == 'downloadable':
self._freeze_downloadable_main_software(out_conf)
else: # vcs
abspath = self.openerp_dir
self.cleanup_openerp_dir()
else:
abspath = self.make_absolute(local_path)
if source_type == 'downloadable':
continue
revision = self._freeze_vcs_source(source_type, abspath)
if local_path is main_software:
addons_option.insert(0, '%s ; main software part' % revision)
# actually, that comment will be lost if this is not the
# last part (dropped upon reread)
else:
addons_option.append(' '.join((local_path, revision)))
if addons_option:
out_conf.set(self.name, 'revisions',
os.linesep.join(addons_option))
if self.local_modifications:
logger.error(
"Uncommitted changes and/or untracked files in: %s"
"Unsafe to freeze. Please commit or revert and test again !",
os.linesep.join(
['', ''] + [' - ' + p
for p in self.local_modifications] + ['', '']))
sys.exit(17) # GR I like that number
with open(self.make_absolute(out_config_path), 'w') as out:
out_conf.write(out)
frozen.add(out_config_path)
def _get_gp_vcs_develops(self):
"""Return a tuple of (raw, parsed) vcs-extends-develop specifications.
"""
lines = self.b_options.get(
GP_VCS_EXTEND_DEVELOP)
if not lines:
return ()
try:
import pip.req
except ImportError:
logger.error("You have vcs-extends-develop distributions "
"but pip is not available. That means that "
"gp.vcsdevelop is not properly installed. Did "
"you ever run that buildout ?")
raise
return tuple((line, pip.req.parse_editable(line))
for line in option_splitlines(lines))
def _prepare_frozen_buildout(self, conf):
"""Create the 'buildout' section in conf."""
conf.add_section('buildout')
conf.set('buildout', 'extends', self.buildout_cfg_name())
conf.add_section('versions')
conf.set('buildout', 'versions', 'versions')
# freezing for gp.vcsdevelop
extends = []
for raw, parsed in self._get_gp_vcs_develops():
local_path = parsed[0]
hash_split = raw.rsplit('#')
url = hash_split[0]
url = url.rsplit('@', 1)[0]
vcs_type = url.split('+', 1)[0]
# vcs-develop process adds .egg-info file (often forgotten in VCS
# ignore files) and changes setup.cfg.
# For now we'll have to allow local modifications.
revision = self._freeze_vcs_source(vcs_type,
self.make_absolute(local_path),
pip_compatible=True,
allow_local_modification=True)
extends.append('%s@%s#%s' % (url, revision, hash_split[1]))
conf.set('buildout', GP_VCS_EXTEND_DEVELOP, os.linesep.join(extends))
def _freeze_downloadable_main_software(self, conf):
"""If needed, sets the main version option in ConfigParser.
Currently does not dump the fully resolved URL, since future
reproduction may be better done with another URL base holding archived
old versions : it's better to let tomorrow logic handle that
from higher level information.
"""
if self.version_wanted == 'latest':
conf.set(self.name, 'version', self.dump_nightly_latest_version())
def _freeze_egg_versions(self, conf, section, exclude=()):
"""Update a ConfigParser section with current working set egg versions.
"""
versions = dict((name, conf.get(section, name))
for name in conf.options(section))
versions.update((name, egg.version)
for name, egg in self.ws.by_key.items()
if name not in exclude
and egg.precedence != pkg_resources.DEVELOP_DIST
)
for name, version in versions.items():
conf.set(section, name, version)
# forbidding picked versions if this zc.buildout supports it right away
# i.e. we are on zc.buildout >= 2.0
allow_picked = self.options.get('freeze-allow-picked-versions', '')
if allow_picked.strip() == 'false':
pick_opt = 'allow-picked-versions'
if pick_opt in self.b_options:
conf.set('buildout', pick_opt, 'false')
def _freeze_vcs_source(self, vcs_type, abspath,
pip_compatible=False,
allow_local_modification=False):
"""Return the current revision for that VCS source.
:param pip_compatible: if ``True``, a pip compatible revision number
is issued. This depends on the precise vcs.
"""
repo = vcs.repo(vcs_type, abspath, '') # no need of remote URL
if not allow_local_modification and repo.uncommitted_changes():
self.local_modifications.append(abspath)
parents = repo.parents(pip_compatible=pip_compatible)
if len(parents) > 1:
self.local_modifications.append(abspath)
return parents[0]
[docs] def extract_downloads_to(self, target_dir, outconf_name='release.cfg'):
"""Extract anything that has been downloaded to target_dir.
This doesn't copy intermediary buildout configurations nor local parts.
In the purpose of making a self-contained and offline playable archive,
these are assumed to be already taken care of.
"""
logger.info("Extracting part %r to directory %r and config file %r "
"therein.", self.name, target_dir, outconf_name)
target_dir = self.make_absolute(target_dir)
out_conf = ConfigParser.ConfigParser()
all_extracted = getattr(self.buildout, '_openerp_recipe_extracted',
None)
if all_extracted is None:
all_extracted = self.buildout._openerp_recipe_extracted = {}
out_config_path = join(target_dir, outconf_name)
# GR TODO this will fail if same target dir has been used with
# a different outconf_name
if target_dir in all_extracted:
# read configuration started by other recipe
out_conf.read(out_config_path)
extracted = all_extracted[target_dir]
else:
self._prepare_extracted_buildout(out_conf, target_dir)
extracted = all_extracted[target_dir] = set()
self._freeze_egg_versions(out_conf, 'versions')
self._extract_sources(out_conf, target_dir, extracted)
with open(out_config_path, 'w') as out:
out_conf.write(out)
def _extract_sources(self, out_conf, target_dir, extracted):
"""Core extraction method.
out_conf is a ConfigParser instance to write to
extracted is a technical set used to know what targets have already
been written by previous parts and store for subsequent ones.
"""
if not os.path.exists(target_dir):
os.mkdir(target_dir)
out_conf.add_section(self.name)
# remove bzr extra if needed
pkg_extras, recipe_cls = self.options['recipe'].split(':')
extra_match = re.match(r'(.*?)\[(.*?)\]', pkg_extras)
if extra_match is not None:
recipe_pkg = extra_match.group(1)
extras = set(e.strip() for e in extra_match.group(2).split(','))
extras.discard('bzr')
extracted_recipe = recipe_pkg
if extras:
extracted_recipe += '[%s]' % ','.join(extras)
extracted_recipe += ':' + recipe_cls
out_conf.set(self.name, 'recipe', extracted_recipe)
addons_option = []
for local_path, source in self.sources.items():
source_type = source[0]
if local_path is main_software:
rel_path = self._extract_main_software(source_type, target_dir,
extracted)
out_conf.set(self.name, 'version', 'local ' + rel_path)
continue
# stripping the group option that won't be usefult
# and actually harming for extracted buildout conf
options = source[2]
group = options.pop('group', None)
if group:
target_local_path = os.path.dirname(local_path)
if group != os.path.basename(target_local_path):
raise RuntimeError(
"Inconsistent configuration that "
"should not happen: group=%r, but resulting path %r "
"does not have it as its parent" % (group, local_path))
else:
target_local_path = local_path
addons_line = ['local', target_local_path]
addons_line.extend('%s=%s' % (opt, val)
for opt, val in options.items())
addons_option.append(' '.join(addons_line))
abspath = self.make_absolute(local_path)
if source_type == 'downloadable':
shutil.copytree(abspath,
os.path.join(target_dir, local_path))
elif source_type != 'local': # vcs
self._extract_vcs_source(source_type, abspath, target_dir,
local_path, extracted)
out_conf.set(self.name, 'addons', os.linesep.join(addons_option))
if self.options.get('revisions'):
out_conf.set(self.name, 'revisions', '')
# GR hacky way to make a comment for a void value. Indeed,
# "revisions = ; comment" is not recognized as an inline comment
# because of overall stripping and a need for whitespace before
# the semicolon (sigh)
out_conf.set(self.name, '; about revisions',
"the extended buildout '%s' uses the 'revisions' "
"option. The present override disables it "
"because it makes no sense after extraction and "
"replacement by the "
"'local' scheme" % self.buildout_cfg_name())
def _extract_vcs_source(self, vcs_type, repo_path, target_dir, local_path,
extracted):
"""Extract a VCS source.
The extracted argument is a set of previously extracted targets.
This is because some VCS will refuse an empty directory (bzr does)
"""
repo_path = self.make_absolute(repo_path)
target_path = os.path.join(target_dir, local_path)
if not os.path.exists(target_path):
os.makedirs(target_path)
if target_path in extracted:
return
repo = vcs.repo(vcs_type, repo_path, '') # no need of remote URL
repo.archive(target_path)
extracted.add(target_path)
def _extract_main_software(self, source_type, target_dir, extracted):
"""Extract the main software to target_dir and return relative path.
As this is for extract_downloads_to, local main software is not
extracted (supposed to be taken care of by the tool that does the
archival of buildout dir itself).
The extracted set avoids extracting twice to same target (refused
by some VCSes anyway)
"""
if not self.openerp_dir.startswith(self.buildout_dir):
raise RuntimeError(
"Main openerp directory %r outside of buildout "
"directory, don't know how to handle that" % self.openerp_dir)
local_path = self.openerp_dir[len(self.buildout_dir + os.sep):]
target_path = join(target_dir, local_path)
if target_path in extracted:
return local_path
if source_type == 'downloadable':
shutil.copytree(self.openerp_dir, target_path)
elif source_type != 'local': # see docstring for 'local'
self._extract_vcs_source(source_type, self.openerp_dir, target_dir,
local_path, extracted)
return local_path
def _prepare_extracted_buildout(self, conf, target_dir):
"""Create the 'buildout' section in ``conf``.
Also takes care of gp.vcsdevelop driven distributions.
In most cases, at this stage, gp.vcsdevelop and regular develop
distributions are expressed with absolute paths. This method will make
them local in the destination ``conf``
Regular develop distributions pointing outside of buildout directory
are kept as is, assuming this has been specified in absolute form
in the config file, hence to some resources that are outside of
the recipe control, that are therefore expected to be deployed before
hand on target systems.
"""
conf.add_section('buildout')
conf.set('buildout', 'extends', self.buildout_cfg_name())
conf.add_section('versions')
conf.set('buildout', 'versions', 'versions')
develops = set(option_splitlines(self.b_options.get('develop')))
extracted = set()
for raw, parsed in self._get_gp_vcs_develops():
local_path = parsed[0]
abs_path = self.make_absolute(local_path)
vcs_type = raw.split('+', 1)[0]
self._extract_vcs_source(vcs_type, abs_path,
target_dir, local_path, extracted)
develops.add(abs_path) # looks silly, but better for uniformity
bdir = os.path.join(self.buildout_dir, '')
conf.set('buildout', 'develop',
os.linesep.join(d[len(bdir):] if d.startswith(bdir) else d
for d in develops))
# remove gp.vcsdevelop from extensions
exts = self.buildout['buildout'].get('extensions', '').split()
if 'gp.vcsdevelop' in exts:
exts.remove('gp.vcsdevelop')
conf.set('buildout', 'extensions', '\n'.join(exts))
def _install_script(self, name, content):
"""Install and register a scripbont with prescribed name and content.
Return the script path
"""
path = join(self.bin_dir, name)
f = open(path, 'w')
f.write(content)
f.close()
os.chmod(path, stat.S_IRWXU)
self.openerp_installed.append(path)
return path
def _install_startup_scripts(self):
raise NotImplementedError
def _create_default_config(self):
raise NotImplementedError
update = install
def _60_fix_root_path(self):
"""Correction of root path for OpenERP 6.0 pure python install
Actual implementation is up to subclasses
"""
def _60_default_addons_path(self):
"""Set the default addons path for OpenERP 6.0 pure python install
Actual implementation is up to subclasses
"""
def _default_addons_path(self):
"""Set the default addons path for OpenERP > 6.0 pure python install
Actual implementation is up to subclasses
"""
[docs] def finalize_addons_paths(self, check_existence=True):
"""Add implicit paths and serialize in the addons_path option.
if check_existence is True, all the paths will be checked for
existence.
"""
opt_key = 'options.addons_path'
if opt_key in self.options:
raise UserError("In part %r, direct use of %s is prohibited. "
"please use addons lines with type 'local' "
"instead." % (self.name, opt_key))
if self.major_version <= (6, 0):
base_addons = join(self.openerp_dir, 'bin', 'addons')
else:
base_addons = join(self.openerp_dir, 'openerp', 'addons')
if os.path.exists(base_addons):
self.addons_paths.insert(0, base_addons)
self.insert_odoo_git_addons(base_addons)
if check_existence:
for path in self.addons_paths:
assert os.path.isdir(path), (
"Not a directory: %r (aborting)" % path)
self.options['options.addons_path'] = ','.join(self.addons_paths)
if self.major_version <= (6, 0):
self._60_fix_root_path()
[docs] def insert_odoo_git_addons(self, base_addons):
"""Insert the standard, non-base addons bundled within Odoo git repo.
See `lp:1327756
<https://bugs.launchpad.net/anybox.recipe.openerp/+bug/1327756>`_
These addons are also part of the Github branch for prior versions,
therefore we cannot rely on version knowledge; we check for existence
instead.
If not found (e.g, we are on a nightly for OpenERP <= 7), this method
does nothing.
The ordering of the different paths of addons is important.
When several addons at different paths have the same name, the first
of them being found is used. This can be used, for instance, to
replace an official addon by another one by placing a different
addons' path before the official one.
If the official addons' path is already set in the config file
(e.g. at the end), it will leave it at the end of the paths list,
if it is not set, it will be placed at the beginning just after
``base`` addons' path.
Care is taken not to break configurations that corrected this manually
with a ``local`` source in the ``addons`` option.
:param base_addons: the path to previously detected ``base`` addons,
to properly insert right after them
"""
odoo_git_addons = join(self.openerp_dir, 'addons')
if not os.path.isdir(odoo_git_addons):
return
self.is_git_layout = True
addons_paths = self.addons_paths
try:
insert_at = addons_paths.index(base_addons) + 1
except ValueError:
insert_at = 0
try:
addons_paths.index(odoo_git_addons)
except ValueError:
addons_paths.insert(insert_at, odoo_git_addons)
[docs] def cleanup_openerp_dir(self):
"""Revert local modifications that have been made during installation.
These can be, e.g., forbidden by the freeze process."""
# from here we can't guess whether it's 'openerp' or 'odoo'.
# Nothing guarantees that this method is called after develop().
# It is in practice now, but one day, the extraction as a separate
# script of freeze/extract will become a reality.
for proj_name in ('openerp', 'odoo'):
egg_info_dir = join(self.openerp_dir, proj_name + '.egg-info')
if os.path.exists(egg_info_dir):
shutil.rmtree(egg_info_dir)
# if setup.py has been rewritten without PIL, it is cleaned
# during the process itself
[docs] def buildout_cfg_name(self, argv=None):
"""Return the name of the config file that's been called.
"""
# not using optparse because it's not obvious how to tell it to
# consider just one option and ignore the others.
if argv is None:
argv = sys.argv[1:]
# -c FILE or --config FILE syntax
for opt in ('-c', '--config'):
try:
i = argv.index(opt)
except ValueError:
continue
else:
return argv[i+1]
# --config=FILE syntax
prefix = "--config="
for a in argv:
if a.startswith(prefix):
return a[len(prefix):]
return 'buildout.cfg'
