1#@!#!123s

: / usr / share / dh-python /
Filename : pybuild
back
#! /usr/bin/python3
# vim: et ts=4 sw=4
# Copyright © 2012-2013 Piotr Ożarowski <piotr@debian.org>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import logging
import argparse
import sys
from os import environ, getcwd, makedirs, remove
from os.path import abspath, exists, join
from shutil import rmtree

logging.basicConfig(format='%(levelname).1s: pybuild '
                           '%(module)s:%(lineno)d: %(message)s')
log = logging.getLogger('dhpython')


def main(cfg):
    log.debug('cfg: %s', cfg)
    from dhpython import build
    from dhpython.version import Version, build_sorted, get_requested_versions
    from dhpython.interpreter import Interpreter
    from dhpython.tools import execute, move_matching_files

    if cfg.list_systems:
        for name, Plugin in sorted(build.plugins.items()):
            print(name, '\t', Plugin.DESCRIPTION)
        exit(0)

    nocheck = False
    if 'DEB_BUILD_OPTIONS' in environ:
        nocheck = 'nocheck' in environ['DEB_BUILD_OPTIONS']

    env = environ.copy()
    if 'LC_ALL' not in env:
        env['LC_ALL'] = 'C.UTF-8'
    if 'no_proxy' not in env:
        env['no_proxy'] = 'localhost'
    if 'http_proxy' not in env:
        env['http_proxy'] = 'http://127.0.0.1:9/'
    elif not env['http_proxy']:
        del env['http_proxy']  # some tools don't like empty var.
    if 'https_proxy' not in env:
        env['https_proxy'] = 'https://127.0.0.1:9/'
    elif not env['https_proxy']:
        del env['https_proxy']  # some tools don't like empty var.

    if cfg.system:
        certainty = 99
        Plugin = build.plugins.get(cfg.system)
        if not Plugin:
            log.error('unrecognized build system: %s', cfg.system)
            exit(10)
        plugin = Plugin(cfg)
        context = {'ENV': env, 'args': {}, 'dir': cfg.dir}
        plugin.detect(context)
    else:
        plugin, certainty, context = None, 0, None
        for Plugin in build.plugins.values():
            try:
                tmp_plugin = Plugin(cfg)
            except Exception as err:
                log.warn('cannot initialize %s plugin: %s', Plugin.NAME,
                         err, exc_info=cfg.verbose)
                continue
            tmp_context = {'ENV': env, 'args': {}, 'dir': cfg.dir}
            tmp_certainty = tmp_plugin.detect(tmp_context)
            if tmp_certainty and tmp_certainty > certainty:
                plugin, certainty, context = tmp_plugin, tmp_certainty, tmp_context
        del Plugin
        if not plugin:
            log.error('cannot detect build system, please use --system option'
                      ' or set PYBUILD_SYSTEM env. variable')
            exit(11)
    for interpreter in cfg.interpreter:
        if plugin.SUPPORTED_INTERPRETERS is not True and interpreter not in plugin.SUPPORTED_INTERPRETERS:
            log.error('interpreter %s not supported by %s', interpreter, plugin)
            exit(12)
    log.debug('detected build system: %s (certainty: %s%%)', plugin.NAME, certainty)

    if cfg.detect_only:
        if not cfg.really_quiet:
            print(plugin.NAME)
        exit(0)

    # reversed so that default Python version will be last
    versions = cfg.versions
    if not versions:
        log.debug('defaulting to all supported Python 3.X versions')
        versions = list(get_requested_versions('cpython3', available=True))
    versions = [Version(v) for v in versions]

    def get_option(name, interpreter=None, version=None, default=None):
        if interpreter:
            # try PYBUILD_NAME_python3.3-dbg (or hardcoded interpreter)
            i = interpreter.format(version=version or '')
            opt = "PYBUILD_{}_{}".format(name.upper(), i)
            if opt in environ:
                return environ[opt]
            # try PYBUILD_NAME_python3-dbg (if not checked above)
            if '{version}' in interpreter and version:
                i = interpreter.format(version=version.major)
                opt = "PYBUILD_{}_{}".format(name.upper(), i)
                if opt in environ:
                    return environ[opt]
        # try PYBUILD_NAME
        opt = "PYBUILD_{}".format(name.upper())
        if opt in environ:
            return environ[opt]
        # try command line args
        return getattr(cfg, name, default) or default

    def get_args(context, step, version, interpreter):
        i = interpreter.format(version=version)
        home_dir = '.pybuild/{}_{}'.format(interpreter.format(version='X.Y'), version)
        build_dir = get_option('build_dir', interpreter, version,
                               default=join(home_dir, 'build'))

        ipreter = Interpreter(i)
        destdir = context['destdir'].format(version=version, interpreter=i)
        if cfg.name:
            package = ipreter.suggest_pkg_name(cfg.name)
        else:
            package = 'PYBUILD_NAME_not_set'
        if cfg.name and destdir.rstrip('/').endswith('debian/tmp'):
            destdir = "debian/{}".format(package)
        destdir = abspath(destdir)

        args = dict(context['args'])
        args.update({
            'package': package,
            'interpreter': ipreter,
            'version': version,
            'args': get_option("%s_args" % step, interpreter, version, ''),
            'dir': abspath(context['dir'].format(version=version, interpreter=i)),
            'destdir': destdir,
            'build_dir': abspath(build_dir.format(version=version, interpreter=i)),
            # versioned dist-packages even for Python 3.X - dh_python3 will fix it later
            # (and will have a chance to compare files)
            'install_dir': get_option('install_dir', interpreter, version,
                                      '/usr/lib/python{version}/dist-packages'
                                      ).format(version=version, interpreter=i),
            'home_dir': abspath(home_dir)})
        if interpreter == 'pypy':
            args['install_dir'] = '/usr/lib/pypy/dist-packages/'
        if step in {'build', 'test'}:
            pp = context['ENV'].get('PYTHONPATH')
            pp = pp.split(':') if pp else []
            if step == 'test':
                args['test_dir'] = join(args['destdir'], args['install_dir'].lstrip('/'))
                if args['test_dir'] not in pp:
                    pp.append(args['test_dir'])
            if args['build_dir'] not in pp:
                pp.append(args['build_dir'])
            args['PYTHONPATH'] = ':'.join(pp)

        if not exists(args['build_dir']):
            makedirs(args['build_dir'])

        return args

    def is_disabled(step, interpreter, version):
        i = interpreter
        prefix = "{}/".format(step)
        disabled = (get_option('disable', i, version) or '').split()
        for item in disabled:
            if item in (step, '1'):
                log.debug('disabling {} step for {} {}'.format(step, i, version))
                return True
            if item.startswith(prefix):
                disabled.append(item[len(prefix):])
        if i in disabled or str(version) in disabled or \
                i.format(version=version) in disabled or \
                i.format(version=version.major) in disabled:
            log.debug('disabling {} step for {} {}'.format(step, i, version))
            return True
        return False

    def run(func, interpreter, version, context):
        step = func.__func__.__name__
        args = get_args(context, step, version, interpreter)
        if 'PYTHONPATH' in args:
            env = dict(context['ENV'])
            env['PYTHONPATH'] = args['PYTHONPATH']
        else:
            env = context['ENV']

        before_cmd = get_option('before_{}'.format(step), interpreter, version)
        if before_cmd:
            if cfg.quiet:
                log_file = join(args['home_dir'], 'before_{}_cmd.log'.format(step))
            else:
                log_file = False
            command = before_cmd.format(**args)
            output = execute(command, context['dir'], env, log_file)
            if output['returncode'] != 0:
                msg = 'exit code={}: {}'.format(output['returncode'], command)
                raise Exception(msg)

        fpath = join(args['home_dir'], 'build_dirs_to_rm_before_install')
        if step == 'install' and exists(fpath):
            with open(fpath) as fp:
                for line in fp:
                    dpath = join(args['build_dir'], line.strip('\n'))
                    if exists(dpath):
                        rmtree(dpath)
            remove(fpath)
        result = func(context, args)

        after_cmd = get_option('after_{}'.format(step), interpreter, version)
        if after_cmd:
            if cfg.quiet:
                log_file = join(args['home_dir'], 'after_{}_cmd.log'.format(step))
            else:
                log_file = False
            command = after_cmd.format(**args)
            output = execute(command, context['dir'], env, log_file)
            if output['returncode'] != 0:
                msg = 'exit code={}: {}'.format(output['returncode'], command)
                raise Exception(msg)
        return result

    func = None
    if cfg.clean_only:
        func = plugin.clean
    elif cfg.configure_only:
        func = plugin.configure
    elif cfg.build_only:
        func = plugin.build
    elif cfg.install_only:
        func = plugin.install
    elif cfg.test_only:
        func = plugin.test

    ### one function for each interpreter at a time mode ###
    if func:
        step = func.__func__.__name__
        if step == 'test' and nocheck:
            exit(0)
        for i in cfg.interpreter:
            ipreter = Interpreter(interpreter.format(version=versions[0]))
            iversions = build_sorted(versions, impl=ipreter.impl)
            if '{version}' not in i and len(versions) > 1:
                log.info('limiting Python versions to %s due to missing {version}'
                         ' in interpreter string', str(versions[-1]))
                iversions = versions[-1:]  # just the default or closest to default
            for version in iversions:
                if is_disabled(step, i, version):
                    continue
                c = dict(context)
                c['dir'] = get_option('dir', i, version, cfg.dir)
                c['destdir'] = get_option('destdir', i, version, cfg.destdir)
                try:
                    run(func, i, version, c)
                except Exception as err:
                    log.error('%s: plugin %s failed with: %s',
                              step, plugin.NAME, err, exc_info=cfg.verbose)
                    exit(13)
                if step == 'install':
                    ext_destdir = get_option('ext_destdir', i, version)
                    if ext_destdir:
                        move_matching_files(c['destdir'], ext_destdir,
                                            get_option('ext_pattern', i, version))
        exit(0)

    ### all functions for interpreters in batches mode ###
    try:
        context_map = {}
        for i in cfg.interpreter:
            ipreter = Interpreter(interpreter.format(version=versions[0]))
            iversions = build_sorted(versions, impl=ipreter.impl)
            if '{version}' not in i and len(versions) > 1:
                log.info('limiting Python versions to %s due to missing {version}'
                         ' in interpreter string', str(versions[-1]))
                iversions = versions[-1:]  # just the default or closest to default
            for version in iversions:
                key = (i, version)
                if key in context_map:
                    c = context_map[key]
                else:
                    c = dict(context)
                    c['dir'] = get_option('dir', i, version, cfg.dir)
                    c['destdir'] = get_option('destdir', i, version, cfg.destdir)
                    context_map[key] = c

                if not is_disabled('clean', i, version):
                    run(plugin.clean, i, version, c)
                if not is_disabled('configure', i, version):
                    run(plugin.configure, i, version, c)
                if not is_disabled('build', i, version):
                    run(plugin.build, i, version, c)
                if not is_disabled('install', i, version):
                    run(plugin.install, i, version, c)
                    ext_destdir = get_option('ext_destdir', i, version)
                    if ext_destdir:
                        move_matching_files(c['destdir'], ext_destdir,
                                            get_option('ext_pattern', i, version))
                if not nocheck and not is_disabled('test', i, version):
                    run(plugin.test, i, version, c)
    except Exception as err:
        log.error('plugin %s failed: %s', plugin.NAME, err,
                  exc_info=cfg.verbose)
        exit(14)


def parse_args(argv):
    usage = '%(prog)s [ACTION] [BUILD SYSTEM ARGS] [DIRECTORIES] [OPTIONS]'
    parser = argparse.ArgumentParser(usage=usage)
    parser.add_argument('-v', '--verbose', action='store_true',
                        default=environ.get('PYBUILD_VERBOSE') == '1',
                        help='turn verbose mode on')
    parser.add_argument('-q', '--quiet', action='store_true',
                        default=environ.get('PYBUILD_QUIET') == '1',
                        help='doesn\'t show external command\'s output')
    parser.add_argument('-qq', '--really-quiet', action='store_true',
                        default=environ.get('PYBUILD_RQUIET') == '1',
                        help='be quiet')
    parser.add_argument('--version', action='version', version='%(prog)s 2.20151103ubuntu1.2')

    action = parser.add_argument_group('ACTION', '''The default is to build,
        install and test the library using detected build system version by
        version. Selecting one of following actions, will invoke given action
        for all versions - one by one - which (contrary to the default action)
        in some build systems can overwrite previous results.''')
    action.add_argument('--detect', action='store_true', dest='detect_only',
                        help='return the name of detected build system')
    action.add_argument('--clean', action='store_true', dest='clean_only',
                        help='clean files using auto-detected build system specific methods')
    action.add_argument('--configure', action='store_true', dest='configure_only',
                        help='invoke configure step for all requested Python versions')
    action.add_argument('--build', action='store_true', dest='build_only',
                        help='invoke build step for all requested Python versions')
    action.add_argument('--install', action='store_true', dest='install_only',
                        help='invoke install step for all requested Python versions')
    action.add_argument('--test', action='store_true', dest='test_only',
                        help='invoke tests for auto-detected build system')
    action.add_argument('--list-systems', action='store_true',
                        help='list available build systems and exit')

    arguments = parser.add_argument_group('BUILD SYSTEM ARGS', '''
        Additional arguments passed to the build system.
        --system=custom requires complete command.''')
    arguments.add_argument('--before-clean', metavar='CMD',
                           help='invoked before the clean command')
    arguments.add_argument('--clean-args', metavar='ARGS')
    arguments.add_argument('--after-clean', metavar='CMD',
                           help='invoked after the clean command')

    arguments.add_argument('--before-configure', metavar='CMD',
                           help='invoked before the configure command')
    arguments.add_argument('--configure-args', metavar='ARGS')
    arguments.add_argument('--after-configure', metavar='CMD',
                           help='invoked after the configure command')

    arguments.add_argument('--before-build', metavar='CMD',
                           help='invoked before the build command')
    arguments.add_argument('--build-args', metavar='ARGS')
    arguments.add_argument('--after-build', metavar='CMD',
                           help='invoked after the build command')

    arguments.add_argument('--before-install', metavar='CMD',
                           help='invoked before the install command')
    arguments.add_argument('--install-args', metavar='ARGS')
    arguments.add_argument('--after-install', metavar='CMD',
                           help='invoked after the install command')

    arguments.add_argument('--before-test', metavar='CMD',
                           help='invoked before the test command')
    arguments.add_argument('--test-args', metavar='ARGS')
    arguments.add_argument('--after-test', metavar='CMD',
                           help='invoked after the test command')

    tests = parser.add_argument_group('TESTS', '''\
        unittest\'s discover is used by default (if available)''')
    tests.add_argument('--test-nose', action='store_true',
                       default=environ.get('PYBUILD_TEST_NOSE') == '1',
                       help='use nose module in --test step')
    tests.add_argument('--test-pytest', action='store_true',
                       default=environ.get('PYBUILD_TEST_PYTEST') == '1',
                       help='use pytest module in --test step')
    tests.add_argument('--test-tox', action='store_true',
                       default=environ.get('PYBUILD_TEST_TOX') == '1',
                       help='use tox in --test step')

    dirs = parser.add_argument_group('DIRECTORIES')
    dirs.add_argument('-d', '--dir', action='store', metavar='DIR',
                      default=getcwd(),
                      help='source files directory - base for other relative dirs [default: CWD]')
    dirs.add_argument('--dest-dir', action='store', metavar='DIR', dest='destdir',
                      default=environ.get('DESTDIR', 'debian/tmp'),
                      help='destination directory [default: debian/tmp]')
    dirs.add_argument('--ext-dest-dir', action='store', metavar='DIR', dest='ext_destdir',
                      default=environ.get('EXT_DESTDIR'),
                      help='destination directory for .so files')
    dirs.add_argument('--ext-pattern', action='store', metavar='PATTERN',
                      default=environ.get('EXT_PATTERN', r'\.so(\.[^/]*)?$'),
                      help='regular expression for files that should be moved'
                      ' if --ext-dest-dir is set [default: .so files]')
    dirs.add_argument('--install-dir', action='store', metavar='DIR',
                      help='installation directory [default: .../dist-packages]')
    dirs.add_argument('--name', action='store',
                      default=environ.get('PYBUILD_NAME'),
                      help='use this name to guess destination directories')

    limit = parser.add_argument_group('LIMITATIONS')
    limit.add_argument('-s', '--system',
                       default=environ.get('PYBUILD_SYSTEM'),
                       help='select a build system [default: auto-detection]')
    limit.add_argument('-p', '--pyver', action='append', dest='versions',
                       help='''build for Python VERSION.
                               This option can be used multiple times
                               [default: all supported Python 3.X versions]''')
    limit.add_argument('-i', '--interpreter', action='append',
                       help='change interpreter [default: python{version}]')
    limit.add_argument('--disable', metavar='ITEMS',
                       help='disable action, interpreter or version')

    args = parser.parse_args()
    if not args.interpreter:
        args.interpreter = environ.get('PYBUILD_INTERPRETERS', 'python{version}').split()
    if not args.versions:
        args.versions = environ.get('PYBUILD_VERSIONS', '').split()
    else:
        # add support for -p `pyversions -rv`
        versions = []
        for version in args.versions:
            versions.extend(version.split())
        args.versions = versions

    if args.test_nose or args.test_pytest or args.test_tox\
       or args.system == 'custom':
        args.custom_tests = True
    else:
        args.custom_tests = False

    return args


if __name__ == '__main__':
    cfg = parse_args(sys.argv)
    if cfg.really_quiet:
        cfg.quiet = True
        log.setLevel(logging.CRITICAL)
    elif cfg.verbose:
        log.setLevel(logging.DEBUG)
    else:
        log.setLevel(logging.INFO)
    log.debug('version: 2.20151103ubuntu1.2')
    log.debug(sys.argv)
    main(cfg)
    # let dh/cdbs clean the .pybuild dir
    # rmtree(join(cfg.dir, '.pybuild'))