Add automatic build of libcapnp if it's not detected

This commit is contained in:
Jason Paryani
2014-12-11 22:04:34 -08:00
parent 02c572fe1e
commit 4c0d4ec290
11 changed files with 818 additions and 1 deletions

12
buildutils/__init__.py Normal file
View File

@@ -0,0 +1,12 @@
"""utilities for building pyzmq.
Largely adapted from h5py
"""
from .msg import *
from .config import *
from .detect import *
from .bundle import *
from .misc import *
from .patch import *
from .build import *

24
buildutils/build.py Normal file
View File

@@ -0,0 +1,24 @@
'Build the bundled capnp distribution'
import subprocess
import os
import tempfile
def build_libcapnp(bundle_dir, build_dir, verbose=False):
bundle_dir = os.path.abspath(bundle_dir)
capnp_dir = os.path.join(bundle_dir, 'capnproto-c++')
build_dir = os.path.abspath(build_dir)
with tempfile.TemporaryFile() as f:
stdout = f
if verbose:
stdout = None
conf = subprocess.Popen(['./configure', '--disable-shared', '--prefix', build_dir], cwd=capnp_dir, stdout=stdout)
returncode = conf.wait()
if returncode != 0:
raise RuntimeError('Configure failed')
make = subprocess.Popen(['make', '-j4', 'install'], cwd=capnp_dir, stdout=stdout)
returncode = make.wait()
if returncode != 0:
raise RuntimeError('Make failed')

165
buildutils/bundle.py Normal file
View File

@@ -0,0 +1,165 @@
"""utilities for fetching build dependencies."""
#-----------------------------------------------------------------------------
# Copyright (C) PyZMQ Developers
# Distributed under the terms of the Modified BSD License.
#
# This bundling code is largely adapted from pyzmq-static's get.sh by
# Brandon Craig-Rhodes, which is itself BSD licensed.
#-----------------------------------------------------------------------------
#
# Adapted for use in pycapnp from pyzmq. See https://github.com/zeromq/pyzmq
# for original project.
import os
import shutil
import stat
import sys
import tarfile
from glob import glob
from subprocess import Popen, PIPE
try:
# py2
from urllib2 import urlopen
except ImportError:
# py3
from urllib.request import urlopen
from .msg import fatal, debug, info, warn
pjoin = os.path.join
#-----------------------------------------------------------------------------
# Constants
#-----------------------------------------------------------------------------
bundled_version = (0,4,1)
libcapnp = "capnproto-c++-%i.%i.%i.tar.gz" % (bundled_version)
libcapnp_url = "https://capnproto.org/" + libcapnp
HERE = os.path.dirname(__file__)
ROOT = os.path.dirname(HERE)
#-----------------------------------------------------------------------------
# Utilities
#-----------------------------------------------------------------------------
def untgz(archive):
return archive.replace('.tar.gz', '')
def localpath(*args):
"""construct an absolute path from a list relative to the root pycapnp directory"""
plist = [ROOT] + list(args)
return os.path.abspath(pjoin(*plist))
def fetch_archive(savedir, url, fname, force=False):
"""download an archive to a specific location"""
dest = pjoin(savedir, fname)
if os.path.exists(dest) and not force:
info("already have %s" % fname)
return dest
info("fetching %s into %s" % (url, savedir))
if not os.path.exists(savedir):
os.makedirs(savedir)
req = urlopen(url)
with open(dest, 'wb') as f:
f.write(req.read())
return dest
#-----------------------------------------------------------------------------
# libcapnp
#-----------------------------------------------------------------------------
def fetch_libcapnp(savedir):
"""download and extract libcapnp"""
dest = pjoin(savedir, 'capnproto-c++')
if os.path.exists(dest):
info("already have %s" % dest)
return
fname = fetch_archive(savedir, libcapnp_url, libcapnp)
tf = tarfile.open(fname)
with_version = pjoin(savedir, tf.firstmember.path)
tf.extractall(savedir)
tf.close()
# remove version suffix:
shutil.move(with_version, dest)
def stage_platform_hpp(capnproot):
"""stage platform.hpp into libcapnp sources
Tries ./configure first (except on Windows),
then falls back on included platform.hpp previously generated.
"""
platform_hpp = pjoin(capnproot, 'src', 'platform.hpp')
if os.path.exists(platform_hpp):
info("already have platform.hpp")
return
if os.name == 'nt':
# stage msvc platform header
platform_dir = pjoin(capnproot, 'builds', 'msvc')
else:
info("attempting ./configure to generate platform.hpp")
p = Popen('./configure', cwd=capnproot, shell=True,
stdout=PIPE, stderr=PIPE,
)
o,e = p.communicate()
if p.returncode:
warn("failed to configure libcapnp:\n%s" % e)
if sys.platform == 'darwin':
platform_dir = pjoin(HERE, 'include_darwin')
elif sys.platform.startswith('freebsd'):
platform_dir = pjoin(HERE, 'include_freebsd')
elif sys.platform.startswith('linux-armv'):
platform_dir = pjoin(HERE, 'include_linux-armv')
else:
platform_dir = pjoin(HERE, 'include_linux')
else:
return
info("staging platform.hpp from: %s" % platform_dir)
shutil.copy(pjoin(platform_dir, 'platform.hpp'), platform_hpp)
def copy_and_patch_libcapnp(capnp, libcapnp):
"""copy libcapnp into source dir, and patch it if necessary.
This command is necessary prior to running a bdist on Linux or OS X.
"""
if sys.platform.startswith('win'):
return
# copy libcapnp into capnp for bdist
local = localpath('capnp',libcapnp)
if not capnp and not os.path.exists(local):
fatal("Please specify capnp prefix via `setup.py configure --capnp=/path/to/capnp` "
"or copy libcapnp into capnp/ manually prior to running bdist.")
try:
# resolve real file through symlinks
lib = os.path.realpath(pjoin(capnp, 'lib', libcapnp))
print ("copying %s -> %s"%(lib, local))
shutil.copy(lib, local)
except Exception:
if not os.path.exists(local):
fatal("Could not copy libcapnp into capnp/, which is necessary for bdist. "
"Please specify capnp prefix via `setup.py configure --capnp=/path/to/capnp` "
"or copy libcapnp into capnp/ manually.")
if sys.platform == 'darwin':
# chmod u+w on the lib,
# which can be user-read-only for some reason
mode = os.stat(local).st_mode
os.chmod(local, mode | stat.S_IWUSR)
# patch install_name on darwin, instead of using rpath
cmd = ['install_name_tool', '-id', '@loader_path/../%s'%libcapnp, local]
try:
p = Popen(cmd, stdout=PIPE,stderr=PIPE)
except OSError:
fatal("install_name_tool not found, cannot patch libcapnp for bundling.")
out,err = p.communicate()
if p.returncode:
fatal("Could not patch bundled libcapnp install_name: %s"%err, p.returncode)

157
buildutils/config.py Normal file
View File

@@ -0,0 +1,157 @@
"""Config functions"""
#-----------------------------------------------------------------------------
# Copyright (C) PyZMQ Developers
#
# This file is part of pyzmq, copied and adapted from h5py.
# h5py source used under the New BSD license
#
# h5py: <http://code.google.com/p/h5py/>
#
# Distributed under the terms of the New BSD License. The full license is in
# the file COPYING.BSD, distributed as part of this software.
#-----------------------------------------------------------------------------
import sys
import os
import json
try:
from configparser import ConfigParser
except:
from ConfigParser import ConfigParser
pjoin = os.path.join
from .msg import debug, fatal, warn
#-----------------------------------------------------------------------------
# Utility functions (adapted from h5py: http://h5py.googlecode.com)
#-----------------------------------------------------------------------------
def load_config(name, base='conf'):
"""Load config dict from JSON"""
fname = pjoin(base, name + '.json')
if not os.path.exists(fname):
return {}
try:
with open(fname) as f:
cfg = json.load(f)
except Exception as e:
warn("Couldn't load %s: %s" % (fname, e))
cfg = {}
return cfg
def save_config(name, data, base='conf'):
"""Save config dict to JSON"""
if not os.path.exists(base):
os.mkdir(base)
fname = pjoin(base, name+'.json')
with open(fname, 'w') as f:
json.dump(data, f, indent=2)
def v_str(v_tuple):
"""turn (2,0,1) into '2.0.1'."""
return ".".join(str(x) for x in v_tuple)
def get_eargs():
""" Look for options in environment vars """
settings = {}
zmq = os.environ.get("ZMQ_PREFIX", None)
if zmq is not None:
debug("Found environ var ZMQ_PREFIX=%s" % zmq)
settings['zmq_prefix'] = zmq
return settings
def cfg2dict(cfg):
"""turn a ConfigParser into a nested dict
because ConfigParser objects are dumb.
"""
d = {}
for section in cfg.sections():
d[section] = dict(cfg.items(section))
return d
def get_cfg_args():
""" Look for options in setup.cfg """
if not os.path.exists('setup.cfg'):
return {}
cfg = ConfigParser()
cfg.read('setup.cfg')
cfg = cfg2dict(cfg)
g = cfg.setdefault('global', {})
# boolean keys:
for key in ['libzmq_extension',
'bundle_libzmq_dylib',
'no_libzmq_extension',
'have_sys_un_h',
'skip_check_zmq',
]:
if key in g:
g[key] = eval(g[key])
# globals go to top level
cfg.update(cfg.pop('global'))
return cfg
def config_from_prefix(prefix):
"""Get config from zmq prefix"""
settings = {}
if prefix.lower() in ('default', 'auto', ''):
settings['zmq_prefix'] = ''
settings['libzmq_extension'] = False
settings['no_libzmq_extension'] = False
elif prefix.lower() in ('bundled', 'extension'):
settings['zmq_prefix'] = ''
settings['libzmq_extension'] = True
settings['no_libzmq_extension'] = False
else:
settings['zmq_prefix'] = prefix
settings['libzmq_extension'] = False
settings['no_libzmq_extension'] = True
return settings
def merge(into, d):
"""merge two containers
into is updated, d has priority
"""
if isinstance(into, dict):
for key in d.keys():
if key not in into:
into[key] = d[key]
else:
into[key] = merge(into[key], d[key])
return into
elif isinstance(into, list):
return into + d
else:
return d
def discover_settings(conf_base=None):
""" Discover custom settings for ZMQ path"""
settings = {
'zmq_prefix': '',
'libzmq_extension': False,
'no_libzmq_extension': False,
'skip_check_zmq': False,
'build_ext': {},
'bdist_egg': {},
}
if sys.platform.startswith('win'):
settings['have_sys_un_h'] = False
if conf_base:
# lowest priority
merge(settings, load_config('config', conf_base))
merge(settings, get_cfg_args())
merge(settings, get_eargs())
return settings

83
buildutils/constants.py Normal file
View File

@@ -0,0 +1,83 @@
"""
script for generating files that involve repetitive updates for zmq constants.
Run this after updating utils/constant_names
Currently generates the following files from templates:
- constant_enums.pxi
- constants.pxi
- zmq_constants.h
"""
# Copyright (C) PyZMQ Developers
# Distributed under the terms of the Modified BSD License.
import os
import sys
from . import info
pjoin = os.path.join
root = os.path.abspath(pjoin(os.path.dirname(__file__), os.path.pardir))
sys.path.insert(0, pjoin(root, 'zmq', 'utils'))
from constant_names import all_names, no_prefix
ifndef_t = """#ifndef {0}
#define {0} (_PYZMQ_UNDEFINED)
#endif
"""
def cython_enums():
"""generate `enum: ZMQ_CONST` block for constant_enums.pxi"""
lines = []
for name in all_names:
if no_prefix(name):
lines.append('enum: ZMQ_{0} "{0}"'.format(name))
else:
lines.append('enum: ZMQ_{0}'.format(name))
return dict(ZMQ_ENUMS='\n '.join(lines))
def ifndefs():
"""generate `#ifndef ZMQ_CONST` block for zmq_constants.h"""
lines = ['#define _PYZMQ_UNDEFINED (-9999)']
for name in all_names:
if not no_prefix(name):
name = 'ZMQ_%s' % name
lines.append(ifndef_t.format(name))
return dict(ZMQ_IFNDEFS='\n'.join(lines))
def constants_pyx():
"""generate CONST = ZMQ_CONST and __all__ for constants.pxi"""
all_lines = []
assign_lines = []
for name in all_names:
if name == "NULL":
# avoid conflict with NULL in Cython
assign_lines.append("globals()['NULL'] = ZMQ_NULL")
else:
assign_lines.append('{0} = ZMQ_{0}'.format(name))
all_lines.append(' "{0}",'.format(name))
return dict(ASSIGNMENTS='\n'.join(assign_lines), ALL='\n'.join(all_lines))
def generate_file(fname, ns_func, dest_dir="."):
"""generate a constants file from its template"""
with open(pjoin(root, 'buildutils', 'templates', '%s' % fname), 'r') as f:
tpl = f.read()
out = tpl.format(**ns_func())
dest = pjoin(dest_dir, fname)
info("generating %s from template" % dest)
with open(dest, 'w') as f:
f.write(out)
def render_constants():
"""render generated constant files from templates"""
generate_file("constant_enums.pxi", cython_enums, pjoin(root, 'zmq', 'backend', 'cython'))
generate_file("constants.pxi", constants_pyx, pjoin(root, 'zmq', 'backend', 'cython'))
generate_file("zmq_constants.h", ifndefs, pjoin(root, 'zmq', 'utils'))
if __name__ == '__main__':
render_constants()

166
buildutils/detect.py Normal file
View File

@@ -0,0 +1,166 @@
"""Detect zmq version"""
#-----------------------------------------------------------------------------
# Copyright (C) PyZMQ Developers
#
# This file is part of pyzmq, copied and adapted from h5py.
# h5py source used under the New BSD license
#
# h5py: <http://code.google.com/p/h5py/>
#
# Distributed under the terms of the New BSD License. The full license is in
# the file COPYING.BSD, distributed as part of this software.
#-----------------------------------------------------------------------------
#
# Adapted for use in pycapnp from pyzmq. See https://github.com/zeromq/pyzmq
# for original project.
import shutil
import sys
import os
import logging
import platform
from distutils import ccompiler
from distutils.ccompiler import get_default_compiler
from subprocess import Popen, PIPE
import tempfile
from .misc import get_compiler, get_output_error
from .patch import patch_lib_paths
pjoin = os.path.join
#-----------------------------------------------------------------------------
# Utility functions (adapted from h5py: http://h5py.googlecode.com)
#-----------------------------------------------------------------------------
def test_compilation(cfile, compiler=None, **compiler_attrs):
"""Test simple compilation with given settings"""
cc = get_compiler(compiler, **compiler_attrs)
efile, ext = os.path.splitext(cfile)
cpreargs = lpreargs = []
if sys.platform == 'darwin':
# use appropriate arch for compiler
if platform.architecture()[0]=='32bit':
if platform.processor() == 'powerpc':
cpu = 'ppc'
else:
cpu = 'i386'
cpreargs = ['-arch', cpu]
lpreargs = ['-arch', cpu, '-undefined', 'dynamic_lookup']
else:
# allow for missing UB arch, since it will still work:
lpreargs = ['-undefined', 'dynamic_lookup']
if sys.platform == 'sunos5':
if platform.architecture()[0]=='32bit':
lpreargs = ['-m32']
else:
lpreargs = ['-m64']
extra = compiler_attrs.get('extra_compile_args', [])
extra += ['--std=c++11']
objs = cc.compile([cfile], extra_preargs=cpreargs, extra_postargs=extra)
cc.link_executable(objs, efile, extra_preargs=lpreargs)
return efile
def compile_and_run(basedir, src, compiler=None, **compiler_attrs):
if not os.path.exists(basedir):
os.makedirs(basedir)
cfile = pjoin(basedir, os.path.basename(src))
shutil.copy(src, cfile)
try:
cc = get_compiler(compiler, **compiler_attrs)
efile = test_compilation(cfile, compiler=cc)
patch_lib_paths(efile, cc.library_dirs)
result = Popen(efile, stdout=PIPE, stderr=PIPE)
so, se = result.communicate()
# for py3k:
so = so.decode()
se = se.decode()
finally:
shutil.rmtree(basedir)
return result.returncode, so, se
def detect_version(basedir, compiler=None, **compiler_attrs):
"""Compile, link & execute a test program, in empty directory `basedir`.
The C compiler will be updated with any keywords given via setattr.
Parameters
----------
basedir : path
The location where the test program will be compiled and run
compiler : str
The distutils compiler key (e.g. 'unix', 'msvc', or 'mingw32')
**compiler_attrs : dict
Any extra compiler attributes, which will be set via ``setattr(cc)``.
Returns
-------
A dict of properties for zmq compilation, with the following two keys:
vers : tuple
The ZMQ version as a tuple of ints, e.g. (2,2,0)
settings : dict
The compiler options used to compile the test function, e.g. `include_dirs`,
`library_dirs`, `libs`, etc.
"""
if compiler is None:
compiler = get_default_compiler()
cfile = pjoin(basedir, 'vers.cpp')
shutil.copy(pjoin(os.path.dirname(__file__), 'vers.cpp'), cfile)
# check if we need to link against Realtime Extensions library
if sys.platform.startswith('linux'):
cc = ccompiler.new_compiler(compiler=compiler)
cc.output_dir = basedir
if not cc.has_function('timer_create'):
compiler_attrs['libraries'].append('rt')
cc = get_compiler(compiler=compiler, **compiler_attrs)
efile = test_compilation(cfile, compiler=cc)
patch_lib_paths(efile, cc.library_dirs)
rc, so, se = get_output_error([efile])
if rc:
msg = "Error running version detection script:\n%s\n%s" % (so,se)
logging.error(msg)
raise IOError(msg)
handlers = {'vers': lambda val: tuple(int(v) for v in val.split('.'))}
props = {}
for line in (x for x in so.split('\n') if x):
key, val = line.split(':')
props[key] = handlers[key](val)
return props
def test_build():
"""do a test build of libcapnp"""
tmp_dir = tempfile.mkdtemp()
# line()
# info("Configure: Autodetecting Cap'n Proto settings...")
# info(" Custom Cap'n Proto dir: %s" % prefix)
try:
detected = detect_version(tmp_dir)
finally:
erase_dir(tmp_dir)
# info(" Cap'n Proto version detected: %s" % v_str(detected['vers']))
return detected
def erase_dir(dir):
try:
shutil.rmtree(dir)
except Exception:
pass

65
buildutils/misc.py Normal file
View File

@@ -0,0 +1,65 @@
"""misc build utility functions"""
# Copyright (c) PyZMQ Developers
# Distributed under the terms of the Modified BSD License.
import os
import sys
import logging
from distutils import ccompiler
from distutils.sysconfig import customize_compiler
from pipes import quote
from subprocess import Popen, PIPE
pjoin = os.path.join
if sys.version_info[0] >= 3:
u = lambda x: x
else:
u = lambda x: x.decode('utf8', 'replace')
def customize_mingw(cc):
# strip -mno-cygwin from mingw32 (Python Issue #12641)
for cmd in [cc.compiler, cc.compiler_cxx, cc.compiler_so, cc.linker_exe, cc.linker_so]:
if '-mno-cygwin' in cmd:
cmd.remove('-mno-cygwin')
# remove problematic msvcr90
if 'msvcr90' in cc.dll_libraries:
cc.dll_libraries.remove('msvcr90')
def get_compiler(compiler, **compiler_attrs):
"""get and customize a compiler"""
if compiler is None or isinstance(compiler, str):
cc = ccompiler.new_compiler(compiler=compiler)
# customize_compiler(cc)
if cc.compiler_type == 'mingw32':
customize_mingw(cc)
else:
cc = compiler
for name, val in compiler_attrs.items():
setattr(cc, name, val)
return cc
def get_output_error(cmd):
"""Return the exit status, stdout, stderr of a command"""
if not isinstance(cmd, list):
cmd = [cmd]
logging.debug("Running: %s", ' '.join(map(quote, cmd)))
try:
result = Popen(cmd, stdout=PIPE, stderr=PIPE)
except IOError as e:
return -1, u(''), u('Failed to run %r: %r' % (cmd, e))
so, se = result.communicate()
# unicode:
so = so.decode('utf8', 'replace')
se = se.decode('utf8', 'replace')
return result.returncode, so, se

39
buildutils/msg.py Normal file
View File

@@ -0,0 +1,39 @@
"""logging"""
# Copyright (c) PyZMQ Developers.
# Distributed under the terms of the Modified BSD License.
from __future__ import division
import os
import sys
import logging
#-----------------------------------------------------------------------------
# Logging (adapted from h5py: http://h5py.googlecode.com)
#-----------------------------------------------------------------------------
logger = logging.getLogger()
if os.environ.get('DEBUG'):
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler(sys.stderr))
def debug(msg):
logger.debug(msg)
def info(msg):
logger.info(msg)
def fatal(msg, code=1):
logger.error("Fatal: " + msg)
exit(code)
def warn(msg):
logger.error("Warning: " + msg)
def line(c='*', width=48):
print(c * (width // len(c)))

61
buildutils/patch.py Normal file
View File

@@ -0,0 +1,61 @@
"""utils for patching libraries"""
# Copyright (c) PyZMQ Developers.
# Distributed under the terms of the Modified BSD License.
import re
import sys
import os
import logging
from .misc import get_output_error
pjoin = os.path.join
# LIB_PAT from delocate
LIB_PAT = re.compile(r"\s*(.*) \(compatibility version (\d+\.\d+\.\d+), "
r"current version (\d+\.\d+\.\d+)\)")
def _get_libs(fname):
rc, so, se = get_output_error(['otool', '-L', fname])
if rc:
logging.error("otool -L %s failed: %r" % (fname, se))
return
for line in so.splitlines()[1:]:
m = LIB_PAT.match(line)
if m:
yield m.group(1)
def _find_library(lib, path):
"""Find a library"""
for d in path[::-1]:
real_lib = os.path.join(d, lib)
if os.path.exists(real_lib):
return real_lib
def _install_name_change(fname, lib, real_lib):
rc, so, se = get_output_error(['install_name_tool', '-change', lib, real_lib, fname])
if rc:
logging.error("Couldn't update load path: %s", se)
def patch_lib_paths(fname, library_dirs):
"""Load any weakly-defined libraries from their real location
(only on OS X)
- Find libraries with `otool -L`
- Update with `install_name_tool -change`
"""
if sys.platform != 'darwin':
return
libs = _get_libs(fname)
for lib in libs:
if not lib.startswith(('@', '/')):
real_lib = _find_library(lib, library_dirs)
if real_lib:
_install_name_change(fname, lib, real_lib)
__all__ = ['patch_lib_paths']

9
buildutils/vers.cpp Normal file
View File

@@ -0,0 +1,9 @@
// check libcapnp version
#include <stdio.h>
#include "capnp/common.h"
int main(int argc, char **argv){
fprintf(stdout, "vers: %d.%d.%d\n", CAPNP_VERSION_MAJOR, CAPNP_VERSION_MINOR, CAPNP_VERSION_MICRO);
return 0;
}