Ticket #13190: trac13190_library.patch

File trac13190_library.patch, 22.6 KB (added by ohanar, 9 years ago)

apply to sage library

  • .hgignore

    # HG changeset patch
    # User R. Andrew Ohana <andrew.ohana@gmail.com>
    # Date 1341131248 25200
    # Node ID 73f068136db2657bb418b1b7c83206389a29b9c8
    # Parent  14a87744d4731f2b3cdfcea5d8d591feba705fe6
    switch from distutils to setuptools for Sage library
    
    diff --git a/.hgignore b/.hgignore
    a b  
    4141dist
    4242sage/.*\.c$
    4343sage/.*\.cpp$
     44sage/.*\.so$
     45sage.egg-info
    4446c_lib/.*\.lo$
    4547c_lib/.*\.os$
    4648c_lib/libcsage.la
  • MANIFEST.in

    diff --git a/MANIFEST.in b/MANIFEST.in
    a b  
    33graft   .hg
    44include .hgignore .hgtags
    55include MANIFEST.in
    6 include bundle export install pull sage-push
     6include bundle export pull sage-push
    77include spkg-delauto spkg-dist spkg-install
    88
    99include c_lib/SConstruct
  • deleted file fpickle_setup.py

    diff --git a/fpickle_setup.py b/fpickle_setup.py
    deleted file mode 100644
    + -  
    1 """
    2 The purpose of this file is to allow instancemethods to be pickable.
    3 This is needed in setup.py only and not installed anywhere, thus is not
    4 a library file. It solves the issue from #11874.
    5 """
    6 
    7 import types, copy_reg, copy
    8 
    9 
    10 # The following four methods are taken from twisted.persisted.styles
    11 # into sage.misc.fpickle, and then here. It was needed due to
    12 # chicken vs egg issue, as we cannot use sage.misc.fpickle in
    13 # setup.py of sage-***.spkg
    14 def pickleMethod(method):
    15     'support function for copy_reg to pickle method refs'
    16     return unpickleMethod, (method.im_func.__name__,
    17                              method.im_self,
    18                              method.im_class)
    19 
    20 def unpickleMethod(im_name,
    21                     im_self,
    22                     im_class):
    23     'support function for copy_reg to unpickle method refs'
    24     try:
    25         unbound = getattr(im_class,im_name)
    26         if im_self is None:
    27             return unbound
    28         bound=types.MethodType(unbound.im_func,
    29                                  im_self)
    30         return bound
    31     except AttributeError:
    32         # assert im_self is not None,"No recourse: no instance to guess from."
    33         # Attempt a common fix before bailing -- if classes have
    34         # changed around since we pickled this method, we may still be
    35         # able to get it by looking on the instance's current class.
    36         unbound = getattr(im_self.__class__,im_name)
    37         if im_self is None:
    38             return unbound
    39         bound=types.MethodType(unbound.im_func,
    40                                  im_self)
    41         return bound
    42 
    43 copy_reg.pickle(types.MethodType,
    44                 pickleMethod,
    45                 unpickleMethod)
    46 
    47 oldModules = {}
    48 
    49 def pickleModule(module):
    50     'support function for copy_reg to pickle module refs'
    51     return unpickleModule, (module.__name__,)
    52 
    53 def unpickleModule(name):
    54     'support function for copy_reg to unpickle module refs'
    55     if oldModules.has_key(name):
    56         name = oldModules[name]
    57     return __import__(name,{},{},'x')
    58 
    59 copy_reg.pickle(types.ModuleType,
    60                 pickleModule,
    61                 unpickleModule)
  • deleted file install

    diff --git a/install b/install
    deleted file mode 100755
    + -  
    1 #!/bin/sh
    2 if [ $# -eq 0 ]; then
    3     echo "Usage: $0 <SAGE_ROOT> [SAGE_VERSION]"
    4     echo "Build and install SAGE."
    5     exit 1
    6 fi
    7 
    8 CUR=`pwd`
    9 cd $1
    10 SAGE_ROOT=`pwd`
    11 cd $CUR
    12 export SAGE_ROOT
    13 
    14 SAGE_VERSION=$2
    15 export SAGE_VERSION
    16 
    17 PATH=$SAGE_ROOT/local/bin/:$PATH
    18 export PATH
    19 
    20 if [ -n "$SAGE_DEBIAN" ]; then
    21     python setup.py install --root="$SAGE_ROOT/" --no-compile -O0
    22 else
    23     python setup.py install
    24 fi
    25 
    26 exit $?
    27 
  • module_list.py

    diff --git a/module_list.py b/module_list.py
    a b  
    11import os, sys
    2 from distutils.core import setup
    3 from distutils.extension import Extension
     2from setuptools import Extension
    43
    54opj = os.path.join
    65
  • sage/interacts/debugger.py

    diff --git a/sage/interacts/debugger.py b/sage/interacts/debugger.py
    a b  
    185185        code = code.replace(TRACEBACK, TRACEBACK[:-1])
    186186
    187187        # Create a hyperlink to the file, if possible.
    188         i = filename.rfind('site-packages/sage')
     188        i = filename.rfind('egg/sage')
    189189        if i != -1:
    190             fname = filename[i+len('site-packages/sage')+1:].rstrip('/')
     190            fname = filename[i+len('egg/sage')+1:].rstrip('/')
    191191            file = '<a href="/src/%s" target="_new">devel/sage/sage/%s</a>'%(fname,fname)
    192192        else:
    193193            file = filename
  • setup.py

    diff --git a/setup.py b/setup.py
    a b  
    11#!/usr/bin/env python
    22
    3 import os, sys, time, errno, platform
    4 from distutils.core import setup
    5 from distutils.extension import Extension
    6 from glob import glob, fnmatch
    7 from warnings import warn
     3import os, sys, time, platform
     4from setuptools import setup
    85
    96#########################################################
    107### List of Extensions
     
    1714
    1815from module_list import ext_modules, exclude_modules
    1916import sage.ext.gen_interpreters
    20 import warnings
    2117
    2218#########################################################
    2319### Configuration
     
    3733    SAGE_DEVEL = SAGE_ROOT + '/devel'
    3834    SAGE_INC   = SAGE_LOCAL + '/include/'
    3935    DOT_SAGE   = os.environ['DOT_SAGE']
    40 
    41 if not os.environ.has_key('SAGE_VERSION'):
    42     SAGE_VERSION=0
    43 else:
    44     SAGE_VERSION = os.environ['SAGE_VERSION']
     36    SAGE_VERSION = os.environ.get('SAGE_VERSION', False)
     37    if not SAGE_VERSION:
     38        import sage.version
     39        SAGE_VERSION = sage.version.version
    4540
    4641try:
    4742    compile_result_dir = os.environ['XML_RESULTS']
     
    5045    compile_result_dir = None
    5146    keep_going = False
    5247
    53 SITE_PACKAGES = '%s/lib/python%s/site-packages/'%(SAGE_LOCAL,platform.python_version().rsplit('.', 1)[0])
    54 if not os.path.exists(SITE_PACKAGES):
    55     raise RuntimeError, "Unable to find site-packages directory (see setup.py file in sage python code)."
    56 
    57 if not os.path.exists('build/sage'):
    58     os.makedirs('build/sage')
    59 
    60 sage_link = SITE_PACKAGES + '/sage'
    61 if not os.path.islink(sage_link) or not os.path.exists(sage_link):
    62     os.system('rm -rf "%s"'%sage_link)
    63     os.system('cd %s; ln -sf ../../../../devel/sage/build/sage .'%SITE_PACKAGES)
    64 
    6548# search for dependencies and add to gcc -I<path>
    6649include_dirs = ['%s/include'%SAGE_LOCAL,
    6750                '%s/include/csage'%SAGE_LOCAL,
     
    8871sage.ext.gen_interpreters.rebuild(SAGE_DEVEL + '/sage/sage/ext/interpreters')
    8972ext_modules = ext_modules + sage.ext.gen_interpreters.modules
    9073
    91 
    92 #########################################################
    93 ### Testing related stuff
    94 #########################################################
    95 
    96 class CompileRecorder(object):
    97 
    98     def __init__(self, f):
    99         self._f = f
    100         self._obj = None
    101 
    102     def __get__(self, obj, type=None):
    103         # Act like a method...
    104         self._obj = obj
    105         return self
    106 
    107     def __call__(self, *args):
    108         t = time.time()
    109         try:
    110             if self._obj:
    111                 res = self._f(self._obj, *args)
    112             else:
    113                 res = self._f(*args)
    114         except Exception, ex:
    115             print ex
    116             res = ex
    117         t = time.time() - t
    118 
    119         errors = failures = 0
    120         if self._f is compile_command0:
    121             name = "cythonize." + args[0][1].name
    122             failures = int(bool(res))
    123         else:
    124             name = "gcc." + args[0][1].name
    125             errors = int(bool(res))
    126         if errors or failures:
    127             type = "failure" if failures else "error"
    128             failure_item = """<%(type)s/>""" % locals()
    129         else:
    130             failure_item = ""
    131         output = open("%s/%s.xml" % (compile_result_dir, name), "w")
    132         output.write("""
    133             <?xml version="1.0" ?>
    134             <testsuite name="%(name)s" errors="%(errors)s" failures="%(failures)s" tests="1" time="%(t)s">
    135             <testcase classname="%(name)s" name="compile">
    136             %(failure_item)s
    137             </testcase>
    138             </testsuite>
    139         """.strip() % locals())
    140         output.close()
    141         return res
    142 
    143 if compile_result_dir:
    144     record_compile = CompileRecorder
    145 else:
    146     record_compile = lambda x: x
    147 
    14874# Remove (potentially invalid) star import caches
    14975import sage.misc.lazy_import_cache
    15076if os.path.exists(sage.misc.lazy_import_cache.get_cache_file()):
     
    180106    m.extra_link_args += extra_link_args
    181107    m.library_dirs += ['%s/lib' % SAGE_LOCAL]
    182108
    183 
    184 
    185 #############################################
    186 ###### Parallel Cython execution
    187 #############################################
    188 
    189 def run_command(cmd):
    190     """
    191     INPUT:
    192         cmd -- a string; a command to run
    193 
    194     OUTPUT:
    195         prints cmd to the console and then runs os.system
    196     """
    197     print cmd
    198     return os.system(cmd)
    199 
    200 def apply_pair(p):
    201     """
    202     Given a pair p consisting of a function and a value, apply
    203     the function to the value.
    204 
    205     This exists solely because we can't pickle an anonymous function
    206     in execute_list_of_commands_in_parallel below.
    207     """
    208     return p[0](p[1])
    209 
    210 def execute_list_of_commands_in_parallel(command_list, nthreads):
    211     """
    212     Execute the given list of commands, possibly in parallel, using
    213     ``nthreads`` threads.  Terminates ``setup.py`` with an exit code
    214     of 1 if an error occurs in any subcommand.
    215 
    216     INPUT:
    217 
    218     - ``command_list`` -- a list of commands, each given as a pair of
    219        the form ``[function, argument]`` of a function to call and its
    220        argument
    221 
    222     - ``nthreads`` -- integer; number of threads to use
    223 
    224     WARNING: commands are run roughly in order, but of course successive
    225     commands may be run at the same time.
    226     """
    227     from multiprocessing import Pool
    228     import fpickle_setup #doing this import will allow instancemethods to be pickable
    229     p = Pool(nthreads)
    230     process_command_results(p.imap(apply_pair, command_list))
    231 
    232 def process_command_results(result_values):
    233     error = None
    234     for r in result_values:
    235         if r:
    236             print "Error running command, failed with status %s."%r
    237             if not keep_going:
    238                 sys.exit(1)
    239             error = r
    240     if error:
    241         sys.exit(1)
    242 
    243 def execute_list_of_commands(command_list):
    244     """
    245     INPUT:
    246 
    247     - ``command_list`` -- a list of strings or pairs
    248 
    249     OUTPUT:
    250 
    251     For each entry in command_list, we attempt to run the command.
    252     If it is a string, we call ``os.system()``. If it is a pair [f, v],
    253     we call f(v).
    254 
    255     If the environment variable :envvar:`SAGE_NUM_THREADS` is set, use
    256     that many threads.
    257     """
    258     t = time.time()
    259     # Determine the number of threads from the environment variable
    260     # SAGE_NUM_THREADS, which is set automatically by sage-env
    261     try:
    262         nthreads = int(os.environ['SAGE_NUM_THREADS'])
    263     except KeyError:
    264         nthreads = 1
    265 
    266     # normalize the command_list to handle strings correctly
    267     command_list = [ [run_command, x] if isinstance(x, str) else x for x in command_list ]
    268 
    269     # No need for more threads than there are commands, but at least one
    270     nthreads = min(len(command_list), nthreads)
    271     nthreads = max(1, nthreads)
    272 
    273     def plural(n,noun):
    274         if n == 1:
    275             return "1 %s"%noun
    276         return "%i %ss"%(n,noun)
    277 
    278     print "Executing %s (using %s)"%(plural(len(command_list),"command"), plural(nthreads,"thread"))
    279     execute_list_of_commands_in_parallel(command_list, nthreads)
    280     print "Time to execute %s: %s seconds"%(plural(len(command_list),"command"), time.time() - t)
    281 
    282 
    283109########################################################################
    284110##
    285111## Parallel gcc execution
    286112##
    287 ## This code is responsible for making distutils dispatch the calls to
    288 ## build_ext in parallel. Since distutils doesn't seem to do this by
    289 ## default, we create our own extension builder and override the
    290 ## appropriate methods.  Unfortunately, in distutils, the logic of
    291 ## deciding whether an extension needs to be recompiled and actually
    292 ## making the call to gcc to recompile the extension are in the same
    293 ## function. As a result, we can't just override one function and have
    294 ## everything magically work. Instead, we split this work between two
    295 ## functions. This works fine for our application, but it means that
    296 ## we can't use this modification to make the other parts of Sage that
    297 ## build with distutils call gcc in parallel.
     113## This code is responsible for making build_ext dispatch the calls to
     114## gcc in parallel. Since Cython's build_ext doesn't do this, we
     115## overwrite the build_extensions method.
    298116##
    299117########################################################################
    300118
    301 from distutils.command.build_ext import build_ext
    302 from distutils.dep_util import newer_group
    303 from types import ListType, TupleType
    304 from distutils import log
     119from Cython.Distutils import build_ext as _build_ext
    305120
    306 class sage_build_ext(build_ext):
     121class build_ext(_build_ext):
    307122
    308123    def build_extensions(self):
    309 
    310         from distutils.debug import DEBUG
    311 
    312         if DEBUG:
    313             print "self.compiler.compiler:"
    314             print self.compiler.compiler
    315             print "self.compiler.compiler_cxx:"
    316             print self.compiler.compiler_cxx # currently not used
    317             print "self.compiler.compiler_so:"
    318             print self.compiler.compiler_so
    319             print "self.compiler.linker_so:"
    320             print self.compiler.linker_so
    321             # There are further interesting variables...
    322             sys.stdout.flush()
    323 
    324 
    325         # At least on MacOS X, the library dir of the *original* Sage
    326         # installation is "hard-coded" into the linker *command*, s.t.
    327         # that directory is always searched *first*, which causes trouble
    328         # after the Sage installation has been moved (or its directory simply
    329         # been renamed), especially in conjunction with upgrades (cf. #9896).
    330         # (In principle, the Python configuration should be modified on
    331         # Sage relocations as well, but until that's done, we simply fix
    332         # the most important.)
    333         # Since the following is performed only once per call to "setup",
    334         # and doesn't hurt on other systems, we unconditionally replace *any*
    335         # library directory specified in the (dynamic) linker command by the
    336         # current Sage library directory (if it doesn't already match that),
    337         # and issue a warning message:
    338 
    339         if True or sys.platform[:6]=="darwin":
    340 
    341             sage_libdir = os.path.realpath(SAGE_LOCAL+"/lib")
    342             ldso_cmd = self.compiler.linker_so # a list of strings, like argv
    343 
    344             for i in range(1, len(ldso_cmd)):
    345 
    346                 if ldso_cmd[i][:2] == "-L":
    347                     libdir = os.path.realpath(ldso_cmd[i][2:])
    348                     self.debug_print(
    349                       "Library dir found in dynamic linker command: " +
    350                       "\"%s\"" % libdir)
    351                     if libdir != sage_libdir:
    352                         self.compiler.warn(
    353                           "Replacing library search directory in linker " +
    354                           "command:\n  \"%s\" -> \"%s\"\n" % (libdir,
    355                                                               sage_libdir))
    356                         ldso_cmd[i] = "-L"+sage_libdir
    357 
    358         if DEBUG:
    359             print "self.compiler.linker_so (after fixing library dirs):"
    360             print self.compiler.linker_so
    361             sys.stdout.flush()
    362 
    363 
    364124        # First, sanity-check the 'extensions' list
    365125        self.check_extensions_list(self.extensions)
    366126
    367         import time
    368         t = time.time()
     127        def process_ext(ext):
     128            ext.sources = self.cython_sources(ext.sources, ext)
     129            self.build_extension(ext)
    369130
    370         compile_commands = []
    371         for ext in self.extensions:
    372             need_to_compile, p = self.prepare_extension(ext)
    373             if need_to_compile:
    374                 compile_commands.append((record_compile(self.build_extension), p))
    375 
    376         execute_list_of_commands(compile_commands)
    377 
    378         print "Total time spent compiling C/C++ extensions: ", time.time() - t, "seconds."
    379 
    380     def prepare_extension(self, ext):
    381         sources = ext.sources
    382         if sources is None or type(sources) not in (ListType, TupleType):
    383             raise DistutilsSetupError, \
    384                   ("in 'ext_modules' option (extension '%s'), " +
    385                    "'sources' must be present and must be " +
    386                    "a list of source filenames") % ext.name
    387         sources = list(sources)
    388 
    389         fullname = self.get_ext_fullname(ext.name)
    390         if self.inplace:
    391             # ignore build-lib -- put the compiled extension into
    392             # the source tree along with pure Python modules
    393 
    394             modpath = string.split(fullname, '.')
    395             package = string.join(modpath[0:-1], '.')
    396             base = modpath[-1]
    397 
    398             build_py = self.get_finalized_command('build_py')
    399             package_dir = build_py.get_package_dir(package)
    400             ext_filename = os.path.join(package_dir,
    401                                         self.get_ext_filename(base))
    402             relative_ext_filename = self.get_ext_filename(base)
     131        nthreads = int(os.environ.get('SAGE_NUM_THREADS', 1))
     132        if nthreads == 1:
     133            for ext in self.extensions:
     134                process_ext(ext)
    403135        else:
    404             ext_filename = os.path.join(self.build_lib,
    405                                         self.get_ext_filename(fullname))
    406             relative_ext_filename = self.get_ext_filename(fullname)
    407 
    408         # while dispatching the calls to gcc in parallel, we sometimes
    409         # hit a race condition where two separate build_ext objects
    410         # try to create a given directory at the same time; whoever
    411         # loses the race then seems to throw an error, saying that
    412         # the directory already exists. so, instead of fighting to
    413         # fix the race condition, we simply make sure the entire
    414         # directory tree exists now, while we're processing the
    415         # extensions in serial.
    416         relative_ext_dir = os.path.split(relative_ext_filename)[0]
    417         prefixes = ['', self.build_lib, self.build_temp]
    418         for prefix in prefixes:
    419             path = os.path.join(prefix, relative_ext_dir)
    420             try:
    421                 os.makedirs(path)
    422             except OSError, e:
    423                 assert e.errno==errno.EEXIST, 'Cannot create %s.' % path
    424         depends = sources + ext.depends
    425         if not (self.force or newer_group(depends, ext_filename, 'newer')):
    426             log.debug("skipping '%s' extension (up-to-date)", ext.name)
    427             need_to_compile = False
    428         else:
    429             log.info("building '%s' extension", ext.name)
    430             need_to_compile = True
    431 
    432         return need_to_compile, (sources, ext, ext_filename)
    433 
    434     def build_extension(self, p):
    435 
    436         sources, ext, ext_filename = p
    437 
    438         # First, scan the sources for SWIG definition files (.i), run
    439         # SWIG on 'em to create .c files, and modify the sources list
    440         # accordingly.
    441         sources = self.swig_sources(sources, ext)
    442 
    443         # Next, compile the source code to object files.
    444 
    445         # XXX not honouring 'define_macros' or 'undef_macros' -- the
    446         # CCompiler API needs to change to accommodate this, and I
    447         # want to do one thing at a time!
    448 
    449         # Two possible sources for extra compiler arguments:
    450         #   - 'extra_compile_args' in Extension object
    451         #   - CFLAGS environment variable (not particularly
    452         #     elegant, but people seem to expect it and I
    453         #     guess it's useful)
    454         # The environment variable should take precedence, and
    455         # any sensible compiler will give precedence to later
    456         # command line args.  Hence we combine them in order:
    457         extra_args = ext.extra_compile_args or []
    458 
    459         macros = ext.define_macros[:]
    460         for undef in ext.undef_macros:
    461             macros.append((undef,))
    462 
    463         objects = self.compiler.compile(sources,
    464                                         output_dir=self.build_temp,
    465                                         macros=macros,
    466                                         include_dirs=ext.include_dirs,
    467                                         debug=self.debug,
    468                                         extra_postargs=extra_args,
    469                                         depends=ext.depends)
    470 
    471         # XXX -- this is a Vile HACK!
    472         #
    473         # The setup.py script for Python on Unix needs to be able to
    474         # get this list so it can perform all the clean up needed to
    475         # avoid keeping object files around when cleaning out a failed
    476         # build of an extension module.  Since Distutils does not
    477         # track dependencies, we have to get rid of intermediates to
    478         # ensure all the intermediates will be properly re-built.
    479         #
    480         self._built_objects = objects[:]
    481 
    482         # Now link the object files together into a "shared object" --
    483         # of course, first we have to figure out all the other things
    484         # that go into the mix.
    485         if ext.extra_objects:
    486             objects.extend(ext.extra_objects)
    487         extra_args = ext.extra_link_args or []
    488 
    489         # Detect target language, if not provided
    490         language = ext.language or self.compiler.detect_language(sources)
    491 
    492         self.compiler.link_shared_object(
    493             objects, ext_filename,
    494             libraries=self.get_libraries(ext),
    495             library_dirs=ext.library_dirs,
    496             runtime_library_dirs=ext.runtime_library_dirs,
    497             extra_postargs=extra_args,
    498             export_symbols=self.get_export_symbols(ext),
    499             debug=self.debug,
    500             build_temp=self.build_temp,
    501             target_lang=language)
    502 
    503 
    504 
     136            from threading import Thread, active_count
     137            ambient_nthreads = active_count()
     138            nthreads += ambient_nthreads
     139            threads = (Thread(target=process_ext, args=(ext,)) for ext in self.extensions)
     140            for thread in threads:
     141                while active_count() == nthreads:
     142                    pass
     143                thread.start()
     144            while active_count() > ambient_nthreads:
     145                pass
    505146
    506147#############################################
    507148###### Cythonize
     
    533174
    534175
    535176#########################################################
    536 ### Distutils
     177### Setuptools
    537178#########################################################
    538179
    539 code = setup(name = 'sage',
     180setup(name = 'sage',
    540181
    541182      version     =  SAGE_VERSION,
    542183
     
    718359
    719360                     'sage.tensor'
    720361                     ],
    721       scripts = [],
    722362
    723       cmdclass = { 'build_ext': sage_build_ext },
    724 
     363      cmdclass = { 'build_ext': build_ext },
    725364      ext_modules = ext_modules,
    726365      include_dirs = include_dirs)
    727