Ticket #12016: 12016-scripts.patch

File 12016-scripts.patch, 16.2 KB (added by jdemeyer, 10 years ago)
  • sage-clone

    # HG changeset patch
    # User J. H. Palmieri <palmieri@math.washington.edu>
    # Date 1321120377 28800
    # Node ID b3d3a170aa4e3fc59c4f59d37560f84dbeb31429
    # Parent  924f1b2c3e20b55565be29c1d56888bfde7cfedf
    Add sage-num-threads.py, set SAGE_NUM_THREADS and SAGE_NUM_THREADS_PARALLEL in sage-sage
    Use SAGE_NUM_THREADS_PARALLEL in sage-ptest
    Fix recursive make by checking/changing MAKE and MAKEFLAGS in sage-env and sage-spkg
    
    diff --git a/sage-clone b/sage-clone
    a b  
    8989copy_dtree('sage/doc/output', branch + '/doc/output')
    9090
    9191print "Building " + branch + "..."
    92 try:
    93     if os.environ["SAGE_PBUILD"] == "yes":
    94         print "Pbuild is currently broken -- defaulting to serial build."
    95         # print "Copying clibsage"
    96         # cmd = 'cp -r sage/c_lib/libcsage.so %s/c_lib/'%branch
    97         #
    98         # print cmd
    99         # if os.system(cmd):
    100         #       print "Error copying clibsage"
    101         #       sys.exit(1)
    102         # cmd = 'ln -snf "sage-%s" sage' % sys.argv[1]
    103         # print cmd
    104         # if os.system(cmd):
    105         #     print "Error building SAGE"
    106         # cmd = 'sage -b'
    107         cmd = 'sage -b %s'%sys.argv[1]
    108     else:
    109         cmd = 'sage -b %s'%sys.argv[1]
    110 except:
    111    cmd = 'sage -b %s'%sys.argv[1]
     92cmd = 'sage -b %s'%sys.argv[1]
    11293print cmd
    11394if os.system(cmd):
    11495    print "Error building Sage"
  • sage-env

    diff --git a/sage-env b/sage-env
    a b  
    338338    MV="mv"  && export MV
    339339fi
    340340
    341 if [ "$MAKE" = "" ]; then
    342     MAKE="make"         && export MAKE
    343 fi
    344 
    345341if [ "$RANLIB" = "" ]; then
    346342    RANLIB="ranlib"  && export RANLIB
    347343fi
     
    362358    TOUCH="touch"  && export TOUCH
    363359fi
    364360
    365 if [ "$MAKEFLAGS" != "" ]; then
    366     MFLAGS="$MAKEFLAGS"  && export MFLAGS
    367 fi
    368 
    369361if [ "$UNAME" = "CYGWIN" ]; then
    370362    PATH="$PATH:$SAGE_LOCAL/lib" && export PATH
    371363fi
     
    373365# See trac 7186 -- this is needed if ecl is moved
    374366ECLDIR="$SAGE_LOCAL/lib/ecl/" && export ECLDIR
    375367
     368# Handle parallel building/testing/...
     369# See Trac Ticket #12016
     370# First, figure out the right values for SAGE_NUM_THREADS (default
     371# number of threads) and SAGE_NUM_THREADS_PARALLEL (default number of
     372# threads when parallel execution is asked explicitly).
     373sage_num_threads_array=(`sage-num-threads.py 2>/dev/null || echo 1 2 1`)
     374SAGE_NUM_THREADS=${sage_num_threads_array[0]}
     375SAGE_NUM_THREADS_PARALLEL=${sage_num_threads_array[1]}
     376export SAGE_NUM_THREADS
     377export SAGE_NUM_THREADS_PARALLEL
     378
     379if [ "$MAKE" = "" ]; then
     380    MAKE="make"
     381fi
     382
     383# If MAKEFLAGS exists, assume it got set by make.
     384# Therefore, remove all flags from $MAKE
     385if [ "${MAKEFLAGS-__unset__}" != "__unset__" ]; then
     386    MAKE=`echo "$MAKE" | sed 's/ .*//'`
     387fi
     388
     389export MAKE
     390
     391
    376392# Export variable names.
    377393export SHAREDFLAGS
    378394export UNAME
  • new file sage-num-threads.py

    diff --git a/sage-num-threads.py b/sage-num-threads.py
    new file mode 100755
    - +  
     1#!/usr/bin/env python
     2#
     3# Determine the number of threads to be used by Sage.
     4#
     5# Outputs three space-separated numbers:
     6# 1) The number of threads to use for Sage, based on MAKE, MAKEFLAGS
     7#    and SAGE_NUM_THREADS
     8# 2) The number of threads to use when parallel execution is explicitly
     9#    asked for (e.g. sage -tp)
     10# 3) The number of CPU cores in the system, as determined by
     11#    multiprocessing.cpu_count()
     12#
     13# AUTHOR: Jeroen Demeyer (2011-12-08): Trac ticket #12016
     14#
     15
     16import os
     17import multiprocessing
     18import re
     19import math
     20
     21def number_of_cores():
     22    """
     23    Try to determine the number of CPU cores in this system.
     24    If successful return that number. Otherwise return 1.
     25    """
     26    # If the environment variable SAGE_NUM_CORES exists, use that value.
     27    # This is useful for testing purposes.
     28    try:
     29        n = int(os.environ["SAGE_NUM_CORES"])
     30        if n > 0:
     31            return n
     32    except (ValueError, KeyError):
     33        pass
     34
     35    try:
     36        n = multiprocessing.cpu_count()
     37        if n > 0:
     38            return n
     39    except NotImplementedError:
     40        pass
     41
     42    try:  # Solaris fix
     43        n = int(os.popen2("sysctl -n hw.ncpu")[1].read().strip())
     44        if n > 0:
     45            return n
     46    except ValueError:
     47        pass
     48
     49    return 1
     50
     51def parse_jobs_from_MAKE(MAKE, unlimited=999999):
     52    """
     53    Parse an environment variable like :envvar:`MAKE` for the number of
     54    jobs specified. This looks at arguments ``-j``, ``--jobs``, ``-l``,
     55    ``--load-average``.
     56
     57    INPUT:
     58
     59    - ``MAKE`` -- The value of :envvar:`MAKE` or :envvar:`MAKEFLAGS`.
     60
     61    - ``unlimited`` -- The value to return when ``MAKE`` contains ``-j``
     62      without argument and no ``-l`` option.  Normally this is interpreted
     63      as "unlimited".
     64
     65    OUTPUT:
     66
     67    The number of jobs specified by that variable.  Raise ``KeyError``
     68    if no number of jobs is specified in ``MAKE``.
     69    """
     70    # First, find value of -j
     71    # Since this is doing a greedy match on the left and non-greedy on the right,
     72    # we find the last -j or --jobs
     73    (j,num) = re.subn(r'^(.* )?(-j *|--jobs(=(?=[0-9])| +))([0-9]*)( .*?)?$', r'\4', MAKE, count=1)
     74    if num < 1:
     75        # No replacement done, i.e. no -j option found
     76        raise KeyError, "No number of jobs specified"
     77    elif j == "":
     78        # j is empty: unlimited number of jobs! :-)
     79        j = unlimited
     80    else:
     81        j = int(j)
     82        if j <= 0:
     83            raise ValueError, "Non-positive value specified for -j"
     84
     85    # Next, find the value of -l
     86    # If it is specified, use this as an upper bound on j
     87    (l,num) = re.subn(r'^(.* )?(-l *|--load-average(=(?=[0-9])| +))([0-9.]*)( .*?)?$', r'\4', MAKE, count=1)
     88    if num < 1:
     89        # No replacement done, i.e. no -l option found
     90        pass
     91    elif l == "":
     92        # No load limit specified
     93        pass
     94    else:
     95        l = int(math.ceil(float(l)))
     96        # A load limit will never prevent starting at least one job
     97        if l <= 1:
     98            l = 1
     99        j = min(j,l)
     100
     101    return j
     102
     103def num_threads():
     104    """
     105    Determine the number of threads from the environment variables
     106    (in decreasing priority) :envvar:`SAGE_NUM_THREADS`, :envvar:`MAKE`,
     107    :envvar:`MAKEFLAGS` and :envvar:`MFLAGS`.
     108
     109    If :envvar:`SAGE_NUM_THREADS` is 0 and neither :envvar:`MAKE` nor
     110    :envvar:`MAKEFLAGS` specifies a number of jobs, the use a default
     111    of ``min(8, number_of_cores)``.
     112
     113    OUTPUT:
     114
     115    a tuple (num_threads, num_threads_parallel, num_cores)
     116    """
     117    num_cores = number_of_cores()
     118
     119    num_threads = None
     120    # Handle MFLAGS only for backwards compatibility
     121    try:
     122        num_threads = parse_jobs_from_MAKE(os.environ["MFLAGS"], unlimited=2)
     123    except (ValueError, KeyError):
     124        pass
     125
     126    try:
     127        # Prepend hyphen to MAKEFLAGS if it does not start with one
     128        MAKEFLAGS=os.environ["MAKEFLAGS"]
     129        if MAKEFLAGS[0] != '-':
     130            MAKEFLAGS = '-' + MAKEFLAGS
     131        # In MAKEFLAGS, "-j" does not mean unlimited.  It probably
     132        # means an inherited number of jobs, let us use 2 for safety.
     133        num_threads = parse_jobs_from_MAKE(MAKEFLAGS, unlimited=2)
     134    except (ValueError, KeyError, IndexError):
     135        pass
     136
     137    try:
     138        num_threads = parse_jobs_from_MAKE(os.environ["MAKE"])
     139    except (ValueError, KeyError):
     140        pass
     141
     142    # Number of threads to use when parallel execution is explicitly
     143    # asked for
     144    num_threads_parallel = num_threads
     145    if num_threads_parallel is None:
     146        num_threads_parallel = max(min(8, num_cores), 2)
     147
     148    try:
     149        sage_num_threads = int(os.environ["SAGE_NUM_THREADS"])
     150        if sage_num_threads == 0:
     151            # If SAGE_NUM_THREADS is 0, use the default only
     152            # if none of the above variables specified anything.
     153            if num_threads is None:
     154                num_threads = min(8, num_cores)
     155        elif sage_num_threads > 0:
     156            # SAGE_NUM_THREADS overrides everything
     157            num_threads = sage_num_threads
     158            num_threads_parallel = sage_num_threads
     159    except (ValueError, KeyError):
     160        pass
     161
     162    # If we still don't know, use 1 thread
     163    if num_threads is None:
     164        num_threads = 1
     165
     166    # Finally, use SAGE_NUM_THREADS_PARALLEL if set.  A user isn't
     167    # likely to set this, but it ensures that sage-env is idempotent
     168    # if called more than once.
     169    try:
     170        sage_num_threads = int(os.environ["SAGE_NUM_THREADS_PARALLEL"])
     171        if sage_num_threads > 0:
     172            num_threads_parallel = sage_num_threads
     173    except (ValueError, KeyError):
     174        pass
     175
     176    return (num_threads, num_threads_parallel, num_cores)
     177
     178print "%i %i %i"%num_threads()
  • sage-ptest

    diff --git a/sage-ptest b/sage-ptest
    a b  
    11#!/usr/bin/env python
    22
     3# Usage: sage -tp N <options> <files>
     4#
     5# <options> may include
     6#      --long                include lines with the phrase 'long time'
     7#      --verbose             debugging output during the test
     8#      --optional            also test all #optional examples
     9#      --only-optional <tag1,...,tagn>    only run tests including one
     10#                                         of the #optional tags
     11#      --randorder[=seed]    randomize order of tests
     12#
     13# This runs doctests on <files> in parallel using N threads.  If N is
     14# zero or omitted, set N to the value of the environment variable
     15# SAGE_NUM_THREADS_PARALLEL, which is set in sage-env.
     16
    317from __future__ import with_statement
    418import os
    519import sys
     
    1226import multiprocessing
    1327import socket
    1428import stat
     29import re
     30
     31def usage():
     32    print """Usage: sage -tp N <options> <files>
     33
     34<options> may include
     35     --long                include lines with the phrase 'long time'
     36     --verbose             debugging output during the test
     37     --optional            also test all #optional examples
     38     --only-optional <tag1,...,tagn>    only run tests including one
     39                                        of the #optional tags
     40     --randorder[=seed]    randomize order of tests
     41
     42This runs doctests on <files> in parallel using N threads.  If N is
     43zero or omitted, try to use a sensible default number of threads: if
     44the '-j' flag of the environment variable 'MAKE' or 'MAKEFLAGS' is set,
     45use that setting. Otherwise, use min(8, number of CPU cores)."""
     46
     47if len(sys.argv) == 1:
     48    usage()
     49    exit(1)
    1550
    1651SAGE_ROOT = os.path.realpath(os.environ['SAGE_ROOT'])
    1752SAGE_SITE = os.path.realpath(os.path.join(os.environ['SAGE_LOCAL'],
     
    266301    filemutex.release()
    267302    return 0
    268303
    269 
    270304for gr in range(0,numglobaliteration):
    271305    argv = sys.argv
    272306    opts = ' '.join([X for X in argv if X[0] == '-'])
    273307    argv = [X for X in argv if X[0] != '-']
    274308
    275309    try:
    276         # FIXME: Nice, but <NUMTHREADS> should immediately follow '-tp' etc.,
    277         #        i.e., be the next argument. We might have file or directory
    278         #        names that properly convert to an int...
    279310        numthreads = int(argv[1])
    280311        infiles = argv[2:]
    281     except ValueError: # can't convert first arg to an integer: arg was probably omitted
    282         numthreads = 1
     312    except ValueError:
     313        # can't convert first arg to an integer: arg was probably omitted
     314        numthreads = 0
    283315        infiles = argv[1:]
    284316
    285317    if '-sagenb' in opts:
     
    298330    verbose = ('-verbose' in opts or '--verbose' in opts)
    299331
    300332    if numthreads == 0:
    301         # Set numthreads to be the number of processors, with a default
    302         # maximum of 8.
    303         #
    304         # The detection of number of processors might not be reliable on some
    305         # platforms. On a Sun SPARC T5240 (t2.math), the reported number of
    306         # processors might not correspond to the actual number of processors.
    307         # See tickets #6283, #7011, and the file SAGE_ROOT/Makefile.
    308         #
    309         # WARNING: If cpu_count() below reports <= 8 for your machine
    310         # and you *don't* want to use all the cores/processors on your
    311         # system for parallel doctesting, use a (sensible) positive
    312         # integer.
    313         #
    314         # If cpu_count() reports > 8 and you want to use that many
    315         # threads, you must manually specify the number of threads.
    316333        try:
    317             numthreads = min(8, multiprocessing.cpu_count())
    318         except NotImplementedError:
     334            numthreads = int(os.environ['SAGE_NUM_THREADS_PARALLEL'])
     335        except KeyError:
    319336            numthreads = 1
    320337
    321338    if numthreads < 1 or len(infiles) == 0:
    322339        if numthreads < 1:
    323             print "Usage: sage -tp <numthreads> <files or directories>: <numthreads> must be positive."
     340            print "Usage: sage -tp <numthreads> <files or directories>: <numthreads> must be non-negative."
    324341        else:
    325342            print "Usage: sage -tp <numthreads> <files or directories>: no files or directories specified."
    326         print "For more information, type 'sage -advanced'."
     343        print "For more information, type 'sage --advanced'."
    327344        sys.exit(1)
    328345
    329346    infiles.sort()
  • sage-sage

    diff --git a/sage-sage b/sage-sage
    a b  
    159159    echo "  -tnew [...]         -- like -t above, but only tests files modified since"
    160160    echo "                         last commit"
    161161    echo "  -tp <N> [...]       -- like -t above, but tests in parallel using N threads"
    162     echo "                         with 0 interpreted as minimum(8, cpu_count())"
     162    echo "                         with 0 interpreted as a sensible default"
    163163    echo "  -testall [options]  -- test all source files, docs, and examples.  options"
    164164    echo "                         like -t"
    165165
     
    593593#####################################################################
    594594
    595595build_sage() {
    596     if [ "$SAGE_PBUILD" == "yes" ]; then
    597         echo >&2 'Pbuild is currently broken -- defaulting to serial build.'
    598         # if [ "$@" ]; then
    599         #     ln -snf "$SAGE_ROOT"/devel/sage-"$@" "$SAGE_ROOT"/devel/sage
    600         # fi
    601         # time python "$SAGE_ROOT"/devel/sage/build.py -b
    602         sage-build "$@"
    603     else
    604         sage-build "$@"
    605     fi
     596    sage-build "$@" || exit $?
    606597}
    607598
    608599if [ "$1" = "-notebook"  -o "$1" = '--notebook' -o "$1" = '-n' ]; then
     
    617608    cd "$CUR"
    618609    shift
    619610    sage-cleaner &>/dev/null &
    620     build_sage && sage-notebook "$@"
     611    build_sage
     612    sage-notebook "$@"
    621613    exit $?
    622614fi
    623615
     
    671663
    672664if [ "$1" = '-br' -o "$1" = "--br" ]; then
    673665    shift
    674     build_sage "$@" && sage
     666    build_sage "$@"
     667    sage
    675668    exit $?
    676669fi
    677670
     
    736729
    737730if [ "$1" = '-t' -o "$1" = '-bt' ]; then
    738731   if [ "$1" = '-bt' ]; then
    739       build_sage || exit $?
     732      build_sage
    740733   fi
    741734   if ! [  -f  "$DOT_SAGE"/init.sage ]; then
    742735      echo >&2 "init.sage does not exist ... creating"
     
    750743
    751744if [ "$1" = '-tp' -o "$1" = '-btp' ]; then
    752745   if [ "$1" = '-btp' ]; then
    753       build_sage || exit $?
     746      build_sage
    754747   fi
    755748   if ! [  -f  "$DOT_SAGE"/init.sage ]; then
    756749      echo >&2 "init.sage does not exist ... creating"
     
    764757
    765758if [ "$1" = '-tnew' -o "$1" = '-btnew' ]; then
    766759   if [ "$1" = '-btnew' ]; then
    767       build_sage || exit $?
     760      build_sage
    768761   fi
    769762   cd "$CUR"
    770763   shift
  • sage-spkg

    diff --git a/sage-spkg b/sage-spkg
    a b  
    6868MESSAGE
    6969}
    7070
    71 # The following sets environment variables for building
    72 # packages.  (Using dot suggested by W. Cheung.)
     71# Handle -n, -t, -q options for recursive make
     72# See Trac #12016.
     73if echo "$MAKE $MAKEFLAGS -$MAKEFLAGS" |grep '[ ]-[A-Za-z]*[qnt]' >/dev/null; then
     74    if echo "$MAKE $MAKEFLAGS -$MAKEFLAGS" |grep '[ ]-[A-Za-z]*q' >/dev/null; then
     75        # Pretend the target is *not* up-to-date
     76        exit 1
     77    else
     78        exit 0
     79    fi
     80fi
    7381
    74 . "$SAGE_ROOT/local/bin/sage-env" # *returns* non-zero value on errors rather than exiting
     82# The following sets environment variables for building packages.
     83# Since this is sourced, it returns a non-zero value on errors rather
     84# than exiting.  Using dot suggested by W. Cheung.
     85. "$SAGE_ROOT/local/bin/sage-env"
    7586 
    7687if [ $? -ne 0 ]; then
    7788    echo "Error setting environment variables by running \"$SAGE_ROOT/local/bin/sage-env\";"
     
    329340
    330341chmod +x spkg-install
    331342
    332 # this is just wrong... (so don't do it)
    333 #echo "TOUCHING"
    334 #touch * */* */*/* */*/*/* 1>/dev/null 2>/dev/null
    335 
    336343echo "****************************************************"
    337344echo "Host system"
    338345echo "uname -a:"