# HG changeset patch
# User J. H. Palmieri <palmieri@math.washington.edu>
# Date 1321120377 28800
# Node ID 14ef3f6422ae3c77d4dba416e50751798cf10ae4
# Parent 4546700d8e80ec452d7d189732e49e9350a88764
#12016: make sage-ptest use the -j flag from MAKE to determine
the number of threads for doctesting. Also add a usage message.
diff --git a/sage-clone b/sage-clone
a
|
b
|
print "Copying over documentation output |
89 | 89 | copy_dtree('sage/doc/output', branch + '/doc/output') |
90 | 90 | |
91 | 91 | print "Building " + branch + "..." |
92 | | try: |
93 | | if os.environ["SAGE_PBUILD"] == "yes": |
94 | | print "Pbuild is currently broken -- defaulting to serial build." |
95 | | # print "Copying clibsage" |
96 | | # cmd = 'cp -r sage/c_lib/libcsage.so %s/c_lib/'%branch |
97 | | # |
98 | | # print cmd |
99 | | # if os.system(cmd): |
100 | | # print "Error copying clibsage" |
101 | | # sys.exit(1) |
102 | | # cmd = 'ln -snf "sage-%s" sage' % sys.argv[1] |
103 | | # print cmd |
104 | | # if os.system(cmd): |
105 | | # print "Error building SAGE" |
106 | | # cmd = 'sage -b' |
107 | | cmd = 'sage -b %s'%sys.argv[1] |
108 | | else: |
109 | | cmd = 'sage -b %s'%sys.argv[1] |
110 | | except: |
111 | | cmd = 'sage -b %s'%sys.argv[1] |
| 92 | cmd = 'sage -b %s'%sys.argv[1] |
112 | 93 | print cmd |
113 | 94 | if os.system(cmd): |
114 | 95 | print "Error building Sage" |
diff --git a/sage-env b/sage-env
a
|
b
|
if [ "$MAKE" = "" ]; then |
339 | 339 | MAKE="make" && export MAKE |
340 | 340 | fi |
341 | 341 | |
| 342 | # Determine the number of threads to use when building in parallel. |
| 343 | # If SAGE_NUM_THREADS is set, leave it alone. Otherwise, if the -j |
| 344 | # flag is set in $MAKE, then use its value; if empty (meaning "run as |
| 345 | # many recipes simultaneously as possible"), then use "0". If it is |
| 346 | # not set, then use "auto". Programs that use this variable should |
| 347 | # interpret 0 and 'auto' appropriately. |
| 348 | if [ -z "$SAGE_NUM_THREADS" ]; then |
| 349 | echo "$MAKE" | grep -q -e "-j" |
| 350 | if [ $? -gt 0 ]; then |
| 351 | # No -j flag: |
| 352 | SAGE_NUM_THREADS='auto' |
| 353 | else |
| 354 | # Look for -j or --jobs, followed by space, tab, and/or '=', |
| 355 | # followed by number. Use the number. |
| 356 | SAGE_NUM_THREADS=`echo "$MAKE" | sed -n -e 's/make[ ]//' -e 's/.*\(-j[obs]*[= ]*\)\([0-9][0-9]*\).*/\2/gp'` |
| 357 | # If no number, then it was 'make -j', so use 0. |
| 358 | if [ -z "$SAGE_NUM_THREADS" ]; then |
| 359 | SAGE_NUM_THREADS='0' |
| 360 | fi |
| 361 | fi |
| 362 | export SAGE_NUM_THREADS |
| 363 | fi |
| 364 | |
342 | 365 | if [ "$RANLIB" = "" ]; then |
343 | 366 | RANLIB="ranlib" && export RANLIB |
344 | 367 | fi |
diff --git a/sage-ptest b/sage-ptest
a
|
b
|
|
1 | 1 | #!/usr/bin/env python |
2 | 2 | |
| 3 | # Usage: sage -tp N <options> <files> |
| 4 | # |
| 5 | # <options> may include |
| 6 | # --long include lines with the phrase 'long time' |
| 7 | # --verbose debugging output during the test |
| 8 | # --optional also test all #optional examples |
| 9 | # --only-optional <tag1,...,tagn> only run tests including one |
| 10 | # of the #optional tags |
| 11 | # --randorder[=seed] randomize order of tests |
| 12 | # |
| 13 | # This runs doctests on <files> in parallel using N threads. If N is |
| 14 | # zero or omitted, set N to the value of the environment variable |
| 15 | # SAGE_NUM_THREADS, which is set in sage-env. If SAGE_NUM_THREADS is |
| 16 | # zero or 'auto', use the maximum of (8, # of cpus), where the latter |
| 17 | # is determined by multiprocessing.cpu_count(). |
| 18 | # |
| 19 | # Note: as discussed on tickets #6283 and #7011, the detection of the |
| 20 | # number of processors might not be reliable on some platforms. On a |
| 21 | # Sun SPARC T5240 (t2.math), the reported number of processors does |
| 22 | # not seem to correspond to the actual number of processors. |
| 23 | |
3 | 24 | from __future__ import with_statement |
4 | 25 | import os |
5 | 26 | import sys |
… |
… |
import subprocess |
12 | 33 | import multiprocessing |
13 | 34 | import socket |
14 | 35 | import stat |
| 36 | import re |
| 37 | |
| 38 | def usage(): |
| 39 | print """Usage: sage -tp N <options> <files> |
| 40 | |
| 41 | <options> may include |
| 42 | --long include lines with the phrase 'long time' |
| 43 | --verbose debugging output during the test |
| 44 | --optional also test all #optional examples |
| 45 | --only-optional <tag1,...,tagn> only run tests including one |
| 46 | of the #optional tags |
| 47 | --randorder[=seed] randomize order of tests |
| 48 | |
| 49 | This runs doctests on <files> in parallel using N threads. If N is |
| 50 | zero or omitted, try to use a sensible default number of threads: if |
| 51 | the '-j' flag of the variable 'MAKE' is set, use that setting. |
| 52 | Otherwise, use min(8, #CPUs).""" |
| 53 | |
| 54 | if len(sys.argv) == 1: |
| 55 | usage() |
| 56 | exit(1) |
15 | 57 | |
16 | 58 | SAGE_ROOT = os.path.realpath(os.environ['SAGE_ROOT']) |
17 | 59 | SAGE_SITE = os.path.realpath(os.path.join(os.environ['SAGE_LOCAL'], |
… |
… |
def populatefilelist(filelist): |
266 | 308 | filemutex.release() |
267 | 309 | return 0 |
268 | 310 | |
269 | | |
270 | 311 | for gr in range(0,numglobaliteration): |
271 | 312 | argv = sys.argv |
272 | 313 | opts = ' '.join([X for X in argv if X[0] == '-']) |
273 | 314 | argv = [X for X in argv if X[0] != '-'] |
274 | 315 | |
275 | 316 | try: |
276 | | # FIXME: Nice, but <NUMTHREADS> should immediately follow '-tp' etc., |
277 | | # i.e., be the next argument. We might have file or directory |
278 | | # names that properly convert to an int... |
279 | 317 | numthreads = int(argv[1]) |
280 | 318 | infiles = argv[2:] |
281 | | except ValueError: # can't convert first arg to an integer: arg was probably omitted |
282 | | numthreads = 1 |
| 319 | except ValueError: |
| 320 | # can't convert first arg to an integer: arg was probably omitted |
| 321 | numthreads = 0 |
283 | 322 | infiles = argv[1:] |
284 | 323 | |
285 | 324 | if '-sagenb' in opts: |
… |
… |
for gr in range(0,numglobaliteration): |
298 | 337 | verbose = ('-verbose' in opts or '--verbose' in opts) |
299 | 338 | |
300 | 339 | if numthreads == 0: |
301 | | # Set numthreads to be the number of processors, with a default |
302 | | # maximum of 8. |
303 | | # |
304 | | # The detection of number of processors might not be reliable on some |
305 | | # platforms. On a Sun SPARC T5240 (t2.math), the reported number of |
306 | | # processors might not correspond to the actual number of processors. |
307 | | # See tickets #6283, #7011, and the file SAGE_ROOT/Makefile. |
308 | | # |
309 | | # WARNING: If cpu_count() below reports <= 8 for your machine |
310 | | # and you *don't* want to use all the cores/processors on your |
311 | | # system for parallel doctesting, use a (sensible) positive |
312 | | # integer. |
313 | | # |
314 | | # If cpu_count() reports > 8 and you want to use that many |
315 | | # threads, you must manually specify the number of threads. |
| 340 | SAGE_NUM_THREADS = os.environ.get('SAGE_NUM_THREADS', 'auto') |
316 | 341 | try: |
317 | | numthreads = min(8, multiprocessing.cpu_count()) |
318 | | except NotImplementedError: |
319 | | numthreads = 1 |
| 342 | numthreads = int(SAGE_NUM_THREADS) |
| 343 | except ValueError: |
| 344 | # SAGE_NUM_THREADS = 'auto' |
| 345 | SAGE_NUM_THREADS == 0 |
| 346 | if SAGE_NUM_THREADS == 0: |
| 347 | # Set numthreads to be the number of processors, with a |
| 348 | # default maximum of 8. |
| 349 | try: |
| 350 | numthreads = min(8, multiprocessing.cpu_count()) |
| 351 | except NotImplementedError: |
| 352 | numthreads = 1 |
320 | 353 | |
321 | 354 | if numthreads < 1 or len(infiles) == 0: |
322 | 355 | if numthreads < 1: |
323 | | print "Usage: sage -tp <numthreads> <files or directories>: <numthreads> must be positive." |
| 356 | print "Usage: sage -tp <numthreads> <files or directories>: <numthreads> must be non-negative." |
324 | 357 | else: |
325 | 358 | print "Usage: sage -tp <numthreads> <files or directories>: no files or directories specified." |
326 | 359 | print "For more information, type 'sage -advanced'." |
diff --git a/sage-sage b/sage-sage
a
|
b
|
usage_advanced() { |
159 | 159 | echo " -tnew [...] -- like -t above, but only tests files modified since" |
160 | 160 | echo " last commit" |
161 | 161 | echo " -tp <N> [...] -- like -t above, but tests in parallel using N threads" |
162 | | echo " with 0 interpreted as minimum(8, cpu_count())" |
| 162 | echo " with 0 interpreted as a sensible default" |
163 | 163 | echo " -testall [options] -- test all source files, docs, and examples. options" |
164 | 164 | echo " like -t" |
165 | 165 | |
… |
… |
fi |
588 | 588 | ##################################################################### |
589 | 589 | |
590 | 590 | build_sage() { |
591 | | if [ "$SAGE_PBUILD" == "yes" ]; then |
592 | | echo >&2 'Pbuild is currently broken -- defaulting to serial build.' |
593 | | # if [ "$@" ]; then |
594 | | # ln -snf "$SAGE_ROOT"/devel/sage-"$@" "$SAGE_ROOT"/devel/sage |
595 | | # fi |
596 | | # time python "$SAGE_ROOT"/devel/sage/build.py -b |
597 | | sage-build "$@" |
598 | | else |
599 | | sage-build "$@" |
600 | | fi |
| 591 | sage-build "$@" |
601 | 592 | } |
602 | 593 | |
603 | 594 | if [ "$1" = "-notebook" -o "$1" = '--notebook' -o "$1" = '-n' ]; then |
… |
… |
if [ "$1" = "-bn" -o "$1" = "--build-and |
612 | 603 | cd "$CUR" |
613 | 604 | shift |
614 | 605 | sage-cleaner &>/dev/null & |
615 | | build_sage && sage-notebook "$@" |
| 606 | build_sage |
| 607 | sage-notebook "$@" |
616 | 608 | exit $? |
617 | 609 | fi |
618 | 610 | |
… |
… |
fi |
666 | 658 | |
667 | 659 | if [ "$1" = '-br' -o "$1" = "--br" ]; then |
668 | 660 | shift |
669 | | build_sage "$@" && sage |
| 661 | build_sage "$@" |
| 662 | sage |
670 | 663 | exit $? |
671 | 664 | fi |
672 | 665 | |
… |
… |
fi |
731 | 724 | |
732 | 725 | if [ "$1" = '-t' -o "$1" = '-bt' ]; then |
733 | 726 | if [ "$1" = '-bt' ]; then |
734 | | build_sage || exit $? |
| 727 | build_sage |
735 | 728 | fi |
736 | 729 | if ! [ -f "$DOT_SAGE"/init.sage ]; then |
737 | 730 | echo >&2 "init.sage does not exist ... creating" |
… |
… |
fi |
745 | 738 | |
746 | 739 | if [ "$1" = '-tp' -o "$1" = '-btp' ]; then |
747 | 740 | if [ "$1" = '-btp' ]; then |
748 | | build_sage || exit $? |
| 741 | build_sage |
749 | 742 | fi |
750 | 743 | if ! [ -f "$DOT_SAGE"/init.sage ]; then |
751 | 744 | echo >&2 "init.sage does not exist ... creating" |
… |
… |
fi |
759 | 752 | |
760 | 753 | if [ "$1" = '-tnew' -o "$1" = '-btnew' ]; then |
761 | 754 | if [ "$1" = '-btnew' ]; then |
762 | | build_sage || exit $? |
| 755 | build_sage |
763 | 756 | fi |
764 | 757 | cd "$CUR" |
765 | 758 | shift |