Ticket #9739: trac_9739-delta.patch

File trac_9739-delta.patch, 11.8 KB (added by jhpalmieri, 9 years ago)
  • sage-doctest

    diff --git a/sage-doctest b/sage-doctest
    a b if 'SAGE_TESTDIR' not in os.environ or o 
    7777else:
    7878    SAGE_TESTDIR = os.environ['SAGE_TESTDIR']
    7979
     80def filename_mangler(name):
     81    """
     82    Replace ``name`` with 'doctest_PID_CWD-name.py', where PID is the
     83    process ID and CWD is the current working directory, which should
     84    be the directory containing ``name``, but with all slashes
     85    replaced by periods (to avoid name clashes between files from
     86    different directories with the same name).
     87    """
     88    new = os.path.join(os.getcwd(), name).replace(os.path.sep, '.')
     89    new = new.lstrip('.') # remove leading '.'
     90    return os.path.join(SAGE_TESTDIR,
     91                        "doctest_%s_%s" % (os.getpid(), new))
     92
    8093tmpfiles = [] # list of temporary files to be deleted if doctesting succeeds
    8194
    8295def delete_tmpfiles():
    def change_warning_output(file): 
    439452            s += "cython(open('%s').read())\n\n" % file_name
    440453
    441454        elif ext in ['.py', '.sage']:
    442             root_base = os.path.basename(root_name)
    443             _, temp_name = tempfile.mkstemp(
    444                 prefix='%s_' % root_base, suffix='.py', dir=SAGE_TESTDIR)
    445             tmpfiles.append(temp_name)
     455
     456            target_name = "%s_%d" % (file_name, os.getpid()) # like 'name', but unique
     457            target_base = os.path.join(SAGE_TESTDIR, target_name) # like 'base', also unique
    446458
    447459            if ext == '.sage':
    448                 # TODO: To avoid (a) possible permission problems and
    449                 # (b) races when testing the same file multiple times
    450                 # in parallel, preparse with a Sage library call
     460                # TODO: preparse "<file>.sage" with a Sage library call
    451461                # instead and write a string into temp_name.
    452                 os.system('sage -preparse %s' % file_name)
    453                 os.system('mv -f %s.py %s' % (root_name, temp_name))
     462
     463                # For now: "sage -preparse <file>.sage" doesn't have any
     464                # output options and always creates <file>.py in the same
     465                # directory, so we first copy the *source* into SAGE_TESTDIR:
     466                os.system("cp '%s' %s.sage" % (file_name, target_base))
     467                # Now create SAGE_TESTDIR/<target_name>.py:
     468                os.system("sage -preparse %s.sage" % target_base)
     469                tmpfiles.append(target_base + ".sage")
    454470            else:
    455                 os.system('cp -f %s %s' % (file_name, temp_name))
     471                # TODO: instead of copying the file, add its source
     472                # directory to PYTHONPATH.  We would also have to
     473                # import from 'name' instead of 'target_name'.
     474                os.system("cp '%s' %s.py" % (source, target_base))
    456475
    457             import_name = os.path.splitext(os.path.basename(temp_name))[0]
    458             s += "from %s import *\n\n" % import_name
    459             tmpfiles.append(temp_name + 'c')
     476            s += "from %s import *\n\n" % target_name
     477
     478            tmpfiles.append(target_base + ".py") # preparsed or copied original
     479            tmpfiles.append(target_base + ".pyc") # compiled version of it
    460480
    461481    # Prefix/suffix for all doctests replacing the starting/ending """
    462482    doc_prefix = 'r""">>> set_random_seed(0L)\n\n>>> change_warning_output(sys.stdout)\n\n'
    def test_file(file, library_code): 
    632652    if os.path.exists(file):
    633653        s = extract_doc(file, library_code=library_code)
    634654        if len(s) == 0:
     655            delete_tmpfiles()
    635656            sys.exit(0)
    636657
    637658        name = os.path.basename(file)
    638659        name = name[:name.find(".")]
    639         _, f = tempfile.mkstemp(prefix='.doctest_%s_' % name, suffix='.py',
    640                                 dir=SAGE_TESTDIR)
     660        f = os.path.splitext(filename_mangler(file))[0] + '.py'
     661
    641662        open(f,"w").write(s)
    642663        tmpfiles.append(f)
    643664
    def test_file(file, library_code): 
    688709                out = outf.read()
    689710                e = proc.poll()
    690711        except KeyboardInterrupt:
     712            # TODO: if tests were interrupted but there were no failures, delete tmpfiles.
    691713            print "KeyboardInterrupt -- interrupted after %.1f seconds!" % (time.time()-tm)
    692714            sys.exit(2)
    693715        if 'raise KeyboardInterrupt' in err:
     716            # TODO: if tests were interrupted but there were no failures, delete tmpfiles.
    694717            print "*"*80 + "Control-C pressed -- interrupting doctests." + "*"*80
    695718            sys.exit(2)
    696719
  • sage-env

    diff --git a/sage-env b/sage-env
    a b if [ -z "${SAGE_ORIG_LD_LIBRARY_PATH_SET 
    191191    SAGE_ORIG_LD_LIBRARY_PATH_SET=True && export SAGE_ORIG_LD_LIBRARY_PATH_SET
    192192fi
    193193
     194# Should this be a temporary directory instead?  Since sage-ptest and
     195# sage-test are written in Python, we can set it to
     196# tempfile.gettempdir() there.
    194197if [ -z "$SAGE_TESTDIR" ]; then
    195198    SAGE_TESTDIR="$DOT_SAGE"/tmp && export SAGE_TESTDIR
    196199fi
  • sage-ptest

    diff --git a/sage-ptest b/sage-ptest
    a b import thread 
    1010import tempfile
    1111import subprocess
    1212import multiprocessing
     13import socket
     14import stat
    1315
    1416SAGE_ROOT = os.path.realpath(os.environ['SAGE_ROOT'])
    1517SAGE_SITE = os.path.realpath(os.path.join(os.environ['SAGE_LOCAL'],
    def skip(F): 
    8284    Returns true if the file should not be tested
    8385    """
    8486    if not os.path.exists(F):
     87        # XXX IMHO this should never happen; in case it does, it's certainly
     88        #     an error to be reported (either filesystem, or bad name specified
     89        #     on the command line). -leif
    8590        return True
    8691    G = abspath(F)
    8792    i = G.rfind(os.path.sep)
     93    # XXX The following should IMHO be performed in populatefilelist():
     94    #     (Currently, populatefilelist() only looks for "__nodoctest__".)
    8895    if os.path.exists(os.path.join(G[:i], 'nodoctest.py')):
    8996        printmutex.acquire()
    9097        print "%s (skipping) -- nodoctest.py file in directory" % test_cmd(F)
     98        sys.stdout.flush()
    9199        printmutex.release()
    92100        return True
    93101    filenm = os.path.split(F)[1]
    def skip(F): 
    96104        return True
    97105    if G.find(os.path.join('doc', 'output')) != -1:
    98106        return True
     107    # XXX The following is (also/already) handled in populatefilelist():
    99108    if not (os.path.splitext(F)[1] in ['.py', '.pyx', '.spyx', '.tex', '.pxi', '.sage', '.rst']):
    100109        return True
    101110    return False
    def test_file(F): 
    116125    for i in range(0,numiteration):
    117126        os.chdir(os.path.dirname(F))
    118127        command = os.path.join(SAGE_ROOT, 'local', 'bin', 'sage-%s' % cmd)
     128        # FIXME: Why call bash here? (Also, we use 'shell=True' below anyway.)
    119129        s = 'bash -c "%s %s > %s" ' % (command, filestr, outfile.name)
    120130        try:
    121131            t = time.time()
    def process_result(result): 
    160170            failed.append(test_cmd(F))
    161171
    162172    print test_cmd(F)
     173    sys.stdout.flush()
    163174
    164175    if ol!="" and (not ol.isspace()):
    165176        if (ol[len(ol)-1]=="\n"):
    166177            ol=ol[0:len(ol)-1]
    167178        print ol
     179        sys.stdout.flush()
    168180    time_dict[abs_sage_path(F)] = finished_time
    169181    if XML_RESULTS:
    170182        t = finished_time
    def process_result(result): 
    192204        """.strip() % locals())
    193205        f.close()
    194206    print "\t [%.1f s]"%(finished_time)
     207    sys.stdout.flush()
    195208
    196209def infiles_cmp(a,b):
    197210    """
    def populatefilelist(filelist): 
    232245                if not (ext in ['.sage', '.py', '.pyx', '.spyx', '.tex', '.pxi', '.rst']):
    233246                    continue
    234247                elif '__nodoctest__' in files:
     248                    # XXX Shouldn't this be 'lfiles'?
     249                    # Also, this test should IMHO be in the outer loop (1 level).
     250                    # Furthermore, the current practice is to put "nodoctest.py"
     251                    # files in the directories that should be skipped, not
     252                    # "__nodoctest__". (I haven't found a single instance of the
     253                    # latter in Sage 4.6.1.alpha3.)
     254                    # "nodoctest.py" is handled in skip() (!), to also be fixed.
     255                    # -leif
    235256                    continue
    236257                appendstr = os.path.join(root,F)
    237258                if skip(appendstr):
    for gr in range(0,numglobaliteration): 
    251272    opts = ' '.join([X for X in argv if X[0] == '-'])
    252273    argv = [X for X in argv if X[0] != '-']
    253274
    254     try:
     275    try:
     276        # FIXME: Nice, but <NUMTHREADS> should immediately follow '-tp' etc.,
     277        #        i.e., be the next argument. We might have file or directory
     278        #        names that properly convert to an int...
    255279        numthreads = int(argv[1])
    256280        infiles = argv[2:]
    257281    except ValueError: # can't convert first arg to an integer: arg was probably omitted
    for gr in range(0,numglobaliteration): 
    307331    t0 = time.time()
    308332    filemutex = thread.allocate_lock()
    309333    printmutex = thread.allocate_lock()
     334    SAGE_TESTDIR = os.environ['SAGE_TESTDIR']
    310335    #Pick a filename for the timing files -- long vs normal
     336    # TODO: perhaps these files shouldn't be hidden?  Also, don't
     337    # store them in SAGE_TESTDIR, in case the user wants to test in
     338    # some temporary directory: store them somewhere more permanent.
    311339    if opts.count("-long"):
    312         time_file_name = os.path.join(os.environ["SAGE_TESTDIR"],
     340        time_file_name = os.path.join(SAGE_TESTDIR,
    313341                                      '.ptest_timing_long')
    314342    else:
    315         time_file_name = os.path.join(os.environ["SAGE_TESTDIR"],
     343        time_file_name = os.path.join(SAGE_TESTDIR,
    316344                                      '.ptest_timing')
    317345    time_dict = { }
    318346    try:
    for gr in range(0,numglobaliteration): 
    337365
    338366    failed = []
    339367
    340     TMP = os.path.join(os.environ['SAGE_TESTDIR'], 'tmp', 'test')
    341     if not os.path.exists(TMP):
    342         os.makedirs(TMP)
     368    HOSTNAME = socket.gethostname().replace('-','_').replace('/','_').replace('\\','_')
     369    # Should TMP be a subdirectory of tempfile.gettempdir() rather than SAGE_TESTDIR?
     370    TMP = os.path.join(SAGE_TESTDIR, '%s-%s' % (HOSTNAME, os.getpid()))
     371    TMP = os.path.abspath(TMP)
     372    if os.path.exists(TMP):
     373        shutil.rmtree(TMP)
     374    os.makedirs(TMP)
     375    # Add rwx permissions for user to TMP:
     376    os.chmod(TMP, os.stat(TMP)[0] | stat.S_IRWXU)
     377    os.environ['SAGE_TESTDIR'] = TMP
     378    print
     379    print "Using the directory"
     380    print "   '%s'." % TMP
     381    print "for doctesting.  If all doctests pass, this directory will"
     382    print "be deleted automatically."
     383    print
    343384
    344385    populatefilelist(infiles)
    345386    #Sort the files by test time
    for gr in range(0,numglobaliteration): 
    374415
    375416    os.chdir(CUR)
    376417   
     418    print
     419    print "Removing the directory '%s'." % TMP
     420    try:
     421        os.rmdir(TMP)
     422    except OSError:
     423        # TODO (probably in sage-doctest): if tests were interrupted
     424        # but there were no failures in the interrupted files, delete
     425        # the temporary files, so that this directory is empty.
     426        print "Warning: the directory was not removed: it is not empty,"
     427        print "probably because doctesting failed or was interrupted."
     428
     429    print
     430    print "-"*int(70)
    377431    if len(failed) == 0:
    378432        if interrupt == False:
    379433            print "All tests passed!"
    for gr in range(0,numglobaliteration): 
    403457        for i in range(len(failed)):
    404458               print "\t", failed[i]
    405459        print "-"*int(70)
     460
    406461    #Only update timings if we are doing something standard
    407462    opts = opts.strip()
    408463    if (opts=="-long" or len(opts)==0) and not interrupt:
  • sage-test

    diff --git a/sage-test b/sage-test
    a b except KeyError: 
    2828    XML_RESULTS = None
    2929
    3030
     31# TODO: should we set SAGE_TESTDIR to a temporary directory like
     32# tempfile.gettempdir()?
    3133if 'SAGE_TESTDIR' not in os.environ:
    3234    os.environ['SAGE_TESTDIR'] = os.path.join(SAGE_ROOT, "tmp")
    3335TMP = os.path.join(os.environ['SAGE_TESTDIR'], "tmp")