Ticket #9739: trac_9739-unique_doctest_names.patch

File trac_9739-unique_doctest_names.patch, 8.8 KB (added by mpatel, 10 years ago)

Doctest with unique temporary files. Apply to scripts repo.

  • sage-doctest

    # HG changeset patch
    # User Mitesh Patel <qed777@gmail.com>
    # Date 1283582692 25200
    # Node ID 606dc76428c40ee24575878c089fe4c043ce06f6
    # Parent  ba92f289904068bf050d5a0385ec082f801a1626
    #9739: Run doctests with unique temporary files, other small improvements
    
    Other improvements:
    
     * Make 'sage -tp' test .spyx files, too.
    
     * Better handling of when to pickle timings.
    
     * Use a better test command string in 'sage -tp'.  This also makes it
       more likely that we'll pickle reusable timings.
    
    diff --git a/sage-doctest b/sage-doctest
    a b def extract_doc(file_name, library_code= 
    409409   
    410410    F = F.replace('\'"""\'','')
    411411
    412     base, ext = os.path.splitext(file_name)
    413     name = os.path.basename(base)
     412    root_name, ext = os.path.splitext(file_name)
    414413    if ext == ".tex":
    415414        F = pythonify_tex(F)
    416415    elif ext == ".rst":
    def change_warning_output(file): 
    436435"""
    437436
    438437    if not library_code:
    439         if ext in ['.py', '.pyx','.spyx']:
    440             os.system('cp -f %s %s' % (file_name, SAGE_TESTDIR))
    441             tmpfiles.append(os.path.join(SAGE_TESTDIR, '%s%s' % (name, ext)))
    442             if ext == '.py':
    443                 s += "from %s import *\n\n" % name
     438        if ext in ['.pyx','.spyx']:
     439            s += "cython(open('%s').read())\n\n" % file_name
     440
     441        elif ext in ['.py', '.sage']:
     442            root_base = os.path.basename(root_name)
     443            _, temp_name = tempfile.mkstemp(
     444                prefix='%s_' % root_base, suffix='.py', dir=SAGE_TESTDIR)
     445            tmpfiles.append(temp_name)
     446
     447            if ext == '.sage':
     448                # TODO: To avoid (a) possible permission problems and
     449                # (b) races when testing the same file multiple times
     450                # in parallel, preparse with a Sage library call
     451                # instead and write a string into temp_name.
     452                os.system('sage -preparse %s' % file_name)
     453                os.system('mv -f %s.py %s' % (root_name, temp_name))
    444454            else:
    445                 s += "cython(open('%s').read())\n\n" % file_name
    446         elif ext == '.sage':
    447             os.system('sage -preparse %s' % file_name)
    448             os.system('mv -f %s.py %s' % (base, SAGE_TESTDIR))
    449             tmpfiles.append(os.path.join(SAGE_TESTDIR, name + '.py'))
    450             s += "from %s import *\n\n" % (name)
    451         if ext in ['.py', '.sage']:
    452             tmpfiles.append(os.path.join(SAGE_TESTDIR, name + '.pyc'))
     455                os.system('cp -f %s %s' % (file_name, temp_name))
     456
     457            import_name = os.path.splitext(os.path.basename(temp_name))[0]
     458            s += "from %s import *\n\n" % import_name
     459            tmpfiles.append(temp_name + 'c')
    453460
    454461    n = 0
    455462    while True:
    456463        i = F.find('"""')
    457464        if i == -1: break
    458         name = "example"       
    459465        k = F[i+3:].find('"""')
    460466        if k == -1: break
    461467        j = i+3 + k
    def change_warning_output(file): 
    469475                n_str = pad_zeros(new_index(n),10)
    470476            else:
    471477                n_str = str(n)
    472             s += "def %s_%s():"%(name,n_str)
     478            s += "def example_%s():" % n_str
    473479            n += 1
    474480            s += "\tr"+ doc + "\n\n"
    475481        F = F[j+3:]
    def post_process(s, file, tmpname): 
    617623
    618624def test_file(file, library_code):
    619625    if os.path.exists(file):
    620         name = os.path.basename(file)
    621         name = name[:name.find(".")]
    622626        s = extract_doc(file, library_code=library_code)
    623627        if len(s) == 0:
    624628            sys.exit(0)
    625629
    626         f = os.path.join(SAGE_TESTDIR, ".doctest_%s.py" % name)
    627 
     630        name = os.path.basename(file)
     631        name = name[:name.find(".")]
     632        _, f = tempfile.mkstemp(prefix='.doctest_%s_' % name, suffix='.py',
     633                                dir=SAGE_TESTDIR)
    628634        open(f,"w").write(s)
    629635        tmpfiles.append(f)
     636
    630637        cmd = "%s %s"%(PYTHON, f)
    631638        if gdb:
    632639            print "*"*80
  • sage-ptest

    diff --git a/sage-ptest b/sage-ptest
    a b import tempfile 
    1111import subprocess
    1212import multiprocessing
    1313
    14 SAGE_ROOT = os.environ['SAGE_ROOT']
     14SAGE_ROOT = os.path.realpath(os.environ['SAGE_ROOT'])
    1515SAGE_SITE = os.path.realpath(os.path.join(os.environ['SAGE_LOCAL'],
    1616                                          'lib', 'python', 'site-packages'))
    1717BUILD_DIR = os.path.realpath(os.path.join(SAGE_ROOT, 'devel', 'sage', 'build'))
    print 'File iterations: ' + str(numitera 
    2424# Exit status for the whole run.
    2525err = 0
    2626
    27 def abspath(x):
    28     """
    29     This function returns the absolute path (adjusted for NFS)
    30     """
    31     return strip_automount_prefix(os.path.abspath(x))
    32 
    3327def strip_automount_prefix(filename):
    3428    """
    3529    Strip prefixes added on automounted filesystems in some cases,
    def strip_automount_prefix(filename): 
    5145            filename = new
    5246    return filename
    5347
    54 def abs(f):
     48def abspath(x):
     49    """
     50    This function returns the absolute path (adjusted for NFS)
     51    """
     52    return strip_automount_prefix(os.path.abspath(x))
     53
     54def abs_sage_path(f):
     55    """
     56    Return the absolute path, relative to the sage root or current directory
     57    """
     58    global CUR
     59
     60    abs_path = abspath(f)
     61    if abs_path.startswith(SAGE_ROOT):
     62        abs_path = abs_path[len(SAGE_ROOT) + 1:]
     63    elif abs_path.startswith(CUR):
     64        abs_path = abs_path[len(CUR) + 1:]
     65
     66    return abs_path
     67
     68def test_cmd(f):
    5569    """
    5670    Return the test command for the file
    5771    """
    5872    global opts
    59     return "sage -t %s %s"%(opts, abspath(f)[len(SAGE_ROOT)+1:])
    60 
    61 def sage_test_cmd(f):
    62     """
    63     Return the test command for the file as given
    64     """
    65     global opts
    66     return "sage -t %s %s"%(opts, f)
    67 
    68 def abs_sage_path(f):
    69     """
    70     Return the absolute path, relative to the sage root directory
    71     """
    72     return abspath(f)[len(SAGE_ROOT)+1:]
     73    return "sage -t %s %s" % (opts, abs_sage_path(f))
    7374
    7475def skip(F):
    7576    """
    def skip(F): 
    8182    i = G.rfind(os.path.sep)
    8283    if os.path.exists(os.path.join(G[:i], 'nodoctest.py')):
    8384        printmutex.acquire()
    84         print "%s (skipping) -- nodoctest.py file in directory"%abs(F)
     85        print "%s (skipping) -- nodoctest.py file in directory" % test_cmd(F)
    8586        printmutex.release()
    8687        return True
    8788    filenm = os.path.split(F)[1]
    def skip(F): 
    9091        return True
    9192    if G.find(os.path.join('doc', 'output')) != -1:
    9293        return True
    93     if not (os.path.splitext(F)[1] in ['.py', '.pyx', '.tex', '.pxi', '.sage', '.rst']):
     94    if not (os.path.splitext(F)[1] in ['.py', '.pyx', '.spyx', '.tex', '.pxi', '.sage', '.rst']):
    9495        return True
    9596    return False
    9697
    def process_result(result): 
    138139    if ret != 0:
    139140        if ret == 128:
    140141            numfail = ol.count('Expected:') + ol.count('Expected nothing') + ol.count('Exception raised:')
    141             failed.append(abs(F)+(" # %s doctests failed" % numfail))
     142            failed.append(test_cmd(F) + (" # %s doctests failed" % numfail))
    142143            ret = numfail
    143144        elif ret == 64:
    144             failed.append(abs(F)+" # Time out")
     145            failed.append(test_cmd(F) + " # Time out")
    145146        elif ret == 8:
    146             failed.append(abs(F)+" # Exception from doctest framework")
     147            failed.append(test_cmd(F) + " # Exception from doctest framework")
    147148        elif ret == 4:
    148             failed.append(abs(F)+" # Killed/crashed")
     149            failed.append(test_cmd(F) + " # Killed/crashed")
    149150        elif ret == 2:
    150             failed.append(abs(F)+" # KeyboardInterrupt")
     151            failed.append(test_cmd(F) + " # KeyboardInterrupt")
    151152        elif ret == 1:
    152             failed.append(abs(F)+" # File not found")
     153            failed.append(test_cmd(F) + " # File not found")
    153154        else:
    154             failed.append(abs(F))
    155     if abspath(F)[:len(CUR)]==CUR:
    156         print sage_test_cmd(F[len(CUR)+1:])
    157     else:
    158         print abs(F)
     155            failed.append(test_cmd(F))
     156
     157    print test_cmd(F)
     158
    159159    if ol!="" and (not ol.isspace()):
    160160        if (ol[len(ol)-1]=="\n"):
    161161            ol=ol[0:len(ol)-1]
    def populatefilelist(filelist): 
    199199        for root, dirs, lfiles in os.walk(walkdir):
    200200            for F in lfiles:
    201201                base, ext = os.path.splitext(F)
    202                 if not (ext in ['.sage', '.py', '.pyx', '.tex', '.pxi', '.rst']):
     202                if not (ext in ['.sage', '.py', '.pyx', '.spyx', '.tex', '.pxi', '.rst']):
    203203                    continue
    204204                elif '__nodoctest__' in files:
    205205                    continue
    for gr in range(0,numglobaliteration): 
    374374               print "\t", failed[i]
    375375        print "-"*int(70)
    376376    #Only update timings if we are doing something standard
     377    opts = opts.strip()
    377378    if (opts=="-long" or len(opts)==0) and not interrupt:
    378379        with open(time_file_name,"w") as time_file:
    379380            pickle.dump(time_dict, time_file)