Ticket #7249: trac_7249_jinja2.patch

File trac_7249_jinja2.patch, 22.8 KB (added by timdumol, 10 years ago)

Converts much storage to unicode, and adds the necessary encoding for storage and networking. Uses jinja2 instead of jinja. Depends on #7835, #7786 and their dependencies.

  • sagenb/data/sage/html/base.html

    diff -r 2444099672b2 sagenb/data/sage/html/base.html
    a b  
    1 {%- macro render_title -%}{% block title %}{% endblock %}{%- endmacro -%}
     1{%- macro render_title() -%}{% block title %}{% endblock %}{%- endmacro -%}
    22<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
    33<html lang="en">
    44    <head>
  • sagenb/data/sage/html/notebook/cell.html

    diff -r 2444099672b2 sagenb/data/sage/html/notebook/cell.html
    a b  
    1212        print or not.
    1313#}
    1414{% if do_print %}
    15     {% set wrap = 68 %}
    16     {% set div_wrap = 68 %}
     15    {% set wrap_ = 68 %}
     16    {% set div_wrap_ = 68 %}
     17{% else %}
     18    {% set wrap_ = wrap %}
     19    {% set div_wrap_ = div_wrap %}
    1720{% endif %}
    1821{% set cell_cls = "cell_evaluated" if cell.evaluated() or do_print else "cell_not_evaluated" %}
    19 
    20 {% if div_wrap %}
     22{% if div_wrap_ %}
    2123<div id="cell_outer_{{ cell.id() }}" class="cell_visible">
    2224    <div id="cell_{{ cell.id() }}" class="{{ cell_cls }}">
    2325{% endif %}
     
    5052                          onFocus="cell_focused(this, {{ cell.id() }}); return true;">{{ cell.input_text().rstrip() }}</textarea>
    5153                <a href="javascript:evaluate_cell({{ cell.id() }},0)"
    5254                   class="eval_button"
    53                    id="eval_button{{ cell.id() }}"
     55                   id="eval_button{{ cell.id() }}" 
    5456                   alt="Click here or press shift-return to evaluate">
    5557                    evaluate
    5658                </a>
     
    7981                                {% if cell.introspect() %}
    8082                                    {{ cell.output_text(0, html=true) }}
    8183                                {% else %}
    82                                     {{ cell.output_text(wrap, html=true) }}
     84                                    {{ cell.output_text(wrap_, html=true) }}
    8385                                {% endif %}
    8486                            </div>
    8587                            {% if not do_print %}
     
    99101        </div>
    100102        {% endif %}
    101103       
    102 {% if div_wrap %}
     104{% if div_wrap_ %}
    103105    </div>
    104106</div>
    105107{% endif %}
  • sagenb/interfaces/expect.py

    diff -r 2444099672b2 sagenb/interfaces/expect.py
    a b  
    241241        # The magic comment at the very start of the file allows utf8 characters.
    242242        open(self._filename,'w').write(
    243243            '# -*- coding: utf_8 -*-\nimport sys;sys.ps1="%s";print "START%s"\n'%(
    244             self._prompt, self._number) + displayhook_hack(string))
     244            self._prompt, self._number) + displayhook_hack(string).encode('utf-8', 'ignore'))
    245245        try:
    246246            self._expect.sendline('\nimport os;os.chdir("%s");\nexecfile("%s")'%(
    247247                              remote, sage_input))
    248         except OSError, msg:
     248        except OSError as msg:
    249249            self._is_computing = False
    250250            self._so_far = str(msg)
    251251
  • sagenb/misc/sageinspect.py

    diff -r 2444099672b2 sagenb/misc/sageinspect.py
    a b  
    483483
    484484    if r is None:
    485485        return ''
    486 
    487     s = format(str(r), embedded=EMBEDDED_MODE)
     486    if isinstance(r, unicode):
     487        r = r.encode('utf-8', 'ignore')
     488    else:
     489        r = str(r)
     490    s = format(r, embedded=EMBEDDED_MODE)
    488491
    489492    # If there is a Cython embedded position, it needs to be stripped
    490493    pos = _extract_embedded_position(s)
  • sagenb/misc/support.py

    diff -r 2444099672b2 sagenb/misc/support.py
    a b  
    331331        filename = sageinspect.sage_getfile(obj)
    332332        try:
    333333            lines, lineno = sageinspect.sage_getsourcelines(obj, is_binary=False)
    334         except IOError, msg:
     334        except IOError as msg:
    335335            return html_markup(str(msg))
    336336        src = indent.join(lines)
    337337        src = indent + format_src(src)
     
    467467    if dir:
    468468        if hasattr(system.__class__, 'chdir'):
    469469            system.chdir(dir)
     470    if isinstance(cmd, unicode):
     471        cmd = cmd.encode('utf-8', 'ignore')
    470472    return system.eval(cmd, sage_globals, locals = sage_globals)
    471473
    472474######################################################################
  • sagenb/notebook/cell.py

    diff -r 2444099672b2 sagenb/notebook/cell.py
    a b  
    9191            sage: C == loads(dumps(C))
    9292            True
    9393        """
     94        if isinstance(text, str):
     95            text = text.decode('utf-8', 'ignore')
    9496        self.__id = int(id)
    9597        self.__text = text
    9698        self.__worksheet = worksheet
     
    136138            sage: C
    137139            TextCell 0: 3+2
    138140        """
     141        if isinstance(input_text, str):
     142            inpit_text = input_text.decode('utf-8', str)
    139143        self.__text = input_text
    140144
    141145    def set_worksheet(self, worksheet, id=None):
     
    297301            sage: C == loads(dumps(C))
    298302            True
    299303        """
     304        if isinstance(out, str):
     305            out = out.encode('utf-8', 'ignore')
     306        else:
     307            out = unicode(out)
     308        if isinstance(input, str):
     309            input = input.encode('utf-8', 'ignore')
     310        else:
     311            input = unicode(input)
     312           
    300313        self.__id    = int(id)
    301         self.__out   = str(out).replace('\r','')
     314        self.__out   = out.replace('\r','')
    302315        self.__worksheet = worksheet
    303316        self.__interrupted = False
    304317        self.__completions = False
     
    355368            sage: C
    356369            Cell 0; in=2+3, out=
    357370        """
    358         self.__out = ''
    359         self.__out_html = ''
     371        self.__out = u''
     372        self.__out_html = u''
    360373        self.__evaluated = False
    361374
    362375    def evaluated(self):
     
    555568            sage: nb.delete()
    556569        """
    557570        if self.is_interactive_cell():
    558             self.__out_html = ""
     571            self.__out_html = u""
    559572        else:
    560573            self.__out_html = self.files_html(output)
    561574
     
    735748        """
    736749        if ncols == 0:
    737750            ncols = self.word_wrap_cols()
    738         s = ''
    739 
     751        s = u''
     752       
     753        if isinstance(self.__in, str):
     754            self.__in = self.__in.decode('utf-8', 'ignore')
     755           
    740756        input_lines = self.__in
     757       
    741758        pr = 'sage: '
    742759
    743760        if prompts:
     
    813830            '{{{id=0|\n2+3\n///\n5\n}}}'
    814831        """
    815832        s = self.plain_text(ncols, prompts, max_out)
    816         return '{{{id=%s|\n%s\n}}}'%(self.id(), s)
     833        return u'{{{id=%s|\n%s\n}}}'%(self.id(), s)
    817834
    818835    def is_last(self):
    819836        """
     
    9971014            sage: nb.delete()
    9981015        """
    9991016        # Stuff to deal with interact
     1017        if isinstance(input, str):
     1018            input = input.decode('utf-8', 'ignore')
     1019       
    10001020        if input.startswith('%__sage_interact__'):
    10011021            self.interact = input[len('%__sage_interact__')+1:]
    10021022            self.__version = self.version() + 1
     
    11841204            sage: C.changed_input_text()
    11851205            '3+3'
    11861206        """
     1207        if isinstance(new_text,str):
     1208            new_text = new_text.decode('utf-8', 'ignore')
    11871209        self.__changed_input = new_text
    11881210        self.__in = new_text
    11891211
     
    12001222            sage: len(C.plain_text())
    12011223            12
    12021224        """
     1225        if isinstance(output, str):
     1226            output = output.decode('utf-8', 'ignore')
     1227        if isinstance(html, str):
     1228            html = html.decode('utf-8', 'ignore')
    12031229        if output.count('<?__SAGE__TEXT>') > 1:
    1204             html = '<h3><font color="red">WARNING: multiple @interacts in one cell disabled (not yet implemented).</font></h3>'
    1205             output = ''
     1230            html = u'<h3><font color="red">WARNING: multiple @interacts in one cell disabled (not yet implemented).</font></h3>'
     1231            output = u''
    12061232
    12071233        # In interacting mode, we just save the computed output
    12081234        # (do not overwrite).
     
    12221248            url = ""
    12231249            if not self.computing():
    12241250                file = os.path.join(self.directory(), "full_output.txt")
    1225                 open(file,"w").write(output)
     1251                open(file,"w").write(output.encode('utf-8', 'ignore'))
    12261252                url = "<a target='_new' href='%s/full_output.txt' class='file_link'>full_output.txt</a>"%(
    12271253                    self.url_to_self())
    12281254                html+="<br>" + url
     
    13481374                # during interact.
    13491375                return ''
    13501376
     1377        if isinstance(self.__out, str):
     1378            self.__out = self.__out.decode('utf-8', 'ignore')
     1379           
    13511380        is_interact = self.is_interactive_cell()
    13521381        if is_interact and ncols == 0:
    13531382            if 'Traceback (most recent call last)' in self.__out:
    13541383                s = self.__out.replace('cell-interact','')
    13551384                is_interact=False
    13561385            else:
    1357                 return '<h2>Click to the left again to hide and once more to show the dynamic interactive window</h2>'
     1386                return u'<h2>Click to the left again to hide and once more to show the dynamic interactive window</h2>'
    13581387        else:
    13591388            s = self.__out
    13601389
     
    15021531            sage: W.quit()
    15031532            sage: nb.delete()
    15041533        """
     1534        if isinstance(html, str):
     1535            html = html.decode('utf-8', 'ignore')
    15051536        self.__introspect_html = html
    15061537        self.introspection_status = 'done'
    15071538
     
    15421573        try:
    15431574            return self.__introspect_html
    15441575        except AttributeError:
    1545             self.__introspect_html = ''
    1546             return ''
     1576            self.__introspect_html = u''
     1577            return u''
    15471578
    15481579    def introspect(self):
    15491580        """
     
    19081939            files  = ''
    19091940        else:
    19101941            files  = ('&nbsp'*3).join(files)
     1942        if isinstance(files, str):
     1943            files = files.decode('utf-8', 'ignore')
     1944        if isinstance(images, str):
     1945            iamges = images.decode('utf-8', 'ignore')
    19111946        return images + files
    19121947
    19131948########
  • sagenb/notebook/notebook.py

    diff -r 2444099672b2 sagenb/notebook/notebook.py
    a b  
    12741274
    12751275        return template(os.path.join("html", "notebook", "worksheet_revision_list.html"),
    12761276                        data = data, worksheet = worksheet,
    1277                         worksheet_filename = worksheet.filename(),
     1277                        notebook = self,
    12781278                        username = username)
    12791279
    12801280
     
    13461346
    13471347        return template(os.path.join("html", "notebook", "worksheet_share.html"),
    13481348                        worksheet = worksheet,
    1349                         worksheet_filename = worksheet.filename(),
     1349                        notebook = self,
    13501350                        username = username, other_users = other_users)
    13511351   
    13521352    def html_download_or_delete_datafile(self, ws, username, filename):
     
    13831383            text_file_content = open(os.path.join(ws.data_directory(), filename)).read()
    13841384
    13851385        return template(os.path.join("html", "notebook", "download_or_delete_datafile.html"),
    1386                         worksheet = ws, worksheet_filename = ws.filename(),
     1386                        worksheet = ws, notebook = self,
    13871387                        username = username,
    13881388                        filename_ = filename,
    13891389                        file_is_image = file_is_image,
     
    15031503
    15041504        return template(os.path.join("html", "notebook", "plain_text_window.html"),
    15051505                        worksheet = worksheet,
    1506                         worksheet_filename = worksheet.filename(),
     1506                        notebook = self,
    15071507                        username = username, plain_text = plain_text,
    15081508                        JSMATH = JSMATH, JEDITABLE_TINYMCE = JEDITABLE_TINYMCE)
    15091509
     
    15311531
    15321532        return template(os.path.join("html", "notebook", "edit_window.html"),
    15331533                        worksheet = worksheet,
    1534                         worksheet_filename = worksheet.filename(),
     1534                        notebook = self,
    15351535                        username = username)
    15361536
    15371537    def html_beforepublish_window(self, worksheet, username):
     
    15681568        """
    15691569        return template(os.path.join("html", "notebook", "beforepublish_window.html"),
    15701570                        worksheet = worksheet,
    1571                         worksheet_filename = worksheet.filename(),
    1572                         username = username,
    1573                         JSMATH = JSMATH, JEDITABLE_TINYMCE = JEDITABLE_TINYMCE)
     1571                        notebook = self,
     1572                        username = username)
    15741573
    15751574    def html_afterpublish_window(self, worksheet, username, url, dtime):
    15761575        r"""
     
    15971596
    15981597        return template(os.path.join("html", "notebook", "afterpublish_window.html"),
    15991598                        worksheet = worksheet,
    1600                         worksheet_filename = worksheet.filename(),
    1601                         username = username, url = url, time = time,
    1602                         JSMATH = JSMATH, JEDITABLE_TINYMCE = JEDITABLE_TINYMCE)
     1599                        notebook = self,
     1600                        username = username, url = url, time = time)
    16031601
    16041602    def html_upload_data_window(self, ws, username):
    16051603        r"""
  • sagenb/notebook/template.py

    diff -r 2444099672b2 sagenb/notebook/template.py
    a b  
    1515#                  http://www.gnu.org/licenses/
    1616#############################################################################
    1717
    18 import jinja
    19 
    20 from jinja.filters import stringfilter, simplefilter
     18import jinja2
    2119
    2220import os, re, sys
    2321
     
    2725
    2826
    2927TEMPLATE_PATH = os.path.join(DATA, 'sage')
    30 env = jinja.Environment(loader=jinja.FileSystemLoader(TEMPLATE_PATH))
     28env = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATE_PATH))
    3129
    3230css_illegal_re = re.compile(r'[^-A-Za-z_0-9]')
    3331
    34 @stringfilter
    3532def css_escape(string):
    3633    r"""
    3734    Returns a string with all characters not legal in a css name
     
    4340
    4441    EXAMPLES::
    4542
    46         sage: from sagenb.notebook.template import env, css_escape
    47         sage: escaper = css_escape()
    48         sage: print(escaper(env, {}, '12abcd'))
    49         12abcd
    50         sage: print(escaper(env, {}, 'abcd'))
    51         abcd
    52         sage: print(escaper(env, {}, r'\'"abcd\'"'))
    53         ---abcd---
    54         sage: print(escaper(env, {}, 'my-invalid/identifier'))
    55         my-invalid-identifier
    56         sage: print(escaper(env, {}, r'quotes"mustbe!escaped'))
    57         quotes-mustbe-escaped
    58 
    59     The following doctests originally accompanied #7269's support for
    60     Jinja2.
    61 
    62         sage: from sagenb.notebook.template import css_escape # not tested
    63         sage: css_escape('abcd')                              # not tested
     43        sage: from sagenb.notebook.template import css_escape
     44        sage: css_escape('abcd')                             
    6445        'abcd'
    65         sage: css_escape('12abcd')                            # not tested
     46        sage: css_escape('12abcd')                           
    6647        '12abcd'
    67         sage: css_escape(r'\'"abcd\'"')                       # not tested
     48        sage: css_escape(r'\'"abcd\'"')                       
    6849        '---abcd---'
    69         sage: css_escape('my-invalid/identifier')             # not tested
     50        sage: css_escape('my-invalid/identifier')             
    7051        'my-invalid-identifier'
    71         sage: css_escape(r'quotes"mustbe!escaped')            # not tested
     52        sage: css_escape(r'quotes"mustbe!escaped')
    7253        'quotes-mustbe-escaped'
    7354    """
    7455    return css_illegal_re.sub('-', string)
     
    119100    return ''.join([x if x.isalnum() else '_' for x in name])
    120101
    121102env.filters['css_escape'] = css_escape
    122 env.filters['number_of_rows'] = simplefilter(number_of_rows)
    123 env.filters['clean_name'] = stringfilter(clean_name)
    124 env.filters['prettify_time_ago'] = simplefilter(prettify_time_ago)
    125 env.filters['math_parse'] = stringfilter(math_parse)
    126 env.filters['max'] = simplefilter(max)
     103env.filters['number_of_rows'] = number_of_rows
     104env.filters['clean_name'] = clean_name
     105env.filters['prettify_time_ago'] = prettify_time_ago
     106env.filters['math_parse'] = math_parse
     107env.filters['max'] = max
    127108
    128109def template(filename, **user_context):
    129110    """
     
    164145                       'conf': notebook.conf() if notebook else None}
    165146    try:
    166147        tmpl = env.get_template(filename)
    167     except jinja.exceptions.TemplateNotFound:
     148    except jinja2.exceptions.TemplateNotFound:
    168149        return "Notebook Bug -- missing template %s"%filename
    169150    context = dict(default_context)
    170151    context.update(user_context)
    171152    r = tmpl.render(**context)
    172153    return r
     154
  • sagenb/notebook/twist.py

    diff -r 2444099672b2 sagenb/notebook/twist.py
    a b  
    8282SEP = '___S_A_G_E___'
    8383
    8484def encode_list(v):
    85     return SEP.join([str(x) for x in v])
     85    seq = []
     86    for x in v:
     87        if isinstance(x, unicode):
     88            x = x.encode('utf-8', 'ignore')
     89        else:
     90            x = str(x)
     91        seq.append(x)
     92    return SEP.join(seq)
    8693
    8794
    8895
     
    10901097    def render(self, ctx):
    10911098        id = self.id(ctx)
    10921099        if not ctx.args.has_key('input'):
    1093             input = ''
     1100            input = u''
    10941101        else:
    10951102            input = ctx.args['input'][0]
     1103            if isinstance(input, str):
     1104                input = input.decode('utf-8')
    10961105
    10971106        cell = self.worksheet.new_cell_before(id, input=input)
    10981107        self.worksheet.increase_state_number()
     
    11091118            input = ''
    11101119        else:
    11111120            input = ctx.args['input'][0]
     1121            if isinstance(input, str):
     1122                input = input.decode('utf-8')
    11121123
    11131124        cell = self.worksheet.new_text_cell_before(id, input=input)
    11141125        s = encode_list([cell.id(), cell.html(editing=True), id])
     
    11251136            input = ''
    11261137        else:
    11271138            input = ctx.args['input'][0]
     1139            if isinstance(input, str):
     1140                input = input.decode('utf-8')
    11281141        cell = self.worksheet.new_cell_after(id, input=input)
    11291142        s = encode_list([cell.id(), cell.html(div_wrap=False), id])
    11301143        return HTMLResponse(stream = s)
     
    11391152            input = ''
    11401153        else:
    11411154            input = ctx.args['input'][0]
     1155            if isinstance(input, str):
     1156                input = input.decode('utf-8')
    11421157
    11431158        cell = self.worksheet.new_text_cell_after(id, input=input)
    11441159        s = encode_list([cell.id(), cell.html(editing=True), id])
     
    12491264    def render(self, ctx):
    12501265        id = self.id(ctx)
    12511266        if not ctx.args.has_key('input'):
    1252             input_text = ''
     1267            input_text = u''
    12531268        else:
    12541269            input_text = ctx.args['input'][0]
    12551270            input_text = input_text.replace('\r\n', '\n')   # DOS
     1271            if isinstance(input_text, str):
     1272                input_text = input_text.decode('utf-8')
     1273
    12561274
    12571275        W = self.worksheet
    12581276        W.increase_state_number()
  • sagenb/notebook/worksheet.py

    diff -r 2444099672b2 sagenb/notebook/worksheet.py
    a b  
    714714            'A renamed worksheet'
    715715        """
    716716        if len(name.strip()) == 0:
    717             name = 'Untitled'
     717            name = u'Untitled'
     718        if isinstance(name, str):
     719            name = name.decode('utf-8', 'ignore')
    718720        self.__name = name
    719721
    720722    def set_filename_without_owner(self, nm):
     
    19121914        if os.path.exists(worksheet_html) and open(worksheet_html).read() == E:
    19131915            # we already wrote it out...
    19141916            return
    1915         open(filename, 'w').write(bz2.compress(E))
    1916         open(worksheet_html, 'w').write(self.body())
     1917        open(filename, 'w').write(bz2.compress(E.encode('utf-8', 'ignore')))
     1918        open(worksheet_html, 'w').write(self.body().encode('utf-8', 'ignore'))
    19171919        self.limit_snapshots()
    19181920        try:
    19191921            X = self.__saved_by_info
     
    34353437        """
    34363438        # The extra newline below is necessary, since otherwise source
    34373439        # code introspection doesn't include the last line.
    3438         return 'open("%s","w").write("# -*- coding: utf-8 -*-\\n" + _support_.preparse_worksheet_cell(base64.b64decode("%s"),globals())+"\\n"); execfile(os.path.abspath("%s"))'%(CODE_PY, base64.b64encode(s), CODE_PY)
     3440        return 'open("%s","w").write("# -*- coding: utf-8 -*-\\n" + _support_.preparse_worksheet_cell(base64.b64decode("%s"),globals())+"\\n"); execfile(os.path.abspath("%s"))'%(CODE_PY, base64.b64encode(s.encode('utf-8', 'ignore')), CODE_PY)
    34393441
    34403442    ##########################################################
    34413443    # Loading and attaching files
     
    36223624            os.makedirs(code)
    36233625        spyx = os.path.abspath(os.path.join(code, 'sage%s.spyx'%id))
    36243626        if not (os.path.exists(spyx) and open(spyx).read() == cmd):
    3625             open(spyx,'w').write(cmd)
     3627            open(spyx,'w').write(cmd.encode('utf-8', 'ignore'))
    36263628        return '_support_.cython_import_all("%s", globals())'%spyx
    36273629
    36283630    def check_for_system_switching(self, input, cell):
  • sagenb/storage/filesystem_storage.py

    diff -r 2444099672b2 sagenb/storage/filesystem_storage.py
    a b  
    291291            # only save if loaded
    292292            # todo -- add check if changed
    293293            filename = self._worksheet_html_filename(username, id_number)
    294             open(self._abspath(filename),'w').write(worksheet.body())
     294            open(self._abspath(filename),'w').write(worksheet.body().encode('utf-8', 'ignore'))
    295295
    296296    def load_worksheet(self, username, id_number):
    297297        """
     
    339339        if title:
    340340            # change the title
    341341            basic['name'] = title
    342 
     342        if isinstance(basic['name'], unicode):
     343            basic['name'] = basic['name'].encode('utf-8', 'ignore')
    343344        # Remove metainformation that perhaps shouldn't be distributed
    344345        for k in ['owner', 'ratings', 'worksheet_that_was_published', 'viewers', 'tags', 'published_id_number',
    345346                  'collaborators', 'auto_publish']:
     
    397398
    398399        worksheet_txt = members[0].name
    399400        W = self.load_worksheet(username, id_number)
    400         W.edit_save_old_format(T.extractfile(worksheet_txt).read())
     401        W.edit_save_old_format(T.extractfile(worksheet_txt).read().decode('utf-8', 'ignore'))
    401402        # '/' is right, since old worksheets always unix
    402403        dir = worksheet_txt.split('/')[0]
    403404