# HG changeset patch
# User Simon King <simon.king@unijena.de>
# Date 1363025892 3600
# Node ID d10aa3f464c3f9f878e7e8f7e7a7cacfa9ab944b
# Parent 02fbdf47d9b8c6d6bad9256009a28ce1965e99c2
#14159: Optional support for weak values in Triple and MonoDict. Safer callback.
diff git a/sage/categories/homset.py b/sage/categories/homset.py
a

b


74  74  ################################### 
75  75  # Use the weak "triple" dictionary 
76  76  # introduced in trac ticket #715 
 77  # with weak values, as introduced in 
 78  # trac ticket #14159 
77  79  
78   from weakref import KeyedRef 
79   from sage.structure.coerce_dict import signed_id, TripleDict 
80   _cache = TripleDict(53) 
 80  from sage.structure.coerce_dict import TripleDict 
 81  _cache = TripleDict(53, weak_values=True) 
81  82  
82  83  def Hom(X, Y, category=None): 
83  84  """ 
… 
… 

216  217  global _cache 
217  218  key = (X,Y,category) 
218  219  try: 
219   H = _cache[key]() 
 220  H = _cache[key] 
220  221  except KeyError: 
221  222  H = None 
222  223  if H is not None: 
… 
… 

244  245  # Now, as the category may have changed, we try to find the hom set in the cache, again: 
245  246  key = (X,Y,category) 
246  247  try: 
247   H = _cache[key]() 
 248  H = _cache[key] 
248  249  except KeyError: 
249  250  H = None 
250  251  if H is not None: 
… 
… 

263  264  H = category.hom_category().parent_class(X, Y, category = category) 
264  265  
265  266  ##_cache[key] = weakref.ref(H) 
266   _cache[key] = KeyedRef(H, _cache.eraser, (signed_id(X),signed_id(Y),signed_id(category))) 
 267  _cache[key] = H 
267  268  return H 
268  269  
269  270  def hom(X, Y, f): 
diff git a/sage/structure/coerce_dict.pxd b/sage/structure/coerce_dict.pxd
a

b


2  2  cdef __weakref__ 
3  3  cdef Py_ssize_t _size 
4  4  cdef buckets 
 5  cdef bint weak_values 
5  6  cdef double threshold 
6  7  cdef public MonoDictEraser eraser 
7  8  cdef get(self, object k) 
… 
… 

14  15  cdef __weakref__ 
15  16  cdef Py_ssize_t _size 
16  17  cdef buckets 
 18  cdef bint weak_values 
17  19  cdef double threshold 
18  20  cdef public TripleDictEraser eraser 
19  21  cdef get(self, object k1, object k2, object k3) 
diff git a/sage/structure/coerce_dict.pyx b/sage/structure/coerce_dict.pyx
a

b


10  10  Containers for storing coercion data 
11  11  
12  12  This module provides :class:`TripleDict` and :class:`MonoDict`. These are 
13   structures similar to ``WeakKeyDictionary`` in Python's weakref module, 
14   and are optimized for lookup speed. The keys for :class:`TripleDict` consist 
15   of triples (k1,k2,k3) and are looked up by identity rather than equality. The 
16   keys are stored by weakrefs if possible. If any one of the components k1, k2, 
17   k3 gets garbage collected, then the entry is removed from the :class:`TripleDict`. 
 13  structures similar to :class:`~weakref.WeakKeyDictionary` in Python's weakref 
 14  module, and are optimized for lookup speed. The keys for :class:`TripleDict` 
 15  consist of triples (k1,k2,k3) and are looked up by identity rather than 
 16  equality. The keys are stored by weakrefs if possible. If any one of the 
 17  components k1, k2, k3 gets garbage collected, then the entry is removed from 
 18  the :class:`TripleDict`. 
18  19  
19  20  Key components that do not allow for weakrefs are stored via a normal 
20  21  refcounted reference. That means that any entry stored using a triple 
… 
… 

22  23  as an entry in a normal dictionary: Its existence in :class:`TripleDict` 
23  24  prevents it from being garbage collected. 
24  25  
25   That container currently is used to store coercion and conversion maps 
26   between two parents (:trac:`715`) and to store homsets of pairs of objects 
27   of a category (:trac:`11521`). In both cases, it is essential that the parent 
28   structures remain garbage collectable, it is essential that the data access 
29   is faster than with a usual ``WeakKeyDictionary``, and we enforce the "unique 
30   parent condition" in Sage (parent structures should be identical if they are 
31   equal). 
 26  That container currently is used to store coercion and conversion maps between 
 27  two parents (:trac:`715`) and to store homsets of pairs of objects of a 
 28  category (:trac:`11521`). In both cases, it is essential that the parent 
 29  structures remain garbage collectable, it is essential that the data access is 
 30  faster than with a usual :class:`~weakref.WeakKeyDictionary`, and we enforce 
 31  the "unique parent condition" in Sage (parent structures should be identical 
 32  if they are equal). 
32  33  
33  34  :class:`MonoDict` behaves similarly, but it takes a single item as a key. It 
34  35  is used for caching the parents which allow a coercion map into a fixed other 
35  36  parent (:trac:`12313`). 
36  37  
 38  By :trac:`14159`, :class:`MonoDict` and :class:`TripleDict` can be optionally 
 39  used with weak references on the values. 
 40  
37  41  """ 
38  42  include "../ext/python_list.pxi" 
39  43  
… 
… 

178  182  cdef list buckets = D.buckets 
179  183  if buckets is None: 
180  184  return 
181   cdef Py_ssize_t h = r.key 
 185  cdef Py_ssize_t h 
 186  cdef int offset 
 187  h,offset = r.key 
182  188  cdef list bucket = <object>PyList_GET_ITEM(buckets, (<size_t>h) % PyList_GET_SIZE(buckets)) 
183  189  cdef Py_ssize_t i 
184  190  for i from 0 <= i < PyList_GET_SIZE(bucket) by 3: 
185  191  if PyInt_AsSsize_t(PyList_GET_ITEM(bucket,i))==h: 
186   del bucket[i:i+3] 
187   D._size = 1 
188   break 
 192  if PyList_GET_ITEM(bucket,i+offset)==<void *>r: 
 193  del bucket[i:i+3] 
 194  D._size = 1 
 195  break 
 196  else: 
 197  break 
189  198  
190  199  cdef class TripleDictEraser: 
191  200  """ 
… 
… 

278  287  # stored key of the unique triple r() had been part of. 
279  288  # We remove that unique triple from self.D 
280  289  cdef Py_ssize_t k1,k2,k3 
281   k1,k2,k3 = r.key 
 290  cdef int offset 
 291  k1,k2,k3,offset = r.key 
282  292  cdef Py_ssize_t h = (k1 + 13*k2 ^ 503*k3) 
283  293  cdef list bucket = <object>PyList_GET_ITEM(buckets, (<size_t>h) % PyList_GET_SIZE(buckets)) 
284  294  cdef Py_ssize_t i 
… 
… 

286  296  if PyInt_AsSsize_t(PyList_GET_ITEM(bucket, i))==k1 and \ 
287  297  PyInt_AsSsize_t(PyList_GET_ITEM(bucket, i+1))==k2 and \ 
288  298  PyInt_AsSsize_t(PyList_GET_ITEM(bucket, i+2))==k3: 
289   del bucket[i:i+7] 
290   D._size = 1 
291   break 
 299  if PyList_GET_ITEM(bucket, i+offset)==<void *>r: 
 300  del bucket[i:i+7] 
 301  D._size = 1 
 302  break 
 303  else: 
 304  break 
292  305  
293  306  cdef class MonoDict: 
294  307  """ 
… 
… 

306  319  It is barebones in the sense that not all dictionary methods are 
307  320  implemented. 
308  321  
 322  IMPLEMENTATION: 
 323  
309  324  It is implemented as a list of lists (hereafter called buckets). The bucket 
310  325  is chosen according to a very simple hash based on the object pointer, 
311  326  and each bucket is of the form [id(k1), r1, value1, id(k2), r2, value2, ...], 
… 
… 

317  332  In the latter case the presence of the key in the dictionary prevents it from 
318  333  being garbage collected. 
319  334  
320   To spread objects evenly, the size should ideally be a prime, and certainly 
321   not divisible by 2. 
 335  INPUT: 
 336  
 337   ``size``  an integer, the initial number of buckets. To spread objects 
 338  evenly, the size should ideally be a prime, and certainly not divisible 
 339  by 2. 
 340   ``data``  optional iterable defining initial data. 
 341   ``threshold``  optional number, default `0.7`. It determines how frequently 
 342  the dictionary will be resized (large threshold implies rare resizing). 
 343   ``weak_values``  optional bool (default False). If it is true, weak references 
 344  to the values in this dictionary will be used, when possible. 
322  345  
323  346  EXAMPLES:: 
324  347  
… 
… 

346  369  Not all features of Python dictionaries are available, but iteration over 
347  370  the dictionary items is possible:: 
348  371  
349   sage: # for some reason the following fails in "make ptest" 
 372  sage: # for some reason the following failed in "make ptest" 
350  373  sage: # on some installations, see #12313 for details 
351  374  sage: sorted(L.iteritems()) # random layout 
352  375  [(15, 3), ('a', 1), ('ab', 2)] 
… 
… 

410  433  sage: len(LE) # indirect doctest 
411  434  1 
412  435  
413   AUTHOR: 
 436  TESTS: 
 437  
 438  Here, we demonstrate the use of weak values. 
 439  :: 
 440  
 441  sage: M = MonoDict(13) 
 442  sage: MW = MonoDict(13, weak_values=True) 
 443  sage: class Foo: pass 
 444  sage: a = Foo() 
 445  sage: b = Foo() 
 446  sage: k = 1 
 447  sage: M[k] = a 
 448  sage: MW[k] = b 
 449  sage: M[k] is a 
 450  True 
 451  sage: MW[k] is b 
 452  True 
 453  sage: k in M 
 454  True 
 455  sage: k in MW 
 456  True 
 457  
 458  While ``M`` uses a strong reference to ``a``, ``MW`` uses a *weak* 
 459  reference to ``b``, and after deleting ``b``, the corresponding item of 
 460  ``MW`` will be removed during the next garbage collection:: 
 461  
 462  sage: import gc 
 463  sage: del a,b 
 464  sage: _ = gc.collect() 
 465  sage: k in M 
 466  True 
 467  sage: k in MW 
 468  False 
 469  sage: len(MW) 
 470  0 
 471  sage: len(M) 
 472  1 
 473  
 474  Note that ``MW`` also accepts values that do not allow for weak references:: 
 475  
 476  sage: MW[k] = int(5) 
 477  sage: MW[k] 
 478  5 
 479  
 480  AUTHORS: 
414  481  
415  482   Simon King (201201) 
416  483   Nils Bruin (201208) 
 484   Simon King (201302) 
417  485  """ 
418   def __init__(self, size, data=None, threshold=0.7): 
 486  def __init__(self, size, data=None, threshold=0.7, weak_values=False): 
419  487  """ 
420  488  Create a special dict using singletons for keys. 
421  489  
… 
… 

432  500  self.threshold = threshold 
433  501  self.buckets = [[] for i from 0 <= i < size] 
434  502  self._size = 0 
 503  self.weak_values = weak_values 
435  504  self.eraser = MonoDictEraser(self) 
436  505  if data is not None: 
437  506  for k, v in data.iteritems(): 
… 
… 

563  632  if isinstance(r, KeyedRef) and PyWeakref_GetObject(r) == Py_None: 
564  633  return False 
565  634  else: 
566   return True 
 635  return (not self.weak_values) or PyWeakref_GetObject(<object>PyList_GET_ITEM(bucket, i+2)) != Py_None 
567  636  return False 
568  637  
569  638  def __getitem__(self, k): 
… 
… 

599  668  cdef Py_ssize_t i 
600  669  cdef list all_buckets = self.buckets 
601  670  cdef list bucket = <object>PyList_GET_ITEM(all_buckets, (<size_t>h) % PyList_GET_SIZE(all_buckets)) 
602   cdef object r 
 671  cdef object r, val 
 672  cdef PyObject * out 
603  673  for i from 0 <= i < PyList_GET_SIZE(bucket) by 3: 
604  674  if PyInt_AsSsize_t(PyList_GET_ITEM(bucket, i)) == h: 
605  675  r = <object>PyList_GET_ITEM(bucket, i+1) 
606  676  if isinstance(r, KeyedRef) and PyWeakref_GetObject(r) == Py_None: 
607  677  raise KeyError, k 
608  678  else: 
609   return <object>PyList_GET_ITEM(bucket, i+2) 
 679  val = <object>PyList_GET_ITEM(bucket, i+2) 
 680  if self.weak_values: 
 681  if not isinstance(val, KeyedRef): 
 682  return val 
 683  out = PyWeakref_GetObject(val) 
 684  if out == Py_None: 
 685  raise KeyError, k 
 686  return <object>out 
 687  else: 
 688  return val 
610  689  raise KeyError, k 
611  690  
612  691  def __setitem__(self, k, value): 
… 
… 

634  713  self.resize() 
635  714  cdef Py_ssize_t h = signed_id(k) 
636  715  cdef Py_ssize_t i 
 716  if self.weak_values: 
 717  try: 
 718  value = KeyedRef(value,self.eraser,(h,2)) 
 719  except TypeError: 
 720  pass 
637  721  cdef list bucket = <object>PyList_GET_ITEM(self.buckets,(<size_t> h) % PyList_GET_SIZE(self.buckets)) 
638  722  cdef object r 
639  723  for i from 0 <= i < PyList_GET_SIZE(bucket) by 3: 
… 
… 

669  753  #investigate our partial entry. 
670  754  PyList_Append(bucket, h) 
671  755  try: 
672   PyList_Append(bucket, KeyedRef(k,self.eraser,h)) 
 756  PyList_Append(bucket, KeyedRef(k,self.eraser,(h,1))) 
673  757  except TypeError: 
674  758  PyList_Append(bucket, k) 
675  759  PyList_Append(bucket, value) 
… 
… 

836  920  If a key component ki supports weak references then ri is a weak reference to 
837  921  ki; otherwise ri is identical to ki. 
838  922  
839   If any of the key components k1,k2,k3 (this can happen for a key component that 
840   supports weak references) gets garbage collected then the entire entry 
841   disappears. In that sense this structure behaves like a nested WeakKeyDictionary. 
 923  INPUT: 
842  924  
843   To spread objects evenly, the size should ideally be a prime, and certainly 
844   not divisible by 2. 
 925   ``size``  an integer, the initial number of buckets. To spread objects 
 926  evenly, the size should ideally be a prime, and certainly not divisible 
 927  by 2. 
 928   ``data``  optional iterable defining initial data. 
 929   ``threshold``  optional number, default `0.7`. It determines how frequently 
 930  the dictionary will be resized (large threshold implies rare resizing). 
 931   ``weak_values``  optional bool (default False). If it is true, weak references 
 932  to the values in this dictionary will be used, when possible. 
 933  
 934  If any of the key components k1,k2,k3 (this can happen for a key component 
 935  that supports weak references) gets garbage collected then the entire 
 936  entry disappears. In that sense this structure behaves like a nested 
 937  :class:`~weakref.WeakKeyDictionary`. 
845  938  
846  939  EXAMPLES:: 
847  940  
… 
… 

920  1013  sage: len(LE) # indirect doctest 
921  1014  1 
922  1015  
 1016  TESTS: 
 1017  
 1018  Here, we demonstrate the use of weak values. 
 1019  :: 
 1020  
 1021  sage: class Foo: pass 
 1022  sage: T = TripleDict(13) 
 1023  sage: TW = TripleDict(13, weak_values=True) 
 1024  sage: a = Foo() 
 1025  sage: b = Foo() 
 1026  sage: k = 1 
 1027  sage: T[a,k,k]=1 
 1028  sage: T[k,a,k]=2 
 1029  sage: T[k,k,a]=3 
 1030  sage: T[k,k,k]=a 
 1031  sage: TW[b,k,k]=1 
 1032  sage: TW[k,b,k]=2 
 1033  sage: TW[k,k,b]=3 
 1034  sage: TW[k,k,k]=b 
 1035  sage: len(T) 
 1036  4 
 1037  sage: len(TW) 
 1038  4 
 1039  sage: (k,k,k) in T 
 1040  True 
 1041  sage: (k,k,k) in TW 
 1042  True 
 1043  sage: T[k,k,k] is a 
 1044  True 
 1045  sage: TW[k,k,k] is b 
 1046  True 
 1047  
 1048  Now, ``T`` holds a strong reference to ``a``, namely in ``T[k,k,k]``. Hence, 
 1049  when we delete ``a``, *all* items of ``T`` survive:: 
 1050  
 1051  sage: del a 
 1052  sage: _ = gc.collect() 
 1053  sage: len(T) 
 1054  4 
 1055  
 1056  Only when we remove the strong reference, the items become collectable:: 
 1057  
 1058  sage: del T[k,k,k] 
 1059  sage: _ = gc.collect() 
 1060  sage: len(T) 
 1061  0 
 1062  
 1063  The situation is different for ``TW``, since it only holds *weak* 
 1064  references to ``a``. Therefore, all items become collectable after 
 1065  deleting ``a``:: 
 1066  
 1067  sage: del b 
 1068  sage: _ = gc.collect() 
 1069  sage: len(TW) 
 1070  0 
 1071  
923  1072  .. NOTE:: 
924  1073  
925  1074  The index `h` corresponding to the key [k1, k2, k3] is computed as a 
… 
… 

949  1098   Simon King, 201201 
950  1099  
951  1100   Nils Bruin, 201208 
 1101  
 1102   Simon King, 201302 
952  1103  """ 
953  1104  
954   def __init__(self, size, data=None, threshold=0.7): 
 1105  def __init__(self, size, data=None, threshold=0.7, weak_values=False): 
955  1106  """ 
956  1107  Create a special dict using triples for keys. 
957  1108  
… 
… 

968  1119  self.threshold = threshold 
969  1120  self.buckets = [[] for i from 0 <= i < size] 
970  1121  self._size = 0 
 1122  self.weak_values = weak_values 
971  1123  self.eraser = TripleDictEraser(self) 
972  1124  if data is not None: 
973  1125  for (k1,k2,k3), v in data.iteritems(): 
… 
… 

1130  1282  cdef Py_ssize_t h3 = signed_id(k3) 
1131  1283  cdef Py_ssize_t h = (h1 + 13*h2 ^ 503*h3) 
1132  1284  
1133   cdef object r1,r2,r3 
 1285  cdef object r1,r2,r3, val 
 1286  cdef PyObject* ref_val 
1134  1287  cdef Py_ssize_t i 
1135  1288  cdef list all_buckets = self.buckets 
1136  1289  cdef list bucket = <object>PyList_GET_ITEM(all_buckets, (<size_t>h )% PyList_GET_SIZE(all_buckets)) 
… 
… 

1147  1300  (isinstance(r3,KeyedRef) and PyWeakref_GetObject(r3) == Py_None): 
1148  1301  raise KeyError, (k1,k2,k3) 
1149  1302  else: 
1150   return <object>PyList_GET_ITEM(bucket, i+6) 
 1303  val = <object>PyList_GET_ITEM(bucket, i+6) 
 1304  if self.weak_values: 
 1305  if not isinstance(val, KeyedRef): 
 1306  return val 
 1307  ref_val = PyWeakref_GetObject(val) 
 1308  if ref_val == Py_None: 
 1309  raise KeyError, (k1,k2,k3) 
 1310  return <object>ref_val 
 1311  else: 
 1312  return val 
1151  1313  raise KeyError, (k1, k2, k3) 
1152  1314  
1153  1315  def __setitem__(self, k, value): 
… 
… 

1177  1339  cdef Py_ssize_t h2 = signed_id(k2) 
1178  1340  cdef Py_ssize_t h3 = signed_id(k3) 
1179  1341  cdef Py_ssize_t h = (h1 + 13*h2 ^ 503*h3) 
 1342  if self.weak_values: 
 1343  try: 
 1344  value = KeyedRef(value,self.eraser,(h1, h2, h3, 6)) 
 1345  except TypeError: 
 1346  pass 
1180  1347  
1181  1348  cdef object r1,r2,r3 
1182  1349  cdef Py_ssize_t i 
… 
… 

1208  1375  #at this point the key triple isn't present so we append a new entry. 
1209  1376  #we first form the appropriate weakrefs to receive callbacks on. 
1210  1377  try: 
1211   r1 = KeyedRef(k1,self.eraser,(h1, h2, h3)) 
 1378  r1 = KeyedRef(k1,self.eraser,(h1, h2, h3, 3)) 
1212  1379  except TypeError: 
1213  1380  r1 = k1 
1214  1381  if k2 is not k1: 
1215  1382  try: 
1216   r2 = KeyedRef(k2,self.eraser,(h1, h2, h3)) 
 1383  r2 = KeyedRef(k2,self.eraser,(h1, h2, h3, 4)) 
1217  1384  except TypeError: 
1218  1385  r2 = k2 
1219  1386  else: 
1220  1387  r2 = None 
1221  1388  if k3 is not k2 or k3 is not k1: 
1222  1389  try: 
1223   r3 = KeyedRef(k3,self.eraser,(h1, h2, h3)) 
 1390  r3 = KeyedRef(k3,self.eraser,(h1, h2, h3, 5)) 
1224  1391  except TypeError: 
1225  1392  r3 = k3 
1226  1393  else: 