| 1 | #---------------------------------------------------------------------- |
| 2 | # Copyright (c) 1999-2001, Digital Creations, Fredericksburg, VA, USA |
| 3 | # and Andrew Kuchling. All rights reserved. |
| 4 | # |
| 5 | # Redistribution and use in source and binary forms, with or without |
| 6 | # modification, are permitted provided that the following conditions are |
| 7 | # met: |
| 8 | # |
| 9 | # o Redistributions of source code must retain the above copyright |
| 10 | # notice, this list of conditions, and the disclaimer that follows. |
| 11 | # |
| 12 | # o Redistributions in binary form must reproduce the above copyright |
| 13 | # notice, this list of conditions, and the following disclaimer in |
| 14 | # the documentation and/or other materials provided with the |
| 15 | # distribution. |
| 16 | # |
| 17 | # o Neither the name of Digital Creations nor the names of its |
| 18 | # contributors may be used to endorse or promote products derived |
| 19 | # from this software without specific prior written permission. |
| 20 | # |
| 21 | # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS AND CONTRIBUTORS *AS |
| 22 | # IS* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
| 23 | # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A |
| 24 | # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL |
| 25 | # CREATIONS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, |
| 26 | # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, |
| 27 | # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS |
| 28 | # OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
| 29 | # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR |
| 30 | # TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE |
| 31 | # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH |
| 32 | # DAMAGE. |
| 33 | #---------------------------------------------------------------------- |
| 34 | |
| 35 | |
| 36 | """Support for BerkeleyDB 3.2 through 4.2. |
| 37 | """ |
| 38 | |
| 39 | try: |
| 40 | if __name__ == 'bsddb3': |
| 41 | # import _pybsddb binary as it should be the more recent version from |
| 42 | # a standalone pybsddb addon package than the version included with |
| 43 | # python as bsddb._bsddb. |
| 44 | import _pybsddb |
| 45 | _bsddb = _pybsddb |
| 46 | else: |
| 47 | import _bsddb |
| 48 | except ImportError: |
| 49 | # Remove ourselves from sys.modules |
| 50 | import sys |
| 51 | del sys.modules[__name__] |
| 52 | raise |
| 53 | |
| 54 | # bsddb3 calls it db, but provide _db for backwards compatibility |
| 55 | db = _db = _bsddb |
| 56 | __version__ = db.__version__ |
| 57 | |
| 58 | error = db.DBError # So bsddb.error will mean something... |
| 59 | |
| 60 | #---------------------------------------------------------------------- |
| 61 | |
| 62 | import sys, os |
| 63 | |
| 64 | # for backwards compatibility with python versions older than 2.3, the |
| 65 | # iterator interface is dynamically defined and added using a mixin |
| 66 | # class. old python can't tokenize it due to the yield keyword. |
| 67 | if sys.version >= '2.3': |
| 68 | import UserDict |
| 69 | from weakref import ref |
| 70 | exec """ |
| 71 | class _iter_mixin(UserDict.DictMixin): |
| 72 | def _make_iter_cursor(self): |
| 73 | cur = self.db.cursor() |
| 74 | key = id(cur) |
| 75 | self._cursor_refs[key] = ref(cur, self._gen_cref_cleaner(key)) |
| 76 | return cur |
| 77 | |
| 78 | def _gen_cref_cleaner(self, key): |
| 79 | # use generate the function for the weakref callback here |
| 80 | # to ensure that we do not hold a strict reference to cur |
| 81 | # in the callback. |
| 82 | return lambda ref: self._cursor_refs.pop(key, None) |
| 83 | |
| 84 | def __iter__(self): |
| 85 | try: |
| 86 | cur = self._make_iter_cursor() |
| 87 | |
| 88 | # FIXME-20031102-greg: race condition. cursor could |
| 89 | # be closed by another thread before this call. |
| 90 | |
| 91 | # since we're only returning keys, we call the cursor |
| 92 | # methods with flags=0, dlen=0, dofs=0 |
| 93 | key = cur.first(0,0,0)[0] |
| 94 | yield key |
| 95 | |
| 96 | next = cur.next |
| 97 | while 1: |
| 98 | try: |
| 99 | key = next(0,0,0)[0] |
| 100 | yield key |
| 101 | except _bsddb.DBCursorClosedError: |
| 102 | cur = self._make_iter_cursor() |
| 103 | # FIXME-20031101-greg: race condition. cursor could |
| 104 | # be closed by another thread before this call. |
| 105 | cur.set(key,0,0,0) |
| 106 | next = cur.next |
| 107 | except _bsddb.DBNotFoundError: |
| 108 | return |
| 109 | except _bsddb.DBCursorClosedError: |
| 110 | # the database was modified during iteration. abort. |
| 111 | return |
| 112 | |
| 113 | def iteritems(self): |
| 114 | try: |
| 115 | cur = self._make_iter_cursor() |
| 116 | |
| 117 | # FIXME-20031102-greg: race condition. cursor could |
| 118 | # be closed by another thread before this call. |
| 119 | |
| 120 | kv = cur.first() |
| 121 | key = kv[0] |
| 122 | yield kv |
| 123 | |
| 124 | next = cur.next |
| 125 | while 1: |
| 126 | try: |
| 127 | kv = next() |
| 128 | key = kv[0] |
| 129 | yield kv |
| 130 | except _bsddb.DBCursorClosedError: |
| 131 | cur = self._make_iter_cursor() |
| 132 | # FIXME-20031101-greg: race condition. cursor could |
| 133 | # be closed by another thread before this call. |
| 134 | cur.set(key,0,0,0) |
| 135 | next = cur.next |
| 136 | except _bsddb.DBNotFoundError: |
| 137 | return |
| 138 | except _bsddb.DBCursorClosedError: |
| 139 | # the database was modified during iteration. abort. |
| 140 | return |
| 141 | """ |
| 142 | else: |
| 143 | class _iter_mixin: pass |
| 144 | |
| 145 | |
| 146 | class _DBWithCursor(_iter_mixin): |
| 147 | """ |
| 148 | A simple wrapper around DB that makes it look like the bsddbobject in |
| 149 | the old module. It uses a cursor as needed to provide DB traversal. |
| 150 | """ |
| 151 | def __init__(self, db): |
| 152 | self.db = db |
| 153 | self.db.set_get_returns_none(0) |
| 154 | |
| 155 | # FIXME-20031101-greg: I believe there is still the potential |
| 156 | # for deadlocks in a multithreaded environment if someone |
| 157 | # attempts to use the any of the cursor interfaces in one |
| 158 | # thread while doing a put or delete in another thread. The |
| 159 | # reason is that _checkCursor and _closeCursors are not atomic |
| 160 | # operations. Doing our own locking around self.dbc, |
| 161 | # self.saved_dbc_key and self._cursor_refs could prevent this. |
| 162 | # TODO: A test case demonstrating the problem needs to be written. |
| 163 | |
| 164 | # self.dbc is a DBCursor object used to implement the |
| 165 | # first/next/previous/last/set_location methods. |
| 166 | self.dbc = None |
| 167 | self.saved_dbc_key = None |
| 168 | |
| 169 | # a collection of all DBCursor objects currently allocated |
| 170 | # by the _iter_mixin interface. |
| 171 | self._cursor_refs = {} |
| 172 | |
| 173 | def __del__(self): |
| 174 | self.close() |
| 175 | |
| 176 | def _checkCursor(self): |
| 177 | if self.dbc is None: |
| 178 | self.dbc = self.db.cursor() |
| 179 | if self.saved_dbc_key is not None: |
| 180 | self.dbc.set(self.saved_dbc_key) |
| 181 | self.saved_dbc_key = None |
| 182 | |
| 183 | # This method is needed for all non-cursor DB calls to avoid |
| 184 | # BerkeleyDB deadlocks (due to being opened with DB_INIT_LOCK |
| 185 | # and DB_THREAD to be thread safe) when intermixing database |
| 186 | # operations that use the cursor internally with those that don't. |
| 187 | def _closeCursors(self, save=1): |
| 188 | if self.dbc: |
| 189 | c = self.dbc |
| 190 | self.dbc = None |
| 191 | if save: |
| 192 | self.saved_dbc_key = c.current(0,0,0)[0] |
| 193 | c.close() |
| 194 | del c |
| 195 | for cref in self._cursor_refs.values(): |
| 196 | c = cref() |
| 197 | if c is not None: |
| 198 | c.close() |
| 199 | |
| 200 | def _checkOpen(self): |
| 201 | if self.db is None: |
| 202 | raise error, "BSDDB object has already been closed" |
| 203 | |
| 204 | def isOpen(self): |
| 205 | return self.db is not None |
| 206 | |
| 207 | def __len__(self): |
| 208 | self._checkOpen() |
| 209 | return len(self.db) |
| 210 | |
| 211 | def __getitem__(self, key): |
| 212 | self._checkOpen() |
| 213 | return self.db[key] |
| 214 | |
| 215 | def __setitem__(self, key, value): |
| 216 | self._checkOpen() |
| 217 | self._closeCursors() |
| 218 | self.db[key] = value |
| 219 | |
| 220 | def __delitem__(self, key): |
| 221 | self._checkOpen() |
| 222 | self._closeCursors() |
| 223 | del self.db[key] |
| 224 | |
| 225 | def close(self): |
| 226 | self._closeCursors(save=0) |
| 227 | if self.dbc is not None: |
| 228 | self.dbc.close() |
| 229 | v = 0 |
| 230 | if self.db is not None: |
| 231 | v = self.db.close() |
| 232 | self.dbc = None |
| 233 | self.db = None |
| 234 | return v |
| 235 | |
| 236 | def keys(self): |
| 237 | self._checkOpen() |
| 238 | return self.db.keys() |
| 239 | |
| 240 | def has_key(self, key): |
| 241 | self._checkOpen() |
| 242 | return self.db.has_key(key) |
| 243 | |
| 244 | def set_location(self, key): |
| 245 | self._checkOpen() |
| 246 | self._checkCursor() |
| 247 | return self.dbc.set_range(key) |
| 248 | |
| 249 | def next(self): |
| 250 | self._checkOpen() |
| 251 | self._checkCursor() |
| 252 | rv = self.dbc.next() |
| 253 | return rv |
| 254 | |
| 255 | def previous(self): |
| 256 | self._checkOpen() |
| 257 | self._checkCursor() |
| 258 | rv = self.dbc.prev() |
| 259 | return rv |
| 260 | |
| 261 | def first(self): |
| 262 | self._checkOpen() |
| 263 | self._checkCursor() |
| 264 | rv = self.dbc.first() |
| 265 | return rv |
| 266 | |
| 267 | def last(self): |
| 268 | self._checkOpen() |
| 269 | self._checkCursor() |
| 270 | rv = self.dbc.last() |
| 271 | return rv |
| 272 | |
| 273 | def sync(self): |
| 274 | self._checkOpen() |
| 275 | return self.db.sync() |
| 276 | |
| 277 | |
| 278 | #---------------------------------------------------------------------- |
| 279 | # Compatibility object factory functions |
| 280 | |
| 281 | def hashopen(file, flag='c', mode=0666, pgsize=None, ffactor=None, nelem=None, |
| 282 | cachesize=None, lorder=None, hflags=0): |
| 283 | |
| 284 | flags = _checkflag(flag, file) |
| 285 | e = _openDBEnv() |
| 286 | d = db.DB(e) |
| 287 | d.set_flags(hflags) |
| 288 | if cachesize is not None: d.set_cachesize(0, cachesize) |
| 289 | if pgsize is not None: d.set_pagesize(pgsize) |
| 290 | if lorder is not None: d.set_lorder(lorder) |
| 291 | if ffactor is not None: d.set_h_ffactor(ffactor) |
| 292 | if nelem is not None: d.set_h_nelem(nelem) |
| 293 | d.open(file, db.DB_HASH, flags, mode) |
| 294 | return _DBWithCursor(d) |
| 295 | |
| 296 | #---------------------------------------------------------------------- |
| 297 | |
| 298 | def btopen(file, flag='c', mode=0666, |
| 299 | btflags=0, cachesize=None, maxkeypage=None, minkeypage=None, |
| 300 | pgsize=None, lorder=None): |
| 301 | |
| 302 | flags = _checkflag(flag, file) |
| 303 | e = _openDBEnv() |
| 304 | d = db.DB(e) |
| 305 | if cachesize is not None: d.set_cachesize(0, cachesize) |
| 306 | if pgsize is not None: d.set_pagesize(pgsize) |
| 307 | if lorder is not None: d.set_lorder(lorder) |
| 308 | d.set_flags(btflags) |
| 309 | if minkeypage is not None: d.set_bt_minkey(minkeypage) |
| 310 | if maxkeypage is not None: d.set_bt_maxkey(maxkeypage) |
| 311 | d.open(file, db.DB_BTREE, flags, mode) |
| 312 | return _DBWithCursor(d) |
| 313 | |
| 314 | #---------------------------------------------------------------------- |
| 315 | |
| 316 | |
| 317 | def rnopen(file, flag='c', mode=0666, |
| 318 | rnflags=0, cachesize=None, pgsize=None, lorder=None, |
| 319 | rlen=None, delim=None, source=None, pad=None): |
| 320 | |
| 321 | flags = _checkflag(flag, file) |
| 322 | e = _openDBEnv() |
| 323 | d = db.DB(e) |
| 324 | if cachesize is not None: d.set_cachesize(0, cachesize) |
| 325 | if pgsize is not None: d.set_pagesize(pgsize) |
| 326 | if lorder is not None: d.set_lorder(lorder) |
| 327 | d.set_flags(rnflags) |
| 328 | if delim is not None: d.set_re_delim(delim) |
| 329 | if rlen is not None: d.set_re_len(rlen) |
| 330 | if source is not None: d.set_re_source(source) |
| 331 | if pad is not None: d.set_re_pad(pad) |
| 332 | d.open(file, db.DB_RECNO, flags, mode) |
| 333 | return _DBWithCursor(d) |
| 334 | |
| 335 | #---------------------------------------------------------------------- |
| 336 | |
| 337 | def _openDBEnv(): |
| 338 | e = db.DBEnv() |
| 339 | e.open('.', db.DB_PRIVATE | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_LOCK | db.DB_INIT_MPOOL) |
| 340 | return e |
| 341 | |
| 342 | def _checkflag(flag, file): |
| 343 | if flag == 'r': |
| 344 | flags = db.DB_RDONLY |
| 345 | elif flag == 'rw': |
| 346 | flags = 0 |
| 347 | elif flag == 'w': |
| 348 | flags = db.DB_CREATE |
| 349 | elif flag == 'c': |
| 350 | flags = db.DB_CREATE |
| 351 | elif flag == 'n': |
| 352 | flags = db.DB_CREATE |
| 353 | #flags = db.DB_CREATE | db.DB_TRUNCATE |
| 354 | # we used db.DB_TRUNCATE flag for this before but BerkeleyDB |
| 355 | # 4.2.52 changed to disallowed truncate with txn environments. |
| 356 | if os.path.isfile(file): |
| 357 | os.unlink(file) |
| 358 | else: |
| 359 | raise error, "flags should be one of 'r', 'w', 'c' or 'n'" |
| 360 | return flags | db.DB_THREAD |
| 361 | |
| 362 | #---------------------------------------------------------------------- |
| 363 | |
| 364 | |
| 365 | # This is a silly little hack that allows apps to continue to use the |
| 366 | # DB_THREAD flag even on systems without threads without freaking out |
| 367 | # BerkeleyDB. |
| 368 | # |
| 369 | # This assumes that if Python was built with thread support then |
| 370 | # BerkeleyDB was too. |
| 371 | |
| 372 | try: |
| 373 | import thread |
| 374 | del thread |
| 375 | except ImportError: |
| 376 | db.DB_THREAD = 0 |
| 377 | |
| 378 | |
| 379 | #---------------------------------------------------------------------- |