Package dbf :: Module tables
[hide private]

Source Code for Module dbf.tables

   1  "table definitions" 
   2  import os 
   3  import sys 
   4  import csv 
   5  import codecs 
   6  import locale 
   7  import unicodedata 
   8  import weakref 
   9  from array import array 
  10  from bisect import bisect_left, bisect_right 
  11  from decimal import Decimal 
  12  from shutil import copyfileobj 
  13  import dbf 
  14  from dbf import _io as io 
  15  from dbf.dates import Date, DateTime, Time 
  16  from dbf.exceptions import Bof, Eof, DbfError, DataOverflow, FieldMissing, NonUnicode, DoNotIndex 
  17   
  18  input_decoding = locale.getdefaultlocale()[1]    # treat non-unicode data as ... 
  19  default_codepage = 'cp1252'  # if no codepage specified on dbf creation, use this 
  20  return_ascii = False         # if True -- convert back to icky ascii, losing chars if no mapping 
  21  temp_dir = os.environ.get("DBF_TEMP") or os.environ.get("TEMP") or "" 
  22   
  23  version_map = { 
  24          '\x02' : 'FoxBASE', 
  25          '\x03' : 'dBase III Plus', 
  26          '\x04' : 'dBase IV', 
  27          '\x05' : 'dBase V', 
  28          '\x30' : 'Visual FoxPro', 
  29          '\x31' : 'Visual FoxPro (auto increment field)', 
  30          '\x43' : 'dBase IV SQL', 
  31          '\x7b' : 'dBase IV w/memos', 
  32          '\x83' : 'dBase III Plus w/memos', 
  33          '\x8b' : 'dBase IV w/memos', 
  34          '\x8e' : 'dBase IV w/SQL table', 
  35          '\xf5' : 'FoxPro w/memos'} 
  36   
  37  code_pages = { 
  38          '\x00' : ('ascii', "plain ol' ascii"), 
  39          '\x01' : ('cp437', 'U.S. MS-DOS'), 
  40          '\x02' : ('cp850', 'International MS-DOS'), 
  41          '\x03' : ('cp1252', 'Windows ANSI'), 
  42          '\x04' : ('mac_roman', 'Standard Macintosh'), 
  43          '\x08' : ('cp865', 'Danish OEM'), 
  44          '\x09' : ('cp437', 'Dutch OEM'), 
  45          '\x0A' : ('cp850', 'Dutch OEM (secondary)'), 
  46          '\x0B' : ('cp437', 'Finnish OEM'), 
  47          '\x0D' : ('cp437', 'French OEM'), 
  48          '\x0E' : ('cp850', 'French OEM (secondary)'), 
  49          '\x0F' : ('cp437', 'German OEM'), 
  50          '\x10' : ('cp850', 'German OEM (secondary)'), 
  51          '\x11' : ('cp437', 'Italian OEM'), 
  52          '\x12' : ('cp850', 'Italian OEM (secondary)'), 
  53          '\x13' : ('cp932', 'Japanese Shift-JIS'), 
  54          '\x14' : ('cp850', 'Spanish OEM (secondary)'), 
  55          '\x15' : ('cp437', 'Swedish OEM'), 
  56          '\x16' : ('cp850', 'Swedish OEM (secondary)'), 
  57          '\x17' : ('cp865', 'Norwegian OEM'), 
  58          '\x18' : ('cp437', 'Spanish OEM'), 
  59          '\x19' : ('cp437', 'English OEM (Britain)'), 
  60          '\x1A' : ('cp850', 'English OEM (Britain) (secondary)'), 
  61          '\x1B' : ('cp437', 'English OEM (U.S.)'), 
  62          '\x1C' : ('cp863', 'French OEM (Canada)'), 
  63          '\x1D' : ('cp850', 'French OEM (secondary)'), 
  64          '\x1F' : ('cp852', 'Czech OEM'), 
  65          '\x22' : ('cp852', 'Hungarian OEM'), 
  66          '\x23' : ('cp852', 'Polish OEM'), 
  67          '\x24' : ('cp860', 'Portugese OEM'), 
  68          '\x25' : ('cp850', 'Potugese OEM (secondary)'), 
  69          '\x26' : ('cp866', 'Russian OEM'), 
  70          '\x37' : ('cp850', 'English OEM (U.S.) (secondary)'), 
  71          '\x40' : ('cp852', 'Romanian OEM'), 
  72          '\x4D' : ('cp936', 'Chinese GBK (PRC)'), 
  73          '\x4E' : ('cp949', 'Korean (ANSI/OEM)'), 
  74          '\x4F' : ('cp950', 'Chinese Big 5 (Taiwan)'), 
  75          '\x50' : ('cp874', 'Thai (ANSI/OEM)'), 
  76          '\x57' : ('cp1252', 'ANSI'), 
  77          '\x58' : ('cp1252', 'Western European ANSI'), 
  78          '\x59' : ('cp1252', 'Spanish ANSI'), 
  79          '\x64' : ('cp852', 'Eastern European MS-DOS'), 
  80          '\x65' : ('cp866', 'Russian MS-DOS'), 
  81          '\x66' : ('cp865', 'Nordic MS-DOS'), 
  82          '\x67' : ('cp861', 'Icelandic MS-DOS'), 
  83          '\x68' : (None, 'Kamenicky (Czech) MS-DOS'), 
  84          '\x69' : (None, 'Mazovia (Polish) MS-DOS'), 
  85          '\x6a' : ('cp737', 'Greek MS-DOS (437G)'), 
  86          '\x6b' : ('cp857', 'Turkish MS-DOS'), 
  87          '\x78' : ('cp950', 'Traditional Chinese (Hong Kong SAR, Taiwan) Windows'), 
  88          '\x79' : ('cp949', 'Korean Windows'), 
  89          '\x7a' : ('cp936', 'Chinese Simplified (PRC, Singapore) Windows'), 
  90          '\x7b' : ('cp932', 'Japanese Windows'), 
  91          '\x7c' : ('cp874', 'Thai Windows'), 
  92          '\x7d' : ('cp1255', 'Hebrew Windows'), 
  93          '\x7e' : ('cp1256', 'Arabic Windows'), 
  94          '\xc8' : ('cp1250', 'Eastern European Windows'), 
  95          '\xc9' : ('cp1251', 'Russian Windows'), 
  96          '\xca' : ('cp1254', 'Turkish Windows'), 
  97          '\xcb' : ('cp1253', 'Greek Windows'), 
  98          '\x96' : ('mac_cyrillic', 'Russian Macintosh'), 
  99          '\x97' : ('mac_latin2', 'Macintosh EE'), 
 100          '\x98' : ('mac_greek', 'Greek Macintosh') } 
 101   
 102  if sys.version_info[:2] < (2, 6): 
103 # define our own property type 104 - class property(object):
105 "Emulate PyProperty_Type() in Objects/descrobject.c" 106
107 - def __init__(self, fget=None, fset=None, fdel=None, doc=None):
108 self.fget = fget 109 self.fset = fset 110 self.fdel = fdel 111 self.__doc__ = doc or fget.__doc__
112 - def __call__(self, func):
113 self.fget = func 114 if not self.__doc__: 115 self.__doc__ = fget.__doc__
116 - def __get__(self, obj, objtype=None):
117 if obj is None: 118 return self 119 if self.fget is None: 120 raise AttributeError, "unreadable attribute" 121 return self.fget(obj)
122 - def __set__(self, obj, value):
123 if self.fset is None: 124 raise AttributeError, "can't set attribute" 125 self.fset(obj, value)
126 - def __delete__(self, obj):
127 if self.fdel is None: 128 raise AttributeError, "can't delete attribute" 129 self.fdel(obj)
130 - def setter(self, func):
131 self.fset = func 132 return self
133 - def deleter(self, func):
134 self.fdel = func 135 return self
136 # Internal classes
137 -class _DbfRecord(object):
138 """Provides routines to extract and save data within the fields of a dbf record.""" 139 __slots__ = ['_recnum', '_layout', '_data', '_dirty', '__weakref__']
140 - def _retrieveFieldValue(yo, record_data, fielddef):
141 """calls appropriate routine to fetch value stored in field from array 142 @param record_data: the data portion of the record 143 @type record_data: array of characters 144 @param fielddef: description of the field definition 145 @type fielddef: dictionary with keys 'type', 'start', 'length', 'end', 'decimals', and 'flags' 146 @returns: python data stored in field""" 147 148 field_type = fielddef['type'] 149 classtype = yo._layout.fieldtypes[field_type]['Class'] 150 retrieve = yo._layout.fieldtypes[field_type]['Retrieve'] 151 if classtype is not None: 152 datum = retrieve(record_data, fielddef, yo._layout.memo, classtype) 153 else: 154 datum = retrieve(record_data, fielddef, yo._layout.memo) 155 if field_type in yo._layout.character_fields: 156 datum = yo._layout.decoder(datum)[0] 157 if yo._layout.return_ascii: 158 try: 159 datum = yo._layout.output_encoder(datum)[0] 160 except UnicodeEncodeError: 161 datum = unicodedata.normalize('NFD', datum).encode('ascii','ignore') 162 return datum
163 - def _updateFieldValue(yo, fielddef, value):
164 "calls appropriate routine to convert value to ascii bytes, and save it in record" 165 field_type = fielddef['type'] 166 update = yo._layout.fieldtypes[field_type]['Update'] 167 if field_type in yo._layout.character_fields: 168 if not isinstance(value, unicode): 169 if yo._layout.input_decoder is None: 170 raise NonUnicode("String not in unicode format, no default encoding specified") 171 value = yo._layout.input_decoder(value)[0] # input ascii => unicode 172 value = yo._layout.encoder(value)[0] # unicode => table ascii 173 bytes = array('c', update(value, fielddef, yo._layout.memo)) 174 size = fielddef['length'] 175 if len(bytes) > size: 176 raise DataOverflow("tried to store %d bytes in %d byte field" % (len(bytes), size)) 177 blank = array('c', ' ' * size) 178 start = fielddef['start'] 179 end = start + size 180 blank[:len(bytes)] = bytes[:] 181 yo._data[start:end] = blank[:] 182 yo._dirty = True
183 - def _update_disk(yo, location='', data=None):
184 if not yo._layout.inmemory: 185 if yo._recnum < 0: 186 raise DbfError("Attempted to update record that has been packed") 187 if location == '': 188 location = yo._recnum * yo._layout.header.record_length + yo._layout.header.start 189 if data is None: 190 data = yo._data 191 yo._layout.dfd.seek(location) 192 yo._layout.dfd.write(data) 193 yo._dirty = False 194 for index in yo.record_table._indexen: 195 index(yo)
196 - def __call__(yo, *specs):
197 results = [] 198 if not specs: 199 specs = yo._layout.index 200 specs = _normalize_tuples(tuples=specs, length=2, filler=[_nop]) 201 for field, func in specs: 202 results.append(func(yo[field])) 203 return tuple(results)
204
205 - def __contains__(yo, key):
206 return key in yo._layout.fields or key in ['record_number','delete_flag']
207 - def __iter__(yo):
208 return (yo[field] for field in yo._layout.fields)
209 - def __getattr__(yo, name):
210 if name[0:2] == '__' and name[-2:] == '__': 211 raise AttributeError, 'Method %s is not implemented.' % name 212 elif name == 'record_number': 213 return yo._recnum 214 elif name == 'delete_flag': 215 return yo._data[0] != ' ' 216 elif not name in yo._layout.fields: 217 raise FieldMissing(name) 218 try: 219 fielddef = yo._layout[name] 220 value = yo._retrieveFieldValue(yo._data[fielddef['start']:fielddef['end']], fielddef) 221 return value 222 except DbfError, error: 223 error.message = "field --%s-- is %s -> %s" % (name, yo._layout.fieldtypes[fielddef['type']]['Type'], error.message) 224 raise
225 - def __getitem__(yo, item):
226 if type(item) in (int, long): 227 if not -yo._layout.header.field_count <= item < yo._layout.header.field_count: 228 raise IndexError("Field offset %d is not in record" % item) 229 return yo[yo._layout.fields[item]] 230 elif type(item) == slice: 231 sequence = [] 232 for index in yo._layout.fields[item]: 233 sequence.append(yo[index]) 234 return sequence 235 elif type(item) == str: 236 return yo.__getattr__(item) 237 else: 238 raise TypeError("%s is not a field name" % item)
239 - def __len__(yo):
240 return yo._layout.header.field_count
241 - def __new__(cls, recnum, layout, kamikaze='', _fromdisk=False):
242 """record = ascii array of entire record; layout=record specification; memo = memo object for table""" 243 record = object.__new__(cls) 244 record._dirty = False 245 record._recnum = recnum 246 record._layout = layout 247 if layout.blankrecord is None and not _fromdisk: 248 record._createBlankRecord() 249 record._data = layout.blankrecord 250 if recnum == -1: # not a disk-backed record 251 return record 252 elif type(kamikaze) == array: 253 record._data = kamikaze[:] 254 elif type(kamikaze) == str: 255 record._data = array('c', kamikaze) 256 else: 257 record._data = kamikaze._data[:] 258 datalen = len(record._data) 259 if datalen < layout.header.record_length: 260 record._data.extend(layout.blankrecord[datalen:]) 261 elif datalen > layout.header.record_length: 262 record._data = record._data[:layout.header.record_length] 263 if not _fromdisk and not layout.inmemory: 264 record._update_disk() 265 return record
266 - def __setattr__(yo, name, value):
267 if name in yo.__slots__: 268 object.__setattr__(yo, name, value) 269 return 270 elif not name in yo._layout.fields: 271 raise FieldMissing(name) 272 fielddef = yo._layout[name] 273 try: 274 yo._updateFieldValue(fielddef, value) 275 except DbfError, error: 276 error.message = "field --%s-- is %s -> %s" % (name, yo._layout.fieldtypes[fielddef['type']]['Type'], error.message) 277 error.data = name 278 raise
279 - def __setitem__(yo, name, value):
280 if type(name) == str: 281 yo.__setattr__(name, value) 282 elif type(name) in (int, long): 283 yo.__setattr__(yo._layout.fields[name], value) 284 elif type(name) == slice: 285 sequence = [] 286 for field in yo._layout.fields[name]: 287 sequence.append(field) 288 if len(sequence) != len(value): 289 raise DbfError("length of slices not equal") 290 for field, val in zip(sequence, value): 291 yo[field] = val 292 else: 293 raise TypeError("%s is not a field name" % name)
294 - def __str__(yo):
295 result = [] 296 for seq, field in enumerate(yo.field_names): 297 result.append("%3d - %-10s: %s" % (seq, field, yo[field])) 298 return '\n'.join(result)
299 - def __repr__(yo):
300 return yo._data.tostring()
301 - def _createBlankRecord(yo):
302 "creates a blank record data chunk" 303 layout = yo._layout 304 ondisk = layout.ondisk 305 layout.ondisk = False 306 yo._data = array('c', ' ' * layout.header.record_length) 307 layout.memofields = [] 308 for field in layout.fields: 309 yo._updateFieldValue(layout[field], layout.fieldtypes[layout[field]['type']]['Blank']()) 310 if layout[field]['type'] in layout.memotypes: 311 layout.memofields.append(field) 312 layout.blankrecord = yo._data[:] 313 layout.ondisk = ondisk
314 - def delete_record(yo):
315 "marks record as deleted" 316 yo._data[0] = '*' 317 yo._dirty = True 318 return yo
319 @property
320 - def field_names(yo):
321 "fields in table/record" 322 return yo._layout.fields[:]
323 - def gather_fields(yo, dictionary, drop=False): # dict, drop_missing=False):
324 "saves a dictionary into a record's fields\nkeys with no matching field will raise a FieldMissing exception unless drop_missing = True" 325 old_data = yo._data[:] 326 try: 327 for key in dictionary: 328 if not key in yo.field_names: 329 if drop: 330 continue 331 raise FieldMissing(key) 332 yo.__setattr__(key, dictionary[key]) 333 except: 334 yo._data[:] = old_data 335 raise 336 return yo
337 @property
338 - def has_been_deleted(yo):
339 "marked for deletion?" 340 return yo._data[0] == '*'
341 - def read_record(yo):
342 "refresh record data from disk" 343 size = yo._layout.header.record_length 344 location = yo._recnum * size + yo._layout.header.start 345 yo._layout.dfd.seek(location) 346 yo._data[:] = yo._meta.dfd.read(size) 347 yo._dirty = False 348 return yo
349 @property
350 - def record_number(yo):
351 "physical record number" 352 return yo._recnum
353 @property
354 - def record_table(yo):
355 table = yo._layout.table() 356 if table is None: 357 raise DbfError("table is no longer available") 358 return table
359 - def check_index(yo):
360 for dbfindex in yo._layout.table()._indexen: 361 dbfindex(yo)
362 - def reset_record(yo, keep_fields=None):
363 "blanks record" 364 if keep_fields is None: 365 keep_fields = [] 366 keep = {} 367 for field in keep_fields: 368 keep[field] = yo[field] 369 if yo._layout.blankrecord == None: 370 yo._createBlankRecord() 371 yo._data[:] = yo._layout.blankrecord[:] 372 for field in keep_fields: 373 yo[field] = keep[field] 374 yo._dirty = True 375 return yo
376 - def scatter_fields(yo, blank=False):
377 "returns a dictionary of fieldnames and values which can be used with gather_fields(). if blank is True, values are empty." 378 keys = yo._layout.fields 379 if blank: 380 values = [yo._layout.fieldtypes[yo._layout[key]['type']]['Blank']() for key in keys] 381 else: 382 values = [yo[field] for field in keys] 383 return dict(zip(keys, values))
384 - def undelete_record(yo):
385 "marks record as active" 386 yo._data[0] = ' ' 387 yo._dirty = True 388 return yo
389 - def write_record(yo, **kwargs):
390 "write record data to disk" 391 if kwargs: 392 yo.gather_fields(kwargs) 393 if yo._dirty: 394 yo._update_disk() 395 return 1 396 return 0
397 -class _DbfMemo(object):
398 """Provides access to memo fields as dictionaries 399 must override _init, _get_memo, and _put_memo to 400 store memo contents to disk"""
401 - def _init(yo):
402 "initialize disk file usage"
403 - def _get_memo(yo, block):
404 "retrieve memo contents from disk"
405 - def _put_memo(yo, data):
406 "store memo contents to disk"
407 - def __init__(yo, meta):
408 "" 409 yo.meta = meta 410 yo.memory = {} 411 yo.nextmemo = 1 412 yo._init() 413 yo.meta.newmemofile = False
414 - def get_memo(yo, block, field):
415 "gets the memo in block" 416 if yo.meta.ignorememos or not block: 417 return '' 418 if yo.meta.ondisk: 419 return yo._get_memo(block) 420 else: 421 return yo.memory[block]
422 - def put_memo(yo, data):
423 "stores data in memo file, returns block number" 424 if yo.meta.ignorememos or data == '': 425 return 0 426 if yo.meta.inmemory: 427 thismemo = yo.nextmemo 428 yo.nextmemo += 1 429 yo.memory[thismemo] = data 430 else: 431 thismemo = yo._put_memo(data) 432 return thismemo
433 -class _Db3Memo(_DbfMemo):
434 - def _init(yo):
435 "dBase III specific" 436 yo.meta.memo_size= 512 437 yo.record_header_length = 2 438 if yo.meta.ondisk and not yo.meta.ignorememos: 439 if yo.meta.newmemofile: 440 yo.meta.mfd = open(yo.meta.memoname, 'w+b') 441 yo.meta.mfd.write(io.packLongInt(1) + '\x00' * 508) 442 else: 443 try: 444 yo.meta.mfd = open(yo.meta.memoname, 'r+b') 445 yo.meta.mfd.seek(0) 446 yo.nextmemo = io.unpackLongInt(yo.meta.mfd.read(4)) 447 except: 448 raise DbfError("memo file appears to be corrupt")
449 - def _get_memo(yo, block):
450 block = int(block) 451 yo.meta.mfd.seek(block * yo.meta.memo_size) 452 eom = -1 453 data = '' 454 while eom == -1: 455 newdata = yo.meta.mfd.read(yo.meta.memo_size) 456 if not newdata: 457 return data 458 data += newdata 459 eom = data.find('\x1a\x1a') 460 return data[:eom].rstrip()
461 - def _put_memo(yo, data):
462 data = data.rstrip() 463 length = len(data) + yo.record_header_length # room for two ^Z at end of memo 464 blocks = length // yo.meta.memo_size 465 if length % yo.meta.memo_size: 466 blocks += 1 467 thismemo = yo.nextmemo 468 yo.nextmemo = thismemo + blocks 469 yo.meta.mfd.seek(0) 470 yo.meta.mfd.write(io.packLongInt(yo.nextmemo)) 471 yo.meta.mfd.seek(thismemo * yo.meta.memo_size) 472 yo.meta.mfd.write(data) 473 yo.meta.mfd.write('\x1a\x1a') 474 double_check = yo._get_memo(thismemo) 475 if len(double_check) != len(data): 476 uhoh = open('dbf_memo_dump.err','wb') 477 uhoh.write('thismemo: %d' % thismemo) 478 uhoh.write('nextmemo: %d' % yo.nextmemo) 479 uhoh.write('saved: %d bytes' % len(data)) 480 uhoh.write(data) 481 uhoh.write('retrieved: %d bytes' % len(double_check)) 482 uhoh.write(double_check) 483 uhoh.close() 484 raise DbfError("unknown error: memo not saved") 485 return thismemo
486 -class _VfpMemo(_DbfMemo):
487 - def _init(yo):
488 "Visual Foxpro 6 specific" 489 if yo.meta.ondisk and not yo.meta.ignorememos: 490 yo.record_header_length = 8 491 if yo.meta.newmemofile: 492 if yo.meta.memo_size == 0: 493 yo.meta.memo_size = 1 494 elif 1 < yo.meta.memo_size < 33: 495 yo.meta.memo_size *= 512 496 yo.meta.mfd = open(yo.meta.memoname, 'w+b') 497 nextmemo = 512 // yo.meta.memo_size 498 if nextmemo * yo.meta.memo_size < 512: 499 nextmemo += 1 500 yo.nextmemo = nextmemo 501 yo.meta.mfd.write(io.packLongInt(nextmemo, bigendian=True) + '\x00\x00' + \ 502 io.packShortInt(yo.meta.memo_size, bigendian=True) + '\x00' * 504) 503 else: 504 try: 505 yo.meta.mfd = open(yo.meta.memoname, 'r+b') 506 yo.meta.mfd.seek(0) 507 header = yo.meta.mfd.read(512) 508 yo.nextmemo = io.unpackLongInt(header[:4], bigendian=True) 509 yo.meta.memo_size = io.unpackShortInt(header[6:8], bigendian=True) 510 except: 511 raise DbfError("memo file appears to be corrupt")
512 - def _get_memo(yo, block):
513 yo.meta.mfd.seek(block * yo.meta.memo_size) 514 header = yo.meta.mfd.read(8) 515 length = io.unpackLongInt(header[4:], bigendian=True) 516 return yo.meta.mfd.read(length)
517 - def _put_memo(yo, data):
518 data = data.rstrip() # no trailing whitespace 519 yo.meta.mfd.seek(0) 520 thismemo = io.unpackLongInt(yo.meta.mfd.read(4), bigendian=True) 521 yo.meta.mfd.seek(0) 522 length = len(data) + yo.record_header_length # room for two ^Z at end of memo 523 blocks = length // yo.meta.memo_size 524 if length % yo.meta.memo_size: 525 blocks += 1 526 yo.meta.mfd.write(io.packLongInt(thismemo+blocks, bigendian=True)) 527 yo.meta.mfd.seek(thismemo*yo.meta.memo_size) 528 yo.meta.mfd.write('\x00\x00\x00\x01' + io.packLongInt(len(data), bigendian=True) + data) 529 return thismemo
530 # Public classes
531 -class DbfTable(object):
532 """Provides a framework for dbf style tables.""" 533 _version = 'basic memory table' 534 _versionabbv = 'dbf' 535 _fieldtypes = { 536 'D' : { 'Type':'Date', 'Init':io.addDate, 'Blank':Date.today, 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Class':None}, 537 'L' : { 'Type':'Logical', 'Init':io.addLogical, 'Blank':bool, 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Class':None}, 538 'M' : { 'Type':'Memo', 'Init':io.addMemo, 'Blank':str, 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Class':None} } 539 _memoext = '' 540 _memotypes = tuple('M', ) 541 _memoClass = _DbfMemo 542 _yesMemoMask = '' 543 _noMemoMask = '' 544 _fixed_fields = ('M','D','L') # always same length in table 545 _variable_fields = tuple() # variable length in table 546 _character_fields = tuple('M', ) # field representing character data 547 _decimal_fields = tuple() # text-based numeric fields 548 _numeric_fields = tuple() # fields representing a number 549 _currency_fields = tuple() 550 _dbfTableHeader = array('c', '\x00' * 32) 551 _dbfTableHeader[0] = '\x00' # table type - none 552 _dbfTableHeader[8:10] = array('c', io.packShortInt(33)) 553 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 554 _dbfTableHeader[29] = '\x00' # code page -- none, using plain ascii 555 _dbfTableHeader = _dbfTableHeader.tostring() 556 _dbfTableHeaderExtra = '' 557 _supported_tables = [] 558 _read_only = False 559 _meta_only = False 560 _use_deleted = True 561 backup = False
562 - class _DbfLists(object):
563 "implements the weakref structure for DbfLists"
564 - def __init__(yo):
565 yo._lists = set()
566 - def __iter__(yo):
567 yo._lists = set([s for s in yo._lists if s() is not None]) 568 return (s() for s in yo._lists if s() is not None)
569 - def __len__(yo):
570 yo._lists = set([s for s in yo._lists if s() is not None]) 571 return len(yo._lists)
572 - def add(yo, new_list):
573 yo._lists.add(weakref.ref(new_list)) 574 yo._lists = set([s for s in yo._lists if s() is not None])
575 - class _Indexen(object):
576 "implements the weakref structure for seperate indexes"
577 - def __init__(yo):
578 yo._indexen = set()
579 - def __iter__(yo):
580 yo._indexen = set([s for s in yo._indexen if s() is not None]) 581 return (s() for s in yo._indexen if s() is not None)
582 - def __len__(yo):
583 yo._indexen = set([s for s in yo._indexen if s() is not None]) 584 return len(yo._indexen)
585 - def add(yo, new_list):
586 yo._indexen.add(weakref.ref(new_list)) 587 yo._indexen = set([s for s in yo._indexen if s() is not None])
588 - class _MetaData(dict):
589 blankrecord = None 590 fields = None 591 filename = None 592 dfd = None 593 memoname = None 594 newmemofile = False 595 memo = None 596 mfd = None 597 ignorememos = False 598 memofields = None 599 current = -1
600 - class _TableHeader(object):
601 - def __init__(yo, data):
602 if len(data) != 32: 603 raise DbfError('table header should be 32 bytes, but is %d bytes' % len(data)) 604 yo._data = array('c', data + '\x0d')
605 - def codepage(yo, cp=None):
606 "get/set code page of table" 607 if cp is None: 608 return yo._data[29] 609 else: 610 cp, sd, ld = _codepage_lookup(cp) 611 yo._data[29] = cp 612 return cp
613 @property
614 - def data(yo):
615 "main data structure" 616 date = io.packDate(Date.today()) 617 yo._data[1:4] = array('c', date) 618 return yo._data.tostring()
619 @data.setter
620 - def data(yo, bytes):
621 if len(bytes) < 32: 622 raise DbfError("length for data of %d is less than 32" % len(bytes)) 623 yo._data[:] = array('c', bytes)
624 @property
625 - def extra(yo):
626 "extra dbf info (located after headers, before data records)" 627 fieldblock = yo._data[32:] 628 for i in range(len(fieldblock)//32+1): 629 cr = i * 32 630 if fieldblock[cr] == '\x0d': 631 break 632 else: 633 raise DbfError("corrupt field structure") 634 cr += 33 # skip past CR 635 return yo._data[cr:].tostring()
636 @extra.setter
637 - def extra(yo, data):
638 fieldblock = yo._data[32:] 639 for i in range(len(fieldblock)//32+1): 640 cr = i * 32 641 if fieldblock[cr] == '\x0d': 642 break 643 else: 644 raise DbfError("corrupt field structure") 645 cr += 33 # skip past CR 646 yo._data[cr:] = array('c', data) # extra 647 yo._data[8:10] = array('c', io.packShortInt(len(yo._data))) # start
648 @property
649 - def field_count(yo):
650 "number of fields (read-only)" 651 fieldblock = yo._data[32:] 652 for i in range(len(fieldblock)//32+1): 653 cr = i * 32 654 if fieldblock[cr] == '\x0d': 655 break 656 else: 657 raise DbfError("corrupt field structure") 658 return len(fieldblock[:cr]) // 32
659 @property
660 - def fields(yo):
661 "field block structure" 662 fieldblock = yo._data[32:] 663 for i in range(len(fieldblock)//32+1): 664 cr = i * 32 665 if fieldblock[cr] == '\x0d': 666 break 667 else: 668 raise DbfError("corrupt field structure") 669 return fieldblock[:cr].tostring()
670 @fields.setter
671 - def fields(yo, block):
672 fieldblock = yo._data[32:] 673 for i in range(len(fieldblock)//32+1): 674 cr = i * 32 675 if fieldblock[cr] == '\x0d': 676 break 677 else: 678 raise DbfError("corrupt field structure") 679 cr += 32 # convert to indexing main structure 680 fieldlen = len(block) 681 if fieldlen % 32 != 0: 682 raise DbfError("fields structure corrupt: %d is not a multiple of 32" % fieldlen) 683 yo._data[32:cr] = array('c', block) # fields 684 yo._data[8:10] = array('c', io.packShortInt(len(yo._data))) # start 685 fieldlen = fieldlen // 32 686 recordlen = 1 # deleted flag 687 for i in range(fieldlen): 688 recordlen += ord(block[i*32+16]) 689 yo._data[10:12] = array('c', io.packShortInt(recordlen))
690 @property
691 - def record_count(yo):
692 "number of records (maximum 16,777,215)" 693 return io.unpackLongInt(yo._data[4:8].tostring())
694 @record_count.setter
695 - def record_count(yo, count):
696 yo._data[4:8] = array('c', io.packLongInt(count))
697 @property
698 - def record_length(yo):
699 "length of a record (read_only) (max of 65,535)" 700 return io.unpackShortInt(yo._data[10:12].tostring())
701 @property
702 - def start(yo):
703 "starting position of first record in file (must be within first 64K)" 704 return io.unpackShortInt(yo._data[8:10].tostring())
705 @start.setter
706 - def start(yo, pos):
707 yo._data[8:10] = array('c', io.packShortInt(pos))
708 @property
709 - def update(yo):
710 "date of last table modification (read-only)" 711 return io.unpackDate(yo._data[1:4].tostring())
712 @property
713 - def version(yo):
714 "dbf version" 715 return yo._data[0]
716 @version.setter
717 - def version(yo, ver):
718 yo._data[0] = ver
719 - class _Table(object):
720 "implements the weakref table for records"
721 - def __init__(yo, count, meta):
722 yo._meta = meta 723 yo._weakref_list = [weakref.ref(lambda x: None)] * count
724 - def __getitem__(yo, index):
725 maybe = yo._weakref_list[index]() 726 if maybe is None: 727 if index < 0: 728 index += yo._meta.header.record_count 729 size = yo._meta.header.record_length 730 location = index * size + yo._meta.header.start 731 yo._meta.dfd.seek(location) 732 if yo._meta.dfd.tell() != location: 733 raise ValueError("unable to seek to offset %d in file" % location) 734 bytes = yo._meta.dfd.read(size) 735 if not bytes: 736 raise ValueError("unable to read record data from %s at location %d" % (yo._meta.filename, location)) 737 maybe = _DbfRecord(recnum=index, layout=yo._meta, kamikaze=bytes, _fromdisk=True) 738 yo._weakref_list[index] = weakref.ref(maybe) 739 return maybe
740 - def append(yo, record):
741 yo._weakref_list.append(weakref.ref(record))
742 - def clear(yo):
743 yo._weakref_list[:] = []
744 - def pop(yo):
745 return yo._weakref_list.pop()
746 - class DbfIterator(object):
747 "returns records using current index"
748 - def __init__(yo, table):
749 yo._table = table 750 yo._index = -1 751 yo._more_records = True
752 - def __iter__(yo):
753 return yo
754 - def next(yo):
755 while yo._more_records: 756 yo._index += 1 757 if yo._index >= len(yo._table): 758 yo._more_records = False 759 continue 760 record = yo._table[yo._index] 761 if not yo._table.use_deleted and record.has_been_deleted: 762 continue 763 return record 764 else: 765 raise StopIteration
766 - def _buildHeaderFields(yo):
767 "constructs fieldblock for disk table" 768 fieldblock = array('c', '') 769 memo = False 770 yo._meta.header.version = chr(ord(yo._meta.header.version) & ord(yo._noMemoMask)) 771 for field in yo._meta.fields: 772 if yo._meta.fields.count(field) > 1: 773 raise DbfError("corrupted field structure (noticed in _buildHeaderFields)") 774 fielddef = array('c', '\x00' * 32) 775 fielddef[:11] = array('c', io.packStr(field)) 776 fielddef[11] = yo._meta[field]['type'] 777 fielddef[12:16] = array('c', io.packLongInt(yo._meta[field]['start'])) 778 fielddef[16] = chr(yo._meta[field]['length']) 779 fielddef[17] = chr(yo._meta[field]['decimals']) 780 fielddef[18] = chr(yo._meta[field]['flags']) 781 fieldblock.extend(fielddef) 782 if yo._meta[field]['type'] in yo._meta.memotypes: 783 memo = True 784 yo._meta.header.fields = fieldblock.tostring() 785 if memo: 786 yo._meta.header.version = chr(ord(yo._meta.header.version) | ord(yo._yesMemoMask)) 787 if yo._meta.memo is None: 788 yo._meta.memo = yo._memoClass(yo._meta)
789 - def _checkMemoIntegrity(yo):
790 "dBase III specific" 791 if yo._meta.header.version == '\x83': 792 try: 793 yo._meta.memo = yo._memoClass(yo._meta) 794 except: 795 yo._meta.dfd.close() 796 yo._meta.dfd = None 797 raise 798 if not yo._meta.ignorememos: 799 for field in yo._meta.fields: 800 if yo._meta[field]['type'] in yo._memotypes: 801 if yo._meta.header.version != '\x83': 802 yo._meta.dfd.close() 803 yo._meta.dfd = None 804 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos") 805 elif not os.path.exists(yo._meta.memoname): 806 yo._meta.dfd.close() 807 yo._meta.dfd = None 808 raise DbfError("Table structure corrupt: memo fields exist without memo file") 809 break
810 - def _initializeFields(yo):
811 "builds the FieldList of names, types, and descriptions from the disk file" 812 yo._meta.fields[:] = [] 813 offset = 1 814 fieldsdef = yo._meta.header.fields 815 if len(fieldsdef) % 32 != 0: 816 raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef)) 817 if len(fieldsdef) // 32 != yo.field_count: 818 raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32)) 819 for i in range(yo.field_count): 820 fieldblock = fieldsdef[i*32:(i+1)*32] 821 name = io.unpackStr(fieldblock[:11]) 822 type = fieldblock[11] 823 if not type in yo._meta.fieldtypes: 824 raise DbfError("Unknown field type: %s" % type) 825 start = offset 826 length = ord(fieldblock[16]) 827 offset += length 828 end = start + length 829 decimals = ord(fieldblock[17]) 830 flags = ord(fieldblock[18]) 831 if name in yo._meta.fields: 832 raise DbfError('Duplicate field name found: %s' % name) 833 yo._meta.fields.append(name) 834 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
835 - def _fieldLayout(yo, i):
836 "Returns field information Name Type(Length[,Decimals])" 837 name = yo._meta.fields[i] 838 type = yo._meta[name]['type'] 839 length = yo._meta[name]['length'] 840 decimals = yo._meta[name]['decimals'] 841 if type in yo._decimal_fields: 842 description = "%s %s(%d,%d)" % (name, type, length, decimals) 843 elif type in yo._fixed_fields: 844 description = "%s %s" % (name, type) 845 else: 846 description = "%s %s(%d)" % (name, type, length) 847 return description
848 - def _loadtable(yo):
849 "loads the records from disk to memory" 850 if yo._meta_only: 851 raise DbfError("%s has been closed, records are unavailable" % yo.filename) 852 dfd = yo._meta.dfd 853 header = yo._meta.header 854 dfd.seek(header.start) 855 allrecords = dfd.read() # kludge to get around mysterious errno 0 problems 856 dfd.seek(0) 857 length = header.record_length 858 for i in range(header.record_count): 859 record_data = allrecords[length*i:length*i+length] 860 yo._table.append(_DbfRecord(i, yo._meta, allrecords[length*i:length*i+length], _fromdisk=True)) 861 dfd.seek(0)
862 - def _list_fields(yo, specs, sep=','):
863 if specs is None: 864 specs = yo.field_names 865 elif isinstance(specs, str): 866 specs = specs.split(sep) 867 else: 868 specs = list(specs) 869 specs = [s.strip() for s in specs] 870 return specs
871 - def _update_disk(yo, headeronly=False):
872 "synchronizes the disk file with current data" 873 if yo._meta.inmemory: 874 return 875 fd = yo._meta.dfd 876 fd.seek(0) 877 fd.write(yo._meta.header.data) 878 if not headeronly: 879 for record in yo._table: 880 record._update_disk() 881 fd.flush() 882 fd.truncate(yo._meta.header.start + yo._meta.header.record_count * yo._meta.header.record_length) 883 if 'db3' in yo._versionabbv: 884 fd.seek(0, os.SEEK_END) 885 fd.write('\x1a') # required for dBase III 886 fd.flush() 887 fd.truncate(yo._meta.header.start + yo._meta.header.record_count * yo._meta.header.record_length + 1)
888
889 - def __contains__(yo, key):
890 return key in yo.field_names
891 - def __enter__(yo):
892 return yo
893 - def __exit__(yo, *exc_info):
894 yo.close()
895 - def __getattr__(yo, name):
896 if name in ('_table'): 897 if yo._meta.ondisk: 898 yo._table = yo._Table(len(yo), yo._meta) 899 else: 900 yo._table = [] 901 yo._loadtable() 902 return object.__getattribute__(yo, name)
903 - def __getitem__(yo, value):
904 if type(value) == int: 905 if not -yo._meta.header.record_count <= value < yo._meta.header.record_count: 906 raise IndexError("Record %d is not in table." % value) 907 return yo._table[value] 908 elif type(value) == slice: 909 sequence = List(desc='%s --> %s' % (yo.filename, value), field_names=yo.field_names) 910 yo._dbflists.add(sequence) 911 for index in range(len(yo))[value]: 912 record = yo._table[index] 913 if yo.use_deleted is True or not record.has_been_deleted: 914 sequence.append(record) 915 return sequence 916 else: 917 raise TypeError('type <%s> not valid for indexing' % type(value))
918 - def __init__(yo, filename=':memory:', field_specs=None, memo_size=128, ignore_memos=False, 919 read_only=False, keep_memos=False, meta_only=False, codepage=None, 920 numbers='default', strings=str, currency=Decimal):
921 """open/create dbf file 922 filename should include path if needed 923 field_specs can be either a ;-delimited string or a list of strings 924 memo_size is always 512 for db3 memos 925 ignore_memos is useful if the memo file is missing or corrupt 926 read_only will load records into memory, then close the disk file 927 keep_memos will also load any memo fields into memory 928 meta_only will ignore all records, keeping only basic table information 929 codepage will override whatever is set in the table itself""" 930 if filename[0] == filename[-1] == ':': 931 if field_specs is None: 932 raise DbfError("field list must be specified for memory tables") 933 elif type(yo) is DbfTable: 934 raise DbfError("only memory tables supported") 935 yo._dbflists = yo._DbfLists() 936 yo._indexen = yo._Indexen() 937 yo._meta = meta = yo._MetaData() 938 for datatypes, classtype in ( 939 (yo._character_fields, strings), 940 (yo._numeric_fields, numbers), 941 (yo._currency_fields, currency), 942 ): 943 for datatype in datatypes: 944 yo._fieldtypes[datatype]['Class'] = classtype 945 meta.numbers = numbers 946 meta.strings = strings 947 meta.currency = currency 948 meta.table = weakref.ref(yo) 949 meta.filename = filename 950 meta.fields = [] 951 meta.fieldtypes = yo._fieldtypes 952 meta.fixed_fields = yo._fixed_fields 953 meta.variable_fields = yo._variable_fields 954 meta.character_fields = yo._character_fields 955 meta.decimal_fields = yo._decimal_fields 956 meta.numeric_fields = yo._numeric_fields 957 meta.memotypes = yo._memotypes 958 meta.ignorememos = ignore_memos 959 meta.memo_size = memo_size 960 meta.input_decoder = codecs.getdecoder(input_decoding) # from ascii to unicode 961 meta.output_encoder = codecs.getencoder(input_decoding) # and back to ascii 962 meta.return_ascii = return_ascii 963 meta.header = header = yo._TableHeader(yo._dbfTableHeader) 964 header.extra = yo._dbfTableHeaderExtra 965 header.data #force update of date 966 if filename[0] == filename[-1] == ':': 967 yo._table = [] 968 meta.ondisk = False 969 meta.inmemory = True 970 meta.memoname = filename 971 else: 972 base, ext = os.path.splitext(filename) 973 if ext == '': 974 meta.filename = base + '.dbf' 975 meta.memoname = base + yo._memoext 976 meta.ondisk = True 977 meta.inmemory = False 978 if field_specs: 979 if meta.ondisk: 980 meta.dfd = open(meta.filename, 'w+b') 981 meta.newmemofile = True 982 yo.add_fields(field_specs) 983 header.codepage(codepage or default_codepage) 984 cp, sd, ld = _codepage_lookup(meta.header.codepage()) 985 meta.decoder = codecs.getdecoder(sd) 986 meta.encoder = codecs.getencoder(sd) 987 return 988 try: 989 dfd = meta.dfd = open(meta.filename, 'r+b') 990 except IOError, e: 991 raise DbfError(str(e)) 992 dfd.seek(0) 993 meta.header = header = yo._TableHeader(dfd.read(32)) 994 if not header.version in yo._supported_tables: 995 dfd.close() 996 dfd = None 997 raise DbfError("Unsupported dbf type: %s [%x]" % (version_map.get(meta.header.version, 'Unknown: %s' % meta.header.version), ord(meta.header.version))) 998 cp, sd, ld = _codepage_lookup(meta.header.codepage()) 999 yo._meta.decoder = codecs.getdecoder(sd) 1000 yo._meta.encoder = codecs.getencoder(sd) 1001 fieldblock = dfd.read(header.start - 32) 1002 for i in range(len(fieldblock)//32+1): 1003 fieldend = i * 32 1004 if fieldblock[fieldend] == '\x0d': 1005 break 1006 else: 1007 raise DbfError("corrupt field structure in header") 1008 if len(fieldblock[:fieldend]) % 32 != 0: 1009 raise DbfError("corrupt field structure in header") 1010 header.fields = fieldblock[:fieldend] 1011 header.extra = fieldblock[fieldend+1:] # skip trailing \r 1012 yo._initializeFields() 1013 yo._checkMemoIntegrity() 1014 meta.current = -1 1015 if len(yo) > 0: 1016 meta.current = 0 1017 dfd.seek(0) 1018 if meta_only: 1019 yo.close(keep_table=False, keep_memos=False) 1020 elif read_only: 1021 yo.close(keep_table=True, keep_memos=keep_memos) 1022 if codepage is not None: 1023 cp, sd, ld = _codepage_lookup(codepage) 1024 yo._meta.decoder = codecs.getdecoder(sd) 1025 yo._meta.encoder = codecs.getencoder(sd)
1026
1027 - def __iter__(yo):
1028 return yo.DbfIterator(yo)
1029 - def __len__(yo):
1030 return yo._meta.header.record_count
1031 - def __nonzero__(yo):
1032 return yo._meta.header.record_count != 0
1033 - def __repr__(yo):
1034 if yo._read_only: 1035 return __name__ + ".Table('%s', read_only=True)" % yo._meta.filename 1036 elif yo._meta_only: 1037 return __name__ + ".Table('%s', meta_only=True)" % yo._meta.filename 1038 else: 1039 return __name__ + ".Table('%s')" % yo._meta.filename
1040 - def __str__(yo):
1041 if yo._read_only: 1042 status = "read-only" 1043 elif yo._meta_only: 1044 status = "meta-only" 1045 else: 1046 status = "read/write" 1047 str = """ 1048 Table: %s 1049 Type: %s 1050 Codepage: %s 1051 Status: %s 1052 Last updated: %s 1053 Record count: %d 1054 Field count: %d 1055 Record length: %d """ % (yo.filename, version_map.get(yo._meta.header.version, 1056 'unknown - ' + hex(ord(yo._meta.header.version))), yo.codepage, status, 1057 yo.last_update, len(yo), yo.field_count, yo.record_length) 1058 str += "\n --Fields--\n" 1059 for i in range(len(yo._meta.fields)): 1060 str += "%11d) %s\n" % (i, yo._fieldLayout(i)) 1061 return str
1062 @property
1063 - def codepage(yo):
1064 return "%s (%s)" % code_pages[yo._meta.header.codepage()]
1065 @codepage.setter
1066 - def codepage(yo, cp):
1067 cp = code_pages[yo._meta.header.codepage(cp)][0] 1068 yo._meta.decoder = codecs.getdecoder(cp) 1069 yo._meta.encoder = codecs.getencoder(cp) 1070 yo._update_disk(headeronly=True)
1071 @property
1072 - def field_count(yo):
1073 "the number of fields in the table" 1074 return yo._meta.header.field_count
1075 @property
1076 - def field_names(yo):
1077 "a list of the fields in the table" 1078 return yo._meta.fields[:]
1079 @property
1080 - def filename(yo):
1081 "table's file name, including path (if specified on open)" 1082 return yo._meta.filename
1083 @property
1084 - def last_update(yo):
1085 "date of last update" 1086 return yo._meta.header.update
1087 @property
1088 - def memoname(yo):
1089 "table's memo name (if path included in filename on open)" 1090 return yo._meta.memoname
1091 @property
1092 - def record_length(yo):
1093 "number of bytes in a record" 1094 return yo._meta.header.record_length
1095 @property
1096 - def record_number(yo):
1097 "index number of the current record" 1098 return yo._meta.current
1099 @property
1100 - def supported_tables(yo):
1101 "allowable table types" 1102 return yo._supported_tables
1103 @property
1104 - def use_deleted(yo):
1105 "process or ignore deleted records" 1106 return yo._use_deleted
1107 @use_deleted.setter
1108 - def use_deleted(yo, new_setting):
1109 yo._use_deleted = new_setting
1110 @property
1111 - def version(yo):
1112 "returns the dbf type of the table" 1113 return yo._version
1114 - def add_fields(yo, field_specs):
1115 """adds field(s) to the table layout; format is Name Type(Length,Decimals)[; Name Type(Length,Decimals)[...]] 1116 backup table is created with _backup appended to name 1117 then modifies current structure""" 1118 all_records = [record for record in yo] 1119 if yo: 1120 yo.create_backup() 1121 yo._meta.blankrecord = None 1122 meta = yo._meta 1123 offset = meta.header.record_length 1124 fields = yo._list_fields(field_specs, sep=';') 1125 for field in fields: 1126 try: 1127 name, format = field.split() 1128 if name[0] == '_' or name[0].isdigit() or not name.replace('_','').isalnum(): 1129 raise DbfError("%s invalid: field names must start with a letter, and can only contain letters, digits, and _" % name) 1130 name = name.lower() 1131 if name in meta.fields: 1132 raise DbfError("Field '%s' already exists" % name) 1133 field_type = format[0].upper() 1134 if len(name) > 10: 1135 raise DbfError("Maximum field name length is 10. '%s' is %d characters long." % (name, len(name))) 1136 if not field_type in meta.fieldtypes.keys(): 1137 raise DbfError("Unknown field type: %s" % field_type) 1138 length, decimals = yo._meta.fieldtypes[field_type]['Init'](format) 1139 except ValueError: 1140 raise DbfError("invalid field specifier: %s (multiple fields should be separated with ';'" % field) 1141 start = offset 1142 end = offset + length 1143 offset = end 1144 meta.fields.append(name) 1145 meta[name] = {'type':field_type, 'start':start, 'length':length, 'end':end, 'decimals':decimals, 'flags':0} 1146 if meta[name]['type'] in yo._memotypes and meta.memo is None: 1147 meta.memo = yo._memoClass(meta) 1148 for record in yo: 1149 record[name] = meta.fieldtypes[field_type]['Blank']() 1150 yo._buildHeaderFields() 1151 yo._update_disk()
1152 - def append(yo, kamikaze='', drop=False, multiple=1):
1153 "adds <multiple> blank records, and fills fields with dict/tuple values if present" 1154 if not yo.field_count: 1155 raise DbfError("No fields defined, cannot append") 1156 empty_table = len(yo) == 0 1157 dictdata = False 1158 tupledata = False 1159 if not isinstance(kamikaze, _DbfRecord): 1160 if isinstance(kamikaze, dict): 1161 dictdata = kamikaze 1162 kamikaze = '' 1163 elif isinstance(kamikaze, tuple): 1164 tupledata = kamikaze 1165 kamikaze = '' 1166 newrecord = _DbfRecord(recnum=yo._meta.header.record_count, layout=yo._meta, kamikaze=kamikaze) 1167 yo._table.append(newrecord) 1168 yo._meta.header.record_count += 1 1169 try: 1170 if dictdata: 1171 newrecord.gather_fields(dictdata, drop=drop) 1172 elif tupledata: 1173 for index, item in enumerate(tupledata): 1174 newrecord[index] = item 1175 elif kamikaze == str: 1176 for field in yo._meta.memofields: 1177 newrecord[field] = '' 1178 elif kamikaze: 1179 for field in yo._meta.memofields: 1180 newrecord[field] = kamikaze[field] 1181 newrecord.write_record() 1182 except Exception: 1183 yo._table.pop() # discard failed record 1184 yo._meta.header.record_count = yo._meta.header.record_count - 1 1185 yo._update_disk() 1186 raise 1187 multiple -= 1 1188 if multiple: 1189 data = newrecord._data 1190 single = yo._meta.header.record_count 1191 total = single + multiple 1192 while single < total: 1193 multi_record = _DbfRecord(single, yo._meta, kamikaze=data) 1194 yo._table.append(multi_record) 1195 for field in yo._meta.memofields: 1196 multi_record[field] = newrecord[field] 1197 single += 1 1198 multi_record.write_record() 1199 yo._meta.header.record_count = total # += multiple 1200 yo._meta.current = yo._meta.header.record_count - 1 1201 newrecord = multi_record 1202 yo._update_disk(headeronly=True) 1203 if empty_table: 1204 yo._meta.current = 0 1205 return newrecord
1206 - def bof(yo, _move=False):
1207 "moves record pointer to previous usable record; returns True if no more usable records" 1208 current = yo._meta.current 1209 try: 1210 while yo._meta.current > 0: 1211 yo._meta.current -= 1 1212 if yo.use_deleted or not yo.current().has_been_deleted: 1213 break 1214 else: 1215 yo._meta.current = -1 1216 return True 1217 return False 1218 finally: 1219 if not _move: 1220 yo._meta.current = current
1221 - def bottom(yo, get_record=False):
1222 """sets record pointer to bottom of table 1223 if get_record, seeks to and returns last (non-deleted) record 1224 DbfError if table is empty 1225 Bof if all records deleted and use_deleted is False""" 1226 yo._meta.current = yo._meta.header.record_count 1227 if get_record: 1228 try: 1229 return yo.prev() 1230 except Bof: 1231 yo._meta.current = yo._meta.header.record_count 1232 raise Eof()
1233 - def close(yo, keep_table=False, keep_memos=False):
1234 """closes disk files 1235 ensures table data is available if keep_table 1236 ensures memo data is available if keep_memos""" 1237 yo._meta.inmemory = True 1238 if keep_table: 1239 replacement_table = [] 1240 for record in yo._table: 1241 replacement_table.append(record) 1242 yo._table = replacement_table 1243 else: 1244 if yo._meta.ondisk: 1245 yo._meta_only = True 1246 if yo._meta.mfd is not None: 1247 if not keep_memos: 1248 yo._meta.ignorememos = True 1249 else: 1250 memo_fields = [] 1251 for field in yo.field_names: 1252 if yo.is_memotype(field): 1253 memo_fields.append(field) 1254 for record in yo: 1255 for field in memo_fields: 1256 record[field] = record[field] 1257 yo._meta.mfd.close() 1258 yo._meta.mfd = None 1259 if yo._meta.ondisk: 1260 yo._meta.dfd.close() 1261 yo._meta.dfd = None 1262 if keep_table: 1263 yo._read_only = True 1264 yo._meta.ondisk = False
1265 - def create_backup(yo, new_name=None, overwrite=False):
1266 "creates a backup table -- ignored if memory table" 1267 if yo.filename[0] == yo.filename[-1] == ':': 1268 return 1269 if new_name is None: 1270 upper = yo.filename.isupper() 1271 name, ext = os.path.splitext(os.path.split(yo.filename)[1]) 1272 extra = '_BACKUP' if upper else '_backup' 1273 new_name = os.path.join(temp_dir, name + extra + ext) 1274 else: 1275 overwrite = True 1276 if overwrite or not yo.backup: 1277 bkup = open(new_name, 'wb') 1278 try: 1279 yo._meta.dfd.seek(0) 1280 copyfileobj(yo._meta.dfd, bkup) 1281 yo.backup = new_name 1282 finally: 1283 bkup.close()
1284 - def create_index(yo, key):
1285 return Index(yo, key)
1286 - def current(yo, index=False):
1287 "returns current logical record, or its index" 1288 if yo._meta.current < 0: 1289 raise Bof() 1290 elif yo._meta.current >= yo._meta.header.record_count: 1291 raise Eof() 1292 if index: 1293 return yo._meta.current 1294 return yo._table[yo._meta.current]
1295 - def delete_fields(yo, doomed):
1296 """removes field(s) from the table 1297 creates backup files with _backup appended to the file name, 1298 then modifies current structure""" 1299 doomed = yo._list_fields(doomed) 1300 for victim in doomed: 1301 if victim not in yo._meta.fields: 1302 raise DbfError("field %s not in table -- delete aborted" % victim) 1303 all_records = [record for record in yo] 1304 yo.create_backup() 1305 for victim in doomed: 1306 yo._meta.fields.pop(yo._meta.fields.index(victim)) 1307 start = yo._meta[victim]['start'] 1308 end = yo._meta[victim]['end'] 1309 for record in yo: 1310 record._data = record._data[:start] + record._data[end:] 1311 for field in yo._meta.fields: 1312 if yo._meta[field]['start'] == end: 1313 end = yo._meta[field]['end'] 1314 yo._meta[field]['start'] = start 1315 yo._meta[field]['end'] = start + yo._meta[field]['length'] 1316 start = yo._meta[field]['end'] 1317 yo._buildHeaderFields() 1318 yo._update_disk()
1319 - def eof(yo, _move=False):
1320 "moves record pointer to next usable record; returns True if no more usable records" 1321 current = yo._meta.current 1322 try: 1323 while yo._meta.current < yo._meta.header.record_count - 1: 1324 yo._meta.current += 1 1325 if yo.use_deleted or not yo.current().has_been_deleted: 1326 break 1327 else: 1328 yo._meta.current = yo._meta.header.record_count 1329 return True 1330 return False 1331 finally: 1332 if not _move: 1333 yo._meta.current = current
1334 - def export(yo, records=None, filename=None, field_specs=None, format='csv', header=True):
1335 """writes the table using CSV or tab-delimited format, using the filename 1336 given if specified, otherwise the table name""" 1337 if filename is not None: 1338 path, filename = os.path.split(filename) 1339 else: 1340 path, filename = os.path.split(yo.filename) 1341 filename = os.path.join(path, filename) 1342 field_specs = yo._list_fields(field_specs) 1343 if records is None: 1344 records = yo 1345 format = format.lower() 1346 if format not in ('csv', 'tab', 'fixed'): 1347 raise DbfError("export format: csv, tab, or fixed -- not %s" % format) 1348 if format == 'fixed': 1349 format = 'txt' 1350 base, ext = os.path.splitext(filename) 1351 if ext.lower() in ('', '.dbf'): 1352 filename = base + "." + format[:3] 1353 fd = open(filename, 'w') 1354 try: 1355 if format == 'csv': 1356 csvfile = csv.writer(fd, dialect='dbf') 1357 if header: 1358 csvfile.writerow(field_specs) 1359 for record in records: 1360 fields = [] 1361 for fieldname in field_specs: 1362 fields.append(record[fieldname]) 1363 csvfile.writerow(fields) 1364 elif format == 'tab': 1365 if header: 1366 fd.write('\t'.join(field_specs) + '\n') 1367 for record in records: 1368 fields = [] 1369 for fieldname in field_specs: 1370 fields.append(str(record[fieldname])) 1371 fd.write('\t'.join(fields) + '\n') 1372 else: # format == 'fixed' 1373 header = open("%s_layout.txt" % os.path.splitext(filename)[0], 'w') 1374 header.write("%-15s Size\n" % "Field Name") 1375 header.write("%-15s ----\n" % ("-" * 15)) 1376 sizes = [] 1377 for field in field_specs: 1378 size = yo.size(field)[0] 1379 sizes.append(size) 1380 header.write("%-15s %3d\n" % (field, size)) 1381 header.write('\nTotal Records in file: %d\n' % len(records)) 1382 header.close() 1383 for record in records: 1384 fields = [] 1385 for i, field_name in enumerate(field_specs): 1386 fields.append("%-*s" % (sizes[i], record[field_name])) 1387 fd.write(''.join(fields) + '\n') 1388 finally: 1389 fd.close() 1390 fd = None 1391 return len(records)
1392 - def get_record(yo, recno):
1393 "returns record at physical_index[recno]" 1394 return yo._table[recno]
1395 - def goto(yo, criteria):
1396 """changes the record pointer to the first matching (non-deleted) record 1397 criteria should be either a tuple of tuple(value, field, func) triples, 1398 or an integer to go to""" 1399 if isinstance(criteria, int): 1400 if not -yo._meta.header.record_count <= criteria < yo._meta.header.record_count: 1401 raise IndexError("Record %d does not exist" % criteria) 1402 if criteria < 0: 1403 criteria += yo._meta.header.record_count 1404 yo._meta.current = criteria 1405 return yo.current() 1406 criteria = _normalize_tuples(tuples=criteria, length=3, filler=[_nop]) 1407 specs = tuple([(field, func) for value, field, func in criteria]) 1408 match = tuple([value for value, field, func in criteria]) 1409 current = yo.current(index=True) 1410 matchlen = len(match) 1411 while not yo.Eof(): 1412 record = yo.current() 1413 results = record(*specs) 1414 if results == match: 1415 return record 1416 return yo.goto(current)
1417 - def is_decimal(yo, name):
1418 "returns True if name is a variable-length field type" 1419 return yo._meta[name]['type'] in yo._decimal_fields
1420 - def is_memotype(yo, name):
1421 "returns True if name is a memo type field" 1422 return yo._meta[name]['type'] in yo._memotypes
1423 - def new(yo, filename, field_specs=None, codepage=None):
1424 "returns a new table of the same type" 1425 if field_specs is None: 1426 field_specs = yo.structure() 1427 if not (filename[0] == filename[-1] == ':'): 1428 path, name = os.path.split(filename) 1429 if path == "": 1430 filename = os.path.join(os.path.split(yo.filename)[0], filename) 1431 elif name == "": 1432 filename = os.path.join(path, os.path.split(yo.filename)[1]) 1433 if codepage is None: 1434 codepage = yo._meta.header.codepage()[0] 1435 return yo.__class__(filename, field_specs, codepage=codepage)
1436 - def next(yo):
1437 "set record pointer to next (non-deleted) record, and return it" 1438 if yo.eof(_move=True): 1439 raise Eof() 1440 return yo.current()
1441 - def open(yo):
1442 meta = yo._meta 1443 meta.inmemory = False 1444 meta.ondisk = True 1445 yo._read_only = False 1446 yo._meta_only = False 1447 if '_table' in dir(yo): 1448 del yo._table 1449 dfd = meta.dfd = open(meta.filename, 'r+b') 1450 dfd.seek(0) 1451 meta.header = header = yo._TableHeader(dfd.read(32)) 1452 if not header.version in yo._supported_tables: 1453 dfd.close() 1454 dfd = None 1455 raise DbfError("Unsupported dbf type: %s [%x]" % (version_map.get(meta.header.version, 'Unknown: %s' % meta.header.version), ord(meta.header.version))) 1456 cp, sd, ld = _codepage_lookup(meta.header.codepage()) 1457 meta.decoder = codecs.getdecoder(sd) 1458 meta.encoder = codecs.getencoder(sd) 1459 fieldblock = dfd.read(header.start - 32) 1460 for i in range(len(fieldblock)//32+1): 1461 fieldend = i * 32 1462 if fieldblock[fieldend] == '\x0d': 1463 break 1464 else: 1465 raise DbfError("corrupt field structure in header") 1466 if len(fieldblock[:fieldend]) % 32 != 0: 1467 raise DbfError("corrupt field structure in header") 1468 header.fields = fieldblock[:fieldend] 1469 header.extra = fieldblock[fieldend+1:] # skip trailing \r 1470 yo._initializeFields() 1471 yo._checkMemoIntegrity() 1472 meta.current = -1 1473 if len(yo) > 0: 1474 meta.current = 0 1475 dfd.seek(0)
1476
1477 - def pack(yo, _pack=True):
1478 "physically removes all deleted records" 1479 for dbfindex in yo._indexen: 1480 dbfindex.clear() 1481 newtable = [] 1482 index = 0 1483 offset = 0 # +1 for each purged record 1484 for record in yo._table: 1485 found = False 1486 if record.has_been_deleted and _pack: 1487 for dbflist in yo._dbflists: 1488 if dbflist._purge(record, record.record_number - offset, 1): 1489 found = True 1490 record._recnum = -1 1491 else: 1492 record._recnum = index 1493 newtable.append(record) 1494 index += 1 1495 if found: 1496 offset += 1 1497 found = False 1498 yo._table.clear() 1499 for record in newtable: 1500 yo._table.append(record) 1501 yo._meta.header.record_count = index 1502 yo._current = -1 1503 yo._update_disk() 1504 yo.reindex()
1505 - def prev(yo):
1506 "set record pointer to previous (non-deleted) record, and return it" 1507 if yo.bof(_move=True): 1508 raise Bof 1509 return yo.current()
1510 - def query(yo, sql_command=None, python=None):
1511 "uses exec to perform queries on the table" 1512 if sql_command: 1513 return sql(yo, sql_command) 1514 elif python is None: 1515 raise DbfError("query: python parameter must be specified") 1516 possible = List(desc="%s --> %s" % (yo.filename, python), field_names=yo.field_names) 1517 yo._dbflists.add(possible) 1518 query_result = {} 1519 select = 'query_result["keep"] = %s' % python 1520 g = {} 1521 use_deleted = yo.use_deleted 1522 for record in yo: 1523 query_result['keep'] = False 1524 g['query_result'] = query_result 1525 exec select in g, record 1526 if query_result['keep']: 1527 possible.append(record) 1528 record.write_record() 1529 return possible
1530 - def reindex(yo):
1531 for dbfindex in yo._indexen: 1532 dbfindex.reindex()
1533 - def rename_field(yo, oldname, newname):
1534 "renames an existing field" 1535 if yo: 1536 yo.create_backup() 1537 if not oldname in yo._meta.fields: 1538 raise DbfError("field --%s-- does not exist -- cannot rename it." % oldname) 1539 if newname[0] == '_' or newname[0].isdigit() or not newname.replace('_','').isalnum(): 1540 raise DbfError("field names cannot start with _ or digits, and can only contain the _, letters, and digits") 1541 newname = newname.lower() 1542 if newname in yo._meta.fields: 1543 raise DbfError("field --%s-- already exists" % newname) 1544 if len(newname) > 10: 1545 raise DbfError("maximum field name length is 10. '%s' is %d characters long." % (newname, len(newname))) 1546 yo._meta[newname] = yo._meta[oldname] 1547 yo._meta.fields[yo._meta.fields.index(oldname)] = newname 1548 yo._buildHeaderFields() 1549 yo._update_disk(headeronly=True)
1550 - def resize_field(yo, doomed, new_size):
1551 """resizes field (C only at this time) 1552 creates backup file, then modifies current structure""" 1553 if not 0 < new_size < 256: 1554 raise DbfError("new_size must be between 1 and 255 (use delete_fields to remove a field)") 1555 doomed = yo._list_fields(doomed) 1556 for victim in doomed: 1557 if victim not in yo._meta.fields: 1558 raise DbfError("field %s not in table -- resize aborted" % victim) 1559 all_records = [record for record in yo] 1560 yo.create_backup() 1561 for victim in doomed: 1562 start = yo._meta[victim]['start'] 1563 end = yo._meta[victim]['end'] 1564 eff_end = min(yo._meta[victim]['length'], new_size) 1565 yo._meta[victim]['length'] = new_size 1566 yo._meta[victim]['end'] = start + new_size 1567 blank = array('c', ' ' * new_size) 1568 for record in yo: 1569 new_data = blank[:] 1570 new_data[:eff_end] = record._data[start:start+eff_end] 1571 record._data = record._data[:start] + new_data + record._data[end:] 1572 for field in yo._meta.fields: 1573 if yo._meta[field]['start'] == end: 1574 end = yo._meta[field]['end'] 1575 yo._meta[field]['start'] = start + new_size 1576 yo._meta[field]['end'] = start + new_size + yo._meta[field]['length'] 1577 start = yo._meta[field]['end'] 1578 yo._buildHeaderFields() 1579 yo._update_disk()
1580 - def size(yo, field):
1581 "returns size of field as a tuple of (length, decimals)" 1582 if field in yo: 1583 return (yo._meta[field]['length'], yo._meta[field]['decimals']) 1584 raise DbfError("%s is not a field in %s" % (field, yo.filename))
1585 - def structure(yo, fields=None):
1586 """return list of fields suitable for creating same table layout 1587 @param fields: list of fields or None for all fields""" 1588 field_specs = [] 1589 fields = yo._list_fields(fields) 1590 try: 1591 for name in fields: 1592 field_specs.append(yo._fieldLayout(yo.field_names.index(name))) 1593 except ValueError: 1594 raise DbfError("field --%s-- does not exist" % name) 1595 return field_specs
1596 - def top(yo, get_record=False):
1597 """sets record pointer to top of table; if get_record, seeks to and returns first (non-deleted) record 1598 DbfError if table is empty 1599 Eof if all records are deleted and use_deleted is False""" 1600 yo._meta.current = -1 1601 if get_record: 1602 try: 1603 return yo.next() 1604 except Eof: 1605 yo._meta.current = -1 1606 raise Bof()
1607 - def type(yo, field):
1608 "returns type of field" 1609 if field in yo: 1610 return yo._meta[field]['type'] 1611 raise DbfError("%s is not a field in %s" % (field, yo.filename))
1612 - def zap(yo, areyousure=False):
1613 """removes all records from table -- this cannot be undone! 1614 areyousure must be True, else error is raised""" 1615 if areyousure: 1616 if yo._meta.inmemory: 1617 yo._table = [] 1618 else: 1619 yo._table.clear() 1620 yo._meta.header.record_count = 0 1621 yo._current = -1 1622 yo._update_disk() 1623 else: 1624 raise DbfError("You must say you are sure to wipe the table")
1625 -class Db3Table(DbfTable):
1626 """Provides an interface for working with dBase III tables.""" 1627 _version = 'dBase III Plus' 1628 _versionabbv = 'db3' 1629 _fieldtypes = { 1630 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter, 'Class':None}, 1631 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate, 'Class':None}, 1632 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical, 'Class':None}, 1633 'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo, 'Class':None}, 1634 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addNumeric, 'Class':None} } 1635 _memoext = '.dbt' 1636 _memotypes = ('M',) 1637 _memoClass = _Db3Memo 1638 _yesMemoMask = '\x80' 1639 _noMemoMask = '\x7f' 1640 _fixed_fields = ('D','L','M') 1641 _variable_fields = ('C','N') 1642 _character_fields = ('C','M') 1643 _decimal_fields = ('N',) 1644 _numeric_fields = ('N',) 1645 _currency_fields = tuple() 1646 _dbfTableHeader = array('c', '\x00' * 32) 1647 _dbfTableHeader[0] = '\x03' # version - dBase III w/o memo's 1648 _dbfTableHeader[8:10] = array('c', io.packShortInt(33)) 1649 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 1650 _dbfTableHeader[29] = '\x03' # code page -- 437 US-MS DOS 1651 _dbfTableHeader = _dbfTableHeader.tostring() 1652 _dbfTableHeaderExtra = '' 1653 _supported_tables = ['\x03', '\x83'] 1654 _read_only = False 1655 _meta_only = False 1656 _use_deleted = True
1657 - def _checkMemoIntegrity(yo):
1658 "dBase III specific" 1659 if yo._meta.header.version == '\x83': 1660 try: 1661 yo._meta.memo = yo._memoClass(yo._meta) 1662 except: 1663 yo._meta.dfd.close() 1664 yo._meta.dfd = None 1665 raise 1666 if not yo._meta.ignorememos: 1667 for field in yo._meta.fields: 1668 if yo._meta[field]['type'] in yo._memotypes: 1669 if yo._meta.header.version != '\x83': 1670 yo._meta.dfd.close() 1671 yo._meta.dfd = None 1672 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos") 1673 elif not os.path.exists(yo._meta.memoname): 1674 yo._meta.dfd.close() 1675 yo._meta.dfd = None 1676 raise DbfError("Table structure corrupt: memo fields exist without memo file") 1677 break
1678 - def _initializeFields(yo):
1679 "builds the FieldList of names, types, and descriptions" 1680 yo._meta.fields[:] = [] 1681 offset = 1 1682 fieldsdef = yo._meta.header.fields 1683 if len(fieldsdef) % 32 != 0: 1684 raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef)) 1685 if len(fieldsdef) // 32 != yo.field_count: 1686 raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32)) 1687 for i in range(yo.field_count): 1688 fieldblock = fieldsdef[i*32:(i+1)*32] 1689 name = io.unpackStr(fieldblock[:11]) 1690 type = fieldblock[11] 1691 if not type in yo._meta.fieldtypes: 1692 raise DbfError("Unknown field type: %s" % type) 1693 start = offset 1694 length = ord(fieldblock[16]) 1695 offset += length 1696 end = start + length 1697 decimals = ord(fieldblock[17]) 1698 flags = ord(fieldblock[18]) 1699 yo._meta.fields.append(name) 1700 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
1701 -class FpTable(DbfTable):
1702 'Provides an interface for working with FoxPro 2 tables' 1703 _version = 'Foxpro' 1704 _versionabbv = 'fp' 1705 _fieldtypes = { 1706 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter, 'Class':None}, 1707 'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric, 'Class':None}, 1708 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric, 'Class':None}, 1709 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical, 'Class':None}, 1710 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate, 'Class':None}, 1711 'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addVfpMemo, 'Class':None}, 1712 'G' : {'Type':'General', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo, 'Class':None}, 1713 'P' : {'Type':'Picture', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo, 'Class':None}, 1714 '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None, 'Class':None} } 1715 _memoext = '.fpt' 1716 _memotypes = ('G','M','P') 1717 _memoClass = _VfpMemo 1718 _yesMemoMask = '\xf5' # 1111 0101 1719 _noMemoMask = '\x03' # 0000 0011 1720 _fixed_fields = ('B','D','G','I','L','M','P','T','Y') 1721 _variable_fields = ('C','F','N') 1722 _character_fields = ('C','M') # field representing character data 1723 _decimal_fields = ('F','N') 1724 _numeric_fields = ('F','N') 1725 _currency_fields = tuple() 1726 _supported_tables = ('\x03', '\xf5') 1727 _dbfTableHeader = array('c', '\x00' * 32) 1728 _dbfTableHeader[0] = '\x30' # version - Foxpro 6 0011 0000 1729 _dbfTableHeader[8:10] = array('c', io.packShortInt(33+263)) 1730 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 1731 _dbfTableHeader[29] = '\x03' # code page -- 437 US-MS DOS 1732 _dbfTableHeader = _dbfTableHeader.tostring() 1733 _dbfTableHeaderExtra = '\x00' * 263 1734 _use_deleted = True
1735 - def _checkMemoIntegrity(yo):
1736 if os.path.exists(yo._meta.memoname): 1737 try: 1738 yo._meta.memo = yo._memoClass(yo._meta) 1739 except: 1740 yo._meta.dfd.close() 1741 yo._meta.dfd = None 1742 raise 1743 if not yo._meta.ignorememos: 1744 for field in yo._meta.fields: 1745 if yo._meta[field]['type'] in yo._memotypes: 1746 if not os.path.exists(yo._meta.memoname): 1747 yo._meta.dfd.close() 1748 yo._meta.dfd = None 1749 raise DbfError("Table structure corrupt: memo fields exist without memo file") 1750 break
1751 - def _initializeFields(yo):
1752 "builds the FieldList of names, types, and descriptions" 1753 yo._meta.fields[:] = [] 1754 offset = 1 1755 fieldsdef = yo._meta.header.fields 1756 if len(fieldsdef) % 32 != 0: 1757 raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef)) 1758 if len(fieldsdef) // 32 != yo.field_count: 1759 raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32)) 1760 for i in range(yo.field_count): 1761 fieldblock = fieldsdef[i*32:(i+1)*32] 1762 name = io.unpackStr(fieldblock[:11]) 1763 type = fieldblock[11] 1764 if not type in yo._meta.fieldtypes: 1765 raise DbfError("Unknown field type: %s" % type) 1766 elif type == '0': 1767 return # ignore nullflags 1768 start = offset 1769 length = ord(fieldblock[16]) 1770 offset += length 1771 end = start + length 1772 decimals = ord(fieldblock[17]) 1773 flags = ord(fieldblock[18]) 1774 yo._meta.fields.append(name) 1775 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
1776
1777 -class VfpTable(DbfTable):
1778 'Provides an interface for working with Visual FoxPro 6 tables' 1779 _version = 'Visual Foxpro v6' 1780 _versionabbv = 'vfp' 1781 _fieldtypes = { 1782 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter, 'Class':None}, 1783 'Y' : {'Type':'Currency', 'Retrieve':io.retrieveCurrency, 'Update':io.updateCurrency, 'Blank':Decimal(), 'Init':io.addVfpCurrency, 'Class':None}, 1784 'B' : {'Type':'Double', 'Retrieve':io.retrieveDouble, 'Update':io.updateDouble, 'Blank':float, 'Init':io.addVfpDouble, 'Class':None}, 1785 'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric, 'Class':None}, 1786 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric, 'Class':None}, 1787 'I' : {'Type':'Integer', 'Retrieve':io.retrieveInteger, 'Update':io.updateInteger, 'Blank':int, 'Init':io.addVfpInteger, 'Class':None}, 1788 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical, 'Class':None}, 1789 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate, 'Class':None}, 1790 'T' : {'Type':'DateTime', 'Retrieve':io.retrieveVfpDateTime, 'Update':io.updateVfpDateTime, 'Blank':DateTime.now, 'Init':io.addVfpDateTime, 'Class':None}, 1791 'M' : {'Type':'Memo', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo, 'Class':None}, 1792 'G' : {'Type':'General', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo, 'Class':None}, 1793 'P' : {'Type':'Picture', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo, 'Class':None}, 1794 '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None, 'Class':None} } 1795 _memoext = '.fpt' 1796 _memotypes = ('G','M','P') 1797 _memoClass = _VfpMemo 1798 _yesMemoMask = '\x30' # 0011 0000 1799 _noMemoMask = '\x30' # 0011 0000 1800 _fixed_fields = ('B','D','G','I','L','M','P','T','Y') 1801 _variable_fields = ('C','F','N') 1802 _character_fields = ('C','M') # field representing character data 1803 _decimal_fields = ('F','N') 1804 _numeric_fields = ('B','F','I','N','Y') 1805 _currency_fields = ('Y',) 1806 _supported_tables = ('\x30',) 1807 _dbfTableHeader = array('c', '\x00' * 32) 1808 _dbfTableHeader[0] = '\x30' # version - Foxpro 6 0011 0000 1809 _dbfTableHeader[8:10] = array('c', io.packShortInt(33+263)) 1810 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 1811 _dbfTableHeader[29] = '\x03' # code page -- 437 US-MS DOS 1812 _dbfTableHeader = _dbfTableHeader.tostring() 1813 _dbfTableHeaderExtra = '\x00' * 263 1814 _use_deleted = True
1815 - def _checkMemoIntegrity(yo):
1816 if os.path.exists(yo._meta.memoname): 1817 try: 1818 yo._meta.memo = yo._memoClass(yo._meta) 1819 except: 1820 yo._meta.dfd.close() 1821 yo._meta.dfd = None 1822 raise 1823 if not yo._meta.ignorememos: 1824 for field in yo._meta.fields: 1825 if yo._meta[field]['type'] in yo._memotypes: 1826 if not os.path.exists(yo._meta.memoname): 1827 yo._meta.dfd.close() 1828 yo._meta.dfd = None 1829 raise DbfError("Table structure corrupt: memo fields exist without memo file") 1830 break
1831 - def _initializeFields(yo):
1832 "builds the FieldList of names, types, and descriptions" 1833 yo._meta.fields[:] = [] 1834 offset = 1 1835 fieldsdef = yo._meta.header.fields 1836 for i in range(yo.field_count): 1837 fieldblock = fieldsdef[i*32:(i+1)*32] 1838 name = io.unpackStr(fieldblock[:11]) 1839 type = fieldblock[11] 1840 if not type in yo._meta.fieldtypes: 1841 raise DbfError("Unknown field type: %s" % type) 1842 elif type == '0': 1843 return # ignore nullflags 1844 start = io.unpackLongInt(fieldblock[12:16]) 1845 length = ord(fieldblock[16]) 1846 offset += length 1847 end = start + length 1848 decimals = ord(fieldblock[17]) 1849 flags = ord(fieldblock[18]) 1850 yo._meta.fields.append(name) 1851 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
1852 -class List(object):
1853 "list of Dbf records, with set-like behavior" 1854 _desc = ''
1855 - def __init__(yo, new_records=None, desc=None, key=None, field_names=None):
1856 yo.field_names = field_names 1857 yo._list = [] 1858 yo._set = set() 1859 if key is not None: 1860 yo.key = key 1861 if key.__doc__ is None: 1862 key.__doc__ = 'unknown' 1863 key = yo.key 1864 yo._current = -1 1865 if isinstance(new_records, yo.__class__) and key is new_records.key: 1866 yo._list = new_records._list[:] 1867 yo._set = new_records._set.copy() 1868 yo._current = 0 1869 elif new_records is not None: 1870 for record in new_records: 1871 value = key(record) 1872 item = (record.record_table, record.record_number, value) 1873 if value not in yo._set: 1874 yo._set.add(value) 1875 yo._list.append(item) 1876 yo._current = 0 1877 if desc is not None: 1878 yo._desc = desc
1879 - def __add__(yo, other):
1880 key = yo.key 1881 if isinstance(other, (DbfTable, list)): 1882 other = yo.__class__(other, key=key) 1883 if isinstance(other, yo.__class__): 1884 result = yo.__class__() 1885 result._set = yo._set.copy() 1886 result._list[:] = yo._list[:] 1887 result.key = yo.key 1888 if key is other.key: # same key? just compare key values 1889 for item in other._list: 1890 if item[2] not in result._set: 1891 result._set.add(item[2]) 1892 result._list.append(item) 1893 else: # different keys, use this list's key on other's records 1894 for rec in other: 1895 value = key(rec) 1896 if value not in result._set: 1897 result._set.add(value) 1898 result._list.append((rec.record_table, rec.record_number, value)) 1899 result._current = 0 if result else -1 1900 return result 1901 return NotImplemented
1902 - def __contains__(yo, record):
1903 if isinstance(record, tuple): 1904 item = record 1905 else: 1906 item = yo.key(record) 1907 return item in yo._set
1908 - def __delitem__(yo, key):
1909 if isinstance(key, int): 1910 item = yo._list.pop[key] 1911 yo._set.remove(item[2]) 1912 elif isinstance(key, slice): 1913 yo._set.difference_update([item[2] for item in yo._list[key]]) 1914 yo._list.__delitem__(key) 1915 else: 1916 raise TypeError
1917 - def __getitem__(yo, key):
1918 if isinstance(key, int): 1919 count = len(yo._list) 1920 if not -count <= key < count: 1921 raise IndexError("Record %d is not in list." % key) 1922 return yo._get_record(*yo._list[key]) 1923 elif isinstance(key, slice): 1924 result = yo.__class__() 1925 result._list[:] = yo._list[key] 1926 result._set = set(result._list) 1927 result.key = yo.key 1928 result._current = 0 if result else -1 1929 return result 1930 else: 1931 raise TypeError('indices must be integers')
1932 - def __iter__(yo):
1933 return (table.get_record(recno) for table, recno, value in yo._list)
1934 - def __len__(yo):
1935 return len(yo._list)
1936 - def __nonzero__(yo):
1937 return len(yo) > 0
1938 - def __radd__(yo, other):
1939 return yo.__add__(other)
1940 - def __repr__(yo):
1941 if yo._desc: 1942 return "%s(key=%s - %s - %d records)" % (yo.__class__, yo.key.__doc__, yo._desc, len(yo._list)) 1943 else: 1944 return "%s(key=%s - %d records)" % (yo.__class__, yo.key.__doc__, len(yo._list))
1945 - def __rsub__(yo, other):
1946 key = yo.key 1947 if isinstance(other, (DbfTable, list)): 1948 other = yo.__class__(other, key=key) 1949 if isinstance(other, yo.__class__): 1950 result = yo.__class__() 1951 result._list[:] = other._list[:] 1952 result._set = other._set.copy() 1953 result.key = key 1954 lost = set() 1955 if key is other.key: 1956 for item in yo._list: 1957 if item[2] in result._list: 1958 result._set.remove(item[2]) 1959 lost.add(item) 1960 else: 1961 for rec in other: 1962 value = key(rec) 1963 if value in result._set: 1964 result._set.remove(value) 1965 lost.add((rec.record_table, rec.record_number, value)) 1966 result._list = [item for item in result._list if item not in lost] 1967 result._current = 0 if result else -1 1968 return result 1969 return NotImplemented
1970 - def __sub__(yo, other):
1971 key = yo.key 1972 if isinstance(other, (DbfTable, list)): 1973 other = yo.__class__(other, key=key) 1974 if isinstance(other, yo.__class__): 1975 result = yo.__class__() 1976 result._list[:] = yo._list[:] 1977 result._set = yo._set.copy() 1978 result.key = key 1979 lost = set() 1980 if key is other.key: 1981 for item in other._list: 1982 if item[2] in result._set: 1983 result._set.remove(item[2]) 1984 lost.add(item[2]) 1985 else: 1986 for rec in other: 1987 value = key(rec) 1988 if value in result._set: 1989 result._set.remove(value) 1990 lost.add(value) 1991 result._list = [item for item in result._list if item[2] not in lost] 1992 result._current = 0 if result else -1 1993 return result 1994 return NotImplemented
1995 - def _maybe_add(yo, item):
1996 if item[2] not in yo._set: 1997 yo._set.add(item[2]) 1998 yo._list.append(item)
1999 - def _get_record(yo, table=None, rec_no=None, value=None):
2000 if table is rec_no is None: 2001 table, rec_no, value = yo._list[yo._current] 2002 return table.get_record(rec_no)
2003 - def _purge(yo, record, old_record_number, offset):
2004 partial = record.record_table, old_record_number 2005 records = sorted(yo._list, key=lambda item: (item[0], item[1])) 2006 for item in records: 2007 if partial == item[:2]: 2008 found = True 2009 break 2010 elif partial[0] is item[0] and partial[1] < item[1]: 2011 found = False 2012 break 2013 else: 2014 found = False 2015 if found: 2016 yo._list.pop(yo._list.index(item)) 2017 yo._set.remove(item[2]) 2018 start = records.index(item) + found 2019 for item in records[start:]: 2020 if item[0] is not partial[0]: # into other table's records 2021 break 2022 i = yo._list.index(item) 2023 yo._set.remove(item[2]) 2024 item = item[0], (item[1] - offset), item[2] 2025 yo._list[i] = item 2026 yo._set.add(item[2]) 2027 return found
2028 - def append(yo, new_record):
2029 yo._maybe_add((new_record.record_table, new_record.record_number, yo.key(new_record))) 2030 if yo._current == -1 and yo._list: 2031 yo._current = 0 2032 return new_record
2033 - def bottom(yo):
2034 if yo._list: 2035 yo._current = len(yo._list) - 1 2036 return yo._get_record() 2037 raise DbfError("dbf.List is empty")
2038 - def clear(yo):
2039 yo._list = [] 2040 yo._set = set() 2041 yo._current = -1
2042 - def current(yo):
2043 if yo._current < 0: 2044 raise Bof() 2045 elif yo._current == len(yo._list): 2046 raise Eof() 2047 return yo._get_record()
2048 - def extend(yo, new_records):
2049 key = yo.key 2050 if isinstance(new_records, yo.__class__): 2051 if key is new_records.key: # same key? just compare key values 2052 for item in new_records._list: 2053 yo._maybe_add(item) 2054 else: # different keys, use this list's key on other's records 2055 for rec in new_records: 2056 value = key(rec) 2057 yo._maybe_add((rec.record_table, rec.record_number, value)) 2058 else: 2059 for record in new_records: 2060 value = key(rec) 2061 yo._maybe_add((rec.record_table, rec.record_number, value)) 2062 if yo._current == -1 and yo._list: 2063 yo._current = 0
2064 - def goto(yo, index_number):
2065 if yo._list: 2066 if 0 <= index_number <= len(yo._list): 2067 yo._current = index_number 2068 return yo._get_record() 2069 raise DbfError("index %d not in dbf.List of %d records" % (index_number, len(yo._list))) 2070 raise DbfError("dbf.List is empty")
2071 - def index(yo, sort=None, reverse=False):
2072 "sort= ((field_name, func), (field_name, func),) | 'ORIGINAL'" 2073 if sort is None: 2074 results = [] 2075 for field, func in yo._meta.index: 2076 results.append("%s(%s)" % (func.__name__, field)) 2077 return ', '.join(results + ['reverse=%s' % yo._meta.index_reversed]) 2078 yo._meta.index_reversed = reverse 2079 if sort == 'ORIGINAL': 2080 yo._index = range(yo._meta.header.record_count) 2081 yo._meta.index = 'ORIGINAL' 2082 if reverse: 2083 yo._index.reverse() 2084 return 2085 new_sort = _normalize_tuples(tuples=sort, length=2, filler=[_nop]) 2086 yo._meta.index = tuple(new_sort) 2087 yo._meta.orderresults = [''] * len(yo) 2088 for record in yo: 2089 yo._meta.orderresults[record.record_number] = record() 2090 yo._index.sort(key=lambda i: yo._meta.orderresults[i], reverse=reverse)
2091 - def index(yo, record, start=None, stop=None):
2092 item = record.record_table, record.record_number, yo.key(record) 2093 if start is None: 2094 start = 0 2095 if stop is None: 2096 stop = len(yo._list) 2097 return yo._list.index(item, start, stop)
2098 - def insert(yo, i, record):
2099 item = record.record_table, record.record_number, yo.key(record) 2100 if item not in yo._set: 2101 yo._set.add(item[2]) 2102 yo._list.insert(i, item)
2103 - def key(yo, record):
2104 "table_name, record_number" 2105 return record.record_table, record.record_number
2106 - def next(yo):
2107 if yo._current < len(yo._list): 2108 yo._current += 1 2109 if yo._current < len(yo._list): 2110 return yo._get_record() 2111 raise Eof()
2112 - def pop(yo, index=None):
2113 if index is None: 2114 table, recno, value = yo._list.pop() 2115 else: 2116 table, recno, value = yo._list.pop(index) 2117 yo._set.remove(value) 2118 return yo._get_record(table, recno, value)
2119 - def prev(yo):
2120 if yo._current >= 0: 2121 yo._current -= 1 2122 if yo._current > -1: 2123 return yo._get_record() 2124 raise Bof()
2125 - def remove(yo, record):
2126 item = record.record_table, record.record_number, yo.key(record) 2127 yo._list.remove(item) 2128 yo._set.remove(item[2])
2129 - def reverse(yo):
2130 return yo._list.reverse()
2131 - def top(yo):
2132 if yo._list: 2133 yo._current = 0 2134 return yo._get_record() 2135 raise DbfError("dbf.List is empty")
2136 - def sort(yo, key=None, reverse=False):
2137 if key is None: 2138 return yo._list.sort(reverse=reverse) 2139 return yo._list.sort(key=lambda item: key(item[0].get_record(item[1])), reverse=reverse)
2140
2141 -class DbfCsv(csv.Dialect):
2142 "csv format for exporting tables" 2143 delimiter = ',' 2144 doublequote = True 2145 escapechar = None 2146 lineterminator = '\n' 2147 quotechar = '"' 2148 skipinitialspace = True 2149 quoting = csv.QUOTE_NONNUMERIC
2150 -class Index(object):
2151 - class IndexIterator(object):
2152 "returns records using this index"
2153 - def __init__(yo, table, records):
2154 yo.table = table 2155 yo.records = records 2156 yo.index = 0
2157 - def __iter__(yo):
2158 return yo
2159 - def next(yo):
2160 while yo.index < len(yo.records): 2161 record = yo.table.get_record(yo.records[yo.index]) 2162 yo.index += 1 2163 if not yo.table.use_deleted and record.has_been_deleted: 2164 continue 2165 return record 2166 else: 2167 raise StopIteration
2168 - def __init__(yo, table, key, field_names=None):
2169 yo._table = table 2170 yo._values = [] # ordered list of values 2171 yo._rec_by_val = [] # matching record numbers 2172 yo._records = {} # record numbers:values 2173 yo.__doc__ = key.__doc__ or 'unknown' 2174 yo.key = key 2175 yo.field_names = field_names or table.field_names 2176 for record in table: 2177 value = key(record) 2178 if value is DoNotIndex: 2179 continue 2180 rec_num = record.record_number 2181 if not isinstance(value, tuple): 2182 value = (value, ) 2183 vindex = bisect_right(yo._values, value) 2184 yo._values.insert(vindex, value) 2185 yo._rec_by_val.insert(vindex, rec_num) 2186 yo._records[rec_num] = value 2187 table._indexen.add(yo)
2188 - def __call__(yo, record):
2189 rec_num = record.record_number 2190 if rec_num in yo._records: 2191 value = yo._records[rec_num] 2192 vindex = bisect_left(yo._values, value) 2193 yo._values.pop(vindex) 2194 yo._rec_by_val.pop(vindex) 2195 value = yo.key(record) 2196 if value is DoNotIndex: 2197 return 2198 if not isinstance(value, tuple): 2199 value = (value, ) 2200 vindex = bisect_right(yo._values, value) 2201 yo._values.insert(vindex, value) 2202 yo._rec_by_val.insert(vindex, rec_num) 2203 yo._records[rec_num] = value
2204 - def __contains__(yo, match):
2205 if isinstance(match, _DbfRecord): 2206 if match.record_table is yo._table: 2207 return match.record_number in yo._records 2208 match = yo.key(match) 2209 elif not isinstance(match, tuple): 2210 match = (match, ) 2211 return yo.find(match) != -1
2212 - def __getitem__(yo, key):
2213 if isinstance(key, int): 2214 count = len(yo._values) 2215 if not -count <= key < count: 2216 raise IndexError("Record %d is not in list." % key) 2217 rec_num = yo._rec_by_val[key] 2218 return yo._table.get_record(rec_num) 2219 elif isinstance(key, slice): 2220 result = List(field_names=yo._table.field_names) 2221 yo._table._dbflists.add(result) 2222 start, stop, step = key.start, key.stop, key.step 2223 if start is None: start = 0 2224 if stop is None: stop = len(yo._rec_by_val) 2225 if step is None: step = 1 2226 for loc in range(start, stop, step): 2227 record = yo._table.get_record(yo._rec_by_val[loc]) 2228 result._maybe_add(item=(yo._table, yo._rec_by_val[loc], result.key(record))) 2229 result._current = 0 if result else -1 2230 return result 2231 elif isinstance (key, (str, unicode, tuple, _DbfRecord)): 2232 if isinstance(key, _DbfRecord): 2233 key = yo.key(key) 2234 elif not isinstance(key, tuple): 2235 key = (key, ) 2236 loc = yo.find(key) 2237 if loc == -1: 2238 raise KeyError(key) 2239 return yo._table.get_record(yo._rec_by_val[loc]) 2240 else: 2241 raise TypeError('indices must be integers, match objects must by strings or tuples')
2242 - def __enter__(yo):
2243 return yo
2244 - def __exit__(yo, *exc_info):
2245 yo._table.close() 2246 yo._values[:] = [] 2247 yo._rec_by_val[:] = [] 2248 yo._records.clear() 2249 return False
2250 - def __iter__(yo):
2251 return yo.IndexIterator(yo._table, yo._rec_by_val)
2252 - def __len__(yo):
2253 return len(yo._records)
2254 - def _partial_match(yo, target, match):
2255 target = target[:len(match)] 2256 if isinstance(match[-1], (str, unicode)): 2257 target = list(target) 2258 target[-1] = target[-1][:len(match[-1])] 2259 target = tuple(target) 2260 return target == match
2261 - def _purge(yo, rec_num):
2262 value = yo._records.get(rec_num) 2263 if value is not None: 2264 vindex = bisect_left(yo._values, value) 2265 del yo._records[rec_num] 2266 yo._values.pop(vindex) 2267 yo._rec_by_val.pop(vindex)
2268 - def _search(yo, match, lo=0, hi=None):
2269 if hi is None: 2270 hi = len(yo._values) 2271 return bisect_left(yo._values, match, lo, hi)
2272 - def clear(yo):
2273 "removes all entries from index" 2274 yo._values[:] = [] 2275 yo._rec_by_val[:] = [] 2276 yo._records.clear()
2277 - def close(yo):
2278 yo._table.close()
2279 - def find(yo, match, partial=False):
2280 "returns numeric index of (partial) match, or -1" 2281 if isinstance(match, _DbfRecord): 2282 if match.record_number in yo._records: 2283 return yo._values.index(yo._records[match.record_number]) 2284 else: 2285 return -1 2286 if not isinstance(match, tuple): 2287 match = (match, ) 2288 loc = yo._search(match) 2289 while loc < len(yo._values) and yo._values[loc] == match: 2290 if not yo._table.use_deleted and yo._table.get_record(yo._rec_by_val[loc]).has_been_deleted: 2291 loc += 1 2292 continue 2293 return loc 2294 if partial: 2295 while loc < len(yo._values) and yo._partial_match(yo._values[loc], match): 2296 if not yo._table.use_deleted and yo._table.get_record(yo._rec_by_val[loc]).has_been_deleted: 2297 loc += 1 2298 continue 2299 return loc 2300 return -1
2301 - def find_index(yo, match):
2302 "returns numeric index of either (partial) match, or position of where match would be" 2303 if isinstance(match, _DbfRecord): 2304 if match.record_number in yo._records: 2305 return yo._values.index(yo._records[match.record_number]) 2306 else: 2307 match = yo.key(match) 2308 if not isinstance(match, tuple): 2309 match = (match, ) 2310 loc = yo._search(match) 2311 return loc
2312 - def index(yo, match, partial=False):
2313 "returns numeric index of (partial) match, or raises ValueError" 2314 loc = yo.find(match, partial) 2315 if loc == -1: 2316 if isinstance(match, _DbfRecord): 2317 raise ValueError("table <%s> record [%d] not in index <%s>" % (yo._table.filename, match.record_number, yo.__doc__)) 2318 else: 2319 raise ValueError("match criteria <%s> not in index" % (match, )) 2320 return loc
2321 - def reindex(yo):
2322 "reindexes all records" 2323 for record in yo._table: 2324 yo(record)
2325 - def query(yo, sql_command=None, python=None):
2326 """recognized sql commands are SELECT, UPDATE, REPLACE, INSERT, DELETE, and RECALL""" 2327 if sql_command: 2328 return sql(yo, sql_command) 2329 elif python is None: 2330 raise DbfError("query: python parameter must be specified") 2331 possible = List(desc="%s --> %s" % (yo._table.filename, python), field_names=yo._table.field_names) 2332 yo._table._dbflists.add(possible) 2333 query_result = {} 2334 select = 'query_result["keep"] = %s' % python 2335 g = {} 2336 for record in yo: 2337 query_result['keep'] = False 2338 g['query_result'] = query_result 2339 exec select in g, record 2340 if query_result['keep']: 2341 possible.append(record) 2342 record.write_record() 2343 return possible
2344 - def search(yo, match, partial=False):
2345 "returns dbf.List of all (partially) matching records" 2346 result = List(field_names=yo._table.field_names) 2347 yo._table._dbflists.add(result) 2348 if not isinstance(match, tuple): 2349 match = (match, ) 2350 loc = yo._search(match) 2351 if loc == len(yo._values): 2352 return result 2353 while loc < len(yo._values) and yo._values[loc] == match: 2354 record = yo._table.get_record(yo._rec_by_val[loc]) 2355 if not yo._table.use_deleted and record.has_been_deleted: 2356 loc += 1 2357 continue 2358 result._maybe_add(item=(yo._table, yo._rec_by_val[loc], result.key(record))) 2359 loc += 1 2360 if partial: 2361 while loc < len(yo._values) and yo._partial_match(yo._values[loc], match): 2362 record = yo._table.get_record(yo._rec_by_val[loc]) 2363 if not yo._table.use_deleted and record.has_been_deleted: 2364 loc += 1 2365 continue 2366 result._maybe_add(item=(yo._table, yo._rec_by_val[loc], result.key(record))) 2367 loc += 1 2368 return result
2369 2370 csv.register_dialect('dbf', DbfCsv)
2371 2372 -def sql_select(records, chosen_fields, condition, field_names):
2373 if chosen_fields != '*': 2374 field_names = chosen_fields.replace(' ','').split(',') 2375 result = condition(records) 2376 result.modified = 0, 'record' + ('','s')[len(result)>1] 2377 result.field_names = field_names 2378 return result
2379
2380 -def sql_update(records, command, condition, field_names):
2381 possible = condition(records) 2382 modified = sql_cmd(command, field_names)(possible) 2383 possible.modified = modified, 'record' + ('','s')[modified>1] 2384 return possible
2385
2386 -def sql_delete(records, dead_fields, condition, field_names):
2387 deleted = condition(records) 2388 deleted.modified = len(deleted), 'record' + ('','s')[len(deleted)>1] 2389 deleted.field_names = field_names 2390 if dead_fields == '*': 2391 for record in deleted: 2392 record.delete_record() 2393 record.write_record() 2394 else: 2395 keep = [f for f in field_names if f not in dead_fields.replace(' ','').split(',')] 2396 for record in deleted: 2397 record.reset_record(keep_fields=keep) 2398 record.write_record() 2399 return deleted
2400
2401 -def sql_recall(records, all_fields, condition, field_names):
2402 if all_fields != '*': 2403 raise DbfError('SQL RECALL: fields must be * (only able to recover at the record level)') 2404 revivified = List() 2405 tables = set() 2406 for record in records: 2407 tables.add(record_table) 2408 old_setting = dict() 2409 for table in tables: 2410 old_setting[table] = table.use_deleted 2411 table.use_deleted = True 2412 for record in condition(records): 2413 if record.has_been_deleted: 2414 revivified.append(record) 2415 record.undelete_record() 2416 record.write_record() 2417 for table in tables: 2418 table.use_deleted = old_setting[table] 2419 revivified.modfied = len(revivified), 'record' + ('','s')[len(revivified)>1] 2420 revivified.field_names = field_names 2421 return revivified
2422
2423 -def sql_add(records, new_fields, condition, field_names):
2424 tables = set() 2425 possible = condition(records) 2426 for record in possible: 2427 tables.add(record.record_table) 2428 for table in tables: 2429 table.add_fields(new_fields) 2430 possible.modified = len(tables), 'table' + ('','s')[len(tables)>1] 2431 possible.field_names = field_names 2432 return possible
2433
2434 -def sql_drop(records, dead_fields, condition, field_names):
2435 tables = set() 2436 possible = condition(records) 2437 for record in possible: 2438 tables.add(record.record_table) 2439 for table in tables: 2440 table.delete_fields(dead_fields) 2441 possible.modified = len(tables), 'table' + ('','s')[len(tables)>1] 2442 possible.field_names = field_names 2443 return possible
2444
2445 -def sql_pack(records, command, condition, field_names):
2446 tables = set() 2447 possible = condition(records) 2448 for record in possible: 2449 tables.add(record.record_table) 2450 for table in tables: 2451 table.pack() 2452 possible.modified = len(tables), 'table' + ('','s')[len(tables)>1] 2453 possible.field_names = field_names 2454 return possible
2455
2456 -def sql_resize(records, fieldname_newsize, condition, field_names):
2457 tables = set() 2458 possible = condition(records) 2459 for record in possible: 2460 tables.add(record.record_table) 2461 fieldname, newsize = fieldname_newsize.split() 2462 newsize = int(newsize) 2463 for table in tables: 2464 table.resize_field(fieldname, newsize) 2465 possible.modified = len(tables), 'table' + ('','s')[len(tables)>1] 2466 possible.field_names = field_names 2467 return possible
2468 2469 sql_functions = { 2470 'select' : sql_select, 2471 'update' : sql_update, 2472 'replace': sql_update, 2473 'insert' : None, 2474 'delete' : sql_delete, 2475 'recall' : sql_recall, 2476 'add' : sql_add, 2477 'drop' : sql_drop, 2478 'count' : None, 2479 'pack' : sql_pack, 2480 'resize' : sql_resize, 2481 }
2482 2483 -def sql_criteria(records, criteria):
2484 "creates a function matching the sql criteria" 2485 function = """def func(records): 2486 \"\"\"%s\"\"\" 2487 matched = List(field_names=records[0].field_names) 2488 for rec in records: 2489 %s 2490 2491 if %s: 2492 matched.append(rec) 2493 return matched""" 2494 fields = [] 2495 for field in records[0].field_names: 2496 if field in criteria: 2497 fields.append(field) 2498 fields = '\n '.join(['%s = rec.%s' % (field, field) for field in fields]) 2499 g = dbf.sql_user_functions.copy() 2500 g['List'] = List 2501 function %= (criteria, fields, criteria) 2502 #print function 2503 exec function in g 2504 return g['func']
2505
2506 -def sql_cmd(command, field_names):
2507 "creates a function matching to apply command to each record in records" 2508 function = """def func(records): 2509 \"\"\"%s\"\"\" 2510 changed = 0 2511 for rec in records: 2512 %s 2513 2514 %s 2515 2516 %s 2517 changed += rec.write_record() 2518 return changed""" 2519 fields = [] 2520 for field in field_names: 2521 if field in command: 2522 fields.append(field) 2523 pre_fields = '\n '.join(['%s = rec.%s' % (field, field) for field in fields]) 2524 post_fields = '\n '.join(['rec.%s = %s' % (field, field) for field in fields]) 2525 g = dbf.sql_user_functions.copy() 2526 if '=' not in command and ' with ' in command.lower(): 2527 offset = command.lower().index(' with ') 2528 command = command[:offset] + ' = ' + command[offset+6:] 2529 function %= (command, pre_fields, command, post_fields) 2530 #print function 2531 exec function in g 2532 return g['func']
2533
2534 -def sql(records, command):
2535 """recognized sql commands are SELECT, UPDATE | REPLACE, DELETE, RECALL, ADD, DROP""" 2536 sql_command = command 2537 if ' where ' in command: 2538 command, condition = command.split(' where ', 1) 2539 condition = sql_criteria(records, condition) 2540 else: 2541 def condition(records): 2542 return records[:]
2543 name, command = command.split(' ', 1) 2544 command = command.strip() 2545 name = name.lower() 2546 field_names = records[0].field_names 2547 if sql_functions.get(name) is None: 2548 raise DbfError('unknown SQL command: %s' % name.upper()) 2549 result = sql_functions[name](records, command, condition, field_names) 2550 tables = set() 2551 for record in result: 2552 tables.add(record.record_table) 2553 for list_table in tables: 2554 list_table._dbflists.add(result) 2555 return result
2556 -def _nop(value):
2557 "returns parameter unchanged" 2558 return value
2559 -def _normalize_tuples(tuples, length, filler):
2560 "ensures each tuple is the same length, using filler[-missing] for the gaps" 2561 final = [] 2562 for t in tuples: 2563 if len(t) < length: 2564 final.append( tuple([item for item in t] + filler[len(t)-length:]) ) 2565 else: 2566 final.append(t) 2567 return tuple(final)
2568 -def _codepage_lookup(cp):
2569 if cp not in code_pages: 2570 for code_page in sorted(code_pages.keys()): 2571 sd, ld = code_pages[code_page] 2572 if cp == sd or cp == ld: 2573 if sd is None: 2574 raise DbfError("Unsupported codepage: %s" % ld) 2575 cp = code_page 2576 break 2577 else: 2578 raise DbfError("Unsupported codepage: %s" % cp) 2579 sd, ld = code_pages[cp] 2580 return cp, sd, ld
2581 -def ascii(new_setting=None):
2582 "get/set return_ascii setting" 2583 global return_ascii 2584 if new_setting is None: 2585 return return_ascii 2586 else: 2587 return_ascii = new_setting
2588 -def codepage(cp=None):
2589 "get/set default codepage for any new tables" 2590 global default_codepage 2591 cp, sd, ld = _codepage_lookup(cp or default_codepage) 2592 default_codepage = sd 2593 return "%s (LDID: 0x%02x - %s)" % (sd, ord(cp), ld)
2594 -def encoding(cp=None):
2595 "get/set default encoding for non-unicode strings passed into a table" 2596 global input_decoding 2597 cp, sd, ld = _codepage_lookup(cp or input_decoding) 2598 default_codepage = sd 2599 return "%s (LDID: 0x%02x - %s)" % (sd, ord(cp), ld)
2600 -class _Db4Table(DbfTable):
2601 version = 'dBase IV w/memos (non-functional)' 2602 _versionabbv = 'db4' 2603 _fieldtypes = { 2604 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter}, 2605 'Y' : {'Type':'Currency', 'Retrieve':io.retrieveCurrency, 'Update':io.updateCurrency, 'Blank':Decimal(), 'Init':io.addVfpCurrency}, 2606 'B' : {'Type':'Double', 'Retrieve':io.retrieveDouble, 'Update':io.updateDouble, 'Blank':float, 'Init':io.addVfpDouble}, 2607 'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric}, 2608 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric}, 2609 'I' : {'Type':'Integer', 'Retrieve':io.retrieveInteger, 'Update':io.updateInteger, 'Blank':int, 'Init':io.addVfpInteger}, 2610 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical}, 2611 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate}, 2612 'T' : {'Type':'DateTime', 'Retrieve':io.retrieveVfpDateTime, 'Update':io.updateVfpDateTime, 'Blank':DateTime.now, 'Init':io.addVfpDateTime}, 2613 'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo}, 2614 'G' : {'Type':'General', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo}, 2615 'P' : {'Type':'Picture', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo}, 2616 '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None} } 2617 _memoext = '.dbt' 2618 _memotypes = ('G','M','P') 2619 _memoClass = _VfpMemo 2620 _yesMemoMask = '\x8b' # 0011 0000 2621 _noMemoMask = '\x04' # 0011 0000 2622 _fixed_fields = ('B','D','G','I','L','M','P','T','Y') 2623 _variable_fields = ('C','F','N') 2624 _character_fields = ('C','M') # field representing character data 2625 _decimal_fields = ('F','N') 2626 _numeric_fields = ('B','F','I','N','Y') 2627 _currency_fields = ('Y',) 2628 _supported_tables = ('\x04', '\x8b') 2629 _dbfTableHeader = ['\x00'] * 32 2630 _dbfTableHeader[0] = '\x8b' # version - Foxpro 6 0011 0000 2631 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 2632 _dbfTableHeader[29] = '\x03' # code page -- 437 US-MS DOS 2633 _dbfTableHeader = ''.join(_dbfTableHeader) 2634 _dbfTableHeaderExtra = '' 2635 _use_deleted = True
2636 - def _checkMemoIntegrity(yo):
2637 "dBase III specific" 2638 if yo._meta.header.version == '\x8b': 2639 try: 2640 yo._meta.memo = yo._memoClass(yo._meta) 2641 except: 2642 yo._meta.dfd.close() 2643 yo._meta.dfd = None 2644 raise 2645 if not yo._meta.ignorememos: 2646 for field in yo._meta.fields: 2647 if yo._meta[field]['type'] in yo._memotypes: 2648 if yo._meta.header.version != '\x8b': 2649 yo._meta.dfd.close() 2650 yo._meta.dfd = None 2651 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos") 2652 elif not os.path.exists(yo._meta.memoname): 2653 yo._meta.dfd.close() 2654 yo._meta.dfd = None 2655 raise DbfError("Table structure corrupt: memo fields exist without memo file") 2656 break
2657