Coverage for /home/agp/Documents/me/code/gutools/gutools/recording.py : 38%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import lzma
2import gzip
3import bz2
4from datetime import datetime
7class Recorder(object):
8 """Support events recording to rotating compressed files.
10 By default lzma compression is selected (a.k.a. 'xz' files)
12 The default time_fmt used as timestamp is '%Y-%m-%d %H:%M:%S.%f'
13 allowing microseconds precission
15 The default grouping criteria is records that share the same day.
16 This is made by rotate_by = '%Y%m%d', so all records that share the
17 same day when are saved belongs to the same record file.
19 If you want to record some keys in the same file just use ``aliases``
20 parameter to convert keys before duming record.
22 """
23 INFO = {
24 'xz': lzma,
25 'gz': gzip,
26 'bz2': bz2,
27 }
30 def __init__(self, compression='xz',
31 time_fmt='%Y-%m-%d %H:%M:%S.%f',
32 rotate_by='%Y%m%d',
33 aliases=None, *args, **kw):
34 self.compression = compression
35 self.time_fmt = time_fmt
36 self.rotate_by = rotate_by
37 self.aliases = aliases or dict()
39 self._output = dict()
40 self._last_flush = dict()
42 def record(self, key, *event):
43 # write market data file
44 t = datetime.utcnow()
45 now = t.strftime(self.time_fmt)
46 today = t.strftime(self.rotate_by)
47 outputs = self._output.get(key)
48 if not outputs:
49 outputs = self._output[key] = dict()
51 output = outputs.get(today)
52 if not output:
53 # rotate dump file
54 for k in list(outputs.keys()):
55 output.pop(k).close()
57 # set the new one
58 output = outputs[today] = self.INFO[self.compression].\
59 open(f"{key}.{today}.{self.compression}", 'a')
61 line = [str(k) for k in event]
62 line = '.'.join(line) + '\n'
63 output.write(bytes(line, 'utf-8'))
65 last_flush = self._last_flush.setdefault(key, t)
66 if (t - last_flush).seconds > 120:
67 output._fp.flush()
68 output.close() # will be reopen next write attempt
69 outputs.pop(today)
70 self._last_flush[key] = t
72 def close(self):
73 for _, keys in self._output.items():
74 for k, f in keys.items():
75 print(f"Closing dump file: {f._fp.name}")
76 f._fp.flush()
77 f.close()