Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# -*- coding: utf-8 -*- 

2"""This module contains basic methods and definitions that would be used 

3by other `gutools` modules. 

4""" 

5 

6import sys 

7import os 

8import asyncio 

9import hashlib 

10import random 

11import inspect 

12import types 

13import fnmatch 

14import re 

15import yaml 

16import time 

17import json 

18import stat 

19import math 

20import traceback 

21import dateutil.parser as parser 

22from urllib import parse 

23from io import StringIO 

24from functools import partial 

25from collections.abc import Iterable 

26from weakref import WeakValueDictionary 

27from gutools.colors import * 

28 

29# ------------------------------------------------------- 

30# asyncio nested loops 

31# ------------------------------------------------------- 

32# import nest_asyncio 

33# nest_asyncio.apply() 

34 

35 

36TYPES = [] 

37for k in types.__all__: 

38 klass = getattr(types, k) 

39 try: 

40 if isinstance(TYPES, klass): 

41 pass 

42 TYPES.append(klass) 

43 except Exception as why: 

44 pass 

45TYPES = tuple(TYPES) 

46 

47BASIC_TYPES = tuple([int, float, str, bytes, dict, list, tuple, ]) 

48 

49# ----------------------------------------------------------- 

50# URI handling 

51# ----------------------------------------------------------- 

52reg_uri = re.compile( 

53 r""" 

54 ( 

55 (?P<fscheme> 

56 (?P<direction>[<|>])?(?P<scheme>[^:/]*)) 

57 :// 

58 (?P<fservice> 

59 ( 

60 (?P<auth> 

61 (?P<user>[^:@/]*?) 

62 (:(?P<password>[^@/]*?))? 

63 ) 

64 @)? 

65 (?P<host>[^@:/?]*) 

66 ) 

67 (:(?P<port>\d+))? 

68 )? 

69 

70 (?P<path>/[^?]*)? 

71 (\?(?P<query>[^#]*))? 

72 (\#(?P<fragment>.*))? 

73 """, 

74 re.VERBOSE | re.I | re.DOTALL) 

75 

76def parse_uri(uri, bind=None, **kw): 

77 """Extended version for parsing uris: 

78 

79 Return includes: 

80 

81 - *query_*: dict with all query parameters splited 

82 

83 If `bind` is passed, *localhost* will be replace by argument. 

84 

85 """ 

86 m = reg_uri.match(uri) 

87 if m: 

88 for k, v in m.groupdict().items(): 

89 if k not in kw or v is not None: 

90 kw[k] = v 

91 if bind: 

92 kw['host'] = kw['host'].replace('localhost', bind) 

93 if kw['port']: 

94 kw['port'] = int(kw['port']) 

95 kw['address'] = tuple([kw['host'], kw['port']]) 

96 if kw['query']: 

97 kw['query_'] = dict(parse.parse_qsl(kw['query'])) 

98 

99 return kw 

100 

101def build_uri(fscheme='', direction='', scheme='', 

102 host='', port='', path='', 

103 query='', fragment=''): 

104 """Generate a URI based on individual parameters""" 

105 uri = '' 

106 if fscheme: 

107 uri += fscheme 

108 else: 

109 if not direction: 

110 uri += scheme 

111 else: 

112 uri += f'{direction}{scheme}' 

113 if uri: 

114 uri += '://' 

115 

116 host = host or f'{uuid.getnode():x}' 

117 uri += host 

118 

119 if port: 

120 uri += f':{port}' 

121 if path: 

122 uri += f'{path}' 

123 if query: 

124 uri += f'?{query}' 

125 if fragment: 

126 uri += f'#{fragment}' 

127 

128 return uri 

129 

130 

131# --------------------------------------- 

132 

133def snake_case(name): 

134 s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) 

135 return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() 

136 

137# containers 

138def dset(d, key, value, *args, **kw): 

139 if key not in d: 

140 if args or kw: 

141 value = value(*args, **kw) 

142 d[key] = value 

143 return d[key] 

144 

145# next_lid = random.randint(-10**5, 10**5) 

146next_lid = random.randint(0, 10**5) 

147def new_uid(): 

148 global next_lid 

149 next_lid += 1 

150 return next_lid # TODO: remove, just debuging 

151 # return uidfrom(next_lid) 

152 

153def flatten(iterator, klass=None): 

154 if not isinstance(iterator, Iterable): 

155 yield iterator 

156 return 

157 # from https://stackoverflow.com/questions/2158395/flatten-an-irregular-list-of-lists 

158 for item in iterator: 

159 if isinstance(item, Iterable) and not isinstance(item, (str, bytes)): 

160 yield from flatten(item) 

161 elif klass is None or isinstance(item, klass): 

162 yield item 

163 

164def deep_search(item, patterns, result=None): 

165 """Explore deeply elements hierchachy searching items from 

166 certain classes. 

167 """ 

168 if result is None: 

169 result = dict() 

170 

171 remain = [item] 

172 

173 # TODO: avoid circular references 

174 used = set() 

175 while remain: 

176 item = remain.pop(0) 

177 idx = id(item) 

178 if idx in used: 

179 # print('remain[{}] : {} : <{}> : {}'.format(len(remain), klass, holder, idx)) 

180 continue 

181 used.add(idx) 

182 

183 klass = item.__class__.__name__ 

184 holder, idx = patterns.get(klass, (None, None)) 

185 

186 # print('remain[{}] : {} : <{}> : {}'.format(len(remain), klass, holder, idx)) 

187 if idx: 

188 # found an item of interest. 

189 holder = holder or klass 

190 key = getattr(item, idx) 

191 if key is not None: 

192 result.setdefault(holder, dict())[key] = item 

193 # print('>> [{}][{}] = {}'.format(holder, key, klass)) 

194 foo = 1 

195 

196 if isinstance(item, dict): 

197 # remain.extend(item.keys()) 

198 remain.extend(item.values()) 

199 elif isinstance(item, (list, tuple, set)): 

200 remain.extend(item) 

201 elif hasattr(item, '__dict__'): 

202 remain.append(item.__dict__) 

203 elif hasattr(item, '__slots__'): 

204 remain.append(dict([(k, getattr(item, k, None)) for k in item.__slots__])) 

205 else: 

206 foo = 1 # discard item, we don't know how to go deeper 

207 

208 return result 

209 

210# -------------------------------------------------------------------- 

211# containers helpers 

212# -------------------------------------------------------------------- 

213CONTAINERS = (dict, list, tuple) 

214def walk(container, root=tuple(), includes={}, excludes={}): 

215 """Walk recursive for an arbitrary container """ 

216 

217 def buid_key(*keys): 

218 "creare a key using the right factory" 

219 

220 # if factory in (tuple, ): 

221 # keys = list(flatten(keys)) 

222 # return factory(keys) 

223 # elif factory in (str, ): 

224 # return '/'.join([factory(k) for k in keys]) 

225 # else: 

226 # raise RuntimeError('Unknown factory type') 

227 results = list(keys[0]) 

228 results.extend(keys[1:]) 

229 return tuple(results) 

230 

231 if isinstance(container, dict): 

232 func = container.items 

233 # yield '{}/'.format(root, ), '<{}>'.format(container.__class__.__name__) 

234 yield buid_key(root, ), '<{}>'.format(container.__class__.__name__) 

235 elif isinstance(container, (list, tuple)): 

236 func = lambda : enumerate(container) 

237 yield buid_key(root, ), '<{}>'.format(container.__class__.__name__) 

238 else: 

239 # TODO: apply includes/excludes 

240 yield root, container # container is a single item 

241 return 

242 

243 # EXCLUDES 

244 recursive, match, same_klass = 0, 0, 0 

245 to_exclude = set() 

246 for key, item in func(): 

247 for klass, info in excludes.items(): 

248 if not isinstance(item, klass): 

249 continue 

250 same_klass += 1 

251 for attr, filters in info.items(): 

252 value = getattr(item, attr) 

253 if isinstance(value, CONTAINERS): 

254 recursive += 1 

255 continue 

256 results = [f(value) for f in filters] 

257 if any(match): 

258 match += 1 

259 to_exclude.add(key) 

260 match += any(results) 

261 if recursive == 0 and match == 1 and same_klass == 1: 

262 # discard the whole container because there is 1 single element 

263 # that match the excludes and the rest of element are related with 

264 # different klasses (e.g. a tuple containing related information) 

265 return 

266 

267 # TODO: coding 'includes' filters 

268 

269 # RECURSIVE exploring 

270 for key, item in func(): 

271 if key in to_exclude: 

272 continue 

273 new_key = buid_key(root, key) 

274 

275 if isinstance(item, CONTAINERS): 

276 yield from walk(item, new_key, includes, excludes) 

277 else: 

278 yield new_key, item 

279 

280def dive(container, key): 

281 # if factory in (str, ): 

282 # keys = [k for k in key.split('/') if k] 

283 # else: 

284 # keys = list(key) 

285 keys = list(key) 

286 # keys = [k if k != 'None' else None for k in keys] 

287 while keys: 

288 key = keys.pop(0) 

289 if key in container: 

290 container = container[key] 

291 else: 

292 break 

293 assert len(keys) == 0 

294 return container, key 

295 

296def divepush(container, key, item, default_container=dict): 

297 parent = container 

298 for key in key: 

299 parent, container = container, container.setdefault(key, default_container()) 

300 parent[key] = item 

301 

302def rebuild(walk_iterator, default_converter=None, default_container=dict, 

303 key_converters={}, container_converters={}, converters={}, ): 

304 """rebuild a tree structure from walk result""" 

305 result = None 

306 # if factory in (str, ): 

307 # key_converters = dict([re.compile(k, re.I | re.DOTALL), v] for k, v in key_converters.items()) 

308 

309 def new_container(item): 

310 """Try to get the container described in the item string: 

311 

312 <list> : return the list class 

313 <dict> : return the dict class 

314 

315 """ 

316 

317 if item and item[0] == '<' and item[-1] == '>': 

318 klass = item[1:-1] # the klass name 

319 try: 

320 klass = eval(klass) 

321 except Exception: 

322 klass = default_container # as default building container for non-dict alike containers 

323 return klass 

324 

325 convert = dict() 

326 for key, item in walk_iterator: 

327 klass = new_container(item) # try to guess if is a container 

328 if klass is not None and not issubclass(klass, dict): 

329 convert[key] = klass # for later converting the container 

330 klass = default_container # as default building container for non-dict alike containers 

331 

332 if klass: 

333 item = klass() 

334 

335 conv = converters.get(item.__class__, default_converter) 

336 if conv: 

337 item = conv(item) 

338 

339 if not key: 

340 result = item 

341 continue 

342 

343 assert result is not None, "root element must be found before any other" 

344 divepush(result, key, item) 

345 # we need to convert some nodes from botton-up to preserve 

346 # indexing with keys as str, not integers in case of list/tuples 

347 convert_keys = list(convert.keys()) 

348 # convert_keys.sort(key=lambda x: len(x.split('/')), reverse=True) 

349 convert_keys.sort(key=len, reverse=True) 

350 for key in convert_keys: 

351 klass = convert[key] 

352 # parent_key = key.split('/') 

353 # parent_key.pop(-2) 

354 # parent_key = '/'.join(parent_key) 

355 parent_container, parent_key = dive(result, key[:-1]) 

356 container, child_key = dive(result, key) 

357 keys = list(container.keys()) 

358 

359 if issubclass(klass, (list, tuple)): 

360 keys.sort(key=lambda x: int(x)) 

361 else: 

362 keys.sort() 

363 

364 values = [container[k] for k in keys] 

365 item = klass(values) 

366 # chain converters 

367 for conv in container_converters.get(klass, []): 

368 item = conv(key, item) 

369 

370 # if parent_key: 

371 # foo = 1 

372 # else: 

373 # # result = item 

374 # foo = 1 

375 

376 parent_container[child_key] = item 

377 

378 return result 

379 

380def unflattern(iterator, keys, container=None): 

381 """Build a structure info from a iterator, using some keys that 

382 acts as indexes of the structure. 

383 

384 The value is taken from item itself. 

385 """ 

386 container = container or dict() 

387 for item in iterator: 

388 parent, child = None, container 

389 for key in keys: 

390 if isinstance(item, (list, tuple)): 

391 item = list(item) 

392 key = item.pop(key) 

393 elif isinstance(item, dict): 

394 item = dict(item) 

395 key = item.pop(key) 

396 else: 

397 key = getattr(item, key) 

398 parent, child = child, child.setdefault(key, dict()) 

399 parent[key] = item 

400 return container 

401 

402def serializable_container(container): 

403 def _filter(container): 

404 for path, v in walk(container): 

405 if v in ('<tuple>', ): 

406 yield path, '<list>' 

407 elif v in (None, '<dict>', '<list>'): 

408 yield path, v 

409 elif v in ('<Config>', ): 

410 yield path, '<dict>' 

411 # must be at the end 

412 elif isinstance(v, BASIC_TYPES): 

413 yield path, v 

414 else: 

415 print(f"{RED}Don't know how to _filter{YELLOW} {path} = {v}{RESET}") 

416 

417 result = rebuild(_filter(container)) 

418 return result 

419 

420def update_container(container, value, *keys): 

421 """Try to change a value in a container tree of nested list and dict 

422 returning if tree has been changed or not. 

423 """ 

424 def change(container, key, value): 

425 modified = False 

426 if isinstance(container, (list, )): 

427 assert isinstance(key, int) 

428 if value: 

429 if key not in container: 

430 container.append(key) 

431 modified = True 

432 else: 

433 if key in container: 

434 container.remove(key) 

435 modified = True 

436 else: 

437 old = container.get(key) 

438 if value != old: 

439 container[key] = value 

440 modified = True 

441 return modified 

442 

443 

444 for key in keys[:-1]: 

445 if isinstance(container, (list, )): 

446 key = int(key) 

447 while len(container) < key: 

448 container.append(None) 

449 elif isinstance(container, (dict, )): 

450 container = container.setdefault(key, dict()) 

451 else: 

452 raise RuntimeError(f"Don't know how to handle {container.__class__} in container") 

453 return change(container, keys[-1], value) 

454 

455 

456def flatdict(container): 

457 """Convert any structure in a flat dict that can be 

458 rebuilt later. 

459 

460 flat = flatdict(container) 

461 copy = rebuild(flat.items()) 

462 assert flat == copy 

463 """ 

464 return dict([t for t in walk(container)]) 

465 

466def diffdict(current, last): 

467 """Compare to versions of the same flatdict container, giving: 

468 

469 - new items 

470 - changed items 

471 - deleted items 

472 

473 """ 

474 current_keys = set(current.keys()) 

475 last_keys = set(last.keys()) 

476 

477 new_items = dict([(k, current[k]) for k in current_keys.difference(last_keys)]) 

478 deleted_items = dict([(k, last[k]) for k in last_keys.difference(current_keys)]) 

479 

480 changed_items = dict() 

481 for k in current_keys.intersection(last_keys): 

482 c = current[k] 

483 l = last[k] 

484 if c != l: 

485 changed_items[k] = (c, l) 

486 

487 return new_items, changed_items, deleted_items 

488 

489 

490def sort_iterable(iterable, key=0): 

491 "Specific function for sorting a iterable from a specific key position" 

492 result = list() 

493 iterable = list(iterable) 

494 while iterable: 

495 l = min([len(k[key]) if hasattr(k[key], '__len__') else 0 for k in iterable]) 

496 subset = list() 

497 for i, k in reversed(list(enumerate(iterable))): 

498 if not hasattr(k[key], '__len__') or len(k[key]) == l: 

499 subset.append(k) 

500 iterable.pop(i) 

501 subset.sort(key=str, reverse=False) 

502 result.extend(subset) 

503 return result 

504 

505def merge(base, new, mode='add'): 

506 

507 def next_key_old(key): 

508 key = key.split('/') 

509 key[-1] = int(key[-1]) + 1 

510 return '/'.join(key) 

511 

512 def next_key(key): 

513 key = list(key) 

514 key[-1] += 1 # must be an integer 

515 return tuple(key) 

516 

517 def score(item): 

518 "function for sorting tables" 

519 return str(item) 

520 

521 def _merge(base, new): 

522 base_ = list(walk(base)) 

523 new_ = list(walk(new)) 

524 

525 base_.sort(key=score, reverse=False) 

526 new_.sort(key=score, reverse=False) 

527 

528 last_container = [] 

529 last_parent = None 

530 last_key = None 

531 last_idx = 0 

532 

533 a = b = None 

534 

535 while base_ or new_: 

536 if a is None and base_: 

537 a = base_.pop(0) 

538 if b is None and new_: 

539 b = new_.pop(0) 

540 

541 if a and a[1] in ('<list>', ) or \ 

542 b and b[1] in ('<list>', ): 

543 for b1 in last_container: 

544 last_key = next_key(last_key) 

545 yield (last_key, b1) 

546 

547 last_container = list() 

548 last_parent = b and b[0] 

549 

550 if a is None and b is not None: 

551 yield b 

552 b = None 

553 continue 

554 if b is None and a is not None: 

555 yield a 

556 a = None 

557 continue 

558 

559 a0 = str(a[0]) # to make comparissons possible 

560 b0 = str(b[0]) 

561 if b0 < a0: 

562 yield b 

563 b = None 

564 elif b == a: 

565 yield b 

566 a = b = None 

567 elif b0 == a0: 

568 # share the same key, but values differs 

569 if mode in ('replace', ): 

570 yield b 

571 elif mode in ('add', ): 

572 if isinstance(last_container, list): 

573 last_key = a[0] 

574 last_container.append(b[1]) 

575 yield a 

576 else: 

577 yield b 

578 a = b = None 

579 elif b0 > a0: 

580 yield a 

581 a = None 

582 else: 

583 raise RuntimeError('???') 

584 

585 foo = 1 

586 

587 # data = list() 

588 from pprint import pprint 

589 for pair in _merge(base, new): 

590 print("-"*40) 

591 print(f"{pair[0]}: {pair[1]}") 

592 # data.append(pair) 

593 # result = rebuild(data) 

594 # pprint(result) 

595 foo = 1 

596 

597 result = list(_merge(base, new)) 

598 

599 # key_converters = {'None': None, } 

600 

601 result = sort_iterable(result) 

602 result = rebuild(result) 

603 

604 return result 

605 

606 

607def uidfrom(lid): 

608 if not lid: 

609 return lid 

610 

611 lid = str(lid) 

612 lid = bytes(lid, encoding='UTF-8') 

613 

614 algo = 'md5' 

615 assert algo in hashlib.algorithms_guaranteed 

616 h = hashlib.new(algo) 

617 h.update(lid) 

618 uid = h.hexdigest() 

619 

620 return uid 

621 

622# https://stackoverflow.com/questions/6760685/creating-a-singleton-in-python 

623class Singleton(type): 

624 _instances = {} 

625 _callargs = {} 

626 def __call__(cls, *args, **kwargs): 

627 if cls in cls._instances: 

628 if args or kwargs and cls._callargs[cls] != (args, kwargs): 

629 warn = """ 

630WARNING: Singleton {} called with different args\n 

631 1st: {} 

632 now: {} 

633""".format(cls, cls._callargs[cls], (args, kwargs)) 

634 print(warn) 

635 else: 

636 cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) 

637 cls._callargs[cls] = (args, kwargs) 

638 return cls._instances[cls] 

639 

640 

641 

642# #Python2 

643# class MyClass(BaseClass): 

644 # __metaclass__ = Singleton 

645 

646# #Python3 

647# class MyClass(BaseClass, metaclass=Singleton): 

648 # pass 

649 

650class Xingleton(type): 

651 _instances = {} 

652 def __call__(cls, *args, **kwargs): 

653 __call__ = super(Xingleton, cls).__call__ 

654 __init__ = cls.__init__ 

655 args2 = prepare_call_args(__init__, *args, **kwargs) 

656 # args2 = args 

657 

658 existing = cls._instances.get(cls) # don't use setdefault here 

659 if existing is None: 

660 existing = cls._instances[cls] = WeakValueDictionary() 

661 

662 instance = existing.get(args2) 

663 if instance is None: 

664 instance = existing[args2] = __call__(*args2) 

665 return instance 

666 

667# ---------------------------------------------------------------------- 

668# instrospective and async calls 

669# ---------------------------------------------------------------------- 

670 

671class IntrospCaller(object): 

672 """base class for instrospective calls and async calls. 

673 

674 Requires a context where locate: 

675 - 'converters' dict for mapping arguments call with calleables or data 

676 - 'call_keys' set to extract the key used to store deferred calls 

677 

678 Example: 

679 

680 # context for automatic IntrospCaller 

681 converters = self.context.setdefault('converters', {}) 

682 converters['reqId'] = self._next_rid 

683 converters['contract'] = self._get_contract 

684 

685 self.context.setdefault('call_keys', set()).update(['reqId', ]) 

686 

687 for symbol in ('NQ', 'ES', 'DAX'): 

688 self.make_call(self.protocol.client.reqContractDetails) 

689 

690 It will make 3 calls that will use 'reqId' parameter as key 

691 

692 In order to drop the cache when request is done user must explicit invoke 

693 

694 self._drop_call(kw) 

695 

696 where kw contains the same parameter that has been used to make the call 

697 

698 """ 

699 

700 def __init__(self, context=None): 

701 self.context = context if context is not None else dict() 

702 self._make_request_cache = dict() 

703 self._calls = dict() 

704 

705 def make_call(self, func, **ctx): 

706 frame = inspect.currentframe().f_back 

707 env = dict(frame.f_locals) 

708 context = {} 

709 while frame and not context: 

710 # converters = frame.f_locals.get('kw', {}).get('__context__', {}).get('converters', {}) 

711 for name in ('context', 'ctx', ): 

712 context = frame.f_locals.get(name, {}) 

713 if context: 

714 break 

715 frame = frame.f_back 

716 converters = context.get('converters', {}) 

717 

718 # execute cached strategy 

719 strategy = self._get_strategy(func, env, converters) 

720 kw = execute_strategy(strategy, env, **converters) 

721 keys = self._set_call(func, kw) 

722 return func(**kw), keys 

723 

724 def _get_strategy(self, func, env, converters): 

725 strategy = self._make_request_cache.get(func) 

726 if strategy is not None: 

727 return strategy 

728 

729 strategy = strategy_call(func, env, **converters) 

730 self._make_request_cache[func] = strategy 

731 return strategy 

732 

733 def _set_call(self, func, kw): 

734 # store call parameters for later use 

735 for key in self.context['call_keys'].intersection(kw): 

736 value = kw[key] 

737 self._calls[value] = func, value, kw 

738 

739 # just 1 iteration, maybe more than one for advanced uses 

740 return func, value, kw 

741 

742 def _drop_call(self, kw): 

743 result = [] 

744 for key in self.context['call_keys'].intersection(kw): 

745 result.append(self._calls.pop(kw[key], None)) 

746 return result 

747 

748 

749 

750def strategy_call(func, env, **converters): 

751 """Try to find matching calling arguments making a 

752 deep search in `**kw` arguments.""" 

753 func_info = inspect.getfullargspec(func) 

754 callargs = list(func_info.args) 

755 defaults = list(func_info.defaults or []) 

756 annotations = func_info.annotations or {} 

757 strategy = dict() 

758 while callargs: 

759 attr = callargs.pop(0) 

760 klass = annotations.get(attr) 

761 # find a object that match attr 

762 best_name, best_score = None, 0 

763 for name, value in env.items(): 

764 if name in strategy: 

765 continue 

766 # d1 = likelyhood(attr, name) 

767 # d2 = likelyhood2(attr, name) 

768 # print("<{}, {}> : {} {}".format(attr, name, d1, d2)) 

769 score = (likelyhood4(attr, name) + \ 

770 (klass in (None, value.__class__))) / 2 

771 if 0.50 <= score > best_score: 

772 best_name, best_score = name, score 

773 if best_name: 

774 strategy[attr] = ('env', best_name) 

775 continue 

776 # otherwise we need to use converters 

777 # try to find the best converter possible 

778 # same or similar names, same return values is provided by signature 

779 best_name, best_score = None, 0 

780 for name, conv in converters.items(): 

781 conv_info = inspect.getfullargspec(conv) 

782 ret = conv_info.annotations.get('return') 

783 if ret in (None, klass): 

784 score = likelyhood(attr, name) 

785 if score > best_score: 

786 best_name, best_score = name, score 

787 if best_name: 

788 strategy[attr] = ('conv', best_name) 

789 

790 if isinstance(func, types.MethodType): 

791 bound = func_info.args[0] 

792 assert bound == 'self' # is a convenience criteria 

793 strategy.pop(bound, None) 

794 

795 # self._make_request_cache[request] = strategy 

796 return strategy 

797 

798def execute_strategy(strategy, env, **converters): 

799 kw2 = dict() 

800 for attr, (where, best) in strategy.items(): 

801 if where in ('env'): 

802 value = env.get(best) 

803 elif where in ('conv'): 

804 conv = converters[best] 

805 args, kw = prepare_call(conv, **env) 

806 value = conv(*args, **kw) 

807 kw2[attr] = value 

808 return kw2 

809 

810def update_context(context, *args, **kw): 

811 __avoid_nulls__ = kw.pop('__avoid_nulls__', True) 

812 

813 if __avoid_nulls__: 

814 for k, v in kw.items(): 

815 if v is not None: 

816 context[k] = v 

817 else: 

818 context.update(kw) 

819 

820 for item in args: 

821 if isinstance(item, dict): 

822 d = item 

823 elif hasattr(item, 'as_dict'): 

824 d = item.as_dict() 

825 elif hasattr(item, '__dict__'): 

826 d = item.__dict__ 

827 elif hasattr(item, '__getstate__'): 

828 d = item.__getstate__() 

829 else: 

830 d = dict() 

831 for k in dir(item): 

832 if k.startswith('_'): 

833 continue 

834 v = getattr(item, k) 

835 if v.__class__.__name__ == 'type' or isinstance(v, TYPES):# (types.FunctionType, types.MethodType, types.MethodWrapperType, types.BuiltinFunctionType)): 

836 continue 

837 d[k] = v 

838 

839 if __avoid_nulls__: 

840 for k, v in d.items(): 

841 if v is not None: 

842 context[k] = v 

843 else: 

844 context.update(d) 

845 

846def prepare_call(func, *args, **kw): 

847 # collect available variables from stack 

848 __max_frames_back__ = kw.pop('__max_frames_back__', 1) 

849 frame = sys._getframe(1) 

850 frameN = sys._getframe(__max_frames_back__) 

851 _locals = list() 

852 while __max_frames_back__ > 0 and frame: 

853 _locals.append(frame.f_locals) 

854 __max_frames_back__ -= 1 

855 frame = frame.f_back 

856 _locals.reverse() 

857 kw0 = dict() 

858 for st in _locals: 

859 for item in st.values(): 

860 update_context(kw0, item) 

861 update_context(kw0, st) 

862 kw0.update(kw) 

863 

864 # try to match function calling args 

865 info = inspect.getfullargspec(func) 

866 kw2 = dict() 

867 args = list(args) 

868 callargs = list(info.args) 

869 defaults = list(info.defaults or []) 

870 # remove self for MethodType, and __init__ 

871 if isinstance(func, types.MethodType) or \ 

872 func.__name__ in ('__init__', ) or \ 

873 func.__class__.__name__ in ('type', ): # klass.__call__ 

874 if callargs[0] == 'self': # is a convenience criteria 

875 callargs.pop(0) 

876 kw0.pop('self', None) 

877 

878 while len(defaults) < len(callargs): 

879 defaults.insert(0, None) 

880 

881 while callargs: 

882 attr = callargs.pop(0) 

883 value = defaults.pop(0) 

884 if attr in kw0: 

885 value = kw0.pop(attr) 

886 if args: 

887 if id(value) == id(args[0]): 

888 args.pop(0) 

889 elif args: 

890 value = args.pop(0) 

891 kw2[attr] = value 

892 

893 if not info.varargs: 

894 if len(args) > 0: 

895 raise RuntimeError('too many positional args ({}) for calling {}(...)'.format(args, func.__name__)) 

896 if info.varkw: 

897 kw2.update(kw0) 

898 # if isinstance(func, types.MethodType): 

899 # bound = info.args[0] 

900 # assert bound == 'self' # is a convenience criteria 

901 # kw2.pop(bound) 

902 return args, kw2 

903 

904def prepare_call_args(func, *args, **kw): 

905 args2, kw2 = prepare_call(func, *args, **kw) 

906 kw2.pop('self', None) 

907 info = inspect.getfullargspec(func) 

908 args2.extend([k for k in info.args if k in kw2]) 

909 return tuple([kw2[k] for k in args2]) 

910 

911 

912def _call(func, *args, **kw): 

913 # 1. try to execute directly 

914 if len(args) == 1: 

915 if isinstance(args[0], dict): 

916 args[0].update(kw) 

917 args, kw = [], args[0] 

918 elif isinstance(args[0], (list, tuple)): 

919 args = args[0] 

920 

921 try: 

922 args2, kw2 = prepare_call(func, *args, **kw) 

923 return func(*args2, **kw2) 

924 except Exception as why: # TODO: which is the right exception here? (missing args) 

925 # TypeError 

926 traceback.print_exc() 

927 

928 print(f"{YELLOW}ERROR: _call({func}) -> {BLUE}{why}{RED}") 

929 traceback.print_exc(file=sys.stdout) 

930 # exc_info = sys.exc_info() 

931 # tb = exc_info[-1] 

932 # tb.tb_next 

933 # frame = tb.tb_next.tb_frame 

934 

935 # print(traceback.print_exception(*exc_info)) 

936 print(f"{RESET}") 

937 # del exc_info 

938 foo = 1 

939 

940 # 2. try to find calling arguments from passed dict as env 

941 # TODO: include default **converters as well? 

942 

943 strategy = strategy_call(func, kw) 

944 kw2 = execute_strategy(strategy, kw) 

945 try: 

946 return func(**kw2) 

947 except Exception as why: # TODO: which is the right exception here? (missing args) 

948 foo = 1 

949 

950def async_call(func, *args, **kw): 

951 main = _call(func, *args, **kw) 

952 assert asyncio.iscoroutine(main) 

953 asyncio.run(main) 

954 

955# --------------------------------------------------------------------- 

956# A Dynamic Programming based Python program for edit distance problem 

957# --------------------------------------------------------------------- 

958def editDistDP(name1, name2): 

959 m, n = len(name1), len(name2) 

960 # Create a table to store results of subproblems 

961 dp = [[0 for x in range(n+1)] for x in range(m+1)] 

962 

963 # Fill d[][] in bottom up manner 

964 for i in range(m+1): 

965 for j in range(n+1): 

966 

967 # If first string is empty, only option is to 

968 # insert all characters of second string 

969 if i == 0: 

970 dp[i][j] = j # Min. operations = j 

971 

972 # If second string is empty, only option is to 

973 # remove all characters of second string 

974 elif j == 0: 

975 dp[i][j] = i # Min. operations = i 

976 

977 # If last characters are same, ignore last char 

978 # and recur for remaining string 

979 elif name1[i-1] == name2[j-1]: 

980 dp[i][j] = dp[i-1][j-1] 

981 

982 # If last character are different, consider all 

983 # possibilities and find minimum 

984 else: 

985 dp[i][j] = 1 + min(dp[i][j-1], # Insert 

986 dp[i-1][j], # Remove 

987 dp[i-1][j-1]) # Replace 

988 

989 return dp[m][n] 

990 

991def wlikelyhood(set1, set2, **weights): 

992 W = 0 

993 S = 0 

994 for k, w in weights.items(): 

995 s1, s2 = set1.get(k, []), set2.get(k, []) 

996 if s1 or s2: 

997 # only weight average when any set is not empty 

998 W += w 

999 s = likelyhood4(s1, s2) 

1000 S += w * s 

1001 

1002 return S / (W or 1) 

1003 

1004 

1005def likelyhood4(set1, set2): 

1006 set1, set2 = list(set1), list(set2) 

1007 N = (len(set1) + len(set2)) or 1 

1008 S = 0 

1009 def check(s1, s2): 

1010 s = 0 

1011 while s1: 

1012 item = s1.pop(0) 

1013 if item in s2: 

1014 s2.remove(item) 

1015 s += 2 

1016 else: 

1017 s -= 1 

1018 return s 

1019 S = check(set1, set2) + check(set2, set1) 

1020 return S / N 

1021 

1022def likelyhood3(set1, set2): 

1023 n = min(len(set1), len(set2)) 

1024 if n: 

1025 return 1 - editDistDP(set1, set2) / n 

1026 return 0 

1027 

1028def likelyhood2(name1, name2): 

1029 n = min(len(name1), len(name2)) 

1030 return 1 - editDistDP(name1.lower(), name2.lower()) / n 

1031 

1032def likelyhood(name1, name2): 

1033 if len(name1) > len(name2): 

1034 name1, name2 = list(name2.lower()), list(name1.lower()) 

1035 else: 

1036 name1, name2 = list(name1.lower()), list(name2.lower()) 

1037 

1038 n = len(name1) 

1039 score = 0 

1040 while name1: 

1041 a = name1.pop(0) 

1042 if a in name2: 

1043 idx = name2.index(a) 

1044 name2.pop(idx) 

1045 else: 

1046 idx = n 

1047 score += 1 / (1 + idx) ** 2 

1048 

1049 return score / n 

1050 

1051 

1052 

1053def get_calling_function(level=1): 

1054 """finds the calling function in many decent cases.""" 

1055 # stack = inspect.stack(context) 

1056 # fr = sys._getframe(level) # inspect.stack()[1][0] 

1057 stack = inspect.stack() 

1058 while level < len(stack): 

1059 fr = stack[level][0] 

1060 co = fr.f_code 

1061 for get in ( 

1062 lambda: fr.f_globals[co.co_name], 

1063 lambda: getattr(fr.f_locals['self'], co.co_name), 

1064 lambda: getattr(fr.f_locals['cls'], co.co_name), 

1065 lambda: fr.f_back.f_locals[co.co_name], # nested 

1066 lambda: fr.f_back.f_locals['func'], # decorators 

1067 lambda: fr.f_back.f_locals['meth'], 

1068 lambda: fr.f_back.f_locals['f'], 

1069 ): 

1070 try: 

1071 func = get() 

1072 except (KeyError, AttributeError): 

1073 pass 

1074 else: 

1075 if func.__code__ == co: 

1076 return func 

1077 level += 1 

1078 raise AttributeError("func not found") 

1079 

1080def retry(delay=0.1): 

1081 """Retray the same call where the funcion is invokerd but 

1082 a few instant later""" 

1083 frame = inspect.currentframe().f_back 

1084 calling_args = frame.f_code.co_varnames[:frame.f_code.co_argcount] 

1085 calling_args = [frame.f_locals[k] for k in calling_args] 

1086 func = get_calling_function(2) 

1087 if isinstance(func, types.MethodType): 

1088 calling_args.pop(0) 

1089 

1090 loop = asyncio.get_event_loop() 

1091 loop.call_later(0.2, func, *calling_args) 

1092 

1093 foo = 1 

1094 

1095async def add_to_loop(*aws, delay=0.1, loop=None): 

1096 for coro in aws: 

1097 asyncio.ensure_future(coro, loop=loop) 

1098 await asyncio.sleep(delay, loop=loop) 

1099 

1100# -------------------------------------------------- 

1101# RegExp operations 

1102# -------------------------------------------------- 

1103def _prepare_test_regext(regexp=None, wildcard=None, test=None): 

1104 test = test or list() 

1105 

1106 if isinstance(regexp, (list, tuple)): 

1107 test.extend(regexp) 

1108 else: 

1109 test.append(regexp) 

1110 

1111 if not isinstance(wildcard, (list, tuple)): 

1112 wildcard = [wildcard] 

1113 for wc in wildcard: 

1114 if wc and isinstance(wc, str): 

1115 wc = fnmatch.translate(wc) 

1116 test.append(wc) 

1117 

1118 test = [re.compile(m).search for m in test if m] 

1119 

1120 return test # you can compile a compiled expression 

1121 

1122def _find_match(string, test): 

1123 for match in test: 

1124 m = match(string) 

1125 if m: 

1126 return m 

1127 

1128def _return_matched_info(m, info): 

1129 if info == 'd': 

1130 return m.groupdict() 

1131 elif info == 'g': 

1132 return m.groups() 

1133 

1134def _return_unmatched(info): 

1135 if info == 'd': 

1136 return dict() 

1137 elif info == 'g': 

1138 return list() 

1139 

1140def parse_string(string, regexp=None, wildcard=None, info=None): 

1141 test = _prepare_test_regext(regexp, wildcard) 

1142 m = _find_match(string, test) 

1143 if m: 

1144 return _return_matched_info(m, info) 

1145 return _return_unmatched(info) # to return cpmpatible values in upper stack 

1146 

1147def parse_date(date): 

1148 if isinstance(date, str): 

1149 return parser.parse(date) 

1150 return date 

1151 

1152# -------------------------------------------------- 

1153# Speed control 

1154# -------------------------------------------------- 

1155class SpeedControl(list): 

1156 """Calculate the next pause needed to keep an average 

1157 speed limit between requests to some service. 

1158 """ 

1159 def __init__(self, lapse=0, N=1, *args, **kwargs): 

1160 self.lapse = lapse 

1161 if lapse: 

1162 N = N or int(100/(lapse or 1)) + 1 

1163 N = min(max(N, 10), 100) 

1164 else: 

1165 N = 1 

1166 self.N = N 

1167 

1168 @property 

1169 def pause(self): 

1170 "The next pause before make the next request" 

1171 N = len(self) 

1172 t1 = time.time() 

1173 self.append(t1) 

1174 t0 = self.pop(0) if len(self) > self.N else self[0] 

1175 t1p = self.lapse * N + t0 

1176 return t1p - t1 

1177 

1178 @property 

1179 def speed(self): 

1180 "Return the current average speed" 

1181 elapsed = self[-1] - self[0] 

1182 if elapsed > 0: 

1183 return len(self) / elapsed 

1184 return 0 

1185 # return float('inf') 

1186 

1187 @property 

1188 def round_speed(self): 

1189 # return math.ceil(self.speed * 100) / 100 

1190 return int(self.speed * 100) / 100 

1191 

1192 

1193 

1194 

1195# -------------------------------------------------- 

1196# Progress 

1197# -------------------------------------------------- 

1198class ProgressControl(list): 

1199 def __init__(self, d0, d1): 

1200 self.d0 = d0 

1201 self.range_ = d1 - d0 

1202 # assert self.range_ > 0 

1203 

1204 def progress(self, d): 

1205 return int(10000*(d - self.d0)/self.range_) / 100 

1206 

1207 

1208# -------------------------------------------------- 

1209# File perations 

1210# -------------------------------------------------- 

1211def fileiter(top, regexp=None, wildcard=None, info=None, relative=False): 

1212 """Iterate over files 

1213 info == 'd' : returns filename, regexp.groupdict() 

1214 info == 'g' : returns filename, regexp.groups() 

1215 info == None: : returns filename 

1216 

1217 Allow single expressions or iterator as regexp or wildcard params 

1218 """ 

1219 test = _prepare_test_regext(regexp, wildcard) 

1220 for root, _, files in os.walk(top): 

1221 for name in files: 

1222 filename = os.path.join(root, name) 

1223 m = _find_match(filename, test) 

1224 if m: 

1225 if relative: 

1226 filename = filename.split(top)[-1][1:] 

1227 m = _return_matched_info(m, info) 

1228 if m: 

1229 yield filename, m 

1230 else: 

1231 yield filename 

1232 foo = 1 

1233 

1234def copyfile(src, dest, override=False): 

1235 if os.path.exists(dest) and not override: 

1236 return 

1237 

1238 folder = os.path.dirname(dest) 

1239 if not os.path.exists(folder): 

1240 os.makedirs(folder) 

1241 with open(src, 'rb') as s: 

1242 with open(dest, 'wb') as d: 

1243 d.write(s.read()) 

1244 

1245 st = os.stat(src) 

1246 os.chmod(dest, stat.S_IMODE(st.st_mode)) 

1247 

1248def expandpath(path): 

1249 if path: 

1250 path = os.path.expanduser(path) 

1251 path = os.path.expandvars(path) 

1252 path = os.path.abspath(path) 

1253 while path[-1] == '/': 

1254 path = path[:-1] 

1255 return path 

1256 

1257def load_config(path): 

1258 raise DeprecationWarning() 

1259 loader = { 

1260 'yaml': yaml.load, 

1261 'json': json.load, 

1262 } 

1263 config = dict() 

1264 if os.path.exists(path): 

1265 ext = os.path.splitext(path)[-1][1:] 

1266 loader = loader.get(ext, loader['yaml']) 

1267 with open(path, 'r') as f: 

1268 config = loader(f) or config 

1269 return config 

1270 

1271def save_config(config, path): 

1272 raise DeprecationWarning() 

1273 

1274 saver = { 

1275 'yaml': partial(yaml.dump, default_flow_style=False), 

1276 'json': json.dump, 

1277 } 

1278 ext = os.path.splitext(path)[-1][1:] 

1279 saver = saver.get(ext, saver['yaml']) 

1280 with open(path, 'w') as f: 

1281 config = saver(config, f) 

1282 

1283def yaml_decode(raw: str): 

1284 fd = StringIO(raw) 

1285 return yaml.load(fd) 

1286 

1287def yaml_encode(item: str): 

1288 fd = StringIO() 

1289 yaml.dump(item, fd, default_flow_style=False) 

1290 return fd.getvalue() 

1291 

1292def identity(item): 

1293 "convenience function" 

1294 return item 

1295 

1296def test_pid(pid): 

1297 try: 

1298 pid = int(pid) 

1299 os.kill(pid, 0) 

1300 except OSError: 

1301 return False 

1302 else: 

1303 return True 

1304 

1305# - End -