Coverage for test_stm.py : 0%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import pytest
3import random
4import operator
5from time import time, sleep
6from select import select
8from gutools.tools import get_calling_function, flatten
9from gutools.utests import speed_meter
10from gutools.stm import Layer, Reactor, DebugReactor, STATE_INIT, STATE_READY, STATE_END, \
11 QUITE_STATE, EVENT_QUIT, EVENT_TERM, MERGE_ADD, MERGE_REPLACE_EXISTING
13from gutools.stm import _TestBrowserProtcol
15class _TestReactor(Reactor):
16 def __init__(self, events=None):
17 super().__init__()
18 self._fake_events = events # iterator
19 self._queue = list()
21 def stop(self):
22 print("Stoping reactor")
23 super().stop()
25 # def next_event(self):
26 # try:
27 # while True:
28 # if self._queue:
29 # return self._queue.pop(0)
30 # elif self._fake_events:
31 # return self._fake_events.__next__()
32 # sleep(0.1)
33 # except StopIteration:
34 # self.running = False
35 # return EVENT_TERM, None
37 def next_event(self):
38 """Blocks waiting for an I/O event or timer.
39 Try to compensate delays by substracting time.time() and sub operations."""
40 def close_sock(fd):
41 self._protocols[fd].eof_received()
42 self.close_channel(self._transports[fd])
44 while True:
45 if self._queue:
46 return self._queue.pop(0)
47 elif self._fake_events:
48 try:
49 return self._fake_events.__next__()
50 except StopIteration:
51 self.running = False
52 return EVENT_TERM, None
54 # there is not events to process. look for timers and I/O
55 if self._timers:
56 when, restart, key = timer = self._timers[0]
57 seconds = when - self.time
58 if seconds > 0:
59 rx, _, ex = select(self._transports, [], self._transports, seconds)
60 for fd in rx:
61 try:
62 raw = fd.recv(0xFFFF)
63 if raw:
64 self._protocols[fd].data_received(raw)
65 else:
66 close_sock(fd)
67 except Exception as why:
68 close_sock(fd)
69 pass
71 for fd in ex: # TODO: remove if never is invoked
72 close_sock(fd)
73 foo = 1
74 else:
75 self.publish(key, None)
76 # --------------------------
77 # rearm timer
78 self._timers.pop(0)
79 if restart <= 0:
80 continue # is a timeout timer, don't restart it
81 when += restart
82 timer[0] = when
83 # insert in right position
84 i = 0
85 while i < len(self._timers):
86 if self._timers[i][0] > when:
87 self._timers.insert(i, timer)
88 break
89 i += 1
90 else:
91 self._timers.append(timer)
92 else:
93 # duplicate code for faster execution
94 rx, _, _ = select(self._transports, [], [], 1)
95 for fd in rx:
96 try:
97 raw = fd.recv(0xFFFF)
98 if raw:
99 self._protocols[fd].data_received(raw)
100 else:
101 close_sock(fd)
102 except Exception as why:
103 close_sock(fd)
104 pass
105 foo = 1
106 foo = 1
108def populate1(container, population, n):
109 while n:
110 ctx = container
111 for x in population:
112 key = str(random.randint(0, x))
113 ctx = ctx.setdefault(key, dict())
115 key = str(random.randint(0, population[-1]))
116 ctx = ctx.setdefault(key, list())
117 ctx.append(n)
118 n -= 1
120def populate2(container, population, n):
121 while n:
122 key = tuple([str(random.randint(0, x)) for x in population])
123 container[key] = n
124 n -= 1
126class _TestLayer(Layer):
128 def _setup__testlayer(self):
129 states = {
130 STATE_END: [[], ['bye'], []],
131 }
133 transitions = {
134 STATE_READY: {
135 'each:5,1': [
136 ['READY', [], ['timer']],
137 ],
138 'each:5,21': [
139 [STATE_READY, [], ['bye']],
140 ],
141 },
142 }
143 return states, transitions, MERGE_ADD
145 def start(self, **kw):
146 print("Hello World!!")
147 self.t0 = time()
148 self._func_calls = dict()
149 self._log_function()
151 def term(self, key, **kw):
152 elapsed = time() - self.t0
153 print(f"Term {key}: {elapsed}")
154 super().term(key, **kw)
156 def timer(self, **kw):
157 "Empty timer to be overrided"
159 def _log_function(self):
160 func = get_calling_function(level=2)
161 name = func.__func__.__name__
162 self._func_calls.setdefault(name, 0)
163 self._func_calls[name] += 1
165 def _check_log(self, expected):
166 """Check if observerd calls match expected ones.
167 Expected values can be integer and iterables.
168 Ranges may be defined with strings like '7-8'
169 """
170 for name, _ve in expected.items():
171 ve = set()
172 for v in flatten(list([_ve])):
173 # allow ranges
174 v = str(v).split('-')
175 v.append(v[0])
176 ve.update(range(int(v[0]), int(v[1]) + 1))
178 vr = self._func_calls.get(name, None)
179 if vr not in ve:
180 self.reactor.stop()
181 raise RuntimeError(f"Fuction {name} is expected to be called {ve} time, but where {vr}")
184class Foo(_TestLayer):
185 def __init__(self, states=None, transitions=None, context=None):
186 super().__init__(states, transitions, context)
188 self.b = 0
190 def _setup_test_foo_1(self):
191 states = {
192 STATE_INIT: [[], [], ['hello']],
193 STATE_READY: [[], ['ready'], []],
194 STATE_END: [[], ['stop'], []],
195 }
196 transitions = {
197 }
198 return states, transitions, MERGE_ADD
200 def _setup_test_foo_2(self):
201 states = {
202 }
203 transitions = {
204 STATE_READY: {
205 EVENT_TERM: [
206 [STATE_READY, ['b <= 5'], ['not_yet']],
207 [STATE_READY, ['b > 5'], ['from_init_to_end', 'term']],
208 ],
209 },
210 }
211 return states, transitions, MERGE_REPLACE_EXISTING
213 def timer(self, **kw):
214 self.context['b'] += 1
215 print(f">> timer: b={self.context['b']}")
217 def hello(self, **kw):
218 print(">> Hello!")
219 self._log_function()
221 def ready(self, **kw):
222 print(">> Ready")
223 self._log_function()
225 def stop(self, **kw):
226 print(">> Stop !!")
227 self._log_function()
229 def not_yet(self, key, **kw):
230 print(f'>> Received: {key}, but not_yet()')
231 self._log_function()
233 def from_init_to_end(self, key, **kw):
234 print(f'>> Received: {key}, OK shutdown now')
235 self._log_function()
237 def term(self, **kw):
238 expected = {
239 'from_init_to_end': 1,
240 'hello': 1,
241 'not_yet': 2,
242 'ready': 12,
243 'start': 1
244 }
245 self._check_log(expected)
246 super().term(**kw)
248class iRPNCalc(Layer):
249 operations = {
250 '+': operator.add,
251 '-': operator.sub,
252 '*': operator.mul,
253 '/': operator.truediv,
254 'neg': operator.neg,
255 }
257 def __init__(self, states, transitions):
258 self.stack = []
259 super().__init__(states=states, transitions=transitions)
260 foo = 1
262 def start(self, data, **kw):
263 print("Hello World!!")
265 def push(self, data, **kw):
266 self.stack.append(data)
268 def exec1(self, data, **kw):
269 a = self.stack.pop()
270 r = self.operations[data](a)
271 self.stack.append(r)
273 def exec2(self, data, **kw):
274 b = self.stack.pop()
275 a = self.stack.pop()
276 try:
277 r = self.operations[data](a, b)
278 self.stack.append(r)
279 except ZeroDivisionError:
280 pass
283class RPNCalc1(iRPNCalc):
284 def __init__(self):
285 # Simplier RPN, less states
286 states = {
287 STATE_INIT: [[], ['start'], []],
288 'READY': QUITE_STATE,
289 'PUSH': [[], ['push'], []],
290 'EXEC1': [[], ['exec1'], []],
291 'EXEC2': [[], ['exec2'], []],
292 STATE_END: [[], ['bye'], []],
293 }
295 transitions = {
296 'INIT': {
297 None: [
298 ['READY', [], []],
299 ],
300 },
301 'READY': {
302 'input': [
303 ['PUSH', ['isinstance(data, int)'], []],
304 ['EXEC2', ["data in ('+', '-', '*', '/')", 'len(stack) >= 2'], []],
305 ['EXEC1', ["data in ('neg', )", 'len(stack) >= 1'], []],
306 ],
307 EVENT_TERM: [
308 [STATE_END, [], []],
309 ],
310 },
311 'PUSH': {
312 None: [
313 ['READY', [], []],
314 ],
315 },
316 'EXEC1': {
317 None: [
318 ['READY', [], []],
319 ],
320 },
321 'EXEC2': {
322 None: [
323 ['READY', [], []],
324 ],
325 },
326 }
327 super().__init__(states, transitions)
329class RPNCalc2(iRPNCalc):
330 def __init__(self):
331 # Simplier RPN, less states
332 states = {
333 STATE_INIT: [[], ['start'], [], []],
334 'READY': QUITE_STATE,
335 STATE_END: [[], ['bye'], [], []],
336 }
338 transitions = {
339 'INIT': {
340 None: [
341 ['READY', [], []],
342 ],
343 },
344 'READY': {
345 'input': [
346 ['READY', ['isinstance(data, int)'], ['push']],
347 ['READY', ["data in ('+', '-', '*', '/')", 'len(stack) >= 2'], ['exec2']],
348 ['READY', ["data in ('neg', )", 'len(stack) >= 1'], ['exec1']],
349 ],
350 EVENT_TERM: [
351 [STATE_END, [], []],
352 ],
353 },
354 STATE_END: {
355 },
356 }
357 super().__init__(states, transitions)
361class Clock(_TestLayer):
363 def _setup_test_clock(self):
364 states = {
365 }
366 transitions = {
367 STATE_READY: {
368 # set an additional timer
369 'each:3,2': [
370 [STATE_READY, [], ['timer']],
371 ],
372 },
373 }
374 return states, transitions, MERGE_ADD
376 def timer(self, key, **kw):
377 elapsed = time() - self.t0
378 print(f" > Timer {key}: {elapsed}")
379 restart, offset = [int(x) for x in key[5:].split(',')]
381 cycles = (elapsed - offset) / restart
382 assert cycles - int(cycles) < 0.01
385class Parent(_TestLayer):
387 def _setup_test_parent(self):
388 states = {
389 'FOO': [[], ['bar'], []],
390 }
391 transitions = {
392 }
393 return states, transitions, MERGE_ADD
395 def bar(self, **kw):
396 pass
398class Crawler(_TestLayer):
399 """Dummy Web Crawler for testing dynamic channels creation and
400 protocols.
402 - a timer creates a new channel from time to time.
403 - a timer use a free channel to make a request.
404 - a timeout will term the reactor.
406 """
407 def __init__(self, states=None, transitions=None, context=None):
408 super().__init__(states, transitions, context)
409 self.channels = list() # free channels
410 self.working = list() # in-use channels
411 self.downloaded = 0
413 def _setup_test_crawler(self):
414 states = {
415 'GET': [[], ['get_page'], []],
416 'SAVE': [[], ['save_page'], []],
417 }
418 transitions = {
419 STATE_READY: {
420 'each:3,2': [
421 [STATE_READY, ['len(channels) <= 3'], ['new_channel']],
422 ],
423 'each:1,1': [
424 ['GET', ['len(working) <= 3 and downloaded < 10'], []], # just to use a state
425 ],
426 'each:2,1': [
427 ['SAVE', ['len(working) > 3'], []], # just to use a state
428 ],
429 'each:5,21': [
430 [STATE_READY, ['downloaded >= 10'], ['bye']],
431 ],
432 },
433 'GET': {
434 None: [
435 [STATE_READY, [], []], # going back to READY
436 ],
437 },
438 'SAVE': {
439 None: [
440 [STATE_READY, [], []], # going back to READY
441 ],
442 },
443 }
444 return states, transitions, MERGE_REPLACE_EXISTING
446 def new_channel(self, **kw):
447 print(' > new_channel')
448 self._log_function()
449 self.channels.append(False)
451 def get_page(self, **kw):
452 "Simulate a download page request"
453 for i, working in enumerate(self.channels):
454 if not working:
455 self._log_function()
456 print(' > get_page')
457 self.channels[i] = True
458 self.working.append(i)
459 break
461 def save_page(self, **kw):
462 "Simulate that page has been downloaded and saved to disk"
464 for worker, channel in reversed(list(enumerate(self.working))):
465 if random.randint(0, 100) < 25:
466 self._log_function()
467 self.context['downloaded'] += 1
468 print(f" > save_page: {self.context['downloaded']}")
469 self.channels[channel] = False
470 self.working.pop(worker)
472 def term(self, **kw):
473 expected = {
474 'get_page': '12-13',
475 'new_channel': 4,
476 'save_page': '10-13',
477 'start': 1
478 }
479 self._check_log(expected)
480 super().term(**kw)
483class HashableEvents(Layer):
484 """Test class for testing events using any hashable event
485 - strings
486 - tuples
487 - integers/floats
488 """
489 def _setup_test_hashable_events(self):
490 self.channels = list() # free channels
491 self.working = list() # in-use channels
493 states = {
494 }
495 transitions = {
496 STATE_READY: {
497 ('hello', 'world'): [
498 [STATE_READY, [], ['hello_world']],
499 ],
500 1: [
501 [STATE_READY, [], ['uno']],
502 ],
503 },
504 }
505 return states, transitions, MERGE_ADD
507 def hello_world(self, **kw):
508 print(' > hello_world')
509 foo = 1
511 def uno(self, **kw):
512 print(' > uno')
513 foo = 1
516# -----------------------------------------------------
517# Atlas demo
518# -----------------------------------------------------
519class GraphExample(_TestLayer):
520 def _setup_test_graph(self):
521 states = {
522 'ASK': [[], ['bar'], []],
523 'WAIT': [[], [], []],
524 }
525 transitions = {
526 STATE_READY: {
527 'each:5,1': [
528 ['ASK', [], ['bar']],
529 ],
530 },
531 'ASK': {
532 None: [
533 ['WAIT', [], []],
534 ],
535 },
536 }
537 return states, transitions, MERGE_ADD
539 def bar(self, **kw):
540 pass
543# -----------------------------------------------------
544# tests
545# -----------------------------------------------------
547def test_layer():
549 foo = Foo()
550 reactor = _TestReactor()
551 reactor.attach(foo)
552 # reactor.graph()
553 reactor.run()
554 foo = 1
556def test_timeit():
557 population = [100, 100, 10]
559 states1 = dict()
560 populate1(states1, population, n=10000)
562 N = 1000000
564 i = 0
565 key = [str(random.randint(0, x)) for x in population]
566 t0 = time()
567 while i < N:
568 info = states1
569 for k in key:
570 info = info.get(k, {})
571 i += 1
572 t1 = time()
574 states2 = dict()
575 populate2(states2, population, n=10000)
576 key = [str(random.randint(0, x)) for x in population]
578 i = 0
579 key = [str(random.randint(0, x)) for x in population]
580 info = states2
581 t2 = time()
582 while i < N:
583 k = tuple(key)
584 info = states2.get(k, {})
585 i += 1
586 t3 = time()
588 import pandas as pd
589 t4 = time()
590 df_1 = pd.DataFrame()
592 speed1 = N / (t1 - t0)
593 speed2 = N / (t3 - t2)
595 print(f"{speed1}") # around 28e6 loops/sec
596 print(f"{speed2}") # around 40e6 loops/sec
597 print(f"{speed2/speed1}") # aound 1.5 faster
598 print(f"Pandas: {t4-t3}")
600def test_await_vs_normal():
601 N = 10000000
603 def foo1():
604 return
606 async def foo2():
607 return
609 def timeit0(i):
610 t0 = time()
611 while i > 0:
612 foo1()
613 i -= 1
614 t1 = time()
615 return t1 - t0
617 async def timeit2(i):
618 t0 = time()
619 while i > 0:
620 await foo2()
621 i -= 1
622 t1 = time()
623 return t1 - t0
625 e1 = timeit0(N)
626 import asyncio
627 e2 = asyncio.run(timeit2(N))
629 speed1 = N / (e1)
630 speed2 = N / (e2)
632 print(f"{speed1}") # around 11e6 loops/sec
633 print(f"{speed2}") # around 5e6 loops/sec
634 print(f"{speed1/speed2}") # aound x2 faster
637def test_clock():
638 reactor = _TestReactor()
639 clock = Clock()
640 reactor.attach(clock)
641 reactor.run()
642 foo = 1
644def test_hierarchy():
645 parent = Parent()
647 assert parent.states == {
648 'END': [[], ['bye'], []],
649 'FOO': [[], ['bar'], []],
650 'INIT': [[], [], ['start']],
651 'READY': [[], [], []]
652 }
654 assert parent.transitions == {
655 'END': {},
656 'INIT': {None: [['READY', [], []]]},
657 'READY': {
658 EVENT_QUIT: [['END', [], ['quit']]],
659 EVENT_TERM: [['READY', [], ['term']]],
660 'each:5,1': [['READY', [], ['timer']]],
661 'each:5,21': [['READY', [], ['bye']]]},
662 }
664 foo = 1
667def test_calc():
668 """Compare two different STM definition styles
670 RPNCalc2: loop over same state doing actions on transitions
671 RPNCalc1: has more states and doing actions entering in state
673 Is supposed RPNCalc1 stylel should be faster than RPNCalc2 one
675 Finally, RPNCalc1 stats are saved on a csv file using speed_meter() helper
676 for comparison when changing STM internal libray.
677 """
679 def monkey_calc(N):
680 for i in range(N):
681 r = random.randint(0, 8 + len(iRPNCalc.operations))
682 if r <= 9:
683 yield 'input', r
684 else:
685 ops = list(iRPNCalc.operations.keys())
686 yield 'input', list(iRPNCalc.operations)[r - 9]
688 foo = 1
690 N = 500000
692 # Testing RPNCalc1 definition
693 reactor = _TestReactor(monkey_calc(N))
694 calc = RPNCalc1()
695 reactor.attach(calc)
696 reactor.graph()
698 t0 = time()
699 reactor.run()
700 e1 = time() - t0
701 speed1 = N / e1
702 print(f'Speed: {speed1}')
704 # Just using speed_meter() helper for writedown
705 # RPNCalc1 stats
707 setup = """# This code must be executed for each iteration
708reactor = _TestReactor(monkey_calc(N))
709calc = RPNCalc1()
710reactor.attach(calc)
711"""
712 env = globals()
713 env.update(locals())
715 test = dict(
716 stmt='reactor.run()',
717 setup=setup,
718 globals=env,
719 number=1,
720 repeat=3,
721 N=N,
722 label='RPN1Calc speed'
723 )
725 r = speed_meter(**test)
726 print(r['speed'])
728 # Testing RPNCalc2 definition
729 reactor = _TestReactor(monkey_calc(N))
730 calc = RPNCalc2()
731 reactor.attach(calc)
732 reactor.graph()
734 t0 = time()
735 reactor.run()
736 e2 = time() - t0
738 speed2 = N / e2
739 print(f'Speed: {speed2}')
741 print(f"RPNCalc2 is {speed2/speed1:.4} times faster than RPNCalc1")
742 foo = 1
745def test_graph_layer():
746 monitor = GraphExample()
747 reactor = _TestReactor()
748 reactor.attach(monitor)
749 reactor.graph(view=False)
750 foo = 1
752def test_create_channel():
753 reactor = Reactor()
754 url = 'http://www.debian.org'
756 # create a protocol and transport without layer (persistent until reactor ends)
757 proto, trx = reactor.create_channel(url, factory=_TestBrowserProtcol)
759 reactor.run()
760 foo = 1
762def test_crawler():
763 reactor = Reactor()
764 crawler = Crawler()
765 reactor.attach(crawler)
766 reactor.run()
767 foo = 1
769def test_hashable_events():
770 def events():
771 yield ('hello', 'world'), None
772 yield 1, None
773 foo = 1
775 reactor = _TestReactor(events())
776 layer = HashableEvents()
777 reactor.attach(layer)
778 reactor.run()
779 foo = 1
781def test_decoders():
782 """Use 2 Layers, one send a stream using random lengths
783 The other must recompone message before sending to reactor.
784 """
785 foo = 1
787if __name__ == '__main__':
789 test_decoders()
790 test_calc()
792 # test_timeit()
793 # test_layer()
794 # test_await_vs_normal()
795 test_clock()
796 # test_hierarchy()
797 # test_order_monitor()
798 test_create_channel()
799 test_crawler()
800 # test_hashable_events()
801 pass