From d675f6c61ba1bcc15c28da419461ed50db1c75af Mon Sep 17 00:00:00 2001 From: cheatfate Date: Tue, 24 May 2022 05:57:25 +0300 Subject: [PATCH 01/23] AsyncEventQueue (AsyncEventBus replacement) initial commit. --- chronos/asyncsync.nim | 132 +++++++++++++++++++ tests/testsync.nim | 292 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 423 insertions(+), 1 deletion(-) diff --git a/chronos/asyncsync.nim b/chronos/asyncsync.nim index 824c0df3f..42a5c9ae7 100644 --- a/chronos/asyncsync.nim +++ b/chronos/asyncsync.nim @@ -106,6 +106,19 @@ type eventName: string payload: EventPayloadBase + EventQueueKey* = distinct uint64 + + EventQueueReader* = object + key: EventQueueKey + offset: int + waiter: Future[void] + + AsyncEventQueue*[T] = ref object of RootObj + readers: seq[EventQueueReader] + queue: Deque[T] + counter: uint64 + offset: int + proc newAsyncLock*(): AsyncLock = ## Creates new asynchronous lock ``AsyncLock``. ## @@ -610,3 +623,122 @@ template emitWait*[T](bus: AsyncEventBus, event: string, ## wait until all the subscribers/waiters will receive notification about ## event. emitWait(bus, event, data, getSrcLocation()) + +proc `==`(a, b: EventQueueKey): bool {.borrow.} + +proc compact[T](ab: AsyncEventQueue[T]) = + if len(ab.readers) > 0: + let minOffset = ab.readers[0].offset + doAssert(minOffset >= ab.offset) + if minOffset > ab.offset: + let delta = minOffset - ab.offset + ab.queue.shrink(fromFirst = delta) + ab.offset += delta + else: + ab.queue.clear() + +proc getReaderIndex[T](ab: AsyncEventQueue[T], key: EventQueueKey): int = + for index, value in ab.readers.pairs(): + if value.key == key: + return index + -1 + +proc newAsyncEventQueue*[T](): AsyncEventQueue[T] = + ## Creates new ``AsyncEventBus``. + AsyncEventQueue[T](counter: 0'u64, queue: initDeque[T]()) + +proc len*[T](ab: AsyncEventQueue[T]): int = + len(ab.queue) + +proc register*[T](ab: AsyncEventQueue[T]): EventQueueKey = + inc(ab.counter) + let reader = EventQueueReader(key: EventQueueKey(ab.counter), + offset: ab.offset + len(ab.queue)) + ab.readers.add(reader) + EventQueueKey(ab.counter) + +proc unregister*[T](ab: AsyncEventQueue[T], key: EventQueueKey) = + let index = ab.getReaderIndex(key) + if index >= 0: + let reader = ab.readers[index] + # Completing pending Future to avoid deadlock. + if not(isNil(reader.waiter)) and not(reader.waiter.finished()): + reader.waiter.complete() + ab.readers.delete(index) + ab.compact() + +proc emit*[T](ab: AsyncEventQueue[T], data: T) = + if len(ab.readers) > 0: + # We enqueue `data` only if there active reader present. + ab.queue.addLast(data) + for reader in ab.readers.items(): + if not(isNil(reader.waiter)) and not(reader.waiter.finished()): + reader.waiter.complete() + +proc close*[T](ab: AsyncEventQueue[T]) = + for reader in ab.readers.items(): + if not(isNil(reader.waiter)) and not(reader.waiter.finished()): + reader.waiter.complete() + # This could generate leak, shrink() is not yet implemented for sequences. + ab.readers.setLen(0) + ab.queue.clear() + +proc waitEvents*[T](ab: AsyncEventQueue[T], + key: EventQueueKey, + eventsCount = -1): Future[seq[T]] {.async.} = + ## Wait for events + var + events: seq[T] + resetFuture = false + + while true: + # We need to obtain reader index at every iteration, because `ab.readers` + # sequence could be changed after `await waitFuture` call. + let index = ab.getReaderIndex(key) + if index < 0: + # We going to return everything we have in `events`. + break + + if resetFuture: + resetFuture = false + ab.readers[index].waiter = nil + + let reader = ab.readers[index] + doAssert(isNil(reader.waiter), + "Concurrent waits on same key are not allowed!") + + let length = len(ab.queue) + ab.offset + doAssert(length >= ab.readers[index].offset) + if length == ab.readers[index].offset: + # We are at the end of queue, it means that we should wait for new events. + let waitFuture = newFuture[void]("AsyncEventQueue.waitEvents") + ab.readers[index].waiter = waitFuture + resetFuture = true + await waitFuture + else: + let + itemsInQueue = length - ab.readers[index].offset + itemsOffset = ab.readers[index].offset - ab.offset + itemsCount = + if eventsCount <= 0: + itemsInQueue + else: + min(itemsInQueue, eventsCount - len(events)) + + for i in 0 ..< itemsCount: + events.add(ab.queue[itemsOffset + i]) + ab.readers[index].offset += itemsCount + + # Keep readers sequence sorted by `offset` field. + var slider = index + while (slider + 1 < len(ab.readers)) and + (ab.readers[slider].offset > ab.readers[slider + 1].offset): + swap(ab.readers[slider], ab.readers[slider + 1]) + inc(slider) + # Shrink data queue. + ab.compact() + + if (eventsCount <= 0) or (len(events) == eventsCount): + break + + return events diff --git a/tests/testsync.nim b/tests/testsync.nim index e1e0491fa..7db3da406 100644 --- a/tests/testsync.nim +++ b/tests/testsync.nim @@ -530,4 +530,294 @@ suite "Asynchronous sync primitives test suite": eventBus.emit("EV", 5) await allFutures(futA, futB).wait(1.seconds) - waitFor test() \ No newline at end of file + waitFor test() + + test "AsyncEventQueue() behavior test": + let eventQueue = newAsyncEventQueue[int]() + let key = eventQueue.register() + eventQueue.emit(100) + eventQueue.emit(200) + eventQueue.emit(300) + + proc test1() = + let dataFut = eventQueue.waitEvents(key) + check: + dataFut.finished() == true + dataFut.read() == @[100, 200, 300] + + proc test2() = + let dataFut = eventQueue.waitEvents(key) + check: + dataFut.finished() == false + eventQueue.emit(400) + eventQueue.emit(500) + poll() + check: + dataFut.finished() == true + dataFut.read() == @[400, 500] + + test1() + test2() + eventQueue.close() + + test "AsyncEventQueue() concurrency assert test": + let eventQueue = newAsyncEventQueue[int]() + let key1 = eventQueue.register() + let key2 = eventQueue.register() + + proc test() = + let dataFut = eventQueue.waitEvents(key1) + expect AssertionError: + let failFut {.used.} = eventQueue.waitEvents(key1) + eventQueue.emit(100) + expect AssertionError: + let failFut {.used.} = eventQueue.waitEvents(key1) + let goodFut = eventQueue.waitEvents(key2) + check: + dataFut.finished() == false + goodFut.finished() == true + goodFut.read() == @[100] + poll() + check: + dataFut.finished() == true + dataFut.read() == @[100] + + test() + eventQueue.close() + + test "AsyncEventQueue() concurrency test": + let eventQueue = newAsyncEventQueue[int]() + + let key0 = eventQueue.register() + let key1 = eventQueue.register() + eventQueue.emit(100) + let key2 = eventQueue.register() + eventQueue.emit(200) + eventQueue.emit(300) + let key3 = eventQueue.register() + eventQueue.emit(400) + eventQueue.emit(500) + eventQueue.emit(600) + let key4 = eventQueue.register() + eventQueue.emit(700) + eventQueue.emit(800) + eventQueue.emit(900) + eventQueue.emit(1000) + let key5 = eventQueue.register() + let key6 = eventQueue.register() + + let dataFut1 = eventQueue.waitEvents(key1) + let dataFut2 = eventQueue.waitEvents(key2) + let dataFut3 = eventQueue.waitEvents(key3) + let dataFut4 = eventQueue.waitEvents(key4) + let dataFut5 = eventQueue.waitEvents(key5) + let dataFut6 = eventQueue.waitEvents(key6) + check: + dataFut1.finished() == true + dataFut1.read() == @[100, 200, 300, 400, 500, 600, 700, 800, 900, 1000] + dataFut2.finished() == true + dataFut2.read() == @[200, 300, 400, 500, 600, 700, 800, 900, 1000] + dataFut3.finished() == true + dataFut3.read() == @[400, 500, 600, 700, 800, 900, 1000] + dataFut4.finished() == true + dataFut4.read() == @[700, 800, 900, 1000] + dataFut5.finished() == false + dataFut6.finished() == false + + eventQueue.emit(2000) + poll() + let dataFut0 = eventQueue.waitEvents(key0) + check: + dataFut5.finished() == true + dataFut5.read() == @[2000] + dataFut6.finished() == true + dataFut6.read() == @[2000] + dataFut0.finished() == true + dataFut0.read() == @[100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, + 2000] + eventQueue.close() + + test "AsyncEventQueue() specific number test": + let eventQueue = newAsyncEventQueue[int]() + let key = eventQueue.register() + + let dataFut1 = eventQueue.waitEvents(key, 1) + eventQueue.emit(100) + eventQueue.emit(200) + eventQueue.emit(300) + eventQueue.emit(400) + check dataFut1.finished() == false + poll() + check: + dataFut1.finished() == true + dataFut1.read() == @[100] + let dataFut2 = eventQueue.waitEvents(key, 2) + check: + dataFut2.finished() == true + dataFut2.read() == @[200, 300] + let dataFut3 = eventQueue.waitEvents(key, 5) + check dataFut3.finished() == false + eventQueue.emit(500) + eventQueue.emit(600) + eventQueue.emit(700) + eventQueue.emit(800) + check dataFut3.finished() == false + poll() + check: + dataFut3.finished() == true + dataFut3.read() == @[400, 500, 600, 700, 800] + let dataFut4 = eventQueue.waitEvents(key, -1) + check dataFut4.finished() == false + eventQueue.emit(900) + eventQueue.emit(1000) + eventQueue.emit(1100) + eventQueue.emit(1200) + eventQueue.emit(1300) + eventQueue.emit(1400) + eventQueue.emit(1500) + eventQueue.emit(1600) + check dataFut4.finished() == false + poll() + check: + dataFut4.finished() == true + dataFut4.read() == @[900, 1000, 1100, 1200, 1300, 1400, 1500, 1600] + + eventQueue.close() + + test "AsyncEventQueue() register()/unregister() test": + var emptySeq: seq[int] + let eventQueue = newAsyncEventQueue[int]() + let key1 = eventQueue.register() + + let dataFut1 = eventQueue.waitEvents(key1, 1) + check dataFut1.finished() == false + eventQueue.unregister(key1) + check dataFut1.finished() == false + poll() + check: + dataFut1.finished() == true + dataFut1.read() == emptySeq + + let key2 = eventQueue.register() + let dataFut2 = eventQueue.waitEvents(key2, 5) + check dataFut2.finished() == false + eventQueue.emit(100) + eventQueue.emit(200) + eventQueue.emit(300) + eventQueue.emit(400) + eventQueue.emit(500) + check dataFut2.finished() == false + eventQueue.unregister(key2) + poll() + check: + dataFut2.finished() == true + dataFut2.read() == emptySeq + + let key3 = eventQueue.register() + let dataFut3 = eventQueue.waitEvents(key3, 5) + check dataFut3.finished() == false + eventQueue.emit(100) + eventQueue.emit(200) + eventQueue.emit(300) + check dataFut3.finished() == false + poll() + eventQueue.unregister(key3) + eventQueue.emit(400) + check dataFut3.finished() == false + poll() + check: + dataFut3.finished() == true + dataFut3.read() == @[100, 200, 300] + + eventQueue.close() + + test "AsyncEventQueue() garbage collection test": + let eventQueue = newAsyncEventQueue[int]() + let key1 = eventQueue.register() + check len(eventQueue) == 0 + eventQueue.emit(100) + eventQueue.emit(200) + eventQueue.emit(300) + check len(eventQueue) == 3 + let key2 = eventQueue.register() + eventQueue.emit(400) + eventQueue.emit(500) + eventQueue.emit(600) + eventQueue.emit(700) + check len(eventQueue) == 7 + let key3 = eventQueue.register() + eventQueue.emit(800) + eventQueue.emit(900) + eventQueue.emit(1000) + eventQueue.emit(1100) + eventQueue.emit(1200) + check len(eventQueue) == 12 + let dataFut1 = eventQueue.waitEvents(key1) + check: + dataFut1.finished() == true + dataFut1.read() == @[100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, + 1100, 1200] + len(eventQueue) == 9 + + let dataFut3 = eventQueue.waitEvents(key3) + check: + dataFut3.finished() == true + dataFut3.read() == @[800, 900, 1000, 1100, 1200] + len(eventQueue) == 9 + + let dataFut2 = eventQueue.waitEvents(key2) + check: + dataFut2.finished() == true + dataFut2.read() == @[400, 500, 600, 700, 800, 900, 1000, 1100, 1200] + len(eventQueue) == 0 + + eventQueue.close() + + test "AsyncEventQueue() 1,000,000 of events to 10 clients test": + let eventQueue = newAsyncEventQueue[int]() + let keys = [ + eventQueue.register(), eventQueue.register(), + eventQueue.register(), eventQueue.register(), + eventQueue.register(), eventQueue.register(), + eventQueue.register(), eventQueue.register(), + eventQueue.register(), eventQueue.register() + ] + + proc clientTask(queue: AsyncEventQueue[int], + key: EventQueueKey): Future[seq[int]] {.async.} = + var events: seq[int] + while true: + let res = await queue.waitEvents(key) + if len(res) == 0: + break + events.add(res) + return events + + var futs = [ + clientTask(eventQueue, keys[0]), clientTask(eventQueue, keys[1]), + clientTask(eventQueue, keys[2]), clientTask(eventQueue, keys[3]), + clientTask(eventQueue, keys[4]), clientTask(eventQueue, keys[5]), + clientTask(eventQueue, keys[6]), clientTask(eventQueue, keys[7]), + clientTask(eventQueue, keys[8]), clientTask(eventQueue, keys[9]) + ] + + proc test() {.async.} = + for i in 1 .. 1_000_000: + if (i mod 1000) == 0: + # Give some CPU for clients. + await sleepAsync(0.milliseconds) + eventQueue.emit(i) + + eventQueue.close() + await allFutures(futs) + for fut in futs: + check: + fut.finished() == true + let data = fut.read() + var counter = 1 + for item in data: + check: + item == counter + inc(counter) + + waitFor test() From bda5d38f5591bfbea9216fce79a2a11de8911b2a Mon Sep 17 00:00:00 2001 From: cheatfate Date: Tue, 24 May 2022 12:35:56 +0300 Subject: [PATCH 02/23] Add closeWait() and remove assertion test. --- chronos/asyncsync.nim | 9 +++++++++ tests/testsync.nim | 40 +++++++++------------------------------- 2 files changed, 18 insertions(+), 31 deletions(-) diff --git a/chronos/asyncsync.nim b/chronos/asyncsync.nim index 42a5c9ae7..e1234ea52 100644 --- a/chronos/asyncsync.nim +++ b/chronos/asyncsync.nim @@ -683,6 +683,15 @@ proc close*[T](ab: AsyncEventQueue[T]) = ab.readers.setLen(0) ab.queue.clear() +proc closeWait*[T](ab: AsyncEventQueue[T]): Future[void] = + var retFuture = newFuture[void]("AsyncEventQueue.closeWait()") + proc continuation(udata: pointer) {.gcsafe.} = + if not(retFuture.finished()): + retFuture.complete() + ab.close() + callSoon(continuation) + retFuture + proc waitEvents*[T](ab: AsyncEventQueue[T], key: EventQueueKey, eventsCount = -1): Future[seq[T]] {.async.} = diff --git a/tests/testsync.nim b/tests/testsync.nim index 7db3da406..28e801402 100644 --- a/tests/testsync.nim +++ b/tests/testsync.nim @@ -560,34 +560,8 @@ suite "Asynchronous sync primitives test suite": test2() eventQueue.close() - test "AsyncEventQueue() concurrency assert test": - let eventQueue = newAsyncEventQueue[int]() - let key1 = eventQueue.register() - let key2 = eventQueue.register() - - proc test() = - let dataFut = eventQueue.waitEvents(key1) - expect AssertionError: - let failFut {.used.} = eventQueue.waitEvents(key1) - eventQueue.emit(100) - expect AssertionError: - let failFut {.used.} = eventQueue.waitEvents(key1) - let goodFut = eventQueue.waitEvents(key2) - check: - dataFut.finished() == false - goodFut.finished() == true - goodFut.read() == @[100] - poll() - check: - dataFut.finished() == true - dataFut.read() == @[100] - - test() - eventQueue.close() - test "AsyncEventQueue() concurrency test": let eventQueue = newAsyncEventQueue[int]() - let key0 = eventQueue.register() let key1 = eventQueue.register() eventQueue.emit(100) @@ -635,7 +609,8 @@ suite "Asynchronous sync primitives test suite": dataFut0.finished() == true dataFut0.read() == @[100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 2000] - eventQueue.close() + + waitFor eventQueue.closeWait() test "AsyncEventQueue() specific number test": let eventQueue = newAsyncEventQueue[int]() @@ -651,10 +626,12 @@ suite "Asynchronous sync primitives test suite": check: dataFut1.finished() == true dataFut1.read() == @[100] + let dataFut2 = eventQueue.waitEvents(key, 2) check: dataFut2.finished() == true dataFut2.read() == @[200, 300] + let dataFut3 = eventQueue.waitEvents(key, 5) check dataFut3.finished() == false eventQueue.emit(500) @@ -666,6 +643,7 @@ suite "Asynchronous sync primitives test suite": check: dataFut3.finished() == true dataFut3.read() == @[400, 500, 600, 700, 800] + let dataFut4 = eventQueue.waitEvents(key, -1) check dataFut4.finished() == false eventQueue.emit(900) @@ -682,7 +660,7 @@ suite "Asynchronous sync primitives test suite": dataFut4.finished() == true dataFut4.read() == @[900, 1000, 1100, 1200, 1300, 1400, 1500, 1600] - eventQueue.close() + waitFor eventQueue.closeWait() test "AsyncEventQueue() register()/unregister() test": var emptySeq: seq[int] @@ -729,7 +707,7 @@ suite "Asynchronous sync primitives test suite": dataFut3.finished() == true dataFut3.read() == @[100, 200, 300] - eventQueue.close() + waitFor eventQueue.closeWait() test "AsyncEventQueue() garbage collection test": let eventQueue = newAsyncEventQueue[int]() @@ -771,7 +749,7 @@ suite "Asynchronous sync primitives test suite": dataFut2.read() == @[400, 500, 600, 700, 800, 900, 1000, 1100, 1200] len(eventQueue) == 0 - eventQueue.close() + waitFor eventQueue.closeWait() test "AsyncEventQueue() 1,000,000 of events to 10 clients test": let eventQueue = newAsyncEventQueue[int]() @@ -808,7 +786,7 @@ suite "Asynchronous sync primitives test suite": await sleepAsync(0.milliseconds) eventQueue.emit(i) - eventQueue.close() + await eventQueue.closeWait() await allFutures(futs) for fut in futs: check: From 978df761ca98473a9982a9c2ff7d4aa23e6730ae Mon Sep 17 00:00:00 2001 From: cheatfate Date: Thu, 2 Jun 2022 20:33:59 +0300 Subject: [PATCH 03/23] Fix "possible" memory leak. --- chronos/asyncsync.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/chronos/asyncsync.nim b/chronos/asyncsync.nim index e1234ea52..2f0019e35 100644 --- a/chronos/asyncsync.nim +++ b/chronos/asyncsync.nim @@ -679,8 +679,7 @@ proc close*[T](ab: AsyncEventQueue[T]) = for reader in ab.readers.items(): if not(isNil(reader.waiter)) and not(reader.waiter.finished()): reader.waiter.complete() - # This could generate leak, shrink() is not yet implemented for sequences. - ab.readers.setLen(0) + ab.readers.reset() ab.queue.clear() proc closeWait*[T](ab: AsyncEventQueue[T]): Future[void] = From 4e6ec9ebcf28b50a65e8375391ec41acd2183e01 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Fri, 3 Jun 2022 02:54:39 +0300 Subject: [PATCH 04/23] Add limits to AsyncEventQueue[T]. Add tests for AsyncEventQueue[T] limits. Rebase AsyncSync errors to be children of AsyncError. --- chronos/asyncsync.nim | 57 +++++++++-- tests/testsync.nim | 231 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 277 insertions(+), 11 deletions(-) diff --git a/chronos/asyncsync.nim b/chronos/asyncsync.nim index 2f0019e35..80ce9a8c2 100644 --- a/chronos/asyncsync.nim +++ b/chronos/asyncsync.nim @@ -12,7 +12,7 @@ {.push raises: [Defect].} -import std/[sequtils, deques, tables, typetraits] +import std/[sequtils, math, deques, tables, typetraits] import ./asyncloop export asyncloop @@ -55,11 +55,11 @@ type queue: Deque[T] maxsize: int - AsyncQueueEmptyError* = object of CatchableError + AsyncQueueEmptyError* = object of AsyncError ## ``AsyncQueue`` is empty. - AsyncQueueFullError* = object of CatchableError + AsyncQueueFullError* = object of AsyncError ## ``AsyncQueue`` is full. - AsyncLockError* = object of CatchableError + AsyncLockError* = object of AsyncError ## ``AsyncLock`` is either locked or unlocked. EventBusSubscription*[T] = proc(bus: AsyncEventBus, @@ -106,6 +106,8 @@ type eventName: string payload: EventPayloadBase + AsyncEventQueueFullError* = object of AsyncError + EventQueueKey* = distinct uint64 EventQueueReader* = object @@ -117,6 +119,7 @@ type readers: seq[EventQueueReader] queue: Deque[T] counter: uint64 + limit: int offset: int proc newAsyncLock*(): AsyncLock = @@ -643,9 +646,23 @@ proc getReaderIndex[T](ab: AsyncEventQueue[T], key: EventQueueKey): int = return index -1 -proc newAsyncEventQueue*[T](): AsyncEventQueue[T] = - ## Creates new ``AsyncEventBus``. - AsyncEventQueue[T](counter: 0'u64, queue: initDeque[T]()) +proc newAsyncEventQueue*[T](limitSize = 0): AsyncEventQueue[T] = + ## Creates new ``AsyncEventBus`` maximum size of ``limitSize`` (default is + ## ``0`` which means that there no limits). + ## + ## When number of events emitted exceeds ``limitSize`` - emit() procedure + ## will discard new events, consumers which has number of pending events + ## more than ``limitSize`` will get ``AsyncEventQueueFullError`` + ## error. + doAssert(limitSize >= 0, "Limit size should be non-negative integer") + let queue = + if limitSize == 0: + initDeque[T]() + elif isPowerOfTwo(limitSize + 1): + initDeque[T](limitSize + 1) + else: + initDeque[T](nextPowerOfTwo(limitSize + 1)) + AsyncEventQueue[T](counter: 0'u64, queue: queue, limit: limitSize) proc len*[T](ab: AsyncEventQueue[T]): int = len(ab.queue) @@ -670,10 +687,22 @@ proc unregister*[T](ab: AsyncEventQueue[T], key: EventQueueKey) = proc emit*[T](ab: AsyncEventQueue[T], data: T) = if len(ab.readers) > 0: # We enqueue `data` only if there active reader present. - ab.queue.addLast(data) - for reader in ab.readers.items(): - if not(isNil(reader.waiter)) and not(reader.waiter.finished()): - reader.waiter.complete() + let couldEmit = + if ab.limit == 0: + true + else: + # Because ab.readers is sequence sorted by `offset`, we will apply our + # limit to the most recent consumer. + if ab.limit + (ab.readers[^1].offset - ab.offset) < len(ab.queue): + false + else: + true + + if couldEmit: + ab.queue.addLast(data) + for reader in ab.readers.items(): + if not(isNil(reader.waiter)) and not(reader.waiter.finished()): + reader.waiter.complete() proc close*[T](ab: AsyncEventQueue[T]) = for reader in ab.readers.items(): @@ -727,6 +756,12 @@ proc waitEvents*[T](ab: AsyncEventQueue[T], let itemsInQueue = length - ab.readers[index].offset itemsOffset = ab.readers[index].offset - ab.offset + + if (ab.limit > 0) and (itemsInQueue > ab.limit): + raise newException(AsyncEventQueueFullError, + "AsyncEventQueue size exceeds limits") + + let itemsCount = if eventsCount <= 0: itemsInQueue diff --git a/tests/testsync.nim b/tests/testsync.nim index 28e801402..5b868338a 100644 --- a/tests/testsync.nim +++ b/tests/testsync.nim @@ -799,3 +799,234 @@ suite "Asynchronous sync primitives test suite": inc(counter) waitFor test() + + test "AsyncEventQueue() one consumer limits test": + let eventQueue = newAsyncEventQueue[int](4) + check len(eventQueue) == 0 + eventQueue.emit(100) + eventQueue.emit(200) + eventQueue.emit(300) + eventQueue.emit(400) + # There no consumers, so all the items should be discarded + check len(eventQueue) == 0 + let key1 = eventQueue.register() + check len(eventQueue) == 0 + eventQueue.emit(500) + eventQueue.emit(600) + eventQueue.emit(700) + eventQueue.emit(800) + # So exact `limit` number of items added, consumer should receive all of + # them. + check len(eventQueue) == 4 + let dataFut1 = eventQueue.waitEvents(key1) + check: + dataFut1.finished() == true + dataFut1.read() == @[500, 600, 700, 800] + len(eventQueue) == 0 + + eventQueue.emit(900) + eventQueue.emit(1000) + eventQueue.emit(1100) + eventQueue.emit(1200) + eventQueue.emit(1300) + # So `1300` is overfill item here, but we queue will still accept it so + # consumers could be notified about overfill. + check len(eventQueue) == 5 + eventQueue.emit(1400) + eventQueue.emit(1500) + eventQueue.emit(1600) + eventQueue.emit(1700) + eventQueue.emit(1800) + # No more items should be accepted. + check len(eventQueue) == 5 + let errorFut1 = eventQueue.waitEvents(key1) + check errorFut1.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = errorFut1.read() + # All items are still in queue, because consumer should unregister. + check len(eventQueue) == 5 + eventQueue.unregister(key1) + # All items should be garbage collected after unregister. + check len(eventQueue) == 0 + + waitFor eventQueue.closeWait() + + test "AsyncEventQueue() many consumers limits test": + let eventQueue = newAsyncEventQueue[int](4) + block: + let key1 = eventQueue.register() + eventQueue.emit(100) + check len(eventQueue) == 1 + let key2 = eventQueue.register() + eventQueue.emit(200) + check len(eventQueue) == 2 + let key3 = eventQueue.register() + eventQueue.emit(300) + check len(eventQueue) == 3 + let key4 = eventQueue.register() + eventQueue.emit(400) + check len(eventQueue) == 4 + let key5 = eventQueue.register() + eventQueue.emit(500) + # At this point consumers with key1 is overfilled, but new events should + # be accepted. + check len(eventQueue) == 5 + eventQueue.emit(600) + # At this point consumers with key2 is overfilled, but new events should + # be accepted. + check len(eventQueue) == 6 + eventQueue.emit(700) + # At this point consumers with key3 is overfilled, but new events should + # be accepted. + check len(eventQueue) == 7 + eventQueue.emit(800) + # At this point consumers with key4 is overfilled, but new events should + # be accepted. + check len(eventQueue) == 8 + + # Consumer with key5 is not overfilled. + let dataFut5 = eventQueue.waitEvents(key5) + check: + dataFut5.finished() == true + dataFut5.read() == @[500, 600, 700, 800] + len(eventQueue) == 8 + + eventQueue.unregister(key5) + check len(eventQueue) == 8 + + let dataFut2 = eventQueue.waitEvents(key2) + check dataFut2.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut2.read() + # All items are still in queue, because consumer should unregister. + check len(eventQueue) == 8 + eventQueue.unregister(key2) + check len(eventQueue) == 8 + + let dataFut4 = eventQueue.waitEvents(key4) + check dataFut4.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut4.read() + # All items are still in queue, because consumer should unregister. + check len(eventQueue) == 8 + eventQueue.unregister(key4) + check len(eventQueue) == 8 + + let dataFut3 = eventQueue.waitEvents(key3) + check dataFut3.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut3.read() + # All items are still in queue, because consumer should unregister. + check len(eventQueue) == 8 + eventQueue.unregister(key3) + check len(eventQueue) == 8 + + let dataFut1 = eventQueue.waitEvents(key1) + check dataFut1.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut1.read() + # All items are still in queue, because consumer should unregister. + check len(eventQueue) == 8 + eventQueue.unregister(key1) + # All items should be garbage collected after unregister. + check len(eventQueue) == 0 + + block: + let key1 = eventQueue.register() + eventQueue.emit(100) + check len(eventQueue) == 1 + let key2 = eventQueue.register() + eventQueue.emit(200) + check len(eventQueue) == 2 + let key3 = eventQueue.register() + eventQueue.emit(300) + check len(eventQueue) == 3 + let key4 = eventQueue.register() + eventQueue.emit(400) + check len(eventQueue) == 4 + let key5 = eventQueue.register() + eventQueue.emit(500) + # At this point consumers with key1 is overfilled, but new events should + # be accepted. + check len(eventQueue) == 5 + eventQueue.emit(600) + # At this point consumers with key2 is overfilled, but new events should + # be accepted. + check len(eventQueue) == 6 + eventQueue.emit(700) + # At this point consumers with key3 is overfilled, but new events should + # be accepted. + check len(eventQueue) == 7 + eventQueue.emit(800) + # At this point consumers with key4 is overfilled, but new events should + # be accepted. + check len(eventQueue) == 8 + eventQueue.emit(900) + # At this point all the consumers are overfilled, but single new event still + # should be accepted. + check len(eventQueue) == 9 + eventQueue.emit(1000) + eventQueue.emit(1100) + eventQueue.emit(1200) + eventQueue.emit(1300) + eventQueue.emit(1400) + eventQueue.emit(1500) + eventQueue.emit(1600) + eventQueue.emit(1700) + eventQueue.emit(1800) + eventQueue.emit(1900) + # No more events should be accepted. + check len(eventQueue) == 9 + + let dataFut1 = eventQueue.waitEvents(key1) + check dataFut1.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut1.read() + # All items are still in queue, because consumer should unregister. + check len(eventQueue) == 9 + eventQueue.unregister(key1) + # Only the first item should be garbage collected. + check len(eventQueue) == 8 + + let dataFut2 = eventQueue.waitEvents(key2) + check dataFut2.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut2.read() + # All items are still in queue, because consumer should unregister. + check len(eventQueue) == 8 + eventQueue.unregister(key2) + # Only the first item should be garbage collected. + check len(eventQueue) == 7 + + let dataFut3 = eventQueue.waitEvents(key3) + check dataFut3.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut3.read() + # All items are still in queue, because consumer should unregister. + check len(eventQueue) == 7 + eventQueue.unregister(key3) + # Only the first item should be garbage collected. + check len(eventQueue) == 6 + + let dataFut4 = eventQueue.waitEvents(key4) + check dataFut4.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut4.read() + # All items are still in queue, because consumer should unregister. + check len(eventQueue) == 6 + eventQueue.unregister(key4) + # Only the first item should be garbage collected. + check len(eventQueue) == 5 + + let dataFut5 = eventQueue.waitEvents(key5) + check dataFut5.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut5.read() + # All items are still in queue, because consumer should unregister. + check len(eventQueue) == 5 + eventQueue.unregister(key5) + # All items should be garbage collected after unregister, because there no + # more consumers left. + check len(eventQueue) == 0 + + waitFor eventQueue.closeWait() From f594b3d657e60961d897d0ce45e4ad6dd3f7357c Mon Sep 17 00:00:00 2001 From: cheatfate Date: Wed, 8 Jun 2022 01:21:36 +0300 Subject: [PATCH 05/23] Adopt AsyncEventQueue to garbage collect events on emit() too. Add test from review comment. --- chronos/asyncsync.nim | 62 ++++++++++++---- tests/testsync.nim | 164 +++++++++++++++++++++++------------------- 2 files changed, 139 insertions(+), 87 deletions(-) diff --git a/chronos/asyncsync.nim b/chronos/asyncsync.nim index 80ce9a8c2..36987954f 100644 --- a/chronos/asyncsync.nim +++ b/chronos/asyncsync.nim @@ -114,6 +114,7 @@ type key: EventQueueKey offset: int waiter: Future[void] + overflow: bool AsyncEventQueue*[T] = ref object of RootObj readers: seq[EventQueueReader] @@ -631,12 +632,24 @@ proc `==`(a, b: EventQueueKey): bool {.borrow.} proc compact[T](ab: AsyncEventQueue[T]) = if len(ab.readers) > 0: - let minOffset = ab.readers[0].offset - doAssert(minOffset >= ab.offset) - if minOffset > ab.offset: - let delta = minOffset - ab.offset - ab.queue.shrink(fromFirst = delta) - ab.offset += delta + let minOffset = + block: + var res = -1 + for reader in ab.readers.items(): + if not(reader.overflow): + res = reader.offset + break + res + + if minOffset == -1: + ab.offset += len(ab.queue) + ab.queue.clear() + else: + doAssert(minOffset >= ab.offset) + if minOffset > ab.offset: + let delta = minOffset - ab.offset + ab.queue.shrink(fromFirst = delta) + ab.offset += delta else: ab.queue.clear() @@ -670,7 +683,8 @@ proc len*[T](ab: AsyncEventQueue[T]): int = proc register*[T](ab: AsyncEventQueue[T]): EventQueueKey = inc(ab.counter) let reader = EventQueueReader(key: EventQueueKey(ab.counter), - offset: ab.offset + len(ab.queue)) + offset: ab.offset + len(ab.queue), + overflow: false) ab.readers.add(reader) EventQueueKey(ab.counter) @@ -684,25 +698,44 @@ proc unregister*[T](ab: AsyncEventQueue[T], key: EventQueueKey) = ab.readers.delete(index) ab.compact() +template readerOverflow*[T](ab: AsyncEventQueue[T], + reader: EventQueueReader): bool = + ab.limit + (reader.offset - ab.offset) <= len(ab.queue) + proc emit*[T](ab: AsyncEventQueue[T], data: T) = if len(ab.readers) > 0: # We enqueue `data` only if there active reader present. + var changesPresent = false let couldEmit = if ab.limit == 0: true else: # Because ab.readers is sequence sorted by `offset`, we will apply our # limit to the most recent consumer. - if ab.limit + (ab.readers[^1].offset - ab.offset) < len(ab.queue): + if ab.readerOverflow(ab.readers[^1]): false else: true if couldEmit: + if ab.limit != 0: + for reader in ab.readers.mitems(): + if not(reader.overflow): + if ab.readerOverflow(reader): + reader.overflow = true + changesPresent = true ab.queue.addLast(data) - for reader in ab.readers.items(): + for reader in ab.readers.mitems(): if not(isNil(reader.waiter)) and not(reader.waiter.finished()): reader.waiter.complete() + else: + for reader in ab.readers.mitems(): + if not(reader.overflow): + reader.overflow = true + changesPresent = true + + if changesPresent: + ab.compact() proc close*[T](ab: AsyncEventQueue[T]) = for reader in ab.readers.items(): @@ -744,6 +777,10 @@ proc waitEvents*[T](ab: AsyncEventQueue[T], doAssert(isNil(reader.waiter), "Concurrent waits on same key are not allowed!") + if reader.overflow: + raise newException(AsyncEventQueueFullError, + "AsyncEventQueue size exceeds limits") + let length = len(ab.queue) + ab.offset doAssert(length >= ab.readers[index].offset) if length == ab.readers[index].offset: @@ -756,12 +793,6 @@ proc waitEvents*[T](ab: AsyncEventQueue[T], let itemsInQueue = length - ab.readers[index].offset itemsOffset = ab.readers[index].offset - ab.offset - - if (ab.limit > 0) and (itemsInQueue > ab.limit): - raise newException(AsyncEventQueueFullError, - "AsyncEventQueue size exceeds limits") - - let itemsCount = if eventsCount <= 0: itemsInQueue @@ -778,6 +809,7 @@ proc waitEvents*[T](ab: AsyncEventQueue[T], (ab.readers[slider].offset > ab.readers[slider + 1].offset): swap(ab.readers[slider], ab.readers[slider + 1]) inc(slider) + # Shrink data queue. ab.compact() diff --git a/tests/testsync.nim b/tests/testsync.nim index 5b868338a..0bd63bc65 100644 --- a/tests/testsync.nim +++ b/tests/testsync.nim @@ -828,27 +828,27 @@ suite "Asynchronous sync primitives test suite": eventQueue.emit(1000) eventQueue.emit(1100) eventQueue.emit(1200) + check len(eventQueue) == 4 + # Overfilling queue eventQueue.emit(1300) - # So `1300` is overfill item here, but we queue will still accept it so - # consumers could be notified about overfill. - check len(eventQueue) == 5 + # Because overfill for single consumer happend, whole queue should become + # empty. + check len(eventQueue) == 0 eventQueue.emit(1400) eventQueue.emit(1500) eventQueue.emit(1600) eventQueue.emit(1700) eventQueue.emit(1800) - # No more items should be accepted. - check len(eventQueue) == 5 + check len(eventQueue) == 0 let errorFut1 = eventQueue.waitEvents(key1) check errorFut1.finished() == true expect AsyncEventQueueFullError: let res {.used.} = errorFut1.read() - # All items are still in queue, because consumer should unregister. - check len(eventQueue) == 5 + # There should be no items because consumer was overflowed. + check len(eventQueue) == 0 eventQueue.unregister(key1) # All items should be garbage collected after unregister. check len(eventQueue) == 0 - waitFor eventQueue.closeWait() test "AsyncEventQueue() many consumers limits test": @@ -868,67 +868,65 @@ suite "Asynchronous sync primitives test suite": check len(eventQueue) == 4 let key5 = eventQueue.register() eventQueue.emit(500) - # At this point consumers with key1 is overfilled, but new events should - # be accepted. - check len(eventQueue) == 5 + # At this point consumer with `key1` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [200, 300, 400, 500] + check len(eventQueue) == 4 eventQueue.emit(600) - # At this point consumers with key2 is overfilled, but new events should - # be accepted. - check len(eventQueue) == 6 + # At this point consumers with `key2` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [300, 400, 500, 600] + check len(eventQueue) == 4 eventQueue.emit(700) - # At this point consumers with key3 is overfilled, but new events should - # be accepted. - check len(eventQueue) == 7 + # At this point consumers with `key3` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [400, 500, 600, 700] + check len(eventQueue) == 4 eventQueue.emit(800) - # At this point consumers with key4 is overfilled, but new events should - # be accepted. - check len(eventQueue) == 8 - + # At this point consumers with `key4` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [500, 600, 700, 800] + check len(eventQueue) == 4 # Consumer with key5 is not overfilled. let dataFut5 = eventQueue.waitEvents(key5) check: dataFut5.finished() == true dataFut5.read() == @[500, 600, 700, 800] - len(eventQueue) == 8 - + # No more items should be left because all other consumers are overfilled. + check len(eventQueue) == 0 eventQueue.unregister(key5) - check len(eventQueue) == 8 + check len(eventQueue) == 0 let dataFut2 = eventQueue.waitEvents(key2) check dataFut2.finished() == true expect AsyncEventQueueFullError: let res {.used.} = dataFut2.read() - # All items are still in queue, because consumer should unregister. - check len(eventQueue) == 8 + check len(eventQueue) == 0 eventQueue.unregister(key2) - check len(eventQueue) == 8 + check len(eventQueue) == 0 let dataFut4 = eventQueue.waitEvents(key4) check dataFut4.finished() == true expect AsyncEventQueueFullError: let res {.used.} = dataFut4.read() - # All items are still in queue, because consumer should unregister. - check len(eventQueue) == 8 + check len(eventQueue) == 0 eventQueue.unregister(key4) - check len(eventQueue) == 8 + check len(eventQueue) == 0 let dataFut3 = eventQueue.waitEvents(key3) check dataFut3.finished() == true expect AsyncEventQueueFullError: let res {.used.} = dataFut3.read() - # All items are still in queue, because consumer should unregister. - check len(eventQueue) == 8 + check len(eventQueue) == 0 eventQueue.unregister(key3) - check len(eventQueue) == 8 + check len(eventQueue) == 0 let dataFut1 = eventQueue.waitEvents(key1) check dataFut1.finished() == true expect AsyncEventQueueFullError: let res {.used.} = dataFut1.read() - # All items are still in queue, because consumer should unregister. - check len(eventQueue) == 8 + check len(eventQueue) == 0 eventQueue.unregister(key1) - # All items should be garbage collected after unregister. check len(eventQueue) == 0 block: @@ -946,25 +944,29 @@ suite "Asynchronous sync primitives test suite": check len(eventQueue) == 4 let key5 = eventQueue.register() eventQueue.emit(500) - # At this point consumers with key1 is overfilled, but new events should - # be accepted. - check len(eventQueue) == 5 + # At this point consumer with `key1` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [200, 300, 400, 500] + check len(eventQueue) == 4 eventQueue.emit(600) - # At this point consumers with key2 is overfilled, but new events should - # be accepted. - check len(eventQueue) == 6 + # At this point consumer with `key2` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [300, 400, 500, 600] + check len(eventQueue) == 4 eventQueue.emit(700) - # At this point consumers with key3 is overfilled, but new events should - # be accepted. - check len(eventQueue) == 7 + # At this point consumer with `key3` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [400, 500, 600, 700] + check len(eventQueue) == 4 eventQueue.emit(800) - # At this point consumers with key4 is overfilled, but new events should - # be accepted. - check len(eventQueue) == 8 + # At this point consumer with `key4` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [500, 600, 700, 800] + check len(eventQueue) == 4 eventQueue.emit(900) - # At this point all the consumers are overfilled, but single new event still - # should be accepted. - check len(eventQueue) == 9 + # At this point all consumers are overfilled, so after `emit()` + # queue length should become 0. + check len(eventQueue) == 0 eventQueue.emit(1000) eventQueue.emit(1100) eventQueue.emit(1200) @@ -976,57 +978,75 @@ suite "Asynchronous sync primitives test suite": eventQueue.emit(1800) eventQueue.emit(1900) # No more events should be accepted. - check len(eventQueue) == 9 + check len(eventQueue) == 0 let dataFut1 = eventQueue.waitEvents(key1) check dataFut1.finished() == true expect AsyncEventQueueFullError: let res {.used.} = dataFut1.read() - # All items are still in queue, because consumer should unregister. - check len(eventQueue) == 9 + check len(eventQueue) == 0 eventQueue.unregister(key1) - # Only the first item should be garbage collected. - check len(eventQueue) == 8 + check len(eventQueue) == 0 let dataFut2 = eventQueue.waitEvents(key2) check dataFut2.finished() == true expect AsyncEventQueueFullError: let res {.used.} = dataFut2.read() - # All items are still in queue, because consumer should unregister. - check len(eventQueue) == 8 + check len(eventQueue) == 0 eventQueue.unregister(key2) - # Only the first item should be garbage collected. - check len(eventQueue) == 7 + check len(eventQueue) == 0 let dataFut3 = eventQueue.waitEvents(key3) check dataFut3.finished() == true expect AsyncEventQueueFullError: let res {.used.} = dataFut3.read() - # All items are still in queue, because consumer should unregister. - check len(eventQueue) == 7 + check len(eventQueue) == 0 eventQueue.unregister(key3) - # Only the first item should be garbage collected. - check len(eventQueue) == 6 + check len(eventQueue) == 0 let dataFut4 = eventQueue.waitEvents(key4) check dataFut4.finished() == true expect AsyncEventQueueFullError: let res {.used.} = dataFut4.read() - # All items are still in queue, because consumer should unregister. - check len(eventQueue) == 6 + check len(eventQueue) == 0 eventQueue.unregister(key4) - # Only the first item should be garbage collected. - check len(eventQueue) == 5 + check len(eventQueue) == 0 let dataFut5 = eventQueue.waitEvents(key5) check dataFut5.finished() == true expect AsyncEventQueueFullError: let res {.used.} = dataFut5.read() - # All items are still in queue, because consumer should unregister. - check len(eventQueue) == 5 + check len(eventQueue) == 0 eventQueue.unregister(key5) - # All items should be garbage collected after unregister, because there no - # more consumers left. check len(eventQueue) == 0 waitFor eventQueue.closeWait() + + test "AsyncEventQueue() slow and fast consumer test": + proc test() {.async.} = + let eventQueue = newAsyncEventQueue[int](1) + let + fastConsumer = eventQueue.register() + slowConsumer = eventQueue.register() + slowFut = eventQueue.waitEvents(slowConsumer) + + for i in 0 ..< 1000: + eventQueue.emit(i) + let fastData {.used.} = await eventQueue.waitEvents(fastConsumer) + + check len(eventQueue) == 0 + await allFutures(slowFut) + check len(eventQueue) == 0 + expect AsyncEventQueueFullError: + let res {.used.} = slowFut.read() + + check len(eventQueue) == 0 + eventQueue.unregister(fastConsumer) + check len(eventQueue) == 0 + eventQueue.unregister(slowConsumer) + check len(eventQueue) == 0 + await eventQueue.closeWait() + + waitFor test() + + From c834a0e217654dcbc8e0104f3b6f33d94086f611 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Wed, 8 Jun 2022 12:42:03 +0300 Subject: [PATCH 06/23] Remove AsyncEventBus test suite. --- tests/testsync.nim | 181 --------------------------------------------- 1 file changed, 181 deletions(-) diff --git a/tests/testsync.nim b/tests/testsync.nim index 0bd63bc65..4b74570ef 100644 --- a/tests/testsync.nim +++ b/tests/testsync.nim @@ -352,185 +352,6 @@ suite "Asynchronous sync primitives test suite": check test8() == true test "AsyncQueue() contains test": check test9() == true - test "AsyncEventBus() awaitable primitives test": - const TestsCount = 10 - var bus = newAsyncEventBus() - var flag = "" - - proc waiter(bus: AsyncEventBus) {.async.} = - for i in 0 ..< TestsCount: - let payload = await bus.waitEvent(string, "event") - flag = flag & payload - - proc sender(bus: AsyncEventBus) {.async.} = - for i in 0 ..< (TestsCount + (TestsCount div 2)): - await bus.emitWait("event", $i) - - waitFor allFutures(waiter(bus), sender(bus)) - check flag == "0123456789" - test "AsyncEventBus() waiters test": - var bus = newAsyncEventBus() - let fut11 = bus.waitEvent(int, "event") - let fut12 = bus.waitEvent(int, "event") - let fut13 = bus.waitEvent(int, "event") - let fut21 = bus.waitEvent(string, "event") - let fut22 = bus.waitEvent(string, "event") - let fut23 = bus.waitEvent(string, "event") - bus.emit("event", 65535) - check: - fut11.done() == true - fut12.done() == true - fut13.done() == true - fut21.finished() == false - fut22.finished() == false - fut23.finished() == false - bus.emit("event", "data") - check: - fut21.done() == true - fut22.done() == true - fut23.done() == true - test "AsyncEventBus() subscribers test": - const TestsCount = 10 - var bus = newAsyncEventBus() - var flagInt = 0 - var flagStr = "" - proc eventIntCallback(bus: AsyncEventBus, - payload: EventPayload[int]) {.async.} = - flagInt = payload.get() - proc eventStrCallback(bus: AsyncEventBus, - payload: EventPayload[string]) {.async.} = - flagStr = payload.get() - - let key1 = bus.subscribe("event", eventIntCallback) - let key2 = bus.subscribe("event", eventStrCallback) - - proc test() {.async.} = - check: - flagInt == 0 - flagStr == "" - for i in 0 ..< TestsCount: - await bus.emitWait("event", i) - check: - flagInt == i - flagStr == "" - flagInt = 0 - for i in 0 ..< TestsCount: - await bus.emitWait("event", $i) - check: - flagInt == 0 - flagStr == $i - flagInt = 0 - flagStr = "" - bus.unsubscribe(key1) - for i in 0 ..< TestsCount: - await bus.emitWait("event", i) - check: - flagInt == 0 - flagStr == "" - flagInt = 0 - flagStr = "" - bus.unsubscribe(key2) - for i in 0 ..< TestsCount: - await bus.emitWait("event", $i) - check: - flagInt == 0 - flagStr == "" - waitFor(test()) - test "AsyncEventBus() waiters for all events test": - var bus = newAsyncEventBus() - let fut11 = bus.waitAllEvents() - let fut12 = bus.waitAllEvents() - bus.emit("intevent", 65535) - check: - fut11.done() == true - fut12.done() == true - let event11 = fut11.read() - let event12 = fut12.read() - check: - event11.event() == "intevent" - event12.event() == "intevent" - event11.get(int) == 65535 - event12.get(int) == 65535 - - let fut21 = bus.waitAllEvents() - let fut22 = bus.waitAllEvents() - bus.emit("strevent", "hello") - check: - fut21.done() == true - fut22.done() == true - let event21 = fut21.read() - let event22 = fut22.read() - check: - event21.event() == "strevent" - event22.event() == "strevent" - event21.get(string) == "hello" - event22.get(string) == "hello" - test "AsyncEventBus() subscribers to all events test": - const TestsCount = 10 - var - bus = newAsyncEventBus() - flagInt = 0 - flagStr = "" - - proc eventCallback(bus: AsyncEventBus, event: AwaitableEvent) {.async.} = - case event.event() - of "event1": - flagStr = "" - flagInt = event.get(int) - of "event2": - flagInt = 0 - flagStr = event.get(string) - else: - flagInt = -1 - flagStr = "error" - - proc test() {.async.} = - let key = bus.subscribeAll(eventCallback) - for i in 0 ..< TestsCount: - await bus.emitWait("event1", i) - check: - flagStr == "" - flagInt == i - await bus.emitWait("event2", $i) - check: - flagStr == $i - flagInt == 0 - - bus.unsubscribe(key) - - flagInt = high(int) - flagStr = "empty" - for i in 0 ..< TestsCount: - await bus.emitWait("event1", i) - check: - flagStr == "empty" - flagInt == high(int) - await bus.emitWait("event2", $i) - check: - flagStr == "empty" - flagInt == high(int) - - waitFor(test()) - - test "AsyncEventBus() multiple subscribers test": - let - eventBus = newAsyncEventBus() - futA = newFuture[void]() - futB = newFuture[void]() - - proc eventEV1(bus: AsyncEventBus, payload: EventPayload[int]) {.async.} = - futA.complete() - - proc eventEV2(bus: AsyncEventBus, payload: EventPayload[int]) {.async.} = - futB.complete() - - proc test() {.async.} = - discard eventBus.subscribe("EV", eventEV1) - discard eventBus.subscribe("EV", eventEV2) - eventBus.emit("EV", 5) - await allFutures(futA, futB).wait(1.seconds) - - waitFor test() test "AsyncEventQueue() behavior test": let eventQueue = newAsyncEventQueue[int]() @@ -1048,5 +869,3 @@ suite "Asynchronous sync primitives test suite": await eventQueue.closeWait() waitFor test() - - From 5799173dd56ff44e6e5b8151d4dec841b14c3284 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Wed, 8 Jun 2022 13:20:20 +0300 Subject: [PATCH 07/23] Remove unneeded [T] usage. --- chronos/asyncsync.nim | 46 +++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/chronos/asyncsync.nim b/chronos/asyncsync.nim index 36987954f..19e644688 100644 --- a/chronos/asyncsync.nim +++ b/chronos/asyncsync.nim @@ -630,7 +630,7 @@ template emitWait*[T](bus: AsyncEventBus, event: string, proc `==`(a, b: EventQueueKey): bool {.borrow.} -proc compact[T](ab: AsyncEventQueue[T]) = +proc compact(ab: AsyncEventQueue) = if len(ab.readers) > 0: let minOffset = block: @@ -653,7 +653,7 @@ proc compact[T](ab: AsyncEventQueue[T]) = else: ab.queue.clear() -proc getReaderIndex[T](ab: AsyncEventQueue[T], key: EventQueueKey): int = +proc getReaderIndex(ab: AsyncEventQueue, key: EventQueueKey): int = for index, value in ab.readers.pairs(): if value.key == key: return index @@ -677,10 +677,10 @@ proc newAsyncEventQueue*[T](limitSize = 0): AsyncEventQueue[T] = initDeque[T](nextPowerOfTwo(limitSize + 1)) AsyncEventQueue[T](counter: 0'u64, queue: queue, limit: limitSize) -proc len*[T](ab: AsyncEventQueue[T]): int = +proc len*(ab: AsyncEventQueue): int = len(ab.queue) -proc register*[T](ab: AsyncEventQueue[T]): EventQueueKey = +proc register*(ab: AsyncEventQueue): EventQueueKey = inc(ab.counter) let reader = EventQueueReader(key: EventQueueKey(ab.counter), offset: ab.offset + len(ab.queue), @@ -688,7 +688,7 @@ proc register*[T](ab: AsyncEventQueue[T]): EventQueueKey = ab.readers.add(reader) EventQueueKey(ab.counter) -proc unregister*[T](ab: AsyncEventQueue[T], key: EventQueueKey) = +proc unregister*(ab: AsyncEventQueue, key: EventQueueKey) = let index = ab.getReaderIndex(key) if index >= 0: let reader = ab.readers[index] @@ -698,8 +698,24 @@ proc unregister*[T](ab: AsyncEventQueue[T], key: EventQueueKey) = ab.readers.delete(index) ab.compact() -template readerOverflow*[T](ab: AsyncEventQueue[T], - reader: EventQueueReader): bool = +proc close*(ab: AsyncEventQueue) = + for reader in ab.readers.items(): + if not(isNil(reader.waiter)) and not(reader.waiter.finished()): + reader.waiter.complete() + ab.readers.reset() + ab.queue.clear() + +proc closeWait*(ab: AsyncEventQueue): Future[void] = + var retFuture = newFuture[void]("AsyncEventQueue.closeWait()") + proc continuation(udata: pointer) {.gcsafe.} = + if not(retFuture.finished()): + retFuture.complete() + ab.close() + callSoon(continuation) + retFuture + +template readerOverflow*(ab: AsyncEventQueue, + reader: EventQueueReader): bool = ab.limit + (reader.offset - ab.offset) <= len(ab.queue) proc emit*[T](ab: AsyncEventQueue[T], data: T) = @@ -737,22 +753,6 @@ proc emit*[T](ab: AsyncEventQueue[T], data: T) = if changesPresent: ab.compact() -proc close*[T](ab: AsyncEventQueue[T]) = - for reader in ab.readers.items(): - if not(isNil(reader.waiter)) and not(reader.waiter.finished()): - reader.waiter.complete() - ab.readers.reset() - ab.queue.clear() - -proc closeWait*[T](ab: AsyncEventQueue[T]): Future[void] = - var retFuture = newFuture[void]("AsyncEventQueue.closeWait()") - proc continuation(udata: pointer) {.gcsafe.} = - if not(retFuture.finished()): - retFuture.complete() - ab.close() - callSoon(continuation) - retFuture - proc waitEvents*[T](ab: AsyncEventQueue[T], key: EventQueueKey, eventsCount = -1): Future[seq[T]] {.async.} = From d50f1afefa3b5cc0efbe67d60176937048918523 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Wed, 8 Jun 2022 21:39:03 +0300 Subject: [PATCH 08/23] Optimize memory usage in 1m test. --- tests/testsync.nim | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/testsync.nim b/tests/testsync.nim index 4b74570ef..7838e76d4 100644 --- a/tests/testsync.nim +++ b/tests/testsync.nim @@ -590,9 +590,10 @@ suite "Asynchronous sync primitives test suite": if len(res) == 0: break events.add(res) + queue.unregister(key) return events - var futs = [ + var futs = @[ clientTask(eventQueue, keys[0]), clientTask(eventQueue, keys[1]), clientTask(eventQueue, keys[2]), clientTask(eventQueue, keys[3]), clientTask(eventQueue, keys[4]), clientTask(eventQueue, keys[5]), @@ -606,20 +607,20 @@ suite "Asynchronous sync primitives test suite": # Give some CPU for clients. await sleepAsync(0.milliseconds) eventQueue.emit(i) - await eventQueue.closeWait() await allFutures(futs) - for fut in futs: - check: - fut.finished() == true + for index in 0 ..< len(futs): + let fut = futs[index] + check fut.finished() == true let data = fut.read() var counter = 1 for item in data: - check: - item == counter + check item == counter inc(counter) + futs[index] = nil waitFor test() + futs.reset() test "AsyncEventQueue() one consumer limits test": let eventQueue = newAsyncEventQueue[int](4) From 80c4d11b1777fd15c06fa213f932d7b475484572 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Wed, 8 Jun 2022 22:12:01 +0300 Subject: [PATCH 09/23] Lower number of events in test from 1m to 100k. --- tests/testsync.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testsync.nim b/tests/testsync.nim index 7838e76d4..610003ff2 100644 --- a/tests/testsync.nim +++ b/tests/testsync.nim @@ -572,7 +572,7 @@ suite "Asynchronous sync primitives test suite": waitFor eventQueue.closeWait() - test "AsyncEventQueue() 1,000,000 of events to 10 clients test": + test "AsyncEventQueue() 100,000 of events to 10 clients test": let eventQueue = newAsyncEventQueue[int]() let keys = [ eventQueue.register(), eventQueue.register(), @@ -602,7 +602,7 @@ suite "Asynchronous sync primitives test suite": ] proc test() {.async.} = - for i in 1 .. 1_000_000: + for i in 1 .. 100_000: if (i mod 1000) == 0: # Give some CPU for clients. await sleepAsync(0.milliseconds) From da2106aa57be0a2971e28789c73970f700470471 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Thu, 9 Jun 2022 01:13:37 +0300 Subject: [PATCH 10/23] Deprecate AsyncEventBus. Add some GC debugging for tests. --- chronos/asyncsync.nim | 59 ++++++++++++++++++++++++++++++------------- tests/testsync.nim | 21 ++++++++++++++- 2 files changed, 61 insertions(+), 19 deletions(-) diff --git a/chronos/asyncsync.nim b/chronos/asyncsync.nim index 19e644688..709f3afe9 100644 --- a/chronos/asyncsync.nim +++ b/chronos/asyncsync.nim @@ -465,7 +465,9 @@ proc `$`*[T](aq: AsyncQueue[T]): string = template generateKey(typeName, eventName: string): string = "type[" & typeName & "]-key[" & eventName & "]" -proc newAsyncEventBus*(): AsyncEventBus = +proc newAsyncEventBus*(): AsyncEventBus {. + deprecated: "Implementation has unfixable flaws, please use" & + "AsyncEventQueue[T] instead".} = ## Creates new ``AsyncEventBus``. AsyncEventBus(counter: 0'u64, events: initTable[string, EventItem]()) @@ -477,7 +479,9 @@ template location*(payload: EventPayloadBase): SrcLoc = ## Returns source location address of event emitter. payload.loc[] -proc get*(event: AwaitableEvent, T: typedesc): T = +proc get*(event: AwaitableEvent, T: typedesc): T {. + deprecated: "Implementation has unfixable flaws, please use " & + "AsyncEventQueue[T] instead".} =.} ## Returns event's payload of type ``T`` from event ``event``. cast[EventPayload[T]](event.payload).value @@ -489,7 +493,9 @@ template location*(event: AwaitableEvent): SrcLoc = ## Returns source location address of event emitter. event.payload.loc[] -proc waitEvent*(bus: AsyncEventBus, T: typedesc, event: string): Future[T] = +proc waitEvent*(bus: AsyncEventBus, T: typedesc, event: string): Future[T] {. + deprecated: "Implementation has unfixable flaws, please use " & + "AsyncEventQueue[T] instead".} = ## Wait for the event from AsyncEventBus ``bus`` with name ``event``. ## ## Returned ``Future[T]`` will hold event's payload of type ``T``. @@ -505,7 +511,9 @@ proc waitEvent*(bus: AsyncEventBus, T: typedesc, event: string): Future[T] = bus.events.mgetOrPut(eventKey, default).waiters.add(baseFuture) retFuture -proc waitAllEvents*(bus: AsyncEventBus): Future[AwaitableEvent] = +proc waitAllEvents*(bus: AsyncEventBus): Future[AwaitableEvent] {. + deprecated: "Implementation has unfixable flaws, please use " & + "AsyncEventQueue[T] instead".} = ## Wait for any event from AsyncEventBus ``bus``. ## ## Returns ``Future`` which holds helper object. Using this object you can @@ -519,7 +527,9 @@ proc waitAllEvents*(bus: AsyncEventBus): Future[AwaitableEvent] = retFuture proc subscribe*[T](bus: AsyncEventBus, event: string, - callback: EventBusSubscription[T]): EventBusKey = + callback: EventBusSubscription[T]): EventBusKey {. + deprecated: "Implementation has unfixable flaws, please use " & + "AsyncEventQueue[T] instead".} = ## Subscribe to the event ``event`` passed through eventbus ``bus`` with ## callback ``callback``. ## @@ -541,7 +551,9 @@ proc subscribe*[T](bus: AsyncEventBus, event: string, subkey proc subscribeAll*(bus: AsyncEventBus, - callback: EventBusAllSubscription): EventBusKey = + callback: EventBusAllSubscription): EventBusKey {. + deprecated: "Implementation has unfixable flaws, please use " & + "AsyncEventQueue instead".} = ## Subscribe to all events passed through eventbus ``bus`` with callback ## ``callback``. ## @@ -559,7 +571,9 @@ proc subscribeAll*(bus: AsyncEventBus, bus.subscribers.add(subkey) subkey -proc unsubscribe*(bus: AsyncEventBus, key: EventBusKey) = +proc unsubscribe*(bus: AsyncEventBus, key: EventBusKey) {. + deprecated: "Implementation has unfixable flaws, please use " & + "AsyncEventQueue instead".} = ## Cancel subscription of subscriber with key ``key`` from eventbus ``bus``. let eventKey = generateKey(key.typeName, key.eventName) @@ -570,7 +584,9 @@ proc unsubscribe*(bus: AsyncEventBus, key: EventBusKey) = # Clean subscribers subscribed to all events. bus.subscribers.keepItIf(it.unique != key.unique) -proc emit[T](bus: AsyncEventBus, event: string, data: T, loc: ptr SrcLoc) = +proc emit[T](bus: AsyncEventBus, event: string, data: T, loc: ptr SrcLoc) {. + deprecated: "Implementation has unfixable flaws, please use " & + "AsyncEventQueue instead".} = let eventKey = generateKey(T.name, event) payload = @@ -612,7 +628,9 @@ template emit*[T](bus: AsyncEventBus, event: string, data: T) = emit(bus, event, data, getSrcLocation()) proc emitWait[T](bus: AsyncEventBus, event: string, data: T, - loc: ptr SrcLoc): Future[void] = + loc: ptr SrcLoc): Future[void] {. + deprecated: "Implementation has unfixable flaws, please use " & + "AsyncEventQueue instead".} = var retFuture = newFuture[void]("AsyncEventBus.emitWait") proc continuation(udata: pointer) {.gcsafe.} = if not(retFuture.finished()): @@ -630,7 +648,7 @@ template emitWait*[T](bus: AsyncEventBus, event: string, proc `==`(a, b: EventQueueKey): bool {.borrow.} -proc compact(ab: AsyncEventQueue) = +proc compact(ab: AsyncEventQueue) {.raises: [Defect].} = if len(ab.readers) > 0: let minOffset = block: @@ -653,13 +671,15 @@ proc compact(ab: AsyncEventQueue) = else: ab.queue.clear() -proc getReaderIndex(ab: AsyncEventQueue, key: EventQueueKey): int = +proc getReaderIndex(ab: AsyncEventQueue, key: EventQueueKey): int {. + raises: [Defect].} = for index, value in ab.readers.pairs(): if value.key == key: return index -1 -proc newAsyncEventQueue*[T](limitSize = 0): AsyncEventQueue[T] = +proc newAsyncEventQueue*[T](limitSize = 0): AsyncEventQueue[T] {. + raises: [Defect].} = ## Creates new ``AsyncEventBus`` maximum size of ``limitSize`` (default is ## ``0`` which means that there no limits). ## @@ -677,10 +697,10 @@ proc newAsyncEventQueue*[T](limitSize = 0): AsyncEventQueue[T] = initDeque[T](nextPowerOfTwo(limitSize + 1)) AsyncEventQueue[T](counter: 0'u64, queue: queue, limit: limitSize) -proc len*(ab: AsyncEventQueue): int = +proc len*(ab: AsyncEventQueue): int {.raises: [Defect].} = len(ab.queue) -proc register*(ab: AsyncEventQueue): EventQueueKey = +proc register*(ab: AsyncEventQueue): EventQueueKey {.raises: [Defect].} = inc(ab.counter) let reader = EventQueueReader(key: EventQueueKey(ab.counter), offset: ab.offset + len(ab.queue), @@ -688,7 +708,8 @@ proc register*(ab: AsyncEventQueue): EventQueueKey = ab.readers.add(reader) EventQueueKey(ab.counter) -proc unregister*(ab: AsyncEventQueue, key: EventQueueKey) = +proc unregister*(ab: AsyncEventQueue, key: EventQueueKey) {. + raises: [Defect] .} = let index = ab.getReaderIndex(key) if index >= 0: let reader = ab.readers[index] @@ -698,19 +719,21 @@ proc unregister*(ab: AsyncEventQueue, key: EventQueueKey) = ab.readers.delete(index) ab.compact() -proc close*(ab: AsyncEventQueue) = +proc close*(ab: AsyncEventQueue) {.raises: [Defect].} = for reader in ab.readers.items(): if not(isNil(reader.waiter)) and not(reader.waiter.finished()): reader.waiter.complete() ab.readers.reset() ab.queue.clear() -proc closeWait*(ab: AsyncEventQueue): Future[void] = +proc closeWait*(ab: AsyncEventQueue): Future[void] {.raises: [Defect].} = var retFuture = newFuture[void]("AsyncEventQueue.closeWait()") proc continuation(udata: pointer) {.gcsafe.} = if not(retFuture.finished()): retFuture.complete() ab.close() + # Schedule `continuation` to be called only after all the `reader` + # notifications will be scheduled and processed. callSoon(continuation) retFuture @@ -718,7 +741,7 @@ template readerOverflow*(ab: AsyncEventQueue, reader: EventQueueReader): bool = ab.limit + (reader.offset - ab.offset) <= len(ab.queue) -proc emit*[T](ab: AsyncEventQueue[T], data: T) = +proc emit*[T](ab: AsyncEventQueue[T], data: T) {.raises: [Defect].} = if len(ab.readers) > 0: # We enqueue `data` only if there active reader present. var changesPresent = false diff --git a/tests/testsync.nim b/tests/testsync.nim index 610003ff2..7619a1179 100644 --- a/tests/testsync.nim +++ b/tests/testsync.nim @@ -379,7 +379,9 @@ suite "Asynchronous sync primitives test suite": test1() test2() - eventQueue.close() + waitFor eventQueue.closeWait() + GC_fullCollect() + debugEcho GC_getStatistics() test "AsyncEventQueue() concurrency test": let eventQueue = newAsyncEventQueue[int]() @@ -432,6 +434,8 @@ suite "Asynchronous sync primitives test suite": 2000] waitFor eventQueue.closeWait() + GC_fullCollect() + debugEcho GC_getStatistics() test "AsyncEventQueue() specific number test": let eventQueue = newAsyncEventQueue[int]() @@ -482,6 +486,8 @@ suite "Asynchronous sync primitives test suite": dataFut4.read() == @[900, 1000, 1100, 1200, 1300, 1400, 1500, 1600] waitFor eventQueue.closeWait() + GC_fullCollect() + debugEcho GC_getStatistics() test "AsyncEventQueue() register()/unregister() test": var emptySeq: seq[int] @@ -529,6 +535,8 @@ suite "Asynchronous sync primitives test suite": dataFut3.read() == @[100, 200, 300] waitFor eventQueue.closeWait() + GC_fullCollect() + debugEcho GC_getStatistics() test "AsyncEventQueue() garbage collection test": let eventQueue = newAsyncEventQueue[int]() @@ -571,6 +579,8 @@ suite "Asynchronous sync primitives test suite": len(eventQueue) == 0 waitFor eventQueue.closeWait() + GC_fullCollect() + debugEcho GC_getStatistics() test "AsyncEventQueue() 100,000 of events to 10 clients test": let eventQueue = newAsyncEventQueue[int]() @@ -621,6 +631,8 @@ suite "Asynchronous sync primitives test suite": waitFor test() futs.reset() + GC_fullCollect() + debugEcho GC_getStatistics() test "AsyncEventQueue() one consumer limits test": let eventQueue = newAsyncEventQueue[int](4) @@ -671,7 +683,10 @@ suite "Asynchronous sync primitives test suite": eventQueue.unregister(key1) # All items should be garbage collected after unregister. check len(eventQueue) == 0 + waitFor eventQueue.closeWait() + GC_fullCollect() + debugEcho GC_getStatistics() test "AsyncEventQueue() many consumers limits test": let eventQueue = newAsyncEventQueue[int](4) @@ -843,6 +858,8 @@ suite "Asynchronous sync primitives test suite": check len(eventQueue) == 0 waitFor eventQueue.closeWait() + GC_fullCollect() + debugEcho GC_getStatistics() test "AsyncEventQueue() slow and fast consumer test": proc test() {.async.} = @@ -870,3 +887,5 @@ suite "Asynchronous sync primitives test suite": await eventQueue.closeWait() waitFor test() + GC_fullCollect() + debugEcho GC_getStatistics() From f17ab0370d6799c6d6d8235eeadf1d7779212201 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Thu, 9 Jun 2022 01:28:51 +0300 Subject: [PATCH 11/23] Fix mistype. --- chronos/asyncsync.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chronos/asyncsync.nim b/chronos/asyncsync.nim index 709f3afe9..345e354eb 100644 --- a/chronos/asyncsync.nim +++ b/chronos/asyncsync.nim @@ -481,7 +481,7 @@ template location*(payload: EventPayloadBase): SrcLoc = proc get*(event: AwaitableEvent, T: typedesc): T {. deprecated: "Implementation has unfixable flaws, please use " & - "AsyncEventQueue[T] instead".} =.} + "AsyncEventQueue[T] instead".} = ## Returns event's payload of type ``T`` from event ``event``. cast[EventPayload[T]](event.payload).value From f5a7c4714b0e52aede3905cd14cb5bd1d12f0247 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Thu, 9 Jun 2022 04:27:22 +0300 Subject: [PATCH 12/23] One more attempt to fix crash. --- chronos/asyncsync.nim | 16 ++++++------ tests/testsync.nim | 59 ++++++++++++++++++++++--------------------- 2 files changed, 38 insertions(+), 37 deletions(-) diff --git a/chronos/asyncsync.nim b/chronos/asyncsync.nim index 345e354eb..4d782bebd 100644 --- a/chronos/asyncsync.nim +++ b/chronos/asyncsync.nim @@ -584,9 +584,7 @@ proc unsubscribe*(bus: AsyncEventBus, key: EventBusKey) {. # Clean subscribers subscribed to all events. bus.subscribers.keepItIf(it.unique != key.unique) -proc emit[T](bus: AsyncEventBus, event: string, data: T, loc: ptr SrcLoc) {. - deprecated: "Implementation has unfixable flaws, please use " & - "AsyncEventQueue instead".} = +proc emit[T](bus: AsyncEventBus, event: string, data: T, loc: ptr SrcLoc) = let eventKey = generateKey(T.name, event) payload = @@ -623,14 +621,14 @@ proc emit[T](bus: AsyncEventBus, event: string, data: T, loc: ptr SrcLoc) {. for subscriber in bus.subscribers: triggerSubscriberCallback(subscriber) -template emit*[T](bus: AsyncEventBus, event: string, data: T) = +template emit*[T](bus: AsyncEventBus, event: string, data: T) {. + deprecated: "Implementation has unfixable flaws, please use " & + "AsyncEventQueue instead".} = ## Emit new event ``event`` to the eventbus ``bus`` with payload ``data``. emit(bus, event, data, getSrcLocation()) proc emitWait[T](bus: AsyncEventBus, event: string, data: T, - loc: ptr SrcLoc): Future[void] {. - deprecated: "Implementation has unfixable flaws, please use " & - "AsyncEventQueue instead".} = + loc: ptr SrcLoc): Future[void] = var retFuture = newFuture[void]("AsyncEventBus.emitWait") proc continuation(udata: pointer) {.gcsafe.} = if not(retFuture.finished()): @@ -640,7 +638,9 @@ proc emitWait[T](bus: AsyncEventBus, event: string, data: T, return retFuture template emitWait*[T](bus: AsyncEventBus, event: string, - data: T): Future[void] = + data: T): Future[void] {. + deprecated: "Implementation has unfixable flaws, please use " & + "AsyncEventQueue instead".} = ## Emit new event ``event`` to the eventbus ``bus`` with payload ``data`` and ## wait until all the subscribers/waiters will receive notification about ## event. diff --git a/tests/testsync.nim b/tests/testsync.nim index 7619a1179..3d3d140a1 100644 --- a/tests/testsync.nim +++ b/tests/testsync.nim @@ -583,41 +583,43 @@ suite "Asynchronous sync primitives test suite": debugEcho GC_getStatistics() test "AsyncEventQueue() 100,000 of events to 10 clients test": - let eventQueue = newAsyncEventQueue[int]() - let keys = [ - eventQueue.register(), eventQueue.register(), - eventQueue.register(), eventQueue.register(), - eventQueue.register(), eventQueue.register(), - eventQueue.register(), eventQueue.register(), - eventQueue.register(), eventQueue.register() - ] - - proc clientTask(queue: AsyncEventQueue[int], - key: EventQueueKey): Future[seq[int]] {.async.} = - var events: seq[int] - while true: - let res = await queue.waitEvents(key) - if len(res) == 0: - break - events.add(res) - queue.unregister(key) - return events - - var futs = @[ - clientTask(eventQueue, keys[0]), clientTask(eventQueue, keys[1]), - clientTask(eventQueue, keys[2]), clientTask(eventQueue, keys[3]), - clientTask(eventQueue, keys[4]), clientTask(eventQueue, keys[5]), - clientTask(eventQueue, keys[6]), clientTask(eventQueue, keys[7]), - clientTask(eventQueue, keys[8]), clientTask(eventQueue, keys[9]) - ] - proc test() {.async.} = + let eventQueue = newAsyncEventQueue[int]() + var keys = @[ + eventQueue.register(), eventQueue.register(), + eventQueue.register(), eventQueue.register(), + eventQueue.register(), eventQueue.register(), + eventQueue.register(), eventQueue.register(), + eventQueue.register(), eventQueue.register() + ] + + proc clientTask(queue: AsyncEventQueue[int], + key: EventQueueKey): Future[seq[int]] {.async.} = + var events: seq[int] + while true: + let res = await queue.waitEvents(key) + if len(res) == 0: + break + events.add(res) + queue.unregister(key) + return events + + var futs = @[ + clientTask(eventQueue, keys[0]), clientTask(eventQueue, keys[1]), + clientTask(eventQueue, keys[2]), clientTask(eventQueue, keys[3]), + clientTask(eventQueue, keys[4]), clientTask(eventQueue, keys[5]), + clientTask(eventQueue, keys[6]), clientTask(eventQueue, keys[7]), + clientTask(eventQueue, keys[8]), clientTask(eventQueue, keys[9]) + ] + for i in 1 .. 100_000: if (i mod 1000) == 0: # Give some CPU for clients. await sleepAsync(0.milliseconds) eventQueue.emit(i) + await eventQueue.closeWait() + await allFutures(futs) for index in 0 ..< len(futs): let fut = futs[index] @@ -630,7 +632,6 @@ suite "Asynchronous sync primitives test suite": futs[index] = nil waitFor test() - futs.reset() GC_fullCollect() debugEcho GC_getStatistics() From c543bccb694e5ce474cdbde13b9f521211d2fc64 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Thu, 9 Jun 2022 05:01:14 +0300 Subject: [PATCH 13/23] Add some echo debugging. --- tests/testsync.nim | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/testsync.nim b/tests/testsync.nim index 3d3d140a1..d03491c36 100644 --- a/tests/testsync.nim +++ b/tests/testsync.nim @@ -634,14 +634,18 @@ suite "Asynchronous sync primitives test suite": waitFor test() GC_fullCollect() debugEcho GC_getStatistics() + echo "0" test "AsyncEventQueue() one consumer limits test": + echo "1" let eventQueue = newAsyncEventQueue[int](4) + echo "2" check len(eventQueue) == 0 eventQueue.emit(100) eventQueue.emit(200) eventQueue.emit(300) eventQueue.emit(400) + echo "3" # There no consumers, so all the items should be discarded check len(eventQueue) == 0 let key1 = eventQueue.register() @@ -685,9 +689,12 @@ suite "Asynchronous sync primitives test suite": # All items should be garbage collected after unregister. check len(eventQueue) == 0 + echo "10" waitFor eventQueue.closeWait() + echo "11" GC_fullCollect() debugEcho GC_getStatistics() + echo "12" test "AsyncEventQueue() many consumers limits test": let eventQueue = newAsyncEventQueue[int](4) From ace652d8dc259e558c955dda3dc0afdab84ff5e9 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Thu, 9 Jun 2022 05:19:37 +0300 Subject: [PATCH 14/23] More echo debugging. --- tests/testsync.nim | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/testsync.nim b/tests/testsync.nim index d03491c36..38ac5aa30 100644 --- a/tests/testsync.nim +++ b/tests/testsync.nim @@ -649,11 +649,13 @@ suite "Asynchronous sync primitives test suite": # There no consumers, so all the items should be discarded check len(eventQueue) == 0 let key1 = eventQueue.register() + echo "4" check len(eventQueue) == 0 eventQueue.emit(500) eventQueue.emit(600) eventQueue.emit(700) eventQueue.emit(800) + echo "5" # So exact `limit` number of items added, consumer should receive all of # them. check len(eventQueue) == 4 @@ -662,39 +664,44 @@ suite "Asynchronous sync primitives test suite": dataFut1.finished() == true dataFut1.read() == @[500, 600, 700, 800] len(eventQueue) == 0 + echo "6" eventQueue.emit(900) eventQueue.emit(1000) eventQueue.emit(1100) eventQueue.emit(1200) + echo "7" check len(eventQueue) == 4 # Overfilling queue eventQueue.emit(1300) # Because overfill for single consumer happend, whole queue should become # empty. check len(eventQueue) == 0 + echo "8" eventQueue.emit(1400) eventQueue.emit(1500) eventQueue.emit(1600) eventQueue.emit(1700) eventQueue.emit(1800) check len(eventQueue) == 0 + echo "9" let errorFut1 = eventQueue.waitEvents(key1) check errorFut1.finished() == true expect AsyncEventQueueFullError: let res {.used.} = errorFut1.read() # There should be no items because consumer was overflowed. check len(eventQueue) == 0 + echo "10" eventQueue.unregister(key1) # All items should be garbage collected after unregister. check len(eventQueue) == 0 - echo "10" - waitFor eventQueue.closeWait() echo "11" + waitFor eventQueue.closeWait() + echo "12" GC_fullCollect() debugEcho GC_getStatistics() - echo "12" + echo "13" test "AsyncEventQueue() many consumers limits test": let eventQueue = newAsyncEventQueue[int](4) From 597a3b20f6028f2ee1110cd85f52ac4ea19cfb42 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Thu, 9 Jun 2022 10:17:58 +0300 Subject: [PATCH 15/23] More closer debug echoes. --- tests/testsync.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/testsync.nim b/tests/testsync.nim index 38ac5aa30..c392a62d6 100644 --- a/tests/testsync.nim +++ b/tests/testsync.nim @@ -686,9 +686,12 @@ suite "Asynchronous sync primitives test suite": check len(eventQueue) == 0 echo "9" let errorFut1 = eventQueue.waitEvents(key1) + echo "90" check errorFut1.finished() == true + echo "91" expect AsyncEventQueueFullError: let res {.used.} = errorFut1.read() + echo "92" # There should be no items because consumer was overflowed. check len(eventQueue) == 0 echo "10" From fe85ba9c52b7a5555910393cf106e6b420504f0b Mon Sep 17 00:00:00 2001 From: cheatfate Date: Thu, 9 Jun 2022 11:45:42 +0300 Subject: [PATCH 16/23] Attempt to workaround crash place. --- tests/testsync.nim | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/testsync.nim b/tests/testsync.nim index c392a62d6..ad5c5d192 100644 --- a/tests/testsync.nim +++ b/tests/testsync.nim @@ -689,8 +689,15 @@ suite "Asynchronous sync primitives test suite": echo "90" check errorFut1.finished() == true echo "91" - expect AsyncEventQueueFullError: - let res {.used.} = errorFut1.read() + let checkException = + try: + let res {.used.} = errorFut1.read() + false + except AsyncEventQueueFullError: + true + except CatchableError: + false + check checkException == true echo "92" # There should be no items because consumer was overflowed. check len(eventQueue) == 0 From ddc41a99d34fcf85fb1d23ef8921471b0bf037bc Mon Sep 17 00:00:00 2001 From: cheatfate Date: Thu, 9 Jun 2022 14:09:22 +0300 Subject: [PATCH 17/23] More debugging echoes in exception handlers. --- tests/testsync.nim | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/testsync.nim b/tests/testsync.nim index ad5c5d192..7f99d9215 100644 --- a/tests/testsync.nim +++ b/tests/testsync.nim @@ -691,12 +691,17 @@ suite "Asynchronous sync primitives test suite": echo "91" let checkException = try: + echo "910" let res {.used.} = errorFut1.read() + echo "911" false except AsyncEventQueueFullError: + echo "912" true except CatchableError: + echo "913" false + echo "914" check checkException == true echo "92" # There should be no items because consumer was overflowed. From abe488c019f49c8900111d0de6294a2f28d5069e Mon Sep 17 00:00:00 2001 From: cheatfate Date: Thu, 9 Jun 2022 23:21:23 +0300 Subject: [PATCH 18/23] Convert suspected test into async procedure. --- tests/testsync.nim | 153 +++++++++++++++++++++++---------------------- 1 file changed, 78 insertions(+), 75 deletions(-) diff --git a/tests/testsync.nim b/tests/testsync.nim index 7f99d9215..a58711250 100644 --- a/tests/testsync.nim +++ b/tests/testsync.nim @@ -637,83 +637,86 @@ suite "Asynchronous sync primitives test suite": echo "0" test "AsyncEventQueue() one consumer limits test": - echo "1" - let eventQueue = newAsyncEventQueue[int](4) - echo "2" - check len(eventQueue) == 0 - eventQueue.emit(100) - eventQueue.emit(200) - eventQueue.emit(300) - eventQueue.emit(400) - echo "3" - # There no consumers, so all the items should be discarded - check len(eventQueue) == 0 - let key1 = eventQueue.register() - echo "4" - check len(eventQueue) == 0 - eventQueue.emit(500) - eventQueue.emit(600) - eventQueue.emit(700) - eventQueue.emit(800) - echo "5" - # So exact `limit` number of items added, consumer should receive all of - # them. - check len(eventQueue) == 4 - let dataFut1 = eventQueue.waitEvents(key1) - check: - dataFut1.finished() == true - dataFut1.read() == @[500, 600, 700, 800] - len(eventQueue) == 0 - echo "6" + proc test() {.async.} = + echo "1" + let eventQueue = newAsyncEventQueue[int](4) + echo "2" + check len(eventQueue) == 0 + eventQueue.emit(100) + eventQueue.emit(200) + eventQueue.emit(300) + eventQueue.emit(400) + echo "3" + # There no consumers, so all the items should be discarded + check len(eventQueue) == 0 + let key1 = eventQueue.register() + echo "4" + check len(eventQueue) == 0 + eventQueue.emit(500) + eventQueue.emit(600) + eventQueue.emit(700) + eventQueue.emit(800) + echo "5" + # So exact `limit` number of items added, consumer should receive all of + # them. + check len(eventQueue) == 4 + let dataFut1 = eventQueue.waitEvents(key1) + check: + dataFut1.finished() == true + dataFut1.read() == @[500, 600, 700, 800] + len(eventQueue) == 0 + echo "6" - eventQueue.emit(900) - eventQueue.emit(1000) - eventQueue.emit(1100) - eventQueue.emit(1200) - echo "7" - check len(eventQueue) == 4 - # Overfilling queue - eventQueue.emit(1300) - # Because overfill for single consumer happend, whole queue should become - # empty. - check len(eventQueue) == 0 - echo "8" - eventQueue.emit(1400) - eventQueue.emit(1500) - eventQueue.emit(1600) - eventQueue.emit(1700) - eventQueue.emit(1800) - check len(eventQueue) == 0 - echo "9" - let errorFut1 = eventQueue.waitEvents(key1) - echo "90" - check errorFut1.finished() == true - echo "91" - let checkException = - try: - echo "910" - let res {.used.} = errorFut1.read() - echo "911" - false - except AsyncEventQueueFullError: - echo "912" - true - except CatchableError: - echo "913" - false - echo "914" - check checkException == true - echo "92" - # There should be no items because consumer was overflowed. - check len(eventQueue) == 0 - echo "10" - eventQueue.unregister(key1) - # All items should be garbage collected after unregister. - check len(eventQueue) == 0 + eventQueue.emit(900) + eventQueue.emit(1000) + eventQueue.emit(1100) + eventQueue.emit(1200) + echo "7" + check len(eventQueue) == 4 + # Overfilling queue + eventQueue.emit(1300) + # Because overfill for single consumer happend, whole queue should become + # empty. + check len(eventQueue) == 0 + echo "8" + eventQueue.emit(1400) + eventQueue.emit(1500) + eventQueue.emit(1600) + eventQueue.emit(1700) + eventQueue.emit(1800) + check len(eventQueue) == 0 + echo "9" + let errorFut1 = eventQueue.waitEvents(key1) + echo "90" + check errorFut1.finished() == true + echo "91" + let checkException = + try: + echo "910" + let res {.used.} = await errorFut1 + echo "911" + false + except AsyncEventQueueFullError: + echo "912" + true + except CatchableError: + echo "913" + false + echo "914" + check checkException == true + echo "92" + # There should be no items because consumer was overflowed. + check len(eventQueue) == 0 + echo "10" + eventQueue.unregister(key1) + # All items should be garbage collected after unregister. + check len(eventQueue) == 0 - echo "11" - waitFor eventQueue.closeWait() - echo "12" + echo "11" + await eventQueue.closeWait() + echo "12" + + waitFor test() GC_fullCollect() debugEcho GC_getStatistics() echo "13" From 4e38a7765ec7d89a3bb049b4b5522cc7bbe399bb Mon Sep 17 00:00:00 2001 From: cheatfate Date: Fri, 10 Jun 2022 00:47:23 +0300 Subject: [PATCH 19/23] Make multiple consumers test async. --- tests/testsync.nim | 364 +++++++++++++++++++++------------------------ 1 file changed, 172 insertions(+), 192 deletions(-) diff --git a/tests/testsync.nim b/tests/testsync.nim index a58711250..272c48596 100644 --- a/tests/testsync.nim +++ b/tests/testsync.nim @@ -582,7 +582,7 @@ suite "Asynchronous sync primitives test suite": GC_fullCollect() debugEcho GC_getStatistics() - test "AsyncEventQueue() 100,000 of events to 10 clients test": + test "AsyncEventQueue() 1,000,000 of events to 10 clients test": proc test() {.async.} = let eventQueue = newAsyncEventQueue[int]() var keys = @[ @@ -612,7 +612,7 @@ suite "Asynchronous sync primitives test suite": clientTask(eventQueue, keys[8]), clientTask(eventQueue, keys[9]) ] - for i in 1 .. 100_000: + for i in 1 .. 1_000_000: if (i mod 1000) == 0: # Give some CPU for clients. await sleepAsync(0.milliseconds) @@ -634,29 +634,23 @@ suite "Asynchronous sync primitives test suite": waitFor test() GC_fullCollect() debugEcho GC_getStatistics() - echo "0" test "AsyncEventQueue() one consumer limits test": proc test() {.async.} = - echo "1" let eventQueue = newAsyncEventQueue[int](4) - echo "2" check len(eventQueue) == 0 eventQueue.emit(100) eventQueue.emit(200) eventQueue.emit(300) eventQueue.emit(400) - echo "3" # There no consumers, so all the items should be discarded check len(eventQueue) == 0 let key1 = eventQueue.register() - echo "4" check len(eventQueue) == 0 eventQueue.emit(500) eventQueue.emit(600) eventQueue.emit(700) eventQueue.emit(800) - echo "5" # So exact `limit` number of items added, consumer should receive all of # them. check len(eventQueue) == 4 @@ -665,56 +659,40 @@ suite "Asynchronous sync primitives test suite": dataFut1.finished() == true dataFut1.read() == @[500, 600, 700, 800] len(eventQueue) == 0 - echo "6" eventQueue.emit(900) eventQueue.emit(1000) eventQueue.emit(1100) eventQueue.emit(1200) - echo "7" check len(eventQueue) == 4 # Overfilling queue eventQueue.emit(1300) # Because overfill for single consumer happend, whole queue should become # empty. check len(eventQueue) == 0 - echo "8" eventQueue.emit(1400) eventQueue.emit(1500) eventQueue.emit(1600) eventQueue.emit(1700) eventQueue.emit(1800) check len(eventQueue) == 0 - echo "9" let errorFut1 = eventQueue.waitEvents(key1) - echo "90" check errorFut1.finished() == true - echo "91" let checkException = try: - echo "910" let res {.used.} = await errorFut1 - echo "911" false except AsyncEventQueueFullError: - echo "912" true except CatchableError: - echo "913" false - echo "914" check checkException == true - echo "92" # There should be no items because consumer was overflowed. check len(eventQueue) == 0 - echo "10" eventQueue.unregister(key1) # All items should be garbage collected after unregister. check len(eventQueue) == 0 - - echo "11" await eventQueue.closeWait() - echo "12" waitFor test() GC_fullCollect() @@ -722,175 +700,177 @@ suite "Asynchronous sync primitives test suite": echo "13" test "AsyncEventQueue() many consumers limits test": - let eventQueue = newAsyncEventQueue[int](4) - block: - let key1 = eventQueue.register() - eventQueue.emit(100) - check len(eventQueue) == 1 - let key2 = eventQueue.register() - eventQueue.emit(200) - check len(eventQueue) == 2 - let key3 = eventQueue.register() - eventQueue.emit(300) - check len(eventQueue) == 3 - let key4 = eventQueue.register() - eventQueue.emit(400) - check len(eventQueue) == 4 - let key5 = eventQueue.register() - eventQueue.emit(500) - # At this point consumer with `key1` is overfilled, so after `emit()` - # queue length should be decreased by one item. - # So queue should look like this: [200, 300, 400, 500] - check len(eventQueue) == 4 - eventQueue.emit(600) - # At this point consumers with `key2` is overfilled, so after `emit()` - # queue length should be decreased by one item. - # So queue should look like this: [300, 400, 500, 600] - check len(eventQueue) == 4 - eventQueue.emit(700) - # At this point consumers with `key3` is overfilled, so after `emit()` - # queue length should be decreased by one item. - # So queue should look like this: [400, 500, 600, 700] - check len(eventQueue) == 4 - eventQueue.emit(800) - # At this point consumers with `key4` is overfilled, so after `emit()` - # queue length should be decreased by one item. - # So queue should look like this: [500, 600, 700, 800] - check len(eventQueue) == 4 - # Consumer with key5 is not overfilled. - let dataFut5 = eventQueue.waitEvents(key5) - check: - dataFut5.finished() == true - dataFut5.read() == @[500, 600, 700, 800] - # No more items should be left because all other consumers are overfilled. - check len(eventQueue) == 0 - eventQueue.unregister(key5) - check len(eventQueue) == 0 - - let dataFut2 = eventQueue.waitEvents(key2) - check dataFut2.finished() == true - expect AsyncEventQueueFullError: - let res {.used.} = dataFut2.read() - check len(eventQueue) == 0 - eventQueue.unregister(key2) - check len(eventQueue) == 0 - - let dataFut4 = eventQueue.waitEvents(key4) - check dataFut4.finished() == true - expect AsyncEventQueueFullError: - let res {.used.} = dataFut4.read() - check len(eventQueue) == 0 - eventQueue.unregister(key4) - check len(eventQueue) == 0 - - let dataFut3 = eventQueue.waitEvents(key3) - check dataFut3.finished() == true - expect AsyncEventQueueFullError: - let res {.used.} = dataFut3.read() - check len(eventQueue) == 0 - eventQueue.unregister(key3) - check len(eventQueue) == 0 - - let dataFut1 = eventQueue.waitEvents(key1) - check dataFut1.finished() == true - expect AsyncEventQueueFullError: - let res {.used.} = dataFut1.read() - check len(eventQueue) == 0 - eventQueue.unregister(key1) - check len(eventQueue) == 0 - - block: - let key1 = eventQueue.register() - eventQueue.emit(100) - check len(eventQueue) == 1 - let key2 = eventQueue.register() - eventQueue.emit(200) - check len(eventQueue) == 2 - let key3 = eventQueue.register() - eventQueue.emit(300) - check len(eventQueue) == 3 - let key4 = eventQueue.register() - eventQueue.emit(400) - check len(eventQueue) == 4 - let key5 = eventQueue.register() - eventQueue.emit(500) - # At this point consumer with `key1` is overfilled, so after `emit()` - # queue length should be decreased by one item. - # So queue should look like this: [200, 300, 400, 500] - check len(eventQueue) == 4 - eventQueue.emit(600) - # At this point consumer with `key2` is overfilled, so after `emit()` - # queue length should be decreased by one item. - # So queue should look like this: [300, 400, 500, 600] - check len(eventQueue) == 4 - eventQueue.emit(700) - # At this point consumer with `key3` is overfilled, so after `emit()` - # queue length should be decreased by one item. - # So queue should look like this: [400, 500, 600, 700] - check len(eventQueue) == 4 - eventQueue.emit(800) - # At this point consumer with `key4` is overfilled, so after `emit()` - # queue length should be decreased by one item. - # So queue should look like this: [500, 600, 700, 800] - check len(eventQueue) == 4 - eventQueue.emit(900) - # At this point all consumers are overfilled, so after `emit()` - # queue length should become 0. - check len(eventQueue) == 0 - eventQueue.emit(1000) - eventQueue.emit(1100) - eventQueue.emit(1200) - eventQueue.emit(1300) - eventQueue.emit(1400) - eventQueue.emit(1500) - eventQueue.emit(1600) - eventQueue.emit(1700) - eventQueue.emit(1800) - eventQueue.emit(1900) - # No more events should be accepted. - check len(eventQueue) == 0 - - let dataFut1 = eventQueue.waitEvents(key1) - check dataFut1.finished() == true - expect AsyncEventQueueFullError: - let res {.used.} = dataFut1.read() - check len(eventQueue) == 0 - eventQueue.unregister(key1) - check len(eventQueue) == 0 - - let dataFut2 = eventQueue.waitEvents(key2) - check dataFut2.finished() == true - expect AsyncEventQueueFullError: - let res {.used.} = dataFut2.read() - check len(eventQueue) == 0 - eventQueue.unregister(key2) - check len(eventQueue) == 0 - - let dataFut3 = eventQueue.waitEvents(key3) - check dataFut3.finished() == true - expect AsyncEventQueueFullError: - let res {.used.} = dataFut3.read() - check len(eventQueue) == 0 - eventQueue.unregister(key3) - check len(eventQueue) == 0 - - let dataFut4 = eventQueue.waitEvents(key4) - check dataFut4.finished() == true - expect AsyncEventQueueFullError: - let res {.used.} = dataFut4.read() - check len(eventQueue) == 0 - eventQueue.unregister(key4) - check len(eventQueue) == 0 - - let dataFut5 = eventQueue.waitEvents(key5) - check dataFut5.finished() == true - expect AsyncEventQueueFullError: - let res {.used.} = dataFut5.read() - check len(eventQueue) == 0 - eventQueue.unregister(key5) - check len(eventQueue) == 0 + proc test() {.async.} = + let eventQueue = newAsyncEventQueue[int](4) + block: + let key1 = eventQueue.register() + eventQueue.emit(100) + check len(eventQueue) == 1 + let key2 = eventQueue.register() + eventQueue.emit(200) + check len(eventQueue) == 2 + let key3 = eventQueue.register() + eventQueue.emit(300) + check len(eventQueue) == 3 + let key4 = eventQueue.register() + eventQueue.emit(400) + check len(eventQueue) == 4 + let key5 = eventQueue.register() + eventQueue.emit(500) + # At this point consumer with `key1` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [200, 300, 400, 500] + check len(eventQueue) == 4 + eventQueue.emit(600) + # At this point consumers with `key2` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [300, 400, 500, 600] + check len(eventQueue) == 4 + eventQueue.emit(700) + # At this point consumers with `key3` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [400, 500, 600, 700] + check len(eventQueue) == 4 + eventQueue.emit(800) + # At this point consumers with `key4` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [500, 600, 700, 800] + check len(eventQueue) == 4 + # Consumer with key5 is not overfilled. + let dataFut5 = eventQueue.waitEvents(key5) + check: + dataFut5.finished() == true + dataFut5.read() == @[500, 600, 700, 800] + # No more items should be left because all other consumers are overfilled. + check len(eventQueue) == 0 + eventQueue.unregister(key5) + check len(eventQueue) == 0 + + let dataFut2 = eventQueue.waitEvents(key2) + check dataFut2.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut2.read() + check len(eventQueue) == 0 + eventQueue.unregister(key2) + check len(eventQueue) == 0 + + let dataFut4 = eventQueue.waitEvents(key4) + check dataFut4.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut4.read() + check len(eventQueue) == 0 + eventQueue.unregister(key4) + check len(eventQueue) == 0 + + let dataFut3 = eventQueue.waitEvents(key3) + check dataFut3.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut3.read() + check len(eventQueue) == 0 + eventQueue.unregister(key3) + check len(eventQueue) == 0 + + let dataFut1 = eventQueue.waitEvents(key1) + check dataFut1.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut1.read() + check len(eventQueue) == 0 + eventQueue.unregister(key1) + check len(eventQueue) == 0 + + block: + let key1 = eventQueue.register() + eventQueue.emit(100) + check len(eventQueue) == 1 + let key2 = eventQueue.register() + eventQueue.emit(200) + check len(eventQueue) == 2 + let key3 = eventQueue.register() + eventQueue.emit(300) + check len(eventQueue) == 3 + let key4 = eventQueue.register() + eventQueue.emit(400) + check len(eventQueue) == 4 + let key5 = eventQueue.register() + eventQueue.emit(500) + # At this point consumer with `key1` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [200, 300, 400, 500] + check len(eventQueue) == 4 + eventQueue.emit(600) + # At this point consumer with `key2` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [300, 400, 500, 600] + check len(eventQueue) == 4 + eventQueue.emit(700) + # At this point consumer with `key3` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [400, 500, 600, 700] + check len(eventQueue) == 4 + eventQueue.emit(800) + # At this point consumer with `key4` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [500, 600, 700, 800] + check len(eventQueue) == 4 + eventQueue.emit(900) + # At this point all consumers are overfilled, so after `emit()` + # queue length should become 0. + check len(eventQueue) == 0 + eventQueue.emit(1000) + eventQueue.emit(1100) + eventQueue.emit(1200) + eventQueue.emit(1300) + eventQueue.emit(1400) + eventQueue.emit(1500) + eventQueue.emit(1600) + eventQueue.emit(1700) + eventQueue.emit(1800) + eventQueue.emit(1900) + # No more events should be accepted. + check len(eventQueue) == 0 + + let dataFut1 = eventQueue.waitEvents(key1) + check dataFut1.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut1.read() + check len(eventQueue) == 0 + eventQueue.unregister(key1) + check len(eventQueue) == 0 + + let dataFut2 = eventQueue.waitEvents(key2) + check dataFut2.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut2.read() + check len(eventQueue) == 0 + eventQueue.unregister(key2) + check len(eventQueue) == 0 + + let dataFut3 = eventQueue.waitEvents(key3) + check dataFut3.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut3.read() + check len(eventQueue) == 0 + eventQueue.unregister(key3) + check len(eventQueue) == 0 + + let dataFut4 = eventQueue.waitEvents(key4) + check dataFut4.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut4.read() + check len(eventQueue) == 0 + eventQueue.unregister(key4) + check len(eventQueue) == 0 + + let dataFut5 = eventQueue.waitEvents(key5) + check dataFut5.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut5.read() + check len(eventQueue) == 0 + eventQueue.unregister(key5) + check len(eventQueue) == 0 + await eventQueue.closeWait() - waitFor eventQueue.closeWait() + waitFor test() GC_fullCollect() debugEcho GC_getStatistics() From 22d4c4dffac72564be8f62c57fffa11a1d13eca6 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Fri, 10 Jun 2022 01:39:06 +0300 Subject: [PATCH 20/23] Remove GC debugging. --- tests/testsync.nim | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/tests/testsync.nim b/tests/testsync.nim index 272c48596..9acea50d5 100644 --- a/tests/testsync.nim +++ b/tests/testsync.nim @@ -380,8 +380,6 @@ suite "Asynchronous sync primitives test suite": test1() test2() waitFor eventQueue.closeWait() - GC_fullCollect() - debugEcho GC_getStatistics() test "AsyncEventQueue() concurrency test": let eventQueue = newAsyncEventQueue[int]() @@ -434,8 +432,6 @@ suite "Asynchronous sync primitives test suite": 2000] waitFor eventQueue.closeWait() - GC_fullCollect() - debugEcho GC_getStatistics() test "AsyncEventQueue() specific number test": let eventQueue = newAsyncEventQueue[int]() @@ -486,8 +482,6 @@ suite "Asynchronous sync primitives test suite": dataFut4.read() == @[900, 1000, 1100, 1200, 1300, 1400, 1500, 1600] waitFor eventQueue.closeWait() - GC_fullCollect() - debugEcho GC_getStatistics() test "AsyncEventQueue() register()/unregister() test": var emptySeq: seq[int] @@ -535,8 +529,6 @@ suite "Asynchronous sync primitives test suite": dataFut3.read() == @[100, 200, 300] waitFor eventQueue.closeWait() - GC_fullCollect() - debugEcho GC_getStatistics() test "AsyncEventQueue() garbage collection test": let eventQueue = newAsyncEventQueue[int]() @@ -579,8 +571,6 @@ suite "Asynchronous sync primitives test suite": len(eventQueue) == 0 waitFor eventQueue.closeWait() - GC_fullCollect() - debugEcho GC_getStatistics() test "AsyncEventQueue() 1,000,000 of events to 10 clients test": proc test() {.async.} = @@ -632,8 +622,6 @@ suite "Asynchronous sync primitives test suite": futs[index] = nil waitFor test() - GC_fullCollect() - debugEcho GC_getStatistics() test "AsyncEventQueue() one consumer limits test": proc test() {.async.} = @@ -695,9 +683,6 @@ suite "Asynchronous sync primitives test suite": await eventQueue.closeWait() waitFor test() - GC_fullCollect() - debugEcho GC_getStatistics() - echo "13" test "AsyncEventQueue() many consumers limits test": proc test() {.async.} = @@ -871,8 +856,6 @@ suite "Asynchronous sync primitives test suite": await eventQueue.closeWait() waitFor test() - GC_fullCollect() - debugEcho GC_getStatistics() test "AsyncEventQueue() slow and fast consumer test": proc test() {.async.} = @@ -900,5 +883,3 @@ suite "Asynchronous sync primitives test suite": await eventQueue.closeWait() waitFor test() - GC_fullCollect() - debugEcho GC_getStatistics() From 4e2e7ac79fe6097b5f44b66c75c8c3793a72bf5c Mon Sep 17 00:00:00 2001 From: cheatfate Date: Fri, 10 Jun 2022 02:03:16 +0300 Subject: [PATCH 21/23] Disable added tests. --- tests/testsync.nim | 520 ++++++++++++++++++++++----------------------- 1 file changed, 260 insertions(+), 260 deletions(-) diff --git a/tests/testsync.nim b/tests/testsync.nim index 9acea50d5..c06e536de 100644 --- a/tests/testsync.nim +++ b/tests/testsync.nim @@ -623,263 +623,263 @@ suite "Asynchronous sync primitives test suite": waitFor test() - test "AsyncEventQueue() one consumer limits test": - proc test() {.async.} = - let eventQueue = newAsyncEventQueue[int](4) - check len(eventQueue) == 0 - eventQueue.emit(100) - eventQueue.emit(200) - eventQueue.emit(300) - eventQueue.emit(400) - # There no consumers, so all the items should be discarded - check len(eventQueue) == 0 - let key1 = eventQueue.register() - check len(eventQueue) == 0 - eventQueue.emit(500) - eventQueue.emit(600) - eventQueue.emit(700) - eventQueue.emit(800) - # So exact `limit` number of items added, consumer should receive all of - # them. - check len(eventQueue) == 4 - let dataFut1 = eventQueue.waitEvents(key1) - check: - dataFut1.finished() == true - dataFut1.read() == @[500, 600, 700, 800] - len(eventQueue) == 0 - - eventQueue.emit(900) - eventQueue.emit(1000) - eventQueue.emit(1100) - eventQueue.emit(1200) - check len(eventQueue) == 4 - # Overfilling queue - eventQueue.emit(1300) - # Because overfill for single consumer happend, whole queue should become - # empty. - check len(eventQueue) == 0 - eventQueue.emit(1400) - eventQueue.emit(1500) - eventQueue.emit(1600) - eventQueue.emit(1700) - eventQueue.emit(1800) - check len(eventQueue) == 0 - let errorFut1 = eventQueue.waitEvents(key1) - check errorFut1.finished() == true - let checkException = - try: - let res {.used.} = await errorFut1 - false - except AsyncEventQueueFullError: - true - except CatchableError: - false - check checkException == true - # There should be no items because consumer was overflowed. - check len(eventQueue) == 0 - eventQueue.unregister(key1) - # All items should be garbage collected after unregister. - check len(eventQueue) == 0 - await eventQueue.closeWait() - - waitFor test() - - test "AsyncEventQueue() many consumers limits test": - proc test() {.async.} = - let eventQueue = newAsyncEventQueue[int](4) - block: - let key1 = eventQueue.register() - eventQueue.emit(100) - check len(eventQueue) == 1 - let key2 = eventQueue.register() - eventQueue.emit(200) - check len(eventQueue) == 2 - let key3 = eventQueue.register() - eventQueue.emit(300) - check len(eventQueue) == 3 - let key4 = eventQueue.register() - eventQueue.emit(400) - check len(eventQueue) == 4 - let key5 = eventQueue.register() - eventQueue.emit(500) - # At this point consumer with `key1` is overfilled, so after `emit()` - # queue length should be decreased by one item. - # So queue should look like this: [200, 300, 400, 500] - check len(eventQueue) == 4 - eventQueue.emit(600) - # At this point consumers with `key2` is overfilled, so after `emit()` - # queue length should be decreased by one item. - # So queue should look like this: [300, 400, 500, 600] - check len(eventQueue) == 4 - eventQueue.emit(700) - # At this point consumers with `key3` is overfilled, so after `emit()` - # queue length should be decreased by one item. - # So queue should look like this: [400, 500, 600, 700] - check len(eventQueue) == 4 - eventQueue.emit(800) - # At this point consumers with `key4` is overfilled, so after `emit()` - # queue length should be decreased by one item. - # So queue should look like this: [500, 600, 700, 800] - check len(eventQueue) == 4 - # Consumer with key5 is not overfilled. - let dataFut5 = eventQueue.waitEvents(key5) - check: - dataFut5.finished() == true - dataFut5.read() == @[500, 600, 700, 800] - # No more items should be left because all other consumers are overfilled. - check len(eventQueue) == 0 - eventQueue.unregister(key5) - check len(eventQueue) == 0 - - let dataFut2 = eventQueue.waitEvents(key2) - check dataFut2.finished() == true - expect AsyncEventQueueFullError: - let res {.used.} = dataFut2.read() - check len(eventQueue) == 0 - eventQueue.unregister(key2) - check len(eventQueue) == 0 - - let dataFut4 = eventQueue.waitEvents(key4) - check dataFut4.finished() == true - expect AsyncEventQueueFullError: - let res {.used.} = dataFut4.read() - check len(eventQueue) == 0 - eventQueue.unregister(key4) - check len(eventQueue) == 0 - - let dataFut3 = eventQueue.waitEvents(key3) - check dataFut3.finished() == true - expect AsyncEventQueueFullError: - let res {.used.} = dataFut3.read() - check len(eventQueue) == 0 - eventQueue.unregister(key3) - check len(eventQueue) == 0 - - let dataFut1 = eventQueue.waitEvents(key1) - check dataFut1.finished() == true - expect AsyncEventQueueFullError: - let res {.used.} = dataFut1.read() - check len(eventQueue) == 0 - eventQueue.unregister(key1) - check len(eventQueue) == 0 - - block: - let key1 = eventQueue.register() - eventQueue.emit(100) - check len(eventQueue) == 1 - let key2 = eventQueue.register() - eventQueue.emit(200) - check len(eventQueue) == 2 - let key3 = eventQueue.register() - eventQueue.emit(300) - check len(eventQueue) == 3 - let key4 = eventQueue.register() - eventQueue.emit(400) - check len(eventQueue) == 4 - let key5 = eventQueue.register() - eventQueue.emit(500) - # At this point consumer with `key1` is overfilled, so after `emit()` - # queue length should be decreased by one item. - # So queue should look like this: [200, 300, 400, 500] - check len(eventQueue) == 4 - eventQueue.emit(600) - # At this point consumer with `key2` is overfilled, so after `emit()` - # queue length should be decreased by one item. - # So queue should look like this: [300, 400, 500, 600] - check len(eventQueue) == 4 - eventQueue.emit(700) - # At this point consumer with `key3` is overfilled, so after `emit()` - # queue length should be decreased by one item. - # So queue should look like this: [400, 500, 600, 700] - check len(eventQueue) == 4 - eventQueue.emit(800) - # At this point consumer with `key4` is overfilled, so after `emit()` - # queue length should be decreased by one item. - # So queue should look like this: [500, 600, 700, 800] - check len(eventQueue) == 4 - eventQueue.emit(900) - # At this point all consumers are overfilled, so after `emit()` - # queue length should become 0. - check len(eventQueue) == 0 - eventQueue.emit(1000) - eventQueue.emit(1100) - eventQueue.emit(1200) - eventQueue.emit(1300) - eventQueue.emit(1400) - eventQueue.emit(1500) - eventQueue.emit(1600) - eventQueue.emit(1700) - eventQueue.emit(1800) - eventQueue.emit(1900) - # No more events should be accepted. - check len(eventQueue) == 0 - - let dataFut1 = eventQueue.waitEvents(key1) - check dataFut1.finished() == true - expect AsyncEventQueueFullError: - let res {.used.} = dataFut1.read() - check len(eventQueue) == 0 - eventQueue.unregister(key1) - check len(eventQueue) == 0 - - let dataFut2 = eventQueue.waitEvents(key2) - check dataFut2.finished() == true - expect AsyncEventQueueFullError: - let res {.used.} = dataFut2.read() - check len(eventQueue) == 0 - eventQueue.unregister(key2) - check len(eventQueue) == 0 - - let dataFut3 = eventQueue.waitEvents(key3) - check dataFut3.finished() == true - expect AsyncEventQueueFullError: - let res {.used.} = dataFut3.read() - check len(eventQueue) == 0 - eventQueue.unregister(key3) - check len(eventQueue) == 0 - - let dataFut4 = eventQueue.waitEvents(key4) - check dataFut4.finished() == true - expect AsyncEventQueueFullError: - let res {.used.} = dataFut4.read() - check len(eventQueue) == 0 - eventQueue.unregister(key4) - check len(eventQueue) == 0 - - let dataFut5 = eventQueue.waitEvents(key5) - check dataFut5.finished() == true - expect AsyncEventQueueFullError: - let res {.used.} = dataFut5.read() - check len(eventQueue) == 0 - eventQueue.unregister(key5) - check len(eventQueue) == 0 - await eventQueue.closeWait() - - waitFor test() - - test "AsyncEventQueue() slow and fast consumer test": - proc test() {.async.} = - let eventQueue = newAsyncEventQueue[int](1) - let - fastConsumer = eventQueue.register() - slowConsumer = eventQueue.register() - slowFut = eventQueue.waitEvents(slowConsumer) - - for i in 0 ..< 1000: - eventQueue.emit(i) - let fastData {.used.} = await eventQueue.waitEvents(fastConsumer) - - check len(eventQueue) == 0 - await allFutures(slowFut) - check len(eventQueue) == 0 - expect AsyncEventQueueFullError: - let res {.used.} = slowFut.read() - - check len(eventQueue) == 0 - eventQueue.unregister(fastConsumer) - check len(eventQueue) == 0 - eventQueue.unregister(slowConsumer) - check len(eventQueue) == 0 - await eventQueue.closeWait() - - waitFor test() + # test "AsyncEventQueue() one consumer limits test": + # proc test() {.async.} = + # let eventQueue = newAsyncEventQueue[int](4) + # check len(eventQueue) == 0 + # eventQueue.emit(100) + # eventQueue.emit(200) + # eventQueue.emit(300) + # eventQueue.emit(400) + # # There no consumers, so all the items should be discarded + # check len(eventQueue) == 0 + # let key1 = eventQueue.register() + # check len(eventQueue) == 0 + # eventQueue.emit(500) + # eventQueue.emit(600) + # eventQueue.emit(700) + # eventQueue.emit(800) + # # So exact `limit` number of items added, consumer should receive all of + # # them. + # check len(eventQueue) == 4 + # let dataFut1 = eventQueue.waitEvents(key1) + # check: + # dataFut1.finished() == true + # dataFut1.read() == @[500, 600, 700, 800] + # len(eventQueue) == 0 + + # eventQueue.emit(900) + # eventQueue.emit(1000) + # eventQueue.emit(1100) + # eventQueue.emit(1200) + # check len(eventQueue) == 4 + # # Overfilling queue + # eventQueue.emit(1300) + # # Because overfill for single consumer happend, whole queue should become + # # empty. + # check len(eventQueue) == 0 + # eventQueue.emit(1400) + # eventQueue.emit(1500) + # eventQueue.emit(1600) + # eventQueue.emit(1700) + # eventQueue.emit(1800) + # check len(eventQueue) == 0 + # let errorFut1 = eventQueue.waitEvents(key1) + # check errorFut1.finished() == true + # let checkException = + # try: + # let res {.used.} = await errorFut1 + # false + # except AsyncEventQueueFullError: + # true + # except CatchableError: + # false + # check checkException == true + # # There should be no items because consumer was overflowed. + # check len(eventQueue) == 0 + # eventQueue.unregister(key1) + # # All items should be garbage collected after unregister. + # check len(eventQueue) == 0 + # await eventQueue.closeWait() + + # waitFor test() + + # test "AsyncEventQueue() many consumers limits test": + # proc test() {.async.} = + # let eventQueue = newAsyncEventQueue[int](4) + # block: + # let key1 = eventQueue.register() + # eventQueue.emit(100) + # check len(eventQueue) == 1 + # let key2 = eventQueue.register() + # eventQueue.emit(200) + # check len(eventQueue) == 2 + # let key3 = eventQueue.register() + # eventQueue.emit(300) + # check len(eventQueue) == 3 + # let key4 = eventQueue.register() + # eventQueue.emit(400) + # check len(eventQueue) == 4 + # let key5 = eventQueue.register() + # eventQueue.emit(500) + # # At this point consumer with `key1` is overfilled, so after `emit()` + # # queue length should be decreased by one item. + # # So queue should look like this: [200, 300, 400, 500] + # check len(eventQueue) == 4 + # eventQueue.emit(600) + # # At this point consumers with `key2` is overfilled, so after `emit()` + # # queue length should be decreased by one item. + # # So queue should look like this: [300, 400, 500, 600] + # check len(eventQueue) == 4 + # eventQueue.emit(700) + # # At this point consumers with `key3` is overfilled, so after `emit()` + # # queue length should be decreased by one item. + # # So queue should look like this: [400, 500, 600, 700] + # check len(eventQueue) == 4 + # eventQueue.emit(800) + # # At this point consumers with `key4` is overfilled, so after `emit()` + # # queue length should be decreased by one item. + # # So queue should look like this: [500, 600, 700, 800] + # check len(eventQueue) == 4 + # # Consumer with key5 is not overfilled. + # let dataFut5 = eventQueue.waitEvents(key5) + # check: + # dataFut5.finished() == true + # dataFut5.read() == @[500, 600, 700, 800] + # # No more items should be left because all other consumers are overfilled. + # check len(eventQueue) == 0 + # eventQueue.unregister(key5) + # check len(eventQueue) == 0 + + # let dataFut2 = eventQueue.waitEvents(key2) + # check dataFut2.finished() == true + # expect AsyncEventQueueFullError: + # let res {.used.} = dataFut2.read() + # check len(eventQueue) == 0 + # eventQueue.unregister(key2) + # check len(eventQueue) == 0 + + # let dataFut4 = eventQueue.waitEvents(key4) + # check dataFut4.finished() == true + # expect AsyncEventQueueFullError: + # let res {.used.} = dataFut4.read() + # check len(eventQueue) == 0 + # eventQueue.unregister(key4) + # check len(eventQueue) == 0 + + # let dataFut3 = eventQueue.waitEvents(key3) + # check dataFut3.finished() == true + # expect AsyncEventQueueFullError: + # let res {.used.} = dataFut3.read() + # check len(eventQueue) == 0 + # eventQueue.unregister(key3) + # check len(eventQueue) == 0 + + # let dataFut1 = eventQueue.waitEvents(key1) + # check dataFut1.finished() == true + # expect AsyncEventQueueFullError: + # let res {.used.} = dataFut1.read() + # check len(eventQueue) == 0 + # eventQueue.unregister(key1) + # check len(eventQueue) == 0 + + # block: + # let key1 = eventQueue.register() + # eventQueue.emit(100) + # check len(eventQueue) == 1 + # let key2 = eventQueue.register() + # eventQueue.emit(200) + # check len(eventQueue) == 2 + # let key3 = eventQueue.register() + # eventQueue.emit(300) + # check len(eventQueue) == 3 + # let key4 = eventQueue.register() + # eventQueue.emit(400) + # check len(eventQueue) == 4 + # let key5 = eventQueue.register() + # eventQueue.emit(500) + # # At this point consumer with `key1` is overfilled, so after `emit()` + # # queue length should be decreased by one item. + # # So queue should look like this: [200, 300, 400, 500] + # check len(eventQueue) == 4 + # eventQueue.emit(600) + # # At this point consumer with `key2` is overfilled, so after `emit()` + # # queue length should be decreased by one item. + # # So queue should look like this: [300, 400, 500, 600] + # check len(eventQueue) == 4 + # eventQueue.emit(700) + # # At this point consumer with `key3` is overfilled, so after `emit()` + # # queue length should be decreased by one item. + # # So queue should look like this: [400, 500, 600, 700] + # check len(eventQueue) == 4 + # eventQueue.emit(800) + # # At this point consumer with `key4` is overfilled, so after `emit()` + # # queue length should be decreased by one item. + # # So queue should look like this: [500, 600, 700, 800] + # check len(eventQueue) == 4 + # eventQueue.emit(900) + # # At this point all consumers are overfilled, so after `emit()` + # # queue length should become 0. + # check len(eventQueue) == 0 + # eventQueue.emit(1000) + # eventQueue.emit(1100) + # eventQueue.emit(1200) + # eventQueue.emit(1300) + # eventQueue.emit(1400) + # eventQueue.emit(1500) + # eventQueue.emit(1600) + # eventQueue.emit(1700) + # eventQueue.emit(1800) + # eventQueue.emit(1900) + # # No more events should be accepted. + # check len(eventQueue) == 0 + + # let dataFut1 = eventQueue.waitEvents(key1) + # check dataFut1.finished() == true + # expect AsyncEventQueueFullError: + # let res {.used.} = dataFut1.read() + # check len(eventQueue) == 0 + # eventQueue.unregister(key1) + # check len(eventQueue) == 0 + + # let dataFut2 = eventQueue.waitEvents(key2) + # check dataFut2.finished() == true + # expect AsyncEventQueueFullError: + # let res {.used.} = dataFut2.read() + # check len(eventQueue) == 0 + # eventQueue.unregister(key2) + # check len(eventQueue) == 0 + + # let dataFut3 = eventQueue.waitEvents(key3) + # check dataFut3.finished() == true + # expect AsyncEventQueueFullError: + # let res {.used.} = dataFut3.read() + # check len(eventQueue) == 0 + # eventQueue.unregister(key3) + # check len(eventQueue) == 0 + + # let dataFut4 = eventQueue.waitEvents(key4) + # check dataFut4.finished() == true + # expect AsyncEventQueueFullError: + # let res {.used.} = dataFut4.read() + # check len(eventQueue) == 0 + # eventQueue.unregister(key4) + # check len(eventQueue) == 0 + + # let dataFut5 = eventQueue.waitEvents(key5) + # check dataFut5.finished() == true + # expect AsyncEventQueueFullError: + # let res {.used.} = dataFut5.read() + # check len(eventQueue) == 0 + # eventQueue.unregister(key5) + # check len(eventQueue) == 0 + # await eventQueue.closeWait() + + # waitFor test() + + # test "AsyncEventQueue() slow and fast consumer test": + # proc test() {.async.} = + # let eventQueue = newAsyncEventQueue[int](1) + # let + # fastConsumer = eventQueue.register() + # slowConsumer = eventQueue.register() + # slowFut = eventQueue.waitEvents(slowConsumer) + + # for i in 0 ..< 1000: + # eventQueue.emit(i) + # let fastData {.used.} = await eventQueue.waitEvents(fastConsumer) + + # check len(eventQueue) == 0 + # await allFutures(slowFut) + # check len(eventQueue) == 0 + # expect AsyncEventQueueFullError: + # let res {.used.} = slowFut.read() + + # check len(eventQueue) == 0 + # eventQueue.unregister(fastConsumer) + # check len(eventQueue) == 0 + # eventQueue.unregister(slowConsumer) + # check len(eventQueue) == 0 + # await eventQueue.closeWait() + + # waitFor test() From 70338e5028501073aa712bdc7a31d7234ec00342 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Fri, 10 Jun 2022 10:01:57 +0300 Subject: [PATCH 22/23] Disable AsyncEventQueue tests to confirm that this is an issue. --- tests/testsync.nim | 530 ++++++++++++++++++++++----------------------- 1 file changed, 265 insertions(+), 265 deletions(-) diff --git a/tests/testsync.nim b/tests/testsync.nim index c06e536de..a2441f388 100644 --- a/tests/testsync.nim +++ b/tests/testsync.nim @@ -353,275 +353,275 @@ suite "Asynchronous sync primitives test suite": test "AsyncQueue() contains test": check test9() == true - test "AsyncEventQueue() behavior test": - let eventQueue = newAsyncEventQueue[int]() - let key = eventQueue.register() - eventQueue.emit(100) - eventQueue.emit(200) - eventQueue.emit(300) - - proc test1() = - let dataFut = eventQueue.waitEvents(key) - check: - dataFut.finished() == true - dataFut.read() == @[100, 200, 300] - - proc test2() = - let dataFut = eventQueue.waitEvents(key) - check: - dataFut.finished() == false - eventQueue.emit(400) - eventQueue.emit(500) - poll() - check: - dataFut.finished() == true - dataFut.read() == @[400, 500] - - test1() - test2() - waitFor eventQueue.closeWait() - - test "AsyncEventQueue() concurrency test": - let eventQueue = newAsyncEventQueue[int]() - let key0 = eventQueue.register() - let key1 = eventQueue.register() - eventQueue.emit(100) - let key2 = eventQueue.register() - eventQueue.emit(200) - eventQueue.emit(300) - let key3 = eventQueue.register() - eventQueue.emit(400) - eventQueue.emit(500) - eventQueue.emit(600) - let key4 = eventQueue.register() - eventQueue.emit(700) - eventQueue.emit(800) - eventQueue.emit(900) - eventQueue.emit(1000) - let key5 = eventQueue.register() - let key6 = eventQueue.register() - - let dataFut1 = eventQueue.waitEvents(key1) - let dataFut2 = eventQueue.waitEvents(key2) - let dataFut3 = eventQueue.waitEvents(key3) - let dataFut4 = eventQueue.waitEvents(key4) - let dataFut5 = eventQueue.waitEvents(key5) - let dataFut6 = eventQueue.waitEvents(key6) - check: - dataFut1.finished() == true - dataFut1.read() == @[100, 200, 300, 400, 500, 600, 700, 800, 900, 1000] - dataFut2.finished() == true - dataFut2.read() == @[200, 300, 400, 500, 600, 700, 800, 900, 1000] - dataFut3.finished() == true - dataFut3.read() == @[400, 500, 600, 700, 800, 900, 1000] - dataFut4.finished() == true - dataFut4.read() == @[700, 800, 900, 1000] - dataFut5.finished() == false - dataFut6.finished() == false - - eventQueue.emit(2000) - poll() - let dataFut0 = eventQueue.waitEvents(key0) - check: - dataFut5.finished() == true - dataFut5.read() == @[2000] - dataFut6.finished() == true - dataFut6.read() == @[2000] - dataFut0.finished() == true - dataFut0.read() == @[100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, - 2000] - - waitFor eventQueue.closeWait() - - test "AsyncEventQueue() specific number test": - let eventQueue = newAsyncEventQueue[int]() - let key = eventQueue.register() - - let dataFut1 = eventQueue.waitEvents(key, 1) - eventQueue.emit(100) - eventQueue.emit(200) - eventQueue.emit(300) - eventQueue.emit(400) - check dataFut1.finished() == false - poll() - check: - dataFut1.finished() == true - dataFut1.read() == @[100] - - let dataFut2 = eventQueue.waitEvents(key, 2) - check: - dataFut2.finished() == true - dataFut2.read() == @[200, 300] - - let dataFut3 = eventQueue.waitEvents(key, 5) - check dataFut3.finished() == false - eventQueue.emit(500) - eventQueue.emit(600) - eventQueue.emit(700) - eventQueue.emit(800) - check dataFut3.finished() == false - poll() - check: - dataFut3.finished() == true - dataFut3.read() == @[400, 500, 600, 700, 800] - - let dataFut4 = eventQueue.waitEvents(key, -1) - check dataFut4.finished() == false - eventQueue.emit(900) - eventQueue.emit(1000) - eventQueue.emit(1100) - eventQueue.emit(1200) - eventQueue.emit(1300) - eventQueue.emit(1400) - eventQueue.emit(1500) - eventQueue.emit(1600) - check dataFut4.finished() == false - poll() - check: - dataFut4.finished() == true - dataFut4.read() == @[900, 1000, 1100, 1200, 1300, 1400, 1500, 1600] - - waitFor eventQueue.closeWait() + # test "AsyncEventQueue() behavior test": + # let eventQueue = newAsyncEventQueue[int]() + # let key = eventQueue.register() + # eventQueue.emit(100) + # eventQueue.emit(200) + # eventQueue.emit(300) + + # proc test1() = + # let dataFut = eventQueue.waitEvents(key) + # check: + # dataFut.finished() == true + # dataFut.read() == @[100, 200, 300] - test "AsyncEventQueue() register()/unregister() test": - var emptySeq: seq[int] - let eventQueue = newAsyncEventQueue[int]() - let key1 = eventQueue.register() + # proc test2() = + # let dataFut = eventQueue.waitEvents(key) + # check: + # dataFut.finished() == false + # eventQueue.emit(400) + # eventQueue.emit(500) + # poll() + # check: + # dataFut.finished() == true + # dataFut.read() == @[400, 500] + + # test1() + # test2() + # waitFor eventQueue.closeWait() + + # test "AsyncEventQueue() concurrency test": + # let eventQueue = newAsyncEventQueue[int]() + # let key0 = eventQueue.register() + # let key1 = eventQueue.register() + # eventQueue.emit(100) + # let key2 = eventQueue.register() + # eventQueue.emit(200) + # eventQueue.emit(300) + # let key3 = eventQueue.register() + # eventQueue.emit(400) + # eventQueue.emit(500) + # eventQueue.emit(600) + # let key4 = eventQueue.register() + # eventQueue.emit(700) + # eventQueue.emit(800) + # eventQueue.emit(900) + # eventQueue.emit(1000) + # let key5 = eventQueue.register() + # let key6 = eventQueue.register() + + # let dataFut1 = eventQueue.waitEvents(key1) + # let dataFut2 = eventQueue.waitEvents(key2) + # let dataFut3 = eventQueue.waitEvents(key3) + # let dataFut4 = eventQueue.waitEvents(key4) + # let dataFut5 = eventQueue.waitEvents(key5) + # let dataFut6 = eventQueue.waitEvents(key6) + # check: + # dataFut1.finished() == true + # dataFut1.read() == @[100, 200, 300, 400, 500, 600, 700, 800, 900, 1000] + # dataFut2.finished() == true + # dataFut2.read() == @[200, 300, 400, 500, 600, 700, 800, 900, 1000] + # dataFut3.finished() == true + # dataFut3.read() == @[400, 500, 600, 700, 800, 900, 1000] + # dataFut4.finished() == true + # dataFut4.read() == @[700, 800, 900, 1000] + # dataFut5.finished() == false + # dataFut6.finished() == false + + # eventQueue.emit(2000) + # poll() + # let dataFut0 = eventQueue.waitEvents(key0) + # check: + # dataFut5.finished() == true + # dataFut5.read() == @[2000] + # dataFut6.finished() == true + # dataFut6.read() == @[2000] + # dataFut0.finished() == true + # dataFut0.read() == @[100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, + # 2000] + + # waitFor eventQueue.closeWait() + + # test "AsyncEventQueue() specific number test": + # let eventQueue = newAsyncEventQueue[int]() + # let key = eventQueue.register() + + # let dataFut1 = eventQueue.waitEvents(key, 1) + # eventQueue.emit(100) + # eventQueue.emit(200) + # eventQueue.emit(300) + # eventQueue.emit(400) + # check dataFut1.finished() == false + # poll() + # check: + # dataFut1.finished() == true + # dataFut1.read() == @[100] + + # let dataFut2 = eventQueue.waitEvents(key, 2) + # check: + # dataFut2.finished() == true + # dataFut2.read() == @[200, 300] + + # let dataFut3 = eventQueue.waitEvents(key, 5) + # check dataFut3.finished() == false + # eventQueue.emit(500) + # eventQueue.emit(600) + # eventQueue.emit(700) + # eventQueue.emit(800) + # check dataFut3.finished() == false + # poll() + # check: + # dataFut3.finished() == true + # dataFut3.read() == @[400, 500, 600, 700, 800] + + # let dataFut4 = eventQueue.waitEvents(key, -1) + # check dataFut4.finished() == false + # eventQueue.emit(900) + # eventQueue.emit(1000) + # eventQueue.emit(1100) + # eventQueue.emit(1200) + # eventQueue.emit(1300) + # eventQueue.emit(1400) + # eventQueue.emit(1500) + # eventQueue.emit(1600) + # check dataFut4.finished() == false + # poll() + # check: + # dataFut4.finished() == true + # dataFut4.read() == @[900, 1000, 1100, 1200, 1300, 1400, 1500, 1600] + + # waitFor eventQueue.closeWait() + + # test "AsyncEventQueue() register()/unregister() test": + # var emptySeq: seq[int] + # let eventQueue = newAsyncEventQueue[int]() + # let key1 = eventQueue.register() + + # let dataFut1 = eventQueue.waitEvents(key1, 1) + # check dataFut1.finished() == false + # eventQueue.unregister(key1) + # check dataFut1.finished() == false + # poll() + # check: + # dataFut1.finished() == true + # dataFut1.read() == emptySeq + + # let key2 = eventQueue.register() + # let dataFut2 = eventQueue.waitEvents(key2, 5) + # check dataFut2.finished() == false + # eventQueue.emit(100) + # eventQueue.emit(200) + # eventQueue.emit(300) + # eventQueue.emit(400) + # eventQueue.emit(500) + # check dataFut2.finished() == false + # eventQueue.unregister(key2) + # poll() + # check: + # dataFut2.finished() == true + # dataFut2.read() == emptySeq + + # let key3 = eventQueue.register() + # let dataFut3 = eventQueue.waitEvents(key3, 5) + # check dataFut3.finished() == false + # eventQueue.emit(100) + # eventQueue.emit(200) + # eventQueue.emit(300) + # check dataFut3.finished() == false + # poll() + # eventQueue.unregister(key3) + # eventQueue.emit(400) + # check dataFut3.finished() == false + # poll() + # check: + # dataFut3.finished() == true + # dataFut3.read() == @[100, 200, 300] + + # waitFor eventQueue.closeWait() + + # test "AsyncEventQueue() garbage collection test": + # let eventQueue = newAsyncEventQueue[int]() + # let key1 = eventQueue.register() + # check len(eventQueue) == 0 + # eventQueue.emit(100) + # eventQueue.emit(200) + # eventQueue.emit(300) + # check len(eventQueue) == 3 + # let key2 = eventQueue.register() + # eventQueue.emit(400) + # eventQueue.emit(500) + # eventQueue.emit(600) + # eventQueue.emit(700) + # check len(eventQueue) == 7 + # let key3 = eventQueue.register() + # eventQueue.emit(800) + # eventQueue.emit(900) + # eventQueue.emit(1000) + # eventQueue.emit(1100) + # eventQueue.emit(1200) + # check len(eventQueue) == 12 + # let dataFut1 = eventQueue.waitEvents(key1) + # check: + # dataFut1.finished() == true + # dataFut1.read() == @[100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, + # 1100, 1200] + # len(eventQueue) == 9 + + # let dataFut3 = eventQueue.waitEvents(key3) + # check: + # dataFut3.finished() == true + # dataFut3.read() == @[800, 900, 1000, 1100, 1200] + # len(eventQueue) == 9 + + # let dataFut2 = eventQueue.waitEvents(key2) + # check: + # dataFut2.finished() == true + # dataFut2.read() == @[400, 500, 600, 700, 800, 900, 1000, 1100, 1200] + # len(eventQueue) == 0 + + # waitFor eventQueue.closeWait() + + # test "AsyncEventQueue() 1,000,000 of events to 10 clients test": + # proc test() {.async.} = + # let eventQueue = newAsyncEventQueue[int]() + # var keys = @[ + # eventQueue.register(), eventQueue.register(), + # eventQueue.register(), eventQueue.register(), + # eventQueue.register(), eventQueue.register(), + # eventQueue.register(), eventQueue.register(), + # eventQueue.register(), eventQueue.register() + # ] + + # proc clientTask(queue: AsyncEventQueue[int], + # key: EventQueueKey): Future[seq[int]] {.async.} = + # var events: seq[int] + # while true: + # let res = await queue.waitEvents(key) + # if len(res) == 0: + # break + # events.add(res) + # queue.unregister(key) + # return events + + # var futs = @[ + # clientTask(eventQueue, keys[0]), clientTask(eventQueue, keys[1]), + # clientTask(eventQueue, keys[2]), clientTask(eventQueue, keys[3]), + # clientTask(eventQueue, keys[4]), clientTask(eventQueue, keys[5]), + # clientTask(eventQueue, keys[6]), clientTask(eventQueue, keys[7]), + # clientTask(eventQueue, keys[8]), clientTask(eventQueue, keys[9]) + # ] + + # for i in 1 .. 1_000_000: + # if (i mod 1000) == 0: + # # Give some CPU for clients. + # await sleepAsync(0.milliseconds) + # eventQueue.emit(i) - let dataFut1 = eventQueue.waitEvents(key1, 1) - check dataFut1.finished() == false - eventQueue.unregister(key1) - check dataFut1.finished() == false - poll() - check: - dataFut1.finished() == true - dataFut1.read() == emptySeq - - let key2 = eventQueue.register() - let dataFut2 = eventQueue.waitEvents(key2, 5) - check dataFut2.finished() == false - eventQueue.emit(100) - eventQueue.emit(200) - eventQueue.emit(300) - eventQueue.emit(400) - eventQueue.emit(500) - check dataFut2.finished() == false - eventQueue.unregister(key2) - poll() - check: - dataFut2.finished() == true - dataFut2.read() == emptySeq - - let key3 = eventQueue.register() - let dataFut3 = eventQueue.waitEvents(key3, 5) - check dataFut3.finished() == false - eventQueue.emit(100) - eventQueue.emit(200) - eventQueue.emit(300) - check dataFut3.finished() == false - poll() - eventQueue.unregister(key3) - eventQueue.emit(400) - check dataFut3.finished() == false - poll() - check: - dataFut3.finished() == true - dataFut3.read() == @[100, 200, 300] - - waitFor eventQueue.closeWait() - - test "AsyncEventQueue() garbage collection test": - let eventQueue = newAsyncEventQueue[int]() - let key1 = eventQueue.register() - check len(eventQueue) == 0 - eventQueue.emit(100) - eventQueue.emit(200) - eventQueue.emit(300) - check len(eventQueue) == 3 - let key2 = eventQueue.register() - eventQueue.emit(400) - eventQueue.emit(500) - eventQueue.emit(600) - eventQueue.emit(700) - check len(eventQueue) == 7 - let key3 = eventQueue.register() - eventQueue.emit(800) - eventQueue.emit(900) - eventQueue.emit(1000) - eventQueue.emit(1100) - eventQueue.emit(1200) - check len(eventQueue) == 12 - let dataFut1 = eventQueue.waitEvents(key1) - check: - dataFut1.finished() == true - dataFut1.read() == @[100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, - 1100, 1200] - len(eventQueue) == 9 + # await eventQueue.closeWait() - let dataFut3 = eventQueue.waitEvents(key3) - check: - dataFut3.finished() == true - dataFut3.read() == @[800, 900, 1000, 1100, 1200] - len(eventQueue) == 9 + # await allFutures(futs) + # for index in 0 ..< len(futs): + # let fut = futs[index] + # check fut.finished() == true + # let data = fut.read() + # var counter = 1 + # for item in data: + # check item == counter + # inc(counter) + # futs[index] = nil - let dataFut2 = eventQueue.waitEvents(key2) - check: - dataFut2.finished() == true - dataFut2.read() == @[400, 500, 600, 700, 800, 900, 1000, 1100, 1200] - len(eventQueue) == 0 - - waitFor eventQueue.closeWait() - - test "AsyncEventQueue() 1,000,000 of events to 10 clients test": - proc test() {.async.} = - let eventQueue = newAsyncEventQueue[int]() - var keys = @[ - eventQueue.register(), eventQueue.register(), - eventQueue.register(), eventQueue.register(), - eventQueue.register(), eventQueue.register(), - eventQueue.register(), eventQueue.register(), - eventQueue.register(), eventQueue.register() - ] - - proc clientTask(queue: AsyncEventQueue[int], - key: EventQueueKey): Future[seq[int]] {.async.} = - var events: seq[int] - while true: - let res = await queue.waitEvents(key) - if len(res) == 0: - break - events.add(res) - queue.unregister(key) - return events - - var futs = @[ - clientTask(eventQueue, keys[0]), clientTask(eventQueue, keys[1]), - clientTask(eventQueue, keys[2]), clientTask(eventQueue, keys[3]), - clientTask(eventQueue, keys[4]), clientTask(eventQueue, keys[5]), - clientTask(eventQueue, keys[6]), clientTask(eventQueue, keys[7]), - clientTask(eventQueue, keys[8]), clientTask(eventQueue, keys[9]) - ] - - for i in 1 .. 1_000_000: - if (i mod 1000) == 0: - # Give some CPU for clients. - await sleepAsync(0.milliseconds) - eventQueue.emit(i) - - await eventQueue.closeWait() - - await allFutures(futs) - for index in 0 ..< len(futs): - let fut = futs[index] - check fut.finished() == true - let data = fut.read() - var counter = 1 - for item in data: - check item == counter - inc(counter) - futs[index] = nil - - waitFor test() + # waitFor test() # test "AsyncEventQueue() one consumer limits test": # proc test() {.async.} = From 88d8477fb9dd8498efdbf1e5979c9d30999abdd0 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Wed, 15 Jun 2022 12:52:42 +0300 Subject: [PATCH 23/23] Enable all the tests. --- tests/testsync.nim | 1060 ++++++++++++++++++++++---------------------- 1 file changed, 530 insertions(+), 530 deletions(-) diff --git a/tests/testsync.nim b/tests/testsync.nim index a2441f388..9acea50d5 100644 --- a/tests/testsync.nim +++ b/tests/testsync.nim @@ -353,533 +353,533 @@ suite "Asynchronous sync primitives test suite": test "AsyncQueue() contains test": check test9() == true - # test "AsyncEventQueue() behavior test": - # let eventQueue = newAsyncEventQueue[int]() - # let key = eventQueue.register() - # eventQueue.emit(100) - # eventQueue.emit(200) - # eventQueue.emit(300) - - # proc test1() = - # let dataFut = eventQueue.waitEvents(key) - # check: - # dataFut.finished() == true - # dataFut.read() == @[100, 200, 300] - - # proc test2() = - # let dataFut = eventQueue.waitEvents(key) - # check: - # dataFut.finished() == false - # eventQueue.emit(400) - # eventQueue.emit(500) - # poll() - # check: - # dataFut.finished() == true - # dataFut.read() == @[400, 500] - - # test1() - # test2() - # waitFor eventQueue.closeWait() - - # test "AsyncEventQueue() concurrency test": - # let eventQueue = newAsyncEventQueue[int]() - # let key0 = eventQueue.register() - # let key1 = eventQueue.register() - # eventQueue.emit(100) - # let key2 = eventQueue.register() - # eventQueue.emit(200) - # eventQueue.emit(300) - # let key3 = eventQueue.register() - # eventQueue.emit(400) - # eventQueue.emit(500) - # eventQueue.emit(600) - # let key4 = eventQueue.register() - # eventQueue.emit(700) - # eventQueue.emit(800) - # eventQueue.emit(900) - # eventQueue.emit(1000) - # let key5 = eventQueue.register() - # let key6 = eventQueue.register() - - # let dataFut1 = eventQueue.waitEvents(key1) - # let dataFut2 = eventQueue.waitEvents(key2) - # let dataFut3 = eventQueue.waitEvents(key3) - # let dataFut4 = eventQueue.waitEvents(key4) - # let dataFut5 = eventQueue.waitEvents(key5) - # let dataFut6 = eventQueue.waitEvents(key6) - # check: - # dataFut1.finished() == true - # dataFut1.read() == @[100, 200, 300, 400, 500, 600, 700, 800, 900, 1000] - # dataFut2.finished() == true - # dataFut2.read() == @[200, 300, 400, 500, 600, 700, 800, 900, 1000] - # dataFut3.finished() == true - # dataFut3.read() == @[400, 500, 600, 700, 800, 900, 1000] - # dataFut4.finished() == true - # dataFut4.read() == @[700, 800, 900, 1000] - # dataFut5.finished() == false - # dataFut6.finished() == false - - # eventQueue.emit(2000) - # poll() - # let dataFut0 = eventQueue.waitEvents(key0) - # check: - # dataFut5.finished() == true - # dataFut5.read() == @[2000] - # dataFut6.finished() == true - # dataFut6.read() == @[2000] - # dataFut0.finished() == true - # dataFut0.read() == @[100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, - # 2000] - - # waitFor eventQueue.closeWait() - - # test "AsyncEventQueue() specific number test": - # let eventQueue = newAsyncEventQueue[int]() - # let key = eventQueue.register() - - # let dataFut1 = eventQueue.waitEvents(key, 1) - # eventQueue.emit(100) - # eventQueue.emit(200) - # eventQueue.emit(300) - # eventQueue.emit(400) - # check dataFut1.finished() == false - # poll() - # check: - # dataFut1.finished() == true - # dataFut1.read() == @[100] - - # let dataFut2 = eventQueue.waitEvents(key, 2) - # check: - # dataFut2.finished() == true - # dataFut2.read() == @[200, 300] - - # let dataFut3 = eventQueue.waitEvents(key, 5) - # check dataFut3.finished() == false - # eventQueue.emit(500) - # eventQueue.emit(600) - # eventQueue.emit(700) - # eventQueue.emit(800) - # check dataFut3.finished() == false - # poll() - # check: - # dataFut3.finished() == true - # dataFut3.read() == @[400, 500, 600, 700, 800] - - # let dataFut4 = eventQueue.waitEvents(key, -1) - # check dataFut4.finished() == false - # eventQueue.emit(900) - # eventQueue.emit(1000) - # eventQueue.emit(1100) - # eventQueue.emit(1200) - # eventQueue.emit(1300) - # eventQueue.emit(1400) - # eventQueue.emit(1500) - # eventQueue.emit(1600) - # check dataFut4.finished() == false - # poll() - # check: - # dataFut4.finished() == true - # dataFut4.read() == @[900, 1000, 1100, 1200, 1300, 1400, 1500, 1600] - - # waitFor eventQueue.closeWait() - - # test "AsyncEventQueue() register()/unregister() test": - # var emptySeq: seq[int] - # let eventQueue = newAsyncEventQueue[int]() - # let key1 = eventQueue.register() - - # let dataFut1 = eventQueue.waitEvents(key1, 1) - # check dataFut1.finished() == false - # eventQueue.unregister(key1) - # check dataFut1.finished() == false - # poll() - # check: - # dataFut1.finished() == true - # dataFut1.read() == emptySeq - - # let key2 = eventQueue.register() - # let dataFut2 = eventQueue.waitEvents(key2, 5) - # check dataFut2.finished() == false - # eventQueue.emit(100) - # eventQueue.emit(200) - # eventQueue.emit(300) - # eventQueue.emit(400) - # eventQueue.emit(500) - # check dataFut2.finished() == false - # eventQueue.unregister(key2) - # poll() - # check: - # dataFut2.finished() == true - # dataFut2.read() == emptySeq - - # let key3 = eventQueue.register() - # let dataFut3 = eventQueue.waitEvents(key3, 5) - # check dataFut3.finished() == false - # eventQueue.emit(100) - # eventQueue.emit(200) - # eventQueue.emit(300) - # check dataFut3.finished() == false - # poll() - # eventQueue.unregister(key3) - # eventQueue.emit(400) - # check dataFut3.finished() == false - # poll() - # check: - # dataFut3.finished() == true - # dataFut3.read() == @[100, 200, 300] - - # waitFor eventQueue.closeWait() - - # test "AsyncEventQueue() garbage collection test": - # let eventQueue = newAsyncEventQueue[int]() - # let key1 = eventQueue.register() - # check len(eventQueue) == 0 - # eventQueue.emit(100) - # eventQueue.emit(200) - # eventQueue.emit(300) - # check len(eventQueue) == 3 - # let key2 = eventQueue.register() - # eventQueue.emit(400) - # eventQueue.emit(500) - # eventQueue.emit(600) - # eventQueue.emit(700) - # check len(eventQueue) == 7 - # let key3 = eventQueue.register() - # eventQueue.emit(800) - # eventQueue.emit(900) - # eventQueue.emit(1000) - # eventQueue.emit(1100) - # eventQueue.emit(1200) - # check len(eventQueue) == 12 - # let dataFut1 = eventQueue.waitEvents(key1) - # check: - # dataFut1.finished() == true - # dataFut1.read() == @[100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, - # 1100, 1200] - # len(eventQueue) == 9 - - # let dataFut3 = eventQueue.waitEvents(key3) - # check: - # dataFut3.finished() == true - # dataFut3.read() == @[800, 900, 1000, 1100, 1200] - # len(eventQueue) == 9 - - # let dataFut2 = eventQueue.waitEvents(key2) - # check: - # dataFut2.finished() == true - # dataFut2.read() == @[400, 500, 600, 700, 800, 900, 1000, 1100, 1200] - # len(eventQueue) == 0 - - # waitFor eventQueue.closeWait() - - # test "AsyncEventQueue() 1,000,000 of events to 10 clients test": - # proc test() {.async.} = - # let eventQueue = newAsyncEventQueue[int]() - # var keys = @[ - # eventQueue.register(), eventQueue.register(), - # eventQueue.register(), eventQueue.register(), - # eventQueue.register(), eventQueue.register(), - # eventQueue.register(), eventQueue.register(), - # eventQueue.register(), eventQueue.register() - # ] - - # proc clientTask(queue: AsyncEventQueue[int], - # key: EventQueueKey): Future[seq[int]] {.async.} = - # var events: seq[int] - # while true: - # let res = await queue.waitEvents(key) - # if len(res) == 0: - # break - # events.add(res) - # queue.unregister(key) - # return events - - # var futs = @[ - # clientTask(eventQueue, keys[0]), clientTask(eventQueue, keys[1]), - # clientTask(eventQueue, keys[2]), clientTask(eventQueue, keys[3]), - # clientTask(eventQueue, keys[4]), clientTask(eventQueue, keys[5]), - # clientTask(eventQueue, keys[6]), clientTask(eventQueue, keys[7]), - # clientTask(eventQueue, keys[8]), clientTask(eventQueue, keys[9]) - # ] - - # for i in 1 .. 1_000_000: - # if (i mod 1000) == 0: - # # Give some CPU for clients. - # await sleepAsync(0.milliseconds) - # eventQueue.emit(i) - - # await eventQueue.closeWait() - - # await allFutures(futs) - # for index in 0 ..< len(futs): - # let fut = futs[index] - # check fut.finished() == true - # let data = fut.read() - # var counter = 1 - # for item in data: - # check item == counter - # inc(counter) - # futs[index] = nil - - # waitFor test() - - # test "AsyncEventQueue() one consumer limits test": - # proc test() {.async.} = - # let eventQueue = newAsyncEventQueue[int](4) - # check len(eventQueue) == 0 - # eventQueue.emit(100) - # eventQueue.emit(200) - # eventQueue.emit(300) - # eventQueue.emit(400) - # # There no consumers, so all the items should be discarded - # check len(eventQueue) == 0 - # let key1 = eventQueue.register() - # check len(eventQueue) == 0 - # eventQueue.emit(500) - # eventQueue.emit(600) - # eventQueue.emit(700) - # eventQueue.emit(800) - # # So exact `limit` number of items added, consumer should receive all of - # # them. - # check len(eventQueue) == 4 - # let dataFut1 = eventQueue.waitEvents(key1) - # check: - # dataFut1.finished() == true - # dataFut1.read() == @[500, 600, 700, 800] - # len(eventQueue) == 0 - - # eventQueue.emit(900) - # eventQueue.emit(1000) - # eventQueue.emit(1100) - # eventQueue.emit(1200) - # check len(eventQueue) == 4 - # # Overfilling queue - # eventQueue.emit(1300) - # # Because overfill for single consumer happend, whole queue should become - # # empty. - # check len(eventQueue) == 0 - # eventQueue.emit(1400) - # eventQueue.emit(1500) - # eventQueue.emit(1600) - # eventQueue.emit(1700) - # eventQueue.emit(1800) - # check len(eventQueue) == 0 - # let errorFut1 = eventQueue.waitEvents(key1) - # check errorFut1.finished() == true - # let checkException = - # try: - # let res {.used.} = await errorFut1 - # false - # except AsyncEventQueueFullError: - # true - # except CatchableError: - # false - # check checkException == true - # # There should be no items because consumer was overflowed. - # check len(eventQueue) == 0 - # eventQueue.unregister(key1) - # # All items should be garbage collected after unregister. - # check len(eventQueue) == 0 - # await eventQueue.closeWait() - - # waitFor test() - - # test "AsyncEventQueue() many consumers limits test": - # proc test() {.async.} = - # let eventQueue = newAsyncEventQueue[int](4) - # block: - # let key1 = eventQueue.register() - # eventQueue.emit(100) - # check len(eventQueue) == 1 - # let key2 = eventQueue.register() - # eventQueue.emit(200) - # check len(eventQueue) == 2 - # let key3 = eventQueue.register() - # eventQueue.emit(300) - # check len(eventQueue) == 3 - # let key4 = eventQueue.register() - # eventQueue.emit(400) - # check len(eventQueue) == 4 - # let key5 = eventQueue.register() - # eventQueue.emit(500) - # # At this point consumer with `key1` is overfilled, so after `emit()` - # # queue length should be decreased by one item. - # # So queue should look like this: [200, 300, 400, 500] - # check len(eventQueue) == 4 - # eventQueue.emit(600) - # # At this point consumers with `key2` is overfilled, so after `emit()` - # # queue length should be decreased by one item. - # # So queue should look like this: [300, 400, 500, 600] - # check len(eventQueue) == 4 - # eventQueue.emit(700) - # # At this point consumers with `key3` is overfilled, so after `emit()` - # # queue length should be decreased by one item. - # # So queue should look like this: [400, 500, 600, 700] - # check len(eventQueue) == 4 - # eventQueue.emit(800) - # # At this point consumers with `key4` is overfilled, so after `emit()` - # # queue length should be decreased by one item. - # # So queue should look like this: [500, 600, 700, 800] - # check len(eventQueue) == 4 - # # Consumer with key5 is not overfilled. - # let dataFut5 = eventQueue.waitEvents(key5) - # check: - # dataFut5.finished() == true - # dataFut5.read() == @[500, 600, 700, 800] - # # No more items should be left because all other consumers are overfilled. - # check len(eventQueue) == 0 - # eventQueue.unregister(key5) - # check len(eventQueue) == 0 - - # let dataFut2 = eventQueue.waitEvents(key2) - # check dataFut2.finished() == true - # expect AsyncEventQueueFullError: - # let res {.used.} = dataFut2.read() - # check len(eventQueue) == 0 - # eventQueue.unregister(key2) - # check len(eventQueue) == 0 - - # let dataFut4 = eventQueue.waitEvents(key4) - # check dataFut4.finished() == true - # expect AsyncEventQueueFullError: - # let res {.used.} = dataFut4.read() - # check len(eventQueue) == 0 - # eventQueue.unregister(key4) - # check len(eventQueue) == 0 - - # let dataFut3 = eventQueue.waitEvents(key3) - # check dataFut3.finished() == true - # expect AsyncEventQueueFullError: - # let res {.used.} = dataFut3.read() - # check len(eventQueue) == 0 - # eventQueue.unregister(key3) - # check len(eventQueue) == 0 - - # let dataFut1 = eventQueue.waitEvents(key1) - # check dataFut1.finished() == true - # expect AsyncEventQueueFullError: - # let res {.used.} = dataFut1.read() - # check len(eventQueue) == 0 - # eventQueue.unregister(key1) - # check len(eventQueue) == 0 - - # block: - # let key1 = eventQueue.register() - # eventQueue.emit(100) - # check len(eventQueue) == 1 - # let key2 = eventQueue.register() - # eventQueue.emit(200) - # check len(eventQueue) == 2 - # let key3 = eventQueue.register() - # eventQueue.emit(300) - # check len(eventQueue) == 3 - # let key4 = eventQueue.register() - # eventQueue.emit(400) - # check len(eventQueue) == 4 - # let key5 = eventQueue.register() - # eventQueue.emit(500) - # # At this point consumer with `key1` is overfilled, so after `emit()` - # # queue length should be decreased by one item. - # # So queue should look like this: [200, 300, 400, 500] - # check len(eventQueue) == 4 - # eventQueue.emit(600) - # # At this point consumer with `key2` is overfilled, so after `emit()` - # # queue length should be decreased by one item. - # # So queue should look like this: [300, 400, 500, 600] - # check len(eventQueue) == 4 - # eventQueue.emit(700) - # # At this point consumer with `key3` is overfilled, so after `emit()` - # # queue length should be decreased by one item. - # # So queue should look like this: [400, 500, 600, 700] - # check len(eventQueue) == 4 - # eventQueue.emit(800) - # # At this point consumer with `key4` is overfilled, so after `emit()` - # # queue length should be decreased by one item. - # # So queue should look like this: [500, 600, 700, 800] - # check len(eventQueue) == 4 - # eventQueue.emit(900) - # # At this point all consumers are overfilled, so after `emit()` - # # queue length should become 0. - # check len(eventQueue) == 0 - # eventQueue.emit(1000) - # eventQueue.emit(1100) - # eventQueue.emit(1200) - # eventQueue.emit(1300) - # eventQueue.emit(1400) - # eventQueue.emit(1500) - # eventQueue.emit(1600) - # eventQueue.emit(1700) - # eventQueue.emit(1800) - # eventQueue.emit(1900) - # # No more events should be accepted. - # check len(eventQueue) == 0 - - # let dataFut1 = eventQueue.waitEvents(key1) - # check dataFut1.finished() == true - # expect AsyncEventQueueFullError: - # let res {.used.} = dataFut1.read() - # check len(eventQueue) == 0 - # eventQueue.unregister(key1) - # check len(eventQueue) == 0 - - # let dataFut2 = eventQueue.waitEvents(key2) - # check dataFut2.finished() == true - # expect AsyncEventQueueFullError: - # let res {.used.} = dataFut2.read() - # check len(eventQueue) == 0 - # eventQueue.unregister(key2) - # check len(eventQueue) == 0 - - # let dataFut3 = eventQueue.waitEvents(key3) - # check dataFut3.finished() == true - # expect AsyncEventQueueFullError: - # let res {.used.} = dataFut3.read() - # check len(eventQueue) == 0 - # eventQueue.unregister(key3) - # check len(eventQueue) == 0 - - # let dataFut4 = eventQueue.waitEvents(key4) - # check dataFut4.finished() == true - # expect AsyncEventQueueFullError: - # let res {.used.} = dataFut4.read() - # check len(eventQueue) == 0 - # eventQueue.unregister(key4) - # check len(eventQueue) == 0 - - # let dataFut5 = eventQueue.waitEvents(key5) - # check dataFut5.finished() == true - # expect AsyncEventQueueFullError: - # let res {.used.} = dataFut5.read() - # check len(eventQueue) == 0 - # eventQueue.unregister(key5) - # check len(eventQueue) == 0 - # await eventQueue.closeWait() - - # waitFor test() - - # test "AsyncEventQueue() slow and fast consumer test": - # proc test() {.async.} = - # let eventQueue = newAsyncEventQueue[int](1) - # let - # fastConsumer = eventQueue.register() - # slowConsumer = eventQueue.register() - # slowFut = eventQueue.waitEvents(slowConsumer) - - # for i in 0 ..< 1000: - # eventQueue.emit(i) - # let fastData {.used.} = await eventQueue.waitEvents(fastConsumer) - - # check len(eventQueue) == 0 - # await allFutures(slowFut) - # check len(eventQueue) == 0 - # expect AsyncEventQueueFullError: - # let res {.used.} = slowFut.read() - - # check len(eventQueue) == 0 - # eventQueue.unregister(fastConsumer) - # check len(eventQueue) == 0 - # eventQueue.unregister(slowConsumer) - # check len(eventQueue) == 0 - # await eventQueue.closeWait() - - # waitFor test() + test "AsyncEventQueue() behavior test": + let eventQueue = newAsyncEventQueue[int]() + let key = eventQueue.register() + eventQueue.emit(100) + eventQueue.emit(200) + eventQueue.emit(300) + + proc test1() = + let dataFut = eventQueue.waitEvents(key) + check: + dataFut.finished() == true + dataFut.read() == @[100, 200, 300] + + proc test2() = + let dataFut = eventQueue.waitEvents(key) + check: + dataFut.finished() == false + eventQueue.emit(400) + eventQueue.emit(500) + poll() + check: + dataFut.finished() == true + dataFut.read() == @[400, 500] + + test1() + test2() + waitFor eventQueue.closeWait() + + test "AsyncEventQueue() concurrency test": + let eventQueue = newAsyncEventQueue[int]() + let key0 = eventQueue.register() + let key1 = eventQueue.register() + eventQueue.emit(100) + let key2 = eventQueue.register() + eventQueue.emit(200) + eventQueue.emit(300) + let key3 = eventQueue.register() + eventQueue.emit(400) + eventQueue.emit(500) + eventQueue.emit(600) + let key4 = eventQueue.register() + eventQueue.emit(700) + eventQueue.emit(800) + eventQueue.emit(900) + eventQueue.emit(1000) + let key5 = eventQueue.register() + let key6 = eventQueue.register() + + let dataFut1 = eventQueue.waitEvents(key1) + let dataFut2 = eventQueue.waitEvents(key2) + let dataFut3 = eventQueue.waitEvents(key3) + let dataFut4 = eventQueue.waitEvents(key4) + let dataFut5 = eventQueue.waitEvents(key5) + let dataFut6 = eventQueue.waitEvents(key6) + check: + dataFut1.finished() == true + dataFut1.read() == @[100, 200, 300, 400, 500, 600, 700, 800, 900, 1000] + dataFut2.finished() == true + dataFut2.read() == @[200, 300, 400, 500, 600, 700, 800, 900, 1000] + dataFut3.finished() == true + dataFut3.read() == @[400, 500, 600, 700, 800, 900, 1000] + dataFut4.finished() == true + dataFut4.read() == @[700, 800, 900, 1000] + dataFut5.finished() == false + dataFut6.finished() == false + + eventQueue.emit(2000) + poll() + let dataFut0 = eventQueue.waitEvents(key0) + check: + dataFut5.finished() == true + dataFut5.read() == @[2000] + dataFut6.finished() == true + dataFut6.read() == @[2000] + dataFut0.finished() == true + dataFut0.read() == @[100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, + 2000] + + waitFor eventQueue.closeWait() + + test "AsyncEventQueue() specific number test": + let eventQueue = newAsyncEventQueue[int]() + let key = eventQueue.register() + + let dataFut1 = eventQueue.waitEvents(key, 1) + eventQueue.emit(100) + eventQueue.emit(200) + eventQueue.emit(300) + eventQueue.emit(400) + check dataFut1.finished() == false + poll() + check: + dataFut1.finished() == true + dataFut1.read() == @[100] + + let dataFut2 = eventQueue.waitEvents(key, 2) + check: + dataFut2.finished() == true + dataFut2.read() == @[200, 300] + + let dataFut3 = eventQueue.waitEvents(key, 5) + check dataFut3.finished() == false + eventQueue.emit(500) + eventQueue.emit(600) + eventQueue.emit(700) + eventQueue.emit(800) + check dataFut3.finished() == false + poll() + check: + dataFut3.finished() == true + dataFut3.read() == @[400, 500, 600, 700, 800] + + let dataFut4 = eventQueue.waitEvents(key, -1) + check dataFut4.finished() == false + eventQueue.emit(900) + eventQueue.emit(1000) + eventQueue.emit(1100) + eventQueue.emit(1200) + eventQueue.emit(1300) + eventQueue.emit(1400) + eventQueue.emit(1500) + eventQueue.emit(1600) + check dataFut4.finished() == false + poll() + check: + dataFut4.finished() == true + dataFut4.read() == @[900, 1000, 1100, 1200, 1300, 1400, 1500, 1600] + + waitFor eventQueue.closeWait() + + test "AsyncEventQueue() register()/unregister() test": + var emptySeq: seq[int] + let eventQueue = newAsyncEventQueue[int]() + let key1 = eventQueue.register() + + let dataFut1 = eventQueue.waitEvents(key1, 1) + check dataFut1.finished() == false + eventQueue.unregister(key1) + check dataFut1.finished() == false + poll() + check: + dataFut1.finished() == true + dataFut1.read() == emptySeq + + let key2 = eventQueue.register() + let dataFut2 = eventQueue.waitEvents(key2, 5) + check dataFut2.finished() == false + eventQueue.emit(100) + eventQueue.emit(200) + eventQueue.emit(300) + eventQueue.emit(400) + eventQueue.emit(500) + check dataFut2.finished() == false + eventQueue.unregister(key2) + poll() + check: + dataFut2.finished() == true + dataFut2.read() == emptySeq + + let key3 = eventQueue.register() + let dataFut3 = eventQueue.waitEvents(key3, 5) + check dataFut3.finished() == false + eventQueue.emit(100) + eventQueue.emit(200) + eventQueue.emit(300) + check dataFut3.finished() == false + poll() + eventQueue.unregister(key3) + eventQueue.emit(400) + check dataFut3.finished() == false + poll() + check: + dataFut3.finished() == true + dataFut3.read() == @[100, 200, 300] + + waitFor eventQueue.closeWait() + + test "AsyncEventQueue() garbage collection test": + let eventQueue = newAsyncEventQueue[int]() + let key1 = eventQueue.register() + check len(eventQueue) == 0 + eventQueue.emit(100) + eventQueue.emit(200) + eventQueue.emit(300) + check len(eventQueue) == 3 + let key2 = eventQueue.register() + eventQueue.emit(400) + eventQueue.emit(500) + eventQueue.emit(600) + eventQueue.emit(700) + check len(eventQueue) == 7 + let key3 = eventQueue.register() + eventQueue.emit(800) + eventQueue.emit(900) + eventQueue.emit(1000) + eventQueue.emit(1100) + eventQueue.emit(1200) + check len(eventQueue) == 12 + let dataFut1 = eventQueue.waitEvents(key1) + check: + dataFut1.finished() == true + dataFut1.read() == @[100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, + 1100, 1200] + len(eventQueue) == 9 + + let dataFut3 = eventQueue.waitEvents(key3) + check: + dataFut3.finished() == true + dataFut3.read() == @[800, 900, 1000, 1100, 1200] + len(eventQueue) == 9 + + let dataFut2 = eventQueue.waitEvents(key2) + check: + dataFut2.finished() == true + dataFut2.read() == @[400, 500, 600, 700, 800, 900, 1000, 1100, 1200] + len(eventQueue) == 0 + + waitFor eventQueue.closeWait() + + test "AsyncEventQueue() 1,000,000 of events to 10 clients test": + proc test() {.async.} = + let eventQueue = newAsyncEventQueue[int]() + var keys = @[ + eventQueue.register(), eventQueue.register(), + eventQueue.register(), eventQueue.register(), + eventQueue.register(), eventQueue.register(), + eventQueue.register(), eventQueue.register(), + eventQueue.register(), eventQueue.register() + ] + + proc clientTask(queue: AsyncEventQueue[int], + key: EventQueueKey): Future[seq[int]] {.async.} = + var events: seq[int] + while true: + let res = await queue.waitEvents(key) + if len(res) == 0: + break + events.add(res) + queue.unregister(key) + return events + + var futs = @[ + clientTask(eventQueue, keys[0]), clientTask(eventQueue, keys[1]), + clientTask(eventQueue, keys[2]), clientTask(eventQueue, keys[3]), + clientTask(eventQueue, keys[4]), clientTask(eventQueue, keys[5]), + clientTask(eventQueue, keys[6]), clientTask(eventQueue, keys[7]), + clientTask(eventQueue, keys[8]), clientTask(eventQueue, keys[9]) + ] + + for i in 1 .. 1_000_000: + if (i mod 1000) == 0: + # Give some CPU for clients. + await sleepAsync(0.milliseconds) + eventQueue.emit(i) + + await eventQueue.closeWait() + + await allFutures(futs) + for index in 0 ..< len(futs): + let fut = futs[index] + check fut.finished() == true + let data = fut.read() + var counter = 1 + for item in data: + check item == counter + inc(counter) + futs[index] = nil + + waitFor test() + + test "AsyncEventQueue() one consumer limits test": + proc test() {.async.} = + let eventQueue = newAsyncEventQueue[int](4) + check len(eventQueue) == 0 + eventQueue.emit(100) + eventQueue.emit(200) + eventQueue.emit(300) + eventQueue.emit(400) + # There no consumers, so all the items should be discarded + check len(eventQueue) == 0 + let key1 = eventQueue.register() + check len(eventQueue) == 0 + eventQueue.emit(500) + eventQueue.emit(600) + eventQueue.emit(700) + eventQueue.emit(800) + # So exact `limit` number of items added, consumer should receive all of + # them. + check len(eventQueue) == 4 + let dataFut1 = eventQueue.waitEvents(key1) + check: + dataFut1.finished() == true + dataFut1.read() == @[500, 600, 700, 800] + len(eventQueue) == 0 + + eventQueue.emit(900) + eventQueue.emit(1000) + eventQueue.emit(1100) + eventQueue.emit(1200) + check len(eventQueue) == 4 + # Overfilling queue + eventQueue.emit(1300) + # Because overfill for single consumer happend, whole queue should become + # empty. + check len(eventQueue) == 0 + eventQueue.emit(1400) + eventQueue.emit(1500) + eventQueue.emit(1600) + eventQueue.emit(1700) + eventQueue.emit(1800) + check len(eventQueue) == 0 + let errorFut1 = eventQueue.waitEvents(key1) + check errorFut1.finished() == true + let checkException = + try: + let res {.used.} = await errorFut1 + false + except AsyncEventQueueFullError: + true + except CatchableError: + false + check checkException == true + # There should be no items because consumer was overflowed. + check len(eventQueue) == 0 + eventQueue.unregister(key1) + # All items should be garbage collected after unregister. + check len(eventQueue) == 0 + await eventQueue.closeWait() + + waitFor test() + + test "AsyncEventQueue() many consumers limits test": + proc test() {.async.} = + let eventQueue = newAsyncEventQueue[int](4) + block: + let key1 = eventQueue.register() + eventQueue.emit(100) + check len(eventQueue) == 1 + let key2 = eventQueue.register() + eventQueue.emit(200) + check len(eventQueue) == 2 + let key3 = eventQueue.register() + eventQueue.emit(300) + check len(eventQueue) == 3 + let key4 = eventQueue.register() + eventQueue.emit(400) + check len(eventQueue) == 4 + let key5 = eventQueue.register() + eventQueue.emit(500) + # At this point consumer with `key1` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [200, 300, 400, 500] + check len(eventQueue) == 4 + eventQueue.emit(600) + # At this point consumers with `key2` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [300, 400, 500, 600] + check len(eventQueue) == 4 + eventQueue.emit(700) + # At this point consumers with `key3` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [400, 500, 600, 700] + check len(eventQueue) == 4 + eventQueue.emit(800) + # At this point consumers with `key4` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [500, 600, 700, 800] + check len(eventQueue) == 4 + # Consumer with key5 is not overfilled. + let dataFut5 = eventQueue.waitEvents(key5) + check: + dataFut5.finished() == true + dataFut5.read() == @[500, 600, 700, 800] + # No more items should be left because all other consumers are overfilled. + check len(eventQueue) == 0 + eventQueue.unregister(key5) + check len(eventQueue) == 0 + + let dataFut2 = eventQueue.waitEvents(key2) + check dataFut2.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut2.read() + check len(eventQueue) == 0 + eventQueue.unregister(key2) + check len(eventQueue) == 0 + + let dataFut4 = eventQueue.waitEvents(key4) + check dataFut4.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut4.read() + check len(eventQueue) == 0 + eventQueue.unregister(key4) + check len(eventQueue) == 0 + + let dataFut3 = eventQueue.waitEvents(key3) + check dataFut3.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut3.read() + check len(eventQueue) == 0 + eventQueue.unregister(key3) + check len(eventQueue) == 0 + + let dataFut1 = eventQueue.waitEvents(key1) + check dataFut1.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut1.read() + check len(eventQueue) == 0 + eventQueue.unregister(key1) + check len(eventQueue) == 0 + + block: + let key1 = eventQueue.register() + eventQueue.emit(100) + check len(eventQueue) == 1 + let key2 = eventQueue.register() + eventQueue.emit(200) + check len(eventQueue) == 2 + let key3 = eventQueue.register() + eventQueue.emit(300) + check len(eventQueue) == 3 + let key4 = eventQueue.register() + eventQueue.emit(400) + check len(eventQueue) == 4 + let key5 = eventQueue.register() + eventQueue.emit(500) + # At this point consumer with `key1` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [200, 300, 400, 500] + check len(eventQueue) == 4 + eventQueue.emit(600) + # At this point consumer with `key2` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [300, 400, 500, 600] + check len(eventQueue) == 4 + eventQueue.emit(700) + # At this point consumer with `key3` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [400, 500, 600, 700] + check len(eventQueue) == 4 + eventQueue.emit(800) + # At this point consumer with `key4` is overfilled, so after `emit()` + # queue length should be decreased by one item. + # So queue should look like this: [500, 600, 700, 800] + check len(eventQueue) == 4 + eventQueue.emit(900) + # At this point all consumers are overfilled, so after `emit()` + # queue length should become 0. + check len(eventQueue) == 0 + eventQueue.emit(1000) + eventQueue.emit(1100) + eventQueue.emit(1200) + eventQueue.emit(1300) + eventQueue.emit(1400) + eventQueue.emit(1500) + eventQueue.emit(1600) + eventQueue.emit(1700) + eventQueue.emit(1800) + eventQueue.emit(1900) + # No more events should be accepted. + check len(eventQueue) == 0 + + let dataFut1 = eventQueue.waitEvents(key1) + check dataFut1.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut1.read() + check len(eventQueue) == 0 + eventQueue.unregister(key1) + check len(eventQueue) == 0 + + let dataFut2 = eventQueue.waitEvents(key2) + check dataFut2.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut2.read() + check len(eventQueue) == 0 + eventQueue.unregister(key2) + check len(eventQueue) == 0 + + let dataFut3 = eventQueue.waitEvents(key3) + check dataFut3.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut3.read() + check len(eventQueue) == 0 + eventQueue.unregister(key3) + check len(eventQueue) == 0 + + let dataFut4 = eventQueue.waitEvents(key4) + check dataFut4.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut4.read() + check len(eventQueue) == 0 + eventQueue.unregister(key4) + check len(eventQueue) == 0 + + let dataFut5 = eventQueue.waitEvents(key5) + check dataFut5.finished() == true + expect AsyncEventQueueFullError: + let res {.used.} = dataFut5.read() + check len(eventQueue) == 0 + eventQueue.unregister(key5) + check len(eventQueue) == 0 + await eventQueue.closeWait() + + waitFor test() + + test "AsyncEventQueue() slow and fast consumer test": + proc test() {.async.} = + let eventQueue = newAsyncEventQueue[int](1) + let + fastConsumer = eventQueue.register() + slowConsumer = eventQueue.register() + slowFut = eventQueue.waitEvents(slowConsumer) + + for i in 0 ..< 1000: + eventQueue.emit(i) + let fastData {.used.} = await eventQueue.waitEvents(fastConsumer) + + check len(eventQueue) == 0 + await allFutures(slowFut) + check len(eventQueue) == 0 + expect AsyncEventQueueFullError: + let res {.used.} = slowFut.read() + + check len(eventQueue) == 0 + eventQueue.unregister(fastConsumer) + check len(eventQueue) == 0 + eventQueue.unregister(slowConsumer) + check len(eventQueue) == 0 + await eventQueue.closeWait() + + waitFor test()