Parent Log: http://ci.aztec-labs.com/1440c0fbd00c97c8 Command: fe89a5d9e20c90c9:ISOLATE=1:NAME=p2p/src/services/reqresp/reqresp.test.ts yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts Commit: https://github.com/AztecProtocol/aztec-packages/commit/260d4b0cfbe16cbd8be248715852ee1ff72cc6cd Env: REF_NAME=gh-readonly-queue/master/pr-14716-b9f63e086a00464c0ad1800566372d3b7ddc6746 CURRENT_VERSION=0.87.7 CI_FULL=1 Date: Tue Jun 3 13:33:34 UTC 2025 System: ARCH=amd64 CPUS=128 MEM=493Gi HOSTNAME=pr-14716_amd64_x1-full Resources: CPU_LIST=0-127 CPUS=2 MEM=8g TIMEOUT=600s History: http://ci.aztec-labs.com/list/a57cc872d7daa810 13:33:34 +++ id -u 13:33:34 +++ id -g 13:33:34 ++ docker run -d --name p2p_src_services_reqresp_reqresp.test.ts --net=none --cpuset-cpus=0-127 --cpus=2 --memory=8g --user 1000:1000 -v/home/aztec-dev:/home/aztec-dev --mount type=tmpfs,target=/tmp,tmpfs-size=1g --workdir /home/aztec-dev/aztec-packages -e HOME -e VERBOSE -e GIT_CONFIG_GLOBAL=/home/aztec-dev/aztec-packages/build-images/src/home/.gitconfig -e FORCE_COLOR=true -e CPUS -e MEM aztecprotocol/build:3.0 /bin/bash -c 'timeout -v 600s bash -c '\''yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts'\''' 13:33:34 + cid=aa0c6377566fc7a9d59d1ad76ded35241eb11fef5ce2112fc727e98f2d3300d1 13:33:34 + set +x 13:33:43 [13:33:43.250] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:43 [13:33:43.254] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:43 [13:33:43.258] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:43 [13:33:43.258] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:43 [13:33:43.803] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:43 [13:33:43.804] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:43 [13:33:43.851] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:43 [13:33:43.852] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:43 [13:33:43.853] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:43 [13:33:43.854] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:45 [13:33:45.032] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:45 [13:33:45.033] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:45 [13:33:45.035] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:45 [13:33:45.036] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:45 [13:33:45.154] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:45 [13:33:45.155] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:45 [13:33:45.156] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:45 [13:33:45.156] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:45 [13:33:45.157] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:45 [13:33:45.157] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:45 [13:33:45.162] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:45 [13:33:45.163] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:46 [13:33:46.254] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWBMXz2YXyzv7mDP33pFMxiHEabuLZfZ3JQRa7982YMdLW 13:33:46 [13:33:46.255] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 13:33:46 err: { 13:33:46 "type": "ReqRespStatusError", 13:33:46 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 13:33:46 "stack": 13:33:46 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 13:33:46 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 13:33:46 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 13:33:46 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 13:33:46 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 13:33:46 at processTicksAndRejections (node:internal/process/task_queues:105:5) 13:33:46 "status": 1 13:33:46 } 13:33:46 [13:33:46.263] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWBMXz2YXyzv7mDP33pFMxiHEabuLZfZ3JQRa7982YMdLW 13:33:46 [13:33:46.263] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 13:33:46 err: { 13:33:46 "type": "ReqRespStatusError", 13:33:46 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 13:33:46 "stack": 13:33:46 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 13:33:46 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 13:33:46 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 13:33:46 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 13:33:46 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 13:33:46 at processTicksAndRejections (node:internal/process/task_queues:105:5) 13:33:46 "status": 1 13:33:46 } 13:33:46 [13:33:46.267] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWBMXz2YXyzv7mDP33pFMxiHEabuLZfZ3JQRa7982YMdLW 13:33:46 [13:33:46.267] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 13:33:46 err: { 13:33:46 "type": "ReqRespStatusError", 13:33:46 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 13:33:46 "stack": 13:33:46 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 13:33:46 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 13:33:46 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 13:33:46 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 13:33:46 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 13:33:46 at processTicksAndRejections (node:internal/process/task_queues:105:5) 13:33:46 "status": 1 13:33:46 } 13:33:46 [13:33:46.270] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWBMXz2YXyzv7mDP33pFMxiHEabuLZfZ3JQRa7982YMdLW 13:33:46 [13:33:46.270] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 13:33:46 err: { 13:33:46 "type": "ReqRespStatusError", 13:33:46 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 13:33:46 "stack": 13:33:46 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 13:33:46 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 13:33:46 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 13:33:46 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 13:33:46 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 13:33:46 at processTicksAndRejections (node:internal/process/task_queues:105:5) 13:33:46 "status": 1 13:33:46 } 13:33:46 [13:33:46.274] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWBMXz2YXyzv7mDP33pFMxiHEabuLZfZ3JQRa7982YMdLW 13:33:46 [13:33:46.274] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 13:33:46 err: { 13:33:46 "type": "ReqRespStatusError", 13:33:46 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 13:33:46 "stack": 13:33:46 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 13:33:46 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 13:33:46 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 13:33:46 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 13:33:46 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 13:33:46 at processTicksAndRejections (node:internal/process/task_queues:105:5) 13:33:46 "status": 1 13:33:46 } 13:33:46 [13:33:46.280] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:46 [13:33:46.281] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:46 [13:33:46.285] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:46 [13:33:46.285] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:47 [13:33:47.800] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:47 [13:33:47.801] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:47 [13:33:47.811] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:47 [13:33:47.813] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:48 [13:33:48.904] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:48 [13:33:48.904] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:48 [13:33:48.908] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:48 [13:33:48.908] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:49 [13:33:49.979] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:49 [13:33:49.979] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:49 [13:33:49.983] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:49 [13:33:49.983] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:53 [13:33:53.081] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:53 [13:33:53.081] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:53 [13:33:53.085] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:53 [13:33:53.085] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:58 [13:33:58.296] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:58 [13:33:58.296] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:58 [13:33:58.298] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:58 [13:33:58.298] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:58 [13:33:58.300] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:58 [13:33:58.300] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:58 [13:33:58.306] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:58 [13:33:58.306] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:58 [13:33:58.324] WARN: foundation:memory_fifo Discarding item because queue is flushing 13:33:58 [13:33:58.327] ERROR: p2p:reqresp Unexpected error sending request to peer: Error: Could not enqueue function 13:33:58 at /home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:63:24 13:33:58 at new Promise (<anonymous>) 13:33:58 at SerialQueue.put (/home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:53:16) 13:33:58 at ConnectionSampler.put [as dialProtocol] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/connection-sampler/connection_sampler.ts:204:43) 13:33:58 at ReqResp.dialProtocol (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:449:45) 13:33:58 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 13:33:58 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 13:33:58 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 13:33:58 at TimeoutTask.requestFunction [as fn] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:200:26) {"peerId":"12D3KooWNZ4HY7rzZzELJgpnmeiVci8bWor1sAcP2KUQP3f82Ckv","subProtocol":"/aztec/req/tx/0.1.0"} 13:33:59 [13:33:59.407] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:59 [13:33:59.408] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:59 [13:33:59.412] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:59 [13:33:59.413] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:34:00 [13:34:00.460] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:34:00 [13:34:00.460] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:34:00 [13:34:00.464] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:34:00 [13:34:00.465] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:34:01 [13:34:01.522] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:34:01 [13:34:01.522] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:34:01 [13:34:01.526] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:34:01 [13:34:01.527] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:34:02 [13:34:02.910] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:34:02 [13:34:02.911] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:34:02 [13:34:02.914] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:34:02 [13:34:02.914] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:34:04 [13:34:04.037] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWB1xRdDdUgfYdwTsHTmcenuHp7qwFUNKB51xMwvcL1hpB 13:34:04 [13:34:04.038] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 13:34:04 err: { 13:34:04 "type": "ReqRespStatusError", 13:34:04 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 13:34:04 "stack": 13:34:04 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 13:34:04 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 13:34:04 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 13:34:04 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 13:34:04 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 13:34:04 at processTicksAndRejections (node:internal/process/task_queues:105:5) 13:34:04 "status": 1 13:34:04 } 13:34:04 [13:34:04.062] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:34:04 [13:34:04.063] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:34:04 [13:34:04.064] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:34:04 [13:34:04.064] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:34:04 [13:34:04.067] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:34:04 [13:34:04.068] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:34:05 [13:34:05.322] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:34:05 [13:34:05.323] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:34:05 [13:34:05.324] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:34:05 [13:34:05.324] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:34:05 [13:34:05.325] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:34:05 [13:34:05.325] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:34:05 [13:34:05.329] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:34:05 [13:34:05.329] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:34:06 [13:34:06.445] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWGMfy3Y6Gw5Av5XjH4HQcq57MDLjx97DT42v6hrnQkrhT 13:34:06 [13:34:06.446] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 13:34:06 err: { 13:34:06 "type": "ReqRespStatusError", 13:34:06 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 13:34:06 "stack": 13:34:06 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 13:34:06 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 13:34:06 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 13:34:06 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 13:34:06 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 13:34:06 at processTicksAndRejections (node:internal/process/task_queues:105:5) 13:34:06 "status": 1 13:34:06 } 13:34:06 [13:34:06.450] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWGMfy3Y6Gw5Av5XjH4HQcq57MDLjx97DT42v6hrnQkrhT 13:34:06 [13:34:06.451] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 13:34:06 err: { 13:34:06 "type": "ReqRespStatusError", 13:34:06 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 13:34:06 "stack": 13:34:06 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 13:34:06 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 13:34:06 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 13:34:06 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 13:34:06 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 13:34:06 at processTicksAndRejections (node:internal/process/task_queues:105:5) 13:34:06 "status": 1 13:34:06 } 13:34:06 [13:34:06.467] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWGMfy3Y6Gw5Av5XjH4HQcq57MDLjx97DT42v6hrnQkrhT 13:34:06 [13:34:06.467] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 13:34:06 err: { 13:34:06 "type": "ReqRespStatusError", 13:34:06 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 13:34:06 "stack": 13:34:06 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 13:34:06 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 13:34:06 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 13:34:06 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 13:34:06 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 13:34:06 at processTicksAndRejections (node:internal/process/task_queues:105:5) 13:34:06 "status": 1 13:34:06 } 13:34:06 [13:34:06.477] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:34:06 [13:34:06.478] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:34:06 [13:34:06.478] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:34:06 [13:34:06.479] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:34:06 [13:34:06.482] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:34:06 [13:34:06.482] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:34:06 FAIL src/services/reqresp/reqresp.test.ts (30.906 s) 13:34:06 ReqResp 13:34:06 ✓ should perform a ping request (1210 ms) 13:34:06 ✓ should handle gracefully if a peer connected peer is offline (592 ms) 13:34:06 ✓ should request from a later peer if other peers are offline (1341 ms) 13:34:06 ✕ should hit a rate limit if too many requests are made in quick succession (1100 ms) 13:34:06 Tx req protocol 13:34:06 ✓ can request a Tx from TxHash (1522 ms) 13:34:06 ✓ handles returning empty buffers (1096 ms) 13:34:06 ✓ does not crash if tx hash returns undefined (1076 ms) 13:34:06 ✓ should hit individual timeout if nothing is returned over the stream (3092 ms) 13:34:06 ✓ should hit collective timeout if nothing is returned over the stream from multiple peers (5238 ms) 13:34:06 ✓ should penalize peer if transaction validation fails (1091 ms) 13:34:06 Goodbye protocol 13:34:06 ✓ should send a goodbye message to a peer (1050 ms) 13:34:06 ✓ should not close stream when handling a goodbye message received from peer (1063 ms) 13:34:06 Block protocol 13:34:06 ✓ should handle block requests (1387 ms) 13:34:06 Batch requests 13:34:06 ✕ should send a batch request between many peers (1158 ms) 13:34:06 ✕ should send a batch request with a pinned peer (1267 ms) 13:34:06 ✕ should stop after max retry attempts (1148 ms) 13:34:06 13:34:06 ● ReqResp › should hit a rate limit if too many requests are made in quick succession 13:34:06 13:34:06 expect(received).toBeDefined() 13:34:06 13:34:06 Received: undefined 13:34:06 13:34:06 145 | // Check that one of the responses gets a rate limit response 13:34:06 146 | const rateLimitResponse = responses.find(response => response?.status === ReqRespStatus.RATE_LIMIT_EXCEEDED); 13:34:06 > 147 | expect(rateLimitResponse).toBeDefined(); 13:34:06 | ^ 13:34:06 148 | 13:34:06 149 | // Make sure the error message is logged 13:34:06 150 | const errorMessage = `Rate limit exceeded DeniedPeer for ${ 13:34:06 13:34:06 at Object.toBeDefined (services/reqresp/reqresp.test.ts:147:31) 13:34:06 13:34:06 ● ReqResp › Batch requests › should send a batch request between many peers 13:34:06 13:34:06 expect(jest.fn()).toHaveBeenCalledTimes(expected) 13:34:06 13:34:06 Expected number of calls: 9 13:34:06 Received number of calls: 11 13:34:06 13:34:06 474 | 13:34:06 475 | // Expect one request to have been sent to each peer 13:34:06 > 476 | expect(sendRequestToPeerSpy).toHaveBeenCalledTimes(batchSize); 13:34:06 | ^ 13:34:06 477 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 13:34:06 478 | expect.objectContaining({ 13:34:06 479 | publicKey: nodes[1].p2p.peerId.publicKey, 13:34:06 13:34:06 at Object.toHaveBeenCalledTimes (services/reqresp/reqresp.test.ts:476:36) 13:34:06 13:34:06 ● ReqResp › Batch requests › should send a batch request with a pinned peer 13:34:06 13:34:06 expect(jest.fn()).toHaveBeenCalledWith(...expected) 13:34:06 13:34:06 Expected: ObjectContaining {"publicKey": [8, 1, 18, 32, 61, 141, 73, 44, 61, 32, …]}, "/aztec/req/ping/0.1.0", {"data": [112, 105, 110, 103, 51], "type": "Buffer"} 13:34:06 Received 13:34:06 1 13:34:06 "12D3KooWMoNxMVzcNKzaawjAvxxGoSQiMD7og9hRU86ec8tn1dC8", 13:34:06 "/aztec/req/ping/0.1.0", 13:34:06 @@ -2,9 +2,9 @@ 13:34:06 "data": Array [ 13:34:06 112, 13:34:06 105, 13:34:06 110, 13:34:06 103, 13:34:06 - 51, 13:34:06 + 48, 13:34:06 ], 13:34:06 "type": "Buffer", 13:34:06 }, 13:34:06 2 13:34:06 "12D3KooWPoz9jcXVzHYtv84vMNvs7wvnLkB1XU5P22VhHJAU9nNv", 13:34:06 "/aztec/req/ping/0.1.0", 13:34:06 @@ -2,9 +2,9 @@ 13:34:06 "data": Array [ 13:34:06 112, 13:34:06 105, 13:34:06 110, 13:34:06 103, 13:34:06 - 51, 13:34:06 + 49, 13:34:06 ], 13:34:06 "type": "Buffer", 13:34:06 }, 13:34:06 3 13:34:06 "12D3KooWDxdzWHEgQC8SqRGCztgGEYRg6ELWfBhY5HXfLukj9jmc", 13:34:06 "/aztec/req/ping/0.1.0", 13:34:06 @@ -2,9 +2,9 @@ 13:34:06 "data": Array [ 13:34:06 112, 13:34:06 105, 13:34:06 110, 13:34:06 103, 13:34:06 - 51, 13:34:06 + 48, 13:34:06 ], 13:34:06 "type": "Buffer", 13:34:06 }, 13:34:06 13:34:06 Number of calls: 14 13:34:06 13:34:06 516 | // Expect pinned peer to have received all requests 13:34:06 517 | for (let i = 0; i < batchSize; i++) { 13:34:06 > 518 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 13:34:06 | ^ 13:34:06 519 | expect.objectContaining({ publicKey: nodes[1].p2p.peerId.publicKey }), 13:34:06 520 | ReqRespSubProtocol.PING, 13:34:06 521 | Buffer.from(`ping${i}`), 13:34:06 13:34:06 at Object.toHaveBeenCalledWith (services/reqresp/reqresp.test.ts:518:38) 13:34:06 13:34:06 ● ReqResp › Batch requests › should stop after max retry attempts 13:34:06 13:34:06 expect(received).toEqual(expected) // deep equality 13:34:06 13:34:06 - Expected - 11 13:34:06 + Received + 3 13:34:06 13:34:06 @@ -74,10 +74,11 @@ 13:34:06 103, 13:34:06 ], 13:34:06 "type": "Buffer", 13:34:06 }, 13:34:06 }, 13:34:06 + undefined, 13:34:06 RequestableBuffer { 13:34:06 "buffer": Object { 13:34:06 "data": Array [ 13:34:06 112, 13:34:06 111, 13:34:06 @@ -85,10 +86,11 @@ 13:34:06 103, 13:34:06 ], 13:34:06 "type": "Buffer", 13:34:06 }, 13:34:06 }, 13:34:06 + undefined, 13:34:06 RequestableBuffer { 13:34:06 "buffer": Object { 13:34:06 "data": Array [ 13:34:06 112, 13:34:06 111, 13:34:06 @@ -96,17 +98,7 @@ 13:34:06 103, 13:34:06 ], 13:34:06 "type": "Buffer", 13:34:06 }, 13:34:06 }, 13:34:06 - RequestableBuffer { 13:34:06 - "buffer": Object { 13:34:06 - "data": Array [ 13:34:06 - 112, 13:34:06 - 111, 13:34:06 - 110, 13:34:06 - 103, 13:34:06 - ], 13:34:06 - "type": "Buffer", 13:34:06 - }, 13:34:06 - }, 13:34:06 + undefined, 13:34:06 ] 13:34:06 13:34:06 555 | 13:34:06 556 | const res = await nodes[0].req.sendBatchRequest(ReqRespSubProtocol.PING, requests, undefined); 13:34:06 > 557 | expect(res).toEqual(expectResponses); 13:34:06 | ^ 13:34:06 558 | 13:34:06 559 | // Check that we did detect hitting a rate limit 13:34:06 560 | expect(requesterLoggerSpy).toHaveBeenCalledWith( 13:34:06 13:34:06 at Object.toEqual (services/reqresp/reqresp.test.ts:557:19) 13:34:06 13:34:06 Test Suites: 1 failed, 1 total 13:34:06 Tests: 4 failed, 12 passed, 16 total 13:34:06 Snapshots: 0 total 13:34:06 Time: 30.989 s 13:34:06 Ran all test suites matching /p2p\/src\/services\/reqresp\/reqresp.test.ts/i. 13:34:06 Force exiting Jest: Have you considered using `--detectOpenHandles` to detect async operations that kept running after all tests finished?