ACI ·

Parent Log: http://ci.aztec-labs.com/525be1a7df3beabf Command: cdc0844b6e9d2c1e:ISOLATE=1:NAME=p2p/src/services/reqresp/reqresp.test.ts yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts Commit: https://github.com/AztecProtocol/aztec-packages/commit/260a6c8dbe7557aae065d685de66faa2bc362a9d Env: REF_NAME=gh-readonly-queue/master/pr-14951-64b2b9f661d3e31df0f8ec9120130b753eeb7533 CURRENT_VERSION=0.87.8 CI_FULL=0 Date: Tue Jun 10 22:15:28 UTC 2025 System: ARCH=arm64 CPUS=64 MEM=247Gi HOSTNAME=pr-14951_arm64_a1-fast Resources: CPU_LIST=0-63 CPUS=2 MEM=8g TIMEOUT=600s History: http://ci.aztec-labs.com/list/history_a57cc872d7daa810_master 22:15:28 +++ id -u 22:15:28 +++ id -g 22:15:28 ++ docker run -d --name p2p_src_services_reqresp_reqresp.test.ts --net=none --cpuset-cpus=0-63 --cpus=2 --memory=8g --user 1000:1000 -v/home/aztec-dev:/home/aztec-dev --mount type=tmpfs,target=/tmp,tmpfs-size=1g --workdir /home/aztec-dev/aztec-packages -e HOME -e VERBOSE -e GIT_CONFIG_GLOBAL=/home/aztec-dev/aztec-packages/build-images/src/home/.gitconfig -e FORCE_COLOR=true -e CPUS -e MEM aztecprotocol/build:3.0 /bin/bash -c 'timeout -v 600s bash -c '\''yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts'\''' 22:15:32 + cid=7e39bce79e834198bb80541e16cebf4bff8c46bc0832dc84c27cd97fe0630001 22:15:32 + set +x 22:15:38 [22:15:38.277] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:38 [22:15:38.283] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:38 [22:15:38.287] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:38 [22:15:38.287] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:38 [22:15:38.827] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:38 [22:15:38.828] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:38 [22:15:38.864] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:38 [22:15:38.865] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:38 [22:15:38.866] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:38 [22:15:38.867] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:40 [22:15:40.034] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:40 [22:15:40.035] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:40 [22:15:40.037] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:40 [22:15:40.038] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:40 [22:15:40.133] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:40 [22:15:40.134] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:40 [22:15:40.134] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:40 [22:15:40.135] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:40 [22:15:40.138] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:40 [22:15:40.138] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:40 [22:15:40.142] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:40 [22:15:40.142] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:41 [22:15:41.222] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWPY4r2jPSADniyWFCMsmS1xBBFEb4V9VMQyk384HpdR2H 22:15:41 [22:15:41.223] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 22:15:41 err: { 22:15:41 "type": "ReqRespStatusError", 22:15:41 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 22:15:41 "stack": 22:15:41 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 22:15:41 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 22:15:41 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 22:15:41 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 22:15:41 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 22:15:41 at processTicksAndRejections (node:internal/process/task_queues:105:5) 22:15:41 "status": 1 22:15:41 } 22:15:41 [22:15:41.233] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWPY4r2jPSADniyWFCMsmS1xBBFEb4V9VMQyk384HpdR2H 22:15:41 [22:15:41.233] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 22:15:41 err: { 22:15:41 "type": "ReqRespStatusError", 22:15:41 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 22:15:41 "stack": 22:15:41 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 22:15:41 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 22:15:41 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 22:15:41 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 22:15:41 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 22:15:41 at processTicksAndRejections (node:internal/process/task_queues:105:5) 22:15:41 "status": 1 22:15:41 } 22:15:41 [22:15:41.238] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWPY4r2jPSADniyWFCMsmS1xBBFEb4V9VMQyk384HpdR2H 22:15:41 [22:15:41.238] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 22:15:41 err: { 22:15:41 "type": "ReqRespStatusError", 22:15:41 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 22:15:41 "stack": 22:15:41 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 22:15:41 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 22:15:41 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 22:15:41 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 22:15:41 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 22:15:41 at processTicksAndRejections (node:internal/process/task_queues:105:5) 22:15:41 "status": 1 22:15:41 } 22:15:41 [22:15:41.243] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWPY4r2jPSADniyWFCMsmS1xBBFEb4V9VMQyk384HpdR2H 22:15:41 [22:15:41.243] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 22:15:41 err: { 22:15:41 "type": "ReqRespStatusError", 22:15:41 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 22:15:41 "stack": 22:15:41 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 22:15:41 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 22:15:41 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 22:15:41 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 22:15:41 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 22:15:41 at processTicksAndRejections (node:internal/process/task_queues:105:5) 22:15:41 "status": 1 22:15:41 } 22:15:41 [22:15:41.247] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWPY4r2jPSADniyWFCMsmS1xBBFEb4V9VMQyk384HpdR2H 22:15:41 [22:15:41.247] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 22:15:41 err: { 22:15:41 "type": "ReqRespStatusError", 22:15:41 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 22:15:41 "stack": 22:15:41 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 22:15:41 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 22:15:41 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 22:15:41 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 22:15:41 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 22:15:41 at processTicksAndRejections (node:internal/process/task_queues:105:5) 22:15:41 "status": 1 22:15:41 } 22:15:41 [22:15:41.254] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:41 [22:15:41.255] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:41 [22:15:41.258] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:41 [22:15:41.258] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:42 [22:15:42.607] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:42 [22:15:42.607] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:42 [22:15:42.610] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:42 [22:15:42.610] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:43 [22:15:43.680] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:43 [22:15:43.680] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:43 [22:15:43.683] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:43 [22:15:43.684] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:44 [22:15:44.750] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:44 [22:15:44.751] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:44 [22:15:44.753] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:44 [22:15:44.754] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:47 [22:15:47.790] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:47 [22:15:47.790] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:47 [22:15:47.793] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:47 [22:15:47.793] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:52 [22:15:52.944] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:52 [22:15:52.944] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:52 [22:15:52.945] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:52 [22:15:52.945] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:52 [22:15:52.946] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:52 [22:15:52.946] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:52 [22:15:52.949] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:52 [22:15:52.950] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:52 [22:15:52.962] WARN: foundation:memory_fifo Discarding item because queue is flushing 22:15:52 [22:15:52.964] ERROR: p2p:reqresp Unexpected error sending request to peer: Error: Could not enqueue function 22:15:52 at /home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:65:24 22:15:52 at new Promise (<anonymous>) 22:15:52 at SerialQueue.put (/home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:55:16) 22:15:52 at ConnectionSampler.put [as dialProtocol] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/connection-sampler/connection_sampler.ts:204:43) 22:15:52 at ReqResp.dialProtocol (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:449:45) 22:15:52 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 22:15:52 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 22:15:52 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 22:15:52 at TimeoutTask.requestFunction [as fn] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:200:26) {"peerId":"12D3KooWHsiJpoh33DjQfekpDy3sCMWvLKxgb3Kun4wiYHkEKCXA","subProtocol":"/aztec/req/tx/0.1.0"} 22:15:54 [22:15:54.035] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:54 [22:15:54.035] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:54 [22:15:54.038] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:54 [22:15:54.038] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:55 [22:15:55.077] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:55 [22:15:55.077] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:55 [22:15:55.080] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:55 [22:15:55.081] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:56 [22:15:56.119] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:56 [22:15:56.120] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:56 [22:15:56.123] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:56 [22:15:56.123] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:57 [22:15:57.377] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:57 [22:15:57.377] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:57 [22:15:57.380] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:57 [22:15:57.380] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:58 [22:15:58.472] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWQ7U1EH1JgEZ7iR5NKD4R8YtCgsKJHC6hsbinWJSG7LK2 22:15:58 [22:15:58.473] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 22:15:58 err: { 22:15:58 "type": "ReqRespStatusError", 22:15:58 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 22:15:58 "stack": 22:15:58 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 22:15:58 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 22:15:58 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 22:15:58 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 22:15:58 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 22:15:58 at processTicksAndRejections (node:internal/process/task_queues:105:5) 22:15:58 "status": 1 22:15:58 } 22:15:58 [22:15:58.490] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:58 [22:15:58.491] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:58 [22:15:58.492] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:58 [22:15:58.492] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:58 [22:15:58.495] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:58 [22:15:58.495] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:59 [22:15:59.688] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:59 [22:15:59.689] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:59 [22:15:59.689] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:59 [22:15:59.690] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:59 [22:15:59.690] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:59 [22:15:59.691] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:15:59 [22:15:59.694] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:15:59 [22:15:59.694] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:16:00 [22:16:00.791] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWNHRL5i22Z7zAazhx4L9Kq7YnfLcctLifsJvKx3iBrxMm 22:16:00 [22:16:00.792] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 22:16:00 err: { 22:16:00 "type": "ReqRespStatusError", 22:16:00 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 22:16:00 "stack": 22:16:00 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 22:16:00 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 22:16:00 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 22:16:00 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 22:16:00 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 22:16:00 at processTicksAndRejections (node:internal/process/task_queues:105:5) 22:16:00 "status": 1 22:16:00 } 22:16:00 [22:16:00.795] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWNHRL5i22Z7zAazhx4L9Kq7YnfLcctLifsJvKx3iBrxMm 22:16:00 [22:16:00.795] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 22:16:00 err: { 22:16:00 "type": "ReqRespStatusError", 22:16:00 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 22:16:00 "stack": 22:16:00 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 22:16:00 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 22:16:00 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 22:16:00 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 22:16:00 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 22:16:00 at processTicksAndRejections (node:internal/process/task_queues:105:5) 22:16:00 "status": 1 22:16:00 } 22:16:00 [22:16:00.809] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWNHRL5i22Z7zAazhx4L9Kq7YnfLcctLifsJvKx3iBrxMm 22:16:00 [22:16:00.809] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 22:16:00 err: { 22:16:00 "type": "ReqRespStatusError", 22:16:00 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 22:16:00 "stack": 22:16:00 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 22:16:00 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 22:16:00 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 22:16:00 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 22:16:00 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 22:16:00 at processTicksAndRejections (node:internal/process/task_queues:105:5) 22:16:00 "status": 1 22:16:00 } 22:16:00 [22:16:00.818] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:16:00 [22:16:00.818] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:16:00 [22:16:00.819] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:16:00 [22:16:00.819] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:16:00 [22:16:00.823] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:16:00 [22:16:00.824] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:16:00 FAIL src/services/reqresp/reqresp.test.ts (27.459 s) 22:16:00 ReqResp 22:16:00 ✓ should perform a ping request (1210 ms) 22:16:00 ✓ should handle gracefully if a peer connected peer is offline (574 ms) 22:16:00 ✓ should request from a later peer if other peers are offline (1300 ms) 22:16:00 ✕ should hit a rate limit if too many requests are made in quick succession (1092 ms) 22:16:00 Tx req protocol 22:16:00 ✓ can request a Tx from TxHash (1352 ms) 22:16:00 ✓ handles returning empty buffers (1072 ms) 22:16:00 ✓ does not crash if tx hash returns undefined (1069 ms) 22:16:00 ✓ should hit individual timeout if nothing is returned over the stream (3040 ms) 22:16:00 ✓ should hit collective timeout if nothing is returned over the stream from multiple peers (5168 ms) 22:16:00 ✓ should penalize peer if transaction validation fails (1076 ms) 22:16:00 Goodbye protocol 22:16:00 ✓ should send a goodbye message to a peer (1042 ms) 22:16:00 ✓ should not close stream when handling a goodbye message received from peer (1042 ms) 22:16:00 Block protocol 22:16:00 ✓ should handle block requests (1258 ms) 22:16:00 Batch requests 22:16:00 ✕ should send a batch request between many peers (1119 ms) 22:16:00 ✕ should send a batch request with a pinned peer (1203 ms) 22:16:00 ✕ should stop after max retry attempts (1124 ms) 22:16:00 22:16:00 ● ReqResp › should hit a rate limit if too many requests are made in quick succession 22:16:00 22:16:00 expect(received).toBeDefined() 22:16:00 22:16:00 Received: undefined 22:16:00 22:16:00 145 | // Check that one of the responses gets a rate limit response 22:16:00 146 | const rateLimitResponse = responses.find(response => response?.status === ReqRespStatus.RATE_LIMIT_EXCEEDED); 22:16:00 > 147 | expect(rateLimitResponse).toBeDefined(); 22:16:00 | ^ 22:16:00 148 | 22:16:00 149 | // Make sure the error message is logged 22:16:00 150 | const errorMessage = `Rate limit exceeded DeniedPeer for ${ 22:16:00 22:16:00 at Object.toBeDefined (services/reqresp/reqresp.test.ts:147:31) 22:16:00 22:16:00 ● ReqResp › Batch requests › should send a batch request between many peers 22:16:00 22:16:00 expect(jest.fn()).toHaveBeenCalledTimes(expected) 22:16:00 22:16:00 Expected number of calls: 9 22:16:00 Received number of calls: 11 22:16:00 22:16:00 474 | 22:16:00 475 | // Expect one request to have been sent to each peer 22:16:00 > 476 | expect(sendRequestToPeerSpy).toHaveBeenCalledTimes(batchSize); 22:16:00 | ^ 22:16:00 477 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 22:16:00 478 | expect.objectContaining({ 22:16:00 479 | publicKey: nodes[1].p2p.peerId.publicKey, 22:16:00 22:16:00 at Object.toHaveBeenCalledTimes (services/reqresp/reqresp.test.ts:476:36) 22:16:00 22:16:00 ● ReqResp › Batch requests › should send a batch request with a pinned peer 22:16:00 22:16:00 expect(jest.fn()).toHaveBeenCalledWith(...expected) 22:16:00 22:16:00 Expected: ObjectContaining {"publicKey": [8, 1, 18, 32, 152, 227, 82, 148, 239, 206, …]}, "/aztec/req/ping/0.1.0", {"data": [112, 105, 110, 103, 51], "type": "Buffer"} 22:16:00 Received 22:16:00 1 22:16:00 "12D3KooWPRCzVSt4yjNuFmqXiQWmP4DG3Nu6gHN2F83kMrBmnig4", 22:16:00 "/aztec/req/ping/0.1.0", 22:16:00 @@ -2,9 +2,9 @@ 22:16:00 "data": Array [ 22:16:00 112, 22:16:00 105, 22:16:00 110, 22:16:00 103, 22:16:00 - 51, 22:16:00 + 48, 22:16:00 ], 22:16:00 "type": "Buffer", 22:16:00 }, 22:16:00 2 22:16:00 "12D3KooWQaGZJhAnGDHeiewn1QuAJFANYur3cf6uiQa9StL7PjHN", 22:16:00 "/aztec/req/ping/0.1.0", 22:16:00 @@ -2,9 +2,9 @@ 22:16:00 "data": Array [ 22:16:00 112, 22:16:00 105, 22:16:00 110, 22:16:00 103, 22:16:00 - 51, 22:16:00 + 49, 22:16:00 ], 22:16:00 "type": "Buffer", 22:16:00 }, 22:16:00 3 22:16:00 "12D3KooWL7BBdd2yjyxWqhCshuhhyyazw5QUEeMxKBs2X3bZQXSm", 22:16:00 "/aztec/req/ping/0.1.0", 22:16:00 @@ -2,9 +2,9 @@ 22:16:00 "data": Array [ 22:16:00 112, 22:16:00 105, 22:16:00 110, 22:16:00 103, 22:16:00 - 51, 22:16:00 + 48, 22:16:00 ], 22:16:00 "type": "Buffer", 22:16:00 }, 22:16:00 22:16:00 Number of calls: 12 22:16:00 22:16:00 516 | // Expect pinned peer to have received all requests 22:16:00 517 | for (let i = 0; i < batchSize; i++) { 22:16:00 > 518 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 22:16:00 | ^ 22:16:00 519 | expect.objectContaining({ publicKey: nodes[1].p2p.peerId.publicKey }), 22:16:00 520 | ReqRespSubProtocol.PING, 22:16:00 521 | Buffer.from(`ping${i}`), 22:16:00 22:16:00 at Object.toHaveBeenCalledWith (services/reqresp/reqresp.test.ts:518:38) 22:16:00 22:16:00 ● ReqResp › Batch requests › should stop after max retry attempts 22:16:00 22:16:00 expect(received).toEqual(expected) // deep equality 22:16:00 22:16:00 - Expected - 11 22:16:00 + Received + 3 22:16:00 22:16:00 @@ -74,10 +74,11 @@ 22:16:00 103, 22:16:00 ], 22:16:00 "type": "Buffer", 22:16:00 }, 22:16:00 }, 22:16:00 + undefined, 22:16:00 RequestableBuffer { 22:16:00 "buffer": Object { 22:16:00 "data": Array [ 22:16:00 112, 22:16:00 111, 22:16:00 @@ -85,10 +86,11 @@ 22:16:00 103, 22:16:00 ], 22:16:00 "type": "Buffer", 22:16:00 }, 22:16:00 }, 22:16:00 + undefined, 22:16:00 RequestableBuffer { 22:16:00 "buffer": Object { 22:16:00 "data": Array [ 22:16:00 112, 22:16:00 111, 22:16:00 @@ -96,17 +98,7 @@ 22:16:00 103, 22:16:00 ], 22:16:00 "type": "Buffer", 22:16:00 }, 22:16:00 }, 22:16:00 - RequestableBuffer { 22:16:00 - "buffer": Object { 22:16:00 - "data": Array [ 22:16:00 - 112, 22:16:00 - 111, 22:16:00 - 110, 22:16:00 - 103, 22:16:00 - ], 22:16:00 - "type": "Buffer", 22:16:00 - }, 22:16:00 - }, 22:16:00 + undefined, 22:16:00 ] 22:16:00 22:16:00 555 | 22:16:00 556 | const res = await nodes[0].req.sendBatchRequest(ReqRespSubProtocol.PING, requests, undefined); 22:16:00 > 557 | expect(res).toEqual(expectResponses); 22:16:00 | ^ 22:16:00 558 | 22:16:00 559 | // Check that we did detect hitting a rate limit 22:16:00 560 | expect(requesterLoggerSpy).toHaveBeenCalledWith( 22:16:00 22:16:00 at Object.toEqual (services/reqresp/reqresp.test.ts:557:19) 22:16:00 22:16:00 Test Suites: 1 failed, 1 total 22:16:00 Tests: 4 failed, 12 passed, 16 total 22:16:00 Snapshots: 0 total 22:16:00 Time: 27.528 s 22:16:00 Ran all test suites matching /p2p\/src\/services\/reqresp\/reqresp.test.ts/i. 22:16:00 Force exiting Jest: Have you considered using `--detectOpenHandles` to detect async operations that kept running after all tests finished?