ACI ·

Parent Log: http://ci.aztec-labs.com/b1869ed3b108a335 Command: 3f83527fe89c0976:ISOLATE=1:NAME=p2p/src/services/reqresp/reqresp.test.ts yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts Commit: https://github.com/AztecProtocol/aztec-packages/commit/b1b65bcdbd44a49fbebe5d2265256dc9466e92be Env: REF_NAME=gh-readonly-queue/master/pr-15044-267ac35e8ccb487bdc79bbe12d428f4c8b4d4d9f CURRENT_VERSION=0.87.8 CI_FULL=0 Date: Fri Jun 13 16:27:59 UTC 2025 System: ARCH=arm64 CPUS=64 MEM=247Gi HOSTNAME=pr-15044_arm64_a1-fast Resources: CPU_LIST=0-63 CPUS=2 MEM=8g TIMEOUT=600s History: http://ci.aztec-labs.com/list/history_a57cc872d7daa810_master 16:27:59 +++ id -u 16:27:59 +++ id -g 16:27:59 ++ docker run -d --name p2p_src_services_reqresp_reqresp.test.ts --net=none --cpuset-cpus=0-63 --cpus=2 --memory=8g --user 1000:1000 -v/home/aztec-dev:/home/aztec-dev --mount type=tmpfs,target=/tmp,tmpfs-size=1g --workdir /home/aztec-dev/aztec-packages -e HOME -e VERBOSE -e GIT_CONFIG_GLOBAL=/home/aztec-dev/aztec-packages/build-images/src/home/.gitconfig -e FORCE_COLOR=true -e CPUS -e MEM aztecprotocol/build:3.0 /bin/bash -c 'timeout -v 600s bash -c '\''yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts'\''' 16:27:59 + cid=2729f6b04c28e5b8785b1fb20a507d3c83b4141af828ffce228faf2aa0c57431 16:27:59 + set +x 16:28:05 [16:28:05.949] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:05 [16:28:05.953] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:05 [16:28:05.957] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:05 [16:28:05.958] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:06 [16:28:06.504] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:06 [16:28:06.505] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:06 [16:28:06.545] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:06 [16:28:06.546] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:06 [16:28:06.547] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:06 [16:28:06.548] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:07 [16:28:07.744] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:07 [16:28:07.745] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:07 [16:28:07.747] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:07 [16:28:07.749] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:07 [16:28:07.824] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:07 [16:28:07.825] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:07 [16:28:07.826] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:07 [16:28:07.826] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:07 [16:28:07.827] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:07 [16:28:07.828] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:07 [16:28:07.831] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:07 [16:28:07.831] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:08 [16:28:08.914] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWCjWdqDDdLq1MyutLPzTHN5or1Lt7iuSntYJ4gPavENqz 16:28:08 [16:28:08.915] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 16:28:08 err: { 16:28:08 "type": "ReqRespStatusError", 16:28:08 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 16:28:08 "stack": 16:28:08 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 16:28:08 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 16:28:08 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 16:28:08 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 16:28:08 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 16:28:08 at processTicksAndRejections (node:internal/process/task_queues:105:5) 16:28:08 "status": 1 16:28:08 } 16:28:08 [16:28:08.924] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWCjWdqDDdLq1MyutLPzTHN5or1Lt7iuSntYJ4gPavENqz 16:28:08 [16:28:08.925] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 16:28:08 err: { 16:28:08 "type": "ReqRespStatusError", 16:28:08 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 16:28:08 "stack": 16:28:08 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 16:28:08 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 16:28:08 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 16:28:08 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 16:28:08 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 16:28:08 at processTicksAndRejections (node:internal/process/task_queues:105:5) 16:28:08 "status": 1 16:28:08 } 16:28:08 [16:28:08.930] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWCjWdqDDdLq1MyutLPzTHN5or1Lt7iuSntYJ4gPavENqz 16:28:08 [16:28:08.930] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 16:28:08 err: { 16:28:08 "type": "ReqRespStatusError", 16:28:08 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 16:28:08 "stack": 16:28:08 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 16:28:08 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 16:28:08 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 16:28:08 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 16:28:08 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 16:28:08 at processTicksAndRejections (node:internal/process/task_queues:105:5) 16:28:08 "status": 1 16:28:08 } 16:28:08 [16:28:08.935] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWCjWdqDDdLq1MyutLPzTHN5or1Lt7iuSntYJ4gPavENqz 16:28:08 [16:28:08.935] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 16:28:08 err: { 16:28:08 "type": "ReqRespStatusError", 16:28:08 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 16:28:08 "stack": 16:28:08 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 16:28:08 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 16:28:08 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 16:28:08 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 16:28:08 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 16:28:08 at processTicksAndRejections (node:internal/process/task_queues:105:5) 16:28:08 "status": 1 16:28:08 } 16:28:08 [16:28:08.940] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWCjWdqDDdLq1MyutLPzTHN5or1Lt7iuSntYJ4gPavENqz 16:28:08 [16:28:08.940] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 16:28:08 err: { 16:28:08 "type": "ReqRespStatusError", 16:28:08 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 16:28:08 "stack": 16:28:08 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 16:28:08 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 16:28:08 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 16:28:08 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 16:28:08 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 16:28:08 at processTicksAndRejections (node:internal/process/task_queues:105:5) 16:28:08 "status": 1 16:28:08 } 16:28:08 [16:28:08.947] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:08 [16:28:08.948] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:08 [16:28:08.951] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:08 [16:28:08.952] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:10 [16:28:10.313] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:10 [16:28:10.313] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:10 [16:28:10.316] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:10 [16:28:10.317] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:11 [16:28:11.389] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:11 [16:28:11.389] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:11 [16:28:11.392] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:11 [16:28:11.393] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:12 [16:28:12.461] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:12 [16:28:12.461] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:12 [16:28:12.464] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:12 [16:28:12.464] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:15 [16:28:15.503] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:15 [16:28:15.504] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:15 [16:28:15.507] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:15 [16:28:15.507] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:20 [16:28:20.697] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:20 [16:28:20.697] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:20 [16:28:20.698] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:20 [16:28:20.699] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:20 [16:28:20.699] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:20 [16:28:20.700] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:20 [16:28:20.703] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:20 [16:28:20.703] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:20 [16:28:20.716] WARN: foundation:memory_fifo Discarding item because queue is flushing 16:28:20 [16:28:20.719] ERROR: p2p:reqresp Unexpected error sending request to peer: Error: Could not enqueue function 16:28:20 at /home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:65:24 16:28:20 at new Promise (<anonymous>) 16:28:20 at SerialQueue.put (/home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:55:16) 16:28:20 at ConnectionSampler.put [as dialProtocol] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/connection-sampler/connection_sampler.ts:204:43) 16:28:20 at ReqResp.dialProtocol (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:449:45) 16:28:20 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 16:28:20 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 16:28:20 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 16:28:20 at TimeoutTask.requestFunction [as fn] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:200:26) {"peerId":"12D3KooWLRUSXzerq1eqWC1c3rcExkR5RX36eSFymE8W8yDNbLhV","subProtocol":"/aztec/req/tx/0.1.0"} 16:28:21 [16:28:21.792] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:21 [16:28:21.793] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:21 [16:28:21.796] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:21 [16:28:21.796] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:22 [16:28:22.832] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:22 [16:28:22.833] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:22 [16:28:22.836] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:22 [16:28:22.836] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:23 [16:28:23.875] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:23 [16:28:23.875] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:23 [16:28:23.878] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:23 [16:28:23.878] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:25 [16:28:25.085] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:25 [16:28:25.086] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:25 [16:28:25.089] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:25 [16:28:25.089] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:26 [16:28:26.227] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWNKgG3cbBxHWqAf9eFZ7T3B6gsbzUSHeqa2JinUJXfjhk 16:28:26 [16:28:26.227] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 16:28:26 err: { 16:28:26 "type": "ReqRespStatusError", 16:28:26 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 16:28:26 "stack": 16:28:26 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 16:28:26 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 16:28:26 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 16:28:26 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 16:28:26 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 16:28:26 at processTicksAndRejections (node:internal/process/task_queues:105:5) 16:28:26 "status": 1 16:28:26 } 16:28:26 [16:28:26.248] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:26 [16:28:26.249] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:26 [16:28:26.250] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:26 [16:28:26.250] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:26 [16:28:26.253] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:26 [16:28:26.254] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:27 [16:28:27.508] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:27 [16:28:27.508] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:27 [16:28:27.509] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:27 [16:28:27.509] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:27 [16:28:27.510] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:27 [16:28:27.510] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:27 [16:28:27.514] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:27 [16:28:27.514] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:28 [16:28:28.632] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWFJk4a8tkHhAZqWE1UfMrnPCaviUyneJEaFNccitFacao 16:28:28 [16:28:28.633] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 16:28:28 err: { 16:28:28 "type": "ReqRespStatusError", 16:28:28 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 16:28:28 "stack": 16:28:28 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 16:28:28 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 16:28:28 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 16:28:28 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 16:28:28 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 16:28:28 at processTicksAndRejections (node:internal/process/task_queues:105:5) 16:28:28 "status": 1 16:28:28 } 16:28:28 [16:28:28.636] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWFJk4a8tkHhAZqWE1UfMrnPCaviUyneJEaFNccitFacao 16:28:28 [16:28:28.636] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 16:28:28 err: { 16:28:28 "type": "ReqRespStatusError", 16:28:28 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 16:28:28 "stack": 16:28:28 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 16:28:28 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 16:28:28 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 16:28:28 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 16:28:28 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 16:28:28 at processTicksAndRejections (node:internal/process/task_queues:105:5) 16:28:28 "status": 1 16:28:28 } 16:28:28 [16:28:28.651] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWFJk4a8tkHhAZqWE1UfMrnPCaviUyneJEaFNccitFacao 16:28:28 [16:28:28.651] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 16:28:28 err: { 16:28:28 "type": "ReqRespStatusError", 16:28:28 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 16:28:28 "stack": 16:28:28 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 16:28:28 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 16:28:28 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 16:28:28 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 16:28:28 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 16:28:28 at processTicksAndRejections (node:internal/process/task_queues:105:5) 16:28:28 "status": 1 16:28:28 } 16:28:28 [16:28:28.661] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:28 [16:28:28.662] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:28 [16:28:28.663] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:28 [16:28:28.663] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:28 [16:28:28.666] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 16:28:28 [16:28:28.666] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 16:28:28 FAIL src/services/reqresp/reqresp.test.ts (28.153 s) 16:28:28 ReqResp 16:28:28 ✓ should perform a ping request (1224 ms) 16:28:28 ✓ should handle gracefully if a peer connected peer is offline (584 ms) 16:28:28 ✓ should request from a later peer if other peers are offline (1306 ms) 16:28:28 ✕ should hit a rate limit if too many requests are made in quick succession (1098 ms) 16:28:28 Tx req protocol 16:28:28 ✓ can request a Tx from TxHash (1364 ms) 16:28:28 ✓ handles returning empty buffers (1076 ms) 16:28:28 ✓ does not crash if tx hash returns undefined (1071 ms) 16:28:28 ✓ should hit individual timeout if nothing is returned over the stream (3042 ms) 16:28:28 ✓ should hit collective timeout if nothing is returned over the stream from multiple peers (5209 ms) 16:28:28 ✓ should penalize peer if transaction validation fails (1079 ms) 16:28:28 Goodbye protocol 16:28:28 ✓ should send a goodbye message to a peer (1040 ms) 16:28:28 ✓ should not close stream when handling a goodbye message received from peer (1042 ms) 16:28:28 Block protocol 16:28:28 ✓ should handle block requests (1211 ms) 16:28:28 Batch requests 16:28:28 ✕ should send a batch request between many peers (1168 ms) 16:28:28 ✕ should send a batch request with a pinned peer (1273 ms) 16:28:28 ✕ should stop after max retry attempts (1139 ms) 16:28:28 16:28:28 ● ReqResp › should hit a rate limit if too many requests are made in quick succession 16:28:28 16:28:28 expect(received).toBeDefined() 16:28:28 16:28:28 Received: undefined 16:28:28 16:28:28 145 | // Check that one of the responses gets a rate limit response 16:28:28 146 | const rateLimitResponse = responses.find(response => response?.status === ReqRespStatus.RATE_LIMIT_EXCEEDED); 16:28:28 > 147 | expect(rateLimitResponse).toBeDefined(); 16:28:28 | ^ 16:28:28 148 | 16:28:28 149 | // Make sure the error message is logged 16:28:28 150 | const errorMessage = `Rate limit exceeded DeniedPeer for ${ 16:28:28 16:28:28 at Object.toBeDefined (services/reqresp/reqresp.test.ts:147:31) 16:28:28 16:28:28 ● ReqResp › Batch requests › should send a batch request between many peers 16:28:28 16:28:28 expect(jest.fn()).toHaveBeenCalledTimes(expected) 16:28:28 16:28:28 Expected number of calls: 9 16:28:28 Received number of calls: 11 16:28:28 16:28:28 474 | 16:28:28 475 | // Expect one request to have been sent to each peer 16:28:28 > 476 | expect(sendRequestToPeerSpy).toHaveBeenCalledTimes(batchSize); 16:28:28 | ^ 16:28:28 477 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 16:28:28 478 | expect.objectContaining({ 16:28:28 479 | publicKey: nodes[1].p2p.peerId.publicKey, 16:28:28 16:28:28 at Object.toHaveBeenCalledTimes (services/reqresp/reqresp.test.ts:476:36) 16:28:28 16:28:28 ● ReqResp › Batch requests › should send a batch request with a pinned peer 16:28:28 16:28:28 expect(jest.fn()).toHaveBeenCalledWith(...expected) 16:28:28 16:28:28 Expected: ObjectContaining {"publicKey": [8, 1, 18, 32, 139, 99, 151, 8, 240, 105, …]}, "/aztec/req/ping/0.1.0", {"data": [112, 105, 110, 103, 51], "type": "Buffer"} 16:28:28 Received 16:28:28 1 16:28:28 "12D3KooWJJyJs97E3VKzeEc1DUmXLWkq7H3r4yYCCgQJpgT2K8nJ", 16:28:28 "/aztec/req/ping/0.1.0", 16:28:28 @@ -2,9 +2,9 @@ 16:28:28 "data": Array [ 16:28:28 112, 16:28:28 105, 16:28:28 110, 16:28:28 103, 16:28:28 - 51, 16:28:28 + 48, 16:28:28 ], 16:28:28 "type": "Buffer", 16:28:28 }, 16:28:28 2 16:28:28 "12D3KooWKm9bDV4PPCUDZHHHpoQRZf1kKb4ydpVt66qwmoc8tWt5", 16:28:28 "/aztec/req/ping/0.1.0", 16:28:28 @@ -2,9 +2,9 @@ 16:28:28 "data": Array [ 16:28:28 112, 16:28:28 105, 16:28:28 110, 16:28:28 103, 16:28:28 - 51, 16:28:28 + 49, 16:28:28 ], 16:28:28 "type": "Buffer", 16:28:28 }, 16:28:28 3 16:28:28 "12D3KooWKCUv7yQYBzUDdPT3YpcCdDdGu1BbwknQypNwCHzDdnvm", 16:28:28 "/aztec/req/ping/0.1.0", 16:28:28 @@ -2,9 +2,9 @@ 16:28:28 "data": Array [ 16:28:28 112, 16:28:28 105, 16:28:28 110, 16:28:28 103, 16:28:28 - 51, 16:28:28 + 48, 16:28:28 ], 16:28:28 "type": "Buffer", 16:28:28 }, 16:28:28 16:28:28 Number of calls: 15 16:28:28 16:28:28 516 | // Expect pinned peer to have received all requests 16:28:28 517 | for (let i = 0; i < batchSize; i++) { 16:28:28 > 518 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 16:28:28 | ^ 16:28:28 519 | expect.objectContaining({ publicKey: nodes[1].p2p.peerId.publicKey }), 16:28:28 520 | ReqRespSubProtocol.PING, 16:28:28 521 | Buffer.from(`ping${i}`), 16:28:28 16:28:28 at Object.toHaveBeenCalledWith (services/reqresp/reqresp.test.ts:518:38) 16:28:28 16:28:28 ● ReqResp › Batch requests › should stop after max retry attempts 16:28:28 16:28:28 expect(received).toEqual(expected) // deep equality 16:28:28 16:28:28 - Expected - 11 16:28:28 + Received + 3 16:28:28 16:28:28 @@ -74,10 +74,11 @@ 16:28:28 103, 16:28:28 ], 16:28:28 "type": "Buffer", 16:28:28 }, 16:28:28 }, 16:28:28 + undefined, 16:28:28 RequestableBuffer { 16:28:28 "buffer": Object { 16:28:28 "data": Array [ 16:28:28 112, 16:28:28 111, 16:28:28 @@ -85,10 +86,11 @@ 16:28:28 103, 16:28:28 ], 16:28:28 "type": "Buffer", 16:28:28 }, 16:28:28 }, 16:28:28 + undefined, 16:28:28 RequestableBuffer { 16:28:28 "buffer": Object { 16:28:28 "data": Array [ 16:28:28 112, 16:28:28 111, 16:28:28 @@ -96,17 +98,7 @@ 16:28:28 103, 16:28:28 ], 16:28:28 "type": "Buffer", 16:28:28 }, 16:28:28 }, 16:28:28 - RequestableBuffer { 16:28:28 - "buffer": Object { 16:28:28 - "data": Array [ 16:28:28 - 112, 16:28:28 - 111, 16:28:28 - 110, 16:28:28 - 103, 16:28:28 - ], 16:28:28 - "type": "Buffer", 16:28:28 - }, 16:28:28 - }, 16:28:28 + undefined, 16:28:28 ] 16:28:28 16:28:28 555 | 16:28:28 556 | const res = await nodes[0].req.sendBatchRequest(ReqRespSubProtocol.PING, requests, undefined); 16:28:28 > 557 | expect(res).toEqual(expectResponses); 16:28:28 | ^ 16:28:28 558 | 16:28:28 559 | // Check that we did detect hitting a rate limit 16:28:28 560 | expect(requesterLoggerSpy).toHaveBeenCalledWith( 16:28:28 16:28:28 at Object.toEqual (services/reqresp/reqresp.test.ts:557:19) 16:28:28 16:28:28 Test Suites: 1 failed, 1 total 16:28:28 Tests: 4 failed, 12 passed, 16 total 16:28:28 Snapshots: 0 total 16:28:28 Time: 28.213 s 16:28:28 Ran all test suites matching /p2p\/src\/services\/reqresp\/reqresp.test.ts/i. 16:28:28 Force exiting Jest: Have you considered using `--detectOpenHandles` to detect async operations that kept running after all tests finished?