ACI ·

Parent Log: http://ci.aztec-labs.com/bac80ed53a95468a Command: 53e9b8d3e8ba3411:ISOLATE=1:NAME=p2p/src/services/reqresp/reqresp.test.ts yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts Commit: https://github.com/AztecProtocol/aztec-packages/commit/f4ac84d7ac4041d505399edd8819840bb14d6c9d Env: REF_NAME=gh-readonly-queue/master/pr-15005-b1b65bcdbd44a49fbebe5d2265256dc9466e92be CURRENT_VERSION=0.87.8 CI_FULL=1 Date: Fri Jun 13 17:10:21 UTC 2025 System: ARCH=amd64 CPUS=128 MEM=493Gi HOSTNAME=pr-15005_amd64_x2-full Resources: CPU_LIST=0-127 CPUS=2 MEM=8g TIMEOUT=600s History: http://ci.aztec-labs.com/list/history_a57cc872d7daa810_master 17:10:21 +++ id -u 17:10:21 +++ id -g 17:10:21 ++ docker run -d --name p2p_src_services_reqresp_reqresp.test.ts --net=none --cpuset-cpus=0-127 --cpus=2 --memory=8g --user 1000:1000 -v/home/aztec-dev:/home/aztec-dev --mount type=tmpfs,target=/tmp,tmpfs-size=1g --workdir /home/aztec-dev/aztec-packages -e HOME -e VERBOSE -e GIT_CONFIG_GLOBAL=/home/aztec-dev/aztec-packages/build-images/src/home/.gitconfig -e FORCE_COLOR=true -e CPUS -e MEM aztecprotocol/build:3.0 /bin/bash -c 'timeout -v 600s bash -c '\''yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts'\''' 17:10:21 + cid=e697444708b8c9a3ae56560878f91318fccd7c14da91a5f778592315163a57ef 17:10:21 + set +x 17:10:29 [17:10:29.580] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:29 [17:10:29.585] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:29 [17:10:29.591] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:29 [17:10:29.592] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:30 [17:10:30.161] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:30 [17:10:30.162] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:30 [17:10:30.213] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:30 [17:10:30.214] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:30 [17:10:30.215] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:30 [17:10:30.216] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:31 [17:10:31.426] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:31 [17:10:31.427] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:31 [17:10:31.429] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:31 [17:10:31.431] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:31 [17:10:31.448] ERROR: p2p:reqresp Unexpected error sending request to peer: CodeError: muxer closed locally 17:10:31 at YamuxMuxer.newStream (/home/aztec-dev/aztec-packages/yarn-project/node_modules/@chainsafe/libp2p-yamux/src/muxer.ts:187:13) 17:10:31 at ConnectionImpl.newStream [as _newStream] (/home/aztec-dev/aztec-packages/yarn-project/node_modules/libp2p/src/upgrader.ts:441:41) 17:10:31 at ConnectionImpl.newStream (/home/aztec-dev/aztec-packages/yarn-project/node_modules/libp2p/src/connection/index.ts:134:31) 17:10:31 at Libp2pNode.dialProtocol (/home/aztec-dev/aztec-packages/yarn-project/node_modules/libp2p/src/libp2p.ts:291:23) 17:10:31 at processTicksAndRejections (node:internal/process/task_queues:105:5) 17:10:31 at /home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:58:33 17:10:31 at FifoMemoryQueue.process (/home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/base_memory_queue.js:110:17) { 17:10:31 code: 'ERR_MUXER_LOCAL_CLOSED', 17:10:31 props: {} 17:10:31 } {"peerId":"12D3KooWHUZEEY5ehp1bdm4JZLKvAwqSx1sXoVGMbT6zKqJhdEQA","subProtocol":"/aztec/req/ping/0.1.0"} 17:10:31 [17:10:31.549] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:31 [17:10:31.550] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:31 [17:10:31.551] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:31 [17:10:31.551] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:31 [17:10:31.552] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:31 [17:10:31.552] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:31 [17:10:31.556] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:31 [17:10:31.556] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:32 [17:10:32.644] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWHCLDjbcm7TxHAqXLnBN6m43BbsWH9DST5RUuHXeoQLrE 17:10:32 [17:10:32.645] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 17:10:32 err: { 17:10:32 "type": "ReqRespStatusError", 17:10:32 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 17:10:32 "stack": 17:10:32 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 17:10:32 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 17:10:32 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 17:10:32 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 17:10:32 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 17:10:32 at processTicksAndRejections (node:internal/process/task_queues:105:5) 17:10:32 "status": 1 17:10:32 } 17:10:32 [17:10:32.653] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWHCLDjbcm7TxHAqXLnBN6m43BbsWH9DST5RUuHXeoQLrE 17:10:32 [17:10:32.654] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 17:10:32 err: { 17:10:32 "type": "ReqRespStatusError", 17:10:32 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 17:10:32 "stack": 17:10:32 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 17:10:32 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 17:10:32 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 17:10:32 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 17:10:32 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 17:10:32 at processTicksAndRejections (node:internal/process/task_queues:105:5) 17:10:32 "status": 1 17:10:32 } 17:10:32 [17:10:32.657] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWHCLDjbcm7TxHAqXLnBN6m43BbsWH9DST5RUuHXeoQLrE 17:10:32 [17:10:32.657] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 17:10:32 err: { 17:10:32 "type": "ReqRespStatusError", 17:10:32 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 17:10:32 "stack": 17:10:32 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 17:10:32 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 17:10:32 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 17:10:32 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 17:10:32 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 17:10:32 at processTicksAndRejections (node:internal/process/task_queues:105:5) 17:10:32 "status": 1 17:10:32 } 17:10:32 [17:10:32.661] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWHCLDjbcm7TxHAqXLnBN6m43BbsWH9DST5RUuHXeoQLrE 17:10:32 [17:10:32.661] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 17:10:32 err: { 17:10:32 "type": "ReqRespStatusError", 17:10:32 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 17:10:32 "stack": 17:10:32 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 17:10:32 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 17:10:32 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 17:10:32 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 17:10:32 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 17:10:32 at processTicksAndRejections (node:internal/process/task_queues:105:5) 17:10:32 "status": 1 17:10:32 } 17:10:32 [17:10:32.664] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWHCLDjbcm7TxHAqXLnBN6m43BbsWH9DST5RUuHXeoQLrE 17:10:32 [17:10:32.664] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 17:10:32 err: { 17:10:32 "type": "ReqRespStatusError", 17:10:32 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 17:10:32 "stack": 17:10:32 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 17:10:32 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 17:10:32 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 17:10:32 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 17:10:32 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 17:10:32 at processTicksAndRejections (node:internal/process/task_queues:105:5) 17:10:32 "status": 1 17:10:32 } 17:10:32 [17:10:32.682] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:32 [17:10:32.684] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:32 [17:10:32.690] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:32 [17:10:32.691] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:34 [17:10:34.167] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:34 [17:10:34.168] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:34 [17:10:34.175] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:34 [17:10:34.175] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:35 [17:10:35.254] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:35 [17:10:35.255] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:35 [17:10:35.258] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:35 [17:10:35.259] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:36 [17:10:36.350] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:36 [17:10:36.351] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:36 [17:10:36.355] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:36 [17:10:36.356] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:39 [17:10:39.417] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:39 [17:10:39.417] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:39 [17:10:39.420] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:39 [17:10:39.421] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:44 [17:10:44.734] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:44 [17:10:44.735] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:44 [17:10:44.736] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:44 [17:10:44.737] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:44 [17:10:44.738] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:44 [17:10:44.738] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:44 [17:10:44.742] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:44 [17:10:44.742] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:44 [17:10:44.766] WARN: foundation:memory_fifo Discarding item because queue is flushing 17:10:44 [17:10:44.771] ERROR: p2p:reqresp Unexpected error sending request to peer: Error: Could not enqueue function 17:10:44 at /home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:65:24 17:10:44 at new Promise (<anonymous>) 17:10:44 at SerialQueue.put (/home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:55:16) 17:10:44 at ConnectionSampler.put [as dialProtocol] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/connection-sampler/connection_sampler.ts:204:43) 17:10:44 at ReqResp.dialProtocol (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:449:45) 17:10:44 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 17:10:44 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 17:10:44 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 17:10:44 at TimeoutTask.requestFunction [as fn] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:200:26) {"peerId":"12D3KooWMJJaTP3RUYLm7Vd5N5TD9wiNMyX7rxgc7nrHgeZJgjKb","subProtocol":"/aztec/req/tx/0.1.0"} 17:10:45 [17:10:45.867] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:45 [17:10:45.867] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:45 [17:10:45.871] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:45 [17:10:45.872] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:46 [17:10:46.924] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:46 [17:10:46.924] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:46 [17:10:46.928] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:46 [17:10:46.928] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:47 [17:10:47.967] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:47 [17:10:47.968] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:47 [17:10:47.972] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:47 [17:10:47.972] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:49 [17:10:49.208] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:49 [17:10:49.208] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:49 [17:10:49.211] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:49 [17:10:49.211] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:50 [17:10:50.313] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWGYB8vKLWNSoy3BxMxgX8qLw177fC9vTTQCoBRxkSnk5Z 17:10:50 [17:10:50.313] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 17:10:50 err: { 17:10:50 "type": "ReqRespStatusError", 17:10:50 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 17:10:50 "stack": 17:10:50 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 17:10:50 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 17:10:50 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 17:10:50 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 17:10:50 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 17:10:50 at processTicksAndRejections (node:internal/process/task_queues:105:5) 17:10:50 "status": 1 17:10:50 } 17:10:50 [17:10:50.336] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:50 [17:10:50.337] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:50 [17:10:50.338] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:50 [17:10:50.338] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:50 [17:10:50.341] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:50 [17:10:50.341] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:51 [17:10:51.585] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:51 [17:10:51.586] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:51 [17:10:51.587] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:51 [17:10:51.587] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:51 [17:10:51.588] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:51 [17:10:51.589] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:51 [17:10:51.592] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:51 [17:10:51.593] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:52 [17:10:52.723] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWQmTJzx2uwjNR4rUzW52V1z6WanWb6mUQgYWrBAjvbvXp 17:10:52 [17:10:52.723] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 17:10:52 err: { 17:10:52 "type": "ReqRespStatusError", 17:10:52 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 17:10:52 "stack": 17:10:52 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 17:10:52 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 17:10:52 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 17:10:52 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 17:10:52 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 17:10:52 at processTicksAndRejections (node:internal/process/task_queues:105:5) 17:10:52 "status": 1 17:10:52 } 17:10:52 [17:10:52.726] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWQmTJzx2uwjNR4rUzW52V1z6WanWb6mUQgYWrBAjvbvXp 17:10:52 [17:10:52.726] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 17:10:52 err: { 17:10:52 "type": "ReqRespStatusError", 17:10:52 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 17:10:52 "stack": 17:10:52 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 17:10:52 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 17:10:52 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 17:10:52 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 17:10:52 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 17:10:52 at processTicksAndRejections (node:internal/process/task_queues:105:5) 17:10:52 "status": 1 17:10:52 } 17:10:52 [17:10:52.742] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWQmTJzx2uwjNR4rUzW52V1z6WanWb6mUQgYWrBAjvbvXp 17:10:52 [17:10:52.742] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 17:10:52 err: { 17:10:52 "type": "ReqRespStatusError", 17:10:52 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 17:10:52 "stack": 17:10:52 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 17:10:52 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 17:10:52 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 17:10:52 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 17:10:52 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 17:10:52 at processTicksAndRejections (node:internal/process/task_queues:105:5) 17:10:52 "status": 1 17:10:52 } 17:10:52 [17:10:52.754] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:52 [17:10:52.754] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:52 [17:10:52.756] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:52 [17:10:52.756] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:52 [17:10:52.760] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:10:52 [17:10:52.760] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:10:52 FAIL src/services/reqresp/reqresp.test.ts (30.213 s) 17:10:52 ReqResp 17:10:52 ✓ should perform a ping request (1307 ms) 17:10:52 ✓ should handle gracefully if a peer connected peer is offline (608 ms) 17:10:52 ✓ should request from a later peer if other peers are offline (1375 ms) 17:10:52 ✕ should hit a rate limit if too many requests are made in quick succession (1106 ms) 17:10:52 Tx req protocol 17:10:52 ✓ can request a Tx from TxHash (1478 ms) 17:10:52 ✓ handles returning empty buffers (1087 ms) 17:10:52 ✓ does not crash if tx hash returns undefined (1090 ms) 17:10:52 ✓ should hit individual timeout if nothing is returned over the stream (3064 ms) 17:10:52 ✓ should hit collective timeout if nothing is returned over the stream from multiple peers (5349 ms) 17:10:52 ✓ should penalize peer if transaction validation fails (1106 ms) 17:10:52 Goodbye protocol 17:10:52 ✓ should send a goodbye message to a peer (1050 ms) 17:10:52 ✓ should not close stream when handling a goodbye message received from peer (1044 ms) 17:10:52 Block protocol 17:10:52 ✓ should handle block requests (1242 ms) 17:10:52 Batch requests 17:10:52 ✕ should send a batch request between many peers (1141 ms) 17:10:52 ✕ should send a batch request with a pinned peer (1255 ms) 17:10:52 ✕ should stop after max retry attempts (1156 ms) 17:10:52 17:10:52 ● ReqResp › should hit a rate limit if too many requests are made in quick succession 17:10:52 17:10:52 expect(received).toBeDefined() 17:10:52 17:10:52 Received: undefined 17:10:52 17:10:52 145 | // Check that one of the responses gets a rate limit response 17:10:52 146 | const rateLimitResponse = responses.find(response => response?.status === ReqRespStatus.RATE_LIMIT_EXCEEDED); 17:10:52 > 147 | expect(rateLimitResponse).toBeDefined(); 17:10:52 | ^ 17:10:52 148 | 17:10:52 149 | // Make sure the error message is logged 17:10:52 150 | const errorMessage = `Rate limit exceeded DeniedPeer for ${ 17:10:52 17:10:52 at Object.toBeDefined (services/reqresp/reqresp.test.ts:147:31) 17:10:52 17:10:52 ● ReqResp › Batch requests › should send a batch request between many peers 17:10:52 17:10:52 expect(jest.fn()).toHaveBeenCalledTimes(expected) 17:10:52 17:10:52 Expected number of calls: 9 17:10:52 Received number of calls: 11 17:10:52 17:10:52 474 | 17:10:52 475 | // Expect one request to have been sent to each peer 17:10:52 > 476 | expect(sendRequestToPeerSpy).toHaveBeenCalledTimes(batchSize); 17:10:52 | ^ 17:10:52 477 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 17:10:52 478 | expect.objectContaining({ 17:10:52 479 | publicKey: nodes[1].p2p.peerId.publicKey, 17:10:52 17:10:52 at Object.toHaveBeenCalledTimes (services/reqresp/reqresp.test.ts:476:36) 17:10:52 17:10:52 ● ReqResp › Batch requests › should send a batch request with a pinned peer 17:10:52 17:10:52 expect(jest.fn()).toHaveBeenCalledWith(...expected) 17:10:52 17:10:52 Expected: ObjectContaining {"publicKey": [8, 1, 18, 32, 56, 145, 149, 37, 250, 172, …]}, "/aztec/req/ping/0.1.0", {"data": [112, 105, 110, 103, 51], "type": "Buffer"} 17:10:52 Received 17:10:52 1 17:10:52 "12D3KooWB7WB2fPjtr4RftZqg5q3aF7ufRDTU7j9R3DRMRhEPrKT", 17:10:52 "/aztec/req/ping/0.1.0", 17:10:52 @@ -2,9 +2,9 @@ 17:10:52 "data": Array [ 17:10:52 112, 17:10:52 105, 17:10:52 110, 17:10:52 103, 17:10:52 - 51, 17:10:52 + 48, 17:10:52 ], 17:10:52 "type": "Buffer", 17:10:52 }, 17:10:52 2 17:10:52 "12D3KooWKonmytcYSgNRf3VawE4DvbNAQ1X7uU5TcbVJQh1udvNA", 17:10:52 "/aztec/req/ping/0.1.0", 17:10:52 @@ -2,9 +2,9 @@ 17:10:52 "data": Array [ 17:10:52 112, 17:10:52 105, 17:10:52 110, 17:10:52 103, 17:10:52 - 51, 17:10:52 + 49, 17:10:52 ], 17:10:52 "type": "Buffer", 17:10:52 }, 17:10:52 3 17:10:52 "12D3KooWDdBkgKTpqfpFH9vtDM44UzmRR65qctNyRoxsVCxkjRQL", 17:10:52 "/aztec/req/ping/0.1.0", 17:10:52 @@ -2,9 +2,9 @@ 17:10:52 "data": Array [ 17:10:52 112, 17:10:52 105, 17:10:52 110, 17:10:52 103, 17:10:52 - 51, 17:10:52 + 48, 17:10:52 ], 17:10:52 "type": "Buffer", 17:10:52 }, 17:10:52 17:10:52 Number of calls: 12 17:10:52 17:10:52 516 | // Expect pinned peer to have received all requests 17:10:52 517 | for (let i = 0; i < batchSize; i++) { 17:10:52 > 518 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 17:10:52 | ^ 17:10:52 519 | expect.objectContaining({ publicKey: nodes[1].p2p.peerId.publicKey }), 17:10:52 520 | ReqRespSubProtocol.PING, 17:10:52 521 | Buffer.from(`ping${i}`), 17:10:52 17:10:52 at Object.toHaveBeenCalledWith (services/reqresp/reqresp.test.ts:518:38) 17:10:52 17:10:52 ● ReqResp › Batch requests › should stop after max retry attempts 17:10:52 17:10:52 expect(received).toEqual(expected) // deep equality 17:10:52 17:10:52 - Expected - 11 17:10:52 + Received + 3 17:10:52 17:10:52 @@ -74,10 +74,11 @@ 17:10:52 103, 17:10:52 ], 17:10:52 "type": "Buffer", 17:10:52 }, 17:10:52 }, 17:10:52 + undefined, 17:10:52 RequestableBuffer { 17:10:52 "buffer": Object { 17:10:52 "data": Array [ 17:10:52 112, 17:10:52 111, 17:10:52 @@ -85,10 +86,11 @@ 17:10:52 103, 17:10:52 ], 17:10:52 "type": "Buffer", 17:10:52 }, 17:10:52 }, 17:10:52 + undefined, 17:10:52 RequestableBuffer { 17:10:52 "buffer": Object { 17:10:52 "data": Array [ 17:10:52 112, 17:10:52 111, 17:10:52 @@ -96,17 +98,7 @@ 17:10:52 103, 17:10:52 ], 17:10:52 "type": "Buffer", 17:10:52 }, 17:10:52 }, 17:10:52 - RequestableBuffer { 17:10:52 - "buffer": Object { 17:10:52 - "data": Array [ 17:10:52 - 112, 17:10:52 - 111, 17:10:52 - 110, 17:10:52 - 103, 17:10:52 - ], 17:10:52 - "type": "Buffer", 17:10:52 - }, 17:10:52 - }, 17:10:52 + undefined, 17:10:52 ] 17:10:52 17:10:52 555 | 17:10:52 556 | const res = await nodes[0].req.sendBatchRequest(ReqRespSubProtocol.PING, requests, undefined); 17:10:52 > 557 | expect(res).toEqual(expectResponses); 17:10:52 | ^ 17:10:52 558 | 17:10:52 559 | // Check that we did detect hitting a rate limit 17:10:52 560 | expect(requesterLoggerSpy).toHaveBeenCalledWith( 17:10:52 17:10:52 at Object.toEqual (services/reqresp/reqresp.test.ts:557:19) 17:10:52 17:10:52 Test Suites: 1 failed, 1 total 17:10:52 Tests: 4 failed, 12 passed, 16 total 17:10:52 Snapshots: 0 total 17:10:52 Time: 30.304 s 17:10:52 Ran all test suites matching /p2p\/src\/services\/reqresp\/reqresp.test.ts/i. 17:10:52 Force exiting Jest: Have you considered using `--detectOpenHandles` to detect async operations that kept running after all tests finished?