ACI ·

Parent Log: http://ci.aztec-labs.com/b13745b3a815110f Command: f34dabd8fae2f26b:ISOLATE=1:NAME=p2p/src/services/reqresp/reqresp.test.ts yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts Commit: https://github.com/AztecProtocol/aztec-packages/commit/f4ac84d7ac4041d505399edd8819840bb14d6c9d Env: REF_NAME=gh-readonly-queue/master/pr-15005-b1b65bcdbd44a49fbebe5d2265256dc9466e92be CURRENT_VERSION=0.87.8 CI_FULL=0 Date: Fri Jun 13 17:11:31 UTC 2025 System: ARCH=arm64 CPUS=64 MEM=247Gi HOSTNAME=pr-15005_arm64_a1-fast Resources: CPU_LIST=0-63 CPUS=2 MEM=8g TIMEOUT=600s History: http://ci.aztec-labs.com/list/history_a57cc872d7daa810_master 17:11:31 +++ id -u 17:11:31 +++ id -g 17:11:31 ++ docker run -d --name p2p_src_services_reqresp_reqresp.test.ts --net=none --cpuset-cpus=0-63 --cpus=2 --memory=8g --user 1000:1000 -v/home/aztec-dev:/home/aztec-dev --mount type=tmpfs,target=/tmp,tmpfs-size=1g --workdir /home/aztec-dev/aztec-packages -e HOME -e VERBOSE -e GIT_CONFIG_GLOBAL=/home/aztec-dev/aztec-packages/build-images/src/home/.gitconfig -e FORCE_COLOR=true -e CPUS -e MEM aztecprotocol/build:3.0 /bin/bash -c 'timeout -v 600s bash -c '\''yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts'\''' 17:11:31 + cid=e4fe8b33425a4a12db4b7110fa3586bcbaa09043e6a435cc4e9804d3575fcbe4 17:11:31 + set +x 17:11:37 [17:11:37.526] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:37 [17:11:37.530] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:37 [17:11:37.535] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:37 [17:11:37.535] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:38 [17:11:38.083] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:38 [17:11:38.084] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:38 [17:11:38.125] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:38 [17:11:38.125] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:38 [17:11:38.126] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:38 [17:11:38.127] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:39 [17:11:39.320] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:39 [17:11:39.321] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:39 [17:11:39.323] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:39 [17:11:39.325] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:39 [17:11:39.343] ERROR: p2p:reqresp Unexpected error sending request to peer: CodeError: muxer closed locally 17:11:39 at YamuxMuxer.newStream (/home/aztec-dev/aztec-packages/yarn-project/node_modules/@chainsafe/libp2p-yamux/src/muxer.ts:187:13) 17:11:39 at ConnectionImpl.newStream [as _newStream] (/home/aztec-dev/aztec-packages/yarn-project/node_modules/libp2p/src/upgrader.ts:441:41) 17:11:39 at ConnectionImpl.newStream (/home/aztec-dev/aztec-packages/yarn-project/node_modules/libp2p/src/connection/index.ts:134:31) 17:11:39 at Libp2pNode.dialProtocol (/home/aztec-dev/aztec-packages/yarn-project/node_modules/libp2p/src/libp2p.ts:291:23) 17:11:39 at processTicksAndRejections (node:internal/process/task_queues:105:5) 17:11:39 at /home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:58:33 17:11:39 at FifoMemoryQueue.process (/home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/base_memory_queue.js:110:17) { 17:11:39 code: 'ERR_MUXER_LOCAL_CLOSED', 17:11:39 props: {} 17:11:39 } {"peerId":"12D3KooWQS7Ez5kDRfFSCowiXDEtBTsyVYKcUBbMAeQr2ARh1xv7","subProtocol":"/aztec/req/ping/0.1.0"} 17:11:39 [17:11:39.439] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:39 [17:11:39.440] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:39 [17:11:39.441] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:39 [17:11:39.441] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:39 [17:11:39.442] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:39 [17:11:39.442] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:39 [17:11:39.446] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:39 [17:11:39.446] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:40 [17:11:40.535] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWDHqiVUvx5QrZEEwjjQqsdJZBtW8NTATbzobpDGJZ8sMr 17:11:40 [17:11:40.535] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 17:11:40 err: { 17:11:40 "type": "ReqRespStatusError", 17:11:40 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 17:11:40 "stack": 17:11:40 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 17:11:40 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 17:11:40 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 17:11:40 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 17:11:40 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 17:11:40 at processTicksAndRejections (node:internal/process/task_queues:105:5) 17:11:40 "status": 1 17:11:40 } 17:11:40 [17:11:40.545] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWDHqiVUvx5QrZEEwjjQqsdJZBtW8NTATbzobpDGJZ8sMr 17:11:40 [17:11:40.545] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 17:11:40 err: { 17:11:40 "type": "ReqRespStatusError", 17:11:40 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 17:11:40 "stack": 17:11:40 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 17:11:40 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 17:11:40 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 17:11:40 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 17:11:40 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 17:11:40 at processTicksAndRejections (node:internal/process/task_queues:105:5) 17:11:40 "status": 1 17:11:40 } 17:11:40 [17:11:40.550] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWDHqiVUvx5QrZEEwjjQqsdJZBtW8NTATbzobpDGJZ8sMr 17:11:40 [17:11:40.550] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 17:11:40 err: { 17:11:40 "type": "ReqRespStatusError", 17:11:40 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 17:11:40 "stack": 17:11:40 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 17:11:40 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 17:11:40 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 17:11:40 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 17:11:40 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 17:11:40 at processTicksAndRejections (node:internal/process/task_queues:105:5) 17:11:40 "status": 1 17:11:40 } 17:11:40 [17:11:40.554] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWDHqiVUvx5QrZEEwjjQqsdJZBtW8NTATbzobpDGJZ8sMr 17:11:40 [17:11:40.554] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 17:11:40 err: { 17:11:40 "type": "ReqRespStatusError", 17:11:40 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 17:11:40 "stack": 17:11:40 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 17:11:40 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 17:11:40 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 17:11:40 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 17:11:40 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 17:11:40 at processTicksAndRejections (node:internal/process/task_queues:105:5) 17:11:40 "status": 1 17:11:40 } 17:11:40 [17:11:40.558] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWDHqiVUvx5QrZEEwjjQqsdJZBtW8NTATbzobpDGJZ8sMr 17:11:40 [17:11:40.558] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 17:11:40 err: { 17:11:40 "type": "ReqRespStatusError", 17:11:40 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 17:11:40 "stack": 17:11:40 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 17:11:40 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 17:11:40 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 17:11:40 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 17:11:40 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 17:11:40 at processTicksAndRejections (node:internal/process/task_queues:105:5) 17:11:40 "status": 1 17:11:40 } 17:11:40 [17:11:40.564] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:40 [17:11:40.565] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:40 [17:11:40.568] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:40 [17:11:40.568] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:41 [17:11:41.918] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:41 [17:11:41.918] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:41 [17:11:41.924] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:41 [17:11:41.924] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:42 [17:11:42.993] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:42 [17:11:42.993] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:42 [17:11:42.996] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:42 [17:11:42.997] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:44 [17:11:44.069] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:44 [17:11:44.070] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:44 [17:11:44.073] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:44 [17:11:44.073] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:47 [17:11:47.111] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:47 [17:11:47.111] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:47 [17:11:47.114] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:47 [17:11:47.115] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:52 [17:11:52.278] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:52 [17:11:52.279] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:52 [17:11:52.280] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:52 [17:11:52.280] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:52 [17:11:52.281] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:52 [17:11:52.281] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:52 [17:11:52.284] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:52 [17:11:52.284] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:52 [17:11:52.299] WARN: foundation:memory_fifo Discarding item because queue is flushing 17:11:52 [17:11:52.302] ERROR: p2p:reqresp Unexpected error sending request to peer: Error: Could not enqueue function 17:11:52 at /home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:65:24 17:11:52 at new Promise (<anonymous>) 17:11:52 at SerialQueue.put (/home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:55:16) 17:11:52 at ConnectionSampler.put [as dialProtocol] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/connection-sampler/connection_sampler.ts:204:43) 17:11:52 at ReqResp.dialProtocol (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:449:45) 17:11:52 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 17:11:52 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 17:11:52 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 17:11:52 at TimeoutTask.requestFunction [as fn] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:200:26) {"peerId":"12D3KooWBk9pFthAFVSPg5oQZuBHar2XNLBnqQzbwAZLmHePerMr","subProtocol":"/aztec/req/tx/0.1.0"} 17:11:53 [17:11:53.371] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:53 [17:11:53.371] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:53 [17:11:53.374] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:53 [17:11:53.375] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:54 [17:11:54.427] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:54 [17:11:54.427] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:54 [17:11:54.430] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:54 [17:11:54.431] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:55 [17:11:55.468] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:55 [17:11:55.469] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:55 [17:11:55.472] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:55 [17:11:55.472] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:56 [17:11:56.679] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:56 [17:11:56.679] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:56 [17:11:56.682] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:56 [17:11:56.683] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:57 [17:11:57.868] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWNaqtpMeZXThA7MEDW347g9wRuVzDfarYk1QFUGbfU8JH 17:11:57 [17:11:57.868] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 17:11:57 err: { 17:11:57 "type": "ReqRespStatusError", 17:11:57 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 17:11:57 "stack": 17:11:57 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 17:11:57 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 17:11:57 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 17:11:57 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 17:11:57 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 17:11:57 at processTicksAndRejections (node:internal/process/task_queues:105:5) 17:11:57 "status": 1 17:11:57 } 17:11:57 [17:11:57.889] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:57 [17:11:57.890] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:57 [17:11:57.891] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:57 [17:11:57.891] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:57 [17:11:57.894] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:57 [17:11:57.894] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:59 [17:11:59.124] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:59 [17:11:59.124] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:59 [17:11:59.125] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:59 [17:11:59.125] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:59 [17:11:59.126] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:59 [17:11:59.126] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:11:59 [17:11:59.129] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:11:59 [17:11:59.130] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:12:00 [17:12:00.246] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAbwmxu9miacNJKMmBQBwZyuWoBHY1pic4bWP86UdFTC8 17:12:00 [17:12:00.246] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 17:12:00 err: { 17:12:00 "type": "ReqRespStatusError", 17:12:00 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 17:12:00 "stack": 17:12:00 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 17:12:00 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 17:12:00 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 17:12:00 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 17:12:00 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 17:12:00 at processTicksAndRejections (node:internal/process/task_queues:105:5) 17:12:00 "status": 1 17:12:00 } 17:12:00 [17:12:00.250] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAbwmxu9miacNJKMmBQBwZyuWoBHY1pic4bWP86UdFTC8 17:12:00 [17:12:00.251] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 17:12:00 err: { 17:12:00 "type": "ReqRespStatusError", 17:12:00 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 17:12:00 "stack": 17:12:00 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 17:12:00 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 17:12:00 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 17:12:00 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 17:12:00 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 17:12:00 at processTicksAndRejections (node:internal/process/task_queues:105:5) 17:12:00 "status": 1 17:12:00 } 17:12:00 [17:12:00.265] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAbwmxu9miacNJKMmBQBwZyuWoBHY1pic4bWP86UdFTC8 17:12:00 [17:12:00.265] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 17:12:00 err: { 17:12:00 "type": "ReqRespStatusError", 17:12:00 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 17:12:00 "stack": 17:12:00 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 17:12:00 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 17:12:00 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 17:12:00 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 17:12:00 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 17:12:00 at processTicksAndRejections (node:internal/process/task_queues:105:5) 17:12:00 "status": 1 17:12:00 } 17:12:00 [17:12:00.277] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:12:00 [17:12:00.277] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:12:00 [17:12:00.278] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:12:00 [17:12:00.279] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:12:00 [17:12:00.282] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 17:12:00 [17:12:00.282] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 17:12:00 FAIL src/services/reqresp/reqresp.test.ts (28.09 s) 17:12:00 ReqResp 17:12:00 ✓ should perform a ping request (1210 ms) 17:12:00 ✓ should handle gracefully if a peer connected peer is offline (587 ms) 17:12:00 ✓ should request from a later peer if other peers are offline (1349 ms) 17:12:00 ✕ should hit a rate limit if too many requests are made in quick succession (1092 ms) 17:12:00 Tx req protocol 17:12:00 ✓ can request a Tx from TxHash (1355 ms) 17:12:00 ✓ handles returning empty buffers (1075 ms) 17:12:00 ✓ does not crash if tx hash returns undefined (1073 ms) 17:12:00 ✓ should hit individual timeout if nothing is returned over the stream (3041 ms) 17:12:00 ✓ should hit collective timeout if nothing is returned over the stream from multiple peers (5184 ms) 17:12:00 ✓ should penalize peer if transaction validation fails (1076 ms) 17:12:00 Goodbye protocol 17:12:00 ✓ should send a goodbye message to a peer (1054 ms) 17:12:00 ✓ should not close stream when handling a goodbye message received from peer (1041 ms) 17:12:00 Block protocol 17:12:00 ✓ should handle block requests (1215 ms) 17:12:00 Batch requests 17:12:00 ✕ should send a batch request between many peers (1211 ms) 17:12:00 ✕ should send a batch request with a pinned peer (1241 ms) 17:12:00 ✕ should stop after max retry attempts (1147 ms) 17:12:00 17:12:00 ● ReqResp › should hit a rate limit if too many requests are made in quick succession 17:12:00 17:12:00 expect(received).toBeDefined() 17:12:00 17:12:00 Received: undefined 17:12:00 17:12:00 145 | // Check that one of the responses gets a rate limit response 17:12:00 146 | const rateLimitResponse = responses.find(response => response?.status === ReqRespStatus.RATE_LIMIT_EXCEEDED); 17:12:00 > 147 | expect(rateLimitResponse).toBeDefined(); 17:12:00 | ^ 17:12:00 148 | 17:12:00 149 | // Make sure the error message is logged 17:12:00 150 | const errorMessage = `Rate limit exceeded DeniedPeer for ${ 17:12:00 17:12:00 at Object.toBeDefined (services/reqresp/reqresp.test.ts:147:31) 17:12:00 17:12:00 ● ReqResp › Batch requests › should send a batch request between many peers 17:12:00 17:12:00 expect(jest.fn()).toHaveBeenCalledTimes(expected) 17:12:00 17:12:00 Expected number of calls: 9 17:12:00 Received number of calls: 11 17:12:00 17:12:00 474 | 17:12:00 475 | // Expect one request to have been sent to each peer 17:12:00 > 476 | expect(sendRequestToPeerSpy).toHaveBeenCalledTimes(batchSize); 17:12:00 | ^ 17:12:00 477 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 17:12:00 478 | expect.objectContaining({ 17:12:00 479 | publicKey: nodes[1].p2p.peerId.publicKey, 17:12:00 17:12:00 at Object.toHaveBeenCalledTimes (services/reqresp/reqresp.test.ts:476:36) 17:12:00 17:12:00 ● ReqResp › Batch requests › should send a batch request with a pinned peer 17:12:00 17:12:00 expect(jest.fn()).toHaveBeenCalledWith(...expected) 17:12:00 17:12:00 Expected: ObjectContaining {"publicKey": [8, 1, 18, 32, 177, 63, 202, 112, 235, 131, …]}, "/aztec/req/ping/0.1.0", {"data": [112, 105, 110, 103, 51], "type": "Buffer"} 17:12:00 Received 17:12:00 1 17:12:00 "12D3KooWPQXPfWtWENyDPoAEEY8AyAJWshj1YTGJP4J3HKAcayhh", 17:12:00 "/aztec/req/ping/0.1.0", 17:12:00 @@ -2,9 +2,9 @@ 17:12:00 "data": Array [ 17:12:00 112, 17:12:00 105, 17:12:00 110, 17:12:00 103, 17:12:00 - 51, 17:12:00 + 48, 17:12:00 ], 17:12:00 "type": "Buffer", 17:12:00 }, 17:12:00 2 17:12:00 "12D3KooWP8MmSR4qF4UAtQyG7mgyYdtrd1udEVNhbYCgRFaRHB3n", 17:12:00 "/aztec/req/ping/0.1.0", 17:12:00 @@ -2,9 +2,9 @@ 17:12:00 "data": Array [ 17:12:00 112, 17:12:00 105, 17:12:00 110, 17:12:00 103, 17:12:00 - 51, 17:12:00 + 49, 17:12:00 ], 17:12:00 "type": "Buffer", 17:12:00 }, 17:12:00 3 17:12:00 "12D3KooWMkGkME41WDFBhcFefz2U6vvfhv2dx5GkkMExzQDAvixw", 17:12:00 "/aztec/req/ping/0.1.0", 17:12:00 @@ -2,9 +2,9 @@ 17:12:00 "data": Array [ 17:12:00 112, 17:12:00 105, 17:12:00 110, 17:12:00 103, 17:12:00 - 51, 17:12:00 + 48, 17:12:00 ], 17:12:00 "type": "Buffer", 17:12:00 }, 17:12:00 17:12:00 Number of calls: 15 17:12:00 17:12:00 516 | // Expect pinned peer to have received all requests 17:12:00 517 | for (let i = 0; i < batchSize; i++) { 17:12:00 > 518 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 17:12:00 | ^ 17:12:00 519 | expect.objectContaining({ publicKey: nodes[1].p2p.peerId.publicKey }), 17:12:00 520 | ReqRespSubProtocol.PING, 17:12:00 521 | Buffer.from(`ping${i}`), 17:12:00 17:12:00 at Object.toHaveBeenCalledWith (services/reqresp/reqresp.test.ts:518:38) 17:12:00 17:12:00 ● ReqResp › Batch requests › should stop after max retry attempts 17:12:00 17:12:00 expect(received).toEqual(expected) // deep equality 17:12:00 17:12:00 - Expected - 11 17:12:00 + Received + 3 17:12:00 17:12:00 @@ -74,10 +74,11 @@ 17:12:00 103, 17:12:00 ], 17:12:00 "type": "Buffer", 17:12:00 }, 17:12:00 }, 17:12:00 + undefined, 17:12:00 RequestableBuffer { 17:12:00 "buffer": Object { 17:12:00 "data": Array [ 17:12:00 112, 17:12:00 111, 17:12:00 @@ -85,10 +86,11 @@ 17:12:00 103, 17:12:00 ], 17:12:00 "type": "Buffer", 17:12:00 }, 17:12:00 }, 17:12:00 + undefined, 17:12:00 RequestableBuffer { 17:12:00 "buffer": Object { 17:12:00 "data": Array [ 17:12:00 112, 17:12:00 111, 17:12:00 @@ -96,17 +98,7 @@ 17:12:00 103, 17:12:00 ], 17:12:00 "type": "Buffer", 17:12:00 }, 17:12:00 }, 17:12:00 - RequestableBuffer { 17:12:00 - "buffer": Object { 17:12:00 - "data": Array [ 17:12:00 - 112, 17:12:00 - 111, 17:12:00 - 110, 17:12:00 - 103, 17:12:00 - ], 17:12:00 - "type": "Buffer", 17:12:00 - }, 17:12:00 - }, 17:12:00 + undefined, 17:12:00 ] 17:12:00 17:12:00 555 | 17:12:00 556 | const res = await nodes[0].req.sendBatchRequest(ReqRespSubProtocol.PING, requests, undefined); 17:12:00 > 557 | expect(res).toEqual(expectResponses); 17:12:00 | ^ 17:12:00 558 | 17:12:00 559 | // Check that we did detect hitting a rate limit 17:12:00 560 | expect(requesterLoggerSpy).toHaveBeenCalledWith( 17:12:00 17:12:00 at Object.toEqual (services/reqresp/reqresp.test.ts:557:19) 17:12:00 17:12:00 Test Suites: 1 failed, 1 total 17:12:00 Tests: 4 failed, 12 passed, 16 total 17:12:00 Snapshots: 0 total 17:12:00 Time: 28.151 s 17:12:00 Ran all test suites matching /p2p\/src\/services\/reqresp\/reqresp.test.ts/i. 17:12:00 Force exiting Jest: Have you considered using `--detectOpenHandles` to detect async operations that kept running after all tests finished?