ACI ·

Parent Log: http://ci.aztec-labs.com/7d7b041f79629c7a Command: cdc0844b6e9d2c1e:ISOLATE=1:NAME=p2p/src/services/reqresp/reqresp.test.ts yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts Commit: https://github.com/AztecProtocol/aztec-packages/commit/3a776996a430e63770f252088c2fdd32ae17c1f7 Env: REF_NAME=gh-readonly-queue/master/pr-14716-fc32764490031c0904f476d0e3bcd7e4aa19e719 CURRENT_VERSION=0.87.7 CI_FULL=0 Date: Mon Jun 9 10:57:17 UTC 2025 System: ARCH=arm64 CPUS=64 MEM=247Gi HOSTNAME=pr-14716_arm64_a1-fast Resources: CPU_LIST=0-63 CPUS=2 MEM=8g TIMEOUT=600s History: http://ci.aztec-labs.com/list/history_a57cc872d7daa810_master 10:57:17 +++ id -u 10:57:17 +++ id -g 10:57:17 ++ docker run -d --name p2p_src_services_reqresp_reqresp.test.ts --net=none --cpuset-cpus=0-63 --cpus=2 --memory=8g --user 1000:1000 -v/home/aztec-dev:/home/aztec-dev --mount type=tmpfs,target=/tmp,tmpfs-size=1g --workdir /home/aztec-dev/aztec-packages -e HOME -e VERBOSE -e GIT_CONFIG_GLOBAL=/home/aztec-dev/aztec-packages/build-images/src/home/.gitconfig -e FORCE_COLOR=true -e CPUS -e MEM aztecprotocol/build:3.0 /bin/bash -c 'timeout -v 600s bash -c '\''yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts'\''' 10:57:18 + cid=d1397ce1e1efb3a4d46482cf75ed972f2dd629f9f775808f001c243c82b97a2e 10:57:18 + set +x 10:57:24 [10:57:24.126] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:24 [10:57:24.130] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:24 [10:57:24.135] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:24 [10:57:24.135] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:24 [10:57:24.683] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:24 [10:57:24.684] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:24 [10:57:24.724] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:24 [10:57:24.725] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:24 [10:57:24.725] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:24 [10:57:24.727] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:25 [10:57:25.922] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:25 [10:57:25.923] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:25 [10:57:25.925] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:25 [10:57:25.927] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:25 [10:57:25.944] ERROR: p2p:reqresp Unexpected error sending request to peer: CodeError: muxer closed locally 10:57:25 at YamuxMuxer.newStream (/home/aztec-dev/aztec-packages/yarn-project/node_modules/@chainsafe/libp2p-yamux/src/muxer.ts:187:13) 10:57:25 at ConnectionImpl.newStream [as _newStream] (/home/aztec-dev/aztec-packages/yarn-project/node_modules/libp2p/src/upgrader.ts:441:41) 10:57:25 at ConnectionImpl.newStream (/home/aztec-dev/aztec-packages/yarn-project/node_modules/libp2p/src/connection/index.ts:134:31) 10:57:25 at Libp2pNode.dialProtocol (/home/aztec-dev/aztec-packages/yarn-project/node_modules/libp2p/src/libp2p.ts:291:23) 10:57:25 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:57:25 at /home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:58:33 10:57:25 at FifoMemoryQueue.process (/home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/base_memory_queue.js:110:17) { 10:57:25 code: 'ERR_MUXER_LOCAL_CLOSED', 10:57:25 props: {} 10:57:25 } {"peerId":"12D3KooWMFvbrnd7AVe112vdkaVVbt7wFzo6xZxerL22V2fYtegS","subProtocol":"/aztec/req/ping/0.1.0"} 10:57:26 [10:57:26.043] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:26 [10:57:26.043] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:26 [10:57:26.044] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:26 [10:57:26.045] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:26 [10:57:26.046] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:26 [10:57:26.046] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:26 [10:57:26.049] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:26 [10:57:26.050] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:27 [10:57:27.139] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWDvedtT8hBCHdrPTjxCyL6EoCHQo8bxWZN3RGzEqfH2dV 10:57:27 [10:57:27.139] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:57:27 err: { 10:57:27 "type": "ReqRespStatusError", 10:57:27 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:57:27 "stack": 10:57:27 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:57:27 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:57:27 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:57:27 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:57:27 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:57:27 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:57:27 "status": 1 10:57:27 } 10:57:27 [10:57:27.148] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWDvedtT8hBCHdrPTjxCyL6EoCHQo8bxWZN3RGzEqfH2dV 10:57:27 [10:57:27.148] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:57:27 err: { 10:57:27 "type": "ReqRespStatusError", 10:57:27 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:57:27 "stack": 10:57:27 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:57:27 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:57:27 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:57:27 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:57:27 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:57:27 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:57:27 "status": 1 10:57:27 } 10:57:27 [10:57:27.153] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWDvedtT8hBCHdrPTjxCyL6EoCHQo8bxWZN3RGzEqfH2dV 10:57:27 [10:57:27.153] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:57:27 err: { 10:57:27 "type": "ReqRespStatusError", 10:57:27 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:57:27 "stack": 10:57:27 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:57:27 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:57:27 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:57:27 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:57:27 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:57:27 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:57:27 "status": 1 10:57:27 } 10:57:27 [10:57:27.157] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWDvedtT8hBCHdrPTjxCyL6EoCHQo8bxWZN3RGzEqfH2dV 10:57:27 [10:57:27.157] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:57:27 err: { 10:57:27 "type": "ReqRespStatusError", 10:57:27 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:57:27 "stack": 10:57:27 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:57:27 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:57:27 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:57:27 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:57:27 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:57:27 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:57:27 "status": 1 10:57:27 } 10:57:27 [10:57:27.161] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWDvedtT8hBCHdrPTjxCyL6EoCHQo8bxWZN3RGzEqfH2dV 10:57:27 [10:57:27.162] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:57:27 err: { 10:57:27 "type": "ReqRespStatusError", 10:57:27 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:57:27 "stack": 10:57:27 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:57:27 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:57:27 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:57:27 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:57:27 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:57:27 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:57:27 "status": 1 10:57:27 } 10:57:27 [10:57:27.168] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:27 [10:57:27.169] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:27 [10:57:27.172] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:27 [10:57:27.173] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:28 [10:57:28.513] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:28 [10:57:28.514] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:28 [10:57:28.519] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:28 [10:57:28.520] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:29 [10:57:29.587] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:29 [10:57:29.587] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:29 [10:57:29.590] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:29 [10:57:29.591] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:30 [10:57:30.670] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:30 [10:57:30.670] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:30 [10:57:30.673] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:30 [10:57:30.673] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:33 [10:57:33.712] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:33 [10:57:33.712] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:33 [10:57:33.715] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:33 [10:57:33.715] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:38 [10:57:38.878] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:38 [10:57:38.879] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:38 [10:57:38.880] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:38 [10:57:38.880] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:38 [10:57:38.881] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:38 [10:57:38.882] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:38 [10:57:38.884] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:38 [10:57:38.885] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:38 [10:57:38.900] WARN: foundation:memory_fifo Discarding item because queue is flushing 10:57:38 [10:57:38.902] ERROR: p2p:reqresp Unexpected error sending request to peer: Error: Could not enqueue function 10:57:38 at /home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:65:24 10:57:38 at new Promise (<anonymous>) 10:57:38 at SerialQueue.put (/home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:55:16) 10:57:38 at ConnectionSampler.put [as dialProtocol] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/connection-sampler/connection_sampler.ts:204:43) 10:57:38 at ReqResp.dialProtocol (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:449:45) 10:57:38 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:57:38 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:57:38 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:57:38 at TimeoutTask.requestFunction [as fn] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:200:26) {"peerId":"12D3KooWLrezvmPFMX27zYqJY2kF3UJwWSL7bLRnrSL6jA6u9G7u","subProtocol":"/aztec/req/tx/0.1.0"} 10:57:40 [10:57:40.010] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:40 [10:57:40.011] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:40 [10:57:40.014] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:40 [10:57:40.014] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:41 [10:57:41.052] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:41 [10:57:41.053] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:41 [10:57:41.056] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:41 [10:57:41.056] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:42 [10:57:42.093] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:42 [10:57:42.093] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:42 [10:57:42.096] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:42 [10:57:42.097] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:43 [10:57:43.304] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:43 [10:57:43.304] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:43 [10:57:43.307] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:43 [10:57:43.308] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:44 [10:57:44.419] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWKStyfec82o3Zxxms91KbyQ2KS9bLqQhk4isNdjBxg83t 10:57:44 [10:57:44.419] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:57:44 err: { 10:57:44 "type": "ReqRespStatusError", 10:57:44 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:57:44 "stack": 10:57:44 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:57:44 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:57:44 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:57:44 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:57:44 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:57:44 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:57:44 "status": 1 10:57:44 } 10:57:44 [10:57:44.439] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:44 [10:57:44.440] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:44 [10:57:44.440] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:44 [10:57:44.441] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:44 [10:57:44.443] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:44 [10:57:44.444] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:45 [10:57:45.660] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:45 [10:57:45.660] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:45 [10:57:45.661] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:45 [10:57:45.662] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:45 [10:57:45.663] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:45 [10:57:45.663] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:45 [10:57:45.666] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:45 [10:57:45.666] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:46 [10:57:46.771] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAdF8e9SkYhiGdAUzUrHCpLsiuZf83v85sEyNZpJbnwrR 10:57:46 [10:57:46.771] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:57:46 err: { 10:57:46 "type": "ReqRespStatusError", 10:57:46 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:57:46 "stack": 10:57:46 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:57:46 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:57:46 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:57:46 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:57:46 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:57:46 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:57:46 "status": 1 10:57:46 } 10:57:46 [10:57:46.775] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAdF8e9SkYhiGdAUzUrHCpLsiuZf83v85sEyNZpJbnwrR 10:57:46 [10:57:46.776] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:57:46 err: { 10:57:46 "type": "ReqRespStatusError", 10:57:46 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:57:46 "stack": 10:57:46 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:57:46 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:57:46 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:57:46 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:57:46 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:57:46 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:57:46 "status": 1 10:57:46 } 10:57:46 [10:57:46.790] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAdF8e9SkYhiGdAUzUrHCpLsiuZf83v85sEyNZpJbnwrR 10:57:46 [10:57:46.790] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:57:46 err: { 10:57:46 "type": "ReqRespStatusError", 10:57:46 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:57:46 "stack": 10:57:46 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:57:46 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:57:46 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:57:46 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:57:46 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:57:46 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:57:46 "status": 1 10:57:46 } 10:57:46 [10:57:46.800] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:46 [10:57:46.801] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:46 [10:57:46.801] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:46 [10:57:46.802] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:46 [10:57:46.804] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:57:46 [10:57:46.805] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:57:46 FAIL src/services/reqresp/reqresp.test.ts (28.055 s) 10:57:46 ReqResp 10:57:46 ✓ should perform a ping request (1206 ms) 10:57:46 ✓ should handle gracefully if a peer connected peer is offline (586 ms) 10:57:46 ✓ should request from a later peer if other peers are offline (1355 ms) 10:57:46 ✕ should hit a rate limit if too many requests are made in quick succession (1093 ms) 10:57:46 Tx req protocol 10:57:46 ✓ can request a Tx from TxHash (1345 ms) 10:57:46 ✓ handles returning empty buffers (1073 ms) 10:57:46 ✓ does not crash if tx hash returns undefined (1080 ms) 10:57:46 ✓ should hit individual timeout if nothing is returned over the stream (3042 ms) 10:57:46 ✓ should hit collective timeout if nothing is returned over the stream from multiple peers (5184 ms) 10:57:46 ✓ should penalize peer if transaction validation fails (1115 ms) 10:57:46 Goodbye protocol 10:57:46 ✓ should send a goodbye message to a peer (1040 ms) 10:57:46 ✓ should not close stream when handling a goodbye message received from peer (1041 ms) 10:57:46 Block protocol 10:57:46 ✓ should handle block requests (1215 ms) 10:57:46 Batch requests 10:57:46 ✕ should send a batch request between many peers (1135 ms) 10:57:46 ✕ should send a batch request with a pinned peer (1228 ms) 10:57:46 ✕ should stop after max retry attempts (1133 ms) 10:57:46 10:57:46 ● ReqResp › should hit a rate limit if too many requests are made in quick succession 10:57:46 10:57:46 expect(received).toBeDefined() 10:57:46 10:57:46 Received: undefined 10:57:46 10:57:46 145 | // Check that one of the responses gets a rate limit response 10:57:46 146 | const rateLimitResponse = responses.find(response => response?.status === ReqRespStatus.RATE_LIMIT_EXCEEDED); 10:57:46 > 147 | expect(rateLimitResponse).toBeDefined(); 10:57:46 | ^ 10:57:46 148 | 10:57:46 149 | // Make sure the error message is logged 10:57:46 150 | const errorMessage = `Rate limit exceeded DeniedPeer for ${ 10:57:46 10:57:46 at Object.toBeDefined (services/reqresp/reqresp.test.ts:147:31) 10:57:46 10:57:46 ● ReqResp › Batch requests › should send a batch request between many peers 10:57:46 10:57:46 expect(jest.fn()).toHaveBeenCalledTimes(expected) 10:57:46 10:57:46 Expected number of calls: 9 10:57:46 Received number of calls: 11 10:57:46 10:57:46 474 | 10:57:46 475 | // Expect one request to have been sent to each peer 10:57:46 > 476 | expect(sendRequestToPeerSpy).toHaveBeenCalledTimes(batchSize); 10:57:46 | ^ 10:57:46 477 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 10:57:46 478 | expect.objectContaining({ 10:57:46 479 | publicKey: nodes[1].p2p.peerId.publicKey, 10:57:46 10:57:46 at Object.toHaveBeenCalledTimes (services/reqresp/reqresp.test.ts:476:36) 10:57:46 10:57:46 ● ReqResp › Batch requests › should send a batch request with a pinned peer 10:57:46 10:57:46 expect(jest.fn()).toHaveBeenCalledWith(...expected) 10:57:46 10:57:46 Expected: ObjectContaining {"publicKey": [8, 1, 18, 32, 59, 5, 208, 179, 224, 199, …]}, "/aztec/req/ping/0.1.0", {"data": [112, 105, 110, 103, 51], "type": "Buffer"} 10:57:46 Received 10:57:46 1 10:57:46 "12D3KooWPPnvbsaCZku8H2NrHWJFbhGN9ZEYBfEuTq4u6AMYvEeQ", 10:57:46 "/aztec/req/ping/0.1.0", 10:57:46 @@ -2,9 +2,9 @@ 10:57:46 "data": Array [ 10:57:46 112, 10:57:46 105, 10:57:46 110, 10:57:46 103, 10:57:46 - 51, 10:57:46 + 48, 10:57:46 ], 10:57:46 "type": "Buffer", 10:57:46 }, 10:57:46 2 10:57:46 "12D3KooWEqhXFifCqmg8gQmFtxEi2woeML8moQf98qqMLeW8dFSZ", 10:57:46 "/aztec/req/ping/0.1.0", 10:57:46 @@ -2,9 +2,9 @@ 10:57:46 "data": Array [ 10:57:46 112, 10:57:46 105, 10:57:46 110, 10:57:46 103, 10:57:46 - 51, 10:57:46 + 49, 10:57:46 ], 10:57:46 "type": "Buffer", 10:57:46 }, 10:57:46 3 10:57:46 "12D3KooWDnmNBWdjkmeDEM34Z3EkVqgxuDsjJXwKPTXyEs4SpeKs", 10:57:46 "/aztec/req/ping/0.1.0", 10:57:46 @@ -2,9 +2,9 @@ 10:57:46 "data": Array [ 10:57:46 112, 10:57:46 105, 10:57:46 110, 10:57:46 103, 10:57:46 - 51, 10:57:46 + 48, 10:57:46 ], 10:57:46 "type": "Buffer", 10:57:46 }, 10:57:46 10:57:46 Number of calls: 15 10:57:46 10:57:46 516 | // Expect pinned peer to have received all requests 10:57:46 517 | for (let i = 0; i < batchSize; i++) { 10:57:46 > 518 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 10:57:46 | ^ 10:57:46 519 | expect.objectContaining({ publicKey: nodes[1].p2p.peerId.publicKey }), 10:57:46 520 | ReqRespSubProtocol.PING, 10:57:46 521 | Buffer.from(`ping${i}`), 10:57:46 10:57:46 at Object.toHaveBeenCalledWith (services/reqresp/reqresp.test.ts:518:38) 10:57:46 10:57:46 ● ReqResp › Batch requests › should stop after max retry attempts 10:57:46 10:57:46 expect(received).toEqual(expected) // deep equality 10:57:46 10:57:46 - Expected - 11 10:57:46 + Received + 3 10:57:46 10:57:46 @@ -74,10 +74,11 @@ 10:57:46 103, 10:57:46 ], 10:57:46 "type": "Buffer", 10:57:46 }, 10:57:46 }, 10:57:46 + undefined, 10:57:46 RequestableBuffer { 10:57:46 "buffer": Object { 10:57:46 "data": Array [ 10:57:46 112, 10:57:46 111, 10:57:46 @@ -85,10 +86,11 @@ 10:57:46 103, 10:57:46 ], 10:57:46 "type": "Buffer", 10:57:46 }, 10:57:46 }, 10:57:46 + undefined, 10:57:46 RequestableBuffer { 10:57:46 "buffer": Object { 10:57:46 "data": Array [ 10:57:46 112, 10:57:46 111, 10:57:46 @@ -96,17 +98,7 @@ 10:57:46 103, 10:57:46 ], 10:57:46 "type": "Buffer", 10:57:46 }, 10:57:46 }, 10:57:46 - RequestableBuffer { 10:57:46 - "buffer": Object { 10:57:46 - "data": Array [ 10:57:46 - 112, 10:57:46 - 111, 10:57:46 - 110, 10:57:46 - 103, 10:57:46 - ], 10:57:46 - "type": "Buffer", 10:57:46 - }, 10:57:46 - }, 10:57:46 + undefined, 10:57:46 ] 10:57:46 10:57:46 555 | 10:57:46 556 | const res = await nodes[0].req.sendBatchRequest(ReqRespSubProtocol.PING, requests, undefined); 10:57:46 > 557 | expect(res).toEqual(expectResponses); 10:57:46 | ^ 10:57:46 558 | 10:57:46 559 | // Check that we did detect hitting a rate limit 10:57:46 560 | expect(requesterLoggerSpy).toHaveBeenCalledWith( 10:57:46 10:57:46 at Object.toEqual (services/reqresp/reqresp.test.ts:557:19) 10:57:46 10:57:46 Test Suites: 1 failed, 1 total 10:57:46 Tests: 4 failed, 12 passed, 16 total 10:57:46 Snapshots: 0 total 10:57:46 Time: 28.116 s 10:57:46 Ran all test suites matching /p2p\/src\/services\/reqresp\/reqresp.test.ts/i. 10:57:46 Force exiting Jest: Have you considered using `--detectOpenHandles` to detect async operations that kept running after all tests finished?