ACI ·

Parent Log: http://ci.aztec-labs.com/8776af2186cb3896 Command: 59d7caa3bc5b3609:ISOLATE=1:NAME=p2p/src/services/reqresp/reqresp.test.ts yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts Commit: https://github.com/AztecProtocol/aztec-packages/commit/3a776996a430e63770f252088c2fdd32ae17c1f7 Env: REF_NAME=gh-readonly-queue/master/pr-14716-fc32764490031c0904f476d0e3bcd7e4aa19e719 CURRENT_VERSION=0.87.7 CI_FULL=1 Date: Mon Jun 9 10:52:12 UTC 2025 System: ARCH=amd64 CPUS=128 MEM=493Gi HOSTNAME=pr-14716_amd64_x1-full Resources: CPU_LIST=0-127 CPUS=2 MEM=8g TIMEOUT=600s History: http://ci.aztec-labs.com/list/history_a57cc872d7daa810_master 10:52:12 +++ id -u 10:52:12 +++ id -g 10:52:12 ++ docker run -d --name p2p_src_services_reqresp_reqresp.test.ts --net=none --cpuset-cpus=0-127 --cpus=2 --memory=8g --user 1000:1000 -v/home/aztec-dev:/home/aztec-dev --mount type=tmpfs,target=/tmp,tmpfs-size=1g --workdir /home/aztec-dev/aztec-packages -e HOME -e VERBOSE -e GIT_CONFIG_GLOBAL=/home/aztec-dev/aztec-packages/build-images/src/home/.gitconfig -e FORCE_COLOR=true -e CPUS -e MEM aztecprotocol/build:3.0 /bin/bash -c 'timeout -v 600s bash -c '\''yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts'\''' 10:52:13 + cid=ce20282c53a21c087caff08e6d68a92734cdd8057f904b4570e2af62cb5eaf10 10:52:13 + set +x 10:52:22 [10:52:22.219] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:22 [10:52:22.223] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:22 [10:52:22.228] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:22 [10:52:22.228] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:22 [10:52:22.846] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:22 [10:52:22.851] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:22 [10:52:22.902] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:22 [10:52:22.902] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:22 [10:52:22.903] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:22 [10:52:22.904] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:24 [10:52:24.137] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:24 [10:52:24.137] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:24 [10:52:24.140] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:24 [10:52:24.143] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:24 [10:52:24.233] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:24 [10:52:24.233] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:24 [10:52:24.235] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:24 [10:52:24.235] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:24 [10:52:24.236] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:24 [10:52:24.237] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:24 [10:52:24.243] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:24 [10:52:24.243] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:25 [10:52:25.342] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWEPRAZpQVoTjRgLECoeHhWoKTeHwNitAWRfYWC54UxxKy 10:52:25 [10:52:25.343] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:52:25 err: { 10:52:25 "type": "ReqRespStatusError", 10:52:25 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:52:25 "stack": 10:52:25 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:52:25 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:52:25 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:52:25 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:52:25 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:52:25 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:52:25 "status": 1 10:52:25 } 10:52:25 [10:52:25.351] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWEPRAZpQVoTjRgLECoeHhWoKTeHwNitAWRfYWC54UxxKy 10:52:25 [10:52:25.351] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:52:25 err: { 10:52:25 "type": "ReqRespStatusError", 10:52:25 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:52:25 "stack": 10:52:25 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:52:25 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:52:25 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:52:25 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:52:25 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:52:25 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:52:25 "status": 1 10:52:25 } 10:52:25 [10:52:25.355] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWEPRAZpQVoTjRgLECoeHhWoKTeHwNitAWRfYWC54UxxKy 10:52:25 [10:52:25.355] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:52:25 err: { 10:52:25 "type": "ReqRespStatusError", 10:52:25 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:52:25 "stack": 10:52:25 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:52:25 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:52:25 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:52:25 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:52:25 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:52:25 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:52:25 "status": 1 10:52:25 } 10:52:25 [10:52:25.358] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWEPRAZpQVoTjRgLECoeHhWoKTeHwNitAWRfYWC54UxxKy 10:52:25 [10:52:25.359] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:52:25 err: { 10:52:25 "type": "ReqRespStatusError", 10:52:25 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:52:25 "stack": 10:52:25 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:52:25 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:52:25 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:52:25 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:52:25 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:52:25 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:52:25 "status": 1 10:52:25 } 10:52:25 [10:52:25.362] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWEPRAZpQVoTjRgLECoeHhWoKTeHwNitAWRfYWC54UxxKy 10:52:25 [10:52:25.362] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:52:25 err: { 10:52:25 "type": "ReqRespStatusError", 10:52:25 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:52:25 "stack": 10:52:25 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:52:25 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:52:25 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:52:25 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:52:25 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:52:25 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:52:25 "status": 1 10:52:25 } 10:52:25 [10:52:25.367] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:25 [10:52:25.368] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:25 [10:52:25.371] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:25 [10:52:25.371] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:26 [10:52:26.807] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:26 [10:52:26.808] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:26 [10:52:26.811] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:26 [10:52:26.811] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:27 [10:52:27.904] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:27 [10:52:27.905] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:27 [10:52:27.908] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:27 [10:52:27.908] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:28 [10:52:28.986] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:28 [10:52:28.986] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:28 [10:52:28.993] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:28 [10:52:28.993] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:32 [10:52:32.043] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:32 [10:52:32.043] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:32 [10:52:32.047] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:32 [10:52:32.047] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:37 [10:52:37.234] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:37 [10:52:37.235] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:37 [10:52:37.236] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:37 [10:52:37.236] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:37 [10:52:37.237] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:37 [10:52:37.238] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:37 [10:52:37.240] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:37 [10:52:37.241] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:37 [10:52:37.254] WARN: foundation:memory_fifo Discarding item because queue is flushing 10:52:37 [10:52:37.258] ERROR: p2p:reqresp Unexpected error sending request to peer: Error: Could not enqueue function 10:52:37 at /home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:65:24 10:52:37 at new Promise (<anonymous>) 10:52:37 at SerialQueue.put (/home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:55:16) 10:52:37 at ConnectionSampler.put [as dialProtocol] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/connection-sampler/connection_sampler.ts:204:43) 10:52:37 at ReqResp.dialProtocol (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:449:45) 10:52:37 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:52:37 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:52:37 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:52:37 at TimeoutTask.requestFunction [as fn] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:200:26) {"peerId":"12D3KooWNHXBWZRhaYuJNK65x37WPLfZMPkRmHZ6psHe1WXNJDRP","subProtocol":"/aztec/req/tx/0.1.0"} 10:52:38 [10:52:38.336] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:38 [10:52:38.337] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:38 [10:52:38.341] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:38 [10:52:38.341] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:39 [10:52:39.394] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:39 [10:52:39.394] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:39 [10:52:39.398] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:39 [10:52:39.399] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:40 [10:52:40.474] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:40 [10:52:40.474] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:40 [10:52:40.482] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:40 [10:52:40.482] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:41 [10:52:41.784] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:41 [10:52:41.784] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:41 [10:52:41.787] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:41 [10:52:41.787] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:42 [10:52:42.908] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWBW73Uds12wwYZMmPsrvcSYkmcZoZSur3sWXSgHb6UbLG 10:52:42 [10:52:42.909] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:52:42 err: { 10:52:42 "type": "ReqRespStatusError", 10:52:42 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:52:42 "stack": 10:52:42 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:52:42 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:52:42 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:52:42 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:52:42 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:52:42 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:52:42 "status": 1 10:52:42 } 10:52:42 [10:52:42.941] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:42 [10:52:42.941] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:42 [10:52:42.942] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:42 [10:52:42.943] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:42 [10:52:42.946] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:42 [10:52:42.946] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:44 [10:52:44.204] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:44 [10:52:44.205] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:44 [10:52:44.206] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:44 [10:52:44.206] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:44 [10:52:44.207] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:44 [10:52:44.208] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:44 [10:52:44.211] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:44 [10:52:44.212] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:45 [10:52:45.336] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWSvg6mUTHTfCBUkBCz7WsXEEJ74emj55xAttnKSYfcgLd 10:52:45 [10:52:45.337] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:52:45 err: { 10:52:45 "type": "ReqRespStatusError", 10:52:45 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:52:45 "stack": 10:52:45 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:52:45 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:52:45 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:52:45 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:52:45 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:52:45 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:52:45 "status": 1 10:52:45 } 10:52:45 [10:52:45.340] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWSvg6mUTHTfCBUkBCz7WsXEEJ74emj55xAttnKSYfcgLd 10:52:45 [10:52:45.340] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:52:45 err: { 10:52:45 "type": "ReqRespStatusError", 10:52:45 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:52:45 "stack": 10:52:45 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:52:45 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:52:45 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:52:45 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:52:45 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:52:45 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:52:45 "status": 1 10:52:45 } 10:52:45 [10:52:45.371] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWSvg6mUTHTfCBUkBCz7WsXEEJ74emj55xAttnKSYfcgLd 10:52:45 [10:52:45.371] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:52:45 err: { 10:52:45 "type": "ReqRespStatusError", 10:52:45 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:52:45 "stack": 10:52:45 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:52:45 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:52:45 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:52:45 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:52:45 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:52:45 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:52:45 "status": 1 10:52:45 } 10:52:45 [10:52:45.382] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:45 [10:52:45.382] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:45 [10:52:45.383] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:45 [10:52:45.384] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:45 [10:52:45.386] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:52:45 [10:52:45.387] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:52:45 FAIL src/services/reqresp/reqresp.test.ts (31.232 s) 10:52:45 ReqResp 10:52:45 ✓ should perform a ping request (1439 ms) 10:52:45 ✓ should handle gracefully if a peer connected peer is offline (661 ms) 10:52:45 ✓ should request from a later peer if other peers are offline (1372 ms) 10:52:45 ✕ should hit a rate limit if too many requests are made in quick succession (1095 ms) 10:52:45 Tx req protocol 10:52:45 ✓ can request a Tx from TxHash (1439 ms) 10:52:45 ✓ handles returning empty buffers (1095 ms) 10:52:45 ✓ does not crash if tx hash returns undefined (1086 ms) 10:52:45 ✓ should hit individual timeout if nothing is returned over the stream (3069 ms) 10:52:45 ✓ should hit collective timeout if nothing is returned over the stream from multiple peers (5191 ms) 10:52:45 ✓ should penalize peer if transaction validation fails (1087 ms) 10:52:45 Goodbye protocol 10:52:45 ✓ should send a goodbye message to a peer (1059 ms) 10:52:45 ✓ should not close stream when handling a goodbye message received from peer (1085 ms) 10:52:45 Block protocol 10:52:45 ✓ should handle block requests (1299 ms) 10:52:45 Batch requests 10:52:45 ✕ should send a batch request between many peers (1164 ms) 10:52:45 ✕ should send a batch request with a pinned peer (1276 ms) 10:52:45 ✕ should stop after max retry attempts (1164 ms) 10:52:45 10:52:45 ● ReqResp › should hit a rate limit if too many requests are made in quick succession 10:52:45 10:52:45 expect(received).toBeDefined() 10:52:45 10:52:45 Received: undefined 10:52:45 10:52:45 145 | // Check that one of the responses gets a rate limit response 10:52:45 146 | const rateLimitResponse = responses.find(response => response?.status === ReqRespStatus.RATE_LIMIT_EXCEEDED); 10:52:45 > 147 | expect(rateLimitResponse).toBeDefined(); 10:52:45 | ^ 10:52:45 148 | 10:52:45 149 | // Make sure the error message is logged 10:52:45 150 | const errorMessage = `Rate limit exceeded DeniedPeer for ${ 10:52:45 10:52:45 at Object.toBeDefined (services/reqresp/reqresp.test.ts:147:31) 10:52:45 10:52:45 ● ReqResp › Batch requests › should send a batch request between many peers 10:52:45 10:52:45 expect(jest.fn()).toHaveBeenCalledTimes(expected) 10:52:45 10:52:45 Expected number of calls: 9 10:52:45 Received number of calls: 11 10:52:45 10:52:45 474 | 10:52:45 475 | // Expect one request to have been sent to each peer 10:52:45 > 476 | expect(sendRequestToPeerSpy).toHaveBeenCalledTimes(batchSize); 10:52:45 | ^ 10:52:45 477 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 10:52:45 478 | expect.objectContaining({ 10:52:45 479 | publicKey: nodes[1].p2p.peerId.publicKey, 10:52:45 10:52:45 at Object.toHaveBeenCalledTimes (services/reqresp/reqresp.test.ts:476:36) 10:52:45 10:52:45 ● ReqResp › Batch requests › should send a batch request with a pinned peer 10:52:45 10:52:45 expect(jest.fn()).toHaveBeenCalledWith(...expected) 10:52:45 10:52:45 Expected: ObjectContaining {"publicKey": [8, 1, 18, 32, 11, 214, 248, 206, 20, 236, …]}, "/aztec/req/ping/0.1.0", {"data": [112, 105, 110, 103, 51], "type": "Buffer"} 10:52:45 Received 10:52:45 1 10:52:45 "12D3KooWJmLVrjteABp98Pso4ThiCEUE4nDQSkBR4dqz6s8QANih", 10:52:45 "/aztec/req/ping/0.1.0", 10:52:45 @@ -2,9 +2,9 @@ 10:52:45 "data": Array [ 10:52:45 112, 10:52:45 105, 10:52:45 110, 10:52:45 103, 10:52:45 - 51, 10:52:45 + 48, 10:52:45 ], 10:52:45 "type": "Buffer", 10:52:45 }, 10:52:45 2 10:52:45 "12D3KooWRbsuGkrquRkcX94nyXzqRfibRUFqjapMFjomJ2zVJCsT", 10:52:45 "/aztec/req/ping/0.1.0", 10:52:45 @@ -2,9 +2,9 @@ 10:52:45 "data": Array [ 10:52:45 112, 10:52:45 105, 10:52:45 110, 10:52:45 103, 10:52:45 - 51, 10:52:45 + 49, 10:52:45 ], 10:52:45 "type": "Buffer", 10:52:45 }, 10:52:45 3 10:52:45 "12D3KooWAcamwmkXbxxsUFMxYwSPi3km8mYzwdUprMPsbPfuUwB6", 10:52:45 "/aztec/req/ping/0.1.0", 10:52:45 @@ -2,9 +2,9 @@ 10:52:45 "data": Array [ 10:52:45 112, 10:52:45 105, 10:52:45 110, 10:52:45 103, 10:52:45 - 51, 10:52:45 + 48, 10:52:45 ], 10:52:45 "type": "Buffer", 10:52:45 }, 10:52:45 10:52:45 Number of calls: 14 10:52:45 10:52:45 516 | // Expect pinned peer to have received all requests 10:52:45 517 | for (let i = 0; i < batchSize; i++) { 10:52:45 > 518 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 10:52:45 | ^ 10:52:45 519 | expect.objectContaining({ publicKey: nodes[1].p2p.peerId.publicKey }), 10:52:45 520 | ReqRespSubProtocol.PING, 10:52:45 521 | Buffer.from(`ping${i}`), 10:52:45 10:52:45 at Object.toHaveBeenCalledWith (services/reqresp/reqresp.test.ts:518:38) 10:52:45 10:52:45 ● ReqResp › Batch requests › should stop after max retry attempts 10:52:45 10:52:45 expect(received).toEqual(expected) // deep equality 10:52:45 10:52:45 - Expected - 11 10:52:45 + Received + 3 10:52:45 10:52:45 @@ -74,10 +74,11 @@ 10:52:45 103, 10:52:45 ], 10:52:45 "type": "Buffer", 10:52:45 }, 10:52:45 }, 10:52:45 + undefined, 10:52:45 RequestableBuffer { 10:52:45 "buffer": Object { 10:52:45 "data": Array [ 10:52:45 112, 10:52:45 111, 10:52:45 @@ -85,10 +86,11 @@ 10:52:45 103, 10:52:45 ], 10:52:45 "type": "Buffer", 10:52:45 }, 10:52:45 }, 10:52:45 + undefined, 10:52:45 RequestableBuffer { 10:52:45 "buffer": Object { 10:52:45 "data": Array [ 10:52:45 112, 10:52:45 111, 10:52:45 @@ -96,17 +98,7 @@ 10:52:45 103, 10:52:45 ], 10:52:45 "type": "Buffer", 10:52:45 }, 10:52:45 }, 10:52:45 - RequestableBuffer { 10:52:45 - "buffer": Object { 10:52:45 - "data": Array [ 10:52:45 - 112, 10:52:45 - 111, 10:52:45 - 110, 10:52:45 - 103, 10:52:45 - ], 10:52:45 - "type": "Buffer", 10:52:45 - }, 10:52:45 - }, 10:52:45 + undefined, 10:52:45 ] 10:52:45 10:52:45 555 | 10:52:45 556 | const res = await nodes[0].req.sendBatchRequest(ReqRespSubProtocol.PING, requests, undefined); 10:52:45 > 557 | expect(res).toEqual(expectResponses); 10:52:45 | ^ 10:52:45 558 | 10:52:45 559 | // Check that we did detect hitting a rate limit 10:52:45 560 | expect(requesterLoggerSpy).toHaveBeenCalledWith( 10:52:45 10:52:45 at Object.toEqual (services/reqresp/reqresp.test.ts:557:19) 10:52:45 10:52:45 Test Suites: 1 failed, 1 total 10:52:45 Tests: 4 failed, 12 passed, 16 total 10:52:45 Snapshots: 0 total 10:52:45 Time: 31.306 s 10:52:45 Ran all test suites matching /p2p\/src\/services\/reqresp\/reqresp.test.ts/i. 10:52:45 Force exiting Jest: Have you considered using `--detectOpenHandles` to detect async operations that kept running after all tests finished?