ACI ·

Parent Log: http://ci.aztec-labs.com/5e8add62c497bc1d Command: cdc0844b6e9d2c1e:ISOLATE=1:NAME=p2p/src/services/reqresp/reqresp.test.ts yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts Commit: https://github.com/AztecProtocol/aztec-packages/commit/dada14e07a974e905ef606b4fac7c777b922587f Env: REF_NAME=gh-readonly-queue/master/pr-14898-260a6c8dbe7557aae065d685de66faa2bc362a9d CURRENT_VERSION=0.87.8 CI_FULL=0 Date: Wed Jun 11 19:27:31 UTC 2025 System: ARCH=arm64 CPUS=64 MEM=247Gi HOSTNAME=pr-14898_arm64_a1-fast Resources: CPU_LIST=0-63 CPUS=2 MEM=8g TIMEOUT=600s History: http://ci.aztec-labs.com/list/history_a57cc872d7daa810_master 19:27:31 +++ id -u 19:27:31 +++ id -g 19:27:31 ++ docker run -d --name p2p_src_services_reqresp_reqresp.test.ts --net=none --cpuset-cpus=0-63 --cpus=2 --memory=8g --user 1000:1000 -v/home/aztec-dev:/home/aztec-dev --mount type=tmpfs,target=/tmp,tmpfs-size=1g --workdir /home/aztec-dev/aztec-packages -e HOME -e VERBOSE -e GIT_CONFIG_GLOBAL=/home/aztec-dev/aztec-packages/build-images/src/home/.gitconfig -e FORCE_COLOR=true -e CPUS -e MEM aztecprotocol/build:3.0 /bin/bash -c 'timeout -v 600s bash -c '\''yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts'\''' 19:27:31 + cid=94c7765ab9df72a47f258e1f0ea953533af30c83f6e8b4f0fc6331e2853bf282 19:27:31 + set +x 19:27:38 [19:27:38.279] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:38 [19:27:38.284] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:38 [19:27:38.289] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:38 [19:27:38.290] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:38 [19:27:38.836] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:38 [19:27:38.837] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:38 [19:27:38.878] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:38 [19:27:38.879] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:38 [19:27:38.880] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:38 [19:27:38.881] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:40 [19:27:40.079] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:40 [19:27:40.080] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:40 [19:27:40.082] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:40 [19:27:40.084] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:40 [19:27:40.188] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:40 [19:27:40.189] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:40 [19:27:40.190] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:40 [19:27:40.190] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:40 [19:27:40.191] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:40 [19:27:40.192] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:40 [19:27:40.195] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:40 [19:27:40.198] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:41 [19:27:41.292] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWGQ2fwxwqgGW9KWyaqgZhcuBwq1GfoBFykJr4cRUh2Pqx 19:27:41 [19:27:41.293] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 19:27:41 err: { 19:27:41 "type": "ReqRespStatusError", 19:27:41 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 19:27:41 "stack": 19:27:41 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 19:27:41 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 19:27:41 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 19:27:41 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 19:27:41 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 19:27:41 at processTicksAndRejections (node:internal/process/task_queues:105:5) 19:27:41 "status": 1 19:27:41 } 19:27:41 [19:27:41.302] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWGQ2fwxwqgGW9KWyaqgZhcuBwq1GfoBFykJr4cRUh2Pqx 19:27:41 [19:27:41.303] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 19:27:41 err: { 19:27:41 "type": "ReqRespStatusError", 19:27:41 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 19:27:41 "stack": 19:27:41 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 19:27:41 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 19:27:41 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 19:27:41 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 19:27:41 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 19:27:41 at processTicksAndRejections (node:internal/process/task_queues:105:5) 19:27:41 "status": 1 19:27:41 } 19:27:41 [19:27:41.308] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWGQ2fwxwqgGW9KWyaqgZhcuBwq1GfoBFykJr4cRUh2Pqx 19:27:41 [19:27:41.309] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 19:27:41 err: { 19:27:41 "type": "ReqRespStatusError", 19:27:41 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 19:27:41 "stack": 19:27:41 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 19:27:41 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 19:27:41 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 19:27:41 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 19:27:41 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 19:27:41 at processTicksAndRejections (node:internal/process/task_queues:105:5) 19:27:41 "status": 1 19:27:41 } 19:27:41 [19:27:41.313] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWGQ2fwxwqgGW9KWyaqgZhcuBwq1GfoBFykJr4cRUh2Pqx 19:27:41 [19:27:41.313] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 19:27:41 err: { 19:27:41 "type": "ReqRespStatusError", 19:27:41 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 19:27:41 "stack": 19:27:41 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 19:27:41 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 19:27:41 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 19:27:41 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 19:27:41 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 19:27:41 at processTicksAndRejections (node:internal/process/task_queues:105:5) 19:27:41 "status": 1 19:27:41 } 19:27:41 [19:27:41.317] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWGQ2fwxwqgGW9KWyaqgZhcuBwq1GfoBFykJr4cRUh2Pqx 19:27:41 [19:27:41.318] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 19:27:41 err: { 19:27:41 "type": "ReqRespStatusError", 19:27:41 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 19:27:41 "stack": 19:27:41 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 19:27:41 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 19:27:41 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 19:27:41 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 19:27:41 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 19:27:41 at processTicksAndRejections (node:internal/process/task_queues:105:5) 19:27:41 "status": 1 19:27:41 } 19:27:41 [19:27:41.325] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:41 [19:27:41.325] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:41 [19:27:41.329] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:41 [19:27:41.329] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:42 [19:27:42.693] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:42 [19:27:42.693] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:42 [19:27:42.696] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:42 [19:27:42.697] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:43 [19:27:43.771] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:43 [19:27:43.771] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:43 [19:27:43.775] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:43 [19:27:43.775] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:44 [19:27:44.849] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:44 [19:27:44.850] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:44 [19:27:44.853] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:44 [19:27:44.853] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:47 [19:27:47.892] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:47 [19:27:47.892] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:47 [19:27:47.896] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:47 [19:27:47.896] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:53 [19:27:53.068] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:53 [19:27:53.069] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:53 [19:27:53.070] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:53 [19:27:53.070] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:53 [19:27:53.071] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:53 [19:27:53.072] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:53 [19:27:53.075] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:53 [19:27:53.075] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:53 [19:27:53.088] WARN: foundation:memory_fifo Discarding item because queue is flushing 19:27:53 [19:27:53.091] ERROR: p2p:reqresp Unexpected error sending request to peer: Error: Could not enqueue function 19:27:53 at /home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:65:24 19:27:53 at new Promise (<anonymous>) 19:27:53 at SerialQueue.put (/home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:55:16) 19:27:53 at ConnectionSampler.put [as dialProtocol] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/connection-sampler/connection_sampler.ts:204:43) 19:27:53 at ReqResp.dialProtocol (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:449:45) 19:27:53 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 19:27:53 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 19:27:53 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 19:27:53 at TimeoutTask.requestFunction [as fn] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:200:26) {"peerId":"12D3KooW9rV4Daao5Lbtu1Hn6gpzpmb6x96pxZZqMSMkH99Mf61g","subProtocol":"/aztec/req/tx/0.1.0"} 19:27:54 [19:27:54.167] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:54 [19:27:54.168] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:54 [19:27:54.171] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:54 [19:27:54.171] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:55 [19:27:55.211] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:55 [19:27:55.212] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:55 [19:27:55.215] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:55 [19:27:55.215] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:56 [19:27:56.254] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:56 [19:27:56.255] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:56 [19:27:56.258] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:56 [19:27:56.258] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:57 [19:27:57.469] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:57 [19:27:57.470] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:57 [19:27:57.473] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:57 [19:27:57.473] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:58 [19:27:58.586] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAKemorHGGbbMwP5YUiJ1qp2C7X6W53gRRd6xnh5AdXpT 19:27:58 [19:27:58.586] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 19:27:58 err: { 19:27:58 "type": "ReqRespStatusError", 19:27:58 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 19:27:58 "stack": 19:27:58 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 19:27:58 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 19:27:58 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 19:27:58 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 19:27:58 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 19:27:58 at processTicksAndRejections (node:internal/process/task_queues:105:5) 19:27:58 "status": 1 19:27:58 } 19:27:58 [19:27:58.606] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:58 [19:27:58.607] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:58 [19:27:58.608] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:58 [19:27:58.608] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:58 [19:27:58.611] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:58 [19:27:58.611] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:59 [19:27:59.820] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:59 [19:27:59.821] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:59 [19:27:59.822] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:59 [19:27:59.822] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:59 [19:27:59.823] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:59 [19:27:59.823] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:27:59 [19:27:59.826] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:27:59 [19:27:59.826] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:28:00 [19:28:00.934] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWRX8Q7VjkQsVijbm5QivTnaAvWMqkTiMGY7wVEPa2wwDu 19:28:00 [19:28:00.934] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 19:28:00 err: { 19:28:00 "type": "ReqRespStatusError", 19:28:00 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 19:28:00 "stack": 19:28:00 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 19:28:00 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 19:28:00 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 19:28:00 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 19:28:00 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 19:28:00 at processTicksAndRejections (node:internal/process/task_queues:105:5) 19:28:00 "status": 1 19:28:00 } 19:28:00 [19:28:00.938] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWRX8Q7VjkQsVijbm5QivTnaAvWMqkTiMGY7wVEPa2wwDu 19:28:00 [19:28:00.938] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 19:28:00 err: { 19:28:00 "type": "ReqRespStatusError", 19:28:00 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 19:28:00 "stack": 19:28:00 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 19:28:00 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 19:28:00 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 19:28:00 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 19:28:00 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 19:28:00 at processTicksAndRejections (node:internal/process/task_queues:105:5) 19:28:00 "status": 1 19:28:00 } 19:28:00 [19:28:00.953] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWRX8Q7VjkQsVijbm5QivTnaAvWMqkTiMGY7wVEPa2wwDu 19:28:00 [19:28:00.953] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 19:28:00 err: { 19:28:00 "type": "ReqRespStatusError", 19:28:00 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 19:28:00 "stack": 19:28:00 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 19:28:00 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 19:28:00 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 19:28:00 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 19:28:00 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 19:28:00 at processTicksAndRejections (node:internal/process/task_queues:105:5) 19:28:00 "status": 1 19:28:00 } 19:28:00 [19:28:00.963] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:28:00 [19:28:00.963] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:28:00 [19:28:00.964] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:28:00 [19:28:00.965] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:28:00 [19:28:00.967] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:28:00 [19:28:00.968] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:28:01 FAIL src/services/reqresp/reqresp.test.ts (28.074 s) 19:28:01 ReqResp 19:28:01 ✓ should perform a ping request (1230 ms) 19:28:01 ✓ should handle gracefully if a peer connected peer is offline (586 ms) 19:28:01 ✓ should request from a later peer if other peers are offline (1351 ms) 19:28:01 ✕ should hit a rate limit if too many requests are made in quick succession (1098 ms) 19:28:01 Tx req protocol 19:28:01 ✓ can request a Tx from TxHash (1368 ms) 19:28:01 ✓ handles returning empty buffers (1077 ms) 19:28:01 ✓ does not crash if tx hash returns undefined (1077 ms) 19:28:01 ✓ should hit individual timeout if nothing is returned over the stream (3043 ms) 19:28:01 ✓ should hit collective timeout if nothing is returned over the stream from multiple peers (5192 ms) 19:28:01 ✓ should penalize peer if transaction validation fails (1081 ms) 19:28:01 Goodbye protocol 19:28:01 ✓ should send a goodbye message to a peer (1044 ms) 19:28:01 ✓ should not close stream when handling a goodbye message received from peer (1043 ms) 19:28:01 Block protocol 19:28:01 ✓ should handle block requests (1218 ms) 19:28:01 Batch requests 19:28:01 ✕ should send a batch request between many peers (1137 ms) 19:28:01 ✕ should send a batch request with a pinned peer (1221 ms) 19:28:01 ✕ should stop after max retry attempts (1137 ms) 19:28:01 19:28:01 ● ReqResp › should hit a rate limit if too many requests are made in quick succession 19:28:01 19:28:01 expect(received).toBeDefined() 19:28:01 19:28:01 Received: undefined 19:28:01 19:28:01 145 | // Check that one of the responses gets a rate limit response 19:28:01 146 | const rateLimitResponse = responses.find(response => response?.status === ReqRespStatus.RATE_LIMIT_EXCEEDED); 19:28:01 > 147 | expect(rateLimitResponse).toBeDefined(); 19:28:01 | ^ 19:28:01 148 | 19:28:01 149 | // Make sure the error message is logged 19:28:01 150 | const errorMessage = `Rate limit exceeded DeniedPeer for ${ 19:28:01 19:28:01 at Object.toBeDefined (services/reqresp/reqresp.test.ts:147:31) 19:28:01 19:28:01 ● ReqResp › Batch requests › should send a batch request between many peers 19:28:01 19:28:01 expect(jest.fn()).toHaveBeenCalledTimes(expected) 19:28:01 19:28:01 Expected number of calls: 9 19:28:01 Received number of calls: 11 19:28:01 19:28:01 474 | 19:28:01 475 | // Expect one request to have been sent to each peer 19:28:01 > 476 | expect(sendRequestToPeerSpy).toHaveBeenCalledTimes(batchSize); 19:28:01 | ^ 19:28:01 477 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 19:28:01 478 | expect.objectContaining({ 19:28:01 479 | publicKey: nodes[1].p2p.peerId.publicKey, 19:28:01 19:28:01 at Object.toHaveBeenCalledTimes (services/reqresp/reqresp.test.ts:476:36) 19:28:01 19:28:01 ● ReqResp › Batch requests › should send a batch request with a pinned peer 19:28:01 19:28:01 expect(jest.fn()).toHaveBeenCalledWith(...expected) 19:28:01 19:28:01 Expected: ObjectContaining {"publicKey": [8, 1, 18, 32, 181, 26, 107, 189, 3, 183, …]}, "/aztec/req/ping/0.1.0", {"data": [112, 105, 110, 103, 51], "type": "Buffer"} 19:28:01 Received 19:28:01 1 19:28:01 "12D3KooWEq5f2bNY8PbwdEs16pwFn3pz43Nj4Gege38jNGR5vjos", 19:28:01 "/aztec/req/ping/0.1.0", 19:28:01 @@ -2,9 +2,9 @@ 19:28:01 "data": Array [ 19:28:01 112, 19:28:01 105, 19:28:01 110, 19:28:01 103, 19:28:01 - 51, 19:28:01 + 48, 19:28:01 ], 19:28:01 "type": "Buffer", 19:28:01 }, 19:28:01 2 19:28:01 "12D3KooWBrSsDKcBFu1qpRGmbVhKZGaJaEtudkf7VFJft4nK1gQL", 19:28:01 "/aztec/req/ping/0.1.0", 19:28:01 @@ -2,9 +2,9 @@ 19:28:01 "data": Array [ 19:28:01 112, 19:28:01 105, 19:28:01 110, 19:28:01 103, 19:28:01 - 51, 19:28:01 + 49, 19:28:01 ], 19:28:01 "type": "Buffer", 19:28:01 }, 19:28:01 3 19:28:01 "12D3KooWN1KL2UZ2WUSmy5hGGhSsmjc2Py5T4brgyrGTKJTFRamZ", 19:28:01 "/aztec/req/ping/0.1.0", 19:28:01 @@ -2,9 +2,9 @@ 19:28:01 "data": Array [ 19:28:01 112, 19:28:01 105, 19:28:01 110, 19:28:01 103, 19:28:01 - 51, 19:28:01 + 48, 19:28:01 ], 19:28:01 "type": "Buffer", 19:28:01 }, 19:28:01 19:28:01 Number of calls: 12 19:28:01 19:28:01 516 | // Expect pinned peer to have received all requests 19:28:01 517 | for (let i = 0; i < batchSize; i++) { 19:28:01 > 518 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 19:28:01 | ^ 19:28:01 519 | expect.objectContaining({ publicKey: nodes[1].p2p.peerId.publicKey }), 19:28:01 520 | ReqRespSubProtocol.PING, 19:28:01 521 | Buffer.from(`ping${i}`), 19:28:01 19:28:01 at Object.toHaveBeenCalledWith (services/reqresp/reqresp.test.ts:518:38) 19:28:01 19:28:01 ● ReqResp › Batch requests › should stop after max retry attempts 19:28:01 19:28:01 expect(received).toEqual(expected) // deep equality 19:28:01 19:28:01 - Expected - 11 19:28:01 + Received + 3 19:28:01 19:28:01 @@ -74,10 +74,11 @@ 19:28:01 103, 19:28:01 ], 19:28:01 "type": "Buffer", 19:28:01 }, 19:28:01 }, 19:28:01 + undefined, 19:28:01 RequestableBuffer { 19:28:01 "buffer": Object { 19:28:01 "data": Array [ 19:28:01 112, 19:28:01 111, 19:28:01 @@ -85,10 +86,11 @@ 19:28:01 103, 19:28:01 ], 19:28:01 "type": "Buffer", 19:28:01 }, 19:28:01 }, 19:28:01 + undefined, 19:28:01 RequestableBuffer { 19:28:01 "buffer": Object { 19:28:01 "data": Array [ 19:28:01 112, 19:28:01 111, 19:28:01 @@ -96,17 +98,7 @@ 19:28:01 103, 19:28:01 ], 19:28:01 "type": "Buffer", 19:28:01 }, 19:28:01 }, 19:28:01 - RequestableBuffer { 19:28:01 - "buffer": Object { 19:28:01 - "data": Array [ 19:28:01 - 112, 19:28:01 - 111, 19:28:01 - 110, 19:28:01 - 103, 19:28:01 - ], 19:28:01 - "type": "Buffer", 19:28:01 - }, 19:28:01 - }, 19:28:01 + undefined, 19:28:01 ] 19:28:01 19:28:01 555 | 19:28:01 556 | const res = await nodes[0].req.sendBatchRequest(ReqRespSubProtocol.PING, requests, undefined); 19:28:01 > 557 | expect(res).toEqual(expectResponses); 19:28:01 | ^ 19:28:01 558 | 19:28:01 559 | // Check that we did detect hitting a rate limit 19:28:01 560 | expect(requesterLoggerSpy).toHaveBeenCalledWith( 19:28:01 19:28:01 at Object.toEqual (services/reqresp/reqresp.test.ts:557:19) 19:28:01 19:28:01 Test Suites: 1 failed, 1 total 19:28:01 Tests: 4 failed, 12 passed, 16 total 19:28:01 Snapshots: 0 total 19:28:01 Time: 28.146 s 19:28:01 Ran all test suites matching /p2p\/src\/services\/reqresp\/reqresp.test.ts/i. 19:28:01 Force exiting Jest: Have you considered using `--detectOpenHandles` to detect async operations that kept running after all tests finished?