ACI ·

Parent Log: http://ci.aztec-labs.com/0037d898807e86c0 Command: 4bfd2d3585b9db66:ISOLATE=1:NAME=p2p/src/services/reqresp/reqresp.test.ts yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts Commit: https://github.com/AztecProtocol/aztec-packages/commit/267ac35e8ccb487bdc79bbe12d428f4c8b4d4d9f Env: REF_NAME=gh-readonly-queue/master/pr-15002-1338cef09c870378254c56748f972ae827c4d482 CURRENT_VERSION=0.87.8 CI_FULL=1 Date: Fri Jun 13 14:55:49 UTC 2025 System: ARCH=amd64 CPUS=128 MEM=493Gi HOSTNAME=pr-15002_amd64_x2-full Resources: CPU_LIST=0-127 CPUS=2 MEM=8g TIMEOUT=600s History: http://ci.aztec-labs.com/list/history_a57cc872d7daa810_master 14:55:49 +++ id -u 14:55:49 +++ id -g 14:55:49 ++ docker run -d --name p2p_src_services_reqresp_reqresp.test.ts --net=none --cpuset-cpus=0-127 --cpus=2 --memory=8g --user 1000:1000 -v/home/aztec-dev:/home/aztec-dev --mount type=tmpfs,target=/tmp,tmpfs-size=1g --workdir /home/aztec-dev/aztec-packages -e HOME -e VERBOSE -e GIT_CONFIG_GLOBAL=/home/aztec-dev/aztec-packages/build-images/src/home/.gitconfig -e FORCE_COLOR=true -e CPUS -e MEM aztecprotocol/build:3.0 /bin/bash -c 'timeout -v 600s bash -c '\''yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts'\''' 14:55:49 + cid=5b4e3f64c7a0e37af1da110871bc75b23f5acd4bd16a7eaaf1af2c4af50c6aee 14:55:49 + set +x 14:55:58 [14:55:58.398] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:55:58 [14:55:58.403] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:55:58 [14:55:58.408] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:55:58 [14:55:58.409] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:55:58 [14:55:58.956] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:55:58 [14:55:58.958] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:55:59 [14:55:59.025] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:55:59 [14:55:59.025] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:55:59 [14:55:59.026] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:55:59 [14:55:59.027] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:00 [14:56:00.230] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:00 [14:56:00.231] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:00 [14:56:00.234] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:00 [14:56:00.236] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:00 [14:56:00.348] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:00 [14:56:00.349] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:00 [14:56:00.350] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:00 [14:56:00.351] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:00 [14:56:00.352] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:00 [14:56:00.352] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:00 [14:56:00.355] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:00 [14:56:00.356] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:01 [14:56:01.446] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWJC73GGFowDmeLGaQhZnPbSGVz5CWVvw9yqd4yiJddj5E 14:56:01 [14:56:01.446] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 14:56:01 err: { 14:56:01 "type": "ReqRespStatusError", 14:56:01 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 14:56:01 "stack": 14:56:01 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 14:56:01 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 14:56:01 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 14:56:01 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 14:56:01 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 14:56:01 at processTicksAndRejections (node:internal/process/task_queues:105:5) 14:56:01 "status": 1 14:56:01 } 14:56:01 [14:56:01.457] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWJC73GGFowDmeLGaQhZnPbSGVz5CWVvw9yqd4yiJddj5E 14:56:01 [14:56:01.458] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 14:56:01 err: { 14:56:01 "type": "ReqRespStatusError", 14:56:01 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 14:56:01 "stack": 14:56:01 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 14:56:01 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 14:56:01 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 14:56:01 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 14:56:01 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 14:56:01 at processTicksAndRejections (node:internal/process/task_queues:105:5) 14:56:01 "status": 1 14:56:01 } 14:56:01 [14:56:01.463] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWJC73GGFowDmeLGaQhZnPbSGVz5CWVvw9yqd4yiJddj5E 14:56:01 [14:56:01.463] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 14:56:01 err: { 14:56:01 "type": "ReqRespStatusError", 14:56:01 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 14:56:01 "stack": 14:56:01 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 14:56:01 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 14:56:01 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 14:56:01 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 14:56:01 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 14:56:01 at processTicksAndRejections (node:internal/process/task_queues:105:5) 14:56:01 "status": 1 14:56:01 } 14:56:01 [14:56:01.467] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWJC73GGFowDmeLGaQhZnPbSGVz5CWVvw9yqd4yiJddj5E 14:56:01 [14:56:01.468] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 14:56:01 err: { 14:56:01 "type": "ReqRespStatusError", 14:56:01 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 14:56:01 "stack": 14:56:01 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 14:56:01 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 14:56:01 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 14:56:01 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 14:56:01 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 14:56:01 at processTicksAndRejections (node:internal/process/task_queues:105:5) 14:56:01 "status": 1 14:56:01 } 14:56:01 [14:56:01.471] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWJC73GGFowDmeLGaQhZnPbSGVz5CWVvw9yqd4yiJddj5E 14:56:01 [14:56:01.472] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 14:56:01 err: { 14:56:01 "type": "ReqRespStatusError", 14:56:01 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 14:56:01 "stack": 14:56:01 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 14:56:01 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 14:56:01 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 14:56:01 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 14:56:01 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 14:56:01 at processTicksAndRejections (node:internal/process/task_queues:105:5) 14:56:01 "status": 1 14:56:01 } 14:56:01 [14:56:01.479] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:01 [14:56:01.479] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:01 [14:56:01.483] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:01 [14:56:01.483] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:02 [14:56:02.968] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:02 [14:56:02.969] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:02 [14:56:02.972] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:02 [14:56:02.972] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:04 [14:56:04.057] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:04 [14:56:04.058] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:04 [14:56:04.061] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:04 [14:56:04.062] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:05 [14:56:05.151] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:05 [14:56:05.152] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:05 [14:56:05.156] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:05 [14:56:05.156] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:08 [14:56:08.232] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:08 [14:56:08.233] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:08 [14:56:08.237] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:08 [14:56:08.238] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:13 [14:56:13.457] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:13 [14:56:13.457] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:13 [14:56:13.459] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:13 [14:56:13.459] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:13 [14:56:13.460] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:13 [14:56:13.460] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:13 [14:56:13.463] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:13 [14:56:13.464] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:13 [14:56:13.476] WARN: foundation:memory_fifo Discarding item because queue is flushing 14:56:13 [14:56:13.480] ERROR: p2p:reqresp Unexpected error sending request to peer: Error: Could not enqueue function 14:56:13 at /home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:65:24 14:56:13 at new Promise (<anonymous>) 14:56:13 at SerialQueue.put (/home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:55:16) 14:56:13 at ConnectionSampler.put [as dialProtocol] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/connection-sampler/connection_sampler.ts:204:43) 14:56:13 at ReqResp.dialProtocol (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:449:45) 14:56:13 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 14:56:13 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 14:56:13 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 14:56:13 at TimeoutTask.requestFunction [as fn] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:200:26) {"peerId":"12D3KooWHeV1BGRZAB4vFGPoEWEoQKvdvewdM1GzXqi7DfJ56dSY","subProtocol":"/aztec/req/tx/0.1.0"} 14:56:14 [14:56:14.561] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:14 [14:56:14.562] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:14 [14:56:14.566] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:14 [14:56:14.566] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:15 [14:56:15.757] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:15 [14:56:15.757] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:15 [14:56:15.761] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:15 [14:56:15.761] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:16 [14:56:16.810] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:16 [14:56:16.810] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:16 [14:56:16.814] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:16 [14:56:16.814] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:18 [14:56:18.078] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:18 [14:56:18.078] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:18 [14:56:18.082] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:18 [14:56:18.082] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:19 [14:56:19.226] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWQyEQhzNuB8SNJeVjjvVRMA6XCa15w8YKVwtkd3TUcsDa 14:56:19 [14:56:19.227] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 14:56:19 err: { 14:56:19 "type": "ReqRespStatusError", 14:56:19 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 14:56:19 "stack": 14:56:19 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 14:56:19 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 14:56:19 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 14:56:19 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 14:56:19 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 14:56:19 at processTicksAndRejections (node:internal/process/task_queues:105:5) 14:56:19 "status": 1 14:56:19 } 14:56:19 [14:56:19.253] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:19 [14:56:19.253] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:19 [14:56:19.255] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:19 [14:56:19.255] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:19 [14:56:19.259] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:19 [14:56:19.259] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:20 [14:56:20.524] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:20 [14:56:20.524] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:20 [14:56:20.526] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:20 [14:56:20.526] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:20 [14:56:20.527] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:20 [14:56:20.528] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:20 [14:56:20.532] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:20 [14:56:20.533] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:21 [14:56:21.680] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWKztKZJHaZHi6ZsMmUH8etLfqadDBqhNjPzmQGnYBmLz8 14:56:21 [14:56:21.681] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 14:56:21 err: { 14:56:21 "type": "ReqRespStatusError", 14:56:21 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 14:56:21 "stack": 14:56:21 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 14:56:21 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 14:56:21 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 14:56:21 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 14:56:21 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 14:56:21 at processTicksAndRejections (node:internal/process/task_queues:105:5) 14:56:21 "status": 1 14:56:21 } 14:56:21 [14:56:21.685] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWKztKZJHaZHi6ZsMmUH8etLfqadDBqhNjPzmQGnYBmLz8 14:56:21 [14:56:21.685] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 14:56:21 err: { 14:56:21 "type": "ReqRespStatusError", 14:56:21 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 14:56:21 "stack": 14:56:21 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 14:56:21 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 14:56:21 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 14:56:21 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 14:56:21 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 14:56:21 at processTicksAndRejections (node:internal/process/task_queues:105:5) 14:56:21 "status": 1 14:56:21 } 14:56:21 [14:56:21.704] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWKztKZJHaZHi6ZsMmUH8etLfqadDBqhNjPzmQGnYBmLz8 14:56:21 [14:56:21.705] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 14:56:21 err: { 14:56:21 "type": "ReqRespStatusError", 14:56:21 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 14:56:21 "stack": 14:56:21 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 14:56:21 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 14:56:21 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 14:56:21 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 14:56:21 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 14:56:21 at processTicksAndRejections (node:internal/process/task_queues:105:5) 14:56:21 "status": 1 14:56:21 } 14:56:21 [14:56:21.728] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:21 [14:56:21.729] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:21 [14:56:21.730] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:21 [14:56:21.731] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:21 [14:56:21.735] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:21 [14:56:21.736] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:21 FAIL src/services/reqresp/reqresp.test.ts (31.418 s) 14:56:21 ReqResp 14:56:21 ✓ should perform a ping request (1463 ms) 14:56:21 ✓ should handle gracefully if a peer connected peer is offline (611 ms) 14:56:21 ✓ should request from a later peer if other peers are offline (1358 ms) 14:56:21 ✕ should hit a rate limit if too many requests are made in quick succession (1103 ms) 14:56:21 Tx req protocol 14:56:21 ✓ can request a Tx from TxHash (1485 ms) 14:56:21 ✓ handles returning empty buffers (1092 ms) 14:56:21 ✓ does not crash if tx hash returns undefined (1097 ms) 14:56:21 ✓ should hit individual timeout if nothing is returned over the stream (3081 ms) 14:56:21 ✓ should hit collective timeout if nothing is returned over the stream from multiple peers (5232 ms) 14:56:21 ✓ should penalize peer if transaction validation fails (1090 ms) 14:56:21 Goodbye protocol 14:56:21 ✓ should send a goodbye message to a peer (1193 ms) 14:56:21 ✓ should not close stream when handling a goodbye message received from peer (1060 ms) 14:56:21 Block protocol 14:56:21 ✓ should handle block requests (1262 ms) 14:56:21 Batch requests 14:56:21 ✕ should send a batch request between many peers (1181 ms) 14:56:21 ✕ should send a batch request with a pinned peer (1284 ms) 14:56:21 ✕ should stop after max retry attempts (1195 ms) 14:56:21 14:56:21 ● ReqResp › should hit a rate limit if too many requests are made in quick succession 14:56:21 14:56:21 expect(received).toBeDefined() 14:56:21 14:56:21 Received: undefined 14:56:21 14:56:21 145 | // Check that one of the responses gets a rate limit response 14:56:21 146 | const rateLimitResponse = responses.find(response => response?.status === ReqRespStatus.RATE_LIMIT_EXCEEDED); 14:56:21 > 147 | expect(rateLimitResponse).toBeDefined(); 14:56:21 | ^ 14:56:21 148 | 14:56:21 149 | // Make sure the error message is logged 14:56:21 150 | const errorMessage = `Rate limit exceeded DeniedPeer for ${ 14:56:21 14:56:21 at Object.toBeDefined (services/reqresp/reqresp.test.ts:147:31) 14:56:21 14:56:21 ● ReqResp › Batch requests › should send a batch request between many peers 14:56:21 14:56:21 expect(jest.fn()).toHaveBeenCalledTimes(expected) 14:56:21 14:56:21 Expected number of calls: 9 14:56:21 Received number of calls: 11 14:56:21 14:56:21 474 | 14:56:21 475 | // Expect one request to have been sent to each peer 14:56:21 > 476 | expect(sendRequestToPeerSpy).toHaveBeenCalledTimes(batchSize); 14:56:21 | ^ 14:56:21 477 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 14:56:21 478 | expect.objectContaining({ 14:56:21 479 | publicKey: nodes[1].p2p.peerId.publicKey, 14:56:21 14:56:21 at Object.toHaveBeenCalledTimes (services/reqresp/reqresp.test.ts:476:36) 14:56:21 14:56:21 ● ReqResp › Batch requests › should send a batch request with a pinned peer 14:56:21 14:56:21 expect(jest.fn()).toHaveBeenCalledWith(...expected) 14:56:21 14:56:21 Expected: ObjectContaining {"publicKey": [8, 1, 18, 32, 244, 61, 2, 46, 132, 222, …]}, "/aztec/req/ping/0.1.0", {"data": [112, 105, 110, 103, 51], "type": "Buffer"} 14:56:21 Received 14:56:21 1 14:56:21 "12D3KooWBe9xpupppPdfuxEJafC48Xmnb98XDRYcCxHe8jZg2bb3", 14:56:21 "/aztec/req/ping/0.1.0", 14:56:21 @@ -2,9 +2,9 @@ 14:56:21 "data": Array [ 14:56:21 112, 14:56:21 105, 14:56:21 110, 14:56:21 103, 14:56:21 - 51, 14:56:21 + 48, 14:56:21 ], 14:56:21 "type": "Buffer", 14:56:21 }, 14:56:21 2 14:56:21 "12D3KooWSeriKtkUwDnvWgJ89Kkvev2UaioZMiBXidF9GQF9EkTK", 14:56:21 "/aztec/req/ping/0.1.0", 14:56:21 @@ -2,9 +2,9 @@ 14:56:21 "data": Array [ 14:56:21 112, 14:56:21 105, 14:56:21 110, 14:56:21 103, 14:56:21 - 51, 14:56:21 + 49, 14:56:21 ], 14:56:21 "type": "Buffer", 14:56:21 }, 14:56:21 3 14:56:21 "12D3KooWSFmbwnn52ZmEP1poX2xGsLYnf99a1QuN22XctXn53jXL", 14:56:21 "/aztec/req/ping/0.1.0", 14:56:21 @@ -2,9 +2,9 @@ 14:56:21 "data": Array [ 14:56:21 112, 14:56:21 105, 14:56:21 110, 14:56:21 103, 14:56:21 - 51, 14:56:21 + 48, 14:56:21 ], 14:56:21 "type": "Buffer", 14:56:21 }, 14:56:21 14:56:21 Number of calls: 11 14:56:21 14:56:21 516 | // Expect pinned peer to have received all requests 14:56:21 517 | for (let i = 0; i < batchSize; i++) { 14:56:21 > 518 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 14:56:21 | ^ 14:56:21 519 | expect.objectContaining({ publicKey: nodes[1].p2p.peerId.publicKey }), 14:56:21 520 | ReqRespSubProtocol.PING, 14:56:21 521 | Buffer.from(`ping${i}`), 14:56:21 14:56:21 at Object.toHaveBeenCalledWith (services/reqresp/reqresp.test.ts:518:38) 14:56:21 14:56:21 ● ReqResp › Batch requests › should stop after max retry attempts 14:56:21 14:56:21 expect(received).toEqual(expected) // deep equality 14:56:21 14:56:21 - Expected - 11 14:56:21 + Received + 3 14:56:21 14:56:21 @@ -74,10 +74,11 @@ 14:56:21 103, 14:56:21 ], 14:56:21 "type": "Buffer", 14:56:21 }, 14:56:21 }, 14:56:21 + undefined, 14:56:21 RequestableBuffer { 14:56:21 "buffer": Object { 14:56:21 "data": Array [ 14:56:21 112, 14:56:21 111, 14:56:21 @@ -85,10 +86,11 @@ 14:56:21 103, 14:56:21 ], 14:56:21 "type": "Buffer", 14:56:21 }, 14:56:21 }, 14:56:21 + undefined, 14:56:21 RequestableBuffer { 14:56:21 "buffer": Object { 14:56:21 "data": Array [ 14:56:21 112, 14:56:21 111, 14:56:21 @@ -96,17 +98,7 @@ 14:56:21 103, 14:56:21 ], 14:56:21 "type": "Buffer", 14:56:21 }, 14:56:21 }, 14:56:21 - RequestableBuffer { 14:56:21 - "buffer": Object { 14:56:21 - "data": Array [ 14:56:21 - 112, 14:56:21 - 111, 14:56:21 - 110, 14:56:21 - 103, 14:56:21 - ], 14:56:21 - "type": "Buffer", 14:56:21 - }, 14:56:21 - }, 14:56:21 + undefined, 14:56:21 ] 14:56:21 14:56:21 555 | 14:56:21 556 | const res = await nodes[0].req.sendBatchRequest(ReqRespSubProtocol.PING, requests, undefined); 14:56:21 > 557 | expect(res).toEqual(expectResponses); 14:56:21 | ^ 14:56:21 558 | 14:56:21 559 | // Check that we did detect hitting a rate limit 14:56:21 560 | expect(requesterLoggerSpy).toHaveBeenCalledWith( 14:56:21 14:56:21 at Object.toEqual (services/reqresp/reqresp.test.ts:557:19) 14:56:21 14:56:21 Test Suites: 1 failed, 1 total 14:56:21 Tests: 4 failed, 12 passed, 16 total 14:56:21 Snapshots: 0 total 14:56:21 Time: 31.506 s 14:56:21 Ran all test suites matching /p2p\/src\/services\/reqresp\/reqresp.test.ts/i. 14:56:21 Force exiting Jest: Have you considered using `--detectOpenHandles` to detect async operations that kept running after all tests finished?