ACI ·

Parent Log: http://ci.aztec-labs.com/01950490259eda0b Command: 59d7caa3bc5b3609:ISOLATE=1:NAME=p2p/src/services/reqresp/reqresp.test.ts yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts Commit: https://github.com/AztecProtocol/aztec-packages/commit/dada14e07a974e905ef606b4fac7c777b922587f Env: REF_NAME=gh-readonly-queue/master/pr-14898-260a6c8dbe7557aae065d685de66faa2bc362a9d CURRENT_VERSION=0.87.8 CI_FULL=1 Date: Wed Jun 11 19:36:18 UTC 2025 System: ARCH=amd64 CPUS=128 MEM=493Gi HOSTNAME=pr-14898_amd64_x1-full Resources: CPU_LIST=0-127 CPUS=2 MEM=8g TIMEOUT=600s History: http://ci.aztec-labs.com/list/history_a57cc872d7daa810_master 19:36:18 +++ id -u 19:36:18 +++ id -g 19:36:18 ++ docker run -d --name p2p_src_services_reqresp_reqresp.test.ts --net=none --cpuset-cpus=0-127 --cpus=2 --memory=8g --user 1000:1000 -v/home/aztec-dev:/home/aztec-dev --mount type=tmpfs,target=/tmp,tmpfs-size=1g --workdir /home/aztec-dev/aztec-packages -e HOME -e VERBOSE -e GIT_CONFIG_GLOBAL=/home/aztec-dev/aztec-packages/build-images/src/home/.gitconfig -e FORCE_COLOR=true -e CPUS -e MEM aztecprotocol/build:3.0 /bin/bash -c 'timeout -v 600s bash -c '\''yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts'\''' 19:36:18 + cid=b785a2f1919949f08695162b2356bde56b01d1629c55c0f6e239543cf273bbe8 19:36:18 + set +x 19:36:27 [19:36:27.455] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:27 [19:36:27.462] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:27 [19:36:27.468] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:27 [19:36:27.469] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:28 [19:36:28.024] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:28 [19:36:28.029] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:28 [19:36:28.075] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:28 [19:36:28.076] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:28 [19:36:28.077] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:28 [19:36:28.078] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:29 [19:36:29.342] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:29 [19:36:29.343] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:29 [19:36:29.346] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:29 [19:36:29.348] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:29 [19:36:29.438] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:29 [19:36:29.439] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:29 [19:36:29.440] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:29 [19:36:29.440] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:29 [19:36:29.441] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:29 [19:36:29.441] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:29 [19:36:29.444] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:29 [19:36:29.444] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:30 [19:36:30.522] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWP8wZPGzjB9X3LmhfiwrXucJF4ZBSWBgzXJy9djRz4xnf 19:36:30 [19:36:30.523] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 19:36:30 err: { 19:36:30 "type": "ReqRespStatusError", 19:36:30 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 19:36:30 "stack": 19:36:30 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 19:36:30 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 19:36:30 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 19:36:30 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 19:36:30 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 19:36:30 at processTicksAndRejections (node:internal/process/task_queues:105:5) 19:36:30 "status": 1 19:36:30 } 19:36:30 [19:36:30.534] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWP8wZPGzjB9X3LmhfiwrXucJF4ZBSWBgzXJy9djRz4xnf 19:36:30 [19:36:30.534] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 19:36:30 err: { 19:36:30 "type": "ReqRespStatusError", 19:36:30 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 19:36:30 "stack": 19:36:30 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 19:36:30 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 19:36:30 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 19:36:30 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 19:36:30 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 19:36:30 at processTicksAndRejections (node:internal/process/task_queues:105:5) 19:36:30 "status": 1 19:36:30 } 19:36:30 [19:36:30.538] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWP8wZPGzjB9X3LmhfiwrXucJF4ZBSWBgzXJy9djRz4xnf 19:36:30 [19:36:30.538] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 19:36:30 err: { 19:36:30 "type": "ReqRespStatusError", 19:36:30 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 19:36:30 "stack": 19:36:30 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 19:36:30 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 19:36:30 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 19:36:30 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 19:36:30 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 19:36:30 at processTicksAndRejections (node:internal/process/task_queues:105:5) 19:36:30 "status": 1 19:36:30 } 19:36:30 [19:36:30.541] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWP8wZPGzjB9X3LmhfiwrXucJF4ZBSWBgzXJy9djRz4xnf 19:36:30 [19:36:30.541] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 19:36:30 err: { 19:36:30 "type": "ReqRespStatusError", 19:36:30 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 19:36:30 "stack": 19:36:30 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 19:36:30 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 19:36:30 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 19:36:30 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 19:36:30 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 19:36:30 at processTicksAndRejections (node:internal/process/task_queues:105:5) 19:36:30 "status": 1 19:36:30 } 19:36:30 [19:36:30.544] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWP8wZPGzjB9X3LmhfiwrXucJF4ZBSWBgzXJy9djRz4xnf 19:36:30 [19:36:30.545] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 19:36:30 err: { 19:36:30 "type": "ReqRespStatusError", 19:36:30 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 19:36:30 "stack": 19:36:30 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 19:36:30 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 19:36:30 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 19:36:30 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 19:36:30 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 19:36:30 at processTicksAndRejections (node:internal/process/task_queues:105:5) 19:36:30 "status": 1 19:36:30 } 19:36:30 [19:36:30.550] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:30 [19:36:30.551] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:30 [19:36:30.554] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:30 [19:36:30.555] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:32 [19:36:32.043] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:32 [19:36:32.044] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:32 [19:36:32.048] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:32 [19:36:32.048] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:33 [19:36:33.138] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:33 [19:36:33.138] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:33 [19:36:33.142] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:33 [19:36:33.143] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:34 [19:36:34.230] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:34 [19:36:34.231] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:34 [19:36:34.235] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:34 [19:36:34.235] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:37 [19:36:37.296] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:37 [19:36:37.296] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:37 [19:36:37.299] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:37 [19:36:37.300] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:42 [19:36:42.511] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:42 [19:36:42.512] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:42 [19:36:42.513] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:42 [19:36:42.514] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:42 [19:36:42.515] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:42 [19:36:42.515] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:42 [19:36:42.519] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:42 [19:36:42.520] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:42 [19:36:42.539] WARN: foundation:memory_fifo Discarding item because queue is flushing 19:36:42 [19:36:42.543] ERROR: p2p:reqresp Unexpected error sending request to peer: Error: Could not enqueue function 19:36:42 at /home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:65:24 19:36:42 at new Promise (<anonymous>) 19:36:42 at SerialQueue.put (/home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:55:16) 19:36:42 at ConnectionSampler.put [as dialProtocol] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/connection-sampler/connection_sampler.ts:204:43) 19:36:42 at ReqResp.dialProtocol (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:449:45) 19:36:42 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 19:36:42 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 19:36:42 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 19:36:42 at TimeoutTask.requestFunction [as fn] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:200:26) {"peerId":"12D3KooWLj3NDNkiGzFZRzWGHsp3jVwaq3TiVbfDgE44sD5NNuWt","subProtocol":"/aztec/req/tx/0.1.0"} 19:36:43 [19:36:43.655] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:43 [19:36:43.656] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:43 [19:36:43.660] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:43 [19:36:43.660] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:44 [19:36:44.706] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:44 [19:36:44.706] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:44 [19:36:44.710] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:44 [19:36:44.710] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:45 [19:36:45.757] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:45 [19:36:45.757] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:45 [19:36:45.761] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:45 [19:36:45.761] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:47 [19:36:47.029] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:47 [19:36:47.029] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:47 [19:36:47.033] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:47 [19:36:47.033] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:48 [19:36:48.159] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWSuRzTyMzZ3my6kUS3uSkMYPcamT6swpcb84dosooL6Bf 19:36:48 [19:36:48.160] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 19:36:48 err: { 19:36:48 "type": "ReqRespStatusError", 19:36:48 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 19:36:48 "stack": 19:36:48 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 19:36:48 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 19:36:48 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 19:36:48 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 19:36:48 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 19:36:48 at processTicksAndRejections (node:internal/process/task_queues:105:5) 19:36:48 "status": 1 19:36:48 } 19:36:48 [19:36:48.182] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:48 [19:36:48.182] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:48 [19:36:48.183] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:48 [19:36:48.184] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:48 [19:36:48.187] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:48 [19:36:48.188] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:49 [19:36:49.465] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:49 [19:36:49.465] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:49 [19:36:49.466] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:49 [19:36:49.466] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:49 [19:36:49.467] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:49 [19:36:49.467] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:49 [19:36:49.470] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:49 [19:36:49.470] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:50 [19:36:50.620] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAo5mum6343LAFzCeWpfR4J78EZYP7YL9PY4M16NNTKxV 19:36:50 [19:36:50.621] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 19:36:50 err: { 19:36:50 "type": "ReqRespStatusError", 19:36:50 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 19:36:50 "stack": 19:36:50 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 19:36:50 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 19:36:50 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 19:36:50 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 19:36:50 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 19:36:50 at processTicksAndRejections (node:internal/process/task_queues:105:5) 19:36:50 "status": 1 19:36:50 } 19:36:50 [19:36:50.627] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAo5mum6343LAFzCeWpfR4J78EZYP7YL9PY4M16NNTKxV 19:36:50 [19:36:50.627] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 19:36:50 err: { 19:36:50 "type": "ReqRespStatusError", 19:36:50 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 19:36:50 "stack": 19:36:50 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 19:36:50 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 19:36:50 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 19:36:50 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 19:36:50 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 19:36:50 at processTicksAndRejections (node:internal/process/task_queues:105:5) 19:36:50 "status": 1 19:36:50 } 19:36:50 [19:36:50.646] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAo5mum6343LAFzCeWpfR4J78EZYP7YL9PY4M16NNTKxV 19:36:50 [19:36:50.647] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 19:36:50 err: { 19:36:50 "type": "ReqRespStatusError", 19:36:50 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 19:36:50 "stack": 19:36:50 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 19:36:50 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 19:36:50 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 19:36:50 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 19:36:50 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 19:36:50 at processTicksAndRejections (node:internal/process/task_queues:105:5) 19:36:50 "status": 1 19:36:50 } 19:36:50 [19:36:50.659] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:50 [19:36:50.660] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:50 [19:36:50.661] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:50 [19:36:50.661] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:50 [19:36:50.665] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 19:36:50 [19:36:50.666] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 19:36:50 FAIL src/services/reqresp/reqresp.test.ts (31.006 s) 19:36:50 ReqResp 19:36:50 ✓ should perform a ping request (1320 ms) 19:36:50 ✓ should handle gracefully if a peer connected peer is offline (603 ms) 19:36:50 ✓ should request from a later peer if other peers are offline (1385 ms) 19:36:50 ✕ should hit a rate limit if too many requests are made in quick succession (1099 ms) 19:36:50 Tx req protocol 19:36:50 ✓ can request a Tx from TxHash (1487 ms) 19:36:50 ✓ handles returning empty buffers (1095 ms) 19:36:50 ✓ does not crash if tx hash returns undefined (1097 ms) 19:36:50 ✓ should hit individual timeout if nothing is returned over the stream (3056 ms) 19:36:50 ✓ should hit collective timeout if nothing is returned over the stream from multiple peers (5241 ms) 19:36:50 ✓ should penalize peer if transaction validation fails (1121 ms) 19:36:50 Goodbye protocol 19:36:50 ✓ should send a goodbye message to a peer (1052 ms) 19:36:50 ✓ should not close stream when handling a goodbye message received from peer (1051 ms) 19:36:50 Block protocol 19:36:50 ✓ should handle block requests (1266 ms) 19:36:50 Batch requests 19:36:50 ✕ should send a batch request between many peers (1160 ms) 19:36:50 ✕ should send a batch request with a pinned peer (1299 ms) 19:36:50 ✕ should stop after max retry attempts (1184 ms) 19:36:50 19:36:50 ● ReqResp › should hit a rate limit if too many requests are made in quick succession 19:36:50 19:36:50 expect(received).toBeDefined() 19:36:50 19:36:50 Received: undefined 19:36:50 19:36:50 145 | // Check that one of the responses gets a rate limit response 19:36:50 146 | const rateLimitResponse = responses.find(response => response?.status === ReqRespStatus.RATE_LIMIT_EXCEEDED); 19:36:50 > 147 | expect(rateLimitResponse).toBeDefined(); 19:36:50 | ^ 19:36:50 148 | 19:36:50 149 | // Make sure the error message is logged 19:36:50 150 | const errorMessage = `Rate limit exceeded DeniedPeer for ${ 19:36:50 19:36:50 at Object.toBeDefined (services/reqresp/reqresp.test.ts:147:31) 19:36:50 19:36:50 ● ReqResp › Batch requests › should send a batch request between many peers 19:36:50 19:36:50 expect(jest.fn()).toHaveBeenCalledTimes(expected) 19:36:50 19:36:50 Expected number of calls: 9 19:36:50 Received number of calls: 11 19:36:50 19:36:50 474 | 19:36:50 475 | // Expect one request to have been sent to each peer 19:36:50 > 476 | expect(sendRequestToPeerSpy).toHaveBeenCalledTimes(batchSize); 19:36:50 | ^ 19:36:50 477 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 19:36:50 478 | expect.objectContaining({ 19:36:50 479 | publicKey: nodes[1].p2p.peerId.publicKey, 19:36:50 19:36:50 at Object.toHaveBeenCalledTimes (services/reqresp/reqresp.test.ts:476:36) 19:36:50 19:36:50 ● ReqResp › Batch requests › should send a batch request with a pinned peer 19:36:50 19:36:50 expect(jest.fn()).toHaveBeenCalledWith(...expected) 19:36:50 19:36:50 Expected: ObjectContaining {"publicKey": [8, 1, 18, 32, 185, 106, 95, 109, 235, 194, …]}, "/aztec/req/ping/0.1.0", {"data": [112, 105, 110, 103, 51], "type": "Buffer"} 19:36:50 Received 19:36:50 1 19:36:50 "12D3KooWAyHW4V6MMjad4P98oQbu6jmuK9s4pAewfsfqV9g6vAc3", 19:36:50 "/aztec/req/ping/0.1.0", 19:36:50 @@ -2,9 +2,9 @@ 19:36:50 "data": Array [ 19:36:50 112, 19:36:50 105, 19:36:50 110, 19:36:50 103, 19:36:50 - 51, 19:36:50 + 48, 19:36:50 ], 19:36:50 "type": "Buffer", 19:36:50 }, 19:36:50 2 19:36:50 "12D3KooWSYHqivx7RoENPbz7S6D7iiyGkVUWxrzpmpJSgJi4t1Lw", 19:36:50 "/aztec/req/ping/0.1.0", 19:36:50 @@ -2,9 +2,9 @@ 19:36:50 "data": Array [ 19:36:50 112, 19:36:50 105, 19:36:50 110, 19:36:50 103, 19:36:50 - 51, 19:36:50 + 49, 19:36:50 ], 19:36:50 "type": "Buffer", 19:36:50 }, 19:36:50 3 19:36:50 "12D3KooWNJ9fnVtmXVLkQ6dfFm49ybWpKgrNNuZp5v3gRbnjRkvU", 19:36:50 "/aztec/req/ping/0.1.0", 19:36:50 @@ -2,9 +2,9 @@ 19:36:50 "data": Array [ 19:36:50 112, 19:36:50 105, 19:36:50 110, 19:36:50 103, 19:36:50 - 51, 19:36:50 + 48, 19:36:50 ], 19:36:50 "type": "Buffer", 19:36:50 }, 19:36:50 19:36:50 Number of calls: 15 19:36:50 19:36:50 516 | // Expect pinned peer to have received all requests 19:36:50 517 | for (let i = 0; i < batchSize; i++) { 19:36:50 > 518 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 19:36:50 | ^ 19:36:50 519 | expect.objectContaining({ publicKey: nodes[1].p2p.peerId.publicKey }), 19:36:50 520 | ReqRespSubProtocol.PING, 19:36:50 521 | Buffer.from(`ping${i}`), 19:36:50 19:36:50 at Object.toHaveBeenCalledWith (services/reqresp/reqresp.test.ts:518:38) 19:36:50 19:36:50 ● ReqResp › Batch requests › should stop after max retry attempts 19:36:50 19:36:50 expect(received).toEqual(expected) // deep equality 19:36:50 19:36:50 - Expected - 11 19:36:50 + Received + 3 19:36:50 19:36:50 @@ -74,10 +74,11 @@ 19:36:50 103, 19:36:50 ], 19:36:50 "type": "Buffer", 19:36:50 }, 19:36:50 }, 19:36:50 + undefined, 19:36:50 RequestableBuffer { 19:36:50 "buffer": Object { 19:36:50 "data": Array [ 19:36:50 112, 19:36:50 111, 19:36:50 @@ -85,10 +86,11 @@ 19:36:50 103, 19:36:50 ], 19:36:50 "type": "Buffer", 19:36:50 }, 19:36:50 }, 19:36:50 + undefined, 19:36:50 RequestableBuffer { 19:36:50 "buffer": Object { 19:36:50 "data": Array [ 19:36:50 112, 19:36:50 111, 19:36:50 @@ -96,17 +98,7 @@ 19:36:50 103, 19:36:50 ], 19:36:50 "type": "Buffer", 19:36:50 }, 19:36:50 }, 19:36:50 - RequestableBuffer { 19:36:50 - "buffer": Object { 19:36:50 - "data": Array [ 19:36:50 - 112, 19:36:50 - 111, 19:36:50 - 110, 19:36:50 - 103, 19:36:50 - ], 19:36:50 - "type": "Buffer", 19:36:50 - }, 19:36:50 - }, 19:36:50 + undefined, 19:36:50 ] 19:36:50 19:36:50 555 | 19:36:50 556 | const res = await nodes[0].req.sendBatchRequest(ReqRespSubProtocol.PING, requests, undefined); 19:36:50 > 557 | expect(res).toEqual(expectResponses); 19:36:50 | ^ 19:36:50 558 | 19:36:50 559 | // Check that we did detect hitting a rate limit 19:36:50 560 | expect(requesterLoggerSpy).toHaveBeenCalledWith( 19:36:50 19:36:50 at Object.toEqual (services/reqresp/reqresp.test.ts:557:19) 19:36:50 19:36:50 Test Suites: 1 failed, 1 total 19:36:50 Tests: 4 failed, 12 passed, 16 total 19:36:50 Snapshots: 0 total 19:36:50 Time: 31.088 s 19:36:50 Ran all test suites matching /p2p\/src\/services\/reqresp\/reqresp.test.ts/i. 19:36:50 Force exiting Jest: Have you considered using `--detectOpenHandles` to detect async operations that kept running after all tests finished?