ACI ·

Parent Log: http://ci.aztec-labs.com/b772a3ee9d4bbdac Command: fe89a5d9e20c90c9:ISOLATE=1:NAME=p2p/src/services/reqresp/reqresp.test.ts yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts Commit: https://github.com/AztecProtocol/aztec-packages/commit/260d4b0cfbe16cbd8be248715852ee1ff72cc6cd Env: REF_NAME=gh-readonly-queue/master/pr-14716-b9f63e086a00464c0ad1800566372d3b7ddc6746 CURRENT_VERSION=0.87.7 CI_FULL=1 Date: Tue Jun 3 13:33:13 UTC 2025 System: ARCH=amd64 CPUS=128 MEM=493Gi HOSTNAME=pr-14716_amd64_x2-full Resources: CPU_LIST=0-127 CPUS=2 MEM=8g TIMEOUT=600s History: http://ci.aztec-labs.com/list/a57cc872d7daa810 13:33:13 +++ id -u 13:33:13 +++ id -g 13:33:13 ++ docker run -d --name p2p_src_services_reqresp_reqresp.test.ts --net=none --cpuset-cpus=0-127 --cpus=2 --memory=8g --user 1000:1000 -v/home/aztec-dev:/home/aztec-dev --mount type=tmpfs,target=/tmp,tmpfs-size=1g --workdir /home/aztec-dev/aztec-packages -e HOME -e VERBOSE -e GIT_CONFIG_GLOBAL=/home/aztec-dev/aztec-packages/build-images/src/home/.gitconfig -e FORCE_COLOR=true -e CPUS -e MEM aztecprotocol/build:3.0 /bin/bash -c 'timeout -v 600s bash -c '\''yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts'\''' 13:33:14 + cid=68ecb5b4f06d4bfba0683f0d6a7d966edd07b464dac3baa13892e152b04b1357 13:33:14 + set +x 13:33:22 [13:33:22.905] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:22 [13:33:22.910] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:22 [13:33:22.915] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:22 [13:33:22.915] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:23 [13:33:23.464] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:23 [13:33:23.465] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:23 [13:33:23.522] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:23 [13:33:23.522] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:23 [13:33:23.523] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:23 [13:33:23.524] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:24 [13:33:24.743] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:24 [13:33:24.743] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:24 [13:33:24.746] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:24 [13:33:24.747] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:24 [13:33:24.826] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:24 [13:33:24.827] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:24 [13:33:24.828] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:24 [13:33:24.829] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:24 [13:33:24.830] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:24 [13:33:24.830] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:24 [13:33:24.834] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:24 [13:33:24.835] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:25 [13:33:25.917] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWLygzj97Xkvr4dabMLPPVdhXwPt3zX6FePbeWwQBUY1jS 13:33:25 [13:33:25.918] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 13:33:25 err: { 13:33:25 "type": "ReqRespStatusError", 13:33:25 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 13:33:25 "stack": 13:33:25 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 13:33:25 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 13:33:25 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 13:33:25 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 13:33:25 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 13:33:25 at processTicksAndRejections (node:internal/process/task_queues:105:5) 13:33:25 "status": 1 13:33:25 } 13:33:25 [13:33:25.927] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWLygzj97Xkvr4dabMLPPVdhXwPt3zX6FePbeWwQBUY1jS 13:33:25 [13:33:25.927] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 13:33:25 err: { 13:33:25 "type": "ReqRespStatusError", 13:33:25 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 13:33:25 "stack": 13:33:25 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 13:33:25 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 13:33:25 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 13:33:25 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 13:33:25 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 13:33:25 at processTicksAndRejections (node:internal/process/task_queues:105:5) 13:33:25 "status": 1 13:33:25 } 13:33:25 [13:33:25.932] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWLygzj97Xkvr4dabMLPPVdhXwPt3zX6FePbeWwQBUY1jS 13:33:25 [13:33:25.932] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 13:33:25 err: { 13:33:25 "type": "ReqRespStatusError", 13:33:25 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 13:33:25 "stack": 13:33:25 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 13:33:25 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 13:33:25 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 13:33:25 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 13:33:25 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 13:33:25 at processTicksAndRejections (node:internal/process/task_queues:105:5) 13:33:25 "status": 1 13:33:25 } 13:33:25 [13:33:25.937] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWLygzj97Xkvr4dabMLPPVdhXwPt3zX6FePbeWwQBUY1jS 13:33:25 [13:33:25.937] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 13:33:25 err: { 13:33:25 "type": "ReqRespStatusError", 13:33:25 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 13:33:25 "stack": 13:33:25 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 13:33:25 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 13:33:25 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 13:33:25 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 13:33:25 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 13:33:25 at processTicksAndRejections (node:internal/process/task_queues:105:5) 13:33:25 "status": 1 13:33:25 } 13:33:25 [13:33:25.941] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWLygzj97Xkvr4dabMLPPVdhXwPt3zX6FePbeWwQBUY1jS 13:33:25 [13:33:25.942] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 13:33:25 err: { 13:33:25 "type": "ReqRespStatusError", 13:33:25 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 13:33:25 "stack": 13:33:25 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 13:33:25 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 13:33:25 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 13:33:25 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 13:33:25 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 13:33:25 at processTicksAndRejections (node:internal/process/task_queues:105:5) 13:33:25 "status": 1 13:33:25 } 13:33:25 [13:33:25.949] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:25 [13:33:25.949] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:25 [13:33:25.953] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:25 [13:33:25.954] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:27 [13:33:27.425] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:27 [13:33:27.425] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:27 [13:33:27.428] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:27 [13:33:27.428] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:28 [13:33:28.509] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:28 [13:33:28.509] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:28 [13:33:28.512] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:28 [13:33:28.513] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:29 [13:33:29.597] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:29 [13:33:29.597] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:29 [13:33:29.600] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:29 [13:33:29.600] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:32 [13:33:32.645] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:32 [13:33:32.646] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:32 [13:33:32.648] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:32 [13:33:32.649] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:37 [13:33:37.851] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:37 [13:33:37.852] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:37 [13:33:37.853] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:37 [13:33:37.853] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:37 [13:33:37.854] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:37 [13:33:37.855] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:37 [13:33:37.858] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:37 [13:33:37.859] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:37 [13:33:37.887] WARN: foundation:memory_fifo Discarding item because queue is flushing 13:33:37 [13:33:37.890] ERROR: p2p:reqresp Unexpected error sending request to peer: Error: Could not enqueue function 13:33:37 at /home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:63:24 13:33:37 at new Promise (<anonymous>) 13:33:37 at SerialQueue.put (/home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:53:16) 13:33:37 at ConnectionSampler.put [as dialProtocol] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/connection-sampler/connection_sampler.ts:204:43) 13:33:37 at ReqResp.dialProtocol (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:449:45) 13:33:37 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 13:33:37 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 13:33:37 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 13:33:37 at TimeoutTask.requestFunction [as fn] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:200:26) {"peerId":"12D3KooWSZRZYAYMJLq2JbbvQo4b4asxDyTPyfmG8hrBo2xktktM","subProtocol":"/aztec/req/tx/0.1.0"} 13:33:38 [13:33:38.992] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:38 [13:33:38.993] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:38 [13:33:38.997] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:38 [13:33:38.998] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:40 [13:33:40.051] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:40 [13:33:40.052] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:40 [13:33:40.055] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:40 [13:33:40.056] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:41 [13:33:41.103] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:41 [13:33:41.104] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:41 [13:33:41.107] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:41 [13:33:41.107] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:42 [13:33:42.356] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:42 [13:33:42.356] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:42 [13:33:42.360] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:42 [13:33:42.361] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:43 [13:33:43.494] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooW9tAk4ZYcamjbQz4UQns2Z5j1w6Eu152BE4M8zodjF32E 13:33:43 [13:33:43.495] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 13:33:43 err: { 13:33:43 "type": "ReqRespStatusError", 13:33:43 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 13:33:43 "stack": 13:33:43 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 13:33:43 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 13:33:43 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 13:33:43 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 13:33:43 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 13:33:43 at processTicksAndRejections (node:internal/process/task_queues:105:5) 13:33:43 "status": 1 13:33:43 } 13:33:43 [13:33:43.515] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:43 [13:33:43.516] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:43 [13:33:43.517] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:43 [13:33:43.517] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:43 [13:33:43.520] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:43 [13:33:43.521] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:44 [13:33:44.777] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:44 [13:33:44.777] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:44 [13:33:44.778] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:44 [13:33:44.778] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:44 [13:33:44.779] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:44 [13:33:44.780] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:44 [13:33:44.784] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:44 [13:33:44.785] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:45 [13:33:45.905] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWKsVhAJZyFDA2nErUN6gceiqpmZ2xpqDxX2zVELyjSziz 13:33:45 [13:33:45.906] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 13:33:45 err: { 13:33:45 "type": "ReqRespStatusError", 13:33:45 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 13:33:45 "stack": 13:33:45 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 13:33:45 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 13:33:45 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 13:33:45 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 13:33:45 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 13:33:45 at processTicksAndRejections (node:internal/process/task_queues:105:5) 13:33:45 "status": 1 13:33:45 } 13:33:45 [13:33:45.910] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWKsVhAJZyFDA2nErUN6gceiqpmZ2xpqDxX2zVELyjSziz 13:33:45 [13:33:45.910] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 13:33:45 err: { 13:33:45 "type": "ReqRespStatusError", 13:33:45 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 13:33:45 "stack": 13:33:45 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 13:33:45 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 13:33:45 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 13:33:45 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 13:33:45 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 13:33:45 at processTicksAndRejections (node:internal/process/task_queues:105:5) 13:33:45 "status": 1 13:33:45 } 13:33:45 [13:33:45.927] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWKsVhAJZyFDA2nErUN6gceiqpmZ2xpqDxX2zVELyjSziz 13:33:45 [13:33:45.927] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 13:33:45 err: { 13:33:45 "type": "ReqRespStatusError", 13:33:45 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 13:33:45 "stack": 13:33:45 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 13:33:45 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 13:33:45 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 13:33:45 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 13:33:45 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 13:33:45 at processTicksAndRejections (node:internal/process/task_queues:105:5) 13:33:45 "status": 1 13:33:45 } 13:33:45 [13:33:45.941] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:45 [13:33:45.941] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:45 [13:33:45.943] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:45 [13:33:45.943] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:45 [13:33:45.947] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 13:33:45 [13:33:45.947] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 13:33:46 FAIL src/services/reqresp/reqresp.test.ts (30.969 s) 13:33:46 ReqResp 13:33:46 ✓ should perform a ping request (1258 ms) 13:33:46 ✓ should handle gracefully if a peer connected peer is offline (603 ms) 13:33:46 ✓ should request from a later peer if other peers are offline (1336 ms) 13:33:46 ✕ should hit a rate limit if too many requests are made in quick succession (1095 ms) 13:33:46 Tx req protocol 13:33:46 ✓ can request a Tx from TxHash (1473 ms) 13:33:46 ✓ handles returning empty buffers (1087 ms) 13:33:46 ✓ does not crash if tx hash returns undefined (1080 ms) 13:33:46 ✓ should hit individual timeout if nothing is returned over the stream (3050 ms) 13:33:46 ✓ should hit collective timeout if nothing is returned over the stream from multiple peers (5238 ms) 13:33:46 ✓ should penalize peer if transaction validation fails (1116 ms) 13:33:46 Goodbye protocol 13:33:46 ✓ should send a goodbye message to a peer (1052 ms) 13:33:46 ✓ should not close stream when handling a goodbye message received from peer (1051 ms) 13:33:46 Block protocol 13:33:46 ✓ should handle block requests (1261 ms) 13:33:46 Batch requests 13:33:46 ✕ should send a batch request between many peers (1163 ms) 13:33:46 ✕ should send a batch request with a pinned peer (1269 ms) 13:33:46 ✕ should stop after max retry attempts (1161 ms) 13:33:46 13:33:46 ● ReqResp › should hit a rate limit if too many requests are made in quick succession 13:33:46 13:33:46 expect(received).toBeDefined() 13:33:46 13:33:46 Received: undefined 13:33:46 13:33:46 145 | // Check that one of the responses gets a rate limit response 13:33:46 146 | const rateLimitResponse = responses.find(response => response?.status === ReqRespStatus.RATE_LIMIT_EXCEEDED); 13:33:46 > 147 | expect(rateLimitResponse).toBeDefined(); 13:33:46 | ^ 13:33:46 148 | 13:33:46 149 | // Make sure the error message is logged 13:33:46 150 | const errorMessage = `Rate limit exceeded DeniedPeer for ${ 13:33:46 13:33:46 at Object.toBeDefined (services/reqresp/reqresp.test.ts:147:31) 13:33:46 13:33:46 ● ReqResp › Batch requests › should send a batch request between many peers 13:33:46 13:33:46 expect(jest.fn()).toHaveBeenCalledTimes(expected) 13:33:46 13:33:46 Expected number of calls: 9 13:33:46 Received number of calls: 11 13:33:46 13:33:46 474 | 13:33:46 475 | // Expect one request to have been sent to each peer 13:33:46 > 476 | expect(sendRequestToPeerSpy).toHaveBeenCalledTimes(batchSize); 13:33:46 | ^ 13:33:46 477 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 13:33:46 478 | expect.objectContaining({ 13:33:46 479 | publicKey: nodes[1].p2p.peerId.publicKey, 13:33:46 13:33:46 at Object.toHaveBeenCalledTimes (services/reqresp/reqresp.test.ts:476:36) 13:33:46 13:33:46 ● ReqResp › Batch requests › should send a batch request with a pinned peer 13:33:46 13:33:46 expect(jest.fn()).toHaveBeenCalledWith(...expected) 13:33:46 13:33:46 Expected: ObjectContaining {"publicKey": [8, 1, 18, 32, 107, 97, 48, 119, 217, 116, …]}, "/aztec/req/ping/0.1.0", {"data": [112, 105, 110, 103, 49], "type": "Buffer"} 13:33:46 Received 13:33:46 1 13:33:46 "12D3KooWBzQSR6qeVJJjsNgp56YxMybAhZFpoUJDcoYS34vkvvUX", 13:33:46 "/aztec/req/ping/0.1.0", 13:33:46 @@ -2,9 +2,9 @@ 13:33:46 "data": Array [ 13:33:46 112, 13:33:46 105, 13:33:46 110, 13:33:46 103, 13:33:46 - 49, 13:33:46 + 48, 13:33:46 ], 13:33:46 "type": "Buffer", 13:33:46 }, 13:33:46 2 13:33:46 "12D3KooWLCxW1twXZcV2a8DAauuRXcnhPJMRZpGRhS6L6uygqvRk", 13:33:46 "/aztec/req/ping/0.1.0", 13:33:46 {"data": [112, 105, 110, 103, 49], "type": "Buffer"}, 13:33:46 3 13:33:46 "12D3KooWH3XkHrV2bhrTUhbSbv9BwuECGAEhgQaP9Xth6iBEY16g", 13:33:46 "/aztec/req/ping/0.1.0", 13:33:46 @@ -2,9 +2,9 @@ 13:33:46 "data": Array [ 13:33:46 112, 13:33:46 105, 13:33:46 110, 13:33:46 103, 13:33:46 - 49, 13:33:46 + 48, 13:33:46 ], 13:33:46 "type": "Buffer", 13:33:46 }, 13:33:46 13:33:46 Number of calls: 13 13:33:46 13:33:46 516 | // Expect pinned peer to have received all requests 13:33:46 517 | for (let i = 0; i < batchSize; i++) { 13:33:46 > 518 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 13:33:46 | ^ 13:33:46 519 | expect.objectContaining({ publicKey: nodes[1].p2p.peerId.publicKey }), 13:33:46 520 | ReqRespSubProtocol.PING, 13:33:46 521 | Buffer.from(`ping${i}`), 13:33:46 13:33:46 at Object.toHaveBeenCalledWith (services/reqresp/reqresp.test.ts:518:38) 13:33:46 13:33:46 ● ReqResp › Batch requests › should stop after max retry attempts 13:33:46 13:33:46 expect(received).toEqual(expected) // deep equality 13:33:46 13:33:46 - Expected - 11 13:33:46 + Received + 3 13:33:46 13:33:46 @@ -74,10 +74,11 @@ 13:33:46 103, 13:33:46 ], 13:33:46 "type": "Buffer", 13:33:46 }, 13:33:46 }, 13:33:46 + undefined, 13:33:46 RequestableBuffer { 13:33:46 "buffer": Object { 13:33:46 "data": Array [ 13:33:46 112, 13:33:46 111, 13:33:46 @@ -85,10 +86,11 @@ 13:33:46 103, 13:33:46 ], 13:33:46 "type": "Buffer", 13:33:46 }, 13:33:46 }, 13:33:46 + undefined, 13:33:46 RequestableBuffer { 13:33:46 "buffer": Object { 13:33:46 "data": Array [ 13:33:46 112, 13:33:46 111, 13:33:46 @@ -96,17 +98,7 @@ 13:33:46 103, 13:33:46 ], 13:33:46 "type": "Buffer", 13:33:46 }, 13:33:46 }, 13:33:46 - RequestableBuffer { 13:33:46 - "buffer": Object { 13:33:46 - "data": Array [ 13:33:46 - 112, 13:33:46 - 111, 13:33:46 - 110, 13:33:46 - 103, 13:33:46 - ], 13:33:46 - "type": "Buffer", 13:33:46 - }, 13:33:46 - }, 13:33:46 + undefined, 13:33:46 ] 13:33:46 13:33:46 555 | 13:33:46 556 | const res = await nodes[0].req.sendBatchRequest(ReqRespSubProtocol.PING, requests, undefined); 13:33:46 > 557 | expect(res).toEqual(expectResponses); 13:33:46 | ^ 13:33:46 558 | 13:33:46 559 | // Check that we did detect hitting a rate limit 13:33:46 560 | expect(requesterLoggerSpy).toHaveBeenCalledWith( 13:33:46 13:33:46 at Object.toEqual (services/reqresp/reqresp.test.ts:557:19) 13:33:46 13:33:46 Test Suites: 1 failed, 1 total 13:33:46 Tests: 4 failed, 12 passed, 16 total 13:33:46 Snapshots: 0 total 13:33:46 Time: 31.048 s 13:33:46 Ran all test suites matching /p2p\/src\/services\/reqresp\/reqresp.test.ts/i. 13:33:46 Force exiting Jest: Have you considered using `--detectOpenHandles` to detect async operations that kept running after all tests finished?