ACI ·

Parent Log: http://ci.aztec-labs.com/0270c8bb05dfec64 Command: 59d7caa3bc5b3609:ISOLATE=1:NAME=p2p/src/services/reqresp/reqresp.test.ts yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts Commit: https://github.com/AztecProtocol/aztec-packages/commit/64b2b9f661d3e31df0f8ec9120130b753eeb7533 Env: REF_NAME=gh-readonly-queue/master/pr-14685-5f78d61bb53ab2c2ccd59528d4439a88b2325b96 CURRENT_VERSION=0.87.8 CI_FULL=1 Date: Mon Jun 9 18:55:02 UTC 2025 System: ARCH=amd64 CPUS=128 MEM=493Gi HOSTNAME=pr-14685_amd64_x1-full Resources: CPU_LIST=0-127 CPUS=2 MEM=8g TIMEOUT=600s History: http://ci.aztec-labs.com/list/history_a57cc872d7daa810_master 18:55:02 +++ id -u 18:55:02 +++ id -g 18:55:02 ++ docker run -d --name p2p_src_services_reqresp_reqresp.test.ts --net=none --cpuset-cpus=0-127 --cpus=2 --memory=8g --user 1000:1000 -v/home/aztec-dev:/home/aztec-dev --mount type=tmpfs,target=/tmp,tmpfs-size=1g --workdir /home/aztec-dev/aztec-packages -e HOME -e VERBOSE -e GIT_CONFIG_GLOBAL=/home/aztec-dev/aztec-packages/build-images/src/home/.gitconfig -e FORCE_COLOR=true -e CPUS -e MEM aztecprotocol/build:3.0 /bin/bash -c 'timeout -v 600s bash -c '\''yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts'\''' 18:55:02 + cid=488e3019a96a4262f8e736d50224a7ce34ea335cce05964f6227710a6e464b4c 18:55:02 + set +x 18:55:11 [18:55:11.851] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:11 [18:55:11.858] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:11 [18:55:11.865] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:11 [18:55:11.866] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:12 [18:55:12.451] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:12 [18:55:12.452] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:12 [18:55:12.520] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:12 [18:55:12.521] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:12 [18:55:12.522] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:12 [18:55:12.524] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:13 [18:55:13.784] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:13 [18:55:13.786] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:13 [18:55:13.789] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:13 [18:55:13.792] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:13 [18:55:13.919] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:13 [18:55:13.920] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:13 [18:55:13.921] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:13 [18:55:13.922] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:13 [18:55:13.923] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:13 [18:55:13.923] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:13 [18:55:13.932] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:13 [18:55:13.933] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:15 [18:55:15.024] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAF7YL5Yi852TGpioCQHu6ZUwB57vGLHCC7GYnXCTNwhX 18:55:15 [18:55:15.025] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 18:55:15 err: { 18:55:15 "type": "ReqRespStatusError", 18:55:15 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 18:55:15 "stack": 18:55:15 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 18:55:15 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 18:55:15 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 18:55:15 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 18:55:15 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 18:55:15 at processTicksAndRejections (node:internal/process/task_queues:105:5) 18:55:15 "status": 1 18:55:15 } 18:55:15 [18:55:15.036] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAF7YL5Yi852TGpioCQHu6ZUwB57vGLHCC7GYnXCTNwhX 18:55:15 [18:55:15.037] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 18:55:15 err: { 18:55:15 "type": "ReqRespStatusError", 18:55:15 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 18:55:15 "stack": 18:55:15 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 18:55:15 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 18:55:15 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 18:55:15 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 18:55:15 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 18:55:15 at processTicksAndRejections (node:internal/process/task_queues:105:5) 18:55:15 "status": 1 18:55:15 } 18:55:15 [18:55:15.041] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAF7YL5Yi852TGpioCQHu6ZUwB57vGLHCC7GYnXCTNwhX 18:55:15 [18:55:15.042] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 18:55:15 err: { 18:55:15 "type": "ReqRespStatusError", 18:55:15 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 18:55:15 "stack": 18:55:15 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 18:55:15 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 18:55:15 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 18:55:15 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 18:55:15 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 18:55:15 at processTicksAndRejections (node:internal/process/task_queues:105:5) 18:55:15 "status": 1 18:55:15 } 18:55:15 [18:55:15.046] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAF7YL5Yi852TGpioCQHu6ZUwB57vGLHCC7GYnXCTNwhX 18:55:15 [18:55:15.046] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 18:55:15 err: { 18:55:15 "type": "ReqRespStatusError", 18:55:15 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 18:55:15 "stack": 18:55:15 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 18:55:15 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 18:55:15 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 18:55:15 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 18:55:15 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 18:55:15 at processTicksAndRejections (node:internal/process/task_queues:105:5) 18:55:15 "status": 1 18:55:15 } 18:55:15 [18:55:15.051] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAF7YL5Yi852TGpioCQHu6ZUwB57vGLHCC7GYnXCTNwhX 18:55:15 [18:55:15.051] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 18:55:15 err: { 18:55:15 "type": "ReqRespStatusError", 18:55:15 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 18:55:15 "stack": 18:55:15 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 18:55:15 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 18:55:15 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 18:55:15 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 18:55:15 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 18:55:15 at processTicksAndRejections (node:internal/process/task_queues:105:5) 18:55:15 "status": 1 18:55:15 } 18:55:15 [18:55:15.059] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:15 [18:55:15.059] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:15 [18:55:15.063] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:15 [18:55:15.063] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:16 [18:55:16.478] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:16 [18:55:16.479] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:16 [18:55:16.482] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:16 [18:55:16.482] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:17 [18:55:17.559] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:17 [18:55:17.559] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:17 [18:55:17.562] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:17 [18:55:17.562] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:18 [18:55:18.665] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:18 [18:55:18.666] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:18 [18:55:18.672] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:18 [18:55:18.672] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:21 [18:55:21.740] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:21 [18:55:21.741] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:21 [18:55:21.745] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:21 [18:55:21.746] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:26 [18:55:26.970] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:26 [18:55:26.972] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:26 [18:55:26.975] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:26 [18:55:26.976] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:26 [18:55:26.977] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:26 [18:55:26.978] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:26 [18:55:26.982] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:26 [18:55:26.983] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:27 [18:55:27.003] WARN: foundation:memory_fifo Discarding item because queue is flushing 18:55:27 [18:55:27.009] ERROR: p2p:reqresp Unexpected error sending request to peer: Error: Could not enqueue function 18:55:27 at /home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:65:24 18:55:27 at new Promise (<anonymous>) 18:55:27 at SerialQueue.put (/home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:55:16) 18:55:27 at ConnectionSampler.put [as dialProtocol] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/connection-sampler/connection_sampler.ts:204:43) 18:55:27 at ReqResp.dialProtocol (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:449:45) 18:55:27 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 18:55:27 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 18:55:27 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 18:55:27 at TimeoutTask.requestFunction [as fn] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:200:26) {"peerId":"12D3KooWDPbRsxutQRhRNck77aBczYpifQ5Ebgvub8SotUhHMdRY","subProtocol":"/aztec/req/tx/0.1.0"} 18:55:28 [18:55:28.104] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:28 [18:55:28.105] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:28 [18:55:28.111] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:28 [18:55:28.112] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:29 [18:55:29.160] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:29 [18:55:29.161] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:29 [18:55:29.165] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:29 [18:55:29.166] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:30 [18:55:30.229] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:30 [18:55:30.229] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:30 [18:55:30.233] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:30 [18:55:30.234] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:31 [18:55:31.453] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:31 [18:55:31.454] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:31 [18:55:31.457] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:31 [18:55:31.457] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:32 [18:55:32.575] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWBfwLAtNxe534AqVr5zaTKRjybQvjFTpawpkpME6MoZHT 18:55:32 [18:55:32.576] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 18:55:32 err: { 18:55:32 "type": "ReqRespStatusError", 18:55:32 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 18:55:32 "stack": 18:55:32 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 18:55:32 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 18:55:32 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 18:55:32 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 18:55:32 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 18:55:32 at processTicksAndRejections (node:internal/process/task_queues:105:5) 18:55:32 "status": 1 18:55:32 } 18:55:32 [18:55:32.603] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:32 [18:55:32.604] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:32 [18:55:32.605] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:32 [18:55:32.605] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:32 [18:55:32.609] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:32 [18:55:32.610] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:33 [18:55:33.911] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:33 [18:55:33.912] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:33 [18:55:33.913] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:33 [18:55:33.914] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:33 [18:55:33.915] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:33 [18:55:33.915] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:33 [18:55:33.919] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:33 [18:55:33.920] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:35 [18:55:35.112] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWB7tuN1FR4xcEUt7UqZG5whfQvfg2VbdwwBmo246f4rsV 18:55:35 [18:55:35.113] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 18:55:35 err: { 18:55:35 "type": "ReqRespStatusError", 18:55:35 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 18:55:35 "stack": 18:55:35 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 18:55:35 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 18:55:35 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 18:55:35 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 18:55:35 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 18:55:35 at processTicksAndRejections (node:internal/process/task_queues:105:5) 18:55:35 "status": 1 18:55:35 } 18:55:35 [18:55:35.121] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWB7tuN1FR4xcEUt7UqZG5whfQvfg2VbdwwBmo246f4rsV 18:55:35 [18:55:35.122] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 18:55:35 err: { 18:55:35 "type": "ReqRespStatusError", 18:55:35 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 18:55:35 "stack": 18:55:35 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 18:55:35 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 18:55:35 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 18:55:35 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 18:55:35 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 18:55:35 at processTicksAndRejections (node:internal/process/task_queues:105:5) 18:55:35 "status": 1 18:55:35 } 18:55:35 [18:55:35.149] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWB7tuN1FR4xcEUt7UqZG5whfQvfg2VbdwwBmo246f4rsV 18:55:35 [18:55:35.150] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 18:55:35 err: { 18:55:35 "type": "ReqRespStatusError", 18:55:35 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 18:55:35 "stack": 18:55:35 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 18:55:35 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 18:55:35 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 18:55:35 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 18:55:35 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 18:55:35 at processTicksAndRejections (node:internal/process/task_queues:105:5) 18:55:35 "status": 1 18:55:35 } 18:55:35 [18:55:35.169] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:35 [18:55:35.170] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:35 [18:55:35.172] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:35 [18:55:35.172] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:35 [18:55:35.177] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 18:55:35 [18:55:35.177] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 18:55:35 FAIL src/services/reqresp/reqresp.test.ts (31.469 s) 18:55:35 ReqResp 18:55:35 ✓ should perform a ping request (1410 ms) 18:55:35 ✓ should handle gracefully if a peer connected peer is offline (621 ms) 18:55:35 ✓ should request from a later peer if other peers are offline (1440 ms) 18:55:35 ✕ should hit a rate limit if too many requests are made in quick succession (1100 ms) 18:55:35 Tx req protocol 18:55:35 ✓ can request a Tx from TxHash (1419 ms) 18:55:35 ✓ handles returning empty buffers (1079 ms) 18:55:35 ✓ does not crash if tx hash returns undefined (1111 ms) 18:55:35 ✓ should hit individual timeout if nothing is returned over the stream (3073 ms) 18:55:35 ✓ should hit collective timeout if nothing is returned over the stream from multiple peers (5254 ms) 18:55:35 ✓ should penalize peer if transaction validation fails (1105 ms) 18:55:35 Goodbye protocol 18:55:35 ✓ should send a goodbye message to a peer (1055 ms) 18:55:35 ✓ should not close stream when handling a goodbye message received from peer (1067 ms) 18:55:35 Block protocol 18:55:35 ✓ should handle block requests (1222 ms) 18:55:35 Batch requests 18:55:35 ✕ should send a batch request between many peers (1160 ms) 18:55:35 ✕ should send a batch request with a pinned peer (1320 ms) 18:55:35 ✕ should stop after max retry attempts (1260 ms) 18:55:35 18:55:35 ● ReqResp › should hit a rate limit if too many requests are made in quick succession 18:55:35 18:55:35 expect(received).toBeDefined() 18:55:35 18:55:35 Received: undefined 18:55:35 18:55:35 145 | // Check that one of the responses gets a rate limit response 18:55:35 146 | const rateLimitResponse = responses.find(response => response?.status === ReqRespStatus.RATE_LIMIT_EXCEEDED); 18:55:35 > 147 | expect(rateLimitResponse).toBeDefined(); 18:55:35 | ^ 18:55:35 148 | 18:55:35 149 | // Make sure the error message is logged 18:55:35 150 | const errorMessage = `Rate limit exceeded DeniedPeer for ${ 18:55:35 18:55:35 at Object.toBeDefined (services/reqresp/reqresp.test.ts:147:31) 18:55:35 18:55:35 ● ReqResp › Batch requests › should send a batch request between many peers 18:55:35 18:55:35 expect(jest.fn()).toHaveBeenCalledTimes(expected) 18:55:35 18:55:35 Expected number of calls: 9 18:55:35 Received number of calls: 11 18:55:35 18:55:35 474 | 18:55:35 475 | // Expect one request to have been sent to each peer 18:55:35 > 476 | expect(sendRequestToPeerSpy).toHaveBeenCalledTimes(batchSize); 18:55:35 | ^ 18:55:35 477 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 18:55:35 478 | expect.objectContaining({ 18:55:35 479 | publicKey: nodes[1].p2p.peerId.publicKey, 18:55:35 18:55:35 at Object.toHaveBeenCalledTimes (services/reqresp/reqresp.test.ts:476:36) 18:55:35 18:55:35 ● ReqResp › Batch requests › should send a batch request with a pinned peer 18:55:35 18:55:35 expect(jest.fn()).toHaveBeenCalledWith(...expected) 18:55:35 18:55:35 Expected: ObjectContaining {"publicKey": [8, 1, 18, 32, 227, 45, 6, 132, 202, 179, …]}, "/aztec/req/ping/0.1.0", {"data": [112, 105, 110, 103, 51], "type": "Buffer"} 18:55:35 Received 18:55:35 1 18:55:35 "12D3KooWQYARbKiqefz2eUK99g17iA6pQYG9nA4BKnPw6qaFHD93", 18:55:35 "/aztec/req/ping/0.1.0", 18:55:35 @@ -2,9 +2,9 @@ 18:55:35 "data": Array [ 18:55:35 112, 18:55:35 105, 18:55:35 110, 18:55:35 103, 18:55:35 - 51, 18:55:35 + 48, 18:55:35 ], 18:55:35 "type": "Buffer", 18:55:35 }, 18:55:35 2 18:55:35 "12D3KooWQ2wjA9rECts6p6vVc4HaoVnwSmwARuaHdWCUXTogyBLx", 18:55:35 "/aztec/req/ping/0.1.0", 18:55:35 @@ -2,9 +2,9 @@ 18:55:35 "data": Array [ 18:55:35 112, 18:55:35 105, 18:55:35 110, 18:55:35 103, 18:55:35 - 51, 18:55:35 + 49, 18:55:35 ], 18:55:35 "type": "Buffer", 18:55:35 }, 18:55:35 3 18:55:35 "12D3KooWR7AY6Z9gDK7A9xitBE8iCCiZitK31XMvtVDUepLYnM66", 18:55:35 "/aztec/req/ping/0.1.0", 18:55:35 @@ -2,9 +2,9 @@ 18:55:35 "data": Array [ 18:55:35 112, 18:55:35 105, 18:55:35 110, 18:55:35 103, 18:55:35 - 51, 18:55:35 + 48, 18:55:35 ], 18:55:35 "type": "Buffer", 18:55:35 }, 18:55:35 18:55:35 Number of calls: 14 18:55:35 18:55:35 516 | // Expect pinned peer to have received all requests 18:55:35 517 | for (let i = 0; i < batchSize; i++) { 18:55:35 > 518 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 18:55:35 | ^ 18:55:35 519 | expect.objectContaining({ publicKey: nodes[1].p2p.peerId.publicKey }), 18:55:35 520 | ReqRespSubProtocol.PING, 18:55:35 521 | Buffer.from(`ping${i}`), 18:55:35 18:55:35 at Object.toHaveBeenCalledWith (services/reqresp/reqresp.test.ts:518:38) 18:55:35 18:55:35 ● ReqResp › Batch requests › should stop after max retry attempts 18:55:35 18:55:35 expect(received).toEqual(expected) // deep equality 18:55:35 18:55:35 - Expected - 11 18:55:35 + Received + 3 18:55:35 18:55:35 @@ -74,10 +74,11 @@ 18:55:35 103, 18:55:35 ], 18:55:35 "type": "Buffer", 18:55:35 }, 18:55:35 }, 18:55:35 + undefined, 18:55:35 RequestableBuffer { 18:55:35 "buffer": Object { 18:55:35 "data": Array [ 18:55:35 112, 18:55:35 111, 18:55:35 @@ -85,10 +86,11 @@ 18:55:35 103, 18:55:35 ], 18:55:35 "type": "Buffer", 18:55:35 }, 18:55:35 }, 18:55:35 + undefined, 18:55:35 RequestableBuffer { 18:55:35 "buffer": Object { 18:55:35 "data": Array [ 18:55:35 112, 18:55:35 111, 18:55:35 @@ -96,17 +98,7 @@ 18:55:35 103, 18:55:35 ], 18:55:35 "type": "Buffer", 18:55:35 }, 18:55:35 }, 18:55:35 - RequestableBuffer { 18:55:35 - "buffer": Object { 18:55:35 - "data": Array [ 18:55:35 - 112, 18:55:35 - 111, 18:55:35 - 110, 18:55:35 - 103, 18:55:35 - ], 18:55:35 - "type": "Buffer", 18:55:35 - }, 18:55:35 - }, 18:55:35 + undefined, 18:55:35 ] 18:55:35 18:55:35 555 | 18:55:35 556 | const res = await nodes[0].req.sendBatchRequest(ReqRespSubProtocol.PING, requests, undefined); 18:55:35 > 557 | expect(res).toEqual(expectResponses); 18:55:35 | ^ 18:55:35 558 | 18:55:35 559 | // Check that we did detect hitting a rate limit 18:55:35 560 | expect(requesterLoggerSpy).toHaveBeenCalledWith( 18:55:35 18:55:35 at Object.toEqual (services/reqresp/reqresp.test.ts:557:19) 18:55:35 18:55:35 Test Suites: 1 failed, 1 total 18:55:35 Tests: 4 failed, 12 passed, 16 total 18:55:35 Snapshots: 0 total 18:55:35 Time: 31.549 s 18:55:35 Ran all test suites matching /p2p\/src\/services\/reqresp\/reqresp.test.ts/i. 18:55:35 Force exiting Jest: Have you considered using `--detectOpenHandles` to detect async operations that kept running after all tests finished?