ACI ·

Parent Log: http://ci.aztec-labs.com/f823d0a4ebbfe5c2 Command: 53e9b8d3e8ba3411:ISOLATE=1:NAME=p2p/src/services/reqresp/reqresp.test.ts yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts Commit: https://github.com/AztecProtocol/aztec-packages/commit/2707ec99de02004f3e30e7af0810e0d741ab9794 Env: REF_NAME=gh-readonly-queue/master/pr-15049-f4ac84d7ac4041d505399edd8819840bb14d6c9d CURRENT_VERSION=0.87.8 CI_FULL=1 Date: Sat Jun 14 10:49:17 UTC 2025 System: ARCH=amd64 CPUS=128 MEM=493Gi HOSTNAME=pr-15049_amd64_x2-full Resources: CPU_LIST=0-127 CPUS=2 MEM=8g TIMEOUT=600s History: http://ci.aztec-labs.com/list/history_a57cc872d7daa810_master 10:49:17 +++ id -u 10:49:17 +++ id -g 10:49:17 ++ docker run -d --name p2p_src_services_reqresp_reqresp.test.ts --net=none --cpuset-cpus=0-127 --cpus=2 --memory=8g --user 1000:1000 -v/home/aztec-dev:/home/aztec-dev --mount type=tmpfs,target=/tmp,tmpfs-size=1g --workdir /home/aztec-dev/aztec-packages -e HOME -e VERBOSE -e GIT_CONFIG_GLOBAL=/home/aztec-dev/aztec-packages/build-images/src/home/.gitconfig -e FORCE_COLOR=true -e CPUS -e MEM aztecprotocol/build:3.0 /bin/bash -c 'timeout -v 600s bash -c '\''yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts'\''' 10:49:17 + cid=b73b80a3300bb713fde4b9e5260039d16d53e51245763cff2bed8f929635288d 10:49:17 + set +x 10:49:26 [10:49:26.585] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:26 [10:49:26.590] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:26 [10:49:26.596] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:26 [10:49:26.597] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:27 [10:49:27.147] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:27 [10:49:27.148] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:27 [10:49:27.193] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:27 [10:49:27.193] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:27 [10:49:27.194] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:27 [10:49:27.194] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:28 [10:49:28.389] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:28 [10:49:28.390] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:28 [10:49:28.392] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:28 [10:49:28.393] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:28 [10:49:28.421] ERROR: p2p:reqresp Unexpected error sending request to peer: CodeError: muxer closed locally 10:49:28 at YamuxMuxer.newStream (/home/aztec-dev/aztec-packages/yarn-project/node_modules/@chainsafe/libp2p-yamux/src/muxer.ts:187:13) 10:49:28 at ConnectionImpl.newStream [as _newStream] (/home/aztec-dev/aztec-packages/yarn-project/node_modules/libp2p/src/upgrader.ts:441:41) 10:49:28 at ConnectionImpl.newStream (/home/aztec-dev/aztec-packages/yarn-project/node_modules/libp2p/src/connection/index.ts:134:31) 10:49:28 at Libp2pNode.dialProtocol (/home/aztec-dev/aztec-packages/yarn-project/node_modules/libp2p/src/libp2p.ts:291:23) 10:49:28 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:49:28 at /home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:58:33 10:49:28 at FifoMemoryQueue.process (/home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/base_memory_queue.js:110:17) { 10:49:28 code: 'ERR_MUXER_LOCAL_CLOSED', 10:49:28 props: {} 10:49:28 } {"peerId":"12D3KooWRYrYj9DLcEAzTfrMEjqpMkkYKuTY9J9sdPmXk7LYdngu","subProtocol":"/aztec/req/ping/0.1.0"} 10:49:28 [10:49:28.510] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:28 [10:49:28.511] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:28 [10:49:28.512] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:28 [10:49:28.512] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:28 [10:49:28.513] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:28 [10:49:28.514] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:28 [10:49:28.521] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:28 [10:49:28.521] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:29 [10:49:29.634] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAGWX1Q7VYQVgH44W2daJU1FWgqNjMLDMoE8GsoMJaBaB 10:49:29 [10:49:29.634] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:49:29 err: { 10:49:29 "type": "ReqRespStatusError", 10:49:29 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:49:29 "stack": 10:49:29 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:49:29 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:49:29 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:49:29 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:49:29 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:49:29 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:49:29 "status": 1 10:49:29 } 10:49:29 [10:49:29.647] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAGWX1Q7VYQVgH44W2daJU1FWgqNjMLDMoE8GsoMJaBaB 10:49:29 [10:49:29.647] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:49:29 err: { 10:49:29 "type": "ReqRespStatusError", 10:49:29 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:49:29 "stack": 10:49:29 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:49:29 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:49:29 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:49:29 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:49:29 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:49:29 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:49:29 "status": 1 10:49:29 } 10:49:29 [10:49:29.656] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAGWX1Q7VYQVgH44W2daJU1FWgqNjMLDMoE8GsoMJaBaB 10:49:29 [10:49:29.656] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:49:29 err: { 10:49:29 "type": "ReqRespStatusError", 10:49:29 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:49:29 "stack": 10:49:29 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:49:29 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:49:29 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:49:29 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:49:29 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:49:29 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:49:29 "status": 1 10:49:29 } 10:49:29 [10:49:29.661] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAGWX1Q7VYQVgH44W2daJU1FWgqNjMLDMoE8GsoMJaBaB 10:49:29 [10:49:29.661] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:49:29 err: { 10:49:29 "type": "ReqRespStatusError", 10:49:29 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:49:29 "stack": 10:49:29 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:49:29 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:49:29 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:49:29 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:49:29 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:49:29 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:49:29 "status": 1 10:49:29 } 10:49:29 [10:49:29.666] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAGWX1Q7VYQVgH44W2daJU1FWgqNjMLDMoE8GsoMJaBaB 10:49:29 [10:49:29.666] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:49:29 err: { 10:49:29 "type": "ReqRespStatusError", 10:49:29 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:49:29 "stack": 10:49:29 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:49:29 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:49:29 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:49:29 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:49:29 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:49:29 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:49:29 "status": 1 10:49:29 } 10:49:29 [10:49:29.674] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:29 [10:49:29.674] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:29 [10:49:29.678] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:29 [10:49:29.679] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:31 [10:49:31.107] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:31 [10:49:31.108] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:31 [10:49:31.112] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:31 [10:49:31.113] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:32 [10:49:32.189] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:32 [10:49:32.190] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:32 [10:49:32.194] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:32 [10:49:32.194] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:33 [10:49:33.281] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:33 [10:49:33.282] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:33 [10:49:33.286] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:33 [10:49:33.286] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:36 [10:49:36.342] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:36 [10:49:36.343] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:36 [10:49:36.346] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:36 [10:49:36.346] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:41 [10:49:41.630] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:41 [10:49:41.630] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:41 [10:49:41.632] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:41 [10:49:41.632] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:41 [10:49:41.633] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:41 [10:49:41.633] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:41 [10:49:41.637] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:41 [10:49:41.638] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:41 [10:49:41.652] WARN: foundation:memory_fifo Discarding item because queue is flushing 10:49:41 [10:49:41.654] ERROR: p2p:reqresp Unexpected error sending request to peer: Error: Could not enqueue function 10:49:41 at /home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:65:24 10:49:41 at new Promise (<anonymous>) 10:49:41 at SerialQueue.put (/home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:55:16) 10:49:41 at ConnectionSampler.put [as dialProtocol] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/connection-sampler/connection_sampler.ts:204:43) 10:49:41 at ReqResp.dialProtocol (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:449:45) 10:49:41 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:49:41 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:49:41 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:49:41 at TimeoutTask.requestFunction [as fn] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:200:26) {"peerId":"12D3KooWCGzsEiSmBtVRcbphcDnE2Fg7nfRfMtaBwU2LnRvJ3UQy","subProtocol":"/aztec/req/tx/0.1.0"} 10:49:42 [10:49:42.740] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:42 [10:49:42.741] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:42 [10:49:42.744] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:42 [10:49:42.744] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:43 [10:49:43.789] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:43 [10:49:43.790] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:43 [10:49:43.792] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:43 [10:49:43.793] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:44 [10:49:44.841] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:44 [10:49:44.841] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:44 [10:49:44.844] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:44 [10:49:44.845] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:46 [10:49:46.160] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:46 [10:49:46.160] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:46 [10:49:46.165] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:46 [10:49:46.165] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:47 [10:49:47.281] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAGFW9W6pgFMXxoBJ9Fu5jcNi4TS51oEyPnTSA6svP6zR 10:49:47 [10:49:47.281] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:49:47 err: { 10:49:47 "type": "ReqRespStatusError", 10:49:47 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:49:47 "stack": 10:49:47 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:49:47 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:49:47 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:49:47 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:49:47 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:49:47 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:49:47 "status": 1 10:49:47 } 10:49:47 [10:49:47.299] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:47 [10:49:47.300] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:47 [10:49:47.301] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:47 [10:49:47.301] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:47 [10:49:47.304] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:47 [10:49:47.304] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:48 [10:49:48.684] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:48 [10:49:48.685] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:48 [10:49:48.686] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:48 [10:49:48.686] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:48 [10:49:48.687] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:48 [10:49:48.687] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:48 [10:49:48.691] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:48 [10:49:48.692] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:50 [10:49:50.171] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWCHN7nwa9WJ95iaB5pXF5gbrV65MWNyUqGgSQinTFPabS 10:49:50 [10:49:50.171] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:49:50 err: { 10:49:50 "type": "ReqRespStatusError", 10:49:50 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:49:50 "stack": 10:49:50 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:49:50 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:49:50 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:49:50 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:49:50 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:49:50 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:49:50 "status": 1 10:49:50 } 10:49:50 [10:49:50.175] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWCHN7nwa9WJ95iaB5pXF5gbrV65MWNyUqGgSQinTFPabS 10:49:50 [10:49:50.175] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:49:50 err: { 10:49:50 "type": "ReqRespStatusError", 10:49:50 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:49:50 "stack": 10:49:50 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:49:50 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:49:50 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:49:50 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:49:50 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:49:50 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:49:50 "status": 1 10:49:50 } 10:49:50 [10:49:50.192] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWCHN7nwa9WJ95iaB5pXF5gbrV65MWNyUqGgSQinTFPabS 10:49:50 [10:49:50.193] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 10:49:50 err: { 10:49:50 "type": "ReqRespStatusError", 10:49:50 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 10:49:50 "stack": 10:49:50 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 10:49:50 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 10:49:50 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 10:49:50 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 10:49:50 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 10:49:50 at processTicksAndRejections (node:internal/process/task_queues:105:5) 10:49:50 "status": 1 10:49:50 } 10:49:50 [10:49:50.201] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:50 [10:49:50.202] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:50 [10:49:50.202] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:50 [10:49:50.203] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:50 [10:49:50.205] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 10:49:50 [10:49:50.206] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 10:49:50 FAIL src/services/reqresp/reqresp.test.ts (31.47 s) 10:49:50 ReqResp 10:49:50 ✓ should perform a ping request (1247 ms) 10:49:50 ✓ should handle gracefully if a peer connected peer is offline (589 ms) 10:49:50 ✓ should request from a later peer if other peers are offline (1372 ms) 10:49:50 ✕ should hit a rate limit if too many requests are made in quick succession (1117 ms) 10:49:50 Tx req protocol 10:49:50 ✓ can request a Tx from TxHash (1432 ms) 10:49:50 ✓ handles returning empty buffers (1081 ms) 10:49:50 ✓ does not crash if tx hash returns undefined (1092 ms) 10:49:50 ✓ should hit individual timeout if nothing is returned over the stream (3063 ms) 10:49:50 ✓ should hit collective timeout if nothing is returned over the stream from multiple peers (5299 ms) 10:49:50 ✓ should penalize peer if transaction validation fails (1091 ms) 10:49:50 Goodbye protocol 10:49:50 ✓ should send a goodbye message to a peer (1047 ms) 10:49:50 ✓ should not close stream when handling a goodbye message received from peer (1055 ms) 10:49:50 Block protocol 10:49:50 ✓ should handle block requests (1319 ms) 10:49:50 Batch requests 10:49:50 ✕ should send a batch request between many peers (1141 ms) 10:49:50 ✕ should send a batch request with a pinned peer (1401 ms) 10:49:50 ✕ should stop after max retry attempts (1505 ms) 10:49:50 10:49:50 ● ReqResp › should hit a rate limit if too many requests are made in quick succession 10:49:50 10:49:50 expect(received).toBeDefined() 10:49:50 10:49:50 Received: undefined 10:49:50 10:49:50 145 | // Check that one of the responses gets a rate limit response 10:49:50 146 | const rateLimitResponse = responses.find(response => response?.status === ReqRespStatus.RATE_LIMIT_EXCEEDED); 10:49:50 > 147 | expect(rateLimitResponse).toBeDefined(); 10:49:50 | ^ 10:49:50 148 | 10:49:50 149 | // Make sure the error message is logged 10:49:50 150 | const errorMessage = `Rate limit exceeded DeniedPeer for ${ 10:49:50 10:49:50 at Object.toBeDefined (services/reqresp/reqresp.test.ts:147:31) 10:49:50 10:49:50 ● ReqResp › Batch requests › should send a batch request between many peers 10:49:50 10:49:50 expect(jest.fn()).toHaveBeenCalledTimes(expected) 10:49:50 10:49:50 Expected number of calls: 9 10:49:50 Received number of calls: 11 10:49:50 10:49:50 474 | 10:49:50 475 | // Expect one request to have been sent to each peer 10:49:50 > 476 | expect(sendRequestToPeerSpy).toHaveBeenCalledTimes(batchSize); 10:49:50 | ^ 10:49:50 477 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 10:49:50 478 | expect.objectContaining({ 10:49:50 479 | publicKey: nodes[1].p2p.peerId.publicKey, 10:49:50 10:49:50 at Object.toHaveBeenCalledTimes (services/reqresp/reqresp.test.ts:476:36) 10:49:50 10:49:50 ● ReqResp › Batch requests › should send a batch request with a pinned peer 10:49:50 10:49:50 expect(jest.fn()).toHaveBeenCalledWith(...expected) 10:49:50 10:49:50 Expected: ObjectContaining {"publicKey": [8, 1, 18, 32, 15, 184, 150, 13, 140, 108, …]}, "/aztec/req/ping/0.1.0", {"data": [112, 105, 110, 103, 51], "type": "Buffer"} 10:49:50 Received 10:49:50 1 10:49:50 "12D3KooWNEdGKdb4keH3Sr9SP1v5z9kZ74fcmWiu7CuVxZtcjxQf", 10:49:50 "/aztec/req/ping/0.1.0", 10:49:50 @@ -2,9 +2,9 @@ 10:49:50 "data": Array [ 10:49:50 112, 10:49:50 105, 10:49:50 110, 10:49:50 103, 10:49:50 - 51, 10:49:50 + 48, 10:49:50 ], 10:49:50 "type": "Buffer", 10:49:50 }, 10:49:50 2 10:49:50 "12D3KooWDYCQqhTvawusQYY31EJH92s8j8hb9U324dXKhPDzs9Hv", 10:49:50 "/aztec/req/ping/0.1.0", 10:49:50 @@ -2,9 +2,9 @@ 10:49:50 "data": Array [ 10:49:50 112, 10:49:50 105, 10:49:50 110, 10:49:50 103, 10:49:50 - 51, 10:49:50 + 49, 10:49:50 ], 10:49:50 "type": "Buffer", 10:49:50 }, 10:49:50 3 10:49:50 "12D3KooWAsjXsw5NjhbZvUjDx7M6v4R6zP5nnNXKvRdYwGmV7SvZ", 10:49:50 "/aztec/req/ping/0.1.0", 10:49:50 @@ -2,9 +2,9 @@ 10:49:50 "data": Array [ 10:49:50 112, 10:49:50 105, 10:49:50 110, 10:49:50 103, 10:49:50 - 51, 10:49:50 + 48, 10:49:50 ], 10:49:50 "type": "Buffer", 10:49:50 }, 10:49:50 10:49:50 Number of calls: 13 10:49:50 10:49:50 516 | // Expect pinned peer to have received all requests 10:49:50 517 | for (let i = 0; i < batchSize; i++) { 10:49:50 > 518 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 10:49:50 | ^ 10:49:50 519 | expect.objectContaining({ publicKey: nodes[1].p2p.peerId.publicKey }), 10:49:50 520 | ReqRespSubProtocol.PING, 10:49:50 521 | Buffer.from(`ping${i}`), 10:49:50 10:49:50 at Object.toHaveBeenCalledWith (services/reqresp/reqresp.test.ts:518:38) 10:49:50 10:49:50 ● ReqResp › Batch requests › should stop after max retry attempts 10:49:50 10:49:50 expect(received).toEqual(expected) // deep equality 10:49:50 10:49:50 - Expected - 11 10:49:50 + Received + 3 10:49:50 10:49:50 @@ -74,10 +74,11 @@ 10:49:50 103, 10:49:50 ], 10:49:50 "type": "Buffer", 10:49:50 }, 10:49:50 }, 10:49:50 + undefined, 10:49:50 RequestableBuffer { 10:49:50 "buffer": Object { 10:49:50 "data": Array [ 10:49:50 112, 10:49:50 111, 10:49:50 @@ -85,10 +86,11 @@ 10:49:50 103, 10:49:50 ], 10:49:50 "type": "Buffer", 10:49:50 }, 10:49:50 }, 10:49:50 + undefined, 10:49:50 RequestableBuffer { 10:49:50 "buffer": Object { 10:49:50 "data": Array [ 10:49:50 112, 10:49:50 111, 10:49:50 @@ -96,17 +98,7 @@ 10:49:50 103, 10:49:50 ], 10:49:50 "type": "Buffer", 10:49:50 }, 10:49:50 }, 10:49:50 - RequestableBuffer { 10:49:50 - "buffer": Object { 10:49:50 - "data": Array [ 10:49:50 - 112, 10:49:50 - 111, 10:49:50 - 110, 10:49:50 - 103, 10:49:50 - ], 10:49:50 - "type": "Buffer", 10:49:50 - }, 10:49:50 - }, 10:49:50 + undefined, 10:49:50 ] 10:49:50 10:49:50 555 | 10:49:50 556 | const res = await nodes[0].req.sendBatchRequest(ReqRespSubProtocol.PING, requests, undefined); 10:49:50 > 557 | expect(res).toEqual(expectResponses); 10:49:50 | ^ 10:49:50 558 | 10:49:50 559 | // Check that we did detect hitting a rate limit 10:49:50 560 | expect(requesterLoggerSpy).toHaveBeenCalledWith( 10:49:50 10:49:50 at Object.toEqual (services/reqresp/reqresp.test.ts:557:19) 10:49:50 10:49:50 Test Suites: 1 failed, 1 total 10:49:50 Tests: 4 failed, 12 passed, 16 total 10:49:50 Snapshots: 0 total 10:49:50 Time: 31.563 s 10:49:50 Ran all test suites matching /p2p\/src\/services\/reqresp\/reqresp.test.ts/i. 10:49:50 Force exiting Jest: Have you considered using `--detectOpenHandles` to detect async operations that kept running after all tests finished?