ACI ·

Parent Log: http://ci.aztec-labs.com/203368854ee7e10a Command: 4bfd2d3585b9db66:ISOLATE=1:NAME=p2p/src/services/reqresp/reqresp.test.ts yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts Commit: https://github.com/AztecProtocol/aztec-packages/commit/267ac35e8ccb487bdc79bbe12d428f4c8b4d4d9f Env: REF_NAME=gh-readonly-queue/master/pr-15002-1338cef09c870378254c56748f972ae827c4d482 CURRENT_VERSION=0.87.8 CI_FULL=1 Date: Fri Jun 13 14:55:46 UTC 2025 System: ARCH=amd64 CPUS=128 MEM=493Gi HOSTNAME=pr-15002_amd64_x1-full Resources: CPU_LIST=0-127 CPUS=2 MEM=8g TIMEOUT=600s History: http://ci.aztec-labs.com/list/history_a57cc872d7daa810_master 14:55:46 +++ id -u 14:55:46 +++ id -g 14:55:46 ++ docker run -d --name p2p_src_services_reqresp_reqresp.test.ts --net=none --cpuset-cpus=0-127 --cpus=2 --memory=8g --user 1000:1000 -v/home/aztec-dev:/home/aztec-dev --mount type=tmpfs,target=/tmp,tmpfs-size=1g --workdir /home/aztec-dev/aztec-packages -e HOME -e VERBOSE -e GIT_CONFIG_GLOBAL=/home/aztec-dev/aztec-packages/build-images/src/home/.gitconfig -e FORCE_COLOR=true -e CPUS -e MEM aztecprotocol/build:3.0 /bin/bash -c 'timeout -v 600s bash -c '\''yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts'\''' 14:55:47 + cid=cd7d24f1e6d67781a9b02670a11683224dd96a92f9be7c77139c2aabcde6e97a 14:55:47 + set +x 14:55:55 [14:55:55.582] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:55:55 [14:55:55.586] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:55:55 [14:55:55.590] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:55:55 [14:55:55.591] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:55:56 [14:55:56.152] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:55:56 [14:55:56.153] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:55:56 [14:55:56.201] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:55:56 [14:55:56.202] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:55:56 [14:55:56.203] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:55:56 [14:55:56.204] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:55:57 [14:55:57.392] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:55:57 [14:55:57.393] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:55:57 [14:55:57.395] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:55:57 [14:55:57.397] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:55:57 [14:55:57.546] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:55:57 [14:55:57.547] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:55:57 [14:55:57.548] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:55:57 [14:55:57.548] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:55:57 [14:55:57.549] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:55:57 [14:55:57.549] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:55:57 [14:55:57.553] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:55:57 [14:55:57.554] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:55:58 [14:55:58.658] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAxHSh2bF2uAH51Ecdvf8j8RpHJ84TLSvN1bxAB6pneLb 14:55:58 [14:55:58.659] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 14:55:58 err: { 14:55:58 "type": "ReqRespStatusError", 14:55:58 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 14:55:58 "stack": 14:55:58 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 14:55:58 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 14:55:58 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 14:55:58 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 14:55:58 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 14:55:58 at processTicksAndRejections (node:internal/process/task_queues:105:5) 14:55:58 "status": 1 14:55:58 } 14:55:58 [14:55:58.668] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAxHSh2bF2uAH51Ecdvf8j8RpHJ84TLSvN1bxAB6pneLb 14:55:58 [14:55:58.669] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 14:55:58 err: { 14:55:58 "type": "ReqRespStatusError", 14:55:58 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 14:55:58 "stack": 14:55:58 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 14:55:58 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 14:55:58 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 14:55:58 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 14:55:58 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 14:55:58 at processTicksAndRejections (node:internal/process/task_queues:105:5) 14:55:58 "status": 1 14:55:58 } 14:55:58 [14:55:58.673] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAxHSh2bF2uAH51Ecdvf8j8RpHJ84TLSvN1bxAB6pneLb 14:55:58 [14:55:58.674] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 14:55:58 err: { 14:55:58 "type": "ReqRespStatusError", 14:55:58 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 14:55:58 "stack": 14:55:58 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 14:55:58 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 14:55:58 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 14:55:58 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 14:55:58 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 14:55:58 at processTicksAndRejections (node:internal/process/task_queues:105:5) 14:55:58 "status": 1 14:55:58 } 14:55:58 [14:55:58.677] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAxHSh2bF2uAH51Ecdvf8j8RpHJ84TLSvN1bxAB6pneLb 14:55:58 [14:55:58.678] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 14:55:58 err: { 14:55:58 "type": "ReqRespStatusError", 14:55:58 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 14:55:58 "stack": 14:55:58 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 14:55:58 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 14:55:58 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 14:55:58 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 14:55:58 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 14:55:58 at processTicksAndRejections (node:internal/process/task_queues:105:5) 14:55:58 "status": 1 14:55:58 } 14:55:58 [14:55:58.682] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAxHSh2bF2uAH51Ecdvf8j8RpHJ84TLSvN1bxAB6pneLb 14:55:58 [14:55:58.683] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 14:55:58 err: { 14:55:58 "type": "ReqRespStatusError", 14:55:58 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 14:55:58 "stack": 14:55:58 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 14:55:58 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 14:55:58 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 14:55:58 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 14:55:58 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 14:55:58 at processTicksAndRejections (node:internal/process/task_queues:105:5) 14:55:58 "status": 1 14:55:58 } 14:55:58 [14:55:58.695] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:55:58 [14:55:58.696] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:55:58 [14:55:58.700] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:55:58 [14:55:58.700] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:00 [14:56:00.158] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:00 [14:56:00.159] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:00 [14:56:00.162] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:00 [14:56:00.162] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:01 [14:56:01.228] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:01 [14:56:01.228] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:01 [14:56:01.232] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:01 [14:56:01.232] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:02 [14:56:02.300] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:02 [14:56:02.300] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:02 [14:56:02.304] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:02 [14:56:02.304] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:05 [14:56:05.359] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:05 [14:56:05.359] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:05 [14:56:05.363] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:05 [14:56:05.363] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:10 [14:56:10.567] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:10 [14:56:10.568] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:10 [14:56:10.569] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:10 [14:56:10.570] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:10 [14:56:10.571] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:10 [14:56:10.571] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:10 [14:56:10.575] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:10 [14:56:10.576] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:10 [14:56:10.594] WARN: foundation:memory_fifo Discarding item because queue is flushing 14:56:10 [14:56:10.599] ERROR: p2p:reqresp Unexpected error sending request to peer: Error: Could not enqueue function 14:56:10 at /home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:65:24 14:56:10 at new Promise (<anonymous>) 14:56:10 at SerialQueue.put (/home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:55:16) 14:56:10 at ConnectionSampler.put [as dialProtocol] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/connection-sampler/connection_sampler.ts:204:43) 14:56:10 at ReqResp.dialProtocol (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:449:45) 14:56:10 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 14:56:10 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 14:56:10 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 14:56:10 at TimeoutTask.requestFunction [as fn] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:200:26) {"peerId":"12D3KooWEPWN8mHsw77vCJkq5Ys8vV5PZ2nMKRgoNF9fYDKMV9mj","subProtocol":"/aztec/req/tx/0.1.0"} 14:56:11 [14:56:11.692] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:11 [14:56:11.693] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:11 [14:56:11.699] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:11 [14:56:11.700] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:12 [14:56:12.758] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:12 [14:56:12.759] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:12 [14:56:12.761] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:12 [14:56:12.762] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:13 [14:56:13.802] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:13 [14:56:13.803] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:13 [14:56:13.807] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:13 [14:56:13.807] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:15 [14:56:15.069] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:15 [14:56:15.069] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:15 [14:56:15.073] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:15 [14:56:15.074] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:16 [14:56:16.249] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWAwHf9BBvG4E8d6wQT13bTAMoRRQR3pSjq9KM4pHj4dDg 14:56:16 [14:56:16.250] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 14:56:16 err: { 14:56:16 "type": "ReqRespStatusError", 14:56:16 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 14:56:16 "stack": 14:56:16 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 14:56:16 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 14:56:16 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 14:56:16 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 14:56:16 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 14:56:16 at processTicksAndRejections (node:internal/process/task_queues:105:5) 14:56:16 "status": 1 14:56:16 } 14:56:16 [14:56:16.273] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:16 [14:56:16.273] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:16 [14:56:16.274] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:16 [14:56:16.274] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:16 [14:56:16.277] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:16 [14:56:16.277] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:17 [14:56:17.538] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:17 [14:56:17.538] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:17 [14:56:17.539] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:17 [14:56:17.540] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:17 [14:56:17.541] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:17 [14:56:17.541] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:17 [14:56:17.545] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:17 [14:56:17.546] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:18 [14:56:18.668] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWCQvWJZrpFfEJTncSygActVCuaageRpttahY5p9ZKUUVQ 14:56:18 [14:56:18.669] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 14:56:18 err: { 14:56:18 "type": "ReqRespStatusError", 14:56:18 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 14:56:18 "stack": 14:56:18 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 14:56:18 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 14:56:18 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 14:56:18 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 14:56:18 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 14:56:18 at processTicksAndRejections (node:internal/process/task_queues:105:5) 14:56:18 "status": 1 14:56:18 } 14:56:18 [14:56:18.672] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWCQvWJZrpFfEJTncSygActVCuaageRpttahY5p9ZKUUVQ 14:56:18 [14:56:18.673] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 14:56:18 err: { 14:56:18 "type": "ReqRespStatusError", 14:56:18 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 14:56:18 "stack": 14:56:18 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 14:56:18 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 14:56:18 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 14:56:18 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 14:56:18 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 14:56:18 at processTicksAndRejections (node:internal/process/task_queues:105:5) 14:56:18 "status": 1 14:56:18 } 14:56:18 [14:56:18.687] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWCQvWJZrpFfEJTncSygActVCuaageRpttahY5p9ZKUUVQ 14:56:18 [14:56:18.687] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 14:56:18 err: { 14:56:18 "type": "ReqRespStatusError", 14:56:18 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 14:56:18 "stack": 14:56:18 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 14:56:18 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 14:56:18 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 14:56:18 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 14:56:18 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 14:56:18 at processTicksAndRejections (node:internal/process/task_queues:105:5) 14:56:18 "status": 1 14:56:18 } 14:56:18 [14:56:18.700] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:18 [14:56:18.700] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:18 [14:56:18.701] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:18 [14:56:18.702] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:18 [14:56:18.705] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 14:56:18 [14:56:18.705] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 14:56:18 FAIL src/services/reqresp/reqresp.test.ts (30.619 s) 14:56:18 ReqResp 14:56:18 ✓ should perform a ping request (1242 ms) 14:56:18 ✓ should handle gracefully if a peer connected peer is offline (611 ms) 14:56:18 ✓ should request from a later peer if other peers are offline (1375 ms) 14:56:18 ✕ should hit a rate limit if too many requests are made in quick succession (1121 ms) 14:56:18 Tx req protocol 14:56:18 ✓ can request a Tx from TxHash (1460 ms) 14:56:18 ✓ handles returning empty buffers (1071 ms) 14:56:18 ✓ does not crash if tx hash returns undefined (1070 ms) 14:56:18 ✓ should hit individual timeout if nothing is returned over the stream (3063 ms) 14:56:18 ✓ should hit collective timeout if nothing is returned over the stream from multiple peers (5229 ms) 14:56:18 ✓ should penalize peer if transaction validation fails (1104 ms) 14:56:18 Goodbye protocol 14:56:18 ✓ should send a goodbye message to a peer (1058 ms) 14:56:18 ✓ should not close stream when handling a goodbye message received from peer (1051 ms) 14:56:18 Block protocol 14:56:18 ✓ should handle block requests (1263 ms) 14:56:18 Batch requests 14:56:18 ✕ should send a batch request between many peers (1207 ms) 14:56:18 ✕ should send a batch request with a pinned peer (1277 ms) 14:56:18 ✕ should stop after max retry attempts (1156 ms) 14:56:18 14:56:18 ● ReqResp › should hit a rate limit if too many requests are made in quick succession 14:56:18 14:56:18 expect(received).toBeDefined() 14:56:18 14:56:18 Received: undefined 14:56:18 14:56:18 145 | // Check that one of the responses gets a rate limit response 14:56:18 146 | const rateLimitResponse = responses.find(response => response?.status === ReqRespStatus.RATE_LIMIT_EXCEEDED); 14:56:18 > 147 | expect(rateLimitResponse).toBeDefined(); 14:56:18 | ^ 14:56:18 148 | 14:56:18 149 | // Make sure the error message is logged 14:56:18 150 | const errorMessage = `Rate limit exceeded DeniedPeer for ${ 14:56:18 14:56:18 at Object.toBeDefined (services/reqresp/reqresp.test.ts:147:31) 14:56:18 14:56:18 ● ReqResp › Batch requests › should send a batch request between many peers 14:56:18 14:56:18 expect(jest.fn()).toHaveBeenCalledTimes(expected) 14:56:18 14:56:18 Expected number of calls: 9 14:56:18 Received number of calls: 11 14:56:18 14:56:18 474 | 14:56:18 475 | // Expect one request to have been sent to each peer 14:56:18 > 476 | expect(sendRequestToPeerSpy).toHaveBeenCalledTimes(batchSize); 14:56:18 | ^ 14:56:18 477 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 14:56:18 478 | expect.objectContaining({ 14:56:18 479 | publicKey: nodes[1].p2p.peerId.publicKey, 14:56:18 14:56:18 at Object.toHaveBeenCalledTimes (services/reqresp/reqresp.test.ts:476:36) 14:56:18 14:56:18 ● ReqResp › Batch requests › should send a batch request with a pinned peer 14:56:18 14:56:18 expect(jest.fn()).toHaveBeenCalledWith(...expected) 14:56:18 14:56:18 Expected: ObjectContaining {"publicKey": [8, 1, 18, 32, 33, 55, 103, 39, 219, 213, …]}, "/aztec/req/ping/0.1.0", {"data": [112, 105, 110, 103, 51], "type": "Buffer"} 14:56:18 Received 14:56:18 1 14:56:18 "12D3KooWAXxcWqwwAHKJPPfyZmFguX6TQesvEKbvdhdRkq8UzMQi", 14:56:18 "/aztec/req/ping/0.1.0", 14:56:18 @@ -2,9 +2,9 @@ 14:56:18 "data": Array [ 14:56:18 112, 14:56:18 105, 14:56:18 110, 14:56:18 103, 14:56:18 - 51, 14:56:18 + 48, 14:56:18 ], 14:56:18 "type": "Buffer", 14:56:18 }, 14:56:18 2 14:56:18 "12D3KooWQy5GXUWHRT4364XR6m9DEvy7gS4zKgozTzP7dTTzQWQk", 14:56:18 "/aztec/req/ping/0.1.0", 14:56:18 @@ -2,9 +2,9 @@ 14:56:18 "data": Array [ 14:56:18 112, 14:56:18 105, 14:56:18 110, 14:56:18 103, 14:56:18 - 51, 14:56:18 + 49, 14:56:18 ], 14:56:18 "type": "Buffer", 14:56:18 }, 14:56:18 3 14:56:18 "12D3KooWC42czwRscYy8eTC8PXSeajoycZh5Z3QwjtQssZrxhgSV", 14:56:18 "/aztec/req/ping/0.1.0", 14:56:18 @@ -2,9 +2,9 @@ 14:56:18 "data": Array [ 14:56:18 112, 14:56:18 105, 14:56:18 110, 14:56:18 103, 14:56:18 - 51, 14:56:18 + 48, 14:56:18 ], 14:56:18 "type": "Buffer", 14:56:18 }, 14:56:18 14:56:18 Number of calls: 14 14:56:18 14:56:18 516 | // Expect pinned peer to have received all requests 14:56:18 517 | for (let i = 0; i < batchSize; i++) { 14:56:18 > 518 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 14:56:18 | ^ 14:56:18 519 | expect.objectContaining({ publicKey: nodes[1].p2p.peerId.publicKey }), 14:56:18 520 | ReqRespSubProtocol.PING, 14:56:18 521 | Buffer.from(`ping${i}`), 14:56:18 14:56:18 at Object.toHaveBeenCalledWith (services/reqresp/reqresp.test.ts:518:38) 14:56:18 14:56:18 ● ReqResp › Batch requests › should stop after max retry attempts 14:56:18 14:56:18 expect(received).toEqual(expected) // deep equality 14:56:18 14:56:18 - Expected - 11 14:56:18 + Received + 3 14:56:18 14:56:18 @@ -74,10 +74,11 @@ 14:56:18 103, 14:56:18 ], 14:56:18 "type": "Buffer", 14:56:18 }, 14:56:18 }, 14:56:18 + undefined, 14:56:18 RequestableBuffer { 14:56:18 "buffer": Object { 14:56:18 "data": Array [ 14:56:18 112, 14:56:18 111, 14:56:18 @@ -85,10 +86,11 @@ 14:56:18 103, 14:56:18 ], 14:56:18 "type": "Buffer", 14:56:18 }, 14:56:18 }, 14:56:18 + undefined, 14:56:18 RequestableBuffer { 14:56:18 "buffer": Object { 14:56:18 "data": Array [ 14:56:18 112, 14:56:18 111, 14:56:18 @@ -96,17 +98,7 @@ 14:56:18 103, 14:56:18 ], 14:56:18 "type": "Buffer", 14:56:18 }, 14:56:18 }, 14:56:18 - RequestableBuffer { 14:56:18 - "buffer": Object { 14:56:18 - "data": Array [ 14:56:18 - 112, 14:56:18 - 111, 14:56:18 - 110, 14:56:18 - 103, 14:56:18 - ], 14:56:18 - "type": "Buffer", 14:56:18 - }, 14:56:18 - }, 14:56:18 + undefined, 14:56:18 ] 14:56:18 14:56:18 555 | 14:56:18 556 | const res = await nodes[0].req.sendBatchRequest(ReqRespSubProtocol.PING, requests, undefined); 14:56:18 > 557 | expect(res).toEqual(expectResponses); 14:56:18 | ^ 14:56:18 558 | 14:56:18 559 | // Check that we did detect hitting a rate limit 14:56:18 560 | expect(requesterLoggerSpy).toHaveBeenCalledWith( 14:56:18 14:56:18 at Object.toEqual (services/reqresp/reqresp.test.ts:557:19) 14:56:18 14:56:18 Test Suites: 1 failed, 1 total 14:56:18 Tests: 4 failed, 12 passed, 16 total 14:56:18 Snapshots: 0 total 14:56:18 Time: 30.708 s 14:56:18 Ran all test suites matching /p2p\/src\/services\/reqresp\/reqresp.test.ts/i. 14:56:18 Force exiting Jest: Have you considered using `--detectOpenHandles` to detect async operations that kept running after all tests finished?