ACI ·

Parent Log: http://ci.aztec-labs.com/dcbd399c50cfc3e3 Command: 59d7caa3bc5b3609:ISOLATE=1:NAME=p2p/src/services/reqresp/reqresp.test.ts yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts Commit: https://github.com/AztecProtocol/aztec-packages/commit/5f78d61bb53ab2c2ccd59528d4439a88b2325b96 Env: REF_NAME=gh-readonly-queue/master/pr-14715-3a776996a430e63770f252088c2fdd32ae17c1f7 CURRENT_VERSION=0.87.8 CI_FULL=1 Date: Mon Jun 9 11:25:49 UTC 2025 System: ARCH=amd64 CPUS=128 MEM=493Gi HOSTNAME=pr-14715_amd64_x2-full Resources: CPU_LIST=0-127 CPUS=2 MEM=8g TIMEOUT=600s History: http://ci.aztec-labs.com/list/history_a57cc872d7daa810_master 11:25:50 +++ id -u 11:25:50 +++ id -g 11:25:50 ++ docker run -d --name p2p_src_services_reqresp_reqresp.test.ts --net=none --cpuset-cpus=0-127 --cpus=2 --memory=8g --user 1000:1000 -v/home/aztec-dev:/home/aztec-dev --mount type=tmpfs,target=/tmp,tmpfs-size=1g --workdir /home/aztec-dev/aztec-packages -e HOME -e VERBOSE -e GIT_CONFIG_GLOBAL=/home/aztec-dev/aztec-packages/build-images/src/home/.gitconfig -e FORCE_COLOR=true -e CPUS -e MEM aztecprotocol/build:3.0 /bin/bash -c 'timeout -v 600s bash -c '\''yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts'\''' 11:25:50 + cid=525e2c8dda7889d4f014569f24c98638e8ad1ebf0ad1aa0e8a5ccf794b233a59 11:25:50 + set +x 11:25:58 [11:25:58.594] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:25:58 [11:25:58.599] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:25:58 [11:25:58.604] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:25:58 [11:25:58.605] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:25:59 [11:25:59.154] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:25:59 [11:25:59.155] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:25:59 [11:25:59.227] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:25:59 [11:25:59.228] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:25:59 [11:25:59.229] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:25:59 [11:25:59.231] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:00 [11:26:00.440] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:00 [11:26:00.441] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:00 [11:26:00.443] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:00 [11:26:00.446] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:00 [11:26:00.478] ERROR: p2p:reqresp Unexpected error sending request to peer: CodeError: muxer closed locally 11:26:00 at YamuxMuxer.newStream (/home/aztec-dev/aztec-packages/yarn-project/node_modules/@chainsafe/libp2p-yamux/src/muxer.ts:187:13) 11:26:00 at ConnectionImpl.newStream [as _newStream] (/home/aztec-dev/aztec-packages/yarn-project/node_modules/libp2p/src/upgrader.ts:441:41) 11:26:00 at ConnectionImpl.newStream (/home/aztec-dev/aztec-packages/yarn-project/node_modules/libp2p/src/connection/index.ts:134:31) 11:26:00 at Libp2pNode.dialProtocol (/home/aztec-dev/aztec-packages/yarn-project/node_modules/libp2p/src/libp2p.ts:291:23) 11:26:00 at processTicksAndRejections (node:internal/process/task_queues:105:5) 11:26:00 at /home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:58:33 11:26:00 at FifoMemoryQueue.process (/home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/base_memory_queue.js:110:17) { 11:26:00 code: 'ERR_MUXER_LOCAL_CLOSED', 11:26:00 props: {} 11:26:00 } {"peerId":"12D3KooWDjouyaAArcC7ZT6yqQoTbqeRnnLsidbMP1VTGVD4vX1a","subProtocol":"/aztec/req/ping/0.1.0"} 11:26:00 [11:26:00.592] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:00 [11:26:00.592] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:00 [11:26:00.593] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:00 [11:26:00.594] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:00 [11:26:00.595] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:00 [11:26:00.595] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:00 [11:26:00.600] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:00 [11:26:00.600] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:01 [11:26:01.703] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWQVg1vrbACtdbrWR37gL2NDLQdGfFkTsfd5VPWcFKPQ6v 11:26:01 [11:26:01.703] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 11:26:01 err: { 11:26:01 "type": "ReqRespStatusError", 11:26:01 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 11:26:01 "stack": 11:26:01 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 11:26:01 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 11:26:01 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 11:26:01 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 11:26:01 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 11:26:01 at processTicksAndRejections (node:internal/process/task_queues:105:5) 11:26:01 "status": 1 11:26:01 } 11:26:01 [11:26:01.711] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWQVg1vrbACtdbrWR37gL2NDLQdGfFkTsfd5VPWcFKPQ6v 11:26:01 [11:26:01.712] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 11:26:01 err: { 11:26:01 "type": "ReqRespStatusError", 11:26:01 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 11:26:01 "stack": 11:26:01 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 11:26:01 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 11:26:01 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 11:26:01 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 11:26:01 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 11:26:01 at processTicksAndRejections (node:internal/process/task_queues:105:5) 11:26:01 "status": 1 11:26:01 } 11:26:01 [11:26:01.723] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWQVg1vrbACtdbrWR37gL2NDLQdGfFkTsfd5VPWcFKPQ6v 11:26:01 [11:26:01.723] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 11:26:01 err: { 11:26:01 "type": "ReqRespStatusError", 11:26:01 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 11:26:01 "stack": 11:26:01 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 11:26:01 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 11:26:01 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 11:26:01 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 11:26:01 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 11:26:01 at processTicksAndRejections (node:internal/process/task_queues:105:5) 11:26:01 "status": 1 11:26:01 } 11:26:01 [11:26:01.730] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWQVg1vrbACtdbrWR37gL2NDLQdGfFkTsfd5VPWcFKPQ6v 11:26:01 [11:26:01.730] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 11:26:01 err: { 11:26:01 "type": "ReqRespStatusError", 11:26:01 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 11:26:01 "stack": 11:26:01 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 11:26:01 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 11:26:01 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 11:26:01 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 11:26:01 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 11:26:01 at processTicksAndRejections (node:internal/process/task_queues:105:5) 11:26:01 "status": 1 11:26:01 } 11:26:01 [11:26:01.735] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWQVg1vrbACtdbrWR37gL2NDLQdGfFkTsfd5VPWcFKPQ6v 11:26:01 [11:26:01.736] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 11:26:01 err: { 11:26:01 "type": "ReqRespStatusError", 11:26:01 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 11:26:01 "stack": 11:26:01 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 11:26:01 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 11:26:01 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 11:26:01 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 11:26:01 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 11:26:01 at processTicksAndRejections (node:internal/process/task_queues:105:5) 11:26:01 "status": 1 11:26:01 } 11:26:01 [11:26:01.744] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:01 [11:26:01.745] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:01 [11:26:01.749] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:01 [11:26:01.749] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:03 [11:26:03.193] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:03 [11:26:03.193] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:03 [11:26:03.197] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:03 [11:26:03.198] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:04 [11:26:04.290] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:04 [11:26:04.290] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:04 [11:26:04.294] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:04 [11:26:04.295] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:05 [11:26:05.385] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:05 [11:26:05.386] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:05 [11:26:05.390] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:05 [11:26:05.391] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:08 [11:26:08.445] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:08 [11:26:08.446] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:08 [11:26:08.449] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:08 [11:26:08.449] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:13 [11:26:13.616] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:13 [11:26:13.617] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:13 [11:26:13.618] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:13 [11:26:13.618] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:13 [11:26:13.619] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:13 [11:26:13.619] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:13 [11:26:13.622] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:13 [11:26:13.622] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:13 [11:26:13.635] WARN: foundation:memory_fifo Discarding item because queue is flushing 11:26:13 [11:26:13.638] ERROR: p2p:reqresp Unexpected error sending request to peer: Error: Could not enqueue function 11:26:13 at /home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:65:24 11:26:13 at new Promise (<anonymous>) 11:26:13 at SerialQueue.put (/home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:55:16) 11:26:13 at ConnectionSampler.put [as dialProtocol] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/connection-sampler/connection_sampler.ts:204:43) 11:26:13 at ReqResp.dialProtocol (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:449:45) 11:26:13 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 11:26:13 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 11:26:13 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 11:26:13 at TimeoutTask.requestFunction [as fn] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:200:26) {"peerId":"12D3KooWKXgcW6jgxTfTsc7qTYiGoTYP3Rp5dbkbYzdqTCVgGq8L","subProtocol":"/aztec/req/tx/0.1.0"} 11:26:14 [11:26:14.720] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:14 [11:26:14.721] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:14 [11:26:14.724] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:14 [11:26:14.724] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:15 [11:26:15.805] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:15 [11:26:15.806] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:15 [11:26:15.809] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:15 [11:26:15.810] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:16 [11:26:16.860] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:16 [11:26:16.860] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:16 [11:26:16.871] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:16 [11:26:16.872] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:18 [11:26:18.146] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:18 [11:26:18.147] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:18 [11:26:18.151] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:18 [11:26:18.151] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:19 [11:26:19.287] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWJHnJzS1aLwFu3T3NDwMmXBX99PP2jF9hD65jbqHmgMAs 11:26:19 [11:26:19.288] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 11:26:19 err: { 11:26:19 "type": "ReqRespStatusError", 11:26:19 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 11:26:19 "stack": 11:26:19 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 11:26:19 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 11:26:19 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 11:26:19 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 11:26:19 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 11:26:19 at processTicksAndRejections (node:internal/process/task_queues:105:5) 11:26:19 "status": 1 11:26:19 } 11:26:19 [11:26:19.318] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:19 [11:26:19.318] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:19 [11:26:19.319] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:19 [11:26:19.320] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:19 [11:26:19.323] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:19 [11:26:19.324] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:20 [11:26:20.647] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:20 [11:26:20.648] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:20 [11:26:20.648] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:20 [11:26:20.649] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:20 [11:26:20.649] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:20 [11:26:20.650] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:20 [11:26:20.663] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:20 [11:26:20.664] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:21 [11:26:21.820] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWQsT1C4i3NsCbNVuQwXfxSxV97fTDFT9FFmijDZvjwpCM 11:26:21 [11:26:21.821] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 11:26:21 err: { 11:26:21 "type": "ReqRespStatusError", 11:26:21 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 11:26:21 "stack": 11:26:21 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 11:26:21 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 11:26:21 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 11:26:21 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 11:26:21 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 11:26:21 at processTicksAndRejections (node:internal/process/task_queues:105:5) 11:26:21 "status": 1 11:26:21 } 11:26:21 [11:26:21.825] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWQsT1C4i3NsCbNVuQwXfxSxV97fTDFT9FFmijDZvjwpCM 11:26:21 [11:26:21.825] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 11:26:21 err: { 11:26:21 "type": "ReqRespStatusError", 11:26:21 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 11:26:21 "stack": 11:26:21 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 11:26:21 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 11:26:21 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 11:26:21 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 11:26:21 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 11:26:21 at processTicksAndRejections (node:internal/process/task_queues:105:5) 11:26:21 "status": 1 11:26:21 } 11:26:21 [11:26:21.843] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWQsT1C4i3NsCbNVuQwXfxSxV97fTDFT9FFmijDZvjwpCM 11:26:21 [11:26:21.844] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 11:26:21 err: { 11:26:21 "type": "ReqRespStatusError", 11:26:21 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 11:26:21 "stack": 11:26:21 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 11:26:21 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 11:26:21 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 11:26:21 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 11:26:21 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 11:26:21 at processTicksAndRejections (node:internal/process/task_queues:105:5) 11:26:21 "status": 1 11:26:21 } 11:26:21 [11:26:21.871] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:21 [11:26:21.872] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:21 [11:26:21.873] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:21 [11:26:21.873] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:21 [11:26:21.877] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 11:26:21 [11:26:21.878] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 11:26:21 FAIL src/services/reqresp/reqresp.test.ts (30.722 s) 11:26:21 ReqResp 11:26:21 ✓ should perform a ping request (1247 ms) 11:26:21 ✓ should handle gracefully if a peer connected peer is offline (622 ms) 11:26:21 ✓ should request from a later peer if other peers are offline (1408 ms) 11:26:21 ✕ should hit a rate limit if too many requests are made in quick succession (1114 ms) 11:26:21 Tx req protocol 11:26:21 ✓ can request a Tx from TxHash (1444 ms) 11:26:21 ✓ handles returning empty buffers (1097 ms) 11:26:21 ✓ does not crash if tx hash returns undefined (1097 ms) 11:26:21 ✓ should hit individual timeout if nothing is returned over the stream (3055 ms) 11:26:21 ✓ should hit collective timeout if nothing is returned over the stream from multiple peers (5186 ms) 11:26:21 ✓ should penalize peer if transaction validation fails (1088 ms) 11:26:21 Goodbye protocol 11:26:21 ✓ should send a goodbye message to a peer (1087 ms) 11:26:21 ✓ should not close stream when handling a goodbye message received from peer (1068 ms) 11:26:21 Block protocol 11:26:21 ✓ should handle block requests (1275 ms) 11:26:21 Batch requests 11:26:21 ✕ should send a batch request between many peers (1177 ms) 11:26:21 ✕ should send a batch request with a pinned peer (1350 ms) 11:26:21 ✕ should stop after max retry attempts (1204 ms) 11:26:21 11:26:21 ● ReqResp › should hit a rate limit if too many requests are made in quick succession 11:26:21 11:26:21 expect(received).toBeDefined() 11:26:21 11:26:21 Received: undefined 11:26:21 11:26:21 145 | // Check that one of the responses gets a rate limit response 11:26:21 146 | const rateLimitResponse = responses.find(response => response?.status === ReqRespStatus.RATE_LIMIT_EXCEEDED); 11:26:21 > 147 | expect(rateLimitResponse).toBeDefined(); 11:26:21 | ^ 11:26:21 148 | 11:26:21 149 | // Make sure the error message is logged 11:26:21 150 | const errorMessage = `Rate limit exceeded DeniedPeer for ${ 11:26:21 11:26:21 at Object.toBeDefined (services/reqresp/reqresp.test.ts:147:31) 11:26:21 11:26:21 ● ReqResp › Batch requests › should send a batch request between many peers 11:26:21 11:26:21 expect(jest.fn()).toHaveBeenCalledTimes(expected) 11:26:21 11:26:21 Expected number of calls: 9 11:26:21 Received number of calls: 11 11:26:21 11:26:21 474 | 11:26:21 475 | // Expect one request to have been sent to each peer 11:26:21 > 476 | expect(sendRequestToPeerSpy).toHaveBeenCalledTimes(batchSize); 11:26:21 | ^ 11:26:21 477 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 11:26:21 478 | expect.objectContaining({ 11:26:21 479 | publicKey: nodes[1].p2p.peerId.publicKey, 11:26:21 11:26:21 at Object.toHaveBeenCalledTimes (services/reqresp/reqresp.test.ts:476:36) 11:26:21 11:26:21 ● ReqResp › Batch requests › should send a batch request with a pinned peer 11:26:21 11:26:21 expect(jest.fn()).toHaveBeenCalledWith(...expected) 11:26:21 11:26:21 Expected: ObjectContaining {"publicKey": [8, 1, 18, 32, 62, 106, 177, 201, 201, 203, …]}, "/aztec/req/ping/0.1.0", {"data": [112, 105, 110, 103, 51], "type": "Buffer"} 11:26:21 Received 11:26:21 1 11:26:21 "12D3KooWHcLhZjeU3F2rxX6SjpzQ4sgdVuCwsy6Cuh4dPoVEwV9q", 11:26:21 "/aztec/req/ping/0.1.0", 11:26:21 @@ -2,9 +2,9 @@ 11:26:21 "data": Array [ 11:26:21 112, 11:26:21 105, 11:26:21 110, 11:26:21 103, 11:26:21 - 51, 11:26:21 + 48, 11:26:21 ], 11:26:21 "type": "Buffer", 11:26:21 }, 11:26:21 2 11:26:21 "12D3KooWD47i5dJmmpo5bdFUby8k4yA9vc3zLmYda7Cd5FE327hC", 11:26:21 "/aztec/req/ping/0.1.0", 11:26:21 @@ -2,9 +2,9 @@ 11:26:21 "data": Array [ 11:26:21 112, 11:26:21 105, 11:26:21 110, 11:26:21 103, 11:26:21 - 51, 11:26:21 + 49, 11:26:21 ], 11:26:21 "type": "Buffer", 11:26:21 }, 11:26:21 3 11:26:21 "12D3KooWE21omj2qCWqwhkF8pEwxKJBn92fnTa8fYWvtxL5M19jc", 11:26:21 "/aztec/req/ping/0.1.0", 11:26:21 @@ -2,9 +2,9 @@ 11:26:21 "data": Array [ 11:26:21 112, 11:26:21 105, 11:26:21 110, 11:26:21 103, 11:26:21 - 51, 11:26:21 + 48, 11:26:21 ], 11:26:21 "type": "Buffer", 11:26:21 }, 11:26:21 11:26:21 Number of calls: 14 11:26:21 11:26:21 516 | // Expect pinned peer to have received all requests 11:26:21 517 | for (let i = 0; i < batchSize; i++) { 11:26:21 > 518 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 11:26:21 | ^ 11:26:21 519 | expect.objectContaining({ publicKey: nodes[1].p2p.peerId.publicKey }), 11:26:21 520 | ReqRespSubProtocol.PING, 11:26:21 521 | Buffer.from(`ping${i}`), 11:26:21 11:26:21 at Object.toHaveBeenCalledWith (services/reqresp/reqresp.test.ts:518:38) 11:26:21 11:26:21 ● ReqResp › Batch requests › should stop after max retry attempts 11:26:21 11:26:21 expect(received).toEqual(expected) // deep equality 11:26:21 11:26:21 - Expected - 11 11:26:21 + Received + 3 11:26:21 11:26:21 @@ -74,10 +74,11 @@ 11:26:21 103, 11:26:21 ], 11:26:21 "type": "Buffer", 11:26:21 }, 11:26:21 }, 11:26:21 + undefined, 11:26:21 RequestableBuffer { 11:26:21 "buffer": Object { 11:26:21 "data": Array [ 11:26:21 112, 11:26:21 111, 11:26:21 @@ -85,10 +86,11 @@ 11:26:21 103, 11:26:21 ], 11:26:21 "type": "Buffer", 11:26:21 }, 11:26:21 }, 11:26:21 + undefined, 11:26:21 RequestableBuffer { 11:26:21 "buffer": Object { 11:26:21 "data": Array [ 11:26:21 112, 11:26:21 111, 11:26:21 @@ -96,17 +98,7 @@ 11:26:21 103, 11:26:21 ], 11:26:21 "type": "Buffer", 11:26:21 }, 11:26:21 }, 11:26:21 - RequestableBuffer { 11:26:21 - "buffer": Object { 11:26:21 - "data": Array [ 11:26:21 - 112, 11:26:21 - 111, 11:26:21 - 110, 11:26:21 - 103, 11:26:21 - ], 11:26:21 - "type": "Buffer", 11:26:21 - }, 11:26:21 - }, 11:26:21 + undefined, 11:26:21 ] 11:26:21 11:26:21 555 | 11:26:21 556 | const res = await nodes[0].req.sendBatchRequest(ReqRespSubProtocol.PING, requests, undefined); 11:26:21 > 557 | expect(res).toEqual(expectResponses); 11:26:21 | ^ 11:26:21 558 | 11:26:21 559 | // Check that we did detect hitting a rate limit 11:26:21 560 | expect(requesterLoggerSpy).toHaveBeenCalledWith( 11:26:21 11:26:21 at Object.toEqual (services/reqresp/reqresp.test.ts:557:19) 11:26:21 11:26:21 Test Suites: 1 failed, 1 total 11:26:21 Tests: 4 failed, 12 passed, 16 total 11:26:21 Snapshots: 0 total 11:26:21 Time: 30.8 s 11:26:21 Ran all test suites matching /p2p\/src\/services\/reqresp\/reqresp.test.ts/i. 11:26:21 Force exiting Jest: Have you considered using `--detectOpenHandles` to detect async operations that kept running after all tests finished?