ACI ·

Parent Log: http://ci.aztec-labs.com/91105392ef11184a Command: 59d7caa3bc5b3609:ISOLATE=1:NAME=p2p/src/services/reqresp/reqresp.test.ts yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts Commit: https://github.com/AztecProtocol/aztec-packages/commit/260a6c8dbe7557aae065d685de66faa2bc362a9d Env: REF_NAME=gh-readonly-queue/master/pr-14951-64b2b9f661d3e31df0f8ec9120130b753eeb7533 CURRENT_VERSION=0.87.8 CI_FULL=1 Date: Tue Jun 10 22:19:54 UTC 2025 System: ARCH=amd64 CPUS=128 MEM=493Gi HOSTNAME=pr-14951_amd64_x1-full Resources: CPU_LIST=0-127 CPUS=2 MEM=8g TIMEOUT=600s History: http://ci.aztec-labs.com/list/history_a57cc872d7daa810_master 22:19:54 +++ id -u 22:19:54 +++ id -g 22:19:54 ++ docker run -d --name p2p_src_services_reqresp_reqresp.test.ts --net=none --cpuset-cpus=0-127 --cpus=2 --memory=8g --user 1000:1000 -v/home/aztec-dev:/home/aztec-dev --mount type=tmpfs,target=/tmp,tmpfs-size=1g --workdir /home/aztec-dev/aztec-packages -e HOME -e VERBOSE -e GIT_CONFIG_GLOBAL=/home/aztec-dev/aztec-packages/build-images/src/home/.gitconfig -e FORCE_COLOR=true -e CPUS -e MEM aztecprotocol/build:3.0 /bin/bash -c 'timeout -v 600s bash -c '\''yarn-project/scripts/run_test.sh p2p/src/services/reqresp/reqresp.test.ts'\''' 22:19:55 + cid=a53a074e618d37d88c9b153946a52b29959a1031073eafad67246e3e1b3ebff4 22:19:55 + set +x 22:20:03 [22:20:03.482] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:03 [22:20:03.486] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:03 [22:20:03.490] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:03 [22:20:03.491] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:04 [22:20:04.036] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:04 [22:20:04.039] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:04 [22:20:04.110] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:04 [22:20:04.111] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:04 [22:20:04.112] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:04 [22:20:04.113] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:05 [22:20:05.326] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:05 [22:20:05.326] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:05 [22:20:05.328] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:05 [22:20:05.330] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:05 [22:20:05.348] ERROR: p2p:reqresp Unexpected error sending request to peer: CodeError: muxer closed locally 22:20:05 at YamuxMuxer.newStream (/home/aztec-dev/aztec-packages/yarn-project/node_modules/@chainsafe/libp2p-yamux/src/muxer.ts:187:13) 22:20:05 at ConnectionImpl.newStream [as _newStream] (/home/aztec-dev/aztec-packages/yarn-project/node_modules/libp2p/src/upgrader.ts:441:41) 22:20:05 at ConnectionImpl.newStream (/home/aztec-dev/aztec-packages/yarn-project/node_modules/libp2p/src/connection/index.ts:134:31) 22:20:05 at Libp2pNode.dialProtocol (/home/aztec-dev/aztec-packages/yarn-project/node_modules/libp2p/src/libp2p.ts:291:23) 22:20:05 at processTicksAndRejections (node:internal/process/task_queues:105:5) 22:20:05 at /home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:58:33 22:20:05 at FifoMemoryQueue.process (/home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/base_memory_queue.js:110:17) { 22:20:05 code: 'ERR_MUXER_LOCAL_CLOSED', 22:20:05 props: {} 22:20:05 } {"peerId":"12D3KooWJbwaKSnLE57FbvHAqUyH4HaCHfJtDzWwrJSALGTbkgVg","subProtocol":"/aztec/req/ping/0.1.0"} 22:20:05 [22:20:05.445] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:05 [22:20:05.446] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:05 [22:20:05.447] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:05 [22:20:05.447] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:05 [22:20:05.448] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:05 [22:20:05.448] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:05 [22:20:05.451] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:05 [22:20:05.452] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:06 [22:20:06.551] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWQUz13yaD27oqMwhuugnLSqSpq88yp79FUYcbJfBWkJuS 22:20:06 [22:20:06.552] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 22:20:06 err: { 22:20:06 "type": "ReqRespStatusError", 22:20:06 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 22:20:06 "stack": 22:20:06 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 22:20:06 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 22:20:06 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 22:20:06 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 22:20:06 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 22:20:06 at processTicksAndRejections (node:internal/process/task_queues:105:5) 22:20:06 "status": 1 22:20:06 } 22:20:06 [22:20:06.563] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWQUz13yaD27oqMwhuugnLSqSpq88yp79FUYcbJfBWkJuS 22:20:06 [22:20:06.564] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 22:20:06 err: { 22:20:06 "type": "ReqRespStatusError", 22:20:06 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 22:20:06 "stack": 22:20:06 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 22:20:06 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 22:20:06 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 22:20:06 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 22:20:06 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 22:20:06 at processTicksAndRejections (node:internal/process/task_queues:105:5) 22:20:06 "status": 1 22:20:06 } 22:20:06 [22:20:06.574] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWQUz13yaD27oqMwhuugnLSqSpq88yp79FUYcbJfBWkJuS 22:20:06 [22:20:06.575] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 22:20:06 err: { 22:20:06 "type": "ReqRespStatusError", 22:20:06 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 22:20:06 "stack": 22:20:06 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 22:20:06 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 22:20:06 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 22:20:06 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 22:20:06 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 22:20:06 at processTicksAndRejections (node:internal/process/task_queues:105:5) 22:20:06 "status": 1 22:20:06 } 22:20:06 [22:20:06.579] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWQUz13yaD27oqMwhuugnLSqSpq88yp79FUYcbJfBWkJuS 22:20:06 [22:20:06.579] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 22:20:06 err: { 22:20:06 "type": "ReqRespStatusError", 22:20:06 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 22:20:06 "stack": 22:20:06 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 22:20:06 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 22:20:06 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 22:20:06 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 22:20:06 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 22:20:06 at processTicksAndRejections (node:internal/process/task_queues:105:5) 22:20:06 "status": 1 22:20:06 } 22:20:06 [22:20:06.583] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWQUz13yaD27oqMwhuugnLSqSpq88yp79FUYcbJfBWkJuS 22:20:06 [22:20:06.583] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 22:20:06 err: { 22:20:06 "type": "ReqRespStatusError", 22:20:06 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 22:20:06 "stack": 22:20:06 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 22:20:06 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 22:20:06 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 22:20:06 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 22:20:06 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 22:20:06 at processTicksAndRejections (node:internal/process/task_queues:105:5) 22:20:06 "status": 1 22:20:06 } 22:20:06 [22:20:06.590] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:06 [22:20:06.591] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:06 [22:20:06.595] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:06 [22:20:06.595] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:08 [22:20:08.034] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:08 [22:20:08.034] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:08 [22:20:08.038] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:08 [22:20:08.038] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:09 [22:20:09.110] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:09 [22:20:09.110] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:09 [22:20:09.113] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:09 [22:20:09.113] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:10 [22:20:10.191] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:10 [22:20:10.192] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:10 [22:20:10.195] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:10 [22:20:10.195] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:13 [22:20:13.268] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:13 [22:20:13.269] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:13 [22:20:13.274] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:13 [22:20:13.274] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:18 [22:20:18.492] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:18 [22:20:18.493] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:18 [22:20:18.494] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:18 [22:20:18.494] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:18 [22:20:18.495] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:18 [22:20:18.496] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:18 [22:20:18.499] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:18 [22:20:18.499] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:18 [22:20:18.513] WARN: foundation:memory_fifo Discarding item because queue is flushing 22:20:18 [22:20:18.517] ERROR: p2p:reqresp Unexpected error sending request to peer: Error: Could not enqueue function 22:20:18 at /home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:65:24 22:20:18 at new Promise (<anonymous>) 22:20:18 at SerialQueue.put (/home/aztec-dev/aztec-packages/yarn-project/foundation/dest/queue/serial_queue.js:55:16) 22:20:18 at ConnectionSampler.put [as dialProtocol] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/connection-sampler/connection_sampler.ts:204:43) 22:20:18 at ReqResp.dialProtocol (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:449:45) 22:20:18 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 22:20:18 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 22:20:18 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 22:20:18 at TimeoutTask.requestFunction [as fn] (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:200:26) {"peerId":"12D3KooWBCVE4xwcgZHuRwAaJpubYY5QBC33BcokFJBxPRmia3zY","subProtocol":"/aztec/req/tx/0.1.0"} 22:20:19 [22:20:19.607] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:19 [22:20:19.608] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:19 [22:20:19.611] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:19 [22:20:19.611] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:20 [22:20:20.671] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:20 [22:20:20.671] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:20 [22:20:20.675] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:20 [22:20:20.675] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:21 [22:20:21.718] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:21 [22:20:21.719] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:21 [22:20:21.722] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:21 [22:20:21.722] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:22 [22:20:22.992] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:22 [22:20:22.992] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:22 [22:20:22.995] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:22 [22:20:22.996] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:24 [22:20:24.098] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWRHTPEgS45usqhMVLQpMQ4F9vmHTL6Ajy6Myi9V3MPv3g 22:20:24 [22:20:24.098] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 22:20:24 err: { 22:20:24 "type": "ReqRespStatusError", 22:20:24 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 22:20:24 "stack": 22:20:24 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 22:20:24 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 22:20:24 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 22:20:24 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 22:20:24 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 22:20:24 at processTicksAndRejections (node:internal/process/task_queues:105:5) 22:20:24 "status": 1 22:20:24 } 22:20:24 [22:20:24.119] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:24 [22:20:24.119] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:24 [22:20:24.120] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:24 [22:20:24.120] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:24 [22:20:24.123] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:24 [22:20:24.123] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:26 [22:20:26.267] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:26 [22:20:26.267] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:26 [22:20:26.268] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:26 [22:20:26.268] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:26 [22:20:26.269] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:26 [22:20:26.269] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:26 [22:20:26.272] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:26 [22:20:26.273] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:27 [22:20:27.468] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWDgZw3yGeopMbGNWXkQyiFiWmDcCcLpG7cEUab6gFWYBd 22:20:27 [22:20:27.468] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 22:20:27 err: { 22:20:27 "type": "ReqRespStatusError", 22:20:27 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 22:20:27 "stack": 22:20:27 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 22:20:27 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 22:20:27 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 22:20:27 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 22:20:27 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 22:20:27 at processTicksAndRejections (node:internal/process/task_queues:105:5) 22:20:27 "status": 1 22:20:27 } 22:20:27 [22:20:27.473] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWDgZw3yGeopMbGNWXkQyiFiWmDcCcLpG7cEUab6gFWYBd 22:20:27 [22:20:27.473] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 22:20:27 err: { 22:20:27 "type": "ReqRespStatusError", 22:20:27 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 22:20:27 "stack": 22:20:27 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 22:20:27 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 22:20:27 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 22:20:27 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 22:20:27 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 22:20:27 at processTicksAndRejections (node:internal/process/task_queues:105:5) 22:20:27 "status": 1 22:20:27 } 22:20:27 [22:20:27.489] WARN: p2p:reqresp Rate limit exceeded DeniedPeer for /aztec/req/ping/0.1.0 from 12D3KooWDgZw3yGeopMbGNWXkQyiFiWmDcCcLpG7cEUab6gFWYBd 22:20:27 [22:20:27.489] WARN: p2p:reqresp Reqresp Response error: Error undefined ReqResp Error: RATE_LIMIT_EXCEEDED {"protocol":"/aztec/req/ping/0.1.0"} 22:20:27 err: { 22:20:27 "type": "ReqRespStatusError", 22:20:27 "message": "ReqResp Error: RATE_LIMIT_EXCEEDED", 22:20:27 "stack": 22:20:27 Error: ReqResp Error: RATE_LIMIT_EXCEEDED 22:20:27 at ReqResp.streamHandler (/home/aztec-dev/aztec-packages/yarn-project/p2p/src/services/reqresp/reqresp.ts:630:15) 22:20:27 at /home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:34:54 22:20:27 at NoopTracer.startActiveSpan (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/noop.js:29:16) 22:20:27 at ReqResp.replacementMethod (/home/aztec-dev/aztec-packages/yarn-project/telemetry-client/dest/telemetry.js:31:32) 22:20:27 at processTicksAndRejections (node:internal/process/task_queues:105:5) 22:20:27 "status": 1 22:20:27 } 22:20:27 [22:20:27.501] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:27 [22:20:27.501] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:27 [22:20:27.502] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:27 [22:20:27.502] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:27 [22:20:27.505] INFO: p2p:reqresp:connection-sampler Stopping connection sampler 22:20:27 [22:20:27.505] INFO: p2p:reqresp:connection-sampler Connection sampler stopped 22:20:27 FAIL src/services/reqresp/reqresp.test.ts (31.464 s) 22:20:27 ReqResp 22:20:27 ✓ should perform a ping request (1288 ms) 22:20:27 ✓ should handle gracefully if a peer connected peer is offline (618 ms) 22:20:27 ✓ should request from a later peer if other peers are offline (1373 ms) 22:20:27 ✕ should hit a rate limit if too many requests are made in quick succession (1108 ms) 22:20:27 Tx req protocol 22:20:27 ✓ can request a Tx from TxHash (1442 ms) 22:20:27 ✓ handles returning empty buffers (1074 ms) 22:20:27 ✓ does not crash if tx hash returns undefined (1079 ms) 22:20:27 ✓ should hit individual timeout if nothing is returned over the stream (3089 ms) 22:20:27 ✓ should hit collective timeout if nothing is returned over the stream from multiple peers (5229 ms) 22:20:27 ✓ should penalize peer if transaction validation fails (1098 ms) 22:20:27 Goodbye protocol 22:20:27 ✓ should send a goodbye message to a peer (1065 ms) 22:20:27 ✓ should not close stream when handling a goodbye message received from peer (1050 ms) 22:20:27 Block protocol 22:20:27 ✓ should handle block requests (1267 ms) 22:20:27 Batch requests 22:20:27 ✕ should send a batch request between many peers (1131 ms) 22:20:27 ✕ should send a batch request with a pinned peer (2159 ms) 22:20:27 ✕ should stop after max retry attempts (1225 ms) 22:20:27 22:20:27 ● ReqResp › should hit a rate limit if too many requests are made in quick succession 22:20:27 22:20:27 expect(received).toBeDefined() 22:20:27 22:20:27 Received: undefined 22:20:27 22:20:27 145 | // Check that one of the responses gets a rate limit response 22:20:27 146 | const rateLimitResponse = responses.find(response => response?.status === ReqRespStatus.RATE_LIMIT_EXCEEDED); 22:20:27 > 147 | expect(rateLimitResponse).toBeDefined(); 22:20:27 | ^ 22:20:27 148 | 22:20:27 149 | // Make sure the error message is logged 22:20:27 150 | const errorMessage = `Rate limit exceeded DeniedPeer for ${ 22:20:27 22:20:27 at Object.toBeDefined (services/reqresp/reqresp.test.ts:147:31) 22:20:27 22:20:27 ● ReqResp › Batch requests › should send a batch request between many peers 22:20:27 22:20:27 expect(jest.fn()).toHaveBeenCalledTimes(expected) 22:20:27 22:20:27 Expected number of calls: 9 22:20:27 Received number of calls: 11 22:20:27 22:20:27 474 | 22:20:27 475 | // Expect one request to have been sent to each peer 22:20:27 > 476 | expect(sendRequestToPeerSpy).toHaveBeenCalledTimes(batchSize); 22:20:27 | ^ 22:20:27 477 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 22:20:27 478 | expect.objectContaining({ 22:20:27 479 | publicKey: nodes[1].p2p.peerId.publicKey, 22:20:27 22:20:27 at Object.toHaveBeenCalledTimes (services/reqresp/reqresp.test.ts:476:36) 22:20:27 22:20:27 ● ReqResp › Batch requests › should send a batch request with a pinned peer 22:20:27 22:20:27 expect(jest.fn()).toHaveBeenCalledWith(...expected) 22:20:27 22:20:27 Expected: ObjectContaining {"publicKey": [8, 1, 18, 32, 165, 15, 62, 94, 247, 36, …]}, "/aztec/req/ping/0.1.0", {"data": [112, 105, 110, 103, 51], "type": "Buffer"} 22:20:27 Received 22:20:27 1 22:20:27 "12D3KooWSMCPnJZs4FeKyDxLKznAjhooMpbKVEnRHMpwHwFzF52s", 22:20:27 "/aztec/req/ping/0.1.0", 22:20:27 @@ -2,9 +2,9 @@ 22:20:27 "data": Array [ 22:20:27 112, 22:20:27 105, 22:20:27 110, 22:20:27 103, 22:20:27 - 51, 22:20:27 + 48, 22:20:27 ], 22:20:27 "type": "Buffer", 22:20:27 }, 22:20:27 2 22:20:27 "12D3KooWRwu8fXV5pCozN1rc64hqYSQZznuHdxxmg5HsCMBBbUvU", 22:20:27 "/aztec/req/ping/0.1.0", 22:20:27 @@ -2,9 +2,9 @@ 22:20:27 "data": Array [ 22:20:27 112, 22:20:27 105, 22:20:27 110, 22:20:27 103, 22:20:27 - 51, 22:20:27 + 49, 22:20:27 ], 22:20:27 "type": "Buffer", 22:20:27 }, 22:20:27 3 22:20:27 "12D3KooWLvgvLnP2An8yEJ8hijcYdDukXvHfQtEpRR4Ef6Rax4Aw", 22:20:27 "/aztec/req/ping/0.1.0", 22:20:27 @@ -2,9 +2,9 @@ 22:20:27 "data": Array [ 22:20:27 112, 22:20:27 105, 22:20:27 110, 22:20:27 103, 22:20:27 - 51, 22:20:27 + 48, 22:20:27 ], 22:20:27 "type": "Buffer", 22:20:27 }, 22:20:27 22:20:27 Number of calls: 12 22:20:27 22:20:27 516 | // Expect pinned peer to have received all requests 22:20:27 517 | for (let i = 0; i < batchSize; i++) { 22:20:27 > 518 | expect(sendRequestToPeerSpy).toHaveBeenCalledWith( 22:20:27 | ^ 22:20:27 519 | expect.objectContaining({ publicKey: nodes[1].p2p.peerId.publicKey }), 22:20:27 520 | ReqRespSubProtocol.PING, 22:20:27 521 | Buffer.from(`ping${i}`), 22:20:27 22:20:27 at Object.toHaveBeenCalledWith (services/reqresp/reqresp.test.ts:518:38) 22:20:27 22:20:27 ● ReqResp › Batch requests › should stop after max retry attempts 22:20:27 22:20:27 expect(received).toEqual(expected) // deep equality 22:20:27 22:20:27 - Expected - 11 22:20:27 + Received + 3 22:20:27 22:20:27 @@ -63,10 +63,11 @@ 22:20:27 103, 22:20:27 ], 22:20:27 "type": "Buffer", 22:20:27 }, 22:20:27 }, 22:20:27 + undefined, 22:20:27 RequestableBuffer { 22:20:27 "buffer": Object { 22:20:27 "data": Array [ 22:20:27 112, 22:20:27 111, 22:20:27 @@ -74,10 +75,11 @@ 22:20:27 103, 22:20:27 ], 22:20:27 "type": "Buffer", 22:20:27 }, 22:20:27 }, 22:20:27 + undefined, 22:20:27 RequestableBuffer { 22:20:27 "buffer": Object { 22:20:27 "data": Array [ 22:20:27 112, 22:20:27 111, 22:20:27 @@ -85,21 +87,11 @@ 22:20:27 103, 22:20:27 ], 22:20:27 "type": "Buffer", 22:20:27 }, 22:20:27 }, 22:20:27 - RequestableBuffer { 22:20:27 - "buffer": Object { 22:20:27 - "data": Array [ 22:20:27 - 112, 22:20:27 - 111, 22:20:27 - 110, 22:20:27 - 103, 22:20:27 - ], 22:20:27 - "type": "Buffer", 22:20:27 - }, 22:20:27 - }, 22:20:27 + undefined, 22:20:27 RequestableBuffer { 22:20:27 "buffer": Object { 22:20:27 "data": Array [ 22:20:27 112, 22:20:27 111, 22:20:27 22:20:27 555 | 22:20:27 556 | const res = await nodes[0].req.sendBatchRequest(ReqRespSubProtocol.PING, requests, undefined); 22:20:27 > 557 | expect(res).toEqual(expectResponses); 22:20:27 | ^ 22:20:27 558 | 22:20:27 559 | // Check that we did detect hitting a rate limit 22:20:27 560 | expect(requesterLoggerSpy).toHaveBeenCalledWith( 22:20:27 22:20:27 at Object.toEqual (services/reqresp/reqresp.test.ts:557:19) 22:20:27 22:20:27 Test Suites: 1 failed, 1 total 22:20:27 Tests: 4 failed, 12 passed, 16 total 22:20:27 Snapshots: 0 total 22:20:27 Time: 31.552 s 22:20:27 Ran all test suites matching /p2p\/src\/services\/reqresp\/reqresp.test.ts/i. 22:20:27 Force exiting Jest: Have you considered using `--detectOpenHandles` to detect async operations that kept running after all tests finished?