diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
new file mode 100644
index 0000000..da69f9d
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,28 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ""
+labels: bug
+assignees: ""
+---
+
+**Version**
+List the versions of all `flood` packages you are using.
+
+
+**Platform**
+The output of `uname -a` (UNIX), or version and 32 or 64-bit (Windows)
+
+**Description**
+Enter your issue details here.
+One way to structure the description:
+
+[short summary of the bug]
+
+I tried this code:
+
+[code sample that causes the bug]
+
+I expected to see this happen: [explanation]
+
+Instead, this happened: [explanation]
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
new file mode 100644
index 0000000..df05885
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,19 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ""
+labels: feature-request
+assignees: ""
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.
diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md
new file mode 100644
index 0000000..0052461
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/question.md
@@ -0,0 +1,9 @@
+---
+name: Question
+about: Please use the Telegram group for questions
+title: ""
+labels: ""
+assignees: ""
+---
+
+Please post your question as a discussion in Telegram: https://t.me/paradigm_flood
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
new file mode 100644
index 0000000..e3bcd5e
--- /dev/null
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -0,0 +1,32 @@
+
+
+## Motivation
+
+
+
+## Solution
+
+
+
+## PR Checklist
+
+- [ ] Added Tests
+- [ ] Added Documentation
+- [ ] Breaking changes
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
new file mode 100644
index 0000000..4dfd462
--- /dev/null
+++ b/.github/workflows/docker.yml
@@ -0,0 +1,25 @@
+name: Docker
+
+on:
+ push:
+ branches:
+ - main
+ pull_request:
+
+jobs:
+ docker:
+ strategy:
+ matrix:
+ test: ["eth_getBalance", "eth_getBlockByNumber"]
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - uses: docker/setup-buildx-action@v2
+ - run: docker build --tag paradigmxyz/flood:latest .
+ - run: docker run --rm paradigmxyz/flood:latest version
+ - name: ${{ matrix.test }}
+ run: |
+ docker run --rm paradigmxyz/flood:latest \
+ ${{ matrix.test }} \
+ node1=https://eth.llamarpc.com \
+ --duration 3 --rate 1
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
new file mode 100644
index 0000000..656027b
--- /dev/null
+++ b/.github/workflows/lint.yml
@@ -0,0 +1,9 @@
+name: Lint
+on: [ push, pull_request ]
+
+jobs:
+ ruff:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - uses: chartboost/ruff-action@v1
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
new file mode 100644
index 0000000..24ff929
--- /dev/null
+++ b/.github/workflows/tests.yml
@@ -0,0 +1,30 @@
+name: Pytest
+
+on:
+ push:
+ branches:
+ - main
+ pull_request:
+
+jobs:
+ build:
+
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v2
+
+ - name: Set up Python
+ uses: actions/setup-python@v2
+ with:
+ python-version: 3.x # Specify the desired Python version
+
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install pytest
+ pip install ./
+
+ - name: Run pytest
+ run: pytest
diff --git a/.gitignore b/.gitignore
index 3679cb0..e23ce86 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,3 +21,7 @@ dist/*
notebooks/*
roadmap/*
todo/*
+graveyard.py
+
+*.ipynb
+*.html
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 0000000..0af1e5d
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,156 @@
+## Contributing to flood
+
+Thanks for your interest in improving flood!
+
+There are multiple opportunities to contribute at any level. It doesn't matter if you are just getting started with python or are the most weathered expert, we can use your help.
+
+**No contribution is too small and all contributions are valued.**
+
+This document will help you get started. **Do not let the document intimidate you**.
+It should be considered as a guide to help you navigate the process.
+
+If you contribute to this project, your contributions will be made to the project under both Apache 2.0 and the MIT license.
+
+### Code of Conduct
+
+The flood project adheres to the [Rust Code of Conduct][rust-coc]. This code of conduct describes the _minimum_ behavior expected from all contributors.
+
+Instances of violations of the Code of Conduct can be reported by contacting the team at [storm@paradigm.xyz](mailto:storm@paradigm.xyz).
+
+### Ways to contribute
+
+There are fundamentally three ways an individual can contribute:
+
+1. **By opening an issue:** For example, if you believe that you have uncovered a bug
+ in flood, creating a new issue in the issue tracker is the way to report it.
+2. **By adding context:** Providing additional context to existing issues,
+ such as screenshots and code snippets to help resolve issues.
+3. **By resolving issues:** Typically this is done in the form of either
+ demonstrating that the issue reported is not a problem after all, or more often,
+ by opening a pull request that fixes the underlying problem, in a concrete and
+ reviewable manner.
+
+**Anybody can participate in any stage of contribution**. We urge you to participate in the discussion around bugs and participate in reviewing PRs.
+
+### Asking for help
+
+
+
+
+
+If you have reviewed existing documentation and still have questions, or you are having problems, you can get help by **opening a discussion**. This repository comes with a discussions board where you can also ask for help. Click the "Discussions" tab at the top.
+
+
+### Submitting a bug report
+
+When filing a new bug report in the issue tracker, you will be presented with a basic form to fill out.
+
+If you believe that you have uncovered a bug, please fill out the form to the best of your ability. Do not worry if you cannot answer every detail, just fill in what you can. Contributors will ask follow-up questions if something is unclear.
+
+The most important pieces of information we need in a bug report are:
+
+- The flood version you are on (and that it is up to date)
+- The platform you are on (Windows, macOS, an M1 Mac or Linux)
+- Code snippets if this is happening in relation to testing or building code
+- Concrete steps to reproduce the bug
+
+In order to rule out the possibility of the bug being in your project, the code snippets should be as minimal as possible. It is better if you can reproduce the bug with a small snippet as opposed to an entire project!
+
+See [this guide][mcve] on how to create a minimal, complete, and verifiable example.
+
+### Submitting a feature request
+
+When adding a feature request in the issue tracker, you will be presented with a basic form to fill out.
+
+Please include as detailed of an explanation as possible of the feature you would like, adding additional context if necessary.
+
+If you have examples of other tools that have the feature you are requesting, please include them as well.
+
+### Resolving an issue
+
+Pull requests are the way concrete changes are made to the code, documentation, and dependencies of flood.
+
+Even tiny pull requests, like fixing wording, are greatly appreciated. Before making a large change, it is usually a good idea to first open an issue describing the change to solicit feedback and guidance. This will increase the likelihood of the PR getting merged.
+
+Please also make sure that the following commands pass if you have changed the code:
+
+If you are working on a larger feature, we encourage you to open up a draft pull request, to make sure that other contributors are not duplicating work.
+
+#### Adding tests
+
+If the change being proposed alters code, it is either adding new functionality to flood, or fixing existing, broken functionality.
+In both of these cases, the pull request should include one or more tests to ensure that flood does not regress in the future.
+
+Types of tests include:
+
+- **Unit tests**: Functions which have very specific tasks should be unit tested.
+- **Integration tests**: For general purpose, far reaching functionality,
+ integration tests should be added. The best way to add a new integration test is to look at existing ones and follow the style.
+
+Tests that use forking must contain "fork" in their name.
+
+#### Commits
+
+It is a recommended best practice to keep your changes as logically grouped as possible within individual commits. There is no limit to the number of commits any single pull request may have, and many contributors find it easier to review changes that are split across multiple commits.
+
+That said, if you have a number of commits that are "checkpoints" and don't represent a single logical change, please squash those together.
+
+#### Opening the pull request
+
+From within GitHub, opening a new pull request will present you with a template that should be filled out. Please try your best at filling out the details, but feel free to skip parts if you're not sure what to put.
+
+#### Discuss and update
+
+You will probably get feedback or requests for changes to your pull request.
+This is a big part of the submission process, so don't be discouraged! Some contributors may sign off on the pull request right away, others may have more detailed comments or feedback. This is a necessary part of the process in order to evaluate whether the changes are correct and necessary.
+
+**Any community member can review a PR, so you might get conflicting feedback**. Keep an eye out for comments from code owners to provide guidance on conflicting feedback.
+
+#### Reviewing pull requests
+
+**Any flood community member is welcome to review any pull request**.
+
+All contributors who choose to review and provide feedback on pull requests have a responsibility to both the project and individual making the contribution. Reviews and feedback must be helpful, insightful, and geared towards improving the contribution as opposed to simply blocking it. If there are reasons why you feel the PR should not be merged, explain what those are. Do not expect to be able to block a PR from advancing simply because you say "no" without giving an explanation. Be open to having your mind changed. Be open to working _with_ the contributor to make the pull request better.
+
+Reviews that are dismissive or disrespectful of the contributor or any other reviewers are strictly counter to the [Code of Conduct][coc-header].
+
+When reviewing a pull request, the primary goals are for the codebase to improve and for the person submitting the request to succeed. **Even if a pull request is not merged, the submitter should come away from the experience feeling like their effort was not unappreciated**. Every PR from a new contributor is an opportunity to grow the community.
+
+##### Review a bit at a time
+
+Do not overwhelm new contributors.
+
+It is tempting to micro-optimize and make everything about relative performance, perfect grammar, or exact style matches. Do not succumb to that temptation.
+
+Focus first on the most significant aspects of the change:
+
+1. Does this change make sense for flood?
+2. Does this change make flood better, even if only incrementally?
+3. Are there clear bugs or larger scale issues that need attending?
+4. Are the commit messages readable and correct? If it contains a breaking change, is it clear enough?
+
+Note that only **incremental** improvement is needed to land a PR. This means that the PR does not need to be perfect, only better than the status quo. Follow-up PRs may be opened to continue iterating.
+
+When changes are necessary, _request_ them, do not _demand_ them, and **do not assume that the submitter already knows how to add a test or run a benchmark**.
+
+Specific performance optimization techniques, coding styles and conventions change over time. The first impression you give to a new contributor never does.
+
+Nits (requests for small changes that are not essential) are fine, but try to avoid stalling the pull request. Most nits can typically be fixed by the flood maintainers merging the pull request, but they can also be an opportunity for the contributor to learn a bit more about the project.
+
+It is always good to clearly indicate nits when you comment, e.g.: `Nit: change foo() to bar(). But this is not blocking`.
+
+If your comments were addressed but were not folded after new commits, or if they proved to be mistaken, please, [hide them][hiding-a-comment] with the appropriate reason to keep the conversation flow concise and relevant.
+
+##### Be aware of the person behind the code
+
+Be aware that _how_ you communicate requests and reviews in your feedback can have a significant impact on the success of the pull request. Yes, we may merge a particular change that makes flood better, but the individual might just not want to have anything to do with flood ever again. The goal is not just having good code.
+
+##### Abandoned or stale pull requests
+
+If a pull request appears to be abandoned or stalled, it is polite to first check with the contributor to see if they intend to continue the work before checking if they would mind if you took it over (especially if it just has nits left). When doing so, it is courteous to give the original contributor credit for the work they started, either by preserving their name and e-mail address in the commit log, or by using the `Author: ` or `Co-authored-by: ` metadata tag in the commits.
+
+[rust-coc]: https://github.com/rust-lang/rust/blob/master/CODE_OF_CONDUCT.md
+[coc-header]: #code-of-conduct
+[mcve]: https://stackoverflow.com/help/mcve
+[hiding-a-comment]: https://help.github.com/articles/managing-disruptive-comments/#hiding-a-comment
+
diff --git a/Dockerfile b/Dockerfile
index 08d0259..12f21de 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,32 +1,39 @@
+# uploaded to GitHub Container Registry
+# https://github.com/paradigmxyz/flood/pkgs/container/flood
-FROM python:3.11.3
-
-# create user
-ENV USERNAME="rpc_bench"
+# Stage 1: Install flood
+FROM python:3.11-slim AS flood-builder
+ENV USERNAME="flood"
+ENV PATH="${PATH}:/home/${USERNAME}/.local/bin"
RUN adduser $USERNAME
USER $USERNAME
-# install rpc_bench
-RUN mkdir /home/$USERNAME/repos
-COPY ./ /home/$USERNAME/repos/rpc_bench/
-WORKDIR /home/$USERNAME/repos/rpc_bench
-RUN pip install ./
+COPY ./ /home/$USERNAME/repos/flood/
+WORKDIR /home/$USERNAME/repos/flood
+RUN pip install --user --no-cache-dir ./
-# install ctc
-WORKDIR /home/$USERNAME/repos
-RUN git clone https://github.com/checkthechain/checkthechain
-WORKDIR /home/$USERNAME/repos/checkthechain
-RUN pip install ./
+# Stage 2: Install vegeta
+FROM debian:stable-slim AS vegeta-builder
+ENV VEGETA_VERSION=12.8.4
+RUN apt-get update && apt-get install -y --no-install-recommends wget ca-certificates \
+ && rm -rf /var/lib/apt/lists/* \
+ && mkdir /vegeta
+WORKDIR /vegeta
+RUN wget https://github.com/tsenart/vegeta/releases/download/v${VEGETA_VERSION}/vegeta_${VEGETA_VERSION}_linux_amd64.tar.gz \
+ && tar xzf vegeta_${VEGETA_VERSION}_linux_amd64.tar.gz \
+ && rm vegeta_${VEGETA_VERSION}_linux_amd64.tar.gz
-# # install vegeta
-RUN mkdir -p /home/$USERNAME/bin/vegeta
-WORKDIR /home/$USERNAME/bin/vegeta_files
-RUN wget https://github.com/tsenart/vegeta/releases/download/v12.8.4/vegeta_12.8.4_linux_amd64.tar.gz
-RUN tar xzf vegeta_12.8.4_linux_amd64.tar.gz
-WORKDIR /home/$USERNAME/bin/
-RUN ln -s /home/$USERNAME/bin/vegeta_files/vegeta /home/$USERNAME/bin/vegeta
+# Final stage: Combine flood and vegeta
+FROM python:3.11-slim
+ENV USERNAME="flood"
+ENV PATH="${PATH}:/home/${USERNAME}/.local/bin"
+RUN adduser $USERNAME
-# run rpc_bench
+RUN apt-get update && apt-get install -y --no-install-recommends curl \
+ && rm -rf /var/lib/apt/lists/*
+COPY --from=flood-builder /home/$USERNAME/.local /home/$USERNAME/.local
+COPY --from=vegeta-builder /vegeta/vegeta /home/$USERNAME/.local/bin/vegeta
+
+USER $USERNAME
WORKDIR /home/$USERNAME
-ENTRYPOINT ["python", "-m", "rpc_bench"]
-
+ENTRYPOINT ["python", "-m", "flood"]
diff --git a/LICENSE-APACHE b/LICENSE-APACHE
new file mode 100644
index 0000000..1f050ef
--- /dev/null
+++ b/LICENSE-APACHE
@@ -0,0 +1,202 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+Copyright 2023 flood contributors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
diff --git a/LICENSE-MIT b/LICENSE-MIT
new file mode 100644
index 0000000..c001e3e
--- /dev/null
+++ b/LICENSE-MIT
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 flood contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/README.md b/README.md
index 8921a35..474e25b 100644
--- a/README.md
+++ b/README.md
@@ -1,23 +1,45 @@
-# RPC Bench
+#
🌊🌊 flood 🌊🌊
-A tool for benchmarking and load testing EVM nodes
+**`flood` is a load testing tool for benchmarking EVM nodes over RPC**
-`rpc_bench` can measure many different types of workloads
-- it separately measure performance of every JSON-RPC method type
-- it perform stress tests, spike tests, and soak tests
+
-`rpc_bench` measures a variety of performance metrics including:
+[][gh-ci]
+[][gh-docker]
+[![Telegram Chat][tg-badge]][tg-url]
+
+[gh-ci]: https://github.com/paradigmxyz/flood/actions/workflows/ci.yml
+[gh-docker]: https://github.com/paradigmxyz/flood/actions/workflows/docker.yml
+[tg-badge]: https://img.shields.io/endpoint?color=neon&logo=telegram&label=chat&url=https%3A%2F%2Ftg.sumanjay.workers.dev%2Fparadigm%5Fflood
+[tg-url]: https://t.me/paradigm_flood
+
+For each RPC method, `flood` measures how load affects metrics such as:
1. throughput
2. latency (mean, P50, P90, P95, P99, max)
3. error rate
-`rpc_bench` makes it easy to compare the performance of:
+`flood` makes it easy to compare the performance of:
- different node clients (e.g. geth vs erigon vs reth)
-- different hardware configurations (e.g. local vs cloud vs low memory vs RAID-0)
+- different hardware/software configurations (e.g. local vs cloud vs low memory vs RAID-0)
- different RPC providers (e.g. Alchemy vs Quicknode vs Infura)
-`rpc_bench` can generate tables, figures, and reports to make its results interpretable (example report [here])
+`flood` can generate tables, figures, and reports for easy sharing of results (example report [here](https://datasets.paradigm.xyz/notebooks/flood/example_report.html))
+
+## Contents
+1. ### [Installation](#installation)
+ 1. [Prerequisites](#prerequisites)
+ 2. [Installing Flood](#installing-flood)
+ 3. [Docker](#docker)
+2. ### [Usage Guide](#usage-guide)
+ 1. [Basic Load Tests](#basic-load-tests)
+ 2. [Remote Load Tests](#remote-load-tests)
+ 3. [Printing Test Results](#printing-test-results)
+ 4. [Report Generation](#report-generation)
+ 5. [Differential Tests](#differential-tests)
+ 6. [From Python](#from-python)
+ 7. [Performing Deep Checks](#performing-deep-checks)
+3. ### [Contributing](#contributing)
## Installation
@@ -26,44 +48,91 @@ A tool for benchmarking and load testing EVM nodes
Install [`vegeta`](https://github.com/tsenart/vegeta):
- on mac: `brew update && brew install vegeta`
-- from source: `go install github.com/tsenart/vegeta@latest`
+- on linux: `go install github.com/tsenart/vegeta/v12@v12.8.4`
-Install latest [`ctc`](https://github.com/checkthechain/checkthechain):
-```
-git clone checkthechain/checkthechain
-cd checkthechain
-pip install -e ./
-```
+After installation, make sure `vegeta` is on your `$PATH`. Running `vegeta -h` should output a path. If it does not, you probably have not set up `go` to install items to your `$PATH`. You may need to add something like `export PATH=$PATH:~/go/bin/` to your terminal config file (e.g. `~/.profile`).
-#### Installing `rpc_bench`
+`flood` also requires `python >= 3.7`
+#### Installing `flood`
```
-git clone https://github.com/paradigmxyz/rpc_bench
-cd rpc_bench
-pip install -e ./
+pip install paradigm-flood
```
-## Usage
+Typing `flood help` in your terminal should show help output. If it does not, you probably have not set up `pip` to install items to your `$PATH`. You may need to add something like `export PATH=$PATH:~/.local/bin` to your terminal config file (e.g. `~/.profile`). Alternatively, you can avoid setting up your `$PATH` and just type `python3 -m flood` instead of `flood`.
-#### run test
-```
-rpc_bench eth_getBlockByNumber NODE1_NAME=NODE1_URL NODE2_NAME=NODE2_URL --rates 10 100 1000 --duration 30
-```
+#### Docker
-#### orchestrate tests on remote nodes
-```
-rpc_bench eth_getBlockByNumber NODE1_NAME=NODE1_URL:localhost:8545 NODE2_NAME=NODE2_URL:localhost:8545
-```
+Alternatively, flood can be used as a [Docker image](https://github.com/paradigmxyz/flood/pkgs/container/flood).
-#### create report
-```
-rpc_bench report tests/test1_output tests/test2_output
-````
+## Usage guide
-## Code Layout
-- `rpc_bench/cli`: command line interface
-- `rpc_bench/generators`: utilities for generating sequences of rpc calls with parameterized distributions
-- `rpc_bench/tests`: implementations of tests
-- `rpc_bench/user_io`: utiltiies for parsing user input
+`flood` works by bombarding an RPC endpoint with different patterns of RPC calls. Measurements of the RPC endpoint's performance under different controlled loads are then used to paint a detailed view of the node's performance.
+
+Every time flood runs, it saves its parameters and test results to an output directory. You can specify this output directory with the `--output` parameter, otherwise a temporary directory will be created. Running a test will populate the folder with the following files:
+- `figures/`: directory containing PNG's summarizing node performance
+- `results.json`: results of the test including performance metrics
+- `summary.txt`: printed summary of test that was output to the console
+- `test.json`: metadata and parameters used to create and run the test
+
+### Basic load tests
+
+Here is an example of a basic test with `flood`. It will benchmark block retrieval from two different nodes. It will test at 3 different rates (10, 100, and 1000 requests per second) and it will test them for 30 seconds each.
+
+`flood eth_getBlockByNumber NODE1_NAME=NODE1_URL NODE2_NAME=NODE2_URL --rates 10 100 1000 --duration 30`
+
+To see all of the parameters available for controlling `flood` tests use `flood --help`
+
+### Remote load tests
+
+Instead of broadcasting RPC calls from whatever machine running the `flood` CLI command, `flood` can broadcast the calls from a remote process on a remote machine. In particular, `flood` can broadcast the calls from the same machine that is running the EVM node in order to eliminate any noise or bottlenecks associated with networking.
+
+This can be accomplished by installing flood on the remote machine and then providing `flood` with login credentials and routing details using the following syntax:
+
+`flood [node_name=][username@]hostname:[test_url] ...`
+
+For example, the following command will test a reth node and an erigon node remotely:
+
+`flood eth_call reth=ubuntu@92.92.92.92:localhost:8545 erigon=ubuntu@91.91.91.91:localhost:8545`
+
+If there are multiple remote tests, these tests will be run in parallel. After the tests are complete, `flood` will retrieve the results and summarize using the same methodology as a local test.
+
+### Printing test results
+
+By default `flood` produces verbose output of each test as it runs. This can be disabled with the `--quiet` parameter. To re-print the results of an old test, use `flood print `. To print a summary of multiple tests, use `flood print `.
+
+### Report generation
+
+After running tests, you can generate an HTML + Jupyter report similar to [this](https://datasets.paradigm.xyz/notebooks/flood/example_report.html) one. This is done by running `flood report `. Multiple tests can be combined into one report with `flood repos ...`.
+
+### Differential tests
+
+Instead of testing the raw performance of an RPC node, `flood` can be used to test the correctness of a node's responses using a differential testing approach. This works by using two nodes and making sure that their responses match under a variety of RPC calls. This is done using the `--equality` parameter. For example:
+
+`flood all reth=91.91.91.91 erigon=92.92.92.92 --equality`
+
+### From python
+
+All of `flood`'s functionality can be used from python instead of the CLI. Some functions:
+
+|description|python|
+|-|-|
+|Import flood|`import flood`|
+|Run tests|`flood.run(...)`|
+|Load the parameters of a test|`flood.load_single_run_results_payload(output_dir)`|
+|Load the results of a test|`flood.load_single_run_results_payload(output_dir)`|
+|Run a live version of a results notebook|`jupyter notebook `|
+
+### Performing deep checks
+
+Under normal operation `flood` relies on vegeta to compute performance summaries of each test. This works well, but sometimes it is desirable to implement custom introspection not available in `vegeta`.
+
+In particular, `vegeta` counts any status-200 response as a success, even if the contents of the response is an RPC error. Running with the `--deep-check` command will check every response to make sure that it returns well-formed JSON with no RPC errors. With `--deep-check`, `flood` also computes separate performance statistics successful vs failed calls.
+
+If you want to save the timing information and raw contents of every single response from the test to the `results.json` output, use the `--save-raw-output` argument. This allows for performing own custom analyses on the raw data.
+
+## Contributing
+
+Contributions are welcome in the form of issues, PR's, and commentary. Check out the contributor guide in [CONTRIBUTING.md](CONTRIBUTING.md).
diff --git a/assets/cover.png b/assets/cover.png
new file mode 100644
index 0000000..cb42139
Binary files /dev/null and b/assets/cover.png differ
diff --git a/examples/equality_test.sh b/examples/equality_test.sh
new file mode 100644
index 0000000..b80425f
--- /dev/null
+++ b/examples/equality_test.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+export NODE1=ip.address.1:8545
+export NODE2=ip.address.2:8545
+
+flood all node1=$NODE1 node2=$NODE2 --equality
+
diff --git a/examples/report.sh b/examples/report.sh
new file mode 100755
index 0000000..b66615b
--- /dev/null
+++ b/examples/report.sh
@@ -0,0 +1,82 @@
+#!/bin/bash
+
+export TEST_DATA_DIR=/path/to/some/output/dir
+export NODE1=username@ip.address.1:localhost:8545
+export NODE2=username@ip.address.2:localhost:8545
+
+#
+# # run tests
+#
+
+flood eth_call \
+ node1=$NODE1 \
+ node2=$NODE2 \
+ --output $TEST_DATA_DIR/eth_call \
+ -r 1024 2048 4096 8192 16384
+
+flood eth_getBalance \
+ node1=$NODE1 \
+ node2=$NODE2 \
+ --output $TEST_DATA_DIR/eth_getBalance \
+ -d 1 \
+ -r 1024 2048 4096 8192 16384
+
+flood eth_getBlockByNumber \
+ node1=$NODE1 \
+ node2=$NODE2 \
+ --output $TEST_DATA_DIR/eth_getBlockByNumber \
+ -r 1024 2048 4096 8192 16384
+
+flood eth_getCode \
+ node1=$NODE1 \
+ node2=$NODE2 \
+ --output $TEST_DATA_DIR/eth_getCode \
+ -r 1024 2048 4096 8192 16384
+
+flood eth_getLogs \
+ node1=$NODE1 \
+ node2=$NODE2 \
+ --output $TEST_DATA_DIR/eth_getLogs \
+ -r 64 128 256 512 1024
+
+flood eth_getStorageAt \
+ node1=$NODE1 \
+ node2=$NODE2 \
+ --output $TEST_DATA_DIR/eth_getStorageAt \
+ -r 1024 2048 4096 8192 16384
+
+flood eth_getTransactionByHash \
+ node1=$NODE1 \
+ node2=$NODE2 \
+ --output $TEST_DATA_DIR/eth_getTransactionByHash \
+ -r 1024 2048 4096 8192 16384
+
+flood eth_getTransactionCount \
+ node1=$NODE1 \
+ node2=$NODE2 \
+ --output $TEST_DATA_DIR/eth_getTransactionCount \
+ -r 2048 4096 8192 16384 32768
+
+flood eth_getTransactionReceipt \
+ node1=$NODE1 \
+ node2=$NODE2 \
+ --output $TEST_DATA_DIR/eth_getTransactionReceipt \
+ -r 1024 2048 4096 8192 16384
+
+#
+# # generate report
+#
+
+cd $TEST_DATA_DIR
+
+flood report \
+ $TEST_DATA_DIR/eth_call \
+ $TEST_DATA_DIR/eth_getBalance \
+ $TEST_DATA_DIR/eth_getBlockByNumber \
+ $TEST_DATA_DIR/eth_getCode \
+ $TEST_DATA_DIR/eth_getLogs \
+ $TEST_DATA_DIR/eth_getStorageAt \
+ $TEST_DATA_DIR/eth_getTransactionByHash \
+ $TEST_DATA_DIR/eth_getTransactionCount \
+ $TEST_DATA_DIR/eth_getTransactionReceipt
+
diff --git a/flood/__init__.py b/flood/__init__.py
new file mode 100644
index 0000000..712891a
--- /dev/null
+++ b/flood/__init__.py
@@ -0,0 +1,42 @@
+"""tool for benchmarking RPC endpoints"""
+# ruff: noqa: F401
+
+from .spec import *
+import flood.generators
+import flood.ops
+import flood.runners
+import flood.tests
+import flood.user_io
+
+from flood.generators import generate_test
+from flood.ops import get_flood_version
+from flood.ops import get_local_installation
+from flood.ops import get_remote_installation
+from flood.ops import get_dependency_versions
+from flood.runners import load_single_run_test_payload
+from flood.runners import load_single_run_results_payload
+from flood.runners import run
+from flood.tests.equality_tests import run_equality_test
+from flood.tests.load_tests import run_load_test
+from flood.tests.load_tests import run_load_tests
+
+
+__version__ = '0.3.1'
+
+
+def _clean_package_imports() -> None:
+ """remove deep nested modules from flood namespace"""
+
+ import sys
+
+ flood_module = sys.modules['flood']
+ moduletype = type(flood)
+ delattr(flood, 'annotations')
+ for key, value in tuple(vars(flood_module).items()):
+ if isinstance(value, moduletype):
+ name = value.__name__
+ if not name.startswith('flood') or name.count('.') > 1:
+ delattr(flood, key)
+
+
+_clean_package_imports()
diff --git a/rpc_bench/__main__.py b/flood/__main__.py
similarity index 71%
rename from rpc_bench/__main__.py
rename to flood/__main__.py
index 5eb7660..0c20bde 100644
--- a/rpc_bench/__main__.py
+++ b/flood/__main__.py
@@ -1,8 +1,7 @@
from __future__ import annotations
-from rpc_bench.cli import cli_run
+from flood.cli import cli_run
if __name__ == '__main__':
cli_run.run_cli()
-
diff --git a/flood/cli/cli_run.py b/flood/cli/cli_run.py
new file mode 100644
index 0000000..6968df7
--- /dev/null
+++ b/flood/cli/cli_run.py
@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+import toolcli
+
+import flood
+
+cd_dir_help = {
+ 'samples': 'FLOOD_SAMPLES_PATH env value',
+}
+
+
+def cd_dir_getter(dirname: str) -> str:
+ if dirname == 'samples':
+ return flood.generators.get_flood_samples_dir()
+ else:
+ raise Exception('unknown path: ' + str(dirname))
+
+
+def run_cli(raw_command: str | None = None) -> None:
+ command_index: toolcli.CommandIndex = {
+ (): 'flood.cli.root_command',
+ ('help',): 'toolcli.command_utils.standard_subcommands.help_command',
+ ('ls',): 'flood.cli.ls_command',
+ ('print',): 'flood.cli.print_command',
+ ('report',): 'flood.cli.report_command',
+ ('samples', 'collect'): 'flood.cli.samples_collect_command',
+ ('samples', 'download'): 'flood.cli.samples_download_command',
+ ('samples', 'ls'): 'flood.cli.samples_ls_command',
+ ('version',): 'flood.cli.version_command',
+ ('cd',): 'toolcli.command_utils.standard_subcommands.cd_command',
+ ('update',): 'flood.cli.update_command',
+ }
+
+ config: toolcli.CLIConfig = {
+ 'base_command': 'flood',
+ 'description': flood.__doc__,
+ 'version': flood.__version__,
+ 'default_command_sequence': (),
+ 'root_help_arguments': True,
+ # 'root_help_subcommands': False,
+ 'include_debug_arg': True,
+ 'style_theme': flood.user_io.styles,
+ 'cd_dir_help': cd_dir_help,
+ 'cd_dir_getter': cd_dir_getter,
+ }
+
+ toolcli.run_cli(
+ command_index=command_index,
+ config=config,
+ )
diff --git a/rpc_bench/cli/ls_command.py b/flood/cli/ls_command.py
similarity index 61%
rename from rpc_bench/cli/ls_command.py
rename to flood/cli/ls_command.py
index 4b020ea..cf0e52a 100644
--- a/rpc_bench/cli/ls_command.py
+++ b/flood/cli/ls_command.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import toolcli
-import rpc_bench
+import flood
def get_command_spec() -> toolcli.CommandSpec:
@@ -15,25 +15,26 @@ def get_command_spec() -> toolcli.CommandSpec:
def ls_command() -> None:
import toolstr
+ styles = flood.user_io.styles
+
toolstr.print_text_box(
'Available tests',
- text_style=rpc_bench.styles['metavar'],
- style=rpc_bench.styles['content'],
+ text_style=styles['metavar'],
+ style=styles['content'],
)
print()
toolstr.print_header(
'Single Load Tests',
- style=rpc_bench.styles['content'],
- text_style=rpc_bench.styles['metavar'],
+ style=styles['content'],
+ text_style=styles['metavar'],
)
- for test in rpc_bench.get_single_test_generators():
+ for test in flood.generators.get_single_test_generators():
toolstr.print_bullet(key=test, value='', colon_str='')
print()
toolstr.print_header(
'Multi Load Tests',
- style=rpc_bench.styles['content'],
- text_style=rpc_bench.styles['metavar'],
+ style=styles['content'],
+ text_style=styles['metavar'],
)
toolstr.print_bullet(key='all', value='', colon_str='')
-
diff --git a/flood/cli/print_command.py b/flood/cli/print_command.py
new file mode 100644
index 0000000..e90c676
--- /dev/null
+++ b/flood/cli/print_command.py
@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import typing
+
+import toolcli
+
+import flood
+
+
+def get_command_spec() -> toolcli.CommandSpec:
+ return {
+ 'f': print_command,
+ 'help': 'print output of previous test results',
+ 'args': [
+ {
+ 'name': 'output_dirs',
+ 'help': 'output directory of previous test(s)',
+ 'nargs': '+',
+ },
+ {
+ 'name': ('-m', '--metrics'),
+ 'help': 'space-separated list of performance metrics',
+ 'nargs': '+',
+ },
+ ],
+ }
+
+
+def print_command(
+ output_dirs: typing.Sequence[str], metrics: typing.Sequence[str]
+) -> None:
+ if len(output_dirs) == 1:
+ print_single(output_dirs[0], metrics)
+ elif len(output_dirs) > 1:
+ print_multiple(output_dirs)
+ else:
+ raise Exception('must specify at least one output_dir')
+
+
+def print_single(
+ output_dir: str, metrics: typing.Sequence[str]
+) -> None:
+ test_payload = flood.load_single_run_test_payload(output_dir)
+ test = flood.generate_test(**test_payload['test_parameters'])
+ results_payload = flood.load_single_run_results_payload(output_dir)
+ results = results_payload['results']
+
+ # print test summary
+ flood.runners.single_runner.single_runner_summary._print_single_run_preamble_copy(
+ test_name=test_payload['name'],
+ rates=[subtest['rate'] for subtest in test['attacks']],
+ durations=[subtest['duration'] for subtest in test['attacks']],
+ vegeta_args=[subtest['vegeta_args'] for subtest in test['attacks']],
+ output_dir=output_dir,
+ )
+
+ # print node data
+ print()
+ flood.user_io.print_header('Nodes tested')
+ flood.user_io.print_nodes_table(results_payload['nodes'])
+
+ # decide metrics
+ if metrics is None:
+ metrics = ['success', 'throughput', 'p90']
+
+ # print performance metrics
+ print()
+ print()
+ flood.user_io.print_header('Summarizing performance metrics...')
+ flood.user_io.print_metric_tables(
+ results=results,
+ metrics=metrics,
+ indent=4,
+ )
+
+
+def print_multiple(output_dirs: typing.Sequence[str]) -> None:
+ # load results
+ test_payloads = []
+ outputs = []
+ for output_dir in output_dirs:
+ test_payload = flood.load_single_run_test_payload(output_dir)
+ output = flood.load_single_run_results_payload(output_dir)
+ test_payloads.append(test_payload)
+ outputs.append(output)
+
+ # print max throughput
+ rows = []
+ conditions = list(outputs[0]['results'].keys())
+ for test_payload, output in zip(test_payloads, outputs):
+ test_parameters = test_payload['test_parameters']
+ row: list[typing.Any] = [test_parameters['test_name']]
+ for condition in conditions:
+ results = output['results'][condition]
+ throughputs = [tp for tp in results['throughput'] if tp is not None]
+ if len(throughputs) > 0:
+ row.append(max(throughputs))
+ else:
+ row.append(None)
+ row.append(max(results['target_rate']))
+ rows.append(row)
+ labels = ['test'] + conditions + ['max tested']
+
+ flood.user_io.print_text_box('Max Throughput')
+ print()
+ print('units = requests per second')
+ print()
+ flood.user_io.print_table(rows, labels=labels)
+
diff --git a/rpc_bench/cli/report_command.py b/flood/cli/report_command.py
similarity index 94%
rename from rpc_bench/cli/report_command.py
rename to flood/cli/report_command.py
index f09ced9..85b2d43 100644
--- a/rpc_bench/cli/report_command.py
+++ b/flood/cli/report_command.py
@@ -4,7 +4,7 @@ import typing
import toolcli
-import rpc_bench
+import flood
help_message = """create report of previous test results
@@ -41,15 +41,13 @@ def report_command(
output: str | None,
metrics: typing.Sequence[str] | None,
) -> None:
-
if output is None:
import os
output = os.getcwd()
- rpc_bench.create_load_test_report(
+ flood.tests.load_tests.create_load_test_report(
test_paths=test_dirs,
output_dir=output,
metrics=metrics,
)
-
diff --git a/rpc_bench/cli/root_command.py b/flood/cli/root_command.py
similarity index 63%
rename from rpc_bench/cli/root_command.py
rename to flood/cli/root_command.py
index 2f631ee..77170ed 100644
--- a/rpc_bench/cli/root_command.py
+++ b/flood/cli/root_command.py
@@ -9,12 +9,12 @@ import typing
import toolcli
-import rpc_bench
+import flood
help_message = """Load test JSON RPC endpoints
[bold][title]Test Specification[/bold][/title]
-- [metavar]TEST[/metavar] can be a template, use [metavar]rpc_bench ls[/metavar] to list test templates
+- [metavar]TEST[/metavar] can be a template, use [metavar]flood ls[/metavar] to list test templates
- Alternatively, [metavar]TEST[/metavar] can be a directory path of a previous test
- This will rerun this the previous test, possibly on new nodes
@@ -24,15 +24,12 @@ help_message = """Load test JSON RPC endpoints
- The [metavar]name[/metavar] of each node is used for benchmark summary report
[bold][title]Remote Usage[/bold][/title]
-- [metavar]rpc_bench[/metavar] can be invoked on remote machines
+- [metavar]flood[/metavar] can be invoked on remote machines
- Use node syntax [metavar]user@remote:node_url[/metavar] or [metavar]name=user@remote:node_url[/metavar]
- Can omit the [metavar]user@[/metavar] prefix if ssh config has username specified
-- [metavar]rpc_bench[/metavar] must already be installed on each remote machine
+- [metavar]flood[/metavar] must already be installed on each remote machine
-[bold][title]Parameter Randomization[/bold][/title]
-- [metavar]rpc_bench[/metavar] can call each RPC method multiple times using [metavar]-n [/metavar]
-- For each call, parameters are randomized to minimize caching effects
-- Specify random seed [metavar]-s [/metavar] for repeatable set of randomized calls"""
+For more details, see the [metavar]README.md[/metavar] at [metavar]https://github.com/paradigmxyz/cryo[/metavar]""" # noqa: E501
def get_command_spec() -> toolcli.CommandSpec:
@@ -42,18 +39,18 @@ def get_command_spec() -> toolcli.CommandSpec:
'args': [
{
'name': 'test',
- 'help': 'test to run (use [metavar]rpc_bench ls[/metavar] for list)',
+ 'help': 'test to run (see [metavar]flood ls[/metavar] for list)', # noqa: E501
},
{
'name': 'nodes',
'nargs': '*',
- 'help': 'nodes to test, see syntax above',
+ 'help': 'nodes to test (see syntax above)',
},
{
'name': ['-s', '--seed'],
'dest': 'random_seed',
'type': int,
- 'help': 'random seed to use, default is current timestamp',
+ 'help': 'random seed to use, (default = current timestamp)',
},
{
'name': ['-q', '--quiet'],
@@ -68,22 +65,23 @@ def get_command_spec() -> toolcli.CommandSpec:
{
'name': ['-m', '--mode'],
'choices': ['stress', 'spike', 'soak'],
+ 'hidden': True,
'help': 'load test type: stress, spike, or soak',
},
{
'name': ['-r', '--rates'],
'nargs': '+',
- 'help': 'rates to use in load test (requests per second)',
+ 'help': 'rates to use in load test, units = reqs per second\n(default is test-specific, use [metavar]--dry[/metavar] to view)', # noqa: E501
},
{
'name': ['-d', '--duration'],
'type': int,
- 'help': 'amount of time to test each rate',
+ 'help': 'number of seconds to test each rate (default = [metavar]30[/metavar])', # noqa: E501
},
{
'name': ['-o', '--output'],
'dest': 'output_dir',
- 'help': 'directory to save results, default is tmp dir',
+ 'help': 'directory to save results, (default = new tmp dir)',
},
{
'name': ['--dry'],
@@ -93,7 +91,7 @@ def get_command_spec() -> toolcli.CommandSpec:
{
'name': ['--metrics'],
'nargs': '+',
- 'help': 'space-separated list of performance metrics to show',
+ 'help': 'space-separated list of performance metrics to show\n(default = [metavar]success throughput p90[/metavar])', # noqa: E501
},
{
'name': ['--no-figures'],
@@ -101,6 +99,31 @@ def get_command_spec() -> toolcli.CommandSpec:
'dest': 'figures',
'action': 'store_false',
},
+ {
+ 'name': ['--save-raw-output'],
+ 'help': 'save the contents of every RPC response',
+ 'action': 'store_true',
+ },
+ {
+ 'name': ['--deep-check'],
+ 'help': 'validate the contents of every RPC response',
+ 'action': 'store_true',
+ },
+ {
+ 'name': ['--remote-update'],
+ 'help': 'attempt to update nodes to latest flood version',
+ 'hidden': True,
+ 'action': 'store_true',
+ },
+ {
+ 'name': ['--vegeta-args'],
+ 'help': 'extra args for vegeta, e.g. [metavar]"-timeout 5s -cpus 1"[/metavar]\nfor single args, use [metavar]--vegeta-args="..."[/metavar] (no space)', # noqa: E501
+ },
+ {
+ 'name': ['-V', '--version'],
+ 'help': 'print flood version and exit',
+ 'action': 'store_true',
+ },
],
'examples': [
'eth_getBlockByNumber localhost:8545',
@@ -115,7 +138,7 @@ def root_command(
nodes: typing.Sequence[str] | None,
output_dir: str | None,
metrics: typing.Sequence[str],
- mode: rpc_bench.LoadTestMode | None,
+ mode: flood.LoadTestMode | None,
rates: typing.Sequence[int] | typing.Sequence[str] | None,
duration: int | None,
random_seed: int | None,
@@ -123,7 +146,13 @@ def root_command(
quiet: bool,
figures: bool,
equality: bool,
+ save_raw_output: bool,
+ deep_check: bool,
+ remote_update: bool,
+ vegeta_args: str,
+ version: bool,
) -> None:
+
verbose = not quiet
if nodes is not None and len(nodes) == 0:
nodes = None
@@ -143,7 +172,7 @@ def root_command(
raise Exception('dry not used in equality test')
if not figures:
raise Exception('figures not used in equality test')
- rpc_bench.run_equality_test(
+ flood.run_equality_test(
test_name=test,
nodes=nodes,
random_seed=random_seed,
@@ -152,9 +181,15 @@ def root_command(
else:
+ include_deep_output: typing.List[flood.DeepOutput] = []
+ if deep_check:
+ include_deep_output.append('metrics')
+ if save_raw_output:
+ include_deep_output.append('raw')
+
if rates is not None:
rates = [int(rate) for rate in rates]
- rpc_bench.run(
+ flood.run(
test_name=test,
mode=mode,
nodes=nodes,
@@ -164,7 +199,10 @@ def root_command(
rates=rates,
duration=duration,
dry=dry,
- output_dir=output_dir or True,
+ output_dir=output_dir,
figures=figures,
+ include_deep_output=include_deep_output,
+ deep_check=deep_check,
+ vegeta_args=vegeta_args,
)
diff --git a/flood/cli/samples_collect_command.py b/flood/cli/samples_collect_command.py
new file mode 100644
index 0000000..ba618e4
--- /dev/null
+++ b/flood/cli/samples_collect_command.py
@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+import typing
+
+import toolcli
+
+import flood
+
+
+def get_command_spec() -> toolcli.CommandSpec:
+ return {
+ 'f': samples_collect_command,
+ 'help': 'collect raw EVM samples using Paradigm Data Portal datasets',
+ 'args': [
+ {
+ 'name': ('-n', '--network'),
+ 'help': 'network, default ethereum',
+ },
+ {
+ 'name': ('-s', '--sizes'),
+ 'help': 'sample sizes of {XS, S, M, L, XL}, default all',
+ },
+ {
+ 'name': ('-o', '--output-dir'),
+ 'help': 'download location, default FLOOD_SAMPLES_DIR or cwd',
+ },
+ {
+ 'name': ('-d', '--datatypes'),
+ 'help': 'datatypes to sample, default all',
+ },
+ ],
+ }
+
+
+def samples_collect_command(
+ network: str | None,
+ sizes: typing.Sequence[str] | None,
+ output_dir: str | None,
+ datatypes: typing.Sequence[str] | None,
+) -> None:
+ if output_dir is None:
+ output_dir = flood.generators.get_flood_samples_dir()
+ if network is None:
+ network = 'ethereum'
+ flood.generators.create_samples_dataset(
+ output_dir=output_dir,
+ network=network,
+ sizes=sizes,
+ datatypes=datatypes,
+ )
diff --git a/flood/cli/samples_download_command.py b/flood/cli/samples_download_command.py
new file mode 100644
index 0000000..934bf18
--- /dev/null
+++ b/flood/cli/samples_download_command.py
@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import typing
+
+import toolcli
+
+import flood
+
+
+def get_command_spec() -> toolcli.CommandSpec:
+ return {
+ 'f': download_samples_command,
+ 'help': 'download raw EVM samples for constructing calls',
+ 'args': [
+ {
+ 'name': ('-n', '--network'),
+ 'help': 'network, default ethereum',
+ },
+ {
+ 'name': ('-s', '--sizes'),
+ 'help': 'sample sizes, one of {XS, S, M, L, XL, all}, default L', # noqa: E501
+ 'nargs': '+',
+ },
+ {
+ 'name': ('-o', '--output-dir'),
+ 'help': 'download location, default FLOOD_SAMPLES_DIR or cwd',
+ },
+ {
+ 'name': ('-d', '--datatypes'),
+ 'help': 'datatypes to sample, default all',
+ },
+ {
+ 'name': ('-m', '--missing'),
+ 'help': 'only download missing files',
+ 'action': 'store_true',
+ },
+ ],
+ }
+
+
+def download_samples_command(
+ network: str | None,
+ sizes: typing.Sequence[str] | None,
+ output_dir: str | None,
+ datatypes: typing.Sequence[str] | None,
+ missing: bool,
+) -> None:
+ if network is None:
+ network = 'ethereum'
+ if sizes is None:
+ sizes = ['L']
+ if sizes == 'all':
+ sizes = list(flood.generators.default_sizes.keys())
+ if output_dir is None:
+ output_dir = flood.generators.get_flood_samples_dir()
+
+ flood.generators.download_raw_data(
+ network=network,
+ sizes=sizes,
+ datatypes=datatypes,
+ output_dir=output_dir,
+ only_missing=missing,
+ )
diff --git a/flood/cli/samples_ls_command.py b/flood/cli/samples_ls_command.py
new file mode 100644
index 0000000..1b23dcf
--- /dev/null
+++ b/flood/cli/samples_ls_command.py
@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import os
+import toolcli
+import toolstr
+
+import flood
+
+
+def get_command_spec() -> toolcli.CommandSpec:
+ return {
+ 'f': samples_ls_command,
+ 'help': 'list raw EVM data samples that are downloaded and downloadable', # noqa: E501
+ 'args': [],
+ }
+
+
+def samples_ls_command() -> None:
+ version = 'v1_0_0'
+
+ # parse contents of flood samples dir
+ samples_dir = flood.generators.get_flood_samples_dir()
+ files_by_network: dict[str, list[str]] = {}
+ for filename in os.listdir(samples_dir):
+ if filename.endswith('parquet'):
+ network = filename.split('_')[0]
+ files_by_network.setdefault(network, [])
+ files_by_network[network].append(filename)
+ for network in list(files_by_network.keys()):
+ files_by_network[network] = sorted(files_by_network[network])
+
+ for network in files_by_network.keys():
+ toolstr.print_text_box(network + ' sample data')
+ print()
+ print(samples_dir)
+ print()
+ toolstr.print_header('Local Files')
+ for filename in files_by_network[network]:
+ print('-', filename)
+
+ missing = []
+ for size in flood.generators.default_sizes:
+ for datatype in flood.generators.default_datatypes:
+ filename = flood.generators.raw_data_file_template.format(
+ network=network,
+ size=size,
+ datatype=datatype,
+ version=version,
+ )
+ if filename not in files_by_network[network]:
+ missing.append(filename)
+
+ if len(missing) > 0:
+ print()
+ toolstr.print_header('Available for download')
+ for filename in missing:
+ print('-', filename)
+ print()
+ print('download using `flood samples download` command')
diff --git a/flood/cli/update_command.py b/flood/cli/update_command.py
new file mode 100644
index 0000000..18b76ea
--- /dev/null
+++ b/flood/cli/update_command.py
@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import typing
+
+import toolcli
+
+import flood
+
+help_text = """update flood version
+
+This command only handles the simplest possible use cases of
+- pip installs
+- git dev installs
+ - installed via `flit install --symlink` or `pip install -e ./`
+
+This command cannot:
+- switch a pip install to a git dev install
+- switch a git dev install to a pip install"""
+
+
+def get_command_spec() -> toolcli.CommandSpec:
+ return {
+ 'f': update_command,
+ 'help': help_text,
+ 'hidden': True,
+ 'args': [
+ {
+ 'name': ('nodes'),
+ 'help': 'hostnames to update (requires ssh)',
+ 'nargs': '*',
+ },
+ {
+ 'name': ('--version'),
+ 'help': 'version to update to',
+ 'nargs': '?',
+ },
+ ],
+ }
+
+
+def update_command(version: str | None, nodes: typing.Sequence[str]) -> None:
+ if nodes is None or len(nodes) == 0:
+ flood.ops.update_local(version=version)
+ else:
+ for node in nodes:
+ flood.ops.update_remote(hostname=node, version=version)
+
diff --git a/flood/cli/version_command.py b/flood/cli/version_command.py
new file mode 100644
index 0000000..f3db0b1
--- /dev/null
+++ b/flood/cli/version_command.py
@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+import toolcli
+
+import flood
+
+
+def get_command_spec() -> toolcli.CommandSpec:
+ return {
+ 'f': version_command,
+ 'help': 'get flood version of local or remote installation',
+ 'args': [
+ {
+ 'name': 'node_url',
+ 'help': 'remote node url, if none use local installation',
+ 'nargs': '?',
+ },
+ {
+ 'name': '--json',
+ 'help': 'output version details as json',
+ 'action': 'store_true',
+ 'dest': 'as_json',
+ },
+ {'name': ('-V', '--version'), 'action': 'store_true'},
+ ],
+ }
+
+
+def version_command(node_url: str | None, as_json: bool, version: bool) -> None:
+ if node_url is None:
+ installation = flood.get_local_installation()
+ else:
+ installation = flood.get_remote_installation(node_url)
+
+ if as_json:
+ import json
+
+ print(json.dumps(installation, indent=4, sort_keys=True))
+ else:
+ print(installation['flood_version'])
+
diff --git a/flood/generators/__init__.py b/flood/generators/__init__.py
new file mode 100644
index 0000000..63a48bb
--- /dev/null
+++ b/flood/generators/__init__.py
@@ -0,0 +1,4 @@
+from .object_generators import *
+from .raw_data_sources import *
+from .rng_utils import *
+from .test_generators import *
diff --git a/rpc_bench/generators/__init__.py b/flood/generators/object_generators/__init__.py
similarity index 86%
rename from rpc_bench/generators/__init__.py
rename to flood/generators/object_generators/__init__.py
index cf30d23..bea7bc9 100644
--- a/rpc_bench/generators/__init__.py
+++ b/flood/generators/object_generators/__init__.py
@@ -2,6 +2,5 @@ from .address_generators import *
from .block_generators import *
from .call_generators import *
from .slot_generators import *
-from .test_generators import *
from .timing_generators import *
from .transaction_generators import *
diff --git a/flood/generators/object_generators/address_generators.py b/flood/generators/object_generators/address_generators.py
new file mode 100644
index 0000000..40c2683
--- /dev/null
+++ b/flood/generators/object_generators/address_generators.py
@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+import typing
+
+import flood
+from flood import generators
+
+
+def generate_contract_addresses(
+ n: int,
+ network: str,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[str]:
+ return generators.load_samples(
+ network=network,
+ datatype='contracts',
+ n=n,
+ random_seed=random_seed,
+ )
+
+
+def generate_eoas(
+ n: int,
+ network: str,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[str]:
+ return generators.load_samples(
+ network=network,
+ datatype='eoas',
+ n=n,
+ random_seed=random_seed,
+ )
diff --git a/rpc_bench/generators/block_generators.py b/flood/generators/object_generators/block_generators.py
similarity index 81%
rename from rpc_bench/generators/block_generators.py
rename to flood/generators/object_generators/block_generators.py
index 0151a95..0806c0f 100644
--- a/rpc_bench/generators/block_generators.py
+++ b/flood/generators/object_generators/block_generators.py
@@ -2,7 +2,8 @@ from __future__ import annotations
import typing
-from rpc_bench import spec
+from flood import spec
+from .. import rng_utils
def generate_block_numbers(
@@ -13,28 +14,29 @@ def generate_block_numbers(
replace: bool = False,
sort: bool = False,
random_seed: spec.RandomSeed | None = None,
+ network: str | None = None,
) -> typing.Sequence[int]:
import numpy as np
# seed a generator
- if random_seed is None:
- random_seed = 0
- if isinstance(random_seed, int):
- gen = np.random.Generator(np.random.PCG64(random_seed))
+ rng = rng_utils.get_rng(random_seed=random_seed)
# generate blocks
all_blocks = np.arange(start_block, end_block + 1)
- chosen = gen.choice(all_blocks, size=n, replace=replace)
+ chosen_array = rng.choice(all_blocks, size=n, replace=(n > len(all_blocks)))
+ chosen: list[int] = chosen_array.tolist()
# sort
if sort:
- return sorted(chosen)
- else:
- return list(chosen)
+ chosen = sorted(chosen)
+
+ return chosen
def generate_block_hashes(
n: int,
+ network: str | None = None,
+ random_seed: spec.RandomSeed | None = None,
) -> typing.Sequence[str]:
raise NotImplementedError()
@@ -50,6 +52,7 @@ def generate_block_ranges(
n_attempts: int = 1_000_000,
random_seed: spec.RandomSeed | None = None,
method: str = 'strides',
+ network: str | None = None,
) -> typing.Sequence[tuple[int, int]]:
if method == 'strides':
return _generate_block_ranges_strides(
@@ -90,18 +93,18 @@ def _generate_block_ranges_strides(
) -> typing.Sequence[tuple[int, int]]:
import numpy as np
- if random_seed is None:
- random_seed = 0
- if isinstance(random_seed, int):
- gen = np.random.Generator(np.random.PCG64(random_seed))
+ # seed a generator
+ rng = rng_utils.get_rng(random_seed=random_seed)
block_ranges: list[tuple[int, int]] = []
while len(block_ranges) < n:
random_phases = list(range(range_size))
- gen.shuffle(random_phases)
+ rng.shuffle(random_phases)
for random_phase in random_phases:
- strides = np.arange(start_block + random_phase, end_block, range_size)
- gen.shuffle(strides)
+ strides = np.arange(
+ start_block + random_phase, end_block, range_size
+ )
+ rng.shuffle(strides)
n_missing = n - len(block_ranges)
if n_missing > 0:
for stride in strides[:n_missing]:
@@ -123,13 +126,8 @@ def _generate_block_ranges_individual(
n_attempts: int = 1_000_000,
random_seed: spec.RandomSeed | None = None,
) -> typing.Sequence[tuple[int, int]]:
- import numpy as np
-
# seed a generator
- if random_seed is None:
- random_seed = 0
- if isinstance(random_seed, int):
- gen = np.random.Generator(np.random.PCG64(random_seed))
+ rng = rng_utils.get_rng(random_seed=random_seed)
# create starting sample
start_blocks = generate_block_numbers(
@@ -157,7 +155,7 @@ def _generate_block_ranges_individual(
try:
start = next(candidates)
except StopIteration:
- start = gen.integers(start_block, end_block)
+ start = rng.integers(start_block, end_block)
end = start + range_size
candidate = (start, end)
@@ -182,6 +180,5 @@ def _generate_block_ranges_individual(
return sorted(ranges)
else:
ranges_list = list(ranges)
- gen.shuffle(ranges_list)
+ rng.shuffle(ranges_list)
return ranges_list
-
diff --git a/flood/generators/object_generators/call_generators.py b/flood/generators/object_generators/call_generators.py
new file mode 100644
index 0000000..c8b5dda
--- /dev/null
+++ b/flood/generators/object_generators/call_generators.py
@@ -0,0 +1,631 @@
+from __future__ import annotations
+
+import typing
+
+import flood
+from flood import generators
+from . import address_generators
+from . import block_generators
+from . import slot_generators
+from . import transaction_generators
+
+
+#
+# # blocks
+#
+
+
+def generate_calls_eth_get_block_by_number(
+ n_calls: int | None = None,
+ *,
+ network: str | None = None,
+ block_numbers: typing.Sequence[int] | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.Call]:
+ import ctc.rpc
+
+ if block_numbers is None:
+ if n_calls is None:
+ raise Exception('must floodify more parameters')
+ block_numbers = block_generators.generate_block_numbers(
+ n=n_calls,
+ random_seed=random_seed,
+ start_block=0,
+ end_block=16_000_000,
+ network=network,
+ )
+ return [
+ ctc.rpc.construct_eth_get_block_by_number(block_number=block_number)
+ for block_number in block_numbers
+ ]
+
+
+def generate_calls_eth_get_block_by_hash(
+ n_calls: int | None = None,
+ *,
+ network: str | None = None,
+ block_hashes: typing.Sequence[str] | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.Call]:
+ import ctc.rpc
+
+ if block_hashes is None:
+ if n_calls is None:
+ raise Exception('must floodify more parameters')
+ block_hashes = block_generators.generate_block_hashes(
+ n=n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+ return [
+ ctc.rpc.construct_eth_get_block_by_hash(block_hash=block_hash)
+ for block_hash in block_hashes
+ ]
+
+
+def generate_calls_eth_fee_history(
+ n_calls: int | None = None,
+ *,
+ network: str | None = None,
+ random_seed: flood.RandomSeed | None = None,
+ block_numbers: typing.Sequence[int] | None = None,
+ block_count: int | None = None
+) -> typing.Sequence[flood.Call]:
+ import ctc.rpc
+
+ if block_numbers is None:
+ if n_calls is None:
+ raise Exception('must floodify more parameters')
+ block_numbers = block_generators.generate_block_numbers(
+ n=n_calls,
+ random_seed=random_seed,
+ start_block=13_000_000,
+ end_block=17_000_000,
+ network=network,
+ )
+ if block_count is None:
+ block_count = 1024
+
+ return [
+ ctc.rpc.construct_eth_fee_history(
+ block_number,
+ block_count=block_count,
+ )
+ for block_number in block_numbers
+ ]
+
+
+#
+# # addresses
+#
+
+
+def generate_calls_eth_get_eth_balance(
+ n_calls: int | None = None,
+ *,
+ network: str,
+ addresses: typing.Sequence[str] | None = None,
+ block_numbers: typing.Sequence[int] | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.Call]:
+ import ctc.rpc
+
+ if block_numbers is None:
+ if n_calls is None:
+ raise Exception('must floodify more parameters')
+ block_numbers = block_generators.generate_block_numbers(
+ start_block=10_000_000,
+ end_block=16_000_000,
+ n=n_calls,
+ random_seed=random_seed,
+ network=network,
+ )
+ if addresses is None:
+ if n_calls is None:
+ raise Exception('must floodify more parameters')
+ addresses = address_generators.generate_contract_addresses(
+ n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+
+ return [
+ ctc.rpc.construct_eth_get_balance(
+ address=address, block_number=block_number
+ )
+ for address, block_number in zip(addresses, block_numbers)
+ ]
+
+
+def generate_calls_eth_get_transaction_count(
+ n_calls: int | None = None,
+ *,
+ network: str,
+ addresses: typing.Sequence[str] | None = None,
+ block_numbers: typing.Sequence[int] | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.Call]:
+ import ctc.rpc
+
+ if block_numbers is None:
+ if n_calls is None:
+ raise Exception('must floodify more parameters')
+ block_numbers = block_generators.generate_block_numbers(
+ start_block=10_000_000,
+ end_block=16_000_000,
+ n=n_calls,
+ random_seed=random_seed,
+ network=network,
+ )
+ if addresses is None:
+ if n_calls is None:
+ raise Exception('must floodify more parameters')
+ addresses = address_generators.generate_eoas(
+ n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+
+ return [
+ ctc.rpc.construct_eth_get_transaction_count(
+ from_address=address, block_number=block_number
+ )
+ for address, block_number in zip(addresses, block_numbers)
+ ]
+
+
+#
+# # transactions
+#
+
+
+def generate_calls_eth_get_transaction_by_hash(
+ n_calls: int | None = None,
+ *,
+ network: str,
+ transaction_hashes: typing.Sequence[str] | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.Call]:
+ import ctc.rpc
+
+ if transaction_hashes is None:
+ if n_calls is None:
+ raise Exception('must floodify more parameters')
+ transaction_hashes = transaction_generators.generate_transaction_hashes(
+ n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+ return [
+ ctc.rpc.construct_eth_get_transaction_by_hash(
+ transaction_hash=transaction_hash
+ )
+ for transaction_hash in transaction_hashes
+ ]
+
+
+def generate_calls_eth_get_transaction_receipt(
+ n_calls: int | None = None,
+ *,
+ network: str,
+ transaction_hashes: typing.Sequence[str] | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.Call]:
+ import ctc.rpc
+
+ if transaction_hashes is None:
+ if n_calls is None:
+ raise Exception('must floodify more parameters')
+ transaction_hashes = transaction_generators.generate_transaction_hashes(
+ n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+ return [
+ ctc.rpc.construct_eth_get_transaction_receipt(
+ transaction_hash=transaction_hash
+ )
+ for transaction_hash in transaction_hashes
+ ]
+
+
+#
+# # logs
+#
+
+_default_contracts = {
+ 'USDC': '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48',
+ 'DAI': '0x6b175474e89094c44da98b954eedeac495271d0f',
+ 'LUSD': '0x5f98805a4e8be255a32880fdec7f6728c6568ba0',
+}
+
+_default_event_hashes = {
+ 'Transfer': '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', # noqa: E501
+ 'Approval': '0x8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925', # noqa: E501
+}
+
+
+def generate_calls_eth_get_logs(
+ n_calls: int | None = None,
+ *,
+ contract_address: str | None = None,
+ topics: typing.Sequence[str | None] | None = None,
+ block_ranges: typing.Sequence[tuple[int, int]] | None = None,
+ block_range_size: int | None = None,
+ network: str | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.Call]:
+ import ctc.rpc
+
+ if contract_address is None:
+ contract_address = _default_contracts['USDC']
+ if block_ranges is None:
+ if n_calls is None:
+ raise Exception('must floodify more parameters')
+ if block_range_size is None:
+ block_range_size = 100
+ block_ranges = block_generators.generate_block_ranges(
+ start_block=10_000_000,
+ end_block=16_000_000,
+ n=n_calls,
+ range_size=block_range_size,
+ random_seed=random_seed,
+ network=network,
+ )
+ if topics is None:
+ topics = [_default_event_hashes['Transfer']]
+ return [
+ ctc.rpc.construct_eth_get_logs(
+ address=contract_address,
+ topics=topics,
+ start_block=start_block,
+ end_block=end_block,
+ )
+ for start_block, end_block in block_ranges
+ ]
+
+
+#
+# # contracts
+#
+
+
+def generate_calls_eth_get_code(
+ n_calls: int | None = None,
+ *,
+ network: str,
+ addresses: typing.Sequence[str] | None = None,
+ block_numbers: typing.Sequence[int | typing.Literal['latest']]
+ | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.Call]:
+ import ctc.rpc
+
+ if block_numbers is None:
+ if n_calls is None:
+ raise Exception('must floodify more parameters')
+ block_numbers = block_generators.generate_block_numbers(
+ start_block=10_000_000,
+ end_block=16_000_000,
+ n=n_calls,
+ random_seed=random_seed,
+ network=network,
+ )
+ if addresses is None:
+ if n_calls is None:
+ raise Exception('must floodify more parameters')
+ addresses = address_generators.generate_contract_addresses(
+ n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+ return [
+ ctc.rpc.construct_eth_get_code(
+ address=address, block_number=block_number
+ )
+ for address, block_number in zip(addresses, block_numbers)
+ ]
+
+
+def generate_calls_eth_get_storage_at(
+ n_calls: int | None = None,
+ *,
+ network: str,
+ slots: typing.Sequence[tuple[str, str]] | None = None,
+ block_numbers: typing.Sequence[int | typing.Literal['latest']]
+ | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.Call]:
+ import ctc.rpc
+
+ if block_numbers is None:
+ if n_calls is None:
+ raise Exception('must floodify more parameters')
+ block_numbers = block_generators.generate_block_numbers(
+ start_block=10_000_000,
+ end_block=16_000_000,
+ n=n_calls,
+ random_seed=random_seed,
+ network=network,
+ )
+ if slots is None:
+ if n_calls is None:
+ raise Exception('must floodify more parameters')
+ slots = slot_generators.generate_slots(
+ n_calls, network=network, random_seed=random_seed
+ )
+ return [
+ ctc.rpc.construct_eth_get_storage_at(
+ address=address, position=slot, block_number=block_number
+ )
+ for (address, slot), block_number in zip(slots, block_numbers)
+ ]
+
+
+_default_call_datas = {
+ 'decimals': '0x313ce567',
+ 'totalSupply': '0x18160ddd',
+ 'symbol': '0x95d89b41',
+}
+
+
+def generate_calls_eth_call(
+ n_calls: int,
+ network: str,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.Call]:
+ import ctc.rpc
+
+ if network != 'ethereum':
+ raise Exception('only ethereum supported for eth_call')
+
+ rng = generators.get_rng(random_seed=random_seed)
+ contract_addresses = rng.choice(
+ list(_default_contracts.values()),
+ size=n_calls,
+ )
+ call_datas = rng.choice(
+ list(_default_call_datas.values()),
+ size=n_calls,
+ )
+ block_numbers = block_generators.generate_block_numbers(
+ start_block=10_000_000,
+ end_block=16_000_000,
+ n=n_calls,
+ random_seed=random_seed,
+ network=network,
+ )
+
+ return [
+ ctc.rpc.construct_eth_call(
+ to_address=contract_address,
+ call_data=call_data,
+ block_number=block_number,
+ )
+ for contract_address, call_data, block_number in zip(
+ contract_addresses, call_datas, block_numbers
+ )
+ ]
+
+
+#
+# # traces
+#
+
+
+def generate_calls_trace_block(
+ n_calls: int | None = None,
+ *,
+ block_numbers: typing.Sequence[int] | None = None,
+ network: str | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.Call]:
+ import ctc.rpc
+
+ if block_numbers is None:
+ if n_calls is None:
+ raise Exception('must floodify more parameters')
+ block_numbers = block_generators.generate_block_numbers(
+ n=n_calls,
+ random_seed=0,
+ start_block=0,
+ end_block=16_000_000,
+ network=network,
+ )
+ return [
+ ctc.rpc.construct_trace_block(block_number=block_number)
+ for block_number in block_numbers
+ ]
+
+
+def generate_calls_trace_transaction(
+ n_calls: int | None = None,
+ *,
+ transaction_hashes: typing.Sequence[str] | None = None,
+ network: str | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.Call]:
+ import ctc.rpc
+
+ if transaction_hashes is None:
+ if n_calls is None:
+ raise Exception('must floodify more parameters')
+ if network is None:
+ raise Exception('must floodify network')
+ transaction_hashes = transaction_generators.generate_transaction_hashes(
+ n=n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+ return [
+ ctc.rpc.construct_trace_transaction(transaction_hash=transaction_hash)
+ for transaction_hash in transaction_hashes
+ ]
+
+
+def generate_calls_trace_replay_block_transactions(
+ n_calls: int | None = None,
+ *,
+ block_numbers: typing.Sequence[int] | None = None,
+ network: str | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.Call]:
+ import ctc.rpc
+
+ if block_numbers is None:
+ if n_calls is None:
+ raise Exception('must floodify more parameters')
+ block_numbers = block_generators.generate_block_numbers(
+ n=n_calls,
+ random_seed=random_seed,
+ start_block=0,
+ end_block=16_000_000,
+ network=network,
+ )
+ return [
+ ctc.rpc.construct_trace_replay_block_transactions(
+ block_number=block_number,
+ trace_type=['trace'],
+ )
+ for block_number in block_numbers
+ ]
+
+
+def generate_calls_trace_replay_block_transactions_state_diff(
+ n_calls: int | None = None,
+ *,
+ block_numbers: typing.Sequence[int] | None = None,
+ network: str | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.Call]:
+ import ctc.rpc
+
+ if block_numbers is None:
+ if n_calls is None:
+ raise Exception('must floodify more parameters')
+ block_numbers = block_generators.generate_block_numbers(
+ n=n_calls,
+ random_seed=random_seed,
+ start_block=0,
+ end_block=16_000_000,
+ network=network,
+ )
+ return [
+ ctc.rpc.construct_trace_replay_block_transactions(
+ block_number=block_number,
+ trace_type=['stateDiff'],
+ )
+ for block_number in block_numbers
+ ]
+
+
+def generate_calls_trace_replay_block_transactions_vm_trace(
+ n_calls: int | None = None,
+ *,
+ block_numbers: typing.Sequence[int] | None = None,
+ network: str | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.Call]:
+ import ctc.rpc
+
+ if block_numbers is None:
+ if n_calls is None:
+ raise Exception('must floodify more parameters')
+ block_numbers = block_generators.generate_block_numbers(
+ n=n_calls,
+ random_seed=random_seed,
+ start_block=0,
+ end_block=16_000_000,
+ network=network,
+ )
+ return [
+ ctc.rpc.construct_trace_replay_block_transactions(
+ block_number=block_number,
+ trace_type=['vmTrace'],
+ )
+ for block_number in block_numbers
+ ]
+
+
+def generate_calls_trace_replay_transaction(
+ n_calls: int | None = None,
+ *,
+ transaction_hashes: typing.Sequence[str] | None = None,
+ network: str | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.Call]:
+ import ctc.rpc
+
+ if transaction_hashes is None:
+ if n_calls is None:
+ raise Exception('must floodify more parameters')
+ if network is None:
+ raise Exception('must floodify network')
+ transaction_hashes = transaction_generators.generate_transaction_hashes(
+ n=n_calls,
+ random_seed=random_seed,
+ network=network,
+ )
+ return [
+ ctc.rpc.construct_trace_replay_transaction(
+ transaction_hash=transaction_hash,
+ trace_type=['trace'],
+ )
+ for transaction_hash in transaction_hashes
+ ]
+
+
+def generate_calls_trace_replay_transaction_state_diff(
+ n_calls: int | None = None,
+ *,
+ transaction_hashes: typing.Sequence[str] | None = None,
+ network: str | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.Call]:
+ import ctc.rpc
+
+ if transaction_hashes is None:
+ if n_calls is None:
+ raise Exception('must floodify more parameters')
+ if network is None:
+ raise Exception('must floodify network')
+ transaction_hashes = transaction_generators.generate_transaction_hashes(
+ n=n_calls,
+ random_seed=random_seed,
+ network=network,
+ )
+ return [
+ ctc.rpc.construct_trace_replay_transaction(
+ transaction_hash=transaction_hash,
+ trace_type=['stateDiff'],
+ )
+ for transaction_hash in transaction_hashes
+ ]
+
+
+def generate_calls_trace_replay_transaction_vm_trace(
+ n_calls: int | None = None,
+ *,
+ transaction_hashes: typing.Sequence[str] | None = None,
+ network: str | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.Call]:
+ import ctc.rpc
+
+ if transaction_hashes is None:
+ if n_calls is None:
+ raise Exception('must floodify more parameters')
+ if network is None:
+ raise Exception('must floodify network')
+ transaction_hashes = transaction_generators.generate_transaction_hashes(
+ n=n_calls,
+ random_seed=random_seed,
+ network=network,
+ )
+ return [
+ ctc.rpc.construct_trace_replay_transaction(
+ transaction_hash=transaction_hash,
+ trace_type=['vmTrace'],
+ )
+ for transaction_hash in transaction_hashes
+ ]
diff --git a/flood/generators/object_generators/slot_generators.py b/flood/generators/object_generators/slot_generators.py
new file mode 100644
index 0000000..a811a06
--- /dev/null
+++ b/flood/generators/object_generators/slot_generators.py
@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+import typing
+import flood
+from flood import generators
+
+
+def generate_slots(
+ n: int,
+ network: str,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[tuple[str, str]]:
+ return generators.load_samples(
+ network=network,
+ datatype='slots',
+ n=n,
+ random_seed=random_seed,
+ )
diff --git a/rpc_bench/generators/timing_generators.py b/flood/generators/object_generators/timing_generators.py
similarity index 96%
rename from rpc_bench/generators/timing_generators.py
rename to flood/generators/object_generators/timing_generators.py
index be87148..219c2f2 100644
--- a/rpc_bench/generators/timing_generators.py
+++ b/flood/generators/object_generators/timing_generators.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import typing
-import rpc_bench
+import flood
default_stress_test_rates = [1, 4, 16, 64, 256, 512]
@@ -25,7 +25,7 @@ def generate_timings(
rates: typing.Sequence[int] | None = None,
duration: int | None = None,
durations: typing.Sequence[int] | None = None,
- mode: rpc_bench.LoadTestMode | None = None,
+ mode: flood.LoadTestMode | None = None,
) -> tuple[typing.Sequence[int], typing.Sequence[int]]:
"""create rates and durations for test"""
@@ -90,7 +90,7 @@ def _generate_timings_for_spike_test(
pass
else:
raise Exception(
- 'must specify eith a single spike rate or a [before, during, after] triplet of rates'
+ 'must specify eith a single spike rate or a [before, during, after] triplet of rates' # noqa: E501
)
if duration is not None:
@@ -110,7 +110,7 @@ def _generate_timings_for_spike_test(
pass
else:
raise Exception(
- 'must specify eith a single spike duration or a [before, during, after] triplet of rates'
+ 'must specify eith a single spike duration or a [before, during, after] triplet of rates' # noqa: E501
)
else:
durations = [
@@ -143,4 +143,3 @@ def _generate_timings_for_soak_test(
raise Exception('must specify 1 duration for soak test')
return rates, durations
-
diff --git a/flood/generators/object_generators/transaction_generators.py b/flood/generators/object_generators/transaction_generators.py
new file mode 100644
index 0000000..397e5fb
--- /dev/null
+++ b/flood/generators/object_generators/transaction_generators.py
@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+import typing
+
+import flood
+from flood import generators
+
+
+def generate_transaction_hashes(
+ n: int,
+ network: str,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[str]:
+ return generators.load_samples(
+ network=network,
+ datatype='transactions',
+ n=n,
+ random_seed=random_seed,
+ )
diff --git a/flood/generators/raw_data_sources/__init__.py b/flood/generators/raw_data_sources/__init__.py
new file mode 100644
index 0000000..f800204
--- /dev/null
+++ b/flood/generators/raw_data_sources/__init__.py
@@ -0,0 +1,4 @@
+from .raw_data_spec import *
+from .raw_download_utils import *
+from .raw_gather_utils import *
+from .raw_sample_loading import *
diff --git a/flood/generators/raw_data_sources/raw_data_spec.py b/flood/generators/raw_data_sources/raw_data_spec.py
new file mode 100644
index 0000000..079fdda
--- /dev/null
+++ b/flood/generators/raw_data_sources/raw_data_spec.py
@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+
+raw_data_version = 'v1_0_0'
+
+raw_data_url_template = (
+ 'https://datasets.paradigm.xyz/evm_samples/{network}_samples__{version}/'
+)
+raw_data_file_template = (
+ '{network}_{datatype}_samples__{size}__{version}.parquet'
+)
+
+default_sizes = {
+ 'XS': 1_000,
+ 'S': 10_000,
+ 'M': 100_000,
+ 'L': 1_000_000,
+ 'XL': 10_000_000,
+}
+
+default_datatypes = [
+ 'contracts',
+ 'eoas',
+ 'transactions',
+ 'slots',
+]
diff --git a/flood/generators/raw_data_sources/raw_download_utils.py b/flood/generators/raw_data_sources/raw_download_utils.py
new file mode 100644
index 0000000..d0f7585
--- /dev/null
+++ b/flood/generators/raw_data_sources/raw_download_utils.py
@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+import typing
+
+from . import raw_data_spec
+
+
+def download_raw_data(
+ *,
+ network: str,
+ sizes: str | typing.Sequence[str],
+ datatypes: typing.Sequence[str] | str | None = None,
+ version: str | None = None,
+ output_dir: str | None = None,
+ only_missing: bool = False,
+ verbose: bool = True,
+) -> None:
+ import os
+ import pdp
+
+ # process inputs
+ if version is None:
+ version = 'v1_0_0'
+ if output_dir is None:
+ output_dir = get_flood_samples_dir()
+ if datatypes is None:
+ datatypes = raw_data_spec.default_datatypes
+ if isinstance(datatypes, str):
+ datatypes = [datatypes]
+ if isinstance(sizes, str):
+ sizes = [sizes]
+ elif isinstance(sizes, list):
+ pass
+ elif isinstance(sizes, dict):
+ sizes = list(sizes.keys())
+ else:
+ raise Exception()
+ url_root = raw_data_spec.raw_data_url_template.format(
+ network=network, version=version
+ )
+
+ # download each datatype
+ for datatype in datatypes:
+ for size in sizes:
+ filename = raw_data_spec.raw_data_file_template.format(
+ datatype=datatype,
+ network=network,
+ size=size,
+ version=version,
+ )
+ output_path = os.path.join(output_dir, filename)
+ url = url_root + filename
+
+ if only_missing and os.path.isfile(output_path):
+ continue
+
+ print('downloading', filename)
+ pdp.download_file(url=url, output_path=output_path)
+
+
+def get_flood_samples_dir() -> str:
+ import os
+
+ flood_samples_dir = os.environ.get('FLOOD_SAMPLES_DIR')
+ if flood_samples_dir is None:
+ import warnings
+
+ warnings.warn('FLOOD_SAMPLES_DIR not set in env, defaulting to CWD')
+ return os.path.join(os.getcwd(), 'flood_samples')
+ else:
+ return flood_samples_dir
diff --git a/flood/generators/raw_data_sources/raw_gather_utils.py b/flood/generators/raw_data_sources/raw_gather_utils.py
new file mode 100644
index 0000000..21c6d42
--- /dev/null
+++ b/flood/generators/raw_data_sources/raw_gather_utils.py
@@ -0,0 +1,201 @@
+from __future__ import annotations
+
+import typing
+
+from . import raw_data_spec
+
+if typing.TYPE_CHECKING:
+ import polars as pl
+
+
+def create_samples_dataset(
+ *,
+ output_dir: str,
+ network: str,
+ version: str | None = None,
+ sizes: typing.Mapping[str, int] | typing.Sequence[str] | None = None,
+ datatypes: typing.Sequence[str] | None = None,
+ verbose: bool = True,
+) -> None:
+ # parse inputs
+ if version is None:
+ version = 'v1_0_0'
+ if isinstance(sizes, list):
+ sizes = {
+ k: v for k, v in raw_data_spec.default_sizes.items() if k in sizes
+ }
+ elif isinstance(sizes, dict):
+ pass
+ elif sizes is None:
+ sizes = raw_data_spec.default_sizes
+ else:
+ raise Exception('invalid sizes format: ' + str(type(sizes)))
+ if datatypes is None:
+ datatypes = raw_data_spec.default_datatypes
+ max_size = max(sizes.values())
+
+ # print summary
+ if verbose:
+ print('creating dataset of raw EVM samples')
+ print('- version:', version)
+ print('- datatypes:', datatypes)
+ print('- network:', network)
+ print('- sizes:', sizes)
+ if output_dir is not None:
+ print('- saving to:', output_dir)
+ print()
+ print('collecting samples...')
+
+ # collect samples
+ samples = _create_raw_samples(
+ n=max_size,
+ network='ethereum',
+ datatypes=datatypes,
+ )
+
+ # write sample files
+ if verbose:
+ print('writing samples to disk...')
+ _write_raw_samples(
+ samples=samples,
+ output_dir=output_dir,
+ sizes=sizes,
+ network=network,
+ version=version,
+ )
+
+ if verbose:
+ print('sample dataset completed')
+
+
+def _create_raw_samples(
+ n: int,
+ network: str,
+ datatypes: typing.Sequence[str] | None = None,
+) -> typing.Mapping[str, pl.DataFrame]:
+ import glob
+ import pdp
+ from pdp.datasets import contracts
+ import polars as pl
+
+ if datatypes is None:
+ datatypes = raw_data_spec.default_datatypes
+
+ samples = {}
+
+ # contracts
+ if 'contracts' in datatypes:
+ contracts_df = contracts.query_contracts(
+ columns=['contract_address', 'block_number', 'deployer'],
+ network=network,
+ ).sort('block_number')
+ samples['contracts'] = contracts_df[
+ ['contract_address', 'block_number']
+ ].sample(n=n)
+
+ # slots
+ if 'slots' in datatypes:
+ slot_glob = pdp.get_dataset_glob(
+ network=network,
+ datatype='slots',
+ )
+ pieces = []
+ for file in glob.glob(slot_glob):
+ piece = (
+ pl.scan_parquet(file)
+ .select('contract_address', 'slot', 'first_updated_block')
+ .collect()
+ .sample(int(n / 10))
+ )
+ pieces.append(piece)
+ samples['slots'] = (
+ pl.concat(pieces)
+ .rename({'first_updated_block': 'block_number'})
+ .sample(n)
+ )
+
+ # transactions
+ if 'transactions' in datatypes:
+ native_transfers_glob = pdp.get_dataset_glob(
+ network=network,
+ datatype='native_transfers',
+ )
+ pieces = []
+ for file in glob.glob(native_transfers_glob):
+ piece = (
+ pl.scan_parquet(file)
+ .select('transaction_hash', 'block_number')
+ .collect()
+ )
+ if len(piece) > n / 50:
+ piece = piece.sample(int(n / 50))
+ pieces.append(piece)
+ samples['transactions'] = pl.concat(pieces).sample(n)
+
+ # eoas
+ if 'eoas' in datatypes:
+ native_transfers_glob = pdp.get_dataset_glob(
+ network=network,
+ datatype='native_transfers',
+ )
+ pieces = []
+ for file in glob.glob(native_transfers_glob):
+ piece = (
+ pl.scan_parquet(file)
+ .select('from_address', 'block_number')
+ .unique('from_address', keep='first')
+ .collect()
+ )
+ if len(piece) > n / 50:
+ piece = piece.sample(int(n / 50))
+ pieces.append(piece)
+ eoas_df = (
+ pl.concat(pieces)
+ .filter(
+ ~pl.col('from_address').is_in(contracts_df['contract_address'])
+ )
+ .unique('from_address')
+ .rename({'from_address': 'eoa'})
+ )
+ samples['eoas'] = eoas_df.sample(n)
+
+ return samples
+
+
+def _write_raw_samples(
+ samples: typing.Mapping[str, pl.DataFrame],
+ *,
+ output_dir: str,
+ network: str,
+ version: str,
+ sizes: typing.Mapping[str, int] | None = None,
+) -> None:
+ import os
+
+ path_template = os.path.join(
+ output_dir, raw_data_spec.raw_data_file_template
+ )
+ if sizes is None:
+ sizes = raw_data_spec.default_sizes
+
+ for sample_name, sample_data in samples.items():
+ for size_name, size in sizes.items():
+ # get output path
+ output_path = path_template.format(
+ network=network,
+ datatype=sample_name,
+ size=size_name,
+ version=version,
+ )
+ os.makedirs(os.path.dirname(output_path), exist_ok=True)
+
+ # shrink sample to size
+ if len(sample_data) == size:
+ sized_data = sample_data
+ elif len(sample_data) > size:
+ sized_data = sample_data[:size]
+ else:
+ raise Exception('desired size is too big for sample data')
+
+ # write file
+ sized_data.write_parquet(output_path)
diff --git a/flood/generators/raw_data_sources/raw_sample_loading.py b/flood/generators/raw_data_sources/raw_sample_loading.py
new file mode 100644
index 0000000..7b011ec
--- /dev/null
+++ b/flood/generators/raw_data_sources/raw_sample_loading.py
@@ -0,0 +1,169 @@
+from __future__ import annotations
+
+import typing
+
+import flood
+from flood import generators
+from . import raw_data_spec
+from . import raw_download_utils
+
+if typing.TYPE_CHECKING:
+ import polars as pl
+
+
+def get_raw_samples_path(
+ network: str,
+ datatype: str,
+ *,
+ size: str | None = None,
+ version: str = raw_data_spec.raw_data_version,
+ samples_dir: str | None = None,
+) -> str | None:
+ import os
+
+ if samples_dir is None:
+ samples_dir = generators.get_flood_samples_dir()
+
+ # get largest_available file present
+ if size is None:
+ size = 'largest_available'
+ if size == 'largest_available':
+ import glob
+
+ path_glob = raw_data_spec.raw_data_file_template.format(
+ network=network,
+ datatype=datatype,
+ size='*',
+ version=version,
+ )
+ filepaths = glob.glob(os.path.join(samples_dir, path_glob))
+ if len(filepaths) > 0:
+ sizes = [(os.stat(path).st_size, path) for path in filepaths]
+ return max(sizes)[1]
+ else:
+ return None
+
+ else:
+ filename = raw_data_spec.raw_data_file_template.format(
+ network=network,
+ datatype=datatype,
+ size=size,
+ version=version,
+ )
+ return os.path.join(samples_dir, filename)
+
+
+def load_raw_samples(
+ network: str,
+ datatype: str,
+ *,
+ size: str | None = None,
+ version: str = raw_data_spec.raw_data_version,
+ samples_dir: str | None = None,
+ download_missing: str | None = 'L',
+) -> pl.DataFrame:
+ path = get_raw_samples_path(
+ datatype=datatype,
+ network=network,
+ size=size,
+ version=version,
+ samples_dir=samples_dir,
+ )
+ if path is None:
+ if download_missing is not None:
+ if size is None:
+ size = 'L'
+ raw_download_utils.download_raw_data(
+ network=network,
+ datatypes=[datatype],
+ version=version,
+ output_dir=samples_dir,
+ sizes=[download_missing],
+ )
+ path = get_raw_samples_path(
+ datatype=datatype,
+ network=network,
+ size=size,
+ version=version,
+ samples_dir=samples_dir,
+ )
+ if path is None:
+ raise Exception('could not download necessary data')
+ else:
+ raise Exception('no raw samples found to load')
+ return pl.read_parquet(path)
+
+
+def load_samples(
+ network: str,
+ datatype: str,
+ n: int,
+ *,
+ size: str | None = None,
+ version: str = raw_data_spec.raw_data_version,
+ samples_dir: str | None = None,
+ binary_convert: bool = True,
+ download_missing: bool = True,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[typing.Any]:
+ import polars as pl
+
+ # get path
+ path = get_raw_samples_path(
+ datatype=datatype,
+ network=network,
+ size=size,
+ version=version,
+ samples_dir=samples_dir,
+ )
+ if path is None:
+ # download data if need be
+ if download_missing:
+ if size is None:
+ size = 'L'
+ raw_download_utils.download_raw_data(
+ network=network,
+ datatypes=[datatype],
+ version=version,
+ output_dir=samples_dir,
+ sizes=[size],
+ )
+ path = get_raw_samples_path(
+ datatype=datatype,
+ network=network,
+ size=size,
+ version=version,
+ samples_dir=samples_dir,
+ )
+ if path is None:
+ raise Exception('could not download necessary data')
+ else:
+ raise Exception('no raw samples found to load')
+
+ columns = {
+ 'contracts': ['contract_address'],
+ 'eoas': ['eoa'],
+ 'transactions': ['transaction_hash'],
+ 'slots': ['contract_address', 'slot'],
+ }[datatype]
+
+ df = pl.scan_parquet(path).select(columns).collect()
+ if n > len(df):
+ import math
+
+ n_copies = math.ceil(n / len(df))
+ df = pl.concat(n_copies * [df])
+ if n < len(df):
+ rng = generators.get_rng(random_seed=random_seed)
+ seed = rng.integers(1_000_000_000, size=1)[0]
+ df = df.sample(n, shuffle=True, seed=seed)
+
+ for column in df.select(pl.col(pl.Binary)).columns:
+ df = df.with_columns(
+ ('0x' + pl.col(column).bin.encode('hex')).alias(column)
+ )
+
+ if len(columns) == 1:
+ return df[columns[0]].to_list()
+ else:
+ return df.rows()
diff --git a/flood/generators/rng_utils.py b/flood/generators/rng_utils.py
new file mode 100644
index 0000000..a2b2949
--- /dev/null
+++ b/flood/generators/rng_utils.py
@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+import typing
+
+from flood import spec
+
+if typing.TYPE_CHECKING:
+ import numpy as np
+
+
+def get_rng(random_seed: spec.RandomSeed | None = None) -> np.random.Generator:
+ import numpy as np
+
+ if random_seed is None:
+ import time
+
+ random_seed = int(time.time())
+ if isinstance(random_seed, int):
+ gen = np.random.Generator(np.random.PCG64(random_seed))
+ if isinstance(gen, np.random.Generator):
+ return gen
+ else:
+ raise Exception('invalid seed format: ' + str(type(random_seed)))
diff --git a/rpc_bench/generators/test_generators/__init__.py b/flood/generators/test_generators/__init__.py
similarity index 75%
rename from rpc_bench/generators/test_generators/__init__.py
rename to flood/generators/test_generators/__init__.py
index 52c0262..0359bf3 100644
--- a/rpc_bench/generators/test_generators/__init__.py
+++ b/flood/generators/test_generators/__init__.py
@@ -3,4 +3,6 @@ from .block_test_generators import *
from .contract_test_generators import *
from .generic_test_generators import *
from .log_test_generators import *
+from .multi_test_generators import *
from .transaction_test_generators import *
+from .trace_test_generators import *
diff --git a/flood/generators/test_generators/address_test_generators.py b/flood/generators/test_generators/address_test_generators.py
new file mode 100644
index 0000000..10e23cf
--- /dev/null
+++ b/flood/generators/test_generators/address_test_generators.py
@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+import typing
+
+import flood
+from flood.tests import load_tests
+
+
+def generate_test_eth_get_balance(
+ *,
+ rates: typing.Sequence[int],
+ duration: int | None = None,
+ durations: typing.Sequence[int] | None = None,
+ network: str,
+ vegeta_args: flood.VegetaArgsShorthand | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.VegetaAttack]:
+ n_calls = load_tests.estimate_call_count(
+ rates=rates, duration=duration, durations=durations
+ )
+ calls = flood.generators.generate_calls_eth_get_eth_balance(
+ n_calls=n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+ return load_tests.create_load_test(
+ calls=calls,
+ rates=rates,
+ duration=duration,
+ durations=durations,
+ vegeta_args=vegeta_args,
+ )
+
+
+def generate_test_eth_get_transaction_count(
+ *,
+ rates: typing.Sequence[int],
+ duration: int | None = None,
+ durations: typing.Sequence[int] | None = None,
+ network: str,
+ vegeta_args: flood.VegetaArgsShorthand | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.VegetaAttack]:
+ n_calls = load_tests.estimate_call_count(
+ rates=rates, duration=duration, durations=durations
+ )
+ calls = flood.generators.generate_calls_eth_get_transaction_count(
+ n_calls=n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+ return load_tests.create_load_test(
+ calls=calls,
+ rates=rates,
+ duration=duration,
+ durations=durations,
+ vegeta_args=vegeta_args,
+ )
diff --git a/flood/generators/test_generators/block_test_generators.py b/flood/generators/test_generators/block_test_generators.py
new file mode 100644
index 0000000..988324b
--- /dev/null
+++ b/flood/generators/test_generators/block_test_generators.py
@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+import typing
+
+from flood import spec
+from flood.tests import load_tests
+import flood
+
+
+def generate_test_eth_get_block_by_number(
+ *,
+ rates: typing.Sequence[int],
+ duration: int | None = None,
+ durations: typing.Sequence[int] | None = None,
+ network: str,
+ vegeta_args: flood.VegetaArgsShorthand | None = None,
+ random_seed: spec.RandomSeed | None = None,
+) -> typing.Sequence[flood.VegetaAttack]:
+ n_calls = load_tests.estimate_call_count(
+ rates=rates, duration=duration, durations=durations
+ )
+ calls = flood.generators.generate_calls_eth_get_block_by_number(
+ n_calls=n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+ return load_tests.create_load_test(
+ calls=calls,
+ rates=rates,
+ duration=duration,
+ durations=durations,
+ vegeta_args=vegeta_args,
+ )
+
+
+def generate_test_eth_fee_history(
+ *,
+ rates: typing.Sequence[int],
+ duration: int | None = None,
+ durations: typing.Sequence[int] | None = None,
+ network: str,
+ vegeta_args: flood.VegetaArgsShorthand | None = None,
+ random_seed: spec.RandomSeed | None = None,
+) -> typing.Sequence[flood.VegetaAttack]:
+ n_calls = load_tests.estimate_call_count(
+ rates=rates, duration=duration, durations=durations
+ )
+ calls = flood.generators.generate_calls_eth_fee_history(
+ n_calls=n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+ return load_tests.create_load_test(
+ calls=calls,
+ rates=rates,
+ duration=duration,
+ durations=durations,
+ vegeta_args=vegeta_args,
+ )
+
+
+# def generate_test_eth_get_block_by_hash(
+# *,
+# rates: typing.Sequence[int],
+# duration: int | None = None,
+# durations: typing.Sequence[int] | None = None,
+# network: str,
+# vegeta_args: typing.Mapping[str, str | None] | None = None,
+# random_seed: spec.RandomSeed | None = None,
+# ) -> typing.Sequence[flood.VegetaAttack]:
+# n_calls = load_tests.estimate_call_count(
+# rates=rates, duration=duration, durations=durations
+# )
+# calls = flood.generate_calls_eth_get_block_by_hash(
+# n_calls=n_calls,
+# network=network,
+# random_seed=random_seed,
+# )
+# return load_tests.create_load_test(
+# calls=calls,
+# rates=rates,
+# duration=duration,
+# durations=durations,
+ # vegeta_args=vegeta_args,
+# )
diff --git a/flood/generators/test_generators/contract_test_generators.py b/flood/generators/test_generators/contract_test_generators.py
new file mode 100644
index 0000000..57f02b0
--- /dev/null
+++ b/flood/generators/test_generators/contract_test_generators.py
@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import typing
+
+import flood
+from flood.tests import load_tests
+
+
+def generate_test_eth_get_code(
+ *,
+ rates: typing.Sequence[int],
+ network: str,
+ duration: int | None = None,
+ durations: typing.Sequence[int] | None = None,
+ vegeta_args: flood.VegetaArgsShorthand | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.VegetaAttack]:
+ n_calls = load_tests.estimate_call_count(
+ rates=rates, duration=duration, durations=durations
+ )
+ calls = flood.generators.generate_calls_eth_get_code(
+ n_calls=n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+ return load_tests.create_load_test(
+ calls=calls,
+ rates=rates,
+ duration=duration,
+ durations=durations,
+ vegeta_args=vegeta_args,
+ )
+
+
+def generate_test_eth_get_storage_at(
+ *,
+ rates: typing.Sequence[int],
+ network: str,
+ duration: int | None = None,
+ durations: typing.Sequence[int] | None = None,
+ vegeta_args: flood.VegetaArgsShorthand | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.VegetaAttack]:
+ n_calls = load_tests.estimate_call_count(
+ rates=rates, duration=duration, durations=durations
+ )
+ calls = flood.generators.generate_calls_eth_get_storage_at(
+ n_calls=n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+ return load_tests.create_load_test(
+ calls=calls,
+ rates=rates,
+ duration=duration,
+ durations=durations,
+ vegeta_args=vegeta_args,
+ )
+
+
+def generate_test_eth_call(
+ *,
+ rates: typing.Sequence[int],
+ network: str,
+ duration: int | None = None,
+ durations: typing.Sequence[int] | None = None,
+ vegeta_args: flood.VegetaArgsShorthand | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.VegetaAttack]:
+ n_calls = load_tests.estimate_call_count(
+ rates=rates, duration=duration, durations=durations
+ )
+ calls = flood.generators.generate_calls_eth_call(
+ n_calls=n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+ return load_tests.create_load_test(
+ calls=calls,
+ rates=rates,
+ duration=duration,
+ durations=durations,
+ vegeta_args=vegeta_args,
+ )
diff --git a/flood/generators/test_generators/generic_test_generators.py b/flood/generators/test_generators/generic_test_generators.py
new file mode 100644
index 0000000..f782a85
--- /dev/null
+++ b/flood/generators/test_generators/generic_test_generators.py
@@ -0,0 +1,215 @@
+from __future__ import annotations
+
+import typing
+
+import flood
+
+
+#
+# # bookkeeping
+#
+
+
+def get_single_test_generators() -> (
+ typing.Mapping[str, flood.LoadTestGenerator]
+):
+ """get all single test generators"""
+ return {
+ get_test_generator_display_name(item): item # type: ignore
+ for item in dir(flood.generators)
+ if item.startswith('generate_test_')
+ }
+
+
+def get_multi_test_generators() -> (
+ typing.Mapping[str, flood.MultiLoadTestGenerator]
+):
+ """get all multi test generators"""
+ return {
+ get_test_generator_display_name(item, multi=True): item # type: ignore
+ for item in dir(flood.generators)
+ if item.startswith('generate_tests_')
+ }
+
+
+def get_test_generator(test_name: str) -> flood.LoadTestGenerator:
+ """get particular single test generator"""
+ function_name = get_test_generator_function_name(test_name)
+ if hasattr(flood.generators, function_name):
+ return getattr(flood.generators, function_name) # type: ignore
+ else:
+ raise Exception()
+
+
+def get_tests_generator(test_name: str) -> flood.MultiLoadTestGenerator:
+ """get particular multi test generator"""
+ function_name = get_test_generator_function_name(test_name)
+ if hasattr(flood.generators, function_name):
+ return getattr(flood.generators, function_name) # type: ignore
+ else:
+ raise Exception()
+
+
+#
+# # generation
+#
+
+
+def generate_test(
+ *,
+ test_name: str,
+ random_seed: flood.RandomSeed | None = None,
+ rates: typing.Sequence[int] | None = None,
+ durations: typing.Sequence[int] | None = None,
+ vegeta_args: flood.VegetaArgsShorthand | None = None,
+ network: str,
+ # output_dir: str | None = None,
+ flood_version: str,
+) -> flood.LoadTest:
+ if test_name is None:
+ raise Exception('must specify test_name')
+ test_generator = get_test_generator(test_name)
+ test_parameters: flood.TestGenerationParameters = {
+ 'flood_version': flood.get_flood_version(),
+ 'test_name': test_name,
+ 'random_seed': random_seed,
+ 'rates': rates,
+ 'durations': durations,
+ 'vegeta_args': vegeta_args,
+ 'network': network,
+ }
+ attacks = test_generator(
+ rates=rates,
+ durations=durations,
+ vegeta_args=vegeta_args,
+ network=network,
+ random_seed=random_seed,
+ )
+ return {'attacks': attacks, 'test_parameters': test_parameters}
+
+
+def generate_tests(
+ *,
+ test_name: str,
+ random_seed: flood.RandomSeed | None = None,
+ rates: typing.Sequence[int] | None = None,
+ durations: typing.Sequence[int] | None = None,
+ vegeta_args: flood.VegetaArgsShorthand | None = None,
+ network: str,
+ output_dir: str | None = None,
+ common_parameters: typing.Mapping[str, typing.Any] | None = None,
+ specific_parameters: typing.Mapping[str, typing.Mapping[str, typing.Any]]
+ | None = None,
+) -> typing.Mapping[str, flood.LoadTest]:
+ if test_name is None:
+ raise Exception('must specify test_name')
+ test_generator = get_tests_generator(test_name)
+ tests = test_generator(
+ rates=rates,
+ durations=durations,
+ vegeta_args=vegeta_args,
+ network=network,
+ random_seed=random_seed,
+ common_parameters=common_parameters,
+ specific_parameters=specific_parameters,
+ )
+ if output_dir is not None:
+ flood.runners.multi_runner.multi_runner_io._save_multi_run_tests(
+ output_dir=output_dir,
+ tests=tests,
+ )
+ return tests
+
+
+# def generate_tests(
+# test_name: str,
+# constants: typing.Mapping[str, typing.Any] | None = None,
+# variables: typing.Mapping[
+# str, typing.Mapping[str, typing.Sequence[typing.Any]]
+# ]
+# | None = None,
+# ) -> typing.Mapping[str, flood.LoadTest]:
+# """variables is in format {variable_name: {test_name: variable_value}}"""
+# import itertools
+
+# if constants is None:
+# constants = {}
+# if variables is None:
+# variables = {}
+
+# # compute input variables
+# tests_kwargs = []
+# combinations = itertools.product(*variables.items())
+# for combination in combinations:
+# test_kwargs = dict(combination, **constants)
+# tests_kwargs.append(test_kwargs)
+
+# # use test generator function
+# test_generator = get_test_generator(test_name)
+# tests = []
+# for test_kwargs in tests_kwargs:
+# test = test_generator(**test_kwargs)
+# tests.append(test)
+
+# return tests
+
+
+#
+# # name formatting
+#
+
+
+def get_test_generator_display_name(
+ test: str | flood.LoadTestGenerator,
+ multi: bool = False,
+) -> str:
+ if multi:
+ prefix = 'generate_tests_'
+ else:
+ prefix = 'generate_test_'
+
+ if not isinstance(test, str):
+ test_id = id(test)
+ for key, value in vars(flood.generators).items():
+ if id(value) == test_id:
+ test = key
+ break
+ else:
+ import types
+
+ if isinstance(test, types.FunctionType):
+ test = test.__name__
+ else:
+ raise Exception('should be str or function')
+
+ if not test.startswith(prefix):
+ raise Exception()
+ test = test[len(prefix) :]
+ head, tail = test.split('_', 1)
+ test = head + '_' + _snake_case_to_camel_case(tail)
+ return test
+
+
+def get_test_generator_function_name(
+ display_name: str, multi: bool = False
+) -> str:
+ if multi:
+ prefix = 'generate_tests_'
+ else:
+ prefix = 'generate_test_'
+
+ function_name = prefix + _camel_case_to_snake_case(display_name)
+ return function_name
+
+
+def _camel_case_to_snake_case(string: str) -> str:
+ # adapted from https://stackoverflow.com/a/1176023
+ import re
+
+ return re.sub(r'(? str:
+ pieces = string.split('_')
+ return pieces[0] + ''.join(piece.title() for piece in pieces[1:])
+
diff --git a/rpc_bench/generators/test_generators/log_test_generators.py b/flood/generators/test_generators/log_test_generators.py
similarity index 55%
rename from rpc_bench/generators/test_generators/log_test_generators.py
rename to flood/generators/test_generators/log_test_generators.py
index 8d2491f..61c353a 100644
--- a/rpc_bench/generators/test_generators/log_test_generators.py
+++ b/flood/generators/test_generators/log_test_generators.py
@@ -1,49 +1,79 @@
from __future__ import annotations
+import functools
import typing
-import rpc_bench
+import flood
+from flood.tests import load_tests
def generate_test_eth_get_logs(
+ *,
rates: typing.Sequence[int],
+ network: str,
duration: int | None = None,
durations: typing.Sequence[int] | None = None,
- vegeta_kwargs: typing.Mapping[str, str | None] | None = None,
- random_seed: rpc_bench.RandomSeed | None = None,
-) -> rpc_bench.LoadTest:
- n_calls = rpc_bench.estimate_call_count(
+ vegeta_args: flood.VegetaArgsShorthand | None = None,
+ random_seed: flood.RandomSeed | None = None,
+ contract_address: str | None = None,
+ block_range_size: int | None = None,
+) -> typing.Sequence[flood.VegetaAttack]:
+ n_calls = load_tests.estimate_call_count(
rates=rates, duration=duration, durations=durations
)
- calls = rpc_bench.generate_calls_eth_get_logs(n_calls)
- return rpc_bench.construct_load_test(
+ calls = flood.generators.generate_calls_eth_get_logs(
+ n_calls,
+ network=network,
+ random_seed=random_seed,
+ contract_address=contract_address,
+ block_range_size=block_range_size,
+ )
+ return load_tests.create_load_test(
calls=calls,
rates=rates,
duration=duration,
durations=durations,
+ vegeta_args=vegeta_args,
)
+generate_test_eth_get_logs_lusd_transfers_s = functools.partial(
+ generate_test_eth_get_logs,
+ contract_address='0x5f98805a4e8be255a32880fdec7f6728c6568ba0',
+ block_range_size=100,
+)
+generate_test_eth_get_logs_lusd_transfers_m = functools.partial(
+ generate_test_eth_get_logs,
+ contract_address='0x5f98805a4e8be255a32880fdec7f6728c6568ba0',
+ block_range_size=1000,
+)
+generate_test_eth_get_logs_lusd_transfers_l = functools.partial(
+ generate_test_eth_get_logs,
+ contract_address='0x5f98805a4e8be255a32880fdec7f6728c6568ba0',
+ block_range_size=10000,
+)
+
+
# def generate_tests_eth_get_logs_by_contract(
# url: str,
# rates: typing.Sequence[int],
# duration: int,
# range_size: int,
-# ) -> typing.Mapping[str, rpc_bench.LoadTest]:
+# ) -> typing.Mapping[str, flood.LoadTest]:
# """test: Transfers of USDC vs DAI vs LUSD"""
-# n_calls = rpc_bench.estimate_call_count(rates=rates, duration=duration)
+# n_calls = flood.estimate_call_count(rates=rates, duration=duration)
-# block_ranges = rpc_bench.generate_block_ranges(
+# block_ranges = flood.generate_block_ranges(
# start_block=12_178_594,
# end_block=17_000_000,
# n=n_calls,
# range_size=range_size,
# )
-# tests: typing.MutableMapping[str, rpc_bench.LoadTest] = {}
+# tests: typing.MutableMapping[str, flood.LoadTest] = {}
# for name, contract_address in contracts.items():
-# calls = rpc_bench.generate_calls_eth_get_logs(
+# calls = flood.generate_calls_eth_get_logs(
# contract_address=contract_address,
# topics=[event_hashes['Transfer']],
# block_ranges=block_ranges,
@@ -63,20 +93,20 @@ def generate_test_eth_get_logs(
# rates: typing.Sequence[int],
# duration: int,
# range_sizes: typing.Sequence[int],
-# ) -> typing.Mapping[str, rpc_bench.LoadTest]:
+# ) -> typing.Mapping[str, flood.LoadTest]:
# """test: tiny vs small vs medium vs large block ranges"""
-# n_calls = rpc_bench.estimate_call_count(rates=rates, duration=duration)
+# n_calls = flood.estimate_call_count(rates=rates, duration=duration)
-# tests: typing.MutableMapping[str, rpc_bench.LoadTest] = {}
+# tests: typing.MutableMapping[str, flood.LoadTest] = {}
# for range_size in range_sizes:
-# block_ranges = rpc_bench.generate_block_ranges(
+# block_ranges = flood.generate_block_ranges(
# start_block=12_178_594,
# end_block=17_000_000,
# n=n_calls,
# range_size=range_size,
# )
-# calls = rpc_bench.generate_calls_eth_get_logs(
+# calls = flood.generate_calls_eth_get_logs(
# contract_address=contracts['USDC'],
# topics=[event_hashes['Transfer']],
# block_ranges=block_ranges,
@@ -97,20 +127,20 @@ def generate_test_eth_get_logs(
# rates: typing.Sequence[int],
# duration: int,
# block_bounds: typing.Mapping[str, tuple[int, int]],
-# ) -> typing.Mapping[str, rpc_bench.LoadTest]:
+# ) -> typing.Mapping[str, flood.LoadTest]:
# """test old vs new blocks"""
-# n_calls = rpc_bench.estimate_call_count(rates=rates, duration=duration)
+# n_calls = flood.estimate_call_count(rates=rates, duration=duration)
-# tests: typing.MutableMapping[str, rpc_bench.LoadTest] = {}
+# tests: typing.MutableMapping[str, flood.LoadTest] = {}
# for name, (start_block, end_block) in block_bounds.items():
-# block_ranges = rpc_bench.generate_block_ranges(
+# block_ranges = flood.generate_block_ranges(
# start_block=start_block,
# end_block=end_block,
# n=n_calls,
# range_size=100,
# )
-# calls = rpc_bench.generate_calls_eth_get_logs(
+# calls = flood.generate_calls_eth_get_logs(
# contract_address=contracts['USDC'],
# topics=[event_hashes['Transfer']],
# block_ranges=block_ranges,
@@ -123,4 +153,3 @@ def generate_test_eth_get_logs(
# }
# return tests
-
diff --git a/flood/generators/test_generators/multi_test_generators.py b/flood/generators/test_generators/multi_test_generators.py
new file mode 100644
index 0000000..41ae3f3
--- /dev/null
+++ b/flood/generators/test_generators/multi_test_generators.py
@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+import typing
+
+import flood
+
+
+def generate_tests_all_methods() -> typing.Mapping[str, flood.LoadTest]:
+ raise Exception()
+
diff --git a/flood/generators/test_generators/trace_test_generators.py b/flood/generators/test_generators/trace_test_generators.py
new file mode 100644
index 0000000..0a1398e
--- /dev/null
+++ b/flood/generators/test_generators/trace_test_generators.py
@@ -0,0 +1,215 @@
+from __future__ import annotations
+
+import typing
+
+import flood
+from flood.tests import load_tests
+
+
+def generate_test_trace_block(
+ *,
+ network: str,
+ rates: typing.Sequence[int],
+ duration: int | None = None,
+ durations: typing.Sequence[int] | None = None,
+ vegeta_args: flood.VegetaArgsShorthand | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.VegetaAttack]:
+ n_calls = load_tests.estimate_call_count(
+ rates=rates, duration=duration, durations=durations
+ )
+ calls = flood.generators.generate_calls_trace_block(
+ n_calls=n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+ return load_tests.create_load_test(
+ calls=calls,
+ rates=rates,
+ duration=duration,
+ durations=durations,
+ vegeta_args=vegeta_args,
+ )
+
+
+def generate_test_trace_transaction(
+ *,
+ network: str,
+ rates: typing.Sequence[int],
+ duration: int | None = None,
+ durations: typing.Sequence[int] | None = None,
+ vegeta_args: flood.VegetaArgsShorthand | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.VegetaAttack]:
+ n_calls = load_tests.estimate_call_count(
+ rates=rates, duration=duration, durations=durations
+ )
+ calls = flood.generators.generate_calls_trace_transaction(
+ n_calls=n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+ return load_tests.create_load_test(
+ calls=calls,
+ rates=rates,
+ duration=duration,
+ durations=durations,
+ vegeta_args=vegeta_args,
+ )
+
+
+def generate_test_trace_replay_block_transactions(
+ *,
+ network: str,
+ rates: typing.Sequence[int],
+ duration: int | None = None,
+ durations: typing.Sequence[int] | None = None,
+ vegeta_args: flood.VegetaArgsShorthand | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.VegetaAttack]:
+ n_calls = load_tests.estimate_call_count(
+ rates=rates, duration=duration, durations=durations
+ )
+ calls = flood.generators.generate_calls_trace_replay_block_transactions(
+ n_calls=n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+ return load_tests.create_load_test(
+ calls=calls,
+ rates=rates,
+ duration=duration,
+ durations=durations,
+ vegeta_args=vegeta_args,
+ )
+
+
+def generate_test_trace_replay_block_transactions_state_diff(
+ *,
+ network: str,
+ rates: typing.Sequence[int],
+ duration: int | None = None,
+ durations: typing.Sequence[int] | None = None,
+ vegeta_args: flood.VegetaArgsShorthand | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.VegetaAttack]:
+ n_calls = load_tests.estimate_call_count(
+ rates=rates, duration=duration, durations=durations
+ )
+ calls = flood.generators.generate_calls_trace_replay_block_transactions_state_diff( # noqa: E501
+ n_calls=n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+ return load_tests.create_load_test(
+ calls=calls,
+ rates=rates,
+ duration=duration,
+ durations=durations,
+ vegeta_args=vegeta_args,
+ )
+
+
+def generate_test_trace_replay_block_transactions_vm_trace(
+ *,
+ network: str,
+ rates: typing.Sequence[int],
+ duration: int | None = None,
+ durations: typing.Sequence[int] | None = None,
+ vegeta_args: flood.VegetaArgsShorthand | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.VegetaAttack]:
+ n_calls = load_tests.estimate_call_count(
+ rates=rates, duration=duration, durations=durations
+ )
+ calls = flood.generators.generate_calls_trace_replay_block_transactions_vm_trace( # noqa: E501
+ n_calls=n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+ return load_tests.create_load_test(
+ calls=calls,
+ rates=rates,
+ duration=duration,
+ durations=durations,
+ vegeta_args=vegeta_args,
+ )
+
+
+def generate_test_trace_replay_transaction(
+ *,
+ network: str,
+ rates: typing.Sequence[int],
+ duration: int | None = None,
+ durations: typing.Sequence[int] | None = None,
+ vegeta_args: flood.VegetaArgsShorthand | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.VegetaAttack]:
+ n_calls = load_tests.estimate_call_count(
+ rates=rates, duration=duration, durations=durations
+ )
+ calls = flood.generators.generate_calls_trace_replay_transaction(
+ n_calls=n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+ return load_tests.create_load_test(
+ calls=calls,
+ rates=rates,
+ duration=duration,
+ durations=durations,
+ vegeta_args=vegeta_args,
+ )
+
+
+def generate_test_trace_replay_transaction_state_diff(
+ *,
+ network: str,
+ rates: typing.Sequence[int],
+ duration: int | None = None,
+ durations: typing.Sequence[int] | None = None,
+ vegeta_args: flood.VegetaArgsShorthand | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.VegetaAttack]:
+ n_calls = load_tests.estimate_call_count(
+ rates=rates, duration=duration, durations=durations
+ )
+ calls = flood.generators.generate_calls_trace_replay_transaction_state_diff(
+ n_calls=n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+ return load_tests.create_load_test(
+ calls=calls,
+ rates=rates,
+ duration=duration,
+ durations=durations,
+ vegeta_args=vegeta_args,
+ )
+
+
+def generate_test_trace_replay_transaction_vm_trace(
+ *,
+ network: str,
+ rates: typing.Sequence[int],
+ duration: int | None = None,
+ durations: typing.Sequence[int] | None = None,
+ vegeta_args: flood.VegetaArgsShorthand | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.VegetaAttack]:
+ n_calls = load_tests.estimate_call_count(
+ rates=rates, duration=duration, durations=durations
+ )
+ calls = flood.generators.generate_calls_trace_replay_transaction_vm_trace(
+ n_calls=n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+ return load_tests.create_load_test(
+ calls=calls,
+ rates=rates,
+ duration=duration,
+ durations=durations,
+ vegeta_args=vegeta_args,
+ )
+
diff --git a/flood/generators/test_generators/transaction_test_generators.py b/flood/generators/test_generators/transaction_test_generators.py
new file mode 100644
index 0000000..aeba711
--- /dev/null
+++ b/flood/generators/test_generators/transaction_test_generators.py
@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+import typing
+
+import flood
+from flood.tests import load_tests
+
+
+def generate_test_eth_get_transaction_by_hash(
+ *,
+ rates: typing.Sequence[int],
+ duration: int | None = None,
+ network: str,
+ durations: typing.Sequence[int] | None = None,
+ vegeta_args: flood.VegetaArgsShorthand | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.VegetaAttack]:
+ n_calls = load_tests.estimate_call_count(
+ rates=rates, duration=duration, durations=durations
+ )
+ calls = flood.generators.generate_calls_eth_get_transaction_by_hash(
+ n_calls=n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+ return load_tests.create_load_test(
+ calls=calls,
+ rates=rates,
+ duration=duration,
+ durations=durations,
+ vegeta_args=vegeta_args,
+ )
+
+
+def generate_test_eth_get_transaction_receipt(
+ *,
+ rates: typing.Sequence[int],
+ duration: int | None = None,
+ network: str,
+ durations: typing.Sequence[int] | None = None,
+ vegeta_args: flood.VegetaArgsShorthand | None = None,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.VegetaAttack]:
+ n_calls = load_tests.estimate_call_count(
+ rates=rates, duration=duration, durations=durations
+ )
+ calls = flood.generators.generate_calls_eth_get_transaction_receipt(
+ n_calls=n_calls,
+ network=network,
+ random_seed=random_seed,
+ )
+ return load_tests.create_load_test(
+ calls=calls,
+ rates=rates,
+ duration=duration,
+ durations=durations,
+ vegeta_args=vegeta_args,
+ )
diff --git a/flood/ops/__init__.py b/flood/ops/__init__.py
new file mode 100644
index 0000000..3af1106
--- /dev/null
+++ b/flood/ops/__init__.py
@@ -0,0 +1,2 @@
+from .installation_utils import *
+from .update_utils import *
diff --git a/flood/ops/installation_utils.py b/flood/ops/installation_utils.py
new file mode 100644
index 0000000..590cdf6
--- /dev/null
+++ b/flood/ops/installation_utils.py
@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import subprocess
+import typing
+
+import flood
+
+
+def get_flood_version() -> str:
+ """get flood version including git commit hash if in an untagged commit"""
+ installation = get_local_installation()
+ flood_version = installation['flood_version']
+ if flood_version is None:
+ raise Exception('could not detect flood version')
+ return flood_version
+
+
+def get_local_installation() -> flood.FloodInstallation:
+ result = subprocess.check_output(['which', 'vegeta'])
+ vegeta_path = result.decode().rstrip()
+
+ flood_version = flood.__version__
+
+ # add git commit hash if not in a tagged release
+ use_git_dir = None
+ use_git_commit = None
+ try:
+ import os
+
+ module_dir = os.path.realpath(flood.__path__[0])
+ parent_dir = os.path.dirname(module_dir)
+ parent_dir_files = os.listdir(parent_dir)
+ if 'pyproject.toml' in parent_dir_files and '.git' in parent_dir_files:
+ git_dir = os.path.join(parent_dir, '.git')
+
+ # check whether in tagged release
+ try:
+ cmd = 'git --git-dir={git_dir} describe --tags --exact-match'
+ cmd = cmd.format(git_dir=git_dir)
+ subprocess.check_call(
+ cmd.split(' '),
+ stderr=subprocess.DEVNULL,
+ stdout=subprocess.DEVNULL,
+ )
+ except subprocess.CalledProcessError:
+ # get git commit
+ cmd = 'git --git-dir={git_dir} rev-parse HEAD'.format(
+ git_dir=git_dir
+ )
+ output = subprocess.check_output(cmd.split(' '))
+ git_commit = output.decode().strip()
+
+ # specify commit in flood version
+ flood_version = flood.__version__ + '-' + git_commit[:8]
+ use_git_dir = git_dir
+ use_git_commit = git_commit
+ except Exception:
+ pass
+
+ return {
+ 'flood_version': flood_version,
+ 'vegeta_path': vegeta_path,
+ 'git_dir': use_git_dir,
+ 'git_commit': use_git_commit,
+ 'semver': flood.__version__,
+ }
+
+
+def get_remote_installation(
+ host: str, username: str | None = None
+) -> flood.FloodInstallation:
+ # create command
+ if username is not None:
+ url = username + '@' + host
+ else:
+ url = host
+ template = (
+ "ssh {url} bash -c "
+ "'source ~/.profile; "
+ "echo $(python3 -m flood version --json)'"
+ )
+ cmd = template.format(url=url)
+
+ # run command
+ try:
+ result = subprocess.check_output(
+ cmd.split(' '), stderr=subprocess.DEVNULL
+ )
+ except Exception:
+ raise flood.SshError('could not ssh')
+
+ try:
+ import json
+
+ output: flood.FloodInstallation = json.loads(result.decode().strip())
+ return output
+
+ except Exception:
+ return {
+ 'vegeta_path': None,
+ 'flood_version': None,
+ 'git_dir': None,
+ 'git_commit': None,
+ 'semver': None,
+ }
+
+
+def get_dependency_versions() -> typing.Mapping[str, str | None]:
+ import importlib
+
+ versions = {}
+ for module_name in [
+ 'ctc',
+ 'ipykernel',
+ 'ipython_genutils',
+ 'matplotlib',
+ 'nbconvert',
+ 'nbformat',
+ 'numpy',
+ 'orjson',
+ 'pdp',
+ 'polars',
+ 'requests',
+ 'toolcli',
+ 'toolplot',
+ 'toolstr',
+ 'tqdm',
+ 'typing_extensions',
+ ]:
+ module = importlib.import_module(module_name)
+ versions[module_name] = getattr(module, '__version__', None)
+
+ return versions
+
diff --git a/flood/ops/update_utils.py b/flood/ops/update_utils.py
new file mode 100644
index 0000000..cd99883
--- /dev/null
+++ b/flood/ops/update_utils.py
@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+from . import installation_utils
+
+
+def update_local(version: str | None = None) -> None:
+ installation = installation_utils.get_local_installation()
+ if installation['git_dir'] is not None:
+ _update_local_git(version=version)
+ else:
+ _update_local_pip(version=version)
+
+
+def _update_local_git(version: str | None = None) -> None:
+ import os
+ import subprocess
+
+ installation = installation_utils.get_local_installation()
+ git_dir = installation['git_dir']
+ if git_dir is None:
+ raise Exception('git installation not specified')
+ repo_dir = os.path.dirname(git_dir)
+ if version is None:
+ # git pull
+ cmd = 'git --git-dir={git_dir} --work-tree={repo_dir} pull'.format(
+ git_dir=git_dir, repo_dir=repo_dir
+ )
+ subprocess.call(cmd.split(' '))
+ else:
+ if '.' in version:
+ raise Exception(
+ 'for a pip installation, must specify semantic version number, instead got: ' # noqa: E501
+ + str(version)
+ )
+
+ # git fetch
+ cmd = 'git --git-dir={git_dir} --work-tree={repo_dir} fetch origin'.format( # noqa: E501
+ git_dir=git_dir, repo_dir=repo_dir
+ )
+ subprocess.call(cmd.split(' '))
+
+ # git checkout commit
+ cmd = 'git --git-dir={git_dir} --work-tree={repo_dir} checkout {version}'.format( # noqa: E501
+ git_dir=git_dir,
+ version=version,
+ repo_dir=repo_dir,
+ )
+ subprocess.call(cmd.split(' '))
+
+
+def _update_local_pip(version: str | None = None) -> None:
+ import subprocess
+
+ if version is None:
+ cmd = 'pip install -U flood'
+ else:
+ if '.' not in version:
+ raise Exception(
+ 'for a pip installation, must specify semantic version number, instead got: ' # noqa: E501
+ + str(version)
+ )
+ cmd = 'pip install flood=={version}'.format(version=version)
+
+ subprocess.check_call(cmd.split(' '))
+
+
+def update_remote(hostname: str, version: str | None = None) -> None:
+ import subprocess
+
+ if version is None:
+ cmd = "ssh {host} bash -c 'source ~/.profile; python3 -m flood update'"
+ cmd = cmd.format(host=hostname)
+ else:
+ cmd = "ssh {host} bash -c 'source ~/.profile; python3 -m flood update --version {version}'" # noqa: E501
+ cmd = cmd.format(host=hostname, version=version)
+ subprocess.call(cmd.split(' '))
+
diff --git a/flood/runners/__init__.py b/flood/runners/__init__.py
new file mode 100644
index 0000000..59b8b64
--- /dev/null
+++ b/flood/runners/__init__.py
@@ -0,0 +1,3 @@
+from .generic_runner import *
+from .multi_runner import *
+from .single_runner import *
diff --git a/flood/runners/generic_runner/__init__.py b/flood/runners/generic_runner/__init__.py
new file mode 100644
index 0000000..78a9bb5
--- /dev/null
+++ b/flood/runners/generic_runner/__init__.py
@@ -0,0 +1 @@
+from .generic_runner_execution import *
diff --git a/flood/runners/generic_runner/generic_runner_execution.py b/flood/runners/generic_runner/generic_runner_execution.py
new file mode 100644
index 0000000..e87b6e8
--- /dev/null
+++ b/flood/runners/generic_runner/generic_runner_execution.py
@@ -0,0 +1,121 @@
+from __future__ import annotations
+
+import typing
+
+import flood
+from flood import generators
+from flood.runners.single_runner import single_runner_execution
+
+# from flood.runners.multi_runner import multi_runner_execution
+
+
+def run(
+ test_name: str,
+ *,
+ nodes: flood.NodesShorthand | None,
+ random_seed: flood.RandomSeed | None = None,
+ verbose: bool | int = True,
+ rates: typing.Sequence[int] | None = None,
+ duration: int | None = None,
+ durations: typing.Sequence[int] | None = None,
+ mode: flood.LoadTestMode | None = None,
+ vegeta_args: flood.VegetaArgsShorthand | None = None,
+ dry: bool = False,
+ output_dir: str | None = None,
+ figures: bool = True,
+ metrics: typing.Sequence[str] | None = None,
+ include_deep_output: typing.Sequence[flood.DeepOutput] | None = None,
+ deep_check: bool = False,
+) -> flood.RunOutput:
+ """generate and run tests against nodes"""
+ import os
+
+ # get output_dir
+ use_output_dir = _get_output_dir(output_dir)
+
+ # run test from path
+ if os.path.exists(test_name) or '/' in test_name:
+ (
+ test_name,
+ path_spec,
+ test,
+ nodes,
+ ) = _load_old_test_data(test_name=test_name, nodes=nodes)
+ output = single_runner_execution._run_single(
+ rerun_of=path_spec,
+ test=test,
+ #
+ test_name=test_name,
+ nodes=nodes,
+ random_seed=random_seed,
+ dry=dry,
+ output_dir=use_output_dir,
+ verbose=verbose,
+ metrics=metrics,
+ figures=figures,
+ include_deep_output=include_deep_output,
+ deep_check=deep_check,
+ )
+ return {'single_run': output}
+
+ # generate new test
+ else:
+ if nodes is None:
+ raise Exception('must specify nodes')
+
+ if test_name in generators.get_single_test_generators():
+ output = single_runner_execution._run_single(
+ rates=rates,
+ duration=duration,
+ durations=durations,
+ vegeta_args=vegeta_args,
+ #
+ test_name=test_name,
+ nodes=nodes,
+ random_seed=random_seed,
+ dry=dry,
+ output_dir=use_output_dir,
+ verbose=verbose,
+ metrics=metrics,
+ figures=figures,
+ include_deep_output=include_deep_output,
+ deep_check=deep_check,
+ )
+ return {'single_run': output}
+ elif test_name in generators.get_multi_test_generators():
+ raise NotImplementedError('multi tests not supported yet')
+ else:
+ raise Exception('invalid test name')
+
+
+def _get_output_dir(output_dir: str | None) -> str:
+ import os
+
+ if output_dir is None:
+ import tempfile
+
+ use_output_dir = tempfile.mkdtemp()
+ else:
+ use_output_dir = output_dir
+ return os.path.abspath(os.path.expanduser(use_output_dir))
+
+
+def _load_old_test_data(
+ test_name: str, nodes: flood.NodesShorthand | None
+) -> tuple[str, str, flood.LoadTest, flood.NodesShorthand]:
+ path_spec = test_name
+
+ try:
+ test_payload = flood.load_single_run_test_payload(path_spec)
+ test = generators.generate_test(**test_payload['test_parameters'])
+ test_name = test_payload['name']
+ except Exception:
+ raise Exception('invalid test path: ' + str(path_spec))
+
+ # use old nodes if none specified
+ if nodes is None:
+ results_payload = flood.load_single_run_results_payload(path_spec)
+ nodes = results_payload['nodes']
+
+ return (test_name, path_spec, test, nodes)
+
diff --git a/flood/runners/multi_runner/__init__.py b/flood/runners/multi_runner/__init__.py
new file mode 100644
index 0000000..90abaea
--- /dev/null
+++ b/flood/runners/multi_runner/__init__.py
@@ -0,0 +1 @@
+from .multi_runner_io import *
diff --git a/flood/runners/multi_runner/multi_runner_io.py b/flood/runners/multi_runner/multi_runner_io.py
new file mode 100644
index 0000000..8e2a9f6
--- /dev/null
+++ b/flood/runners/multi_runner/multi_runner_io.py
@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import typing
+
+import flood
+
+
+_path_templates = {
+ 'multi_indicator': 'multi_test.fyi',
+}
+
+
+def _save_multi_run_tests(
+ output_dir: str,
+ tests: typing.Mapping[str, flood.LoadTest],
+) -> None:
+ raise NotImplementedError()
+ # import os
+
+ # # make output dir
+ # if not os.path.isdir(output_dir):
+ # if os.path.exists(output_dir):
+ # raise Exception('output must be a directory path')
+ # else:
+ # os.makedirs(output_dir, exist_ok=True)
+
+ # # create individual tests
+ # for test_name, test in tests.items():
+ # single_runner_io._save_single_run_test(
+ # test_name=test_name,
+ # output_dir=os.path.join(output_dir, test_name),
+ # test=test,
+ # )
+
+ # # create multitest indicator file
+ # indicator_path = os.path.join(
+ # output_dir, _path_templates['multi_indicator']
+ # )
+ # with open(indicator_path, 'a'):
+ # pass
+
+
+def _load_multi_run_tests(
+ output_dir: str,
+) -> typing.Mapping[str, flood.LoadTest]:
+ raise NotImplementedError()
+
diff --git a/flood/runners/single_runner/__init__.py b/flood/runners/single_runner/__init__.py
new file mode 100644
index 0000000..4cc40a1
--- /dev/null
+++ b/flood/runners/single_runner/__init__.py
@@ -0,0 +1 @@
+from .single_runner_io import *
diff --git a/flood/runners/single_runner/single_runner_execution.py b/flood/runners/single_runner/single_runner_execution.py
new file mode 100644
index 0000000..34d81b8
--- /dev/null
+++ b/flood/runners/single_runner/single_runner_execution.py
@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+import typing
+
+import flood
+from . import single_runner_io
+from . import single_runner_summary
+
+
+def _run_single(
+ *,
+ test_name: str,
+ rerun_of: str | None = None,
+ test: flood.LoadTest | None = None,
+ nodes: flood.NodesShorthand,
+ random_seed: flood.RandomSeed | None = None,
+ rates: typing.Sequence[int] | None = None,
+ duration: int | None = None,
+ durations: typing.Sequence[int] | None = None,
+ mode: flood.LoadTestMode | None = None,
+ vegeta_args: flood.VegetaArgsShorthand | None = None,
+ dry: bool,
+ output_dir: str,
+ figures: bool,
+ metrics: typing.Sequence[str] | None = None,
+ verbose: bool | int,
+ include_deep_output: typing.Sequence[flood.DeepOutput] | None = None,
+ deep_check: bool = False,
+) -> flood.SingleRunOutput:
+ import time
+
+ t_start = time.time()
+
+ if include_deep_output is None:
+ include_deep_output = []
+ if deep_check and 'metrics' not in include_deep_output:
+ include_deep_output = list(include_deep_output) + ['metrics']
+
+ # get test parameters
+ rates, durations, vegeta_args = _get_single_test_parameters(
+ test=test,
+ rates=rates,
+ duration=duration,
+ durations=durations,
+ mode=mode,
+ vegeta_args=vegeta_args,
+ )
+
+ # print preamble
+ if verbose:
+ single_runner_summary._print_single_run_preamble_copy(
+ test_name=test_name,
+ rerun_of=rerun_of,
+ rates=rates,
+ durations=durations,
+ vegeta_args=vegeta_args,
+ output_dir=output_dir,
+ )
+
+ # parse nodes
+ nodes = flood.user_io.parse_nodes(
+ nodes, verbose=verbose, request_metadata=True
+ )
+
+ # generate test and save to disk
+ use_test: flood.LoadTest | flood.TestGenerationParameters
+ test_parameters: flood.TestGenerationParameters
+ if test is None:
+ test_parameters = {
+ 'flood_version': flood.get_flood_version(),
+ 'test_name': test_name,
+ 'rates': rates,
+ 'durations': durations,
+ 'vegeta_args': vegeta_args,
+ 'network': flood.user_io.parse_nodes_network(nodes),
+ 'random_seed': random_seed,
+ }
+ flood.runners.single_runner.single_runner_io._save_single_run_test(
+ test_name=test_name,
+ output_dir=output_dir,
+ test_parameters=test_parameters,
+ )
+ use_test = test_parameters
+ else:
+ test_parameters = test['test_parameters']
+ use_test = test
+
+ # skip dry run
+ if dry:
+ print()
+ print('[dry run, exitting]')
+ return # type: ignore
+
+ # run tests
+ if verbose:
+ single_runner_summary._print_run_start()
+ results = flood.run_load_tests(
+ nodes=nodes,
+ test=use_test,
+ verbose=verbose,
+ include_deep_output=include_deep_output,
+ )
+
+ # output results to file
+ payload = single_runner_io._save_single_run_results(
+ output_dir=output_dir,
+ nodes=nodes,
+ results=results,
+ figures=figures,
+ test_name=test_name,
+ t_run_start=t_start,
+ t_run_end=time.time(),
+ )
+
+ # print summary
+ if verbose:
+ single_runner_summary._print_single_run_conclusion(
+ output_dir=output_dir,
+ results=results,
+ metrics=metrics,
+ verbose=verbose,
+ figures=figures,
+ deep_check=deep_check,
+ )
+
+ return {
+ 'output_dir': output_dir,
+ 'test': test,
+ 'test_parameters': test_parameters,
+ 'payload': payload,
+ }
+
+
+def _get_single_test_parameters(
+ test: flood.LoadTest | None = None,
+ rates: typing.Sequence[int] | None = None,
+ duration: int | None = None,
+ durations: typing.Sequence[int] | None = None,
+ mode: flood.LoadTestMode | None = None,
+ vegeta_args: flood.VegetaArgsShorthand | None = None,
+) -> tuple[
+ typing.Sequence[int],
+ typing.Sequence[int],
+ flood.VegetaArgsShorthand | None,
+]:
+ if test is not None:
+ test_data = flood.user_io.parse_test_data(test=test)
+ rates = test_data['rates']
+ durations = test_data['durations']
+ vegeta_args = test_data['vegeta_args']
+ else:
+ rates, durations = flood.generators.generate_timings(
+ rates=rates,
+ duration=duration,
+ durations=durations,
+ mode=mode,
+ )
+ return rates, durations, vegeta_args
+
diff --git a/flood/runners/single_runner/single_runner_io.py b/flood/runners/single_runner/single_runner_io.py
new file mode 100644
index 0000000..eb8b946
--- /dev/null
+++ b/flood/runners/single_runner/single_runner_io.py
@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+import typing
+
+import flood
+
+
+#
+# # path utiltiies
+#
+
+_path_templates = {
+ 'single_run_test': '{output_dir}/test.json',
+ 'single_run_results': '{output_dir}/results.json',
+ 'single_run_figures_dir': '{output_dir}/figures',
+}
+
+
+def get_single_run_test_path(output_dir: str) -> str:
+ return _path_templates['single_run_test'].format(output_dir=output_dir)
+
+
+def get_single_run_results_path(output_dir: str) -> str:
+ return _path_templates['single_run_results'].format(output_dir=output_dir)
+
+
+def get_single_run_figures_path(output_dir: str) -> str:
+ return _path_templates['single_run_figures_dir'].format(
+ output_dir=output_dir
+ )
+
+
+#
+# # save utilities
+#
+
+
+def _save_single_run_test(
+ test_name: str,
+ output_dir: str,
+ test_parameters: flood.TestGenerationParameters,
+) -> None:
+ import os
+ import orjson
+
+ if not os.path.isdir(output_dir):
+ if os.path.exists(output_dir):
+ raise Exception('output must be a directory path')
+ else:
+ os.makedirs(output_dir, exist_ok=True)
+
+ path = _path_templates['single_run_test'].format(output_dir=output_dir)
+ payload: flood.SingleRunTestPayload = {
+ 'flood_version': flood.__version__,
+ 'type': 'single_test',
+ 'name': test_name,
+ 'test_parameters': test_parameters,
+ }
+ with open(path, 'wb') as f:
+ f.write(orjson.dumps(payload))
+
+
+def _save_single_run_results(
+ *,
+ output_dir: str,
+ nodes: flood.Nodes,
+ results: typing.Mapping[str, flood.LoadTestOutput],
+ figures: bool,
+ test_name: str,
+ t_run_start: float,
+ t_run_end: float,
+) -> flood.SingleRunResultsPayload:
+ import os
+ import sys
+
+ import orjson
+
+ if not os.path.isdir(output_dir):
+ if os.path.exists(output_dir):
+ raise Exception('output must be a directory path')
+ else:
+ os.makedirs(output_dir)
+
+ path = _path_templates['single_run_results'].format(output_dir=output_dir)
+ payload: flood.SingleRunResultsPayload = {
+ 'flood_version': flood.get_flood_version(),
+ 'dependency_versions': flood.get_dependency_versions(),
+ 'cli_args': list(sys.argv),
+ 'type': 'single_test',
+ 't_run_start': t_run_start,
+ 't_run_end': t_run_end,
+ 'nodes': nodes,
+ 'results': results,
+ }
+ with open(path, 'wb') as f:
+ f.write(orjson.dumps(payload))
+
+ if figures:
+ figures_dir = get_single_run_figures_path(output_dir=output_dir)
+ colors = flood.user_io.get_nodes_plot_colors(nodes=nodes)
+ flood.tests.load_tests.plot_load_test_results(
+ outputs=results,
+ test_name=test_name,
+ output_dir=figures_dir,
+ colors=colors,
+ )
+
+ return payload
+
+
+#
+# # load utiltiies
+#
+
+
+def load_single_run_test_payload(
+ path_spec: str,
+ allow_other_versions: bool = False,
+) -> flood.SingleRunTestPayload:
+ import os
+ import orjson
+
+ if os.path.isfile(path_spec):
+ path = path_spec
+ else:
+ path = get_single_run_test_path(path_spec)
+ with open(path, 'rb') as f:
+ test: flood.SingleRunTestPayload = orjson.loads(f.read())
+
+ if test['flood_version'] != flood.__version__:
+ if allow_other_versions:
+ pass
+ else:
+ raise Exception(
+ 'loaded test version ('
+ + str(test['flood_version'])
+ + ') does not match current flood version ('
+ + flood.__version__
+ + ')'
+ )
+
+ return test
+
+
+def load_single_run_results_payload(
+ output_dir: str,
+) -> flood.SingleRunResultsPayload:
+ import orjson
+
+ path = get_single_run_results_path(output_dir=output_dir)
+ with open(path, 'rb') as f:
+ results: flood.SingleRunResultsPayload = orjson.loads(f.read())
+ return results
+
diff --git a/flood/runners/single_runner/single_runner_summary.py b/flood/runners/single_runner/single_runner_summary.py
new file mode 100644
index 0000000..6f900aa
--- /dev/null
+++ b/flood/runners/single_runner/single_runner_summary.py
@@ -0,0 +1,258 @@
+from __future__ import annotations
+
+import typing
+
+import flood
+from . import single_runner_io
+
+
+def _print_single_run_preamble(
+ *,
+ test_name: str,
+ rates: typing.Sequence[int],
+ durations: typing.Sequence[int],
+ vegeta_args: flood.VegetaArgsShorthand | None,
+ rerun_of: str | None = None,
+ output_dir: str | None,
+) -> None:
+ import os
+ import toolstr
+
+ _print_single_run_preamble_copy(
+ test_name=test_name,
+ rerun_of=rerun_of,
+ rates=rates,
+ durations=durations,
+ vegeta_args=vegeta_args,
+ output_dir=output_dir,
+ )
+ if output_dir is not None:
+ summary_path = os.path.join(output_dir, 'summary.txt')
+ with toolstr.write_stdout_to_file(summary_path):
+ _print_single_run_preamble_copy(
+ test_name=test_name,
+ rerun_of=rerun_of,
+ rates=rates,
+ durations=durations,
+ vegeta_args=vegeta_args,
+ output_dir=output_dir,
+ )
+
+
+def _print_single_run_preamble_copy(
+ *,
+ test_name: str,
+ rates: typing.Sequence[int],
+ durations: typing.Sequence[int],
+ vegeta_args: flood.VegetaArgsShorthand | None,
+ rerun_of: str | None = None,
+ output_dir: str | None,
+) -> None:
+ import toolstr
+
+ styles = flood.user_io.styles
+
+ toolstr.print_text_box(
+ toolstr.add_style('Load test: ' + test_name, styles['metavar']),
+ style=flood.user_io.styles['content'],
+ )
+ toolstr.print_bullet(key='sample rates', value=rates, styles=styles)
+ if len(set(durations)) == 1:
+ toolstr.print_bullet(
+ key='sample duration',
+ value=durations[0],
+ styles=styles,
+ )
+ else:
+ toolstr.print_bullet(
+ key='sample durations', value=durations, styles=styles
+ )
+ toolstr.print_bullet(key='extra args', value=vegeta_args, styles=styles)
+
+ if rerun_of is not None:
+ toolstr.print_bullet(key='rerun of', value=rerun_of, styles=styles)
+ toolstr.print_bullet(
+ key='output directory', value=output_dir, styles=styles
+ )
+ print()
+
+
+def _print_run_start() -> None:
+ import toolstr
+
+ styles = flood.user_io.styles
+
+ print()
+ print()
+ toolstr.print_header(
+ 'Running load tests...',
+ style=styles['content'],
+ text_style=styles['metavar'],
+ )
+ flood.user_io.print_timestamped('Starting')
+
+
+def _print_single_run_conclusion(
+ *,
+ output_dir: str | None,
+ results: typing.Mapping[str, flood.LoadTestOutput],
+ metrics: typing.Sequence[str] | None,
+ verbose: bool | int,
+ figures: bool,
+ deep_check: bool,
+) -> None:
+ _print_single_run_conclusion_text(
+ output_dir=output_dir,
+ results=results,
+ metrics=metrics,
+ verbose=verbose,
+ figures=figures,
+ deep_check=deep_check,
+ )
+ if output_dir is not None:
+ import os
+ import toolstr
+
+ summary_path = os.path.join(output_dir, 'summary.txt')
+ with toolstr.write_stdout_to_file(summary_path, mode='a'):
+ _print_single_run_conclusion_text(
+ output_dir=output_dir,
+ results=results,
+ metrics=metrics,
+ verbose=verbose,
+ figures=figures,
+ deep_check=deep_check,
+ )
+
+
+def _print_single_run_conclusion_text(
+ output_dir: str | None,
+ results: typing.Mapping[str, flood.LoadTestOutput],
+ metrics: typing.Sequence[str] | None,
+ verbose: bool | int,
+ figures: bool,
+ deep_check: bool,
+) -> None:
+ import os
+ import toolstr
+
+ styles = flood.user_io.styles
+
+ flood.user_io.print_timestamped('Load tests completed.')
+
+ # print message about metrics file
+ if output_dir is not None:
+ test_path = single_runner_io.get_single_run_test_path(
+ output_dir=output_dir
+ )
+ result_path = single_runner_io.get_single_run_results_path(
+ output_dir=output_dir
+ )
+ figures_path = single_runner_io.get_single_run_figures_path(
+ output_dir=output_dir
+ )
+
+ print()
+ print()
+ flood.user_io.print_header('Saving results to output directory...')
+ toolstr.print_bullet(
+ key=os.path.relpath(test_path, output_dir),
+ value='',
+ colon_str='',
+ styles=styles,
+ )
+ toolstr.print_bullet(
+ key=os.path.relpath(result_path, output_dir),
+ value='',
+ colon_str='',
+ styles=styles,
+ )
+ if figures:
+ toolstr.print_bullet(
+ key=os.path.relpath(figures_path, output_dir),
+ value='',
+ colon_str='',
+ styles=styles,
+ )
+
+ # decide metrics
+ if metrics is None:
+ metrics = ['success', 'throughput', 'p90']
+
+ # print metrics
+ print()
+ print()
+ flood.user_io.print_header('Summarizing performance metrics...')
+ if verbose > 1:
+ toolstr.print_bullet(
+ key='metrics shown below',
+ value=', '.join(metrics),
+ styles=styles,
+ )
+ example_result = list(results.values())[0]
+ additional = [
+ key for key in example_result.keys() if key not in metrics
+ ]
+ toolstr.print_bullet(
+ key='additional metrics available',
+ value=', '.join(additional),
+ styles=styles,
+ )
+
+ # print metric values
+ print()
+ flood.user_io.print_metric_tables(
+ results=results, metrics=metrics, indent=4
+ )
+
+ # deep inspection tables
+ if deep_check:
+ print()
+ print()
+ flood.user_io.print_header('Deep inspection of responses...')
+
+ # extract data per category
+ deep_results_by_category: typing.MutableMapping[
+ flood.ResponseCategory,
+ typing.MutableMapping[str, flood.LoadTestDeepOutput],
+ ]
+ deep_results_by_category = {}
+ for result_name, result in results.items():
+ deep_metrics = result['deep_metrics']
+ if deep_metrics is not None:
+ for category, category_results in deep_metrics.items():
+ deep_results_by_category.setdefault(category, {})
+ deep_results_by_category[category][
+ result_name
+ ] = category_results
+ else:
+ raise Exception('deep metrics not available')
+
+ print()
+ flood.user_io.print_metric_tables(
+ results=deep_results_by_category['failed'],
+ metrics=['n_invalid_json_errors'],
+ indent=4,
+ )
+ print()
+ flood.user_io.print_metric_tables(
+ results=deep_results_by_category['failed'],
+ metrics=['n_rpc_errors'],
+ indent=4,
+ )
+
+ metric_names = [
+ m for m in metrics if m not in ['success', 'throughput']
+ ]
+ for (
+ category,
+ result_category_results,
+ ) in deep_results_by_category.items():
+ print()
+ flood.user_io.print_metric_tables(
+ results=result_category_results,
+ metrics=metric_names,
+ suffix=', ' + category + ' calls',
+ indent=4,
+ )
+
diff --git a/flood/spec.py b/flood/spec.py
new file mode 100644
index 0000000..242cb6d
--- /dev/null
+++ b/flood/spec.py
@@ -0,0 +1,292 @@
+"""type definitions for typing annotations"""
+
+from __future__ import annotations
+
+import typing
+
+
+if typing.TYPE_CHECKING:
+ #
+ # # generic types
+ #
+
+ Call = typing.Mapping[str, typing.Mapping[str, typing.Any]]
+
+ MethodCalls = typing.Mapping[str, typing.Sequence[typing.Any]]
+
+ class Node(typing.TypedDict):
+ name: str
+ url: str
+ remote: str | None
+ client_version: str | None
+ network: str | int | None
+
+ NodeShorthand = typing.Union[str, Node]
+
+ NodesShorthand = typing.Union[
+ typing.Sequence[str],
+ typing.Sequence[Node],
+ typing.Mapping[str, str],
+ typing.Mapping[str, Node],
+ ]
+ Nodes = typing.Mapping[str, Node]
+
+ class ProgressBar(typing.TypedDict):
+ desc: str
+ position: int | None
+ leave: bool
+ colour: str
+ disable: bool
+
+ #
+ # # generators
+ #
+
+ import numpy as np
+
+ RandomSeed = typing.Union[int, np.random._generator.Generator]
+
+ #
+ # # latency test types
+ #
+
+ NodeMethodLatencies = typing.Mapping[str, typing.Sequence[float]]
+ NodesMethodLatencies = typing.Mapping[str, NodeMethodLatencies]
+
+ class LatencyBenchmarkResults(typing.TypedDict):
+ nodes: typing.Mapping[str, Node]
+ methods: typing.Sequence[str]
+ samples: int | None
+ calls: typing.Mapping[str, typing.Sequence[str]]
+ latencies: NodesMethodLatencies
+
+ #
+ # # equality test
+ #
+
+ EqualityTest = tuple[
+ str,
+ typing.Callable[..., typing.Any],
+ typing.Sequence[typing.Any],
+ typing.Mapping[str, typing.Any],
+ ]
+
+ #
+ # # load test types
+ #
+
+ class VegetaAttack(typing.TypedDict):
+ rate: int
+ duration: int
+ calls: typing.Sequence[typing.Any]
+ vegeta_args: VegetaArgs
+
+ VegetaArgs = typing.Union[str, None]
+ MultiVegetaArgs = typing.Sequence[VegetaArgs]
+ VegetaArgsShorthand = typing.Union[VegetaArgs, MultiVegetaArgs]
+
+ class TestGenerationParameters(typing.TypedDict):
+ flood_version: str
+ test_name: str
+ random_seed: RandomSeed | None
+ rates: typing.Sequence[int] | None
+ durations: typing.Sequence[int] | None
+ vegeta_args: VegetaArgsShorthand | None
+ network: str
+
+ # LoadTest = typing.Sequence[VegetaAttack]
+ class LoadTest(typing.TypedDict):
+ test_parameters: TestGenerationParameters
+ attacks: typing.Sequence[VegetaAttack]
+
+ class LoadTestColumnWise(typing.TypedDict):
+ rates: typing.Sequence[int]
+ durations: typing.Sequence[int]
+ calls: typing.Sequence[typing.Sequence[typing.Any]]
+ vegeta_args: typing.Sequence[typing.Any]
+
+ LoadTestMode = typing.Literal['stress', 'spike', 'soak']
+
+ LoadTestGenerator = typing.Callable[..., typing.Sequence[VegetaAttack]]
+ MultiLoadTestGenerator = typing.Callable[..., typing.Mapping[str, LoadTest]]
+
+ #
+ # # load tests outputs
+ #
+
+ class RawLoadTestOutputDatum(typing.TypedDict):
+ latencies: typing.Mapping[str, float]
+ bytes_in: typing.Mapping[str, int]
+ bytes_out: typing.Mapping[str, int]
+ earliest: str
+ latest: str
+ end: str
+ duration: int
+ wait: int
+ requests: int
+ rate: float
+ throughput: float
+ success: int
+ status_codes: typing.Mapping[str, int]
+ errors: typing.Sequence[str]
+ first_request_timestamp: str
+ last_request_timestamp: str
+ last_response_timestamp: str
+ final_wait_time: int
+
+ class LoadTestOutputDatum(typing.TypedDict):
+ target_rate: int
+ actual_rate: float | None
+ target_duration: int
+ actual_duration: float | None
+ requests: int
+ throughput: float | None
+ success: float | None
+ min: float | None
+ mean: float | None
+ p50: float | None
+ p90: float | None
+ p95: float | None
+ p99: float | None
+ max: float | None
+ status_codes: typing.Mapping[str, int]
+ errors: typing.Sequence[str]
+ first_request_timestamp: str | None
+ last_request_timestamp: str | None
+ last_response_timestamp: str | None
+ final_wait_time: float | None
+ # additional deep keys
+ deep_raw_output: str | None
+ deep_metrics: typing.Mapping[
+ ResponseCategory, LoadTestDeepOutputDatum
+ ] | None
+ deep_rpc_error_pairs: typing.Sequence[ErrorPair] | None
+
+ ResponseCategory = typing.Literal['all', 'successful', 'failed']
+ ErrorPair = tuple[typing.Any, typing.Any]
+
+ class LoadTestDeepOutputDatum(typing.TypedDict):
+ target_rate: int
+ actual_rate: float | None
+ target_duration: int
+ actual_duration: float | None
+ requests: int
+ throughput: float | None
+ success: float | None
+ min: float | None
+ mean: float | None
+ p50: float | None
+ p90: float | None
+ p95: float | None
+ p99: float | None
+ max: float | None
+ status_codes: typing.Mapping[str, int]
+ errors: typing.Sequence[str]
+ first_request_timestamp: str | None
+ last_request_timestamp: str | None
+ last_response_timestamp: str | None
+ final_wait_time: float | None
+ # additional deep keys:
+ n_invalid_json_errors: int
+ n_rpc_errors: int
+
+ class LoadTestOutput(typing.TypedDict):
+ target_rate: typing.Sequence[int]
+ actual_rate: typing.Sequence[float | None]
+ target_duration: typing.Sequence[int]
+ actual_duration: typing.Sequence[float | None]
+ requests: typing.Sequence[int]
+ throughput: typing.Sequence[float | None]
+ success: typing.Sequence[float | None]
+ min: typing.Sequence[float | None]
+ mean: typing.Sequence[float | None]
+ p50: typing.Sequence[float | None]
+ p90: typing.Sequence[float | None]
+ p95: typing.Sequence[float | None]
+ p99: typing.Sequence[float | None]
+ max: typing.Sequence[float | None]
+ status_codes: typing.Sequence[typing.Mapping[str, int]]
+ errors: typing.Sequence[typing.Sequence[str]]
+ first_request_timestamp: typing.Sequence[str | None]
+ last_request_timestamp: typing.Sequence[str | None]
+ last_response_timestamp: typing.Sequence[str | None]
+ final_wait_time: typing.Sequence[float | None]
+ # additional deep keys
+ deep_raw_output: typing.Sequence[str | None] | None
+ deep_metrics: typing.Mapping[
+ ResponseCategory, LoadTestDeepOutput
+ ] | None
+ deep_rpc_error_pairs: typing.Sequence[
+ typing.Sequence[ErrorPair] | None
+ ] | None
+
+ class LoadTestDeepOutput(typing.TypedDict):
+ target_rate: typing.Sequence[int]
+ actual_rate: typing.Sequence[float | None]
+ target_duration: typing.Sequence[int]
+ actual_duration: typing.Sequence[float | None]
+ requests: typing.Sequence[int]
+ throughput: typing.Sequence[float | None]
+ success: typing.Sequence[float | None]
+ min: typing.Sequence[float | None]
+ mean: typing.Sequence[float | None]
+ p50: typing.Sequence[float | None]
+ p90: typing.Sequence[float | None]
+ p95: typing.Sequence[float | None]
+ p99: typing.Sequence[float | None]
+ max: typing.Sequence[float | None]
+ status_codes: typing.Sequence[typing.Mapping[str, int]]
+ errors: typing.Sequence[typing.Sequence[str]]
+ first_request_timestamp: typing.Sequence[str | None]
+ last_request_timestamp: typing.Sequence[str | None]
+ last_response_timestamp: typing.Sequence[str | None]
+ final_wait_time: typing.Sequence[float | None]
+ # additional deep keys:
+ n_invalid_json_errors: typing.Sequence[int]
+ n_rpc_errors: typing.Sequence[int]
+
+ RunType = typing.Literal['single_test'] # noqa: F821
+ DeepOutput = typing.Literal['raw', 'metrics']
+
+ class SingleRunTestPayload(typing.TypedDict):
+ flood_version: str
+ type: RunType
+ name: str
+ test_parameters: TestGenerationParameters
+
+ class SingleRunResultsPayload(typing.TypedDict):
+ flood_version: str
+ dependency_versions: typing.Mapping[str, str | None]
+ cli_args: typing.Sequence[str]
+ type: RunType
+ t_run_start: float
+ t_run_end: float
+ nodes: Nodes
+ results: typing.Mapping[str, LoadTestOutput]
+
+ # runner outputs
+
+ class RunOutput(typing.TypedDict):
+ single_run: SingleRunOutput
+
+ class SingleRunOutput(typing.TypedDict):
+ output_dir: str
+ test: LoadTest | None
+ test_parameters: TestGenerationParameters
+ payload: SingleRunResultsPayload
+
+ #
+ # # env / ops management
+ #
+
+ class FloodInstallation(typing.TypedDict):
+ vegeta_path: str | None
+ flood_version: str | None
+ git_dir: str | None
+ git_commit: str | None
+ semver: str | None
+
+
+class SshError(Exception):
+ pass
+
diff --git a/flood/tests/__init__.py b/flood/tests/__init__.py
new file mode 100644
index 0000000..9a1eb2c
--- /dev/null
+++ b/flood/tests/__init__.py
@@ -0,0 +1,2 @@
+from .equality_tests import *
+from .load_tests import *
diff --git a/rpc_bench/tests/equality_tests/__init__.py b/flood/tests/equality_tests/__init__.py
similarity index 97%
rename from rpc_bench/tests/equality_tests/__init__.py
rename to flood/tests/equality_tests/__init__.py
index 9c6eb83..98aed45 100644
--- a/rpc_bench/tests/equality_tests/__init__.py
+++ b/flood/tests/equality_tests/__init__.py
@@ -1,2 +1 @@
-
from .equality_test_runs import *
diff --git a/flood/tests/equality_tests/equality_test_runs.py b/flood/tests/equality_tests/equality_test_runs.py
new file mode 100644
index 0000000..4340556
--- /dev/null
+++ b/flood/tests/equality_tests/equality_test_runs.py
@@ -0,0 +1,213 @@
+from __future__ import annotations
+
+import typing
+
+import flood
+from . import equality_test_sets
+
+
+def run_equality_test(
+ test_name: str,
+ nodes: flood.NodesShorthand,
+ *,
+ verbose: bool | int = True,
+ random_seed: flood.RandomSeed | None = None,
+ output_dir: str | None = None,
+) -> None:
+ import json
+ import os
+ import requests
+ import toolstr
+
+ nodes = flood.user_io.parse_nodes(nodes, request_metadata=True)
+ for node in nodes.values():
+ if node['remote'] is not None:
+ raise Exception('remote not supported for equality test')
+ if len(nodes) != 2:
+ raise Exception('should use two nodes in equality test')
+
+ equality_tests = equality_test_sets.get_all_equality_tests()
+
+ # get tests
+ if test_name != 'all':
+ equality_tests = [t for t in equality_tests if t[0] == test_name]
+ if not equality_tests:
+ raise NotImplementedError(
+ 'no matching test found for name "' + test_name + '"'
+ )
+
+ if output_dir is None:
+ import tempfile
+
+ output_dir = tempfile.mkdtemp()
+
+ # print preamble
+ flood.user_io.print_text_box('Equality test: ' + test_name)
+ flood.user_io.print_bullet(key='methods', value='')
+ for test in equality_tests:
+ flood.user_io.print_bullet(
+ key=test[0], value='', colon_str='', indent=4
+ )
+ flood.user_io.print_bullet(key='output_dir', value=output_dir)
+ flood.user_io.print_bullet(key='nodes', value='')
+ styles = flood.user_io.styles
+ for n, node in enumerate(nodes.values()):
+ toolstr.print(
+ toolstr.add_style(str(n + 1), styles['metavar']) + '. ' + str(node),
+ indent=4,
+ style=styles['description'],
+ )
+
+ # run test
+ successful = []
+ calls = {}
+ call_node_responses: typing.Any = {}
+ headers = {'Content-Type': 'application/json', 'User-Agent': 'flood'}
+ for test in equality_tests:
+ # create call
+ test_name, constructor, args, kwargs = test
+ call = constructor(*args, **kwargs)
+ calls[test_name] = call
+ call_node_responses.setdefault(test_name, {})
+
+ # dispatch call
+ results = []
+ responses = []
+ for node in nodes.values():
+ response = None
+ try:
+ response = requests.post(
+ url=node['url'], data=json.dumps(call), headers=headers
+ )
+ response_data = response.json()
+ if 'result' in response_data:
+ result = response_data['result']
+ else:
+ result = None
+ except Exception:
+ result = None
+ responses.append(response)
+ results.append(result)
+ call_node_responses[test_name][node['name']] = result
+
+ # print summary
+ success = _summarize_result(
+ responses=responses,
+ results=results,
+ nodes=nodes,
+ test=test,
+ call=call,
+ )
+ if success:
+ successful.append(test_name)
+ failed = [test[0] for test in equality_tests if test[0] not in successful]
+
+ # save output file
+ summary = {
+ 'calls': calls,
+ 'successful': successful,
+ 'failed': failed,
+ 'responses': call_node_responses,
+ }
+ file_path = os.path.join(output_dir, 'equality_results.json')
+ with open(file_path, 'w') as f:
+ json.dump(summary, f)
+
+ # summarize test
+ print()
+ flood.user_io.print_text_box('Equality Test Summary')
+ print()
+ flood.user_io.print_header(
+ 'No differences detected (n = ' + str(len(successful)) + ')'
+ )
+ if len(successful) == 0:
+ print('[none]')
+ else:
+ for name in sorted(successful):
+ flood.user_io.print_bullet(key=name, value='', colon_str='')
+ print()
+ flood.user_io.print_header(
+ 'Differences detected (n = ' + str(len(failed)) + ')'
+ )
+ if len(failed) == 0:
+ print('[none]')
+ else:
+ for name in sorted(failed):
+ flood.user_io.print_bullet(key=name, value='', colon_str='')
+ print()
+ toolstr.print('summary saved to: ' + file_path, style=styles['comment'])
+
+
+def _summarize_result(
+ responses: typing.Sequence[typing.Any],
+ results: typing.Sequence[typing.Any],
+ nodes: flood.Nodes,
+ test: flood.EqualityTest,
+ call: typing.Mapping[str, typing.Any],
+) -> bool:
+ import toolstr
+
+ styles = flood.user_io.styles
+
+ test_name, constructor, args, kwargs = test
+
+ if results[0] is None or not toolstr.nested_equal(results[0], results[1]):
+ print()
+ flood.user_io.print_text_box('Discrepancies in ' + test_name)
+ print()
+ flood.user_io.print_header('args')
+ if len(args) > 0:
+ for arg in args:
+ flood.user_io.print_bullet(key=arg, value='', colon_str='')
+ if len(kwargs) > 0:
+ for key, value in kwargs.items():
+ flood.user_io.print_bullet(key=key, value=value)
+ print()
+ flood.user_io.print_header('call')
+ print(call)
+
+ if any(result is None for result in results):
+ print()
+ for node, result, response in zip(nodes.values(), results, responses):
+ if result is None:
+ toolstr.print(node['name'] + ' failed', style=styles['title'])
+ if response is None:
+ print('response: no response')
+ elif response.status_code == 200:
+ print('response:', response.json())
+ else:
+ print('response: status code ', response.status_code)
+
+ if results[0] is None or results[1] is None:
+ return False
+
+ if len(results) == 2:
+ node0, node1 = list(nodes.values())
+ print()
+ flood.user_io.print_header('differences in response')
+ as_str = toolstr.create_nested_diff_str(
+ lhs=results[0],
+ rhs=results[1],
+ lhs_name=node0['name'],
+ rhs_name=node1['name'],
+ styles=styles,
+ indent=4,
+ )
+ lines = [line for line in as_str.split('\n')]
+ max_lines = 100
+ if len(lines) > max_lines:
+ as_str = (
+ '\n'.join(lines[:max_lines])
+ + '\n... (skipping '
+ + str(len(lines) - max_lines)
+ + ' lines)'
+ )
+ toolstr.print(as_str)
+ else:
+ print('omitting details when >2 nodes tested')
+
+ return False
+
+ else:
+ return True
+
diff --git a/flood/tests/equality_tests/equality_test_sets.py b/flood/tests/equality_tests/equality_test_sets.py
new file mode 100644
index 0000000..c4289b2
--- /dev/null
+++ b/flood/tests/equality_tests/equality_test_sets.py
@@ -0,0 +1,481 @@
+from __future__ import annotations
+
+import typing
+
+import flood
+
+
+def get_all_equality_tests(
+ start_block: int = 10_000_000,
+ end_block: int = 16_000_000,
+ range_size: int = 100,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.EqualityTest]:
+ return list(
+ get_vanilla_equality_tests(
+ start_block=start_block,
+ end_block=end_block,
+ range_size=range_size,
+ random_seed=random_seed,
+ )
+ ) + list(
+ get_trace_equality_tests(
+ start_block=start_block,
+ end_block=end_block,
+ random_seed=random_seed,
+ )
+ )
+
+
+def get_vanilla_equality_tests(
+ start_block: int = 10_000_000,
+ end_block: int = 16_000_000,
+ range_size: int = 100,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.EqualityTest]:
+ import ctc.rpc
+
+ start_block, end_block = flood.generators.generate_block_ranges(
+ n=1,
+ range_size=range_size,
+ start_block=start_block,
+ end_block=end_block,
+ random_seed=random_seed,
+ )[0]
+
+ return [
+ (
+ 'eth_getChainId',
+ ctc.rpc.construct_eth_chain_id,
+ [],
+ {},
+ ),
+ (
+ 'eth_getLogs',
+ ctc.rpc.construct_eth_get_logs,
+ [],
+ {
+ 'address': '0x6b175474e89094c44da98b954eedeac495271d0f',
+ 'start_block': start_block,
+ 'end_block': end_block,
+ },
+ ),
+ (
+ 'eth_gasPrice',
+ ctc.rpc.construct_eth_chain_id,
+ [],
+ {},
+ ),
+ (
+ 'eth_getBalance',
+ ctc.rpc.construct_eth_get_balance,
+ ['0xeb27d00030033c29307daa483171d22eb0c93342'],
+ # ['0xe540c45c504b348ad4d6eb9344e6cfa07c959be6'],
+ {
+ 'block_number': start_block,
+ },
+ ),
+ (
+ 'eth_getTransactionCount',
+ ctc.rpc.construct_eth_get_transaction_count,
+ ['0xeb27d00030033c29307daa483171d22eb0c93342'],
+ {
+ 'block_number': start_block,
+ },
+ ),
+ (
+ 'eth_getStorageAt',
+ ctc.rpc.construct_eth_get_storage_at,
+ [],
+ {
+ 'address': '0x4d9079bb4165aeb4084c526a32695dcfd2f77381',
+ 'position': '0x0000000000000000000000000000000000000000000000000000000000000002', # noqa: E501
+ 'block_number': start_block,
+ },
+ ),
+ (
+ 'eth_getCode',
+ ctc.rpc.construct_eth_get_code,
+ ['0x4d9079bb4165aeb4084c526a32695dcfd2f77381'],
+ {},
+ ),
+ (
+ 'eth_getBlockByNumber',
+ ctc.rpc.construct_eth_get_block_by_number,
+ [start_block],
+ {},
+ ),
+ (
+ 'eth_getBlockByHash',
+ ctc.rpc.construct_eth_get_block_by_hash,
+ [
+ '0x3dc4ef568ae2635db1419c5fec55c4a9322c05302ae527cd40bff380c1d465dd' # noqa: E501
+ ],
+ {},
+ ),
+ (
+ 'eth_getTransactionByHash',
+ ctc.rpc.construct_eth_get_transaction_by_hash,
+ [
+ '0xd01212e8ab48d2fd2ea9c4f33f8670fd1cf0cfb09d2e3c6ceddfaf54152386e5' # noqa: E501
+ ],
+ {},
+ ),
+ (
+ 'eth_getTransactionReceipt',
+ ctc.rpc.construct_eth_get_transaction_receipt,
+ [
+ '0xd01212e8ab48d2fd2ea9c4f33f8670fd1cf0cfb09d2e3c6ceddfaf54152386e5' # noqa: E501
+ ],
+ {},
+ ),
+ (
+ 'eth_call',
+ ctc.rpc.construct_eth_call,
+ [],
+ {
+ 'to_address': '0x6b175474e89094c44da98b954eedeac495271d0f',
+ 'function_abi': {
+ 'constant': True,
+ 'inputs': [
+ {
+ 'internalType': 'address',
+ 'name': '',
+ 'type': 'address',
+ }
+ ],
+ 'name': 'balanceOf',
+ 'outputs': [
+ {
+ 'internalType': 'uint256',
+ 'name': '',
+ 'type': 'uint256',
+ }
+ ],
+ 'payable': False,
+ 'stateMutability': 'view',
+ 'type': 'function',
+ },
+ 'function_parameters': [
+ '0x5d3a536e4d6dbd6114cc1ead35777bab948e3643'
+ ],
+ 'block_number': start_block,
+ },
+ ),
+ (
+ 'eth_feeHistory',
+ ctc.rpc.construct_eth_fee_history,
+ [
+ 512,
+ int(start_block),
+ ],
+ {},
+ ),
+ ]
+
+
+def get_trace_equality_tests(
+ start_block: int = 10_000_000,
+ end_block: int = 16_000_000,
+ random_seed: flood.RandomSeed | None = None,
+) -> typing.Sequence[flood.EqualityTest]:
+ """
+ ## Trace
+ https://docs.alchemy.com/reference/trace-api
+ https://www.quicknode.com/docs/ethereum/trace_rawTransaction
+ https://openethereum.github.io/JSONRPC-trace-module
+
+ ## Debug
+ https://docs.alchemy.com/reference/debug-api-endpoints
+ https://www.quicknode.com/docs/ethereum/debug_traceBlockByNumber
+
+ not currently testing:
+ - trace_rawTransaction (not sure whether this is working in Erigon)
+ - trace_filter
+ - trace_get
+ """
+ import ctc.rpc
+
+ block_number = flood.generators.generate_block_numbers(
+ n=1,
+ start_block=start_block,
+ end_block=end_block,
+ random_seed=random_seed,
+ )[0]
+
+ function_abi = {
+ 'constant': True,
+ 'inputs': [{'internalType': 'address', 'name': '', 'type': 'address'}],
+ 'name': 'balanceOf',
+ 'outputs': [{'internalType': 'uint256', 'name': '', 'type': 'uint256'}],
+ 'payable': False,
+ 'stateMutability': 'view',
+ 'type': 'function',
+ }
+
+ return [
+ (
+ 'trace_transaction',
+ ctc.rpc.construct_trace_transaction,
+ [
+ '0xd01212e8ab48d2fd2ea9c4f33f8670fd1cf0cfb09d2e3c6ceddfaf54152386e5' # noqa: E501
+ ],
+ {},
+ ),
+ (
+ 'trace_block',
+ ctc.rpc.construct_trace_block,
+ [],
+ {'block_number': block_number},
+ ),
+ (
+ 'trace_call',
+ ctc.rpc.construct_trace_call,
+ [],
+ {
+ 'to_address': '0x6b175474e89094c44da98b954eedeac495271d0f',
+ 'trace_type': ['trace'],
+ 'function_abi': function_abi,
+ 'function_parameters': [
+ '0x5d3a536e4d6dbd6114cc1ead35777bab948e3643'
+ ],
+ 'block_number': block_number,
+ },
+ ),
+ (
+ 'trace_call_state_diff',
+ ctc.rpc.construct_trace_call,
+ [],
+ {
+ 'to_address': '0x6b175474e89094c44da98b954eedeac495271d0f',
+ 'trace_type': ['stateDiff'],
+ 'function_abi': function_abi,
+ 'function_parameters': [
+ '0x5d3a536e4d6dbd6114cc1ead35777bab948e3643'
+ ],
+ 'block_number': block_number,
+ },
+ ),
+ (
+ 'trace_call_vm_trace',
+ ctc.rpc.construct_trace_call,
+ [],
+ {
+ 'to_address': '0x6b175474e89094c44da98b954eedeac495271d0f',
+ 'trace_type': ['vmTrace'],
+ 'function_abi': function_abi,
+ 'function_parameters': [
+ '0x5d3a536e4d6dbd6114cc1ead35777bab948e3643'
+ ],
+ 'block_number': block_number,
+ },
+ ),
+ (
+ 'trace_call_many',
+ ctc.rpc.construct_trace_call_many,
+ [],
+ {
+ 'calls': [
+ {
+ 'to_address': '0x6b175474e89094c44da98b954eedeac495271d0f', # noqa: E501
+ 'function_abi': function_abi,
+ 'function_parameters': [
+ '0x5d3a536e4d6dbd6114cc1ead35777bab948e3643'
+ ],
+ 'trace_type': ['trace'],
+ },
+ ],
+ 'trace_type': ['trace'],
+ },
+ ),
+ (
+ 'trace_call_many_state_diff',
+ ctc.rpc.construct_trace_call_many,
+ [],
+ {
+ 'calls': [
+ {
+ 'to_address': '0x6b175474e89094c44da98b954eedeac495271d0f', # noqa: E501
+ 'function_abi': function_abi,
+ 'function_parameters': [
+ '0x5d3a536e4d6dbd6114cc1ead35777bab948e3643'
+ ],
+ 'trace_type': ['stateDiff'],
+ },
+ ],
+ 'trace_type': ['stateDiff'],
+ },
+ ),
+ (
+ 'trace_call_many_vm_trace',
+ ctc.rpc.construct_trace_call_many,
+ [],
+ {
+ 'calls': [
+ {
+ 'to_address': '0x6b175474e89094c44da98b954eedeac495271d0f', # noqa: E501
+ 'function_abi': function_abi,
+ 'function_parameters': [
+ '0x5d3a536e4d6dbd6114cc1ead35777bab948e3643'
+ ],
+ 'trace_type': ['vmTrace'],
+ },
+ ],
+ 'trace_type': ['vmTrace'],
+ },
+ ),
+ # (
+ # 'trace_filter',
+ # ctc.rpc.construct_trace_filter,
+ # [],
+ # {
+ # 'to_addresses': ['0x6b175474e89094c44da98b954eedeac495271d0f'], # noqa: E501
+ # 'start_block': 10_000_000,
+ # 'end_block': 10_000_001,
+ # 'count': 10,
+ # }
+ # ),
+ # (
+ # 'trace_get',
+ # ctc.rpc.construct_trace_get,
+ # [],
+ # {}
+ # ),
+ # (
+ # 'trace_raw_transaction',
+ # ctc.rpc.construct_trace_raw_transaction,
+ # ['0xd46e8dd67c5d32be8d46e8dd67c5d32be8058bb8eb970870f072445675058bb8eb970870f072445675'], # noqa: E501
+ # {'trace_type': ['trace']},
+ # ),
+ (
+ 'trace_replay_block_transactions',
+ ctc.rpc.construct_trace_replay_block_transactions,
+ [],
+ {
+ 'block_number': block_number,
+ 'trace_type': ['trace'],
+ },
+ ),
+ (
+ 'trace_replay_block_transactions_state_diff',
+ ctc.rpc.construct_trace_replay_block_transactions,
+ [],
+ {
+ 'block_number': block_number,
+ 'trace_type': ['stateDiff'],
+ },
+ ),
+ (
+ 'trace_replay_block_transactions_vm_traces',
+ ctc.rpc.construct_trace_replay_block_transactions,
+ [],
+ {
+ 'block_number': block_number,
+ 'trace_type': ['vmTrace'],
+ },
+ ),
+ (
+ 'trace_replay_transaction',
+ ctc.rpc.construct_trace_replay_transaction,
+ [
+ '0xd01212e8ab48d2fd2ea9c4f33f8670fd1cf0cfb09d2e3c6ceddfaf54152386e5' # noqa: E501
+ ],
+ {'trace_type': ['trace']},
+ ),
+ (
+ 'trace_replay_transaction_state_diff',
+ ctc.rpc.construct_trace_replay_transaction,
+ [
+ '0xd01212e8ab48d2fd2ea9c4f33f8670fd1cf0cfb09d2e3c6ceddfaf54152386e5' # noqa: E501
+ ],
+ {'trace_type': ['stateDiff']},
+ ),
+ (
+ 'trace_replay_transaction_vm_trace',
+ ctc.rpc.construct_trace_replay_transaction,
+ [
+ '0xd01212e8ab48d2fd2ea9c4f33f8670fd1cf0cfb09d2e3c6ceddfaf54152386e5' # noqa: E501
+ ],
+ {'trace_type': ['vmTrace']},
+ ),
+ # debug
+ (
+ 'debug_traceBlockByHash',
+ ctc.rpc.construct_debug_trace_block_by_hash,
+ [
+ "0x97b49e43632ac70c46b4003434058b18db0ad809617bd29f3448d46ca9085576",
+ {"tracer": "callTracer", 'tracerConfig': {}},
+ ],
+ {},
+ ),
+ (
+ 'debug_traceBlockByHash_prestateTracer',
+ ctc.rpc.construct_debug_trace_block_by_hash,
+ [
+ "0x97b49e43632ac70c46b4003434058b18db0ad809617bd29f3448d46ca9085576",
+ {"tracer": "prestateTracer"},
+ ],
+ {},
+ ),
+ (
+ 'debug_traceBlockByNumber',
+ ctc.rpc.construct_debug_trace_block_by_number,
+ [
+ '0xccde12',
+ {'tracer': 'callTracer', 'tracerConfig': {}},
+ ],
+ {},
+ ),
+ (
+ 'debug_traceBlockByNumber_prestateTracer',
+ ctc.rpc.construct_debug_trace_block_by_number,
+ [
+ '0xccde12',
+ {'tracer': 'prestateTracer'},
+ ],
+ {},
+ ),
+ (
+ 'debug_traceCall',
+ ctc.rpc.construct_debug_trace_call,
+ [],
+ {
+ 'to_address': '0x6b175474e89094c44da98b954eedeac495271d0f',
+ 'call_data': '0x70a082310000000000000000000000006e0d01a76c3cf4288372a29124a26d4353ee51be', # noqa: E501
+ 'block_number': '0xc65d40',
+ 'gas_price': '0x6ba9382b14',
+ 'trace_type': {'tracer': 'callTracer'},
+ },
+ ),
+ (
+ 'debug_traceCall_prestateTracer',
+ ctc.rpc.construct_debug_trace_call,
+ [],
+ {
+ 'to_address': '0x6b175474e89094c44da98b954eedeac495271d0f',
+ 'call_data': '0x70a082310000000000000000000000006e0d01a76c3cf4288372a29124a26d4353ee51be', # noqa: E501
+ 'block_number': '0xc65d40',
+ 'gas_price': '0x6ba9382b14',
+ 'trace_type': {'tracer': 'prestateTracer'},
+ },
+ ),
+ (
+ 'debug_traceTransaction',
+ ctc.rpc.construct_debug_trace_transaction,
+ [
+ '0x9e63085271890a141297039b3b711913699f1ee4db1acb667ad7ce304772036b',
+ {'tracer': 'callTracer', 'tracerConfig': {}},
+ ],
+ {},
+ ),
+ (
+ 'debug_traceTransaction_prestateTracer',
+ ctc.rpc.construct_debug_trace_transaction,
+ [
+ '0x9e63085271890a141297039b3b711913699f1ee4db1acb667ad7ce304772036b',
+ {'tracer': 'prestateTracer', 'tracerConfig': {}},
+ ],
+ {},
+ ),
+ ]
+
diff --git a/rpc_bench/tests/load_tests/__init__.py b/flood/tests/load_tests/__init__.py
similarity index 85%
rename from rpc_bench/tests/load_tests/__init__.py
rename to flood/tests/load_tests/__init__.py
index 2a8ec95..49c63d6 100644
--- a/rpc_bench/tests/load_tests/__init__.py
+++ b/flood/tests/load_tests/__init__.py
@@ -1,3 +1,4 @@
+from .deep_utils import *
from .load_test_construction import *
from .load_test_plots import *
from .load_test_reports import *
diff --git a/flood/tests/load_tests/deep_utils.py b/flood/tests/load_tests/deep_utils.py
new file mode 100644
index 0000000..848ae56
--- /dev/null
+++ b/flood/tests/load_tests/deep_utils.py
@@ -0,0 +1,341 @@
+from __future__ import annotations
+
+import typing
+from ... import spec
+
+if typing.TYPE_CHECKING:
+ import polars as pl
+
+
+def compute_deep_datum(
+ raw_output: bytes,
+ target_rate: int,
+ target_duration: int,
+ calls: typing.Sequence[typing.Any],
+) -> tuple[
+ typing.Mapping[spec.ResponseCategory, spec.LoadTestDeepOutputDatum],
+ typing.Sequence[spec.ErrorPair],
+]:
+ import polars as pl
+
+ # convert to dataframe
+ all_df = _convert_raw_vegeta_output_to_dataframe(raw_output)
+
+ # add error columns
+ rpc_error = []
+ invalid_json_error = []
+ for status_code, response in zip(all_df['status_code'], all_df['response']):
+ if status_code == 200:
+ try:
+ import json
+ import base64
+
+ decoded = json.loads(base64.b64decode(response))
+
+ invalid_json_error.append(False)
+ if decoded.get('result') is None:
+ rpc_error.append(True)
+ else:
+ rpc_error.append(False)
+
+ except Exception:
+ invalid_json_error.append(True)
+ rpc_error.append(False)
+ else:
+ invalid_json_error.append(False)
+ rpc_error.append(False)
+ all_df = all_df.with_columns(
+ pl.Series('invalid_json_error', invalid_json_error),
+ pl.Series('rpc_error', rpc_error),
+ )
+ all_df = all_df.with_columns(
+ (
+ (pl.col('status_code') == 200)
+ & ~pl.col('invalid_json_error')
+ & ~pl.col('rpc_error')
+ ).alias('deep_success')
+ )
+
+ # get error pairs
+ rpc_error_pairs: typing.Sequence[spec.ErrorPair] = []
+ rpc_error_pairs = _gather_error_pairs(df=all_df, calls=calls)
+
+ # compute sample metrics
+ category_data = {}
+ dataframes: list[tuple[spec.ResponseCategory, pl.DataFrame]] = [
+ ('all', all_df),
+ ('successful', all_df.filter(pl.col('deep_success'))),
+ ('failed', all_df.filter(~pl.col('deep_success'))),
+ ]
+ for category, df in dataframes:
+ category_data[category] = _compute_raw_output_sample_metrics(
+ df=df, target_rate=target_rate, target_duration=target_duration
+ )
+
+ return category_data, rpc_error_pairs
+
+
+def _convert_raw_vegeta_output_to_dataframe(raw_output: bytes) -> pl.DataFrame:
+ """convert raw vegeta attack output to dataframe, 1 row per response"""
+ import io
+ import subprocess
+ import polars as pl
+
+ cmd = 'vegeta encode --to csv'
+ report_output = (
+ subprocess.check_output(cmd.split(' '), input=raw_output)
+ .decode()
+ .strip()
+ )
+
+ buf = io.StringIO()
+ buf.write(report_output)
+ buf.seek(0)
+
+ schema = [
+ 'timestamp',
+ 'status_code',
+ 'latency',
+ 'bytes_out',
+ 'bytes_in',
+ 'error',
+ 'response',
+ 'name',
+ 'index',
+ 'method',
+ 'url',
+ 'response_headers',
+ ]
+ dtypes = {
+ 'bytes_in': pl.Int64,
+ 'bytes_out': pl.Int64,
+ 'error': pl.Utf8,
+ 'index': pl.Int64,
+ 'latency': pl.Int64,
+ 'method': pl.Utf8,
+ 'name': pl.Utf8,
+ 'response': pl.Utf8,
+ 'response_headers': pl.Utf8,
+ 'status_code': pl.Int64,
+ 'timestamp': pl.Int64,
+ 'url': pl.Utf8,
+ }
+ return pl.read_csv(buf, new_columns=schema, has_header=False, dtypes=dtypes)
+
+
+def _gather_error_pairs(
+ df: pl.DataFrame, calls: typing.Sequence[typing.Any]
+) -> typing.Sequence[spec.ErrorPair]:
+ import polars as pl
+
+ calls_by_id = {}
+ for call in calls:
+ call_id = call.get('id')
+ if call_id is None:
+ raise Exception('id not specified for call')
+ elif call_id in calls_by_id:
+ raise Exception('duplicate call for id')
+ calls_by_id[call_id] = call
+
+ responses = df.filter(pl.col('rpc_error'))['response']
+
+ pairs = []
+ for response in responses:
+ pairs.append((None, response))
+
+ return pairs
+
+
+def _compute_raw_output_sample_metrics(
+ df: pl.DataFrame, target_rate: int, target_duration: int
+) -> spec.LoadTestDeepOutputDatum:
+ """convert standard test metrics from vegeta raw output dataframe"""
+ if len(df) == 0:
+ return {
+ 'target_rate': target_rate,
+ 'actual_rate': 0,
+ 'target_duration': target_duration,
+ 'actual_duration': None,
+ 'requests': 0,
+ 'throughput': None,
+ 'success': None,
+ 'min': None,
+ 'mean': None,
+ 'p50': None,
+ 'p90': None,
+ 'p95': None,
+ 'p99': None,
+ 'max': None,
+ 'status_codes': {},
+ 'errors': [],
+ 'first_request_timestamp': None,
+ 'last_request_timestamp': None,
+ 'last_response_timestamp': None,
+ 'final_wait_time': None,
+ 'n_invalid_json_errors': 0,
+ 'n_rpc_errors': 0,
+ }
+
+ import polars as pl
+
+ actual_rate = (
+ pl.count()
+ / (pl.col('timestamp').max() - pl.col('timestamp').min())
+ * 1e9
+ )
+ actual_rate = actual_rate.alias('actual_rate')
+ successful = df.filter(pl.col('status_code') == 200)
+ total_duration = ( # type: ignore
+ df.select(pl.col('timestamp') + pl.col('latency')).max()
+ ) - df['timestamp'].min()
+ total_duration = total_duration.rows()[0][0]
+ status_codes = {}
+ for k, v in df['status_code'].value_counts().rows():
+ status_codes[str(k)] = v
+ errors = df.filter(~pl.col('error').is_null())['error'].unique().to_list()
+
+ metrics_df = df.select(
+ pl.lit(target_rate).alias('target_rate'),
+ actual_rate,
+ pl.lit(target_duration).alias('target_duration'),
+ (pl.max('timestamp') - pl.min('timestamp')).alias('actual_duration')
+ / 1e9,
+ pl.count().alias('requests'),
+ pl.lit(len(successful) / total_duration).alias('throughput') * 1e9,
+ (len(successful) / pl.count()).alias('success'),
+ pl.min('latency').alias('min') / 1e9,
+ pl.mean('latency').alias('mean') / 1e9,
+ pl.median('latency').alias('p50') / 1e9,
+ pl.quantile('latency', 0.90).alias('p90') / 1e9,
+ pl.quantile('latency', 0.95).alias('p95') / 1e9,
+ pl.quantile('latency', 0.99).alias('p99') / 1e9,
+ pl.max('latency').alias('max') / 1e9,
+ pl.struct(**status_codes, eager=True).alias('status_codes'),
+ pl.Series([errors]).alias('errors'),
+ (pl.col('timestamp').min() / 1e3)
+ .cast(pl.Datetime)
+ .cast(str)
+ .alias('first_request_timestamp'),
+ (pl.col('timestamp').max() / 1e3)
+ .cast(pl.Datetime)
+ .cast(str)
+ .alias('last_request_timestamp'),
+ ((pl.col('timestamp') + pl.col('latency')) / 1e3)
+ .max()
+ .cast(pl.Datetime)
+ .cast(str)
+ .alias('last_response_timestamp'),
+ (
+ (pl.col('timestamp') + pl.col('latency')).max()
+ - pl.max('timestamp')
+ ).alias('final_wait_time')
+ / 1e9,
+ )
+
+ output: spec.LoadTestDeepOutputDatum = metrics_df.to_dicts()[0] # type: ignore # noqa: E501
+ output['n_invalid_json_errors'] = int(df['invalid_json_error'].sum())
+ output['n_rpc_errors'] = int(df['rpc_error'].sum())
+
+ return output
+
+
+# def compute_raw_output_metrics(
+# raw_output: typing.Mapping[str, pl.DataFrame],
+# results: typing.Mapping[str, spec.LoadTestOutput],
+# ) -> typing.Mapping[str, spec.LoadTestOutput]:
+# import polars as pl
+
+# metrics: typing.MutableMapping[str, spec.LoadTestOutput] = {}
+# for node_name, node_results in results.items():
+# node_metrics = []
+# for i, (target_rate, target_duration) in enumerate(
+# zip(node_results['target_rate'], node_results['target_duration'])
+# ):
+# sample_metrics = compute_raw_output_sample_metrics(
+# df=raw_output[node_name].filter(pl.col('sample_index') == i),
+# target_rate=target_rate,
+# target_duration=target_duration,
+# )
+# node_metrics.append(sample_metrics)
+# metrics[node_name] = {
+# metric: [sample_metrics[metric] for sample_metrics in node_metrics] # type: ignore # noqa: E501
+# for metric in node_results.keys()
+# }
+# return metrics
+
+
+# def test_run_to_deep_raw_vegeta_output(
+# results: typing.Mapping[str, spec.LoadTestOutput],
+# sample_index: int | typing.Sequence[int] | None = None,
+# ) -> typing.Mapping[str, pl.DataFrame]:
+# import polars as pl
+
+# node_dfs = {}
+# for node_name, node_results in results.items():
+# raw_output = node_results['deep_raw_output']
+# if raw_output is None:
+# raise Exception('raw_outputs were not saved for test')
+# else:
+# if sample_index is not None:
+# if isinstance(sample_index, int):
+# indices = [sample_index]
+# elif isinstance(sample_index, list):
+# indices = sample_index
+# else:
+# raise Exception('invalid format for sample_index')
+# else:
+# indices = list(range(len(raw_output)))
+# dfs = []
+# for index in indices:
+# item = raw_output[index]
+# if item is None:
+# raise Exception('raw_outputs were not saved for test')
+# decoded = decode_raw_vegeta_output(item)
+# df = convert_raw_vegeta_output_to_dataframe(decoded)
+# df = df.with_columns(pl.lit(index).alias('sample_index'))
+# dfs.append(df)
+# node_dfs[node_name] = pl.concat(dfs)
+# return node_dfs
+
+
+#
+# # serde
+#
+
+
+def encode_raw_vegeta_output(raw_output: bytes) -> str:
+ """encode raw output to str for use in JSON"""
+ import base64
+ import io
+ import gzip
+
+ # gzip compress
+ buf = io.BytesIO()
+ f = gzip.GzipFile(fileobj=buf, mode='wb')
+ f.write(raw_output)
+ f.close()
+ compressed = buf.getvalue()
+
+ # encode as base64
+ as_base64 = base64.b64encode(compressed).decode('utf-8')
+
+ return as_base64
+
+
+def decode_raw_vegeta_output(encoded_output: str) -> bytes:
+ """decode raw output from str for use in JSON"""
+ import base64
+ import io
+ import gzip
+
+ # decode from base64
+ as_bytes = base64.b64decode(encoded_output.encode())
+
+ # gzip decompress
+ buf = io.BytesIO(as_bytes)
+ f = gzip.GzipFile(fileobj=buf, mode='rb')
+ decompressed = f.read()
+ f.close()
+
+ return decompressed
+
diff --git a/rpc_bench/tests/load_tests/load_test_construction.py b/flood/tests/load_tests/load_test_construction.py
similarity index 73%
rename from rpc_bench/tests/load_tests/load_test_construction.py
rename to flood/tests/load_tests/load_test_construction.py
index b4e88aa..6affa2f 100644
--- a/rpc_bench/tests/load_tests/load_test_construction.py
+++ b/flood/tests/load_tests/load_test_construction.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import typing
-import rpc_bench
+import flood
def estimate_call_count(
@@ -29,16 +29,16 @@ def estimate_call_count(
return n_calls
-def construct_load_test(
+def create_load_test(
calls: typing.Sequence[typing.Any],
rates: typing.Sequence[int],
duration: int | None = None,
durations: typing.Sequence[int] | None = None,
- vegeta_kwargs: rpc_bench.VegetaKwargs
- | typing.Sequence[rpc_bench.VegetaKwargs]
+ vegeta_args: flood.VegetaArgs
+ | typing.Sequence[flood.VegetaArgs]
| None = None,
repeat_calls: bool = False,
-) -> rpc_bench.LoadTest:
+) -> typing.Sequence[flood.VegetaAttack]:
# validate inputs
if len(rates) == 0:
raise Exception('must specify at least one rate')
@@ -51,18 +51,19 @@ def construct_load_test(
assert len(durations) == len(rates)
# pluralize singular vegeta kwargs
- if vegeta_kwargs is None:
- vegeta_kwargs = {}
- if isinstance(vegeta_kwargs, dict):
- vegeta_kwargs = [vegeta_kwargs] * len(rates)
- if not isinstance(vegeta_kwargs, list):
+ use_vegeta_args: typing.Sequence[str] | typing.Sequence[None]
+ if vegeta_args is None:
+ use_vegeta_args = [vegeta_args] * len(rates)
+ elif isinstance(vegeta_args, str):
+ use_vegeta_args = [vegeta_args] * len(rates)
+ elif isinstance(vegeta_args, list):
+ use_vegeta_args = vegeta_args
+ else:
raise Exception('invalid input')
# partition calls into individual attacks
if not repeat_calls:
- attacks_calls: typing.MutableSequence[
- typing.Sequence[rpc_bench.Call]
- ] = []
+ attacks_calls: typing.MutableSequence[typing.Sequence[flood.Call]] = []
calls_iter = iter(calls)
for rate, duration in zip(rates, durations):
n_attack_calls = rate * duration
@@ -75,15 +76,15 @@ def construct_load_test(
assert len(attacks_calls) == len(rates)
# create load tests
- load_test: list[rpc_bench.VegetaAttack] = []
+ load_test: list[flood.VegetaAttack] = []
for rate, duration, a_calls, attack_kwargs in zip(
- rates, durations, attacks_calls, vegeta_kwargs
+ rates, durations, attacks_calls, use_vegeta_args
):
- attack: rpc_bench.VegetaAttack = {
+ attack: flood.VegetaAttack = {
'rate': rate,
'duration': duration,
'calls': a_calls,
- 'vegeta_kwargs': attack_kwargs,
+ 'vegeta_args': attack_kwargs,
}
load_test.append(attack)
diff --git a/rpc_bench/tests/load_tests/load_test_plots.py b/flood/tests/load_tests/load_test_plots.py
similarity index 50%
rename from rpc_bench/tests/load_tests/load_test_plots.py
rename to flood/tests/load_tests/load_test_plots.py
index 0561042..d960220 100644
--- a/rpc_bench/tests/load_tests/load_test_plots.py
+++ b/flood/tests/load_tests/load_test_plots.py
@@ -2,53 +2,108 @@ from __future__ import annotations
import typing
-import rpc_bench
+import flood
def plot_load_test_results(
- outputs: typing.Mapping[str, rpc_bench.LoadTestOutput],
+ outputs: typing.Mapping[str, flood.LoadTestOutput]
+ | typing.Mapping[str, flood.LoadTestDeepOutput],
test_name: str,
output_dir: str | None = None,
- latency_yscale_log: bool = False,
+ latency_yscale_log: bool = True,
+ colors: typing.Mapping[str, str] | None = None,
+ title_suffix: str | None = None,
+ file_suffix: str | None = None,
+ plot_success_rate: bool = True,
+ plot_throughput: bool = True,
+ plot_latency: bool = True,
) -> None:
import os
import matplotlib.pyplot as plt # type: ignore
import toolplot
+ if title_suffix is None:
+ title_suffix = ''
+ if file_suffix is None:
+ file_suffix = ''
+
toolplot.setup_plot_formatting()
if output_dir is not None:
os.makedirs(output_dir, exist_ok=True)
- plt.figure()
- plot_load_test_success(outputs, test_name=test_name)
- if output_dir is not None:
- path = os.path.join(output_dir, 'success.png')
- plt.savefig(path)
- else:
- plt.show()
+ if plot_success_rate:
+ plt.figure()
+ plot_load_test_success(outputs, test_name=test_name, colors=colors)
+ if output_dir is not None:
+ path = os.path.join(
+ output_dir, 'success_rate' + file_suffix + '.png'
+ )
+ plt.savefig(path)
+ else:
+ plt.show()
- plt.figure()
- plot_load_test_throughput(outputs, test_name=test_name)
- if output_dir is not None:
- path = os.path.join(output_dir, 'throughput.png')
- plt.savefig(path)
- else:
- plt.show()
+ if plot_throughput:
+ plt.figure()
+ plot_load_test_throughput(outputs, test_name=test_name, colors=colors)
+ if output_dir is not None:
+ path = os.path.join(output_dir, 'throughput' + file_suffix + '.png')
+ plt.savefig(path)
+ else:
+ plt.show()
- plt.figure()
- plot_load_test_latencies(
- outputs, test_name=test_name, yscale_log=latency_yscale_log
+ if plot_latency:
+ plt.figure()
+ plot_load_test_latencies(
+ outputs,
+ test_name=test_name,
+ yscale_log=latency_yscale_log,
+ colors=colors,
+ )
+ if output_dir is not None:
+ path = os.path.join(output_dir, 'latencies' + file_suffix + '.png')
+ plt.savefig(path)
+ else:
+ plt.show()
+
+ # deep graphs
+ has_deep_outputs = any(
+ output.get('deep_metrics') is not None for output in outputs.values()
)
- if output_dir is not None:
- path = os.path.join(output_dir, 'latencies.png')
- plt.savefig(path)
- else:
- plt.show()
+ if has_deep_outputs:
+ plot_load_test_results(
+ outputs={
+ name: output['deep_metrics']['successful'] # type: ignore
+ for name, output in outputs.items()
+ },
+ test_name=test_name,
+ title_suffix=', successful calls only',
+ file_suffix='_successful_calls',
+ plot_success_rate=False,
+ plot_throughput=False,
+ output_dir=output_dir,
+ latency_yscale_log=latency_yscale_log,
+ colors=colors,
+ )
+ plot_load_test_results(
+ outputs={
+ name: output['deep_metrics']['failed'] # type: ignore
+ for name, output in outputs.items()
+ },
+ test_name='failed ' + test_name,
+ title_suffix=', failed calls only',
+ file_suffix='_failed_calls',
+ plot_success_rate=False,
+ plot_throughput=False,
+ output_dir=output_dir,
+ latency_yscale_log=latency_yscale_log,
+ colors=colors,
+ )
def plot_load_test_success(
- results: typing.Mapping[str, rpc_bench.LoadTestOutput],
+ results: typing.Mapping[str, flood.LoadTestOutput]
+ | typing.Mapping[str, flood.LoadTestDeepOutput],
colors: typing.Mapping[str, str] | None = None,
test_name: str | None = None,
) -> None:
@@ -67,10 +122,13 @@ def plot_load_test_success(
def plot_load_test_throughput(
- results: typing.Mapping[str, rpc_bench.LoadTestOutput],
+ results: typing.Mapping[str, flood.LoadTestOutput]
+ | typing.Mapping[str, flood.LoadTestDeepOutput],
colors: typing.Mapping[str, str] | None = None,
test_name: str | None = None,
) -> None:
+ import matplotlib.pyplot as plt
+
plot_load_test_result_metrics(
results=results,
metrics=['throughput'],
@@ -80,10 +138,12 @@ def plot_load_test_throughput(
ylabel='throughput\n(responses per second)',
ymin=0,
)
+ plt.legend(loc='upper left')
def plot_load_test_latencies(
- results: typing.Mapping[str, rpc_bench.LoadTestOutput],
+ results: typing.Mapping[str, flood.LoadTestOutput]
+ | typing.Mapping[str, flood.LoadTestDeepOutput],
colors: typing.Mapping[
str,
str | typing.Sequence[str] | typing.Mapping[str, str],
@@ -93,8 +153,10 @@ def plot_load_test_latencies(
test_name: str | None = None,
yscale_log: bool = False,
) -> None:
+ import matplotlib.pyplot as plt
+
if colors is None:
- colors = dict(zip(results.keys(), rpc_bench.colors.values()))
+ colors = dict(zip(results.keys(), flood.user_io.plot_colors.values()))
if yscale_log:
ymin = None
@@ -111,10 +173,12 @@ def plot_load_test_latencies(
ylabel='latency (seconds)',
yscale_log=yscale_log,
)
+ plt.legend(loc='upper left')
def plot_load_test_result_metrics(
- results: typing.Mapping[str, rpc_bench.LoadTestOutput],
+ results: typing.Mapping[str, flood.LoadTestOutput]
+ | typing.Mapping[str, flood.LoadTestDeepOutput],
metrics: typing.Sequence[str],
*,
colors: typing.Mapping[
@@ -132,17 +196,22 @@ def plot_load_test_result_metrics(
import matplotlib.pyplot as plt
import toolplot
+ plot_colors = flood.user_io.plot_colors
+
if colors is None:
- colors = {
- key: color
- for key, color in zip(results.keys(), rpc_bench.color_defaults)
- }
+ colors = {key: color for key, color in zip(results.keys(), plot_colors)}
for name, result in results.items():
# determine colors
result_colors = colors.get(name)
if isinstance(result_colors, str):
- result_colors = [result_colors] * len(metrics)
+ if result_colors in plot_colors:
+ if len(metrics) == 1:
+ result_colors = [plot_colors[result_colors][1]]
+ else:
+ result_colors = plot_colors[result_colors]
+ else:
+ result_colors = [result_colors] * len(metrics)
elif isinstance(result_colors, list):
assert len(result_colors) >= len(metrics), 'not enough colors'
elif isinstance(result_colors, dict):
diff --git a/flood/tests/load_tests/load_test_reports.py b/flood/tests/load_tests/load_test_reports.py
new file mode 100644
index 0000000..cac2a87
--- /dev/null
+++ b/flood/tests/load_tests/load_test_reports.py
@@ -0,0 +1,507 @@
+from __future__ import annotations
+
+import typing
+
+import flood
+
+if typing.TYPE_CHECKING:
+ import nbformat
+
+
+def create_load_test_report(
+ test_paths: typing.Sequence[str],
+ output_dir: str,
+ metrics: typing.Sequence[str] | None = None,
+) -> None:
+ import os
+
+ ipynb_path = os.path.join(output_dir, 'report.ipynb')
+ _create_load_test_report_ipynb(
+ test_paths=test_paths,
+ output_path=ipynb_path,
+ metrics=metrics,
+ )
+
+ _create_load_test_report_html(ipynb_path=ipynb_path)
+
+
+def print_load_test_summary(test: flood.LoadTest) -> None:
+ import toolstr
+
+ parsed = flood.user_io.parse_test_data(test)
+ rates = parsed['rates']
+ durations = parsed['durations']
+ vegeta_args = parsed['vegeta_args']
+ styles = flood.user_io.styles
+
+ toolstr.print_bullet(key='sample rates', value=rates, styles=styles)
+ if len(set(durations)) == 1:
+ toolstr.print_bullet(
+ key='sample duration',
+ value=durations[0],
+ styles=styles,
+ )
+ else:
+ toolstr.print_bullet(
+ key='sample durations', value=durations, styles=styles
+ )
+ if vegeta_args is None or len(vegeta_args) == 0:
+ toolstr.print_bullet(key='extra args', value=None, styles=styles)
+
+
+def _create_load_test_report_html(
+ ipynb_path: str,
+) -> None:
+ import subprocess
+
+ cmd = 'python3 -m nbconvert --to html --execute ' + ipynb_path
+ subprocess.call(cmd.split(' '))
+
+
+def _create_load_test_report_ipynb(
+ test_paths: typing.Sequence[str],
+ output_path: str,
+ metrics: typing.Sequence[str] | None = None,
+) -> None:
+ from flood.user_io import notebook_io
+
+ if metrics is None:
+ metrics = ['success', 'throughput', 'p50', 'p90', 'p99']
+
+ test_paths_map = {}
+ for test_path in test_paths:
+ payload = flood.load_single_run_test_payload(test_path)
+ name = payload['name']
+ test_paths_map[name] = test_path
+
+ inputs = {
+ 'test_paths': test_paths_map,
+ 'metrics': metrics,
+ 'test_names': list(test_paths_map.keys()),
+ }
+
+ notebook_io.create_notebook(
+ cell_templates=_report_template_cells,
+ output_path=output_path,
+ inputs=inputs,
+ )
+
+
+_header_template = """# `flood` Load Test Report
+
+
+{toc}
+
+### Report Generation
+
+*This report was generated by `flood`. See the `flood report --help` command for report generation options. This report can be executed as a Python notebook using the `.ipynb` version of this file.*""" # noqa: E501
+
+
+_toc_template = """### Contents
+1. [Test Summary](#Test-Summary)
+2. [Tests](#Tests)
+{toc_lines}"""
+
+
+def _create_header_cell(test_names: typing.Sequence[str]) -> str:
+ toc = _create_test_toc(test_names=test_names)
+ return _header_template.format(toc=toc)
+
+
+def _create_test_toc(test_names: typing.Sequence[str]) -> str:
+ line_template = '. [Test: {test_name}](#Test:-{test_name})\n'
+ toc = ''
+ for t, test_name in enumerate(test_names):
+ toc += ' ' + str(t + 1) + line_template.format(test_name=test_name)
+ toc = toc.rstrip()
+ return _toc_template.format(toc_lines=toc)
+
+
+def _create_parameters_cell(
+ test_paths: typing.Mapping[str, str],
+ metrics: typing.Sequence[str],
+) -> str:
+ parameters = '# parameters\n\n'
+
+ parameters += 'test_paths = ' + (
+ repr(test_paths)
+ .replace(',', ',\n ')
+ .replace('{', '{\n ')
+ .replace('}', ',\n}')
+ )
+
+ parameters += '\n\nmetrics = ' + str(metrics)
+
+ return parameters
+
+
+def _create_test_chunks(
+ test_names: typing.Sequence[str],
+) -> typing.Sequence[nbformat.notebooknode.NotebookNode]:
+ return [
+ cell
+ for test_name in test_names
+ for cell in _create_test_chunk(test_name)
+ ]
+
+
+def _create_test_chunk(
+ test_name: str,
+) -> typing.Sequence[nbformat.notebooknode.NotebookNode]:
+ from flood.user_io import notebook_io
+
+ return notebook_io.create_cells(
+ cell_templates=_test_template_cells,
+ inputs={'test_name': test_name},
+ )
+
+
+def print_tests_timing(
+ results_payloads: typing.Mapping[str, flood.SingleRunResultsPayload]
+) -> None:
+ import tooltime
+ import toolstr
+
+ time_per_test = {}
+
+ for test_name in results_payloads:
+ results = results_payloads[test_name]["results"]
+ time_per_test[test_name] = 0.0
+ for condition_name in results.keys():
+ time = sum(
+ item
+ for item in results[condition_name]["actual_duration"]
+ if item is not None
+ ) + sum(
+ item
+ for item in results[condition_name]["final_wait_time"]
+ if item is not None
+ )
+ time_per_test[test_name] = float(time_per_test[test_name] + time)
+
+ test_timing_rows = []
+ t_load_total = 0.0
+ for test_name, payload in results_payloads.items():
+ t_load_total += payload["t_run_end"] - payload["t_run_start"]
+ row = [
+ test_name,
+ payload["t_run_end"] - payload["t_run_start"],
+ time_per_test[test_name],
+ tooltime.timestamp_to_iso_pretty(payload["t_run_start"]).replace(
+ " ", "\n"
+ ),
+ tooltime.timestamp_to_iso_pretty(payload["t_run_end"]).replace(
+ " ", "\n"
+ ),
+ ]
+ test_timing_rows.append(row)
+
+ total_row = ['TOTAL', t_load_total, sum(time_per_test.values()), None, None]
+ test_timing_rows.append(total_row)
+
+ toolstr.print_text_box("Test timing")
+ print("(includes setup and cleanup steps)")
+ toolstr.print_multiline_table(
+ test_timing_rows,
+ labels=[
+ "test",
+ "total\nduration",
+ 'load test\nduration',
+ "t_start",
+ "t_end",
+ ],
+ column_formats={
+ 'load test\nduration': {'postfix': ' s'},
+ 'total\nduration': {'postfix': ' s'},
+ },
+ )
+
+
+def print_tests_cli_commands(
+ results_payloads: typing.Mapping[str, flood.SingleRunResultsPayload]
+) -> None:
+ import toolstr
+
+ toolstr.print_text_box('CLI commands used for each test')
+ for test_name, payload in results_payloads.items():
+ toolstr.print_header(test_name)
+ args = payload['cli_args']
+ args = ['"' + arg + '"' if ' ' in arg else arg for arg in args]
+ print(' '.join(args))
+
+
+def print_tests_nodes(
+ results_payloads: typing.Mapping[str, flood.SingleRunResultsPayload]
+) -> None:
+ import json
+ import toolstr
+
+ nodes_per_test = {}
+ for test_name, payload in results_payloads.items():
+ nodes_per_test[test_name] = payload['nodes']
+ n_unique_node_sets = len(
+ set(
+ json.dumps(nodes, sort_keys=True)
+ for nodes in nodes_per_test.values()
+ )
+ )
+ toolstr.print_text_box('Nodes Used')
+ if n_unique_node_sets == 1:
+ flood.user_io.print_nodes_table(nodes_per_test[test_name])
+ else:
+ for test_name, nodes_used in nodes_per_test.items():
+ flood.user_io.print_header(test_name)
+ flood.user_io.print_nodes_table(nodes_used)
+ print()
+
+
+def print_tests_versions(
+ results_payloads: typing.Mapping[str, flood.SingleRunResultsPayload]
+) -> None:
+ import json
+ import toolstr
+
+ deps_per_test = {}
+ for test_name, payload in results_payloads.items():
+ deps_per_test[test_name] = payload['dependency_versions']
+ n_unique_flood_versions = len(
+ set(payload['flood_version'] for payload in results_payloads.values())
+ )
+ n_unique_deps_sets = len(
+ set(json.dumps(deps, sort_keys=True) for deps in deps_per_test.values())
+ )
+ toolstr.print_text_box('Versions Used')
+ if n_unique_flood_versions == 1 and n_unique_deps_sets == 1:
+ print('flood version:', results_payloads[test_name]['flood_version'])
+ print()
+ toolstr.print_table(
+ list(deps_per_test[test_name].items()),
+ labels=['dependency', 'version'],
+ )
+ else:
+ print('flood version:', results_payloads[test_name]['flood_version'])
+ print()
+ for test_name, deps_used in deps_per_test.items():
+ flood.user_io.print_header(test_name)
+ toolstr.print_table(list(deps_used.items()))
+ print()
+
+
+if typing.TYPE_CHECKING:
+ from flood.user_io import notebook_io
+
+_report_template_cells: notebook_io.NotebookTemplate = [
+ {
+ # header
+ 'type': 'markdown',
+ 'f': _create_header_cell,
+ 'inputs': ['test_names'],
+ },
+ {
+ # imports
+ 'type': 'code',
+ 'content': """
+ import IPython
+ import polars as pl
+ import toolstr
+ import tooltime
+
+ import flood
+
+ flood.user_io.disable_text_colors()
+ """,
+ 'inputs': [],
+ },
+ {
+ # parameters
+ 'type': 'code',
+ 'f': _create_parameters_cell,
+ 'inputs': ['test_paths', 'metrics'],
+ },
+ {
+ # load data
+ 'type': 'code',
+ 'content': """
+ # load data
+
+ test_payloads = {{
+ test_name: flood.load_single_run_test_payload(test_path)
+ for test_name, test_path in test_paths.items()
+ }}
+
+ results_payloads = {{
+ test_name: flood.load_single_run_results_payload(output_dir=test_path)
+ for test_name, test_path in test_paths.items()
+ }}
+ """, # noqa: E501
+ 'inputs': [],
+ },
+ {
+ # Test summary header
+ 'type': 'markdown',
+ 'content': '# Test Summary',
+ 'inputs': [],
+ },
+ {
+ # test list
+ 'type': 'code',
+ 'content': """
+ # test list
+
+ toolstr.print_text_box('Tests')
+ for t, test_name in enumerate(results_payloads.keys()):
+ print(str(t + 1) + '.', test_name)
+ """,
+ 'inputs': [],
+ },
+ {
+ # test list
+ 'type': 'code',
+ 'content': """
+ flood.tests.print_tests_timing(results_payloads)
+ """,
+ 'inputs': [],
+ },
+ {
+ # test list
+ 'type': 'code',
+ 'content': """
+ flood.tests.print_tests_cli_commands(results_payloads)
+ """,
+ 'inputs': [],
+ },
+ {
+ # test list
+ 'type': 'code',
+ 'content': """
+ flood.tests.print_tests_nodes(results_payloads)
+ """,
+ 'inputs': [],
+ },
+ {
+ # test list
+ 'type': 'code',
+ 'content': """
+ flood.tests.print_tests_versions(results_payloads)
+ """,
+ 'inputs': [],
+ },
+ {
+ # Tests header
+ 'type': 'markdown',
+ 'content': '# Tests',
+ 'inputs': [],
+ },
+ {
+ # Tests chunks
+ 'type': 'chunk',
+ 'f': _create_test_chunks,
+ 'inputs': ['test_names'],
+ },
+]
+
+
+_test_template_cells: notebook_io.NotebookTemplate = [
+ {
+ # test header
+ 'type': 'markdown',
+ 'content': '# Test: {test_name}',
+ 'inputs': ['test_name'],
+ },
+ {
+ # load test results
+ 'type': 'code',
+ 'content': """
+ # load test results
+
+ test_name = '{test_name}'
+ test_payload = test_payloads[test_name]
+ results_payload = results_payloads[test_name]
+ results = results_payload['results']
+ """,
+ 'inputs': ['test_name'],
+ },
+ {
+ # show test metadata
+ 'type': 'code',
+ 'content': """
+ # show test metadata
+
+ toolstr.print_text_box(test_name + ' parameters')
+ test = flood.generate_test(**test_payload['test_parameters'])
+ flood.tests.load_tests.print_load_test_summary(test)
+ toolstr.print('- nodes tested:')
+ nodes_df = pl.from_records(list(results_payload['nodes'].values()))
+ toolstr.print_dataframe_as_table(nodes_df)
+ """,
+ 'inputs': [],
+ },
+ {
+ # show result tables
+ 'type': 'code',
+ 'content': """
+ # show result tables
+
+ flood.user_io.print_metric_tables(results, metrics=metrics, comparison=len(results) == 2)
+ """, # noqa: E501
+ 'inputs': [],
+ },
+ {
+ # show result figures
+ 'type': 'code',
+ 'content': """
+ # show result figures
+
+ colors = flood.user_io.get_nodes_plot_colors(nodes=results_payload['nodes'])
+ flood.tests.load_tests.plot_load_test_results(
+ test_name=test_name,
+ outputs=results,
+ latency_yscale_log=True,
+ colors=colors,
+ )
+ """, # noqa: E501
+ 'inputs': [],
+ },
+ {
+ # show errors
+ 'type': 'code',
+ 'content': """
+ # show errors
+
+ toolstr.print_text_box('Error messages present in each test')
+ unique_errors = {{}}
+ for n, name in enumerate(results.keys()):
+ unique_errors.setdefault(name, set())
+ unique_errors[name] |= {{
+ error for error_list in results[name]['errors'] for error in error_list
+ }}
+ print(name)
+ for error in unique_errors[name]:
+ print('-', error)
+ if n != len(results) - 1:
+ print()
+ """, # noqa: E501
+ 'inputs': [],
+ },
+ {
+ # show complete results
+ 'type': 'code',
+ 'content': """
+ # show complete results
+
+ for name in results.keys():
+ toolstr.print_text_box(name + " Complete Results")
+ df = pl.DataFrame(results[name])
+ df = df.drop(
+ "status_codes",
+ "errors",
+ "first_request_timestamp",
+ "last_request_timestamp",
+ "last_response_timestamp",
+ )
+ IPython.display.display(df)
+ """,
+ 'inputs': [],
+ },
+]
+
diff --git a/flood/tests/load_tests/load_test_runs.py b/flood/tests/load_tests/load_test_runs.py
new file mode 100644
index 0000000..0bd1d09
--- /dev/null
+++ b/flood/tests/load_tests/load_test_runs.py
@@ -0,0 +1,390 @@
+from __future__ import annotations
+
+import typing
+
+import flood
+from flood import user_io
+from flood import spec
+from . import vegeta
+
+if typing.TYPE_CHECKING:
+ import multiprocessing
+
+
+def run_load_tests(
+ *,
+ node: spec.NodeShorthand | None = None,
+ nodes: spec.NodesShorthand | None = None,
+ test: spec.LoadTest | spec.TestGenerationParameters | None = None,
+ tests: typing.Mapping[str, spec.LoadTest | spec.TestGenerationParameters]
+ | None = None,
+ verbose: bool | int = False,
+ include_deep_output: typing.Sequence[spec.DeepOutput] | None = None,
+) -> typing.Mapping[str, spec.LoadTestOutput]:
+ """run multiple load tests"""
+ # parse user_io
+ if (node is None) == (nodes is None):
+ raise Exception('must specify either node or nodes')
+ if (test is None) == (tests is None):
+ raise Exception('must specify either test or tests')
+ if node is not None:
+ node = user_io.parse_node(node)
+ if nodes is not None:
+ nodes = user_io.parse_nodes(nodes, request_metadata=False)
+ tqdm = user_io.outputs._get_tqdm()
+ pbar = {
+ 'leave': False,
+ 'desc': 'nodes',
+ 'position': 0,
+ 'colour': flood.user_io.styles['content'],
+ # 'disable': not verbose,
+ 'disable': True,
+ }
+
+ results = {}
+
+ # case: single node and single test
+ if node is not None and test is not None:
+ results[node['name']] = schedule_load_test(
+ node=node,
+ test=test,
+ include_deep_output=include_deep_output,
+ )
+
+ # case: single node and multiple tests
+ elif node is not None and tests is not None:
+ for name, each_test in tqdm.tqdm(tests.items(), **pbar):
+ results[name] = schedule_load_test(
+ node=node,
+ verbose=verbose,
+ test=each_test,
+ include_deep_output=include_deep_output,
+ )
+
+ # case: multiple nodes and single tests
+ elif nodes is not None and test is not None:
+ for name, nd in tqdm.tqdm(nodes.items(), **pbar):
+ results[name] = schedule_load_test(
+ node=nd,
+ verbose=verbose,
+ test=test,
+ include_deep_output=include_deep_output,
+ )
+
+ # case: multiple nodes and multiple tests
+ elif nodes is not None and tests is not None:
+ for node_name, node in nodes.items():
+ for test_name, test in tests.items():
+ results[node_name + '__' + test_name] = schedule_load_test(
+ node=node,
+ verbose=verbose,
+ test=test,
+ include_deep_output=include_deep_output,
+ )
+
+ # case: invalid input
+ else:
+ raise Exception('invalid user_io')
+
+ # join any multiprocessing results
+ joined = {}
+ for name, result in results.items():
+ if isinstance(result, dict):
+ joined[name] = result
+ elif isinstance(result, tuple):
+ process, queue = result
+ process.join()
+ if process.exitcode != 0:
+ import sys
+
+ sys.exit()
+ results_path = queue.get()
+ with open(results_path, 'r') as f:
+ import json
+
+ test_results: spec.SingleRunResultsPayload = json.load(f)
+ joined[name] = test_results['results'][name]
+ else:
+ raise Exception('invalid result type')
+
+ return joined
+
+
+def schedule_load_test(
+ *,
+ node: spec.NodeShorthand,
+ test: spec.LoadTest | spec.TestGenerationParameters,
+ verbose: bool | int = False,
+ include_deep_output: typing.Sequence[spec.DeepOutput] | None = None,
+ _pbar_kwargs: typing.Mapping[str, typing.Any] | None = None,
+) -> (
+ spec.LoadTestOutput
+ | str
+ | tuple[multiprocessing.Process, multiprocessing.Queue[str]]
+):
+ """runs local tests synchronously, remote tests asynchronously"""
+
+ node = user_io.parse_node(node)
+ if node['remote'] is not None:
+ import multiprocessing
+
+ queue: multiprocessing.Queue[str] = multiprocessing.Queue()
+ process = multiprocessing.Process(
+ target=run_load_test,
+ kwargs=dict(
+ node=node,
+ test=test,
+ verbose=verbose,
+ include_deep_output=include_deep_output,
+ _pbar_kwargs=_pbar_kwargs,
+ _container=queue,
+ ),
+ )
+ process.start()
+ return (process, queue)
+ else:
+ return run_load_test(
+ node=node,
+ test=test,
+ verbose=verbose,
+ include_deep_output=include_deep_output,
+ _pbar_kwargs=_pbar_kwargs,
+ )
+
+
+def run_load_test(
+ *,
+ node: spec.NodeShorthand,
+ test: spec.LoadTest | spec.TestGenerationParameters,
+ verbose: bool | int = False,
+ _pbar_kwargs: typing.Mapping[str, typing.Any] | None = None,
+ _container: multiprocessing.Queue[str] | None = None,
+ include_deep_output: typing.Sequence[spec.DeepOutput] | None = None,
+) -> spec.LoadTestOutput | str:
+ """run a load test against a single node"""
+
+ # parse user_io
+ node = user_io.parse_node(node)
+
+ # run tests
+ if node.get('remote') is None:
+ result: spec.LoadTestOutput | str = _run_load_test_locally(
+ node=node,
+ test=test,
+ verbose=verbose,
+ _pbar_kwargs=_pbar_kwargs,
+ include_deep_output=include_deep_output,
+ )
+ else:
+ result = _run_load_test_remotely(
+ node=node,
+ test=test,
+ verbose=verbose,
+ _pbar_kwargs=_pbar_kwargs,
+ include_deep_output=include_deep_output,
+ )
+
+ if _container is not None:
+ if not isinstance(result, str):
+ raise Exception('_container only supportted for remote testing')
+ _container.put(result)
+
+ return result
+
+
+def _run_load_test_locally(
+ *,
+ node: spec.Node,
+ test: spec.LoadTest | spec.TestGenerationParameters,
+ verbose: bool | int = False,
+ _pbar_kwargs: typing.Mapping[str, typing.Any] | None = None,
+ include_deep_output: typing.Sequence[spec.DeepOutput] | None = None,
+) -> spec.LoadTestOutput:
+ """run a load test from local node"""
+
+ if verbose:
+ flood.user_io.print_timestamped('Running load test for ' + node['name'])
+
+ # construct progress bar
+ if _pbar_kwargs is None:
+ _pbar_kwargs = {}
+ tqdm = user_io.outputs._get_tqdm()
+ tqdm_kwargs = dict(
+ leave=False,
+ desc='samples',
+ position=1,
+ colour=flood.user_io.styles['content'],
+ # disable=not verbose,
+ disable=True,
+ **_pbar_kwargs,
+ )
+
+ use_test: spec.LoadTest
+ if 'attacks' in test:
+ use_test = test # type: ignore
+ else:
+ use_test = flood.generate_test(**test)
+
+ # perform tests
+ results = []
+ for attack in tqdm.tqdm(use_test['attacks'], **tqdm_kwargs):
+ if verbose:
+ flood.user_io.print_timestamped(
+ 'Running attack at rate = ' + str(attack['rate']) + ' rps'
+ )
+
+ result = vegeta.run_vegeta_attack(
+ url=node['url'],
+ calls=attack['calls'],
+ duration=attack['duration'],
+ rate=attack['rate'],
+ vegeta_args=attack['vegeta_args'],
+ verbose=verbose >= 2,
+ include_deep_output=include_deep_output,
+ )
+ results.append(result)
+ if verbose >= 2:
+ print()
+
+ # format output
+ output_data: spec.LoadTestOutput = _list_of_maps_to_map_of_lists(results) # type: ignore # noqa: E501
+
+ # format deep output
+ if include_deep_output is None or len(include_deep_output) == 0:
+ for key in list(output_data.keys()):
+ if key.startswith('deep_'):
+ output_data[key] = None # type: ignore
+ else:
+ output_data['deep_rpc_error_pairs'] = [
+ result['deep_rpc_error_pairs'] for result in results
+ ]
+
+ # convert list of map of map into map of map of list
+ categories: list[spec.ResponseCategory] = list(
+ results[0]['deep_metrics'].keys() # type: ignore
+ )
+ deep_metrics = {}
+ for category in categories:
+ deep_metrics[category] = _list_of_maps_to_map_of_lists(
+ [result['deep_metrics'][category] for result in results] # type: ignore # noqa: E501
+ )
+ output_data['deep_metrics'] = deep_metrics # type: ignore
+
+ return output_data
+
+
+def _list_of_maps_to_map_of_lists(
+ list_of_maps: typing.Sequence[typing.Mapping[typing.Any, typing.Any]]
+) -> typing.Mapping[typing.Any, typing.Sequence[typing.Any]]:
+ keys = list_of_maps[0].keys()
+ return {key: [m[key] for m in list_of_maps] for key in keys}
+
+
+def _run_load_test_remotely(
+ *,
+ node: spec.Node,
+ test: spec.LoadTest | spec.TestGenerationParameters,
+ verbose: bool | int = False,
+ _pbar_kwargs: typing.Mapping[str, typing.Any] | None = None,
+ include_deep_output: typing.Sequence[spec.DeepOutput] | None = None,
+) -> str:
+ """run a load test from local node"""
+
+ import os
+ import subprocess
+ import sys
+ import uuid
+ import toolstr
+
+ # parse node specification
+ remote = node['remote']
+ if remote is None:
+ raise Exception('not a remote node')
+
+ # check remote installation
+ local_installation = flood.get_local_installation()
+ remote_installation = flood.get_remote_installation(remote)
+ local_flood_version = local_installation['flood_version']
+ remote_flood_version = remote_installation['flood_version']
+ remote_vegeta_path = remote_installation['vegeta_path']
+ if remote_flood_version is None:
+ raise Exception(
+ 'could not find flood installation on remote host ' + node['name']
+ )
+ sys.exit()
+ if remote_vegeta_path is None:
+ raise Exception(
+ 'could not find vegeta installation on remote host ' + node['name']
+ )
+ sys.exit()
+ if local_flood_version != remote_flood_version:
+ raise Exception(
+ 'local version of flood '
+ + str(local_flood_version)
+ + ' does not match remote version of flood '
+ + remote_flood_version
+ )
+
+ # save call data, saving methods to preserve ordering in json
+ job_id = str(uuid.uuid4())
+ tempdir = '/tmp/flood__' + job_id
+ os.makedirs(tempdir)
+ test_parameters: flood.TestGenerationParameters
+ if 'test_parameters' in test:
+ test_parameters = test['test_parameters'] # type: ignore
+ else:
+ test_parameters = test
+ flood.runners.single_runner.single_runner_io._save_single_run_test(
+ test_name='',
+ test_parameters=test_parameters,
+ output_dir=tempdir,
+ )
+
+ # send call data to remote server
+ if verbose:
+ styles = flood.user_io.styles
+ node_name = (
+ toolstr.add_style('\[', styles['content'])
+ + toolstr.add_style('node', styles['metavar'])
+ + toolstr.add_style('=', styles['content'])
+ + node['name']
+ + toolstr.add_style(']', styles['content'])
+ )
+ flood.user_io.print_timestamped(
+ node_name + ' Sending tests to remote node'
+ )
+ cmd = 'rsync -r ' + tempdir + ' ' + remote + ':' + os.path.dirname(tempdir)
+ subprocess.call(cmd.split(' '), stderr=subprocess.DEVNULL)
+
+ # initiate benchmarks
+ if verbose:
+ flood.user_io.print_timestamped(
+ node_name + ' Executing test on remote node'
+ )
+ cmd_template = "ssh {host} bash -c 'source ~/.profile; python3 -m flood {test} {name}={url} --output {output} --no-figures {extra_kwargs}'" # noqa: E501
+ extra_kwargs = ''
+ if include_deep_output is not None:
+ if 'raw' in include_deep_output:
+ extra_kwargs += ' --save-raw-output'
+ if 'metrics' in include_deep_output:
+ extra_kwargs += ' --deep-check'
+ cmd = cmd_template.format(
+ host=remote,
+ name=node['name'],
+ url=node['url'],
+ test=tempdir,
+ output=tempdir,
+ extra_kwargs=extra_kwargs.lstrip(),
+ )
+ cmd = cmd.strip()
+ subprocess.check_output(cmd.split(' '), stderr=subprocess.DEVNULL)
+
+ # retrieve benchmark results
+ if verbose:
+ flood.user_io.print_timestamped(node_name + ' Retrieving results')
+ results_path = os.path.join(tempdir, 'results.json')
+ cmd = 'rsync ' + remote + ':' + results_path + ' ' + results_path
+ subprocess.call(cmd.split(' '), stderr=subprocess.DEVNULL)
+
+ return results_path
+
diff --git a/rpc_bench/tests/load_tests/vegeta.py b/flood/tests/load_tests/vegeta.py
similarity index 78%
rename from rpc_bench/tests/load_tests/vegeta.py
rename to flood/tests/load_tests/vegeta.py
index 5f48065..be42acf 100644
--- a/rpc_bench/tests/load_tests/vegeta.py
+++ b/flood/tests/load_tests/vegeta.py
@@ -3,7 +3,8 @@ from __future__ import annotations
import typing
-from .. import spec
+from ... import spec
+from . import deep_utils
def run_vegeta_attack(
@@ -12,8 +13,9 @@ def run_vegeta_attack(
rate: int,
calls: typing.Sequence[typing.Any],
duration: int,
- vegeta_kwargs: typing.Mapping[str, str | None] | None = None,
+ vegeta_args: str | None = None,
verbose: bool = False,
+ include_deep_output: typing.Sequence[spec.DeepOutput] | None = None,
) -> spec.LoadTestOutputDatum:
attack = _construct_vegeta_attack(
calls=calls,
@@ -24,13 +26,15 @@ def run_vegeta_attack(
schedule_dir=attack['schedule_dir'],
duration=duration,
rate=rate,
- vegeta_kwargs=vegeta_kwargs,
+ vegeta_args=vegeta_args,
verbose=verbose,
)
report = _create_vegeta_report(
attack_output=attack_output,
target_rate=rate,
target_duration=duration,
+ include_deep_output=include_deep_output,
+ calls=calls,
)
return report
@@ -93,7 +97,7 @@ def _vegeta_attack(
max_workers: int | None = None,
n_cpus: int | None = None,
report_path: str | None = None,
- vegeta_kwargs: typing.Mapping[str, str | None] | None = None,
+ vegeta_args: str | None = None,
verbose: bool = False,
) -> bytes:
import os
@@ -110,11 +114,8 @@ def _vegeta_attack(
cmd += ' -max-connections=' + str(max_connections)
if max_workers is not None:
cmd += ' -max-workers=' + str(max_workers)
- if vegeta_kwargs is not None:
- for key, value in vegeta_kwargs.items():
- cmd += ' -' + key
- if value is not None:
- cmd += '=' + str(value)
+ if vegeta_args is not None:
+ cmd += ' ' + vegeta_args
if verbose:
print('- command:', cmd)
@@ -127,6 +128,8 @@ def _create_vegeta_report(
attack_output: bytes,
target_rate: int,
target_duration: int,
+ include_deep_output: typing.Sequence[spec.DeepOutput] | None,
+ calls: typing.Sequence[typing.Any],
) -> spec.LoadTestOutputDatum:
import json
import subprocess
@@ -144,6 +147,26 @@ def _create_vegeta_report(
else:
latency_min = None
+ # compute deep data
+ deep_raw_output = None
+ deep_metrics = None
+ deep_rpc_error_pairs = None
+ if include_deep_output is None:
+ include_deep_output = []
+ if 'raw' in include_deep_output:
+ deep_raw_output = deep_utils.encode_raw_vegeta_output(attack_output)
+ if 'metrics' in include_deep_output:
+ if 'metrics' in include_deep_output:
+ (
+ deep_metrics,
+ deep_rpc_error_pairs,
+ ) = deep_utils.compute_deep_datum(
+ raw_output=attack_output,
+ target_rate=target_rate,
+ target_duration=target_duration,
+ calls=calls,
+ )
+
return {
'target_rate': target_rate,
'actual_rate': report['rate'],
@@ -166,5 +189,8 @@ def _create_vegeta_report(
'last_request_timestamp': report['latest'],
'last_response_timestamp': report['end'],
'final_wait_time': report['wait'] / 1e9,
+ 'deep_raw_output': deep_raw_output,
+ 'deep_metrics': deep_metrics,
+ 'deep_rpc_error_pairs': deep_rpc_error_pairs,
}
diff --git a/rpc_bench/user_io/__init__.py b/flood/user_io/__init__.py
similarity index 66%
rename from rpc_bench/user_io/__init__.py
rename to flood/user_io/__init__.py
index 3eda352..96743ab 100644
--- a/rpc_bench/user_io/__init__.py
+++ b/flood/user_io/__init__.py
@@ -1,3 +1,2 @@
-from .file_io import *
from .inputs import *
from .outputs import *
diff --git a/flood/user_io/inputs.py b/flood/user_io/inputs.py
new file mode 100644
index 0000000..e719b9f
--- /dev/null
+++ b/flood/user_io/inputs.py
@@ -0,0 +1,216 @@
+from __future__ import annotations
+
+import typing
+
+import flood
+from flood import spec
+from . import outputs
+
+
+def get_ctc_alias_url(url: str) -> str | None:
+ try:
+ import ctc.config
+
+ ctc_providers = ctc.config.get_providers()
+ if url in ctc_providers:
+ url = ctc_providers[url]['url']
+ return url
+ except ImportError:
+ return None
+
+
+def parse_nodes(
+ nodes: spec.NodesShorthand,
+ *,
+ verbose: bool | int = False,
+ request_metadata: bool = False,
+) -> typing.Mapping[str, spec.Node]:
+ """parse given nodes according to input specification"""
+ if verbose:
+ outputs.print_header('Gathering node data...')
+
+ new_nodes: typing.MutableMapping[str, spec.Node] = {}
+ if isinstance(nodes, list):
+ import multiprocessing
+
+ with multiprocessing.Pool() as pool:
+ results = pool.map(parse_node, nodes)
+ new_nodes = {result['name']: result for result in results}
+ elif isinstance(nodes, dict):
+ for key, value in nodes.items():
+ new_nodes[key] = value
+ else:
+ raise Exception('invalid format for nodes')
+
+ if verbose:
+ print_nodes_table(new_nodes)
+
+ return new_nodes
+
+
+def print_nodes_table(nodes: typing.Mapping[str, spec.Node]) -> None:
+ rows = []
+ for node in nodes.values():
+ url = node['url']
+ if node['remote'] is not None:
+ url = node['remote'] + '\n' + url
+ client_version = node['client_version']
+ if client_version is not None:
+ client_version = client_version.replace('/', '\n')
+ row = [
+ node['name'],
+ url,
+ client_version,
+ ]
+ rows.append(row)
+ labels = ['node', 'url', 'metadata']
+ print()
+ outputs.print_multiline_table(
+ rows=rows,
+ labels=labels,
+ indent=4,
+ )
+
+
+def _get_node_str(node: flood.Node) -> str:
+ node_str = '"' + node['name'] + '", url=' + node['url']
+ remote = node['remote']
+ if remote is not None:
+ node_str += ' remote=' + remote
+ if node['client_version'] is not None:
+ node_str = node_str + ' version=' + node['client_version']
+ return node_str
+
+
+def parse_node(
+ node: str | spec.Node, request_metadata: bool = True
+) -> spec.Node:
+ """parse node according to input specification"""
+ prefixes = ['http', 'https', 'ws', 'wss']
+
+ if isinstance(node, dict):
+ return node
+ elif isinstance(node, str):
+ # parse name
+ if '=' in node:
+ name, url = node.split('=', 1)
+ else:
+ name = node
+ url = node
+
+ # check if node is in ctc aliases
+ alias_url = get_ctc_alias_url(url)
+ if alias_url is not None:
+ url = alias_url
+
+ # parse remote and url
+ if ':' in url:
+ head, tail = url.split(':', 1)
+ if head in prefixes:
+ remote = None
+ url = head + ':' + tail
+ else:
+ if tail.split('/')[0].isdecimal():
+ remote = None
+ url = url
+ else:
+ remote = head
+ url = tail
+ else:
+ remote = None
+
+ # add missing prefix
+ if not any(url.startswith(prefix) for prefix in prefixes):
+ # check if is ip
+ pieces = url.split(':')[0].split('.')
+ is_ip = len(pieces) == 4 and all(
+ piece.isdecimal() for piece in pieces
+ )
+
+ # add prefix
+ if url.startswith('localhost') or is_ip:
+ url = 'http://' + url
+ else:
+ url = 'https://' + url
+
+ if request_metadata:
+ # get client version
+ client_version = get_node_client_version(url=url, remote=remote)
+
+ # get network
+ network = 'ethereum'
+
+ else:
+ client_version = None
+ network = None
+
+ return {
+ 'name': name,
+ 'url': url,
+ 'remote': remote,
+ 'client_version': client_version,
+ 'network': network,
+ }
+
+ else:
+ raise Exception('invalid node format')
+
+
+def get_node_client_version(url: str, remote: str | None = None) -> str | None:
+ try:
+ if remote is None:
+ import ctc.rpc
+
+ info: str = ctc.rpc.sync_web3_client_version(
+ context={'provider': url}
+ )
+ return info
+ else:
+ import json
+ import subprocess
+
+ cmd = [
+ """ssh""",
+ remote,
+ """curl -X POST -H 'Content-Type: application/json' -d '{\"jsonrpc\": \"2.0\", \"method\": \"web3_clientVersion\", \"params\": [], \"id\": 1}' """ # noqa: E501
+ + url,
+ ]
+ output = subprocess.check_output(cmd, stderr=subprocess.DEVNULL)
+ response = json.loads(output)
+ info = response['result']
+ return info
+ except Exception:
+ return None
+
+
+def parse_test_data(test: spec.LoadTest) -> spec.LoadTestColumnWise:
+ rates = []
+ durations = []
+ vegeta_args = []
+ calls = []
+ for attack in test['attacks']:
+ rates.append(attack['rate'])
+ durations.append(attack['duration'])
+ vegeta_args.append(attack['vegeta_args'])
+ calls.append(attack['calls'])
+ return {
+ 'rates': rates,
+ 'durations': durations,
+ 'vegeta_args': vegeta_args,
+ 'calls': calls,
+ }
+
+
+def parse_nodes_network(nodes: typing.Mapping[str, spec.Node]) -> str:
+ if len(nodes) == 0:
+ raise Exception('no nodes, cannot get network')
+ networks = [node['network'] for node in nodes.values()]
+ if len(set(networks)) != 1:
+ raise Exception('multiple networks')
+ if networks[0] is None:
+ raise Exception('network could not be determined')
+ network = networks[0]
+ if isinstance(network, str):
+ return network
+ else:
+ raise Exception('need str network')
diff --git a/rpc_bench/user_io/notebook_io.py b/flood/user_io/notebook_io.py
similarity index 95%
rename from rpc_bench/user_io/notebook_io.py
rename to flood/user_io/notebook_io.py
index b259323..faddcb0 100644
--- a/rpc_bench/user_io/notebook_io.py
+++ b/flood/user_io/notebook_io.py
@@ -1,3 +1,4 @@
+"""generic functions for creating notebooks"""
from __future__ import annotations
import typing
@@ -24,7 +25,9 @@ if typing.TYPE_CHECKING:
class NotebookChunkTemplate(typing.TypedDict):
type: typing.Literal['chunk']
- f: typing.Callable[..., typing.Sequence[nbformat.notebooknode.NotebookNode]]
+ f: typing.Callable[
+ ..., typing.Sequence[nbformat.notebooknode.NotebookNode]
+ ]
inputs: typing.Sequence[str]
NotebookCellPrecursor = typing.Union[
@@ -61,7 +64,7 @@ def create_cell(
for line in lines
if not set(line).issubset({' '})
]
- cut = min(left_spaces[1:])
+ cut = min(left_spaces[:])
content = '\n'.join(line[cut:] for line in lines)
content = content.strip()
@@ -132,4 +135,3 @@ def create_notebook(
# write notebook to file
with open(output_path, 'w') as f:
nbformat.write(notebook, f) # type: ignore
-
diff --git a/flood/user_io/outputs.py b/flood/user_io/outputs.py
new file mode 100644
index 0000000..999e2a4
--- /dev/null
+++ b/flood/user_io/outputs.py
@@ -0,0 +1,281 @@
+from __future__ import annotations
+
+import typing
+
+import flood
+from flood import spec
+
+if typing.TYPE_CHECKING:
+ import types
+
+ import toolcli
+
+
+styles: toolcli.StyleTheme = {
+ 'title': 'bold #00e100',
+ 'metavar': 'bold #e5e9f0',
+ 'description': '#aaaaaa',
+ 'content': '#00B400',
+ 'option': 'bold #e5e9f0',
+ 'comment': '#888888',
+}
+
+
+plot_colors = {
+ 'green_shades': [
+ 'forestgreen',
+ 'limegreen',
+ 'chartreuse',
+ ],
+ 'red_shades': [
+ 'firebrick',
+ 'red',
+ 'salmon',
+ ],
+ 'purple_shades': [
+ 'rebeccapurple',
+ 'blueviolet',
+ 'mediumslateblue',
+ ],
+ 'orange_shades': [
+ 'darkgoldenrod',
+ 'darkorange',
+ 'gold',
+ ],
+ 'blue_shades': [
+ 'blue',
+ 'dodgerblue',
+ 'lightskyblue',
+ ],
+ 'streetlight': [
+ 'crimson',
+ 'goldenrod',
+ 'limegreen',
+ ],
+}
+
+
+def get_nodes_plot_colors(
+ nodes: typing.Mapping[str, flood.Node]
+) -> typing.Mapping[str, str]:
+ colors = {}
+ taken = set()
+ for node in nodes.values():
+ # print version
+ version = node['client_version']
+ if version is None:
+ version = ''
+
+ # decide color
+ if (
+ (version is not None and 'reth' in version)
+ or ('reth' in node['name'])
+ ) and 'orange_shades' not in taken:
+ color = 'orange_shades'
+ elif (
+ (version is not None and 'erigon' in version)
+ or ('erigon' in node['name'])
+ ) and 'blue_shades' not in taken:
+ color = 'blue_shades'
+ else:
+ for color_name in plot_colors.keys():
+ if color_name not in taken:
+ color = color_name
+ break
+ else:
+ raise Exception('out of colors')
+
+ colors[node['name']] = color
+ taken.add(color)
+
+ return colors
+
+
+def _get_tqdm() -> types.ModuleType:
+ import sys
+
+ if 'jupyter_client' in sys.modules:
+ try:
+ import ipywidgets # type: ignore # noqa: F401
+ import tqdm.notebook as tqdm
+
+ return tqdm
+ except ImportError:
+ pass
+
+ import tqdm # type: ignore
+
+ return tqdm
+
+
+def print_metric_tables(
+ results: typing.Mapping[str, spec.LoadTestOutput | spec.LoadTestDeepOutput],
+ metrics: typing.Sequence[str],
+ *,
+ suffix: str = '',
+ decimals: int | None = None,
+ comparison: bool | None = None,
+ indent: int | str | None = None,
+) -> None:
+ import toolstr
+
+ if len(results) == 0:
+ toolstr.print('no results', indent=indent)
+ print()
+ if comparison is None:
+ comparison = len(results) == 2
+
+ names = list(results.keys())
+ rates = results[names[0]]['target_rate']
+ for metric in metrics:
+ # create labels
+ if metric in ['success', 'n_invalid_json_errors', 'n_rpc_errors']:
+ metric_suffix = ''
+ elif metric == 'throughput':
+ metric_suffix = ' (rps)'
+ else:
+ metric_suffix = ' (s)'
+ unitted_names = [name + metric_suffix for name in names]
+ labels = ['rate (rps)'] + unitted_names
+ if comparison:
+ if len(results) != 2:
+ raise NotImplementedError('comparison of >2 tests')
+ comparison_label = names[0] + ' / ' + names[1]
+ labels.append(comparison_label)
+ else:
+ comparison_label = None
+
+ # build rows
+ rows: list[list[typing.Any]] = [[rate] for rate in rates]
+ values = []
+ for name, result in results.items():
+ for row, value in zip(rows, result[metric]): # type: ignore
+ row.append(value)
+ values.append(value)
+ if comparison:
+ for row in rows:
+ row.append(row[-2] / row[-1])
+
+ # compute column formats
+ if all(value > 1 for value in values if value is not None):
+ use_decimals = 1
+ else:
+ if decimals is None:
+ use_decimals = 6
+ else:
+ use_decimals = decimals
+ column_formats = {
+ column: {'decimals': use_decimals} for column in unitted_names
+ }
+ if comparison_label is not None:
+ column_formats[comparison_label] = {
+ 'decimals': 1,
+ 'percentage': True,
+ }
+
+ # print header
+ toolstr.print_text_box(
+ toolstr.add_style(
+ metric + ' vs load' + suffix, styles.get('metavar')
+ ),
+ style=styles.get('content'),
+ indent=indent,
+ )
+
+ if metric == 'success':
+ for label in labels[1:]:
+ column_formats.setdefault(label, {})
+ column_formats[label]['percentage'] = True
+ column_formats[label]['decimals'] = 1
+
+ # print table
+ toolstr.print_table(
+ rows,
+ labels=labels,
+ column_formats=column_formats, # type: ignore
+ label_style=styles.get('metavar'),
+ border=styles.get('content'),
+ indent=indent,
+ )
+ if metric != metrics[-1]:
+ print()
+
+
+#
+# # generic restylings of toolstr functions
+#
+
+
+def print_text_box(text: str) -> None:
+ import toolstr
+
+ toolstr.print_text_box(
+ text,
+ text_style=styles.get('metavar'),
+ style=styles.get('content'),
+ )
+
+
+def print_header(text: str) -> None:
+ import toolstr
+
+ toolstr.print_header(
+ text,
+ text_style=styles.get('metavar'),
+ style=styles.get('content'),
+ )
+
+
+def print_bullet(*args: typing.Any, **kwargs: typing.Any) -> None:
+ import toolstr
+
+ toolstr.print_bullet(
+ *args,
+ **kwargs,
+ styles=styles,
+ )
+
+
+def print_table(*args: typing.Any, **kwargs: typing.Any) -> None:
+ import toolstr
+
+ toolstr.print_table(
+ *args,
+ **kwargs,
+ label_style=styles.get('metavar'),
+ border=styles.get('content'),
+ )
+
+
+def print_multiline_table(*args: typing.Any, **kwargs: typing.Any) -> None:
+ import toolstr
+
+ toolstr.print_multiline_table(
+ *args,
+ **kwargs,
+ label_style=styles.get('metavar'),
+ border=styles.get('content'),
+ )
+
+
+def print_timestamped(message: str) -> None:
+ import datetime
+ import toolstr
+
+ dt = datetime.datetime.now()
+ if dt.microsecond >= 500_000:
+ dt = dt + datetime.timedelta(microseconds=1_000_000 - dt.microsecond)
+ else:
+ dt = dt - datetime.timedelta(microseconds=dt.microsecond)
+ timestamp = (
+ toolstr.add_style('\[', styles['content'])
+ + toolstr.add_style(str(dt), styles['metavar'])
+ + toolstr.add_style(']', styles['content'])
+ )
+ toolstr.print(timestamp + ' ' + message)
+
+
+def disable_text_colors() -> None:
+ for key in list(styles.keys()):
+ del styles[key] # type: ignore
+
diff --git a/pyproject.toml b/pyproject.toml
index 65c69ab..7352eff 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ requires = ["flit_core >=3.2, <4"]
build-backend = "flit_core.buildapi"
[project]
-name = "rpc_bench"
+name = "paradigm-flood"
readme = "README.md"
requires-python = ">=3.7"
dynamic = ["version", "description"]
@@ -28,14 +28,22 @@ classifiers = [
"Typing :: Typed",
]
dependencies = [
- 'typing-extensions >=4.2.0, <5',
- 'toolcli >=0.6.13, <0.7',
- 'toolstr >=0.9.5, <0.10',
- 'requests >=2.20.0, <3',
- 'tqdm >=4.65.0, <5',
+ 'checkthechain >= 0.3.9, <0.4.0',
+ 'ipykernel > 6, <7',
+ 'ipython_genutils > 0.1, <1',
+ 'matplotlib >= 3.7.1, <4',
+ 'nbconvert > 6, <7',
+ 'nbformat >= 5.8.0, <6',
'numpy >=1.19.0, <1.25',
- 'nbformat >= 5.8.0',
-# 'checkthechain@git+https://github.com/checkthechain/checkthechain#7c3c48715902ff52392e2a97dc54863e046670d3',
+ 'orjson >=3.0.0, < 4',
+ 'paradigm-data-portal >= 0.2.2, <0.3',
+ 'polars >= 0.17',
+ 'requests >=2.20.0, <3',
+ 'toolcli >=0.6.16, <0.7',
+ 'toolplot >= 0.3.4, <0.4',
+ 'toolstr >=0.9.7, <0.10',
+ 'tqdm >=4.65.0, <5',
+ 'typing-extensions >=4.2.0, <5',
]
[project.optional-dependencies]
@@ -47,16 +55,16 @@ test = [
]
[project.scripts]
-rpc_bench = "rpc_bench.cli.cli_run:run_cli"
+flood = "flood.cli.cli_run:run_cli"
[tool.flit.module]
-name = "rpc_bench"
+name = "flood"
[tool.mypy]
python_version = "3.9"
strict = true
implicit_reexport = true
-files = ["rpc_bench"]
+files = ["flood"]
[tool.pytest.ini_options]
testpaths = [
@@ -64,3 +72,12 @@ testpaths = [
]
asyncio_mode = 'auto'
+[tool.black]
+line-length = 80
+skip-string-normalization = true
+target-version = ['py37']
+
+[tool.ruff]
+line-length = 80
+ignore = ["F403"]
+
diff --git a/rpc_bench/__init__.py b/rpc_bench/__init__.py
deleted file mode 100644
index 98833ae..0000000
--- a/rpc_bench/__init__.py
+++ /dev/null
@@ -1,27 +0,0 @@
-"""tool for benchmarking RPC endpoints"""
-
-from .generators import *
-from .spec import *
-from .tests import *
-from .user_io import *
-
-
-__version__ = '0.1.0'
-
-
-def _clean_package_imports() -> None:
- """remove deep nested modules from rpc_bench namespace"""
-
- import sys
-
- rpc_bench = sys.modules['rpc_bench']
- moduletype = type(rpc_bench)
- delattr(rpc_bench, 'annotations')
- for key, value in tuple(vars(rpc_bench).items()):
- if isinstance(value, moduletype):
- name = value.__name__
- if not name.startswith('rpc_bench') or name.count('.') > 1:
- delattr(rpc_bench, key)
-
-
-_clean_package_imports()
diff --git a/rpc_bench/cli/cli_run.py b/rpc_bench/cli/cli_run.py
deleted file mode 100644
index 481bf2f..0000000
--- a/rpc_bench/cli/cli_run.py
+++ /dev/null
@@ -1,34 +0,0 @@
-from __future__ import annotations
-
-import toolcli
-
-import rpc_bench
-
-
-def run_cli(raw_command: str | None = None) -> None:
- command_index: toolcli.CommandIndex = {
- (): 'rpc_bench.cli.root_command',
- ('ls',): 'rpc_bench.cli.ls_command',
- ('report',): 'rpc_bench.cli.report_command',
- (
- 'version',
- ): 'toolcli.command_utils.standard_subcommands.version_command',
- ('help',): 'toolcli.command_utils.standard_subcommands.help_command',
- }
-
- config: toolcli.CLIConfig = {
- 'base_command': 'rpc_bench',
- 'description': rpc_bench.__doc__,
- 'version': rpc_bench.__version__,
- 'default_command_sequence': (),
- 'root_help_arguments': True,
- # 'root_help_subcommands': False,
- 'include_debug_arg': True,
- 'style_theme': rpc_bench.styles,
- }
-
- toolcli.run_cli(
- command_index=command_index,
- config=config,
- )
-
diff --git a/rpc_bench/generators/address_generators.py b/rpc_bench/generators/address_generators.py
deleted file mode 100644
index a646a18..0000000
--- a/rpc_bench/generators/address_generators.py
+++ /dev/null
@@ -1,18 +0,0 @@
-from __future__ import annotations
-
-import typing
-
-
-def generate_contract_addresses(n: int) -> typing.Sequence[str]:
- import pdp.datasets.contracts
-
- df = pdp.datasets.contracts.query_contracts(
- columns=['contract_address'], network='ethereum'
- )
- series = '0x' + df['contract_address'].sample(n).bin.encode('hex')
- return series.to_list()
-
-
-def generate_eoas(n: int) -> typing.Sequence[str]:
- raise NotImplementedError()
-
diff --git a/rpc_bench/generators/call_generators.py b/rpc_bench/generators/call_generators.py
deleted file mode 100644
index cdfeeeb..0000000
--- a/rpc_bench/generators/call_generators.py
+++ /dev/null
@@ -1,280 +0,0 @@
-from __future__ import annotations
-
-import ctc.rpc
-import typing
-
-from rpc_bench import spec
-from . import address_generators
-from . import block_generators
-from . import slot_generators
-from . import transaction_generators
-
-
-#
-# # blocks
-#
-
-
-def generate_calls_eth_get_block_by_number(
- n_calls: int | None = None,
- *,
- block_numbers: typing.Sequence[int] | None = None,
-) -> typing.Sequence[spec.Call]:
- if block_numbers is None:
- if n_calls is None:
- raise Exception('must specify more parameters')
- block_numbers = block_generators.generate_block_numbers(
- n=n_calls,
- random_seed=0,
- start_block=0,
- end_block=16_000_000,
- )
- return [
- ctc.rpc.construct_eth_get_block_by_number(block_number=block_number)
- for block_number in block_numbers
- ]
-
-
-def generate_calls_eth_get_block_by_hash(
- n_calls: int | None = None,
- *,
- block_hashes: typing.Sequence[str] | None = None,
-) -> typing.Sequence[spec.Call]:
- if block_hashes is None:
- if n_calls is None:
- raise Exception('must specify more parameters')
- block_hashes = block_generators.generate_block_hashes(
- n=n_calls,
- )
- return [
- ctc.rpc.construct_eth_get_block_by_hash(block_hash=block_hash)
- for block_hash in block_hashes
- ]
-
-
-#
-# # addresses
-#
-
-
-def generate_calls_eth_get_eth_balance(
- n_calls: int | None = None,
- *,
- addresses: typing.Sequence[str] | None = None,
- block_numbers: typing.Sequence[int] | None = None,
-) -> typing.Sequence[spec.Call]:
- if block_numbers is None:
- if n_calls is None:
- raise Exception('must specify more parameters')
- block_numbers = block_generators.generate_block_numbers(
- start_block=10_000_000,
- end_block=16_000_000,
- n=n_calls,
- random_seed=0,
- )
- if addresses is None:
- if n_calls is None:
- raise Exception('must specify more parameters')
- addresses = address_generators.generate_contract_addresses(n_calls)
-
- return [
- ctc.rpc.construct_eth_get_balance(
- address=address, block_number=block_number
- )
- for address, block_number in zip(addresses, block_numbers)
- ]
-
-
-def generate_calls_eth_get_transaction_count(
- n_calls: int | None = None,
- *,
- addresses: typing.Sequence[str] | None = None,
- block_numbers: typing.Sequence[int] | None = None,
-) -> typing.Sequence[spec.Call]:
- if block_numbers is None:
- if n_calls is None:
- raise Exception('must specify more parameters')
- block_numbers = block_generators.generate_block_numbers(
- start_block=10_000_000,
- end_block=16_000_000,
- n=n_calls,
- random_seed=0,
- )
- if addresses is None:
- if n_calls is None:
- raise Exception('must specify more parameters')
- addresses = address_generators.generate_eoas(n_calls)
-
- return [
- ctc.rpc.construct_eth_get_transaction_count(
- from_address=address, block_number=block_number
- )
- for address, block_number in zip(addresses, block_numbers)
- ]
-
-
-#
-# # transactions
-#
-
-
-def generate_calls_eth_get_transaction_by_hash(
- n_calls: int | None = None,
- *,
- transaction_hashes: typing.Sequence[str] | None = None,
-) -> typing.Sequence[spec.Call]:
- if transaction_hashes is None:
- if n_calls is None:
- raise Exception('must specify more parameters')
- transaction_hashes = transaction_generators.generate_transaction_hashes(
- n_calls
- )
- return [
- ctc.rpc.construct_eth_get_transaction_by_hash(
- transaction_hash=transaction_hash
- )
- for transaction_hash in transaction_hashes
- ]
-
-
-def generate_calls_eth_get_transaction_receipt(
- n_calls: int | None = None,
- *,
- transaction_hashes: typing.Sequence[str] | None = None,
-) -> typing.Sequence[spec.Call]:
- if transaction_hashes is None:
- if n_calls is None:
- raise Exception('must specify more parameters')
- transaction_hashes = transaction_generators.generate_transaction_hashes(
- n_calls
- )
- return [
- ctc.rpc.construct_eth_get_transaction_receipt(
- transaction_hash=transaction_hash
- )
- for transaction_hash in transaction_hashes
- ]
-
-
-#
-# # logs
-#
-
-contracts = {
- 'USDC': '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48',
- 'DAI': '0x6b175474e89094c44da98b954eedeac495271d0f',
- 'LUSD': '0x5f98805a4e8be255a32880fdec7f6728c6568ba0',
-}
-
-event_hashes = {
- 'Transfer': '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef',
- 'Approval': '0x8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925',
-}
-
-
-def generate_calls_eth_get_logs(
- n_calls: int | None = None,
- *,
- contract_address: str | None = None,
- topics: typing.Sequence[str | None] | None = None,
- block_ranges: typing.Sequence[tuple[int, int]] | None = None,
-) -> typing.Sequence[spec.Call]:
- if contract_address is None:
- contract_address = contracts['USDC']
- if block_ranges is None:
- if n_calls is None:
- raise Exception('must specify more parameters')
- block_ranges = block_generators.generate_block_ranges(
- start_block=10_000_000,
- end_block=16_000_000,
- n=n_calls,
- range_size=100,
- random_seed=0,
- )
- if topics is None:
- topics = [event_hashes['Transfer']]
- return [
- ctc.rpc.construct_eth_get_logs(
- address=contract_address,
- topics=topics,
- start_block=start_block,
- end_block=end_block,
- )
- for start_block, end_block in block_ranges
- ]
-
-
-#
-# # contracts
-#
-
-
-def generate_calls_eth_get_code(
- n_calls: int | None = None,
- *,
- addresses: typing.Sequence[str] | None = None,
- block_numbers: typing.Sequence[int | typing.Literal['latest']]
- | None = None,
-) -> typing.Sequence[spec.Call]:
- if block_numbers is None:
- if n_calls is None:
- raise Exception('must specify more parameters')
- block_numbers = block_generators.generate_block_numbers(
- start_block=10_000_000,
- end_block=16_000_000,
- n=n_calls,
- random_seed=0,
- )
- if addresses is None:
- if n_calls is None:
- raise Exception('must specify more parameters')
- addresses = address_generators.generate_contract_addresses(n_calls)
- return [
- ctc.rpc.construct_eth_get_code(
- address=address, block_number=block_number
- )
- for address, block_number in zip(addresses, block_numbers)
- ]
-
-
-def generate_calls_eth_get_storage_at(
- n_calls: int | None = None,
- *,
- slots: typing.Sequence[tuple[str, str]] | None = None,
- block_numbers: typing.Sequence[int | typing.Literal['latest']]
- | None = None,
-) -> typing.Sequence[spec.Call]:
- if block_numbers is None:
- if n_calls is None:
- raise Exception('must specify more parameters')
- block_numbers = block_generators.generate_block_numbers(
- start_block=10_000_000,
- end_block=16_000_000,
- n=n_calls,
- random_seed=0,
- )
- if slots is None:
- if n_calls is None:
- raise Exception('must specify more parameters')
- slots = slot_generators.generate_slots(n_calls)
- return [
- ctc.rpc.construct_eth_get_storage_at(
- address=address, position=slot, block_number=block_number
- )
- for (address, slot), block_number in zip(slots, block_numbers)
- ]
-
-
-# def generate_calls_eth_call(
-# addresses: typing.Sequence[str],
-# block_numbers: typing.Sequence[int] | None,
-# ) -> typing.Sequence[spec.Call]:
-# if block_numbers is None:
-# block_numbers = [None] * len(addresses)
-# return [
-# ctc.rpc.construct_eth_get_storage_at(
-# address=address, block_number=block_number
-# )
-# for address, block_number in zip(addresses, block_numbers)
-# ]
-
diff --git a/rpc_bench/generators/slot_generators.py b/rpc_bench/generators/slot_generators.py
deleted file mode 100644
index fe60cb2..0000000
--- a/rpc_bench/generators/slot_generators.py
+++ /dev/null
@@ -1,42 +0,0 @@
-from __future__ import annotations
-
-import typing
-
-_slot_sample_path = '/tmp/rpc_bench/slot_samples.parquet'
-
-
-def generate_slots(n: int) -> typing.Sequence[tuple[str, str]]:
- import polars as pl
-
- df = pl.read_parquet(_slot_sample_path)
- df = df[['contract_address', 'slot']]
- df = '0x' + df.select(pl.col('*').bin.encode('hex'))
- rows = df.sample(n).rows()
- return [(contract_address, slot) for contract_address, slot in rows]
-
-
-def _create_slots_samples(
- samples_per_file: int = 100_000,
- output_path: str | None = None,
-) -> None:
- import os
- import glob
- import polars as pl
- import pdp
-
- print('creating slots samples')
-
- source = pdp.get_dataset_glob('ethereum_slots')
- paths = glob.glob(source)
- samples = []
- for p, path in enumerate(paths):
- print(p + 1, '/', len(paths))
- samples.append(pl.read_parquet(path).sample(samples_per_file))
-
- if output_path is None:
- output_path = _slot_sample_path
- os.makedirs(os.path.dirname(output_path), exist_ok=True)
- df = pl.concat(samples)
- df.write_parquet(output_path)
- print('output slot samples to', output_path)
-
diff --git a/rpc_bench/generators/test_generators/address_test_generators.py b/rpc_bench/generators/test_generators/address_test_generators.py
deleted file mode 100644
index a3ed7aa..0000000
--- a/rpc_bench/generators/test_generators/address_test_generators.py
+++ /dev/null
@@ -1,45 +0,0 @@
-from __future__ import annotations
-
-import typing
-
-import rpc_bench
-from rpc_bench import spec
-
-
-def generate_test_eth_get_balance(
- rates: typing.Sequence[int],
- duration: int | None = None,
- durations: typing.Sequence[int] | None = None,
- vegeta_kwargs: typing.Mapping[str, str | None] | None = None,
- random_seed: spec.RandomSeed | None = None,
-) -> spec.LoadTest:
- n_calls = rpc_bench.estimate_call_count(
- rates=rates, duration=duration, durations=durations
- )
- calls = rpc_bench.generate_calls_eth_get_eth_balance(n_calls=n_calls)
- return rpc_bench.construct_load_test(
- calls=calls,
- rates=rates,
- duration=duration,
- durations=durations,
- )
-
-
-def generate_test_eth_get_transaction_count(
- rates: typing.Sequence[int],
- duration: int | None = None,
- durations: typing.Sequence[int] | None = None,
- vegeta_kwargs: typing.Mapping[str, str | None] | None = None,
- random_seed: spec.RandomSeed | None = None,
-) -> spec.LoadTest:
- n_calls = rpc_bench.estimate_call_count(
- rates=rates, duration=duration, durations=durations
- )
- calls = rpc_bench.generate_calls_eth_get_transaction_count(n_calls=n_calls)
- return rpc_bench.construct_load_test(
- calls=calls,
- rates=rates,
- duration=duration,
- durations=durations,
- )
-
diff --git a/rpc_bench/generators/test_generators/block_test_generators.py b/rpc_bench/generators/test_generators/block_test_generators.py
deleted file mode 100644
index 6d0ccfc..0000000
--- a/rpc_bench/generators/test_generators/block_test_generators.py
+++ /dev/null
@@ -1,45 +0,0 @@
-from __future__ import annotations
-
-import typing
-
-from rpc_bench import spec
-import rpc_bench
-
-
-def generate_test_eth_get_block_by_number(
- rates: typing.Sequence[int],
- duration: int | None = None,
- durations: typing.Sequence[int] | None = None,
- vegeta_kwargs: typing.Mapping[str, str | None] | None = None,
- random_seed: spec.RandomSeed | None = None,
-) -> rpc_bench.LoadTest:
- n_calls = rpc_bench.estimate_call_count(
- rates=rates, duration=duration, durations=durations
- )
- calls = rpc_bench.generate_calls_eth_get_block_by_number(n_calls=n_calls)
- return rpc_bench.construct_load_test(
- calls=calls,
- rates=rates,
- duration=duration,
- durations=durations,
- )
-
-
-def generate_test_eth_get_block_by_hash(
- rates: typing.Sequence[int],
- duration: int | None = None,
- durations: typing.Sequence[int] | None = None,
- vegeta_kwargs: typing.Mapping[str, str | None] | None = None,
- random_seed: spec.RandomSeed | None = None,
-) -> rpc_bench.LoadTest:
- n_calls = rpc_bench.estimate_call_count(
- rates=rates, duration=duration, durations=durations
- )
- calls = rpc_bench.generate_calls_eth_get_block_by_hash(n_calls=n_calls)
- return rpc_bench.construct_load_test(
- calls=calls,
- rates=rates,
- duration=duration,
- durations=durations,
- )
-
diff --git a/rpc_bench/generators/test_generators/contract_test_generators.py b/rpc_bench/generators/test_generators/contract_test_generators.py
deleted file mode 100644
index b587383..0000000
--- a/rpc_bench/generators/test_generators/contract_test_generators.py
+++ /dev/null
@@ -1,44 +0,0 @@
-from __future__ import annotations
-
-import typing
-
-import rpc_bench
-
-
-def generate_test_eth_get_code(
- rates: typing.Sequence[int],
- duration: int | None = None,
- durations: typing.Sequence[int] | None = None,
- vegeta_kwargs: typing.Mapping[str, str | None] | None = None,
- random_seed: rpc_bench.RandomSeed | None = None,
-) -> rpc_bench.LoadTest:
- n_calls = rpc_bench.estimate_call_count(
- rates=rates, duration=duration, durations=durations
- )
- calls = rpc_bench.generate_calls_eth_get_code(n_calls=n_calls)
- return rpc_bench.construct_load_test(
- calls=calls,
- rates=rates,
- duration=duration,
- durations=durations,
- )
-
-
-def generate_test_eth_get_storage_at(
- rates: typing.Sequence[int],
- duration: int | None = None,
- durations: typing.Sequence[int] | None = None,
- vegeta_kwargs: typing.Mapping[str, str | None] | None = None,
- random_seed: rpc_bench.RandomSeed | None = None,
-) -> rpc_bench.LoadTest:
- n_calls = rpc_bench.estimate_call_count(
- rates=rates, duration=duration, durations=durations
- )
- calls = rpc_bench.generate_calls_eth_get_storage_at(n_calls=n_calls)
- return rpc_bench.construct_load_test(
- calls=calls,
- rates=rates,
- duration=duration,
- durations=durations,
- )
-
diff --git a/rpc_bench/generators/test_generators/generic_test_generators.py b/rpc_bench/generators/test_generators/generic_test_generators.py
deleted file mode 100644
index b0c24bc..0000000
--- a/rpc_bench/generators/test_generators/generic_test_generators.py
+++ /dev/null
@@ -1,119 +0,0 @@
-from __future__ import annotations
-
-import typing
-
-import rpc_bench
-
-
-#
-# # bookkeeping
-#
-
-
-def get_single_test_generators() -> (
- typing.Mapping[str, rpc_bench.LoadTestGenerator]
-):
- return {
- get_test_generator_display_name(item): item # type: ignore
- for item in dir(rpc_bench)
- if item.startswith('generate_test_')
- }
-
-
-def get_multi_test_generators() -> (
- typing.Mapping[str, rpc_bench.MultiLoadTestGenerator]
-):
- return {
- get_test_generator_display_name(item): item # type: ignore
- for item in dir(rpc_bench)
- if item.startswith('generate_tests_')
- }
-
-
-def get_test_generator(test_name: str) -> rpc_bench.LoadTestGenerator:
- function_name = get_test_generator_function_name(test_name)
- if hasattr(rpc_bench, function_name):
- return getattr(rpc_bench, function_name) # type: ignore
- else:
- raise Exception()
-
-
-def get_test_generator_display_name(test: str | rpc_bench.LoadTestGenerator) -> str:
- if not isinstance(test, str):
- import types
-
- if not isinstance(test, types.FunctionType):
- raise Exception('should be str or function')
- test = test.__name__
-
- if not test.startswith('generate_test_'):
- raise Exception()
- test = test[len('generate_test_') :]
- head, tail = test.split('_', 1)
- test = head + '_' + snake_case_to_camel_case(tail)
- return test
-
-
-def get_test_generator_function_name(display_name: str) -> str:
- function_name = 'generate_test_' + camel_case_to_snake_case(display_name)
- return function_name
-
-
-def camel_case_to_snake_case(string: str) -> str:
- # adapted from https://stackoverflow.com/a/1176023
- import re
-
- return re.sub(r'(? str:
- pieces = string.split('_')
- return pieces[0] + ''.join(piece.title() for piece in pieces[1:])
-
-
-#
-# # generation
-#
-
-
-def generate_test(
- test_name: str,
- constants: typing.Mapping[str, typing.Any],
-) -> rpc_bench.LoadTest:
- test_generator = get_test_generator(test_name)
- test = test_generator(**constants)
- return test
-
-
-# def generate_tests(
-# test_name: str,
-# constants: typing.Mapping[str, typing.Any] | None = None,
-# variables: typing.Mapping[
-# str, typing.Mapping[str, typing.Sequence[typing.Any]]
-# ]
-# | None = None,
-# ) -> typing.Mapping[str, rpc_bench.LoadTest]:
-# """variables is in format {variable_name: {test_name: variable_value}}"""
-# import itertools
-
-# if constants is None:
-# constants = {}
-# if variables is None:
-# variables = {}
-
-# # compute input variables
-# tests_kwargs = []
-# combinations = itertools.product(*variables.items())
-# for combination in combinations:
-# test_kwargs = dict(combination, **constants)
-# tests_kwargs.append(test_kwargs)
-
-# # use test generator function
-# test_generator = get_test_generator(test_name)
-# tests = []
-# for test_kwargs in tests_kwargs:
-# test = test_generator(**test_kwargs)
-# tests.append(test)
-
-# return tests
-
diff --git a/rpc_bench/generators/test_generators/transaction_test_generators.py b/rpc_bench/generators/test_generators/transaction_test_generators.py
deleted file mode 100644
index e3617e5..0000000
--- a/rpc_bench/generators/test_generators/transaction_test_generators.py
+++ /dev/null
@@ -1,48 +0,0 @@
-from __future__ import annotations
-
-import typing
-
-import rpc_bench
-
-
-def generate_test_eth_get_transaction_by_hash(
- rates: typing.Sequence[int],
- duration: int | None = None,
- durations: typing.Sequence[int] | None = None,
- vegeta_kwargs: typing.Mapping[str, str | None] | None = None,
- random_seed: rpc_bench.RandomSeed | None = None,
-) -> rpc_bench.LoadTest:
- n_calls = rpc_bench.estimate_call_count(
- rates=rates, duration=duration, durations=durations
- )
- calls = rpc_bench.generate_calls_eth_get_transaction_by_hash(
- n_calls=n_calls
- )
- return rpc_bench.construct_load_test(
- calls=calls,
- rates=rates,
- duration=duration,
- durations=durations,
- )
-
-
-def generate_test_eth_get_transaction_receipt(
- rates: typing.Sequence[int],
- duration: int | None = None,
- durations: typing.Sequence[int] | None = None,
- vegeta_kwargs: typing.Mapping[str, str | None] | None = None,
- random_seed: rpc_bench.RandomSeed | None = None,
-) -> rpc_bench.LoadTest:
- n_calls = rpc_bench.estimate_call_count(
- rates=rates, duration=duration, durations=durations
- )
- calls = rpc_bench.generate_calls_eth_get_transaction_receipt(
- n_calls=n_calls
- )
- return rpc_bench.construct_load_test(
- calls=calls,
- rates=rates,
- duration=duration,
- durations=durations,
- )
-
diff --git a/rpc_bench/generators/transaction_generators.py b/rpc_bench/generators/transaction_generators.py
deleted file mode 100644
index 35578c9..0000000
--- a/rpc_bench/generators/transaction_generators.py
+++ /dev/null
@@ -1,8 +0,0 @@
-from __future__ import annotations
-
-import typing
-
-
-def generate_transaction_hashes(n: int) -> typing.Sequence[str]:
- raise NotImplementedError()
-
diff --git a/rpc_bench/spec.py b/rpc_bench/spec.py
deleted file mode 100644
index 256444d..0000000
--- a/rpc_bench/spec.py
+++ /dev/null
@@ -1,181 +0,0 @@
-"""type definitions for typing annotations"""
-
-from __future__ import annotations
-
-import typing
-
-
-if typing.TYPE_CHECKING:
- #
- # # generic types
- #
-
- Call = typing.Mapping[str, typing.Mapping[str, typing.Any]]
-
- MethodCalls = typing.Mapping[str, typing.Sequence[typing.Any]]
-
- class Node(typing.TypedDict):
- name: str
- url: str
- remote: str | None
-
- NodeShorthand = typing.Union[str, Node]
-
- NodesShorthand = typing.Union[
- typing.Sequence[str],
- typing.Sequence[Node],
- typing.Mapping[str, str],
- typing.Mapping[str, Node],
- ]
- Nodes = typing.Mapping[str, Node]
-
- class ProgressBar(typing.TypedDict):
- desc: str
- position: int | None
- leave: bool
- colour: str
- disable: bool
-
- #
- # # generators
- #
-
- import numpy as np
-
- RandomSeed = typing.Union[int, np.random._generator.Generator]
-
- #
- # # latency test types
- #
-
- NodeMethodLatencies = typing.Mapping[str, typing.Sequence[float]]
- NodesMethodLatencies = typing.Mapping[str, NodeMethodLatencies]
-
- class LatencyBenchmarkResults(typing.TypedDict):
- nodes: typing.Mapping[str, Node]
- methods: typing.Sequence[str]
- samples: int | None
- calls: typing.Mapping[str, typing.Sequence[str]]
- latencies: NodesMethodLatencies
-
- #
- # # equality test
- #
-
- EqualityTest = tuple[
- str,
- typing.Callable[..., typing.Any],
- typing.Sequence[typing.Any],
- typing.Mapping[str, typing.Any],
- ]
-
- #
- # # load test types
- #
-
- class VegetaAttack(typing.TypedDict):
- rate: int
- duration: int
- calls: typing.Sequence[typing.Any]
- vegeta_kwargs: typing.Mapping[str, typing.Any]
-
- VegetaKwargs = typing.Mapping[str, typing.Union[str, None]]
- MultiVegetaKwargs = typing.Sequence[VegetaKwargs]
- VegetaKwargsShorthand = typing.Union[VegetaKwargs, MultiVegetaKwargs]
-
- LoadTest = typing.Sequence[VegetaAttack]
-
- class LoadTestColumnWise(typing.TypedDict):
- rates: typing.Sequence[int]
- durations: typing.Sequence[int]
- calls: typing.Sequence[typing.Sequence[typing.Any]]
- vegeta_kwargs: typing.Sequence[typing.Any]
-
- LoadTestMode = typing.Literal['stress', 'spike', 'soak']
-
- LoadTestGenerator = typing.Callable[..., LoadTest]
- MultiLoadTestGenerator = typing.Callable[..., typing.Mapping[str, LoadTest]]
-
- #
- # # load tests outputs
- #
-
- class RawLoadTestOutputDatum(typing.TypedDict):
- latencies: typing.Mapping[str, float]
- bytes_in: typing.Mapping[str, int]
- bytes_out: typing.Mapping[str, int]
- earliest: str
- latest: str
- end: str
- duration: int
- wait: int
- requests: int
- rate: float
- throughput: float
- success: int
- status_codes: typing.Mapping[str, int]
- errors: typing.Sequence[str]
- first_request_timestamp: str
- last_request_timestamp: str
- last_response_timestamp: str
- final_wait_time: int
-
- class LoadTestOutputDatum(typing.TypedDict):
- target_rate: int
- actual_rate: float
- target_duration: int
- actual_duration: float
- requests: int
- throughput: float
- success: float
- min: float | None
- mean: float
- p50: float
- p90: float
- p95: float
- p99: float
- max: float
- status_codes: typing.Mapping[str, int]
- errors: typing.Sequence[str]
- first_request_timestamp: str
- last_request_timestamp: str
- last_response_timestamp: str
- final_wait_time: float
-
- class LoadTestOutput(typing.TypedDict):
- target_rate: typing.Sequence[int]
- actual_rate: typing.Sequence[float]
- target_duration: typing.Sequence[int]
- actual_duration: typing.Sequence[float]
- requests: typing.Sequence[int]
- throughput: typing.Sequence[float]
- success: typing.Sequence[float]
- min: typing.Sequence[float | None]
- mean: typing.Sequence[float]
- p50: typing.Sequence[float]
- p90: typing.Sequence[float]
- p95: typing.Sequence[float]
- p99: typing.Sequence[float]
- max: typing.Sequence[float]
- status_codes: typing.Sequence[typing.Mapping[str, int]]
- errors: typing.Sequence[typing.Sequence[str]]
- first_request_timestamp: typing.Sequence[str]
- last_request_timestamp: typing.Sequence[str]
- last_response_timestamp: typing.Sequence[str]
- final_wait_time: typing.Sequence[float]
-
- RunType = typing.Literal['single_run']
-
- class SingleRunTestPayload(typing.TypedDict):
- version: str
- type: RunType
- name: str
- test: LoadTest
-
- class SingleRunResultsPayload(typing.TypedDict):
- version: str
- type: RunType
- test: LoadTest
- nodes: Nodes
- results: typing.Mapping[str, LoadTestOutput]
-
diff --git a/rpc_bench/tests/__init__.py b/rpc_bench/tests/__init__.py
deleted file mode 100644
index 9b7f267..0000000
--- a/rpc_bench/tests/__init__.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from .equality_tests import *
-from .generic_tests import *
-from .latency_tests import *
-from .load_tests import *
-
diff --git a/rpc_bench/tests/equality_tests/equality_test_runs.py b/rpc_bench/tests/equality_tests/equality_test_runs.py
deleted file mode 100644
index 566a581..0000000
--- a/rpc_bench/tests/equality_tests/equality_test_runs.py
+++ /dev/null
@@ -1,214 +0,0 @@
-from __future__ import annotations
-
-import typing
-
-import rpc_bench
-from . import equality_test_sets
-
-
-def run_equality_test(
- test_name: str,
- nodes: rpc_bench.NodesShorthand,
- *,
- verbose: bool | int = True,
- random_seed: rpc_bench.RandomSeed | None = None,
-) -> None:
- import json
- import requests
- import toolstr
-
- nodes = rpc_bench.parse_nodes(nodes)
- for node in nodes.values():
- if node['remote'] is not None:
- raise Exception('remote not supported for equality test')
- if len(nodes) != 2:
- raise Exception('should use two nodes in equality test')
-
- # get tests
- if test_name != 'all':
- raise NotImplementedError('must use test_name="all" for equality test')
- equality_tests = equality_test_sets.get_all_equality_tests()
-
- # print preamble
- rpc_bench.print_text_box('Running equality test')
- rpc_bench.print_bullet(key='nodes', value='')
- for n, node in enumerate(nodes.values()):
- toolstr.print(
- toolstr.add_style(str(n + 1), rpc_bench.styles['metavar'])
- + '. '
- + str(node),
- indent=4,
- style=rpc_bench.styles['description'],
- )
- rpc_bench.print_bullet(key='methods', value='')
- for test in equality_tests:
- rpc_bench.print_bullet(key=test[0], value='', colon_str='', indent=4)
-
- successful = []
- headers = {'Content-Type': 'application/json', 'User-Agent': 'rpc_bench'}
- for test in equality_tests:
- # create call
- test_name, constructor, args, kwargs = test
- call = constructor(*args, **kwargs)
-
- # dispatch call
- results = []
- for node in nodes.values():
- try:
- response = requests.post(
- url=node['url'], data=json.dumps(call), headers=headers
- )
- response_data = response.json()
- if 'result' in response_data:
- result = response_data['result']
- else:
- result = None
- except Exception:
- result = None
- results.append(result)
-
- # print summary
- success = _summarize_result(
- results=results, nodes=nodes, test=test, call=call
- )
- if success:
- successful.append(test_name)
-
- print()
- rpc_bench.print_text_box('Equality Test Summary')
- print()
- rpc_bench.print_header('No differences detected (n = ' + str(len(successful)) + ')')
- if len(successful) == 0:
- print('[none]')
- else:
- for name in sorted(successful):
- rpc_bench.print_bullet(key=name, value='', colon_str='')
- print()
-
- failed = [test[0] for test in equality_tests if test[0] not in successful]
- rpc_bench.print_header('Differences detected (n = ' + str(len(failed)) + ')')
- if len(failed) == 0:
- print('[none]')
- else:
- for name in sorted(failed):
- rpc_bench.print_bullet(key=name, value='', colon_str='')
-
-
-def json_equal(lhs: typing.Any, rhs: typing.Any) -> bool:
- import json
-
- return json.dumps(lhs, sort_keys=True) == json.dumps(rhs, sort_keys=True)
-
-
-def _summarize_result(
- results: typing.Sequence[typing.Any],
- nodes: rpc_bench.Nodes,
- test: rpc_bench.EqualityTest,
- call: typing.Mapping[str, typing.Any],
-) -> bool:
- import toolstr
-
- test_name, constructor, args, kwargs = test
-
- if not json_equal(results[0], results[1]):
- print()
- rpc_bench.print_text_box('Discrepancies in ' + test_name)
- print()
- rpc_bench.print_header('args')
- if len(args) > 0:
- for arg in args:
- rpc_bench.print_bullet(key=arg, value='', colon_str='')
- if len(kwargs) > 0:
- for key, value in kwargs.items():
- rpc_bench.print_bullet(key=key, value=value)
- print()
- rpc_bench.print_header('call')
- print(call)
-
- if any(result is None for result in results):
- print()
- for node, result in zip(nodes.values(), results):
- if result is None:
- toolstr.print(
- node['name'] + ' failed', style=rpc_bench.styles['title']
- )
-
- if results[0] is None or results[1] is None:
- return False
-
- _print_result_diff(results=results, nodes=list(nodes.values()))
-
- return False
-
- else:
- return True
-
-
-def _print_result_diff(
- results: typing.Sequence[typing.Any],
- nodes: typing.Sequence[rpc_bench.Node],
-) -> None:
- import toolstr
-
- result0 = results[0]
- result1 = results[1]
-
- if type(result0) is not type(result1):
- for node, result in zip(nodes, results):
- print(node['name'], 'type:', type(result))
- return
-
- if isinstance(result0, dict):
- keys0 = set(result0.keys())
- keys1 = set(result1.keys())
- if keys0 != keys1:
- only_in_0 = keys0 - keys1
- only_in_1 = keys1 - keys0
- print()
- rpc_bench.print_header('different fields in results')
- if len(only_in_0) > 0:
- print(
- '- present only in',
- nodes[0]['name'] + ':',
- ', '.join(only_in_0),
- )
- if len(only_in_1) > 0:
- print(
- '- present only in',
- nodes[1]['name'] + ':',
- ', '.join(only_in_1),
- )
-
- rows = []
- for key in keys0 & keys1:
- if not json_equal(result0[key], result1[key]):
- row = [key, result0[key], result1[key]]
- rows.append(row)
- if len(rows) > 0:
- print()
- rpc_bench.print_header('differences in values')
- rpc_bench.print_table(
- rows,
- labels=['field', nodes[0]['name'], nodes[1]['name']],
- compact=3,
- max_column_widths=[20, 30, 30],
- indent=4,
- )
-
- elif isinstance(result0, list):
- if len(result1) != len(result0):
- print()
- rpc_bench.print_header('different number of results')
- for node, result in zip(nodes, results):
- rpc_bench.print_bullet(
- key=node['name'], value=str(len(result)) + ' results'
- )
- else:
- raise NotImplementedError()
-
- else:
- if result1 != result0:
- rpc_bench.print_header('differences in values')
- for node, result in zip(nodes, results):
- rpc_bench.print_bullet(key=node['name'], value=result)
-
diff --git a/rpc_bench/tests/equality_tests/equality_test_sets.py b/rpc_bench/tests/equality_tests/equality_test_sets.py
deleted file mode 100644
index 7eb56ee..0000000
--- a/rpc_bench/tests/equality_tests/equality_test_sets.py
+++ /dev/null
@@ -1,281 +0,0 @@
-from __future__ import annotations
-
-import typing
-
-import rpc_bench
-from rpc_bench import block_generators
-
-
-def get_all_equality_tests(
- start_block: int = 10_000_000,
- end_block: int = 16_000_000,
- range_size: int = 100,
- random_seed: rpc_bench.RandomSeed | None = None,
-) -> typing.Sequence[rpc_bench.EqualityTest]:
- return list(
- get_vanilla_equality_tests(
- start_block=start_block,
- end_block=end_block,
- range_size=range_size,
- random_seed=random_seed,
- )
- ) + list(
- get_trace_equality_tests(
- start_block=start_block,
- end_block=end_block,
- random_seed=random_seed,
- )
- )
-
-
-def get_vanilla_equality_tests(
- start_block: int = 10_000_000,
- end_block: int = 16_000_000,
- range_size: int = 100,
- random_seed: rpc_bench.RandomSeed | None = None,
-) -> typing.Sequence[rpc_bench.EqualityTest]:
- import ctc.rpc
-
- start_block, end_block = block_generators.generate_block_ranges(
- n=1,
- range_size=range_size,
- start_block=start_block,
- end_block=end_block,
- random_seed=random_seed,
- )[0]
-
- return [
- (
- 'eth_getChainId',
- ctc.rpc.construct_eth_chain_id,
- [],
- {},
- ),
- (
- 'eth_getLogs',
- ctc.rpc.construct_eth_get_logs,
- [],
- {
- 'address': '0x6b175474e89094c44da98b954eedeac495271d0f',
- 'start_block': start_block,
- 'end_block': end_block,
- },
- ),
- (
- 'eth_gasPrice',
- ctc.rpc.construct_eth_chain_id,
- [],
- {},
- ),
- (
- 'eth_getBalance',
- ctc.rpc.construct_eth_get_balance,
- ['0xeb27d00030033c29307daa483171d22eb0c93342'],
- # ['0xe540c45c504b348ad4d6eb9344e6cfa07c959be6'],
- {
- 'block_number': start_block,
- },
- ),
- (
- 'eth_getTransactionCount',
- ctc.rpc.construct_eth_get_transaction_count,
- ['0xeb27d00030033c29307daa483171d22eb0c93342'],
- {
- 'block_number': start_block,
- },
- ),
- (
- 'eth_getStorageAt',
- ctc.rpc.construct_eth_get_storage_at,
- [],
- {
- 'address': '0x4d9079bb4165aeb4084c526a32695dcfd2f77381',
- 'position': '0x0000000000000000000000000000000000000000000000000000000000000002',
- 'block_number': start_block,
- },
- ),
- (
- 'eth_getCode',
- ctc.rpc.construct_eth_get_code,
- ['0x4d9079bb4165aeb4084c526a32695dcfd2f77381'],
- {},
- ),
- (
- 'eth_getBlockByNumber',
- ctc.rpc.construct_eth_get_block_by_number,
- [start_block],
- {},
- ),
- (
- 'eth_getBlockByHash',
- ctc.rpc.construct_eth_get_block_by_hash,
- [
- '0x3dc4ef568ae2635db1419c5fec55c4a9322c05302ae527cd40bff380c1d465dd'
- ],
- {},
- ),
- (
- 'eth_getTransactionByHash',
- ctc.rpc.construct_eth_get_transaction_by_hash,
- [
- '0xd01212e8ab48d2fd2ea9c4f33f8670fd1cf0cfb09d2e3c6ceddfaf54152386e5'
- ],
- {},
- ),
- (
- 'eth_getTransactionReceipt',
- ctc.rpc.construct_eth_get_transaction_receipt,
- [
- '0xd01212e8ab48d2fd2ea9c4f33f8670fd1cf0cfb09d2e3c6ceddfaf54152386e5'
- ],
- {},
- ),
- (
- 'eth_call',
- ctc.rpc.construct_eth_call,
- [],
- {
- 'to_address': '0x6b175474e89094c44da98b954eedeac495271d0f',
- 'function_abi': {
- 'constant': True,
- 'inputs': [
- {
- 'internalType': 'address',
- 'name': '',
- 'type': 'address',
- }
- ],
- 'name': 'balanceOf',
- 'outputs': [
- {
- 'internalType': 'uint256',
- 'name': '',
- 'type': 'uint256',
- }
- ],
- 'payable': False,
- 'stateMutability': 'view',
- 'type': 'function',
- },
- 'function_parameters': [
- '0x5d3a536e4d6dbd6114cc1ead35777bab948e3643'
- ],
- 'block_number': start_block,
- },
- ),
- ]
-
-
-def get_trace_equality_tests(
- start_block: int = 10_000_000,
- end_block: int = 16_000_000,
- random_seed: rpc_bench.RandomSeed | None = None,
-) -> typing.Sequence[rpc_bench.EqualityTest]:
- import ctc.rpc
-
- block_number = block_generators.generate_block_numbers(
- n=1,
- start_block=start_block,
- end_block=end_block,
- random_seed=random_seed,
- )[0]
-
- function_abi = {
- 'constant': True,
- 'inputs': [{'internalType': 'address', 'name': '', 'type': 'address'}],
- 'name': 'balanceOf',
- 'outputs': [{'internalType': 'uint256', 'name': '', 'type': 'uint256'}],
- 'payable': False,
- 'stateMutability': 'view',
- 'type': 'function',
- }
-
- return [
- (
- 'trace_block',
- ctc.rpc.construct_trace_block,
- [],
- {'block_number': block_number},
- ),
- (
- 'trace_call',
- ctc.rpc.construct_trace_call,
- [],
- {
- 'to_address': '0x6b175474e89094c44da98b954eedeac495271d0f',
- 'trace_type': ['trace'],
- 'function_abi': function_abi,
- 'function_parameters': [
- '0x5d3a536e4d6dbd6114cc1ead35777bab948e3643'
- ],
- 'block_number': block_number,
- },
- ),
- (
- 'trace_call_many',
- ctc.rpc.construct_trace_call_many,
- [],
- {
- 'calls': [
- {
- 'to_address': '0x6b175474e89094c44da98b954eedeac495271d0f',
- 'function_abi': function_abi,
- 'function_parameters': [
- '0x5d3a536e4d6dbd6114cc1ead35777bab948e3643'
- ],
- 'trace_type': ['trace'],
- },
- ],
- 'trace_type': ['trace'],
- },
- ),
- # (
- # 'trace_filter',
- # ctc.rpc.construct_trace_filter,
- # [],
- # {
- # 'to_addresses': ['0x6b175474e89094c44da98b954eedeac495271d0f'],
- # 'start_block': 10_000_000,
- # 'end_block': 10_000_001,
- # 'count': 10,
- # }
- # ),
- # (
- # 'trace_get',
- # ctc.rpc.construct_trace_get,
- # [],
- # {}
- # ),
- # (
- # 'trace_raw_transaction',
- # ctc.rpc.construct_trace_raw_transaction,
- # ['0xd01212e8ab48d2fd2ea9c4f33f8670fd1cf0cfb09d2e3c6ceddfaf54152386e5'],
- # {'trace_type': ['trace']},
- # ),
- (
- 'trace_replay_block_transactions',
- ctc.rpc.construct_trace_replay_block_transactions,
- [],
- {
- 'block_number': block_number,
- 'trace_type': ['trace'],
- },
- ),
- (
- 'trace_replay_transaction',
- ctc.rpc.construct_trace_replay_transaction,
- [
- '0xd01212e8ab48d2fd2ea9c4f33f8670fd1cf0cfb09d2e3c6ceddfaf54152386e5'
- ],
- {'trace_type': ['trace']},
- ),
- (
- 'trace_transaction',
- ctc.rpc.construct_trace_transaction,
- [
- '0xd01212e8ab48d2fd2ea9c4f33f8670fd1cf0cfb09d2e3c6ceddfaf54152386e5'
- ],
- {},
- ),
- ]
-
diff --git a/rpc_bench/tests/generic_tests/__init__.py b/rpc_bench/tests/generic_tests/__init__.py
deleted file mode 100644
index ef708b0..0000000
--- a/rpc_bench/tests/generic_tests/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from .runner import *
diff --git a/rpc_bench/tests/generic_tests/runner.py b/rpc_bench/tests/generic_tests/runner.py
deleted file mode 100644
index 33a94e0..0000000
--- a/rpc_bench/tests/generic_tests/runner.py
+++ /dev/null
@@ -1,409 +0,0 @@
-from __future__ import annotations
-
-import typing
-
-import rpc_bench
-from rpc_bench.user_io import file_io
-
-
-def run(
- test_name: str,
- *,
- nodes: rpc_bench.NodesShorthand | None,
- random_seed: rpc_bench.RandomSeed | None = None,
- verbose: bool | int = True,
- rates: typing.Sequence[int] | None = None,
- duration: int | None = None,
- durations: typing.Sequence[int] | None = None,
- mode: rpc_bench.LoadTestMode | None = None,
- vegeta_kwargs: rpc_bench.VegetaKwargsShorthand | None = None,
- dry: bool,
- output_dir: str | bool | None = None,
- figures: bool = True,
- metrics: typing.Sequence[str] | None = None,
-) -> None:
- """generate and run load test(s) against node(s)"""
- import json
- import os
-
- if isinstance(output_dir, bool):
- if output_dir:
- import tempfile
-
- output_dir = tempfile.mkdtemp()
- else:
- output_dir = None
- if output_dir is not None:
- output_dir = os.path.abspath(os.path.expanduser(output_dir))
-
- # run test from path
- if os.path.exists(test_name) or '/' in test_name:
- path_spec = test_name
- if os.path.isdir(path_spec):
- test_path = rpc_bench.get_single_run_test_path(output_dir=path_spec)
- else:
- test_path = path_spec
- try:
- with open(test_path, 'r') as f:
- test_payload = json.load(f)
- test = test_payload['test']
- test_name = test_payload['name']
- except Exception:
- raise Exception('invalid test path: ' + str(test_path))
-
- if nodes is None:
- if os.path.isdir(path_spec):
- result_path = rpc_bench.get_single_run_results_path(
- output_dir=path_spec,
- )
- else:
- result_path = path_spec
- try:
- with open(result_path, 'r') as f:
- test_payload = json.load(f)
- nodes = test_payload['nodes']
- except Exception:
- raise Exception('invalid test path: ' + str(result_path))
-
- return _run_single(
- test_name=test_name,
- rerun_of=path_spec,
- test=test,
- nodes=nodes,
- random_seed=random_seed,
- dry=dry,
- output_dir=output_dir,
- verbose=verbose,
- metrics=metrics,
- figures=figures,
- )
-
- if nodes is None:
- raise Exception('must specify nodes')
-
- if test_name in rpc_bench.get_single_test_generators():
- _run_single(
- test_name=test_name,
- nodes=nodes,
- random_seed=random_seed,
- rates=rates,
- duration=duration,
- durations=durations,
- vegeta_kwargs=vegeta_kwargs,
- dry=dry,
- output_dir=output_dir,
- verbose=verbose,
- metrics=metrics,
- figures=figures,
- )
- elif test_name in rpc_bench.get_multi_test_generators():
- raise NotImplementedError()
- else:
- raise Exception('invalid test name')
-
-
-def _run_single(
- *,
- test_name: str,
- rerun_of: str | None = None,
- test: rpc_bench.LoadTest | None = None,
- nodes: rpc_bench.NodesShorthand,
- random_seed: rpc_bench.RandomSeed | None = None,
- rates: typing.Sequence[int] | None = None,
- duration: int | None = None,
- durations: typing.Sequence[int] | None = None,
- mode: rpc_bench.LoadTestMode | None = None,
- vegeta_kwargs: rpc_bench.VegetaKwargsShorthand | None = None,
- dry: bool,
- output_dir: str | None = None,
- figures: bool,
- metrics: typing.Sequence[str] | None = None,
- verbose: bool | int,
-) -> None:
- import os
- import toolstr
-
- # parse inputs
- nodes = rpc_bench.parse_nodes(nodes)
-
- # create test
- if test is not None:
- test_data = rpc_bench.parse_test_data(test=test)
- rates = test_data['rates']
- durations = test_data['durations']
- vegeta_kwargs = test_data['vegeta_kwargs']
- else:
- if test_name is None:
- raise Exception('must specify test or test_name')
- rates, durations = rpc_bench.generate_timings(
- rates=rates,
- duration=duration,
- durations=durations,
- mode=mode,
- )
- test = rpc_bench.generate_test(
- test_name=test_name,
- constants={
- 'rates': rates,
- 'durations': durations,
- 'vegeta_kwargs': vegeta_kwargs,
- },
- )
-
- # print preamble
- if verbose:
- _print_single_run_preamble(
- test_name=test_name,
- rerun_of=rerun_of,
- rates=rates,
- durations=durations,
- vegeta_kwargs=vegeta_kwargs,
- output_dir=output_dir,
- nodes=nodes,
- )
-
- if output_dir is not None:
- summary_path = os.path.join(output_dir, 'summary.txt')
- with toolstr.write_stdout_to_file(summary_path):
- _print_single_run_preamble(
- test_name=test_name,
- rerun_of=rerun_of,
- rates=rates,
- durations=durations,
- vegeta_kwargs=vegeta_kwargs,
- output_dir=output_dir,
- nodes=nodes,
- )
-
- # save test to disk
- if output_dir is not None:
- file_io._save_single_run_test(
- test_name=test_name, output_dir=output_dir, test=test
- )
-
- # skip dry run
- if dry:
- return
-
- # run tests
- try:
- results = rpc_bench.run_load_tests(
- nodes=nodes,
- test=test,
- verbose=verbose,
- )
- except Exception as e:
- import traceback
- print('THIS WAS THE ERROR:', e.args)
- print()
- print(traceback.format_exc())
- print()
- print("ERROR OVER")
- raise e
-
- # output results to file
- if output_dir is not None:
- file_io._save_single_run_results(
- output_dir=output_dir,
- test=test,
- nodes=nodes,
- results=results,
- )
-
- if figures:
- figures_dir = file_io.get_single_run_figures_path(
- output_dir=output_dir
- )
- rpc_bench.plot_load_test_results(
- outputs=results,
- test_name=test_name,
- output_dir=figures_dir,
- )
-
- # print summary
- if verbose:
- _print_single_run_conclusion(
- output_dir=output_dir,
- results=results,
- metrics=metrics,
- verbose=verbose,
- figures=figures,
- )
- with toolstr.write_stdout_to_file(summary_path, mode='a'):
- _print_single_run_conclusion(
- output_dir=output_dir,
- results=results,
- metrics=metrics,
- verbose=verbose,
- figures=figures,
- )
-
-
-#
-# # printing outputs
-#
-
-
-def _print_single_run_preamble(
- *,
- test_name: str,
- rates: typing.Sequence[int],
- durations: typing.Sequence[int],
- vegeta_kwargs: rpc_bench.VegetaKwargsShorthand | None,
- nodes: rpc_bench.Nodes,
- rerun_of: str | None = None,
- output_dir: str | None,
-) -> None:
- import toolstr
-
- toolstr.print_text_box(
- toolstr.add_style(
- 'Load test: ' + test_name, rpc_bench.styles['metavar']
- ),
- style=rpc_bench.styles['content'],
- )
- toolstr.print_bullet(
- key='sample rates', value=rates, styles=rpc_bench.styles
- )
- if len(set(durations)) == 1:
- toolstr.print_bullet(
- key='sample duration',
- value=durations[0],
- styles=rpc_bench.styles,
- )
- else:
- toolstr.print_bullet(
- key='sample durations', value=durations, styles=rpc_bench.styles
- )
- if vegeta_kwargs is None or len(vegeta_kwargs) == 0:
- toolstr.print_bullet(
- key='extra args', value=None, styles=rpc_bench.styles
- )
-
- if rerun_of is not None:
- toolstr.print_bullet(
- key='rerun of', value=rerun_of, styles=rpc_bench.styles
- )
- toolstr.print_bullet(
- key='output directory', value=output_dir, styles=rpc_bench.styles
- )
-
- if len(nodes) == 1:
- node = list(nodes.values())[0]
- toolstr.print_bullet(
- key='node',
- value=_get_node_str(node),
- styles=rpc_bench.styles,
- )
- else:
- toolstr.print_bullet(key='nodes', value='', styles=rpc_bench.styles)
- for n, node in enumerate(nodes.values()):
- toolstr.print(
- toolstr.add_style(str(n + 1), rpc_bench.styles['metavar'])
- + '. '
- + _get_node_str(node),
- indent=4,
- style=rpc_bench.styles['description'],
- )
-
- print()
- print()
- toolstr.print_header(
- 'Running load tests...',
- style=rpc_bench.styles['content'],
- text_style=rpc_bench.styles['metavar'],
- )
-
-
-def _get_node_str(node: rpc_bench.Node) -> str:
- node_str = '"' + node['name'] + '", url=' + node['url']
- remote = node['remote']
- if remote is not None:
- node_str += ' remote=' + remote
- return node_str
-
-
-def _print_single_run_conclusion(
- output_dir: str | None,
- results: typing.Mapping[str, rpc_bench.LoadTestOutput],
- metrics: typing.Sequence[str] | None,
- verbose: bool | int,
- figures: bool,
-) -> None:
- import os
- import toolstr
-
- print('Load tests completed.')
-
- # print message about metrics file
- if output_dir is not None:
-
- test_path = file_io.get_single_run_test_path(
- output_dir=output_dir
- )
- result_path = file_io.get_single_run_results_path(
- output_dir=output_dir
- )
- figures_path = file_io.get_single_run_figures_path(
- output_dir=output_dir
- )
-
- print()
- print()
- toolstr.print_bullet(
- key='Saving results',
- value='',
- styles=rpc_bench.styles,
- bullet_str='',
- )
- toolstr.print_bullet(
- key=os.path.relpath(test_path, output_dir),
- value='',
- colon_str='',
- styles=rpc_bench.styles,
- )
- toolstr.print_bullet(
- key=os.path.relpath(result_path, output_dir),
- value='',
- colon_str='',
- styles=rpc_bench.styles,
- )
- if figures:
- toolstr.print_bullet(
- key=os.path.relpath(figures_path, output_dir),
- value='',
- colon_str='',
- styles=rpc_bench.styles,
- )
-
- # decide metrics
- if metrics is None:
- metrics = ['success', 'throughput', 'p90']
-
- # print metrics
- print()
- print()
- print('Summarizing performance metrics...')
- if verbose > 1:
- toolstr.print_bullet(
- key='metrics shown below',
- value=', '.join(metrics),
- styles=rpc_bench.styles,
- )
- example_result = list(results.values())[0]
- additional = [
- key for key in example_result.keys() if key not in metrics
- ]
- toolstr.print_bullet(
- key='additional metrics available',
- value=', '.join(additional),
- styles=rpc_bench.styles,
- )
-
- # print metric values
- print()
- rpc_bench.print_metric_tables(
- results=results,
- metrics=metrics,
- )
-
diff --git a/rpc_bench/tests/latency_tests/__init__.py b/rpc_bench/tests/latency_tests/__init__.py
deleted file mode 100644
index 35333d5..0000000
--- a/rpc_bench/tests/latency_tests/__init__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from .latency_test_execution import *
-from .latency_test_outputs import *
-from .latency_test_runs import *
diff --git a/rpc_bench/tests/latency_tests/call_creation.py b/rpc_bench/tests/latency_tests/call_creation.py
deleted file mode 100644
index f3f4a06..0000000
--- a/rpc_bench/tests/latency_tests/call_creation.py
+++ /dev/null
@@ -1,65 +0,0 @@
-from __future__ import annotations
-
-import ctc.rpc
-import typing
-
-from rpc_bench import spec
-
-
-def _get_all_methods() -> typing.Sequence[str]:
- return [
- 'eth_blockNumber',
- 'eth_chainId',
- 'eth_hashRate',
- ]
-
-
-def _create_calls(
- methods: typing.Sequence[str] | None = None,
- samples: int | None = None,
- *,
- random_seed: int | None = None,
- calls_file: str | None = None,
-) -> spec.MethodCalls:
- calls: spec.MethodCalls
-
- # load calls from file
- if calls_file is not None:
- import json
-
- with open(calls_file, 'r') as f:
- calls_data = json.load(f)
- return {
- method: calls_data['calls'][method]
- for method in calls_data['methods']
- }
-
- # parse inputs
- if samples is None:
- samples = 1
- if methods is None:
- methods = _get_all_methods()
- if random_seed is None:
- random_seed = 0
-
- # generate calls
- calls = {method: [] for method in methods}
- for method in methods:
- call_creator = _get_call_creator(method)
- for sample in range(samples):
- call = call_creator()
- calls[method].append(call) # type: ignore
-
- return calls
-
-
-def _get_call_creator(method: str) -> typing.Callable[..., typing.Any]:
- if method == 'eth_blockNumber':
- return ctc.rpc.construct_eth_block_number
- elif method == 'eth_chainId':
- return ctc.rpc.construct_eth_chain_id
- elif method == 'eth_hashRate':
- return ctc.rpc.construct_eth_hashrate
- else:
- raise ValueError('unknown method: ' + str(method))
-
diff --git a/rpc_bench/tests/latency_tests/latency_test_execution.py b/rpc_bench/tests/latency_tests/latency_test_execution.py
deleted file mode 100644
index 7819c27..0000000
--- a/rpc_bench/tests/latency_tests/latency_test_execution.py
+++ /dev/null
@@ -1,184 +0,0 @@
-from __future__ import annotations
-
-import typing
-
-import rpc_bench
-from rpc_bench import spec
-from . import latency_test_outputs
-
-
-def _execute_calls(
- nodes: typing.Mapping[str, spec.Node],
- calls: spec.MethodCalls,
- verbose: bool,
-) -> spec.NodesMethodLatencies:
- """perform RPC calls"""
- # local calls
- local_nodes = {k: v for k, v in nodes.items() if v['remote'] is None}
- local_latencies = _execute_local_calls(
- nodes=local_nodes, calls=calls, verbose=verbose
- )
-
- # remote calls
- remote_nodes = {k: v for k, v in nodes.items() if v['remote'] is not None}
- remote_latencies = _execute_remote_calls(
- nodes=remote_nodes, calls=calls, verbose=verbose
- )
-
- return dict(local_latencies, **remote_latencies)
-
-
-def _execute_local_calls(
- nodes: typing.Mapping[str, spec.Node],
- calls: spec.MethodCalls,
- verbose: bool,
-) -> spec.NodesMethodLatencies:
- """perform local RPC calls"""
-
- import json
- import requests
- import tqdm
-
- if len(nodes) == 0:
- return {}
-
- # print prelude and get progress bars
- if verbose:
- latency_test_outputs._print_local_execution_prelude()
- node_bar, method_bar, sample_bar = latency_test_outputs._get_progress_bars(
- nodes=nodes, verbose=verbose
- )
-
- # specify headers
- headers = {
- 'Content-Type': 'application/json',
- 'User-Agent': 'rpc_bench',
- }
-
- # perform calls
- methods = list(calls.keys())
- latencies: spec.NodesMethodLatencies = {
- name: {method: [] for method in methods} for name in nodes.keys()
- }
- for name, node in tqdm.tqdm(nodes.items(), **node_bar):
- for method in tqdm.tqdm(methods, **method_bar):
- for call in tqdm.tqdm(calls[method], **sample_bar):
- response = requests.post(
- url=node['url'],
- data=json.dumps(call),
- headers=headers,
- )
- latency = response.elapsed.total_seconds()
- latencies[name][method].append(latency) # type: ignore
-
- # print summary
- if verbose:
- latency_test_outputs._print_local_execution_summary()
-
- return latencies
-
-
-def _execute_remote_calls(
- nodes: typing.Mapping[str, spec.Node],
- calls: spec.MethodCalls,
- verbose: bool,
-) -> spec.NodesMethodLatencies:
- """perform remote RPC calls"""
-
- import toolstr
-
- if verbose and len(nodes) > 0:
- print()
- print()
- toolstr.print_header(
- toolstr.add_style(
- 'Performing Remote Benchmarks...', rpc_bench.styles['metavar']
- ),
- style=rpc_bench.styles['content'],
- )
-
- latencies: typing.MutableMapping[
- str,
- typing.Mapping[str, typing.Sequence[float]],
- ]
- latencies = {}
- for name, node in nodes.items():
- remote = node['remote']
- if remote is None:
- raise Exception('not a remote node')
- result = _execute_node_remote_calls(
- node=node, calls=calls, verbose=verbose
- )
- latencies[name] = result
- return latencies
-
-
-def _execute_node_remote_calls(
- node: spec.Node,
- calls: spec.MethodCalls,
- verbose: bool,
-) -> spec.NodeMethodLatencies:
- """perform remote RPC calls on particular node"""
-
- import json
- import os
- import subprocess
- import uuid
- import toolstr
-
- # parse node specification
- remote = node['remote']
- if remote is None:
- raise Exception('not a remote node')
-
- # save call data, saving methods to preserve ordering in json
- job_id = str(uuid.uuid4())
- tempdir = '/tmp/rpc_bench__' + job_id
- os.makedirs(tempdir)
- calls_path = os.path.join(tempdir, 'calls.json')
- calls_data = {'methods': list(calls.keys()), 'calls': calls}
- with open(calls_path, 'w') as f:
- json.dump(calls_data, f)
-
- # send call data to remote server
- if verbose:
- toolstr.print(
- toolstr.add_style('- ', rpc_bench.styles['title'])
- + toolstr.add_style(node['name'], rpc_bench.styles['metavar'])
- + toolstr.add_style(':', rpc_bench.styles['title'])
- + ' host'
- + toolstr.add_style('=', rpc_bench.styles['title'])
- + toolstr.add_style(remote, rpc_bench.styles['metavar'])
- + ', url'
- + toolstr.add_style('=', rpc_bench.styles['title'])
- + toolstr.add_style(node['url'], rpc_bench.styles['metavar'])
- )
- print(' Sending call schedule')
- cmd = 'ssh {host} mkdir {tempdir}'.format(host=remote, tempdir=tempdir)
- subprocess.call(cmd.split(' '), stderr=subprocess.DEVNULL)
- cmd = 'rsync ' + calls_path + ' ' + remote + ':' + calls_path
- subprocess.call(cmd.split(' '), stderr=subprocess.DEVNULL)
-
- # initiate benchmarks
- if verbose:
- print(' Executing remote calls')
- results_path = os.path.join(tempdir, 'results.json')
- cmd = 'ssh {host} python3 -m rpc_bench {url} --calls {calls_path} --output {output}'.format(
- host=remote,
- url=node['url'],
- calls_path=calls_path,
- output=results_path,
- )
- subprocess.check_output(cmd.split(' '), stderr=subprocess.DEVNULL)
-
- # retrieve benchmark results
- if verbose:
- print(' Retrieving results')
- cmd = 'rsync ' + remote + ':' + results_path + ' ' + results_path
- subprocess.call(cmd.split(' '), stderr=subprocess.DEVNULL)
-
- # return results
- with open(results_path, 'r') as f:
- results: spec.LatencyBenchmarkResults = json.load(f)
- return results['latencies'][node['url']]
-
diff --git a/rpc_bench/tests/latency_tests/latency_test_outputs.py b/rpc_bench/tests/latency_tests/latency_test_outputs.py
deleted file mode 100644
index 78a4235..0000000
--- a/rpc_bench/tests/latency_tests/latency_test_outputs.py
+++ /dev/null
@@ -1,209 +0,0 @@
-from __future__ import annotations
-
-import typing
-
-import rpc_bench
-from rpc_bench import spec
-
-if typing.TYPE_CHECKING:
- import datetime
-
-
-def _print_benchmark_prelude(
- *,
- nodes: typing.Mapping[str, spec.Node],
- methods: typing.Sequence[str] | None,
- samples: int | None,
- random_seed: int | str | None,
- calls: spec.MethodCalls,
- output_file: str | None = None,
- start_time: datetime.datetime,
-) -> None:
- import toolstr
-
- toolstr.print_text_box(
- toolstr.add_style('RPC Benchmark', rpc_bench.styles['metavar']),
- style=rpc_bench.styles['content'],
- double=True,
- )
-
- if len(nodes) == 1:
- node = list(nodes.values())[0]
- name = node['name']
- url = node['url']
- if name == url:
- label = url
- else:
- label = name + ' (' + url + ')'
- toolstr.print_bullet(key='node', value=label, styles=rpc_bench.styles)
- elif len(nodes) > 1:
- toolstr.print_bullet(key='nodes', value='', styles=rpc_bench.styles)
- for node in nodes.values():
- name = node['name']
- url = node['url']
- if name == url:
- label = name
- else:
- label = name + ' (' + url + ')'
- toolstr.print_bullet(
- key=label,
- value='',
- colon_str='',
- indent=4,
- styles=rpc_bench.styles,
- key_style=rpc_bench.styles['description'],
- )
- else:
- raise Exception('no nodes specified')
-
- if methods is None:
- methods = sorted(list(calls.keys()))
- toolstr.print_bullet(
- key='methods', value=', '.join(methods), styles=rpc_bench.styles
- )
-
- if samples is None:
- example_calls = next(iter(calls.values()))
- samples = len(example_calls)
- toolstr.print_bullet(key='samples', value=samples, styles=rpc_bench.styles)
-
- if random_seed is None:
- random_seed = 'not specified'
- toolstr.print_bullet(
- key='random_seed', value=random_seed, styles=rpc_bench.styles
- )
- toolstr.print_bullet(
- key='output_file', value=output_file, styles=rpc_bench.styles
- )
- toolstr.print_bullet(
- key='start_time',
- value=start_time,
- styles=rpc_bench.styles,
- )
-
-
-def _print_benchmark_summary(
- latencies: spec.NodesMethodLatencies,
- start_time: datetime.datetime,
- calls: spec.MethodCalls,
-) -> None:
- import datetime
- import time
- import numpy as np
- import toolstr
-
- print()
- print()
- toolstr.print_header(
- toolstr.add_style('Benchmark Summary', rpc_bench.styles['metavar']),
- style=rpc_bench.styles['content'],
- )
- end_time = datetime.datetime.fromtimestamp(int(time.time()))
- toolstr.print_bullet(
- key='start_time',
- value=start_time,
- styles=rpc_bench.styles,
- )
- toolstr.print_bullet(
- key='end_time',
- value=' ' + str(end_time),
- styles=rpc_bench.styles,
- )
- toolstr.print_bullet(
- key='duration',
- value=' ' + str(end_time - start_time).split('.')[0],
- styles=rpc_bench.styles,
- )
-
- # print table of latency per provider
- all_latencies: list[float] = []
- rows = []
- for method in calls.keys():
- row = [method]
- for node in latencies.keys():
- array = np.array(latencies[node][method])
- all_latencies.extend(latencies[node][method])
- row.append(array.mean())
-
- if len(latencies.keys()) == 2:
- row.append(row[1] / row[2]) # type: ignore
- rows.append(row)
-
- print()
- print()
- toolstr.print_text_box(
- toolstr.add_style(
- 'Mean Latencies (seconds)', rpc_bench.styles['metavar']
- ),
- style=rpc_bench.styles['content'],
- )
-
- node_names = list(latencies.keys())
- labels = ['method'] + node_names
- if len(latencies) == 2:
- labels.append(node_names[0] + ' / ' + node_names[1])
-
- if any(latency < 0.001 for latency in all_latencies):
- decimals = 4
- else:
- decimals = 3
-
- toolstr.print_table(
- rows,
- indent=4,
- labels=labels,
- border=rpc_bench.styles['content'],
- label_style=rpc_bench.styles['metavar'],
- column_styles={'method': rpc_bench.styles['metavar']},
- column_formats={name: {'decimals': decimals} for name in node_names},
- )
-
-
-def _print_local_execution_prelude() -> None:
- import toolstr
-
- print()
- print()
- toolstr.print_header(
- toolstr.add_style(
- 'Performing Local Benchmarks...', rpc_bench.styles['metavar']
- ),
- style=rpc_bench.styles['content'],
- )
-
-
-def _print_local_execution_summary() -> None:
- pass
-
-
-def _get_progress_bars(
- nodes: typing.Mapping[str, spec.Node],
- verbose: bool,
-) -> tuple[spec.ProgressBar, spec.ProgressBar, spec.ProgressBar]:
- if len(nodes) == 1:
- positions = [None, 0, 1]
- else:
- positions = [0, 1, 2]
- node_bar: spec.ProgressBar = {
- 'desc': ' nodes',
- 'position': positions[0],
- 'leave': True,
- 'colour': rpc_bench.styles['content'],
- 'disable': (not verbose) or (len(nodes) == 1),
- }
- method_bar: spec.ProgressBar = {
- 'desc': 'methods',
- 'position': positions[1],
- 'leave': len(nodes) == 1,
- 'colour': rpc_bench.styles['content'],
- 'disable': not verbose,
- }
- sample_bar: spec.ProgressBar = {
- 'desc': 'samples',
- 'position': positions[2],
- 'leave': False,
- 'colour': rpc_bench.styles['content'],
- 'disable': not verbose,
- }
- return node_bar, method_bar, sample_bar
-
diff --git a/rpc_bench/tests/latency_tests/latency_test_runs.py b/rpc_bench/tests/latency_tests/latency_test_runs.py
deleted file mode 100644
index e1f3b28..0000000
--- a/rpc_bench/tests/latency_tests/latency_test_runs.py
+++ /dev/null
@@ -1,77 +0,0 @@
-from __future__ import annotations
-
-import json
-import typing
-
-from rpc_bench import user_io
-from rpc_bench import spec
-from . import call_creation
-from . import latency_test_execution
-from . import latency_test_outputs
-
-
-def run_latency_test(
- *,
- nodes: spec.NodesShorthand,
- methods: typing.Sequence[str] | None,
- samples: int | None = None,
- calls: spec.MethodCalls | None = None,
- calls_file: str | None = None,
- verbose: bool = True,
- output_file: str | None = None,
- random_seed: int | None = None,
-) -> None:
- """run latency benchmark on specified nodes and methods"""
-
- import datetime
- import time
-
- # parse inputs
- if not isinstance(nodes, dict):
- nodes = user_io.parse_nodes(nodes)
- start_time = datetime.datetime.fromtimestamp(int(time.time()))
-
- # create calls
- if calls is None:
- calls = call_creation._create_calls(
- methods=methods,
- samples=samples,
- random_seed=random_seed,
- calls_file=calls_file,
- )
-
- # print prelude
- if verbose:
- latency_test_outputs._print_benchmark_prelude(
- nodes=nodes,
- methods=methods,
- samples=samples,
- random_seed=random_seed,
- calls=calls,
- output_file=output_file,
- start_time=start_time,
- )
-
- # perform calls
- latencies = latency_test_execution._execute_calls(
- nodes=nodes, calls=calls, verbose=verbose
- )
-
- # print summary
- if verbose:
- latency_test_outputs._print_benchmark_summary(
- latencies=latencies, calls=calls, start_time=start_time
- )
-
- # output file
- if output_file is not None:
- summary: spec.LatencyBenchmarkResults = {
- 'nodes': nodes,
- 'methods': list(calls.keys()),
- 'samples': samples,
- 'calls': calls,
- 'latencies': latencies,
- }
- with open(output_file, 'w') as f:
- json.dump(summary, f)
-
diff --git a/rpc_bench/tests/load_tests/load_test_reports.py b/rpc_bench/tests/load_tests/load_test_reports.py
deleted file mode 100644
index 7a0c728..0000000
--- a/rpc_bench/tests/load_tests/load_test_reports.py
+++ /dev/null
@@ -1,348 +0,0 @@
-from __future__ import annotations
-
-import typing
-
-import rpc_bench
-
-if typing.TYPE_CHECKING:
- import nbformat
-
-
-def create_load_test_report(
- test_paths: typing.Sequence[str],
- output_dir: str,
- metrics: typing.Sequence[str] | None = None,
-) -> None:
-
- import os
-
- ipynb_path = os.path.join(output_dir, 'report.ipynb')
- _create_load_test_report_ipynb(
- test_paths=test_paths,
- output_path=ipynb_path,
- metrics=metrics,
- )
-
- _create_load_test_report_html(ipynb_path=ipynb_path)
-
-
-def _create_load_test_report_html(
- ipynb_path: str,
-) -> None:
-
- import subprocess
-
- cmd = 'python3 -m nbconvert --to html --execute ' + ipynb_path
- subprocess.call(cmd.split(' '))
-
-
-def _create_load_test_report_ipynb(
- test_paths: typing.Sequence[str],
- output_path: str,
- metrics: typing.Sequence[str] | None = None,
-) -> None:
- from rpc_bench.user_io import notebook_io
-
- if metrics is None:
- metrics = ['success', 'throughput', 'p50', 'p90', 'p99']
-
- test_paths_map = {}
- for test_path in test_paths:
- payload = rpc_bench.load_single_run_test_payload(output_dir=test_path)
- name = payload['name']
- test_paths_map[name] = test_path
-
- inputs = {
- 'test_paths': test_paths_map,
- 'metrics': metrics,
- 'test_names': list(test_paths_map.keys()),
- }
-
- notebook_io.create_notebook(
- cell_templates=report_template_cells,
- output_path=output_path,
- inputs=inputs,
- )
-
-
-header_template = """# `rpc_bench` Load Test Report
-
-
-{toc}
-
-### Report Generation
-
-*This report was generated by `rpc_bench`. See the `rpc_bench report --help` command for report generation options. This report can be executed as a Python notebook using the `.ipynb` version of this file.*"""
-
-
-toc_template = """### Contents
-1. [Test Summary](#Test-Summary)
-2. [Tests](#Tests)
-{toc_lines}"""
-
-
-def create_header_cell(test_names: typing.Sequence[str]) -> str:
- toc = create_test_toc(test_names=test_names)
- return header_template.format(toc=toc)
-
-
-def create_test_toc(test_names: typing.Sequence[str]) -> str:
- line_template = '. [Test: {test_name}](#Test:-{test_name})\n'
- toc = ''
- for t, test_name in enumerate(test_names):
- toc += ' ' + str(t + 1) + line_template.format(test_name=test_name)
- toc = toc.rstrip()
- return toc_template.format(toc_lines=toc)
-
-
-def create_parameters_cell(
- test_paths: typing.Mapping[str, str],
- metrics: typing.Sequence[str],
-) -> str:
- parameters = '# parameters\n\n'
-
- parameters += 'test_paths = ' + (
- repr(test_paths)
- .replace(',', ',\n ')
- .replace('{', '{\n ')
- .replace('}', ',\n}')
- )
-
- parameters += '\n\nmetrics = ' + str(metrics)
-
- return parameters
-
-
-def create_test_chunks(
- test_names: typing.Sequence[str],
-) -> typing.Sequence[nbformat.notebooknode.NotebookNode]:
- return [
- cell
- for test_name in test_names
- for cell in create_test_chunk(test_name)
- ]
-
-
-def create_test_chunk(
- test_name: str,
-) -> typing.Sequence[nbformat.notebooknode.NotebookNode]:
- from rpc_bench.user_io import notebook_io
-
- return notebook_io.create_cells(
- cell_templates=test_template_cells,
- inputs={'test_name': test_name},
- )
-
-
-if typing.TYPE_CHECKING:
- from rpc_bench.user_io import notebook_io
-
-report_template_cells: notebook_io.NotebookTemplate = [
- {
- # header
- 'type': 'markdown',
- 'f': create_header_cell,
- 'inputs': ['test_names'],
- },
- {
- # imports
- 'type': 'code',
- 'content': """
- import IPython
- import polars as pl
- import toolstr
- import tooltime
-
- import rpc_bench
-
- rpc_bench.styles = {{}}
- """,
- 'inputs': [],
- },
- {
- # parameters
- 'type': 'code',
- 'f': create_parameters_cell,
- 'inputs': ['test_paths', 'metrics'],
- },
- {
- # load data
- 'type': 'code',
- 'content': """
- # load data
-
- test_payloads = {{
- test_name: rpc_bench.load_single_run_test_payload(output_dir=test_path)
- for test_name, test_path in test_paths.items()
- }}
-
- results_payloads = {{
- test_name: rpc_bench.load_single_run_results_payload(output_dir=test_path)
- for test_name, test_path in test_paths.items()
- }}
- """,
- 'inputs': [],
- },
- {
- # Test summary header
- 'type': 'markdown',
- 'content': '# Test Summary',
- 'inputs': [],
- },
- {
- # test list
- 'type': 'code',
- 'content': """
- # test list
-
- toolstr.print_text_box('Tests')
- for t, test_name in enumerate(results_payloads.keys()):
- print(str(t + 1) + '.', test_name)
- """,
- 'inputs': [],
- },
- {
- # test durations
- 'type': 'code',
- 'content': """
- # test durations
-
- time_per_test = {{}}
- time_per_condition = {{}}
-
- for test_name in results_payloads:
- results = results_payloads[test_name]["results"]
- time_per_test[test_name] = 0
- for condition_name in results.keys():
- time_per_condition.setdefault(condition_name, 0)
- time = sum(results[condition_name]["actual_duration"]) + sum(
- results[condition_name]["final_wait_time"]
- )
- time_per_test[test_name] += time
- time_per_condition[condition_name] += time
-
- toolstr.print_text_box('Total time')
- toolstr.print(tooltime.timelength_to_phrase(int(sum(time_per_test.values()))))
- print()
-
- toolstr.print_text_box('Total time per test')
- rows = list(time_per_test.items())
- toolstr.print_table(rows, labels=['test', 'time (s)'])
-
- toolstr.print_text_box('Total time per condition')
- rows = list(time_per_condition.items())
- toolstr.print_table(rows, labels=['condition', 'time (s)'])
- """,
- 'inputs': [],
- },
- {
- # Tests header
- 'type': 'markdown',
- 'content': '# Tests',
- 'inputs': [],
- },
- {
- # Tests chunks
- 'type': 'chunk',
- 'f': create_test_chunks,
- 'inputs': ['test_names'],
- },
-]
-
-
-test_template_cells: notebook_io.NotebookTemplate = [
- {
- # test header
- 'type': 'markdown',
- 'content': '# Test: {test_name}',
- 'inputs': ['test_name'],
- },
- {
- # load test results
- 'type': 'code',
- 'content': """
- # load test results
-
- test_name = '{test_name}'
- results_payload = results_payloads[test_name]
- results = results_payload['results']
- """,
- 'inputs': ['test_name'],
- },
- {
- # show test metadata
- 'type': 'code',
- 'content': """
- # show test metadata
-
- toolstr.print_text_box(test_name + ' parameters')
- rpc_bench.print_load_test_summary(results_payload['test'])
- toolstr.print('- nodes tested:')
- nodes_df = pl.from_records(list(results_payload['nodes'].values()))
- toolstr.print_dataframe_as_table(nodes_df)
- """,
- 'inputs': [],
- },
- {
- # show result tables
- 'type': 'code',
- 'content': """
- # show result tables
-
- rpc_bench.print_metric_tables(results, metrics=metrics, comparison=True)
- """,
- 'inputs': [],
- },
- {
- # show result figures
- 'type': 'code',
- 'content': """
- # show result figures
-
- rpc_bench.plot_load_test_results(test_name=test_name, outputs=results, latency_yscale_log=True)
- """,
- 'inputs': [],
- },
- {
- # show errors
- 'type': 'code',
- 'content': """
- # show errors
-
- toolstr.print_text_box('Error messages present in each test')
- unique_errors = {{}}
- for n, name in enumerate(results.keys()):
- unique_errors.setdefault(name, set())
- unique_errors[name] |= {{
- error for error_list in results[name]['errors'] for error in error_list
- }}
- print(name)
- for error in unique_errors[name]:
- print('-', error)
- if n != len(results) - 1:
- print()
- """,
- 'inputs': [],
- },
- {
- # show complete results
- 'type': 'code',
- 'content': """
- # show complete results
-
- for name in results.keys():
- toolstr.print_text_box(name + " Complete Results")
- df = pl.DataFrame(results[name])
- df = df.drop(
- "status_codes",
- "errors",
- "first_request_timestamp",
- "last_request_timestamp",
- "last_response_timestamp",
- )
- IPython.display.display(df)
- """,
- 'inputs': [],
- },
-]
-
diff --git a/rpc_bench/tests/load_tests/load_test_runs.py b/rpc_bench/tests/load_tests/load_test_runs.py
deleted file mode 100644
index cb9d9f2..0000000
--- a/rpc_bench/tests/load_tests/load_test_runs.py
+++ /dev/null
@@ -1,261 +0,0 @@
-from __future__ import annotations
-
-import typing
-
-import rpc_bench
-from rpc_bench import user_io
-from rpc_bench import spec
-from . import load_test_construction
-from . import vegeta
-
-
-def run_load_tests(
- *,
- node: spec.NodeShorthand | None = None,
- nodes: spec.NodesShorthand | None = None,
- test: spec.LoadTest | None = None,
- tests: typing.Mapping[str, spec.LoadTest] | None = None,
- verbose: bool | int = False,
-) -> typing.Mapping[str, spec.LoadTestOutput]:
- """run multiple load tests"""
- # parse user_io
- if (node is None) == (nodes is None):
- raise Exception('must specify either node or nodes')
- if (test is None) == (tests is None):
- raise Exception('must specify either test or tests')
- if node is not None:
- node = user_io.parse_node(node)
- if nodes is not None:
- nodes = user_io.parse_nodes(nodes)
- tqdm = user_io.outputs._get_tqdm()
- pbar = {
- 'leave': False,
- 'desc': 'nodes',
- 'position': 0,
- 'colour': rpc_bench.styles['content'],
- 'disable': not verbose,
- }
-
- # case: single node and single test
- if node is not None and test is not None:
- result = run_load_test(node=node, test=test)
- return {node['name']: result}
-
- # case: single node and multiple tests
- elif node is not None and tests is not None:
- results = {}
- for name, test in tqdm.tqdm(tests.items(), **pbar):
- results[name] = run_load_test(node=node, verbose=verbose, test=test)
- return results
-
- # case: multiple nodes and single tests
- elif nodes is not None and test is not None:
- results = {}
- for name, nd in tqdm.tqdm(nodes.items(), **pbar):
- results[name] = run_load_test(node=nd, verbose=verbose, test=test)
- return results
-
- # case: multiple nodes and multiple tests
- elif nodes is not None and tests is not None:
- results = {}
- for node_name, node in nodes.items():
- for test_name, test in tests.items():
- results[node_name + '__' + test_name] = run_load_test(
- node=node,
- verbose=verbose,
- test=test,
- )
- return results
-
- # case: invalid input
- else:
- raise Exception('invalid user_io')
-
-
-def run_load_test(
- *,
- node: spec.NodeShorthand,
- test: spec.LoadTest | None = None,
- rates: typing.Sequence[int] | None = None,
- calls: typing.Sequence[typing.Any] | None = None,
- duration: int | None = None,
- durations: typing.Sequence[int] | None = None,
- vegeta_kwargs: typing.Mapping[str, str | None] | None = None,
- verbose: bool | int = False,
- _pbar_kwargs: typing.Mapping[str, typing.Any] | None = None,
-) -> spec.LoadTestOutput:
- """run a load test against a single node"""
-
- # parse user_io
- node = user_io.parse_node(node)
- if test is None:
- if rates is None or calls is None:
- raise Exception('specify rates and calls')
- test = load_test_construction.construct_load_test(
- rates=rates,
- calls=calls,
- duration=duration,
- durations=durations,
- vegeta_kwargs=vegeta_kwargs,
- )
-
- # run tests
- if node.get('remote') is None:
- return _run_load_test_locally(
- node=node,
- test=test,
- verbose=verbose,
- _pbar_kwargs=_pbar_kwargs,
- )
- else:
- return _run_load_test_remotely(
- node=node,
- test=test,
- verbose=verbose,
- _pbar_kwargs=_pbar_kwargs,
- )
-
-
-def _run_load_test_locally(
- *,
- node: spec.Node,
- test: spec.LoadTest | None,
- verbose: bool | int = False,
- _pbar_kwargs: typing.Mapping[str, typing.Any] | None = None,
-) -> spec.LoadTestOutput:
- """run a load test from local node"""
-
- # construct progress bar
- if _pbar_kwargs is None:
- _pbar_kwargs = {}
- tqdm = user_io.outputs._get_tqdm()
- tqdm_kwargs = dict(
- leave=False,
- desc='samples',
- position=1,
- colour=rpc_bench.styles['content'],
- disable=not verbose,
- **_pbar_kwargs,
- )
-
- # perform tests
- results = []
- for attack in tqdm.tqdm(test, **tqdm_kwargs):
- result = vegeta.run_vegeta_attack(
- url=node['url'],
- calls=attack['calls'],
- duration=attack['duration'],
- rate=attack['rate'],
- vegeta_kwargs=attack['vegeta_kwargs'],
- verbose=verbose >= 2,
- )
- results.append(result)
- if verbose >= 2:
- print()
-
- # format output
- keys = results[0].keys()
- return {key: [result[key] for result in results] for key in keys} # type: ignore
-
-
-def _run_load_test_remotely(
- *,
- node: spec.Node,
- test: spec.LoadTest,
- verbose: bool | int = False,
- _pbar_kwargs: typing.Mapping[str, typing.Any] | None = None,
-) -> spec.LoadTestOutput:
- """run a load test from local node"""
-
- import json
- import os
- import subprocess
- import uuid
- import toolstr
-
- # parse node specification
- remote = node['remote']
- if remote is None:
- raise Exception('not a remote node')
-
- # save call data, saving methods to preserve ordering in json
- job_id = str(uuid.uuid4())
- tempdir = '/tmp/rpc_bench__' + job_id
- os.makedirs(tempdir)
- rpc_bench.user_io.file_io._save_single_run_test(
- test_name='',
- test=test,
- output_dir=tempdir,
- )
-
- # send call data to remote server
- if verbose:
- import datetime
-
- dt = datetime.datetime.now()
- if dt.microsecond >= 500_000:
- dt = dt + datetime.timedelta(microseconds=1_000_000 - dt.microsecond)
- else:
- dt = dt - datetime.timedelta(microseconds=dt.microsecond)
- timestamp = (
- toolstr.add_style('\[', rpc_bench.styles['content'])
- + toolstr.add_style(str(dt), rpc_bench.styles['metavar'])
- + toolstr.add_style(']', rpc_bench.styles['content'])
- )
- node_name = (
- toolstr.add_style('\[', rpc_bench.styles['content'])
- + toolstr.add_style('node', rpc_bench.styles['metavar'])
- + toolstr.add_style('=', rpc_bench.styles['content'])
- + node['name']
- + toolstr.add_style(']', rpc_bench.styles['content'])
- )
- toolstr.print(
- timestamp + ' ' + node_name + ' Sending tests to remote node'
- )
- cmd = 'rsync -r ' + tempdir + ' ' + remote + ':' + os.path.dirname(tempdir)
- subprocess.call(cmd.split(' '), stderr=subprocess.DEVNULL)
-
- # initiate benchmarks
- if verbose:
- dt = datetime.datetime.now()
- if dt.microsecond >= 500_000:
- dt = dt + datetime.timedelta(microseconds=1_000_000 - dt.microsecond)
- else:
- dt = dt - datetime.timedelta(microseconds=dt.microsecond)
- timestamp = (
- toolstr.add_style('\[', rpc_bench.styles['content'])
- + toolstr.add_style(str(dt), rpc_bench.styles['metavar'])
- + toolstr.add_style(']', rpc_bench.styles['content'])
- )
- toolstr.print(timestamp + ' ' + node_name + ' Executing test on remote node')
- cmd = "ssh {host} bash -c 'source ~/.profile; python3 -m rpc_bench {test} {name}={url} --output {output} --no-figures'".format(
- host=remote,
- name=node['name'],
- url=node['url'],
- test=tempdir,
- output=tempdir,
- )
- subprocess.check_output(cmd.split(' '), stderr=subprocess.DEVNULL)
-
- # retrieve benchmark results
- if verbose:
- dt = datetime.datetime.now()
- if dt.microsecond >= 500_000:
- dt = dt + datetime.timedelta(microseconds=1_000_000 - dt.microsecond)
- else:
- dt = dt - datetime.timedelta(microseconds=dt.microsecond)
- timestamp = (
- toolstr.add_style('\[', rpc_bench.styles['content'])
- + toolstr.add_style(str(dt), rpc_bench.styles['metavar'])
- + toolstr.add_style(']', rpc_bench.styles['content'])
- )
- toolstr.print(timestamp + ' ' + node_name + ' Retrieving results')
- results_path = os.path.join(tempdir, 'results.json')
- cmd = 'rsync ' + remote + ':' + results_path + ' ' + results_path
- subprocess.call(cmd.split(' '), stderr=subprocess.DEVNULL)
-
- # return results
- with open(results_path, 'r') as f:
- results: spec.SingleRunResultsPayload = json.load(f)
- return results['results'][node['name']]
-
diff --git a/rpc_bench/user_io/file_io.py b/rpc_bench/user_io/file_io.py
deleted file mode 100644
index 7e4d612..0000000
--- a/rpc_bench/user_io/file_io.py
+++ /dev/null
@@ -1,115 +0,0 @@
-from __future__ import annotations
-
-import typing
-
-import rpc_bench
-
-
-_path_templates = {
- 'single_run_test': '{output_dir}/test.json',
- 'single_run_results': '{output_dir}/results.json',
- 'single_run_figures_dir': '{output_dir}/figures',
-}
-
-
-def get_single_run_test_path(output_dir: str) -> str:
- return _path_templates['single_run_test'].format(output_dir=output_dir)
-
-
-def get_single_run_results_path(output_dir: str) -> str:
- return _path_templates['single_run_results'].format(output_dir=output_dir)
-
-
-def get_single_run_figures_path(output_dir: str) -> str:
- return _path_templates['single_run_figures_dir'].format(
- output_dir=output_dir
- )
-
-
-def _save_single_run_test(
- test_name: str,
- output_dir: str,
- test: rpc_bench.LoadTest,
-) -> None:
- import os
- import json
-
- if not os.path.isdir(output_dir):
- if os.path.exists(output_dir):
- raise Exception('output must be a directory path')
- else:
- os.makedirs(output_dir, exist_ok=True)
-
- path = _path_templates['single_run_test'].format(output_dir=output_dir)
- payload = {
- 'version': rpc_bench.__version__,
- 'type': 'single_test',
- 'name': test_name,
- 'test': test,
- }
- with open(path, 'w') as f:
- json.dump(payload, f)
-
-
-def _save_single_run_results(
- output_dir: str,
- test: rpc_bench.LoadTest,
- nodes: rpc_bench.Nodes,
- results: typing.Mapping[str, rpc_bench.LoadTestOutput],
-) -> None:
- import os
- import json
-
- if not os.path.isdir(output_dir):
- if os.path.exists(output_dir):
- raise Exception('output must be a directory path')
- else:
- os.makedirs(output_dir)
-
- path = _path_templates['single_run_results'].format(output_dir=output_dir)
- payload = {
- 'version': rpc_bench.__version__,
- 'type': 'single_test',
- 'test': test,
- 'nodes': nodes,
- 'results': results,
- }
- with open(path, 'w') as f:
- json.dump(payload, f)
-
-
-def load_single_run_test_payload(
- output_dir: str,
-) -> rpc_bench.SingleRunTestPayload:
- import json
-
- path = get_single_run_test_path(output_dir=output_dir)
- with open(path) as f:
- test: rpc_bench.SingleRunTestPayload = json.load(f)
- return test
-
-
-def load_single_run_test(
- output_dir: str,
-) -> rpc_bench.LoadTest:
- payload = load_single_run_test_payload(output_dir=output_dir)
- return payload['test']
-
-
-def load_single_run_results_payload(
- output_dir: str,
-) -> rpc_bench.SingleRunResultsPayload:
- import json
-
- path = get_single_run_results_path(output_dir=output_dir)
- with open(path) as f:
- results: rpc_bench.SingleRunResultsPayload = json.load(f)
- return results
-
-
-def load_single_run_results(
- output_dir: str,
-) -> typing.Mapping[str, rpc_bench.LoadTestOutput]:
- payload = load_single_run_results_payload(output_dir=output_dir)
- return payload['results']
-
diff --git a/rpc_bench/user_io/inputs.py b/rpc_bench/user_io/inputs.py
deleted file mode 100644
index 6b4464d..0000000
--- a/rpc_bench/user_io/inputs.py
+++ /dev/null
@@ -1,110 +0,0 @@
-from __future__ import annotations
-
-import typing
-
-from rpc_bench import spec
-
-
-def get_ctc_alias_url(url: str) -> str | None:
- try:
- import ctc.config
-
- ctc_providers = ctc.config.get_providers()
- if url in ctc_providers:
- url = ctc_providers[url]['url']
- return url
- except ImportError:
- return None
-
-
-def parse_nodes(nodes: spec.NodesShorthand) -> typing.Mapping[str, spec.Node]:
- """parse given nodes according to input specification"""
- new_nodes: typing.MutableMapping[str, spec.Node] = {}
- if isinstance(nodes, list):
- for node in nodes:
- new_node = parse_node(node)
- new_nodes[new_node['name']] = new_node
- elif isinstance(nodes, dict):
- for key, value in nodes.items():
- new_nodes[key] = value
- else:
- raise Exception('invalid format for nodes')
- return new_nodes
-
-
-def parse_node(node: str | spec.Node) -> spec.Node:
- """parse node according to input specification"""
- prefixes = ['http', 'https', 'ws', 'wss']
-
- if isinstance(node, dict):
- return node
- elif isinstance(node, str):
- # parse name
- if '=' in node:
- name, url = node.split('=')
- else:
- name = node
- url = node
-
- # check if node is in ctc aliases
- alias_url = get_ctc_alias_url(url)
- if alias_url is not None:
- url = alias_url
-
- # parse remote and url
- if ':' in url:
- head, tail = url.split(':', 1)
- if head in prefixes:
- remote = None
- url = head + ':' + tail
- else:
- if tail.split('/')[0].isdecimal():
- remote = None
- url = url
- else:
- remote = head
- url = tail
- else:
- remote = None
-
- # add missing prefix
- if not any(url.startswith(prefix) for prefix in prefixes):
- # check if is ip
- pieces = url.split(':')[0].split('.')
- is_ip = len(pieces) == 4 and all(
- piece.isdecimal() for piece in pieces
- )
-
- # add prefix
- if url.startswith('localhost') or is_ip:
- url = 'http://' + url
- else:
- url = 'https://' + url
-
- return {
- 'name': name,
- 'url': url,
- 'remote': remote,
- }
-
- else:
- raise Exception('invalid node format')
-
-
-def parse_test_data(test: spec.LoadTest) -> spec.LoadTestColumnWise:
- rates = []
- durations = []
- vegeta_kwargs = []
- calls = []
- for attack in test:
- rates.append(attack['rate'])
- durations.append(attack['duration'])
- vegeta_kwargs.append(attack['vegeta_kwargs'])
- calls.append(attack['calls'])
- return {
- 'rates': rates,
- 'durations': durations,
- 'vegeta_kwargs': vegeta_kwargs,
- 'calls': calls,
- }
-
diff --git a/rpc_bench/user_io/outputs.py b/rpc_bench/user_io/outputs.py
deleted file mode 100644
index a9c3eea..0000000
--- a/rpc_bench/user_io/outputs.py
+++ /dev/null
@@ -1,227 +0,0 @@
-from __future__ import annotations
-
-import typing
-
-import rpc_bench
-from rpc_bench import spec
-
-if typing.TYPE_CHECKING:
- import types
-
- import polars as pl
- import toolcli
-
-
-styles: toolcli.StyleTheme = {
- 'title': 'bold #00e100',
- 'metavar': 'bold #e5e9f0',
- 'description': '#aaaaaa',
- 'content': '#00B400',
- 'option': 'bold #e5e9f0',
- 'comment': '#888888',
-}
-
-
-colors = {
- 'orange_shades': [
- 'darkgoldenrod',
- 'darkorange',
- 'gold',
- ],
- 'blue_shades': [
- 'blue',
- 'dodgerblue',
- 'lightskyblue',
- ],
- 'streetlight': [
- 'crimson',
- 'goldenrod',
- 'limegreen',
- ],
-}
-
-color_defaults = [
- 'darkorange',
- 'dodgerblue',
-]
-
-
-def _get_tqdm() -> types.ModuleType:
- import sys
-
- if 'jupyter_client' in sys.modules:
- try:
- import ipywidgets # type: ignore
- import tqdm.notebook as tqdm
-
- return tqdm
- except ImportError:
- pass
-
- import tqdm # type: ignore
-
- return tqdm
-
-
-def outputs_to_dataframe(
- outputs: typing.Mapping[str, spec.LoadTestOutput]
-) -> pl.DataFrame:
- import polars as pl
-
- return pl.concat(
- [
- pl.DataFrame(data).with_columns(pl.lit(name).alias('test'))
- for name, data in outputs.items()
- ]
- )
-
-
-def print_load_test_summary(test: rpc_bench.LoadTest) -> None:
- import toolstr
-
- parsed = rpc_bench.parse_test_data(test)
- rates = parsed['rates']
- durations = parsed['durations']
- vegeta_kwargs = parsed['vegeta_kwargs']
-
- toolstr.print_bullet(
- key='sample rates', value=rates, styles=rpc_bench.styles
- )
- if len(set(durations)) == 1:
- toolstr.print_bullet(
- key='sample duration',
- value=durations[0],
- styles=rpc_bench.styles,
- )
- else:
- toolstr.print_bullet(
- key='sample durations', value=durations, styles=rpc_bench.styles
- )
- if vegeta_kwargs is None or len(vegeta_kwargs) == 0:
- toolstr.print_bullet(
- key='extra args', value=None, styles=rpc_bench.styles
- )
-
-
-def print_metric_tables(
- results: typing.Mapping[str, spec.LoadTestOutput],
- metrics: typing.Sequence[str],
- *,
- decimals: int | None = None,
- comparison: bool = False,
-) -> None:
- import toolstr
-
- if len(results) == 0:
- print('no results')
- print()
-
- names = list(results.keys())
- rates = results[names[0]]['target_rate']
- for metric in metrics:
- # create labels
- if metric == 'success':
- suffix = ''
- else:
- suffix = ' (s)'
- unitted_names = [name + suffix for name in names]
- labels = ['rate (rps)'] + unitted_names
- if comparison:
- if len(results) != 2:
- raise NotImplementedError('comparison of >2 tests')
- comparison_label = names[0] + ' / ' + names[1]
- labels.append(comparison_label)
- else:
- comparison_label = None
-
- # build rows
- rows: list[list[typing.Any]] = [[rate] for rate in rates]
- values = []
- for name, result in results.items():
- for row, value in zip(rows, result[metric]): # type: ignore
- row.append(value)
- values.append(value)
- if comparison:
- for row in rows:
- row.append(row[-2] / row[-1])
-
- # compute column formats
- if all(value > 1 for value in values):
- use_decimals = 1
- else:
- if decimals is None:
- use_decimals = 6
- else:
- use_decimals = decimals
- column_formats = {
- column: {'decimals': use_decimals} for column in unitted_names
- }
- if comparison_label is not None:
- column_formats[comparison_label] = {
- 'decimals': 1,
- 'percentage': True,
- }
-
- # print header
- toolstr.print_text_box(
- toolstr.add_style(
- metric + ' vs load', rpc_bench.styles.get('metavar')
- ),
- style=rpc_bench.styles.get('content'),
- )
-
- # print table
- toolstr.print_table(
- rows,
- labels=labels,
- column_formats=column_formats, # type: ignore
- label_style=rpc_bench.styles.get('metavar'),
- border=rpc_bench.styles.get('content'),
- )
- if metric != metrics[-1]:
- print()
-
-#
-# # generic restylings of toolstr functions
-#
-
-def print_text_box(text: str) -> None:
- import toolstr
-
- toolstr.print_text_box(
- text,
- text_style=rpc_bench.styles.get('metavar'),
- style=rpc_bench.styles.get('content'),
- )
-
-
-def print_header(text: str) -> None:
- import toolstr
-
- toolstr.print_header(
- text,
- text_style=rpc_bench.styles.get('metavar'),
- style=rpc_bench.styles.get('content'),
- )
-
-
-def print_bullet(*args: typing.Any, **kwargs: typing.Any) -> None:
- import toolstr
-
- toolstr.print_bullet(
- *args,
- **kwargs,
- styles=rpc_bench.styles,
- )
-
-
-def print_table(*args: typing.Any, **kwargs: typing.Any) -> None:
- import toolstr
-
- toolstr.print_table(
- *args,
- **kwargs,
- label_style=rpc_bench.styles.get('metavar'),
- border=rpc_bench.styles.get('content'),
- )
-
diff --git a/tests/README.md b/tests/README.md
new file mode 100644
index 0000000..961f12f
--- /dev/null
+++ b/tests/README.md
@@ -0,0 +1,8 @@
+
+# Environment variables
+The following env variables should be set for complete testing of `flood`:
+- `FLOOD_TEST_LOCAL_NODE_1`: example = `localhost:8545`
+- `FLOOD_TEST_LOCAL_NODE_2`: example = `123.123.123.123:8888`
+- `FLOOD_TEST_REMOTE_NODE_1`: example = `ubuntu@123.123.123.123:localhost:8545`
+- `FLOOD_TEST_REMOTE_NODE_2`: example = `ubuntu@123.123.123.123:45.45.45.45:8545`
+
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 0000000..73adbd9
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,46 @@
+import os
+
+import pytest
+
+
+def get_env_value(name):
+ value = os.environ.get(name)
+ if value in [None, '']:
+ raise Exception(name + ' not set')
+ return value
+
+
+@pytest.fixture
+def flood_test_local_node_1():
+ skip_if_env_nodes_unset()
+ return get_env_value('FLOOD_TEST_LOCAL_NODE_1')
+
+
+@pytest.fixture
+def flood_test_local_node_2():
+ skip_if_env_nodes_unset()
+ return get_env_value('FLOOD_TEST_LOCAL_NODE_2')
+
+
+@pytest.fixture
+def flood_test_remote_node_1():
+ skip_if_env_nodes_unset()
+ return get_env_value('FLOOD_TEST_REMOTE_NODE_1')
+
+
+@pytest.fixture
+def flood_test_remote_node_2():
+ skip_if_env_nodes_unset()
+ return get_env_value('FLOOD_TEST_REMOTE_NODE_2')
+
+
+def skip_if_env_nodes_unset():
+ for var in [
+ 'FLOOD_TEST_LOCAL_NODE_1',
+ 'FLOOD_TEST_LOCAL_NODE_2',
+ 'FLOOD_TEST_REMOTE_NODE_1',
+ 'FLOOD_TEST_REMOTE_NODE_2',
+ ]:
+ if os.getenv(var) is None:
+ pytest.skip(reason=var + ' env var not set')
+
diff --git a/tests/test_env_vars.py b/tests/test_env_vars.py
new file mode 100644
index 0000000..c8a2327
--- /dev/null
+++ b/tests/test_env_vars.py
@@ -0,0 +1,22 @@
+import flood
+
+
+def test_local_node_1(flood_test_local_node_1):
+ result = flood.user_io.parse_node(flood_test_local_node_1)
+ assert result['remote'] is None
+
+
+def test_local_node_2(flood_test_local_node_2):
+ result = flood.user_io.parse_node(flood_test_local_node_2)
+ assert result['remote'] is None
+
+
+def test_remote_node_1(flood_test_remote_node_1):
+ result = flood.user_io.parse_node(flood_test_remote_node_1)
+ assert result['remote'] is not None
+
+
+def test_remote_node_2(flood_test_remote_node_2):
+ result = flood.user_io.parse_node(flood_test_remote_node_2)
+ assert result['remote'] is not None
+
diff --git a/tests/test_equality_tests.py b/tests/test_equality_tests.py
new file mode 100644
index 0000000..da23caa
--- /dev/null
+++ b/tests/test_equality_tests.py
@@ -0,0 +1,10 @@
+import subprocess
+
+
+def test_equality_tests(flood_test_local_node_1, flood_test_local_node_2):
+ cmd = 'flood all node1={} node2={} --equality'.format(
+ flood_test_local_node_1,
+ flood_test_local_node_2,
+ )
+ subprocess.check_call(cmd.split(' '))
+
diff --git a/tests/test_load_tests.py b/tests/test_load_tests.py
new file mode 100644
index 0000000..5b11766
--- /dev/null
+++ b/tests/test_load_tests.py
@@ -0,0 +1,36 @@
+import subprocess
+
+import pytest
+
+import flood
+
+
+cmd_templates = {
+ 'local_bare': 'flood {test_name} {local_node_1} {local_node_2} -d 1 -r 1 2 4', # noqa: E501
+ # 'local_alias': 'flood {test_name} node1={local_node_1} node2={local_node_2} -d 1 -r 1 2 4', # noqa: E501
+ 'remote_bare': 'flood {test_name} {remote_node_1} {remote_node_2} -d 1 -r 20 40 60', # noqa: E501
+ # 'remote_alias': 'flood {test_name} node1={remote_node_1} node2={remote_node_2} -d 1 -r 1 2 4', # noqa: E501
+}
+
+
+@pytest.mark.parametrize(
+ 'test_name', flood.generators.get_single_test_generators().keys()
+)
+@pytest.mark.parametrize('cmd_template', cmd_templates.keys())
+def test_load_test(
+ test_name,
+ cmd_template,
+ flood_test_local_node_1,
+ flood_test_local_node_2,
+ flood_test_remote_node_1,
+ flood_test_remote_node_2,
+):
+ cmd = cmd_templates[cmd_template].format(
+ test_name=test_name,
+ local_node_1=flood_test_local_node_1,
+ local_node_2=flood_test_local_node_2,
+ remote_node_1=flood_test_remote_node_1,
+ remote_node_2=flood_test_remote_node_2,
+ )
+ subprocess.check_call(cmd.split(' '))
+
diff --git a/tests/test_node_parsing.py b/tests/test_node_parsing.py
index b0f39d6..928353a 100644
--- a/tests/test_node_parsing.py
+++ b/tests/test_node_parsing.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import pytest
-import rpc_bench
+import flood
names = [None, 'test_node', 'backup_node']
@@ -43,19 +43,22 @@ def test_parse_node(
if name is not None:
node_str = name + '=' + node_str
- parsed = rpc_bench.parse_node(node_str)
+ parsed = flood.user_io.parse_node(node_str, request_metadata=False)
if name is not None:
assert parsed['name'] == name
else:
assert parsed['name'] == unnamed_node_str
+ pieces = node_url.split(':')[0].split('.')
+ is_ip = len(pieces) == 4 and all(
+ piece.isdecimal() for piece in pieces
+ )
+
if url_prefix != '':
assert parsed['url'] == node_url
elif (
- node_url.startswith('localhost')
- or node_url.startswith('0.0.0.0')
- or node_url.startswith('127.0.0.1')
+ node_url.startswith('localhost') or is_ip
):
assert parsed['url'] == 'http://' + node_url
else: