From a7af1cac489715093a16d84472a140dfc928caeb Mon Sep 17 00:00:00 2001 From: zerosnacks <95942363+zerosnacks@users.noreply.github.com> Date: Mon, 15 Apr 2024 19:05:51 +0200 Subject: [PATCH] Bug: Etherscan cache is constantly invalidated (#40) Closes: https://github.com/foundry-rs/foundry/issues/6792 Due to a small logical error and a missing smoke test the Etherscan cache was constantly being invalidated. On a large codebase with 197 tests, without this optimization: https://github.com/foundry-rs/foundry/pull/7606 Comparison: 1. Ran 17 test suites in 53.41s (64.87s CPU time): 197 tests passed, 0 failed, 0 skipped (197 total tests) (no cache) 2. Ran 17 test suites in 6.29s (49.10s CPU time): 197 tests passed, 0 failed, 0 skipped (197 total tests) (with cache) --- src/lib.rs | 30 ++++++++++-- ...00000219ab540356cbb839cbe05303d7705fa.json | 1 + ...00000219ab540356cbb839cbe05303d7705fa.json | 1 + tests/it/contract.rs | 47 ++++++++++++++++++- tests/it/main.rs | 23 +++++++++ 5 files changed, 97 insertions(+), 5 deletions(-) create mode 100644 test-data/cache/abi/0x00000000219ab540356cbb839cbe05303d7705fa.json create mode 100644 test-data/cache/sources/0x00000000219ab540356cbb839cbe05303d7705fa.json diff --git a/src/lib.rs b/src/lib.rs index 0b2c85f..751f989 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -388,16 +388,31 @@ impl ClientBuilder { } /// A wrapper around an Etherscan cache object with an expiry +/// time for each item. #[derive(Clone, Debug, Deserialize, Serialize)] struct CacheEnvelope { + // The expiry time is the time the cache item was created + the cache TTL. + // The cache item is considered expired if the current time is greater than the expiry time. expiry: u64, + // The cached data. data: T, } -/// Simple cache for etherscan requests +/// Simple cache for Etherscan requests. +/// +/// The cache is stored at the defined `root` with the following structure: +/// +/// - $root/abi/$address.json +/// - $root/sources/$address.json +/// +/// Each cache item is stored as a JSON file with the following structure: +/// +/// - { "expiry": $expiry, "data": $data } #[derive(Clone, Debug)] struct Cache { + // Path to the cache directory root. root: PathBuf, + // Time to live for each cache item. ttl: Duration, } @@ -444,18 +459,25 @@ impl Cache { fn get(&self, prefix: &str, address: Address) -> Option { let path = self.root.join(prefix).join(format!("{address:?}.json")); + let Ok(contents) = std::fs::read_to_string(path) else { return None; }; + let Ok(inner) = serde_json::from_str::>(&contents) else { return None; }; - // If this does not return None then we have passed the expiry + + // Check if the cache item is still valid. SystemTime::now() .duration_since(UNIX_EPOCH) .expect("system time is before unix epoch") - .checked_sub(Duration::from_secs(inner.expiry)) - .map(|_| inner.data) + // Check if the current time is less than the expiry time + // to determine if the cache item is still valid. + .lt(&Duration::from_secs(inner.expiry)) + // If the cache item is still valid, return the data. + // Otherwise, return None. + .then_some(inner.data) } } diff --git a/test-data/cache/abi/0x00000000219ab540356cbb839cbe05303d7705fa.json b/test-data/cache/abi/0x00000000219ab540356cbb839cbe05303d7705fa.json new file mode 100644 index 0000000..c6d1ab5 --- /dev/null +++ b/test-data/cache/abi/0x00000000219ab540356cbb839cbe05303d7705fa.json @@ -0,0 +1 @@ +{"expiry":1713286751,"data":[{"type":"constructor","inputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"deposit","inputs":[{"name":"pubkey","type":"bytes","internalType":"bytes"},{"name":"withdrawal_credentials","type":"bytes","internalType":"bytes"},{"name":"signature","type":"bytes","internalType":"bytes"},{"name":"deposit_data_root","type":"bytes32","internalType":"bytes32"}],"outputs":[],"stateMutability":"payable"},{"type":"function","name":"get_deposit_count","inputs":[],"outputs":[{"name":"","type":"bytes","internalType":"bytes"}],"stateMutability":"view"},{"type":"function","name":"get_deposit_root","inputs":[],"outputs":[{"name":"","type":"bytes32","internalType":"bytes32"}],"stateMutability":"view"},{"type":"function","name":"supportsInterface","inputs":[{"name":"interfaceId","type":"bytes4","internalType":"bytes4"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"pure"},{"type":"event","name":"DepositEvent","inputs":[{"name":"pubkey","type":"bytes","indexed":false,"internalType":"bytes"},{"name":"withdrawal_credentials","type":"bytes","indexed":false,"internalType":"bytes"},{"name":"amount","type":"bytes","indexed":false,"internalType":"bytes"},{"name":"signature","type":"bytes","indexed":false,"internalType":"bytes"},{"name":"index","type":"bytes","indexed":false,"internalType":"bytes"}],"anonymous":false}]} \ No newline at end of file diff --git a/test-data/cache/sources/0x00000000219ab540356cbb839cbe05303d7705fa.json b/test-data/cache/sources/0x00000000219ab540356cbb839cbe05303d7705fa.json new file mode 100644 index 0000000..d2fc5b5 --- /dev/null +++ b/test-data/cache/sources/0x00000000219ab540356cbb839cbe05303d7705fa.json @@ -0,0 +1 @@ +{"expiry":1713286754,"data":[{"SourceCode":"// ┏━━━┓━┏┓━┏┓━━┏━━━┓━━┏━━━┓━━━━┏━━━┓━━━━━━━━━━━━━━━━━━━┏┓━━━━━┏━━━┓━━━━━━━━━┏┓━━━━━━━━━━━━━━┏┓━\r\n// ┃┏━━┛┏┛┗┓┃┃━━┃┏━┓┃━━┃┏━┓┃━━━━┗┓┏┓┃━━━━━━━━━━━━━━━━━━┏┛┗┓━━━━┃┏━┓┃━━━━━━━━┏┛┗┓━━━━━━━━━━━━┏┛┗┓\r\n// ┃┗━━┓┗┓┏┛┃┗━┓┗┛┏┛┃━━┃┃━┃┃━━━━━┃┃┃┃┏━━┓┏━━┓┏━━┓┏━━┓┏┓┗┓┏┛━━━━┃┃━┗┛┏━━┓┏━┓━┗┓┏┛┏━┓┏━━┓━┏━━┓┗┓┏┛\r\n// ┃┏━━┛━┃┃━┃┏┓┃┏━┛┏┛━━┃┃━┃┃━━━━━┃┃┃┃┃┏┓┃┃┏┓┃┃┏┓┃┃━━┫┣┫━┃┃━━━━━┃┃━┏┓┃┏┓┃┃┏┓┓━┃┃━┃┏┛┗━┓┃━┃┏━┛━┃┃━\r\n// ┃┗━━┓━┃┗┓┃┃┃┃┃┃┗━┓┏┓┃┗━┛┃━━━━┏┛┗┛┃┃┃━┫┃┗┛┃┃┗┛┃┣━━┃┃┃━┃┗┓━━━━┃┗━┛┃┃┗┛┃┃┃┃┃━┃┗┓┃┃━┃┗┛┗┓┃┗━┓━┃┗┓\r\n// ┗━━━┛━┗━┛┗┛┗┛┗━━━┛┗┛┗━━━┛━━━━┗━━━┛┗━━┛┃┏━┛┗━━┛┗━━┛┗┛━┗━┛━━━━┗━━━┛┗━━┛┗┛┗┛━┗━┛┗┛━┗━━━┛┗━━┛━┗━┛\r\n// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┃┃━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\r\n// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┗┛━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\r\n\r\n// SPDX-License-Identifier: CC0-1.0\r\n\r\npragma solidity 0.6.11;\r\n\r\n// This interface is designed to be compatible with the Vyper version.\r\n/// @notice This is the Ethereum 2.0 deposit contract interface.\r\n/// For more information see the Phase 0 specification under https://github.com/ethereum/eth2.0-specs\r\ninterface IDepositContract {\r\n /// @notice A processed deposit event.\r\n event DepositEvent(\r\n bytes pubkey,\r\n bytes withdrawal_credentials,\r\n bytes amount,\r\n bytes signature,\r\n bytes index\r\n );\r\n\r\n /// @notice Submit a Phase 0 DepositData object.\r\n /// @param pubkey A BLS12-381 public key.\r\n /// @param withdrawal_credentials Commitment to a public key for withdrawals.\r\n /// @param signature A BLS12-381 signature.\r\n /// @param deposit_data_root The SHA-256 hash of the SSZ-encoded DepositData object.\r\n /// Used as a protection against malformed input.\r\n function deposit(\r\n bytes calldata pubkey,\r\n bytes calldata withdrawal_credentials,\r\n bytes calldata signature,\r\n bytes32 deposit_data_root\r\n ) external payable;\r\n\r\n /// @notice Query the current deposit root hash.\r\n /// @return The deposit root hash.\r\n function get_deposit_root() external view returns (bytes32);\r\n\r\n /// @notice Query the current deposit count.\r\n /// @return The deposit count encoded as a little endian 64-bit number.\r\n function get_deposit_count() external view returns (bytes memory);\r\n}\r\n\r\n// Based on official specification in https://eips.ethereum.org/EIPS/eip-165\r\ninterface ERC165 {\r\n /// @notice Query if a contract implements an interface\r\n /// @param interfaceId The interface identifier, as specified in ERC-165\r\n /// @dev Interface identification is specified in ERC-165. This function\r\n /// uses less than 30,000 gas.\r\n /// @return `true` if the contract implements `interfaceId` and\r\n /// `interfaceId` is not 0xffffffff, `false` otherwise\r\n function supportsInterface(bytes4 interfaceId) external pure returns (bool);\r\n}\r\n\r\n// This is a rewrite of the Vyper Eth2.0 deposit contract in Solidity.\r\n// It tries to stay as close as possible to the original source code.\r\n/// @notice This is the Ethereum 2.0 deposit contract interface.\r\n/// For more information see the Phase 0 specification under https://github.com/ethereum/eth2.0-specs\r\ncontract DepositContract is IDepositContract, ERC165 {\r\n uint constant DEPOSIT_CONTRACT_TREE_DEPTH = 32;\r\n // NOTE: this also ensures `deposit_count` will fit into 64-bits\r\n uint constant MAX_DEPOSIT_COUNT = 2**DEPOSIT_CONTRACT_TREE_DEPTH - 1;\r\n\r\n bytes32[DEPOSIT_CONTRACT_TREE_DEPTH] branch;\r\n uint256 deposit_count;\r\n\r\n bytes32[DEPOSIT_CONTRACT_TREE_DEPTH] zero_hashes;\r\n\r\n constructor() public {\r\n // Compute hashes in empty sparse Merkle tree\r\n for (uint height = 0; height < DEPOSIT_CONTRACT_TREE_DEPTH - 1; height++)\r\n zero_hashes[height + 1] = sha256(abi.encodePacked(zero_hashes[height], zero_hashes[height]));\r\n }\r\n\r\n function get_deposit_root() override external view returns (bytes32) {\r\n bytes32 node;\r\n uint size = deposit_count;\r\n for (uint height = 0; height < DEPOSIT_CONTRACT_TREE_DEPTH; height++) {\r\n if ((size & 1) == 1)\r\n node = sha256(abi.encodePacked(branch[height], node));\r\n else\r\n node = sha256(abi.encodePacked(node, zero_hashes[height]));\r\n size /= 2;\r\n }\r\n return sha256(abi.encodePacked(\r\n node,\r\n to_little_endian_64(uint64(deposit_count)),\r\n bytes24(0)\r\n ));\r\n }\r\n\r\n function get_deposit_count() override external view returns (bytes memory) {\r\n return to_little_endian_64(uint64(deposit_count));\r\n }\r\n\r\n function deposit(\r\n bytes calldata pubkey,\r\n bytes calldata withdrawal_credentials,\r\n bytes calldata signature,\r\n bytes32 deposit_data_root\r\n ) override external payable {\r\n // Extended ABI length checks since dynamic types are used.\r\n require(pubkey.length == 48, \"DepositContract: invalid pubkey length\");\r\n require(withdrawal_credentials.length == 32, \"DepositContract: invalid withdrawal_credentials length\");\r\n require(signature.length == 96, \"DepositContract: invalid signature length\");\r\n\r\n // Check deposit amount\r\n require(msg.value >= 1 ether, \"DepositContract: deposit value too low\");\r\n require(msg.value % 1 gwei == 0, \"DepositContract: deposit value not multiple of gwei\");\r\n uint deposit_amount = msg.value / 1 gwei;\r\n require(deposit_amount <= type(uint64).max, \"DepositContract: deposit value too high\");\r\n\r\n // Emit `DepositEvent` log\r\n bytes memory amount = to_little_endian_64(uint64(deposit_amount));\r\n emit DepositEvent(\r\n pubkey,\r\n withdrawal_credentials,\r\n amount,\r\n signature,\r\n to_little_endian_64(uint64(deposit_count))\r\n );\r\n\r\n // Compute deposit data root (`DepositData` hash tree root)\r\n bytes32 pubkey_root = sha256(abi.encodePacked(pubkey, bytes16(0)));\r\n bytes32 signature_root = sha256(abi.encodePacked(\r\n sha256(abi.encodePacked(signature[:64])),\r\n sha256(abi.encodePacked(signature[64:], bytes32(0)))\r\n ));\r\n bytes32 node = sha256(abi.encodePacked(\r\n sha256(abi.encodePacked(pubkey_root, withdrawal_credentials)),\r\n sha256(abi.encodePacked(amount, bytes24(0), signature_root))\r\n ));\r\n\r\n // Verify computed and expected deposit data roots match\r\n require(node == deposit_data_root, \"DepositContract: reconstructed DepositData does not match supplied deposit_data_root\");\r\n\r\n // Avoid overflowing the Merkle tree (and prevent edge case in computing `branch`)\r\n require(deposit_count < MAX_DEPOSIT_COUNT, \"DepositContract: merkle tree full\");\r\n\r\n // Add deposit data root to Merkle tree (update a single `branch` node)\r\n deposit_count += 1;\r\n uint size = deposit_count;\r\n for (uint height = 0; height < DEPOSIT_CONTRACT_TREE_DEPTH; height++) {\r\n if ((size & 1) == 1) {\r\n branch[height] = node;\r\n return;\r\n }\r\n node = sha256(abi.encodePacked(branch[height], node));\r\n size /= 2;\r\n }\r\n // As the loop should always end prematurely with the `return` statement,\r\n // this code should be unreachable. We assert `false` just to be safe.\r\n assert(false);\r\n }\r\n\r\n function supportsInterface(bytes4 interfaceId) override external pure returns (bool) {\r\n return interfaceId == type(ERC165).interfaceId || interfaceId == type(IDepositContract).interfaceId;\r\n }\r\n\r\n function to_little_endian_64(uint64 value) internal pure returns (bytes memory ret) {\r\n ret = new bytes(8);\r\n bytes8 bytesValue = bytes8(value);\r\n // Byteswapping during copying to bytes.\r\n ret[0] = bytesValue[7];\r\n ret[1] = bytesValue[6];\r\n ret[2] = bytesValue[5];\r\n ret[3] = bytesValue[4];\r\n ret[4] = bytesValue[3];\r\n ret[5] = bytesValue[2];\r\n ret[6] = bytesValue[1];\r\n ret[7] = bytesValue[0];\r\n }\r\n}","ABI":"[{\"inputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"constructor\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":false,\"internalType\":\"bytes\",\"name\":\"pubkey\",\"type\":\"bytes\"},{\"indexed\":false,\"internalType\":\"bytes\",\"name\":\"withdrawal_credentials\",\"type\":\"bytes\"},{\"indexed\":false,\"internalType\":\"bytes\",\"name\":\"amount\",\"type\":\"bytes\"},{\"indexed\":false,\"internalType\":\"bytes\",\"name\":\"signature\",\"type\":\"bytes\"},{\"indexed\":false,\"internalType\":\"bytes\",\"name\":\"index\",\"type\":\"bytes\"}],\"name\":\"DepositEvent\",\"type\":\"event\"},{\"inputs\":[{\"internalType\":\"bytes\",\"name\":\"pubkey\",\"type\":\"bytes\"},{\"internalType\":\"bytes\",\"name\":\"withdrawal_credentials\",\"type\":\"bytes\"},{\"internalType\":\"bytes\",\"name\":\"signature\",\"type\":\"bytes\"},{\"internalType\":\"bytes32\",\"name\":\"deposit_data_root\",\"type\":\"bytes32\"}],\"name\":\"deposit\",\"outputs\":[],\"stateMutability\":\"payable\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"get_deposit_count\",\"outputs\":[{\"internalType\":\"bytes\",\"name\":\"\",\"type\":\"bytes\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"get_deposit_root\",\"outputs\":[{\"internalType\":\"bytes32\",\"name\":\"\",\"type\":\"bytes32\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"bytes4\",\"name\":\"interfaceId\",\"type\":\"bytes4\"}],\"name\":\"supportsInterface\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"pure\",\"type\":\"function\"}]","ContractName":"DepositContract","CompilerVersion":"v0.6.11+commit.5ef660b1","OptimizationUsed":1,"Runs":5000000,"ConstructorArguments":"0x","EVMVersion":"Default","Library":"","LicenseType":"Unlicense","Proxy":0,"SwarmSource":"ipfs://dceca8706b29e917dacf25fceef95acac8d90d765ac926663ce4096195952b61"}]} \ No newline at end of file diff --git a/tests/it/contract.rs b/tests/it/contract.rs index 5a9b0a0..e8b47f6 100644 --- a/tests/it/contract.rs +++ b/tests/it/contract.rs @@ -1,4 +1,4 @@ -use crate::{init_tracing, run_with_client}; +use crate::{init_tracing, run_with_client, run_with_client_cached}; use alloy_chains::{Chain, NamedChain}; use foundry_block_explorers::{contract::SourceCodeMetadata, errors::EtherscanError, Client}; use serial_test::serial; @@ -31,6 +31,51 @@ async fn can_fetch_contract_abi() { .await; } +#[tokio::test] +#[serial] +async fn can_fetch_and_cache_contract_abi() { + async fn fetch_abi(client: &Client, addr: &str) { + let abi = client.contract_abi(addr.parse().unwrap()).await.unwrap(); + assert_eq!(abi, serde_json::from_str(DEPOSIT_CONTRACT_ABI).unwrap()); + } + + run_with_client_cached(Chain::mainnet(), |client| async move { + // Fetch the abi and cache it. + fetch_abi(&client, "0x00000000219ab540356cBB839Cbe05303d7705Fa").await; + + // Repeated calls on the cached abi should not trigger a new request. + for _ in 0..10 { + fetch_abi(&client, "0x00000000219ab540356cBB839Cbe05303d7705Fa").await; + } + }) + .await; +} + +#[tokio::test] +#[serial] +async fn can_fetch_and_cache_contract_source_code() { + async fn fetch_source_code(client: &Client, addr: &str) { + let meta = client.contract_source_code(addr.parse().unwrap()).await.unwrap(); + assert_eq!(meta.items.len(), 1); + + let item = &meta.items[0]; + assert!(matches!(item.source_code, SourceCodeMetadata::SourceCode(_))); + assert_eq!(item.source_code.sources().len(), 1); + assert_eq!(item.abi().unwrap(), serde_json::from_str(DEPOSIT_CONTRACT_ABI).unwrap()); + } + + run_with_client_cached(Chain::mainnet(), |client| async move { + // Fetch the source code and cache it. + fetch_source_code(&client, "0x00000000219ab540356cBB839Cbe05303d7705Fa").await; + + // Repeated calls on the cached source code should not trigger a new request. + for _ in 0..10 { + fetch_source_code(&client, "0x00000000219ab540356cBB839Cbe05303d7705Fa").await; + } + }) + .await; +} + #[tokio::test] #[serial] async fn can_fetch_deposit_contract_source_code_from_blockscout() { diff --git a/tests/it/main.rs b/tests/it/main.rs index 3001137..d8aaff6 100644 --- a/tests/it/main.rs +++ b/tests/it/main.rs @@ -6,6 +6,7 @@ use alloy_chains::{Chain, ChainKind, NamedChain}; use foundry_block_explorers::{errors::EtherscanError, Client}; use std::{ future::Future, + path::PathBuf, time::{Duration, Instant}, }; @@ -34,6 +35,28 @@ where run_at_least_duration(duration, f(client)).await } +/// Calls the function with a new cached Etherscan Client. +pub async fn run_with_client_cached(chain: Chain, f: F) -> T +where + F: FnOnce(Client) -> Fut, + Fut: Future, +{ + init_tracing(); + let cache_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("test-data/cache"); + let (client, duration) = match Client::builder() + .chain(chain) + .unwrap() + .with_cache(Some(cache_path), Duration::from_secs(24 * 60 * 60)) + .build() + { + Ok(c) => (c, rate_limit(chain, true)), + Err(_) => { + (Client::builder().chain(chain).unwrap().build().unwrap(), rate_limit(chain, false)) + } + }; + run_at_least_duration(duration, f(client)).await +} + #[track_caller] fn rate_limit(chain: Chain, key: bool) -> Duration { match (chain.kind(), key) {