diff --git a/v-next/example-project/contracts/Counter.t.sol b/v-next/example-project/contracts/Counter.t.sol index b59fca158d..6345d43724 100644 --- a/v-next/example-project/contracts/Counter.t.sol +++ b/v-next/example-project/contracts/Counter.t.sol @@ -14,6 +14,10 @@ contract CounterTest { require(counter.x() == 0, "Initial value should be 0"); } + function testFailInitialValue() public view { + require(counter.x() == 1, "Initial value should be 1"); + } + function testFuzzInc(uint8 x) public { for (uint8 i = 0; i < x; i++) { counter.inc(); @@ -21,6 +25,13 @@ contract CounterTest { require(counter.x() == x, "Value after calling inc x times should be x"); } + function testFailFuzzInc(uint8 x) public { + for (uint8 i = 0; i < x; i++) { + counter.inc(); + } + require(counter.x() == x + 1, "Value after calling inc x times should be x + 1"); + } + // function invariant() public pure { // assert(true); // } @@ -47,6 +58,13 @@ contract FailingCounterTest { ); } + function testFailFuzzInc(uint8 x) public { + for (uint8 i = 0; i < x; i++) { + counter.inc(); + } + require(counter.x() == x, "Value after calling inc x times should be x"); + } + // function invariant() public pure { // assert(false); // } diff --git a/v-next/example-project/hardhat.config.ts b/v-next/example-project/hardhat.config.ts index b65dca7018..37402f2db3 100644 --- a/v-next/example-project/hardhat.config.ts +++ b/v-next/example-project/hardhat.config.ts @@ -179,6 +179,9 @@ const config: HardhatUserConfig = { version: "0.8.1", }, }, + test: { + testFail: true, + } }, test: { version: "0.8.2", diff --git a/v-next/hardhat/src/internal/builtin-plugins/solidity-test/helpers.ts b/v-next/hardhat/src/internal/builtin-plugins/solidity-test/helpers.ts index a5f71e45ce..d8c725ad7d 100644 --- a/v-next/hardhat/src/internal/builtin-plugins/solidity-test/helpers.ts +++ b/v-next/hardhat/src/internal/builtin-plugins/solidity-test/helpers.ts @@ -1,10 +1,109 @@ +import type { RunOptions } from "./runner.js"; import type { ArtifactsManager } from "../../../types/artifacts.js"; -import type { Artifact } from "@ignored/edr"; +import type { SolidityTestUserConfig } from "../../../types/config.js"; +import type { + Artifact, + SolidityTestRunnerConfigArgs, + CachedChains, + CachedEndpoints, + PathPermission, + StorageCachingConfig, + AddressLabel, +} from "@ignored/edr"; import { HardhatError } from "@ignored/hardhat-vnext-errors"; import { exists } from "@ignored/hardhat-vnext-utils/fs"; +import { hexStringToBytes } from "@ignored/hardhat-vnext-utils/hex"; import { resolveFromRoot } from "@ignored/hardhat-vnext-utils/path"; +function hexStringToBuffer(hexString: string): Buffer { + return Buffer.from(hexStringToBytes(hexString)); +} + +export function solidityTestUserConfigToRunOptions( + config: SolidityTestUserConfig, +): RunOptions { + return config; +} + +export function solidityTestUserConfigToSolidityTestRunnerConfigArgs( + projectRoot: string, + config: SolidityTestUserConfig, +): SolidityTestRunnerConfigArgs { + const fsPermissions: PathPermission[] | undefined = [ + config.fsPermissions?.readWrite?.map((path) => ({ access: 0, path })) ?? [], + config.fsPermissions?.read?.map((path) => ({ access: 0, path })) ?? [], + config.fsPermissions?.write?.map((path) => ({ access: 0, path })) ?? [], + ].flat(1); + + const labels: AddressLabel[] | undefined = config.labels?.map( + ({ address, label }) => ({ + address: hexStringToBuffer(address), + label, + }), + ); + + let rpcStorageCaching: StorageCachingConfig | undefined; + if (config.rpcStorageCaching !== undefined) { + let chains: CachedChains | string[]; + if (Array.isArray(config.rpcStorageCaching.chains)) { + chains = config.rpcStorageCaching.chains; + } else { + const rpcStorageCachingChains: "All" | "None" = + config.rpcStorageCaching.chains; + switch (rpcStorageCachingChains) { + case "All": + chains = 0; + break; + case "None": + chains = 1; + break; + } + } + let endpoints: CachedEndpoints | string; + if (config.rpcStorageCaching.endpoints instanceof RegExp) { + endpoints = config.rpcStorageCaching.endpoints.source; + } else { + const rpcStorageCachingEndpoints: "All" | "Remote" = + config.rpcStorageCaching.endpoints; + switch (rpcStorageCachingEndpoints) { + case "All": + endpoints = 0; + break; + case "Remote": + endpoints = 1; + break; + } + } + rpcStorageCaching = { + chains, + endpoints, + }; + } + + const sender: Buffer | undefined = + config.sender === undefined ? undefined : hexStringToBuffer(config.sender); + const txOrigin: Buffer | undefined = + config.txOrigin === undefined + ? undefined + : hexStringToBuffer(config.txOrigin); + const blockCoinbase: Buffer | undefined = + config.blockCoinbase === undefined + ? undefined + : hexStringToBuffer(config.blockCoinbase); + + return { + projectRoot, + ...config, + fsPermissions, + labels, + sender, + txOrigin, + blockCoinbase, + rpcStorageCaching, + }; +} + export async function getArtifacts( hardhatArtifacts: ArtifactsManager, ): Promise { diff --git a/v-next/hardhat/src/internal/builtin-plugins/solidity-test/index.ts b/v-next/hardhat/src/internal/builtin-plugins/solidity-test/index.ts index 8211f9fbdc..0901c45c56 100644 --- a/v-next/hardhat/src/internal/builtin-plugins/solidity-test/index.ts +++ b/v-next/hardhat/src/internal/builtin-plugins/solidity-test/index.ts @@ -1,6 +1,5 @@ import type { HardhatPlugin } from "../../../types/plugins.js"; -import { ArgumentType } from "../../../types/arguments.js"; import { task } from "../../core/config.js"; const hardhatPlugin: HardhatPlugin = { @@ -8,13 +7,6 @@ const hardhatPlugin: HardhatPlugin = { tasks: [ task(["test", "solidity"], "Run the Solidity tests") .setAction(import.meta.resolve("./task-action.js")) - .addOption({ - name: "timeout", - description: - "The maximum time in milliseconds to wait for all the test suites to finish", - type: ArgumentType.INT, - defaultValue: 60 * 60 * 1000, - }) .addFlag({ name: "noCompile", description: "Don't compile the project before running the tests", diff --git a/v-next/hardhat/src/internal/builtin-plugins/solidity-test/runner.ts b/v-next/hardhat/src/internal/builtin-plugins/solidity-test/runner.ts index cce59bc417..dea51a124d 100644 --- a/v-next/hardhat/src/internal/builtin-plugins/solidity-test/runner.ts +++ b/v-next/hardhat/src/internal/builtin-plugins/solidity-test/runner.ts @@ -25,8 +25,6 @@ function getEdrContext(): EdrContext { export interface RunOptions { /** * The maximum time in milliseconds to wait for all the test suites to finish. - * - * If not provided, the default is 1 hour. */ timeout?: number; } @@ -62,18 +60,20 @@ export function run( const remainingSuites = new Set(testSuiteIds.map(formatArtifactId)); - // NOTE: The timeout prevents the situation in which the stream is never - // closed. This can happen if we receive fewer suite results than the - // number of test suites. The timeout is set to 1 hour. - const duration = options?.timeout ?? 60 * 60 * 1000; - const timeout = setTimeout(() => { - controller.error( - new HardhatError(HardhatError.ERRORS.SOLIDITY_TESTS.RUNNER_TIMEOUT, { - duration, - suites: Array.from(remainingSuites).join(", "), - }), - ); - }, duration); + let timeout: NodeJS.Timeout | undefined; + if (options?.timeout !== undefined) { + timeout = setTimeout(() => { + controller.error( + new HardhatError( + HardhatError.ERRORS.SOLIDITY_TESTS.RUNNER_TIMEOUT, + { + duration: options.timeout, + suites: Array.from(remainingSuites).join(", "), + }, + ), + ); + }, options.timeout); + } // TODO: Just getting the context here to get it initialized, but this // is not currently tied to the `runSolidityTests` function. diff --git a/v-next/hardhat/src/internal/builtin-plugins/solidity-test/task-action.ts b/v-next/hardhat/src/internal/builtin-plugins/solidity-test/task-action.ts index 02b8e14019..55660179dd 100644 --- a/v-next/hardhat/src/internal/builtin-plugins/solidity-test/task-action.ts +++ b/v-next/hardhat/src/internal/builtin-plugins/solidity-test/task-action.ts @@ -1,22 +1,27 @@ import type { RunOptions } from "./runner.js"; import type { TestEvent } from "./types.js"; import type { NewTaskActionFunction } from "../../../types/tasks.js"; +import type { SolidityTestRunnerConfigArgs } from "@ignored/edr"; import { finished } from "node:stream/promises"; import { createNonClosingWriter } from "@ignored/hardhat-vnext-utils/stream"; -import { getArtifacts, isTestArtifact } from "./helpers.js"; +import { + getArtifacts, + isTestArtifact, + solidityTestUserConfigToRunOptions, + solidityTestUserConfigToSolidityTestRunnerConfigArgs, +} from "./helpers.js"; import { testReporter } from "./reporter.js"; import { run } from "./runner.js"; interface TestActionArguments { - timeout: number; noCompile: boolean; } const runSolidityTests: NewTaskActionFunction = async ( - { timeout, noCompile }, + { noCompile }, hre, ) => { if (!noCompile) { @@ -42,14 +47,19 @@ const runSolidityTests: NewTaskActionFunction = async ( console.log("Running Solidity tests"); console.log(); - const config = { - projectRoot: hre.config.paths.root, - }; - let includesFailures = false; let includesErrors = false; - const options: RunOptions = { timeout }; + const profileName = hre.globalOptions.buildProfile; + const profile = hre.config.solidity.profiles[profileName]; + const testOptions = profile.test; + + const config: SolidityTestRunnerConfigArgs = + solidityTestUserConfigToSolidityTestRunnerConfigArgs( + hre.config.paths.root, + testOptions, + ); + const options: RunOptions = solidityTestUserConfigToRunOptions(testOptions); const runStream = run(artifacts, testSuiteIds, config, options); diff --git a/v-next/hardhat/src/internal/builtin-plugins/solidity/config.ts b/v-next/hardhat/src/internal/builtin-plugins/solidity/config.ts index e95ba2814d..d3f802e946 100644 --- a/v-next/hardhat/src/internal/builtin-plugins/solidity/config.ts +++ b/v-next/hardhat/src/internal/builtin-plugins/solidity/config.ts @@ -12,6 +12,7 @@ import { resolveFromRoot } from "@ignored/hardhat-vnext-utils/path"; import { conditionalUnionType, incompatibleFieldType, + unionType, validateUserConfigZodType, } from "@ignored/hardhat-vnext-zod-utils"; import { z } from "zod"; @@ -34,22 +35,114 @@ const solcUserConfigType = z.object({ profiles: incompatibleFieldType("This field is incompatible with `version`"), }); +const solidityTestUserConfigType = z.object({ + timeout: z.number().optional(), + fsPermissions: z + .object({ + readWrite: z.array(z.string()).optional(), + read: z.array(z.string()).optional(), + write: z.array(z.string()).optional(), + }) + .optional(), + trace: z.boolean().optional(), + testFail: z.boolean().optional(), + labels: z + .array( + z.object({ + address: z.string().startsWith("0x"), + label: z.string(), + }), + ) + .optional(), + isolate: z.boolean().optional(), + ffi: z.boolean().optional(), + sender: z.string().startsWith("0x").optional(), + txOrigin: z.string().startsWith("0x").optional(), + initialBalance: z.bigint().optional(), + blockBaseFeePerGas: z.bigint().optional(), + blockCoinbase: z.string().startsWith("0x").optional(), + blockTimestamp: z.bigint().optional(), + blockDifficulty: z.bigint().optional(), + blockGasLimit: z.bigint().optional(), + disableBlockGasLimit: z.boolean().optional(), + memoryLimit: z.bigint().optional(), + ethRpcUrl: z.string().optional(), + forkBlockNumber: z.bigint().optional(), + rpcEndpoints: z.record(z.string()).optional(), + rpcCachePath: z.string().optional(), + rpcStorageCaching: z + .object({ + chains: unionType( + [z.enum(["All", "None"]), z.array(z.string())], + "Expected `All`, `None` or a list of chain names to cache", + ), + endpoints: unionType( + [ + z.enum(["All", "Remote"]), + z.object({ + source: z.string(), + }), + ], + "Expected `All`, `Remote` or a RegExp object matching endpoints to cacche", + ), + }) + .optional(), + promptTimeout: z.number().optional(), + fuzz: z + .object({ + failurePersistDir: z.string().optional(), + failurePersistFile: z.string().optional(), + runs: z.number().optional(), + maxTestRejects: z.number().optional(), + seed: z.string().optional(), + dictionaryWeight: z.number().optional(), + includeStorage: z.boolean().optional(), + includePushBytes: z.boolean().optional(), + }) + .optional(), + invariant: z + .object({ + failurePersistDir: z.string().optional(), + runs: z.number().optional(), + depth: z.number().optional(), + failOnRevert: z.boolean().optional(), + callOverride: z.boolean().optional(), + dictionaryWeight: z.number().optional(), + includeStorage: z.boolean().optional(), + includePushBytes: z.boolean().optional(), + shrinkRunLimit: z.number().optional(), + }) + .optional(), +}); + +const singleVersionSolcUserConfigType = solcUserConfigType.extend({ + test: solidityTestUserConfigType.optional(), +}); + const multiVersionSolcUserConfigType = z.object({ compilers: z.array(solcUserConfigType).nonempty(), overrides: z.record(z.string(), solcUserConfigType).optional(), + test: solidityTestUserConfigType.optional(), version: incompatibleFieldType("This field is incompatible with `compilers`"), settings: incompatibleFieldType( "This field is incompatible with `compilers`", ), }); -const singleVersionSolidityUserConfigType = solcUserConfigType.extend({ - dependenciesToCompile: z.array(z.string()).optional(), - remappings: z.array(z.string()).optional(), - compilers: incompatibleFieldType("This field is incompatible with `version`"), - overrides: incompatibleFieldType("This field is incompatible with `version`"), - profiles: incompatibleFieldType("This field is incompatible with `version`"), -}); +const singleVersionSolidityUserConfigType = + singleVersionSolcUserConfigType.extend({ + dependenciesToCompile: z.array(z.string()).optional(), + remappings: z.array(z.string()).optional(), + compilers: incompatibleFieldType( + "This field is incompatible with `version`", + ), + overrides: incompatibleFieldType( + "This field is incompatible with `version`", + ), + profiles: incompatibleFieldType( + "This field is incompatible with `version`", + ), + }); const multiVersionSolidityUserConfigType = multiVersionSolcUserConfigType.extend({ @@ -68,7 +161,10 @@ const buildProfilesSolidityUserConfigType = z.object({ z.string(), conditionalUnionType( [ - [(data) => isObject(data) && "version" in data, solcUserConfigType], + [ + (data) => isObject(data) && "version" in data, + singleVersionSolcUserConfigType, + ], [ (data) => isObject(data) && "compilers" in data, multiVersionSolcUserConfigType, @@ -196,6 +292,7 @@ function resolveSolidityConfig( settings: {}, })), overrides: {}, + test: {}, }, }, dependenciesToCompile: [], @@ -214,6 +311,7 @@ function resolveSolidityConfig( }, ], overrides: {}, + test: solidityConfig.test ?? {}, }, }, dependenciesToCompile: solidityConfig.dependenciesToCompile ?? [], @@ -242,6 +340,7 @@ function resolveSolidityConfig( }, ), ), + test: solidityConfig.test ?? {}, }, }, dependenciesToCompile: solidityConfig.dependenciesToCompile ?? [], @@ -264,6 +363,7 @@ function resolveSolidityConfig( }, ], overrides: {}, + test: {}, }; continue; } @@ -286,6 +386,7 @@ function resolveSolidityConfig( }, ), ), + test: profile.test ?? {}, }; } diff --git a/v-next/hardhat/src/internal/builtin-plugins/solidity/type-extensions.ts b/v-next/hardhat/src/internal/builtin-plugins/solidity/type-extensions.ts index 7e64585791..f87ea46bc0 100644 --- a/v-next/hardhat/src/internal/builtin-plugins/solidity/type-extensions.ts +++ b/v-next/hardhat/src/internal/builtin-plugins/solidity/type-extensions.ts @@ -13,12 +13,75 @@ declare module "../../../types/config.js" { settings?: any; } + export interface SolidityTestUserConfig { + timeout?: number; + fsPermissions?: { + readWrite?: string[]; + read?: string[]; + write?: string[]; + }; + trace?: boolean; + testFail?: boolean; + labels?: Array<{ + address: string; // 0x-prefixed hex string + label: string; + }>; + isolate?: boolean; + ffi?: boolean; + sender?: string; // 0x-prefixed hex string + txOrigin?: string; // 0x-prefixed hex string + initialBalance?: bigint; + blockBaseFeePerGas?: bigint; + blockCoinbase?: string; // 0x-prefixed hex string + blockTimestamp?: bigint; + blockDifficulty?: bigint; + blockGasLimit?: bigint; + disableBlockGasLimit?: boolean; + memoryLimit?: bigint; + ethRpcUrl?: string; + forkBlockNumber?: bigint; + rpcEndpoints?: Record; + rpcCachePath?: string; + rpcStorageCaching?: { + chains: "All" | "None" | string[]; + endpoints: "All" | "Remote" | RegExp; + }; + promptTimeout?: number; + fuzz?: { + failurePersistDir?: string; + failurePersistFile?: string; + runs?: number; + maxTestRejects?: number; + seed?: string; + dictionaryWeight?: number; + includeStorage?: boolean; + includePushBytes?: boolean; + }; + invariant?: { + failurePersistDir?: string; + runs?: number; + depth?: number; + failOnRevert?: boolean; + callOverride?: boolean; + dictionaryWeight?: number; + includeStorage?: boolean; + includePushBytes?: boolean; + shrinkRunLimit?: number; + }; + } + + export interface SingleVersionSolcUserConfig extends SolcUserConfig { + test?: SolidityTestUserConfig; + } + export interface MultiVersionSolcUserConfig { compilers: SolcUserConfig[]; overrides?: Record; + test?: SolidityTestUserConfig; } - export interface SingleVersionSolidityUserConfig extends SolcUserConfig { + export interface SingleVersionSolidityUserConfig + extends SingleVersionSolcUserConfig { dependenciesToCompile?: string[]; remappings?: string[]; } @@ -30,7 +93,10 @@ declare module "../../../types/config.js" { } export interface BuildProfilesSolidityUserConfig { - profiles: Record; + profiles: Record< + string, + SingleVersionSolcUserConfig | MultiVersionSolcUserConfig + >; dependenciesToCompile?: string[]; remappings?: string[]; } @@ -44,9 +110,13 @@ declare module "../../../types/config.js" { settings: any; } + // eslint-disable-next-line @typescript-eslint/no-empty-interface -- This could be an extension point + export interface SolidityTestConfig extends SolidityTestUserConfig {} + export interface SolidityBuildProfileConfig { compilers: SolcConfig[]; overrides: Record; + test: SolidityTestConfig; } export interface SolidityConfig {