Add e2e test for the CLI (microsoft#2878)

fix microsoft#489

e2e test were also not running at all and the `emitter-ts` template was
failing due to importing `vitest` instead of `node:test`

.chronus/changes/tests-cli-2024-1-9-1-13-28.md

@@ -0,0 +1,8 @@
+---
+# Change versionKind to one of: internal, fix, dependencies, feature, deprecation, breaking
+changeKind: internal
+packages:
+  - "@typespec/compiler"
+---
+
+Add e2e test for the CLI

e2e/package.json

@@ -4,6 +4,6 @@
   "version": "0.0.1",
   "private": true,
   "scripts": {
-    "e2e": "node ./e2e-tests.js"
+    "test:e2e": "node ./e2e-tests.js"
   }
 }

package.json

@@ -23,7 +23,7 @@
     "regen-docs": "pnpm -r --parallel --aggregate-output --reporter=append-only run regen-docs",
     "regen-samples": "pnpm -r run regen-samples",
     "test-official": "pnpm -r --aggregate-output --reporter=append-only test-official",
-    "test:e2e": "pnpm -r run e2e",
+    "test:e2e": "pnpm -r run test:e2e",
     "test": "pnpm -r --aggregate-output --reporter=append-only run test",
     "update-latest-docs": "pnpm -r run update-latest-docs",
     "watch": "tsc --build ./tsconfig.ws.json --watch"

packages/compiler/package.json

@@ -69,7 +69,7 @@
     "test:ui": "vitest --ui",
     "test:watch": "vitest -w",
     "test-official": "vitest run --coverage --reporter=junit --reporter=default --no-file-parallelism",
-    "e2e:disabled": "vitest run --config ./vitest.config.e2e.ts",
+    "test:e2e": "vitest run --config ./vitest.config.e2e.ts",
     "gen-manifest": "node scripts/generate-manifest.js",
     "regen-nonascii": "node scripts/regen-nonascii.js",
     "fuzz": "node dist/test/manual/fuzz.js run",

packages/compiler/src/core/cli/actions/compile/args.ts

@@ -56,6 +56,10 @@ export async function getCompilerOptions(
       }),
     })
   );
+  if (args["no-emit"]) {
+    resolvedOptions.noEmit = true;
+  }
+
   return diagnostics.wrap(
     omitUndefined({
       ...resolvedOptions,

packages/compiler/templates/__snapshots__/emitter-ts/test/hello.test.ts

@@ -1,5 +1,5 @@
 import { strictEqual } from "node:assert";
-import { describe, it } from "vitest";
+import { describe, it } from "node:test";
 import { emit } from "./test-host.js";
 
 describe("hello", () => {

packages/compiler/templates/emitter-ts/test/hello.test.ts

@@ -1,5 +1,5 @@
 import { strictEqual } from "node:assert";
-import { describe, it } from "vitest";
+import { describe, it } from "node:test";
 import { emit } from "./test-host.js";
 
 describe("hello", () => {

packages/compiler/test/e2e/cli/cli.e2e.ts

@@ -0,0 +1,193 @@
+import { ChildProcess, SpawnOptions, spawn } from "child_process";
+import { access, readFile, rm } from "fs/promises";
+import { beforeEach, describe, expect, it } from "vitest";
+import { resolvePath } from "../../../src/index.js";
+import { findTestPackageRoot } from "../../../src/testing/test-utils.js";
+
+const pkgRoot = await findTestPackageRoot(import.meta.url);
+const scenarioRoot = resolvePath(pkgRoot, "test/e2e/cli/scenarios");
+
+function getScenarioDir(name: string) {
+  return resolvePath(scenarioRoot, name);
+}
+interface ExecCliOptions {
+  cwd?: string;
+}
+
+async function execCli(args: string[], { cwd }: ExecCliOptions) {
+  const node = process.platform === "win32" ? "node.exe" : "node";
+  return execAsync(node, [resolvePath(pkgRoot, "entrypoints/cli.js"), ...args], { cwd });
+}
+async function execCliSuccess(args: string[], { cwd }: ExecCliOptions) {
+  const result = await execCli(args, { cwd });
+  if (result.exitCode !== 0) {
+    throw new Error(`Failed to execute cli: ${result.stdio}`);
+  }
+
+  return result;
+}
+async function execCliFail(args: string[], { cwd }: ExecCliOptions) {
+  const result = await execCli(args, { cwd });
+  if (result.exitCode === 0) {
+    throw new Error(`Cli succeeded but expected failure: ${result.stdio}`);
+  }
+  return result;
+}
+
+export interface ExecResult {
+  exitCode: number;
+  stdout: string;
+  stderr: string;
+  stdio: string;
+  proc: ChildProcess;
+}
+export async function execAsync(
+  command: string,
+  args: string[],
+  options: SpawnOptions
+): Promise<ExecResult> {
+  const child = spawn(command, args, options);
+
+  return new Promise((resolve, reject) => {
+    child.on("error", (error) => {
+      reject(error);
+    });
+    const stdio: Buffer[] = [];
+    const stdout: Buffer[] = [];
+    const stderr: Buffer[] = [];
+    child.stdout?.on("data", (data) => {
+      stdout.push(data);
+      stdio.push(data);
+    });
+    child.stderr?.on("data", (data) => {
+      stderr.push(data);
+      stdio.push(data);
+    });
+
+    child.on("exit", (exitCode) => {
+      resolve({
+        exitCode: exitCode ?? -1,
+        stdout: Buffer.concat(stdout).toString(),
+        stderr: Buffer.concat(stderr).toString(),
+        stdio: Buffer.concat(stdio).toString(),
+        proc: child,
+      });
+    });
+  });
+}
+
+async function cleanOutputDir(scenarioName: string) {
+  const dir = resolvePath(getScenarioDir(scenarioName), "tsp-output");
+  await rm(dir, { recursive: true, force: true });
+}
+describe("cli", () => {
+  it("shows help", async () => {
+    const { stdout } = await execCliSuccess(["--help"], {
+      cwd: getScenarioDir("simple"),
+    });
+    expect(stdout).toContain("tsp <command>");
+    expect(stdout).toContain("tsp compile <path>       Compile TypeSpec source.");
+    expect(stdout).toContain("tsp format <include...>  Format given list of TypeSpec files.");
+  });
+
+  describe("compiling spec with warning", () => {
+    it("logs warning and succeed", async () => {
+      const { stdout } = await execCliSuccess(["compile", ".", "--pretty", "false"], {
+        cwd: getScenarioDir("warn"),
+      });
+
+      // eslint-disable-next-line no-console
+      console.log("Stdout", stdout);
+      expect(stdout).toContain("main.tsp:5:8 - warning deprecated: Deprecated: Deprecated");
+      expect(stdout).toContain("Found 1 warning.");
+    });
+
+    it("logs warning as error(and fail) when using --warn-as-error", async () => {
+      const { stdout } = await execCliFail(
+        ["compile", ".", "--warn-as-error", "--pretty", "false"],
+        {
+          cwd: getScenarioDir("warn"),
+        }
+      );
+      // eslint-disable-next-line no-console
+      console.log("Stdout", stdout);
+      expect(stdout).toContain("main.tsp:5:8 - error deprecated: Deprecated: Deprecated");
+      expect(stdout).toContain("Found 1 error.");
+    });
+  });
+
+  describe("compiling with an emitter", () => {
+    beforeEach(async () => {
+      await cleanOutputDir("with-emitter");
+    });
+
+    it("emits output", async () => {
+      const { stdout } = await execCliSuccess(["compile", ".", "--emit", "./emitter.js"], {
+        cwd: getScenarioDir("with-emitter"),
+      });
+      expect(stdout).toContain("Compilation completed successfully.");
+      const file = await readFile(
+        resolvePath(getScenarioDir("with-emitter"), "tsp-output/out.txt")
+      );
+      expect(file.toString()).toEqual("Hello, world!");
+    });
+
+    it("doesn't emit output when --noEmit is set", async () => {
+      const { stdout } = await execCliSuccess(
+        ["compile", ".", "--emit", "./emitter.js", "--no-emit"],
+        {
+          cwd: getScenarioDir("with-emitter"),
+        }
+      );
+      expect(stdout).toContain("Compilation completed successfully.");
+      await expect(() =>
+        access(resolvePath(getScenarioDir("with-emitter"), "tsp-output/out.txt"))
+      ).rejects.toEqual(expect.any(Error));
+    });
+  });
+
+  describe("compiling with no emitter", () => {
+    it("logs warnings", async () => {
+      const { stdout } = await execCliSuccess(["compile", "."], {
+        cwd: getScenarioDir("simple"),
+      });
+      expect(stdout).toContain(
+        "No emitter was configured, no output was generated. Use `--emit <emitterName>` to pick emitter or specify it in the TypeSpec config."
+      );
+    });
+
+    it("doesn't log warning when --noEmit is set", async () => {
+      const { stdout } = await execCliSuccess(["compile", ".", "--no-emit"], {
+        cwd: getScenarioDir("simple"),
+      });
+      expect(stdout).not.toContain(
+        "No emitter was configured, no output was generated. Use `--emit <emitterName>` to pick emitter or specify it in the TypeSpec config."
+      );
+    });
+  });
+
+  it("can provide emitter options", async () => {
+    const { stdout } = await execCliSuccess(
+      ["compile", ".", "--emit", "./emitter.js", "--option", "test-emitter.text=foo"],
+      {
+        cwd: getScenarioDir("with-emitter"),
+      }
+    );
+    expect(stdout).toContain("Compilation completed successfully.");
+    const file = await readFile(resolvePath(getScenarioDir("with-emitter"), "tsp-output/out.txt"));
+    expect(file.toString()).toEqual("foo");
+  });
+
+  it("set config parmaeter with --arg", async () => {
+    await cleanOutputDir("with-config");
+
+    const { stdout } = await execCliSuccess(
+      ["compile", ".", "--emit", "./emitter.js", "--arg", "custom-dir=custom-dir-name"],
+      {
+        cwd: getScenarioDir("with-config"),
+      }
+    );
+    expect(stdout).toContain("Compilation completed successfully.");
+    await access(resolvePath(getScenarioDir("with-config"), "tsp-output/custom-dir-name/out.txt"));
+  });
+});

packages/compiler/test/e2e/cli/scenarios/simple/main.tsp

@@ -0,0 +1 @@
+model Foo {}

packages/compiler/test/e2e/cli/scenarios/warn/main.tsp

@@ -0,0 +1,6 @@
+#deprecated "Deprecated"
+model Foo {}
+
+model Bar {
+  foo: Foo;
+}

packages/compiler/test/e2e/cli/scenarios/with-config/emitter.js

@@ -0,0 +1,6 @@
+import { mkdir, writeFile } from "fs/promises";
+
+export async function $onEmit(context) {
+  await mkdir(context.program.compilerOptions.outputDir, { recursive: true });
+  await writeFile(context.program.compilerOptions.outputDir + "/out.txt", "");
+}

packages/compiler/test/e2e/cli/scenarios/with-config/main.tsp

@@ -0,0 +1 @@
+model Foo {}

packages/compiler/test/e2e/cli/scenarios/with-config/tspconfig.yaml

@@ -0,0 +1,8 @@
+parameters:
+  custom-dir:
+    default: default-value
+
+emit:
+  - ./emitter.js
+
+output-dir: "{project-root}/tsp-output/{custom-dir}"

packages/compiler/test/e2e/cli/scenarios/with-emitter/emitter.js

@@ -0,0 +1,15 @@
+import { mkdir, writeFile } from "fs/promises";
+
+export async function $onEmit(context) {
+  if (!context.program.compilerOptions.noEmit) {
+    await mkdir(context.program.compilerOptions.outputDir, { recursive: true });
+    await writeFile(
+      context.program.compilerOptions.outputDir + "/out.txt",
+      context.options["text"] ?? "Hello, world!"
+    );
+  }
+}
+
+export const $lib = {
+  name: "test-emitter",
+};

packages/compiler/test/e2e/cli/scenarios/with-emitter/main.tsp

@@ -0,0 +1 @@
+model Foo {}

packages/compiler/test/e2e/init-templates.e2e.ts

@@ -87,7 +87,8 @@ describe("Init templates e2e tests", () => {
     return {
       directory: targetFolder,
       checkCommand: async (command: string, args: string[] = [], options: SpawnOptions = {}) => {
-        const result = await execAsync(command, args, {
+        const xplatCmd = process.platform === "win32" ? `${command}.cmd` : command;
+        const result = await execAsync(xplatCmd, args, {
           ...options,
           cwd: targetFolder,
         });

packages/playground-website/package.json

@@ -38,8 +38,8 @@
     "preview": "npm run build && vite preview",
     "start": "vite",
     "watch": "vite",
-    "e2e": "cross-env PW_EXPERIMENTAL_TS_ESM=1 playwright test -c e2e ",
-    "e2e:headed": "cross-env PW_EXPERIMENTAL_TS_ESM=1 playwright test -c e2e --headed",
+    "test:e2e": "cross-env PW_EXPERIMENTAL_TS_ESM=1 playwright test -c e2e ",
+    "test:e2e:headed": "cross-env PW_EXPERIMENTAL_TS_ESM=1 playwright test -c e2e --headed",
     "lint": "eslint . --max-warnings=0",
     "lint:fix": "eslint . --fix"
   },