[TypeSpecRequirement] Add tests (Azure#29639)

.github/workflows/typespec-requirement-test.yaml

@@ -0,0 +1,23 @@
+name: TypeSpec Requirement - Test
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - package-lock.json
+      - package.json
+      - tsconfig.json
+      - .github/workflows/_reusable-eng-tools-test.yaml
+      - .github/workflows/typespec-requirement-test.yaml
+      - eng/scripts/**
+      - eng/tools/package.json
+      - eng/tools/tsconfig.json
+      - eng/tools/typespec-requirement/**
+
+jobs:
+  typespec-requirement:
+    uses: ./.github/workflows/_reusable-eng-tools-test.yaml
+    with:
+      package: typespec-requirement

eng/scripts/Logging-Functions.ps1

@@ -15,7 +15,7 @@ function LogWarning {
     Write-Host ("##vso[task.LogIssue type=warning;]$args" -replace "`n", "%0D%0A")
   }
   elseif (Test-SupportsGitHubLogging) {
-    Write-Host ("::warning::$args" -replace "`n", "%0D%0A")
+    Write-Warning ("::warning::$args" -replace "`n", "%0D%0A")
   }
   else {
     Write-Warning "$args"
@@ -28,7 +28,7 @@ function LogErrorForFile($file, $errorString)
     Write-Host ("##vso[task.logissue type=error;sourcepath=$file;linenumber=1;columnnumber=1;]$errorString" -replace "`n", "%0D%0A")
   }
   elseif (Test-SupportsGitHubLogging) {
-    Write-Host ("::error file=$file,line=1,col=1::$errorString" -replace "`n", "%0D%0A")
+    Write-Error ("::error file=$file,line=1,col=1::$errorString" -replace "`n", "%0D%0A")
   }
   else {
     Write-Error "[Error in file $file]$errorString"
@@ -39,7 +39,7 @@ function LogError {
     Write-Host ("##vso[task.LogIssue type=error;]$args" -replace "`n", "%0D%0A")
   }
   elseif (Test-SupportsGitHubLogging) {
-    Write-Host ("::error::$args" -replace "`n", "%0D%0A")
+    Write-Error ("::error::$args" -replace "`n", "%0D%0A")
   }
   else {
     Write-Error "$args"
@@ -51,7 +51,7 @@ function LogDebug {
     Write-Host "[debug]$args"
   }
   elseif (Test-SupportsGitHubLogging) {
-    Write-Host "::debug::$args"
+    Write-Debug "::debug::$args"
   }
   else {
     Write-Debug "$args"

eng/scripts/TypeSpec-Requirement.ps1

@@ -6,7 +6,11 @@ param (
   [string] $TargetCommitish = "HEAD",
   [Parameter(Position = 2)]
   [string] $SpecType = "data-plane|resource-manager",
-  [string] $CheckAllUnder
+  [string] $CheckAllUnder,
+  # Reserved for testing.  Call using:
+  # $ pwsh -Command '... -_ResponseCache @{"url"=200}'
+  [Parameter(DontShow)]
+  [hashtable] $_ResponseCache = @{}
 )
 Set-StrictMode -Version 3
 
@@ -59,7 +63,7 @@ $repoPath = Resolve-Path "$PSScriptRoot/../.."
 $pathsWithErrors = @()
 
 $filesToCheck = $CheckAllUnder ?
-  (Get-ChildItem -Path $CheckAllUnder -Recurse -File | Resolve-Path -Relative | ForEach-Object { $_ -replace '\\', '/' }) :
+  (Get-ChildItem -Path $CheckAllUnder -Recurse -File | Resolve-Path -Relative -RelativeBasePath $repoPath | ForEach-Object { $_ -replace '\\', '/' }) :
   (Get-ChangedSwaggerFiles (Get-ChangedFiles $BaseCommitish $TargetCommitish))
 
 $filesToCheck = $filesToCheck.Where({
@@ -72,7 +76,7 @@ if (!$filesToCheck) {
 }
 else {
   # Cache responses to GitHub web requests, for efficiency and to prevent rate limiting
-  $responseCache = @{}
+  $responseCache = $_ResponseCache
 
   # - Forward slashes on both Linux and Windows
   # - May be nested 4 or 5 levels deep, perhaps even deeper
@@ -107,16 +111,15 @@ else {
       if ($null -ne ${jsonContent}?["info"]?["x-typespec-generated"]) {
         LogInfo "  OpenAPI was generated from TypeSpec (contains '/info/x-typespec-generated')"
 
-        if ($file -match "specification/(?<rpFolder>[^/]+)/") {
-          $rpFolder = $Matches["rpFolder"];
-          $tspConfigs = @(Get-ChildItem -Path (Join-Path $repoPath "specification" $rpFolder) -Recurse -File
-            | Where-Object { $_.Name -eq "tspconfig.yaml" })
+        if ($file -match "^.*specification/[^/]+/") {
+          $rpFolder = $Matches[0];
+          $tspConfigs = @(Get-ChildItem -Path $rpFolder -Recurse -File | Where-Object { $_.Name -eq "tspconfig.yaml" })
 
           if ($tspConfigs) {
-            LogInfo "  Folder 'specification/$rpFolder' contains $($tspConfigs.Count) file(s) named 'tspconfig.yaml'"
+            LogInfo "  Folder '$rpFolder' contains $($tspConfigs.Count) file(s) named 'tspconfig.yaml'"
           }
           else {
-            LogError ("OpenAPI was generated from TypeSpec, but folder 'specification/$rpFolder' contains no files named 'tspconfig.yaml'." `
+            LogError ("OpenAPI was generated from TypeSpec, but folder '$rpFolder' contains no files named 'tspconfig.yaml'." `
                 + "  The TypeSpec used to generate OpenAPI must be added to this folder.")
             LogJobFailure
             exit 1
@@ -176,12 +179,12 @@ else {
     if ($responseStatus -eq 200) {
       LogInfo "  Branch 'main' contains path '$servicePath/stable', so spec already exists and is not required to use TypeSpec"
     }
-    elseif ($response.StatusCode -eq 404) {
+    elseif ($responseStatus -eq 404) {
       LogInfo "  Branch 'main' does not contain path '$servicePath/stable', so spec is new and must use TypeSpec"
       $pathsWithErrors += $file
     }
     else {
-      LogError "Unexpected response from ${logUrlToStableFolder}: ${response.StatusCode}"
+      LogError "Unexpected response from ${logUrlToStableFolder}: ${responseStatus}"
       LogJobFailure
       exit 1
     }

eng/tools/package.json

@@ -2,6 +2,7 @@
   "name": "azure-rest-api-specs-eng-tools",
   "devDependencies": {
     "@azure-tools/suppressions": "file:suppressions",
+    "@azure-tools/typespec-requirement": "file:typespec-requirement",
     "@azure-tools/typespec-validation": "file:typespec-validation"
   },
   "private": true

eng/tools/typespec-requirement/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "@azure-tools/typespec-requirement",
+  "private": true,
+  "type": "module",
+  "devDependencies": {
+    "@types/node": "^18.19.31",
+    "@vitest/coverage-v8": "^1.6.0",
+    "execa": "^9.3.0",
+    "typescript": "~5.4.5",
+    "vitest": "^1.6.0"
+  },
+  "scripts": {
+    "build": "tsc",
+    "postinstall": "npm run build",
+    "test": "vitest",
+    "test:ci": "vitest run --coverage --reporter=verbose"
+  },
+  "engines": {
+    "node": ">= 18.0.0"
+  }
+}

eng/tools/typespec-requirement/test/specification/empty/placeholder.txt

@@ -0,0 +1 @@
+Test

eng/tools/typespec-requirement/test/specification/hand-written/resource-manager/Microsoft.HandWritten/preview/2024-01-01-preview/hand-written.json

@@ -0,0 +1,2 @@
+{
+}

eng/tools/typespec-requirement/test/specification/no-tspconfig/resource-manager/Microsoft.NoTspConfig/preview/2024-01-01-preview/no-tspconfig.json

@@ -0,0 +1,9 @@
+{
+  "info": {
+    "x-typespec-generated": [
+      {
+        "emitter": "@azure-tools/typespec-autorest"
+      }
+    ]
+  }
+}

eng/tools/typespec-requirement/test/specification/parse-error/resource-manager/Microsoft.ParseError/preview/2024-01-01-preview/parse-error.json

@@ -0,0 +1 @@
+{

eng/tools/typespec-requirement/test/specification/suppression/resource-manager/Microsoft.Suppression/preview/2024-01-01-preview/suppression.json

@@ -0,0 +1,2 @@
+{
+}

eng/tools/typespec-requirement/test/specification/suppression/suppressions.yaml

@@ -0,0 +1,3 @@
+- tool: TypeSpecRequirement
+  path: ./resource-manager/Microsoft.Suppression/preview/2024-01-01-preview/*.json
+  reason: Test

eng/tools/typespec-requirement/test/specification/typespec-generated/Microsoft.TypeSpecGenerated.Management/tspconfig.yaml

@@ -0,0 +1 @@
+# Test

eng/tools/typespec-requirement/test/specification/typespec-generated/resource-manager/Microsoft.TypeSpecGenerated/preview/2024-01-01-preview/valid.json

@@ -0,0 +1,9 @@
+{
+  "info": {
+    "x-typespec-generated": [
+      {
+        "emitter": "@azure-tools/typespec-autorest"
+      }
+    ]
+  }
+}

eng/tools/typespec-requirement/test/typespec-requirement.test.ts

@@ -0,0 +1,83 @@
+import { execa } from 'execa';
+import { join } from 'path';
+import { test } from 'vitest';
+
+async function checkAllUnder(path: string, responseCache?: string) {
+  const repoRoot = join(__dirname, '..', '..', '..', '..');
+  const script = join('eng', 'scripts', 'TypeSpec-Requirement.ps1');
+
+  let command = `${script} -CheckAllUnder ${join(__dirname, path)}`;
+  if (responseCache) {
+    command += ` -_ResponseCache ${responseCache}`;
+  }
+
+  return await execa("pwsh", ["-Command", command], { cwd: repoRoot, reject: false });
+}
+
+test.concurrent("No files to check", async ({ expect }) => {
+  const { stdout, exitCode } = await checkAllUnder("specification/empty");
+
+  expect(stdout).toMatchInlineSnapshot(`"No OpenAPI files found to check"`);
+  expect(exitCode).toBe(0);
+});
+
+test.concurrent("Suppression", async ({ expect }) => {
+  const { stdout, exitCode } = await checkAllUnder("specification/suppression");
+
+  expect(stdout).toContain("Suppressed");
+  expect(exitCode).toBe(0);
+});
+
+test.concurrent("Parse error", async ({ expect }) => {
+  const { stdout, exitCode } = await checkAllUnder("specification/parse-error");
+
+  expect(stdout).toContain("cannot be parsed as JSON");
+  expect(exitCode).toBe(1);
+});
+
+test.concurrent("No tspconfig.yaml", async ({ expect }) => {
+  const { stderr, exitCode } = await checkAllUnder("specification/no-tspconfig");
+
+  expect(stderr).toContain("no files named 'tspconfig.yaml'");
+  expect(exitCode).toBe(1);
+});
+
+test.concurrent("Generated from TypeSpec", async ({ expect }) => {
+  const { stdout, exitCode } = await checkAllUnder("specification/typespec-generated");
+
+  expect(stdout).toContain("was generated from TypeSpec");
+  expect(exitCode).toBe(0);
+});
+
+test.concurrent("Hand-written, exists in main", async ({ expect }) => {
+  const { stdout, exitCode } = await checkAllUnder(
+    "specification/hand-written",
+    '@{"https://github.com/Azure/azure-rest-api-specs/tree/main/specification/hand-written/resource-manager/Microsoft.HandWritten/stable"=200}'
+  );
+
+  expect(stdout).toContain("was not generated from TypeSpec");
+  expect(stdout).toContain("'main' contains path");
+  expect(exitCode).toBe(0);
+});
+
+test.concurrent("Hand-written, does not exist in main", async ({ expect }) => {
+  const { stdout, exitCode } = await checkAllUnder(
+    "specification/hand-written",
+    '@{"https://github.com/Azure/azure-rest-api-specs/tree/main/specification/hand-written/resource-manager/Microsoft.HandWritten/stable"=404}'
+  );
+
+  expect(stdout).toContain("was not generated from TypeSpec");
+  expect(stdout).toContain("'main' does not contain path");
+  expect(exitCode).toBe(1);
+});
+
+test.concurrent("Hand-written, unexpected response checking main", async ({ expect }) => {
+  const { stdout, stderr, exitCode } = await checkAllUnder(
+    "specification/hand-written",
+    '@{"https://github.com/Azure/azure-rest-api-specs/tree/main/specification/hand-written/resource-manager/Microsoft.HandWritten/stable"=519}'
+  );
+
+  expect(stdout).toContain("was not generated from TypeSpec");
+  expect(stderr).toContain("Unexpected response");
+  expect(exitCode).toBe(1);
+});

eng/tools/typespec-requirement/tsconfig.json

@@ -0,0 +1,6 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+  }
+}

eng/tools/typespec-requirement/vite.config.ts

@@ -0,0 +1,8 @@
+import { defineConfig } from 'vite'
+
+export default defineConfig({
+  test: {
+    // Default timeout of 5 seconds is too low
+    testTimeout: 20000
+  }
+})