Add catalog-server and catalog-api packages (#1323)

package.json

@@ -36,7 +36,8 @@
   "wireit": {
     "test": {
       "dependencies": [
-        "./packages/custom-elements-manifest-tools:test"
+        "./packages/custom-elements-manifest-tools:test",
+        "./packages/catalog-server:test"
       ]
     },
     "serve:prod": {

packages/catalog-api/.gitignore

@@ -0,0 +1,7 @@
+/lib/
+/out/
+/gen/
+/node_modules/
+/test/
+/index.*
+/tsconfig.tsbuildinfo

packages/catalog-api/codegen.yml

@@ -0,0 +1,19 @@
+schema: ./src/lib/schema.graphql
+
+generates:
+  gen/lib/schema.ts:
+    plugins:
+      - typescript
+      - typed-document-node
+      - typescript-resolvers
+    config:
+      declarationKind: interface
+      dedupeOperationSuffix: true
+      immutableTypes: true
+      namingConvention: keep
+      noSchemaStitching: true
+      operationResultSuffix: Data
+      wrapFieldDefinitions: true
+      scalars:
+        Date: Date
+        JSON: String

packages/catalog-api/package.json

@@ -0,0 +1,72 @@
+{
+  "name": "@webcomponents/catalog-api",
+  "version": "0.0.0",
+  "private": true,
+  "description": "Schema and interfaces for the @webcomponents/catalog-server",
+  "author": "Google LLC",
+  "homepage": "",
+  "license": "Apache-2.0",
+  "type": "module",
+  "main": "index.js",
+  "scripts": {
+    "build": "wireit",
+    "build:ts": "wireit",
+    "build:graphql": "wireit"
+  },
+  "wireit": {
+    "build": {
+      "command": "cp -r out/src/* . ; cp -r out/gen/* .",
+      "dependencies": [
+        "build:graphql",
+        "build:ts"
+      ],
+      "files": [
+        "out/"
+      ],
+      "output": [
+        "lib",
+        "index.{js,js.map,d.ts,d.ts.map}"
+      ],
+      "clean": "if-file-deleted"
+    },
+    "build:ts": {
+      "command": "tsc --pretty",
+      "dependencies": [
+        "build:graphql"
+      ],
+      "files": [
+        "tsconfig.json",
+        "src/**/*.ts",
+        "gen/**/*.ts"
+      ],
+      "output": [
+        "out/",
+        "tsconfig.tsbuildinfo"
+      ],
+      "clean": "if-file-deleted"
+    },
+    "build:graphql": {
+      "command": "graphql-codegen -c codegen.yml",
+      "files": [
+        "codegen.yml",
+        "src/lib/schema.graphql"
+      ],
+      "output": [
+        "gen/lib/schema.ts"
+      ],
+      "clean": "if-file-deleted"
+    }
+  },
+  "devDependencies": {
+    "@graphql-codegen/cli": "^2.6.2",
+    "@graphql-codegen/typed-document-node": "^2.2.8",
+    "@graphql-codegen/typescript": "^2.4.8",
+    "@graphql-codegen/typescript-resolvers": "^2.6.1",
+    "uvu": "^0.5.6"
+  },
+  "dependencies": {
+    "@graphql-tools/schema": "^8.3.6",
+    "custom-elements-manifest": "^2.0.0",
+    "graphql": "^16.3.0"
+  }
+}

packages/catalog-api/src/index.ts

@@ -0,0 +1,2 @@
+//Placeholder file to make copy script work
+export {};

packages/catalog-api/src/lib/schema.graphql

@@ -0,0 +1,250 @@
+type Query {
+
+  """
+  Retrieves currently indexed top-level package metadata.
+  """
+  package(packageName: String!): PackageInfo
+
+  """
+  Queries custom elements in the entire catalog, from the latest version of each
+  package.
+
+  Eventually this will use some sort of ranking algorithm, otherwise the order
+  will just be defined by the database implementation.
+  """
+  elements(tag: String = "latest", limit: Int): [CustomElement!]!
+
+  """
+  Retrieves the custom element data for a single element.
+
+  NOT IMPLEMENTED
+  DO_NOT_LAUNCH
+  """
+  element(packageName: String!, elementName: String!, tag: String = "latest"): CustomElement
+}
+
+type Mutation {
+  """
+  NOT IMPLEMENTED
+  DO_NOT_LAUNCH
+  """
+  importPackage(packageName: String!): PackageInfo
+}
+
+"""
+Information about an npm package that's either version-independent
+or related to the latest version of the package.
+"""
+type PackageInfo {
+  name: ID!
+
+  status: PackageStatus!
+
+  lastUpdate: Date!
+
+  """
+  The package description derived from the latest version
+  """
+  description: String
+
+  """
+  The dist tag mappings for the package.
+  """
+  distTags: [DistTag!]! @map
+
+  """
+  Retreives a published versioned package. The default is "latest" for the
+  current version.
+  """
+  version(versionOrTag: String = "latest"): PackageVersion @collection
+}
+
+type DistTag {
+  tag: String!
+  version: String!
+}
+
+"""
+The status of a package.
+
+Concurrent requests for package metadata, which may trigger fetching from
+npm, are possible. We need to ensure there's only one writing request via
+transactions that lock on the status field.
+
+Because packages have versions that update over a time, a package may have
+a pending mutation task from initial fetching, or updating by querying and
+fetching new version metadata.
+
+Packages may not exist, in which case we record that they're not found in
+order to avoid flooding npm with invalid requests.
+
+Errors while fetching packages are divided into two categories:
+ - Invalid packages for which we were able to fetch metadata, but something
+in it was incorrect. Hopefully this never happens, but is included for
+completeness.
+ - Errors, such as network failures. In these cases we should be able to
+retry fetching package metadata.
+
+TODO: we need time and possibly version associtated with the last impmort
+attempt.
+"""
+enum PackageStatus {
+  """
+  The package is being downloaded and indexed for the first time.
+  """
+  INITIALIZING
+
+  """
+  Packages may be read from, but there is a pending update task and
+  new versions of the package are being downloaded and indexed.
+  """
+  UPDATING
+
+  """
+  The package is indexed and read to be read.
+  """
+  READY
+
+  """
+  The package was not found on npm.
+  """
+  NOT_FOUND
+
+  """
+  The package metadata was invalid. This is not a recoverable error unless a
+  new package version is published.
+  """
+  INVALID
+
+  """
+  An other recoverable error occured, such as network failure.
+  """
+  ERROR
+}
+
+"""
+Information about a specific version of a package
+"""
+type PackageVersion {
+  version: String!
+
+  """
+  The dist tags assigned to this version
+  """
+  distTags: [String!]!
+
+  """
+  The time this version was written to the registry
+  """
+  lastUpdate: Date!
+
+  status: VersionStatus!
+
+  description: String!
+
+  """
+  The npm package type: 'module' or 'commonjs'.
+  """
+  type: String!
+
+  author: String!
+
+  """
+  The package publication time reported by npm
+  """
+  time: Date!
+
+  homepage: String
+
+  customElements(tagName: String): [CustomElement!]
+
+  customElementsManifest: String
+
+  problems: [ValidationProblem]
+}
+
+enum VersionStatus {
+  """
+  The version is being downloaded and indexed.
+  """
+  INITIALIZING
+
+  """
+  The version is indexed and read to be read.
+  """
+  READY
+
+  """
+  A recoverable error.
+  """
+  ERROR
+}
+
+type ValidationProblem {
+  code: String!
+  severity: String!
+  message: String!
+  filePath: String!
+  start: Int!
+  length: Int!
+}
+
+"""
+Holds important per-element data extracted from the exports of the
+package's custom elements manifest.
+"""
+type CustomElement {
+
+  """
+  The package name containing this element
+  """
+  package: String!
+
+  """
+  The package version containing this element
+  """
+  version: String!
+
+  """
+  The dist tags assigned to this version
+  """
+  distTags: [String!]!
+
+  author: String!
+
+  tagName: String
+
+  className: String
+
+  """
+  Reference to the custom element export.
+
+  Reference strings have the format: `{package-name}/{module-path}#{tag-name}`
+  """
+  customElementExport: String
+
+  """
+  Reference to the JavaScript class export for the custom element class, if it
+  exists.
+
+  Reference strings have the format: `{package-name}/{module-path}#{export-name}`
+  """
+  jsExport: String
+
+  """
+  Reference to the class declaration of the custom element class. This is
+  different than the jsExport, as it's possible for a custom element to not
+  export its class. The declaration might not be directly importable.
+
+  Reference strings have the format: `{package-name}/{module-path}#{identifier}`
+  """
+  declaration: String
+}
+
+scalar Date
+
+scalar JSON
+
+directive @collection on FIELD_DEFINITION
+
+directive @map on FIELD_DEFINITION

packages/catalog-api/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "composite": true,
+    "rootDirs": ["./src", "./gen"],
+    "outDir": "./out"
+  },
+  "include": ["src/**/*.ts", "gen/**/*.ts"],
+  "exclude": []
+}

packages/catalog-server/.gitignore

@@ -0,0 +1,6 @@
+/lib/
+/node_modules/
+/test/*
+!/test/test-packages
+/tsconfig.tsbuildinfo
+*-debug.log

packages/catalog-server/README.md

@@ -0,0 +1,3 @@
+# `@webcomponents/catalog-server`
+
+This package contains the server that runs the component catalog API and database.