From 3e1eb526591565001a17de5545ec40fe664b1de0 Mon Sep 17 00:00:00 2001 From: Justin Fagnani Date: Wed, 12 Oct 2022 18:20:16 -0700 Subject: [PATCH 1/5] Add 'isLatest' field to reduce array-contains uses --- .../src/lib/firestore/firestore-repository.ts | 24 ++++++++++++++----- .../firestore/package-version-converter.ts | 1 + 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/packages/catalog-server/src/lib/firestore/firestore-repository.ts b/packages/catalog-server/src/lib/firestore/firestore-repository.ts index 612368a8..14eb9ba4 100644 --- a/packages/catalog-server/src/lib/firestore/firestore-repository.ts +++ b/packages/catalog-server/src/lib/firestore/firestore-repository.ts @@ -160,16 +160,22 @@ export class FirestoreRepository implements Repository { // collection first. const elements = await t.get(customElementsRef); + const isLatest = versionDistTags.includes('latest'); + // Update the PackageVersion doc - await t.update(versionRef, { + // We remove the converter to fix the types: + // https://github.com/googleapis/nodejs-firestore/issues/1745 + await t.update(versionRef.withConverter(null), { distTags: versionDistTags, + isLatest, }); // Update all elements await Promise.all( elements.docs.map(async (element) => { - await t.update(element.ref, { + await t.update(element.ref.withConverter(null), { distTags: versionDistTags, + isLatest, }); }) ); @@ -284,6 +290,7 @@ export class FirestoreRepository implements Repository { // Store custom elements data in subcollection const versionRef = this.getPackageVersionRef(packageName, version); const customElementsRef = versionRef.collection('customElements'); + const isLatest = distTags.includes('latest'); const batch = db.batch(); for (const c of customElements) { @@ -291,6 +298,7 @@ export class FirestoreRepository implements Repository { package: packageName, version, distTags, + isLatest, author, tagName: c.export.name, className: c.declaration.name, @@ -394,10 +402,14 @@ export class FirestoreRepository implements Repository { } // Now query for a version that's assigned this dist-tag - const result = await this.getPackageVersionCollectionRef(packageName) - .where('distTags', 'array-contains', versionOrTag) - .limit(1) - .get(); + let query: CollectionReference | Query = + this.getPackageVersionCollectionRef(packageName); + if (versionOrTag === 'latest') { + query = query.where('isLatest', '==', true); + } else { + query = query.where('distTags', 'array-contains', versionOrTag); + } + const result = await query.limit(1).get(); if (result.size !== 0) { return result.docs[0]!.data(); } diff --git a/packages/catalog-server/src/lib/firestore/package-version-converter.ts b/packages/catalog-server/src/lib/firestore/package-version-converter.ts index cb77df6a..8800d7ee 100644 --- a/packages/catalog-server/src/lib/firestore/package-version-converter.ts +++ b/packages/catalog-server/src/lib/firestore/package-version-converter.ts @@ -39,6 +39,7 @@ export const packageVersionConverter: FirestoreDataConverter = { customElementsManifest: packageVersion.customElementsManifest, description: packageVersion.description, distTags: packageVersion.distTags, + isLatest: packageVersion.distTags.includes('latest'), homepage: packageVersion.homepage, lastUpdate: packageVersion.lastUpdate, status: packageVersion.status, From 9a80564e6dc3ff5dac5de5988582ca624469c6a8 Mon Sep 17 00:00:00 2001 From: Justin Fagnani Date: Sat, 8 Oct 2022 11:46:44 -0700 Subject: [PATCH 2/5] Implement basic full-text-search --- package-lock.json | 121 ++++++++++++++++++ packages/catalog-api/src/lib/schema.graphql | 21 +-- packages/catalog-server/package.json | 2 + packages/catalog-server/src/lib/catalog.ts | 13 +- .../src/lib/firestore/firestore-repository.ts | 72 ++++++++++- packages/catalog-server/src/lib/graphql.ts | 27 +++- packages/catalog-server/src/lib/repository.ts | 18 ++- .../src/test/lib/catalog_test.ts | 50 +++++++- .../test-1/0.0.0/custom-elements.json | 28 +++- .../test-packages/test-1/0.0.0/package.json | 1 + .../test-1/1.0.0/custom-elements.json | 26 +++- .../test-packages/test-1/1.0.0/package.json | 1 + .../src/lib/npm.ts | 6 +- .../src/test/local-fs-package-files.ts | 2 +- 14 files changed, 341 insertions(+), 47 deletions(-) diff --git a/package-lock.json b/package-lock.json index 173d53ad..a9713fd7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3734,6 +3734,14 @@ "version": "3.0.5", "license": "MIT" }, + "node_modules/@types/natural": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@types/natural/-/natural-5.1.1.tgz", + "integrity": "sha512-BqppT8eHJvc0pN81XE7Rx5P8Osg1TSnOx3iwzLWImIO+6DwNUfpKR20tvg713O2eqHoxLcqrBaL10foo1mw/Xw==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/node": { "version": "18.8.3", "license": "MIT" @@ -4265,6 +4273,15 @@ "node": ">=0.4.0" } }, + "node_modules/afinn-165": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/afinn-165/-/afinn-165-1.0.4.tgz", + "integrity": "sha512-7+Wlx3BImrK0HiG6y3lU4xX7SpBPSSu8T9iguPMlaueRFxjbYwAQrp9lqZUuFikqKbd/en8lVREILvP2J80uJA==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/agent-base": { "version": "6.0.2", "license": "MIT", @@ -4372,6 +4389,17 @@ "node": ">= 8" } }, + "node_modules/apparatus": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/apparatus/-/apparatus-0.0.10.tgz", + "integrity": "sha512-KLy/ugo33KZA7nugtQ7O0E1c8kQ52N3IvD/XgIh4w/Nr28ypfkwDfA67F1ev4N1m5D+BOk1+b2dEJDfpj/VvZg==", + "dependencies": { + "sylvester": ">= 0.0.8" + }, + "engines": { + "node": ">=0.2.6" + } + }, "node_modules/arg": { "version": "4.1.3", "dev": true, @@ -17110,6 +17138,22 @@ "node": ">=0.10.0" } }, + "node_modules/natural": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/natural/-/natural-5.2.3.tgz", + "integrity": "sha512-fsGGpbU15YBc2oQCEsi0t7ZeF3VmKyxDhgWucQTPk4zaDFzeZtquRbZt4xlznN2ZUlH88215HcThMYaDHFM48Q==", + "dependencies": { + "afinn-165": "^1.0.2", + "apparatus": "^0.0.10", + "safe-stable-stringify": "^2.2.0", + "sylvester": "^0.0.12", + "underscore": "^1.9.1", + "wordnet-db": "^3.1.11" + }, + "engines": { + "node": ">=0.4.10" + } + }, "node_modules/natural-compare": { "version": "1.4.0", "license": "MIT" @@ -18810,6 +18854,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/safe-stable-stringify": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.0.tgz", + "integrity": "sha512-eehKHKpab6E741ud7ZIMcXhKcP6TSIezPkNZhy5U8xC6+VvrRdUA2tMgxGxaGl4cz7c2Ew5+mg5+wNB16KQqrA==", + "engines": { + "node": ">=10" + } + }, "node_modules/safer-buffer": { "version": "2.1.2", "license": "MIT" @@ -19732,6 +19784,14 @@ "dev": true, "license": "0BSD" }, + "node_modules/sylvester": { + "version": "0.0.12", + "resolved": "https://registry.npmjs.org/sylvester/-/sylvester-0.0.12.tgz", + "integrity": "sha512-SzRP5LQ6Ts2G5NyAa/jg16s8e3R7rfdFjizy1zeoecYWw+nGL+YA1xZvW/+iJmidBGSdLkuvdwTYEyJEb+EiUw==", + "engines": { + "node": ">=0.2.6" + } + }, "node_modules/symbol-observable": { "version": "4.0.0", "license": "MIT", @@ -20842,6 +20902,14 @@ "node": ">=0.10.0" } }, + "node_modules/wordnet-db": { + "version": "3.1.14", + "resolved": "https://registry.npmjs.org/wordnet-db/-/wordnet-db-3.1.14.tgz", + "integrity": "sha512-zVyFsvE+mq9MCmwXUWHIcpfbrHHClZWZiVOzKSxNJruIcFn2RbY55zkhiAMMxM8zCVSmtNiViq8FsAZSFpMYag==", + "engines": { + "node": ">=0.6.0" + } + }, "node_modules/wordwrap": { "version": "1.0.0", "license": "MIT" @@ -21049,6 +21117,7 @@ "@koa/router": "^12.0.0", "@types/koa__router": "^12.0.0", "@types/koa-bodyparser": "^4.3.8", + "@types/natural": "^5.1.1", "@webcomponents/catalog-api": "0.0.0", "@webcomponents/custom-elements-manifest-tools": "0.0.0", "custom-elements-manifest": "^2.0.0", @@ -21056,6 +21125,7 @@ "firebase-admin": "^11.0.0", "graphql-helix": "^1.13.0", "koa-bodyparser": "^4.3.0", + "natural": "^5.2.3", "node-fetch": "^3.2.3", "npm-registry-fetch": "^13.1.0", "semver": "^7.3.7" @@ -23742,6 +23812,14 @@ "@types/minimatch": { "version": "3.0.5" }, + "@types/natural": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@types/natural/-/natural-5.1.1.tgz", + "integrity": "sha512-BqppT8eHJvc0pN81XE7Rx5P8Osg1TSnOx3iwzLWImIO+6DwNUfpKR20tvg713O2eqHoxLcqrBaL10foo1mw/Xw==", + "requires": { + "@types/node": "*" + } + }, "@types/node": { "version": "18.8.3" }, @@ -23983,6 +24061,7 @@ "@koa/router": "^12.0.0", "@types/koa__router": "^12.0.0", "@types/koa-bodyparser": "^4.3.8", + "@types/natural": "*", "@types/node": "^18.0.6", "@types/npm-registry-fetch": "^8.0.0", "@types/source-map-support": "^0.5.3", @@ -23994,6 +24073,7 @@ "firebase-tools": "^11.3.0", "graphql-helix": "^1.13.0", "koa-bodyparser": "^4.3.0", + "natural": "^5.2.3", "node-fetch": "^3.2.3", "npm-registry-fetch": "^13.1.0", "semver": "^7.3.7" @@ -24142,6 +24222,11 @@ "version": "8.2.0", "dev": true }, + "afinn-165": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/afinn-165/-/afinn-165-1.0.4.tgz", + "integrity": "sha512-7+Wlx3BImrK0HiG6y3lU4xX7SpBPSSu8T9iguPMlaueRFxjbYwAQrp9lqZUuFikqKbd/en8lVREILvP2J80uJA==" + }, "agent-base": { "version": "6.0.2", "requires": { @@ -24206,6 +24291,14 @@ "picomatch": "^2.0.4" } }, + "apparatus": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/apparatus/-/apparatus-0.0.10.tgz", + "integrity": "sha512-KLy/ugo33KZA7nugtQ7O0E1c8kQ52N3IvD/XgIh4w/Nr28ypfkwDfA67F1ev4N1m5D+BOk1+b2dEJDfpj/VvZg==", + "requires": { + "sylvester": ">= 0.0.8" + } + }, "arg": { "version": "4.1.3", "dev": true @@ -32692,6 +32785,19 @@ } } }, + "natural": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/natural/-/natural-5.2.3.tgz", + "integrity": "sha512-fsGGpbU15YBc2oQCEsi0t7ZeF3VmKyxDhgWucQTPk4zaDFzeZtquRbZt4xlznN2ZUlH88215HcThMYaDHFM48Q==", + "requires": { + "afinn-165": "^1.0.2", + "apparatus": "^0.0.10", + "safe-stable-stringify": "^2.2.0", + "sylvester": "^0.0.12", + "underscore": "^1.9.1", + "wordnet-db": "^3.1.11" + } + }, "natural-compare": { "version": "1.4.0" }, @@ -33754,6 +33860,11 @@ "is-regex": "^1.1.4" } }, + "safe-stable-stringify": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.0.tgz", + "integrity": "sha512-eehKHKpab6E741ud7ZIMcXhKcP6TSIezPkNZhy5U8xC6+VvrRdUA2tMgxGxaGl4cz7c2Ew5+mg5+wNB16KQqrA==" + }, "safer-buffer": { "version": "2.1.2" }, @@ -34389,6 +34500,11 @@ } } }, + "sylvester": { + "version": "0.0.12", + "resolved": "https://registry.npmjs.org/sylvester/-/sylvester-0.0.12.tgz", + "integrity": "sha512-SzRP5LQ6Ts2G5NyAa/jg16s8e3R7rfdFjizy1zeoecYWw+nGL+YA1xZvW/+iJmidBGSdLkuvdwTYEyJEb+EiUw==" + }, "symbol-observable": { "version": "4.0.0" }, @@ -35082,6 +35198,11 @@ "word-wrap": { "version": "1.2.3" }, + "wordnet-db": { + "version": "3.1.14", + "resolved": "https://registry.npmjs.org/wordnet-db/-/wordnet-db-3.1.14.tgz", + "integrity": "sha512-zVyFsvE+mq9MCmwXUWHIcpfbrHHClZWZiVOzKSxNJruIcFn2RbY55zkhiAMMxM8zCVSmtNiViq8FsAZSFpMYag==" + }, "wordwrap": { "version": "1.0.0" }, diff --git a/packages/catalog-api/src/lib/schema.graphql b/packages/catalog-api/src/lib/schema.graphql index 1ce3ab61..d9c2af85 100644 --- a/packages/catalog-api/src/lib/schema.graphql +++ b/packages/catalog-api/src/lib/schema.graphql @@ -8,27 +8,8 @@ type Query { """ Queries custom elements in the entire catalog, from the latest version of each package. - - Eventually this will have more query parameters, and use some sort of ranking - algorithm, otherwise the order will just be defined by the database - implementation. - - NOT IMPLEMENTED - DO_NOT_LAUNCH - """ - elements(distTag: String = "latest", limit: Int): [CustomElement!]! - - """ - Retrieves the custom element data for a single element. - - NOT IMPLEMENTED - DO_NOT_LAUNCH """ - element( - packageName: String! - elementName: String! - tag: String = "latest" - ): CustomElement + elements(query: String, limit: Int): [CustomElement!]! } type Mutation { diff --git a/packages/catalog-server/package.json b/packages/catalog-server/package.json index 5bc10e9b..f31e85fe 100644 --- a/packages/catalog-server/package.json +++ b/packages/catalog-server/package.json @@ -84,6 +84,7 @@ "@koa/router": "^12.0.0", "@types/koa__router": "^12.0.0", "@types/koa-bodyparser": "^4.3.8", + "@types/natural": "^5.1.1", "@webcomponents/catalog-api": "0.0.0", "@webcomponents/custom-elements-manifest-tools": "0.0.0", "custom-elements-manifest": "^2.0.0", @@ -91,6 +92,7 @@ "firebase-admin": "^11.0.0", "graphql-helix": "^1.13.0", "koa-bodyparser": "^4.3.0", + "natural": "^5.2.3", "node-fetch": "^3.2.3", "npm-registry-fetch": "^13.1.0", "semver": "^7.3.7" diff --git a/packages/catalog-server/src/lib/catalog.ts b/packages/catalog-server/src/lib/catalog.ts index 5c8c2def..386bf7ee 100644 --- a/packages/catalog-server/src/lib/catalog.ts +++ b/packages/catalog-server/src/lib/catalog.ts @@ -272,8 +272,7 @@ export class Catalog { console.log('Writing custom elements...'); await this.#repository.writeCustomElements( - packageName, - version, + packageVersionMetadata, customElements, versionDistTags, author @@ -307,6 +306,9 @@ export class Catalog { return this.#repository.getPackageVersion(packageName, version); } + /** + * Gets the custom elements for a package + */ async getCustomElements( packageName: string, version: string, @@ -314,4 +316,11 @@ export class Catalog { ): Promise> { return this.#repository.getCustomElements(packageName, version, tagName); } + + async queryElements({query, limit}: {query?: string; limit?: number}) { + // TODO (justinfagnani): The catalog should parse out GitHub-style search + // operators (like "author:yogibear") and pass structured + text queries + // to the repository + return this.#repository.queryElements({query, limit}); + } } diff --git a/packages/catalog-server/src/lib/firestore/firestore-repository.ts b/packages/catalog-server/src/lib/firestore/firestore-repository.ts index 14eb9ba4..90c0fc18 100644 --- a/packages/catalog-server/src/lib/firestore/firestore-repository.ts +++ b/packages/catalog-server/src/lib/firestore/firestore-repository.ts @@ -4,13 +4,19 @@ * SPDX-License-Identifier: Apache-2.0 */ -import {FieldValue, Query, CollectionReference} from '@google-cloud/firestore'; +import { + FieldValue, + Query, + CollectionReference, + CollectionGroup, +} from '@google-cloud/firestore'; import {Firestore} from '@google-cloud/firestore'; import firebase from 'firebase-admin'; import {CustomElementInfo} from '@webcomponents/custom-elements-manifest-tools'; import {referenceString} from '@webcomponents/custom-elements-manifest-tools/lib/reference-string.js'; import clean from 'semver/functions/clean.js'; import semverValidRange from 'semver/ranges/valid.js'; +import natural from 'natural'; import {distTagListToMap, getDistTagsForVersion} from '../npm.js'; @@ -24,7 +30,10 @@ import { ReadablePackageVersion, ReadablePackageInfo, } from '@webcomponents/catalog-api/lib/schema.js'; -import {Package} from '@webcomponents/custom-elements-manifest-tools/lib/npm.js'; +import { + Package, + Version, +} from '@webcomponents/custom-elements-manifest-tools/lib/npm.js'; import {Repository} from '../repository.js'; import { packageInfoConverter, @@ -281,26 +290,52 @@ export class FirestoreRepository implements Repository { } async writeCustomElements( - packageName: string, - version: string, + packageVersionMetadata: Version, customElements: CustomElementInfo[], distTags: string[], author: string ): Promise { // Store custom elements data in subcollection + const {name: packageName, version, description} = packageVersionMetadata; const versionRef = this.getPackageVersionRef(packageName, version); const customElementsRef = versionRef.collection('customElements'); const isLatest = distTags.includes('latest'); const batch = db.batch(); + // Stem the package description + const packageDescriptionStems = natural.PorterStemmer.tokenizeAndStem( + description ?? '' + ); + for (const c of customElements) { + const tagName = c.export.name; + // Grab longer tag name parts for searching. We want "button" from + // md-button, etc. + const tagNameParts = tagName.split('-').filter((s) => s.length > 3); + const descriptionStems = natural.PorterStemmer.tokenizeAndStem( + c.declaration.description ?? '' + ); + const summaryStems = natural.PorterStemmer.tokenizeAndStem( + c.declaration.summary ?? '' + ); + + // Combine and deduplicate terms + const searchTerms = [ + ...new Set([ + ...packageDescriptionStems, + ...descriptionStems, + ...summaryStems, + ...tagNameParts, + ]), + ]; + batch.create(customElementsRef.doc(), { package: packageName, version, distTags, isLatest, author, - tagName: c.export.name, + tagName, className: c.declaration.name, customElementExport: referenceString( packageName, @@ -308,6 +343,7 @@ export class FirestoreRepository implements Repository { c.export.name ), declaration: referenceString(packageName, c.module, c.declaration.name), + searchTerms, }); } @@ -453,6 +489,32 @@ export class FirestoreRepository implements Repository { return customElementsResults.docs.map((d) => d.data()); } + async queryElements({ + query, + limit, + }: { + query?: string; + limit?: number; + }): Promise> { + let dbQuery: Query | CollectionGroup = db + .collectionGroup('customElements') + .withConverter(customElementConverter) + .where('isLatest', '==', true) + .limit(limit ?? 25); + + if (query !== undefined) { + // Split query + const queryTerms = natural.PorterStemmer.tokenizeAndStem(query); + if (queryTerms.length > 10) { + queryTerms.length = 10; + } + dbQuery = dbQuery.where('searchTerms', 'array-contains-any', queryTerms); + } + + const result = await (await dbQuery.get()).docs.map((d) => d.data()); + return result; + } + getPackageRef(packageName: string) { return db .collection('packages' + (this.namespace ? `-${this.namespace}` : '')) diff --git a/packages/catalog-server/src/lib/graphql.ts b/packages/catalog-server/src/lib/graphql.ts index 2d53a2a2..5d0784d9 100644 --- a/packages/catalog-server/src/lib/graphql.ts +++ b/packages/catalog-server/src/lib/graphql.ts @@ -7,7 +7,13 @@ import {readFile} from 'fs/promises'; import {createRequire} from 'module'; import {makeExecutableSchema} from '@graphql-tools/schema'; -import {isReadablePackage, isReadablePackageVersion, PackageInfo, Resolvers} from '@webcomponents/catalog-api/lib/schema.js'; +import { + CustomElement, + isReadablePackage, + isReadablePackageVersion, + PackageInfo, + Resolvers, +} from '@webcomponents/catalog-api/lib/schema.js'; import {Catalog} from './catalog.js'; const require = createRequire(import.meta.url); @@ -25,14 +31,17 @@ export const makeExecutableCatalogSchema = async (catalog: Catalog) => { // an explanation of the role of resolvers in performing GraphQL queries. const resolvers: Resolvers = { Query: { - async package(_parent, {packageName}: {packageName: string}): Promise { + async package( + _parent, + {packageName}: {packageName: string} + ): Promise { console.log('query package', packageName); const packageInfo = await catalog.getPackageInfo(packageName); if (packageInfo === undefined) { console.log(`package ${packageName} not found in db, importing`); let result; try { - result = await catalog.importPackage(packageName); + result = await catalog.importPackage(packageName); } catch (e) { console.error(e); throw e; @@ -53,6 +62,13 @@ export const makeExecutableCatalogSchema = async (catalog: Catalog) => { return null; } }, + async elements(_parent, {query, limit}): Promise> { + console.log('query elements', {query, limit}); + return catalog.queryElements({ + query: query ?? undefined, + limit: limit ?? 25, + }); + }, }, PackageInfo: { __resolveType(obj) { @@ -75,7 +91,10 @@ export const makeExecutableCatalogSchema = async (catalog: Catalog) => { distTags?.find((distTag) => distTag.tag === versionOrTag)?.version ?? versionOrTag; - const packageVersion = await catalog.getPackageVersion(packageInfo.name, version); + const packageVersion = await catalog.getPackageVersion( + packageInfo.name, + version + ); if (packageVersion === undefined) { throw new Error(`tag ${packageInfo.name}@${versionOrTag} not found`); } diff --git a/packages/catalog-server/src/lib/repository.ts b/packages/catalog-server/src/lib/repository.ts index d0671be9..eb2e0178 100644 --- a/packages/catalog-server/src/lib/repository.ts +++ b/packages/catalog-server/src/lib/repository.ts @@ -13,7 +13,10 @@ import type { ValidationProblem, } from '@webcomponents/catalog-api/lib/schema'; import type {CustomElementInfo} from '@webcomponents/custom-elements-manifest-tools'; -import type {Package} from '@webcomponents/custom-elements-manifest-tools/lib/npm.js'; +import type { + Package, + Version, +} from '@webcomponents/custom-elements-manifest-tools/lib/npm.js'; /** * Interface for a database that stores package and custom element data. @@ -91,8 +94,7 @@ export interface Repository { ): Promise; writeCustomElements( - packageName: string, - version: string, + packageVersionMetadata: Version, customElements: CustomElementInfo[], distTags: string[], author: string @@ -105,7 +107,15 @@ export interface Repository { packageName: string, version: string, tagName?: string - ): Promise; + ): Promise>; + + queryElements({ + query, + limit, + }: { + query?: string; + limit?: number; + }): Promise>; writeProblems( packageName: string, diff --git a/packages/catalog-server/src/test/lib/catalog_test.ts b/packages/catalog-server/src/test/lib/catalog_test.ts index 25bae124..08ca6843 100644 --- a/packages/catalog-server/src/test/lib/catalog_test.ts +++ b/packages/catalog-server/src/test/lib/catalog_test.ts @@ -25,6 +25,13 @@ const testPackage1Path = fileURLToPath( new URL('../test-packages/test-1/', import.meta.url) ); +// A set of import, fetch, search tests that use the same data +const TEST_SEQUENCE_ONE = 'test-data-1'; + +// Other tests than can run independently +const TEST_SEQUENCE_TWO = 'test-data-2'; + + test('Imports a package with no problems', async () => { const packageName = 'test-1'; const version = '0.0.0'; @@ -36,7 +43,7 @@ test('Imports a package with no problems', async () => { latest: '0.0.0', }, }); - const repository = new FirestoreRepository('catalog-test-1'); + const repository = new FirestoreRepository(TEST_SEQUENCE_ONE); const catalog = new Catalog({files, repository}); const importResult = await catalog.importPackage(packageName); @@ -73,7 +80,7 @@ test('A second import does nothing', async () => { }, }); // This must use the same namespace as in the previous test - const repository = new FirestoreRepository('catalog-test-1'); + const repository = new FirestoreRepository(TEST_SEQUENCE_ONE); const catalog = new Catalog({files, repository}); const importResult = await catalog.importPackage(packageName); @@ -84,6 +91,37 @@ test('A second import does nothing', async () => { assert.equal(importResult.problems, undefined); }); +test('Full text search', async () => { + const packageName = 'test-1'; + const files = new LocalFsPackageFiles({ + path: testPackage1Path, + packageName, + publishedVersions: ['0.0.0'], + distTags: { + latest: '0.0.0', + }, + }); + // This must use the same namespace as in the previous test + const repository = new FirestoreRepository(TEST_SEQUENCE_ONE); + const catalog = new Catalog({files, repository}); + + // Use a term in the package description - it should match all elements + const resultOne = await catalog.queryElements({ + query: 'cool', + limit: 10 + }); + assert.ok(resultOne); + assert.equal(resultOne.length, 2); + + // Use a term in an element description + const resultTwo = await catalog.queryElements({ + query: 'incredible', + limit: 10 + }); + assert.ok(resultTwo); + assert.equal(resultTwo.length, 1); +}); + test('Gets package version data from imported package', async () => { const packageName = 'test-1'; const version = '0.0.0'; @@ -95,7 +133,7 @@ test('Gets package version data from imported package', async () => { latest: '0.0.0', }, }); - const repository = new FirestoreRepository('catalog-test-2'); + const repository = new FirestoreRepository(TEST_SEQUENCE_TWO); const catalog = new Catalog({files, repository}); const importResult = await catalog.importPackageVersion(packageName, version); const {problems} = importResult; @@ -123,7 +161,7 @@ test('Gets package version data from imported package', async () => { version, undefined ); - assert.equal(customElements?.length, 1); + assert.equal(customElements?.length, 2); // TODO (justinfagnani): add assertion when we have catalog.getPackageVersionProblems // const problems = await catalog.getPackageVersionProblems(packageName, version); @@ -142,11 +180,11 @@ test('Updates a package', async () => { }); // This must use the same namespace as in the first (import) test - const repository = new FirestoreRepository('catalog-test-1'); + const repository = new FirestoreRepository(TEST_SEQUENCE_ONE); const catalog = new Catalog({files, repository}); const importResult = await catalog.importPackage( packageName, - Temporal.Duration.from({minutes: 0}) + Temporal.Duration.from({nanoseconds: 1}) ); assert.ok(importResult.packageInfo); diff --git a/packages/catalog-server/test/test-packages/test-1/0.0.0/custom-elements.json b/packages/catalog-server/test/test-packages/test-1/0.0.0/custom-elements.json index ab806988..9a218534 100644 --- a/packages/catalog-server/test/test-packages/test-1/0.0.0/custom-elements.json +++ b/packages/catalog-server/test/test-packages/test-1/0.0.0/custom-elements.json @@ -1,5 +1,6 @@ { "schemaVersion": "1.0.0", + "description": "A set of cool elements", "modules": [ { "kind": "javascript-module", @@ -18,14 +19,39 @@ "declaration": { "name": "FooElement" } + }, + { + "kind": "js", + "name": "BarElement", + "declaration": { + "name": "BarElement" + } + }, + { + "kind": "custom-element-definition", + "name": "bar-element", + "declaration": { + "name": "BarElement" + } } ], "declarations": [ { "kind": "class", "customElement": true, - "tagName": "my-element", + "tagName": "foo-element", "name": "FooElement", + "description": "A incredible element", + "superclass": { + "name": "HTMLElement" + } + }, + { + "kind": "class", + "customElement": true, + "tagName": "bar-element", + "name": "BarElement", + "description": "An amazing element", "superclass": { "name": "HTMLElement" } diff --git a/packages/catalog-server/test/test-packages/test-1/0.0.0/package.json b/packages/catalog-server/test/test-packages/test-1/0.0.0/package.json index 1f9346ae..bd5be689 100644 --- a/packages/catalog-server/test/test-packages/test-1/0.0.0/package.json +++ b/packages/catalog-server/test/test-packages/test-1/0.0.0/package.json @@ -1,5 +1,6 @@ { "name": "test-1", "version": "0.0.0", + "description": "A set of cool elements", "customElements": "custom-elements.json" } diff --git a/packages/catalog-server/test/test-packages/test-1/1.0.0/custom-elements.json b/packages/catalog-server/test/test-packages/test-1/1.0.0/custom-elements.json index ab806988..dfb5b90f 100644 --- a/packages/catalog-server/test/test-packages/test-1/1.0.0/custom-elements.json +++ b/packages/catalog-server/test/test-packages/test-1/1.0.0/custom-elements.json @@ -1,5 +1,6 @@ { "schemaVersion": "1.0.0", + "description": "A set of cool elements", "modules": [ { "kind": "javascript-module", @@ -18,17 +19,40 @@ "declaration": { "name": "FooElement" } + }, + { + "kind": "js", + "name": "BarElement", + "declaration": { + "name": "BarElement" + } + }, + { + "kind": "custom-element-definition", + "name": "bar-element", + "declaration": { + "name": "BarElement" + } } ], "declarations": [ { "kind": "class", "customElement": true, - "tagName": "my-element", + "tagName": "bar-element", "name": "FooElement", "superclass": { "name": "HTMLElement" } + }, + { + "kind": "class", + "customElement": true, + "tagName": "bar-element", + "name": "BarElement", + "superclass": { + "name": "HTMLElement" + } } ] } diff --git a/packages/catalog-server/test/test-packages/test-1/1.0.0/package.json b/packages/catalog-server/test/test-packages/test-1/1.0.0/package.json index 41b55c8c..45f91eb6 100644 --- a/packages/catalog-server/test/test-packages/test-1/1.0.0/package.json +++ b/packages/catalog-server/test/test-packages/test-1/1.0.0/package.json @@ -1,5 +1,6 @@ { "name": "test-1", "version": "1.0.0", + "description": "A set of cool elements", "customElements": "custom-elements.json" } diff --git a/packages/custom-elements-manifest-tools/src/lib/npm.ts b/packages/custom-elements-manifest-tools/src/lib/npm.ts index 5418159c..df6541c8 100644 --- a/packages/custom-elements-manifest-tools/src/lib/npm.ts +++ b/packages/custom-elements-manifest-tools/src/lib/npm.ts @@ -38,16 +38,16 @@ export interface Package { export interface Version { name: string; version: string; - description: string; + description?: string; dist: Dist; type?: 'module' | 'commonjs'; - main: string; + main?: string; module?: string; author?: {name: string}; homepage?: string; - repository: { + repository?: { type: 'git' | 'svn'; url: string; }; diff --git a/packages/custom-elements-manifest-tools/src/test/local-fs-package-files.ts b/packages/custom-elements-manifest-tools/src/test/local-fs-package-files.ts index a7fa901d..68b71e43 100644 --- a/packages/custom-elements-manifest-tools/src/test/local-fs-package-files.ts +++ b/packages/custom-elements-manifest-tools/src/test/local-fs-package-files.ts @@ -51,7 +51,7 @@ export class LocalFsPackageFiles implements PackageFiles { modified: now, // ...Object.fromEntries(this.publishedVersions.map((v) => [v, now])), }; - let description!: string; + let description: string | undefined; let foundLatest = false; await Promise.all( this.publishedVersions.map(async (v) => { From 0897748006bf6a500834fbbcde0ba0868c045ddd Mon Sep 17 00:00:00 2001 From: Justin Fagnani Date: Fri, 14 Oct 2022 10:22:08 -0700 Subject: [PATCH 3/5] Address feedback --- .../catalog-server/src/lib/firestore/firestore-repository.ts | 4 ++-- packages/catalog-server/src/lib/graphql.ts | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/catalog-server/src/lib/firestore/firestore-repository.ts b/packages/catalog-server/src/lib/firestore/firestore-repository.ts index 90c0fc18..07279019 100644 --- a/packages/catalog-server/src/lib/firestore/firestore-repository.ts +++ b/packages/catalog-server/src/lib/firestore/firestore-repository.ts @@ -290,13 +290,12 @@ export class FirestoreRepository implements Repository { } async writeCustomElements( - packageVersionMetadata: Version, + {name: packageName, version, description}: Version, customElements: CustomElementInfo[], distTags: string[], author: string ): Promise { // Store custom elements data in subcollection - const {name: packageName, version, description} = packageVersionMetadata; const versionRef = this.getPackageVersionRef(packageName, version); const customElementsRef = versionRef.collection('customElements'); const isLatest = distTags.includes('latest'); @@ -326,6 +325,7 @@ export class FirestoreRepository implements Repository { ...descriptionStems, ...summaryStems, ...tagNameParts, + tagName ]), ]; diff --git a/packages/catalog-server/src/lib/graphql.ts b/packages/catalog-server/src/lib/graphql.ts index 5d0784d9..1711a62e 100644 --- a/packages/catalog-server/src/lib/graphql.ts +++ b/packages/catalog-server/src/lib/graphql.ts @@ -63,7 +63,6 @@ export const makeExecutableCatalogSchema = async (catalog: Catalog) => { } }, async elements(_parent, {query, limit}): Promise> { - console.log('query elements', {query, limit}); return catalog.queryElements({ query: query ?? undefined, limit: limit ?? 25, From 7589b7d919d6a19a725ac45d7b1a24b2e0f77928 Mon Sep 17 00:00:00 2001 From: Justin Fagnani Date: Fri, 14 Oct 2022 14:20:37 -0700 Subject: [PATCH 4/5] Add more tests --- .../src/lib/firestore/firestore-repository.ts | 6 ++- .../src/test/lib/catalog_test.ts | 40 +++++++++++++------ 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/packages/catalog-server/src/lib/firestore/firestore-repository.ts b/packages/catalog-server/src/lib/firestore/firestore-repository.ts index 07279019..7e06a2bf 100644 --- a/packages/catalog-server/src/lib/firestore/firestore-repository.ts +++ b/packages/catalog-server/src/lib/firestore/firestore-repository.ts @@ -325,7 +325,10 @@ export class FirestoreRepository implements Repository { ...descriptionStems, ...summaryStems, ...tagNameParts, - tagName + tagName, + // TODO (justinfagnani): tokenizing the package name is temporary + // until we don't tokenize the *entire* query + ...natural.PorterStemmer.tokenizeAndStem(packageName), ]), ]; @@ -508,6 +511,7 @@ export class FirestoreRepository implements Repository { if (queryTerms.length > 10) { queryTerms.length = 10; } + console.log('queryTerms', queryTerms); dbQuery = dbQuery.where('searchTerms', 'array-contains-any', queryTerms); } diff --git a/packages/catalog-server/src/test/lib/catalog_test.ts b/packages/catalog-server/src/test/lib/catalog_test.ts index 08ca6843..f62a6d8a 100644 --- a/packages/catalog-server/src/test/lib/catalog_test.ts +++ b/packages/catalog-server/src/test/lib/catalog_test.ts @@ -31,7 +31,6 @@ const TEST_SEQUENCE_ONE = 'test-data-1'; // Other tests than can run independently const TEST_SEQUENCE_TWO = 'test-data-2'; - test('Imports a package with no problems', async () => { const packageName = 'test-1'; const version = '0.0.0'; @@ -106,20 +105,35 @@ test('Full text search', async () => { const catalog = new Catalog({files, repository}); // Use a term in the package description - it should match all elements - const resultOne = await catalog.queryElements({ - query: 'cool', - limit: 10 - }); - assert.ok(resultOne); - assert.equal(resultOne.length, 2); + let result = await catalog.queryElements({query: 'cool'}); + assert.equal(result.length, 2); // Use a term in an element description - const resultTwo = await catalog.queryElements({ - query: 'incredible', - limit: 10 - }); - assert.ok(resultTwo); - assert.equal(resultTwo.length, 1); + result = await catalog.queryElements({query: 'incredible'}); + assert.equal(result.length, 1); + + // Use a term not found + result = await catalog.queryElements({query: 'jandgslwijd'}); + assert.equal(result.length, 0); + + // Use an element name + result = await catalog.queryElements({query: '"foo-element"'}); + // TODO (justinfagnani): this isn't what we want. We really just want + // The element to be returned, but the tokenizer we're + // using is splitting "foo-element" into ["foo", "element"] and "element" + // is matching against bar-element's search terms. + // If we keep our own search index, we'll want to use or write a tokenizer + // that preserves quoted sections for exact matches: + // http://naturalnode.github.io/natural/Tokenizers.html + assert.equal(result.length, 2); + + // Use part of an element name + result = await catalog.queryElements({query: 'element'}); + assert.equal(result.length, 2); + + // Use a package name + result = await catalog.queryElements({query: 'test-1'}); + assert.equal(result.length, 2); }); test('Gets package version data from imported package', async () => { From 7b801457de52c6ee3d7fd565f0d5b618f3e7973c Mon Sep 17 00:00:00 2001 From: Justin Fagnani Date: Fri, 14 Oct 2022 17:23:06 -0700 Subject: [PATCH 5/5] Remove console log --- .../catalog-server/src/lib/firestore/firestore-repository.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/catalog-server/src/lib/firestore/firestore-repository.ts b/packages/catalog-server/src/lib/firestore/firestore-repository.ts index 7e06a2bf..b8592948 100644 --- a/packages/catalog-server/src/lib/firestore/firestore-repository.ts +++ b/packages/catalog-server/src/lib/firestore/firestore-repository.ts @@ -511,7 +511,6 @@ export class FirestoreRepository implements Repository { if (queryTerms.length > 10) { queryTerms.length = 10; } - console.log('queryTerms', queryTerms); dbQuery = dbQuery.where('searchTerms', 'array-contains-any', queryTerms); }