Merge pull request #24 from jmelis/graphql-schema-dynamic

Dynamic GraphQL Schema - intermediate format

assets/schema.yml

@@ -0,0 +1,188 @@
+- name: VaultSecret
+  version: '1'
+  fields:
+  - { name: path, type: string, isRequired: true }
+  - { name: field, type: string, isRequired: true }
+  - { name: format, type: string }
+
+- name: QuayOrg
+  version: '1'
+  fields:
+  - { name: schema, type: string, isRequired: true }
+  - { name: path, type: string, isRequired: true }
+  - { name: labels, type: json }
+  - { name: name, type: string, isRequired: true }
+  - { name: description, type: string, isRequired: true }
+  - { name: managedTeams, type: string, isList: true, isRequired: true }
+  - { name: automationToken, type: VaultSecret }
+
+- name: ClusterManagedRole
+  version: '1'
+  fields:
+  - { name: namespace, type: string, isRequired: true }
+  - { name: role, type: string, isRequired: true }
+
+- name: Cluster
+  version: '1'
+  fields:
+  - { name: schema, type: string, isRequired: true }
+  - { name: path, type: string, isRequired: true }
+  - { name: labels, type: json }
+  - { name: name, type: string, isRequired: true }
+  - { name: description, type: string, isRequired: true }
+  - { name: serverUrl, type: string, isRequired: true }
+  - { name: automationToken, type: VaultSecret }
+  - { name: managedRoles, type: ClusterManagedRole, isList: true }
+
+- name: AppServiceOwner
+  version: '1'
+  fields:
+  - { name: name, type: string, isRequired: true }
+  - { name: email, type: string, isRequired: true }
+
+- name: AppPerformanceParameters
+  version: '1'
+  fields:
+  - { name: SLO, type: float, isRequired: true }
+  - { name: SLA, type: float }
+  - { name: statusPage, type: string }
+
+- name: AppDependencies
+  version: '1'
+  fields:
+  - { name: name, type: string, isRequired: true }
+  - { name: statefulness, type: string, isRequired: true }
+  - { name: opsModel, type: string, isRequired: true }
+  - { name: statusPage, type: string }
+  - { name: SLA, type: float, isRequired: true }
+  - { name: dependencyFailureImpact, type: string, isRequired: true }
+
+- name: AppQuayReposItems
+  version: '1'
+  fields:
+  - { name: name, type: string, isRequired: true }
+  - { name: description, type: string, isRequired: true }
+  - { name: public, type: boolean, isRequired: true }
+
+- name: AppQuayRepos
+  version: '1'
+  fields:
+  - { name: org, type: QuayOrg, isRequired: true }
+  - { name: items, type: AppQuayReposItems, isRequired: true, isList: true }
+
+- name: App
+  version: '1'
+  fields:
+  - { name: schema, type: string, isRequired: true }
+  - { name: path, type: string, isRequired: true }
+  - { name: labels, type: json }
+  - { name: title, type: string, isRequired: true }
+  - { name: serviceOwner, type: AppServiceOwner, isRequired: true }
+  - { name: dependencies, type: AppDependencies, isList: true }
+  - { name: quayRepos, type: AppQuayRepos, isList: true }
+
+- name: Permission
+  version: '1'
+  isInterface: true
+  interfaceResolve:
+    strategy: fieldMap
+    field: service
+    fieldMap:
+      aws-analytics: PermissionAWSAnalytics
+      github-org: PermissionGithubOrg
+      github-org-team: PermissionGithubOrgTeam
+      openshift-rolebinding: PermissionOpenshiftRolebinding
+      quay-membership: PermissionQuayOrgTeam
+  fields:
+  - { name: service, type: string, isRequired: true }
+
+- name: PermissionAWSAnalytics
+  version: '1'
+  interface: Permission
+  fields:
+  - { name: service, type: string, isRequired: true }
+
+- name: PermissionGithubOrg
+  version: '1'
+  interface: Permission
+  fields:
+  - { name: service, type: string, isRequired: true }
+  - { name: org, type: string, isRequired: true }
+
+- name: PermissionGithubOrgTeam
+  version: '1'
+  interface: Permission
+  fields:
+  - { name: service, type: string, isRequired: true }
+  - { name: org, type: string, isRequired: true }
+  - { name: team, type: string, isRequired: true }
+
+- name: PermissionOpenshiftRolebinding
+  version: '1'
+  interface: Permission
+  fields:
+  - { name: service, type: string, isRequired: true }
+  - { name: cluster, type: string, isRequired: true }
+  - { name: namespace, type: string, isRequired: true }
+  - { name: role, type: string, isRequired: true }
+
+- name: PermissionQuayOrgTeam
+  version: '1'
+  interface: Permission
+  fields:
+  - { name: service, type: string, isRequired: true }
+  - { name: org, type: string, isRequired: true }
+  - { name: team, type: string, isRequired: true }
+
+- name: User
+  version: '1'
+  fields:
+  - { name: schema, type: string, isRequired: true }
+  - { name: path, type: string, isRequired: true }
+  - { name: labels, type: json }
+  - { name: name, type: string, isRequired: true }
+  - { name: redhat_username, type: string, isRequired: true }
+  - { name: github_username, type: string, isRequired: true }
+  - { name: quay_username, type: string }
+
+- name: Bot
+  version: '1'
+  fields:
+  - { name: schema, type: string, isRequired: true }
+  - { name: path, type: string, isRequired: true }
+  - { name: labels, type: json }
+  - { name: name, type: string, isRequired: true }
+  - { name: github_username, type: string }
+  - { name: quay_username, type: string }
+  - { name: owner, type: User }
+
+- name: Role
+  version: '1'
+  datafile: /access/role-1.yml
+  fields:
+  - { name: schema, type: string, isRequired: true }
+  - { name: path, type: string, isRequired: true }
+  - { name: labels, type: json }
+  - { name: name, type: string, isRequired: true }
+  - { name: permissions, type: Permission, isList: true, isInterface: true }
+  - name: users
+    type: User
+    isList: true
+    synthetic:
+      schema: /access/user-1.yml
+      subAttr: roles
+  - name: bots
+    type: Bot
+    isList: true
+    synthetic:
+      schema: /access/bot-1.yml
+      subAttr: roles
+
+- name: Query
+  fields:
+  - { name: user, type: User, isList: true, datafileSchema: /access/user-1.yml }
+  - { name: bot, type: Bot, isList: true, datafileSchema: /access/bot-1.yml }
+  - { name: role, type: Role, isList: true, datafileSchema: /access/role-1.yml }
+  - { name: cluster, type: Cluster, isList: true, datafileSchema: /openshift/cluster-1.yml }
+  - { name: quay_org, type: QuayOrg, isList: true, datafileSchema: /dependencies/quay-org-1.yml }
+  - { name: app, type: App, isList: true, datafileSchema: /app-sre/app-1.yml }

package.json

@@ -12,7 +12,7 @@
     "build": "rm -rf ./dist && NODE_ENV=production ts-node -O '{\"module\":\"commonjs\"}' ./node_modules/.bin/webpack --mode=production",
     "server": "node ./dist/main-bundle.js",
     "lint": "./node_modules/.bin/tslint --project .",
-    "test": "mocha -r ts-node/register --recursive"
+    "test": "mocha -r ts-node/register $(find test -type f -name '*.ts')"
   },
   "dependencies": {
     "apollo-server-express": "^2.2.0",
@@ -22,6 +22,7 @@
     "express": "^4.16.4",
     "fs": "0.0.1-security",
     "graphql": "^14.0.2",
+    "js-yaml": "^3.12.1",
     "jsonpointer": "^4.0.1",
     "node-forge": "^0.7.6",
     "path": "^0.12.7",
@@ -32,6 +33,7 @@
     "@types/express": "^4.16.0",
     "@types/graphql": "^14.0.5",
     "@types/mocha": "^5.2.5",
+    "@types/node-forge": "^0.7.11",
     "@types/webpack": "^4.4.22",
     "chai": "^4.2.0",
     "chai-http": "^4.2.1",

src/db.ts

@@ -0,0 +1,146 @@
+import { readFileSync } from 'fs';
+import { S3 } from 'aws-sdk';
+import { md as forgeMd } from 'node-forge';
+
+// cannot use `import` (old package with no associated types)
+const jsonpointer = require('jsonpointer');
+
+// interfaces
+interface IDatafile {
+  $schema: string;
+  path: string;
+}
+
+interface IDatafilesDict {
+  [key: string]: any;
+}
+
+// module variables
+let datafiles: IDatafilesDict = new Map<string, IDatafile>();
+let sha256sum: string = '';
+
+// utils
+const getRefPath = (ref: string): string => /^[^$]*/.exec(ref)[0];
+
+const getRefExpr = (ref: string): string => {
+  const m = /[$#].*/.exec(ref);
+  return m ? m[0] : '';
+};
+
+// filters
+export function getDatafilesBySchema(schema: string): IDatafile[] {
+  return Object.values(datafiles).filter((d: any) => d.$schema === schema);
+}
+
+export function resolveRef(itemRef: any) {
+  const path = getRefPath(itemRef.$ref);
+  const expr = getRefExpr(itemRef.$ref);
+
+  const datafile: any = datafiles[path];
+
+  if (typeof (datafile) === 'undefined') {
+    console.log(`Error retrieving datafile '${path}'.`);
+  }
+
+  const resolvedData = jsonpointer.get(datafile, expr);
+
+  if (typeof (resolvedData) === 'undefined') {
+    console.log(`Error resolving ref: datafile: '${JSON.stringify(datafile)}', expr: '${expr}'.`);
+  }
+
+  return resolvedData;
+}
+
+function validateDatafile(d: any) {
+  const datafilePath: any = d[0];
+  const datafileData: any = d[1];
+
+  if (typeof (datafilePath) !== 'string') {
+    throw new Error('Expecting string for datafilePath');
+  }
+
+  if (typeof (datafileData) !== 'object' ||
+    Object.keys(datafileData).length === 0 ||
+    !('$schema' in datafileData)) {
+    throw new Error('Invalid datafileData object');
+  }
+
+}
+// datafile Loading functions
+function loadUnpack(raw: string) {
+  const dbDatafilesNew: any = {};
+
+  const bundle = JSON.parse(raw);
+
+  const sha256hex = forgeMd.sha256.create().update(raw).digest().toHex();
+
+  Object.entries(bundle).forEach((d) => {
+    validateDatafile(d);
+
+    const datafilePath: string = d[0];
+    const datafileData: any = d[1];
+
+    datafileData.path = datafilePath;
+
+    dbDatafilesNew[datafilePath] = datafileData;
+  });
+
+  datafiles = dbDatafilesNew;
+  sha256sum = sha256hex;
+
+  console.log(`End datafile reload: ${new Date()}`);
+}
+
+function loadFromS3() {
+  const s3 = new S3({
+    accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+    region: process.env.AWS_REGION,
+  });
+
+  const s3params = {
+    Bucket: process.env.AWS_S3_BUCKET,
+    Key: process.env.AWS_S3_KEY,
+  };
+
+  s3.getObject(s3params, (err: any, data: any) => {
+    if (err) {
+      console.log(err, err.stack);
+    } else {
+      loadUnpack(data.Body.toString('utf-8'));
+    }
+  });
+}
+
+export function loadFromFile(path: string) {
+  let loadPath: string;
+
+  if (typeof (path) === 'undefined') {
+    loadPath = process.env.DATAFILES_FILE;
+  } else {
+    loadPath = path;
+  }
+
+  const raw = readFileSync(loadPath);
+  loadUnpack(String(raw));
+}
+
+export function load() {
+  console.log(`Start datafile reload: ${new Date()}`);
+
+  switch (process.env.LOAD_METHOD) {
+    case 'fs':
+      console.log('Loading from fs.');
+      loadFromFile(undefined);
+      break;
+    case 's3':
+      console.log('Loading from s3.');
+      loadFromS3();
+      break;
+    default:
+      console.log('Skip data loading.');
+  }
+}
+
+export const sha256 = (): string => sha256sum;
+export const datafilesLength = (): number => Object.keys(datafiles).length;