INIT_BUNDLES from s3 (#146)

renamed INIT_DISK_BUNDLES to INIT_BUNDLES and allow pre-populating a qontract-server with bundles from fs or s3 `INIT_BUNDLES` is a CSV of bundle references of the following formats * fs:\\path\to\bundle * s3:\\bundle-key The s3 flavour relies on the `AWS_*` env variables to specify the bucket and configure authentication. the specified `bundle-key` is used as `AWS_S3_KEY`. The bundles listed in `INIT_BUNDLE` can come from different sources and are added to the `qontract-server` in the order they are specified. This means that the bundle listed last is also the one returned by the `/sha256` endpoint. Signed-off-by: Gerd Oberlechner <[email protected]>
app-sre · Jun 23, 2022 · 6df67b6 · 6df67b6
1 parent 6ca0601
commit 6df67b6
Show file tree

Hide file tree

Showing 5 changed files with 56 additions and 25 deletions.
diff --git a/README.md b/README.md
@@ -155,6 +155,18 @@ yarn build
 # Start the server
 make run
 ```
+
+### Preload bundles
+
+Especially during PR checks, a `qontract-server` with multiple bundles preloaded simplifies test infra setup. For this the env variable `INIT_BUNDLE` to specify a comma separated list of bundle references of the following form
+
+- fs://path/to/bundle
+- s3://bundle-key
+
+The s3 flavour relies on the `AWS_*` env variables to specify the bucket and configure authentication. the specified `bundle-key` is used as `AWS_S3_KEY`.
+
+The bundles listed in `INIT_BUNDLE` are added to the `qontract-server` in the order they are specified. This means that the bundle listed last is also the one returned by the `/sha256` endpoint.
+
 ## Style
 
 All code should follow the [airbnb style guide], which is enforced by this

diff --git a/src/db.ts b/src/db.ts
@@ -125,6 +125,14 @@ const hashDatafile = (contents: string) => {
 const bundleFromS3 = async(accessKeyId: string, secretAccessKey: string, region: string,
                            bucket: string, key: string) => {
   const s3 = new aws.S3({ accessKeyId, secretAccessKey, region });
+  try {
+    await s3.waitFor(
+      'objectExists',
+      { Bucket: bucket, Key: key, $waiter: { delay: 5, maxAttempts: 20 } },
+    ).promise();
+  } catch (error) {
+    throw new Error(`key ${key} not found in s3 bucket ${bucket}`);
+  }
   const getObject = util.promisify(s3.getObject.bind(s3));
   const response = await getObject({ Bucket: bucket, Key: key });
   const contents = response.Body.toString('utf-8');
@@ -140,8 +148,22 @@ export const bundleFromDisk = async(path: string) => {
 };
 
 export const getInitialBundles = () => {
-  if (process.env.INIT_DISK_BUNDLES) {
-    return process.env.INIT_DISK_BUNDLES.split(',').map((path: any) => bundleFromDisk(path));
+  if (process.env.INIT_BUNDLES) {
+    return process.env.INIT_BUNDLES.split(',').map((bundleUrl: any) => {
+      const urlParts = bundleUrl.split('://');
+      switch (urlParts[0]) {
+        case 'fs':
+          return bundleFromDisk(urlParts[1]);
+        case 's3':
+          return bundleFromS3(process.env.AWS_ACCESS_KEY_ID,
+                              process.env.AWS_SECRET_ACCESS_KEY,
+                              process.env.AWS_REGION,
+                              process.env.AWS_S3_BUCKET,
+                              urlParts[1]);
+        default:
+          throw new Error(`incorrect bundle ${bundleUrl} specified`);
+      }
+    });
   }
   return [bundleFromEnvironment()];
 };

diff --git a/test/diff/diff.test.ts b/test/diff/diff.test.ts
@@ -13,14 +13,14 @@ import { logger } from '../../src/logger';
 
 const should = chai.should();
 
-const diskBundles = 'test/diff/old.data.json,test/diff/new.data.json';
+const diskBundles = 'fs://test/diff/old.data.json,fs://test/diff/new.data.json';
 const oldSha = 'bf56095bf2ada36a6b2deca9cb9b6616d536b5c9ce230f0905296165d221a66b';
 const newSha = '302071115aa5dda8559f6e582fa7b6db7e0b64b5a9a6a9e3e9c22e2f86567f4b';
 
 describe('diff', async() => {
   let srv: http.Server;
   before(async() => {
-    process.env.INIT_DISK_BUNDLES = diskBundles;
+    process.env.INIT_BUNDLES = diskBundles;
     const app = await server.appFromBundle(db.getInitialBundles());
     srv = app.listen({ port: 4000 });
   });
@@ -52,6 +52,6 @@ describe('diff', async() => {
   });
 
   after(() => {
-    delete process.env.INIT_DISK_BUNDLES;
+    delete process.env.INIT_BUNDLES;
   });
 });
diff --git a/test/schemas/schemas.test.ts b/test/schemas/schemas.test.ts
@@ -32,23 +32,3 @@ describe('clusters', async() => {
     return resp.body.data.clusters[0].name.should.equal('example cluster');
   });
 });
-
-describe('clusters', async() => {
-  let srv: http.Server;
-  before(async() => {
-    process.env.INIT_DISK_BUNDLES = 'test/schemas/schemas.data.json';
-    const app = await server.appFromBundle(db.getInitialBundles());
-    srv = app.listen({ port: 4000 });
-  });
-
-  it('check if init disk bundle loads the bundle', async() => {
-    const resp = await chai.request(srv)
-                        .get('/sha256');
-    resp.should.have.status(200);
-    return resp.text.should.eql('242acb1998e9d37c26186ba9be0262fb34e3ef388b503390d143164f7658c24e');
-  });
-
-  after(() => {
-    delete process.env.INIT_DISK_BUNDLES;
-  });
-});
diff --git a/test/server.test.ts b/test/server.test.ts
@@ -251,3 +251,20 @@ describe('server', async () => {
     expect(perm.org).to.equal('org-A');
   });
 });
+
+describe('bundle loading', async() => {
+
+  it('check if init disk bundle is loaded', async() => {
+    process.env.INIT_BUNDLES = 'fs://test/schemas/schemas.data.json';
+    const app = await server.appFromBundle(db.getInitialBundles());
+    const srv = app.listen({ port: 4000 });
+    const resp = await chai.request(srv)
+                        .get('/sha256');
+    resp.should.have.status(200);
+    return resp.text.should.eql('242acb1998e9d37c26186ba9be0262fb34e3ef388b503390d143164f7658c24e');
+  });
+
+  after(() => {
+    delete process.env.INIT_BUNDLES;
+  });
+});