Skip to content

Commit

Permalink
chore(infra): Refactor workflows into separate ones from projen gener…
Browse files Browse the repository at this point in the history
…ated [experiment]
  • Loading branch information
Justinon committed Sep 11, 2023
1 parent 35c657d commit 3085e9c
Show file tree
Hide file tree
Showing 8 changed files with 310 additions and 479 deletions.
4 changes: 2 additions & 2 deletions src/datadog.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { stringify } from 'cson-parser';
import { JobPermission } from 'projen/lib/github/workflows-model';
import { NodeProject } from 'projen/lib/javascript';
import { getVersionStep } from './workflows/utils/getVersionStep';
import { getVersionSteps } from './workflows/utils/getVersion';

export module datadog {
export interface ReleaseEventTags {
Expand Down Expand Up @@ -84,7 +84,7 @@ export module datadog {
CI: 'true',
},
steps: [
...getVersionStep(project),
...getVersionSteps(project),
{
name: 'Send Datadog event',
// https://github.com/Glennmen/datadog-event-action/releases/tag/1.1.0
Expand Down
52 changes: 14 additions & 38 deletions src/workflows/ecs-service-build.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,30 +44,6 @@ export module ecsServiceBuildPublishWorkflow {
type: 'boolean',
default: false,
},
'buildcache-aws-access-key-id': {
type: 'string',
required: true,
},
'buildcache-aws-secret-access-key': {
type: 'string',
required: true,
},
'buildcache-aws-region': {
type: 'string',
required: true,
},
'lacework-account-name': {
type: 'string',
required: true,
},
'lacework-access-token': {
type: 'string',
required: true,
},
'all-package-read-token': {
type: 'string',
required: true,
},
version: {
description: 'The version number that was created for the deploy.',
required: true,
Expand Down Expand Up @@ -180,7 +156,7 @@ export module ecsServiceBuildPublishWorkflow {
getBuildcacheLoginStep(),
{
name: 'Configure NPM',
run: 'echo "//npm.pkg.github.com/:_authToken=${{ inputs.all-package-read-token }}" >> ~/.npmrc',
run: 'echo "//npm.pkg.github.com/:_authToken=${{ secrets.ALL_PACKAGE_READ_TOKEN }}" >> ~/.npmrc',
},
{
uses: 'pnpm/action-setup@d882d12c64e032187b2edb46d3a0d003b7a43598', // 2.4.0
Expand All @@ -204,8 +180,8 @@ export module ecsServiceBuildPublishWorkflow {
{
name: 'Publish image as draft',
// nrh: CLK-254084: temp fix to stop image creaation proliferation
if: 'startsWith(github.ref, "refs/heads/")',
runNow: [
if: "startsWith(github.ref, 'refs/heads/')",
run: [
...dockerTags.map(
(tag) =>
`docker tag \${{ env.DOCKER_IMAGE_ID }} "\${{ steps.buildcache.outputs.docker-repo-uri }}:${tag}"`,
Expand Down Expand Up @@ -239,13 +215,13 @@ export module ecsServiceBuildPublishWorkflow {
uses: 'time-loop/github-actions/dist/docker-vulnerability-scan@53f03c9bc36389e6b4e671bd8a3cc6286ef37044', // TODO: Do not use prerelease from branch
id: 'vulnerability-scan',
with: {
'buildcache-aws-access-key-id': '${{ inputs.buildcache-aws-access-key-id }}',
'buildcache-aws-secret-access-key': '${{ inputs.buildcache-aws-secret-access-key }}',
'buildcache-aws-region': '${{ inputs.buildcache-aws-region }}',
'buildcache-aws-access-key-id': '${{ secrets.ECR_BUILD_CACHE_AWS_ACCESS_KEY_ID }}',
'buildcache-aws-secret-access-key': '${{ secrets.ECR_BUILD_CACHE_AWS_SECRET_ACCESS_KEY }}',
'buildcache-aws-region': '${{ secrets.ECR_BUILD_CACHE_AWS_REGION }}',
'buildcache-ecr-repo-name': 'monorepo-buildcache',
'buildcache-pull-tag': '${{ env.DOCKER_TAG_DRAFT_COMMIT }}-amd64', // FIXME:
'lacework-account-name': '${{ inputs.lacework-account-name }}',
'lacework-access-token': '${{ inputs.lacework-account-token }}',
'lacework-account-name': '${{ secrets.LW_ACCOUNT_NAME }}',
'lacework-access-token': '${{ secrets.LW_ACCESS_TOKEN }}',
},
},
],
Expand All @@ -258,7 +234,7 @@ export module ecsServiceBuildPublishWorkflow {
const job = {
'docker-validate': {
// nrh: CLK-254084: temp fix to stop image creaation proliferation
if: 'startsWith(github.ref, "refs/heads/")',
if: "startsWith(github.ref, 'refs/heads/')",
needs: buildJobNames,
name: 'Validate',
strategy: {
Expand All @@ -284,7 +260,7 @@ export module ecsServiceBuildPublishWorkflow {
},
{
name: 'healthcheck verification setup (nestjs)',
if: 'matrix.step == "healthcheck-verification" && inputs.is-nest-app && !inputs.skip-health-check',
if: "matrix.step == 'healthcheck-verification' && inputs.is-nest-app && !inputs.skip-health-check",
uses: DEFAULT_RETRY_ACTION,
with: {
max_attempts: 3,
Expand All @@ -310,7 +286,7 @@ export module ecsServiceBuildPublishWorkflow {
},
{
name: 'healthcheck verification setup (generic)',
if: 'matrix.step == "healthcheck-verification" && !inputs.is-nest-app && !inputs.skip-health-check',
if: "matrix.step == 'healthcheck-verification' && !inputs.is-nest-app && !inputs.skip-health-check",
run: [
'docker run \\',
'-d \\',
Expand All @@ -320,7 +296,7 @@ export module ecsServiceBuildPublishWorkflow {
},
{
name: 'healthcheck verification',
if: 'matrix.step == "healthcheck-verification" && !inputs.skip-health-check',
if: "matrix.step == 'healthcheck-verification' && !inputs.skip-health-check",
run: [
'DOCKER_CONTAINER_ID="$(cat /tmp/CONTAINER_ID)"',
// 40x5 = 200s
Expand Down Expand Up @@ -365,7 +341,7 @@ done;`,
const job = {
'docker-publish-buildcache': {
// nrh: CLK-254084: temp fix to stop image creaation proliferation
if: 'startsWith(github.ref, "refs/heads/")',
if: "startsWith(github.ref, 'refs/heads/')",
needs: [...buildJobNames, 'docker-validate'],
name: 'Publish to Buildcache',
'runs-on': 'ubuntu-latest',
Expand Down Expand Up @@ -435,7 +411,7 @@ done;`,
options: BuildPublishOptionsConfig,
override?: any,
): void {
new YamlFile(project, WORKFLOW_LOCATION, {
new YamlFile(project, ecsServiceBuildPublishWorkflow.WORKFLOW_LOCATION, {
obj: { ...createBuildPublishWorkflow({ nodeVersion: project.workflowNodeVersion, ...options }), ...override },
});
}
Expand Down
55 changes: 32 additions & 23 deletions src/workflows/ecs-service-cd.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import { JobPermission } from 'projen/lib/github/workflows-model';
import { YamlFile } from 'projen';
import { ecsServiceDeployWorkflow } from './ecs-service-deploy';
import { getVersionStep } from './utils/getVersionStep';
import { getVersionJob } from './utils/getVersion';
import { clickupEcsService } from '../clickup-ecs-service';

export module ecsServiceCDWorkflow {
export const WORKFLOW_LOCATION = '.github/workflows/ecs-service-cd.yml';
export interface CDOptionsConfig extends ecsServiceDeployWorkflow.EcsServiceDeployOptionsConfig {}

export function createContinuousDeliveryWorkflow(
Expand All @@ -12,30 +13,38 @@ export module ecsServiceCDWorkflow {
override?: any,
) {
ecsServiceDeployWorkflow.addEcsServiceDeployWorkflowYml(project, options, override);
new YamlFile(project, ecsServiceCDWorkflow.WORKFLOW_LOCATION, {
obj: {
...getContinuousDeliveryWorkflow(project),
...override,
},
});
}

project.release?.addJobs({
cd: {
name: `Deploy ${project.serviceName} to ECS`,
runsOn: ['ubuntu-latest'],
needs: ['release'], // Release job name from project.release
permissions: {
actions: JobPermission.READ,
contents: JobPermission.READ,
packages: JobPermission.READ,
function getContinuousDeliveryWorkflow(project: clickupEcsService.ClickUpTypeScriptEcsServiceProject) {
const workflow = {
name: `Deploy ${project.serviceName} to ECS`,
on: {
push: {
branches: project.release?.branches,
},
steps: [
...getVersionStep(project),
{
name: 'Deploy',
uses: `./${ecsServiceDeployWorkflow.WORKFLOW_LOCATION}`,
with: {
'service-name': project.serviceName,
version: '${{ steps.event_metadata.outputs.release_tag }}',
'harness-account-identifier': '${{ secrets.HARNESS_ACCOUNT_IDENTIFIER }}',
},
},
env: {
RELEASE: true,
},
jobs: {
...getVersionJob(project),
Deploy: {
uses: `./${ecsServiceDeployWorkflow.WORKFLOW_LOCATION}`,
needs: ['get-version'],
with: {
'service-name': project.serviceName,
version: '${{ needs.get-version.outputs.version }}',
'harness-account-identifier': '${{ secrets.HARNESS_ACCOUNT_IDENTIFIER }}',
},
],
},
},
});
};
return workflow;
}
}
80 changes: 48 additions & 32 deletions src/workflows/ecs-service-ci.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
import { YamlFile } from 'projen';
import { JobPermission } from 'projen/lib/github/workflows-model';
import { ecsServiceBuildPublishWorkflow } from './ecs-service-build';
import { getCheckoutStep } from './utils/getCheckoutStep';
import { getVersionStep as getVersionSteps } from './utils/getVersionStep';
import { getVersionJob } from './utils/getVersion';
import { clickupEcsService } from '../clickup-ecs-service';

export module ecsServiceCIWorkflow {
export const WORKFLOW_LOCATION = '.github/workflows/ecs-service-ci.yml';
export interface CIOptionsConfig extends ecsServiceBuildPublishWorkflow.BuildPublishOptionsConfig {}

/**
Expand Down Expand Up @@ -42,44 +43,59 @@ export module ecsServiceCIWorkflow {

export function createContinuousIntegrationWorkflow(
project: clickupEcsService.ClickUpTypeScriptEcsServiceProject,
options: CIOptionsConfig,
options: ecsServiceBuildPublishWorkflow.BuildPublishOptionsConfig,
override?: any,
) {
): void {
ecsServiceBuildPublishWorkflow.addBuildPublishWorkflowYml(project, options, override);
setBuildReleaseTasks(project);
project.buildWorkflow?.addPostBuildJob('ci', {
name: 'Docker Build and Publish',
new YamlFile(project, ecsServiceCIWorkflow.WORKFLOW_LOCATION, {
obj: {
...getContinuousIntegrationWorkflow(project),
...override,
},
});
}

export function getContinuousIntegrationWorkflow(project: clickupEcsService.ClickUpTypeScriptEcsServiceProject) {
const workflow = {
name: 'Continuous Integration',
on: {
pull_request: {},
workflow_dispatch: {},
},
env: {
FORCE_COLOR: '3', // Fix terminal color output
GIT_TAG_PREFIX: 'backend',
},
runsOn: ['ubuntu-latest'],
permissions: {
contents: JobPermission.READ,
actions: JobPermission.READ,
packages: JobPermission.READ,
jobs: {
...getVersionJob(project),
...createBuildPublishJob(project),
},
steps: [
getCheckoutStep(),
...getVersionSteps(project),
{
name: 'Build and Publish Docker Image to ECR',
uses: `./${ecsServiceBuildPublishWorkflow.WORKFLOW_LOCATION}`,
with: {
version: '${{ steps.event_metadata.outputs.release_tag }}',
'service-name': project.serviceName,
'all-package-read-token': '${{ secrets.ALL_PACKAGE_READ_TOKEN }}',
'buildcache-aws-access-key-id': '${{ secrets.ECR_BUILD_CACHE_AWS_ACCESS_KEY_ID }}',
'buildcache-aws-secret-access-key': '${{ secrets.ECR_BUILD_CACHE_AWS_SECRET_ACCESS_KEY }}',
'buildcache-aws-region': '${{ secrets.ECR_BUILD_CACHE_AWS_REGION }}',
'lacework-account-name': '${{ secrets.LW_ACCOUNT_NAME }}',
'lacework-access-token': '${{ secrets.LW_ACCESS_TOKEN }}',
'skip-health-check': false, // TODO: Not hardcode false
'is-nest-app': true, // TODO: Not hardcode true
'skip-vulnerability-scan': true, // TODO: Not hardcode true
},
};
return workflow;
}

function createBuildPublishJob(project: clickupEcsService.ClickUpTypeScriptEcsServiceProject) {
const job = {
'build-publish-docker': {
name: 'Build and Publish Docker Image to ECR',
permissions: {
contents: JobPermission.READ,
actions: JobPermission.READ,
packages: JobPermission.READ,
},
],
});
needs: ['get-version'],
uses: `./${ecsServiceBuildPublishWorkflow.WORKFLOW_LOCATION}`,
secrets: 'inherit',
with: {
version: '${{ needs.get-version.outputs.version }}',
'service-name': project.serviceName,
'skip-health-check': false, // TODO: Not hardcode false
'is-nest-app': true, // TODO: Not hardcode true
'skip-vulnerability-scan': true, // TODO: Not hardcode true
},
},
};
return job;
}
}
72 changes: 72 additions & 0 deletions src/workflows/utils/getVersion.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
import { JobPermission } from 'projen/lib/github/workflows-model';
import { NodeProject } from 'projen/lib/javascript';
import { getCheckoutStep } from './getCheckoutStep';

/**
* To reference the version, use ${{ steps.event_metadata.outputs.release_tag }}
* @param project
* @returns
*/
export const getVersionSteps = (project: NodeProject, download: boolean = true) => {
const downloadStep = {
name: 'Download build artifacts',
uses: 'actions/download-artifact@v3',
with: {
name: 'build-artifact',
path: project.release!.artifactsDirectory,
},
};
const inlineGetVersionStep = {
name: 'Bump to Create Version Tag',
run: 'test "$RELEASE" = "true" && npx projen bump || PRERELEASE=$(git rev-parse HEAD) npx projen bump',
};
const inlineUndoGetVersionStep = {
name: 'Unbump to Return to Previous State',
run: 'test "$RELEASE" = "true" && npx projen unbump || PRERELEASE=$(git rev-parse HEAD) npx projen unbump',
};
const versionStep = {
name: 'Get version',
id: 'event_metadata',
run: `echo "release_tag=$(cat ${project.release!.artifactsDirectory}/releasetag.txt)" >> $GITHUB_OUTPUT`,
};
const steps = download ? [downloadStep, versionStep] : [inlineGetVersionStep, versionStep, inlineUndoGetVersionStep];
return steps;
};

export const getVersionJob = (project: NodeProject) => {
const job = {
'get-version': {
name: 'Get Version',
'runs-on': 'ubuntu-latest',
permissions: {
contents: JobPermission.READ,
actions: JobPermission.READ,
packages: JobPermission.READ,
},
outputs: {
version: '${{ steps.event_metadata.outputs.release_tag }}',
},
steps: [
getCheckoutStep(),
{
name: 'GitHub Packages authorization',
env: {
NPM_TOKEN: '${{ secrets.ALL_PACKAGE_READ_TOKEN }}',
},
run: [
'cat > .npmrc <<EOF',
'//npm.pkg.github.com/:_authToken=${NPM_TOKEN}',
'@time-loop:registry=https://npm.pkg.github.com/',
'EOF',
].join('\n'),
},
{
name: 'Install dependencies',
run: 'yarn install --check-files --frozen-lockfile',
},
...getVersionSteps(project, false),
],
},
};
return job;
};
Loading

0 comments on commit 3085e9c

Please sign in to comment.