From 089cb44c6c9870f93a37b820f96ae0ce1abe9945 Mon Sep 17 00:00:00 2001 From: Kamakshee Samant Date: Thu, 4 Jan 2024 16:57:05 +1100 Subject: [PATCH 1/5] chore: in progress --- spa/src/api/subscriptions/index.ts | 1 + .../services/subscription-manager/index.ts | 25 ++- src/rest-interfaces/index.ts | 27 +++ .../subscriptions/backfill-status.test.ts | 188 ++++++++++++++++++ .../routes/subscriptions/backfill-status.ts | 121 +++++++++++ src/rest/routes/subscriptions/index.ts | 3 + src/util/github-installations-helper.ts | 5 +- src/util/handlebars/handlebar-helpers.ts | 2 +- 8 files changed, 366 insertions(+), 6 deletions(-) create mode 100644 src/rest/routes/subscriptions/backfill-status.test.ts create mode 100644 src/rest/routes/subscriptions/backfill-status.ts diff --git a/spa/src/api/subscriptions/index.ts b/spa/src/api/subscriptions/index.ts index 50e615d51..44b8f24df 100644 --- a/spa/src/api/subscriptions/index.ts +++ b/spa/src/api/subscriptions/index.ts @@ -3,6 +3,7 @@ import { RestSyncReqBody } from "~/src/rest-interfaces"; export default { getSubscriptions: () => axiosRest.get("/rest/subscriptions"), + getSubscriptionsBackfillStatus: () => axiosRest.get("/rest/subscriptions//backfill-status"), deleteGHEServer: (serverUrl: string) => axiosRest.delete(`/rest/ghes-servers/${serverUrl}`), deleteGHEApp: (uuid: string) => diff --git a/spa/src/services/subscription-manager/index.ts b/spa/src/services/subscription-manager/index.ts index 01e9221b3..f94d28097 100644 --- a/spa/src/services/subscription-manager/index.ts +++ b/spa/src/services/subscription-manager/index.ts @@ -2,7 +2,7 @@ import Api from "../../api"; import { AxiosError } from "axios"; import { reportError } from "../../utils"; import { GHSubscriptions } from "../../../../src/rest-interfaces"; -import { RestSyncReqBody } from "~/src/rest-interfaces"; +import { BackfillStatusResp, RestSyncReqBody } from "~/src/rest-interfaces"; async function syncSubscription(subscriptionId:number, reqBody: RestSyncReqBody): Promise { try { @@ -26,14 +26,32 @@ async function getSubscriptions(): Promise { const isSuccessful = response.status === 200; if(!isSuccessful) { reportError( - { message: "Response status for getting subscriptions is not 204", status: response.status }, + { message: "Response status for getting subscriptions is not 200", status: response.status }, { path: "getSubscriptions" } ); } return response.data; } catch (e: unknown) { - reportError(new Error("Unable to delete subscription", { cause: e }), { path: "getSubscriptions" }); + reportError(new Error("Unable to get subscription", { cause: e }), { path: "getSubscriptions" }); + return e as AxiosError; + } +} + +async function getSubscriptionsBackfillStatus(): Promise { + try { + const response= await Api.subscriptions.getSubscriptionsBackfillStatus(); + const isSuccessful = response.status === 200; + if(!isSuccessful) { + reportError( + { message: "Response status for getting subscriptions backfill status is not 200", status: response.status }, + { path: "getSubscriptionsBackfillStatus" } + ); + } + + return response.data; + } catch (e: unknown) { + reportError(new Error("Unable to Get subscription backfill status update", { cause: e }), { path: "getSubscriptionsBackfillStatus" }); return e as AxiosError; } } @@ -92,6 +110,7 @@ async function deleteGHEApp(uuid: string): Promise { } export default { getSubscriptions, + getSubscriptionsBackfillStatus, deleteSubscription, deleteGHEServer, syncSubscription, diff --git a/src/rest-interfaces/index.ts b/src/rest-interfaces/index.ts index 03aedd0dc..1544fd529 100644 --- a/src/rest-interfaces/index.ts +++ b/src/rest-interfaces/index.ts @@ -187,3 +187,30 @@ export type GHSubscriptions = { }; export type BackfillPageModalTypes = "BACKFILL" | "DISCONNECT_SUBSCRIPTION" | "DISCONNECT_SERVER_APP" | "DISCONNECT_SERVER" | "DELETE_GHE_APP"; + +export type ConnectionSyncStatus = "IN PROGRESS" | "FINISHED" | "PENDING" | "FAILED" | undefined; + +export type SubscriptionBackfillState = { + totalRepos?: number; + syncedRepos?: number; + syncStatus: ConnectionSyncStatus; + isSyncComplete: boolean; + backfillSince?: Date; + failedSyncErrors?: Record; + syncWarning?: string; +}; + +export type BackfillStatusError = { + subscriptionId: string; + error: string; +}; +export type BackFillType = { + [key: string]: SubscriptionBackfillState; +}; + +export type BackfillStatusResp = { + subscriptions: BackFillType; + isBackfillComplete: boolean; + subscriptionIds: Array; + errors: BackfillStatusError; +}; diff --git a/src/rest/routes/subscriptions/backfill-status.test.ts b/src/rest/routes/subscriptions/backfill-status.test.ts new file mode 100644 index 000000000..78e1b46b6 --- /dev/null +++ b/src/rest/routes/subscriptions/backfill-status.test.ts @@ -0,0 +1,188 @@ +import { getFrontendApp } from "~/src/app"; +import { Installation } from "models/installation"; +import { encodeSymmetric } from "atlassian-jwt"; +import { getLogger } from "config/logger"; +import { Subscription } from "models/subscription"; +import { DatabaseStateCreator } from "test/utils/database-state-creator"; +import supertest from "supertest"; +import { booleanFlag, BooleanFlags } from "config/feature-flags"; +import { when } from "jest-when"; +import { RepoSyncState } from "models/reposyncstate"; + +jest.mock("config/feature-flags"); + +describe("jira-get-connections-backfillStatus.test", () => { + let app; + let installation: Installation; + let subscription: Subscription; + let repoSyncState: RepoSyncState; + const generateJwt = async () => { + return encodeSymmetric( + { + qsh: "context-qsh", + iss: installation.plainClientKey, + sub: "myAccountId" + }, + await installation.decrypt("encryptedSharedSecret", getLogger("test")) + ); + }; + + beforeEach(async () => { + app = getFrontendApp(); + const result = await new DatabaseStateCreator() + .withActiveRepoSyncState() + .create(); + installation = result.installation; + subscription = result.subscription; + repoSyncState = result.repoSyncState!; + when(booleanFlag) + .calledWith(BooleanFlags.JIRA_ADMIN_CHECK) + .mockResolvedValue(true); + }); + + it("should return 401 when no JWT was provided", async () => { + const resp = await supertest(app).get( + `/jira/subscriptions/backfill-status/?subscriptionIds=${subscription.id}` + ); + expect(resp.status).toStrictEqual(401); + expect(resp.text).toBe("Unauthorised"); + }); + + it("should return 403 when not an admin", async () => { + const resp = await supertest(app) + .get( + `/jira/subscriptions/backfill-status?subscriptionIds=${subscription.id}` + ) + .set( + "authorization", + `JWT ${await generateJwt()}` + ); + expect(resp.status).toStrictEqual(403); + }); + + describe("admin and JWT are OK", () => { + beforeEach(() => { + const payload = { + accountId: "myAccountId", + globalPermissions: ["ADMINISTER"] + }; + jiraNock + .post("/rest/api/latest/permissions/check", payload) + .reply(200, { globalPermissions: ["ADMINISTER"] }); + }); + + it("should return 400 when no subscriptions were found", async () => { + const resp = await supertest(app) + .get( + `/jira/subscriptions/backfill-status?subscriptionIds=${ + subscription.id + 1 + }` + ) + .set( + "authorization", + `JWT ${await generateJwt()}` + ); + expect(resp.status).toStrictEqual(400); + }); + + it("should return 400 when no Missing Subscription IDs were found in query", async () => { + const resp = await supertest(app) + .get(`/jira/subscriptions/backfill-status`) + .set("authorization", `JWT ${await generateJwt()}`); + expect(resp.status).toStrictEqual(400); + expect(resp.text).toBe("Missing Subscription IDs"); + }); + + it("should return 403 if the subscription belongs to a different user", async () => { + const result = await new DatabaseStateCreator() + .forJiraHost("https://another-one.atlassian.net") + .create(); + const resp = await supertest(app) + .get( + `/jira/subscriptions/backfill-status?subscriptionIds=${result.subscription.id}` + ) + .set( + "authorization", + `JWT ${await generateJwt()}` + ); + + expect(resp.status).toStrictEqual(403); + }); + + it("should return 200 if the subscription belongs to the same user", async () => { + const resp = await supertest(app) + .get( + `/jira/subscriptions/backfill-status?subscriptionIds=${subscription.id}` + ) + .set( + "authorization", + `JWT ${await generateJwt()}` + ); + expect(resp.status).toStrictEqual(200); + }); + + describe("happy paths", () => { + beforeEach(async () => { + const newRepoSyncStatesData: any[] = []; + for (let newRepoStateNo = 1; newRepoStateNo < 50; newRepoStateNo++) { + const newRepoSyncState = { ...repoSyncState.dataValues }; + newRepoSyncState["repoId"] = repoSyncState.repoId + newRepoStateNo; + newRepoSyncState["repoName"] = + repoSyncState.repoName + newRepoStateNo.toString(); + newRepoSyncState["repoFullName"] = + repoSyncState.repoFullName + + String(newRepoStateNo).padStart(3, "0"); + if (newRepoStateNo === 1) { + newRepoSyncState["commitStatus"] = "pending"; + newRepoSyncState["branchStatus"] = "complete"; + newRepoSyncState["pullStatus"] = "complete"; + newRepoSyncState["buildStatus"] = "complete"; + newRepoSyncState["deploymentStatus"] = "pending"; + } else if (newRepoStateNo % 3 == 1) { + newRepoSyncState["commitStatus"] = "complete"; + newRepoSyncState["branchStatus"] = "complete"; + newRepoSyncState["pullStatus"] = "complete"; + newRepoSyncState["buildStatus"] = "complete"; + newRepoSyncState["deploymentStatus"] = "complete"; + } else if (newRepoStateNo % 3 == 2) { + newRepoSyncState["commitStatus"] = "failed"; + newRepoSyncState["branchStatus"] = "complete"; + newRepoSyncState["pullStatus"] = "complete"; + newRepoSyncState["buildStatus"] = "complete"; + newRepoSyncState["deploymentStatus"] = "failed"; + } + newRepoSyncStatesData.push(newRepoSyncState); + } + await RepoSyncState.bulkCreate(newRepoSyncStatesData); + }); + + it("should return 200 if the subscription belongs to the same user", async () => { + const resp = await supertest(app) + .get( + `/jira/subscriptions/backfill-status?subscriptionIds=${subscription.id}` + ) + .set( + "authorization", + `JWT ${await generateJwt()}` + ); + expect(resp.status).toStrictEqual(200); + + expect(resp.body).toMatchObject({ + data: { + subscriptions: { + [subscription.id]: { + isSyncComplete: false, + syncStatus: "IN PROGRESS", + totalRepos: 33, + syncedRepos: 17, + backfillSince: null + } + }, + isBackfillComplete: false, + subscriptionIds: [subscription.id] + } + }); + }); + }); + }); +}); \ No newline at end of file diff --git a/src/rest/routes/subscriptions/backfill-status.ts b/src/rest/routes/subscriptions/backfill-status.ts new file mode 100644 index 000000000..cf199dff9 --- /dev/null +++ b/src/rest/routes/subscriptions/backfill-status.ts @@ -0,0 +1,121 @@ +import { Request, Response } from "express"; +import { groupBy } from "lodash"; +import { RepoSyncState } from "~/src/models/reposyncstate"; +import { Subscription, SyncStatus } from "~/src/models/subscription"; +import { + mapSyncStatus, + getRetryableFailedSyncErrors +} from "~/src/util/github-installations-helper"; +import { errorWrapper } from "../../helper"; +import { + BackFillType, + SubscriptionBackfillState, + BackfillStatusError +} from "../../../../spa/src/rest-interfaces"; + +const GetSubscriptionsBackfillStatus = async (req: Request, res: Response) => { + try { + const { jiraHost: localJiraHost } = res.locals; + const subscriptionIds = String(req.query.subscriptionIds) + .split(",") + .map(Number) + .filter(Boolean); + + if (subscriptionIds.length === 0) { + req.log.warn("Missing Subscription IDs"); + res.status(400).send("Missing Subscription IDs"); + return; + } + + const subscriptions = await Subscription.findAllForSubscriptionIds( + subscriptionIds + ); + + const resultSubscriptionIds: Array = subscriptions.map( + (subscription) => subscription.id + ); + + if (subscriptions.length === 0) { + req.log.error("Missing Subscription"); + res.status(400).send("Missing Subscription"); + return; + } + + const jiraHostsMatched = subscriptions.every( + (subscription) => subscription.jiraHost === localJiraHost + ); + + if (!jiraHostsMatched) { + req.log.error("mismatched Jira Host"); + res.status(403).send("mismatched Jira Host"); + return; + } + const subscriptionsById = groupBy(subscriptions, "id"); + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const { backfillStatus, errors } = await getBackfillStatus( + subscriptionsById + ); + const isBackfillComplete = getBackfillCompletionStatus(backfillStatus); + res.status(200).send({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + subscriptions: backfillStatus, + isBackfillComplete, + subscriptionIds: resultSubscriptionIds, + errors + }); + } catch (error) { + req.log.error( + "Failed to poll repo backfill status for provided subscription ID" + ); + } +}; + +const getBackfillCompletionStatus = (backfillStatus: BackFillType): boolean => + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + Object.values(backfillStatus).every( + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + (backFill: SubscriptionBackfillState): boolean => backFill.isSyncComplete + ); + +const getBackfillStatus = async ( + subscriptionsById +): Promise<{ + backfillStatus: BackFillType; + errors?: BackfillStatusError[]; +}> => { + const backfillStatus: BackFillType = {}; + const errors: BackfillStatusError[] = []; + for (const subscriptionId in subscriptionsById) { + try { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access + const subscription: Subscription = subscriptionsById[subscriptionId][0]; + const isSyncComplete = + subscription.syncStatus === SyncStatus.COMPLETE || + subscription.syncStatus === SyncStatus.FAILED; + const failedSyncErrors = await getRetryableFailedSyncErrors(subscription); + + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + backfillStatus[subscriptionId] = { + isSyncComplete, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + syncStatus: mapSyncStatus(subscription.syncStatus), + totalRepos: subscription.totalNumberOfRepos, + syncedRepos: await RepoSyncState.countFullySyncedReposForSubscription( + subscription + ), + failedSyncErrors, + backfillSince: subscription.backfillSince, + syncWarning: subscription.syncWarning + }; + } catch (error: unknown) { + errors.push({ subscriptionId, error: JSON.stringify(error) }); + } + } + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + return { backfillStatus, errors }; +}; + +export const GetSubBackfillStatusHandler = errorWrapper( + "SyncRouterHandler", + GetSubscriptionsBackfillStatus +); diff --git a/src/rest/routes/subscriptions/index.ts b/src/rest/routes/subscriptions/index.ts index da165809c..5833cb0e2 100644 --- a/src/rest/routes/subscriptions/index.ts +++ b/src/rest/routes/subscriptions/index.ts @@ -6,6 +6,7 @@ import { removeSubscription } from "utils/jira-utils"; import { GitHubServerApp } from "models/github-server-app"; import { InvalidArgumentError } from "config/errors"; import { SyncRouterHandler } from "./sync"; +import { GetSubBackfillStatusHandler } from "./backfill-status"; export const SubscriptionsRouter = Router({ mergeParams: true }); @@ -19,6 +20,8 @@ SubscriptionsRouter.get("/", errorWrapper("SubscriptionsGet", async (req: Reques }); })); +SubscriptionsRouter.get("/backfill-status", GetSubBackfillStatusHandler); + /** * This delete endpoint only handles Github cloud subscriptions */ diff --git a/src/util/github-installations-helper.ts b/src/util/github-installations-helper.ts index 59b1ed959..b5ec188dd 100644 --- a/src/util/github-installations-helper.ts +++ b/src/util/github-installations-helper.ts @@ -8,6 +8,7 @@ import { RepoSyncState } from "models/reposyncstate"; import { statsd } from "config/statsd"; import { metricError } from "config/metric-names"; import { groupBy, countBy } from "lodash"; +import { ConnectionSyncStatus } from "~/spa/src/rest-interfaces"; interface FailedConnection { id: number; @@ -30,8 +31,7 @@ interface GitHubCloudObj { failedConnections: FailedConnection[] } -export type ConnectionSyncStatus = "IN PROGRESS" | "FINISHED" | "PENDING" | "FAILED" | undefined; -const mapSyncStatus = (syncStatus: SyncStatus = SyncStatus.PENDING): ConnectionSyncStatus => { +export const mapSyncStatus = (syncStatus: SyncStatus = SyncStatus.PENDING): ConnectionSyncStatus => { switch (syncStatus) { case "ACTIVE": return "IN PROGRESS"; @@ -76,6 +76,7 @@ const getInstallation = async (subscription: Subscription, gitHubAppId: number | return { ...response.data, subscriptionId: subscription.id, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment syncStatus: mapSyncStatus(subscription.syncStatus), syncWarning: subscription.syncWarning, totalNumberOfRepos: subscription.totalNumberOfRepos, diff --git a/src/util/handlebars/handlebar-helpers.ts b/src/util/handlebars/handlebar-helpers.ts index ba733a224..546fb6063 100644 --- a/src/util/handlebars/handlebar-helpers.ts +++ b/src/util/handlebars/handlebar-helpers.ts @@ -1,6 +1,6 @@ import hbs from "hbs"; import { isPlainObject } from "lodash"; -import { ConnectionSyncStatus } from "utils/github-installations-helper"; +import { ConnectionSyncStatus } from "~/spa/src/rest-interfaces"; export const concatStringHelper = (...strings: string[]) => strings.filter((arg: unknown) => typeof arg !== "object").join(" "); // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition From 4b862111c10908b362335c8968a16f7f23a22259 Mon Sep 17 00:00:00 2001 From: Kamakshee Samant Date: Fri, 5 Jan 2024 10:28:53 +1100 Subject: [PATCH 2/5] chore: in progress --- spa/src/api/subscriptions/index.ts | 16 +++++++++----- spa/src/pages/Connections/index.tsx | 22 +++++++++++++++++++ .../services/subscription-manager/index.ts | 4 ++-- src/rest-interfaces/index.ts | 4 ++++ .../subscriptions/backfill-status.test.ts | 14 ++++++------ .../routes/subscriptions/backfill-status.ts | 14 +++++++----- test/snapshots/app.test.ts.snap | 4 ++++ 7 files changed, 58 insertions(+), 20 deletions(-) diff --git a/spa/src/api/subscriptions/index.ts b/spa/src/api/subscriptions/index.ts index 44b8f24df..d17cfd5e1 100644 --- a/spa/src/api/subscriptions/index.ts +++ b/spa/src/api/subscriptions/index.ts @@ -1,15 +1,21 @@ import { axiosRest } from "../axiosInstance"; -import { RestSyncReqBody } from "~/src/rest-interfaces"; +import { + BackfillStatusUrlParams, + RestSyncReqBody, +} from "~/src/rest-interfaces"; export default { getSubscriptions: () => axiosRest.get("/rest/subscriptions"), - getSubscriptionsBackfillStatus: () => axiosRest.get("/rest/subscriptions//backfill-status"), + getSubscriptionsBackfillStatus: (params: BackfillStatusUrlParams) => + axiosRest.get(`/rest/subscriptions/backfill-status`, { params }), deleteGHEServer: (serverUrl: string) => axiosRest.delete(`/rest/ghes-servers/${serverUrl}`), - deleteGHEApp: (uuid: string) => - axiosRest.delete(`/rest/app/${uuid}`), + deleteGHEApp: (uuid: string) => axiosRest.delete(`/rest/app/${uuid}`), deleteSubscription: (subscriptionId: number) => axiosRest.delete(`/rest/app/cloud/subscriptions/${subscriptionId}`), syncSubscriptions: (subscriptionId: number, reqBody: RestSyncReqBody) => - axiosRest.post(`/rest/app/cloud/subscriptions/${subscriptionId}/sync`, reqBody), + axiosRest.post( + `/rest/app/cloud/subscriptions/${subscriptionId}/sync`, + reqBody + ), }; diff --git a/spa/src/pages/Connections/index.tsx b/spa/src/pages/Connections/index.tsx index 694831ff0..eaafe2ddd 100644 --- a/spa/src/pages/Connections/index.tsx +++ b/spa/src/pages/Connections/index.tsx @@ -89,6 +89,7 @@ const Connections = () => { try { setIsLoading(true); const response = await SubscriptionManager.getSubscriptions(); + console.log(">>>>>",response); if (response instanceof AxiosError) { // TODO: Handle the error once we have the designs console.error("Error", response); @@ -102,10 +103,31 @@ const Connections = () => { } }; + const fetchBackfillStatus = async () => { + try { + setIsLoading(true); + const response = await SubscriptionManager.getSubscriptionsBackfillStatus("41034311"); + if (response instanceof AxiosError) { + // TODO: Handle the error once we have the designs + console.error("Error", response); + } + } catch (e) { + // TODO: handle this error in UI/Modal ? + console.error("Could not fetch ghe subscriptions: ", e); + } finally { + setIsLoading(false); + } + }; + useEffect(() => { + fetchBackfillStatus(); fetchGHSubscriptions(); }, []); + // useEffect(() => { + // fetchBackfillStatus(); + // }, [subscriptions]); + // If there are no connections then go back to the start page useEffect(() => { if ( diff --git a/spa/src/services/subscription-manager/index.ts b/spa/src/services/subscription-manager/index.ts index f94d28097..eeadcb755 100644 --- a/spa/src/services/subscription-manager/index.ts +++ b/spa/src/services/subscription-manager/index.ts @@ -38,9 +38,9 @@ async function getSubscriptions(): Promise { } } -async function getSubscriptionsBackfillStatus(): Promise { +async function getSubscriptionsBackfillStatus(subscriptionIds: string): Promise { try { - const response= await Api.subscriptions.getSubscriptionsBackfillStatus(); + const response= await Api.subscriptions.getSubscriptionsBackfillStatus({ subscriptionIds }); const isSuccessful = response.status === 200; if(!isSuccessful) { reportError( diff --git a/src/rest-interfaces/index.ts b/src/rest-interfaces/index.ts index 1544fd529..4ad7e8354 100644 --- a/src/rest-interfaces/index.ts +++ b/src/rest-interfaces/index.ts @@ -24,6 +24,10 @@ export type DeferredInstallationUrlParams = { gitHubOrgName: string; }; +export type BackfillStatusUrlParams = { + subscriptionIds: string; +}; + export type DeferralParsedRequest = { orgName: string; jiraHost: string; diff --git a/src/rest/routes/subscriptions/backfill-status.test.ts b/src/rest/routes/subscriptions/backfill-status.test.ts index 78e1b46b6..34cf01746 100644 --- a/src/rest/routes/subscriptions/backfill-status.test.ts +++ b/src/rest/routes/subscriptions/backfill-status.test.ts @@ -42,7 +42,7 @@ describe("jira-get-connections-backfillStatus.test", () => { it("should return 401 when no JWT was provided", async () => { const resp = await supertest(app).get( - `/jira/subscriptions/backfill-status/?subscriptionIds=${subscription.id}` + `/rest/subscriptions/backfill-status?subscriptionIds=${subscription.id}` ); expect(resp.status).toStrictEqual(401); expect(resp.text).toBe("Unauthorised"); @@ -51,7 +51,7 @@ describe("jira-get-connections-backfillStatus.test", () => { it("should return 403 when not an admin", async () => { const resp = await supertest(app) .get( - `/jira/subscriptions/backfill-status?subscriptionIds=${subscription.id}` + `/rest/subscriptions/backfill-status?subscriptionIds=${subscription.id}` ) .set( "authorization", @@ -74,7 +74,7 @@ describe("jira-get-connections-backfillStatus.test", () => { it("should return 400 when no subscriptions were found", async () => { const resp = await supertest(app) .get( - `/jira/subscriptions/backfill-status?subscriptionIds=${ + `/rest/subscriptions/backfill-status?subscriptionIds=${ subscription.id + 1 }` ) @@ -87,7 +87,7 @@ describe("jira-get-connections-backfillStatus.test", () => { it("should return 400 when no Missing Subscription IDs were found in query", async () => { const resp = await supertest(app) - .get(`/jira/subscriptions/backfill-status`) + .get(`/rest/subscriptions/backfill-status`) .set("authorization", `JWT ${await generateJwt()}`); expect(resp.status).toStrictEqual(400); expect(resp.text).toBe("Missing Subscription IDs"); @@ -99,7 +99,7 @@ describe("jira-get-connections-backfillStatus.test", () => { .create(); const resp = await supertest(app) .get( - `/jira/subscriptions/backfill-status?subscriptionIds=${result.subscription.id}` + `/rest/subscriptions/backfill-status?subscriptionIds=${result.subscription.id}` ) .set( "authorization", @@ -112,7 +112,7 @@ describe("jira-get-connections-backfillStatus.test", () => { it("should return 200 if the subscription belongs to the same user", async () => { const resp = await supertest(app) .get( - `/jira/subscriptions/backfill-status?subscriptionIds=${subscription.id}` + `/rest/subscriptions/backfill-status?subscriptionIds=${subscription.id}` ) .set( "authorization", @@ -159,7 +159,7 @@ describe("jira-get-connections-backfillStatus.test", () => { it("should return 200 if the subscription belongs to the same user", async () => { const resp = await supertest(app) .get( - `/jira/subscriptions/backfill-status?subscriptionIds=${subscription.id}` + `/rest/subscriptions/backfill-status?subscriptionIds=${subscription.id}` ) .set( "authorization", diff --git a/src/rest/routes/subscriptions/backfill-status.ts b/src/rest/routes/subscriptions/backfill-status.ts index cf199dff9..7cf12fab1 100644 --- a/src/rest/routes/subscriptions/backfill-status.ts +++ b/src/rest/routes/subscriptions/backfill-status.ts @@ -12,23 +12,25 @@ import { SubscriptionBackfillState, BackfillStatusError } from "../../../../spa/src/rest-interfaces"; +import { BaseLocals } from ".."; -const GetSubscriptionsBackfillStatus = async (req: Request, res: Response) => { +const GetSubscriptionsBackfillStatus = async (req: Request, res: Response) => { try { const { jiraHost: localJiraHost } = res.locals; - const subscriptionIds = String(req.query.subscriptionIds) + const { subscriptionIds } = req.query; + + const subIds = String(subscriptionIds) .split(",") .map(Number) .filter(Boolean); - - if (subscriptionIds.length === 0) { + if (subIds.length === 0) { req.log.warn("Missing Subscription IDs"); res.status(400).send("Missing Subscription IDs"); return; } const subscriptions = await Subscription.findAllForSubscriptionIds( - subscriptionIds + subIds ); const resultSubscriptionIds: Array = subscriptions.map( @@ -116,6 +118,6 @@ const getBackfillStatus = async ( }; export const GetSubBackfillStatusHandler = errorWrapper( - "SyncRouterHandler", + "GetSubBackfillStatusHandler", GetSubscriptionsBackfillStatus ); diff --git a/test/snapshots/app.test.ts.snap b/test/snapshots/app.test.ts.snap index c81a6a48a..574e2e21e 100644 --- a/test/snapshots/app.test.ts.snap +++ b/test/snapshots/app.test.ts.snap @@ -15,6 +15,8 @@ exports[`app getFrontendApp please review routes and update snapshot when adding query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,serveStatic :GET ^/?(?=/|$)^/rest/?(?=/|$)^/subscriptions/?(?=/|$)^/?$ query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,JwtHandler,jiraAdminEnforceMiddleware,SubscriptionsGet +:GET ^/?(?=/|$)^/rest/?(?=/|$)^/subscriptions/?(?=/|$)^/backfill-status/?$ + query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,JwtHandler,jiraAdminEnforceMiddleware,SyncRouterHandler :DELETE ^/?(?=/|$)^/rest/?(?=/|$)^/subscriptions/?(?=/|$)^/?$ query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,JwtHandler,jiraAdminEnforceMiddleware,SubscriptionDelete :POST ^/?(?=/|$)^/rest/?(?=/|$)^/subscriptions/?(?=/|$)^/sync/?$ @@ -51,6 +53,8 @@ exports[`app getFrontendApp please review routes and update snapshot when adding query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,JwtHandler,jiraAdminEnforceMiddleware,tempReplaceUUID,GithubServerAppMiddleware,JiraCloudIDGet :GET ^/?(?=/|$)^/rest/?(?=/|$)^/app/(?:([^/]+?))/?(?=/|$)^/subscriptions/(?:([^/]+?))/?(?=/|$)^/?$ query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,JwtHandler,jiraAdminEnforceMiddleware,tempReplaceUUID,GithubServerAppMiddleware,SubscriptionsGet +:GET ^/?(?=/|$)^/rest/?(?=/|$)^/app/(?:([^/]+?))/?(?=/|$)^/subscriptions/(?:([^/]+?))/?(?=/|$)^/backfill-status/?$ + query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,JwtHandler,jiraAdminEnforceMiddleware,tempReplaceUUID,GithubServerAppMiddleware,SyncRouterHandler :DELETE ^/?(?=/|$)^/rest/?(?=/|$)^/app/(?:([^/]+?))/?(?=/|$)^/subscriptions/(?:([^/]+?))/?(?=/|$)^/?$ query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,JwtHandler,jiraAdminEnforceMiddleware,tempReplaceUUID,GithubServerAppMiddleware,SubscriptionDelete :POST ^/?(?=/|$)^/rest/?(?=/|$)^/app/(?:([^/]+?))/?(?=/|$)^/subscriptions/(?:([^/]+?))/?(?=/|$)^/sync/?$ From 3f85c5dbe9714ab13e0ddbaf902d0059c261b8a3 Mon Sep 17 00:00:00 2001 From: Kamakshee Samant Date: Tue, 9 Jan 2024 12:45:07 +1100 Subject: [PATCH 3/5] chore: in progress --- spa/src/pages/Connections/index.tsx | 61 ++++++---- spa/src/utils/index.ts | 105 ++++++++++++++++++ src/rest-interfaces/index.ts | 7 +- .../routes/subscriptions/backfill-status.ts | 5 +- 4 files changed, 154 insertions(+), 24 deletions(-) diff --git a/spa/src/pages/Connections/index.tsx b/spa/src/pages/Connections/index.tsx index eaafe2ddd..16a9c5fb6 100644 --- a/spa/src/pages/Connections/index.tsx +++ b/spa/src/pages/Connections/index.tsx @@ -11,13 +11,18 @@ import { GHSubscriptions, BackfillPageModalTypes, SuccessfulConnection, - GitHubEnterpriseApplication, + GitHubEnterpriseApplication } from "../../rest-interfaces"; import SkeletonForLoading from "./SkeletonForLoading"; import SubscriptionManager from "../../services/subscription-manager"; import RestartBackfillModal from "./Modals/RestartBackfillModal"; import DisconnectSubscriptionModal from "./Modals/DisconnectSubscriptionModal"; -import { DisconnectGHEServerModal, DeleteAppInGitHubModal, DisconnectGHEServerAppModal } from "./Modals/DisconnectGHEServerModal"; +import { + DisconnectGHEServerModal, + DeleteAppInGitHubModal, + DisconnectGHEServerAppModal, +} from "./Modals/DisconnectGHEServerModal"; +import { getInProgressSubIds, getUpdatedSubscriptions } from "~/src/utils"; const hasGHCloudConnections = (subscriptions: GHSubscriptions): boolean => subscriptions?.ghCloudSubscriptions && @@ -33,6 +38,13 @@ const Connections = () => { const [dataForModal, setDataForModal] = useState< SuccessfulConnection | GitHubEnterpriseApplication | undefined >(undefined); + const [isLoading, setIsLoading] = useState(false); + const [subscriptions, setSubscriptions] = useState( + null + ); + const [inProgressSubs, setInProgressSubs] = useState | null>( + null + ); const openedModal = () => { switch (selectedModal) { case "BACKFILL": @@ -80,21 +92,20 @@ const Connections = () => { } }; - const [isLoading, setIsLoading] = useState(false); - const [subscriptions, setSubscriptions] = useState( - null - ); - const fetchGHSubscriptions = async () => { try { setIsLoading(true); const response = await SubscriptionManager.getSubscriptions(); - console.log(">>>>>",response); if (response instanceof AxiosError) { // TODO: Handle the error once we have the designs console.error("Error", response); + } else { + const inProgressSubIds = getInProgressSubIds(response); + if (inProgressSubIds && inProgressSubIds.length > 0) { + setInProgressSubs(inProgressSubIds); + } + setSubscriptions(response as GHSubscriptions); } - setSubscriptions(response as GHSubscriptions); } catch (e) { // TODO: handle this error in UI/Modal ? console.error("Could not fetch ghe subscriptions: ", e); @@ -103,31 +114,43 @@ const Connections = () => { } }; - const fetchBackfillStatus = async () => { + const fetchBackfillStatus = async (inProgressSubs: Array) => { try { - setIsLoading(true); - const response = await SubscriptionManager.getSubscriptionsBackfillStatus("41034311"); + const response = await SubscriptionManager.getSubscriptionsBackfillStatus( + inProgressSubs.toString() + ); if (response instanceof AxiosError) { // TODO: Handle the error once we have the designs console.error("Error", response); + } else { + if(subscriptions){ + const newSubscriptions = getUpdatedSubscriptions(response, subscriptions); + if(newSubscriptions) { + setSubscriptions(newSubscriptions); + } + } + if (!response.isBackfillComplete) { + setTimeout(() => { + fetchBackfillStatus(inProgressSubs); + }, 3000); + } } } catch (e) { // TODO: handle this error in UI/Modal ? console.error("Could not fetch ghe subscriptions: ", e); - } finally { - setIsLoading(false); } }; useEffect(() => { - fetchBackfillStatus(); + if (inProgressSubs && inProgressSubs.length > 0) { + fetchBackfillStatus(inProgressSubs); + } + }, [inProgressSubs]); + + useEffect(() => { fetchGHSubscriptions(); }, []); - // useEffect(() => { - // fetchBackfillStatus(); - // }, [subscriptions]); - // If there are no connections then go back to the start page useEffect(() => { if ( diff --git a/spa/src/utils/index.ts b/spa/src/utils/index.ts index 0d4254022..ba019ad65 100644 --- a/spa/src/utils/index.ts +++ b/spa/src/utils/index.ts @@ -1,5 +1,6 @@ import * as Sentry from "@sentry/react"; import { AxiosError } from "axios"; +import { BackfillStatusResp, GHSubscriptions } from "../rest-interfaces"; export const getJiraJWT = (): Promise => new Promise(resolve => { return AP.context.getToken((token: string) => { @@ -58,3 +59,107 @@ export function openChildWindow(url: string) { }, 100); return child; } + +export const getInProgressSubIds = (response: GHSubscriptions): Array => { + const successfulCloudConnections = + response.ghCloudSubscriptions.successfulCloudConnections; + const inProgressCloudConnections = successfulCloudConnections.filter( + (connection) => + connection.syncStatus === "IN PROGRESS" || + connection.syncStatus === "PENDING" + ); + const inProgressCloudSubIds = inProgressCloudConnections.map( + (connection) => connection.subscriptionId + ); + let inProgressGHESubIds: Array = []; + const ghEnterpriseServers = response.ghEnterpriseServers; + for (const ghEnterpriseServer of ghEnterpriseServers) { + const applications = ghEnterpriseServer.applications; + for (const application of applications) { + const successfulGHEConnections = application.successfulConnections; + const inProgressGHEConnections = successfulGHEConnections.filter( + (connection) => + connection.syncStatus === "IN PROGRESS" || + connection.syncStatus === "PENDING" + ); + inProgressGHESubIds = inProgressGHEConnections.map( + (connection) => connection.subscriptionId + ); + } + } + return [...inProgressCloudSubIds, ...inProgressGHESubIds]; +}; + +export const getUpdatedSubscriptions = (response: BackfillStatusResp, subscriptions: GHSubscriptions): GHSubscriptions | undefined => { + const currentSubs = subscriptions; + let newSubs; + const subscriptionIds = response.subscriptionIds || []; + if (currentSubs) { + for (const subscriptionId of subscriptionIds) { + const subscriptions = response.subscriptions; + const subscription = subscriptions[subscriptionId]; + if (!subscription.gitHubAppId) { + let matchedIndex; + const successfulCloudConnections = + currentSubs?.ghCloudSubscriptions.successfulCloudConnections; + if (successfulCloudConnections) { + matchedIndex = successfulCloudConnections.findIndex( + (connection) => connection.subscriptionId === subscriptionId + ); + successfulCloudConnections[matchedIndex] = { + ...successfulCloudConnections[matchedIndex], + numberOfSyncedRepos: subscription.syncedRepos, + syncStatus: subscription.syncStatus, + }; + if(subscription.backfillSince){ + successfulCloudConnections[matchedIndex]["backfillSince"] = subscription.backfillSince; + } + newSubs = { + ...currentSubs, + ghCloudSubscriptions: { + ...currentSubs.ghCloudSubscriptions, + successfulCloudConnections: successfulCloudConnections, + }, + }; + } + } else { + const ghEnterpriseServers = currentSubs.ghEnterpriseServers; + let ghEnterpriseServerIndex; + let applicationIndex; + for (const [ + ghEnterpriseServerI, + ghEnterpriseServer, + ] of ghEnterpriseServers.entries()) { + const applications = ghEnterpriseServer.applications; + for (const [applicationI, app] of applications.entries()) { + if (app.id === subscription.gitHubAppId) { + ghEnterpriseServerIndex = ghEnterpriseServerI; + applicationIndex = applicationI; + break; + } + } + } + console.log("::::: ghEnterpriseServerIndex",ghEnterpriseServerIndex); + console.log("::::: applicationIndex", applicationIndex); + if (ghEnterpriseServerIndex && applicationIndex) { + const newGHEnterpriseServers = ghEnterpriseServers; + const newApps = + ghEnterpriseServers[ghEnterpriseServerIndex].applications; + newApps[applicationIndex] = { + ...newApps[applicationIndex], + successfulConnections: [], + }; + newGHEnterpriseServers[ghEnterpriseServerIndex] = { + ...ghEnterpriseServers[ghEnterpriseServerIndex], + applications: newApps, + }; + newSubs = { + ...currentSubs, + ghEnterpriseServers: newGHEnterpriseServers, + }; + } + } + } + return newSubs; + } +}; diff --git a/src/rest-interfaces/index.ts b/src/rest-interfaces/index.ts index 4ad7e8354..776f638b2 100644 --- a/src/rest-interfaces/index.ts +++ b/src/rest-interfaces/index.ts @@ -192,16 +192,17 @@ export type GHSubscriptions = { export type BackfillPageModalTypes = "BACKFILL" | "DISCONNECT_SUBSCRIPTION" | "DISCONNECT_SERVER_APP" | "DISCONNECT_SERVER" | "DELETE_GHE_APP"; -export type ConnectionSyncStatus = "IN PROGRESS" | "FINISHED" | "PENDING" | "FAILED" | undefined; +export type ConnectionSyncStatus = "IN PROGRESS" | "FINISHED" | "PENDING" | "FAILED"; export type SubscriptionBackfillState = { totalRepos?: number; - syncedRepos?: number; + syncedRepos: number; syncStatus: ConnectionSyncStatus; isSyncComplete: boolean; - backfillSince?: Date; + backfillSince?: string; failedSyncErrors?: Record; syncWarning?: string; + gitHubAppId?: number; }; export type BackfillStatusError = { diff --git a/src/rest/routes/subscriptions/backfill-status.ts b/src/rest/routes/subscriptions/backfill-status.ts index 7cf12fab1..a35e3bea4 100644 --- a/src/rest/routes/subscriptions/backfill-status.ts +++ b/src/rest/routes/subscriptions/backfill-status.ts @@ -106,8 +106,9 @@ const getBackfillStatus = async ( subscription ), failedSyncErrors, - backfillSince: subscription.backfillSince, - syncWarning: subscription.syncWarning + backfillSince: subscription.backfillSince?.toString(), + syncWarning: subscription.syncWarning, + gitHubAppId: subscription.gitHubAppId }; } catch (error: unknown) { errors.push({ subscriptionId, error: JSON.stringify(error) }); From 773d20bf3eca6c2a44ff01d11f77c02e1d437e85 Mon Sep 17 00:00:00 2001 From: Kamakshee Samant Date: Wed, 10 Jan 2024 18:58:58 +1100 Subject: [PATCH 4/5] chore: in progress --- spa/src/pages/Connections/index.tsx | 2 +- spa/src/utils/index.ts | 175 ++++++++++++------ src/rest-interfaces/index.ts | 1 + .../routes/subscriptions/backfill-status.ts | 1 + 4 files changed, 117 insertions(+), 62 deletions(-) diff --git a/spa/src/pages/Connections/index.tsx b/spa/src/pages/Connections/index.tsx index 16a9c5fb6..2e02f0727 100644 --- a/spa/src/pages/Connections/index.tsx +++ b/spa/src/pages/Connections/index.tsx @@ -22,7 +22,7 @@ import { DeleteAppInGitHubModal, DisconnectGHEServerAppModal, } from "./Modals/DisconnectGHEServerModal"; -import { getInProgressSubIds, getUpdatedSubscriptions } from "~/src/utils"; +import { getInProgressSubIds, getUpdatedSubscriptions } from "../../utils"; const hasGHCloudConnections = (subscriptions: GHSubscriptions): boolean => subscriptions?.ghCloudSubscriptions && diff --git a/spa/src/utils/index.ts b/spa/src/utils/index.ts index ba019ad65..50c12e3b9 100644 --- a/spa/src/utils/index.ts +++ b/spa/src/utils/index.ts @@ -1,6 +1,6 @@ import * as Sentry from "@sentry/react"; import { AxiosError } from "axios"; -import { BackfillStatusResp, GHSubscriptions } from "../rest-interfaces"; +import { BackfillStatusResp, GHSubscriptions, SubscriptionBackfillState } from "../rest-interfaces"; export const getJiraJWT = (): Promise => new Promise(resolve => { return AP.context.getToken((token: string) => { @@ -23,7 +23,7 @@ export function reportError(err: unknown, extra: { try { const cause = (err as Record).cause || {}; - delete (err as Record).cause; //so that Sentry doesn't group all axios error together + delete (err as Record).cause; //so that Sentry doesn"t group all axios error together Sentry.captureException(err, { extra: { @@ -90,7 +90,112 @@ export const getInProgressSubIds = (response: GHSubscriptions): Array => return [...inProgressCloudSubIds, ...inProgressGHESubIds]; }; -export const getUpdatedSubscriptions = (response: BackfillStatusResp, subscriptions: GHSubscriptions): GHSubscriptions | undefined => { +const getUpdatedCloudSubs = ( + currentSubs: GHSubscriptions, + subscriptionId: number, + subscription: SubscriptionBackfillState +) => { + let matchedIndex; + const successfulCloudConnections = + currentSubs?.ghCloudSubscriptions.successfulCloudConnections; + if (successfulCloudConnections) { + matchedIndex = successfulCloudConnections.findIndex( + (connection) => connection.subscriptionId === subscriptionId + ); + successfulCloudConnections[matchedIndex] = { + ...successfulCloudConnections[matchedIndex], + numberOfSyncedRepos: subscription.syncedRepos, + syncStatus: subscription.syncStatus, + }; + if (subscription.backfillSince) { + successfulCloudConnections[matchedIndex]["backfillSince"] = + subscription.backfillSince; + } + return { + ...currentSubs, + ghCloudSubscriptions: { + ...currentSubs.ghCloudSubscriptions, + successfulCloudConnections: successfulCloudConnections, + }, + }; + } +}; + +const getUpdatedGHESubs = ( + currentSubs: GHSubscriptions, + subscription: SubscriptionBackfillState +) => { + const ghEnterpriseServers = currentSubs.ghEnterpriseServers; + let ghEnterpriseServerIndex; + let applicationIndex; + for (const [ + ghEnterpriseServerI, + ghEnterpriseServer, + ] of ghEnterpriseServers.entries()) { + const applications = ghEnterpriseServer.applications; + + for (const [appIndex, app] of applications.entries()) { + if (app.id === subscription.gitHubAppId) { + ghEnterpriseServerIndex = ghEnterpriseServerI; + applicationIndex = appIndex; + break; + } + } + } + if ( + typeof ghEnterpriseServerIndex === "number" && + !isNaN(ghEnterpriseServerIndex) && + typeof applicationIndex === "number" && + !isNaN(applicationIndex) + ) { + const newGHEnterpriseServers = ghEnterpriseServers; + const apps = ghEnterpriseServers[ghEnterpriseServerIndex].applications; + const newApps = [...apps]; + + if (subscription.gitHubAppId) { + const successfulConnections = + newApps[applicationIndex]?.successfulConnections; + const newSuccessfulConnections = successfulConnections.map( + (connection) => { + if (connection.subscriptionId === subscription.id) { + const result = { + ...connection, + syncStatus: subscription.syncStatus, + numberOfSyncedRepos: subscription.syncedRepos, + + }; + if (subscription.backfillSince) { + result["backfillSince"] = + subscription.backfillSince; + } + return result; + } + return connection; + } + ); + + newApps[applicationIndex] = { + ...newApps[applicationIndex], + successfulConnections: [...newSuccessfulConnections], + }; + } + + newGHEnterpriseServers[ghEnterpriseServerIndex] = { + ...ghEnterpriseServers[ghEnterpriseServerIndex], + applications: newApps, + }; + const result = { + ...currentSubs, + ghEnterpriseServers: newGHEnterpriseServers, + }; + return result; + } +}; + +export const getUpdatedSubscriptions = ( + response: BackfillStatusResp, + subscriptions: GHSubscriptions +): GHSubscriptions | undefined => { const currentSubs = subscriptions; let newSubs; const subscriptionIds = response.subscriptionIds || []; @@ -99,65 +204,13 @@ export const getUpdatedSubscriptions = (response: BackfillStatusResp, subscripti const subscriptions = response.subscriptions; const subscription = subscriptions[subscriptionId]; if (!subscription.gitHubAppId) { - let matchedIndex; - const successfulCloudConnections = - currentSubs?.ghCloudSubscriptions.successfulCloudConnections; - if (successfulCloudConnections) { - matchedIndex = successfulCloudConnections.findIndex( - (connection) => connection.subscriptionId === subscriptionId - ); - successfulCloudConnections[matchedIndex] = { - ...successfulCloudConnections[matchedIndex], - numberOfSyncedRepos: subscription.syncedRepos, - syncStatus: subscription.syncStatus, - }; - if(subscription.backfillSince){ - successfulCloudConnections[matchedIndex]["backfillSince"] = subscription.backfillSince; - } - newSubs = { - ...currentSubs, - ghCloudSubscriptions: { - ...currentSubs.ghCloudSubscriptions, - successfulCloudConnections: successfulCloudConnections, - }, - }; - } + newSubs = getUpdatedCloudSubs( + currentSubs, + subscriptionId, + subscription + ); } else { - const ghEnterpriseServers = currentSubs.ghEnterpriseServers; - let ghEnterpriseServerIndex; - let applicationIndex; - for (const [ - ghEnterpriseServerI, - ghEnterpriseServer, - ] of ghEnterpriseServers.entries()) { - const applications = ghEnterpriseServer.applications; - for (const [applicationI, app] of applications.entries()) { - if (app.id === subscription.gitHubAppId) { - ghEnterpriseServerIndex = ghEnterpriseServerI; - applicationIndex = applicationI; - break; - } - } - } - console.log("::::: ghEnterpriseServerIndex",ghEnterpriseServerIndex); - console.log("::::: applicationIndex", applicationIndex); - if (ghEnterpriseServerIndex && applicationIndex) { - const newGHEnterpriseServers = ghEnterpriseServers; - const newApps = - ghEnterpriseServers[ghEnterpriseServerIndex].applications; - newApps[applicationIndex] = { - ...newApps[applicationIndex], - successfulConnections: [], - }; - newGHEnterpriseServers[ghEnterpriseServerIndex] = { - ...ghEnterpriseServers[ghEnterpriseServerIndex], - applications: newApps, - }; - newSubs = { - ...currentSubs, - ghEnterpriseServers: newGHEnterpriseServers, - }; - } + newSubs = getUpdatedGHESubs(currentSubs, subscription); } } return newSubs; diff --git a/src/rest-interfaces/index.ts b/src/rest-interfaces/index.ts index 776f638b2..cf3814cd4 100644 --- a/src/rest-interfaces/index.ts +++ b/src/rest-interfaces/index.ts @@ -195,6 +195,7 @@ export type BackfillPageModalTypes = "BACKFILL" | "DISCONNECT_SUBSCRIPTION" | "D export type ConnectionSyncStatus = "IN PROGRESS" | "FINISHED" | "PENDING" | "FAILED"; export type SubscriptionBackfillState = { + id: number; totalRepos?: number; syncedRepos: number; syncStatus: ConnectionSyncStatus; diff --git a/src/rest/routes/subscriptions/backfill-status.ts b/src/rest/routes/subscriptions/backfill-status.ts index a35e3bea4..465b1d386 100644 --- a/src/rest/routes/subscriptions/backfill-status.ts +++ b/src/rest/routes/subscriptions/backfill-status.ts @@ -98,6 +98,7 @@ const getBackfillStatus = async ( // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access backfillStatus[subscriptionId] = { + id: parseInt(subscriptionId), isSyncComplete, // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment syncStatus: mapSyncStatus(subscription.syncStatus), From c39f86c1268af862c3043cd6cbeaf7c5d4d3bf7b Mon Sep 17 00:00:00 2001 From: Kamakshee Samant Date: Wed, 10 Jan 2024 22:41:26 +1100 Subject: [PATCH 5/5] chore: backfill status api --- .../subscriptions/backfill-status.test.ts | 188 ------------------ .../routes/subscriptions/backfill-status.ts | 88 ++++---- test/snapshots/app.test.ts.snap | 4 +- 3 files changed, 42 insertions(+), 238 deletions(-) delete mode 100644 src/rest/routes/subscriptions/backfill-status.test.ts diff --git a/src/rest/routes/subscriptions/backfill-status.test.ts b/src/rest/routes/subscriptions/backfill-status.test.ts deleted file mode 100644 index 34cf01746..000000000 --- a/src/rest/routes/subscriptions/backfill-status.test.ts +++ /dev/null @@ -1,188 +0,0 @@ -import { getFrontendApp } from "~/src/app"; -import { Installation } from "models/installation"; -import { encodeSymmetric } from "atlassian-jwt"; -import { getLogger } from "config/logger"; -import { Subscription } from "models/subscription"; -import { DatabaseStateCreator } from "test/utils/database-state-creator"; -import supertest from "supertest"; -import { booleanFlag, BooleanFlags } from "config/feature-flags"; -import { when } from "jest-when"; -import { RepoSyncState } from "models/reposyncstate"; - -jest.mock("config/feature-flags"); - -describe("jira-get-connections-backfillStatus.test", () => { - let app; - let installation: Installation; - let subscription: Subscription; - let repoSyncState: RepoSyncState; - const generateJwt = async () => { - return encodeSymmetric( - { - qsh: "context-qsh", - iss: installation.plainClientKey, - sub: "myAccountId" - }, - await installation.decrypt("encryptedSharedSecret", getLogger("test")) - ); - }; - - beforeEach(async () => { - app = getFrontendApp(); - const result = await new DatabaseStateCreator() - .withActiveRepoSyncState() - .create(); - installation = result.installation; - subscription = result.subscription; - repoSyncState = result.repoSyncState!; - when(booleanFlag) - .calledWith(BooleanFlags.JIRA_ADMIN_CHECK) - .mockResolvedValue(true); - }); - - it("should return 401 when no JWT was provided", async () => { - const resp = await supertest(app).get( - `/rest/subscriptions/backfill-status?subscriptionIds=${subscription.id}` - ); - expect(resp.status).toStrictEqual(401); - expect(resp.text).toBe("Unauthorised"); - }); - - it("should return 403 when not an admin", async () => { - const resp = await supertest(app) - .get( - `/rest/subscriptions/backfill-status?subscriptionIds=${subscription.id}` - ) - .set( - "authorization", - `JWT ${await generateJwt()}` - ); - expect(resp.status).toStrictEqual(403); - }); - - describe("admin and JWT are OK", () => { - beforeEach(() => { - const payload = { - accountId: "myAccountId", - globalPermissions: ["ADMINISTER"] - }; - jiraNock - .post("/rest/api/latest/permissions/check", payload) - .reply(200, { globalPermissions: ["ADMINISTER"] }); - }); - - it("should return 400 when no subscriptions were found", async () => { - const resp = await supertest(app) - .get( - `/rest/subscriptions/backfill-status?subscriptionIds=${ - subscription.id + 1 - }` - ) - .set( - "authorization", - `JWT ${await generateJwt()}` - ); - expect(resp.status).toStrictEqual(400); - }); - - it("should return 400 when no Missing Subscription IDs were found in query", async () => { - const resp = await supertest(app) - .get(`/rest/subscriptions/backfill-status`) - .set("authorization", `JWT ${await generateJwt()}`); - expect(resp.status).toStrictEqual(400); - expect(resp.text).toBe("Missing Subscription IDs"); - }); - - it("should return 403 if the subscription belongs to a different user", async () => { - const result = await new DatabaseStateCreator() - .forJiraHost("https://another-one.atlassian.net") - .create(); - const resp = await supertest(app) - .get( - `/rest/subscriptions/backfill-status?subscriptionIds=${result.subscription.id}` - ) - .set( - "authorization", - `JWT ${await generateJwt()}` - ); - - expect(resp.status).toStrictEqual(403); - }); - - it("should return 200 if the subscription belongs to the same user", async () => { - const resp = await supertest(app) - .get( - `/rest/subscriptions/backfill-status?subscriptionIds=${subscription.id}` - ) - .set( - "authorization", - `JWT ${await generateJwt()}` - ); - expect(resp.status).toStrictEqual(200); - }); - - describe("happy paths", () => { - beforeEach(async () => { - const newRepoSyncStatesData: any[] = []; - for (let newRepoStateNo = 1; newRepoStateNo < 50; newRepoStateNo++) { - const newRepoSyncState = { ...repoSyncState.dataValues }; - newRepoSyncState["repoId"] = repoSyncState.repoId + newRepoStateNo; - newRepoSyncState["repoName"] = - repoSyncState.repoName + newRepoStateNo.toString(); - newRepoSyncState["repoFullName"] = - repoSyncState.repoFullName + - String(newRepoStateNo).padStart(3, "0"); - if (newRepoStateNo === 1) { - newRepoSyncState["commitStatus"] = "pending"; - newRepoSyncState["branchStatus"] = "complete"; - newRepoSyncState["pullStatus"] = "complete"; - newRepoSyncState["buildStatus"] = "complete"; - newRepoSyncState["deploymentStatus"] = "pending"; - } else if (newRepoStateNo % 3 == 1) { - newRepoSyncState["commitStatus"] = "complete"; - newRepoSyncState["branchStatus"] = "complete"; - newRepoSyncState["pullStatus"] = "complete"; - newRepoSyncState["buildStatus"] = "complete"; - newRepoSyncState["deploymentStatus"] = "complete"; - } else if (newRepoStateNo % 3 == 2) { - newRepoSyncState["commitStatus"] = "failed"; - newRepoSyncState["branchStatus"] = "complete"; - newRepoSyncState["pullStatus"] = "complete"; - newRepoSyncState["buildStatus"] = "complete"; - newRepoSyncState["deploymentStatus"] = "failed"; - } - newRepoSyncStatesData.push(newRepoSyncState); - } - await RepoSyncState.bulkCreate(newRepoSyncStatesData); - }); - - it("should return 200 if the subscription belongs to the same user", async () => { - const resp = await supertest(app) - .get( - `/rest/subscriptions/backfill-status?subscriptionIds=${subscription.id}` - ) - .set( - "authorization", - `JWT ${await generateJwt()}` - ); - expect(resp.status).toStrictEqual(200); - - expect(resp.body).toMatchObject({ - data: { - subscriptions: { - [subscription.id]: { - isSyncComplete: false, - syncStatus: "IN PROGRESS", - totalRepos: 33, - syncedRepos: 17, - backfillSince: null - } - }, - isBackfillComplete: false, - subscriptionIds: [subscription.id] - } - }); - }); - }); - }); -}); \ No newline at end of file diff --git a/src/rest/routes/subscriptions/backfill-status.ts b/src/rest/routes/subscriptions/backfill-status.ts index 465b1d386..029780481 100644 --- a/src/rest/routes/subscriptions/backfill-status.ts +++ b/src/rest/routes/subscriptions/backfill-status.ts @@ -13,63 +13,55 @@ import { BackfillStatusError } from "../../../../spa/src/rest-interfaces"; import { BaseLocals } from ".."; +import { InsufficientPermissionError, InvalidArgumentError } from "~/src/config/errors"; const GetSubscriptionsBackfillStatus = async (req: Request, res: Response) => { - try { - const { jiraHost: localJiraHost } = res.locals; - const { subscriptionIds } = req.query; + const { jiraHost: localJiraHost } = res.locals; + const { subscriptionIds } = req.query; - const subIds = String(subscriptionIds) - .split(",") - .map(Number) - .filter(Boolean); - if (subIds.length === 0) { - req.log.warn("Missing Subscription IDs"); - res.status(400).send("Missing Subscription IDs"); - return; - } + const subIds = String(subscriptionIds) + .split(",") + .map(Number) + .filter(Boolean); + if (subIds.length === 0) { + req.log.warn("Missing Subscription IDs"); + throw new InvalidArgumentError("Missing Subscription IDs"); + } - const subscriptions = await Subscription.findAllForSubscriptionIds( - subIds - ); + const subscriptions = await Subscription.findAllForSubscriptionIds( + subIds + ); - const resultSubscriptionIds: Array = subscriptions.map( - (subscription) => subscription.id - ); + const resultSubscriptionIds: Array = subscriptions.map( + (subscription) => subscription.id + ); - if (subscriptions.length === 0) { - req.log.error("Missing Subscription"); - res.status(400).send("Missing Subscription"); - return; - } + if (subscriptions.length === 0) { + req.log.error("Missing Subscription"); + throw new InvalidArgumentError("Missing Subscription"); + } - const jiraHostsMatched = subscriptions.every( - (subscription) => subscription.jiraHost === localJiraHost - ); + const jiraHostsMatched = subscriptions.every( + (subscription) => subscription.jiraHost === localJiraHost + ); - if (!jiraHostsMatched) { - req.log.error("mismatched Jira Host"); - res.status(403).send("mismatched Jira Host"); - return; - } - const subscriptionsById = groupBy(subscriptions, "id"); - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const { backfillStatus, errors } = await getBackfillStatus( - subscriptionsById - ); - const isBackfillComplete = getBackfillCompletionStatus(backfillStatus); - res.status(200).send({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - subscriptions: backfillStatus, - isBackfillComplete, - subscriptionIds: resultSubscriptionIds, - errors - }); - } catch (error) { - req.log.error( - "Failed to poll repo backfill status for provided subscription ID" - ); + if (!jiraHostsMatched) { + req.log.error("mismatched Jira Host"); + throw new InsufficientPermissionError("mismatched Jira Host"); } + const subscriptionsById = groupBy(subscriptions, "id"); + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const { backfillStatus, errors } = await getBackfillStatus( + subscriptionsById + ); + const isBackfillComplete = getBackfillCompletionStatus(backfillStatus); + res.status(200).send({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + subscriptions: backfillStatus, + isBackfillComplete, + subscriptionIds: resultSubscriptionIds, + errors + }); }; const getBackfillCompletionStatus = (backfillStatus: BackFillType): boolean => diff --git a/test/snapshots/app.test.ts.snap b/test/snapshots/app.test.ts.snap index 574e2e21e..95d6fe285 100644 --- a/test/snapshots/app.test.ts.snap +++ b/test/snapshots/app.test.ts.snap @@ -16,7 +16,7 @@ exports[`app getFrontendApp please review routes and update snapshot when adding :GET ^/?(?=/|$)^/rest/?(?=/|$)^/subscriptions/?(?=/|$)^/?$ query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,JwtHandler,jiraAdminEnforceMiddleware,SubscriptionsGet :GET ^/?(?=/|$)^/rest/?(?=/|$)^/subscriptions/?(?=/|$)^/backfill-status/?$ - query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,JwtHandler,jiraAdminEnforceMiddleware,SyncRouterHandler + query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,JwtHandler,jiraAdminEnforceMiddleware,GetSubBackfillStatusHandler :DELETE ^/?(?=/|$)^/rest/?(?=/|$)^/subscriptions/?(?=/|$)^/?$ query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,JwtHandler,jiraAdminEnforceMiddleware,SubscriptionDelete :POST ^/?(?=/|$)^/rest/?(?=/|$)^/subscriptions/?(?=/|$)^/sync/?$ @@ -54,7 +54,7 @@ exports[`app getFrontendApp please review routes and update snapshot when adding :GET ^/?(?=/|$)^/rest/?(?=/|$)^/app/(?:([^/]+?))/?(?=/|$)^/subscriptions/(?:([^/]+?))/?(?=/|$)^/?$ query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,JwtHandler,jiraAdminEnforceMiddleware,tempReplaceUUID,GithubServerAppMiddleware,SubscriptionsGet :GET ^/?(?=/|$)^/rest/?(?=/|$)^/app/(?:([^/]+?))/?(?=/|$)^/subscriptions/(?:([^/]+?))/?(?=/|$)^/backfill-status/?$ - query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,JwtHandler,jiraAdminEnforceMiddleware,tempReplaceUUID,GithubServerAppMiddleware,SyncRouterHandler + query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,JwtHandler,jiraAdminEnforceMiddleware,tempReplaceUUID,GithubServerAppMiddleware,GetSubBackfillStatusHandler :DELETE ^/?(?=/|$)^/rest/?(?=/|$)^/app/(?:([^/]+?))/?(?=/|$)^/subscriptions/(?:([^/]+?))/?(?=/|$)^/?$ query,expressInit,elapsedTimeMetrics,sentryRequestMiddleware,urlencodedParser,jsonParser,cookieParser,LogMiddleware,JwtHandler,jiraAdminEnforceMiddleware,tempReplaceUUID,GithubServerAppMiddleware,SubscriptionDelete :POST ^/?(?=/|$)^/rest/?(?=/|$)^/app/(?:([^/]+?))/?(?=/|$)^/subscriptions/(?:([^/]+?))/?(?=/|$)^/sync/?$