Improve previous sessions list performance (#298)

.github/workflows/alpha.yml

@@ -2,7 +2,7 @@ name: 'Alpha Build'
 
 on:
   push:
-    branches: [v470/slack-bot]
+    branches: [v470/prev-sessions-view]
 
 jobs:
   build:

backend/src/db/actions/sessions.ts

@@ -5,15 +5,14 @@ import {
   ColumnDefinitionEntity,
   SessionEntity,
   SessionTemplateEntity,
+  SessionView,
 } from '../entities';
 import {
   Session,
   defaultSession,
   ColumnDefinition,
   SessionOptions,
   SessionMetadata,
-  VoteType,
-  User,
   AccessErrorType,
   FullUser,
 } from '@retrospected/common';
@@ -27,11 +26,10 @@ import {
   PostGroupRepository,
   ColumnRepository,
 } from '../repositories';
-import { orderBy } from 'lodash';
 import { transaction } from './transaction';
-import { EntityManager } from 'typeorm';
+import { EntityManager, In } from 'typeorm';
 import { getUserViewInner, isUserPro } from './users';
-import { ALL_FIELDS } from '../entities/User';
+import { uniq } from 'lodash';
 
 export async function createSessionFromSlack(
   slackUserId: string,
@@ -243,65 +241,36 @@ export async function deleteSessions(
   });
 }
 
-function numberOfVotes(type: VoteType, session: SessionEntity) {
-  if (!session.posts) {
-    return 0;
-  }
-  return session.posts.reduce<number>((prev, cur) => {
-    return cur.votes
-      ? prev + cur.votes.filter((v) => v.type === type).length
-      : prev;
-  }, 0);
-}
-
-function numberOfActions(posts: PostEntity[] | undefined) {
-  if (!posts) {
-    return 0;
-  }
-  return posts.filter((p) => p.action !== null).length;
-}
-
-function getParticipants(visitors: UserEntity[] | undefined): User[] {
-  if (!visitors) {
-    return [];
-  }
-  return visitors.map((u) => u.toJson());
-}
-
 export async function previousSessions(
   userId: string
 ): Promise<SessionMetadata[]> {
   return await transaction(async (manager) => {
-    const userRepository = manager.getCustomRepository(UserRepository);
-    const loadedUser = await userRepository.findOne(userId, {
-      relations: ['sessions', 'sessions.posts', 'sessions.visitors'],
-      select: ALL_FIELDS,
-    });
-    if (loadedUser && loadedUser.sessions) {
-      return orderBy(loadedUser.sessions, (s) => s.updated, 'desc').map(
-        (session) =>
-          ({
-            created: session.created,
-            createdBy: session.createdBy.toJson(),
-            encrypted: session.encrypted,
-            id: session.id,
-            name: session.name,
-            numberOfNegativeVotes: numberOfVotes('dislike', session),
-            numberOfPositiveVotes: numberOfVotes('like', session),
-            numberOfPosts: session.posts?.length,
-            numberOfActions: numberOfActions(session.posts),
-            locked: session.locked,
-            lockedForUser:
-              session.locked && session.visitors
-                ? !session.visitors.map((v) => v.id).includes(userId)
-                : false,
-            participants: getParticipants(session.visitors),
-            canBeDeleted: userId === session.createdBy.id,
-          } as SessionMetadata)
-      );
-    }
+    const sessionsAsVisitors: { sessionsId: string }[] = await manager.query(
+      `
+      select distinct v."sessionsId" from visitors v
+      where v."usersId" = $1
+    `,
+      [userId]
+    );
 
-    return [];
+    const sessionsAsOwner: { id: string }[] = await manager.query(
+      `
+      select s.id from sessions s
+      where s."createdById" = $1
+    `,
+      [userId]
+    );
+
+    const ids = uniq([
+      ...sessionsAsVisitors.map((s) => s.sessionsId),
+      ...sessionsAsOwner.map((s) => s.id),
+    ]);
+
+    const sessionViewRepository = manager.getRepository(SessionView);
+    const sessions = await sessionViewRepository.find({
+      where: { id: In(ids) },
+    });
+    return sessions.map((s) => s.toJson(userId));
   });
 }
 

backend/src/db/entities/Post.ts

@@ -20,6 +20,7 @@ export default class PostEntity {
   @PrimaryColumn({ primary: true, generated: false, unique: true })
   public id: string;
   @ManyToOne(() => SessionEntity, { nullable: false })
+  @Index()
   public session: SessionEntity;
   @ManyToOne(() => PostGroupEntity, {
     nullable: true,
@@ -28,6 +29,7 @@ export default class PostEntity {
     onDelete: 'SET NULL',
     onUpdate: 'CASCADE',
   })
+  @Index()
   public group: PostGroupEntity | null;
   @Column({ default: 0 })
   public column: number;
@@ -41,6 +43,7 @@ export default class PostEntity {
   @Column({ nullable: true, type: 'character varying' })
   public giphy: null | string;
   @ManyToOne(() => UserEntity, { eager: true, cascade: true, nullable: false })
+  @Index()
   public user: UserEntity;
   @OneToMany(() => VoteEntity, (vote) => vote.post, {
     cascade: true,

backend/src/db/entities/PostGroup.ts

@@ -19,6 +19,7 @@ export default class PostGroupEntity {
   @PrimaryColumn({ primary: true, generated: false, unique: true })
   public id: string;
   @ManyToOne(() => SessionEntity, { nullable: false })
+  @Index()
   public session: SessionEntity;
   @Column({ default: 0 })
   public column: number;
@@ -28,6 +29,7 @@ export default class PostGroupEntity {
   @Column()
   public label: string;
   @ManyToOne(() => UserEntity, { eager: true, cascade: true, nullable: false })
+  @Index()
   public user: UserEntity;
   @OneToMany(() => PostEntity, (post) => post.session, {
     cascade: true,

backend/src/db/entities/Session.ts

@@ -28,6 +28,7 @@ export default class SessionEntity {
   @Index()
   public name: string | null;
   @ManyToOne(() => UserEntity, { eager: true, cascade: true, nullable: false })
+  @Index()
   public createdBy: UserEntity;
   @OneToMany(() => PostEntity, (post) => post.session, {
     cascade: true,

backend/src/db/entities/SessionTemplate.ts

@@ -24,6 +24,7 @@ export default class SessionTemplateEntity {
   @Index()
   public name: string | null;
   @ManyToOne(() => UserEntity, { eager: true, cascade: true, nullable: false })
+  @Index()
   public createdBy: UserEntity;
   @OneToMany(
     () => TemplateColumnDefinitionEntity,

backend/src/db/entities/SessionView.ts

@@ -0,0 +1,84 @@
+import { ViewEntity, ViewColumn } from 'typeorm';
+import { User, SessionMetadata } from '@retrospected/common';
+
+@ViewEntity({
+  expression: `
+select 
+	s.id,
+	s.name,
+	s.created,
+	(
+		select to_jsonb(cb) from (
+			select cbu.id, cbu.name, cbu.photo from users cbu
+			where cbu.id = s."createdById"
+		) as cb
+	) as "createdBy",
+	s.encrypted,
+	s.locked,
+	(select count(*) from posts p where p."sessionId" = s.id and p.action is not null) as "numberOfActions",	
+	(select count(*) from posts p where p."sessionId" = s.id) as "numberOfPosts",
+	(
+		select count(*) from votes vv
+		left join posts vp on vp.id = vv."postId"
+		where vp."sessionId" = s.id
+	) as "numberOfVotes",
+	(
+		select json_agg(vis) from (
+			select vu.id, vu.name, vu.photo from visitors v
+			join users vu on vu.id = v."usersId"
+			where v."sessionsId" = s.id
+		) as vis
+	) as participants
+
+from sessions s
+left join users u on s."createdById" = u.id
+
+order by s.updated desc
+  `,
+})
+export default class SessionView {
+  @ViewColumn()
+  public id: string;
+  @ViewColumn()
+  public name: string;
+  @ViewColumn()
+  public created: Date;
+  @ViewColumn()
+  public createdBy: User;
+  @ViewColumn()
+  public encrypted: string | null;
+  @ViewColumn()
+  public locked: boolean;
+  @ViewColumn()
+  public numberOfActions: number;
+  @ViewColumn()
+  public numberOfPosts: number;
+  @ViewColumn()
+  public numberOfVotes: number;
+  @ViewColumn()
+  public participants: User[];
+
+  constructor(id: string, name: string) {
+    this.id = id;
+    this.name = name;
+    this.created = new Date();
+    this.createdBy = { id: '0', name: '', photo: '' };
+    this.encrypted = null;
+    this.locked = false;
+    this.numberOfActions = 0;
+    this.numberOfPosts = 0;
+    this.numberOfVotes = 0;
+    this.participants = [];
+  }
+
+  toJson(userId: string): SessionMetadata {
+    return {
+      ...this,
+      canBeDeleted: userId === this.createdBy.id,
+      lockedForUser:
+        this.locked && this.participants
+          ? !this.participants.map((v) => v.id).includes(userId)
+          : false,
+    };
+  }
+}

backend/src/db/entities/Subscription.ts

@@ -5,6 +5,7 @@ import {
   CreateDateColumn,
   UpdateDateColumn,
   ManyToOne,
+  Index,
 } from 'typeorm';
 import UserEntity from './User';
 import { Plan } from '@retrospected/common';
@@ -18,6 +19,7 @@ export default class SubscriptionEntity {
   @Column({ nullable: false, type: 'character varying' })
   public plan: Plan;
   @ManyToOne(() => UserEntity, { eager: true, cascade: true, nullable: false })
+  @Index()
   public owner: UserEntity;
   @Column({ nullable: true, type: 'character varying' })
   public domain: string | null;

backend/src/db/entities/User.ts

@@ -49,6 +49,7 @@ export default class UserEntity {
   @Column({ nullable: false, type: 'character varying', default: 'en' })
   public language: string;
   @ManyToOne(() => SessionTemplateEntity, { nullable: true, eager: false })
+  @Index()
   public defaultTemplate: SessionTemplateEntity | null | undefined;
   @ManyToMany(() => SessionEntity, (session) => session.visitors, {
     eager: false,

backend/src/db/entities/UserIdentity.ts

@@ -33,6 +33,7 @@ export default class UserIdentityEntity {
     onDelete: 'CASCADE',
     onUpdate: 'CASCADE',
   })
+  @Index()
   public user: UserEntity;
   @Column({ default: 'anonymous' })
   public accountType: AccountType;

backend/src/db/entities/Vote.ts

@@ -5,6 +5,7 @@ import {
   ManyToOne,
   CreateDateColumn,
   UpdateDateColumn,
+  Index,
 } from 'typeorm';
 import { VoteType, Vote, VoteExtract } from '@retrospected/common';
 import UserEntity from './User';
@@ -15,13 +16,15 @@ export default class VoteEntity {
   @PrimaryColumn({ primary: true, generated: false, unique: true })
   public id: string;
   @ManyToOne(() => UserEntity, { eager: true, nullable: false })
+  @Index()
   public user: UserEntity;
   @ManyToOne(() => PostEntity, {
     eager: false,
     nullable: false,
     onDelete: 'CASCADE',
     onUpdate: 'CASCADE',
   })
+  @Index()
   public post: PostEntity;
   @Column({ type: 'character varying' })
   public type: VoteType;

backend/src/db/entities/index.ts

@@ -1,6 +1,7 @@
 export { default as PostEntity } from './Post';
 export { default as PostGroupEntity } from './PostGroup';
 export { default as SessionEntity } from './Session';
+export { default as SessionView } from './SessionView';
 export { default as UserEntity } from './User';
 export {
   ColumnDefinitionEntity,

backend/src/db/migrations/1632066879336-IndexPost.ts

@@ -0,0 +1,14 @@
+import {MigrationInterface, QueryRunner} from "typeorm";
+
+export class IndexPost1632066879336 implements MigrationInterface {
+    name = 'IndexPost1632066879336'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`CREATE INDEX "IDX_764b665f832e28c01595ec15cf" ON "public"."posts" ("sessionId") `);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP INDEX "public"."IDX_764b665f832e28c01595ec15cf"`);
+    }
+
+}

backend/src/db/migrations/1632067126663-IndexEverything.ts

@@ -0,0 +1,34 @@
+import {MigrationInterface, QueryRunner} from "typeorm";
+
+export class IndexEverything1632067126663 implements MigrationInterface {
+    name = 'IndexEverything1632067126663'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`CREATE INDEX "IDX_9ad140fba0b4ad9559e6530282" ON "public"."groups" ("sessionId") `);
+        await queryRunner.query(`CREATE INDEX "IDX_898cf6af34722df13f760cc364" ON "public"."groups" ("userId") `);
+        await queryRunner.query(`CREATE INDEX "IDX_5169384e31d0989699a318f3ca" ON "public"."votes" ("userId") `);
+        await queryRunner.query(`CREATE INDEX "IDX_b5b05adc89dda0614276a13a59" ON "public"."votes" ("postId") `);
+        await queryRunner.query(`CREATE INDEX "IDX_c58b12b1a7b4012bb238bc2654" ON "public"."templates" ("createdById") `);
+        await queryRunner.query(`CREATE INDEX "IDX_331400b3a08ee505d42ddba1db" ON "public"."subscriptions" ("ownerId") `);
+        await queryRunner.query(`CREATE INDEX "IDX_a10dca6aa6bda5865287bf2792" ON "public"."users_identities" ("userId") `);
+        await queryRunner.query(`CREATE INDEX "IDX_5b086b03bb64304390cec7635e" ON "public"."users" ("defaultTemplateId") `);
+        await queryRunner.query(`CREATE INDEX "IDX_d10acbe503da4c56853181efc9" ON "public"."posts" ("groupId") `);
+        await queryRunner.query(`CREATE INDEX "IDX_ae05faaa55c866130abef6e1fe" ON "public"."posts" ("userId") `);
+        await queryRunner.query(`CREATE INDEX "IDX_d26fe2e6102cd9c47650a0d7a6" ON "public"."sessions" ("createdById") `);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP INDEX "public"."IDX_d26fe2e6102cd9c47650a0d7a6"`);
+        await queryRunner.query(`DROP INDEX "public"."IDX_ae05faaa55c866130abef6e1fe"`);
+        await queryRunner.query(`DROP INDEX "public"."IDX_d10acbe503da4c56853181efc9"`);
+        await queryRunner.query(`DROP INDEX "public"."IDX_5b086b03bb64304390cec7635e"`);
+        await queryRunner.query(`DROP INDEX "public"."IDX_a10dca6aa6bda5865287bf2792"`);
+        await queryRunner.query(`DROP INDEX "public"."IDX_331400b3a08ee505d42ddba1db"`);
+        await queryRunner.query(`DROP INDEX "public"."IDX_c58b12b1a7b4012bb238bc2654"`);
+        await queryRunner.query(`DROP INDEX "public"."IDX_b5b05adc89dda0614276a13a59"`);
+        await queryRunner.query(`DROP INDEX "public"."IDX_5169384e31d0989699a318f3ca"`);
+        await queryRunner.query(`DROP INDEX "public"."IDX_898cf6af34722df13f760cc364"`);
+        await queryRunner.query(`DROP INDEX "public"."IDX_9ad140fba0b4ad9559e6530282"`);
+    }
+
+}