Skip to content

Commit

Permalink
Merge pull request #46 from Shinsina/seed-via-json
Browse files Browse the repository at this point in the history 
Seed via JSON, remove multi-class fields from sessions where not applicable, correct sorting of subsessions on season page
  • Loading branch information
@Shinsina
Shinsina authored May 3, 2024
2 parents 311c5d7 + f5eca3b commit c5d4ef7
Show file tree
Hide file tree
Showing 16 changed files with 4,117 additions and 4,363 deletions.
2 changes: 0 additions & 2 deletions .github/workflows/deploy.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,6 @@ jobs:
- name: Install, build, and upload your site
env:
ASTRO_STUDIO_APP_TOKEN: ${{secrets.ASTRO_STUDIO_APP_TOKEN }}
MONGODB_URI: ${{ secrets.MONGODB_URI }}
MONGODB_DB: ${{ secrets.MONGODB_DB }}
uses: withastro/action@v0
with:
# path: . # The root location of your Astro project inside the repository. (optional)
Expand Down
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,6 @@ pnpm-debug.log*

# macOS-specific files
.DS_Store

# Database seed files
db/*.json
2 changes: 1 addition & 1 deletion db/config.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ const Standing = defineTable({
country: column.text(),
season_id: column.number({ references: () => PastSeason.columns.season_id }),
season_name: column.text(),
division_rank: column.number(),
division_rank: column.number({ default: 0 }),
},
});

Expand Down
175 changes: 49 additions & 126 deletions db/seed.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,36 +13,18 @@ import {
User,
} from "astro:db";
import type { Subsession as SubsessionType } from "$lib/types";
import { connectToDatabase } from "$lib/mongodb";
import { Collection } from "mongodb";
import type { Document } from "mongodb";
import users from "./users.json";
import cars from "./cars.json";
import carClasses from "./car-classes.json";
import seasons from "./seasons.json";
import pastSeasons from "./past-seasons-output.json";
import jackStandings from "./jack-standings-output.json";
import jakeStandings from "./jake-standings-output.json";
import seasonIdArray from "./distinct-season-ids.json";
import subsessions from "./subsessions-output.json";

async function paginatedQuery(
collection: Collection<Document>
): Promise<Array<Document>> {
const size = 100;
const count = await collection.countDocuments({});
const batches = Math.ceil(count / size);
const results = await Promise.all(
Array(batches)
.fill({})
.map((_, index) => {
console.log(
`Processing batch ${index + 1} of ${batches} for ${
collection.collectionName
}...`
);
return collection
.find({})
.skip(index * size)
.limit(size)
.toArray();
})
);
return results.flat();
}
function batchInserts(array: Array<any>): Array<Array<any>> {
const size = 2000;
const size = 200;
const count = array.length;
const batches = Math.ceil(count / size);
return Array(batches)
Expand All @@ -54,18 +36,20 @@ function batchInserts(array: Array<any>): Array<Array<any>> {

// https://astro.build/db/seed
export default async function seed() {
console.log("Attempting database connection...");
const dbConnection = await connectToDatabase();
const mongoDb = dbConnection.db;
console.log("Connection made!");
const userCollection = mongoDb.collection("users");
const users = await userCollection.find({}).toArray();
console.log("Seeding Users...");
await db.delete(SubsessionRaceResults);
await db.delete(SubsessionQualifyingResults);
await db.delete(SubsessionPracticeResults);
await db.delete(Subsession);
await db.delete(Standing);
await db.delete(PastSeason);
await db.delete(Season);
await db.delete(CarClass);
await db.delete(Car);
await db.delete(User);
console.log("Seeding Users...");
await db.insert(User).values(
users.map((user: any) => {
const { _id, last_login, read_comp_rules, read_pp, read_tc, ...rest } =
user;
const { last_login, read_comp_rules, read_pp, read_tc, ...rest } = user;
return {
...rest,
last_login: new Date(last_login),
Expand All @@ -76,13 +60,10 @@ export default async function seed() {
})
);
console.log("Users Seeded!");
const carCollection = mongoDb.collection("cars");
const cars = await carCollection.find({}).toArray();
console.log("Seeding Cars...");
await db.delete(Car);
await db.insert(Car).values(
cars.map((car: any) => {
const { _id, created, first_sale, ...rest } = car;
const { created, first_sale, ...rest } = car;
return {
...rest,
created: new Date(created),
Expand All @@ -91,57 +72,38 @@ export default async function seed() {
})
);
console.log("Cars Seeded!");
const carClassCollection = mongoDb.collection("carclasses");
const carClasses = await paginatedQuery(carClassCollection);
console.log("Seeding Car Classes...");
await db.delete(CarClass);
await db.insert(CarClass).values(
carClasses.map((carClass: any) => {
const { _id, ...rest } = carClass;
return rest;
})
);
await db.insert(CarClass).values(carClasses);
console.log("Car Classes Seeded!");
const seasonsCollection = mongoDb.collection("seasons");
const seasons = await paginatedQuery(seasonsCollection);
console.log("Seeding Seasons...");
await db.delete(Season);
await db.insert(Season).values(
seasons.map((season: any) => {
const { _id, start_date, ...rest } = season;
const { start_date, ...rest } = season;
return {
...rest,
start_date: new Date(start_date),
};
})
);
console.log("Seeded Seasons!");
const pastSeasonsCollection = mongoDb.collection("pastseasons");
const pastSeasons = await paginatedQuery(pastSeasonsCollection);
console.log("Seeding Past Seasons...");
await db.delete(PastSeason);
await db.insert(PastSeason).values(
pastSeasons.map((pastSeason: any) => {
const { _id, ...rest } = pastSeason;
return rest;
})
Object.keys(pastSeasons)
.map((key) =>
pastSeasons[key].series.seasons.filter((season) =>
seasonIdArray.includes(season.season_id)
)
)
.flat()
);
console.log("Past Seasons Seeded!");
const standingsCollection = mongoDb.collection("standings");
const standings = await paginatedQuery(standingsCollection);
const standings = [...jackStandings, ...jakeStandings];
console.log("Seeding Standings...");
await db.delete(Standing);
await db.insert(Standing).values(
standings.map((standing: any) => {
const {
_id: id,
car_class_id,
season_driver_data,
season_id,
...rest
} = standing;
const { car_class_id, season_driver_data, season_id, ...rest } = standing;
return {
id,
id: `${season_id}_${car_class_id}_${season_driver_data.cust_id}`,
car_class_id: Number(car_class_id),
season_id: Number(season_id),
...season_driver_data,
Expand All @@ -150,30 +112,25 @@ export default async function seed() {
})
);
console.log("Standings Seeded!");
const subsessionsCollection = mongoDb.collection("subsessions");
// @ts-ignore
const subsessions: Array<SubsessionType> = await paginatedQuery(
subsessionsCollection
);
console.log("Seeding Subsessions...");
const {
allPracticeResults,
allQualifyingResults,
allRaceResults,
allSubsessions,
} = subsessions.reduce(
} = Object.keys(subsessions).reduce(
// @todo Provide an accurate type for the unknown here
(object: Record<string, Array<any>>, subsession) => {
const { _id, session_results, end_time, start_time, ...rest } =
subsession;
(object: Record<string, Array<any>>, key) => {
const { session_results, end_time, start_time, ...rest } =
subsessions[key];
session_results.forEach((sessionResult) => {
const { results, ...rest } = sessionResult;
results.forEach((result) => {
const resultWithSession = {
subsession_id: _id,
subsession_id: key,
...result,
...rest,
id: `${_id}_${rest.simsession_name}_${result.cust_id}`,
id: `${key}_${rest.simsession_name}_${result.cust_id}`,
};
if (rest.simsession_type_name.match("Practice")) {
object.allPracticeResults.push(resultWithSession);
Expand Down Expand Up @@ -219,22 +176,10 @@ export default async function seed() {
);
if (practiceResultCount !== allPracticeResults.length) {
await db.delete(SubsessionPracticeResults);
const practiceResultInserts: Array<any> = [];
allPracticeResults.forEach((result) => {
practiceResultInserts.push(
db.insert(SubsessionPracticeResults).values(result)
);
});
await Promise.all(
batchInserts(practiceResultInserts).map((batch, index) => {
console.log(
`Processing batch ${index + 1} of ${Math.ceil(
practiceResultInserts.length / 2000
)} for practiceResults`
);
// @ts-expect-error
return db.batch(batch);
})
batchInserts(allPracticeResults).map((batch) =>
db.insert(SubsessionPracticeResults).values(batch)
)
);
}
console.log("Subsession Practice Results Seeded!");
Expand All @@ -248,22 +193,10 @@ export default async function seed() {
);
if (qualifyingResultCount !== allQualifyingResults.length) {
await db.delete(SubsessionQualifyingResults);
const qualifyingResultInserts: Array<any> = [];
allQualifyingResults.forEach((result) => {
qualifyingResultInserts.push(
db.insert(SubsessionQualifyingResults).values(result)
);
});
await Promise.all(
batchInserts(qualifyingResultInserts).map((batch, index) => {
console.log(
`Processing batch ${index + 1} of ${Math.ceil(
qualifyingResultInserts.length / 2000
)} for qualifyingResults`
);
// @ts-expect-error
return db.batch(batch);
})
batchInserts(allQualifyingResults).map((batch) =>
db.insert(SubsessionQualifyingResults).values(batch)
)
);
}
console.log("Subsession Qualifying Results Seeded!");
Expand All @@ -277,20 +210,10 @@ export default async function seed() {
);
if (raceResultCount !== allRaceResults.length) {
await db.delete(SubsessionRaceResults);
const raceResultInserts: Array<any> = [];
allRaceResults.forEach((result) => {
raceResultInserts.push(db.insert(SubsessionRaceResults).values(result));
});
await Promise.all(
batchInserts(raceResultInserts).map((batch, index) => {
console.log(
`Processing batch ${index + 1} of ${Math.ceil(
raceResultInserts.length / 2000
)} for raceResults`
);
// @ts-expect-error
return db.batch(batch);
})
batchInserts(allRaceResults).map((batch) =>
db.insert(SubsessionRaceResults).values(batch)
)
);
}
console.log("Subsession Race Results Seeded!");
Expand Down
Loading

0 comments on commit c5d4ef7

Please sign in to comment.