Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Migrate Grades v2 to Next.js #795

Draft
wants to merge 23 commits into
base: main
Choose a base branch
from
Draft
Changes from 1 commit
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
ccb859a
Bootstrap rust application
junlarsen Feb 9, 2024
3d13526
Implement crawling of all faculties
junlarsen Feb 9, 2024
7c69bfe
Implement synchronization of departments
junlarsen Feb 9, 2024
dd78366
Chunk jobs across multiple threads
junlarsen Feb 9, 2024
b996bb5
Ship MVP version of the Rust syncer
junlarsen Feb 9, 2024
fedb20b
Fix group by chunking
junlarsen Feb 10, 2024
8f623bf
Replace theading with futures join_all
junlarsen Feb 10, 2024
869f99d
Synchronize everything since 2004
junlarsen Feb 10, 2024
7588bec
Bootstrap Next.js application
junlarsen Feb 7, 2024
63bfc1c
Implement hkdir query for subjects and departments
junlarsen Feb 7, 2024
4ce4b11
Implement fetcing subject grdaes
junlarsen Feb 7, 2024
0dd8efa
Rename trpc to server
junlarsen Feb 7, 2024
e5de81e
Populate faculty names into database
junlarsen Feb 7, 2024
d72033b
Refactor faculty population into job service
junlarsen Feb 7, 2024
87d02ef
Add logging to subject sync job
junlarsen Feb 7, 2024
2929ef6
Speed up subject synchronization with async priority queue
junlarsen Feb 7, 2024
5b03125
Fetch departments in parallel as well
junlarsen Feb 7, 2024
f7c9d75
Prevent duplicate write on subject grades
junlarsen Feb 8, 2024
185621a
Query most popular or filtered subjects
junlarsen Feb 8, 2024
ad52270
Build average grade for subject while populating
junlarsen Feb 8, 2024
ea0f1ff
Short circuit grades without any registrations
junlarsen Feb 8, 2024
63b1903
Fix parameters
junlarsen Feb 9, 2024
332cbdb
Ship Rust synchronizer
junlarsen Feb 10, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Ship MVP version of the Rust syncer
junlarsen committed Feb 9, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
commit b996bb5784e24a5eb1d8b8ba0137dcc4878e2499
25 changes: 25 additions & 0 deletions apps/grades-sync/migrations/20240209212106_grades.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
START TRANSACTION;

CREATE TYPE subject_grading_season AS ENUM ('WINTER', 'SPRING', 'SUMMER', 'AUTUMN');

CREATE TABLE IF NOT EXISTS subject_season_grade
(
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
subject_id UUID NOT NULL,
season subject_grading_season NOT NULL,
year INTEGER NOT NULL,

graded_a INTEGER,
graded_b INTEGER,
graded_c INTEGER,
graded_d INTEGER,
graded_e INTEGER,
graded_f INTEGER,
graded_pass INTEGER,
graded_fail INTEGER,

CONSTRAINT subject_season_grade_fk_subject_id FOREIGN KEY (subject_id) REFERENCES subject (id),
CONSTRAINT subject_season_grade_unique UNIQUE (subject_id, season, year)
);

COMMIT;
39 changes: 39 additions & 0 deletions apps/grades-sync/queries/departments.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
{
"tabell_id": 210,
"api_versjon": 1,
"statuslinje": "N",
"kodetekst": "J",
"desimal_separator": ".",
"variabler": [
"*"
],
"sortBy": [
"Nivå"
],
"filter": [
{
"variabel": "Institusjonskode",
"selection": {
"filter": "item",
"values": [
"1150"
],
"exclude": [
""
]
}
},
{
"variabel": "Avdelingskode",
"selection": {
"filter": "all",
"values": [
"*"
],
"exclude": [
"000000"
]
}
}
]
}
68 changes: 68 additions & 0 deletions apps/grades-sync/queries/grades.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
{
"tabell_id": 308,
"api_versjon": 1,
"statuslinje": "N",
"kodetekst": "J",
"desimal_separator": ".",
"variabler": [
"*"
],
"groupBy": [
"Årstall",
"Semester",
"Karakter",
"Emnekode",
"Institusjonskode"
],
"filter": [
{
"variabel": "Institusjonskode",
"selection": {
"filter": "item",
"values": [
"1150"
],
"exclude": [
""
]
}
},
{
"variabel": "Emnekode",
"selection": {
"filter": "all",
"values": [
"*"
],
"exclude": [
""
]
}
},
{
"variabel": "Semester",
"selection": {
"filter": "all",
"values": [
"*"
],
"exclude": [
""
]
}
},

{
"variabel": "Årstall",
"selection": {
"filter": "top",
"values": [
"2"
],
"exclude": [
""
]
}
}
]
}
92 changes: 92 additions & 0 deletions apps/grades-sync/queries/subjects.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
{
"tabell_id": 208,
"api_versjon": 1,
"statuslinje": "N",
"kodetekst": "J",
"desimal_separator": ".",
"variabler": [
"*"
],
"sortBy": [
"Årstall",
"Institusjonskode",
"Avdelingskode"
],
"filter": [
{
"variabel": "Institusjonskode",
"selection": {
"filter": "item",
"values": [
"1150"
],
"exclude": [
""
]
}
},
{
"variabel": "Nivåkode",
"selection": {
"filter": "item",
"values": [
"HN",
"LN"
],
"exclude": [
""
]
}
},
{
"variabel": "Status",
"selection": {
"filter": "item",
"values": [
"1",
"2"
],
"exclude": [
""
]
}
},
{
"variabel": "Avdelingskode",
"selection": {
"filter": "all",
"values": [
"*"
],
"exclude": [
"000000"
]
}
},
{
"variabel": "Oppgave (ny fra h2012)",
"selection": {
"filter": "all",
"values": [
"*"
],
"exclude": [
"1",
"2"
]
}
},
{
"variabel": "Årstall",
"selection": {
"filter": "top",
"values": [
"2"
],
"exclude": [
""
]
}
}
]
}
Loading