From ccb859a7ca361fb71113b81a6a36c5534c667e31 Mon Sep 17 00:00:00 2001 From: Mats Larsen Date: Fri, 9 Feb 2024 20:01:23 +0100 Subject: [PATCH 01/23] Bootstrap rust application --- apps/grades-sync/Cargo.lock | 2312 ++++++++++++++++++++ apps/grades-sync/Cargo.toml | 17 + apps/grades-sync/README.md | 11 + apps/grades-sync/src/faculty_repository.rs | 56 + apps/grades-sync/src/hkdir.rs | 39 + apps/grades-sync/src/job.rs | 52 + apps/grades-sync/src/json.rs | 37 + apps/grades-sync/src/main.rs | 25 + apps/grades-sync/src/pg.rs | 14 + 9 files changed, 2563 insertions(+) create mode 100644 apps/grades-sync/Cargo.lock create mode 100644 apps/grades-sync/Cargo.toml create mode 100644 apps/grades-sync/README.md create mode 100644 apps/grades-sync/src/faculty_repository.rs create mode 100644 apps/grades-sync/src/hkdir.rs create mode 100644 apps/grades-sync/src/job.rs create mode 100644 apps/grades-sync/src/json.rs create mode 100644 apps/grades-sync/src/main.rs create mode 100644 apps/grades-sync/src/pg.rs diff --git a/apps/grades-sync/Cargo.lock b/apps/grades-sync/Cargo.lock new file mode 100644 index 000000000..a3573d56a --- /dev/null +++ b/apps/grades-sync/Cargo.lock @@ -0,0 +1,2312 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "ahash" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + +[[package]] +name = "anstream" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" + +[[package]] +name = "anstyle-parse" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + +[[package]] +name = "anyhow" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-write-file" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edcdbedc2236483ab103a53415653d6b4442ea6141baf1ffa85df29635e88436" +dependencies = [ + "nix", + "rand", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" +dependencies = [ + "serde", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "der" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +dependencies = [ + "serde", +] + +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "env_filter" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05e7cf40684ae96ade6232ed84582f40ce0a66efcd43a5117aef610534f8e0b8" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "finl_unicode" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" + +[[package]] +name = "flume" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +dependencies = [ + "futures-core", + "futures-sink", + "spin 0.9.8", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-executor" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + +[[package]] +name = "futures-task" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" + +[[package]] +name = "futures-util" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +dependencies = [ + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "grades-sync" +version = "0.1.0" +dependencies = [ + "anyhow", + "dotenv", + "env_logger", + "log", + "reqwest", + "serde", + "serde_json", + "sqlx", + "tokio", +] + +[[package]] +name = "h2" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0c62115964e08cb8039170eb33c1d0e2388a256930279edca206fff675f82c3" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "http" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "0.14.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824b2ae422412366ba479e8111fd301f7b5faece8149317bb81925979a53f520" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" + +[[package]] +name = "js-sys" +version = "0.3.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +dependencies = [ + "spin 0.5.2", +] + +[[package]] +name = "libc" +version = "0.2.153" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + +[[package]] +name = "libsqlite3-sys" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" + +[[package]] +name = "lock_api" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.48.0", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nix" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +dependencies = [ + "bitflags 2.4.2", + "cfg-if", + "libc", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "openssl" +version = "0.10.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15c9d69dd87a29568d4d017cfe8ec518706046a05184e5aea92d0af890b803c8" +dependencies = [ + "bitflags 2.4.2", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e1bf214306098e4832460f797824c05d25aacdf896f64a985fb0fd992454ae" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.48.5", +] + +[[package]] +name = "paste" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "regex" +version = "1.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + +[[package]] +name = "reqwest" +version = "0.11.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "ring" +version = "0.17.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "684d5e6e18f669ccebf64a92236bb7db9a34f07be010e3627368182027180866" +dependencies = [ + "cc", + "getrandom", + "libc", + "spin 0.9.8", + "untrusted", + "windows-sys 0.48.0", +] + +[[package]] +name = "rsa" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "rustix" +version = "0.38.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +dependencies = [ + "bitflags 2.4.2", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.21.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" +dependencies = [ + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "ryu" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" + +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.196" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.196" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "serde_json" +version = "1.0.113" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" + +[[package]] +name = "socket2" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlformat" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c" +dependencies = [ + "itertools", + "nom", + "unicode_categories", +] + +[[package]] +name = "sqlx" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dba03c279da73694ef99763320dea58b51095dfe87d001b1d4b5fe78ba8763cf" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d84b0a3c3739e220d94b3239fd69fb1f74bc36e16643423bd99de3b43c21bfbd" +dependencies = [ + "ahash", + "atoi", + "byteorder", + "bytes", + "crc", + "crossbeam-queue", + "dotenvy", + "either", + "event-listener", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashlink", + "hex", + "indexmap", + "log", + "memchr", + "once_cell", + "paste", + "percent-encoding", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlformat", + "thiserror", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid", + "webpki-roots", +] + +[[package]] +name = "sqlx-macros" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89961c00dc4d7dffb7aee214964b065072bff69e36ddb9e2c107541f75e4f2a5" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 1.0.109", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0bd4519486723648186a08785143599760f7cc81c52334a55d6a83ea1e20841" +dependencies = [ + "atomic-write-file", + "dotenvy", + "either", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 1.0.109", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e37195395df71fd068f6e2082247891bc11e3289624bbc776a0cdfa1ca7f1ea4" +dependencies = [ + "atoi", + "base64", + "bitflags 2.4.2", + "byteorder", + "bytes", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ac0ac3b7ccd10cc96c7ab29791a7dd236bd94021f31eec7ba3d46a74aa1c24" +dependencies = [ + "atoi", + "base64", + "bitflags 2.4.2", + "byteorder", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand", + "serde", + "serde_json", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "210976b7d948c7ba9fced8ca835b11cbb2d677c59c79de41ac0d397e14547490" +dependencies = [ + "atoi", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "tracing", + "url", + "urlencoding", + "uuid", +] + +[[package]] +name = "stringprep" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" +dependencies = [ + "finl_unicode", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a365e8cd18e44762ef95d87f284f4b5cd04107fec2ff3052bd6a3e6069669e67" +dependencies = [ + "cfg-if", + "fastrand", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "thiserror" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "num_cpus", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + +[[package]] +name = "uuid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.48", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877b9c3f61ceea0e56331985743b13f3d25c406a7098d45180fb5f09bc19ed97" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" + +[[package]] +name = "web-sys" +version = "0.3.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96565907687f7aceb35bc5fc03770a8a0471d82e479f25832f54a0e3f4b28446" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" + +[[package]] +name = "whoami" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + +[[package]] +name = "zerocopy" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "zeroize" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" diff --git a/apps/grades-sync/Cargo.toml b/apps/grades-sync/Cargo.toml new file mode 100644 index 000000000..1492e8f58 --- /dev/null +++ b/apps/grades-sync/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "grades-sync" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +sqlx = { version = "0.7", features = ["runtime-tokio", "tls-rustls", "uuid", "json", "migrate", "postgres"] } +tokio = { version = "1", features = ["full"] } +reqwest = { version = "0.11", features = ["json"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +anyhow = "1.0.79" +dotenv = "0.15.0" +log = { version = "0.4.20", features = [] } +env_logger = "0.11.1" diff --git a/apps/grades-sync/README.md b/apps/grades-sync/README.md new file mode 100644 index 000000000..d96d63a34 --- /dev/null +++ b/apps/grades-sync/README.md @@ -0,0 +1,11 @@ +# Grades Sync + +Asynchronous grades synchronizer against HKDir's API. + +## Environment variables + +```env +DATABASE_URL=postgres://... +``` + +The `RUST_LOG` variable can be set to `info` or `debug` to enable logging. diff --git a/apps/grades-sync/src/faculty_repository.rs b/apps/grades-sync/src/faculty_repository.rs new file mode 100644 index 000000000..bc09a36cb --- /dev/null +++ b/apps/grades-sync/src/faculty_repository.rs @@ -0,0 +1,56 @@ +use crate::pg::Database; +use sqlx::types::Uuid; +use sqlx::FromRow; + +#[derive(Debug, FromRow)] +pub struct Faculty { + pub id: Uuid, + pub name: String, + pub ref_id: String, +} + +pub trait FacultyRepository: Sync { + async fn create_faculty(&self, name: String, ref_id: String) -> Result + where + Self: Sized; + async fn get_faculty_by_ref_id(&self, ref_id: &str) -> Result, sqlx::Error> + where + Self: Sized; +} + +pub struct FacultyRepositoryImpl<'a> { + db: &'a Database, +} + +impl<'a> FacultyRepositoryImpl<'a> { + pub fn new(db: &'a Database) -> Self { + Self { db } + } +} + +impl<'a> FacultyRepository for FacultyRepositoryImpl<'a> { + async fn create_faculty(&self, name: String, ref_id: String) -> Result { + sqlx::query_as::<_, Faculty>( + r#" + INSERT INTO ntnu_faculty (name, ref_id) VALUES ($1, $2) + ON CONFLICT (ref_id) DO UPDATE SET name = $1, ref_id = $2 + RETURNING ALL + "#, + ) + .bind(name) + .bind(ref_id) + .fetch_one(self.db) + .await + } + + async fn get_faculty_by_ref_id(&self, ref_id: &str) -> Result, sqlx::Error> { + sqlx::query_as::<_, Faculty>( + r#" + SELECT * FROM ntnu_faculty WHERE ref_id = $1 + "#, + ) + .bind(ref_id) + .fetch_optional(self.db) + .await + } +} diff --git a/apps/grades-sync/src/hkdir.rs b/apps/grades-sync/src/hkdir.rs new file mode 100644 index 000000000..44a2ec23e --- /dev/null +++ b/apps/grades-sync/src/hkdir.rs @@ -0,0 +1,39 @@ +use crate::json::HkdirDepartment; +use reqwest::{Client}; +use serde_json::{json, Value}; + +const HKDIR_API_URL: &str = "https://dbh.hkdir.no/api/Tabeller/hentJSONTabellData"; + +fn build_get_departments_request() -> Value { + let json = json!({ + "tabell_id": 210, + "api_versjon": 1, + "statuslinje": "N", + "kodetekst": "J", + "desimal_separator": ".", + "variabler": ["*"], + "sortBy": ["Nivå"], + "filter": [ + { + "variabel": "Institusjonskode", + "selection": { + "filter": "item", + "values": ["1150"], + "exclude": [""], + }, + }, + ], + }); + json +} + +pub async fn get_departments() -> reqwest::Result> { + let request_body = build_get_departments_request(); + let client = Client::new(); + let response = client + .post(HKDIR_API_URL) + .json(&request_body) + .send() + .await?; + response.json::>().await +} diff --git a/apps/grades-sync/src/job.rs b/apps/grades-sync/src/job.rs new file mode 100644 index 000000000..2338009c9 --- /dev/null +++ b/apps/grades-sync/src/job.rs @@ -0,0 +1,52 @@ +use crate::faculty_repository::{FacultyRepository}; +use crate::hkdir::get_departments; + +use log::info; +use tokio::task::JoinSet; + +pub trait JobService { + async fn perform_faculty_synchronization(&self) -> anyhow::Result<()>; + async fn perform_department_synchronization(&self) -> anyhow::Result<()>; + async fn perform_grade_synchronization(&self) -> anyhow::Result<()>; +} + +pub struct JobServiceImpl<'a> { + faculty_repository: &'a dyn FacultyRepository, +} + +impl<'a> JobServiceImpl<'a> { + pub fn new(faculty_repository: &'a dyn FacultyRepository) -> Self { + Self { faculty_repository } + } +} + +impl<'a> JobService for JobServiceImpl<'a> { + async fn perform_faculty_synchronization(&self) -> anyhow::Result<()> { + info!("performing faculty synchronization"); + let departments = get_departments().await?; + let _processed_count = 0; + info!( + "performing synchronization for {} departments", + departments.len() + ); + let mut set = JoinSet::new(); + for (i, _department) in departments.iter().enumerate() { + set.spawn(async move { + info!("processing department {}", i); + }); + } + + while let Some(result) = set.join_next().await { + result?; + } + Ok(()) + } + + async fn perform_department_synchronization(&self) -> anyhow::Result<()> { + Ok(()) + } + + async fn perform_grade_synchronization(&self) -> anyhow::Result<()> { + Ok(()) + } +} diff --git a/apps/grades-sync/src/json.rs b/apps/grades-sync/src/json.rs new file mode 100644 index 000000000..a170f0eb3 --- /dev/null +++ b/apps/grades-sync/src/json.rs @@ -0,0 +1,37 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize)] +pub struct HkdirDepartment { + #[serde(rename = "Nivå")] + pub level: String, + #[serde(rename = "Nivå_tekst")] + pub level_text: String, + #[serde(rename = "Institusjonskode")] + pub institution_code: String, + #[serde(rename = "Institusjonsnavn")] + pub institution_name: String, + #[serde(rename = "Avdelingskode")] + pub department_code: String, + #[serde(rename = "Avdelingsnavn")] + pub department_name: String, + #[serde(rename = "Gyldig_fra")] + pub valid_from: Option, + #[serde(rename = "Gyldig_til")] + pub valid_to: Option, + #[serde(rename = "fagkode_avdeling")] + pub department_subject_code: Option, + #[serde(rename = "fagnavn_avdeling")] + pub department_subject_name: Option, + #[serde(rename = "Fakultetskode")] + pub faculty_code: String, + #[serde(rename = "Fakultetsnavn")] + pub faculty_name: String, + #[serde(rename = "Avdelingskode (3 siste siffer)")] + pub department_code_3: String, +} + +#[derive(Serialize, Deserialize)] +pub struct HkdirSubject {} + +#[derive(Serialize, Deserialize)] +pub struct HkdirGrade {} diff --git a/apps/grades-sync/src/main.rs b/apps/grades-sync/src/main.rs new file mode 100644 index 000000000..2ced7e884 --- /dev/null +++ b/apps/grades-sync/src/main.rs @@ -0,0 +1,25 @@ +use crate::faculty_repository::FacultyRepositoryImpl; +use crate::job::{JobService, JobServiceImpl}; + +mod faculty_repository; +mod hkdir; +mod job; +mod json; +mod pg; + +fn bootstrap_environment() { + dotenv::dotenv().ok(); + env_logger::init() +} + +#[tokio::main] +async fn main() { + bootstrap_environment(); + let connection = pg::create_postgres_pool().await.unwrap(); + let faculty_repository = FacultyRepositoryImpl::new(&connection); + let job_service = JobServiceImpl::new(&faculty_repository); + + job_service.perform_faculty_synchronization().await.unwrap(); + + println!("Hello, world!"); +} diff --git a/apps/grades-sync/src/pg.rs b/apps/grades-sync/src/pg.rs new file mode 100644 index 000000000..7f90e74d2 --- /dev/null +++ b/apps/grades-sync/src/pg.rs @@ -0,0 +1,14 @@ +use sqlx::postgres::PgPoolOptions; +use sqlx::{Pool, Postgres}; + +pub type Database = Pool; + +pub async fn create_postgres_pool() -> Result, sqlx::Error> { + let database_url = + std::env::var("DATABASE_URL").expect("missing DATABASE_URL environment variable"); + let pool = PgPoolOptions::new() + .max_connections(5) + .connect(&database_url) + .await?; + Ok(pool) +} From 3d13526b9bf401d833da4f3a8f07879fd26a57c1 Mon Sep 17 00:00:00 2001 From: Mats Larsen Date: Fri, 9 Feb 2024 20:44:12 +0100 Subject: [PATCH 02/23] Implement crawling of all faculties --- apps/grades-sync/Cargo.lock | 72 +++++++++++++++++++ apps/grades-sync/Cargo.toml | 2 + .../migrations/20240209190449_schema.sql | 25 +++++++ apps/grades-sync/src/faculty_repository.rs | 17 +++-- apps/grades-sync/src/hkdir.rs | 2 +- apps/grades-sync/src/job.rs | 34 +++++---- apps/grades-sync/src/main.rs | 2 +- apps/grades-sync/src/pg.rs | 2 +- 8 files changed, 130 insertions(+), 26 deletions(-) create mode 100644 apps/grades-sync/migrations/20240209190449_schema.sql diff --git a/apps/grades-sync/Cargo.lock b/apps/grades-sync/Cargo.lock index a3573d56a..137014e41 100644 --- a/apps/grades-sync/Cargo.lock +++ b/apps/grades-sync/Cargo.lock @@ -99,6 +99,28 @@ version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" +[[package]] +name = "async-scoped" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4042078ea593edffc452eef14e99fdb2b120caa4ad9618bcdeabc4a023b98740" +dependencies = [ + "futures", + "pin-project", + "tokio", +] + +[[package]] +name = "async-trait" +version = "0.1.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "atoi" version = "2.0.0" @@ -447,6 +469,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.30" @@ -491,6 +528,17 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "futures-sink" version = "0.3.30" @@ -509,8 +557,10 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ + "futures-channel", "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -551,6 +601,8 @@ name = "grades-sync" version = "0.1.0" dependencies = [ "anyhow", + "async-scoped", + "async-trait", "dotenv", "env_logger", "log", @@ -1075,6 +1127,26 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pin-project" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "pin-project-lite" version = "0.2.13" diff --git a/apps/grades-sync/Cargo.toml b/apps/grades-sync/Cargo.toml index 1492e8f58..cc7f23fdd 100644 --- a/apps/grades-sync/Cargo.toml +++ b/apps/grades-sync/Cargo.toml @@ -15,3 +15,5 @@ anyhow = "1.0.79" dotenv = "0.15.0" log = { version = "0.4.20", features = [] } env_logger = "0.11.1" +async-trait = "0.1.77" +async-scoped = { version = "0.9.0", features = ["use-tokio"] } diff --git a/apps/grades-sync/migrations/20240209190449_schema.sql b/apps/grades-sync/migrations/20240209190449_schema.sql new file mode 100644 index 000000000..7e2ef2341 --- /dev/null +++ b/apps/grades-sync/migrations/20240209190449_schema.sql @@ -0,0 +1,25 @@ +START TRANSACTION; + +CREATE EXTENSION IF NOT EXISTS "fuzzystrmatch"; + +CREATE TABLE IF NOT EXISTS faculty +( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + ref_id TEXT NOT NULL, + name TEXT NOT NULL, + + CONSTRAINT faculty_ref_id_unique UNIQUE (ref_id) +); + +CREATE TABLE IF NOT EXISTS department +( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + ref_id TEXT NOT NULL, + faculty_id UUID NOT NULL, + name TEXT NOT NULL, + + CONSTRAINT department_fk_faculty_id FOREIGN KEY (faculty_id) REFERENCES faculty (id), + CONSTRAINT department_uq_ref_id UNIQUE (ref_id) +); + +COMMIT; diff --git a/apps/grades-sync/src/faculty_repository.rs b/apps/grades-sync/src/faculty_repository.rs index bc09a36cb..54f696c32 100644 --- a/apps/grades-sync/src/faculty_repository.rs +++ b/apps/grades-sync/src/faculty_repository.rs @@ -1,3 +1,4 @@ +use async_trait::async_trait; use crate::pg::Database; use sqlx::types::Uuid; use sqlx::FromRow; @@ -9,13 +10,10 @@ pub struct Faculty { pub ref_id: String, } +#[async_trait] pub trait FacultyRepository: Sync { - async fn create_faculty(&self, name: String, ref_id: String) -> Result - where - Self: Sized; - async fn get_faculty_by_ref_id(&self, ref_id: &str) -> Result, sqlx::Error> - where - Self: Sized; + async fn create_faculty(&self, name: String, ref_id: String) -> Result; + async fn get_faculty_by_ref_id(&self, ref_id: &str) -> Result, sqlx::Error>; } pub struct FacultyRepositoryImpl<'a> { @@ -28,13 +26,14 @@ impl<'a> FacultyRepositoryImpl<'a> { } } +#[async_trait] impl<'a> FacultyRepository for FacultyRepositoryImpl<'a> { async fn create_faculty(&self, name: String, ref_id: String) -> Result { sqlx::query_as::<_, Faculty>( r#" - INSERT INTO ntnu_faculty (name, ref_id) VALUES ($1, $2) + INSERT INTO faculty (name, ref_id) VALUES ($1, $2) ON CONFLICT (ref_id) DO UPDATE SET name = $1, ref_id = $2 - RETURNING ALL + RETURNING *; "#, ) .bind(name) @@ -46,7 +45,7 @@ impl<'a> FacultyRepository for FacultyRepositoryImpl<'a> { async fn get_faculty_by_ref_id(&self, ref_id: &str) -> Result, sqlx::Error> { sqlx::query_as::<_, Faculty>( r#" - SELECT * FROM ntnu_faculty WHERE ref_id = $1 + SELECT * FROM faculty WHERE ref_id = $1 "#, ) .bind(ref_id) diff --git a/apps/grades-sync/src/hkdir.rs b/apps/grades-sync/src/hkdir.rs index 44a2ec23e..8be041e1c 100644 --- a/apps/grades-sync/src/hkdir.rs +++ b/apps/grades-sync/src/hkdir.rs @@ -1,5 +1,5 @@ use crate::json::HkdirDepartment; -use reqwest::{Client}; +use reqwest::Client; use serde_json::{json, Value}; const HKDIR_API_URL: &str = "https://dbh.hkdir.no/api/Tabeller/hentJSONTabellData"; diff --git a/apps/grades-sync/src/job.rs b/apps/grades-sync/src/job.rs index 2338009c9..39fea7816 100644 --- a/apps/grades-sync/src/job.rs +++ b/apps/grades-sync/src/job.rs @@ -1,8 +1,9 @@ -use crate::faculty_repository::{FacultyRepository}; + +use crate::faculty_repository::FacultyRepository; use crate::hkdir::get_departments; use log::info; -use tokio::task::JoinSet; + pub trait JobService { async fn perform_faculty_synchronization(&self) -> anyhow::Result<()>; @@ -24,21 +25,26 @@ impl<'a> JobService for JobServiceImpl<'a> { async fn perform_faculty_synchronization(&self) -> anyhow::Result<()> { info!("performing faculty synchronization"); let departments = get_departments().await?; - let _processed_count = 0; + let department_count = departments.len(); info!( "performing synchronization for {} departments", - departments.len() + department_count ); - let mut set = JoinSet::new(); - for (i, _department) in departments.iter().enumerate() { - set.spawn(async move { - info!("processing department {}", i); - }); - } - - while let Some(result) = set.join_next().await { - result?; - } + + async_scoped::TokioScope::scope_and_block(|s| { + for department in departments { + s.spawn(async move { + // Create the faculty if it doesn't exist. The underlying query performs an on + // conflict update, which means that we do not need to check if the faculty + // exists before creating it. + self + .faculty_repository + .create_faculty(department.faculty_code, department.faculty_name) + .await + .unwrap(); + }); + } + }); Ok(()) } diff --git a/apps/grades-sync/src/main.rs b/apps/grades-sync/src/main.rs index 2ced7e884..da955ecc8 100644 --- a/apps/grades-sync/src/main.rs +++ b/apps/grades-sync/src/main.rs @@ -21,5 +21,5 @@ async fn main() { job_service.perform_faculty_synchronization().await.unwrap(); - println!("Hello, world!"); + connection.close().await; } diff --git a/apps/grades-sync/src/pg.rs b/apps/grades-sync/src/pg.rs index 7f90e74d2..90589da0a 100644 --- a/apps/grades-sync/src/pg.rs +++ b/apps/grades-sync/src/pg.rs @@ -7,7 +7,7 @@ pub async fn create_postgres_pool() -> Result, sqlx::Error> { let database_url = std::env::var("DATABASE_URL").expect("missing DATABASE_URL environment variable"); let pool = PgPoolOptions::new() - .max_connections(5) + .max_connections(100) .connect(&database_url) .await?; Ok(pool) From 7c69bfe8f58239457fd44cfd2358545051d0ab36 Mon Sep 17 00:00:00 2001 From: Mats Larsen Date: Fri, 9 Feb 2024 20:59:37 +0100 Subject: [PATCH 03/23] Implement synchronization of departments --- .../migrations/20240209195028_subject.sql | 1 + apps/grades-sync/src/department_repository.rs | 55 +++++++++++++++++++ apps/grades-sync/src/faculty_repository.rs | 14 +---- apps/grades-sync/src/job.rs | 36 +++++++++--- apps/grades-sync/src/main.rs | 5 +- apps/grades-sync/src/pg.rs | 11 +++- 6 files changed, 97 insertions(+), 25 deletions(-) create mode 100644 apps/grades-sync/migrations/20240209195028_subject.sql create mode 100644 apps/grades-sync/src/department_repository.rs diff --git a/apps/grades-sync/migrations/20240209195028_subject.sql b/apps/grades-sync/migrations/20240209195028_subject.sql new file mode 100644 index 000000000..8ddc1d3ff --- /dev/null +++ b/apps/grades-sync/migrations/20240209195028_subject.sql @@ -0,0 +1 @@ +-- Add migration script here diff --git a/apps/grades-sync/src/department_repository.rs b/apps/grades-sync/src/department_repository.rs new file mode 100644 index 000000000..64c64ee30 --- /dev/null +++ b/apps/grades-sync/src/department_repository.rs @@ -0,0 +1,55 @@ +use crate::pg::Database; +use async_trait::async_trait; +use sqlx::types::Uuid; +use sqlx::FromRow; + +#[derive(Debug, FromRow)] +pub struct Department { + pub id: Uuid, + pub name: String, + pub ref_id: String, + pub faculty_id: Uuid, +} + +#[async_trait] +pub trait DepartmentRepository: Sync { + async fn create_department( + &self, + name: String, + ref_id: String, + faculty_id: Uuid, + ) -> Result; +} + +pub struct DepartmentRepositoryImpl<'a> { + db: &'a Database, +} + +impl<'a> DepartmentRepositoryImpl<'a> { + pub fn new(db: &'a Database) -> Self { + Self { db } + } +} + +#[async_trait] +impl<'a> DepartmentRepository for DepartmentRepositoryImpl<'a> { + async fn create_department( + &self, + name: String, + ref_id: String, + faculty_id: Uuid, + ) -> Result { + sqlx::query_as::<_, Department>( + r#" + INSERT INTO department (name, ref_id, faculty_id) VALUES ($1, $2, $3) + ON CONFLICT (ref_id) DO UPDATE SET name = $1, ref_id = $2, faculty_id = $3 + RETURNING *; + "#, + ) + .bind(name) + .bind(ref_id) + .bind(faculty_id) + .fetch_one(self.db) + .await + } +} diff --git a/apps/grades-sync/src/faculty_repository.rs b/apps/grades-sync/src/faculty_repository.rs index 54f696c32..0a8392f64 100644 --- a/apps/grades-sync/src/faculty_repository.rs +++ b/apps/grades-sync/src/faculty_repository.rs @@ -1,5 +1,5 @@ -use async_trait::async_trait; use crate::pg::Database; +use async_trait::async_trait; use sqlx::types::Uuid; use sqlx::FromRow; @@ -13,7 +13,6 @@ pub struct Faculty { #[async_trait] pub trait FacultyRepository: Sync { async fn create_faculty(&self, name: String, ref_id: String) -> Result; - async fn get_faculty_by_ref_id(&self, ref_id: &str) -> Result, sqlx::Error>; } pub struct FacultyRepositoryImpl<'a> { @@ -41,15 +40,4 @@ impl<'a> FacultyRepository for FacultyRepositoryImpl<'a> { .fetch_one(self.db) .await } - - async fn get_faculty_by_ref_id(&self, ref_id: &str) -> Result, sqlx::Error> { - sqlx::query_as::<_, Faculty>( - r#" - SELECT * FROM faculty WHERE ref_id = $1 - "#, - ) - .bind(ref_id) - .fetch_optional(self.db) - .await - } } diff --git a/apps/grades-sync/src/job.rs b/apps/grades-sync/src/job.rs index 39fea7816..15152947d 100644 --- a/apps/grades-sync/src/job.rs +++ b/apps/grades-sync/src/job.rs @@ -1,26 +1,35 @@ - use crate::faculty_repository::FacultyRepository; use crate::hkdir::get_departments; +use async_trait::async_trait; +use crate::department_repository::DepartmentRepository; use log::info; - -pub trait JobService { +#[async_trait] +pub trait JobService: Sync { async fn perform_faculty_synchronization(&self) -> anyhow::Result<()>; - async fn perform_department_synchronization(&self) -> anyhow::Result<()>; + async fn perform_subject_synchronization(&self) -> anyhow::Result<()>; async fn perform_grade_synchronization(&self) -> anyhow::Result<()>; } pub struct JobServiceImpl<'a> { faculty_repository: &'a dyn FacultyRepository, + department_repository: &'a dyn DepartmentRepository, } impl<'a> JobServiceImpl<'a> { - pub fn new(faculty_repository: &'a dyn FacultyRepository) -> Self { - Self { faculty_repository } + pub fn new( + faculty_repository: &'a dyn FacultyRepository, + department_repository: &'a dyn DepartmentRepository, + ) -> Self { + Self { + faculty_repository, + department_repository, + } } } +#[async_trait] impl<'a> JobService for JobServiceImpl<'a> { async fn perform_faculty_synchronization(&self) -> anyhow::Result<()> { info!("performing faculty synchronization"); @@ -37,18 +46,29 @@ impl<'a> JobService for JobServiceImpl<'a> { // Create the faculty if it doesn't exist. The underlying query performs an on // conflict update, which means that we do not need to check if the faculty // exists before creating it. - self + let faculty = self .faculty_repository .create_faculty(department.faculty_code, department.faculty_name) .await .unwrap(); + + // Create the department. The same rules apply here as for the faculty. + self.department_repository + .create_department( + department.department_code, + department.department_name, + faculty.id, + ) + .await + .unwrap(); }); } }); + info!("synchronization complete"); Ok(()) } - async fn perform_department_synchronization(&self) -> anyhow::Result<()> { + async fn perform_subject_synchronization(&self) -> anyhow::Result<()> { Ok(()) } diff --git a/apps/grades-sync/src/main.rs b/apps/grades-sync/src/main.rs index da955ecc8..fa3f4e724 100644 --- a/apps/grades-sync/src/main.rs +++ b/apps/grades-sync/src/main.rs @@ -1,6 +1,8 @@ +use crate::department_repository::DepartmentRepositoryImpl; use crate::faculty_repository::FacultyRepositoryImpl; use crate::job::{JobService, JobServiceImpl}; +mod department_repository; mod faculty_repository; mod hkdir; mod job; @@ -17,7 +19,8 @@ async fn main() { bootstrap_environment(); let connection = pg::create_postgres_pool().await.unwrap(); let faculty_repository = FacultyRepositoryImpl::new(&connection); - let job_service = JobServiceImpl::new(&faculty_repository); + let department_repository = DepartmentRepositoryImpl::new(&connection); + let job_service = JobServiceImpl::new(&faculty_repository, &department_repository); job_service.perform_faculty_synchronization().await.unwrap(); diff --git a/apps/grades-sync/src/pg.rs b/apps/grades-sync/src/pg.rs index 90589da0a..c830c3507 100644 --- a/apps/grades-sync/src/pg.rs +++ b/apps/grades-sync/src/pg.rs @@ -1,14 +1,19 @@ -use sqlx::postgres::PgPoolOptions; -use sqlx::{Pool, Postgres}; +use std::time::Duration; +use log::{LevelFilter}; +use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; +use sqlx::{ConnectOptions, Pool, Postgres}; pub type Database = Pool; pub async fn create_postgres_pool() -> Result, sqlx::Error> { let database_url = std::env::var("DATABASE_URL").expect("missing DATABASE_URL environment variable"); + let opts: PgConnectOptions = database_url.parse()?; + let opts = opts.log_statements(LevelFilter::Debug) + .log_slow_statements(LevelFilter::Warn, Duration::from_secs(1)); let pool = PgPoolOptions::new() .max_connections(100) - .connect(&database_url) + .connect_with(opts) .await?; Ok(pool) } From dd78366395774d43a20ffa2addad7e29b5c9e7b5 Mon Sep 17 00:00:00 2001 From: Mats Larsen Date: Fri, 9 Feb 2024 22:17:25 +0100 Subject: [PATCH 04/23] Chunk jobs across multiple threads --- apps/grades-sync/Cargo.lock | 2 + apps/grades-sync/Cargo.toml | 2 + .../migrations/20240209195028_subject.sql | 24 ++- apps/grades-sync/src/department_repository.rs | 12 ++ apps/grades-sync/src/hkdir.rs | 84 ++++++++++- apps/grades-sync/src/job.rs | 138 +++++++++++++++--- apps/grades-sync/src/json.rs | 49 ++++++- apps/grades-sync/src/main.rs | 18 ++- apps/grades-sync/src/pg.rs | 7 +- apps/grades-sync/src/subject_repository.rs | 84 +++++++++++ 10 files changed, 385 insertions(+), 35 deletions(-) create mode 100644 apps/grades-sync/src/subject_repository.rs diff --git a/apps/grades-sync/Cargo.lock b/apps/grades-sync/Cargo.lock index 137014e41..07d39f46a 100644 --- a/apps/grades-sync/Cargo.lock +++ b/apps/grades-sync/Cargo.lock @@ -605,7 +605,9 @@ dependencies = [ "async-trait", "dotenv", "env_logger", + "itertools", "log", + "regex", "reqwest", "serde", "serde_json", diff --git a/apps/grades-sync/Cargo.toml b/apps/grades-sync/Cargo.toml index cc7f23fdd..365934377 100644 --- a/apps/grades-sync/Cargo.toml +++ b/apps/grades-sync/Cargo.toml @@ -17,3 +17,5 @@ log = { version = "0.4.20", features = [] } env_logger = "0.11.1" async-trait = "0.1.77" async-scoped = { version = "0.9.0", features = ["use-tokio"] } +regex = "1.10.3" +itertools = "0.12.1" diff --git a/apps/grades-sync/migrations/20240209195028_subject.sql b/apps/grades-sync/migrations/20240209195028_subject.sql index 8ddc1d3ff..e7f4a9792 100644 --- a/apps/grades-sync/migrations/20240209195028_subject.sql +++ b/apps/grades-sync/migrations/20240209195028_subject.sql @@ -1 +1,23 @@ --- Add migration script here +BEGIN TRANSACTION; + +CREATE TABLE IF NOT EXISTS subject +( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + ref_id TEXT NOT NULL, + name TEXT NOT NULL, + slug TEXT NOT NULL, + department_id UUID NOT NULL, + + instruction_language TEXT NOT NULL, + educational_level TEXT NOT NULL, + credits REAL NOT NULL, + + average_grade REAL NOT NULL DEFAULT 0.0, + total_students INT NOT NULL DEFAULT 0, + failed_students INT NOT NULL DEFAULT 0, + + CONSTRAINT subject_fk_department_id FOREIGN KEY (department_id) REFERENCES department (id), + CONSTRAINT subject_uq_ref_id UNIQUE (ref_id) +); + +COMMIT; diff --git a/apps/grades-sync/src/department_repository.rs b/apps/grades-sync/src/department_repository.rs index 64c64ee30..3f1f488cc 100644 --- a/apps/grades-sync/src/department_repository.rs +++ b/apps/grades-sync/src/department_repository.rs @@ -19,6 +19,7 @@ pub trait DepartmentRepository: Sync { ref_id: String, faculty_id: Uuid, ) -> Result; + async fn get_department_by_ref_id(&self, ref_id: String) -> Result; } pub struct DepartmentRepositoryImpl<'a> { @@ -52,4 +53,15 @@ impl<'a> DepartmentRepository for DepartmentRepositoryImpl<'a> { .fetch_one(self.db) .await } + + async fn get_department_by_ref_id(&self, ref_id: String) -> Result { + sqlx::query_as::<_, Department>( + r#" + SELECT * FROM department WHERE ref_id = $1; + "#, + ) + .bind(ref_id) + .fetch_one(self.db) + .await + } } diff --git a/apps/grades-sync/src/hkdir.rs b/apps/grades-sync/src/hkdir.rs index 8be041e1c..7bbbd4965 100644 --- a/apps/grades-sync/src/hkdir.rs +++ b/apps/grades-sync/src/hkdir.rs @@ -1,4 +1,4 @@ -use crate::json::HkdirDepartment; +use crate::json::{HkdirDepartment, HkdirSubject}; use reqwest::Client; use serde_json::{json, Value}; @@ -22,6 +22,77 @@ fn build_get_departments_request() -> Value { "exclude": [""], }, }, + { + "variabel": "Avdelingskode", + "selection": { + "filter": "all", + "values": ["*"], + "exclude": ["000000"], + }, + }, + ], + }); + json +} + +fn build_get_subjects_request() -> Value { + let json = json!({ + "tabell_id": 208, + "api_versjon": 1, + "statuslinje": "N", + "kodetekst": "J", + "desimal_separator": ".", + "variabler": ["*"], + "sortBy": ["Årstall", "Institusjonskode", "Avdelingskode"], + "filter": [ + { + "variabel": "Institusjonskode", + "selection": { + "filter": "item", + "values": ["1150"], + "exclude": [""], + }, + }, + { + "variabel": "Nivåkode", + "selection": { + "filter": "item", + "values": ["HN", "LN"], + "exclude": [""], + }, + }, + { + "variabel": "Status", + "selection": { + "filter": "item", + "values": ["1", "2"], + "exclude": [""], + }, + }, + { + "variabel": "Avdelingskode", + "selection": { + "filter": "all", + "values": ["*"], + "exclude": ["000000"], + }, + }, + { + "variabel": "Oppgave (ny fra h2012)", + "selection": { + "filter": "all", + "values": ["*"], + "exclude": ["1", "2"], + }, + }, + { + "variabel": "Årstall", + "selection": { + "filter": "top", + "values": ["5"], + "exclude": [""], + }, + } ], }); json @@ -37,3 +108,14 @@ pub async fn get_departments() -> reqwest::Result> { .await?; response.json::>().await } + +pub async fn get_subjects() -> reqwest::Result> { + let request_body = build_get_subjects_request(); + let client = Client::new(); + let response = client + .post(HKDIR_API_URL) + .json(&request_body) + .send() + .await?; + response.json::>().await +} diff --git a/apps/grades-sync/src/job.rs b/apps/grades-sync/src/job.rs index 15152947d..671998135 100644 --- a/apps/grades-sync/src/job.rs +++ b/apps/grades-sync/src/job.rs @@ -1,66 +1,73 @@ -use crate::faculty_repository::FacultyRepository; -use crate::hkdir::get_departments; +use std::cmp::{min}; +use crate::faculty_repository::{FacultyRepository}; +use crate::hkdir::{get_departments, get_subjects}; use async_trait::async_trait; +use itertools::Itertools; -use crate::department_repository::DepartmentRepository; + +use crate::department_repository::{DepartmentRepository}; +use crate::json::{HkdirDepartment, HkdirSubject}; +use crate::subject_repository::{SubjectRepository}; use log::info; +use regex::Regex; #[async_trait] pub trait JobService: Sync { async fn perform_faculty_synchronization(&self) -> anyhow::Result<()>; async fn perform_subject_synchronization(&self) -> anyhow::Result<()>; async fn perform_grade_synchronization(&self) -> anyhow::Result<()>; + + async fn synchronize_single_faculty(&self, faculty: HkdirDepartment) -> anyhow::Result<()>; + async fn synchronize_single_subject(&self, subject: HkdirSubject) -> anyhow::Result<()>; } pub struct JobServiceImpl<'a> { faculty_repository: &'a dyn FacultyRepository, department_repository: &'a dyn DepartmentRepository, + subject_repository: &'a dyn SubjectRepository, } impl<'a> JobServiceImpl<'a> { pub fn new( faculty_repository: &'a dyn FacultyRepository, department_repository: &'a dyn DepartmentRepository, + subject_repository: &'a dyn SubjectRepository, ) -> Self { Self { faculty_repository, department_repository, + subject_repository, } } } +const MAX_TASK_COUNT: usize = 100; + #[async_trait] impl<'a> JobService for JobServiceImpl<'a> { async fn perform_faculty_synchronization(&self) -> anyhow::Result<()> { info!("performing faculty synchronization"); let departments = get_departments().await?; let department_count = departments.len(); + let chunks_count = min(department_count / MAX_TASK_COUNT, 1000); info!( - "performing synchronization for {} departments", - department_count + "performing synchronization for {} departments across {} threads", + department_count, + chunks_count ); + let departments_chunked = departments + .into_iter() + .chunks(chunks_count) + .into_iter() + .map(|chunk| chunk.collect()) + .collect::>>(); async_scoped::TokioScope::scope_and_block(|s| { - for department in departments { + for departments in departments_chunked { s.spawn(async move { - // Create the faculty if it doesn't exist. The underlying query performs an on - // conflict update, which means that we do not need to check if the faculty - // exists before creating it. - let faculty = self - .faculty_repository - .create_faculty(department.faculty_code, department.faculty_name) - .await - .unwrap(); - - // Create the department. The same rules apply here as for the faculty. - self.department_repository - .create_department( - department.department_code, - department.department_name, - faculty.id, - ) - .await - .unwrap(); + for department in departments { + self.synchronize_single_faculty(department).await.unwrap(); + } }); } }); @@ -69,10 +76,93 @@ impl<'a> JobService for JobServiceImpl<'a> { } async fn perform_subject_synchronization(&self) -> anyhow::Result<()> { + info!("performing subject synchronization"); + let subjects = get_subjects().await?; + let subject_count = subjects.len(); + let chunks_count = min(subject_count / MAX_TASK_COUNT, 1000); + info!( + "performing synchronization for {} subjects across {} threads", + subject_count, + chunks_count + ); + + let subjects_chunked = subjects + .into_iter() + .chunks(chunks_count) + .into_iter() + .map(|chunk| chunk.collect()) + .collect::>>(); + async_scoped::TokioScope::scope_and_block(|s| { + for subjects in subjects_chunked { + s.spawn(async move { + for subject in subjects { + self.synchronize_single_subject(subject).await.unwrap(); + } + }); + } + }); + + info!("synchronization complete"); Ok(()) } async fn perform_grade_synchronization(&self) -> anyhow::Result<()> { Ok(()) } + + async fn synchronize_single_faculty(&self, department: HkdirDepartment) -> anyhow::Result<()> { + // Create the faculty if it doesn't exist. The underlying query performs an on + // conflict update, which means that we do not need to check if the faculty + // exists before creating it. + let faculty = self + .faculty_repository + .create_faculty(department.faculty_code, department.faculty_name) + .await + .unwrap(); + + // Create the department. The same rules apply here as for the faculty. + self.department_repository + .create_department( + department.department_name, + department.department_code, + faculty.id, + ) + .await + .unwrap(); + Ok(()) + } + + async fn synchronize_single_subject(&self, subject: HkdirSubject) -> anyhow::Result<()> { + // Forge a slug from the subject name. A lot of the subject code fields from + // HKDir have a -1 or another number appended to them, which is not useful + // because we're using to seeing 'TDT4120' instead of 'TDT4120-1'. + let re = Regex::new("-[A-Za-z0-9]+$").unwrap(); + let slug = re.replace_all(&subject.subject_code, "").to_lowercase(); + // Find a reference to the department. + let department = self + .department_repository + .get_department_by_ref_id(subject.department_code) + .await + .unwrap(); + + // Create the subject if it doesn't exist. The underlying query performs an on + // conflict update, which means that we do not need to check if the subject + // exists before creating it. + self.subject_repository + .create_subject( + subject.subject_code, + subject.subject_name, + department.id, + slug, + subject.instruction_language, + subject.level_code, + subject.credits.parse().unwrap(), + 0f32, + 0, + 0, + ) + .await + .unwrap(); + Ok(()) + } } diff --git a/apps/grades-sync/src/json.rs b/apps/grades-sync/src/json.rs index a170f0eb3..8756424bd 100644 --- a/apps/grades-sync/src/json.rs +++ b/apps/grades-sync/src/json.rs @@ -31,7 +31,54 @@ pub struct HkdirDepartment { } #[derive(Serialize, Deserialize)] -pub struct HkdirSubject {} +pub struct HkdirSubject { + #[serde(rename = "Institusjonskode")] + pub institution_code: String, + #[serde(rename = "Institusjonsnavn")] + pub institution_name: String, + #[serde(rename = "Avdelingskode")] + pub department_code: String, + #[serde(rename = "Avdelingsnavn")] + pub department_name: String, + #[serde(rename = "Avdelingskode_SSB")] + pub department_code_ssb: String, + #[serde(rename = "Årstall")] + pub year: String, + #[serde(rename = "Semester")] + pub semester: String, + #[serde(rename = "Semesternavn")] + pub semester_name: String, + #[serde(rename = "Studieprogramkode")] + pub study_program_code: String, + #[serde(rename = "Studieprogramnavn")] + pub study_program_name: String, + #[serde(rename = "Emnekode")] + pub subject_code: String, + #[serde(rename = "Emnenavn")] + pub subject_name: String, + #[serde(rename = "Nivåkode")] + pub level_code: String, + #[serde(rename = "Nivånavn")] + pub level_name: String, + #[serde(rename = "Studiepoeng")] + pub credits: String, + #[serde(rename = "NUS-kode")] + pub nus_code: String, + #[serde(rename = "Status")] + pub status: String, + #[serde(rename = "Statusnavn")] + pub status_name: String, + #[serde(rename = "Underv.språk")] + pub instruction_language: String, + #[serde(rename = "Navn")] + pub name: String, + #[serde(rename = "Fagkode")] + pub subject_code_department: Option, + #[serde(rename = "Fagnavn")] + pub subject_name_department: Option, + #[serde(rename = "Oppgave (ny fra h2012)")] + pub task: Option, +} #[derive(Serialize, Deserialize)] pub struct HkdirGrade {} diff --git a/apps/grades-sync/src/main.rs b/apps/grades-sync/src/main.rs index fa3f4e724..fd46b1ff3 100644 --- a/apps/grades-sync/src/main.rs +++ b/apps/grades-sync/src/main.rs @@ -1,6 +1,7 @@ use crate::department_repository::DepartmentRepositoryImpl; use crate::faculty_repository::FacultyRepositoryImpl; use crate::job::{JobService, JobServiceImpl}; +use crate::subject_repository::SubjectRepositoryImpl; mod department_repository; mod faculty_repository; @@ -8,6 +9,7 @@ mod hkdir; mod job; mod json; mod pg; +mod subject_repository; fn bootstrap_environment() { dotenv::dotenv().ok(); @@ -17,12 +19,18 @@ fn bootstrap_environment() { #[tokio::main] async fn main() { bootstrap_environment(); - let connection = pg::create_postgres_pool().await.unwrap(); - let faculty_repository = FacultyRepositoryImpl::new(&connection); - let department_repository = DepartmentRepositoryImpl::new(&connection); - let job_service = JobServiceImpl::new(&faculty_repository, &department_repository); + let pool = pg::create_postgres_pool().await.unwrap(); + let faculty_repository = FacultyRepositoryImpl::new(&pool); + let department_repository = DepartmentRepositoryImpl::new(&pool); + let subject_repository = SubjectRepositoryImpl::new(&pool); + let job_service = JobServiceImpl::new( + &faculty_repository, + &department_repository, + &subject_repository, + ); job_service.perform_faculty_synchronization().await.unwrap(); + job_service.perform_subject_synchronization().await.unwrap(); - connection.close().await; + pool.close().await; } diff --git a/apps/grades-sync/src/pg.rs b/apps/grades-sync/src/pg.rs index c830c3507..644a5e5b6 100644 --- a/apps/grades-sync/src/pg.rs +++ b/apps/grades-sync/src/pg.rs @@ -1,7 +1,7 @@ -use std::time::Duration; -use log::{LevelFilter}; +use log::LevelFilter; use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; use sqlx::{ConnectOptions, Pool, Postgres}; +use std::time::Duration; pub type Database = Pool; @@ -9,7 +9,8 @@ pub async fn create_postgres_pool() -> Result, sqlx::Error> { let database_url = std::env::var("DATABASE_URL").expect("missing DATABASE_URL environment variable"); let opts: PgConnectOptions = database_url.parse()?; - let opts = opts.log_statements(LevelFilter::Debug) + let opts = opts + .log_statements(LevelFilter::Debug) .log_slow_statements(LevelFilter::Warn, Duration::from_secs(1)); let pool = PgPoolOptions::new() .max_connections(100) diff --git a/apps/grades-sync/src/subject_repository.rs b/apps/grades-sync/src/subject_repository.rs new file mode 100644 index 000000000..0a2105cce --- /dev/null +++ b/apps/grades-sync/src/subject_repository.rs @@ -0,0 +1,84 @@ +use crate::pg::Database; +use async_trait::async_trait; +use sqlx::types::Uuid; +use sqlx::FromRow; + +#[derive(Debug, FromRow)] +pub struct Subject { + pub id: Uuid, + pub ref_id: String, + pub name: String, + pub department_id: Uuid, + pub slug: String, + pub instruction_language: String, + pub educational_level: String, + pub credits: f32, + pub average_grade: f32, + pub total_students: i32, + pub failed_students: i32, +} + +#[async_trait] +pub trait SubjectRepository: Sync { + async fn create_subject( + &self, + ref_id: String, + name: String, + department_id: Uuid, + slug: String, + instruction_language: String, + educational_level: String, + credits: f32, + average_grade: f32, + total_students: i32, + failed_students: i32, + ) -> Result; +} + +pub struct SubjectRepositoryImpl<'a> { + db: &'a Database, +} + +impl<'a> SubjectRepositoryImpl<'a> { + pub fn new(db: &'a Database) -> Self { + Self { db } + } +} + +#[async_trait] +impl<'a> SubjectRepository for SubjectRepositoryImpl<'a> { + async fn create_subject( + &self, + ref_id: String, + name: String, + department_id: Uuid, + slug: String, + instruction_language: String, + educational_level: String, + credits: f32, + average_grade: f32, + total_students: i32, + failed_students: i32, + ) -> Result { + sqlx::query_as::<_, Subject>( + r#" + INSERT INTO subject (ref_id, name, department_id, slug, instruction_language, educational_level, credits, average_grade, total_students, failed_students) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ON CONFLICT (ref_id) DO UPDATE SET name = $2, department_id = $3, slug = $4, instruction_language = $5, educational_level = $6, credits = $7 + RETURNING *; + "#, + ) + .bind(ref_id) + .bind(name) + .bind(department_id) + .bind(slug) + .bind(instruction_language) + .bind(educational_level) + .bind(credits) + .bind(average_grade) + .bind(total_students) + .bind(failed_students) + .fetch_one(self.db) + .await + } +} From b996bb5784e24a5eb1d8b8ba0137dcc4878e2499 Mon Sep 17 00:00:00 2001 From: Mats Larsen Date: Sat, 10 Feb 2024 00:46:06 +0100 Subject: [PATCH 05/23] Ship MVP version of the Rust syncer --- .../migrations/20240209212106_grades.sql | 25 ++ apps/grades-sync/queries/departments.json | 39 +++ apps/grades-sync/queries/grades.json | 68 +++++ apps/grades-sync/queries/subjects.json | 92 ++++++ apps/grades-sync/src/grade_repository.rs | 269 ++++++++++++++++++ apps/grades-sync/src/hkdir.rs | 130 +++------ apps/grades-sync/src/job.rs | 93 +++++- apps/grades-sync/src/json.rs | 16 +- apps/grades-sync/src/main.rs | 9 +- apps/grades-sync/src/pg.rs | 2 +- apps/grades-sync/src/subject_repository.rs | 38 ++- 11 files changed, 667 insertions(+), 114 deletions(-) create mode 100644 apps/grades-sync/migrations/20240209212106_grades.sql create mode 100644 apps/grades-sync/queries/departments.json create mode 100644 apps/grades-sync/queries/grades.json create mode 100644 apps/grades-sync/queries/subjects.json create mode 100644 apps/grades-sync/src/grade_repository.rs diff --git a/apps/grades-sync/migrations/20240209212106_grades.sql b/apps/grades-sync/migrations/20240209212106_grades.sql new file mode 100644 index 000000000..67ab37e4d --- /dev/null +++ b/apps/grades-sync/migrations/20240209212106_grades.sql @@ -0,0 +1,25 @@ +START TRANSACTION; + +CREATE TYPE subject_grading_season AS ENUM ('WINTER', 'SPRING', 'SUMMER', 'AUTUMN'); + +CREATE TABLE IF NOT EXISTS subject_season_grade +( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + subject_id UUID NOT NULL, + season subject_grading_season NOT NULL, + year INTEGER NOT NULL, + + graded_a INTEGER, + graded_b INTEGER, + graded_c INTEGER, + graded_d INTEGER, + graded_e INTEGER, + graded_f INTEGER, + graded_pass INTEGER, + graded_fail INTEGER, + + CONSTRAINT subject_season_grade_fk_subject_id FOREIGN KEY (subject_id) REFERENCES subject (id), + CONSTRAINT subject_season_grade_unique UNIQUE (subject_id, season, year) +); + +COMMIT; diff --git a/apps/grades-sync/queries/departments.json b/apps/grades-sync/queries/departments.json new file mode 100644 index 000000000..e89052972 --- /dev/null +++ b/apps/grades-sync/queries/departments.json @@ -0,0 +1,39 @@ +{ + "tabell_id": 210, + "api_versjon": 1, + "statuslinje": "N", + "kodetekst": "J", + "desimal_separator": ".", + "variabler": [ + "*" + ], + "sortBy": [ + "Nivå" + ], + "filter": [ + { + "variabel": "Institusjonskode", + "selection": { + "filter": "item", + "values": [ + "1150" + ], + "exclude": [ + "" + ] + } + }, + { + "variabel": "Avdelingskode", + "selection": { + "filter": "all", + "values": [ + "*" + ], + "exclude": [ + "000000" + ] + } + } + ] +} \ No newline at end of file diff --git a/apps/grades-sync/queries/grades.json b/apps/grades-sync/queries/grades.json new file mode 100644 index 000000000..120efbd59 --- /dev/null +++ b/apps/grades-sync/queries/grades.json @@ -0,0 +1,68 @@ +{ + "tabell_id": 308, + "api_versjon": 1, + "statuslinje": "N", + "kodetekst": "J", + "desimal_separator": ".", + "variabler": [ + "*" + ], + "groupBy": [ + "Årstall", + "Semester", + "Karakter", + "Emnekode", + "Institusjonskode" + ], + "filter": [ + { + "variabel": "Institusjonskode", + "selection": { + "filter": "item", + "values": [ + "1150" + ], + "exclude": [ + "" + ] + } + }, + { + "variabel": "Emnekode", + "selection": { + "filter": "all", + "values": [ + "*" + ], + "exclude": [ + "" + ] + } + }, + { + "variabel": "Semester", + "selection": { + "filter": "all", + "values": [ + "*" + ], + "exclude": [ + "" + ] + } + }, + + { + "variabel": "Årstall", + "selection": { + "filter": "top", + "values": [ + "2" + ], + "exclude": [ + "" + ] + } + } + ] +} \ No newline at end of file diff --git a/apps/grades-sync/queries/subjects.json b/apps/grades-sync/queries/subjects.json new file mode 100644 index 000000000..3d0682bad --- /dev/null +++ b/apps/grades-sync/queries/subjects.json @@ -0,0 +1,92 @@ +{ + "tabell_id": 208, + "api_versjon": 1, + "statuslinje": "N", + "kodetekst": "J", + "desimal_separator": ".", + "variabler": [ + "*" + ], + "sortBy": [ + "Årstall", + "Institusjonskode", + "Avdelingskode" + ], + "filter": [ + { + "variabel": "Institusjonskode", + "selection": { + "filter": "item", + "values": [ + "1150" + ], + "exclude": [ + "" + ] + } + }, + { + "variabel": "Nivåkode", + "selection": { + "filter": "item", + "values": [ + "HN", + "LN" + ], + "exclude": [ + "" + ] + } + }, + { + "variabel": "Status", + "selection": { + "filter": "item", + "values": [ + "1", + "2" + ], + "exclude": [ + "" + ] + } + }, + { + "variabel": "Avdelingskode", + "selection": { + "filter": "all", + "values": [ + "*" + ], + "exclude": [ + "000000" + ] + } + }, + { + "variabel": "Oppgave (ny fra h2012)", + "selection": { + "filter": "all", + "values": [ + "*" + ], + "exclude": [ + "1", + "2" + ] + } + }, + { + "variabel": "Årstall", + "selection": { + "filter": "top", + "values": [ + "2" + ], + "exclude": [ + "" + ] + } + } + ] +} \ No newline at end of file diff --git a/apps/grades-sync/src/grade_repository.rs b/apps/grades-sync/src/grade_repository.rs new file mode 100644 index 000000000..0b7d8dd6f --- /dev/null +++ b/apps/grades-sync/src/grade_repository.rs @@ -0,0 +1,269 @@ +use crate::pg::Database; +use crate::subject_repository::Subject; +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use sqlx::types::Uuid; +use sqlx::FromRow; + +#[derive(Debug, PartialEq, PartialOrd, sqlx::Type, Deserialize, Serialize, Copy, Clone)] +#[sqlx(type_name = "subject_grading_season", rename_all = "UPPERCASE")] +pub enum SubjectGradingSeason { + Winter, + Spring, + Summer, + Autumn, +} + +#[derive(Serialize, Deserialize, Copy, Clone, Debug)] +pub enum SubjectGradingKey { + A, + B, + C, + D, + E, + F, + G, + H, +} + +impl SubjectGradingKey { + pub fn to_column_key_name(self) -> &'static str { + match self { + SubjectGradingKey::A => "graded_a", + SubjectGradingKey::B => "graded_b", + SubjectGradingKey::C => "graded_c", + SubjectGradingKey::D => "graded_d", + SubjectGradingKey::E => "graded_e", + SubjectGradingKey::F => "graded_f", + SubjectGradingKey::G => "graded_pass", + SubjectGradingKey::H => "graded_fail", + } + } + + pub fn to_multiplication_factor(self) -> f32 { + match self { + SubjectGradingKey::A => 5.0, + SubjectGradingKey::B => 4.0, + SubjectGradingKey::C => 3.0, + SubjectGradingKey::D => 2.0, + SubjectGradingKey::E => 1.0, + SubjectGradingKey::F => 0.0, + x => panic!("invalid grading key: {:?}", x), + } + } + + pub fn is_pass_or_fail_key(self) -> bool { + matches!(self, SubjectGradingKey::G | SubjectGradingKey::H) + } + + pub fn is_evaluated_as_failed(self) -> bool { + matches!(self, SubjectGradingKey::F | SubjectGradingKey::H) + } +} + +#[derive(Debug, FromRow)] +pub struct Grade { + pub id: Uuid, + pub subject_id: Uuid, + pub season: SubjectGradingSeason, + pub year: i32, + pub graded_a: Option, + pub graded_b: Option, + pub graded_c: Option, + pub graded_d: Option, + pub graded_e: Option, + pub graded_f: Option, + pub graded_pass: Option, + pub graded_fail: Option, +} + +impl Grade { + pub fn has_previously_been_graded(&self, key: SubjectGradingKey) -> bool { + match key { + SubjectGradingKey::A => self.graded_a.is_some(), + SubjectGradingKey::B => self.graded_b.is_some(), + SubjectGradingKey::C => self.graded_c.is_some(), + SubjectGradingKey::D => self.graded_d.is_some(), + SubjectGradingKey::E => self.graded_e.is_some(), + SubjectGradingKey::F => self.graded_f.is_some(), + SubjectGradingKey::G => self.graded_pass.is_some(), + SubjectGradingKey::H => self.graded_fail.is_some(), + } + } +} + +#[async_trait] +pub trait GradeRepository: Sync { + async fn create_grade( + &self, + subject_id: Uuid, + season: SubjectGradingSeason, + year: i32, + graded_a: Option, + graded_b: Option, + graded_c: Option, + graded_d: Option, + graded_f: Option, + graded_pass: Option, + graded_fail: Option, + ) -> Result; + async fn update_grade_record( + &self, + subject_id: Uuid, + season: SubjectGradingSeason, + year: i32, + key: SubjectGradingKey, + count: i32, + ) -> Result; + async fn find_grade_by_season( + &self, + subject_id: Uuid, + season: SubjectGradingSeason, + year: i32, + ) -> Result, sqlx::Error>; +} + +pub struct GradeRepositoryImpl<'a> { + db: &'a Database, +} + +impl<'a> GradeRepositoryImpl<'a> { + pub fn new(db: &'a Database) -> Self { + Self { db } + } +} + +#[async_trait] +impl<'a> GradeRepository for GradeRepositoryImpl<'a> { + async fn create_grade( + &self, + subject_id: Uuid, + season: SubjectGradingSeason, + year: i32, + graded_a: Option, + graded_b: Option, + graded_c: Option, + graded_d: Option, + graded_f: Option, + graded_pass: Option, + graded_fail: Option, + ) -> Result { + sqlx::query_as::<_, Grade>( + r#" + INSERT INTO subject_season_grade (subject_id, season, year, graded_a, graded_b, graded_c, graded_d, graded_f, graded_pass, graded_fail) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + RETURNING *; + "#, + ) + .bind(subject_id) + .bind(season) + .bind(year) + .bind(graded_a) + .bind(graded_b) + .bind(graded_c) + .bind(graded_d) + .bind(graded_f) + .bind(graded_pass) + .bind(graded_fail) + .fetch_one(self.db) + .await + } + + async fn update_grade_record( + &self, + subject_id: Uuid, + season: SubjectGradingSeason, + year: i32, + key: SubjectGradingKey, + count: i32, + ) -> Result { + // TODO: Refactor this logic out, it's only here because it's a transaction right now + let mut tx = self.db.begin().await?; + let grade = sqlx::query_as::<_, Grade>( + r#" + SELECT * FROM subject_season_grade WHERE subject_id = $1 AND season = $2 AND year = $3 + FOR UPDATE; + "#, + ) + .bind(subject_id) + .bind(season) + .bind(year) + .fetch_optional(&mut *tx) + .await?; + + let grade = match grade { + Some(grade) => grade, + None => { + sqlx::query_as::<_, Grade>( + r#" + INSERT INTO subject_season_grade (subject_id, season, year, graded_a, graded_b, graded_c, graded_d, graded_f, graded_pass, graded_fail) + VALUES ($1, $2, $3, 0, 0, 0, 0, 0, 0, 0) + RETURNING *; + "# + ).bind(subject_id) + .bind(season) + .bind(year) + .fetch_one(&mut *tx) + .await? + } + }; + + // If there is already an existing grade record for the current combination of subject, + // season and year, then we skip. + if grade.has_previously_been_graded(key) { + return Ok(grade); + } + + let subject = + sqlx::query_as::<_, Subject>(r#"SELECT * FROM subject WHERE id = $1 FOR UPDATE;"#) + .bind(grade.subject_id) + .fetch_one(&mut *tx) + .await?; + + sqlx::query_as::<_, Subject>( + r#" + UPDATE subject SET total_students = $1, average_grade = $2, failed_students = $3 WHERE id = $4; + "# + ) + .bind(subject.get_next_total_students(count)) + .bind(subject.get_next_average(key, count)) + .bind(subject.get_next_failed_students(key, count)) + .bind(subject.id) + .fetch_optional(&mut *tx) + .await?; + + let grade = sqlx::query_as::<_, Grade>(&format!( + r#" + UPDATE subject_season_grade SET {} = $1 + WHERE id = $2 + RETURNING *; + "#, + key.to_column_key_name() + )) + .bind(count) + .bind(grade.id) + .fetch_one(&mut *tx) + .await; + + tx.commit().await?; + grade + } + + async fn find_grade_by_season( + &self, + subject_id: Uuid, + season: SubjectGradingSeason, + year: i32, + ) -> Result, sqlx::Error> { + sqlx::query_as::<_, Grade>( + r#" + SELECT * FROM subject_season_grade WHERE subject_id = $1 AND season = $2 AND year = $3; + "#, + ) + .bind(subject_id) + .bind(season) + .bind(year) + .fetch_optional(self.db) + .await + } +} diff --git a/apps/grades-sync/src/hkdir.rs b/apps/grades-sync/src/hkdir.rs index 7bbbd4965..9691c92cd 100644 --- a/apps/grades-sync/src/hkdir.rs +++ b/apps/grades-sync/src/hkdir.rs @@ -1,105 +1,14 @@ -use crate::json::{HkdirDepartment, HkdirSubject}; +use crate::grade_repository::SubjectGradingSeason; +use crate::json::{HkdirDepartment, HkdirGrade, HkdirSubject}; use reqwest::Client; -use serde_json::{json, Value}; +use serde_json::Value; const HKDIR_API_URL: &str = "https://dbh.hkdir.no/api/Tabeller/hentJSONTabellData"; -fn build_get_departments_request() -> Value { - let json = json!({ - "tabell_id": 210, - "api_versjon": 1, - "statuslinje": "N", - "kodetekst": "J", - "desimal_separator": ".", - "variabler": ["*"], - "sortBy": ["Nivå"], - "filter": [ - { - "variabel": "Institusjonskode", - "selection": { - "filter": "item", - "values": ["1150"], - "exclude": [""], - }, - }, - { - "variabel": "Avdelingskode", - "selection": { - "filter": "all", - "values": ["*"], - "exclude": ["000000"], - }, - }, - ], - }); - json -} - -fn build_get_subjects_request() -> Value { - let json = json!({ - "tabell_id": 208, - "api_versjon": 1, - "statuslinje": "N", - "kodetekst": "J", - "desimal_separator": ".", - "variabler": ["*"], - "sortBy": ["Årstall", "Institusjonskode", "Avdelingskode"], - "filter": [ - { - "variabel": "Institusjonskode", - "selection": { - "filter": "item", - "values": ["1150"], - "exclude": [""], - }, - }, - { - "variabel": "Nivåkode", - "selection": { - "filter": "item", - "values": ["HN", "LN"], - "exclude": [""], - }, - }, - { - "variabel": "Status", - "selection": { - "filter": "item", - "values": ["1", "2"], - "exclude": [""], - }, - }, - { - "variabel": "Avdelingskode", - "selection": { - "filter": "all", - "values": ["*"], - "exclude": ["000000"], - }, - }, - { - "variabel": "Oppgave (ny fra h2012)", - "selection": { - "filter": "all", - "values": ["*"], - "exclude": ["1", "2"], - }, - }, - { - "variabel": "Årstall", - "selection": { - "filter": "top", - "values": ["5"], - "exclude": [""], - }, - } - ], - }); - json -} - pub async fn get_departments() -> reqwest::Result> { - let request_body = build_get_departments_request(); + let request_body = include_str!("../queries/departments.json") + .parse::() + .expect("invalid json"); let client = Client::new(); let response = client .post(HKDIR_API_URL) @@ -110,7 +19,9 @@ pub async fn get_departments() -> reqwest::Result> { } pub async fn get_subjects() -> reqwest::Result> { - let request_body = build_get_subjects_request(); + let request_body = include_str!("../queries/subjects.json") + .parse::() + .expect("invalid json"); let client = Client::new(); let response = client .post(HKDIR_API_URL) @@ -119,3 +30,26 @@ pub async fn get_subjects() -> reqwest::Result> { .await?; response.json::>().await } + +pub async fn get_grades() -> reqwest::Result> { + let request_body = include_str!("../queries/grades.json") + .parse::() + .expect("invalid json"); + let client = Client::new(); + let response = client + .post(HKDIR_API_URL) + .json(&request_body) + .send() + .await?; + response.json::>().await +} + +pub fn map_season_index_to(index: &str) -> SubjectGradingSeason { + match index { + "0" => SubjectGradingSeason::Winter, + "1" => SubjectGradingSeason::Spring, + "2" => SubjectGradingSeason::Summer, + "3" => SubjectGradingSeason::Autumn, + x => panic!("attempted to parse invalid season {}", x), + } +} diff --git a/apps/grades-sync/src/job.rs b/apps/grades-sync/src/job.rs index 671998135..e34c50c33 100644 --- a/apps/grades-sync/src/job.rs +++ b/apps/grades-sync/src/job.rs @@ -1,13 +1,13 @@ -use std::cmp::{min}; -use crate::faculty_repository::{FacultyRepository}; -use crate::hkdir::{get_departments, get_subjects}; +use crate::faculty_repository::FacultyRepository; +use crate::hkdir::{get_departments, get_grades, get_subjects, map_season_index_to}; use async_trait::async_trait; use itertools::Itertools; +use std::cmp::min; - -use crate::department_repository::{DepartmentRepository}; -use crate::json::{HkdirDepartment, HkdirSubject}; -use crate::subject_repository::{SubjectRepository}; +use crate::department_repository::DepartmentRepository; +use crate::grade_repository::GradeRepository; +use crate::json::{HkdirDepartment, HkdirGrade, HkdirSubject}; +use crate::subject_repository::SubjectRepository; use log::info; use regex::Regex; @@ -19,12 +19,14 @@ pub trait JobService: Sync { async fn synchronize_single_faculty(&self, faculty: HkdirDepartment) -> anyhow::Result<()>; async fn synchronize_single_subject(&self, subject: HkdirSubject) -> anyhow::Result<()>; + async fn synchronize_single_grade(&self, grade: HkdirGrade) -> anyhow::Result<()>; } pub struct JobServiceImpl<'a> { faculty_repository: &'a dyn FacultyRepository, department_repository: &'a dyn DepartmentRepository, subject_repository: &'a dyn SubjectRepository, + grade_repository: &'a dyn GradeRepository, } impl<'a> JobServiceImpl<'a> { @@ -32,11 +34,13 @@ impl<'a> JobServiceImpl<'a> { faculty_repository: &'a dyn FacultyRepository, department_repository: &'a dyn DepartmentRepository, subject_repository: &'a dyn SubjectRepository, + grade_repository: &'a dyn GradeRepository, ) -> Self { Self { faculty_repository, department_repository, subject_repository, + grade_repository, } } } @@ -52,8 +56,7 @@ impl<'a> JobService for JobServiceImpl<'a> { let chunks_count = min(department_count / MAX_TASK_COUNT, 1000); info!( "performing synchronization for {} departments across {} threads", - department_count, - chunks_count + department_count, chunks_count ); let departments_chunked = departments @@ -82,8 +85,7 @@ impl<'a> JobService for JobServiceImpl<'a> { let chunks_count = min(subject_count / MAX_TASK_COUNT, 1000); info!( "performing synchronization for {} subjects across {} threads", - subject_count, - chunks_count + subject_count, chunks_count ); let subjects_chunked = subjects @@ -107,6 +109,46 @@ impl<'a> JobService for JobServiceImpl<'a> { } async fn perform_grade_synchronization(&self) -> anyhow::Result<()> { + info!("performing grade synchronization"); + let grades = get_grades().await?; + // Here, we also skip the grades where the number of students is 0. + let grades = grades + .into_iter() + .filter(|grade| { + grade + .total_candidates + .parse::() + .expect("invalid number for total candidates") + > 0 + }) + .collect::>(); + let grade_count = grades.len(); + let chunks_count = min(grade_count / MAX_TASK_COUNT, 1000); + info!( + "performing synchronization for {} grades across {} threads", + grade_count, chunks_count + ); + + let grades_chunked = grades + .into_iter() + .chunks(chunks_count) + .into_iter() + .map(|chunk| chunk.collect()) + .collect::>>(); + + async_scoped::TokioScope::scope_and_block(|s| { + for grades in grades_chunked { + s.spawn(async move { + for grade in grades { + self.synchronize_single_grade(grade) + .await + .expect("failed to synchronize grade"); + } + }); + } + }); + + info!("synchronization complete"); Ok(()) } @@ -165,4 +207,33 @@ impl<'a> JobService for JobServiceImpl<'a> { .unwrap(); Ok(()) } + + async fn synchronize_single_grade(&self, grade: HkdirGrade) -> anyhow::Result<()> { + // First we find the matching subject, or else we return early. + let subject = match self + .subject_repository + .find_subject_by_ref_id(grade.subject_code) + .await? + { + Some(subject) => subject, + None => return Ok(()), + }; + let year = grade.year.parse::().unwrap(); + let season_key = map_season_index_to(grade.semester.as_str()); + // Then we find a matching grade record. + self.grade_repository + .update_grade_record( + subject.id, + season_key, + year, + grade.grade, + grade + .total_candidates + .parse::() + .expect("invalid number for total candidates"), + ) + .await?; + + Ok(()) + } } diff --git a/apps/grades-sync/src/json.rs b/apps/grades-sync/src/json.rs index 8756424bd..bdf049107 100644 --- a/apps/grades-sync/src/json.rs +++ b/apps/grades-sync/src/json.rs @@ -1,3 +1,4 @@ +use crate::grade_repository::SubjectGradingKey; use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] @@ -81,4 +82,17 @@ pub struct HkdirSubject { } #[derive(Serialize, Deserialize)] -pub struct HkdirGrade {} +pub struct HkdirGrade { + #[serde(rename = "Årstall")] + pub year: String, + #[serde(rename = "Semester")] + pub semester: String, + #[serde(rename = "Semesternavn")] + pub semester_name: String, + #[serde(rename = "Karakter")] + pub grade: SubjectGradingKey, + #[serde(rename = "Emnekode")] + pub subject_code: String, + #[serde(rename = "Antall kandidater totalt")] + pub total_candidates: String, +} diff --git a/apps/grades-sync/src/main.rs b/apps/grades-sync/src/main.rs index fd46b1ff3..fd3f4dc8b 100644 --- a/apps/grades-sync/src/main.rs +++ b/apps/grades-sync/src/main.rs @@ -1,10 +1,12 @@ use crate::department_repository::DepartmentRepositoryImpl; use crate::faculty_repository::FacultyRepositoryImpl; +use crate::grade_repository::GradeRepositoryImpl; use crate::job::{JobService, JobServiceImpl}; use crate::subject_repository::SubjectRepositoryImpl; mod department_repository; mod faculty_repository; +mod grade_repository; mod hkdir; mod job; mod json; @@ -23,14 +25,17 @@ async fn main() { let faculty_repository = FacultyRepositoryImpl::new(&pool); let department_repository = DepartmentRepositoryImpl::new(&pool); let subject_repository = SubjectRepositoryImpl::new(&pool); + let grade_repository = GradeRepositoryImpl::new(&pool); let job_service = JobServiceImpl::new( &faculty_repository, &department_repository, &subject_repository, + &grade_repository, ); - job_service.perform_faculty_synchronization().await.unwrap(); - job_service.perform_subject_synchronization().await.unwrap(); + // job_service.perform_faculty_synchronization().await.unwrap(); + // job_service.perform_subject_synchronization().await.unwrap(); + job_service.perform_grade_synchronization().await.unwrap(); pool.close().await; } diff --git a/apps/grades-sync/src/pg.rs b/apps/grades-sync/src/pg.rs index 644a5e5b6..f62682a04 100644 --- a/apps/grades-sync/src/pg.rs +++ b/apps/grades-sync/src/pg.rs @@ -13,7 +13,7 @@ pub async fn create_postgres_pool() -> Result, sqlx::Error> { .log_statements(LevelFilter::Debug) .log_slow_statements(LevelFilter::Warn, Duration::from_secs(1)); let pool = PgPoolOptions::new() - .max_connections(100) + .max_connections(80) .connect_with(opts) .await?; Ok(pool) diff --git a/apps/grades-sync/src/subject_repository.rs b/apps/grades-sync/src/subject_repository.rs index 0a2105cce..e267007d4 100644 --- a/apps/grades-sync/src/subject_repository.rs +++ b/apps/grades-sync/src/subject_repository.rs @@ -1,7 +1,8 @@ +use crate::grade_repository::SubjectGradingKey; use crate::pg::Database; use async_trait::async_trait; use sqlx::types::Uuid; -use sqlx::FromRow; +use sqlx::{Error, FromRow}; #[derive(Debug, FromRow)] pub struct Subject { @@ -18,6 +19,29 @@ pub struct Subject { pub failed_students: i32, } +impl Subject { + pub fn get_next_average(&self, key: SubjectGradingKey, count: i32) -> f32 { + if key.is_pass_or_fail_key() { + return self.average_grade; + } + + let complete_factor = self.average_grade * self.total_students as f32 + + count as f32 * key.to_multiplication_factor(); + complete_factor / (self.total_students + count) as f32 + } + + pub fn get_next_failed_students(&self, key: SubjectGradingKey, count: i32) -> i32 { + if key.is_evaluated_as_failed() { + return self.failed_students + count; + } + self.failed_students + } + + pub fn get_next_total_students(&self, count: i32) -> i32 { + self.total_students + count + } +} + #[async_trait] pub trait SubjectRepository: Sync { async fn create_subject( @@ -33,6 +57,7 @@ pub trait SubjectRepository: Sync { total_students: i32, failed_students: i32, ) -> Result; + async fn find_subject_by_ref_id(&self, ref_id: String) -> Result, sqlx::Error>; } pub struct SubjectRepositoryImpl<'a> { @@ -81,4 +106,15 @@ impl<'a> SubjectRepository for SubjectRepositoryImpl<'a> { .fetch_one(self.db) .await } + + async fn find_subject_by_ref_id(&self, ref_id: String) -> Result, Error> { + sqlx::query_as::<_, Subject>( + r#" + SELECT * FROM subject WHERE ref_id = $1; + "#, + ) + .bind(ref_id) + .fetch_optional(self.db) + .await + } } From fedb20b9871b028ae8e0bc6e4d1d0c7b60854c73 Mon Sep 17 00:00:00 2001 From: Mats Larsen Date: Sat, 10 Feb 2024 14:34:48 +0100 Subject: [PATCH 06/23] Fix group by chunking --- apps/grades-sync/src/grade_repository.rs | 5 +++-- apps/grades-sync/src/job.rs | 18 +++++++++++++++--- apps/grades-sync/src/json.rs | 2 +- apps/grades-sync/src/main.rs | 2 +- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/apps/grades-sync/src/grade_repository.rs b/apps/grades-sync/src/grade_repository.rs index 0b7d8dd6f..77eb4dae8 100644 --- a/apps/grades-sync/src/grade_repository.rs +++ b/apps/grades-sync/src/grade_repository.rs @@ -3,7 +3,7 @@ use crate::subject_repository::Subject; use async_trait::async_trait; use serde::{Deserialize, Serialize}; use sqlx::types::Uuid; -use sqlx::FromRow; +use sqlx::{Executor, FromRow}; #[derive(Debug, PartialEq, PartialOrd, sqlx::Type, Deserialize, Serialize, Copy, Clone)] #[sqlx(type_name = "subject_grading_season", rename_all = "UPPERCASE")] @@ -179,6 +179,7 @@ impl<'a> GradeRepository for GradeRepositoryImpl<'a> { ) -> Result { // TODO: Refactor this logic out, it's only here because it's a transaction right now let mut tx = self.db.begin().await?; + tx.execute("SET TRANSACTION ISOLATION LEVEL READ COMMITTED;").await?; let grade = sqlx::query_as::<_, Grade>( r#" SELECT * FROM subject_season_grade WHERE subject_id = $1 AND season = $2 AND year = $3 @@ -197,7 +198,7 @@ impl<'a> GradeRepository for GradeRepositoryImpl<'a> { sqlx::query_as::<_, Grade>( r#" INSERT INTO subject_season_grade (subject_id, season, year, graded_a, graded_b, graded_c, graded_d, graded_f, graded_pass, graded_fail) - VALUES ($1, $2, $3, 0, 0, 0, 0, 0, 0, 0) + VALUES ($1, $2, $3, NULL, NULL, NULL, NULL, NULL, NULL, NULL) RETURNING *; "# ).bind(subject_id) diff --git a/apps/grades-sync/src/job.rs b/apps/grades-sync/src/job.rs index e34c50c33..5611c41c8 100644 --- a/apps/grades-sync/src/job.rs +++ b/apps/grades-sync/src/job.rs @@ -1,9 +1,11 @@ use crate::faculty_repository::FacultyRepository; use crate::hkdir::{get_departments, get_grades, get_subjects, map_season_index_to}; use async_trait::async_trait; -use itertools::Itertools; +use itertools::{Itertools}; use std::cmp::min; + + use crate::department_repository::DepartmentRepository; use crate::grade_repository::GradeRepository; use crate::json::{HkdirDepartment, HkdirGrade, HkdirSubject}; @@ -129,11 +131,21 @@ impl<'a> JobService for JobServiceImpl<'a> { grade_count, chunks_count ); - let grades_chunked = grades + let grades_grouped_by_subject = grades + .into_iter() + // Group all the grades by the subject code to prevent them to be split across threads + .into_grouping_map_by(|grade| grade.subject_code.clone()) + .collect::>() + .values() + .cloned() + .collect::>>(); + + let grades_chunked = grades_grouped_by_subject .into_iter() + // Split the chunks into smaller chunks, each of which is to be put in a separate worker .chunks(chunks_count) .into_iter() - .map(|chunk| chunk.collect()) + .map(|chunk| chunk.flatten().collect()) .collect::>>(); async_scoped::TokioScope::scope_and_block(|s| { diff --git a/apps/grades-sync/src/json.rs b/apps/grades-sync/src/json.rs index bdf049107..21e492135 100644 --- a/apps/grades-sync/src/json.rs +++ b/apps/grades-sync/src/json.rs @@ -81,7 +81,7 @@ pub struct HkdirSubject { pub task: Option, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Clone)] pub struct HkdirGrade { #[serde(rename = "Årstall")] pub year: String, diff --git a/apps/grades-sync/src/main.rs b/apps/grades-sync/src/main.rs index fd3f4dc8b..4c4342e0b 100644 --- a/apps/grades-sync/src/main.rs +++ b/apps/grades-sync/src/main.rs @@ -34,7 +34,7 @@ async fn main() { ); // job_service.perform_faculty_synchronization().await.unwrap(); - // job_service.perform_subject_synchronization().await.unwrap(); + job_service.perform_subject_synchronization().await.unwrap(); job_service.perform_grade_synchronization().await.unwrap(); pool.close().await; From 8f623bf61e02e9ade43fce00552b3b0968d87509 Mon Sep 17 00:00:00 2001 From: Mats Larsen Date: Sat, 10 Feb 2024 14:58:00 +0100 Subject: [PATCH 07/23] Replace theading with futures join_all --- apps/grades-sync/Cargo.lock | 33 +---------- apps/grades-sync/Cargo.toml | 2 +- apps/grades-sync/src/job.rs | 110 ++++++++++-------------------------- 3 files changed, 33 insertions(+), 112 deletions(-) diff --git a/apps/grades-sync/Cargo.lock b/apps/grades-sync/Cargo.lock index 07d39f46a..5607890f8 100644 --- a/apps/grades-sync/Cargo.lock +++ b/apps/grades-sync/Cargo.lock @@ -99,17 +99,6 @@ version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" -[[package]] -name = "async-scoped" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4042078ea593edffc452eef14e99fdb2b120caa4ad9618bcdeabc4a023b98740" -dependencies = [ - "futures", - "pin-project", - "tokio", -] - [[package]] name = "async-trait" version = "0.1.77" @@ -601,10 +590,10 @@ name = "grades-sync" version = "0.1.0" dependencies = [ "anyhow", - "async-scoped", "async-trait", "dotenv", "env_logger", + "futures", "itertools", "log", "regex", @@ -1129,26 +1118,6 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" -[[package]] -name = "pin-project" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", -] - [[package]] name = "pin-project-lite" version = "0.2.13" diff --git a/apps/grades-sync/Cargo.toml b/apps/grades-sync/Cargo.toml index 365934377..baca1f6dc 100644 --- a/apps/grades-sync/Cargo.toml +++ b/apps/grades-sync/Cargo.toml @@ -16,6 +16,6 @@ dotenv = "0.15.0" log = { version = "0.4.20", features = [] } env_logger = "0.11.1" async-trait = "0.1.77" -async-scoped = { version = "0.9.0", features = ["use-tokio"] } regex = "1.10.3" itertools = "0.12.1" +futures = "0.3.30" diff --git a/apps/grades-sync/src/job.rs b/apps/grades-sync/src/job.rs index 5611c41c8..e700ecd67 100644 --- a/apps/grades-sync/src/job.rs +++ b/apps/grades-sync/src/job.rs @@ -4,8 +4,6 @@ use async_trait::async_trait; use itertools::{Itertools}; use std::cmp::min; - - use crate::department_repository::DepartmentRepository; use crate::grade_repository::GradeRepository; use crate::json::{HkdirDepartment, HkdirGrade, HkdirSubject}; @@ -55,27 +53,18 @@ impl<'a> JobService for JobServiceImpl<'a> { info!("performing faculty synchronization"); let departments = get_departments().await?; let department_count = departments.len(); - let chunks_count = min(department_count / MAX_TASK_COUNT, 1000); + let _chunks_count = min(department_count / MAX_TASK_COUNT, 1000); info!( - "performing synchronization for {} departments across {} threads", - department_count, chunks_count + "performing synchronization for {} departments", department_count ); - let departments_chunked = departments - .into_iter() - .chunks(chunks_count) + futures::future::join_all(departments .into_iter() - .map(|chunk| chunk.collect()) - .collect::>>(); - async_scoped::TokioScope::scope_and_block(|s| { - for departments in departments_chunked { - s.spawn(async move { - for department in departments { - self.synchronize_single_faculty(department).await.unwrap(); - } - }); - } - }); + .map(|department| { + async move { + self.synchronize_single_faculty(department).await.unwrap(); + } + })).await; info!("synchronization complete"); Ok(()) } @@ -84,28 +73,18 @@ impl<'a> JobService for JobServiceImpl<'a> { info!("performing subject synchronization"); let subjects = get_subjects().await?; let subject_count = subjects.len(); - let chunks_count = min(subject_count / MAX_TASK_COUNT, 1000); info!( - "performing synchronization for {} subjects across {} threads", - subject_count, chunks_count + "performing synchronization for {} subjects", + subject_count ); - let subjects_chunked = subjects - .into_iter() - .chunks(chunks_count) + futures::future::join_all(subjects .into_iter() - .map(|chunk| chunk.collect()) - .collect::>>(); - async_scoped::TokioScope::scope_and_block(|s| { - for subjects in subjects_chunked { - s.spawn(async move { - for subject in subjects { - self.synchronize_single_subject(subject).await.unwrap(); - } - }); - } - }); - + .map(|subject| { + async move { + self.synchronize_single_subject(subject).await.unwrap(); + } + })).await; info!("synchronization complete"); Ok(()) } @@ -113,53 +92,26 @@ impl<'a> JobService for JobServiceImpl<'a> { async fn perform_grade_synchronization(&self) -> anyhow::Result<()> { info!("performing grade synchronization"); let grades = get_grades().await?; - // Here, we also skip the grades where the number of students is 0. - let grades = grades - .into_iter() - .filter(|grade| { - grade - .total_candidates - .parse::() - .expect("invalid number for total candidates") - > 0 - }) - .collect::>(); - let grade_count = grades.len(); - let chunks_count = min(grade_count / MAX_TASK_COUNT, 1000); info!( - "performing synchronization for {} grades across {} threads", - grade_count, chunks_count + "performing synchronization for {} grades", + grades.len() ); - - let grades_grouped_by_subject = grades - .into_iter() - // Group all the grades by the subject code to prevent them to be split across threads + let map = grades.into_iter() + .filter(|grade| grade + .total_candidates + .parse::() + .expect("invalid number for total candidates") > 0 + ) .into_grouping_map_by(|grade| grade.subject_code.clone()) - .collect::>() - .values() - .cloned() - .collect::>>(); - - let grades_chunked = grades_grouped_by_subject - .into_iter() - // Split the chunks into smaller chunks, each of which is to be put in a separate worker - .chunks(chunks_count) - .into_iter() - .map(|chunk| chunk.flatten().collect()) - .collect::>>(); - - async_scoped::TokioScope::scope_and_block(|s| { - for grades in grades_chunked { - s.spawn(async move { + // TODO: Maybe there is a way to avoid this collect? + .collect::>(); + futures::future::join_all(map.into_values().map(|grades| { + async move { for grade in grades { - self.synchronize_single_grade(grade) - .await - .expect("failed to synchronize grade"); + self.synchronize_single_grade(grade).await.unwrap(); } - }); - } - }); - + } + })).await; info!("synchronization complete"); Ok(()) } From 869f99d80be9effbe5404e06b1bf8a541c83a7f8 Mon Sep 17 00:00:00 2001 From: Mats Larsen Date: Sat, 10 Feb 2024 15:54:19 +0100 Subject: [PATCH 08/23] Synchronize everything since 2004 --- apps/grades-sync/queries/grades.json | 13 ------------- apps/grades-sync/queries/subjects.json | 25 ------------------------- apps/grades-sync/src/json.rs | 2 +- apps/grades-sync/src/pg.rs | 8 +++++++- 4 files changed, 8 insertions(+), 40 deletions(-) diff --git a/apps/grades-sync/queries/grades.json b/apps/grades-sync/queries/grades.json index 120efbd59..be027787d 100644 --- a/apps/grades-sync/queries/grades.json +++ b/apps/grades-sync/queries/grades.json @@ -50,19 +50,6 @@ "" ] } - }, - - { - "variabel": "Årstall", - "selection": { - "filter": "top", - "values": [ - "2" - ], - "exclude": [ - "" - ] - } } ] } \ No newline at end of file diff --git a/apps/grades-sync/queries/subjects.json b/apps/grades-sync/queries/subjects.json index 3d0682bad..6124a9ff2 100644 --- a/apps/grades-sync/queries/subjects.json +++ b/apps/grades-sync/queries/subjects.json @@ -38,19 +38,6 @@ ] } }, - { - "variabel": "Status", - "selection": { - "filter": "item", - "values": [ - "1", - "2" - ], - "exclude": [ - "" - ] - } - }, { "variabel": "Avdelingskode", "selection": { @@ -75,18 +62,6 @@ "2" ] } - }, - { - "variabel": "Årstall", - "selection": { - "filter": "top", - "values": [ - "2" - ], - "exclude": [ - "" - ] - } } ] } \ No newline at end of file diff --git a/apps/grades-sync/src/json.rs b/apps/grades-sync/src/json.rs index 21e492135..67aa7ddb2 100644 --- a/apps/grades-sync/src/json.rs +++ b/apps/grades-sync/src/json.rs @@ -42,7 +42,7 @@ pub struct HkdirSubject { #[serde(rename = "Avdelingsnavn")] pub department_name: String, #[serde(rename = "Avdelingskode_SSB")] - pub department_code_ssb: String, + pub department_code_ssb: Option, #[serde(rename = "Årstall")] pub year: String, #[serde(rename = "Semester")] diff --git a/apps/grades-sync/src/pg.rs b/apps/grades-sync/src/pg.rs index f62682a04..799ac090f 100644 --- a/apps/grades-sync/src/pg.rs +++ b/apps/grades-sync/src/pg.rs @@ -8,12 +8,18 @@ pub type Database = Pool; pub async fn create_postgres_pool() -> Result, sqlx::Error> { let database_url = std::env::var("DATABASE_URL").expect("missing DATABASE_URL environment variable"); + let max_connections = std::env::var("MAX_CONNECTIONS") + .unwrap_or_else(|_| "10".to_string()) + .parse::() + .expect("MAX_CONNECTIONS must be a number"); + let opts: PgConnectOptions = database_url.parse()?; let opts = opts .log_statements(LevelFilter::Debug) .log_slow_statements(LevelFilter::Warn, Duration::from_secs(1)); let pool = PgPoolOptions::new() - .max_connections(80) + .max_connections(max_connections) + .acquire_timeout(Duration::from_secs(10 * 60)) .connect_with(opts) .await?; Ok(pool) From 7588becd7ad19bb526e6a28a0d527d5fe2d89e8d Mon Sep 17 00:00:00 2001 From: Mats Larsen Date: Wed, 7 Feb 2024 19:31:12 +0100 Subject: [PATCH 09/23] Bootstrap Next.js application --- apps/grades/.eslintrc.cjs | 6 + apps/grades/.gitignore | 36 +++ apps/grades/README.md | 36 +++ apps/grades/next.config.mjs | 4 + apps/grades/package.json | 29 +++ apps/grades/postcss.config.js | 6 + apps/grades/public/next.svg | 1 + apps/grades/public/vercel.svg | 1 + apps/grades/src/app/favicon.ico | Bin 0 -> 25931 bytes apps/grades/src/app/globals.css | 33 +++ apps/grades/src/app/layout.tsx | 22 ++ apps/grades/src/app/page.tsx | 103 ++++++++ apps/grades/tailwind.config.ts | 19 ++ apps/grades/tsconfig.json | 26 +++ pnpm-lock.yaml | 402 ++++++++++++++++++++++++++------ 15 files changed, 657 insertions(+), 67 deletions(-) create mode 100644 apps/grades/.eslintrc.cjs create mode 100644 apps/grades/.gitignore create mode 100644 apps/grades/README.md create mode 100644 apps/grades/next.config.mjs create mode 100644 apps/grades/package.json create mode 100644 apps/grades/postcss.config.js create mode 100644 apps/grades/public/next.svg create mode 100644 apps/grades/public/vercel.svg create mode 100644 apps/grades/src/app/favicon.ico create mode 100644 apps/grades/src/app/globals.css create mode 100644 apps/grades/src/app/layout.tsx create mode 100644 apps/grades/src/app/page.tsx create mode 100644 apps/grades/tailwind.config.ts create mode 100644 apps/grades/tsconfig.json diff --git a/apps/grades/.eslintrc.cjs b/apps/grades/.eslintrc.cjs new file mode 100644 index 000000000..cfe9f4a1e --- /dev/null +++ b/apps/grades/.eslintrc.cjs @@ -0,0 +1,6 @@ +const config = require("@dotkomonline/config/eslint-preset") + +module.exports = { + ...config, + extends: [...config.extends, "next"], +} diff --git a/apps/grades/.gitignore b/apps/grades/.gitignore new file mode 100644 index 000000000..fd3dbb571 --- /dev/null +++ b/apps/grades/.gitignore @@ -0,0 +1,36 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +/node_modules +/.pnp +.pnp.js +.yarn/install-state.gz + +# testing +/coverage + +# next.js +/.next/ +/out/ + +# production +/build + +# misc +.DS_Store +*.pem + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# local env files +.env*.local + +# vercel +.vercel + +# typescript +*.tsbuildinfo +next-env.d.ts diff --git a/apps/grades/README.md b/apps/grades/README.md new file mode 100644 index 000000000..c4033664f --- /dev/null +++ b/apps/grades/README.md @@ -0,0 +1,36 @@ +This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app). + +## Getting Started + +First, run the development server: + +```bash +npm run dev +# or +yarn dev +# or +pnpm dev +# or +bun dev +``` + +Open [http://localhost:3000](http://localhost:3000) with your browser to see the result. + +You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file. + +This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font. + +## Learn More + +To learn more about Next.js, take a look at the following resources: + +- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API. +- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial. + +You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome! + +## Deploy on Vercel + +The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js. + +Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details. diff --git a/apps/grades/next.config.mjs b/apps/grades/next.config.mjs new file mode 100644 index 000000000..4678774e6 --- /dev/null +++ b/apps/grades/next.config.mjs @@ -0,0 +1,4 @@ +/** @type {import('next').NextConfig} */ +const nextConfig = {}; + +export default nextConfig; diff --git a/apps/grades/package.json b/apps/grades/package.json new file mode 100644 index 000000000..3df68b49f --- /dev/null +++ b/apps/grades/package.json @@ -0,0 +1,29 @@ +{ + "name": "@dotkomonline/grades", + "version": "0.1.0", + "private": true, + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start", + "lint": "eslint --max-warnings 0 .", + "lint:fix": "eslint --fix .", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "react": "^18", + "react-dom": "^18", + "next": "14.1.0" + }, + "devDependencies": { + "@dotkomonline/config": "workspace:*", + "@types/node": "^20", + "@types/react": "^18", + "@types/react-dom": "^18", + "eslint": "^8.56.0", + "autoprefixer": "^10.0.1", + "postcss": "^8", + "tailwindcss": "^3.3.0", + "typescript": "^5" + } +} diff --git a/apps/grades/postcss.config.js b/apps/grades/postcss.config.js new file mode 100644 index 000000000..33ad091d2 --- /dev/null +++ b/apps/grades/postcss.config.js @@ -0,0 +1,6 @@ +module.exports = { + plugins: { + tailwindcss: {}, + autoprefixer: {}, + }, +} diff --git a/apps/grades/public/next.svg b/apps/grades/public/next.svg new file mode 100644 index 000000000..5174b28c5 --- /dev/null +++ b/apps/grades/public/next.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/apps/grades/public/vercel.svg b/apps/grades/public/vercel.svg new file mode 100644 index 000000000..d2f842227 --- /dev/null +++ b/apps/grades/public/vercel.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/apps/grades/src/app/favicon.ico b/apps/grades/src/app/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..718d6fea4835ec2d246af9800eddb7ffb276240c GIT binary patch literal 25931 zcmeHv30#a{`}aL_*G&7qml|y<+KVaDM2m#dVr!KsA!#An?kSQM(q<_dDNCpjEux83 zLb9Z^XxbDl(w>%i@8hT6>)&Gu{h#Oeyszu?xtw#Zb1mO{pgX9699l+Qppw7jXaYf~-84xW z)w4x8?=youko|}Vr~(D$UXIbiXABHh`p1?nn8Po~fxRJv}|0e(BPs|G`(TT%kKVJAdg5*Z|x0leQq0 zkdUBvb#>9F()jo|T~kx@OM8$9wzs~t2l;K=woNssA3l6|sx2r3+kdfVW@e^8e*E}v zA1y5{bRi+3Z`uD3{F7LgFJDdvm;nJilkzDku>BwXH(8ItVCXk*-lSJnR?-2UN%hJ){&rlvg`CDTj z)Bzo!3v7Ou#83zEDEFcKt(f1E0~=rqeEbTnMvWR#{+9pg%7G8y>u1OVRUSoox-ovF z2Ydma(;=YuBY(eI|04{hXzZD6_f(v~H;C~y5=DhAC{MMS>2fm~1H_t2$56pc$NH8( z5bH|<)71dV-_oCHIrzrT`2s-5w_+2CM0$95I6X8p^r!gHp+j_gd;9O<1~CEQQGS8) zS9Qh3#p&JM-G8rHekNmKVewU;pJRcTAog68KYo^dRo}(M>36U4Us zfgYWSiHZL3;lpWT=zNAW>Dh#mB!_@Lg%$ms8N-;aPqMn+C2HqZgz&9~Eu z4|Kp<`$q)Uw1R?y(~S>ePdonHxpV1#eSP1B;Ogo+-Pk}6#0GsZZ5!||ev2MGdh}_m z{DeR7?0-1^zVs&`AV6Vt;r3`I`OI_wgs*w=eO%_#7Kepl{B@xiyCANc(l zzIyd4y|c6PXWq9-|KM8(zIk8LPk(>a)zyFWjhT!$HJ$qX1vo@d25W<fvZQ2zUz5WRc(UnFMKHwe1| zWmlB1qdbiA(C0jmnV<}GfbKtmcu^2*P^O?MBLZKt|As~ge8&AAO~2K@zbXelK|4T<{|y4`raF{=72kC2Kn(L4YyenWgrPiv z@^mr$t{#X5VuIMeL!7Ab6_kG$&#&5p*Z{+?5U|TZ`B!7llpVmp@skYz&n^8QfPJzL z0G6K_OJM9x+Wu2gfN45phANGt{7=C>i34CV{Xqlx(fWpeAoj^N0Biu`w+MVcCUyU* zDZuzO0>4Z6fbu^T_arWW5n!E45vX8N=bxTVeFoep_G#VmNlQzAI_KTIc{6>c+04vr zx@W}zE5JNSU>!THJ{J=cqjz+4{L4A{Ob9$ZJ*S1?Ggg3klFp!+Y1@K+pK1DqI|_gq z5ZDXVpge8-cs!o|;K73#YXZ3AShj50wBvuq3NTOZ`M&qtjj#GOFfgExjg8Gn8>Vq5 z`85n+9|!iLCZF5$HJ$Iu($dm?8~-ofu}tEc+-pyke=3!im#6pk_Wo8IA|fJwD&~~F zc16osQ)EBo58U7XDuMexaPRjU@h8tXe%S{fA0NH3vGJFhuyyO!Uyl2^&EOpX{9As0 zWj+P>{@}jxH)8|r;2HdupP!vie{sJ28b&bo!8`D^x}TE$%zXNb^X1p@0PJ86`dZyj z%ce7*{^oo+6%&~I!8hQy-vQ7E)0t0ybH4l%KltWOo~8cO`T=157JqL(oq_rC%ea&4 z2NcTJe-HgFjNg-gZ$6!Y`SMHrlj}Etf7?r!zQTPPSv}{so2e>Fjs1{gzk~LGeesX%r(Lh6rbhSo_n)@@G-FTQy93;l#E)hgP@d_SGvyCp0~o(Y;Ee8{ zdVUDbHm5`2taPUOY^MAGOw*>=s7=Gst=D+p+2yON!0%Hk` zz5mAhyT4lS*T3LS^WSxUy86q&GnoHxzQ6vm8)VS}_zuqG?+3td68_x;etQAdu@sc6 zQJ&5|4(I?~3d-QOAODHpZ=hlSg(lBZ!JZWCtHHSj`0Wh93-Uk)_S%zsJ~aD>{`A0~ z9{AG(e|q3g5B%wYKRxiL2Y$8(4w6bzchKuloQW#e&S3n+P- z8!ds-%f;TJ1>)v)##>gd{PdS2Oc3VaR`fr=`O8QIO(6(N!A?pr5C#6fc~Ge@N%Vvu zaoAX2&(a6eWy_q&UwOhU)|P3J0Qc%OdhzW=F4D|pt0E4osw;%<%Dn58hAWD^XnZD= z>9~H(3bmLtxpF?a7su6J7M*x1By7YSUbxGi)Ot0P77`}P3{)&5Un{KD?`-e?r21!4vTTnN(4Y6Lin?UkSM z`MXCTC1@4A4~mvz%Rh2&EwY))LeoT=*`tMoqcEXI>TZU9WTP#l?uFv+@Dn~b(>xh2 z;>B?;Tz2SR&KVb>vGiBSB`@U7VIWFSo=LDSb9F{GF^DbmWAfpms8Sx9OX4CnBJca3 zlj9(x!dIjN?OG1X4l*imJNvRCk}F%!?SOfiOq5y^mZW)jFL@a|r-@d#f7 z2gmU8L3IZq0ynIws=}~m^#@&C%J6QFo~Mo4V`>v7MI-_!EBMMtb%_M&kvAaN)@ZVw z+`toz&WG#HkWDjnZE!6nk{e-oFdL^$YnbOCN}JC&{$#$O27@|Tn-skXr)2ml2~O!5 zX+gYoxhoc7qoU?C^3~&!U?kRFtnSEecWuH0B0OvLodgUAi}8p1 zrO6RSXHH}DMc$&|?D004DiOVMHV8kXCP@7NKB zgaZq^^O<7PoKEp72kby@W0Z!Y*Ay{&vfg#C&gG@YVR9g?FEocMUi1gSN$+V+ayF45{a zuDZDTN}mS|;BO%gEf}pjBfN2-gIrU#G5~cucA;dokXW89%>AyXJJI z9X4UlIWA|ZYHgbI z5?oFk@A=Ik7lrEQPDH!H+b`7_Y~aDb_qa=B2^Y&Ow41cU=4WDd40dp5(QS-WMN-=Y z9g;6_-JdNU;|6cPwf$ak*aJIcwL@1n$#l~zi{c{EW?T;DaW*E8DYq?Umtz{nJ&w-M zEMyTDrC&9K$d|kZe2#ws6)L=7K+{ zQw{XnV6UC$6-rW0emqm8wJoeZK)wJIcV?dST}Z;G0Arq{dVDu0&4kd%N!3F1*;*pW zR&qUiFzK=@44#QGw7k1`3t_d8&*kBV->O##t|tonFc2YWrL7_eqg+=+k;!F-`^b8> z#KWCE8%u4k@EprxqiV$VmmtiWxDLgnGu$Vs<8rppV5EajBXL4nyyZM$SWVm!wnCj-B!Wjqj5-5dNXukI2$$|Bu3Lrw}z65Lc=1G z^-#WuQOj$hwNGG?*CM_TO8Bg-1+qc>J7k5c51U8g?ZU5n?HYor;~JIjoWH-G>AoUP ztrWWLbRNqIjW#RT*WqZgPJXU7C)VaW5}MiijYbABmzoru6EmQ*N8cVK7a3|aOB#O& zBl8JY2WKfmj;h#Q!pN%9o@VNLv{OUL?rixHwOZuvX7{IJ{(EdPpuVFoQqIOa7giLVkBOKL@^smUA!tZ1CKRK}#SSM)iQHk)*R~?M!qkCruaS!#oIL1c z?J;U~&FfH#*98^G?i}pA{ z9Jg36t4=%6mhY(quYq*vSxptes9qy|7xSlH?G=S@>u>Ebe;|LVhs~@+06N<4CViBk zUiY$thvX;>Tby6z9Y1edAMQaiH zm^r3v#$Q#2T=X>bsY#D%s!bhs^M9PMAcHbCc0FMHV{u-dwlL;a1eJ63v5U*?Q_8JO zT#50!RD619#j_Uf))0ooADz~*9&lN!bBDRUgE>Vud-i5ck%vT=r^yD*^?Mp@Q^v+V zG#-?gKlr}Eeqifb{|So?HM&g91P8|av8hQoCmQXkd?7wIJwb z_^v8bbg`SAn{I*4bH$u(RZ6*xUhuA~hc=8czK8SHEKTzSxgbwi~9(OqJB&gwb^l4+m`k*Q;_?>Y-APi1{k zAHQ)P)G)f|AyjSgcCFps)Fh6Bca*Xznq36!pV6Az&m{O8$wGFD? zY&O*3*J0;_EqM#jh6^gMQKpXV?#1?>$ml1xvh8nSN>-?H=V;nJIwB07YX$e6vLxH( zqYwQ>qxwR(i4f)DLd)-$P>T-no_c!LsN@)8`e;W@)-Hj0>nJ-}Kla4-ZdPJzI&Mce zv)V_j;(3ERN3_@I$N<^|4Lf`B;8n+bX@bHbcZTopEmDI*Jfl)-pFDvo6svPRoo@(x z);_{lY<;);XzT`dBFpRmGrr}z5u1=pC^S-{ce6iXQlLGcItwJ^mZx{m$&DA_oEZ)B{_bYPq-HA zcH8WGoBG(aBU_j)vEy+_71T34@4dmSg!|M8Vf92Zj6WH7Q7t#OHQqWgFE3ARt+%!T z?oLovLVlnf?2c7pTc)~cc^($_8nyKwsN`RA-23ed3sdj(ys%pjjM+9JrctL;dy8a( z@en&CQmnV(()bu|Y%G1-4a(6x{aLytn$T-;(&{QIJB9vMox11U-1HpD@d(QkaJdEb zG{)+6Dos_L+O3NpWo^=gR?evp|CqEG?L&Ut#D*KLaRFOgOEK(Kq1@!EGcTfo+%A&I z=dLbB+d$u{sh?u)xP{PF8L%;YPPW53+@{>5W=Jt#wQpN;0_HYdw1{ksf_XhO4#2F= zyPx6Lx2<92L-;L5PD`zn6zwIH`Jk($?Qw({erA$^bC;q33hv!d!>%wRhj# zal^hk+WGNg;rJtb-EB(?czvOM=H7dl=vblBwAv>}%1@{}mnpUznfq1cE^sgsL0*4I zJ##!*B?=vI_OEVis5o+_IwMIRrpQyT_Sq~ZU%oY7c5JMIADzpD!Upz9h@iWg_>>~j zOLS;wp^i$-E?4<_cp?RiS%Rd?i;f*mOz=~(&3lo<=@(nR!_Rqiprh@weZlL!t#NCc zO!QTcInq|%#>OVgobj{~ixEUec`E25zJ~*DofsQdzIa@5^nOXj2T;8O`l--(QyU^$t?TGY^7#&FQ+2SS3B#qK*k3`ye?8jUYSajE5iBbJls75CCc(m3dk{t?- zopcER9{Z?TC)mk~gpi^kbbu>b-+a{m#8-y2^p$ka4n60w;Sc2}HMf<8JUvhCL0B&Btk)T`ctE$*qNW8L$`7!r^9T+>=<=2qaq-;ll2{`{Rg zc5a0ZUI$oG&j-qVOuKa=*v4aY#IsoM+1|c4Z)<}lEDvy;5huB@1RJPquU2U*U-;gu z=En2m+qjBzR#DEJDO`WU)hdd{Vj%^0V*KoyZ|5lzV87&g_j~NCjwv0uQVqXOb*QrQ zy|Qn`hxx(58c70$E;L(X0uZZ72M1!6oeg)(cdKO ze0gDaTz+ohR-#d)NbAH4x{I(21yjwvBQfmpLu$)|m{XolbgF!pmsqJ#D}(ylp6uC> z{bqtcI#hT#HW=wl7>p!38sKsJ`r8}lt-q%Keqy%u(xk=yiIJiUw6|5IvkS+#?JTBl z8H5(Q?l#wzazujH!8o>1xtn8#_w+397*_cy8!pQGP%K(Ga3pAjsaTbbXJlQF_+m+-UpUUent@xM zg%jqLUExj~o^vQ3Gl*>wh=_gOr2*|U64_iXb+-111aH}$TjeajM+I20xw(((>fej-@CIz4S1pi$(#}P7`4({6QS2CaQS4NPENDp>sAqD z$bH4KGzXGffkJ7R>V>)>tC)uax{UsN*dbeNC*v}#8Y#OWYwL4t$ePR?VTyIs!wea+ z5Urmc)X|^`MG~*dS6pGSbU+gPJoq*^a=_>$n4|P^w$sMBBy@f*Z^Jg6?n5?oId6f{ z$LW4M|4m502z0t7g<#Bx%X;9<=)smFolV&(V^(7Cv2-sxbxopQ!)*#ZRhTBpx1)Fc zNm1T%bONzv6@#|dz(w02AH8OXe>kQ#1FMCzO}2J_mST)+ExmBr9cva-@?;wnmWMOk z{3_~EX_xadgJGv&H@zK_8{(x84`}+c?oSBX*Ge3VdfTt&F}yCpFP?CpW+BE^cWY0^ zb&uBN!Ja3UzYHK-CTyA5=L zEMW{l3Usky#ly=7px648W31UNV@K)&Ub&zP1c7%)`{);I4b0Q<)B}3;NMG2JH=X$U zfIW4)4n9ZM`-yRj67I)YSLDK)qfUJ_ij}a#aZN~9EXrh8eZY2&=uY%2N0UFF7<~%M zsB8=erOWZ>Ct_#^tHZ|*q`H;A)5;ycw*IcmVxi8_0Xk}aJA^ath+E;xg!x+As(M#0=)3!NJR6H&9+zd#iP(m0PIW8$ z1Y^VX`>jm`W!=WpF*{ioM?C9`yOR>@0q=u7o>BP-eSHqCgMDj!2anwH?s%i2p+Q7D zzszIf5XJpE)IG4;d_(La-xenmF(tgAxK`Y4sQ}BSJEPs6N_U2vI{8=0C_F?@7<(G; zo$~G=8p+076G;`}>{MQ>t>7cm=zGtfbdDXm6||jUU|?X?CaE?(<6bKDYKeHlz}DA8 zXT={X=yp_R;HfJ9h%?eWvQ!dRgz&Su*JfNt!Wu>|XfU&68iRikRrHRW|ZxzRR^`eIGt zIeiDgVS>IeExKVRWW8-=A=yA`}`)ZkWBrZD`hpWIxBGkh&f#ijr449~m`j6{4jiJ*C!oVA8ZC?$1RM#K(_b zL9TW)kN*Y4%^-qPpMP7d4)o?Nk#>aoYHT(*g)qmRUb?**F@pnNiy6Fv9rEiUqD(^O zzyS?nBrX63BTRYduaG(0VVG2yJRe%o&rVrLjbxTaAFTd8s;<<@Qs>u(<193R8>}2_ zuwp{7;H2a*X7_jryzriZXMg?bTuegABb^87@SsKkr2)0Gyiax8KQWstw^v#ix45EVrcEhr>!NMhprl$InQMzjSFH54x5k9qHc`@9uKQzvL4ihcq{^B zPrVR=o_ic%Y>6&rMN)hTZsI7I<3&`#(nl+3y3ys9A~&^=4?PL&nd8)`OfG#n zwAMN$1&>K++c{^|7<4P=2y(B{jJsQ0a#U;HTo4ZmWZYvI{+s;Td{Yzem%0*k#)vjpB zia;J&>}ICate44SFYY3vEelqStQWFihx%^vQ@Do(sOy7yR2@WNv7Y9I^yL=nZr3mb zXKV5t@=?-Sk|b{XMhA7ZGB@2hqsx}4xwCW!in#C zI@}scZlr3-NFJ@NFaJlhyfcw{k^vvtGl`N9xSo**rDW4S}i zM9{fMPWo%4wYDG~BZ18BD+}h|GQKc-g^{++3MY>}W_uq7jGHx{mwE9fZiPCoxN$+7 zrODGGJrOkcPQUB(FD5aoS4g~7#6NR^ma7-!>mHuJfY5kTe6PpNNKC9GGRiu^L31uG z$7v`*JknQHsYB!Tm_W{a32TM099djW%5e+j0Ve_ct}IM>XLF1Ap+YvcrLV=|CKo6S zb+9Nl3_YdKP6%Cxy@6TxZ>;4&nTneadr z_ES90ydCev)LV!dN=#(*f}|ZORFdvkYBni^aLbUk>BajeWIOcmHP#8S)*2U~QKI%S zyrLmtPqb&TphJ;>yAxri#;{uyk`JJqODDw%(Z=2`1uc}br^V%>j!gS)D*q*f_-qf8&D;W1dJgQMlaH5er zN2U<%Smb7==vE}dDI8K7cKz!vs^73o9f>2sgiTzWcwY|BMYHH5%Vn7#kiw&eItCqa zIkR2~Q}>X=Ar8W|^Ms41Fm8o6IB2_j60eOeBB1Br!boW7JnoeX6Gs)?7rW0^5psc- zjS16yb>dFn>KPOF;imD}e!enuIniFzv}n$m2#gCCv4jM#ArwlzZ$7@9&XkFxZ4n!V zj3dyiwW4Ki2QG{@i>yuZXQizw_OkZI^-3otXC{!(lUpJF33gI60ak;Uqitp74|B6I zgg{b=Iz}WkhCGj1M=hu4#Aw173YxIVbISaoc z-nLZC*6Tgivd5V`K%GxhBsp@SUU60-rfc$=wb>zdJzXS&-5(NRRodFk;Kxk!S(O(a0e7oY=E( zAyS;Ow?6Q&XA+cnkCb{28_1N8H#?J!*$MmIwLq^*T_9-z^&UE@A(z9oGYtFy6EZef LrJugUA?W`A8`#=m literal 0 HcmV?d00001 diff --git a/apps/grades/src/app/globals.css b/apps/grades/src/app/globals.css new file mode 100644 index 000000000..875c01e81 --- /dev/null +++ b/apps/grades/src/app/globals.css @@ -0,0 +1,33 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; + +:root { + --foreground-rgb: 0, 0, 0; + --background-start-rgb: 214, 219, 220; + --background-end-rgb: 255, 255, 255; +} + +@media (prefers-color-scheme: dark) { + :root { + --foreground-rgb: 255, 255, 255; + --background-start-rgb: 0, 0, 0; + --background-end-rgb: 0, 0, 0; + } +} + +body { + color: rgb(var(--foreground-rgb)); + background: linear-gradient( + to bottom, + transparent, + rgb(var(--background-end-rgb)) + ) + rgb(var(--background-start-rgb)); +} + +@layer utilities { + .text-balance { + text-wrap: balance; + } +} diff --git a/apps/grades/src/app/layout.tsx b/apps/grades/src/app/layout.tsx new file mode 100644 index 000000000..3a62e1caf --- /dev/null +++ b/apps/grades/src/app/layout.tsx @@ -0,0 +1,22 @@ +import { type Metadata } from "next" +import { Inter } from "next/font/google" +import "./globals.css" + +const inter = Inter({ subsets: ["latin"] }) + +export const metadata: Metadata = { + title: "Create Next App", + description: "Generated by create next app", +} + +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode +}>) { + return ( + + {children} + + ) +} diff --git a/apps/grades/src/app/page.tsx b/apps/grades/src/app/page.tsx new file mode 100644 index 000000000..ddcf1170b --- /dev/null +++ b/apps/grades/src/app/page.tsx @@ -0,0 +1,103 @@ +import Image from "next/image" + +export default function Home() { + return ( +
+
+

+ Get started by editing  + src/app/page.tsx +

+ +
+ +
+ Next.js Logo +
+ + +
+ ) +} diff --git a/apps/grades/tailwind.config.ts b/apps/grades/tailwind.config.ts new file mode 100644 index 000000000..0768354a9 --- /dev/null +++ b/apps/grades/tailwind.config.ts @@ -0,0 +1,19 @@ +import { type Config } from "tailwindcss" + +const config: Config = { + content: [ + "./src/pages/**/*.{js,ts,jsx,tsx,mdx}", + "./src/components/**/*.{js,ts,jsx,tsx,mdx}", + "./src/app/**/*.{js,ts,jsx,tsx,mdx}", + ], + theme: { + extend: { + backgroundImage: { + "gradient-radial": "radial-gradient(var(--tw-gradient-stops))", + "gradient-conic": "conic-gradient(from 180deg at 50% 50%, var(--tw-gradient-stops))", + }, + }, + }, + plugins: [], +} +export default config diff --git a/apps/grades/tsconfig.json b/apps/grades/tsconfig.json new file mode 100644 index 000000000..7b2858930 --- /dev/null +++ b/apps/grades/tsconfig.json @@ -0,0 +1,26 @@ +{ + "compilerOptions": { + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "plugins": [ + { + "name": "next" + } + ], + "paths": { + "@/*": ["./src/*"] + } + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "exclude": ["node_modules"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 62f6cc5f8..0150b133a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -234,6 +234,46 @@ importers: specifier: 5.2.2 version: 5.2.2 + apps/grades: + dependencies: + next: + specifier: 14.1.0 + version: 14.1.0(react-dom@18.2.0)(react@18.2.0) + react: + specifier: ^18 + version: 18.2.0 + react-dom: + specifier: ^18 + version: 18.2.0(react@18.2.0) + devDependencies: + '@dotkomonline/config': + specifier: workspace:* + version: link:../../packages/config + '@types/node': + specifier: ^20 + version: 20.11.16 + '@types/react': + specifier: ^18 + version: 18.2.38 + '@types/react-dom': + specifier: ^18 + version: 18.2.17 + autoprefixer: + specifier: ^10.0.1 + version: 10.4.16(postcss@8.4.32) + eslint: + specifier: ^8.56.0 + version: 8.56.0 + postcss: + specifier: ^8 + version: 8.4.32 + tailwindcss: + specifier: ^3.3.0 + version: 3.3.5(ts-node@10.9.1) + typescript: + specifier: ^5 + version: 5.2.2 + apps/rif: dependencies: '@dotkomonline/env': @@ -4125,6 +4165,16 @@ packages: eslint-visitor-keys: 3.4.3 dev: true + /@eslint-community/eslint-utils@4.4.0(eslint@8.56.0): + resolution: {integrity: sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + peerDependencies: + eslint: ^6.0.0 || ^7.0.0 || >=8.0.0 + dependencies: + eslint: 8.56.0 + eslint-visitor-keys: 3.4.3 + dev: true + /@eslint-community/regexpp@4.6.2: resolution: {integrity: sha512-pPTNuaAG3QMH+buKyBIGJs3g/S5y0caxw0ygM3YyE6yJFySwiGGSzA+mM3KJ8QQvzeLh3blwgSonkFjgQdxzMw==} engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0} @@ -4147,11 +4197,33 @@ packages: - supports-color dev: true + /@eslint/eslintrc@2.1.4: + resolution: {integrity: sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + dependencies: + ajv: 6.12.6 + debug: 4.3.4 + espree: 9.6.1 + globals: 13.20.0 + ignore: 5.2.4 + import-fresh: 3.3.0 + js-yaml: 4.1.0 + minimatch: 3.1.2 + strip-json-comments: 3.1.1 + transitivePeerDependencies: + - supports-color + dev: true + /@eslint/js@8.54.0: resolution: {integrity: sha512-ut5V+D+fOoWPgGGNj83GGjnntO39xDy6DWxO0wb7Jp3DcMX0TfIqdzHF85VTQkerdyGmuuMD9AKAo5KiNlf/AQ==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} dev: true + /@eslint/js@8.56.0: + resolution: {integrity: sha512-gMsVel9D7f2HLkBma9VbtzZRehRogVRfbr++f06nL2vnCGCNlzOD+/MUov/F4p8myyAHspEhVobgjpX64q5m6A==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + dev: true + /@floating-ui/core@1.4.1: resolution: {integrity: sha512-jk3WqquEJRlcyu7997NtR5PibI+y5bi+LS3hPmguVClypenMsCY3CBa3LAQnozRCtCrYWSEtAdiskpamuJRFOQ==} dependencies: @@ -4575,6 +4647,10 @@ packages: resolution: {integrity: sha512-7xRqh9nMvP5xrW4/+L0jgRRX+HoNRGnfJpD+5Wq6/13j3dsdzxO3BCXn7D3hMqsDb+vjZnJq+vI7+EtgrYZTeA==} dev: false + /@next/env@14.1.0: + resolution: {integrity: sha512-Py8zIo+02ht82brwwhTg36iogzFqGLPXlRGKQw5s+qP/kMNc4MAyDeEwBKDijk6zTIbegEgu8Qy7C1LboslQAw==} + dev: false + /@next/eslint-plugin-next@14.0.3: resolution: {integrity: sha512-j4K0n+DcmQYCVnSAM+UByTVfIHnYQy2ODozfQP+4RdwtRDfobrIvKq1K4Exb2koJ79HSSa7s6B2SA8T/1YR3RA==} dependencies: @@ -4590,6 +4666,15 @@ packages: dev: false optional: true + /@next/swc-darwin-arm64@14.1.0: + resolution: {integrity: sha512-nUDn7TOGcIeyQni6lZHfzNoo9S0euXnu0jhsbMOmMJUBfgsnESdjN97kM7cBqQxZa8L/bM9om/S5/1dzCrW6wQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: false + optional: true + /@next/swc-darwin-x64@14.0.3: resolution: {integrity: sha512-RkTf+KbAD0SgYdVn1XzqE/+sIxYGB7NLMZRn9I4Z24afrhUpVJx6L8hsRnIwxz3ERE2NFURNliPjJ2QNfnWicQ==} engines: {node: '>= 10'} @@ -4599,6 +4684,15 @@ packages: dev: false optional: true + /@next/swc-darwin-x64@14.1.0: + resolution: {integrity: sha512-1jgudN5haWxiAl3O1ljUS2GfupPmcftu2RYJqZiMJmmbBT5M1XDffjUtRUzP4W3cBHsrvkfOFdQ71hAreNQP6g==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: false + optional: true + /@next/swc-linux-arm64-gnu@14.0.3: resolution: {integrity: sha512-3tBWGgz7M9RKLO6sPWC6c4pAw4geujSwQ7q7Si4d6bo0l6cLs4tmO+lnSwFp1Tm3lxwfMk0SgkJT7EdwYSJvcg==} engines: {node: '>= 10'} @@ -4608,6 +4702,15 @@ packages: dev: false optional: true + /@next/swc-linux-arm64-gnu@14.1.0: + resolution: {integrity: sha512-RHo7Tcj+jllXUbK7xk2NyIDod3YcCPDZxj1WLIYxd709BQ7WuRYl3OWUNG+WUfqeQBds6kvZYlc42NJJTNi4tQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: false + optional: true + /@next/swc-linux-arm64-musl@14.0.3: resolution: {integrity: sha512-v0v8Kb8j8T23jvVUWZeA2D8+izWspeyeDGNaT2/mTHWp7+37fiNfL8bmBWiOmeumXkacM/AB0XOUQvEbncSnHA==} engines: {node: '>= 10'} @@ -4617,6 +4720,15 @@ packages: dev: false optional: true + /@next/swc-linux-arm64-musl@14.1.0: + resolution: {integrity: sha512-v6kP8sHYxjO8RwHmWMJSq7VZP2nYCkRVQ0qolh2l6xroe9QjbgV8siTbduED4u0hlk0+tjS6/Tuy4n5XCp+l6g==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: false + optional: true + /@next/swc-linux-x64-gnu@14.0.3: resolution: {integrity: sha512-VM1aE1tJKLBwMGtyBR21yy+STfl0MapMQnNrXkxeyLs0GFv/kZqXS5Jw/TQ3TSUnbv0QPDf/X8sDXuMtSgG6eg==} engines: {node: '>= 10'} @@ -4626,6 +4738,15 @@ packages: dev: false optional: true + /@next/swc-linux-x64-gnu@14.1.0: + resolution: {integrity: sha512-zJ2pnoFYB1F4vmEVlb/eSe+VH679zT1VdXlZKX+pE66grOgjmKJHKacf82g/sWE4MQ4Rk2FMBCRnX+l6/TVYzQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + requiresBuild: true + dev: false + optional: true + /@next/swc-linux-x64-musl@14.0.3: resolution: {integrity: sha512-64EnmKy18MYFL5CzLaSuUn561hbO1Gk16jM/KHznYP3iCIfF9e3yULtHaMy0D8zbHfxset9LTOv6cuYKJgcOxg==} engines: {node: '>= 10'} @@ -4635,6 +4756,15 @@ packages: dev: false optional: true + /@next/swc-linux-x64-musl@14.1.0: + resolution: {integrity: sha512-rbaIYFt2X9YZBSbH/CwGAjbBG2/MrACCVu2X0+kSykHzHnYH5FjHxwXLkcoJ10cX0aWCEynpu+rP76x0914atg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + requiresBuild: true + dev: false + optional: true + /@next/swc-win32-arm64-msvc@14.0.3: resolution: {integrity: sha512-WRDp8QrmsL1bbGtsh5GqQ/KWulmrnMBgbnb+59qNTW1kVi1nG/2ndZLkcbs2GX7NpFLlToLRMWSQXmPzQm4tog==} engines: {node: '>= 10'} @@ -4644,6 +4774,15 @@ packages: dev: false optional: true + /@next/swc-win32-arm64-msvc@14.1.0: + resolution: {integrity: sha512-o1N5TsYc8f/HpGt39OUQpQ9AKIGApd3QLueu7hXk//2xq5Z9OxmV6sQfNp8C7qYmiOlHYODOGqNNa0e9jvchGQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + requiresBuild: true + dev: false + optional: true + /@next/swc-win32-ia32-msvc@14.0.3: resolution: {integrity: sha512-EKffQeqCrj+t6qFFhIFTRoqb2QwX1mU7iTOvMyLbYw3QtqTw9sMwjykyiMlZlrfm2a4fA84+/aeW+PMg1MjuTg==} engines: {node: '>= 10'} @@ -4653,6 +4792,15 @@ packages: dev: false optional: true + /@next/swc-win32-ia32-msvc@14.1.0: + resolution: {integrity: sha512-XXIuB1DBRCFwNO6EEzCTMHT5pauwaSj4SWs7CYnME57eaReAKBXCnkUE80p/pAZcewm7hs+vGvNqDPacEXHVkw==} + engines: {node: '>= 10'} + cpu: [ia32] + os: [win32] + requiresBuild: true + dev: false + optional: true + /@next/swc-win32-x64-msvc@14.0.3: resolution: {integrity: sha512-ERhKPSJ1vQrPiwrs15Pjz/rvDHZmkmvbf/BjPN/UCOI++ODftT0GtasDPi0j+y6PPJi5HsXw+dpRaXUaw4vjuQ==} engines: {node: '>= 10'} @@ -4662,6 +4810,15 @@ packages: dev: false optional: true + /@next/swc-win32-x64-msvc@14.1.0: + resolution: {integrity: sha512-9WEbVRRAqJ3YFVqEZIxUqkiO8l1nool1LmNxygr5HWF8AcSYsEpneUDhmjUVJEzO2A04+oPtZdombzzPPkTtgg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + requiresBuild: true + dev: false + optional: true + /@nicolo-ribaudo/eslint-scope-5-internals@5.1.1-v1: resolution: {integrity: sha512-54/JRvkLIzzDWshCWfuhadfrfZVPiElY8Fcgmg1HroEly/EDSszzhBAsarCux+D/kOslTRquNzuyGSmUSTTHGg==} dependencies: @@ -6126,7 +6283,7 @@ packages: /@sanity/bifur-client@0.3.1: resolution: {integrity: sha512-GlY9+tUmM0Vye64BHwIYLOivuRL37ucW/sj/D9MYqBmjgBnTRrjfmg8NR7qoodZuJ5nYJ5qpGMsVIBLP4Plvnw==} dependencies: - nanoid: 3.3.6 + nanoid: 3.3.7 rxjs: 7.8.1 dev: false @@ -7328,7 +7485,7 @@ packages: resolution: {integrity: sha512-Lz5J+NFgZS4cEVhquwjIGH4oQwlVn2h7LXD3boitujBnzOE5o7s9H8hchEjoDK2SlRsJTogdKnQeiJgPPKLIEw==} dependencies: '@types/docker-modem': 3.0.6 - '@types/node': 18.18.9 + '@types/node': 18.18.13 dev: true /@types/eslint@8.44.7: @@ -7378,7 +7535,7 @@ packages: /@types/ignore-walk@4.0.3: resolution: {integrity: sha512-6V7wDsk0nz8LtRC7qeC0GfXadFLT4FdCtVbXhxoIGRdkn2kLr20iMLupRGiBhlZ79WWWqaObIdR3nkXfUrBPdQ==} dependencies: - '@types/node': 18.18.9 + '@types/node': 18.18.13 dev: true /@types/is-hotkey@0.1.10: @@ -7438,12 +7595,8 @@ packages: dependencies: undici-types: 5.26.5 - /@types/node@18.18.4: - resolution: {integrity: sha512-t3rNFBgJRugIhackit2mVcLfF6IRc0JE4oeizPQL8Zrm8n2WY/0wOdpOPhdtG0V9Q2TlW/axbF1MJ6z+Yj/kKQ==} - dev: true - - /@types/node@18.18.9: - resolution: {integrity: sha512-0f5klcuImLnG4Qreu9hPj/rEfFq6YRc5n2mAjSsH+ec/mJL+3voBH0+8T7o8RpFjH7ovc+TRsL/c7OYIQsPTfQ==} + /@types/node@20.11.16: + resolution: {integrity: sha512-gKb0enTmRCzXSSUJDq6/sPcqrfCv2mkkG6Jt/clpn5eiCbKTY+SgZUxo+p8ZKMof5dCp9vHQUAB7wOUTod22wQ==} dependencies: undici-types: 5.26.5 dev: true @@ -7454,7 +7607,7 @@ packages: /@types/pg@8.10.9: resolution: {integrity: sha512-UksbANNE/f8w0wOMxVKKIrLCbEMV+oM1uKejmwXr39olg4xqcfBDbXxObJAt6XxHbDa4XTKOlUEcEltXDX+XLQ==} dependencies: - '@types/node': 18.18.4 + '@types/node': 18.18.13 pg-protocol: 1.6.0 pg-types: 4.0.1 dev: true @@ -8266,6 +8419,22 @@ packages: postcss-value-parser: 4.2.0 dev: true + /autoprefixer@10.4.16(postcss@8.4.32): + resolution: {integrity: sha512-7vd3UC6xKp0HLfua5IjZlcXvGAGy7cBAXTg2lyQ/8WpNhd6SiZ8Be+xm3FyBSYJx5GKcpRCzBh7RH4/0dnY+uQ==} + engines: {node: ^10 || ^12 || >=14} + hasBin: true + peerDependencies: + postcss: ^8.1.0 + dependencies: + browserslist: 4.21.11 + caniuse-lite: 1.0.30001538 + fraction.js: 4.3.6 + normalize-range: 0.1.2 + picocolors: 1.0.0 + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: true + /available-typed-arrays@1.0.5: resolution: {integrity: sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==} engines: {node: '>= 0.4'} @@ -8552,6 +8721,10 @@ packages: /caniuse-lite@1.0.30001538: resolution: {integrity: sha512-HWJnhnID+0YMtGlzcp3T9drmBJUVDchPJ08tpUGFLs9CYlwWPH2uLgpHn8fND5pCgXVtnGS3H4QR9XLMHVNkHw==} + /caniuse-lite@1.0.30001585: + resolution: {integrity: sha512-yr2BWR1yLXQ8fMpdS/4ZZXpseBgE7o4g41x3a6AJOqZuOi+iE/WdJYAuZ6Y95i4Ohd2Y+9MzIWRR+uGABH4s3Q==} + dev: false + /ccount@2.0.1: resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} dev: true @@ -8960,7 +9133,6 @@ packages: loose-envify: 1.4.0 object-assign: 4.1.1 dev: false - bundledDependencies: false /create-require@1.1.1: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} @@ -10139,6 +10311,53 @@ packages: - supports-color dev: true + /eslint@8.56.0: + resolution: {integrity: sha512-Go19xM6T9puCOWntie1/P997aXxFsOi37JIHRWI514Hc6ZnaHGKY9xFhrU65RT6CcBEzZoGG1e6Nq+DT04ZtZQ==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + hasBin: true + dependencies: + '@eslint-community/eslint-utils': 4.4.0(eslint@8.56.0) + '@eslint-community/regexpp': 4.6.2 + '@eslint/eslintrc': 2.1.4 + '@eslint/js': 8.56.0 + '@humanwhocodes/config-array': 0.11.13 + '@humanwhocodes/module-importer': 1.0.1 + '@nodelib/fs.walk': 1.2.8 + '@ungap/structured-clone': 1.2.0 + ajv: 6.12.6 + chalk: 4.1.2 + cross-spawn: 7.0.3 + debug: 4.3.4 + doctrine: 3.0.0 + escape-string-regexp: 4.0.0 + eslint-scope: 7.2.2 + eslint-visitor-keys: 3.4.3 + espree: 9.6.1 + esquery: 1.5.0 + esutils: 2.0.3 + fast-deep-equal: 3.1.3 + file-entry-cache: 6.0.1 + find-up: 5.0.0 + glob-parent: 6.0.2 + globals: 13.20.0 + graphemer: 1.4.0 + ignore: 5.2.4 + imurmurhash: 0.1.4 + is-glob: 4.0.3 + is-path-inside: 3.0.3 + js-yaml: 4.1.0 + json-stable-stringify-without-jsonify: 1.0.1 + levn: 0.4.1 + lodash.merge: 4.6.2 + minimatch: 3.1.2 + natural-compare: 1.4.0 + optionator: 0.9.3 + strip-ansi: 6.0.1 + text-table: 0.2.0 + transitivePeerDependencies: + - supports-color + dev: true + /espree@9.6.1: resolution: {integrity: sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -10347,7 +10566,6 @@ packages: glob-parent: 5.1.2 merge2: 1.4.1 micromatch: 4.0.5 - dev: true /fast-json-stable-stringify@2.1.0: resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} @@ -13021,7 +13239,6 @@ packages: resolution: {integrity: sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true - dev: true /natural-compare-lite@1.4.0: resolution: {integrity: sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g==} @@ -13123,6 +13340,45 @@ packages: - babel-plugin-macros dev: false + /next@14.1.0(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-wlzrsbfeSU48YQBjZhDzOwhWhGsy+uQycR8bHAOt1LY1bn3zZEcDyHQOEoN3aWzQ8LHCAJ1nqrWCc9XF2+O45Q==} + engines: {node: '>=18.17.0'} + hasBin: true + peerDependencies: + '@opentelemetry/api': ^1.1.0 + react: ^18.2.0 + react-dom: ^18.2.0 + sass: ^1.3.0 + peerDependenciesMeta: + '@opentelemetry/api': + optional: true + sass: + optional: true + dependencies: + '@next/env': 14.1.0 + '@swc/helpers': 0.5.2 + busboy: 1.6.0 + caniuse-lite: 1.0.30001585 + graceful-fs: 4.2.11 + postcss: 8.4.31 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + styled-jsx: 5.1.1(react@18.2.0) + optionalDependencies: + '@next/swc-darwin-arm64': 14.1.0 + '@next/swc-darwin-x64': 14.1.0 + '@next/swc-linux-arm64-gnu': 14.1.0 + '@next/swc-linux-arm64-musl': 14.1.0 + '@next/swc-linux-x64-gnu': 14.1.0 + '@next/swc-linux-x64-musl': 14.1.0 + '@next/swc-win32-arm64-msvc': 14.1.0 + '@next/swc-win32-ia32-msvc': 14.1.0 + '@next/swc-win32-x64-msvc': 14.1.0 + transitivePeerDependencies: + - '@babel/core' + - babel-plugin-macros + dev: false + /node-cleanup@2.1.2: resolution: {integrity: sha512-qN8v/s2PAJwGUtr1/hYTpNKlD6Y9rc4p8KSmJXyGdYGZsDGKXrGThikLFP9OCHFeLeEpQzPwiAtdIvBLqm//Hw==} dev: true @@ -13766,6 +14022,18 @@ packages: read-cache: 1.0.0 resolve: 1.22.2 + /postcss-import@15.1.0(postcss@8.4.32): + resolution: {integrity: sha512-hpr+J05B2FVYUAXHeK1YyI267J/dDDhMU6B6civm8hSY1jYJnBXxzKDKDswzJmtLHryrjhnDjqqp/49t8FALew==} + engines: {node: '>=14.0.0'} + peerDependencies: + postcss: ^8.0.0 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + read-cache: 1.0.0 + resolve: 1.22.2 + dev: false + /postcss-js@4.0.1(postcss@8.4.31): resolution: {integrity: sha512-dDLF8pEO191hJMtlHFPRa8xsizHaM82MLfNkUHdUtVEV3tgTp5oj+8qbEqYM57SLfc74KSbw//4SeJma2LRVIw==} engines: {node: ^12 || ^14 || >= 16} @@ -13775,6 +14043,16 @@ packages: camelcase-css: 2.0.1 postcss: 8.4.31 + /postcss-js@4.0.1(postcss@8.4.32): + resolution: {integrity: sha512-dDLF8pEO191hJMtlHFPRa8xsizHaM82MLfNkUHdUtVEV3tgTp5oj+8qbEqYM57SLfc74KSbw//4SeJma2LRVIw==} + engines: {node: ^12 || ^14 || >= 16} + peerDependencies: + postcss: ^8.4.21 + dependencies: + camelcase-css: 2.0.1 + postcss: 8.4.32 + dev: false + /postcss-load-config@4.0.1(postcss@8.4.31)(ts-node@10.9.1): resolution: {integrity: sha512-vEJIc8RdiBRu3oRAI0ymerOn+7rPuMvRXslTvZUKZonDHFIczxztIyJ1urxM1x9JXEikvpWWTUUqal5j/8QgvA==} engines: {node: '>= 14'} @@ -13792,6 +14070,23 @@ packages: ts-node: 10.9.1(@types/node@18.18.13)(typescript@5.2.2) yaml: 2.3.1 + /postcss-load-config@4.0.1(postcss@8.4.32)(ts-node@10.9.1): + resolution: {integrity: sha512-vEJIc8RdiBRu3oRAI0ymerOn+7rPuMvRXslTvZUKZonDHFIczxztIyJ1urxM1x9JXEikvpWWTUUqal5j/8QgvA==} + engines: {node: '>= 14'} + peerDependencies: + postcss: '>=8.0.9' + ts-node: '>=9.0.0' + peerDependenciesMeta: + postcss: + optional: true + ts-node: + optional: true + dependencies: + lilconfig: 2.1.0 + postcss: 8.4.32 + ts-node: 10.9.1(@types/node@18.18.13)(typescript@5.2.2) + yaml: 2.3.1 + /postcss-mixins@9.0.4(postcss@8.4.31): resolution: {integrity: sha512-XVq5jwQJDRu5M1XGkdpgASqLk37OqkH4JCFDXl/Dn7janOJjCTEKL+36cnRVy7bMtoBzALfO7bV7nTIsFnUWLA==} engines: {node: '>=14.0'} @@ -13814,6 +14109,16 @@ packages: postcss: 8.4.31 postcss-selector-parser: 6.0.13 + /postcss-nested@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-mEp4xPMi5bSWiMbsgoPfcP74lsWLHkQbZc3sY+jWYd65CUwXrUaTp0fmNpa01ZcETKlIgUdFN/MpS2xZtqL9dQ==} + engines: {node: '>=12.0'} + peerDependencies: + postcss: ^8.2.14 + dependencies: + postcss: 8.4.32 + postcss-selector-parser: 6.0.13 + dev: false + /postcss-preset-mantine@1.11.0(postcss@8.4.31): resolution: {integrity: sha512-drhCJAd8Rrn5ulRX5cP6DM6nMLrACAsBU73MfXNI3c77p4dodO36KcfEMY0WqaQkd/E2oODkTm7gVYOzjs45Gw==} peerDependencies: @@ -13865,7 +14170,6 @@ packages: nanoid: 3.3.7 picocolors: 1.0.0 source-map-js: 1.0.2 - dev: true /postgres-array@2.0.0: resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} @@ -15468,7 +15772,7 @@ packages: '@types/stylis': 4.2.1 css-to-react-native: 3.2.0 csstype: 3.1.2 - postcss: 8.4.31 + postcss: 8.4.32 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) shallowequal: 1.1.0 @@ -15593,7 +15897,7 @@ packages: chokidar: 3.5.3 didyoumean: 1.2.2 dlv: 1.1.3 - fast-glob: 3.3.1 + fast-glob: 3.3.2 glob-parent: 6.0.2 is-glob: 4.0.3 jiti: 1.19.1 @@ -15602,11 +15906,11 @@ packages: normalize-path: 3.0.0 object-hash: 3.0.0 picocolors: 1.0.0 - postcss: 8.4.31 - postcss-import: 15.1.0(postcss@8.4.31) - postcss-js: 4.0.1(postcss@8.4.31) - postcss-load-config: 4.0.1(postcss@8.4.31)(ts-node@10.9.1) - postcss-nested: 6.0.1(postcss@8.4.31) + postcss: 8.4.32 + postcss-import: 15.1.0(postcss@8.4.32) + postcss-js: 4.0.1(postcss@8.4.32) + postcss-load-config: 4.0.1(postcss@8.4.32)(ts-node@10.9.1) + postcss-nested: 6.0.1(postcss@8.4.32) postcss-selector-parser: 6.0.13 postcss-value-parser: 4.2.0 resolve: 1.22.8 @@ -16031,7 +16335,7 @@ packages: execa: 5.1.1 globby: 11.1.0 joycon: 3.1.1 - postcss-load-config: 4.0.1(postcss@8.4.31)(ts-node@10.9.1) + postcss-load-config: 4.0.1(postcss@8.4.32)(ts-node@10.9.1) resolve-from: 5.0.0 rollup: 3.29.2 source-map: 0.8.0-beta.0 @@ -16617,7 +16921,7 @@ packages: vfile-message: 4.0.2 dev: true - /vite-node@0.34.6(@types/node@18.18.4): + /vite-node@0.34.6(@types/node@18.18.13): resolution: {integrity: sha512-nlBMJ9x6n7/Amaz6F3zJ97EBwR2FkzhBRxF5e+jE6LA3yi6Wtc2lyTij1OnDMIr34v5g/tVQtsVAzhT0jc5ygA==} engines: {node: '>=v14.18.0'} hasBin: true @@ -16627,7 +16931,7 @@ packages: mlly: 1.4.0 pathe: 1.1.1 picocolors: 1.0.0 - vite: 4.5.0(@types/node@18.18.4) + vite: 4.5.0(@types/node@18.18.13) transitivePeerDependencies: - '@types/node' - less @@ -16656,7 +16960,7 @@ packages: - typescript dev: true - /vite@4.4.11(@types/node@18.18.4): + /vite@4.4.11(@types/node@18.18.13): resolution: {integrity: sha512-ksNZJlkcU9b0lBwAGZGGaZHCMqHsc8OpgtoYhsQ4/I2v5cnpmmmqe5pM4nv/4Hn6G/2GhTdj0DhZh2e+Er1q5A==} engines: {node: ^14.18.0 || >=16.0.0} hasBin: true @@ -16684,9 +16988,9 @@ packages: terser: optional: true dependencies: - '@types/node': 18.18.4 + '@types/node': 18.18.13 esbuild: 0.18.20 - postcss: 8.4.31 + postcss: 8.4.32 rollup: 3.29.2 optionalDependencies: fsevents: 2.3.3 @@ -16722,46 +17026,10 @@ packages: dependencies: '@types/node': 18.18.13 esbuild: 0.18.20 - postcss: 8.4.31 - rollup: 3.29.2 - optionalDependencies: - fsevents: 2.3.3 - - /vite@4.5.0(@types/node@18.18.4): - resolution: {integrity: sha512-ulr8rNLA6rkyFAlVWw2q5YJ91v098AFQ2R0PRFwPzREXOUJQPtFUG0t+/ZikhaOCDqFoDhN6/v8Sq0o4araFAw==} - engines: {node: ^14.18.0 || >=16.0.0} - hasBin: true - peerDependencies: - '@types/node': '>= 14' - less: '*' - lightningcss: ^1.21.0 - sass: '*' - stylus: '*' - sugarss: '*' - terser: ^5.4.0 - peerDependenciesMeta: - '@types/node': - optional: true - less: - optional: true - lightningcss: - optional: true - sass: - optional: true - stylus: - optional: true - sugarss: - optional: true - terser: - optional: true - dependencies: - '@types/node': 18.18.4 - esbuild: 0.18.20 - postcss: 8.4.31 + postcss: 8.4.32 rollup: 3.29.2 optionalDependencies: fsevents: 2.3.3 - dev: true /vite@5.0.7(@types/node@18.18.13): resolution: {integrity: sha512-B4T4rJCDPihrQo2B+h1MbeGL/k/GMAHzhQ8S0LjQ142s6/+l3hHTT095ORvsshj4QCkoWu3Xtmob5mazvakaOw==} @@ -16832,7 +17100,7 @@ packages: dependencies: '@types/chai': 4.3.5 '@types/chai-subset': 1.3.3 - '@types/node': 18.18.4 + '@types/node': 18.18.13 '@vitest/expect': 0.34.6 '@vitest/runner': 0.34.6 '@vitest/snapshot': 0.34.6 @@ -16852,8 +17120,8 @@ packages: strip-literal: 1.0.1 tinybench: 2.5.0 tinypool: 0.7.0 - vite: 4.4.11(@types/node@18.18.4) - vite-node: 0.34.6(@types/node@18.18.4) + vite: 4.4.11(@types/node@18.18.13) + vite-node: 0.34.6(@types/node@18.18.13) why-is-node-running: 2.2.2 transitivePeerDependencies: - less From 63bfc1c9da9860a560398f315c5c9d8ce30aec0f Mon Sep 17 00:00:00 2001 From: Mats Larsen Date: Wed, 7 Feb 2024 20:48:47 +0100 Subject: [PATCH 10/23] Implement hkdir query for subjects and departments --- apps/grades/migrations/0001_schema.sql | 60 +++++++++ apps/grades/package.json | 11 +- apps/grades/src/db.generated.d.ts | 54 ++++++++ apps/grades/src/pages/api/departments.ts | 9 ++ apps/grades/src/pages/api/subjects.ts | 9 ++ apps/grades/src/trpc/hkdir-service.ts | 161 +++++++++++++++++++++++ apps/grades/src/trpc/kysely.ts | 18 +++ pnpm-lock.yaml | 44 +++++++ 8 files changed, 363 insertions(+), 3 deletions(-) create mode 100644 apps/grades/migrations/0001_schema.sql create mode 100644 apps/grades/src/db.generated.d.ts create mode 100644 apps/grades/src/pages/api/departments.ts create mode 100644 apps/grades/src/pages/api/subjects.ts create mode 100644 apps/grades/src/trpc/hkdir-service.ts create mode 100644 apps/grades/src/trpc/kysely.ts diff --git a/apps/grades/migrations/0001_schema.sql b/apps/grades/migrations/0001_schema.sql new file mode 100644 index 000000000..09417f08f --- /dev/null +++ b/apps/grades/migrations/0001_schema.sql @@ -0,0 +1,60 @@ +START TRANSACTION; + +CREATE EXTENSION IF NOT EXISTS fuzzystrmatch; + +CREATE TABLE IF NOT EXISTS ntnu_faculty +( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + ref_id TEXT NOT NULL, + name TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS ntnu_faculty_department +( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + ref_id TEXT NOT NULL, + faculty_id UUID NOT NULL, + name TEXT NOT NULL, + + CONSTRAINT fk_faculty_id FOREIGN KEY (faculty_id) REFERENCES ntnu_faculty (id) +); + +CREATE TABLE IF NOT EXISTS subject +( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + ref_id TEXT NOT NULL, + name TEXT NOT NULL, + slug TEXT NOT NULL, + department_id UUID NOT NULL, + + instruction_language TEXT NOT NULL, + educational_level TEXT NOT NULL, + credits FLOAT NOT NULL, + + CONSTRAINT fk_department_id FOREIGN KEY (department_id) REFERENCES ntnu_faculty_department (id) +); + +CREATE TYPE subject_season AS ENUM ('SPRING', 'AUTUMN', 'WINTER', 'SUMMER'); + +CREATE TABLE IF NOT EXISTS subject_season_grade +( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + subject_id UUID NOT NULL, + season subject_season NOT NULL, + year INTEGER NOT NULL, + grade FLOAT NOT NULL, + + graded_a INTEGER, + graded_b INTEGER, + graded_c INTEGER, + graded_d INTEGER, + graded_e INTEGER, + graded_f INTEGER, + + graded_fail INTEGER, + graded_pass INTEGER, + + CONSTRAINT fk_subject_id FOREIGN KEY (subject_id) REFERENCES subject (id) +); + +COMMIT; diff --git a/apps/grades/package.json b/apps/grades/package.json index 3df68b49f..ab89f38dd 100644 --- a/apps/grades/package.json +++ b/apps/grades/package.json @@ -8,20 +8,25 @@ "start": "next start", "lint": "eslint --max-warnings 0 .", "lint:fix": "eslint --fix .", - "type-check": "tsc --noEmit" + "type-check": "tsc --noEmit", + "generate": "kysely-codegen --dialect postgres --camel-case --out-file src/db.generated.d.ts" }, "dependencies": { + "kysely": "^0.27.2", + "pg": "^8.11.3", "react": "^18", "react-dom": "^18", - "next": "14.1.0" + "next": "14.1.0", + "zod": "^3.22.4" }, "devDependencies": { "@dotkomonline/config": "workspace:*", "@types/node": "^20", "@types/react": "^18", "@types/react-dom": "^18", - "eslint": "^8.56.0", "autoprefixer": "^10.0.1", + "eslint": "^8.56.0", + "kysely-codegen": "^0.11.0", "postcss": "^8", "tailwindcss": "^3.3.0", "typescript": "^5" diff --git a/apps/grades/src/db.generated.d.ts b/apps/grades/src/db.generated.d.ts new file mode 100644 index 000000000..5c2695332 --- /dev/null +++ b/apps/grades/src/db.generated.d.ts @@ -0,0 +1,54 @@ +import { type ColumnType } from "kysely" + +export type Generated = T extends ColumnType + ? ColumnType + : ColumnType + +export type SubjectSeason = "AUTUMN" | "SPRING" | "SUMMER" | "WINTER" + +export interface NtnuFaculty { + id: Generated + name: string + refId: string +} + +export interface NtnuFacultyDepartment { + facultyId: string + id: Generated + name: string + refId: string +} + +export interface Subject { + credits: number + departmentId: string + educationalLevel: string + id: Generated + instructionLanguage: string + name: string + refId: string + slug: string +} + +export interface SubjectSeasonGrade { + grade: number + gradedA: number | null + gradedB: number | null + gradedC: number | null + gradedD: number | null + gradedE: number | null + gradedF: number | null + gradedFail: number | null + gradedPass: number | null + id: Generated + season: SubjectSeason + subjectId: string + year: number +} + +export interface DB { + ntnuFaculty: NtnuFaculty + ntnuFacultyDepartment: NtnuFacultyDepartment + subject: Subject + subjectSeasonGrade: SubjectSeasonGrade +} diff --git a/apps/grades/src/pages/api/departments.ts b/apps/grades/src/pages/api/departments.ts new file mode 100644 index 000000000..eea82ce4f --- /dev/null +++ b/apps/grades/src/pages/api/departments.ts @@ -0,0 +1,9 @@ +import { type NextApiRequest, type NextApiResponse } from "next" +import { HkdirServiceImpl } from "@/trpc/hkdir-service" + +export default async function route(req: NextApiRequest, res: NextApiResponse) { + const service = new HkdirServiceImpl(fetch) + const departments = await service.getDepartments("1150") + + res.json(departments) +} diff --git a/apps/grades/src/pages/api/subjects.ts b/apps/grades/src/pages/api/subjects.ts new file mode 100644 index 000000000..329edcf1e --- /dev/null +++ b/apps/grades/src/pages/api/subjects.ts @@ -0,0 +1,9 @@ +import { type NextApiRequest, type NextApiResponse } from "next" +import { HkdirServiceImpl } from "@/trpc/hkdir-service" + +export default async function route(req: NextApiRequest, res: NextApiResponse) { + const service = new HkdirServiceImpl(fetch) + const subjects = await service.getSubjects("1150") + + res.json(subjects) +} diff --git a/apps/grades/src/trpc/hkdir-service.ts b/apps/grades/src/trpc/hkdir-service.ts new file mode 100644 index 000000000..855a99176 --- /dev/null +++ b/apps/grades/src/trpc/hkdir-service.ts @@ -0,0 +1,161 @@ +import { z, type ZodSchema } from "zod" + +export type HkdirDepartment = z.infer +export const HkdirDepartment = z.object({ + "Nivå": z.string(), + "Nivå_tekst": z.string(), + "Institusjonskode": z.string(), + "Institusjonsnavn": z.string(), + "Avdelingskode": z.string(), + "Avdelingsnavn": z.string(), + "Gyldig_fra": z.string().nullish().default(null), + "Gyldig_til": z.string().nullish().default(null), + "fagkode_avdeling": z.string().nullish().default(null), + "fagnavn_avdeling": z.string().nullish().default(null), + "Fakultetskode": z.string().nullish().default(null), + "Fakueltetsnavn": z.string().nullish().default(null), + "Avdelingskode (3 siste siffer)": z.string(), +}) + +export type HkdirSubject = z.infer +export const HkdirSubject = z.object({ + "Institusjonskode": z.string(), + "Institusjonsnavn": z.string(), + "Avdelingskode": z.string(), + "Avdelingsnavn": z.string(), + "Avdelingskode_SSB": z.string(), + "Årstall": z.string(), + "Semester": z.string(), + "Semesternavn": z.string(), + "Studieprogramkode": z.string(), + "Studieprogramnavn": z.string(), + "Emnekode": z.string(), + "Emnenavn": z.string(), + "Nivåkode": z.string(), + "Nivånavn": z.string(), + "Studiepoeng": z.string(), + "NUS-kode": z.string(), + "Status": z.string(), + "Statusnavn": z.string(), + "Underv.språk": z.string(), + "Navn": z.string(), + "Fagkode": z.string().nullish().default(null), + "Fagnavn": z.string().nullish().default(null), + "Oppgave (ny fra h2012)": z.string().nullish().default(null), +}) + +const HKDIR_API_BASE_URL = "https://dbh.hkdir.no/api/Tabeller/hentJSONTabellData" + +type QueryOption = Record +interface QueryOptions { + sortBy?: string[] + groupBy?: QueryOption[] + filter: QueryOption[] +} + +export interface HkdirService { + getDepartments(institution: string): Promise + getSubjects(institution: string): Promise +} + +export class HkdirServiceImpl implements HkdirService { + public constructor(private readonly fetch: WindowOrWorkerGlobalScope["fetch"]) {} + + public async getDepartments(institution: string): Promise { + const query = this.createQuery(210, { + sortBy: ["Nivå"], + filter: [ + this.createQueryFilter("Institusjonskode", [institution.toString()]), + this.createExcludeQueryFilter("Avdelingskode", ["000000"]), + ], + }) + return await this.query(HkdirDepartment.array(), query) + } + + async getSubjects(institution: string): Promise { + const query = this.createQuery(208, { + sortBy: ["Årstall", "Institusjonskode", "Avdelingskode"], + filter: [ + this.createQueryFilter("Institusjonskode", [institution.toString()]), + this.createQueryFilter("Nivåkode", ["HN", "LN"]), + this.createQueryFilter("Status", ["1", "2"]), + this.createTopQueryFilter("Årstall", 3), + this.createExcludeQueryFilter("Avdelingskode", ["000000"]), + ], + }) + return await this.query(HkdirSubject.array(), query) + } + + private createQuery(tableId: number, options: QueryOptions): Record { + return { + tabell_id: tableId, + api_versjon: 1, + statuslinje: "N", + kodetekst: "J", + desimal_separator: ".", + variabler: ["*"], + sortBy: options.sortBy, + groupBy: options.groupBy, + filter: options.filter, + } + } + + private async query(schema: T, parameters: Record): Promise> { + const response = await this.fetch(HKDIR_API_BASE_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + "Accept": "application/json", + }, + body: JSON.stringify(parameters), + }) + if (response.body === null) { + throw new Error("HKDir API Response did not contain a body.") + } + + return schema.parse(await response.json()) + } + + private createQueryFilter(variableName: string, values: string[]) { + return { + variabel: variableName, + selection: { + filter: "item", + values, + }, + } + } + + private createTopQueryFilter(variableName: string, query: number) { + return { + variabel: variableName, + selection: { + filter: "top", + values: [query.toString()], + exclude: [""], + }, + } + } + + private createExcludeQueryFilter(variableName: string, values: string[]) { + return { + variabel: variableName, + selection: { + filter: "all", + values: ["*"], + exclude: values, + }, + } + } + + private createAllQueryFilter(variableName: string) { + return { + variabel: variableName, + selection: { + filter: "all", + values: ["*"], + exclude: [""], + }, + } + } +} diff --git a/apps/grades/src/trpc/kysely.ts b/apps/grades/src/trpc/kysely.ts new file mode 100644 index 000000000..ec58d326f --- /dev/null +++ b/apps/grades/src/trpc/kysely.ts @@ -0,0 +1,18 @@ +import process from "node:process" +import pg from "pg" +import { CamelCasePlugin, Kysely, PostgresDialect } from "kysely" +import { type DB } from "@/db.generated" + +export type Database = Awaited> + +export const createKysely = () => { + const conn = new pg.Pool({ + connectionString: process.env.DATABASE_URL ?? "postgres://postgres:postgres@localhost:5433/postgres", + }) + return new Kysely({ + dialect: new PostgresDialect({ + pool: conn, + }), + plugins: [new CamelCasePlugin()], + }) +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0150b133a..116968659 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -236,15 +236,24 @@ importers: apps/grades: dependencies: + kysely: + specifier: ^0.27.2 + version: 0.27.2 next: specifier: 14.1.0 version: 14.1.0(react-dom@18.2.0)(react@18.2.0) + pg: + specifier: ^8.11.3 + version: 8.11.3 react: specifier: ^18 version: 18.2.0 react-dom: specifier: ^18 version: 18.2.0(react@18.2.0) + zod: + specifier: ^3.22.4 + version: 3.22.4 devDependencies: '@dotkomonline/config': specifier: workspace:* @@ -264,6 +273,9 @@ importers: eslint: specifier: ^8.56.0 version: 8.56.0 + kysely-codegen: + specifier: ^0.11.0 + version: 0.11.0(kysely@0.27.2)(pg@8.11.3) postcss: specifier: ^8 version: 8.4.32 @@ -12295,10 +12307,42 @@ packages: pg: 8.11.3 dev: true + /kysely-codegen@0.11.0(kysely@0.27.2)(pg@8.11.3): + resolution: {integrity: sha512-8aklzXygjANshk5BoGSQ0BWukKIoPL4/k1iFWyteGUQ/VtB1GlyrELBZv1GglydjLGECSSVDpsOgEXyWQmuksg==} + hasBin: true + peerDependencies: + '@libsql/kysely-libsql': ^0.3.0 + better-sqlite3: '>=7.6.2' + kysely: '>=0.19.12' + mysql2: ^2.3.3 || ^3.0.0 + pg: ^8.8.0 + peerDependenciesMeta: + '@libsql/kysely-libsql': + optional: true + better-sqlite3: + optional: true + mysql2: + optional: true + pg: + optional: true + dependencies: + chalk: 4.1.2 + dotenv: 16.3.1 + git-diff: 2.0.6 + kysely: 0.27.2 + micromatch: 4.0.5 + minimist: 1.2.8 + pg: 8.11.3 + dev: true + /kysely@0.26.3: resolution: {integrity: sha512-yWSgGi9bY13b/W06DD2OCDDHQmq1kwTGYlQ4wpZkMOJqMGCstVCFIvxCCVG4KfY1/3G0MhDAcZsip/Lw8/vJWw==} engines: {node: '>=14.0.0'} + /kysely@0.27.2: + resolution: {integrity: sha512-DmRvEfiR/NLpgsTbSxma2ldekhsdcd65+MNiKXyd/qj7w7X5e3cLkXxcj+MypsRDjPhHQ/CD5u3Eq1sBYzX0bw==} + engines: {node: '>=14.0.0'} + /language-subtag-registry@0.3.22: resolution: {integrity: sha512-tN0MCzyWnoz/4nHS6uxdlFWoUZT7ABptwKPQ52Ea7URk6vll88bWBVhodtnlfEuCcKWNGoc+uGbw1cwa9IKh/w==} dev: true From 4ce4b113d176a733428b7dc9ecc3ea5852f86d8b Mon Sep 17 00:00:00 2001 From: Mats Larsen Date: Wed, 7 Feb 2024 21:08:33 +0100 Subject: [PATCH 11/23] Implement fetcing subject grdaes --- apps/grades/src/pages/api/grades.ts | 9 +++++++++ apps/grades/src/trpc/hkdir-service.ts | 26 +++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 apps/grades/src/pages/api/grades.ts diff --git a/apps/grades/src/pages/api/grades.ts b/apps/grades/src/pages/api/grades.ts new file mode 100644 index 000000000..7e9fa1c0d --- /dev/null +++ b/apps/grades/src/pages/api/grades.ts @@ -0,0 +1,9 @@ +import { type NextApiRequest, type NextApiResponse } from "next" +import { HkdirServiceImpl } from "@/trpc/hkdir-service" + +export default async function route(req: NextApiRequest, res: NextApiResponse) { + const service = new HkdirServiceImpl(fetch) + const grades = await service.getSubjectGrades("1150") + + res.json(grades) +} diff --git a/apps/grades/src/trpc/hkdir-service.ts b/apps/grades/src/trpc/hkdir-service.ts index 855a99176..6e3e54c5f 100644 --- a/apps/grades/src/trpc/hkdir-service.ts +++ b/apps/grades/src/trpc/hkdir-service.ts @@ -44,18 +44,29 @@ export const HkdirSubject = z.object({ "Oppgave (ny fra h2012)": z.string().nullish().default(null), }) +export type HkdirGrade = z.infer +export const HkdirGrade = z.object({ + "Årstall": z.string(), + "Semester": z.string(), + "Semesternavn": z.string(), + "Karakter": z.string(), + "Emnekode": z.string(), + "Antall kandidater totalt": z.string(), +}) + const HKDIR_API_BASE_URL = "https://dbh.hkdir.no/api/Tabeller/hentJSONTabellData" type QueryOption = Record interface QueryOptions { sortBy?: string[] - groupBy?: QueryOption[] + groupBy?: string[] filter: QueryOption[] } export interface HkdirService { getDepartments(institution: string): Promise getSubjects(institution: string): Promise + getSubjectGrades(institution: string): Promise } export class HkdirServiceImpl implements HkdirService { @@ -86,6 +97,19 @@ export class HkdirServiceImpl implements HkdirService { return await this.query(HkdirSubject.array(), query) } + async getSubjectGrades(institution: string): Promise { + const query = this.createQuery(308, { + groupBy: ["Årstall", "Semester", "Karakter", "Emnekode", "Institusjonskode"], + filter: [ + this.createQueryFilter("Institusjonskode", [institution.toString()]), + this.createTopQueryFilter("Årstall", 3), + this.createAllQueryFilter("Emnekode"), + this.createAllQueryFilter("Semester"), + ], + }) + return await this.query(HkdirGrade.array(), query) + } + private createQuery(tableId: number, options: QueryOptions): Record { return { tabell_id: tableId, From 0dd8efa9baf7ef78c6541970e6f4995ce0c0d8a0 Mon Sep 17 00:00:00 2001 From: Mats Larsen Date: Wed, 7 Feb 2024 21:15:08 +0100 Subject: [PATCH 12/23] Rename trpc to server --- apps/grades/src/pages/api/departments.ts | 2 +- apps/grades/src/pages/api/grades.ts | 2 +- apps/grades/src/pages/api/subjects.ts | 2 +- apps/grades/src/{trpc => server}/hkdir-service.ts | 0 apps/grades/src/{trpc => server}/kysely.ts | 0 5 files changed, 3 insertions(+), 3 deletions(-) rename apps/grades/src/{trpc => server}/hkdir-service.ts (100%) rename apps/grades/src/{trpc => server}/kysely.ts (100%) diff --git a/apps/grades/src/pages/api/departments.ts b/apps/grades/src/pages/api/departments.ts index eea82ce4f..40263b01f 100644 --- a/apps/grades/src/pages/api/departments.ts +++ b/apps/grades/src/pages/api/departments.ts @@ -1,5 +1,5 @@ import { type NextApiRequest, type NextApiResponse } from "next" -import { HkdirServiceImpl } from "@/trpc/hkdir-service" +import { HkdirServiceImpl } from "@/server/hkdir-service" export default async function route(req: NextApiRequest, res: NextApiResponse) { const service = new HkdirServiceImpl(fetch) diff --git a/apps/grades/src/pages/api/grades.ts b/apps/grades/src/pages/api/grades.ts index 7e9fa1c0d..6e93d6899 100644 --- a/apps/grades/src/pages/api/grades.ts +++ b/apps/grades/src/pages/api/grades.ts @@ -1,5 +1,5 @@ import { type NextApiRequest, type NextApiResponse } from "next" -import { HkdirServiceImpl } from "@/trpc/hkdir-service" +import { HkdirServiceImpl } from "@/server/hkdir-service" export default async function route(req: NextApiRequest, res: NextApiResponse) { const service = new HkdirServiceImpl(fetch) diff --git a/apps/grades/src/pages/api/subjects.ts b/apps/grades/src/pages/api/subjects.ts index 329edcf1e..e4f64f42e 100644 --- a/apps/grades/src/pages/api/subjects.ts +++ b/apps/grades/src/pages/api/subjects.ts @@ -1,5 +1,5 @@ import { type NextApiRequest, type NextApiResponse } from "next" -import { HkdirServiceImpl } from "@/trpc/hkdir-service" +import { HkdirServiceImpl } from "@/server/hkdir-service" export default async function route(req: NextApiRequest, res: NextApiResponse) { const service = new HkdirServiceImpl(fetch) diff --git a/apps/grades/src/trpc/hkdir-service.ts b/apps/grades/src/server/hkdir-service.ts similarity index 100% rename from apps/grades/src/trpc/hkdir-service.ts rename to apps/grades/src/server/hkdir-service.ts diff --git a/apps/grades/src/trpc/kysely.ts b/apps/grades/src/server/kysely.ts similarity index 100% rename from apps/grades/src/trpc/kysely.ts rename to apps/grades/src/server/kysely.ts From e5de81ec8d032ad355fa5ebce40f3d6081bb0613 Mon Sep 17 00:00:00 2001 From: Mats Larsen Date: Wed, 7 Feb 2024 21:37:56 +0100 Subject: [PATCH 13/23] Populate faculty names into database --- apps/grades/src/pages/api/departments.ts | 9 ----- apps/grades/src/pages/api/grades.ts | 9 ----- apps/grades/src/pages/api/repopulate.ts | 23 +++++++++++++ apps/grades/src/pages/api/subjects.ts | 9 ----- apps/grades/src/server/core.ts | 20 +++++++++++ apps/grades/src/server/faculty-repository.ts | 36 ++++++++++++++++++++ apps/grades/src/server/hkdir-service.ts | 4 +-- 7 files changed, 81 insertions(+), 29 deletions(-) delete mode 100644 apps/grades/src/pages/api/departments.ts delete mode 100644 apps/grades/src/pages/api/grades.ts create mode 100644 apps/grades/src/pages/api/repopulate.ts delete mode 100644 apps/grades/src/pages/api/subjects.ts create mode 100644 apps/grades/src/server/core.ts create mode 100644 apps/grades/src/server/faculty-repository.ts diff --git a/apps/grades/src/pages/api/departments.ts b/apps/grades/src/pages/api/departments.ts deleted file mode 100644 index 40263b01f..000000000 --- a/apps/grades/src/pages/api/departments.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { type NextApiRequest, type NextApiResponse } from "next" -import { HkdirServiceImpl } from "@/server/hkdir-service" - -export default async function route(req: NextApiRequest, res: NextApiResponse) { - const service = new HkdirServiceImpl(fetch) - const departments = await service.getDepartments("1150") - - res.json(departments) -} diff --git a/apps/grades/src/pages/api/grades.ts b/apps/grades/src/pages/api/grades.ts deleted file mode 100644 index 6e93d6899..000000000 --- a/apps/grades/src/pages/api/grades.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { type NextApiRequest, type NextApiResponse } from "next" -import { HkdirServiceImpl } from "@/server/hkdir-service" - -export default async function route(req: NextApiRequest, res: NextApiResponse) { - const service = new HkdirServiceImpl(fetch) - const grades = await service.getSubjectGrades("1150") - - res.json(grades) -} diff --git a/apps/grades/src/pages/api/repopulate.ts b/apps/grades/src/pages/api/repopulate.ts new file mode 100644 index 000000000..e84f80e5a --- /dev/null +++ b/apps/grades/src/pages/api/repopulate.ts @@ -0,0 +1,23 @@ +import { type NextApiRequest, type NextApiResponse } from "next" +import { createKysely } from "@/server/kysely" +import { createServiceLayer } from "@/server/core" + +export default async function route(req: NextApiRequest, res: NextApiResponse) { + const kysely = await createKysely() + const core = createServiceLayer({ fetch, db: kysely }) + + const departments = await core.hkdirService.getDepartments("1150") + + for (const department of departments) { + const faculty = await core.facultyRepository.getFacultyByReferenceId(department.Fakultetskode) + if (faculty !== null) { + continue + } + await core.facultyRepository.createFaculty({ + refId: department.Fakultetskode, + name: department.Fakultetsnavn, + }) + } + + res.status(200).send({}) +} diff --git a/apps/grades/src/pages/api/subjects.ts b/apps/grades/src/pages/api/subjects.ts deleted file mode 100644 index e4f64f42e..000000000 --- a/apps/grades/src/pages/api/subjects.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { type NextApiRequest, type NextApiResponse } from "next" -import { HkdirServiceImpl } from "@/server/hkdir-service" - -export default async function route(req: NextApiRequest, res: NextApiResponse) { - const service = new HkdirServiceImpl(fetch) - const subjects = await service.getSubjects("1150") - - res.json(subjects) -} diff --git a/apps/grades/src/server/core.ts b/apps/grades/src/server/core.ts new file mode 100644 index 000000000..fdd9a56cb --- /dev/null +++ b/apps/grades/src/server/core.ts @@ -0,0 +1,20 @@ +import { type Database } from "@/server/kysely" +import { type FacultyRepository, FacultyRepositoryImpl } from "@/server/faculty-repository" +import { type HkdirService, HkdirServiceImpl } from "@/server/hkdir-service" + +export interface CreateServiceLayerOptions { + fetch: WindowOrWorkerGlobalScope["fetch"] + db: Database +} + +export type ServiceLayer = Awaited> + +export const createServiceLayer = (opts: CreateServiceLayerOptions) => { + const facultyRepository: FacultyRepository = new FacultyRepositoryImpl(opts.db) + const hkdirService: HkdirService = new HkdirServiceImpl(opts.fetch) + + return { + facultyRepository, + hkdirService, + } +} diff --git a/apps/grades/src/server/faculty-repository.ts b/apps/grades/src/server/faculty-repository.ts new file mode 100644 index 000000000..88e9eb8d3 --- /dev/null +++ b/apps/grades/src/server/faculty-repository.ts @@ -0,0 +1,36 @@ +import { z } from "zod" +import { type Insertable } from "kysely" +import { type NtnuFaculty } from "@/db.generated" +import { type Database } from "@/server/kysely" + +export type Faculty = z.infer +export const Faculty = z.object({ + id: z.string().uuid(), + refId: z.string(), + name: z.string(), +}) + +export interface FacultyRepository { + createFaculty(input: Insertable): Promise + getFacultyByReferenceId(refId: string): Promise + getFacultyById(id: string): Promise +} + +export class FacultyRepositoryImpl implements FacultyRepository { + constructor(private readonly db: Database) {} + + async createFaculty(input: Insertable): Promise { + const faculty = await this.db.insertInto("ntnuFaculty").values(input).returningAll().executeTakeFirstOrThrow() + return Faculty.parse(faculty) + } + + async getFacultyByReferenceId(refId: string): Promise { + const faculty = await this.db.selectFrom("ntnuFaculty").selectAll().where("refId", "=", refId).executeTakeFirst() + return faculty ? Faculty.parse(faculty) : null + } + + async getFacultyById(id: string): Promise { + const faculty = await this.db.selectFrom("ntnuFaculty").selectAll().where("id", "=", id).executeTakeFirst() + return faculty ? Faculty.parse(faculty) : null + } +} diff --git a/apps/grades/src/server/hkdir-service.ts b/apps/grades/src/server/hkdir-service.ts index 6e3e54c5f..940cfa447 100644 --- a/apps/grades/src/server/hkdir-service.ts +++ b/apps/grades/src/server/hkdir-service.ts @@ -12,8 +12,8 @@ export const HkdirDepartment = z.object({ "Gyldig_til": z.string().nullish().default(null), "fagkode_avdeling": z.string().nullish().default(null), "fagnavn_avdeling": z.string().nullish().default(null), - "Fakultetskode": z.string().nullish().default(null), - "Fakueltetsnavn": z.string().nullish().default(null), + "Fakultetskode": z.string(), + "Fakultetsnavn": z.string(), "Avdelingskode (3 siste siffer)": z.string(), }) From d72033b4fb6aa597abc90a822851d68a2190255f Mon Sep 17 00:00:00 2001 From: Mats Larsen Date: Wed, 7 Feb 2024 22:05:49 +0100 Subject: [PATCH 14/23] Refactor faculty population into job service --- apps/grades/src/pages/api/repopulate.ts | 14 +----- apps/grades/src/server/core.ts | 6 ++- .../src/server/department-repository.ts | 49 +++++++++++++++++++ apps/grades/src/server/job-service.ts | 48 ++++++++++++++++++ 4 files changed, 103 insertions(+), 14 deletions(-) create mode 100644 apps/grades/src/server/department-repository.ts create mode 100644 apps/grades/src/server/job-service.ts diff --git a/apps/grades/src/pages/api/repopulate.ts b/apps/grades/src/pages/api/repopulate.ts index e84f80e5a..52702bf02 100644 --- a/apps/grades/src/pages/api/repopulate.ts +++ b/apps/grades/src/pages/api/repopulate.ts @@ -6,18 +6,6 @@ export default async function route(req: NextApiRequest, res: NextApiResponse) { const kysely = await createKysely() const core = createServiceLayer({ fetch, db: kysely }) - const departments = await core.hkdirService.getDepartments("1150") - - for (const department of departments) { - const faculty = await core.facultyRepository.getFacultyByReferenceId(department.Fakultetskode) - if (faculty !== null) { - continue - } - await core.facultyRepository.createFaculty({ - refId: department.Fakultetskode, - name: department.Fakultetsnavn, - }) - } - + await core.jobService.performFacultySynchronizationJob() res.status(200).send({}) } diff --git a/apps/grades/src/server/core.ts b/apps/grades/src/server/core.ts index fdd9a56cb..ca9f69c70 100644 --- a/apps/grades/src/server/core.ts +++ b/apps/grades/src/server/core.ts @@ -1,6 +1,8 @@ import { type Database } from "@/server/kysely" import { type FacultyRepository, FacultyRepositoryImpl } from "@/server/faculty-repository" import { type HkdirService, HkdirServiceImpl } from "@/server/hkdir-service" +import { type DepartmentRepository, DepartmentRepositoryImpl } from "@/server/department-repository" +import { type JobService, JobServiceImpl } from "@/server/job-service" export interface CreateServiceLayerOptions { fetch: WindowOrWorkerGlobalScope["fetch"] @@ -11,10 +13,12 @@ export type ServiceLayer = Awaited> export const createServiceLayer = (opts: CreateServiceLayerOptions) => { const facultyRepository: FacultyRepository = new FacultyRepositoryImpl(opts.db) + const departmentRepository: DepartmentRepository = new DepartmentRepositoryImpl(opts.db) const hkdirService: HkdirService = new HkdirServiceImpl(opts.fetch) + const jobService: JobService = new JobServiceImpl(facultyRepository, departmentRepository, hkdirService) return { - facultyRepository, hkdirService, + jobService, } } diff --git a/apps/grades/src/server/department-repository.ts b/apps/grades/src/server/department-repository.ts new file mode 100644 index 000000000..4faf90094 --- /dev/null +++ b/apps/grades/src/server/department-repository.ts @@ -0,0 +1,49 @@ +import { z } from "zod" +import { type Insertable } from "kysely" +import { type NtnuFacultyDepartment } from "@/db.generated" +import { type Database } from "@/server/kysely" + +export type Department = z.infer +export const Department = z.object({ + id: z.string().uuid(), + refId: z.string(), + facultyId: z.string().uuid(), + name: z.string(), +}) + +export interface DepartmentRepository { + createDepartment(input: Insertable): Promise + getDepartmentByReferenceId(refId: string): Promise + getDepartmentById(id: string): Promise +} + +export class DepartmentRepositoryImpl implements DepartmentRepository { + constructor(private readonly db: Database) {} + + async createDepartment(input: Insertable): Promise { + const department = await this.db + .insertInto("ntnuFacultyDepartment") + .values(input) + .returningAll() + .executeTakeFirstOrThrow() + return Department.parse(department) + } + + async getDepartmentByReferenceId(refId: string): Promise { + const department = await this.db + .selectFrom("ntnuFacultyDepartment") + .selectAll() + .where("refId", "=", refId) + .executeTakeFirst() + return department ? Department.parse(department) : null + } + + async getDepartmentById(id: string): Promise { + const department = await this.db + .selectFrom("ntnuFacultyDepartment") + .selectAll() + .where("id", "=", id) + .executeTakeFirst() + return department ? Department.parse(department) : null + } +} diff --git a/apps/grades/src/server/job-service.ts b/apps/grades/src/server/job-service.ts new file mode 100644 index 000000000..8717226b1 --- /dev/null +++ b/apps/grades/src/server/job-service.ts @@ -0,0 +1,48 @@ +import { type FacultyRepository } from "@/server/faculty-repository" +import { type HkdirService } from "@/server/hkdir-service" +import { type DepartmentRepository } from "@/server/department-repository" + +export interface JobService { + performFacultySynchronizationJob(): Promise +} + +export class JobServiceImpl implements JobService { + public constructor( + private readonly facultyRepository: FacultyRepository, + private readonly departmentRepository: DepartmentRepository, + private readonly hkdirService: HkdirService + ) {} + + /** + * Synchronize faculties from HKDir to the database. + * + * HKDir departments are used to represent both faculties and departments in our data model. In HKDir, a department is + * of either level two or three. A level two department is a faculty in our model, while level three departments are + * the equivalent of departments. + * + * Because of this hierarchy, a level three department MUST have a matching level two department before creation. + */ + public async performFacultySynchronizationJob(): Promise { + const faculties = await this.hkdirService.getDepartments("1150") + for (const faculty of faculties) { + // Attempt to register the faculty (level two institution) + let existingFaculty = await this.facultyRepository.getFacultyByReferenceId(faculty.Fakultetskode) + if (existingFaculty === null) { + existingFaculty = await this.facultyRepository.createFaculty({ + name: faculty.Fakultetsnavn, + refId: faculty.Fakultetskode, + }) + } + + // Attempt to register the department (level three institution) + let existingDepartment = await this.departmentRepository.getDepartmentByReferenceId(faculty.Avdelingskode) + if (existingDepartment === null) { + existingDepartment = await this.departmentRepository.createDepartment({ + name: faculty.Avdelingsnavn, + refId: faculty.Avdelingskode, + facultyId: existingFaculty.id, + }) + } + } + } +} From 87d02ef8b27201c20ae900072a300d7c21587305 Mon Sep 17 00:00:00 2001 From: Mats Larsen Date: Wed, 7 Feb 2024 23:40:25 +0100 Subject: [PATCH 15/23] Add logging to subject sync job --- apps/grades/next.config.mjs | 4 +- apps/grades/package.json | 1 + apps/grades/src/pages/api/repopulate.ts | 15 +++++- apps/grades/src/server/core.ts | 9 +++- apps/grades/src/server/hkdir-service.ts | 2 +- apps/grades/src/server/job-service.ts | 55 ++++++++++++++++++++ apps/grades/src/server/subject-repository.ts | 41 +++++++++++++++ pnpm-lock.yaml | 3 ++ 8 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 apps/grades/src/server/subject-repository.ts diff --git a/apps/grades/next.config.mjs b/apps/grades/next.config.mjs index 4678774e6..ece3de141 100644 --- a/apps/grades/next.config.mjs +++ b/apps/grades/next.config.mjs @@ -1,4 +1,6 @@ /** @type {import('next').NextConfig} */ -const nextConfig = {}; +const nextConfig = { + transpilePackages: ['@dotkomonline/logger'] +}; export default nextConfig; diff --git a/apps/grades/package.json b/apps/grades/package.json index ab89f38dd..483c26344 100644 --- a/apps/grades/package.json +++ b/apps/grades/package.json @@ -12,6 +12,7 @@ "generate": "kysely-codegen --dialect postgres --camel-case --out-file src/db.generated.d.ts" }, "dependencies": { + "@dotkomonline/logger": "workspace:*", "kysely": "^0.27.2", "pg": "^8.11.3", "react": "^18", diff --git a/apps/grades/src/pages/api/repopulate.ts b/apps/grades/src/pages/api/repopulate.ts index 52702bf02..4f3b602ea 100644 --- a/apps/grades/src/pages/api/repopulate.ts +++ b/apps/grades/src/pages/api/repopulate.ts @@ -1,11 +1,24 @@ import { type NextApiRequest, type NextApiResponse } from "next" +import { getLogger } from "@dotkomonline/logger" import { createKysely } from "@/server/kysely" import { createServiceLayer } from "@/server/core" +const logger = getLogger("api/repopulate") + export default async function route(req: NextApiRequest, res: NextApiResponse) { + const skip = (req.query.skip as string | undefined)?.split(",") + logger.info(`Initializing repopulation job with skip ${skip?.join(", ")}) ?? ""}`) + const kysely = await createKysely() const core = createServiceLayer({ fetch, db: kysely }) - await core.jobService.performFacultySynchronizationJob() + if (!skip?.includes("faculty")) { + await core.jobService.performFacultySynchronizationJob() + } + + if (!skip?.includes("subject")) { + await core.jobService.performSubjectSynchronizationJob() + } + res.status(200).send({}) } diff --git a/apps/grades/src/server/core.ts b/apps/grades/src/server/core.ts index ca9f69c70..a3405332c 100644 --- a/apps/grades/src/server/core.ts +++ b/apps/grades/src/server/core.ts @@ -3,6 +3,7 @@ import { type FacultyRepository, FacultyRepositoryImpl } from "@/server/faculty- import { type HkdirService, HkdirServiceImpl } from "@/server/hkdir-service" import { type DepartmentRepository, DepartmentRepositoryImpl } from "@/server/department-repository" import { type JobService, JobServiceImpl } from "@/server/job-service" +import { type SubjectRepository, SubjectRepositoryImpl } from "@/server/subject-repository" export interface CreateServiceLayerOptions { fetch: WindowOrWorkerGlobalScope["fetch"] @@ -14,8 +15,14 @@ export type ServiceLayer = Awaited> export const createServiceLayer = (opts: CreateServiceLayerOptions) => { const facultyRepository: FacultyRepository = new FacultyRepositoryImpl(opts.db) const departmentRepository: DepartmentRepository = new DepartmentRepositoryImpl(opts.db) + const subjectRepository: SubjectRepository = new SubjectRepositoryImpl(opts.db) const hkdirService: HkdirService = new HkdirServiceImpl(opts.fetch) - const jobService: JobService = new JobServiceImpl(facultyRepository, departmentRepository, hkdirService) + const jobService: JobService = new JobServiceImpl( + facultyRepository, + departmentRepository, + subjectRepository, + hkdirService + ) return { hkdirService, diff --git a/apps/grades/src/server/hkdir-service.ts b/apps/grades/src/server/hkdir-service.ts index 940cfa447..eebd4df41 100644 --- a/apps/grades/src/server/hkdir-service.ts +++ b/apps/grades/src/server/hkdir-service.ts @@ -90,7 +90,7 @@ export class HkdirServiceImpl implements HkdirService { this.createQueryFilter("Institusjonskode", [institution.toString()]), this.createQueryFilter("Nivåkode", ["HN", "LN"]), this.createQueryFilter("Status", ["1", "2"]), - this.createTopQueryFilter("Årstall", 3), + this.createTopQueryFilter("Årstall", 1), this.createExcludeQueryFilter("Avdelingskode", ["000000"]), ], }) diff --git a/apps/grades/src/server/job-service.ts b/apps/grades/src/server/job-service.ts index 8717226b1..adf910d04 100644 --- a/apps/grades/src/server/job-service.ts +++ b/apps/grades/src/server/job-service.ts @@ -1,15 +1,21 @@ +import { getLogger } from "@dotkomonline/logger" import { type FacultyRepository } from "@/server/faculty-repository" import { type HkdirService } from "@/server/hkdir-service" import { type DepartmentRepository } from "@/server/department-repository" +import { type SubjectRepository } from "@/server/subject-repository" export interface JobService { performFacultySynchronizationJob(): Promise + performSubjectSynchronizationJob(): Promise } export class JobServiceImpl implements JobService { + private readonly logger = getLogger("JobService") + public constructor( private readonly facultyRepository: FacultyRepository, private readonly departmentRepository: DepartmentRepository, + private readonly subjectRepository: SubjectRepository, private readonly hkdirService: HkdirService ) {} @@ -23,8 +29,16 @@ export class JobServiceImpl implements JobService { * Because of this hierarchy, a level three department MUST have a matching level two department before creation. */ public async performFacultySynchronizationJob(): Promise { + this.logger.info("Synchronizing faculties from HKDir") const faculties = await this.hkdirService.getDepartments("1150") + this.logger.info(`Beginning synchronization of ${faculties.length} faculties`) + + let count = 0 for (const faculty of faculties) { + if (count !== 0 && count % 100 === 0) { + this.logger.info(`Synchronized ${count} faculties`) + } + // Attempt to register the faculty (level two institution) let existingFaculty = await this.facultyRepository.getFacultyByReferenceId(faculty.Fakultetskode) if (existingFaculty === null) { @@ -43,6 +57,47 @@ export class JobServiceImpl implements JobService { facultyId: existingFaculty.id, }) } + + count++ + } + this.logger.info("Synchronization of faculties complete") + } + + public async performSubjectSynchronizationJob() { + this.logger.info("Synchronizing subjects from HKDir") + const subjects = await this.hkdirService.getSubjects("1150") + this.logger.info(`Beginning synchronization of ${subjects.length} subjects`) + + let count = 0 + for (const subject of subjects) { + if (count !== 0 && count % 100 === 0) { + this.logger.info(`Synchronized ${count} subjects`) + } + + // Pull the department for the given subject from the database + const department = await this.departmentRepository.getDepartmentByReferenceId(subject.Avdelingskode) + if (department === null) { + throw new Error(`Department with reference ID ${subject.Avdelingskode} not found`) + } + + // Attempt to register the subject + let existingSubject = await this.subjectRepository.getSubjectByReferenceId(subject.Emnekode) + if (existingSubject === null) { + // A lot of subjects have a -x suffix, which we want to remove for the slug + const slug = subject.Emnekode.replace(/-(1|2|a|b|k)$/, "").toLowerCase() + existingSubject = await this.subjectRepository.createSubject({ + name: subject.Emnenavn, + refId: subject.Emnekode, + educationalLevel: subject.Nivåkode, + instructionLanguage: subject["Underv.språk"], + credits: parseFloat(subject.Studiepoeng), + departmentId: department.id, + slug, + }) + } + + count++ } + this.logger.info("Synchronization of subjects complete") } } diff --git a/apps/grades/src/server/subject-repository.ts b/apps/grades/src/server/subject-repository.ts new file mode 100644 index 000000000..9310b09f4 --- /dev/null +++ b/apps/grades/src/server/subject-repository.ts @@ -0,0 +1,41 @@ +import { z } from "zod" +import { type Insertable } from "kysely" +import { type Subject as DatabaseSubject } from "@/db.generated" +import { type Database } from "@/server/kysely" + +export type Subject = z.infer +export const Subject = z.object({ + id: z.string().uuid(), + refId: z.string(), + departmentId: z.string().uuid(), + name: z.string(), + slug: z.string(), + instructionLanguage: z.string(), + educationalLevel: z.string(), + credits: z.number(), +}) + +export interface SubjectRepository { + createSubject(input: Insertable): Promise + getSubjectByReferenceId(refId: string): Promise + getSubjectById(id: string): Promise +} + +export class SubjectRepositoryImpl implements SubjectRepository { + constructor(private readonly db: Database) {} + + async createSubject(input: Insertable): Promise { + const subject = await this.db.insertInto("subject").values(input).returningAll().executeTakeFirstOrThrow() + return Subject.parse(subject) + } + + async getSubjectByReferenceId(refId: string): Promise { + const subject = await this.db.selectFrom("subject").selectAll().where("refId", "=", refId).executeTakeFirst() + return subject ? Subject.parse(subject) : null + } + + async getSubjectById(id: string): Promise { + const subject = await this.db.selectFrom("subject").selectAll().where("id", "=", id).executeTakeFirst() + return subject ? Subject.parse(subject) : null + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 116968659..f6c26ff1e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -236,6 +236,9 @@ importers: apps/grades: dependencies: + '@dotkomonline/logger': + specifier: workspace:* + version: link:../../packages/logger kysely: specifier: ^0.27.2 version: 0.27.2 From 2929ef6785f6b00a70469f49b6366535075b18bb Mon Sep 17 00:00:00 2001 From: Mats Larsen Date: Wed, 7 Feb 2024 23:56:17 +0100 Subject: [PATCH 16/23] Speed up subject synchronization with async priority queue --- apps/grades/package.json | 3 +- apps/grades/src/server/job-service.ts | 61 +++++++++++++++------------ pnpm-lock.yaml | 21 +++++++++ turbo.json | 3 +- 4 files changed, 60 insertions(+), 28 deletions(-) diff --git a/apps/grades/package.json b/apps/grades/package.json index 483c26344..bd33da203 100644 --- a/apps/grades/package.json +++ b/apps/grades/package.json @@ -14,10 +14,11 @@ "dependencies": { "@dotkomonline/logger": "workspace:*", "kysely": "^0.27.2", + "next": "14.1.0", + "p-queue": "^8.0.1", "pg": "^8.11.3", "react": "^18", "react-dom": "^18", - "next": "14.1.0", "zod": "^3.22.4" }, "devDependencies": { diff --git a/apps/grades/src/server/job-service.ts b/apps/grades/src/server/job-service.ts index adf910d04..38fdfbcd2 100644 --- a/apps/grades/src/server/job-service.ts +++ b/apps/grades/src/server/job-service.ts @@ -1,4 +1,5 @@ import { getLogger } from "@dotkomonline/logger" +import Queue from "p-queue" import { type FacultyRepository } from "@/server/faculty-repository" import { type HkdirService } from "@/server/hkdir-service" import { type DepartmentRepository } from "@/server/department-repository" @@ -11,6 +12,7 @@ export interface JobService { export class JobServiceImpl implements JobService { private readonly logger = getLogger("JobService") + private readonly concurrencyLevel = parseInt(process.env.MAX_CONCURRENCY ?? "10") public constructor( private readonly facultyRepository: FacultyRepository, @@ -66,38 +68,45 @@ export class JobServiceImpl implements JobService { public async performSubjectSynchronizationJob() { this.logger.info("Synchronizing subjects from HKDir") const subjects = await this.hkdirService.getSubjects("1150") - this.logger.info(`Beginning synchronization of ${subjects.length} subjects`) + this.logger.info( + `Beginning synchronization of ${subjects.length} subjects with concurrency level of ${this.concurrencyLevel}` + ) + + const queue = new Queue({ concurrency: this.concurrencyLevel }) - let count = 0 for (const subject of subjects) { - if (count !== 0 && count % 100 === 0) { - this.logger.info(`Synchronized ${count} subjects`) - } + queue.add(async () => { + // Pull the department for the given subject from the database + const department = await this.departmentRepository.getDepartmentByReferenceId(subject.Avdelingskode) + if (department === null) { + throw new Error(`Department with reference ID ${subject.Avdelingskode} not found`) + } - // Pull the department for the given subject from the database - const department = await this.departmentRepository.getDepartmentByReferenceId(subject.Avdelingskode) - if (department === null) { - throw new Error(`Department with reference ID ${subject.Avdelingskode} not found`) - } + // Attempt to register the subject + let existingSubject = await this.subjectRepository.getSubjectByReferenceId(subject.Emnekode) + if (existingSubject === null) { + // A lot of subjects have a -x suffix, which we want to remove for the slug + const slug = subject.Emnekode.replace(/-(1|2|a|b|k)$/, "").toLowerCase() + existingSubject = await this.subjectRepository.createSubject({ + name: subject.Emnenavn, + refId: subject.Emnekode, + educationalLevel: subject.Nivåkode, + instructionLanguage: subject["Underv.språk"], + credits: parseFloat(subject.Studiepoeng), + departmentId: department.id, + slug, + }) + } + }) + } - // Attempt to register the subject - let existingSubject = await this.subjectRepository.getSubjectByReferenceId(subject.Emnekode) - if (existingSubject === null) { - // A lot of subjects have a -x suffix, which we want to remove for the slug - const slug = subject.Emnekode.replace(/-(1|2|a|b|k)$/, "").toLowerCase() - existingSubject = await this.subjectRepository.createSubject({ - name: subject.Emnenavn, - refId: subject.Emnekode, - educationalLevel: subject.Nivåkode, - instructionLanguage: subject["Underv.språk"], - credits: parseFloat(subject.Studiepoeng), - departmentId: department.id, - slug, - }) + queue.on("next", () => { + if (queue.size % 100 !== 0 && queue.size % 100 === 0) { + this.logger.info(`Queue processing for subjects reached ${queue.size} jobs left`) } + }) - count++ - } + await queue.onIdle() this.logger.info("Synchronization of subjects complete") } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f6c26ff1e..a2b14fa15 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -245,6 +245,9 @@ importers: next: specifier: 14.1.0 version: 14.1.0(react-dom@18.2.0)(react@18.2.0) + p-queue: + specifier: ^8.0.1 + version: 8.0.1 pg: specifier: ^8.11.3 version: 8.11.3 @@ -9148,6 +9151,7 @@ packages: loose-envify: 1.4.0 object-assign: 4.1.1 dev: false + bundledDependencies: false /create-require@1.1.1: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} @@ -10470,6 +10474,10 @@ packages: through: 2.3.8 dev: true + /eventemitter3@5.0.1: + resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==} + dev: false + /eventsource@2.0.2: resolution: {integrity: sha512-IzUmBGPR3+oUG9dUeXynyNmf91/3zUSJg1lCktzKw47OXuhco54U3r9B7O4XX+Rb1Itm9OZ2b0RkTs10bICOxA==} engines: {node: '>=12.0.0'} @@ -13760,6 +13768,19 @@ packages: engines: {node: '>=4'} dev: false + /p-queue@8.0.1: + resolution: {integrity: sha512-NXzu9aQJTAzbBqOt2hwsR63ea7yvxJc0PwN/zobNAudYfb1B7R08SzB4TsLeSbUCuG467NhnoT0oO6w1qRO+BA==} + engines: {node: '>=18'} + dependencies: + eventemitter3: 5.0.1 + p-timeout: 6.1.2 + dev: false + + /p-timeout@6.1.2: + resolution: {integrity: sha512-UbD77BuZ9Bc9aABo74gfXhNvzC9Tx7SxtHSh1fxvx3jTLLYvmVhiQZZrJzqqU0jKbN32kb5VOKiLEQI/3bIjgQ==} + engines: {node: '>=14.16'} + dev: false + /p-try@2.2.0: resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} engines: {node: '>=6'} diff --git a/turbo.json b/turbo.json index d3b3e3802..080a0ad49 100644 --- a/turbo.json +++ b/turbo.json @@ -69,6 +69,7 @@ "EMAIL_TOKEN", "EMAIL_ENDPOINT", "INTEREST_FORM_SERVICE_ACCOUNT", - "INTEREST_FORM_SPREADSHEET_ID" + "INTEREST_FORM_SPREADSHEET_ID", + "MAX_CONCURRENCY" ] } From 5b03125eac38fb71e6ed2491c6f24131965ae54e Mon Sep 17 00:00:00 2001 From: Mats Larsen Date: Thu, 8 Feb 2024 00:34:54 +0100 Subject: [PATCH 17/23] Fetch departments in parallel as well --- apps/grades/migrations/0001_schema.sql | 10 +- apps/grades/src/pages/api/repopulate.ts | 2 +- .../src/server/department-repository.ts | 1 + apps/grades/src/server/faculty-repository.ts | 7 +- apps/grades/src/server/job-service.ts | 126 +++++++++--------- apps/grades/src/server/subject-repository.ts | 7 +- apps/grades/src/server/util.ts | 19 +++ 7 files changed, 104 insertions(+), 68 deletions(-) create mode 100644 apps/grades/src/server/util.ts diff --git a/apps/grades/migrations/0001_schema.sql b/apps/grades/migrations/0001_schema.sql index 09417f08f..cf88d16c1 100644 --- a/apps/grades/migrations/0001_schema.sql +++ b/apps/grades/migrations/0001_schema.sql @@ -6,7 +6,9 @@ CREATE TABLE IF NOT EXISTS ntnu_faculty ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), ref_id TEXT NOT NULL, - name TEXT NOT NULL + name TEXT NOT NULL, + + CONSTRAINT ntnu_faculty_uq_ref_id UNIQUE (ref_id) ); CREATE TABLE IF NOT EXISTS ntnu_faculty_department @@ -16,7 +18,8 @@ CREATE TABLE IF NOT EXISTS ntnu_faculty_department faculty_id UUID NOT NULL, name TEXT NOT NULL, - CONSTRAINT fk_faculty_id FOREIGN KEY (faculty_id) REFERENCES ntnu_faculty (id) + CONSTRAINT ntnu_faculty_department_fk_faculty_id FOREIGN KEY (faculty_id) REFERENCES ntnu_faculty (id), + CONSTRAINT ntnu_faculty_department_uq_ref_id UNIQUE (ref_id) ); CREATE TABLE IF NOT EXISTS subject @@ -31,7 +34,8 @@ CREATE TABLE IF NOT EXISTS subject educational_level TEXT NOT NULL, credits FLOAT NOT NULL, - CONSTRAINT fk_department_id FOREIGN KEY (department_id) REFERENCES ntnu_faculty_department (id) + CONSTRAINT subject_fk_department_id FOREIGN KEY (department_id) REFERENCES ntnu_faculty_department (id), + CONSTRAINT subject_uq_ref_id UNIQUE (ref_id) ); CREATE TYPE subject_season AS ENUM ('SPRING', 'AUTUMN', 'WINTER', 'SUMMER'); diff --git a/apps/grades/src/pages/api/repopulate.ts b/apps/grades/src/pages/api/repopulate.ts index 4f3b602ea..1089e3c0f 100644 --- a/apps/grades/src/pages/api/repopulate.ts +++ b/apps/grades/src/pages/api/repopulate.ts @@ -7,7 +7,7 @@ const logger = getLogger("api/repopulate") export default async function route(req: NextApiRequest, res: NextApiResponse) { const skip = (req.query.skip as string | undefined)?.split(",") - logger.info(`Initializing repopulation job with skip ${skip?.join(", ")}) ?? ""}`) + logger.info(`Initializing repopulation job with skip ${skip?.join(", ") ?? ""}`) const kysely = await createKysely() const core = createServiceLayer({ fetch, db: kysely }) diff --git a/apps/grades/src/server/department-repository.ts b/apps/grades/src/server/department-repository.ts index 4faf90094..225ad41ee 100644 --- a/apps/grades/src/server/department-repository.ts +++ b/apps/grades/src/server/department-repository.ts @@ -24,6 +24,7 @@ export class DepartmentRepositoryImpl implements DepartmentRepository { const department = await this.db .insertInto("ntnuFacultyDepartment") .values(input) + .onConflict((eb) => eb.columns(["refId"]).doUpdateSet({ ...input })) .returningAll() .executeTakeFirstOrThrow() return Department.parse(department) diff --git a/apps/grades/src/server/faculty-repository.ts b/apps/grades/src/server/faculty-repository.ts index 88e9eb8d3..4e05c073d 100644 --- a/apps/grades/src/server/faculty-repository.ts +++ b/apps/grades/src/server/faculty-repository.ts @@ -20,7 +20,12 @@ export class FacultyRepositoryImpl implements FacultyRepository { constructor(private readonly db: Database) {} async createFaculty(input: Insertable): Promise { - const faculty = await this.db.insertInto("ntnuFaculty").values(input).returningAll().executeTakeFirstOrThrow() + const faculty = await this.db + .insertInto("ntnuFaculty") + .values(input) + .onConflict((eb) => eb.columns(["refId"]).doUpdateSet({ ...input })) + .returningAll() + .executeTakeFirstOrThrow() return Faculty.parse(faculty) } diff --git a/apps/grades/src/server/job-service.ts b/apps/grades/src/server/job-service.ts index 38fdfbcd2..8fa52ace6 100644 --- a/apps/grades/src/server/job-service.ts +++ b/apps/grades/src/server/job-service.ts @@ -1,9 +1,9 @@ import { getLogger } from "@dotkomonline/logger" -import Queue from "p-queue" import { type FacultyRepository } from "@/server/faculty-repository" import { type HkdirService } from "@/server/hkdir-service" import { type DepartmentRepository } from "@/server/department-repository" import { type SubjectRepository } from "@/server/subject-repository" +import { executeWithAsyncQueue } from "@/server/util" export interface JobService { performFacultySynchronizationJob(): Promise @@ -12,7 +12,6 @@ export interface JobService { export class JobServiceImpl implements JobService { private readonly logger = getLogger("JobService") - private readonly concurrencyLevel = parseInt(process.env.MAX_CONCURRENCY ?? "10") public constructor( private readonly facultyRepository: FacultyRepository, @@ -35,78 +34,81 @@ export class JobServiceImpl implements JobService { const faculties = await this.hkdirService.getDepartments("1150") this.logger.info(`Beginning synchronization of ${faculties.length} faculties`) - let count = 0 - for (const faculty of faculties) { - if (count !== 0 && count % 100 === 0) { - this.logger.info(`Synchronized ${count} faculties`) - } + await executeWithAsyncQueue({ + build: (queue, concurrency) => { + this.logger.info(`Processing faculties using async queue with ${concurrency} concurrency`) + for (const faculty of faculties) { + queue.add(async () => { + // Attempt to register the faculty (level two institution) + let existingFaculty = await this.facultyRepository.getFacultyByReferenceId(faculty.Fakultetskode) + if (existingFaculty === null) { + existingFaculty = await this.facultyRepository.createFaculty({ + name: faculty.Fakultetsnavn, + refId: faculty.Fakultetskode, + }) + } - // Attempt to register the faculty (level two institution) - let existingFaculty = await this.facultyRepository.getFacultyByReferenceId(faculty.Fakultetskode) - if (existingFaculty === null) { - existingFaculty = await this.facultyRepository.createFaculty({ - name: faculty.Fakultetsnavn, - refId: faculty.Fakultetskode, - }) - } - - // Attempt to register the department (level three institution) - let existingDepartment = await this.departmentRepository.getDepartmentByReferenceId(faculty.Avdelingskode) - if (existingDepartment === null) { - existingDepartment = await this.departmentRepository.createDepartment({ - name: faculty.Avdelingsnavn, - refId: faculty.Avdelingskode, - facultyId: existingFaculty.id, - }) - } - - count++ - } + // Attempt to register the department (level three institution) + let existingDepartment = await this.departmentRepository.getDepartmentByReferenceId(faculty.Avdelingskode) + if (existingDepartment === null) { + existingDepartment = await this.departmentRepository.createDepartment({ + name: faculty.Avdelingsnavn, + refId: faculty.Avdelingskode, + facultyId: existingFaculty.id, + }) + } + }) + } + }, + onTaskComplete: (remainingTasks) => { + if (remainingTasks && remainingTasks % 100 === 0) { + this.logger.info(`Queue processing for faculties reached ${remainingTasks} jobs left`) + } + }, + }) this.logger.info("Synchronization of faculties complete") } public async performSubjectSynchronizationJob() { this.logger.info("Synchronizing subjects from HKDir") const subjects = await this.hkdirService.getSubjects("1150") - this.logger.info( - `Beginning synchronization of ${subjects.length} subjects with concurrency level of ${this.concurrencyLevel}` - ) + this.logger.info(`Beginning synchronization of ${subjects.length} subjects`) - const queue = new Queue({ concurrency: this.concurrencyLevel }) + await executeWithAsyncQueue({ + build: (queue, concurrency) => { + this.logger.info(`Processing subjects using async queue with ${concurrency} concurrency`) + for (const subject of subjects) { + queue.add(async () => { + // Pull the department for the given subject from the database + const department = await this.departmentRepository.getDepartmentByReferenceId(subject.Avdelingskode) + if (department === null) { + throw new Error(`Department with reference ID ${subject.Avdelingskode} not found`) + } - for (const subject of subjects) { - queue.add(async () => { - // Pull the department for the given subject from the database - const department = await this.departmentRepository.getDepartmentByReferenceId(subject.Avdelingskode) - if (department === null) { - throw new Error(`Department with reference ID ${subject.Avdelingskode} not found`) - } - - // Attempt to register the subject - let existingSubject = await this.subjectRepository.getSubjectByReferenceId(subject.Emnekode) - if (existingSubject === null) { - // A lot of subjects have a -x suffix, which we want to remove for the slug - const slug = subject.Emnekode.replace(/-(1|2|a|b|k)$/, "").toLowerCase() - existingSubject = await this.subjectRepository.createSubject({ - name: subject.Emnenavn, - refId: subject.Emnekode, - educationalLevel: subject.Nivåkode, - instructionLanguage: subject["Underv.språk"], - credits: parseFloat(subject.Studiepoeng), - departmentId: department.id, - slug, + // Attempt to register the subject + let existingSubject = await this.subjectRepository.getSubjectByReferenceId(subject.Emnekode) + if (existingSubject === null) { + // A lot of subjects have a -x suffix, which we want to remove for the slug + const slug = subject.Emnekode.replace(/-[12ABK]$/, "").toLowerCase() + existingSubject = await this.subjectRepository.createSubject({ + name: subject.Emnenavn, + refId: subject.Emnekode, + educationalLevel: subject.Nivåkode, + instructionLanguage: subject["Underv.språk"], + credits: parseFloat(subject.Studiepoeng), + departmentId: department.id, + slug, + }) + } }) } - }) - } - - queue.on("next", () => { - if (queue.size % 100 !== 0 && queue.size % 100 === 0) { - this.logger.info(`Queue processing for subjects reached ${queue.size} jobs left`) - } + }, + onTaskComplete: (remainingTasks) => { + if (remainingTasks && remainingTasks % 100 === 0) { + this.logger.info(`Queue processing for subjects reached ${remainingTasks} jobs left`) + } + }, }) - - await queue.onIdle() this.logger.info("Synchronization of subjects complete") } } diff --git a/apps/grades/src/server/subject-repository.ts b/apps/grades/src/server/subject-repository.ts index 9310b09f4..e765ead88 100644 --- a/apps/grades/src/server/subject-repository.ts +++ b/apps/grades/src/server/subject-repository.ts @@ -25,7 +25,12 @@ export class SubjectRepositoryImpl implements SubjectRepository { constructor(private readonly db: Database) {} async createSubject(input: Insertable): Promise { - const subject = await this.db.insertInto("subject").values(input).returningAll().executeTakeFirstOrThrow() + const subject = await this.db + .insertInto("subject") + .values(input) + .onConflict((eb) => eb.columns(["refId"]).doUpdateSet({ ...input })) + .returningAll() + .executeTakeFirstOrThrow() return Subject.parse(subject) } diff --git a/apps/grades/src/server/util.ts b/apps/grades/src/server/util.ts new file mode 100644 index 000000000..bc0d39cbf --- /dev/null +++ b/apps/grades/src/server/util.ts @@ -0,0 +1,19 @@ +import Queue from "p-queue" +const defaultConcurrnecy = parseInt(process.env.MAX_CONCURRENCY ?? "10") + +export interface AsyncExecuteOptions { + build(queue: Queue, concurrency: number): void + onTaskComplete(remainingTasks: number): void + concurrency?: number +} + +export const executeWithAsyncQueue = async ({ + build, + onTaskComplete, + concurrency = defaultConcurrnecy, +}: AsyncExecuteOptions) => { + const queue = new Queue({ concurrency }) + queue.on("next", () => onTaskComplete(queue.size)) + build(queue, concurrency) + await queue.onIdle() +} From f7c9d75b11cf054ab08ebeac91580b278c20f44b Mon Sep 17 00:00:00 2001 From: Mats Larsen Date: Thu, 8 Feb 2024 01:55:30 +0100 Subject: [PATCH 18/23] Prevent duplicate write on subject grades --- .../0002_unique_constraint_grades.sql | 5 + .../0003_grade_submission_tracking.sql | 18 +++ apps/grades/src/db.generated.d.ts | 14 ++ apps/grades/src/pages/api/repopulate.ts | 4 + apps/grades/src/server/core.ts | 3 + apps/grades/src/server/grade-repository.ts | 129 ++++++++++++++++++ apps/grades/src/server/hkdir-service.ts | 5 +- apps/grades/src/server/hkdir-util.ts | 39 ++++++ apps/grades/src/server/job-service.ts | 78 +++++++++++ apps/grades/src/server/util.ts | 4 +- 10 files changed, 295 insertions(+), 4 deletions(-) create mode 100644 apps/grades/migrations/0002_unique_constraint_grades.sql create mode 100644 apps/grades/migrations/0003_grade_submission_tracking.sql create mode 100644 apps/grades/src/server/grade-repository.ts create mode 100644 apps/grades/src/server/hkdir-util.ts diff --git a/apps/grades/migrations/0002_unique_constraint_grades.sql b/apps/grades/migrations/0002_unique_constraint_grades.sql new file mode 100644 index 000000000..8146e6c75 --- /dev/null +++ b/apps/grades/migrations/0002_unique_constraint_grades.sql @@ -0,0 +1,5 @@ +BEGIN TRANSACTION; + +CREATE UNIQUE INDEX subject_season_grade_uq_subject_id_season_year ON subject_season_grade (subject_id, season, year); + +COMMIT; diff --git a/apps/grades/migrations/0003_grade_submission_tracking.sql b/apps/grades/migrations/0003_grade_submission_tracking.sql new file mode 100644 index 000000000..9a05c9e6d --- /dev/null +++ b/apps/grades/migrations/0003_grade_submission_tracking.sql @@ -0,0 +1,18 @@ +BEGIN TRANSACTION; + +CREATE TYPE subject_write_log_grade AS ENUM ('A', 'B', 'C', 'D', 'E', 'F', 'G', 'H'); + +CREATE TABLE IF NOT EXISTS subject_season_grade_write_log +( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + subject_id UUID NOT NULL, + season subject_season NOT NULL, + year INTEGER NOT NULL, + grade subject_write_log_grade NOT NULL, + + CONSTRAINT subject_season_grade_write_log_fk_subject_id FOREIGN KEY (subject_id) REFERENCES subject (id) ON DELETE CASCADE, + CONSTRAINT subject_season_grade_write_log_uq_subject_season_grade_year_gra UNIQUE (subject_id, season, year, grade) +); + +COMMIT; diff --git a/apps/grades/src/db.generated.d.ts b/apps/grades/src/db.generated.d.ts index 5c2695332..f5beb1321 100644 --- a/apps/grades/src/db.generated.d.ts +++ b/apps/grades/src/db.generated.d.ts @@ -6,6 +6,10 @@ export type Generated = T extends ColumnType export type SubjectSeason = "AUTUMN" | "SPRING" | "SUMMER" | "WINTER" +export type SubjectWriteLogGrade = "A" | "B" | "C" | "D" | "E" | "F" | "G" | "H" + +export type Timestamp = ColumnType + export interface NtnuFaculty { id: Generated name: string @@ -46,9 +50,19 @@ export interface SubjectSeasonGrade { year: number } +export interface SubjectSeasonGradeWriteLog { + createdAt: Generated + grade: SubjectWriteLogGrade + id: Generated + season: SubjectSeason + subjectId: string + year: number +} + export interface DB { ntnuFaculty: NtnuFaculty ntnuFacultyDepartment: NtnuFacultyDepartment subject: Subject subjectSeasonGrade: SubjectSeasonGrade + subjectSeasonGradeWriteLog: SubjectSeasonGradeWriteLog } diff --git a/apps/grades/src/pages/api/repopulate.ts b/apps/grades/src/pages/api/repopulate.ts index 1089e3c0f..e0bd8cf6b 100644 --- a/apps/grades/src/pages/api/repopulate.ts +++ b/apps/grades/src/pages/api/repopulate.ts @@ -20,5 +20,9 @@ export default async function route(req: NextApiRequest, res: NextApiResponse) { await core.jobService.performSubjectSynchronizationJob() } + if (!skip?.includes("grade")) { + await core.jobService.performGradeSynchronizationJob() + } + res.status(200).send({}) } diff --git a/apps/grades/src/server/core.ts b/apps/grades/src/server/core.ts index a3405332c..12f194f75 100644 --- a/apps/grades/src/server/core.ts +++ b/apps/grades/src/server/core.ts @@ -4,6 +4,7 @@ import { type HkdirService, HkdirServiceImpl } from "@/server/hkdir-service" import { type DepartmentRepository, DepartmentRepositoryImpl } from "@/server/department-repository" import { type JobService, JobServiceImpl } from "@/server/job-service" import { type SubjectRepository, SubjectRepositoryImpl } from "@/server/subject-repository" +import { type GradeRepository, GradeRepositoryImpl } from "@/server/grade-repository" export interface CreateServiceLayerOptions { fetch: WindowOrWorkerGlobalScope["fetch"] @@ -16,11 +17,13 @@ export const createServiceLayer = (opts: CreateServiceLayerOptions) => { const facultyRepository: FacultyRepository = new FacultyRepositoryImpl(opts.db) const departmentRepository: DepartmentRepository = new DepartmentRepositoryImpl(opts.db) const subjectRepository: SubjectRepository = new SubjectRepositoryImpl(opts.db) + const gradeRepository: GradeRepository = new GradeRepositoryImpl(opts.db) const hkdirService: HkdirService = new HkdirServiceImpl(opts.fetch) const jobService: JobService = new JobServiceImpl( facultyRepository, departmentRepository, subjectRepository, + gradeRepository, hkdirService ) diff --git a/apps/grades/src/server/grade-repository.ts b/apps/grades/src/server/grade-repository.ts new file mode 100644 index 000000000..598e6e067 --- /dev/null +++ b/apps/grades/src/server/grade-repository.ts @@ -0,0 +1,129 @@ +import { z } from "zod" +import { type Insertable, type Updateable } from "kysely" +import { type SubjectSeasonGrade } from "@/db.generated" +import { type Database } from "@/server/kysely" +import { HkdirGrade, type HkdirGradeKey } from "@/server/hkdir-service" + +export type Grade = z.infer +export type Season = Grade["season"] +export const Grade = z.object({ + id: z.string().uuid(), + subjectId: z.string().uuid(), + season: z.enum(["SPRING", "AUTUMN", "WINTER", "SUMMER"]), + year: z.number(), + + gradedA: z.number().int().nullable().default(null), + gradedB: z.number().int().nullable().default(null), + gradedC: z.number().int().nullable().default(null), + gradedD: z.number().int().nullable().default(null), + gradedE: z.number().int().nullable().default(null), + gradedF: z.number().int().nullable().default(null), + + gradedFail: z.number().int().nullable().default(null), + gradedPass: z.number().int().nullable().default(null), +}) + +export type GradeWriteLog = z.infer +export const GradeWriteLog = z.object({ + id: z.string().uuid(), + subjectId: z.string().uuid(), + season: Grade.shape.season, + year: z.number(), + grade: HkdirGrade.shape.Karakter, + createdAt: z.date().nullable(), +}) + +export interface GradeRepository { + createGrade(input: Insertable): Promise + updateGradeWithExplicitLock( + id: string, + input: Updateable, + hkdirKey: HkdirGradeKey + ): Promise + getGradeBySemester(subjectId: string, season: Season, year: number): Promise + getPreviousWriteLogEntry( + subjectId: string, + season: Season, + year: number, + grade: HkdirGradeKey + ): Promise +} + +export class GradeRepositoryImpl implements GradeRepository { + constructor(private readonly db: Database) {} + + async createGrade(input: Insertable): Promise { + const grade = await this.db + .insertInto("subjectSeasonGrade") + .values(input) + .onConflict((eb) => eb.columns(["subjectId", "season", "year"]).doUpdateSet({ ...input })) + .returningAll() + .executeTakeFirstOrThrow() + return Grade.parse(grade) + } + + async getGradeBySemester(subjectId: string, season: Season, year: number): Promise { + const grade = await this.db + .selectFrom("subjectSeasonGrade") + .selectAll() + .where("subjectId", "=", subjectId) + .where("season", "=", season) + .where("year", "=", year) + .executeTakeFirst() + return grade ? Grade.parse(grade) : null + } + + async updateGradeWithExplicitLock( + id: string, + input: Updateable, + hkdirKey: HkdirGradeKey + ): Promise { + const grade = await this.db + .transaction() + .setIsolationLevel("read committed") + .execute(async (tx) => { + // If this throws, then it means we have a read inconsistency from the caller. + const grade = await tx + .selectFrom("subjectSeasonGrade") + .selectAll() + .where("id", "=", id) + .forUpdate() + .executeTakeFirstOrThrow() + await tx + .insertInto("subjectSeasonGradeWriteLog") + .values({ + subjectId: grade.subjectId, + season: grade.season, + year: grade.year, + grade: hkdirKey, + }) + .execute() + return await tx + .updateTable("subjectSeasonGrade") + .set(input) + .where("id", "=", id) + .returningAll() + .executeTakeFirstOrThrow() + }) + return Grade.parse(grade) + } + + async getPreviousWriteLogEntry( + subjectId: string, + season: Season, + year: number, + grade: HkdirGradeKey + ): Promise { + const writeLog = await this.db + .selectFrom("subjectSeasonGradeWriteLog") + .selectAll() + .where("subjectId", "=", subjectId) + .where("season", "=", season) + .where("year", "=", year) + .where("grade", "=", grade) + .orderBy("createdAt", "desc") + .limit(1) + .executeTakeFirst() + return writeLog ? GradeWriteLog.parse(writeLog) : null + } +} diff --git a/apps/grades/src/server/hkdir-service.ts b/apps/grades/src/server/hkdir-service.ts index eebd4df41..3a5c62a4c 100644 --- a/apps/grades/src/server/hkdir-service.ts +++ b/apps/grades/src/server/hkdir-service.ts @@ -45,11 +45,12 @@ export const HkdirSubject = z.object({ }) export type HkdirGrade = z.infer +export type HkdirGradeKey = HkdirGrade["Karakter"] export const HkdirGrade = z.object({ "Årstall": z.string(), "Semester": z.string(), "Semesternavn": z.string(), - "Karakter": z.string(), + "Karakter": z.enum(["A", "B", "C", "D", "E", "F", "G", "H"]), "Emnekode": z.string(), "Antall kandidater totalt": z.string(), }) @@ -102,7 +103,7 @@ export class HkdirServiceImpl implements HkdirService { groupBy: ["Årstall", "Semester", "Karakter", "Emnekode", "Institusjonskode"], filter: [ this.createQueryFilter("Institusjonskode", [institution.toString()]), - this.createTopQueryFilter("Årstall", 3), + this.createTopQueryFilter("Årstall", 1), this.createAllQueryFilter("Emnekode"), this.createAllQueryFilter("Semester"), ], diff --git a/apps/grades/src/server/hkdir-util.ts b/apps/grades/src/server/hkdir-util.ts new file mode 100644 index 000000000..8415897bb --- /dev/null +++ b/apps/grades/src/server/hkdir-util.ts @@ -0,0 +1,39 @@ +import { type Grade, type Season } from "@/server/grade-repository" + +export const mapHkdirSemesterToSeason = (semester: string): Season => { + switch (semester) { + case "1": + return "SPRING" + case "3": + return "AUTUMN" + case "2": + return "SUMMER" + case "0": + return "WINTER" + default: + throw new Error(`Unknown semester: ${semester}`) + } +} + +export const mapHkdirGradeToGrade = (grade: string): keyof Grade => { + switch (grade) { + case "A": + return "gradedA" + case "B": + return "gradedB" + case "C": + return "gradedC" + case "D": + return "gradedD" + case "E": + return "gradedE" + case "F": + return "gradedF" + case "G": + return "gradedPass" + case "H": + return "gradedFail" + default: + throw new Error(`Unknown grade: ${grade}`) + } +} diff --git a/apps/grades/src/server/job-service.ts b/apps/grades/src/server/job-service.ts index 8fa52ace6..a1155d2f4 100644 --- a/apps/grades/src/server/job-service.ts +++ b/apps/grades/src/server/job-service.ts @@ -4,12 +4,16 @@ import { type HkdirService } from "@/server/hkdir-service" import { type DepartmentRepository } from "@/server/department-repository" import { type SubjectRepository } from "@/server/subject-repository" import { executeWithAsyncQueue } from "@/server/util" +import { type GradeRepository } from "@/server/grade-repository" +import { mapHkdirGradeToGrade, mapHkdirSemesterToSeason } from "@/server/hkdir-util" export interface JobService { performFacultySynchronizationJob(): Promise performSubjectSynchronizationJob(): Promise + performGradeSynchronizationJob(): Promise } +// TODO: Evaluate whether all of these should run in PostGreSQL transactions export class JobServiceImpl implements JobService { private readonly logger = getLogger("JobService") @@ -17,6 +21,7 @@ export class JobServiceImpl implements JobService { private readonly facultyRepository: FacultyRepository, private readonly departmentRepository: DepartmentRepository, private readonly subjectRepository: SubjectRepository, + private readonly gradeRepository: GradeRepository, private readonly hkdirService: HkdirService ) {} @@ -111,4 +116,77 @@ export class JobServiceImpl implements JobService { }) this.logger.info("Synchronization of subjects complete") } + + /** + * Synchronize grades from HKDir to the database. + * + * This job is a bit more complex than the others, because HKDir doesn't provide a result-set for the grades. Instead + * they provide a list of subjects, and for each subject a list of grades. This means we have to iterate over all + * subjects, and for each subject iterate over all grades. + */ + public async performGradeSynchronizationJob() { + this.logger.info("Synchronizing grades from HKDir") + const grades = await this.hkdirService.getSubjectGrades("1150") + this.logger.info(`Beginning synchronization of ${grades.length} grades`) + + await executeWithAsyncQueue({ + build: (queue, concurrency) => { + this.logger.info(`Processing grades using async queue with ${concurrency} concurrency`) + for (const grade of grades) { + queue.add(async () => { + // We need a reference to the subject from the reference id given by HKDir + const subject = await this.subjectRepository.getSubjectByReferenceId(grade.Emnekode) + if (subject === null) { + // TODO: Evaluate whether we should create the subject if it doesn't exist + return + } + + // First we need to get or insert the matching grade. + let existingGrade = await this.gradeRepository.getGradeBySemester( + subject.id, + mapHkdirSemesterToSeason(grade.Semester), + parseInt(grade.Årstall) + ) + if (existingGrade === null) { + existingGrade = await this.gradeRepository.createGrade({ + subjectId: subject.id, + season: mapHkdirSemesterToSeason(grade.Semester), + year: parseInt(grade.Årstall), + grade: 0, + }) + } + + // If there exists a previous write log entry for the subject, season, year and grade, we need to skip this + // grade, as it has already been processed. + const previousWriteLogEntry = await this.gradeRepository.getPreviousWriteLogEntry( + existingGrade.subjectId, + mapHkdirSemesterToSeason(grade.Semester), + parseInt(grade.Årstall), + grade.Karakter + ) + if (previousWriteLogEntry !== null) { + return + } + + // Then we need to update the grade distribution for the current grade + const key = mapHkdirGradeToGrade(grade.Karakter) + const studentsWithGrade = parseInt(grade["Antall kandidater totalt"]) + await this.gradeRepository.updateGradeWithExplicitLock( + existingGrade.id, + { + [key]: studentsWithGrade, + }, + grade.Karakter + ) + }) + } + }, + onTaskComplete: (remainingTasks) => { + if (remainingTasks && remainingTasks % 100 === 0) { + this.logger.info(`Queue processing for grades reached ${remainingTasks} jobs left`) + } + }, + }) + this.logger.info("Synchronization of grades complete") + } } diff --git a/apps/grades/src/server/util.ts b/apps/grades/src/server/util.ts index bc0d39cbf..b015172ed 100644 --- a/apps/grades/src/server/util.ts +++ b/apps/grades/src/server/util.ts @@ -1,5 +1,5 @@ import Queue from "p-queue" -const defaultConcurrnecy = parseInt(process.env.MAX_CONCURRENCY ?? "10") +const defaultConcurrency = parseInt(process.env.MAX_CONCURRENCY ?? "10") export interface AsyncExecuteOptions { build(queue: Queue, concurrency: number): void @@ -10,7 +10,7 @@ export interface AsyncExecuteOptions { export const executeWithAsyncQueue = async ({ build, onTaskComplete, - concurrency = defaultConcurrnecy, + concurrency = defaultConcurrency, }: AsyncExecuteOptions) => { const queue = new Queue({ concurrency }) queue.on("next", () => onTaskComplete(queue.size)) From 185621abde3c5a9110a69198ac4d5c66a35e35ab Mon Sep 17 00:00:00 2001 From: Mats Larsen Date: Thu, 8 Feb 2024 17:43:47 +0100 Subject: [PATCH 19/23] Query most popular or filtered subjects --- apps/grades/src/pages/api/subjects.ts | 19 +++++++ apps/grades/src/server/core.ts | 3 ++ apps/grades/src/server/kysely.ts | 6 +++ apps/grades/src/server/subject-repository.ts | 54 +++++++++++++++++++- apps/grades/src/server/subject-service.ts | 16 ++++++ 5 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 apps/grades/src/pages/api/subjects.ts create mode 100644 apps/grades/src/server/subject-service.ts diff --git a/apps/grades/src/pages/api/subjects.ts b/apps/grades/src/pages/api/subjects.ts new file mode 100644 index 000000000..59b32fd3e --- /dev/null +++ b/apps/grades/src/pages/api/subjects.ts @@ -0,0 +1,19 @@ +import { type NextApiRequest, type NextApiResponse } from "next" +import { z } from "zod" +import { createKysely } from "@/server/kysely" +import { createServiceLayer } from "@/server/core" + +const QuerySchema = z.object({ + q: z.string().nullish().default(null), + take: z.number().int().positive().default(30), + skip: z.number().int().nonnegative().default(0), +}) + +export default async function route(req: NextApiRequest, res: NextApiResponse) { + const query = QuerySchema.parse(req.query) + const kysely = await createKysely() + const core = createServiceLayer({ fetch, db: kysely }) + + const subjects = await core.subjectService.search(query.q, query.take, query.skip) + res.status(200).json(subjects) +} diff --git a/apps/grades/src/server/core.ts b/apps/grades/src/server/core.ts index 12f194f75..832bb7fe4 100644 --- a/apps/grades/src/server/core.ts +++ b/apps/grades/src/server/core.ts @@ -5,6 +5,7 @@ import { type DepartmentRepository, DepartmentRepositoryImpl } from "@/server/de import { type JobService, JobServiceImpl } from "@/server/job-service" import { type SubjectRepository, SubjectRepositoryImpl } from "@/server/subject-repository" import { type GradeRepository, GradeRepositoryImpl } from "@/server/grade-repository" +import { type SubjectService, SubjectServiceImpl } from "@/server/subject-service" export interface CreateServiceLayerOptions { fetch: WindowOrWorkerGlobalScope["fetch"] @@ -26,9 +27,11 @@ export const createServiceLayer = (opts: CreateServiceLayerOptions) => { gradeRepository, hkdirService ) + const subjectService: SubjectService = new SubjectServiceImpl(subjectRepository) return { hkdirService, jobService, + subjectService, } } diff --git a/apps/grades/src/server/kysely.ts b/apps/grades/src/server/kysely.ts index ec58d326f..3b4600761 100644 --- a/apps/grades/src/server/kysely.ts +++ b/apps/grades/src/server/kysely.ts @@ -1,10 +1,13 @@ import process from "node:process" import pg from "pg" import { CamelCasePlugin, Kysely, PostgresDialect } from "kysely" +import { getLogger } from "@dotkomonline/logger" import { type DB } from "@/db.generated" export type Database = Awaited> +const logger = getLogger("server/kysely") + export const createKysely = () => { const conn = new pg.Pool({ connectionString: process.env.DATABASE_URL ?? "postgres://postgres:postgres@localhost:5433/postgres", @@ -14,5 +17,8 @@ export const createKysely = () => { pool: conn, }), plugins: [new CamelCasePlugin()], + log: (evt) => { + logger.info(evt.query.sql) + }, }) } diff --git a/apps/grades/src/server/subject-repository.ts b/apps/grades/src/server/subject-repository.ts index e765ead88..efa61306d 100644 --- a/apps/grades/src/server/subject-repository.ts +++ b/apps/grades/src/server/subject-repository.ts @@ -1,5 +1,5 @@ import { z } from "zod" -import { type Insertable } from "kysely" +import { type Insertable, sql } from "kysely" import { type Subject as DatabaseSubject } from "@/db.generated" import { type Database } from "@/server/kysely" @@ -19,6 +19,8 @@ export interface SubjectRepository { createSubject(input: Insertable): Promise getSubjectByReferenceId(refId: string): Promise getSubjectById(id: string): Promise + getSubjectsBySearchExpression(expression: string, take: number, skip: number): Promise + getSubjectsByPopularity(take: number, skip: number): Promise } export class SubjectRepositoryImpl implements SubjectRepository { @@ -43,4 +45,54 @@ export class SubjectRepositoryImpl implements SubjectRepository { const subject = await this.db.selectFrom("subject").selectAll().where("id", "=", id).executeTakeFirst() return subject ? Subject.parse(subject) : null } + + async getSubjectsBySearchExpression(expression: string, take: number, skip: number): Promise { + // We want to match the expression case-insensitively, and we will also attempt to match the slug for the subject. + // We should also find close matches using levenstein distance + const subjects = await this.db + .selectFrom("subject") + .selectAll() + .where((eb) => + eb.or([ + eb("name", "ilike", `%${expression}%`), + eb("slug", "ilike", `%${expression}%`), + eb(sql`levenshtein(subject.name, ${expression}::text)`, "<=", 1), + eb(sql`levenshtein(subject.slug, ${expression}::text)`, "<=", 1), + ]) + ) + .limit(take) + .offset(skip) + .execute() + return subjects.map((x) => Subject.parse(x)) + } + + async getSubjectsByPopularity(take: number, skip: number): Promise { + // We determine popularity by the number of grades given to the subject + const subjects = await this.db + .selectFrom("subject") + .leftJoin("subjectSeasonGrade", "subjectSeasonGrade.subjectId", "subject.id") + .selectAll("subject") + .select(({ eb }) => [ + eb + .parens( + sql` + ${eb.fn.sum(eb.fn.coalesce("subjectSeasonGrade.gradedA", sql.lit(0)))} + + ${eb.fn.sum(eb.fn.coalesce("subjectSeasonGrade.gradedB", sql.lit(0)))} + + ${eb.fn.sum(eb.fn.coalesce("subjectSeasonGrade.gradedC", sql.lit(0)))} + + ${eb.fn.sum(eb.fn.coalesce("subjectSeasonGrade.gradedD", sql.lit(0)))} + + ${eb.fn.sum(eb.fn.coalesce("subjectSeasonGrade.gradedE", sql.lit(0)))} + + ${eb.fn.sum(eb.fn.coalesce("subjectSeasonGrade.gradedF", sql.lit(0)))} + + ${eb.fn.sum(eb.fn.coalesce("subjectSeasonGrade.gradedPass", sql.lit(0)))} + + ${eb.fn.sum(eb.fn.coalesce("subjectSeasonGrade.gradedFail", sql.lit(0)))} + ` + ) + .as("students"), + ]) + .groupBy("subject.id") + .orderBy("students", "desc") + .limit(take) + .offset(skip) + .execute() + return subjects.map((x) => Subject.parse(x)) + } } diff --git a/apps/grades/src/server/subject-service.ts b/apps/grades/src/server/subject-service.ts new file mode 100644 index 000000000..77be62b80 --- /dev/null +++ b/apps/grades/src/server/subject-service.ts @@ -0,0 +1,16 @@ +import { type Subject, type SubjectRepository } from "@/server/subject-repository" + +export interface SubjectService { + search(expression: string | null, take: number, skip: number): Promise +} + +export class SubjectServiceImpl implements SubjectService { + constructor(private readonly subjectRepository: SubjectRepository) {} + + async search(expression: string | null, take: number, skip: number): Promise { + if (expression !== null) { + return this.subjectRepository.getSubjectsBySearchExpression(expression.trim(), take, skip) + } + return this.subjectRepository.getSubjectsByPopularity(take, skip) + } +} From ad52270b4c8b8d410dd57d374ac07129794afbfe Mon Sep 17 00:00:00 2001 From: Mats Larsen Date: Thu, 8 Feb 2024 19:02:12 +0100 Subject: [PATCH 20/23] Build average grade for subject while populating --- .../grades/migrations/0004_average_grades.sql | 6 +++ apps/grades/src/db.generated.d.ts | 2 + apps/grades/src/server/grade-repository.ts | 40 ++++++++++++++----- apps/grades/src/server/hkdir-util.ts | 20 ++++++++++ apps/grades/src/server/job-service.ts | 5 ++- apps/grades/src/server/kysely.ts | 6 --- apps/grades/src/server/subject-repository.ts | 37 ++++++++--------- 7 files changed, 77 insertions(+), 39 deletions(-) create mode 100644 apps/grades/migrations/0004_average_grades.sql diff --git a/apps/grades/migrations/0004_average_grades.sql b/apps/grades/migrations/0004_average_grades.sql new file mode 100644 index 000000000..1e6887a11 --- /dev/null +++ b/apps/grades/migrations/0004_average_grades.sql @@ -0,0 +1,6 @@ +BEGIN TRANSACTION; + +ALTER TABLE subject ADD COLUMN average_grade FLOAT NOT NULL DEFAULT 0.0; +ALTER TABLE subject ADD COLUMN total_registered INT NOT NULL DEFAULT 0; + +COMMIT; diff --git a/apps/grades/src/db.generated.d.ts b/apps/grades/src/db.generated.d.ts index f5beb1321..2bd8cf24c 100644 --- a/apps/grades/src/db.generated.d.ts +++ b/apps/grades/src/db.generated.d.ts @@ -24,6 +24,7 @@ export interface NtnuFacultyDepartment { } export interface Subject { + averageGrade: Generated credits: number departmentId: string educationalLevel: string @@ -32,6 +33,7 @@ export interface Subject { name: string refId: string slug: string + totalRegistered: Generated } export interface SubjectSeasonGrade { diff --git a/apps/grades/src/server/grade-repository.ts b/apps/grades/src/server/grade-repository.ts index 598e6e067..c940e45c0 100644 --- a/apps/grades/src/server/grade-repository.ts +++ b/apps/grades/src/server/grade-repository.ts @@ -3,6 +3,7 @@ import { type Insertable, type Updateable } from "kysely" import { type SubjectSeasonGrade } from "@/db.generated" import { type Database } from "@/server/kysely" import { HkdirGrade, type HkdirGradeKey } from "@/server/hkdir-service" +import { mapHkdirGradeToGradeFactor } from "@/server/hkdir-util" export type Grade = z.infer export type Season = Grade["season"] @@ -35,11 +36,7 @@ export const GradeWriteLog = z.object({ export interface GradeRepository { createGrade(input: Insertable): Promise - updateGradeWithExplicitLock( - id: string, - input: Updateable, - hkdirKey: HkdirGradeKey - ): Promise + updateGrade(id: string, input: Updateable, hkdirKey: HkdirGradeKey, count: number): Promise getGradeBySemester(subjectId: string, season: Season, year: number): Promise getPreviousWriteLogEntry( subjectId: string, @@ -73,19 +70,20 @@ export class GradeRepositoryImpl implements GradeRepository { return grade ? Grade.parse(grade) : null } - async updateGradeWithExplicitLock( + async updateGrade( id: string, input: Updateable, - hkdirKey: HkdirGradeKey - ): Promise { - const grade = await this.db + hkdirKey: HkdirGradeKey, + count: number + ): Promise { + await this.db .transaction() .setIsolationLevel("read committed") .execute(async (tx) => { // If this throws, then it means we have a read inconsistency from the caller. const grade = await tx .selectFrom("subjectSeasonGrade") - .selectAll() + .select(["subjectId", "season", "year"]) .where("id", "=", id) .forUpdate() .executeTakeFirstOrThrow() @@ -98,6 +96,27 @@ export class GradeRepositoryImpl implements GradeRepository { grade: hkdirKey, }) .execute() + // We need to update the grade distribution for the current grade as well. + if (count !== 0 && hkdirKey !== "H" && hkdirKey !== "G") { + const { totalRegistered, averageGrade } = await tx + .selectFrom("subject") + .select(["totalRegistered", "averageGrade"]) + .where("id", "=", grade.subjectId) + .executeTakeFirstOrThrow() + const newTotalRegistered = totalRegistered + count + const multiplicationFactor = mapHkdirGradeToGradeFactor(hkdirKey) + const newAverageGrade = (averageGrade * totalRegistered + count * multiplicationFactor) / newTotalRegistered + // Set the new average grade and total registered count + await tx + .updateTable("subject") + .set({ + totalRegistered: newTotalRegistered, + averageGrade: newAverageGrade, + }) + .where("id", "=", grade.subjectId) + .execute() + } + return await tx .updateTable("subjectSeasonGrade") .set(input) @@ -105,7 +124,6 @@ export class GradeRepositoryImpl implements GradeRepository { .returningAll() .executeTakeFirstOrThrow() }) - return Grade.parse(grade) } async getPreviousWriteLogEntry( diff --git a/apps/grades/src/server/hkdir-util.ts b/apps/grades/src/server/hkdir-util.ts index 8415897bb..209de9557 100644 --- a/apps/grades/src/server/hkdir-util.ts +++ b/apps/grades/src/server/hkdir-util.ts @@ -1,4 +1,5 @@ import { type Grade, type Season } from "@/server/grade-repository" +import { type HkdirGradeKey } from "@/server/hkdir-service" export const mapHkdirSemesterToSeason = (semester: string): Season => { switch (semester) { @@ -15,6 +16,25 @@ export const mapHkdirSemesterToSeason = (semester: string): Season => { } } +export const mapHkdirGradeToGradeFactor = (grade: Exclude): number => { + switch (grade) { + case "A": + return 5 + case "B": + return 4 + case "C": + return 3 + case "D": + return 2 + case "E": + return 1 + case "F": + return 0 + default: + throw new Error(`Unknown grade: ${grade}`) + } +} + export const mapHkdirGradeToGrade = (grade: string): keyof Grade => { switch (grade) { case "A": diff --git a/apps/grades/src/server/job-service.ts b/apps/grades/src/server/job-service.ts index a1155d2f4..f85320f2c 100644 --- a/apps/grades/src/server/job-service.ts +++ b/apps/grades/src/server/job-service.ts @@ -171,12 +171,13 @@ export class JobServiceImpl implements JobService { // Then we need to update the grade distribution for the current grade const key = mapHkdirGradeToGrade(grade.Karakter) const studentsWithGrade = parseInt(grade["Antall kandidater totalt"]) - await this.gradeRepository.updateGradeWithExplicitLock( + await this.gradeRepository.updateGrade( existingGrade.id, { [key]: studentsWithGrade, }, - grade.Karakter + grade.Karakter, + studentsWithGrade ) }) } diff --git a/apps/grades/src/server/kysely.ts b/apps/grades/src/server/kysely.ts index 3b4600761..ec58d326f 100644 --- a/apps/grades/src/server/kysely.ts +++ b/apps/grades/src/server/kysely.ts @@ -1,13 +1,10 @@ import process from "node:process" import pg from "pg" import { CamelCasePlugin, Kysely, PostgresDialect } from "kysely" -import { getLogger } from "@dotkomonline/logger" import { type DB } from "@/db.generated" export type Database = Awaited> -const logger = getLogger("server/kysely") - export const createKysely = () => { const conn = new pg.Pool({ connectionString: process.env.DATABASE_URL ?? "postgres://postgres:postgres@localhost:5433/postgres", @@ -17,8 +14,5 @@ export const createKysely = () => { pool: conn, }), plugins: [new CamelCasePlugin()], - log: (evt) => { - logger.info(evt.query.sql) - }, }) } diff --git a/apps/grades/src/server/subject-repository.ts b/apps/grades/src/server/subject-repository.ts index efa61306d..279d03803 100644 --- a/apps/grades/src/server/subject-repository.ts +++ b/apps/grades/src/server/subject-repository.ts @@ -13,6 +13,8 @@ export const Subject = z.object({ instructionLanguage: z.string(), educationalLevel: z.string(), credits: z.number(), + averageGrade: z.number().nonnegative(), + totalRegistered: z.number().int().nonnegative(), }) export interface SubjectRepository { @@ -21,6 +23,7 @@ export interface SubjectRepository { getSubjectById(id: string): Promise getSubjectsBySearchExpression(expression: string, take: number, skip: number): Promise getSubjectsByPopularity(take: number, skip: number): Promise + getSubjectsByAverageGrade(take: number, skip: number): Promise } export class SubjectRepositoryImpl implements SubjectRepository { @@ -70,26 +73,20 @@ export class SubjectRepositoryImpl implements SubjectRepository { // We determine popularity by the number of grades given to the subject const subjects = await this.db .selectFrom("subject") - .leftJoin("subjectSeasonGrade", "subjectSeasonGrade.subjectId", "subject.id") - .selectAll("subject") - .select(({ eb }) => [ - eb - .parens( - sql` - ${eb.fn.sum(eb.fn.coalesce("subjectSeasonGrade.gradedA", sql.lit(0)))} + - ${eb.fn.sum(eb.fn.coalesce("subjectSeasonGrade.gradedB", sql.lit(0)))} + - ${eb.fn.sum(eb.fn.coalesce("subjectSeasonGrade.gradedC", sql.lit(0)))} + - ${eb.fn.sum(eb.fn.coalesce("subjectSeasonGrade.gradedD", sql.lit(0)))} + - ${eb.fn.sum(eb.fn.coalesce("subjectSeasonGrade.gradedE", sql.lit(0)))} + - ${eb.fn.sum(eb.fn.coalesce("subjectSeasonGrade.gradedF", sql.lit(0)))} + - ${eb.fn.sum(eb.fn.coalesce("subjectSeasonGrade.gradedPass", sql.lit(0)))} + - ${eb.fn.sum(eb.fn.coalesce("subjectSeasonGrade.gradedFail", sql.lit(0)))} - ` - ) - .as("students"), - ]) - .groupBy("subject.id") - .orderBy("students", "desc") + .selectAll() + .orderBy("totalRegistered", "desc") + .limit(take) + .offset(skip) + .execute() + return subjects.map((x) => Subject.parse(x)) + } + + async getSubjectsByAverageGrade(take: number, skip: number): Promise { + // We determine popularity by the number of grades given to the subject + const subjects = await this.db + .selectFrom("subject") + .selectAll() + .orderBy("averageGrade", "desc") .limit(take) .offset(skip) .execute() From ea0f1ffb8684d46c1c7a09da28d485f1d8fb0f38 Mon Sep 17 00:00:00 2001 From: Mats Larsen Date: Thu, 8 Feb 2024 19:07:03 +0100 Subject: [PATCH 21/23] Short circuit grades without any registrations --- apps/grades/src/server/job-service.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/apps/grades/src/server/job-service.ts b/apps/grades/src/server/job-service.ts index f85320f2c..88ce22c9d 100644 --- a/apps/grades/src/server/job-service.ts +++ b/apps/grades/src/server/job-service.ts @@ -133,6 +133,11 @@ export class JobServiceImpl implements JobService { build: (queue, concurrency) => { this.logger.info(`Processing grades using async queue with ${concurrency} concurrency`) for (const grade of grades) { + const studentsWithGrade = parseInt(grade["Antall kandidater totalt"]) + if (studentsWithGrade === 0) { + continue + } + queue.add(async () => { // We need a reference to the subject from the reference id given by HKDir const subject = await this.subjectRepository.getSubjectByReferenceId(grade.Emnekode) @@ -141,17 +146,19 @@ export class JobServiceImpl implements JobService { return } + const year = parseInt(grade.Årstall) + // First we need to get or insert the matching grade. let existingGrade = await this.gradeRepository.getGradeBySemester( subject.id, mapHkdirSemesterToSeason(grade.Semester), - parseInt(grade.Årstall) + year ) if (existingGrade === null) { existingGrade = await this.gradeRepository.createGrade({ subjectId: subject.id, season: mapHkdirSemesterToSeason(grade.Semester), - year: parseInt(grade.Årstall), + year, grade: 0, }) } @@ -161,7 +168,7 @@ export class JobServiceImpl implements JobService { const previousWriteLogEntry = await this.gradeRepository.getPreviousWriteLogEntry( existingGrade.subjectId, mapHkdirSemesterToSeason(grade.Semester), - parseInt(grade.Årstall), + year, grade.Karakter ) if (previousWriteLogEntry !== null) { @@ -170,7 +177,6 @@ export class JobServiceImpl implements JobService { // Then we need to update the grade distribution for the current grade const key = mapHkdirGradeToGrade(grade.Karakter) - const studentsWithGrade = parseInt(grade["Antall kandidater totalt"]) await this.gradeRepository.updateGrade( existingGrade.id, { From 63b190342fea20251791d18c66ec5cb7004093ff Mon Sep 17 00:00:00 2001 From: Mats Larsen Date: Fri, 9 Feb 2024 17:31:23 +0100 Subject: [PATCH 22/23] Fix parameters --- apps/grades/src/server/hkdir-service.ts | 6 +++--- apps/grades/src/server/job-service.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/grades/src/server/hkdir-service.ts b/apps/grades/src/server/hkdir-service.ts index 3a5c62a4c..8f6db4dbd 100644 --- a/apps/grades/src/server/hkdir-service.ts +++ b/apps/grades/src/server/hkdir-service.ts @@ -23,7 +23,7 @@ export const HkdirSubject = z.object({ "Institusjonsnavn": z.string(), "Avdelingskode": z.string(), "Avdelingsnavn": z.string(), - "Avdelingskode_SSB": z.string(), + "Avdelingskode_SSB": z.string().nullish().default(null), "Årstall": z.string(), "Semester": z.string(), "Semesternavn": z.string(), @@ -91,7 +91,7 @@ export class HkdirServiceImpl implements HkdirService { this.createQueryFilter("Institusjonskode", [institution.toString()]), this.createQueryFilter("Nivåkode", ["HN", "LN"]), this.createQueryFilter("Status", ["1", "2"]), - this.createTopQueryFilter("Årstall", 1), + // this.createTopQueryFilter("Årstall", 1), this.createExcludeQueryFilter("Avdelingskode", ["000000"]), ], }) @@ -103,7 +103,7 @@ export class HkdirServiceImpl implements HkdirService { groupBy: ["Årstall", "Semester", "Karakter", "Emnekode", "Institusjonskode"], filter: [ this.createQueryFilter("Institusjonskode", [institution.toString()]), - this.createTopQueryFilter("Årstall", 1), + // this.createTopQueryFilter("Årstall", 1), this.createAllQueryFilter("Emnekode"), this.createAllQueryFilter("Semester"), ], diff --git a/apps/grades/src/server/job-service.ts b/apps/grades/src/server/job-service.ts index 88ce22c9d..2944cdc6b 100644 --- a/apps/grades/src/server/job-service.ts +++ b/apps/grades/src/server/job-service.ts @@ -94,7 +94,7 @@ export class JobServiceImpl implements JobService { let existingSubject = await this.subjectRepository.getSubjectByReferenceId(subject.Emnekode) if (existingSubject === null) { // A lot of subjects have a -x suffix, which we want to remove for the slug - const slug = subject.Emnekode.replace(/-[12ABK]$/, "").toLowerCase() + const slug = subject.Emnekode.replace(/-[123ABK]$/, "").toLowerCase() existingSubject = await this.subjectRepository.createSubject({ name: subject.Emnenavn, refId: subject.Emnekode, From 332cbdb7e49a95522e1e385fdfcd14af36777390 Mon Sep 17 00:00:00 2001 From: Mats Larsen Date: Sat, 10 Feb 2024 16:16:42 +0100 Subject: [PATCH 23/23] Ship Rust synchronizer --- apps/grades/migrations/0001_schema.sql | 64 ------ .../0002_unique_constraint_grades.sql | 5 - .../0003_grade_submission_tracking.sql | 18 -- .../grades/migrations/0004_average_grades.sql | 6 - apps/grades/package.json | 1 - apps/grades/src/db.generated.d.ts | 40 ++-- apps/grades/src/pages/api/repopulate.ts | 28 --- apps/grades/src/pages/api/subjects.ts | 2 +- apps/grades/src/server/core.ts | 18 +- .../src/server/department-repository.ts | 29 +-- apps/grades/src/server/faculty-repository.ts | 21 +- apps/grades/src/server/grade-repository.ts | 107 ---------- apps/grades/src/server/hkdir-service.ts | 186 ---------------- apps/grades/src/server/hkdir-util.ts | 59 ------ apps/grades/src/server/job-service.ts | 199 ------------------ apps/grades/src/server/kysely.ts | 2 +- apps/grades/src/server/subject-repository.ts | 19 +- apps/grades/src/server/util.ts | 19 -- pnpm-lock.yaml | 20 -- 19 files changed, 31 insertions(+), 812 deletions(-) delete mode 100644 apps/grades/migrations/0001_schema.sql delete mode 100644 apps/grades/migrations/0002_unique_constraint_grades.sql delete mode 100644 apps/grades/migrations/0003_grade_submission_tracking.sql delete mode 100644 apps/grades/migrations/0004_average_grades.sql delete mode 100644 apps/grades/src/pages/api/repopulate.ts delete mode 100644 apps/grades/src/server/hkdir-service.ts delete mode 100644 apps/grades/src/server/hkdir-util.ts delete mode 100644 apps/grades/src/server/job-service.ts delete mode 100644 apps/grades/src/server/util.ts diff --git a/apps/grades/migrations/0001_schema.sql b/apps/grades/migrations/0001_schema.sql deleted file mode 100644 index cf88d16c1..000000000 --- a/apps/grades/migrations/0001_schema.sql +++ /dev/null @@ -1,64 +0,0 @@ -START TRANSACTION; - -CREATE EXTENSION IF NOT EXISTS fuzzystrmatch; - -CREATE TABLE IF NOT EXISTS ntnu_faculty -( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - ref_id TEXT NOT NULL, - name TEXT NOT NULL, - - CONSTRAINT ntnu_faculty_uq_ref_id UNIQUE (ref_id) -); - -CREATE TABLE IF NOT EXISTS ntnu_faculty_department -( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - ref_id TEXT NOT NULL, - faculty_id UUID NOT NULL, - name TEXT NOT NULL, - - CONSTRAINT ntnu_faculty_department_fk_faculty_id FOREIGN KEY (faculty_id) REFERENCES ntnu_faculty (id), - CONSTRAINT ntnu_faculty_department_uq_ref_id UNIQUE (ref_id) -); - -CREATE TABLE IF NOT EXISTS subject -( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - ref_id TEXT NOT NULL, - name TEXT NOT NULL, - slug TEXT NOT NULL, - department_id UUID NOT NULL, - - instruction_language TEXT NOT NULL, - educational_level TEXT NOT NULL, - credits FLOAT NOT NULL, - - CONSTRAINT subject_fk_department_id FOREIGN KEY (department_id) REFERENCES ntnu_faculty_department (id), - CONSTRAINT subject_uq_ref_id UNIQUE (ref_id) -); - -CREATE TYPE subject_season AS ENUM ('SPRING', 'AUTUMN', 'WINTER', 'SUMMER'); - -CREATE TABLE IF NOT EXISTS subject_season_grade -( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - subject_id UUID NOT NULL, - season subject_season NOT NULL, - year INTEGER NOT NULL, - grade FLOAT NOT NULL, - - graded_a INTEGER, - graded_b INTEGER, - graded_c INTEGER, - graded_d INTEGER, - graded_e INTEGER, - graded_f INTEGER, - - graded_fail INTEGER, - graded_pass INTEGER, - - CONSTRAINT fk_subject_id FOREIGN KEY (subject_id) REFERENCES subject (id) -); - -COMMIT; diff --git a/apps/grades/migrations/0002_unique_constraint_grades.sql b/apps/grades/migrations/0002_unique_constraint_grades.sql deleted file mode 100644 index 8146e6c75..000000000 --- a/apps/grades/migrations/0002_unique_constraint_grades.sql +++ /dev/null @@ -1,5 +0,0 @@ -BEGIN TRANSACTION; - -CREATE UNIQUE INDEX subject_season_grade_uq_subject_id_season_year ON subject_season_grade (subject_id, season, year); - -COMMIT; diff --git a/apps/grades/migrations/0003_grade_submission_tracking.sql b/apps/grades/migrations/0003_grade_submission_tracking.sql deleted file mode 100644 index 9a05c9e6d..000000000 --- a/apps/grades/migrations/0003_grade_submission_tracking.sql +++ /dev/null @@ -1,18 +0,0 @@ -BEGIN TRANSACTION; - -CREATE TYPE subject_write_log_grade AS ENUM ('A', 'B', 'C', 'D', 'E', 'F', 'G', 'H'); - -CREATE TABLE IF NOT EXISTS subject_season_grade_write_log -( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - subject_id UUID NOT NULL, - season subject_season NOT NULL, - year INTEGER NOT NULL, - grade subject_write_log_grade NOT NULL, - - CONSTRAINT subject_season_grade_write_log_fk_subject_id FOREIGN KEY (subject_id) REFERENCES subject (id) ON DELETE CASCADE, - CONSTRAINT subject_season_grade_write_log_uq_subject_season_grade_year_gra UNIQUE (subject_id, season, year, grade) -); - -COMMIT; diff --git a/apps/grades/migrations/0004_average_grades.sql b/apps/grades/migrations/0004_average_grades.sql deleted file mode 100644 index 1e6887a11..000000000 --- a/apps/grades/migrations/0004_average_grades.sql +++ /dev/null @@ -1,6 +0,0 @@ -BEGIN TRANSACTION; - -ALTER TABLE subject ADD COLUMN average_grade FLOAT NOT NULL DEFAULT 0.0; -ALTER TABLE subject ADD COLUMN total_registered INT NOT NULL DEFAULT 0; - -COMMIT; diff --git a/apps/grades/package.json b/apps/grades/package.json index bd33da203..9fe06e592 100644 --- a/apps/grades/package.json +++ b/apps/grades/package.json @@ -15,7 +15,6 @@ "@dotkomonline/logger": "workspace:*", "kysely": "^0.27.2", "next": "14.1.0", - "p-queue": "^8.0.1", "pg": "^8.11.3", "react": "^18", "react-dom": "^18", diff --git a/apps/grades/src/db.generated.d.ts b/apps/grades/src/db.generated.d.ts index 2bd8cf24c..142da039b 100644 --- a/apps/grades/src/db.generated.d.ts +++ b/apps/grades/src/db.generated.d.ts @@ -4,20 +4,29 @@ export type Generated = T extends ColumnType ? ColumnType : ColumnType -export type SubjectSeason = "AUTUMN" | "SPRING" | "SUMMER" | "WINTER" +export type Int8 = ColumnType -export type SubjectWriteLogGrade = "A" | "B" | "C" | "D" | "E" | "F" | "G" | "H" +export type SubjectGradingSeason = "AUTUMN" | "SPRING" | "SUMMER" | "WINTER" export type Timestamp = ColumnType -export interface NtnuFaculty { +export interface _SqlxMigrations { + checksum: Buffer + description: string + executionTime: Int8 + installedOn: Generated + success: boolean + version: Int8 +} + +export interface Department { + facultyId: string id: Generated name: string refId: string } -export interface NtnuFacultyDepartment { - facultyId: string +export interface Faculty { id: Generated name: string refId: string @@ -28,16 +37,16 @@ export interface Subject { credits: number departmentId: string educationalLevel: string + failedStudents: Generated id: Generated instructionLanguage: string name: string refId: string slug: string - totalRegistered: Generated + totalStudents: Generated } export interface SubjectSeasonGrade { - grade: number gradedA: number | null gradedB: number | null gradedC: number | null @@ -47,24 +56,15 @@ export interface SubjectSeasonGrade { gradedFail: number | null gradedPass: number | null id: Generated - season: SubjectSeason - subjectId: string - year: number -} - -export interface SubjectSeasonGradeWriteLog { - createdAt: Generated - grade: SubjectWriteLogGrade - id: Generated - season: SubjectSeason + season: SubjectGradingSeason subjectId: string year: number } export interface DB { - ntnuFaculty: NtnuFaculty - ntnuFacultyDepartment: NtnuFacultyDepartment + _SqlxMigrations: _SqlxMigrations + department: Department + faculty: Faculty subject: Subject subjectSeasonGrade: SubjectSeasonGrade - subjectSeasonGradeWriteLog: SubjectSeasonGradeWriteLog } diff --git a/apps/grades/src/pages/api/repopulate.ts b/apps/grades/src/pages/api/repopulate.ts deleted file mode 100644 index e0bd8cf6b..000000000 --- a/apps/grades/src/pages/api/repopulate.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { type NextApiRequest, type NextApiResponse } from "next" -import { getLogger } from "@dotkomonline/logger" -import { createKysely } from "@/server/kysely" -import { createServiceLayer } from "@/server/core" - -const logger = getLogger("api/repopulate") - -export default async function route(req: NextApiRequest, res: NextApiResponse) { - const skip = (req.query.skip as string | undefined)?.split(",") - logger.info(`Initializing repopulation job with skip ${skip?.join(", ") ?? ""}`) - - const kysely = await createKysely() - const core = createServiceLayer({ fetch, db: kysely }) - - if (!skip?.includes("faculty")) { - await core.jobService.performFacultySynchronizationJob() - } - - if (!skip?.includes("subject")) { - await core.jobService.performSubjectSynchronizationJob() - } - - if (!skip?.includes("grade")) { - await core.jobService.performGradeSynchronizationJob() - } - - res.status(200).send({}) -} diff --git a/apps/grades/src/pages/api/subjects.ts b/apps/grades/src/pages/api/subjects.ts index 59b32fd3e..c147a65c9 100644 --- a/apps/grades/src/pages/api/subjects.ts +++ b/apps/grades/src/pages/api/subjects.ts @@ -11,7 +11,7 @@ const QuerySchema = z.object({ export default async function route(req: NextApiRequest, res: NextApiResponse) { const query = QuerySchema.parse(req.query) - const kysely = await createKysely() + const kysely = createKysely() const core = createServiceLayer({ fetch, db: kysely }) const subjects = await core.subjectService.search(query.q, query.take, query.skip) diff --git a/apps/grades/src/server/core.ts b/apps/grades/src/server/core.ts index 832bb7fe4..5ac44a9e8 100644 --- a/apps/grades/src/server/core.ts +++ b/apps/grades/src/server/core.ts @@ -1,8 +1,6 @@ import { type Database } from "@/server/kysely" import { type FacultyRepository, FacultyRepositoryImpl } from "@/server/faculty-repository" -import { type HkdirService, HkdirServiceImpl } from "@/server/hkdir-service" import { type DepartmentRepository, DepartmentRepositoryImpl } from "@/server/department-repository" -import { type JobService, JobServiceImpl } from "@/server/job-service" import { type SubjectRepository, SubjectRepositoryImpl } from "@/server/subject-repository" import { type GradeRepository, GradeRepositoryImpl } from "@/server/grade-repository" import { type SubjectService, SubjectServiceImpl } from "@/server/subject-service" @@ -15,23 +13,13 @@ export interface CreateServiceLayerOptions { export type ServiceLayer = Awaited> export const createServiceLayer = (opts: CreateServiceLayerOptions) => { - const facultyRepository: FacultyRepository = new FacultyRepositoryImpl(opts.db) - const departmentRepository: DepartmentRepository = new DepartmentRepositoryImpl(opts.db) + const _facultyRepository: FacultyRepository = new FacultyRepositoryImpl(opts.db) + const _departmentRepository: DepartmentRepository = new DepartmentRepositoryImpl(opts.db) const subjectRepository: SubjectRepository = new SubjectRepositoryImpl(opts.db) - const gradeRepository: GradeRepository = new GradeRepositoryImpl(opts.db) - const hkdirService: HkdirService = new HkdirServiceImpl(opts.fetch) - const jobService: JobService = new JobServiceImpl( - facultyRepository, - departmentRepository, - subjectRepository, - gradeRepository, - hkdirService - ) + const _gradeRepository: GradeRepository = new GradeRepositoryImpl(opts.db) const subjectService: SubjectService = new SubjectServiceImpl(subjectRepository) return { - hkdirService, - jobService, subjectService, } } diff --git a/apps/grades/src/server/department-repository.ts b/apps/grades/src/server/department-repository.ts index 225ad41ee..f831f16b9 100644 --- a/apps/grades/src/server/department-repository.ts +++ b/apps/grades/src/server/department-repository.ts @@ -1,6 +1,4 @@ import { z } from "zod" -import { type Insertable } from "kysely" -import { type NtnuFacultyDepartment } from "@/db.generated" import { type Database } from "@/server/kysely" export type Department = z.infer @@ -12,39 +10,14 @@ export const Department = z.object({ }) export interface DepartmentRepository { - createDepartment(input: Insertable): Promise - getDepartmentByReferenceId(refId: string): Promise getDepartmentById(id: string): Promise } export class DepartmentRepositoryImpl implements DepartmentRepository { constructor(private readonly db: Database) {} - async createDepartment(input: Insertable): Promise { - const department = await this.db - .insertInto("ntnuFacultyDepartment") - .values(input) - .onConflict((eb) => eb.columns(["refId"]).doUpdateSet({ ...input })) - .returningAll() - .executeTakeFirstOrThrow() - return Department.parse(department) - } - - async getDepartmentByReferenceId(refId: string): Promise { - const department = await this.db - .selectFrom("ntnuFacultyDepartment") - .selectAll() - .where("refId", "=", refId) - .executeTakeFirst() - return department ? Department.parse(department) : null - } - async getDepartmentById(id: string): Promise { - const department = await this.db - .selectFrom("ntnuFacultyDepartment") - .selectAll() - .where("id", "=", id) - .executeTakeFirst() + const department = await this.db.selectFrom("department").selectAll().where("id", "=", id).executeTakeFirst() return department ? Department.parse(department) : null } } diff --git a/apps/grades/src/server/faculty-repository.ts b/apps/grades/src/server/faculty-repository.ts index 4e05c073d..ac246e09c 100644 --- a/apps/grades/src/server/faculty-repository.ts +++ b/apps/grades/src/server/faculty-repository.ts @@ -1,6 +1,4 @@ import { z } from "zod" -import { type Insertable } from "kysely" -import { type NtnuFaculty } from "@/db.generated" import { type Database } from "@/server/kysely" export type Faculty = z.infer @@ -11,31 +9,14 @@ export const Faculty = z.object({ }) export interface FacultyRepository { - createFaculty(input: Insertable): Promise - getFacultyByReferenceId(refId: string): Promise getFacultyById(id: string): Promise } export class FacultyRepositoryImpl implements FacultyRepository { constructor(private readonly db: Database) {} - async createFaculty(input: Insertable): Promise { - const faculty = await this.db - .insertInto("ntnuFaculty") - .values(input) - .onConflict((eb) => eb.columns(["refId"]).doUpdateSet({ ...input })) - .returningAll() - .executeTakeFirstOrThrow() - return Faculty.parse(faculty) - } - - async getFacultyByReferenceId(refId: string): Promise { - const faculty = await this.db.selectFrom("ntnuFaculty").selectAll().where("refId", "=", refId).executeTakeFirst() - return faculty ? Faculty.parse(faculty) : null - } - async getFacultyById(id: string): Promise { - const faculty = await this.db.selectFrom("ntnuFaculty").selectAll().where("id", "=", id).executeTakeFirst() + const faculty = await this.db.selectFrom("faculty").selectAll().where("id", "=", id).executeTakeFirst() return faculty ? Faculty.parse(faculty) : null } } diff --git a/apps/grades/src/server/grade-repository.ts b/apps/grades/src/server/grade-repository.ts index c940e45c0..23747830a 100644 --- a/apps/grades/src/server/grade-repository.ts +++ b/apps/grades/src/server/grade-repository.ts @@ -1,9 +1,5 @@ import { z } from "zod" -import { type Insertable, type Updateable } from "kysely" -import { type SubjectSeasonGrade } from "@/db.generated" import { type Database } from "@/server/kysely" -import { HkdirGrade, type HkdirGradeKey } from "@/server/hkdir-service" -import { mapHkdirGradeToGradeFactor } from "@/server/hkdir-util" export type Grade = z.infer export type Season = Grade["season"] @@ -24,41 +20,13 @@ export const Grade = z.object({ gradedPass: z.number().int().nullable().default(null), }) -export type GradeWriteLog = z.infer -export const GradeWriteLog = z.object({ - id: z.string().uuid(), - subjectId: z.string().uuid(), - season: Grade.shape.season, - year: z.number(), - grade: HkdirGrade.shape.Karakter, - createdAt: z.date().nullable(), -}) - export interface GradeRepository { - createGrade(input: Insertable): Promise - updateGrade(id: string, input: Updateable, hkdirKey: HkdirGradeKey, count: number): Promise getGradeBySemester(subjectId: string, season: Season, year: number): Promise - getPreviousWriteLogEntry( - subjectId: string, - season: Season, - year: number, - grade: HkdirGradeKey - ): Promise } export class GradeRepositoryImpl implements GradeRepository { constructor(private readonly db: Database) {} - async createGrade(input: Insertable): Promise { - const grade = await this.db - .insertInto("subjectSeasonGrade") - .values(input) - .onConflict((eb) => eb.columns(["subjectId", "season", "year"]).doUpdateSet({ ...input })) - .returningAll() - .executeTakeFirstOrThrow() - return Grade.parse(grade) - } - async getGradeBySemester(subjectId: string, season: Season, year: number): Promise { const grade = await this.db .selectFrom("subjectSeasonGrade") @@ -69,79 +37,4 @@ export class GradeRepositoryImpl implements GradeRepository { .executeTakeFirst() return grade ? Grade.parse(grade) : null } - - async updateGrade( - id: string, - input: Updateable, - hkdirKey: HkdirGradeKey, - count: number - ): Promise { - await this.db - .transaction() - .setIsolationLevel("read committed") - .execute(async (tx) => { - // If this throws, then it means we have a read inconsistency from the caller. - const grade = await tx - .selectFrom("subjectSeasonGrade") - .select(["subjectId", "season", "year"]) - .where("id", "=", id) - .forUpdate() - .executeTakeFirstOrThrow() - await tx - .insertInto("subjectSeasonGradeWriteLog") - .values({ - subjectId: grade.subjectId, - season: grade.season, - year: grade.year, - grade: hkdirKey, - }) - .execute() - // We need to update the grade distribution for the current grade as well. - if (count !== 0 && hkdirKey !== "H" && hkdirKey !== "G") { - const { totalRegistered, averageGrade } = await tx - .selectFrom("subject") - .select(["totalRegistered", "averageGrade"]) - .where("id", "=", grade.subjectId) - .executeTakeFirstOrThrow() - const newTotalRegistered = totalRegistered + count - const multiplicationFactor = mapHkdirGradeToGradeFactor(hkdirKey) - const newAverageGrade = (averageGrade * totalRegistered + count * multiplicationFactor) / newTotalRegistered - // Set the new average grade and total registered count - await tx - .updateTable("subject") - .set({ - totalRegistered: newTotalRegistered, - averageGrade: newAverageGrade, - }) - .where("id", "=", grade.subjectId) - .execute() - } - - return await tx - .updateTable("subjectSeasonGrade") - .set(input) - .where("id", "=", id) - .returningAll() - .executeTakeFirstOrThrow() - }) - } - - async getPreviousWriteLogEntry( - subjectId: string, - season: Season, - year: number, - grade: HkdirGradeKey - ): Promise { - const writeLog = await this.db - .selectFrom("subjectSeasonGradeWriteLog") - .selectAll() - .where("subjectId", "=", subjectId) - .where("season", "=", season) - .where("year", "=", year) - .where("grade", "=", grade) - .orderBy("createdAt", "desc") - .limit(1) - .executeTakeFirst() - return writeLog ? GradeWriteLog.parse(writeLog) : null - } } diff --git a/apps/grades/src/server/hkdir-service.ts b/apps/grades/src/server/hkdir-service.ts deleted file mode 100644 index 8f6db4dbd..000000000 --- a/apps/grades/src/server/hkdir-service.ts +++ /dev/null @@ -1,186 +0,0 @@ -import { z, type ZodSchema } from "zod" - -export type HkdirDepartment = z.infer -export const HkdirDepartment = z.object({ - "Nivå": z.string(), - "Nivå_tekst": z.string(), - "Institusjonskode": z.string(), - "Institusjonsnavn": z.string(), - "Avdelingskode": z.string(), - "Avdelingsnavn": z.string(), - "Gyldig_fra": z.string().nullish().default(null), - "Gyldig_til": z.string().nullish().default(null), - "fagkode_avdeling": z.string().nullish().default(null), - "fagnavn_avdeling": z.string().nullish().default(null), - "Fakultetskode": z.string(), - "Fakultetsnavn": z.string(), - "Avdelingskode (3 siste siffer)": z.string(), -}) - -export type HkdirSubject = z.infer -export const HkdirSubject = z.object({ - "Institusjonskode": z.string(), - "Institusjonsnavn": z.string(), - "Avdelingskode": z.string(), - "Avdelingsnavn": z.string(), - "Avdelingskode_SSB": z.string().nullish().default(null), - "Årstall": z.string(), - "Semester": z.string(), - "Semesternavn": z.string(), - "Studieprogramkode": z.string(), - "Studieprogramnavn": z.string(), - "Emnekode": z.string(), - "Emnenavn": z.string(), - "Nivåkode": z.string(), - "Nivånavn": z.string(), - "Studiepoeng": z.string(), - "NUS-kode": z.string(), - "Status": z.string(), - "Statusnavn": z.string(), - "Underv.språk": z.string(), - "Navn": z.string(), - "Fagkode": z.string().nullish().default(null), - "Fagnavn": z.string().nullish().default(null), - "Oppgave (ny fra h2012)": z.string().nullish().default(null), -}) - -export type HkdirGrade = z.infer -export type HkdirGradeKey = HkdirGrade["Karakter"] -export const HkdirGrade = z.object({ - "Årstall": z.string(), - "Semester": z.string(), - "Semesternavn": z.string(), - "Karakter": z.enum(["A", "B", "C", "D", "E", "F", "G", "H"]), - "Emnekode": z.string(), - "Antall kandidater totalt": z.string(), -}) - -const HKDIR_API_BASE_URL = "https://dbh.hkdir.no/api/Tabeller/hentJSONTabellData" - -type QueryOption = Record -interface QueryOptions { - sortBy?: string[] - groupBy?: string[] - filter: QueryOption[] -} - -export interface HkdirService { - getDepartments(institution: string): Promise - getSubjects(institution: string): Promise - getSubjectGrades(institution: string): Promise -} - -export class HkdirServiceImpl implements HkdirService { - public constructor(private readonly fetch: WindowOrWorkerGlobalScope["fetch"]) {} - - public async getDepartments(institution: string): Promise { - const query = this.createQuery(210, { - sortBy: ["Nivå"], - filter: [ - this.createQueryFilter("Institusjonskode", [institution.toString()]), - this.createExcludeQueryFilter("Avdelingskode", ["000000"]), - ], - }) - return await this.query(HkdirDepartment.array(), query) - } - - async getSubjects(institution: string): Promise { - const query = this.createQuery(208, { - sortBy: ["Årstall", "Institusjonskode", "Avdelingskode"], - filter: [ - this.createQueryFilter("Institusjonskode", [institution.toString()]), - this.createQueryFilter("Nivåkode", ["HN", "LN"]), - this.createQueryFilter("Status", ["1", "2"]), - // this.createTopQueryFilter("Årstall", 1), - this.createExcludeQueryFilter("Avdelingskode", ["000000"]), - ], - }) - return await this.query(HkdirSubject.array(), query) - } - - async getSubjectGrades(institution: string): Promise { - const query = this.createQuery(308, { - groupBy: ["Årstall", "Semester", "Karakter", "Emnekode", "Institusjonskode"], - filter: [ - this.createQueryFilter("Institusjonskode", [institution.toString()]), - // this.createTopQueryFilter("Årstall", 1), - this.createAllQueryFilter("Emnekode"), - this.createAllQueryFilter("Semester"), - ], - }) - return await this.query(HkdirGrade.array(), query) - } - - private createQuery(tableId: number, options: QueryOptions): Record { - return { - tabell_id: tableId, - api_versjon: 1, - statuslinje: "N", - kodetekst: "J", - desimal_separator: ".", - variabler: ["*"], - sortBy: options.sortBy, - groupBy: options.groupBy, - filter: options.filter, - } - } - - private async query(schema: T, parameters: Record): Promise> { - const response = await this.fetch(HKDIR_API_BASE_URL, { - method: "POST", - headers: { - "Content-Type": "application/json", - "Accept": "application/json", - }, - body: JSON.stringify(parameters), - }) - if (response.body === null) { - throw new Error("HKDir API Response did not contain a body.") - } - - return schema.parse(await response.json()) - } - - private createQueryFilter(variableName: string, values: string[]) { - return { - variabel: variableName, - selection: { - filter: "item", - values, - }, - } - } - - private createTopQueryFilter(variableName: string, query: number) { - return { - variabel: variableName, - selection: { - filter: "top", - values: [query.toString()], - exclude: [""], - }, - } - } - - private createExcludeQueryFilter(variableName: string, values: string[]) { - return { - variabel: variableName, - selection: { - filter: "all", - values: ["*"], - exclude: values, - }, - } - } - - private createAllQueryFilter(variableName: string) { - return { - variabel: variableName, - selection: { - filter: "all", - values: ["*"], - exclude: [""], - }, - } - } -} diff --git a/apps/grades/src/server/hkdir-util.ts b/apps/grades/src/server/hkdir-util.ts deleted file mode 100644 index 209de9557..000000000 --- a/apps/grades/src/server/hkdir-util.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { type Grade, type Season } from "@/server/grade-repository" -import { type HkdirGradeKey } from "@/server/hkdir-service" - -export const mapHkdirSemesterToSeason = (semester: string): Season => { - switch (semester) { - case "1": - return "SPRING" - case "3": - return "AUTUMN" - case "2": - return "SUMMER" - case "0": - return "WINTER" - default: - throw new Error(`Unknown semester: ${semester}`) - } -} - -export const mapHkdirGradeToGradeFactor = (grade: Exclude): number => { - switch (grade) { - case "A": - return 5 - case "B": - return 4 - case "C": - return 3 - case "D": - return 2 - case "E": - return 1 - case "F": - return 0 - default: - throw new Error(`Unknown grade: ${grade}`) - } -} - -export const mapHkdirGradeToGrade = (grade: string): keyof Grade => { - switch (grade) { - case "A": - return "gradedA" - case "B": - return "gradedB" - case "C": - return "gradedC" - case "D": - return "gradedD" - case "E": - return "gradedE" - case "F": - return "gradedF" - case "G": - return "gradedPass" - case "H": - return "gradedFail" - default: - throw new Error(`Unknown grade: ${grade}`) - } -} diff --git a/apps/grades/src/server/job-service.ts b/apps/grades/src/server/job-service.ts deleted file mode 100644 index 2944cdc6b..000000000 --- a/apps/grades/src/server/job-service.ts +++ /dev/null @@ -1,199 +0,0 @@ -import { getLogger } from "@dotkomonline/logger" -import { type FacultyRepository } from "@/server/faculty-repository" -import { type HkdirService } from "@/server/hkdir-service" -import { type DepartmentRepository } from "@/server/department-repository" -import { type SubjectRepository } from "@/server/subject-repository" -import { executeWithAsyncQueue } from "@/server/util" -import { type GradeRepository } from "@/server/grade-repository" -import { mapHkdirGradeToGrade, mapHkdirSemesterToSeason } from "@/server/hkdir-util" - -export interface JobService { - performFacultySynchronizationJob(): Promise - performSubjectSynchronizationJob(): Promise - performGradeSynchronizationJob(): Promise -} - -// TODO: Evaluate whether all of these should run in PostGreSQL transactions -export class JobServiceImpl implements JobService { - private readonly logger = getLogger("JobService") - - public constructor( - private readonly facultyRepository: FacultyRepository, - private readonly departmentRepository: DepartmentRepository, - private readonly subjectRepository: SubjectRepository, - private readonly gradeRepository: GradeRepository, - private readonly hkdirService: HkdirService - ) {} - - /** - * Synchronize faculties from HKDir to the database. - * - * HKDir departments are used to represent both faculties and departments in our data model. In HKDir, a department is - * of either level two or three. A level two department is a faculty in our model, while level three departments are - * the equivalent of departments. - * - * Because of this hierarchy, a level three department MUST have a matching level two department before creation. - */ - public async performFacultySynchronizationJob(): Promise { - this.logger.info("Synchronizing faculties from HKDir") - const faculties = await this.hkdirService.getDepartments("1150") - this.logger.info(`Beginning synchronization of ${faculties.length} faculties`) - - await executeWithAsyncQueue({ - build: (queue, concurrency) => { - this.logger.info(`Processing faculties using async queue with ${concurrency} concurrency`) - for (const faculty of faculties) { - queue.add(async () => { - // Attempt to register the faculty (level two institution) - let existingFaculty = await this.facultyRepository.getFacultyByReferenceId(faculty.Fakultetskode) - if (existingFaculty === null) { - existingFaculty = await this.facultyRepository.createFaculty({ - name: faculty.Fakultetsnavn, - refId: faculty.Fakultetskode, - }) - } - - // Attempt to register the department (level three institution) - let existingDepartment = await this.departmentRepository.getDepartmentByReferenceId(faculty.Avdelingskode) - if (existingDepartment === null) { - existingDepartment = await this.departmentRepository.createDepartment({ - name: faculty.Avdelingsnavn, - refId: faculty.Avdelingskode, - facultyId: existingFaculty.id, - }) - } - }) - } - }, - onTaskComplete: (remainingTasks) => { - if (remainingTasks && remainingTasks % 100 === 0) { - this.logger.info(`Queue processing for faculties reached ${remainingTasks} jobs left`) - } - }, - }) - this.logger.info("Synchronization of faculties complete") - } - - public async performSubjectSynchronizationJob() { - this.logger.info("Synchronizing subjects from HKDir") - const subjects = await this.hkdirService.getSubjects("1150") - this.logger.info(`Beginning synchronization of ${subjects.length} subjects`) - - await executeWithAsyncQueue({ - build: (queue, concurrency) => { - this.logger.info(`Processing subjects using async queue with ${concurrency} concurrency`) - for (const subject of subjects) { - queue.add(async () => { - // Pull the department for the given subject from the database - const department = await this.departmentRepository.getDepartmentByReferenceId(subject.Avdelingskode) - if (department === null) { - throw new Error(`Department with reference ID ${subject.Avdelingskode} not found`) - } - - // Attempt to register the subject - let existingSubject = await this.subjectRepository.getSubjectByReferenceId(subject.Emnekode) - if (existingSubject === null) { - // A lot of subjects have a -x suffix, which we want to remove for the slug - const slug = subject.Emnekode.replace(/-[123ABK]$/, "").toLowerCase() - existingSubject = await this.subjectRepository.createSubject({ - name: subject.Emnenavn, - refId: subject.Emnekode, - educationalLevel: subject.Nivåkode, - instructionLanguage: subject["Underv.språk"], - credits: parseFloat(subject.Studiepoeng), - departmentId: department.id, - slug, - }) - } - }) - } - }, - onTaskComplete: (remainingTasks) => { - if (remainingTasks && remainingTasks % 100 === 0) { - this.logger.info(`Queue processing for subjects reached ${remainingTasks} jobs left`) - } - }, - }) - this.logger.info("Synchronization of subjects complete") - } - - /** - * Synchronize grades from HKDir to the database. - * - * This job is a bit more complex than the others, because HKDir doesn't provide a result-set for the grades. Instead - * they provide a list of subjects, and for each subject a list of grades. This means we have to iterate over all - * subjects, and for each subject iterate over all grades. - */ - public async performGradeSynchronizationJob() { - this.logger.info("Synchronizing grades from HKDir") - const grades = await this.hkdirService.getSubjectGrades("1150") - this.logger.info(`Beginning synchronization of ${grades.length} grades`) - - await executeWithAsyncQueue({ - build: (queue, concurrency) => { - this.logger.info(`Processing grades using async queue with ${concurrency} concurrency`) - for (const grade of grades) { - const studentsWithGrade = parseInt(grade["Antall kandidater totalt"]) - if (studentsWithGrade === 0) { - continue - } - - queue.add(async () => { - // We need a reference to the subject from the reference id given by HKDir - const subject = await this.subjectRepository.getSubjectByReferenceId(grade.Emnekode) - if (subject === null) { - // TODO: Evaluate whether we should create the subject if it doesn't exist - return - } - - const year = parseInt(grade.Årstall) - - // First we need to get or insert the matching grade. - let existingGrade = await this.gradeRepository.getGradeBySemester( - subject.id, - mapHkdirSemesterToSeason(grade.Semester), - year - ) - if (existingGrade === null) { - existingGrade = await this.gradeRepository.createGrade({ - subjectId: subject.id, - season: mapHkdirSemesterToSeason(grade.Semester), - year, - grade: 0, - }) - } - - // If there exists a previous write log entry for the subject, season, year and grade, we need to skip this - // grade, as it has already been processed. - const previousWriteLogEntry = await this.gradeRepository.getPreviousWriteLogEntry( - existingGrade.subjectId, - mapHkdirSemesterToSeason(grade.Semester), - year, - grade.Karakter - ) - if (previousWriteLogEntry !== null) { - return - } - - // Then we need to update the grade distribution for the current grade - const key = mapHkdirGradeToGrade(grade.Karakter) - await this.gradeRepository.updateGrade( - existingGrade.id, - { - [key]: studentsWithGrade, - }, - grade.Karakter, - studentsWithGrade - ) - }) - } - }, - onTaskComplete: (remainingTasks) => { - if (remainingTasks && remainingTasks % 100 === 0) { - this.logger.info(`Queue processing for grades reached ${remainingTasks} jobs left`) - } - }, - }) - this.logger.info("Synchronization of grades complete") - } -} diff --git a/apps/grades/src/server/kysely.ts b/apps/grades/src/server/kysely.ts index ec58d326f..4bf874517 100644 --- a/apps/grades/src/server/kysely.ts +++ b/apps/grades/src/server/kysely.ts @@ -7,7 +7,7 @@ export type Database = Awaited> export const createKysely = () => { const conn = new pg.Pool({ - connectionString: process.env.DATABASE_URL ?? "postgres://postgres:postgres@localhost:5433/postgres", + connectionString: process.env.DATABASE_URL ?? "__MISSING_DATABASE_URL__", }) return new Kysely({ dialect: new PostgresDialect({ diff --git a/apps/grades/src/server/subject-repository.ts b/apps/grades/src/server/subject-repository.ts index 279d03803..3c82fd2f7 100644 --- a/apps/grades/src/server/subject-repository.ts +++ b/apps/grades/src/server/subject-repository.ts @@ -1,6 +1,5 @@ import { z } from "zod" -import { type Insertable, sql } from "kysely" -import { type Subject as DatabaseSubject } from "@/db.generated" +import { sql } from "kysely" import { type Database } from "@/server/kysely" export type Subject = z.infer @@ -14,11 +13,11 @@ export const Subject = z.object({ educationalLevel: z.string(), credits: z.number(), averageGrade: z.number().nonnegative(), - totalRegistered: z.number().int().nonnegative(), + totalStudents: z.number().int().nonnegative(), + failedStudents: z.number().int().nonnegative(), }) export interface SubjectRepository { - createSubject(input: Insertable): Promise getSubjectByReferenceId(refId: string): Promise getSubjectById(id: string): Promise getSubjectsBySearchExpression(expression: string, take: number, skip: number): Promise @@ -29,16 +28,6 @@ export interface SubjectRepository { export class SubjectRepositoryImpl implements SubjectRepository { constructor(private readonly db: Database) {} - async createSubject(input: Insertable): Promise { - const subject = await this.db - .insertInto("subject") - .values(input) - .onConflict((eb) => eb.columns(["refId"]).doUpdateSet({ ...input })) - .returningAll() - .executeTakeFirstOrThrow() - return Subject.parse(subject) - } - async getSubjectByReferenceId(refId: string): Promise { const subject = await this.db.selectFrom("subject").selectAll().where("refId", "=", refId).executeTakeFirst() return subject ? Subject.parse(subject) : null @@ -74,7 +63,7 @@ export class SubjectRepositoryImpl implements SubjectRepository { const subjects = await this.db .selectFrom("subject") .selectAll() - .orderBy("totalRegistered", "desc") + .orderBy("totalStudents", "desc") .limit(take) .offset(skip) .execute() diff --git a/apps/grades/src/server/util.ts b/apps/grades/src/server/util.ts deleted file mode 100644 index b015172ed..000000000 --- a/apps/grades/src/server/util.ts +++ /dev/null @@ -1,19 +0,0 @@ -import Queue from "p-queue" -const defaultConcurrency = parseInt(process.env.MAX_CONCURRENCY ?? "10") - -export interface AsyncExecuteOptions { - build(queue: Queue, concurrency: number): void - onTaskComplete(remainingTasks: number): void - concurrency?: number -} - -export const executeWithAsyncQueue = async ({ - build, - onTaskComplete, - concurrency = defaultConcurrency, -}: AsyncExecuteOptions) => { - const queue = new Queue({ concurrency }) - queue.on("next", () => onTaskComplete(queue.size)) - build(queue, concurrency) - await queue.onIdle() -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a2b14fa15..03f8b95df 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -245,9 +245,6 @@ importers: next: specifier: 14.1.0 version: 14.1.0(react-dom@18.2.0)(react@18.2.0) - p-queue: - specifier: ^8.0.1 - version: 8.0.1 pg: specifier: ^8.11.3 version: 8.11.3 @@ -10474,10 +10471,6 @@ packages: through: 2.3.8 dev: true - /eventemitter3@5.0.1: - resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==} - dev: false - /eventsource@2.0.2: resolution: {integrity: sha512-IzUmBGPR3+oUG9dUeXynyNmf91/3zUSJg1lCktzKw47OXuhco54U3r9B7O4XX+Rb1Itm9OZ2b0RkTs10bICOxA==} engines: {node: '>=12.0.0'} @@ -13768,19 +13761,6 @@ packages: engines: {node: '>=4'} dev: false - /p-queue@8.0.1: - resolution: {integrity: sha512-NXzu9aQJTAzbBqOt2hwsR63ea7yvxJc0PwN/zobNAudYfb1B7R08SzB4TsLeSbUCuG467NhnoT0oO6w1qRO+BA==} - engines: {node: '>=18'} - dependencies: - eventemitter3: 5.0.1 - p-timeout: 6.1.2 - dev: false - - /p-timeout@6.1.2: - resolution: {integrity: sha512-UbD77BuZ9Bc9aABo74gfXhNvzC9Tx7SxtHSh1fxvx3jTLLYvmVhiQZZrJzqqU0jKbN32kb5VOKiLEQI/3bIjgQ==} - engines: {node: '>=14.16'} - dev: false - /p-try@2.2.0: resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} engines: {node: '>=6'}