Rewrite Grades v2 data synchronizer in async Rust

apps/grades-sync/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "grades-sync"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+sqlx = { version = "0.7", features = ["runtime-tokio", "tls-rustls", "uuid", "json", "migrate", "postgres"] }
+tokio = { version = "1", features = ["full"] }
+reqwest = { version = "0.11", features = ["json"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+anyhow = "1.0.79"
+dotenv = "0.15.0"
+log = { version = "0.4.20", features = [] }
+env_logger = "0.11.1"
+async-trait = "0.1.77"
+regex = "1.10.3"
+itertools = "0.12.1"
+futures = "0.3.30"
+lazy_static = "1.4.0"

apps/grades-sync/README.md

@@ -0,0 +1,11 @@
+# Grades Sync
+
+Asynchronous grades synchronizer against HKDir's API.
+
+## Environment variables
+
+```env
+DATABASE_URL=postgres://...
+```
+
+The `RUST_LOG` variable can be set to `info` or `debug` to enable logging.

apps/grades-sync/migrations/20240209190449_schema.sql

@@ -0,0 +1,25 @@
+START TRANSACTION;
+
+CREATE EXTENSION IF NOT EXISTS "fuzzystrmatch";
+
+CREATE TABLE IF NOT EXISTS faculty
+(
+    id     UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    ref_id TEXT NOT NULL,
+    name   TEXT NOT NULL,
+
+    CONSTRAINT faculty_ref_id_unique UNIQUE (ref_id)
+);
+
+CREATE TABLE IF NOT EXISTS department
+(
+    id         UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    ref_id     TEXT NOT NULL,
+    faculty_id UUID NOT NULL,
+    name       TEXT NOT NULL,
+
+    CONSTRAINT department_fk_faculty_id FOREIGN KEY (faculty_id) REFERENCES faculty (id),
+    CONSTRAINT department_uq_ref_id UNIQUE (ref_id)
+);
+
+COMMIT;

apps/grades-sync/migrations/20240209195028_subject.sql

@@ -0,0 +1,23 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS subject
+(
+    id                   UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    ref_id               TEXT NOT NULL,
+    name                 TEXT NOT NULL,
+    slug                 TEXT NOT NULL,
+    department_id        UUID NOT NULL,
+
+    instruction_language TEXT NOT NULL,
+    educational_level    TEXT NOT NULL,
+    credits              REAL NOT NULL,
+
+    average_grade        REAL NOT NULL    DEFAULT 0.0,
+    total_students       INT  NOT NULL    DEFAULT 0,
+    failed_students      INT  NOT NULL    DEFAULT 0,
+
+    CONSTRAINT subject_fk_department_id FOREIGN KEY (department_id) REFERENCES department (id),
+    CONSTRAINT subject_uq_ref_id UNIQUE (ref_id)
+);
+
+COMMIT;

apps/grades-sync/migrations/20240209212106_grades.sql

@@ -0,0 +1,25 @@
+START TRANSACTION;
+
+CREATE TYPE subject_grading_season AS ENUM ('WINTER', 'SPRING', 'SUMMER', 'AUTUMN');
+
+CREATE TABLE IF NOT EXISTS subject_season_grade
+(
+    id          UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    subject_id  UUID                   NOT NULL,
+    season      subject_grading_season NOT NULL,
+    year        INTEGER                NOT NULL,
+
+    graded_a    INTEGER,
+    graded_b    INTEGER,
+    graded_c    INTEGER,
+    graded_d    INTEGER,
+    graded_e    INTEGER,
+    graded_f    INTEGER,
+    graded_pass INTEGER,
+    graded_fail INTEGER,
+
+    CONSTRAINT subject_season_grade_fk_subject_id FOREIGN KEY (subject_id) REFERENCES subject (id),
+    CONSTRAINT subject_season_grade_unique UNIQUE (subject_id, season, year)
+);
+
+COMMIT;

apps/grades-sync/queries/departments.json

@@ -0,0 +1,39 @@
+{
+  "tabell_id": 210,
+  "api_versjon": 1,
+  "statuslinje": "N",
+  "kodetekst": "J",
+  "desimal_separator": ".",
+  "variabler": [
+    "*"
+  ],
+  "sortBy": [
+    "Nivå"
+  ],
+  "filter": [
+    {
+      "variabel": "Institusjonskode",
+      "selection": {
+        "filter": "item",
+        "values": [
+          "1150"
+        ],
+        "exclude": [
+          ""
+        ]
+      }
+    },
+    {
+      "variabel": "Avdelingskode",
+      "selection": {
+        "filter": "all",
+        "values": [
+          "*"
+        ],
+        "exclude": [
+          "000000"
+        ]
+      }
+    }
+  ]
+}

apps/grades-sync/queries/grades.json

@@ -0,0 +1,55 @@
+{
+  "tabell_id": 308,
+  "api_versjon": 1,
+  "statuslinje": "N",
+  "kodetekst": "J",
+  "desimal_separator": ".",
+  "variabler": [
+    "*"
+  ],
+  "groupBy": [
+    "Årstall",
+    "Semester",
+    "Karakter",
+    "Emnekode",
+    "Institusjonskode"
+  ],
+  "filter": [
+    {
+      "variabel": "Institusjonskode",
+      "selection": {
+        "filter": "item",
+        "values": [
+          "1150"
+        ],
+        "exclude": [
+          ""
+        ]
+      }
+    },
+    {
+      "variabel": "Emnekode",
+      "selection": {
+        "filter": "all",
+        "values": [
+          "*"
+        ],
+        "exclude": [
+          ""
+        ]
+      }
+    },
+    {
+      "variabel": "Semester",
+      "selection": {
+        "filter": "all",
+        "values": [
+          "*"
+        ],
+        "exclude": [
+          ""
+        ]
+      }
+    }
+  ]
+}

apps/grades-sync/queries/subjects.json

@@ -0,0 +1,67 @@
+{
+  "tabell_id": 208,
+  "api_versjon": 1,
+  "statuslinje": "N",
+  "kodetekst": "J",
+  "desimal_separator": ".",
+  "variabler": [
+    "*"
+  ],
+  "sortBy": [
+    "Årstall",
+    "Institusjonskode",
+    "Avdelingskode"
+  ],
+  "filter": [
+    {
+      "variabel": "Institusjonskode",
+      "selection": {
+        "filter": "item",
+        "values": [
+          "1150"
+        ],
+        "exclude": [
+          ""
+        ]
+      }
+    },
+    {
+      "variabel": "Nivåkode",
+      "selection": {
+        "filter": "item",
+        "values": [
+          "HN",
+          "LN"
+        ],
+        "exclude": [
+          ""
+        ]
+      }
+    },
+    {
+      "variabel": "Avdelingskode",
+      "selection": {
+        "filter": "all",
+        "values": [
+          "*"
+        ],
+        "exclude": [
+          "000000"
+        ]
+      }
+    },
+    {
+      "variabel": "Oppgave (ny fra h2012)",
+      "selection": {
+        "filter": "all",
+        "values": [
+          "*"
+        ],
+        "exclude": [
+          "1",
+          "2"
+        ]
+      }
+    }
+  ]
+}

apps/grades-sync/src/department_repository.rs

@@ -0,0 +1,67 @@
+use crate::pg::Database;
+use async_trait::async_trait;
+use sqlx::types::Uuid;
+use sqlx::FromRow;
+
+#[derive(Debug, FromRow)]
+pub struct Department {
+    pub id: Uuid,
+    pub name: String,
+    pub ref_id: String,
+    pub faculty_id: Uuid,
+}
+
+#[async_trait]
+pub trait DepartmentRepository: Sync {
+    async fn create_department(
+        &self,
+        name: String,
+        ref_id: String,
+        faculty_id: Uuid,
+    ) -> Result<Department, sqlx::Error>;
+    async fn get_department_by_ref_id(&self, ref_id: String) -> Result<Department, sqlx::Error>;
+}
+
+pub struct DepartmentRepositoryImpl<'a> {
+    db: &'a Database,
+}
+
+impl<'a> DepartmentRepositoryImpl<'a> {
+    pub fn new(db: &'a Database) -> Self {
+        Self { db }
+    }
+}
+
+#[async_trait]
+impl<'a> DepartmentRepository for DepartmentRepositoryImpl<'a> {
+    async fn create_department(
+        &self,
+        name: String,
+        ref_id: String,
+        faculty_id: Uuid,
+    ) -> Result<Department, sqlx::Error> {
+        sqlx::query_as::<_, Department>(
+            r#"
+            INSERT INTO department (name, ref_id, faculty_id) VALUES ($1, $2, $3)
+            ON CONFLICT (ref_id) DO UPDATE SET name = $1, ref_id = $2, faculty_id = $3
+            RETURNING *;
+            "#,
+        )
+        .bind(name)
+        .bind(ref_id)
+        .bind(faculty_id)
+        .fetch_one(self.db)
+        .await
+    }
+
+    async fn get_department_by_ref_id(&self, ref_id: String) -> Result<Department, sqlx::Error> {
+        sqlx::query_as::<_, Department>(
+            r#"
+            SELECT * FROM department WHERE ref_id = $1;
+            "#,
+        )
+        .bind(ref_id)
+        .fetch_one(self.db)
+        .await
+    }
+}

apps/grades-sync/src/faculty_repository.rs

@@ -0,0 +1,43 @@
+use crate::pg::Database;
+use async_trait::async_trait;
+use sqlx::types::Uuid;
+use sqlx::FromRow;
+
+#[derive(Debug, FromRow)]
+pub struct Faculty {
+    pub id: Uuid,
+    pub name: String,
+    pub ref_id: String,
+}
+
+#[async_trait]
+pub trait FacultyRepository: Sync {
+    async fn create_faculty(&self, name: String, ref_id: String) -> Result<Faculty, sqlx::Error>;
+}
+
+pub struct FacultyRepositoryImpl<'a> {
+    db: &'a Database,
+}
+
+impl<'a> FacultyRepositoryImpl<'a> {
+    pub fn new(db: &'a Database) -> Self {
+        Self { db }
+    }
+}
+
+#[async_trait]
+impl<'a> FacultyRepository for FacultyRepositoryImpl<'a> {
+    async fn create_faculty(&self, name: String, ref_id: String) -> Result<Faculty, sqlx::Error> {
+        sqlx::query_as::<_, Faculty>(
+            r#"
+            INSERT INTO faculty (name, ref_id) VALUES ($1, $2)
+            ON CONFLICT (ref_id) DO UPDATE SET name = $1, ref_id = $2
+            RETURNING *;
+            "#,
+        )
+        .bind(name)
+        .bind(ref_id)
+        .fetch_one(self.db)
+        .await
+    }
+}