From e4ea1f475a4921e736b54155312656b65f65e5e0 Mon Sep 17 00:00:00 2001 From: 9FS <9fs@sussywussy.de> Date: Tue, 24 Dec 2024 21:47:59 +0000 Subject: [PATCH] added proper database setup via migrations; removed DATABASE_URL setting; WARNING: recreation of the database is necessary, just delete the "./db/" directory --- Cargo.lock | 153 ++++++++++++++++++------------------ Cargo.toml | 2 +- db_migrations/001_3.0.0.sql | 31 ++++++++ docker-compose.yaml | 2 +- readme.md | 7 -- src/config.rs | 2 - src/connect_to_db.rs | 91 ++++++--------------- src/main.rs | 6 +- src/main_inner.rs | 6 +- 9 files changed, 141 insertions(+), 159 deletions(-) create mode 100644 db_migrations/001_3.0.0.sql diff --git a/Cargo.lock b/Cargo.lock index 6a2664c..1d96703 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -147,9 +147,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytemuck" -version = "1.20.0" +version = "1.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a" +checksum = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3" [[package]] name = "byteorder" @@ -186,9 +186,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.2" +version = "1.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" +checksum = "c31a0499c1dc64f458ad13872de75c0eb7e3fdb0e67964610c914b034fc5956e" dependencies = [ "jobserver", "libc", @@ -209,9 +209,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -334,18 +334,18 @@ dependencies = [ [[package]] name = "crossbeam-queue" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crypto-common" @@ -481,9 +481,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "fern" @@ -709,11 +709,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -758,9 +758,9 @@ checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "hyper" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" +checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" dependencies = [ "bytes", "futures-channel", @@ -777,9 +777,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.3" +version = "0.27.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" +checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", "http", @@ -1033,9 +1033,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.74" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a865e038f7f6ed956f788f0d7d60c541fff74c7bd74272c5d4cf15c63743e705" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" dependencies = [ "once_cell", "wasm-bindgen", @@ -1052,9 +1052,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.167" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libm" @@ -1164,9 +1164,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" dependencies = [ "adler2", ] @@ -1184,7 +1184,7 @@ dependencies = [ [[package]] name = "nhentai_archivist" -version = "3.7.0" +version = "3.8.0" dependencies = [ "chrono", "load_config", @@ -1196,7 +1196,7 @@ dependencies = [ "serde_json", "setup_logging", "sqlx", - "thiserror 2.0.4", + "thiserror 2.0.9", "tokio", "unicode-segmentation", "zip", @@ -1267,9 +1267,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.5" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] @@ -1468,7 +1468,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.4", + "thiserror 2.0.9", "tokio", "tracing", ] @@ -1487,7 +1487,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.4", + "thiserror 2.0.9", "tinyvec", "tracing", "web-time", @@ -1495,9 +1495,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.7" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5a626c6807713b15cac82a6acaccd6043c9a5408c24baae07611fec3f243da" +checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904" dependencies = [ "cfg_aliases", "libc", @@ -1548,9 +1548,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ "bitflags", ] @@ -1648,22 +1648,22 @@ checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" [[package]] name = "rustix" -version = "0.38.41" +version = "0.38.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" +checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "rustls" -version = "0.23.19" +version = "0.23.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "934b404430bb06b3fae2cba809eb45a1ab1aecd64491213d7c3301b88393f8d1" +checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" dependencies = [ "once_cell", "ring", @@ -1684,9 +1684,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" dependencies = [ "web-time", ] @@ -1725,9 +1725,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.215" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" dependencies = [ "serde_derive", ] @@ -1746,9 +1746,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.215" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" dependencies = [ "proc-macro2", "quote", @@ -1757,9 +1757,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.133" +version = "1.0.134" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" dependencies = [ "itoa", "memchr", @@ -2126,9 +2126,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.90" +version = "2.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" +checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" dependencies = [ "proc-macro2", "quote", @@ -2179,11 +2179,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.4" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f49a1853cf82743e3b7950f77e0f4d622ca36cf4317cba00c767838bac8d490" +checksum = "f072643fd0190df67a8bab670c20ef5d8737177d6ac6b2e9a236cb096206b2cc" dependencies = [ - "thiserror-impl 2.0.4", + "thiserror-impl 2.0.9", ] [[package]] @@ -2199,9 +2199,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.4" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8381894bb3efe0c4acac3ded651301ceee58a15d47c2e34885ed1908ad667061" +checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4" dependencies = [ "proc-macro2", "quote", @@ -2251,9 +2251,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" dependencies = [ "tinyvec_macros", ] @@ -2395,9 +2395,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" @@ -2496,9 +2496,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d15e63b4482863c109d70a7b8706c1e364eb6ea449b201a76c5b89cedcec2d5c" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" dependencies = [ "cfg-if", "once_cell", @@ -2507,13 +2507,12 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d36ef12e3aaca16ddd3f67922bc63e48e953f126de60bd33ccc0101ef9998cd" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", "syn", @@ -2522,9 +2521,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.47" +version = "0.4.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dfaf8f50e5f293737ee323940c7d8b08a66a95a419223d9f41610ca08b0833d" +checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" dependencies = [ "cfg-if", "js-sys", @@ -2535,9 +2534,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "705440e08b42d3e4b36de7d66c944be628d579796b8090bfa3471478a2260051" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2545,9 +2544,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c9ae5a76e46f4deecd0f0255cc223cfa18dc9b261213b8aa0c7b36f61b3f1d" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", @@ -2558,15 +2557,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" [[package]] name = "web-sys" -version = "0.3.74" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a98bc3c33f0fe7e59ad7cd041b89034fa82a7c2d4365ca538dda6cdaf513863c" +checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" dependencies = [ "js-sys", "wasm-bindgen", @@ -2953,9 +2952,9 @@ dependencies = [ [[package]] name = "zip" -version = "2.2.1" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d52293fc86ea7cf13971b3bb81eb21683636e7ae24c729cdaf1b7c4157a352" +checksum = "ae9c1ea7b3a5e1f4b922ff856a129881167511563dc219869afe3787fc0c1a45" dependencies = [ "aes", "arbitrary", @@ -2973,7 +2972,7 @@ dependencies = [ "pbkdf2", "rand", "sha1", - "thiserror 2.0.4", + "thiserror 2.0.9", "time", "zeroize", "zopfli", diff --git a/Cargo.toml b/Cargo.toml index 82d7180..18cdac0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ license = "MIT" name = "nhentai_archivist" readme = "readme.md" repository = "https://github.com/9-FS/nhentai_archivist" -version = "3.7.0" +version = "3.8.0" [dependencies] chrono = { version = "^0.4.0", features = ["serde"] } diff --git a/db_migrations/001_3.0.0.sql b/db_migrations/001_3.0.0.sql new file mode 100644 index 0000000..38a8335 --- /dev/null +++ b/db_migrations/001_3.0.0.sql @@ -0,0 +1,31 @@ +CREATE TABLE Hentai +( + id INTEGER NOT NULL, + cover_type TEXT NOT NULL, + media_id INTEGER NOT NULL, + num_favorites INTEGER NOT NULL, + num_pages INTEGER NOT NULL, + page_types TEXT NOT NULL, + scanlator TEXT, + title_english TEXT, + title_japanese TEXT, + title_pretty TEXT, + upload_date TEXT NOT NULL, + PRIMARY KEY(id) +); +CREATE TABLE Tag +( + id INTEGER NOT NULL, + name TEXT NOT NULL, + type TEXT NOT NULL, + url TEXT NOT NULL, + PRIMARY KEY(id) +); +CREATE TABLE Hentai_Tag +( + hentai_id INTEGER NOT NULL, + tag_id INTEGER NOT NULL, + PRIMARY KEY(hentai_id, tag_id), + FOREIGN KEY(hentai_id) REFERENCES Hentai(id), + FOREIGN KEY(tag_id) REFERENCES Tag(id) +); \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index cce8149..a58ad40 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,7 +1,7 @@ services: nhentai_archivist: container_name: "nhentai_archivist" - image: "ghcr.io/9-fs/nhentai_archivist:3.7.0" + image: "ghcr.io/9-fs/nhentai_archivist:3.8.0" environment: HOST_OS: "Unraid" TZ: "UTC" diff --git a/readme.md b/readme.md index c841ebe..63e567d 100644 --- a/readme.md +++ b/readme.md @@ -45,10 +45,6 @@ I'm happy about anyone who finds my software useful and feedback is also always Setting this to `false` prevents the temporary directory containing the original images from being deleted after the CBZ file has been created. In addition to that it also saves a `ComicBook.xml` in the directory. This can be useful to improve compatibility with third party readers or deduplication software. -- `DATABASE_URL` - - This is the URL to the SQLite database file. If you changed `DATABASE_URL`, confirm the database directory already exists. It is possible that it is not created automatically because the URL could point to a remote directory. The database file will and should be created automatically. - - `DONTDOWNLOADME_FILEPATH`, optional, defaults to `None` This is the path to the file containing the nHentai ID you explicitly do not want to download, separated by line breaks. It has priority over all input methods. If you want to systematically exclude hentai by tag, use the `-` operator in the tag search instead. @@ -120,7 +116,6 @@ I'm happy about anyone who finds my software useful and feedback is also always Example `./config/.env`: ```TOML -DATABASE_URL = "./db/db.sqlite" LIBRARY_PATH = "./hentai/" ``` @@ -134,7 +129,6 @@ Example `./config/.env`: ```TOML CSRFTOKEN = "your token here" -DATABASE_URL = "./db/db.sqlite" DONTDOWNLOADME_FILEPATH = "./config/dontdownloadme.txt" DOWNLOADME_FILEPATH = "./config/downloadme.txt" LIBRARY_PATH = "./hentai/" @@ -154,7 +148,6 @@ Example `./config/.env`: ```TOML CF_CLEARANCE = "" CSRFTOKEN = "your token here" -DATABASE_URL = "./db/db.sqlite" DONTDOWNLOADME_FILEPATH = "./config/dontdownloadme.txt" DOWNLOADME_FILEPATH = "./config/downloadme.txt" LIBRARY_PATH = "./hentai/" diff --git a/src/config.rs b/src/config.rs index b307479..df88dd6 100644 --- a/src/config.rs +++ b/src/config.rs @@ -10,7 +10,6 @@ pub struct Config pub CF_CLEARANCE: Option, // bypass bot protection pub CLEANUP_TEMPORARY_FILES: Option, // clean up temporary files after downloading? some prefer off for deduplication or compatibility with other tools pub CSRFTOKEN: Option, // bypass bot protection - pub DATABASE_URL: String, // url to database file pub DEBUG: Option, // debug mode? pub DONTDOWNLOADME_FILEPATH: Option, // path to file containing hentai ID to not download, blacklist pub DOWNLOADME_FILEPATH: Option, // path to file containing hentai ID to download @@ -31,7 +30,6 @@ impl Default for Config CF_CLEARANCE: None, CLEANUP_TEMPORARY_FILES: None, CSRFTOKEN: Some("".to_owned()), - DATABASE_URL: "./db/db.sqlite".to_owned(), DEBUG: None, // no entry in default config, defaults to false DONTDOWNLOADME_FILEPATH: Some("./config/dontdownloadme.txt".to_owned()), DOWNLOADME_FILEPATH: Some("./config/downloadme.txt".to_owned()), diff --git a/src/connect_to_db.rs b/src/connect_to_db.rs index 6bfa9ca..f4fd3fa 100644 --- a/src/connect_to_db.rs +++ b/src/connect_to_db.rs @@ -4,60 +4,29 @@ use sqlx::migrate::MigrateDatabase; /// # Summary -/// Connects to database at `database_url` and returns a connection pool. If database does not exist, creates a new database and initialises it with the instructions in `./db/create_db.sql`. +/// Creates a new database or connects to an existing one at `database_url`, runs the instructions in `migrations_path`, and returns a connection pool. /// /// # Arguments -/// - `database_url`: path to database file +/// - `database_url`: url to database file, might not be local but is recommended to be so +/// - `migrations_path`: path to directory containing migration files /// /// # Returns /// - connection pool to database or error -pub async fn connect_to_db(database_url: &str) -> Result +pub async fn connect_to_db(db_url: &str, db_migrations_path: &str) -> Result { - const CREATE_DB_QUERY_STRING: &str = // query string to create all tables except the dynamically created Hentai_{id}_Pages - "CREATE TABLE Hentai - ( - id INTEGER NOT NULL, - cover_type TEXT NOT NULL, - media_id INTEGER NOT NULL, - num_favorites INTEGER NOT NULL, - num_pages INTEGER NOT NULL, - page_types TEXT NOT NULL, - scanlator TEXT, - title_english TEXT, - title_japanese TEXT, - title_pretty TEXT, - upload_date TEXT NOT NULL, - PRIMARY KEY(id) - ); - CREATE TABLE Tag - ( - id INTEGER NOT NULL, - name TEXT NOT NULL, - type TEXT NOT NULL, - url TEXT NOT NULL, - PRIMARY KEY(id) - ); - CREATE TABLE Hentai_Tag - ( - hentai_id INTEGER NOT NULL, - tag_id INTEGER NOT NULL, - PRIMARY KEY(hentai_id, tag_id), - FOREIGN KEY(hentai_id) REFERENCES Hentai(id), - FOREIGN KEY(tag_id) REFERENCES Tag(id) - );"; - let db: sqlx::sqlite::SqlitePool; // database containing all metadata from nhentai.net api + let db: sqlx::SqlitePool; // database connection pool - if !sqlx::sqlite::Sqlite::database_exists(database_url).await? // if database does not exist + if !sqlx::Sqlite::database_exists(db_url).await? // if database does not exist { - match std::path::Path::new(database_url).parent() + match std::path::Path::new(db_url).parent() { Some(parent) => { #[cfg(target_family = "unix")] if let Err(e) = tokio::fs::DirBuilder::new().recursive(true).mode(0o777).create(parent).await // create all parent directories with permissions "drwxrwxrwx" { - log::warn!("Creating parent directories for new database at \"{database_url}\" failed with {e}.\nThis could be expected behaviour, usually if this is a remote pointing URL and not a local filepath. In that case create the parent directories manually."); + log::warn!("Creating parent directories for new database at \"{db_url}\" failed with {e}.\nThis could be expected behaviour, usually if this is a remote pointing URL and not a local filepath. In that case create the parent directories manually."); } #[cfg(not(target_family = "unix"))] if let Err(e) = tokio::fs::DirBuilder::new().recursive(true).create(parent).await // create all parent directories @@ -65,37 +34,27 @@ pub async fn connect_to_db(database_url: &str) -> Result log::warn!("Creating parent directories for new database at \"{database_url}\", because the directory part could not be parsed.\nThis could be expected behaviour, usually if this is a remote pointing URL and not a local filepath. In that case create the parent directories manually."), + None => log::warn!("Creating parent directories for new database at \"{db_url}\", because the directory part could not be parsed.\nThis could be expected behaviour, usually if this is a remote pointing URL and not a local filepath. In that case create the parent directories manually."), } - sqlx::sqlite::Sqlite::create_database(database_url).await?; // create new database - log::info!("Created new database at \"{}\".", database_url); + sqlx::Sqlite::create_database(db_url).await?; // create new database + log::info!("Created new database at \"{db_url}\"."); + } - db = sqlx::sqlite::SqlitePoolOptions::new() - .max_connections(1) // only 1 connection to database at the same time, otherwise concurrent writers fail - .max_lifetime(None) // keep connection open indefinitely otherwise database locks up after lifetime, closing and reconnecting manually - .connect(database_url).await?; // connect to database - db.set_connect_options(sqlx::sqlite::SqliteConnectOptions::new() - .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal) // use write-ahead journal for better performance - .locking_mode(sqlx::sqlite::SqliteLockingMode::Exclusive) // do not release file lock until all transactions are complete - .log_slow_statements(log::LevelFilter::Warn, std::time::Duration::from_secs(5)) // log slow statements only after 5 s - .synchronous(sqlx::sqlite::SqliteSynchronous::Normal)); // ensure data is written to disk after each transaction for consistent state - log::info!("Connected to database at \"{}\".", database_url); + db = sqlx::sqlite::SqlitePoolOptions::new() + .max_connections(1) // only 1 connection to database at the same time, otherwise concurrent writers fail + .max_lifetime(None) // keep connection open indefinitely otherwise database locks up after lifetime, closing and reconnecting manually + .connect(db_url).await?; // connect to database + db.set_connect_options(sqlx::sqlite::SqliteConnectOptions::new() + .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal) // use write-ahead journal for better performance + .locking_mode(sqlx::sqlite::SqliteLockingMode::Exclusive) // do not release file lock until all transactions are complete + .log_slow_statements(log::LevelFilter::Warn, std::time::Duration::from_secs(5)) // log slow statements only after 5 s + .synchronous(sqlx::sqlite::SqliteSynchronous::Normal)); // ensure data is written to disk after each transaction for consistent state + log::info!("Connected to database at \"{db_url}\"."); - sqlx::query(CREATE_DB_QUERY_STRING).execute(&db).await?; // initialise database by creating tables - log::info!("Created database tables."); - } - else // if database already exists + if std::path::Path::new(db_migrations_path).exists() // if migrations path exists { - db = sqlx::sqlite::SqlitePoolOptions::new() - .max_connections(1) // only 1 connection to database at the same time, otherwise concurrent writers fail - .max_lifetime(None) // keep connection open indefinitely otherwise database locks up after lifetime, closing and reconnecting manually - .connect(database_url).await?; // connect to database - db.set_connect_options(sqlx::sqlite::SqliteConnectOptions::new() - .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal) // use write-ahead journal for better performance - .locking_mode(sqlx::sqlite::SqliteLockingMode::Exclusive) // do not release file lock until all transactions are complete - .log_slow_statements(log::LevelFilter::Warn, std::time::Duration::from_secs(5)) // log slow statements only after 5 s - .synchronous(sqlx::sqlite::SqliteSynchronous::Normal)); // ensure data is written to disk after each transaction for consistent state - log::info!("Connected to database at \"{}\".", database_url); + sqlx::migrate::Migrator::new(std::path::Path::new(db_migrations_path)).await?.run(&db).await?; // run migrations to create and update tables + log::debug!("Executed migrations at \"{db_migrations_path}\"."); } return Ok(db); diff --git a/src/main.rs b/src/main.rs index f48d82c..30e80e0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -52,14 +52,14 @@ fn main() -> std::process::ExitCode crate_logging_level.insert("hyper_util".to_owned(), log::Level::Info); // shut up crate_logging_level.insert("serde_xml_rs".to_owned(), log::Level::Error); // shut up - crate_logging_level.insert("sqlx::query".to_owned(), log::Level::Error); // shut up + crate_logging_level.insert("sqlx::query".to_owned(), log::Level::Warn); // shut up if config.DEBUG.unwrap_or(false) // setup logging, if DEBUG unset default to false { - setup_logging::setup_logging(log::Level::Debug, None, "./log/%Y-%m-%dT%H_%M.log"); + setup_logging::setup_logging(log::Level::Debug, Some(crate_logging_level), "./log/%Y-%m-%dT%H_%M.log"); } else { - setup_logging::setup_logging(log::Level::Info, None, "./log/%Y-%m-%d.log"); + setup_logging::setup_logging(log::Level::Info, Some(crate_logging_level), "./log/%Y-%m-%d.log"); } log::debug!("Loaded {config:?}."); // log loaded config diff --git a/src/main_inner.rs b/src/main_inner.rs index 78ab646..4fd8950 100644 --- a/src/main_inner.rs +++ b/src/main_inner.rs @@ -8,6 +8,8 @@ use crate::hentai::*; pub async fn main_inner(config: Config) -> Result<(), Error> { + const DB_FILEPATH: &str = "./db/db.sqlite"; // database filepath + const DB_MIGRATIONS_PATH: &str = "./db_migrations/"; // migrations path const HTTP_TIMEOUT: u64 = 30; // connection timeout const NHENTAI_HENTAI_SEARCH_URL: &str = "https://nhentai.net/api/gallery/"; // nhentai search by id api url const NHENTAI_TAG_SEARCH_URL: &str = "https://nhentai.net/api/galleries/search"; // nhentai search by tag api url @@ -76,7 +78,7 @@ pub async fn main_inner(config: Config) -> Result<(), Error> break 'iteration; // if server mode: only abort iteration, go straight to sleeping } - match connect_to_db(&config.DATABASE_URL).await // connect to database + match connect_to_db(DB_FILEPATH, DB_MIGRATIONS_PATH).await // connect to database { Ok(o) => db = o, Err(e) => @@ -132,7 +134,7 @@ pub async fn main_inner(config: Config) -> Result<(), Error> db.close().await; // close database connection - log::info!("Disconnected from database at \"{}\".", config.DATABASE_URL); + log::info!("Disconnected from database at \"{}\".", DB_FILEPATH); if config.NHENTAI_TAGS.is_none() {break 'program;} // if tag not set: client mode, exit