From 6dceb9e32776f1678bfb10c144b87d1bba25c0ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Kleinb=C3=B6lting?= Date: Mon, 29 Jul 2024 14:03:56 +0200 Subject: [PATCH] feat: Add robots.txt and sitemap.xml (DEV-3931) (#152) --- dsp-meta-cmd/src/main-server.rs | 6 +++ dsp-meta/src/api/handler/health.rs | 2 +- dsp-meta/src/api/handler/mod.rs | 4 +- dsp-meta/src/api/handler/robots_txt.rs | 26 +++++++++++ dsp-meta/src/api/handler/sitemap_xml.rs | 43 +++++++++++++++++++ dsp-meta/src/api/handler/v1/mod.rs | 1 + .../projects/handlers.rs} | 8 ++-- dsp-meta/src/api/handler/v1/projects/mod.rs | 2 + .../v1/projects/responses.rs} | 0 dsp-meta/src/api/mod.rs | 1 - dsp-meta/src/api/model/mod.rs | 2 - .../api/model/project_metadata_graph_dto.rs | 18 -------- dsp-meta/src/api/router.rs | 19 +++++--- dsp-meta/src/app_state.rs | 3 ++ .../service/project_metadata_api_contract.rs | 1 + .../service/project_metadata_service.rs | 4 ++ .../src/domain/service/repository_contract.rs | 5 ++- .../service/project_metadata_repository.rs | 6 +++ web-frontend/src/Snackbar.svelte | 2 +- .../src/project-page/ProjectPage.svelte | 2 +- .../ProjectsRepository.svelte | 2 +- 21 files changed, 121 insertions(+), 36 deletions(-) create mode 100644 dsp-meta/src/api/handler/robots_txt.rs create mode 100644 dsp-meta/src/api/handler/sitemap_xml.rs create mode 100644 dsp-meta/src/api/handler/v1/mod.rs rename dsp-meta/src/api/handler/{project_metadata_handler.rs => v1/projects/handlers.rs} (91%) create mode 100644 dsp-meta/src/api/handler/v1/projects/mod.rs rename dsp-meta/src/api/{model/project_metadata_dto.rs => handler/v1/projects/responses.rs} (100%) delete mode 100644 dsp-meta/src/api/model/mod.rs delete mode 100644 dsp-meta/src/api/model/project_metadata_graph_dto.rs diff --git a/dsp-meta-cmd/src/main-server.rs b/dsp-meta-cmd/src/main-server.rs index b552ad77..308b6820 100644 --- a/dsp-meta-cmd/src/main-server.rs +++ b/dsp-meta-cmd/src/main-server.rs @@ -12,6 +12,7 @@ use tokio::net::TcpListener; use tracing::info; use tracing_subscriber::prelude::*; use tracing_subscriber::{fmt, EnvFilter}; +use url::Url; fn main() { // Do the pid1 magic. Needs to be the first thing executed. @@ -64,12 +65,17 @@ async fn init_server() { .get::("public_dir") .unwrap_or("/public".to_string()); + let base_url = settings + .get::("base_url") + .unwrap_or(Url::parse("http://localhost:3000").unwrap()); + let shared_state = Arc::new(AppState { project_metadata_service: ProjectMetadataService::new(ProjectMetadataRepository::new( Path::new(&data_dir), )), public_dir, version: VERSION, + base_url, }); // start the server diff --git a/dsp-meta/src/api/handler/health.rs b/dsp-meta/src/api/handler/health.rs index 50d9b26d..ff97b68b 100644 --- a/dsp-meta/src/api/handler/health.rs +++ b/dsp-meta/src/api/handler/health.rs @@ -1,6 +1,6 @@ use tracing::trace; -pub(crate) async fn health_handler() -> &'static str { +pub(crate) async fn health() -> &'static str { trace!("entered health_handler()"); "healthy" } diff --git a/dsp-meta/src/api/handler/mod.rs b/dsp-meta/src/api/handler/mod.rs index d67ef8a5..91f59bf3 100644 --- a/dsp-meta/src/api/handler/mod.rs +++ b/dsp-meta/src/api/handler/mod.rs @@ -1,2 +1,4 @@ pub mod health; -pub mod project_metadata_handler; +pub mod robots_txt; +pub mod sitemap_xml; +pub mod v1; diff --git a/dsp-meta/src/api/handler/robots_txt.rs b/dsp-meta/src/api/handler/robots_txt.rs new file mode 100644 index 00000000..beba0c6d --- /dev/null +++ b/dsp-meta/src/api/handler/robots_txt.rs @@ -0,0 +1,26 @@ +use std::sync::Arc; + +use axum::extract::State; +use axum::http::{Response, StatusCode}; + +use crate::app_state::AppState; +use crate::error::DspMetaError; + +pub async fn robots_txt( + State(state): State>, +) -> Result, DspMetaError> { + let sitemap_xml = state + .base_url + .join("sitemap.xml") + .expect("valid url") + .to_string(); + let response = Response::builder() + .status(StatusCode::OK) + .header("Content-Type", "text/plain") + .body(format!( + "Sitemap: {}\nUser-agent: *\nDisallow:\n", + sitemap_xml + )) + .expect("Failed to build response"); + Ok(response) +} diff --git a/dsp-meta/src/api/handler/sitemap_xml.rs b/dsp-meta/src/api/handler/sitemap_xml.rs new file mode 100644 index 00000000..8adc0110 --- /dev/null +++ b/dsp-meta/src/api/handler/sitemap_xml.rs @@ -0,0 +1,43 @@ +use std::sync::Arc; + +use axum::extract::State; +use axum::http::StatusCode; +use axum::response::Response; +use tracing::instrument; + +use crate::app_state::AppState; +use crate::domain::service::project_metadata_api_contract::ProjectMetadataApiContract; +use crate::error::DspMetaError; + +#[instrument(skip(state))] +pub async fn sitemap_xml( + State(state): State>, +) -> Result, DspMetaError> { + let base_url = state.base_url.clone(); + let mut xml = String::from("\n"); + xml.push_str("\n"); + xml.push_str( + format!( + "{}weekly\n", + base_url + ) + .as_str(), + ); + for meta in state.project_metadata_service.find_all()? { + let mut url = base_url.to_string() + "projects/"; + url.push_str(&meta.project.shortcode.as_string()); + let line = format!( + "{}weekly\n", + url + ); + xml.push_str(line.as_str()); + } + xml.push_str("\n"); + + let resp = Response::builder() + .status(StatusCode::OK) + .header("Content-Type", "application/xml") + .body(xml) + .expect("Failed to build response"); + Ok(resp) +} diff --git a/dsp-meta/src/api/handler/v1/mod.rs b/dsp-meta/src/api/handler/v1/mod.rs new file mode 100644 index 00000000..b42e1c67 --- /dev/null +++ b/dsp-meta/src/api/handler/v1/mod.rs @@ -0,0 +1 @@ +pub mod projects; diff --git a/dsp-meta/src/api/handler/project_metadata_handler.rs b/dsp-meta/src/api/handler/v1/projects/handlers.rs similarity index 91% rename from dsp-meta/src/api/handler/project_metadata_handler.rs rename to dsp-meta/src/api/handler/v1/projects/handlers.rs index eb1fea54..85e52e1f 100644 --- a/dsp-meta/src/api/handler/project_metadata_handler.rs +++ b/dsp-meta/src/api/handler/v1/projects/handlers.rs @@ -6,7 +6,9 @@ use axum::response::{IntoResponse, Response}; use axum::Json; use tracing::{instrument, trace}; -use crate::api::model::project_metadata_dto::{ProjectMetadataDto, ProjectMetadataWithInfoDto}; +use crate::api::handler::v1::projects::responses::{ + ProjectMetadataDto, ProjectMetadataWithInfoDto, +}; use crate::app_state::AppState; use crate::domain::model::draft_model::Shortcode; use crate::domain::service::project_metadata_api_contract::ProjectMetadataApiContract; @@ -18,7 +20,7 @@ use crate::error::DspMetaError; /// /// TODO: Add error handling with correct status codes #[instrument(skip(state))] -pub async fn get_project_metadata_by_shortcode( +pub async fn get_by_shortcode( Path(shortcode): Path, State(state): State>, ) -> Result { @@ -37,7 +39,7 @@ pub async fn get_project_metadata_by_shortcode( } #[instrument(skip(state))] -pub async fn get_all_project_metadata( +pub async fn get_by_page_and_filter( State(state): State>, pagination: Option>, filter: Option>, diff --git a/dsp-meta/src/api/handler/v1/projects/mod.rs b/dsp-meta/src/api/handler/v1/projects/mod.rs new file mode 100644 index 00000000..6108acf9 --- /dev/null +++ b/dsp-meta/src/api/handler/v1/projects/mod.rs @@ -0,0 +1,2 @@ +pub mod handlers; +pub mod responses; diff --git a/dsp-meta/src/api/model/project_metadata_dto.rs b/dsp-meta/src/api/handler/v1/projects/responses.rs similarity index 100% rename from dsp-meta/src/api/model/project_metadata_dto.rs rename to dsp-meta/src/api/handler/v1/projects/responses.rs diff --git a/dsp-meta/src/api/mod.rs b/dsp-meta/src/api/mod.rs index 1c78b6d7..f984e2d6 100644 --- a/dsp-meta/src/api/mod.rs +++ b/dsp-meta/src/api/mod.rs @@ -1,4 +1,3 @@ pub mod convert; mod handler; -mod model; pub mod router; diff --git a/dsp-meta/src/api/model/mod.rs b/dsp-meta/src/api/model/mod.rs deleted file mode 100644 index 97b864c9..00000000 --- a/dsp-meta/src/api/model/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod project_metadata_dto; -pub mod project_metadata_graph_dto; diff --git a/dsp-meta/src/api/model/project_metadata_graph_dto.rs b/dsp-meta/src/api/model/project_metadata_graph_dto.rs deleted file mode 100644 index 624541bf..00000000 --- a/dsp-meta/src/api/model/project_metadata_graph_dto.rs +++ /dev/null @@ -1,18 +0,0 @@ -use axum::http::StatusCode; -use axum::response::{IntoResponse, Response}; - -use crate::api::convert::rdf::project_metadata::ProjectMetadataGraph; - -pub struct ProjectMetadataGraphDto(pub Option); - -/// Convert `ProjectMetadataGraph` into a response. -impl IntoResponse for ProjectMetadataGraphDto { - fn into_response(self) -> Response { - match self.0 { - Some(metadata_graph) => { - (StatusCode::OK, metadata_graph.to_turtle_string()).into_response() - } - None => StatusCode::NOT_FOUND.into_response(), - } - } -} diff --git a/dsp-meta/src/api/router.rs b/dsp-meta/src/api/router.rs index 14fad46b..938edb4f 100644 --- a/dsp-meta/src/api/router.rs +++ b/dsp-meta/src/api/router.rs @@ -13,7 +13,7 @@ use tower_http::services::{ServeDir, ServeFile}; use tower_http::trace::TraceLayer; use tracing::{error, info_span, warn, Span}; -use crate::api::handler::{health, project_metadata_handler}; +use crate::api::handler::{health, robots_txt, sitemap_xml, v1}; use crate::app_state::AppState; /// Having a function that produces our router makes it easy to call it from tests @@ -27,14 +27,16 @@ pub fn router(shared_state: Arc) -> Router { Router::new() .route( "/api/v1/projects", - get(project_metadata_handler::get_all_project_metadata), + get(v1::projects::handlers::get_by_page_and_filter), ) .route( "/api/v1/projects/:shortcode", - get(project_metadata_handler::get_project_metadata_by_shortcode), + get(v1::projects::handlers::get_by_shortcode), ) - .route("/health", get(health::health_handler)) + .route("/health", get(health::health)) .route("/version.txt", get(shared_state.version)) + .route("/robots.txt", get(robots_txt::robots_txt)) + .route("/sitemap.xml", get(sitemap_xml::sitemap_xml)) .fallback_service( ServeDir::new(shared_state.public_dir.as_str()).fallback(ServeFile::new(format!( "{}/index.html", @@ -84,9 +86,12 @@ mod tests { use axum::body::Body; use axum::http::StatusCode; - use http_body_util::BodyExt; // for `collect` - use tower::ServiceExt; // for `oneshot` and `ready` + use http_body_util::BodyExt; + // for `collect` + use tower::ServiceExt; + use url::Url; + // for `oneshot` and `ready` use super::*; use crate::domain::service::project_metadata_service::ProjectMetadataService; use crate::repo::service::project_metadata_repository::ProjectMetadataRepository; @@ -101,6 +106,7 @@ mod tests { )), public_dir: "".to_string(), version: "", + base_url: Url::parse("http://localhost:3000").unwrap(), }); let router = router(shared_state); @@ -133,6 +139,7 @@ mod tests { )), public_dir: "".to_string(), version: "", + base_url: Url::parse("http://localhost:3000").unwrap(), }); let router = router(shared_state); diff --git a/dsp-meta/src/app_state.rs b/dsp-meta/src/app_state.rs index cb4ad636..868ce50a 100644 --- a/dsp-meta/src/app_state.rs +++ b/dsp-meta/src/app_state.rs @@ -1,3 +1,5 @@ +use url::Url; + use crate::domain::service::project_metadata_service::ProjectMetadataService; use crate::repo::service::project_metadata_repository::ProjectMetadataRepository; @@ -6,4 +8,5 @@ pub struct AppState { pub project_metadata_service: ProjectMetadataService, pub public_dir: String, pub version: &'static str, + pub base_url: Url, } diff --git a/dsp-meta/src/domain/service/project_metadata_api_contract.rs b/dsp-meta/src/domain/service/project_metadata_api_contract.rs index fc3b577a..2c31d86c 100644 --- a/dsp-meta/src/domain/service/project_metadata_api_contract.rs +++ b/dsp-meta/src/domain/service/project_metadata_api_contract.rs @@ -4,6 +4,7 @@ use crate::error::DspMetaError; pub trait ProjectMetadataApiContract { fn find_by_id(&self, id: &Shortcode) -> Result, DspMetaError>; + fn find_all(&self) -> Result, DspMetaError>; fn find( &self, filter: &Filter, diff --git a/dsp-meta/src/domain/service/project_metadata_service.rs b/dsp-meta/src/domain/service/project_metadata_service.rs index be375f0f..32519d10 100644 --- a/dsp-meta/src/domain/service/project_metadata_service.rs +++ b/dsp-meta/src/domain/service/project_metadata_service.rs @@ -28,6 +28,10 @@ where self.repo.find_by_id(id) } + fn find_all(&self) -> Result, DspMetaError> { + self.repo.find_all() + } + #[instrument(skip(self))] fn find( &self, diff --git a/dsp-meta/src/domain/service/repository_contract.rs b/dsp-meta/src/domain/service/repository_contract.rs index 4dd15460..284ae79c 100644 --- a/dsp-meta/src/domain/service/repository_contract.rs +++ b/dsp-meta/src/domain/service/repository_contract.rs @@ -34,9 +34,12 @@ pub trait RepositoryContract { /// If the entity does not exist, `None` is returned. fn find_by_id(&self, id: &Id) -> Result, Error>; - /// Returns all entities. + /// Returns all entities with filter and pagination. fn find(&self, filter: &Filter, pagination: &Pagination) -> Result, Error>; + /// Returns all entities. + fn find_all(&self) -> Result, Error>; + /// Returns the number of entities. fn count(&self) -> Result; } diff --git a/dsp-meta/src/repo/service/project_metadata_repository.rs b/dsp-meta/src/repo/service/project_metadata_repository.rs index 7c8f90c9..815c8793 100644 --- a/dsp-meta/src/repo/service/project_metadata_repository.rs +++ b/dsp-meta/src/repo/service/project_metadata_repository.rs @@ -106,6 +106,12 @@ impl RepositoryContract for ProjectMetad Ok(Page { data, total }) } + fn find_all(&self) -> Result, DspMetaError> { + let db = self.db.read().unwrap(); + let v = db.iter().map(|(_, v)| v.clone()).collect(); + Ok(v) + } + fn count(&self) -> Result { let db = self.db.read().unwrap(); Ok(db.len()) diff --git a/web-frontend/src/Snackbar.svelte b/web-frontend/src/Snackbar.svelte index 3f786c12..eec1be5f 100644 --- a/web-frontend/src/Snackbar.svelte +++ b/web-frontend/src/Snackbar.svelte @@ -18,7 +18,7 @@ }) -
+
{$handleSnackbar.message}
diff --git a/web-frontend/src/project-page/ProjectPage.svelte b/web-frontend/src/project-page/ProjectPage.svelte index d3ff1446..d64c1ca8 100644 --- a/web-frontend/src/project-page/ProjectPage.svelte +++ b/web-frontend/src/project-page/ProjectPage.svelte @@ -94,7 +94,7 @@ {/if} {#if $projectMetadata} -
+
{#if mobileResolution}
{/if} -
+
{#if $pagedResults && $pagedResults.length} {#each $pagedResults as project}