feat: Add robots.txt and sitemap.xml (DEV-3931) (#152)

dsp-meta-cmd/src/main-server.rs

@@ -12,6 +12,7 @@ use tokio::net::TcpListener;
 use tracing::info;
 use tracing_subscriber::prelude::*;
 use tracing_subscriber::{fmt, EnvFilter};
+use url::Url;
 
 fn main() {
     // Do the pid1 magic. Needs to be the first thing executed.
@@ -64,12 +65,17 @@ async fn init_server() {
         .get::<String>("public_dir")
         .unwrap_or("/public".to_string());
 
+    let base_url = settings
+        .get::<Url>("base_url")
+        .unwrap_or(Url::parse("http://localhost:3000").unwrap());
+
     let shared_state = Arc::new(AppState {
         project_metadata_service: ProjectMetadataService::new(ProjectMetadataRepository::new(
             Path::new(&data_dir),
         )),
         public_dir,
         version: VERSION,
+        base_url,
     });
 
     // start the server

dsp-meta/src/api/handler/health.rs

@@ -1,6 +1,6 @@
 use tracing::trace;
 
-pub(crate) async fn health_handler() -> &'static str {
+pub(crate) async fn health() -> &'static str {
     trace!("entered health_handler()");
     "healthy"
 }

dsp-meta/src/api/handler/mod.rs

@@ -1,2 +1,4 @@
 pub mod health;
-pub mod project_metadata_handler;
+pub mod robots_txt;
+pub mod sitemap_xml;
+pub mod v1;

dsp-meta/src/api/handler/robots_txt.rs

@@ -0,0 +1,26 @@
+use std::sync::Arc;
+
+use axum::extract::State;
+use axum::http::{Response, StatusCode};
+
+use crate::app_state::AppState;
+use crate::error::DspMetaError;
+
+pub async fn robots_txt(
+    State(state): State<Arc<AppState>>,
+) -> Result<Response<String>, DspMetaError> {
+    let sitemap_xml = state
+        .base_url
+        .join("sitemap.xml")
+        .expect("valid url")
+        .to_string();
+    let response = Response::builder()
+        .status(StatusCode::OK)
+        .header("Content-Type", "text/plain")
+        .body(format!(
+            "Sitemap: {}\nUser-agent: *\nDisallow:\n",
+            sitemap_xml
+        ))
+        .expect("Failed to build response");
+    Ok(response)
+}

dsp-meta/src/api/handler/sitemap_xml.rs

@@ -0,0 +1,43 @@
+use std::sync::Arc;
+
+use axum::extract::State;
+use axum::http::StatusCode;
+use axum::response::Response;
+use tracing::instrument;
+
+use crate::app_state::AppState;
+use crate::domain::service::project_metadata_api_contract::ProjectMetadataApiContract;
+use crate::error::DspMetaError;
+
+#[instrument(skip(state))]
+pub async fn sitemap_xml(
+    State(state): State<Arc<AppState>>,
+) -> Result<Response<String>, DspMetaError> {
+    let base_url = state.base_url.clone();
+    let mut xml = String::from("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+    xml.push_str("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n");
+    xml.push_str(
+        format!(
+            "<url><loc>{}</loc><changefreq>weekly</changefreq></url>\n",
+            base_url
+        )
+        .as_str(),
+    );
+    for meta in state.project_metadata_service.find_all()? {
+        let mut url = base_url.to_string() + "projects/";
+        url.push_str(&meta.project.shortcode.as_string());
+        let line = format!(
+            "<url><loc>{}</loc><changefreq>weekly</changefreq></url>\n",
+            url
+        );
+        xml.push_str(line.as_str());
+    }
+    xml.push_str("</urlset>\n");
+
+    let resp = Response::builder()
+        .status(StatusCode::OK)
+        .header("Content-Type", "application/xml")
+        .body(xml)
+        .expect("Failed to build response");
+    Ok(resp)
+}

dsp-meta/src/api/handler/v1/mod.rs

@@ -0,0 +1 @@
+pub mod projects;

dsp-meta/src/api/handler/project_metadata_handler.rs → dsp-meta/src/api/handler/v1/projects/handlers.rs

@@ -6,7 +6,9 @@ use axum::response::{IntoResponse, Response};
 use axum::Json;
 use tracing::{instrument, trace};
 
-use crate::api::model::project_metadata_dto::{ProjectMetadataDto, ProjectMetadataWithInfoDto};
+use crate::api::handler::v1::projects::responses::{
+    ProjectMetadataDto, ProjectMetadataWithInfoDto,
+};
 use crate::app_state::AppState;
 use crate::domain::model::draft_model::Shortcode;
 use crate::domain::service::project_metadata_api_contract::ProjectMetadataApiContract;
@@ -18,7 +20,7 @@ use crate::error::DspMetaError;
 ///
 /// TODO: Add error handling with correct status codes
 #[instrument(skip(state))]
-pub async fn get_project_metadata_by_shortcode(
+pub async fn get_by_shortcode(
     Path(shortcode): Path<Shortcode>,
     State(state): State<Arc<AppState>>,
 ) -> Result<Response, DspMetaError> {
@@ -37,7 +39,7 @@ pub async fn get_project_metadata_by_shortcode(
 }
 
 #[instrument(skip(state))]
-pub async fn get_all_project_metadata(
+pub async fn get_by_page_and_filter(
     State(state): State<Arc<AppState>>,
     pagination: Option<Query<Pagination>>,
     filter: Option<Query<Filter>>,

dsp-meta/src/api/handler/v1/projects/mod.rs

@@ -0,0 +1,2 @@
+pub mod handlers;
+pub mod responses;

dsp-meta/src/api/mod.rs

@@ -1,4 +1,3 @@
 pub mod convert;
 mod handler;
-mod model;
 pub mod router;

dsp-meta/src/api/router.rs

@@ -13,7 +13,7 @@ use tower_http::services::{ServeDir, ServeFile};
 use tower_http::trace::TraceLayer;
 use tracing::{error, info_span, warn, Span};
 
-use crate::api::handler::{health, project_metadata_handler};
+use crate::api::handler::{health, robots_txt, sitemap_xml, v1};
 use crate::app_state::AppState;
 
 /// Having a function that produces our router makes it easy to call it from tests
@@ -27,14 +27,16 @@ pub fn router(shared_state: Arc<AppState>) -> Router {
     Router::new()
         .route(
             "/api/v1/projects",
-            get(project_metadata_handler::get_all_project_metadata),
+            get(v1::projects::handlers::get_by_page_and_filter),
         )
         .route(
             "/api/v1/projects/:shortcode",
-            get(project_metadata_handler::get_project_metadata_by_shortcode),
+            get(v1::projects::handlers::get_by_shortcode),
         )
-        .route("/health", get(health::health_handler))
+        .route("/health", get(health::health))
         .route("/version.txt", get(shared_state.version))
+        .route("/robots.txt", get(robots_txt::robots_txt))
+        .route("/sitemap.xml", get(sitemap_xml::sitemap_xml))
         .fallback_service(
             ServeDir::new(shared_state.public_dir.as_str()).fallback(ServeFile::new(format!(
                 "{}/index.html",
@@ -84,9 +86,12 @@ mod tests {
 
     use axum::body::Body;
     use axum::http::StatusCode;
-    use http_body_util::BodyExt; // for `collect`
-    use tower::ServiceExt; // for `oneshot` and `ready`
+    use http_body_util::BodyExt;
+    // for `collect`
+    use tower::ServiceExt;
+    use url::Url;
 
+    // for `oneshot` and `ready`
     use super::*;
     use crate::domain::service::project_metadata_service::ProjectMetadataService;
     use crate::repo::service::project_metadata_repository::ProjectMetadataRepository;
@@ -101,6 +106,7 @@ mod tests {
             )),
             public_dir: "".to_string(),
             version: "",
+            base_url: Url::parse("http://localhost:3000").unwrap(),
         });
 
         let router = router(shared_state);
@@ -133,6 +139,7 @@ mod tests {
             )),
             public_dir: "".to_string(),
             version: "",
+            base_url: Url::parse("http://localhost:3000").unwrap(),
         });
 
         let router = router(shared_state);

dsp-meta/src/app_state.rs

@@ -1,3 +1,5 @@
+use url::Url;
+
 use crate::domain::service::project_metadata_service::ProjectMetadataService;
 use crate::repo::service::project_metadata_repository::ProjectMetadataRepository;
 
@@ -6,4 +8,5 @@ pub struct AppState {
     pub project_metadata_service: ProjectMetadataService<ProjectMetadataRepository>,
     pub public_dir: String,
     pub version: &'static str,
+    pub base_url: Url,
 }

dsp-meta/src/domain/service/project_metadata_api_contract.rs

@@ -4,6 +4,7 @@ use crate::error::DspMetaError;
 
 pub trait ProjectMetadataApiContract {
     fn find_by_id(&self, id: &Shortcode) -> Result<Option<DraftMetadata>, DspMetaError>;
+    fn find_all(&self) -> Result<Vec<DraftMetadata>, DspMetaError>;
     fn find(
         &self,
         filter: &Filter,

dsp-meta/src/domain/service/project_metadata_service.rs

@@ -28,6 +28,10 @@ where
         self.repo.find_by_id(id)
     }
 
+    fn find_all(&self) -> Result<Vec<DraftMetadata>, DspMetaError> {
+        self.repo.find_all()
+    }
+
     #[instrument(skip(self))]
     fn find(
         &self,

dsp-meta/src/domain/service/repository_contract.rs

@@ -34,9 +34,12 @@ pub trait RepositoryContract<Entity, Id, Error> {
     /// If the entity does not exist, `None` is returned.
     fn find_by_id(&self, id: &Id) -> Result<Option<Entity>, Error>;
 
-    /// Returns all entities.
+    /// Returns all entities with filter and pagination.
     fn find(&self, filter: &Filter, pagination: &Pagination) -> Result<Page<Entity>, Error>;
 
+    /// Returns all entities.
+    fn find_all(&self) -> Result<Vec<Entity>, Error>;
+
     /// Returns the number of entities.
     fn count(&self) -> Result<usize, Error>;
 }

dsp-meta/src/repo/service/project_metadata_repository.rs

@@ -106,6 +106,12 @@ impl RepositoryContract<DraftMetadata, Shortcode, DspMetaError> for ProjectMetad
         Ok(Page { data, total })
     }
 
+    fn find_all(&self) -> Result<Vec<DraftMetadata>, DspMetaError> {
+        let db = self.db.read().unwrap();
+        let v = db.iter().map(|(_, v)| v.clone()).collect();
+        Ok(v)
+    }
+
     fn count(&self) -> Result<usize, DspMetaError> {
         let db = self.db.read().unwrap();
         Ok(db.len())

web-frontend/src/Snackbar.svelte

@@ -18,7 +18,7 @@
   })
 </script>
 
-<div in:fade={{duration: 250}}>
+<div transition:fade={{duration: 400}}>
   {$handleSnackbar.message}
 </div>
 

web-frontend/src/project-page/ProjectPage.svelte

@@ -94,7 +94,7 @@
 {/if}
 
 {#if $projectMetadata}
-  <div class="container" in:fade={{ duration: 200 }}>
+  <div class="container" in:fade={{duration: 250}}>
     {#if mobileResolution}
       <button on:click={() => history.back()} class=goback-button title="go back to the projects list" disabled={!$previousRoute && window.history.length <= 2}>
         <svg class=icon fill="none" stroke="currentColor" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">

web-frontend/src/projects-repository/ProjectsRepository.svelte

@@ -51,7 +51,7 @@
   </div>
 {/if}
 
-<main in:fade="{{duration: 200}}">
+<main in:fade={{duration: 250}}>
   <div class=tile-container>
     {#if $pagedResults && $pagedResults.length}
       {#each $pagedResults as project}