Merge pull request #231 from earthpulse/225-improve-markdown-metadata…

…-workflow

improve markdown metadata workflow

eotdl/eotdl/datasets/ingest.py

@@ -3,10 +3,6 @@
 from tqdm import tqdm
 import json
 import frontmatter
-import markdown
-from markdown.extensions.tables import TableExtension 
-from markdown.extensions.fenced_code import FencedCodeExtension
-
 
 from ..auth import with_auth
 from .metadata import Metadata
@@ -75,8 +71,6 @@ def ingest_folder(
         raise Exception("Error loading metadata")
     # retrieve dataset (create if doesn't exist)
     dataset = retrieve_dataset(metadata, user)
-    if content:
-        content = markdown.markdown(content, extensions=[TableExtension(),FencedCodeExtension()])
 
     update_metadata = True
     if "description" in dataset:

eotdl/eotdl/datasets/metadata.py

@@ -1,6 +1,5 @@
 from pydantic import BaseModel, validator
 from typing import List, Optional
-import markdownify
 from pathlib import Path
 
 
@@ -39,7 +38,7 @@ def generate_metadata(download_path, dataset):
         for author in dataset["authors"]:
             f.write(f"  - {author}\n")
         f.write("---\n")
-        f.write(markdownify.markdownify(dataset["description"], heading_style="ATX"))
+        f.write(dataset["description"])
     # remove metadata.yml if exists
     if Path(download_path + "/metadata.yml").exists():
         Path(download_path + "/metadata.yml").unlink()

eotdl/eotdl/models/ingest.py

@@ -1,7 +1,6 @@
 from pathlib import Path
 import yaml
 import frontmatter
-import markdown
 from tqdm import tqdm
 import json
 
@@ -67,8 +66,7 @@ def ingest_folder(
         raise Exception(f"Error loading metadata: {e}")
     # retrieve model (create if doesn't exist)
     model = retrieve_model(metadata, user)
-    if content:
-        content = markdown.markdown(content)
+
     update_metadata = True
     if "description" in model:
         # do not do this if the model is new, only if it already exists

eotdl/eotdl/models/metadata.py

@@ -1,6 +1,5 @@
 from pydantic import BaseModel, validator
 from typing import List, Optional
-import markdownify
 from pathlib import Path
 
 
@@ -37,7 +36,7 @@ def generate_metadata(download_path, model):
         for author in model["authors"]:
             f.write(f"  - {author}\n")
         f.write("---\n")
-        f.write(markdownify.markdownify(model["description"], heading_style="ATX"))
+        f.write(model["description"])
     # remove metadata.yml if exists
     if Path(download_path + "/metadata.yml").exists():
         Path(download_path + "/metadata.yml").unlink()

ui/src/routes/datasets/IngestForm.svelte

@@ -16,12 +16,15 @@
   export let license;
   export let content;
   export let quality = 0;
+  let value = content
+  console.log(value)
   let files;
   let loading = false;
   let error = null;
   $: selected_tags = current_tags;
 
   const ingest = async () => {
+    console.log(value)
     error = null;
     if (source && !validate_source(source)) return;
     loading = true;
@@ -32,7 +35,7 @@
         await submit(
           files,
           name,
-          content,
+          value,
           authors?.split(","),
           source,
           license,
@@ -43,7 +46,7 @@
         await submit(
           //files,
           name,
-          content,
+          value,
           authors?.split(","),
           source,
           license,
@@ -156,7 +159,7 @@
       </span>
     {/if}
     <p>Description</p>
-    <TextEditor bind:content />
+    <TextEditor bind:value />
     {#if forCreate}
       <p>Files</p>
       <input id="uploadfiles" class="hidden" bind:files={files} type="file" multiple directory webkitdirectory/>

ui/src/routes/datasets/TextEditor.svelte

@@ -3,8 +3,6 @@
   import "carta-md/default.css"; /* Default theme */
   import "$styles/carta-md.css";
   import DOMPurify from "isomorphic-dompurify";
-  import TurndownService from "turndown";
-  import { tables } from "turndown-plugin-gfm";
   import { browser } from "$app/environment";
 
   let x;
@@ -31,16 +29,12 @@
     sanitizer: DOMPurify.sanitize,
   });
 
-  export let content;
-  var turndownService = new TurndownService({
-    codeBlockStyle: "fenced",
-    preformattedCode: true,
-  });
-  turndownService.use(tables);
+  export let value;
+
+  let html = null;
 
-  let value = turndownService.turndown(content);
   const renderHtml = async () => {
-    content = await carta.render(value);
+    html = await carta.render(value);
   };
 </script>
 

ui/src/routes/datasets/[name]/+page.svelte

@@ -11,15 +11,27 @@
 	import { fade } from "svelte/transition";
 	import retrieveDatasetFiles from "$lib/datasets/retrieveDatasetFiles";
 	import Map from "$components/Map.svelte";
+	import { Carta } from "carta-md";
+	import DOMPurify from "isomorphic-dompurify";
+
+	const carta = new Carta({
+		extensions: [],
+		sanitizer: DOMPurify.sanitize,
+	});
 
 	export let data;
 
 	let dataset = null;
 	let version = null;
 	let message = null;
+	let description = null;
 
 	const load = async () => {
 		dataset = await retrieveDataset(data.name);
+		description =  await carta.render(dataset.description)
+		if (!description){
+			description = dataset.description;
+		}
 	};
 
 	$: if (browser) load();
@@ -106,8 +118,8 @@
 			>
 				<div class="w-full overflow-auto">
 					<div class="content">
-						{#if dataset.description}
-							{@html dataset.description}
+						{#if description}
+							{@html description}
 						{:else}
 							<p class="italic">No description.</p>
 						{/if}

ui/src/routes/models/[name]/+page.svelte

@@ -10,15 +10,27 @@
 	import { fade } from "svelte/transition";
 	import retrieveModelFiles from "$lib/models/retrieveModelFiles";
 	import Update from "$components/Update.svelte";
+	import { Carta } from "carta-md";
+	import DOMPurify from "isomorphic-dompurify";
+
+	const carta = new Carta({
+		extensions: [],
+		sanitizer: DOMPurify.sanitize,
+	});
 
 	export let data;
 
 	let model = null;
 	let version = null;
 	let message = null;
+	let description = null;
 
 	const load = async () => {
 		model = await retrieveModel(data.name);
+		description =  await carta.render(model.description)
+		if (!description){
+			description = model.description;
+		}
 	};
 
 	$: if (browser) load();
@@ -94,8 +106,8 @@
 			<div class="grid grid-cols-[auto,350px] gap-3 mt-5">
 				<div class="w-full overflow-auto">
 					<div class="content">
-						{#if model.description}
-							{@html model.description}
+						{#if description}
+							{@html description}
 						{:else}
 							<p class="italic">No description.</p>
 						{/if}