From 8cc6bda5f69f6ff0fc44ae8682faf534c275b67a Mon Sep 17 00:00:00 2001
From: Fernando Hernandez <ferndohm99emf@gmail.com>
Date: Tue, 27 Aug 2024 12:46:33 +0200
Subject: [PATCH] added model ingestion through ui

---
 ui/src/components/Create.svelte          |  78 ---------
 ui/src/lib/models/createModel.js         |  10 ++
 ui/src/lib/models/ingestFile.js          |  12 ++
 ui/src/lib/models/setModelVersion.js     |   9 ++
 ui/src/routes/datasets/IngestForm.svelte |   2 +-
 ui/src/routes/models/+page.svelte        |   7 +-
 ui/src/routes/models/Ingest.svelte       |  72 +++++++++
 ui/src/routes/models/IngestForm.svelte   | 198 +++++++++++++++++++++++
 ui/src/routes/models/TextEditor.svelte   |  56 +++++++
 ui/src/stores/models.js                  |  28 +++-
 10 files changed, 387 insertions(+), 85 deletions(-)
 delete mode 100644 ui/src/components/Create.svelte
 create mode 100644 ui/src/lib/models/createModel.js
 create mode 100644 ui/src/lib/models/ingestFile.js
 create mode 100644 ui/src/lib/models/setModelVersion.js
 create mode 100644 ui/src/routes/models/Ingest.svelte
 create mode 100644 ui/src/routes/models/IngestForm.svelte
 create mode 100644 ui/src/routes/models/TextEditor.svelte

diff --git a/ui/src/components/Create.svelte b/ui/src/components/Create.svelte
deleted file mode 100644
index 199f6bf8..00000000
--- a/ui/src/components/Create.svelte
+++ /dev/null
@@ -1,78 +0,0 @@
-<script>
-	import { user, id_token } from "$stores/auth";
-	import IngestForm from "../routes/datasets/IngestForm.svelte";
-	import { goto } from "$app/navigation";
-
-	export let tags;
-	export let id;
-	export let current_tags;
-	export let name;
-	export let authors;
-	export let license;
-	export let description;
-	export let source;
-	export let selected_tags;
-	export let quality;
-	export let store;
-	export let route;
-	export let version;
-	const submit = async (
-		_name,
-		content,
-		_authors,
-		_source,
-		_license,
-		_selected_tags
-	) => {
-		// const current = $datasets.data.find((d) => d.id == id);
-		// if (current.name == name) name = null;
-		const data = await store.update(
-			id,
-			_name,
-			content,
-			_authors,
-			_source,
-			_license,
-			_selected_tags,
-			$id_token
-		);
-		if (_name) name = _name;
-		if (_authors) authors = _authors;
-		if (_source) source = _source;
-		if (_license) license = _license;
-		if (content) description = content;
-		selected_tags = _selected_tags;
-		if (name) goto(`/${route}/${name}`, { replaceState: true });
-	};
-	const uploadFile = async (_file) => {
-		const data = await store.ingest(
-			id,
-			_file,
-			$id_token,
-			version.version_id,
-			name)
-	}
-</script>
-
-{#if $user}
-	<IngestForm
-		{tags}
-		{submit}
-		{uploadFile}
-        forCreate = {true}
-		text="EDIT"
-		{current_tags}
-		content={description}
-		{authors}
-		{source}
-		{license}
-		{name}
-		{quality}
-	>
-		<h3 class="text-lg font-bold">Edit</h3>
-		<!-- <p>
-			⚠ You can overwrite existing files by uploading a new file with the
-			same name.
-		</p> -->
-	</IngestForm>
-{/if}
\ No newline at end of file
diff --git a/ui/src/lib/models/createModel.js b/ui/src/lib/models/createModel.js
new file mode 100644
index 00000000..7bbc1681
--- /dev/null
+++ b/ui/src/lib/models/createModel.js
@@ -0,0 +1,10 @@
+import { PUBLIC_EOTDL_API } from '$env/static/public';
+import fetchEOTDL from '$lib/shared/fetchEOTDL';
+
+export default async (name, authors, source, license, token) => {
+	const url = `${PUBLIC_EOTDL_API}/models`;
+	const body = {name, authors, license, source};
+	const { data, error } = await fetchEOTDL(url, token, 'POST', body);
+	if (error) throw new Error(error);
+	return data;
+};
diff --git a/ui/src/lib/models/ingestFile.js b/ui/src/lib/models/ingestFile.js
new file mode 100644
index 00000000..a15f1f95
--- /dev/null
+++ b/ui/src/lib/models/ingestFile.js
@@ -0,0 +1,12 @@
+import { PUBLIC_EOTDL_API } from '$env/static/public';
+import EOTDLFileUpload from '$lib/shared/EOTDLFileUpload';
+import fileChecksum from '../datasets/fileChecksum';
+export default async (model_id, file, token, version) => {
+	const url = `${PUBLIC_EOTDL_API}/models/${model_id}?version=${version.version}`;
+	const formData = new FormData();
+	formData.append("file", file);
+	formData.append("checksum", fileChecksum(await file.arrayBuffer()));
+	const { data, error } = await EOTDLFileUpload(url, token, formData);
+	if (error) throw new Error(error);
+	return data;
+};
diff --git a/ui/src/lib/models/setModelVersion.js b/ui/src/lib/models/setModelVersion.js
new file mode 100644
index 00000000..ac808631
--- /dev/null
+++ b/ui/src/lib/models/setModelVersion.js
@@ -0,0 +1,9 @@
+import { PUBLIC_EOTDL_API } from '$env/static/public';
+import fetchEOTDL from '$lib/shared/fetchEOTDL';
+
+export default async (model_id, token) => {
+	const url = `${PUBLIC_EOTDL_API}/models/version/${model_id}`;
+	const { data, error } = await fetchEOTDL(url, token, 'POST');
+	if (error) throw new Error(error);
+	return data;
+};
\ No newline at end of file
diff --git a/ui/src/routes/datasets/IngestForm.svelte b/ui/src/routes/datasets/IngestForm.svelte
index 7ffd08d7..a2692a24 100755
--- a/ui/src/routes/datasets/IngestForm.svelte
+++ b/ui/src/routes/datasets/IngestForm.svelte
@@ -188,7 +188,7 @@
       >
       <button
         class="btn btn-ghost btn-outline {loading && 'loading'}"
-        type="submit">Update</button
+        type="submit">Ingest</button
       >
     </span>
     {#if error}
diff --git a/ui/src/routes/models/+page.svelte b/ui/src/routes/models/+page.svelte
index 061d1e96..e8fe1412 100755
--- a/ui/src/routes/models/+page.svelte
+++ b/ui/src/routes/models/+page.svelte
@@ -9,7 +9,7 @@
 	import Skeleton from "$components/Skeleton.svelte";
 	import ModelsLeaderboard from "../ModelsLeaderboard.svelte";
 	import QualitySelector from "$components/QualitySelector.svelte";
-
+	import Ingest from "./Ingest.svelte";
 	export let data;
 
 	let loading = true;
@@ -103,14 +103,11 @@
 					>
 					<QualitySelector bind:selected_qualities />
 				</span>
-				<!-- <Ingest tags={data?.tags} /> -->
 			</div>
 			<Tags tags={data?.tags} bind:selected_tags {onToggleTag} />
 		</div>
 		<span class="flex flex-row w-full justify-between items-center">
-			<a href="/docs/models/ingest" class="text-green-200 hover:underline"
-				>Ingest model</a
-			>
+			<Ingest tags={data?.tags} />
 		</span>
 		{#if loading}
 			<div class="grid grid-cols-1 sm:grid-cols-3 gap-3 w-full mt-3">
diff --git a/ui/src/routes/models/Ingest.svelte b/ui/src/routes/models/Ingest.svelte
new file mode 100644
index 00000000..b66d5a54
--- /dev/null
+++ b/ui/src/routes/models/Ingest.svelte
@@ -0,0 +1,72 @@
+<script>
+    import { user, id_token } from "$stores/auth";
+    import { models } from "../../stores/models";
+    import IngestForm from "./IngestForm.svelte";
+
+    export let tags;
+
+    const submit = async (
+        files,
+        name,
+        content,
+        authors,
+        source,
+        license,
+        selected_tags
+    ) => {
+        if (
+            !name ||
+            files?.length == 0 ||
+            authors.length == 0 ||
+            !license ||
+            !source ||
+            !content ||
+            content.length == 0
+        )
+            throw new Error("Please fill in all fields");
+        let data;
+        if (files.length == 0) throw new Error("Please upload a file");
+        const modelExists = $models.data.find(
+            (d) => d.name == name && d.uid == $user.uid
+        );
+        if (modelExists)
+            throw new Error("Model already exists, choose a different name");
+        const model_id = await models.create(
+            name,
+            authors,
+            source,
+            license,
+            $id_token
+        );
+        const version = await models.setVersion(model_id, $id_token);
+        for (var i = 0; i < files.length; i++) {
+            data = await models.ingest(model_id, files[i], $id_token, version, name);
+        }
+        await models.update(
+            model_id,
+            name,
+            content,
+            authors,
+            source,
+            license,
+            selected_tags,
+            $id_token
+        );
+    };
+</script>
+
+{#if $user}
+    <IngestForm 
+    {tags} 
+    {submit} 
+    text="Ingest model" 
+    forCreate={true} 
+    required={true}
+    name={""}
+    authors={""}
+    source={""}
+    license={""}
+    content={""}>
+        <h3 class="text-lg font-bold">Ingest model</h3>
+    </IngestForm>
+{/if}
diff --git a/ui/src/routes/models/IngestForm.svelte b/ui/src/routes/models/IngestForm.svelte
new file mode 100644
index 00000000..922cefcb
--- /dev/null
+++ b/ui/src/routes/models/IngestForm.svelte
@@ -0,0 +1,198 @@
+<script>
+  import Code from "../docs/components/Code.svelte";
+  import CLI from "../docs/components/CLI.svelte";
+  import TextEditor from "./TextEditor.svelte";
+  // import formatFileSize from "../../lib/datasets/formatFileSize.js";
+
+  export let tags;
+  export let submit;
+  export let text;
+  export let forCreate = false;
+  export let required = false;
+  export let current_tags = [];
+  export let name;
+  export let authors;
+  export let source;
+  export let license;
+  export let content;
+  export let quality = 0;
+  let files;
+  let loading = false;
+  let error = null;
+  $: selected_tags = current_tags;
+
+  const ingest = async () => {
+    error = null;
+    if (source && !validate_source(source)) return;
+    loading = true;
+    try {
+      if (authors instanceof Array) authors = authors.join(",");
+      if (forCreate){
+        if (!validateModelSize(files)) return;
+        await submit(
+          files,
+          name,
+          content,
+          authors?.split(","),
+          source,
+          license,
+          selected_tags,
+        );
+      }
+      else {
+        await submit(
+          //files,
+          name,
+          content,
+          authors?.split(","),
+          source,
+          license,
+          selected_tags,
+        );
+      }
+      document.getElementById("ingest-model").checked = false;
+      // name = "";
+      // authors = "";
+      // source = "";
+      // license = "";
+      // content = "";
+      // files = null;
+      // input.value = ""; // reset files in input
+      // selected_tags = [];
+    } catch (e) {
+      // alert(e.message);
+      error = e.message;
+    }
+    loading = false;
+  };
+
+  const validateModelSize = (files) => {
+    let totalSize = 0;
+    for (let i = 0; i < files.length; i++) {
+      console.log(files[i].size)
+      totalSize += files[i].size;
+      console.log(totalSize);
+    }
+    console.log(totalSize);
+    if (totalSize < 500 * 1024 * 1024) return true; //max 500MB
+    else {
+      alert("Size must be les than 500MB, current size: " +Math.round(totalSize/1024/1024*100)/100 + "MB");
+      return false;
+    } 
+  }
+  const validate_source = (link) => {
+    if (!link.startsWith("http://") && !link.startsWith("https://")) {
+      alert("Link should start with http:// or https://");
+      return false;
+    }
+    return true;
+  };
+
+  const toggleTag = (tag) => {
+    if (selected_tags.includes(tag)) {
+      selected_tags = selected_tags.filter((t) => t !== tag);
+    } else {
+      selected_tags = [...selected_tags, tag];
+    }
+  };
+</script>
+
+<label for="ingest-model" class="btn btn-ghost btn-outline">{text}</label>
+<input type="checkbox" id="ingest-model" class="modal-toggle" />
+<div class="modal">
+  <form
+    on:submit|preventDefault={ingest}
+    class="flex flex-col gap-2 text-sm modal-box w-[95%] sm:w-[75%] max-w-none"
+  >
+    <slot />
+    <span>
+      <p>Name</p>
+      <input
+        class="w-full input input-bordered"
+        type="text"
+        placeholder="Model name"
+        {required}
+        bind:value={name}
+      />
+      <p class="text-sm text-gray-400">*Name should be unique</p>
+    </span>
+    {#if quality == 0}
+      <span>
+        <p>Authors (use comma for multiple authors)</p>
+        <input
+          class="w-full input input-bordered"
+          type="text"
+          placeholder="Model authors"
+          {required}
+          bind:value={authors}
+        />
+        <p class="text-sm text-gray-400">
+          *If you are not the author, provide the correct attribution
+        </p>
+      </span>
+      <span>
+        <p>Link to source data</p>
+        <input
+          class="w-full input input-bordered"
+          type="text"
+          placeholder="Link to source data"
+          {required}
+          bind:value={source}
+        />
+        <p class="text-sm text-gray-400">
+          *Link to the original source of the data.
+        </p>
+      </span>
+      <span>
+        <p>License</p>
+        <input
+          class="w-full input input-bordered"
+          type="text"
+          placeholder="License"
+          {required}
+          bind:value={license}
+        />
+        <p class="text-sm text-gray-400">*Provide a license for the model.</p>
+      </span>
+    {/if}
+    <p>Description</p>
+    <TextEditor bind:content />
+    {#if forCreate}
+      <p>Files</p>
+      <input id="uploadfiles" class="hidden" bind:files={files} type="file" multiple directory webkitdirectory/>
+      <label for="uploadfiles" class="btn btn-outline btn-ghost">Select the model directory</label>
+      {#if files}        
+        <p class="text-sm text-gray-400">Total files: {files.length}</p>
+      {/if}
+    {/if}
+    <p>Select the appropriate tags:</p>
+    <div class="flex flex-wrap gap-1">
+      {#each tags as tag}
+        <p
+          class="badge cursor-pointer text-slate-400 text-xs {selected_tags.includes(
+            tag.name,
+          )
+            ? 'text-slate-200 border-0'
+            : 'badge-outline'}"
+          on:click={() => toggleTag(tag.name)}
+          on:keyup={() => {}}
+          style={`${selected_tags.includes(tag.name) && `background-color: ${tag.color};`}`}
+        >
+          {tag.name}
+        </p>
+      {/each}
+    </div>
+    <span class="self-end">
+      <label for="ingest-model" class="btn btn-ghost btn-outline btn-error"
+        >Close</label
+      >
+      <button
+        class="btn btn-ghost btn-outline {loading && 'loading'}"
+        type="submit">Ingest</button
+      >
+    </span>
+    {#if error}
+      <p class="text-sm text-error">{error}</p>
+    {/if}
+  </form>
+</div>
diff --git a/ui/src/routes/models/TextEditor.svelte b/ui/src/routes/models/TextEditor.svelte
new file mode 100644
index 00000000..b081ea55
--- /dev/null
+++ b/ui/src/routes/models/TextEditor.svelte
@@ -0,0 +1,56 @@
+<script>
+  import { Carta, MarkdownEditor } from "carta-md";
+  import "carta-md/default.css"; /* Default theme */
+  import "$styles/carta-md.css";
+  import DOMPurify from "isomorphic-dompurify";
+  import TurndownService from "turndown";
+  import { tables } from "turndown-plugin-gfm";
+  import { browser } from "$app/environment";
+
+  let x;
+  let sm;
+
+  $: if (browser) {
+    x = window?.matchMedia("(max-width: 640px)");
+    let sm = x.matches ? true : false;
+    let normal = x.matches ? false : true;
+    // Attach listener function on state changes
+    x.addEventListener("change", function () {
+      if (x.matches) {
+        sm = true;
+        normal = false;
+      } else {
+        sm = false;
+        normal = true;
+      }
+    });
+  }
+
+  const carta = new Carta({
+    extensions: [],
+    sanitizer: DOMPurify.sanitize,
+  });
+
+  export let content;
+  var turndownService = new TurndownService({
+    codeBlockStyle: "fenced",
+    preformattedCode: true,
+  });
+  turndownService.use(tables);
+
+  let value = turndownService.turndown(content);
+  const renderHtml = async () => {
+    content = await carta.render(value);
+  };
+</script>
+
+<div class="flex justify-center">
+  <div
+    on:change={renderHtml}
+    class="w-[62rem] flex flex-col items-center justify-center p-2 rounded-xl"
+  >
+    <MarkdownEditor mode={sm ? "tabs" : "split"} {carta} bind:value />
+  </div>
+</div>
+
+<slot />
diff --git a/ui/src/stores/models.js b/ui/src/stores/models.js
index 7d960169..01c0710c 100755
--- a/ui/src/stores/models.js
+++ b/ui/src/stores/models.js
@@ -1,8 +1,11 @@
 import { writable } from "svelte/store";
 import retrieveModels from "$lib/models/retrieveModels";
+import retrieveModel from "$lib/models/retrieveModel";
 import likeModel from "$lib/models/likeModel";
 import updateModel from "$lib/models/updateModel";
-
+import createModel from "$lib/models/createModel";
+import setModelVersion from "$lib/models/setModelVersion";
+import ingestFile from "$lib/models/ingestFile";
 
 const createModels = () => {
   const { subscribe, set, update } = writable({
@@ -21,6 +24,26 @@ const createModels = () => {
         set({ loading: false, error: e.message });
       }
     },
+    create: async (name, authors, source, license, token) => {
+      const { model_id } = await createModel(name, authors, source, license, token);
+      return model_id
+    },
+    ingest: async (model_id, file, token, version, name) => {
+      await ingestFile(model_id, file, token, version);
+      const data = await retrieveModel(name);
+      update((current) => {
+        const modelExists = current.data.find((model) => model.id === data.id)
+        if (modelExists) {
+          return {
+            data: current.data.map((model) => model.id === data.id ? data : model)
+          }
+        }
+        return {
+          data: [data, ...current.data],
+        }
+      });
+      return data
+    },
     update: async (model_id, name, content, authors, source, license, tags, token) => {
       const data = await updateModel(model_id, name, content, authors, source, license, tags, token);
       update((current) => ({
@@ -33,6 +56,9 @@ const createModels = () => {
     like: async (id, token) => {
       likeModel(id, token);
     },
+    setVersion: async (id, token) => {
+      return await setModelVersion(id, token);
+    },
   };
 };