feat: add Sources Page (#144)

* feat: add Sources page Create new Sources page to display data source information from Elasticsearch * Add expandable document view for each source Implement a new feature that allows users to quickly view documents associated with each source directly from the sources table, allowing for better understanding of the current scraped content for each data source. * Add document content viewer to Sources page Implement a modal that displays full document details when clicking the view icon in the Sources table. This allows to inspect all fields of individual documents fetched from Elasticsearch, providing deeper insights into the indexed content.
bitcoinsearch · Sep 3, 2024 · ade1520 · ade1520
1 parent b6126e5
commit ade1520
Show file tree

Hide file tree

Showing 11 changed files with 772 additions and 0 deletions.
diff --git a/src/components/sources/DocumentModal.tsx b/src/components/sources/DocumentModal.tsx
@@ -0,0 +1,126 @@
+import React from "react";
+import {
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalHeader,
+  ModalBody,
+  ModalCloseButton,
+  Text,
+  Link,
+  UnorderedList,
+  ListItem,
+  Box,
+  VStack,
+} from "@chakra-ui/react";
+
+interface DocumentModalProps {
+  isOpen: boolean;
+  onClose: () => void;
+  document: Record<string, any> | null;
+  isLoading: boolean;
+  isError: boolean;
+  error?: string;
+}
+
+const formatValue = (value: any): string => {
+  if (typeof value === "string") {
+    return value;
+  } else if (typeof value === "number" || typeof value === "boolean") {
+    return value.toString();
+  } else if (value instanceof Date) {
+    return value.toISOString();
+  }
+  return "";
+};
+
+const RenderField = ({ name, value }: { name: string; value: any }) => {
+  if (Array.isArray(value)) {
+    return (
+      <Box mb={2}>
+        <Text fontWeight="bold">{name}:</Text>
+        <UnorderedList ml={5}>
+          {value.map((item, index) => (
+            <ListItem key={index}>
+              {typeof item === "object" ? (
+                <RenderObject object={item} />
+              ) : (
+                formatValue(item)
+              )}
+            </ListItem>
+          ))}
+        </UnorderedList>
+      </Box>
+    );
+  } else if (typeof value === "object" && value !== null) {
+    return (
+      <Box mb={2}>
+        <Text fontWeight="bold">{name}:</Text>
+        <Box ml={4}>
+          <RenderObject object={value} />
+        </Box>
+      </Box>
+    );
+  } else {
+    return (
+      <Text mb={2}>
+        <Text as="span" fontWeight="bold">
+          {name}:
+        </Text>{" "}
+        {formatValue(value)}
+      </Text>
+    );
+  }
+};
+
+const RenderObject = ({ object }: { object: Record<string, any> }) => {
+  return (
+    <VStack align="stretch" spacing={2}>
+      {Object.entries(object).map(([key, value]) => (
+        <RenderField key={key} name={key} value={value} />
+      ))}
+    </VStack>
+  );
+};
+
+const DocumentModal: React.FC<DocumentModalProps> = ({
+  isOpen,
+  onClose,
+  document,
+  isLoading,
+  isError,
+  error,
+}) => {
+  return (
+    <Modal isOpen={isOpen} onClose={onClose} size="xl" scrollBehavior="inside">
+      <ModalOverlay />
+      <ModalContent>
+        <ModalHeader>{document?.title || "Document Details"}</ModalHeader>
+        <ModalCloseButton />
+        <ModalBody>
+          {isLoading && <Text>Loading document content...</Text>}
+          {isError && (
+            <Text color="red.500">Error loading document: {error}</Text>
+          )}
+          {!isLoading && !isError && document && (
+            <VStack align="stretch" spacing={4}>
+              {document.url && (
+                <Link
+                  href={document.url}
+                  isExternal
+                  color="blue.500"
+                  _hover={{ textDecoration: "underline" }}
+                >
+                  {document.url}
+                </Link>
+              )}
+              <RenderObject object={document} />
+            </VStack>
+          )}
+        </ModalBody>
+      </ModalContent>
+    </Modal>
+  );
+};
+
+export default DocumentModal;
diff --git a/src/components/sources/Documents.tsx b/src/components/sources/Documents.tsx
@@ -0,0 +1,146 @@
+import React, { useState } from "react";
+import { FaEye } from "react-icons/fa";
+
+import { useSourceDocuments } from "@/hooks/useSourceDocuments";
+import { useDocumentContent } from "@/hooks/useDocumentContent";
+import { formatTimeAgo } from "@/utils/dateUtils";
+import DocumentModal from "./DocumentModal";
+
+interface SourceDocumentsProps {
+  domain: string;
+}
+
+const trimUrl = (url: string, domain: string): string => {
+  const domainPattern = new RegExp(
+    `^(https?:\/\/)?(www\.)?${domain.replace(".", ".")}/?`,
+    "i"
+  );
+  const trimmed = url.replace(domainPattern, "");
+  return trimmed.startsWith("/") ? trimmed : `/${trimmed}`;
+};
+
+const Documents: React.FC<SourceDocumentsProps> = ({ domain }) => {
+  const [page, setPage] = useState(1);
+  const { sourceDocuments, total, isLoading, isError, error } =
+    useSourceDocuments(domain, page);
+  const [selectedDocumentUrl, setSelectedDocumentUrl] = useState<string | null>(
+    null
+  );
+  const {
+    documentContent,
+    isLoading: isContentLoading,
+    isError: isContentError,
+    error: contentError,
+  } = useDocumentContent(selectedDocumentUrl);
+
+  if (isLoading) return <div>Loading source documents...</div>;
+  if (isError)
+    return <div>Error loading source documents: {error.message}</div>;
+
+  const totalPages = Math.ceil(total / 10);
+
+  const handleViewDocument = (url: string) => {
+    setSelectedDocumentUrl(url);
+  };
+
+  return (
+    <div className="mt-4">
+      <h3 className="text-lg font-semibold mb-2">Documents for {domain}</h3>
+      <div className="overflow-x-auto">
+        <table className="min-w-full border border-custom-stroke">
+          <thead>
+            <tr className="bg-custom-hover-state dark:bg-custom-hover-primary">
+              <th className="px-4 py-2 border-b border-custom-stroke whitespace-nowrap">
+                Title
+              </th>
+              <th className="px-4 py-2 border-b border-custom-stroke whitespace-nowrap">
+                URL
+              </th>
+              <th className="px-4 py-2 border-b border-custom-stroke whitespace-nowrap">
+                Indexed At
+              </th>
+              <th className="w-10 px-2 py-2 border-b border-custom-stroke"></th>
+            </tr>
+          </thead>
+          <tbody>
+            {sourceDocuments?.map((doc, index) => (
+              <tr
+                key={index}
+                className={
+                  index % 2 === 0
+                    ? "bg-custom-hover-state dark:bg-custom-hover-primary"
+                    : "bg-custom-background"
+                }
+              >
+                <td className="px-4 py-2 border-b border-custom-stroke max-w-xs">
+                  <div className="truncate" title={doc.title}>
+                    {doc.title}
+                  </div>
+                </td>
+                <td className="px-4 py-2 border-b border-custom-stroke">
+                  <div className="truncate max-w-md">
+                    <a
+                      href={doc.url}
+                      target="_blank"
+                      rel="noopener noreferrer"
+                      className="text-custom-accent hover:underline"
+                      title={doc.url}
+                    >
+                      {trimUrl(doc.url, domain)}
+                    </a>
+                  </div>
+                </td>
+                <td className="px-4 py-2 border-b border-custom-stroke whitespace-nowrap">
+                  <div className="group relative inline-block">
+                    <span>{formatTimeAgo(doc.indexed_at)}</span>
+                    <span className="invisible group-hover:visible absolute left-0 -translate-x-1/2 bottom-full mb-2 px-2 py-1 text-sm bg-custom-black text-custom-white rounded opacity-0 group-hover:opacity-100 transition-opacity duration-300 whitespace-nowrap z-10">
+                      {new Date(doc.indexed_at).toLocaleString()}
+                    </span>
+                  </div>
+                </td>
+                <td className="w-10 px-2 py-2 border-b border-custom-stroke text-center">
+                  <button
+                    onClick={() => handleViewDocument(doc.url)}
+                    className="text-custom-accent hover:text-custom-accent-dark"
+                    title="View document"
+                  >
+                    <FaEye className="w-5 h-5 inline-block" />
+                  </button>
+                </td>
+              </tr>
+            ))}
+          </tbody>
+        </table>
+      </div>
+      <div className="mt-4 flex justify-between items-center">
+        <button
+          onClick={() => setPage((p) => Math.max(1, p - 1))}
+          disabled={page === 1}
+          className="px-4 py-2 bg-custom-button text-custom-white rounded disabled:opacity-50"
+        >
+          Previous
+        </button>
+        <span>
+          Page {page} of {totalPages}
+        </span>
+        <button
+          onClick={() => setPage((p) => Math.min(totalPages, p + 1))}
+          disabled={page === totalPages}
+          className="px-4 py-2 bg-custom-button text-custom-white rounded disabled:opacity-50"
+        >
+          Next
+        </button>
+      </div>
+      <DocumentModal
+        isOpen={!!selectedDocumentUrl}
+        onClose={() => setSelectedDocumentUrl(null)}
+        document={documentContent}
+        isLoading={isContentLoading}
+        isError={isContentError}
+        error={contentError?.message}
+      />
+    </div>
+  );
+};
+
+export default Documents;
diff --git a/src/hooks/useDocumentContent.ts b/src/hooks/useDocumentContent.ts
@@ -0,0 +1,39 @@
+import { useQuery } from "@tanstack/react-query";
+
+interface DocumentContent {
+  title: string;
+  url: string;
+  content: string;
+  indexed_at: string;
+}
+
+const fetchDocumentContent = async (url: string): Promise<DocumentContent> => {
+  const response = await fetch("/api/elasticSearchProxy/getDocumentContent", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({ url }),
+  });
+  const data = await response.json();
+  if (!data.success) throw new Error(data.message);
+  return data.data;
+};
+
+export const useDocumentContent = (url: string) => {
+  const { data, isLoading, isError, error } = useQuery<DocumentContent, Error>({
+    queryKey: ["documentContent", url],
+    queryFn: () => fetchDocumentContent(url),
+    enabled: !!url,
+    cacheTime: Infinity,
+    staleTime: Infinity,
+    refetchOnWindowFocus: false,
+  });
+
+  return {
+    documentContent: data,
+    isLoading,
+    isError,
+    error,
+  };
+};
diff --git a/src/hooks/useSourceDocuments.ts b/src/hooks/useSourceDocuments.ts
@@ -0,0 +1,47 @@
+import { useQuery } from "@tanstack/react-query";
+
+interface Document {
+  title: string;
+  url: string;
+  indexed_at: string;
+}
+
+interface SourceDocumentsResponse {
+  documents: Document[];
+  total: number;
+}
+
+const fetchSourceDocuments = async (
+  domain: string,
+  page: number
+): Promise<SourceDocumentsResponse> => {
+  const response = await fetch("/api/elasticSearchProxy/sourceDocuments", {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ domain, page }),
+  });
+  const data = await response.json();
+  if (!data.success) throw new Error(data.message);
+  return data.data;
+};
+
+export const useSourceDocuments = (domain: string, page: number) => {
+  const { data, isLoading, isError, error } = useQuery<
+    SourceDocumentsResponse,
+    Error
+  >({
+    queryKey: ["sourceDocuments", domain, page],
+    queryFn: () => fetchSourceDocuments(domain, page),
+    cacheTime: Infinity,
+    staleTime: Infinity,
+    refetchOnWindowFocus: false,
+  });
+
+  return {
+    sourceDocuments: data?.documents,
+    total: data?.total,
+    isLoading,
+    isError,
+    error,
+  };
+};