From 04f1f36b326ae0a5dc247b9901d5bae23b9f2ea9 Mon Sep 17 00:00:00 2001 From: Andreas Date: Tue, 26 Nov 2024 15:25:58 +0200 Subject: [PATCH 1/2] Enhanced Document Views for Source Explorer (#150) * feat(sources): add view modes and refactor Documents component - Add threaded and combined summaries view modes for documents - Filter out combined-summary documents from default view - Break down Documents component into smaller, reusable components - Add source metadata for available view modes (threads/summaries) - Improve component organization and maintainability * fix build error with more readable code not related with the change but resulting in build error --- src/components/sources/DocumentTable.tsx | 90 ++++++++ src/components/sources/Documents.tsx | 192 +++++++++--------- src/components/sources/PaginationControls.tsx | 37 ++++ .../sources/ThreadedDocumentTable.tsx | 141 +++++++++++++ src/components/sources/ViewModeSelector.tsx | 57 ++++++ src/context/SearchQueryContext.tsx | 12 +- src/hooks/useSourceDocuments.ts | 26 ++- .../api/elasticSearchProxy/sourceDocuments.ts | 170 +++++++++++++--- src/pages/api/elasticSearchProxy/sources.ts | 22 ++ src/pages/index.tsx | 8 +- src/pages/sources.tsx | 6 +- src/types.ts | 20 +- 12 files changed, 640 insertions(+), 141 deletions(-) create mode 100644 src/components/sources/DocumentTable.tsx create mode 100644 src/components/sources/PaginationControls.tsx create mode 100644 src/components/sources/ThreadedDocumentTable.tsx create mode 100644 src/components/sources/ViewModeSelector.tsx diff --git a/src/components/sources/DocumentTable.tsx b/src/components/sources/DocumentTable.tsx new file mode 100644 index 0000000..d4f813e --- /dev/null +++ b/src/components/sources/DocumentTable.tsx @@ -0,0 +1,90 @@ +import React from "react"; +import { FaEye } from "react-icons/fa"; +import { formatTimeAgo } from "@/utils/dateUtils"; +import { Document } from "@/types"; + +interface DocumentTableProps { + documents: Document[]; + domain: string; + onViewDocument: (url: string) => void; +} + +const DocumentTable: React.FC = ({ + documents, + domain, + onViewDocument, +}) => { + const trimUrl = (url: string, domain: string): string => { + const domainPattern = new RegExp( + `^(https?:\/\/)?(www\.)?${domain.replace(".", ".")}/?`, + "i" + ); + const trimmed = url.replace(domainPattern, ""); + return trimmed.startsWith("/") ? trimmed : `/${trimmed}`; + }; + + return ( + + + + + + + + + + + {documents?.map((doc, index) => ( + + + + + + + ))} + +
TitleURL + Indexed At +
+
+ {doc.title} +
+ 		+ + +
+ {formatTimeAgo(doc.indexed_at)} + + {new Date(doc.indexed_at).toLocaleString()} + +
+ 		+ +
+ ); +}; + +export default DocumentTable; diff --git a/src/components/sources/Documents.tsx b/src/components/sources/Documents.tsx index 948bb20..1b2a730 100644 --- a/src/components/sources/Documents.tsx +++ b/src/components/sources/Documents.tsx @@ -1,28 +1,32 @@ import React, { useState } from "react"; -import { FaEye } from "react-icons/fa"; - import { useSourceDocuments } from "@/hooks/useSourceDocuments"; import { useDocumentContent } from "@/hooks/useDocumentContent"; -import { formatTimeAgo } from "@/utils/dateUtils"; import DocumentModal from "./DocumentModal"; +import ViewModeSelector from "./ViewModeSelector"; +import PaginationControls from "./PaginationControls"; +import DocumentTable from "./DocumentTable"; +import ThreadedDocumentTable from "./ThreadedDocumentTable"; +import { ViewMode, ViewModeType } from "@/types"; interface SourceDocumentsProps { domain: string; + hasSummaries: boolean; + hasThreads: boolean; } -const trimUrl = (url: string, domain: string): string => { - const domainPattern = new RegExp( - `^(https?:\/\/)?(www\.)?${domain.replace(".", ".")}/?`, - "i" - ); - const trimmed = url.replace(domainPattern, ""); - return trimmed.startsWith("/") ? trimmed : `/${trimmed}`; -}; - -const Documents: React.FC = ({ domain }) => { +const Documents: React.FC = ({ + domain, + hasSummaries, + hasThreads, +}) => { const [page, setPage] = useState(1); + const [threadsPage, setThreadsPage] = useState(1); + const [viewMode, setViewMode] = useState(ViewMode.FLAT); + const [expandedThreads, setExpandedThreads] = useState(new Set()); + const { sourceDocuments, total, isLoading, isError, error } = - useSourceDocuments(domain, page); + useSourceDocuments(domain, page, viewMode, threadsPage); + const [selectedDocumentUrl, setSelectedDocumentUrl] = useState( null ); @@ -39,98 +43,90 @@ const Documents: React.FC = ({ domain }) => { const totalPages = Math.ceil(total / 10); + const handleViewModeChange = (newMode: ViewModeType) => { + // Reset to flat view if trying to switch to an unavailable mode + if ( + (newMode === ViewMode.THREADED && !hasThreads) || + (newMode === ViewMode.SUMMARIES && !hasSummaries) + ) { + newMode = ViewMode.FLAT; + } + setViewMode(newMode); + setPage(1); + setThreadsPage(1); + setExpandedThreads(new Set()); + }; + + const toggleThread = (threadUrl: string) => { + const newExpanded = new Set(expandedThreads); + if (newExpanded.has(threadUrl)) { + newExpanded.delete(threadUrl); + } else { + newExpanded.add(threadUrl); + } + setExpandedThreads(newExpanded); + }; + const handleViewDocument = (url: string) => { setSelectedDocumentUrl(url); }; + // Group documents by thread_url for threaded view + const threadGroups = + viewMode === ViewMode.THREADED + ? sourceDocuments?.reduce((acc, doc) => { + const threadUrl = doc.thread_url || "ungrouped"; + if (!acc[threadUrl]) { + acc[threadUrl] = []; + } + acc[threadUrl].push(doc); + return acc; + }, {} as Record) + : {}; + + const handlePageChange = (newPage: number) => { + if (viewMode === ViewMode.THREADED) { + setThreadsPage(newPage); + } else { + setPage(newPage); + } + }; + return (
-

Documents for {domain}

-
- - - - - - - - - - - {sourceDocuments?.map((doc, index) => ( - - - - - - - ))} - -
- Title - - URL - - Indexed At -
-
- {doc.title} -
- 		- - -
- {formatTimeAgo(doc.indexed_at)} - - {new Date(doc.indexed_at).toLocaleString()} - -
- 		- -
+
+

Documents for {domain}

+
-
- - - Page {page} of {totalPages} - - + +
+ {viewMode === ViewMode.THREADED ? ( + + ) : ( + + )}
+ + + setSelectedDocumentUrl(null)} diff --git a/src/components/sources/PaginationControls.tsx b/src/components/sources/PaginationControls.tsx new file mode 100644 index 0000000..face303 --- /dev/null +++ b/src/components/sources/PaginationControls.tsx @@ -0,0 +1,37 @@ +import React from "react"; + +interface PaginationControlsProps { + currentPage: number; + totalPages: number; + onPageChange: (newPage: number) => void; +} + +const PaginationControls: React.FC = ({ + currentPage, + totalPages, + onPageChange, +}) => { + return ( +
+ + + Page {currentPage} of {totalPages} + + +
+ ); +}; + +export default PaginationControls; diff --git a/src/components/sources/ThreadedDocumentTable.tsx b/src/components/sources/ThreadedDocumentTable.tsx new file mode 100644 index 0000000..68cef2e --- /dev/null +++ b/src/components/sources/ThreadedDocumentTable.tsx @@ -0,0 +1,141 @@ +import React from "react"; +import { FaEye, FaChevronDown, FaChevronRight } from "react-icons/fa"; +import { formatTimeAgo } from "@/utils/dateUtils"; +import { Document } from "@/types"; + +interface ThreadGroup { + [key: string]: Document[]; +} + +interface ThreadedDocumentTableProps { + threadGroups: ThreadGroup; + expandedThreads: Set; + onToggleThread: (threadUrl: string) => void; + onViewDocument: (url: string) => void; +} + +const ThreadedDocumentTable: React.FC = ({ + threadGroups, + expandedThreads, + onToggleThread, + onViewDocument, +}) => { + return ( + + + + + + + + + + + {Object.entries(threadGroups).map(([threadUrl, docs], index) => ( + + + + + + + + {expandedThreads.has(threadUrl) && ( + + + + )} + + ))} + +
+ Thread URL + Documents + Latest Update +
+ + + {threadUrl !== "ungrouped" ? ( + + {threadUrl} + + ) : ( + + Ungrouped Documents + + )} + + {docs.length} + + {formatTimeAgo( + Math.max(...docs.map((d) => new Date(d.indexed_at).getTime())) + )} +
+
+ + + {docs.map((doc, docIndex) => ( + + + + + + + ))} + +
+
+ {doc.title} +
+ 		+ + {doc.url} + + + {formatTimeAgo(doc.indexed_at)} + + +
+
+
+ ); +}; + +export default ThreadedDocumentTable; diff --git a/src/components/sources/ViewModeSelector.tsx b/src/components/sources/ViewModeSelector.tsx new file mode 100644 index 0000000..fcac712 --- /dev/null +++ b/src/components/sources/ViewModeSelector.tsx @@ -0,0 +1,57 @@ +import React from "react"; +import { ViewMode, ViewModeType } from "@/types"; + +interface ViewModeSelectorProps { + viewMode: ViewModeType; + onViewModeChange: (mode: ViewModeType) => void; + hasThreads: boolean; + hasSummaries: boolean; +} + +const ViewModeSelector: React.FC = ({ + viewMode, + onViewModeChange, + hasThreads, + hasSummaries, +}) => { + return ( +
+ + {hasThreads && ( + + )} + {hasSummaries && ( + + )} +
+ ); +}; + +export default ViewModeSelector; diff --git a/src/context/SearchQueryContext.tsx b/src/context/SearchQueryContext.tsx index e45c3aa..3245b1c 100644 --- a/src/context/SearchQueryContext.tsx +++ b/src/context/SearchQueryContext.tsx @@ -80,7 +80,17 @@ export const SearchQueryProvider = ({ }, [rawSearchQuery]); const page = useMemo(() => { - return pageQuery ? parseInt(pageQuery) - 1 ?? 0 : 0; + // Handle empty or invalid input + if (!pageQuery) { + return 0; + } + + // Convert to number and validate + const parsedPage = Number(pageQuery); + const isValidPage = !isNaN(parsedPage) && parsedPage > 0; + + // Convert from 1-based to 0-based index, or default to 0 + return isValidPage ? parsedPage - 1 : 0; }, [pageQuery]); const resultsPerPage = sizeQuery diff --git a/src/hooks/useSourceDocuments.ts b/src/hooks/useSourceDocuments.ts index 0500a95..213738a 100644 --- a/src/hooks/useSourceDocuments.ts +++ b/src/hooks/useSourceDocuments.ts @@ -1,37 +1,40 @@ import { useQuery } from "@tanstack/react-query"; - -interface Document { - title: string; - url: string; - indexed_at: string; -} +import { ViewModeType, Document } from "@/types"; interface SourceDocumentsResponse { documents: Document[]; total: number; + viewMode: ViewModeType; } const fetchSourceDocuments = async ( domain: string, - page: number + page: number, + viewMode: ViewModeType, + threadsPage: number ): Promise => { const response = await fetch("/api/elasticSearchProxy/sourceDocuments", { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ domain, page }), + body: JSON.stringify({ domain, page, viewMode, threadsPage }), }); const data = await response.json(); if (!data.success) throw new Error(data.message); return data.data; }; -export const useSourceDocuments = (domain: string, page: number) => { +export const useSourceDocuments = ( + domain: string, + page: number, + viewMode: ViewModeType, + threadsPage: number +) => { const { data, isLoading, isError, error } = useQuery< SourceDocumentsResponse, Error >({ - queryKey: ["sourceDocuments", domain, page], - queryFn: () => fetchSourceDocuments(domain, page), + queryKey: ["sourceDocuments", domain, page, viewMode, threadsPage], + queryFn: () => fetchSourceDocuments(domain, page, viewMode, threadsPage), cacheTime: Infinity, staleTime: Infinity, refetchOnWindowFocus: false, @@ -40,6 +43,7 @@ export const useSourceDocuments = (domain: string, page: number) => { return { sourceDocuments: data?.documents, total: data?.total, + viewMode: data?.viewMode, isLoading, isError, error, diff --git a/src/pages/api/elasticSearchProxy/sourceDocuments.ts b/src/pages/api/elasticSearchProxy/sourceDocuments.ts index 3590b59..d5d69eb 100644 --- a/src/pages/api/elasticSearchProxy/sourceDocuments.ts +++ b/src/pages/api/elasticSearchProxy/sourceDocuments.ts @@ -1,5 +1,31 @@ import type { NextApiRequest, NextApiResponse } from "next"; import { client } from "@/config/elasticsearch"; +import { ViewMode } from "@/types"; + +interface ThreadBucket { + key: string; + doc_count: number; + latest_doc: { + value: number; + }; + docs: { + hits: { + hits: Array<{ + _source: { + thread_url: string; + }; + }>; + }; + }; +} + +interface ThreadAggregationResponse { + aggregations: { + threads: { + buckets: ThreadBucket[]; + }; + }; +} export default async function handler( req: NextApiRequest, @@ -12,7 +38,12 @@ export default async function handler( }); } - const { domain, page = 1 } = req.body; + const { + domain, + page = 1, + viewMode = ViewMode.FLAT, + threadsPage = 1, + } = req.body; if (!domain) { return res.status(400).json({ @@ -23,35 +54,124 @@ export default async function handler( const size = 10; const from = (page - 1) * size; - try { - const result = await client.search({ - index: process.env.INDEX, - body: { - from, - size, - query: { - term: { "domain.keyword": domain }, + // Create query based on view mode + const createQuery = (additionalTerms = {}) => { + if (viewMode === ViewMode.SUMMARIES) { + return { + bool: { + must: [ + { term: { "domain.keyword": domain } }, + { term: { "type.keyword": "combined-summary" } }, + ], }, - _source: ["title", "url", "indexed_at"], - sort: [{ indexed_at: "desc" }], + }; + } + + return { + bool: { + must: [ + { term: { "domain.keyword": domain } }, + ...(Object.keys(additionalTerms).length > 0 ? [additionalTerms] : []), + ], + must_not: [{ term: { "type.keyword": "combined-summary" } }], }, - }); + }; + }; - const documents = result.hits.hits.map((hit) => hit._source); + try { + if (viewMode === ViewMode.THREADED) { + // Thread view + const threadAggregation = (await client.search({ + index: process.env.INDEX, + body: { + query: createQuery(), + size: 0, + aggs: { + threads: { + terms: { + field: "thread_url.keyword", + size: 10000, + order: { "latest_doc.value": "desc" }, + }, + aggs: { + latest_doc: { + max: { field: "indexed_at" }, + }, + docs: { + top_hits: { + size: 1, + sort: [{ indexed_at: "desc" }], + _source: ["thread_url"], + }, + }, + }, + }, + }, + }, + })) as unknown as ThreadAggregationResponse; - // Handle both possible types of total - const total = - typeof result.hits.total === "number" - ? result.hits.total - : result.hits.total.value; + const threadBuckets = threadAggregation.aggregations.threads.buckets; + const totalThreads = threadBuckets.length; + const threadsPerPage = 10; + const startThread = (threadsPage - 1) * threadsPerPage; + const endThread = startThread + threadsPerPage; + const paginatedThreads = threadBuckets.slice(startThread, endThread); - return res.status(200).json({ - success: true, - data: { - documents, - total, - }, - }); + const threadUrls = paginatedThreads.map((bucket) => bucket.key); + const documentsResult = await client.search({ + index: process.env.INDEX, + body: { + query: createQuery({ + terms: { + "thread_url.keyword": threadUrls, + }, + }), + size: 1000, + sort: [{ indexed_at: "desc" }], + _source: ["title", "url", "indexed_at", "thread_url", "type"], + }, + }); + + const documents = documentsResult.hits.hits.map((hit) => hit._source); + + return res.status(200).json({ + success: true, + data: { + documents, + total: totalThreads, + viewMode: ViewMode.THREADED, + }, + }); + } else { + // Flat or summaries view + const result = await client.search({ + index: process.env.INDEX, + body: { + from, + size, + query: createQuery(), + _source: ["title", "url", "indexed_at", "thread_url", "type"], + sort: [{ indexed_at: "desc" }], + }, + }); + + const documents = result.hits.hits.map((hit) => hit._source); + + // Handle both possible types of total + const total = + typeof result.hits.total === "number" + ? result.hits.total + : result.hits.total.value; + + return res.status(200).json({ + success: true, + data: { + documents, + total, + viewMode, + }, + }); + } } catch (error) { console.error(error); return res.status(400).json({ diff --git a/src/pages/api/elasticSearchProxy/sources.ts b/src/pages/api/elasticSearchProxy/sources.ts index 8c5ff7b..5ef5027 100644 --- a/src/pages/api/elasticSearchProxy/sources.ts +++ b/src/pages/api/elasticSearchProxy/sources.ts @@ -7,6 +7,12 @@ interface DomainAggregationBucket { last_indexed: { value: number; }; + has_summaries: { + doc_count: number; + }; + has_threads: { + doc_count: number; + }; } export default async function handler( @@ -37,6 +43,20 @@ export default async function handler( field: "indexed_at", }, }, + has_summaries: { + filter: { + term: { + "type.keyword": "combined-summary", + }, + }, + }, + has_threads: { + filter: { + exists: { + field: "thread_url", + }, + }, + }, }, }, }, @@ -53,6 +73,8 @@ export default async function handler( domain: bucket.key, documentCount: bucket.doc_count, lastScraped: bucket.last_indexed.value || null, + hasSummaries: bucket.has_summaries.doc_count > 0, + hasThreads: bucket.has_threads.doc_count > 0, })); return res.status(200).json({ diff --git a/src/pages/index.tsx b/src/pages/index.tsx index 1215b1f..aace459 100644 --- a/src/pages/index.tsx +++ b/src/pages/index.tsx @@ -55,10 +55,10 @@ export const getServerSideProps: GetServerSideProps = async ( }; } - const page = pageQuery ? parseInt(pageQuery) - 1 ?? 0 : 0; - const size = sizeQuery - ? parseInt(sizeQuery) ?? defaultParam[URLSearchParamsKeyword.SIZE] - : defaultParam[URLSearchParamsKeyword.SIZE]; + const defaultPage = 0; + const page = pageQuery ? parseInt(pageQuery) - 1 : defaultPage; + const defaultSize = defaultParam[URLSearchParamsKeyword.SIZE]; + const size = sizeQuery ? parseInt(sizeQuery) || defaultSize : defaultSize; const options = { queryString, diff --git a/src/pages/sources.tsx b/src/pages/sources.tsx index 1861d4c..88fc762 100644 --- a/src/pages/sources.tsx +++ b/src/pages/sources.tsx @@ -135,7 +135,11 @@ const SourcesPage: React.FC = () => { colSpan={4} className="px-4 py-2 border-b border-custom-stroke" > - + )} diff --git a/src/types.ts b/src/types.ts index 83496af..25dd772 100644 --- a/src/types.ts +++ b/src/types.ts @@ -61,10 +61,28 @@ export type EsSearchResponse = SearchResponse< Record >; +export const ViewMode = { + FLAT: "flat", + THREADED: "threaded", + SUMMARIES: "summaries", +} as const; + +export type ViewModeType = (typeof ViewMode)[keyof typeof ViewMode]; + export interface Source { domain: string; - lastScraped: string; documentCount: number; + lastScraped: string; + hasSummaries: boolean; + hasThreads: boolean; +} + +export interface Document { + title: string; + url: string; + indexed_at: string; + thread_url?: string; + type?: string; } export type EsSourcesResponse = Source[]; From bf233c40ca421e712ec9069941734130ace44fb7 Mon Sep 17 00:00:00 2001 From: 0tuedon <90271995+0tuedon@users.noreply.github.com> Date: Fri, 29 Nov 2024 02:51:49 +0100 Subject: [PATCH 2/2] Addition of boss banner (#151) * chore(banner): add boss banner * chore(version): updated bdp-ui patch fix --- package.json | 1 + src/components/Banner.tsx | 56 ++++++++------------------------ src/components/navBar/NavBar.tsx | 4 ++- src/pages/_app.tsx | 1 + 4 files changed, 19 insertions(+), 43 deletions(-) diff --git a/package.json b/package.json index 49cda94..f36b513 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "prettier:fix": "prettier --write . --ignore-path .gitignore" }, "dependencies": { + "@bitcoin-dev-project/bdp-ui": "^1.5.2", "@chakra-ui/react": "^2.0.0", "@elastic/elasticsearch": "^8.8.0", "@elastic/react-search-ui": "1.20.2", diff --git a/src/components/Banner.tsx b/src/components/Banner.tsx index d5be3cf..3341201 100644 --- a/src/components/Banner.tsx +++ b/src/components/Banner.tsx @@ -1,48 +1,20 @@ -import Image from "next/image"; -import Link from "next/link"; -import React, { useEffect, useState } from "react"; +import { Banner } from "@bitcoin-dev-project/bdp-ui"; -const BANNER_KEY = "FOSS-banner"; - -const Banner = () => { - const [isOpen, setIsOpen] = useState(false); - - const handleClose = () => { - if (typeof window !== "undefined") { - sessionStorage.setItem(BANNER_KEY, "hidden"); - } - setIsOpen(false); - }; - - useEffect(() => { - const banner_in_session = window.sessionStorage.getItem(BANNER_KEY); - if (banner_in_session === "hidden") { - setIsOpen(false); - } else { - setIsOpen(true); - } - }, []); - - if (!isOpen) return null; +const BossBanner = () => { return ( -
-
-

{`Start Your Career in Bitcoin Open Source`}

-
- {`Development in 2024 `} - - {`Apply Today!`} - -
-
- +
+
); }; -export default Banner; +export default BossBanner; diff --git a/src/components/navBar/NavBar.tsx b/src/components/navBar/NavBar.tsx index bcf507f..edc5d15 100644 --- a/src/components/navBar/NavBar.tsx +++ b/src/components/navBar/NavBar.tsx @@ -12,6 +12,7 @@ import useSearchQuery from "@/hooks/useSearchQuery"; import { removeMarkdownCharacters } from "@/utils/elastic-search-ui-functions"; import { useTheme } from "@/context/Theme"; import { Tooltip } from "@chakra-ui/react"; +import BossBanner from "../Banner"; function ThemeSwitcher() { const { theme, toggleTheme } = useTheme(); @@ -128,10 +129,11 @@ const NavBar = () => { return (