diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 176922885b83f..2b1777df0b542 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -123,3 +123,8 @@ jobs: # this should be a last resort in case by any chances the build memory gets too high # but in general this should never happen NODE_OPTIONS: '--max_old_space_size=4096' + + - name: Sync Orama Cloud + if: github.ref == 'refs/heads/main' + run: | + npm run sync-orama diff --git a/.gitignore b/.gitignore index e137b9768de66..7f47879b973a9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ node_modules npm-debug.log .npm +.env.local # Next.js Build Output .next diff --git a/app/[locale]/next-data/page-data/route.ts b/app/[locale]/next-data/page-data/route.ts index 2b21630b19398..34a715f5973de 100644 --- a/app/[locale]/next-data/page-data/route.ts +++ b/app/[locale]/next-data/page-data/route.ts @@ -15,30 +15,38 @@ export const GET = async () => { defaultLocale.code ); - const availablePagesMetadata = allAvailbleRoutes.map(async pathname => { - const { source, filename } = await dynamicRouter.getMarkdownFile( - defaultLocale.code, - pathname - ); + const availablePagesMetadata = allAvailbleRoutes + .filter(route => !route.startsWith('blog')) + .map(async pathname => { + const { source, filename } = await dynamicRouter.getMarkdownFile( + defaultLocale.code, + pathname + ); - // Gets the title and the Description from the Page Metadata - const { title, description } = await dynamicRouter.getPageMetadata( - defaultLocale.code, - pathname - ); + // Gets the title and the Description from the Page Metadata + const { title, description } = await dynamicRouter.getPageMetadata( + defaultLocale.code, + pathname + ); - // Parser the Markdown source with `gray-matter` and then only - // grabs the markdown content and cleanses it by removing HTML/JSX tags - // removing empty/blank lines or lines just with spaces and trims each line - // from leading and trailing paddings/spaces - const cleanedContent = parseRichTextIntoPlainText(matter(source).content); + // Parser the Markdown source with `gray-matter` and then only + // grabs the markdown content and cleanses it by removing HTML/JSX tags + // removing empty/blank lines or lines just with spaces and trims each line + // from leading and trailing paddings/spaces + const cleanedContent = parseRichTextIntoPlainText(matter(source).content); - // Deflates a String into a base64 string-encoded (zlib compressed) - const deflatedSource = deflateSync(cleanedContent).toString('base64'); + // Deflates a String into a base64 string-encoded (zlib compressed) + const deflatedSource = deflateSync(cleanedContent).toString('base64'); - // Returns metadata of each page available on the Website - return { filename, pathname, title, description, content: deflatedSource }; - }); + // Returns metadata of each page available on the Website + return { + filename, + pathname, + title, + description, + content: deflatedSource, + }; + }); return Response.json(await Promise.all(availablePagesMetadata)); }; diff --git a/components/Common/Search/States/WithAllResults.tsx b/components/Common/Search/States/WithAllResults.tsx new file mode 100644 index 0000000000000..481edb8cc3966 --- /dev/null +++ b/components/Common/Search/States/WithAllResults.tsx @@ -0,0 +1,40 @@ +import type { Results } from '@orama/orama'; +import NextLink from 'next/link'; +import { useParams } from 'next/navigation'; +import { useTranslations } from 'next-intl'; +import type { FC } from 'react'; + +import type { SearchDoc } from '@/types'; + +import styles from './index.module.css'; + +type SearchResults = Results; + +type SeeAllProps = { + searchResults: SearchResults; + searchTerm: string; + selectedFacetName: string; + onSeeAllClick: () => void; +}; + +export const WithAllResults: FC = props => { + const t = useTranslations(); + const params = useParams(); + + const locale = params?.locale ?? 'en'; + const resultsCount = props.searchResults?.count?.toLocaleString('en') ?? 0; + const searchParams = new URLSearchParams(); + + searchParams.set('q', props.searchTerm); + searchParams.set('section', props.selectedFacetName); + + const allResultsURL = `/${locale}/search?${searchParams.toString()}`; + + return ( +
+ + {t('components.search.seeAll.text', { count: resultsCount })} + +
+ ); +}; diff --git a/components/Common/Search/States/WithEmptyState.tsx b/components/Common/Search/States/WithEmptyState.tsx new file mode 100644 index 0000000000000..f554a0bec206b --- /dev/null +++ b/components/Common/Search/States/WithEmptyState.tsx @@ -0,0 +1,14 @@ +import { useTranslations } from 'next-intl'; +import type { FC } from 'react'; + +import styles from './index.module.css'; + +export const WithEmptyState: FC = () => { + const t = useTranslations(); + + return ( +
+ {t('components.search.emptyState.text')} +
+ ); +}; diff --git a/components/Common/Search/States/WithError.tsx b/components/Common/Search/States/WithError.tsx new file mode 100644 index 0000000000000..33eecbabd147d --- /dev/null +++ b/components/Common/Search/States/WithError.tsx @@ -0,0 +1,14 @@ +import { useTranslations } from 'next-intl'; +import type { FC } from 'react'; + +import styles from './index.module.css'; + +export const WithError: FC = () => { + const t = useTranslations(); + + return ( +
+ {t('components.search.searchError.text')} +
+ ); +}; diff --git a/components/Common/Search/States/WithNoResults.tsx b/components/Common/Search/States/WithNoResults.tsx new file mode 100644 index 0000000000000..5b55c60469c4b --- /dev/null +++ b/components/Common/Search/States/WithNoResults.tsx @@ -0,0 +1,16 @@ +import { useTranslations } from 'next-intl'; +import type { FC } from 'react'; + +import styles from './index.module.css'; + +type NoResultsProps = { searchTerm: string }; + +export const WithNoResults: FC = props => { + const t = useTranslations(); + + return ( +
+ {t('components.search.noResults.text', { query: props.searchTerm })} +
+ ); +}; diff --git a/components/Common/Search/States/WithPoweredBy.tsx b/components/Common/Search/States/WithPoweredBy.tsx new file mode 100644 index 0000000000000..3986280d5d7a6 --- /dev/null +++ b/components/Common/Search/States/WithPoweredBy.tsx @@ -0,0 +1,41 @@ +'use client'; + +import Image from 'next/image'; +import { useTranslations } from 'next-intl'; +import { useTheme } from 'next-themes'; +import { useEffect, useState } from 'react'; + +import styles from './index.module.css'; + +const getLogoURL = (theme: string = 'dark') => + `https://website-assets.oramasearch.com/orama-when-${theme}.svg`; + +export const WithPoweredBy = () => { + const t = useTranslations(); + const { resolvedTheme } = useTheme(); + const [logoURL, setLogoURL] = useState(); + + useEffect(() => setLogoURL(getLogoURL(resolvedTheme)), [resolvedTheme]); + + return ( +
+ {t('components.search.poweredBy.text')} + + + {logoURL && ( + Powered by OramaSearch + )} + +
+ ); +}; diff --git a/components/Common/Search/States/WithSearchBox.tsx b/components/Common/Search/States/WithSearchBox.tsx new file mode 100644 index 0000000000000..685650aedc907 --- /dev/null +++ b/components/Common/Search/States/WithSearchBox.tsx @@ -0,0 +1,193 @@ +'use client'; + +import { + MagnifyingGlassIcon, + ChevronLeftIcon, +} from '@heroicons/react/24/outline'; +import type { Results, Nullable } from '@orama/orama'; +import classNames from 'classnames'; +import { useState, useRef, useEffect } from 'react'; +import type { FC } from 'react'; + +import styles from '@/components/Common/Search/States/index.module.css'; +import { WithAllResults } from '@/components/Common/Search/States/WithAllResults'; +import { WithEmptyState } from '@/components/Common/Search/States/WithEmptyState'; +import { WithError } from '@/components/Common/Search/States/WithError'; +import { WithNoResults } from '@/components/Common/Search/States/WithNoResults'; +import { WithPoweredBy } from '@/components/Common/Search/States/WithPoweredBy'; +import { WithSearchResult } from '@/components/Common/Search/States/WithSearchResult'; +import { useClickOutside } from '@/hooks/react-client'; +import { useRouter } from '@/navigation.mjs'; +import { DEFAULT_ORAMA_QUERY_PARAMS } from '@/next.constants.mjs'; +import { search as oramaSearch, getInitialFacets } from '@/next.orama.mjs'; +import type { SearchDoc } from '@/types'; +import { debounce } from '@/util/debounce'; + +type Facets = { [key: string]: number }; + +type SearchResults = Nullable>; + +type SearchBoxProps = { onClose: () => void }; + +export const WithSearchBox: FC = ({ onClose }) => { + const [searchTerm, setSearchTerm] = useState(''); + const [searchResults, setSearchResults] = useState(null); + const [selectedFacet, setSelectedFacet] = useState(0); + const [searchError, setSearchError] = useState>(null); + + const router = useRouter(); + const searchInputRef = useRef(null); + const searchBoxRef = useRef(null); + + const search = (term: string) => { + oramaSearch({ + term, + ...DEFAULT_ORAMA_QUERY_PARAMS, + mode: 'fulltext', + returning: [ + 'path', + 'pageSectionTitle', + 'pageTitle', + 'path', + 'siteSection', + ], + ...filterBySection(), + }) + .then(setSearchResults) + .catch(setSearchError); + }; + + useClickOutside(searchBoxRef, () => { + reset(); + onClose(); + }); + + useEffect(() => { + searchInputRef.current?.focus(); + + getInitialFacets().then(setSearchResults).catch(setSearchError); + + return reset; + }, []); + + useEffect( + () => debounce(() => search(searchTerm), 1000), + // we don't need to care about memoization of search function + // eslint-disable-next-line react-hooks/exhaustive-deps + [searchTerm, selectedFacet] + ); + + const reset = () => { + setSearchTerm(''); + setSearchResults(null); + setSelectedFacet(0); + }; + + const onSubmit = (e: React.FormEvent) => { + e.preventDefault(); + router.push(`/search?q=${searchTerm}§ion=${selectedFacetName}`); + onClose(); + }; + + const changeFacet = (idx: number) => setSelectedFacet(idx); + + const filterBySection = () => { + if (selectedFacet === 0) { + return {}; + } + + return { where: { siteSection: { eq: selectedFacetName } } }; + }; + + const facets: Facets = { + all: searchResults?.count ?? 0, + ...(searchResults?.facets?.siteSection?.values ?? {}), + }; + + const selectedFacetName = Object.keys(facets)[selectedFacet]; + + return ( +
+
+
+
+ + + + +
+ setSearchTerm(event.target.value)} + value={searchTerm} + /> +
+
+ +
+ {Object.keys(facets).map((facetName, idx) => ( + + ))} +
+ +
+ {searchError && } + + {!searchError && !searchTerm && } + + {!searchError && searchTerm && ( + <> + {searchResults && + searchResults.count > 0 && + searchResults.hits.map(hit => ( + + ))} + + {searchResults && searchResults.count === 0 && ( + + )} + + {searchResults && searchResults.count > 8 && ( + + )} + + )} +
+ +
+ +
+
+
+
+ ); +}; diff --git a/components/Common/Search/States/WithSearchResult.tsx b/components/Common/Search/States/WithSearchResult.tsx new file mode 100644 index 0000000000000..76757dafa4d1b --- /dev/null +++ b/components/Common/Search/States/WithSearchResult.tsx @@ -0,0 +1,42 @@ +import type { Result } from '@orama/orama'; +import type { FC } from 'react'; + +import { pathToBreadcrumbs } from '@/components/Common/Search/utils'; +import Link from '@/components/Link'; +import { highlighter } from '@/next.orama.mjs'; +import type { SearchDoc } from '@/types'; + +import styles from './index.module.css'; + +type SearchResultProps = { + hit: Result; + searchTerm: string; +}; + +export const WithSearchResult: FC = props => { + const isAPIResult = props.hit.document.siteSection.toLowerCase() === 'api'; + const basePath = isAPIResult ? 'https://nodejs.org' : ''; + const path = `${basePath}/${props.hit.document.path}`; + + return ( + +
+
+ {pathToBreadcrumbs(props.hit.document.path).join(' > ')} + {' > '} + {props.hit.document.pageTitle} +
+ + ); +}; diff --git a/components/Common/Search/States/index.module.css b/components/Common/Search/States/index.module.css new file mode 100644 index 0000000000000..c711eac2f7d63 --- /dev/null +++ b/components/Common/Search/States/index.module.css @@ -0,0 +1,223 @@ +.searchBoxModalContainer { + @apply fixed + inset-0 + z-50 + flex + items-center + justify-center + bg-neutral-900 + bg-opacity-90 + dark:bg-neutral-900 + dark:bg-opacity-90; +} + +.searchBoxModalPanel { + @apply fixed + h-screen + w-full + bg-neutral-100 + dark:bg-neutral-950 + md:top-60 + md:h-[450px] + md:max-w-3xl + md:rounded-xl + md:shadow-lg; +} + +.searchBoxInnerPanel { + @apply pt-12 + text-neutral-800 + dark:text-neutral-400 + md:pt-2; +} + +.searchBoxMagnifyingGlassIcon { + @apply absolute + top-[10px] + hidden + size-6 + md:block; +} + +.searchBoxBackIconContainer { + @apply block + md:hidden; +} + +.searchBoxBackIcon { + @apply absolute + top-[7px] + block + size-6 + md:hidden; +} + +.searchBoxInputContainer { + @apply relative + px-2 + md:px-4; +} + +.searchBoxInput { + @apply w-full + rounded-b-none + border-b + border-neutral-300 + bg-transparent + py-2 + pl-8 + pr-4 + focus:outline-none + dark:border-neutral-900 + dark:text-neutral-300 + dark:placeholder-neutral-300; +} + +.fulltextResultsContainer { + @apply h-80 + overflow-auto + md:px-4; +} + +.fulltextSearchResult { + @apply flex + flex-col + rounded-md + p-2 + text-left + text-sm + hover:bg-neutral-300 + dark:hover:bg-neutral-900; +} + +.fulltextSearchResultTitle { + @apply text-neutral-800 + dark:text-neutral-300; +} + +.fulltextSearchResultBreadcrumb { + @apply mt-1 + text-xs + capitalize + text-neutral-800 + dark:text-neutral-600; +} + +.fulltextSearchSections { + @apply mb-1 + mt-2 + flex + gap-2 + overflow-x-auto + p-2 + text-xs + font-semibold + text-neutral-700 + dark:text-neutral-600 + md:px-4; +} + +.fulltextSearchSection { + @apply rounded-lg + border-b + border-transparent + px-2 + py-1 + capitalize + hover:bg-neutral-200 + dark:border-neutral-900 + dark:border-b-transparent + dark:hover:bg-neutral-900; +} + +.fulltextSearchSectionSelected { + @apply rounded-b-none + border-neutral-700 + text-neutral-900 + dark:border-neutral-700 + dark:text-neutral-300; +} + +.fulltextSearchSectionCount { + @apply ml-1 + text-neutral-500 + dark:text-neutral-800; +} + +.seeAllFulltextSearchResults { + @apply m-auto + mb-2 + mt-4 + w-full + text-center + text-sm + text-neutral-700 + hover:underline + dark:text-neutral-600; +} + +.poweredBy { + @apply flex + text-xs + text-neutral-950 + dark:text-neutral-200; +} + +.poweredByLogo { + @apply ml-2 + w-16; +} + +.emptyStateContainer { + @apply flex + h-[80%] + w-full + flex-col + items-center + justify-center + text-center + text-sm + text-neutral-600 + dark:text-neutral-500; +} + +.noResultsContainer { + @apply flex + h-[80%] + w-full + items-center + justify-center + text-center + text-sm + text-neutral-600 + dark:text-neutral-500; +} + +.noResultsTerm { + @apply font-semibold; +} + +.searchErrorContainer { + @apply flex + h-[80%] + w-full + items-center + justify-center + text-center + text-sm + text-neutral-600 + dark:text-neutral-500; +} + +.fulltextSearchFooter { + @apply flex + w-full + justify-end + rounded-b-xl + border-t + border-neutral-300 + bg-neutral-100 + p-4 + dark:border-neutral-900 + dark:bg-neutral-950; +} diff --git a/components/Common/Search/index.module.css b/components/Common/Search/index.module.css new file mode 100644 index 0000000000000..cd9dbcac7e9b9 --- /dev/null +++ b/components/Common/Search/index.module.css @@ -0,0 +1,28 @@ +.searchButton { + @apply relative + w-52 + rounded-md + bg-neutral-100 + py-2 + pl-9 + pr-4 + text-left + text-sm + text-neutral-700 + transition-colors + duration-200 + ease-in-out + hover:bg-neutral-200 + hover:text-neutral-800 + dark:bg-neutral-900 + dark:text-neutral-600 + dark:hover:bg-neutral-800 + dark:hover:text-neutral-500; +} + +.magnifyingGlassIcon { + @apply absolute + left-2 + top-[8px] + size-5; +} diff --git a/components/Common/Search/index.tsx b/components/Common/Search/index.tsx new file mode 100644 index 0000000000000..b82628e3ac629 --- /dev/null +++ b/components/Common/Search/index.tsx @@ -0,0 +1,43 @@ +'use client'; + +import { MagnifyingGlassIcon } from '@heroicons/react/24/outline'; +import { useTranslations } from 'next-intl'; +import { useState, type FC } from 'react'; + +import { WithSearchBox } from '@/components/Common/Search/States/WithSearchBox'; +import { useKeyboardCommands } from '@/hooks/react-client'; + +import styles from './index.module.css'; + +export const SearchButton: FC = () => { + const [isOpen, setIsOpen] = useState(false); + const t = useTranslations(); + const openSearchBox = () => setIsOpen(true); + const closeSearchBox = () => setIsOpen(false); + + useKeyboardCommands(cmd => { + switch (cmd) { + case 'cmd-k': + openSearchBox(); + break; + case 'escape': + closeSearchBox(); + break; + default: + } + }); + + return ( + <> + + {isOpen ? : null} + + ); +}; diff --git a/components/Common/Search/utils.ts b/components/Common/Search/utils.ts new file mode 100644 index 0000000000000..ca204dda9b64f --- /dev/null +++ b/components/Common/Search/utils.ts @@ -0,0 +1,7 @@ +export const pathToBreadcrumbs = (path: string) => + path + .replace(/#.+$/, '') + .split('/') + .slice(0, -1) + .map(element => element.replaceAll('-', ' ')) + .filter(Boolean); diff --git a/components/Containers/NavBar/index.tsx b/components/Containers/NavBar/index.tsx index f465b8765804d..64ec8e21b62e5 100644 --- a/components/Containers/NavBar/index.tsx +++ b/components/Containers/NavBar/index.tsx @@ -7,6 +7,7 @@ import { useState } from 'react'; import type { FC, ComponentProps } from 'react'; import LanguageDropdown from '@/components/Common/LanguageDropDown'; +import { SearchButton } from '@/components/Common/Search'; import ThemeToggle from '@/components/Common/ThemeToggle'; import NavItem from '@/components/Containers/NavBar/NavItem'; import NodejsDark from '@/components/Icons/Logos/NodejsDark'; @@ -64,6 +65,8 @@ const NavBar: FC = ({
+ + >; +type Hit = Result; + +const SearchPage: FC = () => { + const t = useTranslations(); + const searchParams = useSearchParams(); + const [searchResults, setSearchResults] = useState(null); + const [hits, setHits] = useState>([]); + const [offset, setOffset] = useState(0); + + const searchTerm = searchParams?.get('q'); + const searchSection = searchParams?.get('section'); + + useBottomScrollListener(() => setOffset(offset => offset + 10)); + + // eslint-disable-next-line react-hooks/exhaustive-deps + useEffect(() => search(offset), [offset]); + + useEffect(() => { + setHits([]); + search(0); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [searchSection, searchTerm]); + + const uniqueHits = (newHits: Array) => + newHits.filter( + (obj, index) => newHits.findIndex(item => item.id === obj.id) === index + ); + + const search = (resultsOffset = 0) => { + oramaSearch({ + ...DEFAULT_ORAMA_QUERY_PARAMS, + mode: 'fulltext', + term: searchTerm || '', + limit: 10, + offset: resultsOffset, + ...filterBySection(), + }) + .then(results => { + setSearchResults(results); + setHits(hits => uniqueHits([...hits, ...(results?.hits ?? [])])); + }) + .catch(); + }; + + const facets = { + all: searchResults?.count ?? 0, + ...(searchResults?.facets?.siteSection?.values ?? {}), + }; + + const filterBySection = () => + searchSection && searchSection !== 'all' + ? { where: { siteSection: { eq: searchSection } } } + : {}; + + const getDocumentURL = (path: string) => + path.startsWith('api/') ? `https://nodejs.org/${path}` : path; + + return ( +
+
+

+ {t('components.search.searchPage.title', { query: searchTerm })} +

+ + +
+ +
+
+ {Object.keys(facets).map(facetName => ( + + {facetName} + + ({facets[facetName as keyof typeof facets]}) + + + ))} +
+ +
+ {hits?.map(hit => ( + +
+

+ {hit.document.pageSectionTitle} +

+ +

+ +

+ Home {'>'} {pathToBreadcrumbs(hit.document.path).join(' > ')} +
+
+ + ))} +
+
+
+ ); +}; + +export default SearchPage; diff --git a/components/withLayout.tsx b/components/withLayout.tsx index 771a57a7d702f..6f082c2c38713 100644 --- a/components/withLayout.tsx +++ b/components/withLayout.tsx @@ -15,6 +15,7 @@ import DocsLayout from '@/layouts/New/Docs'; import HomeLayout from '@/layouts/New/Home'; import LearnLayout from '@/layouts/New/Learn'; import PostLayout from '@/layouts/New/Post'; +import SearchLayout from '@/layouts/New/Search'; import { ENABLE_WEBSITE_REDESIGN } from '@/next.constants.mjs'; import type { Layouts, LegacyLayouts } from '@/types'; @@ -39,6 +40,7 @@ const redesignLayouts = { 'page.hbs': DefaultLayout, 'blog-post.hbs': PostLayout, 'blog-category.hbs': BlogLayout, + 'search.hbs': SearchLayout, } satisfies Record; type WithLayout = PropsWithChildren<{ layout: L }>; diff --git a/hooks/react-client/index.ts b/hooks/react-client/index.ts index a6cf9825ba2c2..a4423177fef31 100644 --- a/hooks/react-client/index.ts +++ b/hooks/react-client/index.ts @@ -3,3 +3,6 @@ export { default as useDetectOS } from './useDetectOS'; export { default as useMediaQuery } from './useMediaQuery'; export { default as useNotification } from './useNotification'; export { default as useClientContext } from './useClientContext'; +export { default as useKeyboardCommands } from './useKeyboardCommands'; +export { default as useClickOutside } from './useClickOutside'; +export { default as useBottomScrollListener } from './useBottomScrollListener'; diff --git a/hooks/react-client/useBottomScrollListener.ts b/hooks/react-client/useBottomScrollListener.ts new file mode 100644 index 0000000000000..a380a0e495303 --- /dev/null +++ b/hooks/react-client/useBottomScrollListener.ts @@ -0,0 +1,38 @@ +import { useState, useEffect } from 'react'; + +import { debounce } from '@/util/debounce'; + +type CallbackFunction = () => void; + +const useBottomScrollListener = ( + callback: CallbackFunction, + debounceTime = 300 +) => { + const [bottomReached, setBottomReached] = useState(false); + + const debouncedCallback = debounce(callback, debounceTime); + + const handleScroll = () => { + const scrollTop = document.documentElement.scrollTop; + const windowHeight = window.innerHeight; + const height = document.documentElement.scrollHeight; + + const bottomOfWindow = Math.ceil(scrollTop + windowHeight) >= height; + + if (bottomOfWindow) { + setBottomReached(true); + debouncedCallback(); + } else { + setBottomReached(false); + } + }; + + useEffect(() => { + window.addEventListener('scroll', handleScroll, { passive: true }); + return () => window.removeEventListener('scroll', handleScroll); + }, []); + + return bottomReached; +}; + +export default useBottomScrollListener; diff --git a/hooks/react-client/useClickOutside.ts b/hooks/react-client/useClickOutside.ts new file mode 100644 index 0000000000000..88337392b9b48 --- /dev/null +++ b/hooks/react-client/useClickOutside.ts @@ -0,0 +1,20 @@ +import type { RefObject } from 'react'; +import { useEffect } from 'react'; + +const useClickOutside = ( + ref: RefObject, + fn: () => void +) => { + useEffect(() => { + const element = ref?.current; + const handleClickOutside = (event: Event) => { + if (element && !element.contains(event.target as Node)) { + fn(); + } + }; + document.addEventListener('click', handleClickOutside); + return () => document.removeEventListener('click', handleClickOutside); + }, [ref, fn]); +}; + +export default useClickOutside; diff --git a/hooks/react-client/useKeyboardCommands.ts b/hooks/react-client/useKeyboardCommands.ts new file mode 100644 index 0000000000000..34a9123ef7baf --- /dev/null +++ b/hooks/react-client/useKeyboardCommands.ts @@ -0,0 +1,36 @@ +import { useEffect } from 'react'; + +type KeyboardCommand = 'cmd-k' | 'escape' | 'down' | 'up' | 'enter'; + +type KeyboardCommandCallback = (key: KeyboardCommand) => void; + +const useKeyboardCommands = (fn: KeyboardCommandCallback) => { + useEffect(() => { + document.addEventListener('keydown', event => { + // Detect ⌘ + k on Mac, Ctrl + k on Windows + if ((event.metaKey || event.ctrlKey) && event.key === 'k') { + event.preventDefault(); + fn('cmd-k'); + } + + switch (event.key) { + case 'Escape': + fn('escape'); + break; + case 'Enter': + fn('enter'); + break; + case 'ArrowDown': + fn('down'); + break; + case 'ArrowUp': + fn('up'); + break; + } + }); + + return () => document.removeEventListener('keydown', () => {}); + }, []); +}; + +export default useKeyboardCommands; diff --git a/i18n/locales/en.json b/i18n/locales/en.json index 584ca6ba7b178..c6854003a9a1e 100644 --- a/i18n/locales/en.json +++ b/i18n/locales/en.json @@ -200,6 +200,29 @@ "changelogModal": { "startContributing": "Start Contributing" } + }, + "search": { + "searchBox": { + "placeholder": "Start typing..." + }, + "seeAll": { + "text": "See all {count} results" + }, + "searchError": { + "text": "An error occurred while searching. Please try again later." + }, + "poweredBy": { + "text": "Powered by" + }, + "noResults": { + "text": "No results found for \"{query}\"." + }, + "emptyState": { + "text": "Search something..." + }, + "searchPage": { + "title": "You're searching: {query}" + } } }, "layouts": { diff --git a/layouts/New/Search.tsx b/layouts/New/Search.tsx new file mode 100644 index 0000000000000..e2b3bb982ef6e --- /dev/null +++ b/layouts/New/Search.tsx @@ -0,0 +1,14 @@ +import type { FC, PropsWithChildren } from 'react'; + +import WithFooter from '@/components/withFooter'; +import WithNavBar from '@/components/withNavBar'; + +const SearchLayout: FC = ({ children }) => ( + <> + +
{children}
+ + +); + +export default SearchLayout; diff --git a/next.constants.mjs b/next.constants.mjs index 1a0c07f9af41a..04beebf490015 100644 --- a/next.constants.mjs +++ b/next.constants.mjs @@ -122,7 +122,7 @@ export const BLOG_POSTS_PER_PAGE = ENABLE_WEBSITE_REDESIGN ? 6 : 20; */ export const THEME_STORAGE_KEY = 'theme'; -/*** +/** * This is a list of all external links that are used on website sitemap. * @see https://github.com/nodejs/nodejs.org/issues/5813 for more context */ @@ -135,3 +135,45 @@ export const EXTERNAL_LINKS_SITEMAP = [ 'https://trademark-list.openjsf.org/', 'https://www.linuxfoundation.org/cookies', ]; + +/** + * These are the default Orama Query Parameters that are used by the Website + * @see https://docs.oramasearch.com/open-source/usage/search/introduction + */ +export const DEFAULT_ORAMA_QUERY_PARAMS = { + mode: 'fulltext', + limit: 8, + threshold: 0, + boost: { + pageSectionTitle: 4, + pageSectionContent: 2.5, + pageTitle: 1.5, + }, + facets: { + siteSection: {}, + }, +}; + +/** + * The default batch size to use when syncing Orama Cloud + */ +export const ORAMA_SYNC_BATCH_SIZE = 50; + +/** + * The default heartbeat interval to use when communicating with Orama Cloud. + * Default should be 3500ms (3.5 seconds). + */ +export const ORAMA_CLOUD_HEARTBEAT_INTERVAL = 3500; + +/** + * The default Orama Cloud endpoint to use when searching with Orama Cloud. + */ +export const ORAMA_CLOUD_ENDPOINT = + process.env.NEXT_PUBLIC_ORAMA_ENDPOINT || + 'https://cloud.orama.run/v1/indexes/nodejs-org-dev-hhqrzv'; + +/** + * The default Orama Cloud API Key to use when searching with Orama Cloud. + * This is a public API key and can be shared publicly on the frontend. + */ +export const ORAMA_CLOUD_API_KEY = process.env.NEXT_PUBLIC_ORAMA_API_KEY || ''; diff --git a/next.mdx.use.mjs b/next.mdx.use.mjs index 0aad7237587d4..68d23b56a088b 100644 --- a/next.mdx.use.mjs +++ b/next.mdx.use.mjs @@ -11,6 +11,7 @@ import UpcomingEvents from './components/MDX/Calendar/UpcomingEvents'; import UpcomingSummits from './components/MDX/Calendar/UpcomingSummits'; import MDXCodeBox from './components/MDX/CodeBox'; import MDXCodeTabs from './components/MDX/CodeTabs'; +import SearchPage from './components/MDX/SearchPage'; import WithBadge from './components/withBadge'; import WithBanner from './components/withBanner'; import WithNodeRelease from './components/withNodeRelease'; @@ -40,6 +41,8 @@ export const mdxComponents = { DownloadLink: DownloadLink, // Renders a Button Component for `button` tags Button: Button, + // Renders a Search Page + SearchPage: SearchPage, // Renders an container for Upcoming Node.js Summits UpcomingSummits: UpcomingSummits, // Renders an container for Upcoming Node.js Events diff --git a/next.orama.mjs b/next.orama.mjs new file mode 100644 index 0000000000000..b01319925ba14 --- /dev/null +++ b/next.orama.mjs @@ -0,0 +1,37 @@ +import { Highlight } from '@orama/highlight'; +import { OramaClient } from '@oramacloud/client'; + +import { + DEFAULT_ORAMA_QUERY_PARAMS, + ORAMA_CLOUD_HEARTBEAT_INTERVAL, + ORAMA_CLOUD_ENDPOINT, + ORAMA_CLOUD_API_KEY, +} from './next.constants.mjs'; + +// Provides a safe-wrapper that initialises the OramaClient +// based on the presence of environmental variables +const { search, getInitialFacets } = (() => { + if (ORAMA_CLOUD_ENDPOINT && ORAMA_CLOUD_API_KEY) { + const orama = new OramaClient({ + endpoint: ORAMA_CLOUD_ENDPOINT, + api_key: ORAMA_CLOUD_API_KEY, + }); + + orama.startHeartBeat({ frequency: ORAMA_CLOUD_HEARTBEAT_INTERVAL }); + + return { + search: orama.search.bind(orama), + getInitialFacets: async () => + orama.search({ term: '', ...DEFAULT_ORAMA_QUERY_PARAMS }).catch(), + }; + } + + return { search: async () => null, getInitialFacets: async () => null }; +})(); + +export { search, getInitialFacets }; + +export const highlighter = new Highlight({ + CSSClass: 'font-bold', + HTMLTag: 'span', +}); diff --git a/package-lock.json b/package-lock.json index 22bf7f5d39a13..b53bab752ad1a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,5 +1,5 @@ { - "name": "nodejsorg", + "name": "nodejs.org", "lockfileVersion": 3, "requires": true, "packages": { @@ -9,6 +9,8 @@ "@heroicons/react": "~2.1.1", "@mdx-js/mdx": "^3.0.0", "@nodevu/core": "~0.1.0", + "@orama/highlight": "^0.1.3", + "@oramacloud/client": "^1.0.9", "@radix-ui/react-accessible-icon": "^1.0.3", "@radix-ui/react-avatar": "^1.0.4", "@radix-ui/react-dialog": "^1.0.5", @@ -730,7 +732,6 @@ "version": "7.23.6", "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.6.tgz", "integrity": "sha512-Z2uID7YJ7oNvAI20O9X0bblw7Qqs8Q2hFy0R9tAfnfLkp5MW0UH9eUvnDSnFwKZ0AvgS1ucqR4KzvVHgnke1VQ==", - "dev": true, "bin": { "parser": "bin/babel-parser.js" }, @@ -4032,6 +4033,17 @@ "node": ">= 10" } }, + "node_modules/@noble/hashes": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.3.3.tgz", + "integrity": "sha512-V7/fPHgl+jsVPXqqeOzT8egNj2iBIVt+ECeMMG8TdcnTikP3oaBtUVqpT/gYCR68aEBJSF+XbYUxStjbFMqIIA==", + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -4147,6 +4159,44 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/@orama/highlight": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@orama/highlight/-/highlight-0.1.3.tgz", + "integrity": "sha512-KmqMkSaGZxKnS2UiK1/nacu7+D+wadT+irgBdIBoda5BkDVPPsIXwIta0ISKmZRaM3GnUs2oKx3KteYojBkIVA==", + "dependencies": { + "@orama/orama": "^2.0.0-beta.1" + } + }, + "node_modules/@orama/orama": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@orama/orama/-/orama-2.0.3.tgz", + "integrity": "sha512-8BXTrXqP+kcyIExipZyf6voB3pzGPREh1BUrIqEP7V4PJwN/SnEcLJsafyPiPFM23fPSyH9krwLrXzvisLL19A==", + "engines": { + "node": ">= 16.0.0" + } + }, + "node_modules/@oramacloud/client": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@oramacloud/client/-/client-1.0.9.tgz", + "integrity": "sha512-qBYzppjtFfINYHoBRito8hLKJO5KbYswzZYvldBrLZoxSLrPluqt+vW4Ex8E0VhyvqPaezu8koYc79aqBLLEHA==", + "dependencies": { + "@orama/orama": "^2.0.1", + "@paralleldrive/cuid2": "^2.2.1", + "lodash": "^4.17.21", + "lodash.debounce": "^4.0.8", + "lodash.throttle": "^4.1.1", + "react": "^18.2.0", + "vue": "^3.3.4" + } + }, + "node_modules/@paralleldrive/cuid2": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/@paralleldrive/cuid2/-/cuid2-2.2.2.tgz", + "integrity": "sha512-ZOBkgDwEdoYVlSeRbYYXs0S9MejQofiVYoTbKzy/6GQa39/q5tQU2IX46+shYnUkpEl3wc+J6wRlar7r2EK2xA==", + "dependencies": { + "@noble/hashes": "^1.1.5" + } + }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", @@ -8530,6 +8580,117 @@ } } }, + "node_modules/@vue/compiler-core": { + "version": "3.4.14", + "resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.4.14.tgz", + "integrity": "sha512-ro4Zzl/MPdWs7XwxT7omHRxAjMbDFRZEEjD+2m3NBf8YzAe3HuoSEZosXQo+m1GQ1G3LQ1LdmNh1RKTYe+ssEg==", + "dependencies": { + "@babel/parser": "^7.23.6", + "@vue/shared": "3.4.14", + "entities": "^4.5.0", + "estree-walker": "^2.0.2", + "source-map-js": "^1.0.2" + } + }, + "node_modules/@vue/compiler-core/node_modules/estree-walker": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", + "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==" + }, + "node_modules/@vue/compiler-dom": { + "version": "3.4.14", + "resolved": "https://registry.npmjs.org/@vue/compiler-dom/-/compiler-dom-3.4.14.tgz", + "integrity": "sha512-nOZTY+veWNa0DKAceNWxorAbWm0INHdQq7cejFaWM1WYnoNSJbSEKYtE7Ir6lR/+mo9fttZpPVI9ZFGJ1juUEQ==", + "dependencies": { + "@vue/compiler-core": "3.4.14", + "@vue/shared": "3.4.14" + } + }, + "node_modules/@vue/compiler-sfc": { + "version": "3.4.14", + "resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-3.4.14.tgz", + "integrity": "sha512-1vHc9Kv1jV+YBZC/RJxQJ9JCxildTI+qrhtDh6tPkR1O8S+olBUekimY0km0ZNn8nG1wjtFAe9XHij+YLR8cRQ==", + "dependencies": { + "@babel/parser": "^7.23.6", + "@vue/compiler-core": "3.4.14", + "@vue/compiler-dom": "3.4.14", + "@vue/compiler-ssr": "3.4.14", + "@vue/shared": "3.4.14", + "estree-walker": "^2.0.2", + "magic-string": "^0.30.5", + "postcss": "^8.4.33", + "source-map-js": "^1.0.2" + } + }, + "node_modules/@vue/compiler-sfc/node_modules/estree-walker": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", + "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==" + }, + "node_modules/@vue/compiler-sfc/node_modules/magic-string": { + "version": "0.30.5", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.5.tgz", + "integrity": "sha512-7xlpfBaQaP/T6Vh8MO/EqXSW5En6INHEvEXQiuff7Gku0PWjU3uf6w/j9o7O+SpB5fOAkrI5HeoNgwjEO0pFsA==", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.4.15" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@vue/compiler-ssr": { + "version": "3.4.14", + "resolved": "https://registry.npmjs.org/@vue/compiler-ssr/-/compiler-ssr-3.4.14.tgz", + "integrity": "sha512-bXT6+oAGlFjTYVOTtFJ4l4Jab1wjsC0cfSfOe2B4Z0N2vD2zOBSQ9w694RsCfhjk+bC2DY5Gubb1rHZVii107Q==", + "dependencies": { + "@vue/compiler-dom": "3.4.14", + "@vue/shared": "3.4.14" + } + }, + "node_modules/@vue/reactivity": { + "version": "3.4.14", + "resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.4.14.tgz", + "integrity": "sha512-xRYwze5Q4tK7tT2J4uy4XLhK/AIXdU5EBUu9PLnIHcOKXO0uyXpNNMzlQKuq7B+zwtq6K2wuUL39pHA6ZQzObw==", + "dependencies": { + "@vue/shared": "3.4.14" + } + }, + "node_modules/@vue/runtime-core": { + "version": "3.4.14", + "resolved": "https://registry.npmjs.org/@vue/runtime-core/-/runtime-core-3.4.14.tgz", + "integrity": "sha512-qu+NMkfujCoZL6cfqK5NOfxgXJROSlP2ZPs4CTcVR+mLrwl4TtycF5Tgo0QupkdBL+2kigc6EsJlTcuuZC1NaQ==", + "dependencies": { + "@vue/reactivity": "3.4.14", + "@vue/shared": "3.4.14" + } + }, + "node_modules/@vue/runtime-dom": { + "version": "3.4.14", + "resolved": "https://registry.npmjs.org/@vue/runtime-dom/-/runtime-dom-3.4.14.tgz", + "integrity": "sha512-B85XmcR4E7XsirEHVqhmy4HPbRT9WLFWV9Uhie3OapV9m1MEN9+Er6hmUIE6d8/l2sUygpK9RstFM2bmHEUigA==", + "dependencies": { + "@vue/runtime-core": "3.4.14", + "@vue/shared": "3.4.14", + "csstype": "^3.1.3" + } + }, + "node_modules/@vue/server-renderer": { + "version": "3.4.14", + "resolved": "https://registry.npmjs.org/@vue/server-renderer/-/server-renderer-3.4.14.tgz", + "integrity": "sha512-pwSKXQfYdJBTpvWHGEYI+akDE18TXAiLcGn+Q/2Fj8wQSHWztoo7PSvfMNqu6NDhp309QXXbPFEGCU5p85HqkA==", + "dependencies": { + "@vue/compiler-ssr": "3.4.14", + "@vue/shared": "3.4.14" + }, + "peerDependencies": { + "vue": "3.4.14" + } + }, + "node_modules/@vue/shared": { + "version": "3.4.14", + "resolved": "https://registry.npmjs.org/@vue/shared/-/shared-3.4.14.tgz", + "integrity": "sha512-nmi3BtLpvqXAWoRZ6HQ+pFJOHBU4UnH3vD3opgmwXac7vhaHKA9nj1VeGjMggdB9eLtW83eHyPCmOU1qzdsC7Q==" + }, "node_modules/@webassemblyjs/ast": { "version": "1.11.6", "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.6.tgz", @@ -11309,8 +11470,7 @@ "node_modules/csstype": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz", - "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==", - "devOptional": true + "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==" }, "node_modules/damerau-levenshtein": { "version": "1.0.8", @@ -12031,7 +12191,6 @@ "version": "4.5.0", "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", - "dev": true, "engines": { "node": ">=0.12" }, @@ -18166,14 +18325,12 @@ "node_modules/lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", - "dev": true + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "node_modules/lodash.debounce": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", - "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", - "dev": true + "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==" }, "node_modules/lodash.merge": { "version": "4.6.2", @@ -18181,6 +18338,11 @@ "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, + "node_modules/lodash.throttle": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.throttle/-/lodash.throttle-4.1.1.tgz", + "integrity": "sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ==" + }, "node_modules/lodash.truncate": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz", @@ -34403,6 +34565,26 @@ "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==", "dev": true }, + "node_modules/vue": { + "version": "3.4.14", + "resolved": "https://registry.npmjs.org/vue/-/vue-3.4.14.tgz", + "integrity": "sha512-Rop5Al/ZcBbBz+KjPZaZDgHDX0kUP4duEzDbm+1o91uxYUNmJrZSBuegsNIJvUGy+epLevNRNhLjm08VKTgGyw==", + "dependencies": { + "@vue/compiler-dom": "3.4.14", + "@vue/compiler-sfc": "3.4.14", + "@vue/runtime-dom": "3.4.14", + "@vue/server-renderer": "3.4.14", + "@vue/shared": "3.4.14" + }, + "peerDependencies": { + "typescript": "*" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, "node_modules/w3c-xmlserializer": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-4.0.0.tgz", diff --git a/package.json b/package.json index 01662c45b1c0c..217613f56160a 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "prettier": "prettier \"**/*.{js,mjs,ts,tsx,md,mdx,json,yml,css}\" --check --cache --cache-strategy=content --cache-location=.prettiercache", "prettier:fix": "npm run prettier -- --write", "format": "npm run lint:fix && npm run prettier:fix", + "sync-orama": "node ./scripts/orama-search/sync-orama-cloud.mjs", "storybook": "cross-env NODE_NO_WARNINGS=1 storybook dev -p 6006 --quiet --no-open", "storybook:build": "cross-env NODE_NO_WARNINGS=1 storybook build --quiet --webpack-stats-json", "test:unit": "cross-env NODE_NO_WARNINGS=1 jest", @@ -40,6 +41,8 @@ "@heroicons/react": "~2.1.1", "@mdx-js/mdx": "^3.0.0", "@nodevu/core": "~0.1.0", + "@orama/highlight": "^0.1.3", + "@oramacloud/client": "^1.0.9", "@radix-ui/react-accessible-icon": "^1.0.3", "@radix-ui/react-avatar": "^1.0.4", "@radix-ui/react-dialog": "^1.0.5", diff --git a/pages/en/search.mdx b/pages/en/search.mdx new file mode 100644 index 0000000000000..211fe28e6084e --- /dev/null +++ b/pages/en/search.mdx @@ -0,0 +1,6 @@ +--- +layout: search.hbs +title: Search Results +--- + + diff --git a/scripts/orama-search/get-documents.mjs b/scripts/orama-search/get-documents.mjs new file mode 100644 index 0000000000000..617ea11301a91 --- /dev/null +++ b/scripts/orama-search/get-documents.mjs @@ -0,0 +1,76 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import zlib from 'node:zlib'; + +import { slug } from 'github-slugger'; + +import { getRelativePath } from '../../next.helpers.mjs'; + +const currentRoot = getRelativePath(import.meta.url); +const dataBasePath = join(currentRoot, '../../.next/server/app/en/next-data'); + +if (!existsSync(dataBasePath)) { + throw new Error( + 'The data directory does not exist. Please run `npm run build` first.' + ); +} + +const nextPageData = readFileSync(`${dataBasePath}/page-data.body`, 'utf-8'); +const nextAPIPageData = readFileSync(`${dataBasePath}/api-data.body`, 'utf-8'); + +const pageData = JSON.parse(nextPageData); +const apiData = JSON.parse(nextAPIPageData); + +const splitIntoSections = markdownContent => { + const lines = markdownContent.split(/\n/gm); + const sections = []; + + let section = null; + + for (const line of lines) { + if (line.match(/^#{1,6}\s/)) { + section = { + pageSectionTitle: line.replace(/^#{1,6}\s*/, ''), + pageSectionContent: [], + }; + + sections.push(section); + } else if (section) { + section.pageSectionContent.push(line); + } + } + + return sections.map(section => ({ + ...section, + pageSectionContent: section.pageSectionContent.join('\n'), + })); +}; + +const getPageTitle = data => + data.title || + data.pathname + .split('/') + .pop() + .replace(/\.html$/, '') + .replace(/-/g, ' '); + +export const siteContent = [...pageData, ...apiData] + .map(data => { + const { pathname, title = getPageTitle(data), content } = data; + const markdownContent = zlib + .inflateSync(Buffer.from(content, 'base64')) + .toString('utf-8'); + + const siteSection = pathname.split('/').shift(); + const subSections = splitIntoSections(markdownContent); + + return subSections.map(section => { + return { + path: pathname + '#' + slug(section.pageSectionTitle), + siteSection, + pageTitle: title, + ...section, + }; + }); + }) + .flat(); diff --git a/scripts/orama-search/sync-orama-cloud.mjs b/scripts/orama-search/sync-orama-cloud.mjs new file mode 100644 index 0000000000000..91a734cc13877 --- /dev/null +++ b/scripts/orama-search/sync-orama-cloud.mjs @@ -0,0 +1,65 @@ +import { siteContent } from './get-documents.mjs'; +import { ORAMA_SYNC_BATCH_SIZE } from '../../next.constants.mjs'; + +// The following follows the instructions at https://docs.oramasearch.com/cloud/data-sources/custom-integrations/webhooks + +const INDEX_ID = process.env.ORAMA_INDEX_ID; +const API_KEY = process.env.ORAMA_SECRET_KEY; +const ORAMA_API_BASE_URL = `https://api.oramasearch.com/api/v1/webhooks/${INDEX_ID}`; + +const oramaHeaders = { + 'Content-Type': 'application/json', + Authorization: `Bearer ${API_KEY}`, +}; + +// Orama allows to send several documents at once, so we batch them in groups of 50. +// This is not strictly necessary, but it makes the process faster. +const runUpdate = async () => { + const batchSize = ORAMA_SYNC_BATCH_SIZE; + const batches = []; + + for (let i = 0; i < siteContent.length; i += batchSize) { + batches.push(siteContent.slice(i, i + batchSize)); + } + + await Promise.all(batches.map(insertBatch)); +}; + +// We call the "notify" API to upsert the documents in the index. +// Orama will keep a queue of all the documents we send, and will process them once we call the "deploy" API. +// Full docs on the "notify" API: https://docs.oramasearch.com/cloud/data-sources/custom-integrations/webhooks#updating-removing-inserting-elements-in-a-live-index +const insertBatch = async batch => + await fetch(`${ORAMA_API_BASE_URL}/notify`, { + method: 'POST', + headers: oramaHeaders, + body: JSON.stringify({ upsert: batch }), + }); + +// We call the "deploy" API to trigger a deployment of the index, which will process all the documents in the queue. +// Full docs on the "deploy" API: https://docs.oramasearch.com/cloud/data-sources/custom-integrations/webhooks#deploying-the-index +const triggerDeployment = async () => + await fetch(`${ORAMA_API_BASE_URL}/deploy`, { + method: 'POST', + headers: oramaHeaders, + }); + +// We call the "snapshot" API to empty the index before inserting the new documents. +// The "snapshot" API is typically used to replace the entire index with a fresh set of documents, but we use it here to empty the index. +// This operation gets queued, so the live index will still be available until we call the "deploy" API and redeploy the index. +// Full docs on the "snapshot" API: https://docs.oramasearch.com/cloud/data-sources/custom-integrations/webhooks#inserting-a-snapshot +const emptyOramaIndex = async () => + await fetch(`${ORAMA_API_BASE_URL}/snapshot`, { + method: 'POST', + headers: oramaHeaders, + body: JSON.stringify([]), + }); + +// Now we proceed to call the APIs in order: +// 1. Empty the index +// 2. Insert the documents +// 3. Trigger a deployment +// Once all these steps are done, the new documents will be available in the live index. +// Allow Orama up to 1 minute to distribute the documents to all the 300+ nodes worldwide. +await emptyOramaIndex(); +await runUpdate(); +await triggerDeployment(); diff --git a/turbo.json b/turbo.json index 808bffd7ab199..f702d2230c4e5 100644 --- a/turbo.json +++ b/turbo.json @@ -13,6 +13,8 @@ "NEXT_PUBLIC_DIST_URL", "NEXT_PUBLIC_DOCS_URL", "NEXT_PUBLIC_BASE_PATH", + "NEXT_PUBLIC_ORAMA_API_KEY", + "NEXT_PUBLIC_ORAMA_ENDPOINT", "NEXT_PUBLIC_VERCEL_REVALIDATE_TIME", "NEXT_PUBLIC_ENABLE_REDESIGN", "NEXT_PUBLIC_DATA_URL" @@ -35,6 +37,8 @@ "NEXT_PUBLIC_DIST_URL", "NEXT_PUBLIC_DOCS_URL", "NEXT_PUBLIC_BASE_PATH", + "NEXT_PUBLIC_ORAMA_API_KEY", + "NEXT_PUBLIC_ORAMA_ENDPOINT", "NEXT_PUBLIC_VERCEL_REVALIDATE_TIME", "NEXT_PUBLIC_ENABLE_REDESIGN", "NEXT_PUBLIC_DATA_URL" @@ -51,6 +55,8 @@ "NEXT_PUBLIC_DIST_URL", "NEXT_PUBLIC_DOCS_URL", "NEXT_PUBLIC_BASE_PATH", + "NEXT_PUBLIC_ORAMA_API_KEY", + "NEXT_PUBLIC_ORAMA_ENDPOINT", "NEXT_PUBLIC_VERCEL_REVALIDATE_TIME", "NEXT_PUBLIC_ENABLE_REDESIGN", "NEXT_PUBLIC_DATA_URL" @@ -73,6 +79,8 @@ "NEXT_PUBLIC_DIST_URL", "NEXT_PUBLIC_DOCS_URL", "NEXT_PUBLIC_BASE_PATH", + "NEXT_PUBLIC_ORAMA_API_KEY", + "NEXT_PUBLIC_ORAMA_ENDPOINT", "NEXT_PUBLIC_VERCEL_REVALIDATE_TIME", "NEXT_PUBLIC_ENABLE_REDESIGN", "NEXT_PUBLIC_DATA_URL" diff --git a/types/index.ts b/types/index.ts index c26a8e205a2b9..567d44d0e088a 100644 --- a/types/index.ts +++ b/types/index.ts @@ -10,3 +10,4 @@ export * from './redirects'; export * from './server'; export * from './github'; export * from './calendar'; +export * from './search'; diff --git a/types/layouts.ts b/types/layouts.ts index ec7c518c6bc92..41177c34d9858 100644 --- a/types/layouts.ts +++ b/types/layouts.ts @@ -6,7 +6,8 @@ export type Layouts = | 'learn.hbs' | 'page.hbs' | 'blog-category.hbs' - | 'blog-post.hbs'; + | 'blog-post.hbs' + | 'search.hbs'; // @TODO: These are legacy layouts that are going to be replaced with the `nodejs/nodejs.dev` Layouts in the future export type LegacyLayouts = diff --git a/types/search.ts b/types/search.ts new file mode 100644 index 0000000000000..03ac4e67a4a18 --- /dev/null +++ b/types/search.ts @@ -0,0 +1,8 @@ +export interface SearchDoc { + id: string; + path: string; + pageTitle: string; + siteSection: string; + pageSectionTitle: string; + pageSectionContent: string; +} diff --git a/util/__tests__/stringUtils.test.mjs b/util/__tests__/stringUtils.test.mjs index 5a04cd095c606..05362d4938f7c 100644 --- a/util/__tests__/stringUtils.test.mjs +++ b/util/__tests__/stringUtils.test.mjs @@ -1,4 +1,7 @@ -import { getAcronymFromString } from '@/util/stringUtils'; +import { + getAcronymFromString, + parseRichTextIntoPlainText, +} from '@/util/stringUtils'; describe('String utils', () => { it('getAcronymFromString returns the correct acronym', () => { @@ -12,4 +15,50 @@ describe('String utils', () => { it('getAcronymFromString if the string is empty, it returns NA', () => { expect(getAcronymFromString('')).toBe(''); }); + + it('parseRichTextIntoPlainText returns the correct plain text from an HTML tag', () => { + expect(parseRichTextIntoPlainText('

John Doe

')).toBe('John Doe'); + }); + + it('parseRichTextIntoPlainText returns only the text of a link tag', () => { + expect( + parseRichTextIntoPlainText('[this is a link](https://www.google.com)') + ).toBe('this is a link'); + }); + + it('parseRichTextIntoPlainText replaces markdown lists with their content', () => { + expect( + parseRichTextIntoPlainText('- this is a list item\n- this is another') + ).toBe('this is a list item\nthis is another'); + }); + + it('parseRichTextIntoPlainText removes underscore, bold and italic with their content', () => { + expect( + parseRichTextIntoPlainText( + '**bold content**, *italic content*, _underscore content_' + ) + ).toBe('bold content, italic content, underscore content'); + }); + + it('parseRichTextIntoPlainText removes code blocks with their content', () => { + expect( + parseRichTextIntoPlainText('this is a\n```code block```\nwith content') + ).toBe('this is a\nwith content'); + }); + + it('parseRichTextIntoPlainText replaces empty lines or lines just with spaces with an empty string', () => { + expect(parseRichTextIntoPlainText('\n \n')).toBe(''); + }); + + it('parseRichTextIntoPlainText replaces leading and trailing spaces from each line with an empty string', () => { + expect(parseRichTextIntoPlainText(' this is a line ')).toBe( + 'this is a line' + ); + }); + + it('parseRichTextIntoPlainText replaces leading numbers and dots from each line with an empty string', () => { + expect( + parseRichTextIntoPlainText('1. this is a line\n2. this is a second line') + ).toBe('this is a line\nthis is a second line'); + }); }); diff --git a/util/debounce.ts b/util/debounce.ts new file mode 100644 index 0000000000000..73472b35e07e4 --- /dev/null +++ b/util/debounce.ts @@ -0,0 +1,16 @@ +type DebounceFunction = (...args: Array) => void; + +export const debounce = ( + func: T, + delay: number +): ((...args: Parameters) => void) => { + let timeoutId: NodeJS.Timeout; + + return (...args: Parameters) => { + clearTimeout(timeoutId); + + timeoutId = setTimeout(() => { + func(...args); + }, delay); + }; +}; diff --git a/util/stringUtils.ts b/util/stringUtils.ts index 73d825e053c44..4dc48a046bf19 100644 --- a/util/stringUtils.ts +++ b/util/stringUtils.ts @@ -12,10 +12,12 @@ export const parseRichTextIntoPlainText = (richText: string) => // replaces Markdown lists with their content .replace(/^[*-] (.*)$/gm, '$1') // replaces Markdown underscore, bold and italic with their content - .replace(/[_*]{1,2}(.*)[_*]{1,2}/gm, '$1') + .replace(/(\*\*|\*|__|_)(.*?)\1/gm, '$2') // replaces Markdown multiline codeblocks with their content .replace(/```.+?```/gms, '') - // replaces emppty lines or lines just with spaces with an empty string + // replaces empty lines or lines just with spaces with an empty string .replace(/^\s*\n/gm, '') // replaces leading and trailing spaces from each line with an empty string - .replace(/^[ ]+|[ ]+$/gm, ''); + .replace(/^[ ]+|[ ]+$/gm, '') + // replaces leading numbers and dots from each line with an empty string + .replace(/^\d+\.\s/gm, '');