feat: fetch paths from UDR all-docs-paths (#2654)

* feat: fetch paths from UDR all-docs-paths

* add check for sandbox env

* PR feedback: only show paths from products in config

* filter results from content API

* add tests

* remove comment

src/lib/__tests__/docs-content-fields.test.ts

@@ -0,0 +1,79 @@
+import { describe, it, expect, vi } from 'vitest'
+import { allDocsFields } from '../sitemap/docs-content-fields'
+
+describe('allDocsFields', () => {
+	beforeEach(() => {
+		vi.resetAllMocks()
+	})
+
+	it('should fetch and return all docs fields from content API when unified-docs-sandbox is not set', async () => {
+		process.env.HASHI_ENV = 'production'
+		process.env.MKTG_CONTENT_DOCS_API = 'https://content-api.example.com'
+		const mockContentAPIDocsResult = [
+			{ path: 'doc1', created_at: '2025-01-07T18:44:51.431Z' },
+			{ path: 'doc2', created_at: '2025-01-07T18:44:51.431Z' },
+		]
+		global.fetch = vi.fn().mockResolvedValue({
+			json: vi.fn().mockResolvedValue({ result: mockContentAPIDocsResult }),
+		})
+
+		const result = await allDocsFields()
+
+		expect(fetch).toHaveBeenCalledWith(
+			'https://content-api.example.com/api/all-docs-paths'
+		)
+		expect(result).toEqual(
+			mockContentAPIDocsResult.map((page) => ({
+				loc: `https://developer.hashicorp.com/${page.path}`,
+				lastmod: page.created_at,
+				priority: 1,
+				changefreq: 'daily',
+			}))
+		)
+	})
+
+	it('should fetch and return all docs fields from both content API and UDR when unified-docs-sandbox is set', async () => {
+		process.env.HASHI_ENV = 'unified-docs-sandbox'
+		process.env.MKTG_CONTENT_DOCS_API = 'https://content-api.example.com'
+		process.env.UNIFIED_DOCS_API = 'https://udr-api.example.com'
+		__config.flags = {
+			enable_datadog: false,
+			enable_io_beta_cta: false,
+			enable_hvd_on_preview_branch: false,
+			unified_docs_migrated_repos: ['repo1', 'repo2'],
+		}
+		const mockContentAPIDocsResult = [
+			{ path: 'doc1', created_at: '2025-01-07T18:44:51.431Z' },
+			{ path: 'doc2', created_at: '2025-01-07T18:44:51.431Z' },
+		]
+		const mockUDRDocsResult = [
+			{ path: 'udr-doc1', created_at: '2025-01-07T18:44:51.431Z' },
+			{ path: 'udr-doc2', created_at: '2023-01-04' },
+		]
+		global.fetch = vi
+			.fn()
+			.mockResolvedValueOnce({
+				json: vi.fn().mockResolvedValue({ result: mockContentAPIDocsResult }),
+			})
+			.mockResolvedValueOnce({
+				json: vi.fn().mockResolvedValue({ result: mockUDRDocsResult }),
+			})
+
+		const result = await allDocsFields()
+
+		expect(fetch).toHaveBeenCalledWith(
+			'https://content-api.example.com/api/all-docs-paths?filterOut=repo1&filterOut=repo2'
+		)
+		expect(fetch).toHaveBeenCalledWith(
+			'https://udr-api.example.com/api/all-docs-paths?products=repo1&products=repo2'
+		)
+		expect(result).toEqual(
+			[...mockContentAPIDocsResult, ...mockUDRDocsResult].map((page) => ({
+				loc: `https://developer.hashicorp.com/${page.path}`,
+				lastmod: page.created_at,
+				priority: 1,
+				changefreq: 'daily',
+			}))
+		)
+	})
+})

src/lib/sitemap/docs-content-fields.ts

@@ -6,14 +6,51 @@
 import { makeSitemapField } from './helpers'
 
 export async function allDocsFields() {
-	const getDocsPaths = await fetch(
+	// If there are docs that have been migrated to the unified docs repo, get the paths for those docs
+	// and merge them with the paths for the docs that haven't been migrated from the content API
+	if (
+		process.env.HASHI_ENV === 'unified-docs-sandbox' &&
+		__config.flags?.unified_docs_migrated_repos.length > 0
+	) {
+		const contentAPIFilter = __config.flags.unified_docs_migrated_repos.map(
+			(repo) => {
+				return `filterOut=${repo}`
+			}
+		)
+		const contentAPIFilterString = contentAPIFilter.join('&')
+		const getContentAPIDocsPaths = await fetch(
+			`${process.env.MKTG_CONTENT_DOCS_API}/api/all-docs-paths?${contentAPIFilterString}`
+		)
+		const { result: contentAPIDocsResult } = await getContentAPIDocsPaths.json()
+
+		const UDRFilter = __config.flags.unified_docs_migrated_repos.map((repo) => {
+			return `products=${repo}`
+		})
+		const UDRFilterString = UDRFilter.join('&')
+		const getUDRDocsPaths = await fetch(
+			`${process.env.UNIFIED_DOCS_API}/api/all-docs-paths?${UDRFilterString}`
+		)
+		const { result: udrDocsResult } = await getUDRDocsPaths.json()
+
+		const allDocsData = [...contentAPIDocsResult, ...udrDocsResult]
+		return allDocsData.map((page: { path: string; created_at: string }) =>
+			makeSitemapField({
+				slug: `${page.path}`,
+				lastmodDate: page.created_at,
+			})
+		)
+	}
+
+	const getContentAPIDocsPaths = await fetch(
 		`${process.env.MKTG_CONTENT_DOCS_API}/api/all-docs-paths`
 	)
-	const { result: docsResult } = await getDocsPaths.json()
-	return docsResult.map((page: { path: string; created_at: string }) =>
-		makeSitemapField({
-			slug: `${page.path}`,
-			lastmodDate: page.created_at,
-		})
+	const { result: contentAPIDocsResult } = await getContentAPIDocsPaths.json()
+
+	return contentAPIDocsResult.map(
+		(page: { path: string; created_at: string }) =>
+			makeSitemapField({
+				slug: `${page.path}`,
+				lastmodDate: page.created_at,
+			})
 	)
 }