feat: crowdin updating

.github/renovate.json

@@ -6,14 +6,14 @@
 	"packageRules": [
 		{
 			"groupName": "patched packages",
-			"matchPackageNames": ["@crowdin/ota-client", "trpc-panel", "json-schema-to-zod"],
+			"matchDepNames": ["@crowdin/ota-client", "trpc-panel", "json-schema-to-zod"],
 			"matchUpdateTypes": ["major", "minor", "patch"]
 		},
 		{
 			"enabled": false,
 			"groupName": "Ignored Versions",
 			"matchCurrentVersion": "0.9.2",
-			"matchPackageNames": ["@t3-oss/env-nextjs"]
+			"matchDepNames": ["@t3-oss/env-nextjs"]
 		}
 	],
 	"semanticCommitScope": "{{parentDir}}"

apps/app/lib/generate.ts

@@ -14,19 +14,21 @@ import { generateTranslationKeys } from 'lib/generators'
 const program = new Command()
 
 export type PassedTask = ListrTaskWrapper<unknown, ListrDefaultRenderer, ListrSimpleRenderer>
+type TaskDef = ListrTask<unknown, ListrDefaultRenderer, ListrSimpleRenderer>
 
-const options = {
+const rendererOptions: TaskDef['rendererOptions'] = {
 	bottomBar: 10,
 	persistentOutput: true,
+	outputBar: true,
 }
 const translation = [
 	{
 		title: 'Translation definitions from DB',
 		task: (_ctx: ListrContext, task: PassedTask) => generateTranslationKeys(task),
 		skip: !process.env.DATABASE_URL,
-		options,
+		rendererOptions,
 	},
-]
+] satisfies TaskDef[]
 
 program
 	.name('generate')
@@ -47,6 +49,7 @@ if (Object.keys(cliOpts).length === 0) {
 
 const tasks = new Listr(tasklist, {
 	exitOnError: false,
+	rendererOptions: { collapseSubtasks: false },
 })
 
 tasks.run()

apps/app/lib/generators/translationKeys.ts

@@ -21,27 +21,28 @@ const isObject = (data: unknown): data is Record<string, string> =>
 
 const countKeys = (obj: Output): number => Object.keys(flatten(obj)).length
 
-export const generateTranslationKeys = async (task: PassedTask) => {
-	const prettierOpts = (await prettier.resolveConfig(__filename)) ?? undefined
-
-	const where = (): Prisma.TranslationNamespaceWhereInput | undefined => {
-		switch (true) {
-			case !!process.env.EXPORT_ALL: {
-				return undefined
-			}
-			case !!process.env.EXPORT_DB: {
-				return { name: 'org-data' }
-			}
-			default: {
-				return { exportFile: true }
+const where = (): Prisma.TranslationNamespaceWhereInput | undefined => {
+	switch (true) {
+		case !!process.env.EXPORT_ALL: {
+			return undefined
+		}
+		case !!process.env.EXPORT_DB: {
+			return {
+				name: 'org-data',
 			}
 		}
+		default: {
+			return { exportFile: true }
+		}
 	}
+}
 
-	const data = await prisma.translationNamespace.findMany({
+const getKeysFromDb = async () =>
+	await prisma.translationNamespace.findMany({
 		where: where(),
 		include: {
 			keys: {
+				...(!process.env.EXPORT_INACTIVE && { where: { active: true } }),
 				orderBy: {
 					key: 'asc',
 				},
@@ -51,52 +52,59 @@ export const generateTranslationKeys = async (task: PassedTask) => {
 			name: 'asc',
 		},
 	})
+
+type DBKeys = Prisma.PromiseReturnType<typeof getKeysFromDb>[number]['keys']
+
+const processKeys = (keys: DBKeys) => {
+	const outputData: Output = {}
+	for (const item of keys) {
+		if (item.interpolation && isObject(item.interpolationValues)) {
+			for (const [context, textContent] of Object.entries(item.interpolationValues)) {
+				if (typeof textContent !== 'string') {
+					throw new Error('Invalid nested plural item')
+				}
+				outputData[`${item.key}_${context}`] = textContent
+			}
+		}
+		if (!item.interpolation || item.interpolation === 'CONTEXT') {
+			outputData[item.key] = item.text
+		}
+	}
+	return outputData
+}
+
+export const generateTranslationKeys = async (task: PassedTask) => {
+	const prettierConfig = (await prettier.resolveConfig(__filename, { editorconfig: true })) ?? undefined
+	const prettierOpts = prettierConfig ? { ...prettierConfig, parser: 'json' } : undefined
+	const data = await getKeysFromDb()
 	let logMessage = ''
 	let i = 0
 	task.output = `Fetched ${data.length} namespaces from DB`
 	for (const namespace of data) {
-		const outputData: Output = {}
-		for (const item of namespace.keys) {
-			if (item.interpolation && isObject(item.interpolationValues)) {
-				for (const [key, value] of Object.entries(item.interpolationValues)) {
-					if (typeof value !== 'string') {
-						throw new Error('Invalid nested plural item')
-					}
-					outputData[`${item.key}_${key}`] = value
-				}
-			}
-			if (item.ns === 'attribute') {
-				outputData[item.key] = item.text
-			}
-		}
+		const outputData = processKeys(namespace.keys)
 		const filename = `${localePath}/${namespace.name}.json`
 
 		let existingFile: unknown = {}
-		if (fs.existsSync(filename)) {
+		if (fs.existsSync(filename) && !namespace.overwriteFileOnExport) {
 			existingFile = flatten(JSON.parse(fs.readFileSync(filename, 'utf-8')))
 		}
 		if (!isOutput(existingFile)) {
 			throw new Error("tried to load file, but it's empty")
 		}
-		// const existingLength = Object.keys(existingFile).length
+
 		const existingLength = countKeys(existingFile)
 		let outputFile: Output = unflatten(Object.assign(existingFile, outputData), { overwrite: true })
 		outputFile = Object.keys(outputFile)
-			.sort((a, b) => a.localeCompare(b))
+			.toSorted((a, b) => a.localeCompare(b))
 			.reduce((obj: Record<string, string>, key) => {
 				obj[key] = outputFile[key] as string
 				return obj
 			}, {})
 
 		const newKeys = countKeys(outputFile) - existingLength
-		logMessage = `${filename} generated with ${newKeys} new ${newKeys === 1 ? 'key' : 'keys'}.`
-
-		const formattedOutput = await prettier.format(JSON.stringify(outputFile), {
-			...prettierOpts,
-			parser: 'json',
-		})
-		fs.writeFileSync(filename, formattedOutput)
-
+		logMessage = `${filename} generated with ${newKeys} ${namespace.overwriteFileOnExport ? 'total' : 'new'} ${newKeys === 1 ? 'key' : 'keys'}.`
+		const formattedOutput = await prettier.format(JSON.stringify(outputFile), prettierOpts)
+		fs.writeFileSync(filename, formattedOutput, 'utf-8')
 		task.output = logMessage
 		i++
 	}

apps/app/package.json

@@ -13,8 +13,8 @@
 		"postdev": "pnpm -w docker:down",
 		"dev:verbose": "NEXT_VERBOSE=1 next dev",
 		"format": "prettier --write --ignore-unknown .",
-		"generate:all": "tsx ./lib/generate.ts",
-		"generate:i18n": "tsx ./lib/generate.ts -t",
+		"generate:all": "pnpm with-env tsx ./lib/generate.ts",
+		"generate:i18n": "pnpm with-env tsx ./lib/generate.ts -t",
 		"generate:i18nTypes": "i18next-resources-for-ts interface -i ./public/locales/en -o ./src/types/resources.d.ts",
 		"preinstall": "npx only-allow pnpm",
 		"lint": "next lint",

apps/app/src/pages/api/i18n/load.ts

@@ -17,8 +17,8 @@ export const config = {
 }
 
 const QuerySchema = z.object({
-	lng: z.string(),
-	ns: z.string(),
+	lng: z.string().transform((s) => s.split(' ')),
+	ns: z.string().transform((s) => s.split(' ')),
 })
 const tracer = trace.getTracer('inreach-app')
 const log = createLoggerInstance('i18n Loader')
@@ -34,10 +34,9 @@ export default async function handler(req: NextRequest) {
 			})
 		}
 		const query = parsedQuery.data
-		const namespaces = query.ns.split(' ')
-		const langs = query.lng.split(' ')
+		const { ns: namespaces, lng: langs } = query
 		const cacheWriteQueue: WriteCacheArgs[] = []
-		const otaManifestTimestamp = await crowdinDistTimestamp()
+		const otaManifestTimestamps = await crowdinDistTimestamp()
 		const results = new Map<string, object>()
 
 		for (const lang of langs) {
@@ -46,19 +45,20 @@ export default async function handler(req: NextRequest) {
 				if (lang === 'en') {
 					continue
 				}
-				const databaseFile = sourceFiles(lang).databaseStrings
-				const cached = await redisReadCache(namespaces, lang, otaManifestTimestamp)
+				const cached = await redisReadCache(namespaces, lang, otaManifestTimestamps)
 				const langResult = new Map<string, object | string>(cached)
 
 				const fetchCrowdin = async (ns: string) => {
 					const crowdinSpan = tracer.startSpan('Crowdin OTA', undefined, context.active())
 					try {
 						crowdinSpan.setAttributes({ ns })
 						switch (true) {
+							// Check if the namespace is already in the cache
 							case langResult.has(ns): {
 								return
 							}
-							case Object.hasOwn(nsFileMap, ns): {
+							// Check if the namespace is file based
+							case ns in nsFileMap: {
 								const file = nsFileMap[ns as keyof typeof nsFileMap] ?? ''
 								const strings = await fetchCrowdinFile(file, lang)
 								if (strings && Object.keys(strings).length) {
@@ -67,9 +67,9 @@ export default async function handler(req: NextRequest) {
 								langResult.set(ns, strings)
 								break
 							}
+							// Otherwise, it must be a database key
 							default: {
-								const file = databaseFile
-								const strings = await fetchCrowdinDbKey(ns, file, lang)
+								const strings = await fetchCrowdinDbKey(ns, lang)
 								if (strings) {
 									cacheWriteQueue.push({ lang, ns, strings })
 								}

packages/api/router/orgEmail/mutation.create.handler.ts

@@ -1,14 +1,27 @@
+import { addSingleKey } from '@weareinreach/crowdin/api'
 import { getAuditedClient } from '@weareinreach/db'
 import { type TRPCHandlerParams } from '~api/types/handler'
 
 import { type TCreateSchema } from './mutation.create.schema'
 
 const create = async ({ ctx, input }: TRPCHandlerParams<TCreateSchema, 'protected'>) => {
 	const prisma = getAuditedClient(ctx.actorId)
-	const newEmail = await prisma.orgEmail.create({
-		data: input,
-		select: { id: true },
+
+	const result = await prisma.$transaction(async (tx) => {
+		if (input.description) {
+			const crowdinId = await addSingleKey({
+				isDatabaseString: true,
+				key: input.description.create.tsKey.create.key,
+				text: input.description.create.tsKey.create.text,
+			})
+			input.description.create.tsKey.create.crowdinId = crowdinId.id
+		}
+		const newEmail = await tx.orgEmail.create({
+			data: input,
+			select: { id: true },
+		})
+		return newEmail
 	})
-	return newEmail
+	return result
 }
 export default create

packages/api/router/orgEmail/mutation.create.schema.ts

@@ -26,27 +26,33 @@ export const ZCreateSchema = z
 	.transform(({ orgId, data, title, titleId, description }) => {
 		const id = generateId('orgEmail')
 
+		const handleTitle = () => {
+			if (title) {
+				return {
+					create: {
+						title,
+						key: {
+							create: {
+								text: title,
+								key: slug(title),
+								namespace: { connect: { name: namespace.userTitle } },
+							},
+						},
+					},
+				}
+			}
+			if (titleId) {
+				return { connect: { id: titleId } }
+			}
+			return undefined
+		}
+
 		return Prisma.validator<Prisma.OrgEmailCreateInput>()({
 			...data,
 			description: description
 				? generateNestedFreeText({ orgId, itemId: id, text: description, type: 'emailDesc' })
 				: undefined,
-			title: title
-				? {
-						create: {
-							title,
-							key: {
-								create: {
-									text: title,
-									key: slug(title),
-									namespace: { connect: { name: namespace.userTitle } },
-								},
-							},
-						},
-					}
-				: titleId
-					? { connect: { id: titleId } }
-					: undefined,
+			title: handleTitle(),
 		})
 	})
 export type TCreateSchema = z.infer<typeof ZCreateSchema>