Setup Inngest. Create thumbhash Inngest function.

inngest/client.ts

@@ -0,0 +1,7 @@
+import { Inngest, EventSchemas } from 'inngest';
+import { type Events } from './events';
+
+export const inngest = new Inngest({
+  id: 'book-inventory',
+  schemas: new EventSchemas().fromRecord<Events>(),
+});

inngest/events.ts

@@ -0,0 +1,10 @@
+type BookCreated = {
+  data: {
+    title: string;
+    image_url: string | null;
+  };
+};
+
+export type Events = {
+  'book.created': BookCreated;
+};

inngest/functions/generateThumbhash.ts

@@ -0,0 +1,47 @@
+import { inngest } from '../client';
+import { processBook, updateThumbhashQuery } from '@/lib/db/seed-thumbhash';
+import { sql } from '@/lib/db/drizzle';
+
+export const generateThumbhash = inngest.createFunction(
+  {
+    id: 'generate-thumbhash',
+    concurrency: 10,
+    batchEvents: {
+      maxSize: 100,
+      timeout: '60s',
+    },
+  },
+  { event: 'book.created' },
+  async ({ events, step }) => {
+    let queryValues = [];
+
+    // Loop over all books in this batch
+    for (const event of events) {
+      // Fetch the image and generate the thumbhash in a single step
+      // Errors will be retried automatically
+      // Successful results will be cached if a later step fails
+      const result = await step.run('generate-thumbhash', async () => {
+        return await processBook(event.data);
+      });
+      if (result) {
+        queryValues.push(result);
+      }
+    }
+
+    // Build the query
+    const queries = queryValues.map((values) => {
+      return sql(updateThumbhashQuery, values);
+    });
+
+    // Execute the db transaction within a step as it's a side effect
+    // and should be retried if it fails, but not re-executed if it succeeds
+    await step.run('update-db', async () => {
+      return await sql.transaction((tx) => queries);
+    });
+
+    return {
+      status: 'success',
+      message: `Created thumbhashes for ${queryValues.length} out of ${events.length} books`,
+    };
+  },
+);

inngest/index.ts

@@ -0,0 +1,7 @@
+import { type GetEvents } from 'inngest';
+import { inngest } from './client';
+import { generateThumbhash } from './functions/generateThumbhash';
+
+export { inngest };
+export const functions = [generateThumbhash];
+export type Events = GetEvents<typeof inngest>;

lib/db/seed-books.ts

@@ -2,6 +2,7 @@ import path from 'path';
 import { sql } from './drizzle';
 import { NeonQueryFunction } from '@neondatabase/serverless';
 import { processEntities } from './seed-utils';
+import { inngest, type Events } from '@/inngest';
 
 const BATCH_SIZE = 900;
 const CHECKPOINT_FILE = 'book_import_checkpoint.json';
@@ -29,7 +30,7 @@ interface BookData {
 
 async function batchInsertBooks(
   batch: BookData[],
-  sqlQuery: NeonQueryFunction<false, false>
+  sqlQuery: NeonQueryFunction<false, false>,
 ) {
   const insertBookAndAuthorsQuery = `
     WITH inserted_book AS (
@@ -45,7 +46,7 @@ async function batchInsertBooks(
     ON CONFLICT DO NOTHING
   `;
 
-  return sqlQuery.transaction((tx) => {
+  const query = await sqlQuery.transaction((tx) => {
     return batch.map((book) =>
       tx(insertBookAndAuthorsQuery, [
         book.isbn || null,
@@ -63,9 +64,21 @@ async function batchInsertBooks(
         book.series || null,
         JSON.stringify(book.popular_shelves),
         book.authors.map((author) => author.author_id),
-      ])
+      ]),
     );
   });
+
+  const events: Events['book.created'][] = batch.map((book) => ({
+    name: 'book.created',
+    data: {
+      title: book.title,
+      image_url: book.image_url,
+    },
+  }));
+
+  await inngest.send(events);
+
+  return query;
 }
 
 async function main() {

lib/db/seed-thumbhash.ts

@@ -54,7 +54,9 @@ async function generateThumbHash(imageBuffer: Buffer): Promise<string | null> {
   }
 }
 
-async function processBook(book: BookData): Promise<[string, string] | null> {
+export async function processBook(
+  book: BookData,
+): Promise<[string, string] | null> {
   if (book.image_url && book.image_url !== EMPTY_IMAGE_URL) {
     const imageBuffer = await fetchImage(book.image_url);
     if (imageBuffer) {
@@ -67,24 +69,24 @@ async function processBook(book: BookData): Promise<[string, string] | null> {
   return null;
 }
 
+export const updateThumbhashQuery = `
+UPDATE books
+SET thumbhash = $1
+WHERE image_url = $2
+`;
+
 async function batchUpdateThumbHash(
   batch: BookData[],
-  sqlQuery: NeonQueryFunction<false, false>
+  sqlQuery: NeonQueryFunction<false, false>,
 ) {
-  const updateThumbhashQuery = `
-    UPDATE books
-    SET thumbhash = $1
-    WHERE image_url = $2
-  `;
-
   const processedBooks = await Promise.all(
-    batch.map((book) => limit(() => processBook(book)))
+    batch.map((book) => limit(() => processBook(book))),
   );
 
   const queries = processedBooks
     .filter((result): result is [string, string] => result !== null)
     .map(([thumbHash, imageUrl]) =>
-      sqlQuery(updateThumbhashQuery, [thumbHash, imageUrl])
+      sqlQuery(updateThumbhashQuery, [thumbHash, imageUrl]),
     );
 
   return sqlQuery.transaction((tx) => queries);

package.json

@@ -10,7 +10,8 @@
     "db:seed-authors": "tsx lib/db/seed-authors.ts",
     "db:seed-books": "tsx lib/db/seed-books.ts",
     "db:seed-thumbhash": "tsx lib/db/seed-thumbhash.ts",
-    "db:embeddings": "tsx lib/ai/create-embeddings.ts"
+    "db:embeddings": "tsx lib/ai/create-embeddings.ts",
+    "inngest": "npx inngest-cli@latest dev -u http://localhost:3000/api/inngest --no-discovery"
   },
   "dependencies": {
     "@ai-sdk/openai": "^0.0.53",
@@ -36,6 +37,7 @@
     "drizzle-kit": "^0.24.1",
     "drizzle-orm": "^0.33.0",
     "geist": "^1.3.1",
+    "inngest": "^3.22.8",
     "lucide-react": "^0.435.0",
     "next": "15.0.0-canary.127",
     "p-limit": "^6.1.0",