diff --git a/src/db.ts b/src/db.ts
index f266feac..2a94b1de 100644
--- a/src/db.ts
+++ b/src/db.ts
@@ -18,25 +18,82 @@ await client.schema.createTableIfNotExists('key_value', function (table) {
table.datetime('expires').nullable();
});
-// Helper function to get the meta data for a list of activity URIs
+// Helper function to get the meta data for an array of activity URIs
// from the database. This allows us to fetch information about the activities
// without having to fetch the full activity object. This is a bit of a hack to
// support sorting / filtering of the activities and should be replaced when we
// have a proper db schema
-export async function getActivityMeta(uris: string[]): Promise<Map<string, { id: number, type: string }>> {
+
+type ActivityMeta = {
+ id: number; // Used for sorting
+ activity_type: string; // Used for filtering by activity type
+ object_type: string; // Used for filtering by object type
+ reply_object_url: string; // Used for filtering by isReplyToOwn criteria
+ reply_object_name: string; // Used for filtering by isReplyToOwn criteria
+};
+
+type getActivityMetaQueryResult = {
+ key: string,
+ left_id: number,
+ activity_type: string,
+ object_type: string,
+ reply_object_url: string,
+ reply_object_name: string
+}
+
+export async function getActivityMeta(uris: string[]): Promise<Map<string, ActivityMeta>> {
const results = await client
- .select('key', 'id', client.raw('JSON_EXTRACT(value, "$.type") as type'))
- .from('key_value')
- .whereIn('key', uris.map(uri => `["${uri}"]`));
+ .select(
+ 'left.key',
+ 'left.id as left_id',
+ client.raw('JSON_EXTRACT(left.value, "$.type") as activity_type'),
+ client.raw('JSON_EXTRACT(left.value, "$.object.type") as object_type'),
+ client.raw('JSON_EXTRACT(right.value, "$.object.url") as reply_object_url'),
+ client.raw('JSON_EXTRACT(right.value, "$.object.name") as reply_object_name')
+ )
+ .from({ left: 'key_value' })
+ // @ts-ignore: This works as expected but the type definitions complain 🤔
+ .leftJoin(
+ { right: 'key_value' },
+ client.raw('JSON_UNQUOTE(JSON_EXTRACT(right.value, "$.object.id"))'),
+ '=',
+ client.raw('JSON_UNQUOTE(JSON_EXTRACT(left.value, "$.object.inReplyTo"))')
+ )
+ .whereIn('left.key', uris.map(uri => `["${uri}"]`));
- const map = new Map<string, { id: number, type: string }>();
+ const map = new Map<string, ActivityMeta>();
- for (const result of results) {
+ for (const result of results as getActivityMetaQueryResult[]) {
map.set(result.key.substring(2, result.key.length - 2), {
- id: result.id,
- type: result.type,
+ id: result.left_id,
+ activity_type: result.activity_type,
+ object_type: result.object_type,
+ reply_object_url: result.reply_object_url,
+ reply_object_name: result.reply_object_name,
});
}
return map;
}
+
+// Helper function to retrieve a map of replies for an array of activity URIs
+// from the database
+export async function getRepliesMap (uris: string[]): Promise<Map<string, any>> {
+ const map = new Map<string, any>();
+
+ const results = await client
+ .select('value')
+ .from('key_value')
+ .where(client.raw('JSON_EXTRACT(value, "$.object.inReplyTo") IS NOT NULL'))
+ .whereIn('key', uris.map(uri => `["${uri}"]`));
+
+ for (const {value: result} of results) {
+ const replies = map.get(result.object.inReplyTo) ?? [];
+
+ replies.push(result);
+
+ map.set(result.object.inReplyTo, replies);
+ }
+
+ return map;
+}
diff --git a/src/handlers.ts b/src/handlers.ts
index 6ed8cd7c..478233cd 100644
--- a/src/handlers.ts
+++ b/src/handlers.ts
@@ -18,7 +18,7 @@ import { Buffer } from 'node:buffer';
import { Context, Next } from 'hono';
import sanitizeHtml from 'sanitize-html';
import { v4 as uuidv4 } from 'uuid';
-import { getActivityMeta } from './db';
+import { getActivityMeta, getRepliesMap } from './db';
import { addToList, removeFromList } from './kv-helpers';
import { toURL } from './toURL';
import { ContextData, HonoContextVariables, fedify } from './app';
@@ -489,6 +489,7 @@ async function buildActivity(
db: KvStore,
apCtx: APContext<ContextData>,
liked: string[] = [],
+ repliesMap: Map<string, any> | null = null,
): Promise<InboxItem | null> {
const item = await db.get<InboxItem>([uri]);
@@ -559,6 +560,29 @@ async function buildActivity(
}
}
+ // If a replies map has been provided, the item is not a string, and the
+ // item has an id, we should nest any replies recursively (which involves
+ // calling this function again for each reply)
+ if (repliesMap && typeof item.object !== 'string' && item.object.id) {
+ item.object.replies = [];
+
+ const replies = repliesMap.get(item.object.id);
+
+ if (replies) {
+ const builtReplies = [];
+
+ for (const reply of replies) {
+ const builtReply = await buildActivity(reply.id, db, apCtx, liked, repliesMap);
+
+ if (builtReply) {
+ builtReplies.push(builtReply);
+ }
+ }
+
+ item.object.replies = builtReplies;
+ }
+ }
+
// Return the built item
return item;
}
@@ -622,15 +646,56 @@ export async function getActivities(
const globaldb = ctx.get('globaldb');
const apCtx = fedify.createContext(ctx.req.raw as Request, {db, globaldb});
- // Parse cursor and limit from query parameters
+ // -------------------------------------------------------------------------
+ // Process query parameters
+ // -------------------------------------------------------------------------
+
+ // Parse "cursor" and "limit" from query parameters
+ // These are used to paginate the results
+ // ?cursor=<string>
+ // ?limit=<number>
const queryCursor = ctx.req.query('cursor')
const cursor = queryCursor ? Buffer.from(queryCursor, 'base64url').toString('utf-8') : null;
const limit = Number.parseInt(ctx.req.query('limit') || DEFAULT_LIMIT.toString(), 10);
- // Parse includeOwn from query parameters
+ // Parse "includeOwn" from query parameters
// This is used to include the user's own activities in the results
+ // ?includeOwn=<boolean>
const includeOwn = ctx.req.query('includeOwn') === 'true';
+ // Parse includeReplies from query parameters
+ // This is used to include nested replies in the results
+ // ?includeReplies=<boolean>
+ const includeReplies = ctx.req.query('includeReplies') === 'true';
+
+ // Parse "filter" from query parameters
+ // This is used to filter the activities by various criteria
+ // ?filter={type: ['<activityType>', '<activityType>:<objectType>', '<activityType>:<objectType>:<criteria>']}
+ const queryFilters = ctx.req.query('filter') || '[]';
+ const filters = JSON.parse(decodeURI(queryFilters))
+
+ const typeFilters = (filters.type || []).map((filter: string) => {
+ const [activityType, objectType = null, criteria = null] = filter.split(':');
+
+ return {
+ activity: activityType,
+ object: objectType,
+ criteria,
+ }
+ });
+
+ console.log('Request query =', ctx.req.query());
+ console.log('Processed query params =', JSON.stringify({
+ cursor,
+ limit,
+ includeOwn,
+ typeFilters,
+ }, null, 2));
+
+ // -------------------------------------------------------------------------
+ // Fetch required data from the database
+ // -------------------------------------------------------------------------
+
// Fetch the liked object refs from the database:
// - Data is structured as an array of strings
// - Each string is a URI to an object in the database
@@ -654,19 +719,64 @@ export async function getActivities(
outboxRefs = await db.get<string[]>(['outbox']) || [];
}
- // To be able to return a sorted / filtered "feed" of activities, we need to
+ // To be able to return a sorted / filtered list of activities, we need to
// fetch some additional meta data about the referenced activities. Doing this
// upfront allows us to sort, filter and paginate the activities before
// building them for the response which saves us from having to perform
// unnecessary database lookups for referenced activities that will not be
- // included in the response. If we can't find the meta data in the database
- // for an activity, we skip it as this is unexpected
+ // included in the response
let activityRefs = [...inboxRefs, ...outboxRefs];
const activityMeta = await getActivityMeta(activityRefs);
+ // If we can't find the meta data in the database for an activity, we skip
+ // it as this is unexpected
activityRefs = activityRefs.filter(ref => activityMeta.has(ref));
- // Sort the activity refs by the id of the activity (newest first)
+ // -------------------------------------------------------------------------
+ // Apply filtering and sorting
+ // -------------------------------------------------------------------------
+
+ // Filter the activity refs by any provided type filters
+ if (typeFilters.length > 0) {
+ activityRefs = activityRefs.filter(ref => {
+ const activity = activityMeta.get(ref)!;
+
+ return typeFilters.some((filter: { activity: string; object: string | null, criteria: string | null }) => {
+ // ?filter={type: ['<activityType>']}
+ if (filter.activity && activity.activity_type !== filter.activity) {
+ return false;
+ }
+
+ // ?filter={type: ['<activityType>:<objectType>']}
+ if (filter.object && activity.object_type !== filter.object) {
+ return false;
+ }
+
+ // ?filter={type: ['<activityType>:<objectType>:isReplyToOwn,<siteHost>']}
+ if (filter.criteria && filter.criteria.startsWith('isReplyToOwn,')) {
+ // If the activity does not have a reply object url or name,
+ // we can't determine if it's a reply to an own object so
+ // we skip it
+ if (!activity.reply_object_url || !activity.reply_object_name) {
+ return false;
+ }
+
+ // Verify that the reply is to an object created by the user by
+ // checking that the hostname associated with the reply object
+ // is the same as the hostname of the site. This is not a bullet
+ // proof check, but it's a good enough for now (i think 😅)
+ const [_, siteHost] = filter.criteria.split(',');
+ const { hostname: replyHost } = new URL(activity.reply_object_url);
+
+ return siteHost === replyHost;
+ }
+
+ return true;
+ });
+ });
+ }
+
+ // Sort the activity refs by the id of the activity (newest first).
// We are using the id to sort because currently not all activity types have
// a timestamp. The id property is a unique auto incremented number at the
// database level
@@ -674,6 +784,10 @@ export async function getActivities(
return activityMeta.get(b)!.id - activityMeta.get(a)!.id;
});
+ // -------------------------------------------------------------------------
+ // Paginate
+ // -------------------------------------------------------------------------
+
// Find the starting index based on the cursor
const startIndex = cursor ? activityRefs.findIndex(ref => ref === cursor) + 1 : 0;
@@ -685,12 +799,22 @@ export async function getActivities(
? Buffer.from(paginatedRefs[paginatedRefs.length - 1]).toString('base64url')
: null;
- // Build the activities for the response
+ // -------------------------------------------------------------------------
+ // Build the activities and return the response
+ // -------------------------------------------------------------------------
+
const activities = [];
+ // If we need to include replies, fetch the replies map based on the paginated
+ // activity refs, which will be utilised when building the activities
+ const repliesMap = includeReplies
+ ? await getRepliesMap(paginatedRefs)
+ : null;
+
+ // Build the activities
for (const ref of paginatedRefs) {
try {
- const builtActivity = await buildActivity(ref, globaldb, apCtx, likedRefs);
+ const builtActivity = await buildActivity(ref, globaldb, apCtx, likedRefs, repliesMap);
if (builtActivity) {
activities.push(builtActivity);
@@ -700,7 +824,7 @@ export async function getActivities(
}
}
- // Return the built activities and the next cursor
+ // Return the response
return new Response(JSON.stringify({
items: activities,
nextCursor,