Contract deletion

backend/database.js

@@ -40,10 +40,13 @@ if (!fs.existsSync(dataFolder)) {
 }
 
 // Streams stored contract log entries since the given entry hash (inclusive!).
-sbp('sbp/selectors/register', {
+export default ((sbp('sbp/selectors/register', {
   'backend/db/streamEntriesAfter': async function (contractID: string, height: string, requestedLimit: ?number): Promise<*> {
     const limit = Math.min(requestedLimit ?? Number.POSITIVE_INFINITY, process.env.MAX_EVENTS_BATCH_SIZE ?? 500)
     const latestHEADinfo = await sbp('chelonia/db/latestHEADinfo', contractID)
+    if (latestHEADinfo === '') {
+      throw Boom.resourceGone(`contractID ${contractID} has been deleted!`)
+    }
     if (!latestHEADinfo) {
       throw Boom.notFound(`contractID ${contractID} doesn't exist!`)
     }
@@ -113,13 +116,13 @@ sbp('sbp/selectors/register', {
     const value = await sbp('chelonia/db/get', namespaceKey(name))
     return value || Boom.notFound()
   }
-})
+}): any): string[])
 
 function namespaceKey (name: string): string {
   return 'name=' + name
 }
 
-export default async () => {
+export const initDB = async () => {
   // If persistence must be enabled:
   // - load and initialize the selected storage backend
   // - then overwrite 'chelonia/db/get' and '-set' to use it with an LRU cache
@@ -214,3 +217,101 @@ export default async () => {
   }
   await Promise.all([initVapid(), initZkpp()])
 }
+
+// Index management
+
+/**
+ * Creates a factory function that appends a value to a string index in a
+ * database.
+ * The index is identified by the provided key. The value is appended only if it
+ * does not already exist in the index.
+ *
+ * @param key - The key that identifies the index in the database.
+ * @returns A function that takes a value to append to the index.
+ */
+export const appendToIndexFactory = (key: string): (value: string) => Promise<void> => {
+  return (value: string) => {
+    // We want to ensure that the index is updated atomically (i.e., if there
+    // are multiple additions, all of them should be handled), so a queue
+    // is needed for the load & store operation.
+    return sbp('okTurtles.eventQueue/queueEvent', key, async () => {
+      // Retrieve the current index from the database using the provided key
+      const currentIndex = await sbp('chelonia/db/get', key)
+
+      // If the current index exists, check if the value is already present
+      if (currentIndex) {
+        // Check for existing value to avoid duplicates
+        if (
+          // Check if the value is at the end
+          currentIndex.endsWith('\x00' + value) ||
+          // Check if the value is at the start
+          currentIndex.startsWith(value + '\x00') ||
+          // Check if the current index is exactly the value
+          currentIndex === value
+        ) {
+          // Exit if the value already exists
+          return
+        }
+
+        // Append the new value to the current index, separated by NUL
+        await sbp('chelonia/db/set', key, `${currentIndex}\x00${value}`)
+        return
+      }
+
+      // If the current index does not exist, set it to the new value
+      await sbp('chelonia/db/set', key, value)
+    })
+  }
+}
+
+/**
+ * Creates a factory function that removes a value from a string index in a
+ * database.
+ * The index is identified by the provided key. The function handles various
+ * cases to ensure the value is correctly removed from the index.
+ *
+ * @param key - The key that identifies the index in the database.
+ * @returns A function that takes a value to remove from the index.
+ */
+export const removeFromIndexFactory = (key: string): (value: string) => Promise<void> => {
+  return (value: string) => {
+    return sbp('okTurtles.eventQueue/queueEvent', key, async () => {
+      // Retrieve the existing entries from the database using the provided key
+      const existingEntries = await sbp('chelonia/db/get', key)
+      // Exit if there are no existing entries
+      if (!existingEntries) return
+
+      // Handle the case where the value is at the end of the entries
+      if (existingEntries.endsWith('\x00' + value)) {
+        await sbp('chelonia/db/set', key, existingEntries.slice(0, -value.length - 1))
+        return
+      }
+
+      // Handle the case where the value is at the start of the entries
+      if (existingEntries.startsWith(value + '\x00')) {
+        await sbp('chelonia/db/set', key, existingEntries.slice(value.length + 1))
+        return
+      }
+
+      // Handle the case where the existing entries are exactly the value
+      if (existingEntries === value) {
+        await sbp('chelonia/db/delete', key)
+        return
+      }
+
+      // Find the index of the value in the existing entries
+      const entryIndex = existingEntries.indexOf('\x00' + value + '\x00')
+      if (entryIndex === -1) return
+
+      // Create an updated index by removing the value
+      const updatedIndex = existingEntries.slice(0, entryIndex) + existingEntries.slice(entryIndex + value.length + 1)
+
+      // Update the index in the database or delete it if empty
+      if (updatedIndex) {
+        await sbp('chelonia/db/set', key, updatedIndex)
+      } else {
+        await sbp('chelonia/db/delete', key)
+      }
+    })
+  }
+}

backend/errors.js

@@ -0,0 +1,7 @@
+'use strict'
+
+import { ChelErrorGenerator } from '~/shared/domains/chelonia/errors.js'
+
+export const BackendErrorNotFound: typeof Error = ChelErrorGenerator('BackendErrorNotFound')
+export const BackendErrorGone: typeof Error = ChelErrorGenerator('BackendErrorGone')
+export const BackendErrorBadData: typeof Error = ChelErrorGenerator('BackendErrorBadData')

backend/index.js

@@ -50,7 +50,11 @@ console.error = logger.error.bind(logger)
 
 console.info('NODE_ENV =', process.env.NODE_ENV)
 
-const dontLog = { 'backend/server/broadcastEntry': true, 'backend/server/broadcastKV': true }
+const dontLog = {
+  'backend/server/broadcastEntry': true,
+  'backend/server/broadcastDeletion': true,
+  'backend/server/broadcastKV': true
+}
 
 function logSBP (domain, selector, data: Array<*>) {
   if (!dontLog[selector]) {

backend/routes.js

@@ -8,7 +8,7 @@ import { createCID } from '~/shared/functions.js'
 import { SERVER_INSTANCE } from './instance-keys.js'
 import path from 'path'
 import chalk from 'chalk'
-import './database.js'
+import { appendToIndexFactory } from './database.js'
 import { registrationKey, register, getChallenge, getContractSalt, updateContractSalt, redeemSaltUpdateToken } from './zkppSalt.js'
 import Bottleneck from 'bottleneck'
 
@@ -157,6 +157,10 @@ route.POST('/event', {
             console.info(`new user: ${name}=${deserializedHEAD.contractID} (${ip})`)
           }
         }
+        const deletionToken = request.headers['shelter-deletion-token']
+        if (deletionToken) {
+          await sbp('chelonia/db/set', `_private_deletionToken_${deserializedHEAD.contractID}`, deletionToken)
+        }
       }
       // Store size information
       await sbp('backend/server/updateSize', deserializedHEAD.contractID, Buffer.byteLength(request.payload))
@@ -275,6 +279,9 @@ route.GET('/latestHEADinfo/{contractID}', {
   try {
     if (contractID.startsWith('_private')) return Boom.notFound()
     const HEADinfo = await sbp('chelonia/db/latestHEADinfo', contractID)
+    if (HEADinfo === '') {
+      return Boom.resourceGone()
+    }
     if (!HEADinfo) {
       console.warn(`[backend] latestHEADinfo not found for ${contractID}`)
       return Boom.notFound()
@@ -472,7 +479,9 @@ route.GET('/file/{hash}', {
   }
 
   const blobOrString = await sbp('chelonia/db/get', `any:${hash}`)
-  if (!blobOrString) {
+  if (blobOrString === '') {
+    return Boom.resourceGone()
+  } else if (!blobOrString) {
     return Boom.notFound()
   }
   return h.response(blobOrString).etag(hash)
@@ -535,40 +544,98 @@ route.POST('/deleteFile/{hash}', {
 
   // Authentication passed, now proceed to delete the file and its associated
   // keys
-  const rawManifest = await sbp('chelonia/db/get', hash)
-  if (!rawManifest) return Boom.notFound()
   try {
-    const manifest = JSON.parse(rawManifest)
-    if (!manifest || typeof manifest !== 'object') return Boom.badData('manifest format is invalid')
-    if (manifest.version !== '1.0.0') return Boom.badData('unsupported manifest version')
-    if (!Array.isArray(manifest.chunks) || !manifest.chunks.length) return Boom.badData('missing chunks')
-    // Delete all chunks
-    await Promise.all(manifest.chunks.map(([, cid]) => sbp('chelonia/db/delete', cid)))
+    await sbp('backend/deleteFile', hash)
+    return h.response()
   } catch (e) {
-    console.warn(e, `Error parsing manifest for ${hash}. It's probably not a file manifest.`)
-    return Boom.notFound()
+    switch (e.name) {
+      case 'BackendErrorNotFound':
+        return Boom.notFound()
+      case 'BackendErrorGone':
+        return Boom.resourceGone()
+      case 'BackendErrorBadData':
+        return Boom.badData(e.message)
+      default:
+        console.error(e, 'Error during deletion')
+        return Boom.internal(e.message ?? 'internal error')
+    }
   }
-  // The keys to be deleted are not read from or updated, so they can be deleted
-  // without using a queue
-  await sbp('chelonia/db/delete', hash)
-  await sbp('chelonia/db/delete', `_private_owner_${hash}`)
-  await sbp('chelonia/db/delete', `_private_size_${hash}`)
-  await sbp('chelonia/db/delete', `_private_deletionToken_${hash}`)
-  const resourcesKey = `_private_resources_${owner}`
-  // Use a queue for atomicity
-  await sbp('okTurtles.eventQueue/queueEvent', resourcesKey, async () => {
-    const existingResources = await sbp('chelonia/db/get', resourcesKey)
-    if (!existingResources) return
-    if (existingResources.endsWith(hash)) {
-      await sbp('chelonia/db/set', resourcesKey, existingResources.slice(0, -hash.length - 1))
-      return
+})
+
+route.POST('/deleteContract/{hash}', {
+  auth: {
+    // Allow file deletion, and allow either the bearer of the deletion token or
+    // the file owner to delete it
+    strategies: ['chel-shelter', 'chel-bearer'],
+    mode: 'required'
+  }
+}, async function (request, h) {
+  const { hash } = request.params
+  const strategy = request.auth.strategy
+  if (!hash || hash.startsWith('_private')) return Boom.notFound()
+
+  switch (strategy) {
+    case 'chel-shelter': {
+      const owner = await sbp('chelonia/db/get', `_private_owner_${hash}`)
+      if (!owner) {
+        return Boom.notFound()
+      }
+
+      let ultimateOwner = owner
+      let count = 0
+      // Walk up the ownership tree
+      do {
+        const owner = await sbp('chelonia/db/get', `_private_owner_${ultimateOwner}`)
+        if (owner) {
+          ultimateOwner = owner
+          count++
+        } else {
+          break
+        }
+      // Prevent an infinite loop
+      } while (count < 128)
+      // Check that the user making the request is the ultimate owner (i.e.,
+      // that they have permission to delete this file)
+      if (!ctEq(request.auth.credentials.billableContractID, ultimateOwner)) {
+        return Boom.unauthorized('Invalid token', 'bearer')
+      }
+      break
     }
-    const hashIndex = existingResources.indexOf(hash + '\x00')
-    if (hashIndex === -1) return
-    await sbp('chelonia/db/set', resourcesKey, existingResources.slice(0, hashIndex) + existingResources.slice(hashIndex + hash.length + 1))
-  })
+    case 'chel-bearer': {
+      const expectedToken = await sbp('chelonia/db/get', `_private_deletionToken_${hash}`)
+      if (!expectedToken) {
+        return Boom.notFound()
+      }
+      const token = request.auth.credentials.token
+      // Constant-time comparison
+      // Check that the token provided matches the deletion token for this contract
+      if (!ctEq(expectedToken, token)) {
+        return Boom.unauthorized('Invalid token', 'bearer')
+      }
+      break
+    }
+    default:
+      return Boom.unauthorized('Missing or invalid auth strategy')
+  }
 
-  return h.response()
+  // Authentication passed, now proceed to delete the contract and its associated
+  // keys
+  try {
+    const [id] = sbp('chelonia.persistentActions/enqueue', ['backend/deleteContract', hash])
+    return h.response({ id }).code(202)
+  } catch (e) {
+    switch (e.name) {
+      case 'BackendErrorNotFound':
+        return Boom.notFound()
+      case 'BackendErrorGone':
+        return Boom.resourceGone()
+      case 'BackendErrorBadData':
+        return Boom.badData(e.message)
+      default:
+        console.error(e, 'Error during deletion')
+        return Boom.internal(e.message ?? 'internal error')
+    }
+  }
 })
 
 route.POST('/kv/{contractID}/{key}', {
@@ -651,6 +718,7 @@ route.POST('/kv/{contractID}/{key}', {
   const existingSize = existing ? Buffer.from(existing).byteLength : 0
   await sbp('chelonia/db/set', `_private_kv_${contractID}_${key}`, request.payload)
   await sbp('backend/server/updateSize', contractID, request.payload.byteLength - existingSize)
+  await appendToIndexFactory(`_private_kvIdx_${contractID}`)(key)
   await sbp('backend/server/broadcastKV', contractID, key, request.payload.toString())
 
   return h.response().code(204)