Merge pull request #44 from DIG-Network/release/v0.0.1-alpha.48

Release/v0.0.1 alpha.48

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [0.0.1-alpha.48](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.47...v0.0.1-alpha.48) (2024-09-20)
+
+
+### Bug Fixes
+
+* sync process ([935fd3f](https://github.com/DIG-Network/dig-chia-sdk/commit/935fd3ff312f87f0cbbfdefb143f68f87e82c3ed))
+* sync process ([b57a8e0](https://github.com/DIG-Network/dig-chia-sdk/commit/b57a8e0572ba28abae81fe02179a9c4b9a1b4ea0))
+
 ### [0.0.1-alpha.47](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.46...v0.0.1-alpha.47) (2024-09-20)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dignetwork/dig-sdk",
-  "version": "0.0.1-alpha.47",
+  "version": "0.0.1-alpha.48",
   "description": "",
   "type": "commonjs",
   "main": "./dist/index.js",

src/DataIntegrityTree/DataIntegrityTree.ts

@@ -775,13 +775,14 @@ class DataIntegrityTree {
    * @param expectedRootHash - The expected root hash of the Merkle tree.
    * @returns A boolean indicating if the SHA-256 is present in the foreign tree and the root hash matches.
    */
-  static validateKeyIntegrityWithForeignTree(
-    key: string,
+  static async validateKeyIntegrityWithForeignTree(
+    hexkey: string,
     sha256: string,
     serializedTree: object,
-    expectedRootHash: string
-  ): boolean {
-    if (!isHexString(key)) {
+    expectedRootHash: string,
+    dataDir: string
+  ): Promise<boolean> {
+    if (!isHexString(hexkey)) {
       throw new Error("key must be a valid hex string");
     }
     if (!isHexString(sha256)) {
@@ -791,42 +792,83 @@ class DataIntegrityTree {
       throw new Error("expectedRootHash must be a valid hex string");
     }
 
-    // Deserialize the foreign tree
-    const leaves = (serializedTree as any).leaves.map((leaf: string) =>
-      Buffer.from(leaf, "hex")
-    );
-    const tree = new MerkleTree(leaves, SHA256, { sortPairs: true });
+    // File path based on sha256
+    const filePath = path.join(dataDir, sha256.match(/.{1,2}/g)!.join("/"));
 
-    // Verify that the deserialized tree's root matches the expected root hash
-    const treeRootHash = tree.getRoot().toString("hex");
-    if (treeRootHash !== expectedRootHash) {
-      console.warn(
-        `Expected root hash ${expectedRootHash}, but got ${treeRootHash}`
-      );
-      return false;
+    // Check if the file exists
+    if (!fs.existsSync(filePath)) {
+      throw new Error(`File at path ${filePath} does not exist`);
     }
 
-    // Rebuild the files map from the serialized tree
-    // @ts-ignore
-    tree.files = new Map(
-      Object.entries((serializedTree as any).files).map(
-        ([key, value]: [string, any]) => [
-          key,
-          { hash: value.hash, sha256: value.sha256 },
-        ]
-      )
-    );
+    const compressedReadStream = fs.createReadStream(filePath);
+    const decompressStream = zlib.createGunzip();
+    const hash = crypto.createHash("sha256");
 
-    // Check if the SHA-256 exists in the foreign tree's files
-    const combinedHash = crypto
-      .createHash("sha256")
-      .update(`${toHex(key)}/${sha256}`)
-      .digest("hex");
+    // Process file decompression and hash comparison
+    return new Promise((resolve, reject) => {
+      compressedReadStream.pipe(decompressStream);
 
-    const leaf = Buffer.from(combinedHash, "hex");
-    const isInTree = tree.getLeafIndex(leaf) !== -1;
+      decompressStream.on("data", (chunk) => {
+        hash.update(chunk);
+      });
+
+      decompressStream.on("end", () => {
+        const uncompressedSha256 = hash.digest("hex");
+        console.log(`SHA-256 of uncompressed file: ${uncompressedSha256}`);
+
+        if (uncompressedSha256 !== sha256) {
+          console.warn(
+            `File hash mismatch. Expected: ${sha256}, got: ${uncompressedSha256}`
+          );
+          return resolve(false);
+        }
 
-    return isInTree;
+        // Deserialize the foreign tree
+        const leaves = (serializedTree as any).leaves.map((leaf: string) =>
+          Buffer.from(leaf, "hex")
+        );
+        const tree = new MerkleTree(leaves, SHA256, { sortPairs: true });
+
+        // Verify that the deserialized tree's root matches the expected root hash
+        const treeRootHash = tree.getRoot().toString("hex");
+        if (treeRootHash !== expectedRootHash) {
+          console.warn(
+            `Expected root hash ${expectedRootHash}, but got ${treeRootHash}`
+          );
+          return resolve(false);
+        }
+
+        // Rebuild the files map from the serialized tree
+        // @ts-ignore
+        tree.files = new Map(
+          Object.entries((serializedTree as any).files).map(
+            ([key, value]: [string, any]) => [
+              key,
+              { hash: value.hash, sha256: value.sha256 },
+            ]
+          )
+        );
+
+        // Check if the SHA-256 exists in the foreign tree's files
+        const combinedHash = crypto
+          .createHash("sha256")
+          .update(`${hexkey}/${sha256}`)
+          .digest("hex");
+
+        const leaf = Buffer.from(combinedHash, "hex");
+        const isInTree = tree.getLeafIndex(leaf) !== -1;
+
+        resolve(isInTree);
+      });
+
+      decompressStream.on("error", (err) => {
+        reject(err);
+      });
+
+      compressedReadStream.on("error", (err) => {
+        reject(err);
+      });
+    });
   }
 }
 

src/DigNetwork/DigNetwork.ts

@@ -8,6 +8,7 @@ import { DataStore, ServerCoin } from "../blockchain";
 import { DIG_FOLDER_PATH } from "../utils/config";
 import { RootHistoryItem } from "../types";
 import { promisify } from "util";
+import { DataIntegrityTree } from "../DataIntegrityTree";
 
 const rename = promisify(fs.rename);
 const unlink = promisify(fs.unlink);
@@ -213,7 +214,7 @@ export class DigNetwork {
         // Add peer to blacklist if it doesn't meet criteria
         peerBlackList.push(peerIp);
       } catch (error) {
-        console.error(`Error connecting to peer ${peerIp}. Resampling...`);
+        console.error(`Error connecting to DIG Peer ${peerIp}. Resampling...`);
         if (peerIp) {
           peerBlackList.push(peerIp); // Add to blacklist if error occurs
         }
@@ -259,6 +260,7 @@ export class DigNetwork {
         .filter(
           (item) => !fs.existsSync(`${this.storeDir}/${item.root_hash}.dat`)
         )
+        // Reverse to download the latest first
         .reverse();
 
       if (!rootHistoryFiltered.length) {
@@ -308,15 +310,36 @@ export class DigNetwork {
                   `${this.storeDir}/data`
                 );
 
-                if (!fs.existsSync(filePath) || forceDownload) {
-                  console.log(
-                    `Downloading file with sha256: ${file.sha256}...`
+                console.log(`Downloading file with sha256: ${file.sha256}...`);
+
+                await selectedPeer.downloadData(
+                  this.dataStore.StoreId,
+                  `data/${file.sha256.match(/.{1,2}/g)!.join("/")}`
+                );
+
+                const integrityCheck =
+                  await DataIntegrityTree.validateKeyIntegrityWithForeignTree(
+                    storeKey,
+                    file.sha256,
+                    root,
+                    rootInfo.root_hash,
+                    `${this.storeDir}/data`
                   );
-                  await selectedPeer.downloadData(
-                    this.dataStore.StoreId,
-                    `data/${file.sha256.match(/.{1,2}/g)!.join("/")}`
+
+                if (integrityCheck) {
+                  console.log(
+                    `\x1b[32mIntegrity check passed for file with sha256: ${file.sha256}.\x1b[0m`
                   );
+                  continue;
                 }
+
+                console.error(
+                  `\x1b[31mIntegrity check failed for file with sha256: ${file.sha256}.\x1b[0m`
+                );
+                await unlink(filePath);
+                throw new Error(
+                  `Store Integrity check failed. Syncing file from another peer.`
+                );
               }
             }
 
@@ -339,15 +362,22 @@ export class DigNetwork {
             }
           }
         }
+
+        // Only process the first root hash so other stores can sync the latest.
+        // This has an effect where the latest roothash will always be synced first, even if new ones come in.
+        // Then it will backfill historical roothashes
+        break;
       }
 
       await this.downloadManifestFile(true);
 
       console.log("Syncing store complete.");
     } catch (error: any) {
       if (selectedPeer) {
-        peerBlackList.push(selectedPeer.IpAddress);
+        peerBlackList.push((selectedPeer as DigPeer).IpAddress);
       }
+
+      console.trace(error);
       throw error;
     }
   }
@@ -396,15 +426,15 @@ export class DigNetwork {
       const blacklist = this.peerBlacklist.get(dataPath) || new Set<string>();
 
       for (const digPeer of digPeers) {
-        if (blacklist.has(digPeer.IpAddress)) continue;
+        try {
+          if (blacklist.has(digPeer.IpAddress)) continue;
 
-        const response = await digPeer.propagationServer.headStore();
+          const response = await digPeer.propagationServer.headStore();
 
-        if (!response.success) {
-          continue;
-        }
+          if (!response.success) {
+            continue;
+          }
 
-        try {
           // Create directory if it doesn't exist
           const directory = path.dirname(tempFilePath);
           if (!fs.existsSync(directory)) {

src/DigNetwork/DigPeer.ts

@@ -6,6 +6,7 @@ import { PropagationServer } from "./PropagationServer";
 import { IncentiveServer } from "./IncentiveServer";
 import { DataStore } from "../blockchain";
 import { DataIntegrityTree } from "../DataIntegrityTree";
+import { DIG_FOLDER_PATH } from "../utils/config";
 import fs from "fs";
 import {
   sendXch,
@@ -150,7 +151,8 @@ export class DigPeer {
           key,
           fileData.sha256,
           datFileContent,
-          rootHash
+          rootHash,
+          path.resolve(DIG_FOLDER_PATH, "stores", this.storeId, 'data')
         );
 
         if (!treeCheck) {