diff --git a/package-lock.json b/package-lock.json index ddcabca..f8d746e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "@dignetwork/datalayer-driver": "^0.1.24", "archiver": "^7.0.1", + "axios": "^1.7.7", "bip39": "^3.1.0", "chia-bls": "^1.0.2", "chia-config-loader": "^1.0.1", @@ -1155,6 +1156,16 @@ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, + "node_modules/axios": { + "version": "1.7.7", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", + "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/b4a": { "version": "1.6.6", "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.6.tgz", @@ -2828,6 +2839,25 @@ "flat": "cli.js" } }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/foreground-child": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.0.tgz", @@ -4574,6 +4604,11 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "node_modules/q": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", diff --git a/package.json b/package.json index 0df75ab..7f8260b 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "dependencies": { "@dignetwork/datalayer-driver": "^0.1.24", "archiver": "^7.0.1", + "axios": "^1.7.7", "bip39": "^3.1.0", "chia-bls": "^1.0.2", "chia-config-loader": "^1.0.1", diff --git a/src/DigNetwork/DigPeer.ts b/src/DigNetwork/DigPeer.ts index ef7b2c1..c67fe23 100644 --- a/src/DigNetwork/DigPeer.ts +++ b/src/DigNetwork/DigPeer.ts @@ -169,14 +169,15 @@ export class DigPeer { public async syncStore(): Promise { const dataStore = DataStore.from(this.storeId); const rootHistory = await dataStore.getRootHistory(); - rootHistory - .filter((root) => root.synced) - .forEach((item) => { - this.pushStoreRoot(this.storeId, item.root_hash); - }); - } + + for (const item of rootHistory.filter((root) => root.synced)) { + await this.pushStoreRoot(this.storeId, item.root_hash); + } +} + public async pushStoreRoot(storeId: string, rootHash: string): Promise { + console.log(`Pushing root hash ${rootHash} to ${this.IpAddress}`); const dataStore = DataStore.from(storeId); const alreadySynced = await this.contentServer.hasRootHash(rootHash); @@ -188,6 +189,9 @@ export class DigPeer { const tree = await dataStore.Tree.serialize(rootHash); + // @ts-ignore + console.log(tree.files); + // @ts-ignore tree.files.forEach(async (file) => { await dataStore.Tree.verifyKeyIntegrity(file.sha256, rootHash); diff --git a/src/utils/ContentScanner.ts b/src/utils/ContentScanner.ts new file mode 100644 index 0000000..34085d4 --- /dev/null +++ b/src/utils/ContentScanner.ts @@ -0,0 +1,181 @@ +import { exec } from "child_process"; +import fs from "fs"; +import path from "path"; +import axios from "axios"; + +/** + * @class IllegalContentScanner + * Scans files for illegal content using ClamAV running in a Docker container and deletes files if flagged. + */ +export class IllegalContentScanner { + private clamAVContainerName: string; + private virusTotalApiKey?: string; + private openAiApiKey?: string; + + /** + * @constructor + * @param clamAVContainerName The name of the Docker container running ClamAV (default: 'clamav'). + */ + constructor(clamAVContainerName: string = "clamav") { + this.clamAVContainerName = clamAVContainerName; + this.virusTotalApiKey = process.env.VIRUSTOTAL_API_KEY || undefined; + this.openAiApiKey = process.env.OPENAI_API_KEY || undefined; + } + + /** + * Scans a file for illegal content using ClamAV, VirusTotal, and OpenAI Moderation API. + * If a service is missing its API key, the scan for that service is skipped. + * @param filePath The path of the file to scan. + * @returns {Promise} A promise that resolves after the scan and potential deletion. + */ + public async scanFile(filePath: string): Promise { + try { + // 1. ClamAV Scan + console.log(`Starting ClamAV scan for file: ${filePath}`); + const clamAvResult = await this.scanWithClamAV(filePath); + if (this.isClamAVResultMalicious(clamAvResult)) { + console.log(`ClamAV flagged file: ${filePath}. Deleting...`); + this.deleteFile(filePath); + return; + } else { + console.log(`ClamAV did not flag the file: ${filePath}.`); + } + + // 2. VirusTotal Scan (if API key is provided) + if (this.virusTotalApiKey) { + console.log(`Submitting file to VirusTotal for scanning: ${filePath}`); + const virusTotalResult = await this.scanWithVirusTotal(filePath); + if (this.isVirusTotalResultMalicious(virusTotalResult)) { + console.log(`VirusTotal flagged file: ${filePath}. Deleting...`); + this.deleteFile(filePath); + return; + } else { + console.log(`VirusTotal did not flag the file: ${filePath}.`); + } + } else { + console.log(`VirusTotal API key not provided. Skipping VirusTotal scan.`); + } + + // 3. OpenAI Moderation API (if API key is provided and file is text-based) + const fileExtension = path.extname(filePath).toLowerCase(); + if (this.openAiApiKey && (fileExtension === ".txt" || fileExtension === ".json")) { + console.log(`Scanning file content with OpenAI Moderation API: ${filePath}`); + const fileContent = await fs.promises.readFile(filePath, "utf-8"); + const openAiResult = await this.scanWithOpenAI(fileContent); + if (this.isOpenAiResultMalicious(openAiResult)) { + console.log(`OpenAI flagged file: ${filePath}. Deleting...`); + this.deleteFile(filePath); + return; + } + } else { + if (!this.openAiApiKey) { + console.log(`OpenAI API key not provided. Skipping OpenAI scan.`); + } else { + console.log(`OpenAI scan skipped: File type not supported (${fileExtension}).`); + } + } + + console.log(`File ${filePath} passed all multilayer scans.`); + } catch (error) { + console.error(`Error scanning file: ${filePath}`, error); + } + } + + /** + * Scans a file using ClamAV running in Docker. + * @param filePath The path of the file to scan. + * @returns {Promise} The result of the ClamAV scan. + */ + private scanWithClamAV(filePath: string): Promise { + return new Promise((resolve, reject) => { + exec(`docker exec ${this.clamAVContainerName} clamdscan ${filePath}`, (error, stdout, stderr) => { + if (error) { + reject(`ClamAV scan error: ${stderr}`); + } else { + resolve(stdout); + } + }); + }); + } + + /** + * Submits a file to VirusTotal for scanning. + * @param filePath The path of the file to scan. + * @returns {Promise} The result from VirusTotal. + */ + private async scanWithVirusTotal(filePath: string): Promise { + const fileContent = await fs.promises.readFile(filePath); + const formData = new FormData(); + // @ts-ignore + formData.append("file", fileContent); + + const response = await axios.post("https://www.virustotal.com/vtapi/v2/file/scan", formData, { + headers: { + "x-apikey": this.virusTotalApiKey, + // @ts-ignore + ...formData.getHeaders(), + }, + }); + return response.data; + } + + /** + * Scans file content (text) using OpenAI's Moderation API. + * @param content The text content to scan. + * @returns {Promise} The result from OpenAI's Moderation API. + */ + private async scanWithOpenAI(content: string): Promise { + const response = await axios.post( + "https://api.openai.com/v1/moderations", + { input: content }, + { + headers: { + Authorization: `Bearer ${this.openAiApiKey}`, + }, + } + ); + return response.data; + } + + /** + * Checks if the ClamAV scan result indicates a malicious file. + * @param result The ClamAV scan output. + * @returns {boolean} True if the file is flagged, false otherwise. + */ + private isClamAVResultMalicious(result: string): boolean { + return result.includes("FOUND"); + } + + /** + * Checks if the VirusTotal result indicates a malicious file. + * @param result The VirusTotal scan result. + * @returns {boolean} True if the file is flagged, false otherwise. + */ + private isVirusTotalResultMalicious(result: any): boolean { + // Check if VirusTotal flagged the file based on its multiple antivirus results + return result.positives > 0; + } + + /** + * Checks if the OpenAI Moderation result indicates harmful content. + * @param result The OpenAI Moderation result. + * @returns {boolean} True if harmful content is detected, false otherwise. + */ + private isOpenAiResultMalicious(result: any): boolean { + return result.flagged === true; + } + + /** + * Deletes a file from the filesystem. + * @param filePath The file to delete. + */ + private deleteFile(filePath: string): void { + fs.unlink(filePath, (err) => { + if (err) { + console.error(`Error deleting file: ${filePath}`, err); + } else { + console.log(`File ${filePath} was deleted successfully.`); + } + }); + } +} \ No newline at end of file