// ! Hashed File Store (not file system!!) import { join } from "path"; import { existsSync, mkdirSync, createWriteStream, rename, stat, writeFile, rm, rmSync, createReadStream } from "fs"; import { Console } from "hsconsole"; import { yellow } from "dyetty"; import { createHash, randomBytes } from "crypto"; import FunkyArray from "funky-array"; import HashFSFileInformation from "./HashFSFileInformation"; import type { BusboyFileStream } from "@fastify/busboy"; import { pipeline } from "stream/promises"; export default class HashFS { public static STARTUP_DIR: string; private static HASHFS_INSTANCES: FunkyArray = new FunkyArray(); public static GetHashFSInstance(name: string) { const instance = this.HASHFS_INSTANCES.get(name); if (!instance) { throw `Attempted to get nonexistent HashFS instance "${name}"`; } return instance; } public readonly path: string; private readonly tempPath: string; private readonly folder: string; private logInfo(logText: string) { Console.printInfo(`[ ${yellow(`HashFS: ${this.folder}`)} ] ${logText}`); } public constructor(folder: string) { HashFS.HASHFS_INSTANCES.set(folder, this); this.folder = folder; this.path = join(HashFS.STARTUP_DIR, folder); let firstCreation = false; if (!existsSync(this.path)) { this.logInfo(`Creating HashFS for "${folder}"...`); mkdirSync(this.path); firstCreation = true; } this.logInfo(`Validating file store...`); let issuesRepaired = 0; for (let i = 0; i < 16; i++) { const hashRootFolderPath = join(this.path, i.toString(16)); if (!existsSync(hashRootFolderPath)) { mkdirSync(hashRootFolderPath); this.logInfo(`"${i.toString(16)}" does not exist, creating...`); issuesRepaired++; } for (let i1 = 0; i1 < 16; i1++) { const subFolderPath = join(hashRootFolderPath, (i * 16 + i1).toString(16).padStart(2, "0")); if (!existsSync(subFolderPath)) { this.logInfo(`"${i.toString(16)}/${(i * 16 + i1).toString(16).padStart(2, "0")}" does not exist, creating...`); mkdirSync(subFolderPath); issuesRepaired++; } } } // TODO: Validate the files in the file store this.logInfo(`File Store Validated Successfully${!firstCreation && issuesRepaired > 0 ? `. Repaired ${issuesRepaired} issues.` : " with no issues."}`); this.tempPath = join(this.path, "temp"); if (existsSync(this.tempPath)) { rmSync(this.tempPath, { recursive: true }); } mkdirSync(this.tempPath); this.logInfo(`Created temp working folder at "${this.tempPath}"`); this.logInfo(`Ready!`); } public GetFilePath(hash: string) { return join(this.path, hash[0], `${hash[0]}${hash[1]}`, hash); } public GetRelativePath(hash: string) { return join(hash[0], `${hash[0]}${hash[1]}`, hash); } public AddFile(contents: Buffer | string) { return new Promise(async (resolve, reject) => { const hasher = createHash("sha1"); hasher.setEncoding("hex"); hasher.write(contents); hasher.end(); const hash: string = hasher.read(); const fileInfo = new HashFSFileInformation(); fileInfo.fileHash = hash; fileInfo.fileSize = contents.length; if (await this.FileExists(hash)) { fileInfo.fileExistsAlready = true; this.logInfo(`File with hash "${hash}" already exists.`); return resolve(fileInfo); } writeFile(this.GetFilePath(hash), contents, (err) => { if (err) { return reject(err); } resolve(fileInfo); }) }); } public AddFromStream(stream: BusboyFileStream) { return new Promise(async (resolve, reject) => { const hasher = createHash("sha1"); hasher.setEncoding("hex"); let fileSize = 0; const tempFilePath = join(this.tempPath, randomBytes(16).toString("base64url")); await pipeline(stream, createWriteStream(tempFilePath)); const readStream = createReadStream(tempFilePath); for await (const chunk of readStream) { fileSize += chunk.length; hasher.write(chunk) } hasher.end(); const hash: string = hasher.read(); const fileInfo = new HashFSFileInformation(); fileInfo.fileHash = hash; fileInfo.fileSize = fileSize; if (await this.FileExists(hash)) { fileInfo.fileExistsAlready = true; rm(tempFilePath, err => { if (err) { return reject(err); } this.logInfo(`File with hash "${hash}" already exists.`); resolve(fileInfo); }); } else { rename(tempFilePath, this.GetFilePath(hash), err => { if (err) { return reject(err); } this.logInfo(`Stored file as ${hash}`); resolve(fileInfo); }); } }); } public FileExists(hash: string) { return new Promise((resolve, reject) => { stat(this.GetFilePath(hash), (err, _stat) => { if (err) { if (err.code === "ENOENT") { resolve(false); } else { reject(err); } } else { resolve(true); } }); }); } }