From c9688ec899322df191973620253469cf77ae4817 Mon Sep 17 00:00:00 2001 From: Ethan Date: Mon, 15 Feb 2021 08:31:47 +0000 Subject: [PATCH 01/24] Update LICENSE --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index ced896f..ca96115 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2020 Ethan Stubbs +Copyright (c) 2021 Ethan Stubbs Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal -- 2.47.0 From 8bbb2de40f405686195397c085b22f457c0b2893 Mon Sep 17 00:00:00 2001 From: tgpethan Date: Sun, 21 Feb 2021 02:52:47 +0000 Subject: [PATCH 02/24] Add Permissions-Policy --- EUS.js | 1 + 1 file changed, 1 insertion(+) diff --git a/EUS.js b/EUS.js index fbd6c8f..b35e72a 100644 --- a/EUS.js +++ b/EUS.js @@ -184,6 +184,7 @@ module.exports = { res.set("Strict-Transport-Security", "max-age=31536000; includeSubDomains"); res.set("X-XSS-Protection", "1; mode=block"); res.set("Feature-Policy", "fullscreen 'none'"); + res.set("Permissions-Policy", "microphone=(), geolocation=(), magnetometer=(), camera=(), payment=(), usb=(), accelerometer=(), gyroscope=()"); res.set("Referrer-Policy", "strict-origin-when-cross-origin"); res.set("Content-Security-Policy", "block-all-mixed-content;frame-ancestors 'self'"); res.set("X-Frame-Options", "SAMEORIGIN"); -- 2.47.0 From f1c5019b07e4e25704aba3b25e5f2357344e02c3 Mon Sep 17 00:00:00 2001 From: tgpethan Date: Thu, 11 Mar 2021 19:26:42 +0000 Subject: [PATCH 03/24] Update EUS.js --- EUS.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/EUS.js b/EUS.js index b35e72a..6cdc89d 100644 --- a/EUS.js +++ b/EUS.js @@ -64,6 +64,15 @@ if (!fs.existsSync(__dirname + BASE_PATH + "/config.json")) { if (validateConfig(eusConfig)) console.log("[EUS] EUS config passed all checks"); } +function cleanURL(url = "") { + return url.split("%20").join(" ").split("%22").join("\"").split("%23").join("#").split("%24").join("$").split("%25").join("%").split("%26").join("&").split("%27").join("'").split("%2B").join("+").split("%2C").join(",").split("%2F").join("/") + .split("%3A").join(":").split("%3B").join(";").split("%3C").join("<").split("%3D").join("=").split("%3E").join(">").split("%3F").join("?") + .split("%40").join("@") + .split("%5B").join("[").split("%5C").join("\\").split("%5D").join("]").split("%5E").join("^") + .split("%60").join("`") + .split("%7B").join("{").split("%7C").join("|").split("%7D").join("}").split("%7E").join("~"); +} + // Cache for the file count and space usage, this takes a while to do so it's best to cache the result let cacheIsReady = false; async function cacheFilesAndSpace() { @@ -190,6 +199,8 @@ module.exports = { res.set("X-Frame-Options", "SAMEORIGIN"); res.set("X-Content-Type-Options", "nosniff"); + req.url = cleanURL(req.url); + // Check if returned value is true. if (req.url.includes("/api/")) return handleAPI(req, res); -- 2.47.0 From df9d3fbd092071fab4af0acaf391c285c6de9c0a Mon Sep 17 00:00:00 2001 From: tgpethan Date: Tue, 16 Mar 2021 18:48:55 +0000 Subject: [PATCH 04/24] Asyncify --- EUS.js | 653 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 333 insertions(+), 320 deletions(-) diff --git a/EUS.js b/EUS.js index 6cdc89d..7853547 100644 --- a/EUS.js +++ b/EUS.js @@ -1,383 +1,396 @@ const fs = require("fs"), - config = require("../config/config.json"), - chalk = require("chalk"), - busboy = require("connect-busboy"), - randomstring = require("randomstring"), - getSize = require("get-folder-size"), - diskUsage = require("diskusage"), - emoji = require("../misc/emoji_list.json"); + config = require("../config/config.json"), + chalk = require("chalk"), + busboy = require("connect-busboy"), + randomstring = require("randomstring"), + getSize = require("get-folder-size"), + diskUsage = require("diskusage-ng"), + emoji = require("../misc/emoji_list.json"); // Defines the function of this module const MODULE_FUNCTION = "handle_requests", - // Base path for module folder creation and navigation - BASE_PATH = "/EUS"; + // Base path for module folder creation and navigation + BASE_PATH = "/EUS"; let eusConfig = {}, - image_json = {}, - d = new Date(), - startTime, - endTime, - useUploadKey = true, - cacheJSON = ""; + image_json = {}, + d = new Date(), + startTime, + endTime, + useUploadKey = true, + cacheJSON = ""; // Only ran on startup so using sync functions is fine // Makes the folder for files of the module if (!fs.existsSync(__dirname + BASE_PATH)) { - fs.mkdirSync(__dirname + BASE_PATH); - console.log(`[EUS] Made EUS module folder`); + fs.mkdirSync(__dirname + BASE_PATH); + console.log(`[EUS] Made EUS module folder`); } // Makes the folder for frontend files if (!fs.existsSync(__dirname + BASE_PATH + "/files")) { - fs.mkdirSync(__dirname + BASE_PATH + "/files"); - console.log(`[EUS] Made EUS web files folder`); + fs.mkdirSync(__dirname + BASE_PATH + "/files"); + console.log(`[EUS] Made EUS web files folder`); } // Makes the folder for images if (!fs.existsSync(__dirname + BASE_PATH + "/i")) { - fs.mkdirSync(__dirname + BASE_PATH + "/i"); - console.log(`[EUS] Made EUS images folder`); + fs.mkdirSync(__dirname + BASE_PATH + "/i"); + console.log(`[EUS] Made EUS images folder`); } // Makes the image-type file if (!fs.existsSync(__dirname + BASE_PATH + "/image-type.json")) { - // Doesn't exist, create it. - fs.writeFileSync(`${__dirname}${BASE_PATH}/image-type.json`, '{}'); - console.log("[EUS] Made EUS image-type File!"); - // File has been created, load it. - image_json = require(`${__dirname}${BASE_PATH}/image-type.json`); + // Doesn't exist, create it. + fs.writeFileSync(`${__dirname}${BASE_PATH}/image-type.json`, '{}'); + console.log("[EUS] Made EUS image-type File!"); + // File has been created, load it. + image_json = require(`${__dirname}${BASE_PATH}/image-type.json`); } else { - // File already exists, load it. - const ijLoadStartTime = new Date().getTime(); - image_json = require(`${__dirname}${BASE_PATH}/image-type.json`); - console.log(`[EUS] Loaded image-type file, took ${new Date().getTime() - ijLoadStartTime}ms`); + // File already exists, load it. + const ijLoadStartTime = new Date().getTime(); + image_json = require(`${__dirname}${BASE_PATH}/image-type.json`); + console.log(`[EUS] Loaded image-type file, took ${new Date().getTime() - ijLoadStartTime}ms`); } // Makes the config file if (!fs.existsSync(__dirname + BASE_PATH + "/config.json")) { - // Config doesn't exist, make it. - fs.writeFileSync(`${__dirname}${BASE_PATH}/config.json`, '{\n\t"baseURL":"http://example.com/",\n\t"acceptedTypes": [\n\t\t".png",\n\t\t".jpg",\n\t\t".jpeg",\n\t\t".gif"\n\t],\n\t"uploadKey": ""\n}'); - console.log("[EUS] Made EUS config File!"); - console.log("[EUS] Please edit the EUS Config file before restarting."); - // Config has been made, close framework. - process.exit(0); + // Config doesn't exist, make it. + fs.writeFileSync(`${__dirname}${BASE_PATH}/config.json`, '{\n\t"baseURL":"http://example.com/",\n\t"acceptedTypes": [\n\t\t".png",\n\t\t".jpg",\n\t\t".jpeg",\n\t\t".gif"\n\t],\n\t"uploadKey": ""\n}'); + console.log("[EUS] Made EUS config File!"); + console.log("[EUS] Please edit the EUS Config file before restarting."); + // Config has been made, close framework. + process.exit(0); } else { - eusConfig = require(`${__dirname}${BASE_PATH}/config.json`); - if (validateConfig(eusConfig)) console.log("[EUS] EUS config passed all checks"); -} - -function cleanURL(url = "") { - return url.split("%20").join(" ").split("%22").join("\"").split("%23").join("#").split("%24").join("$").split("%25").join("%").split("%26").join("&").split("%27").join("'").split("%2B").join("+").split("%2C").join(",").split("%2F").join("/") - .split("%3A").join(":").split("%3B").join(";").split("%3C").join("<").split("%3D").join("=").split("%3E").join(">").split("%3F").join("?") - .split("%40").join("@") - .split("%5B").join("[").split("%5C").join("\\").split("%5D").join("]").split("%5E").join("^") - .split("%60").join("`") - .split("%7B").join("{").split("%7C").join("|").split("%7D").join("}").split("%7E").join("~"); + eusConfig = require(`${__dirname}${BASE_PATH}/config.json`); + if (validateConfig(eusConfig)) console.log("[EUS] EUS config passed all checks"); } // Cache for the file count and space usage, this takes a while to do so it's best to cache the result let cacheIsReady = false; async function cacheFilesAndSpace() { - const startCacheTime = new Date().getTime(); - let cachedFilesAndSpace = { - files: {} - }, + return new Promise((resolve, reject) => { + const startCacheTime = new Date().getTime(); + let cachedFilesAndSpace = { + files: {} + }, - // Cache file totals - total = 0; - // Add each accepted file type to the json - for (var i2 = 0; i2 < eusConfig.acceptedTypes.length; i2++) { - cachedFilesAndSpace["files"][`${eusConfig.acceptedTypes[i2]}`.replace(".", "")] = 0; - } - // Read all files from the images directory - fs.readdir(__dirname + BASE_PATH + "/i", async (err, files) => { - if (err) throw err; - // Loop through all files - for (var i = 0; i < files.length; i++) { - // Loop through all accepted file types to check for a match - for (var i1 = 0; i1 < eusConfig.acceptedTypes.length; i1++) { - const jsudfg = files[i].split("."); - if (`.${jsudfg[jsudfg.length-1]}` == eusConfig.acceptedTypes[i1]) { - // There is a match! Add it to the json - cachedFilesAndSpace["files"][eusConfig.acceptedTypes[i1].replace(".", "")]++; - // Also increase the total - total++; - } - } - } - // Set the total in the json to the calculated total value - cachedFilesAndSpace["files"]["total"] = total; + // Cache file totals + total = 0; + // Add each accepted file type to the json + for (var i2 = 0; i2 < eusConfig.acceptedTypes.length; i2++) { + cachedFilesAndSpace["files"][`${eusConfig.acceptedTypes[i2]}`.replace(".", "")] = 0; + } + // Read all files from the images directory + fs.readdir(__dirname + BASE_PATH + "/i", async (err, files) => { + if (err) reject(err); + else { + // Loop through all files + for (var i = 0; i < files.length; i++) { + // Loop through all accepted file types to check for a match + for (var i1 = 0; i1 < eusConfig.acceptedTypes.length; i1++) { + const readFileName = files[i].split("."); + if (`.${readFileName[readFileName.length-1]}` == eusConfig.acceptedTypes[i1]) { + // There is a match! Add it to the json + cachedFilesAndSpace["files"][eusConfig.acceptedTypes[i1].replace(".", "")]++; + // Also increase the total + total++; + } + } + } + // Set the total in the json to the calculated total value + cachedFilesAndSpace["files"]["total"] = total; - // Cache usage - cachedFilesAndSpace["space"] = { - usage: {} - }; - // Get the space used on the disk - getSize(__dirname + BASE_PATH + "/i", async (err, size) => { - if (err) throw err; - // Calculate in different units the space taken up on disk - let sizeOfFolder = (size / 1024 / 1024); - cachedFilesAndSpace["space"]["usage"]["mb"] = sizeOfFolder; - sizeOfFolder = (size / 1024 / 1024 / 1024); - cachedFilesAndSpace["space"]["usage"]["gb"] = sizeOfFolder; - cachedFilesAndSpace["space"]["usage"]["string"] = spaceToLowest(size, true); - // Get total disk space - diskUsage.check(__dirname, async (err, data) => { - if (err) throw err; - cachedFilesAndSpace["space"]["total"] = { - value: spaceToLowest(data["total"], false), - mbvalue: (data["total"] / 1024 / 1024), - gbvalue: (data["total"] / 1024 / 1024 / 1024), - stringValue: spaceToLowest(data["total"], true).split(" ")[1].toLowerCase(), - string: spaceToLowest(data["total"], true) - }; + // Cache usage + cachedFilesAndSpace["space"] = { + usage: {} + }; + // Get the space used on the disk + getSize(__dirname + BASE_PATH + "/i", async (err, size) => { + if (err) reject(err); + else { + // Calculate in different units the space taken up on disk + let sizeOfFolder = (size / 2048); + cachedFilesAndSpace["space"]["usage"]["mb"] = sizeOfFolder; + sizeOfFolder = (size / 3072); + cachedFilesAndSpace["space"]["usage"]["gb"] = sizeOfFolder; + cachedFilesAndSpace["space"]["usage"]["string"] = await spaceToLowest(size, true); + // Get total disk space + diskUsage(__dirname, async (err, data) => { + if (err) reject(err); + else { + cachedFilesAndSpace["space"]["total"] = { + value: await spaceToLowest(data["total"], false), + mbvalue: (data["total"] / 2048), + gbvalue: (data["total"] / 3072), + stringValue: (await spaceToLowest(data["total"], true)).split(" ")[1].toLowerCase(), + string: await spaceToLowest(data["total"], true) + }; - cacheIsReady = true; - cacheJSON = JSON.stringify(cachedFilesAndSpace); - global.modules.consoleHelper.printInfo(emoji.folder, `Stats api cache took ${new Date().getTime() - startCacheTime}ms`); - }); - }); - }); + resolve(cachedFilesAndSpace); + global.modules.consoleHelper.printInfo(emoji.folder, `Stats api cache took ${new Date().getTime() - startCacheTime}ms`); + } + }); + } + }); + } + }); + }); } function validateConfig(json) { - let performShutdownAfterValidation = false; - // URL Tests - if (json["baseURL"] == null) { - console.error("EUS baseURL property does not exist!"); - performShutdownAfterValidation = true; - } else { - if (json["baseURL"] == "") console.warn("EUS baseURL property is blank"); - const bURL = `${json["baseURL"]}`.split(""); - if (bURL.length > 1) { - if (bURL[bURL.length-1] != "/") console.warn("EUS baseURL property doesn't have a / at the end, this can lead to unpredictable results!"); - } - else { - if (json["baseURL"] != "http://" || json["baseURL"] != "https://") console.warn("EUS baseURL property is possibly invalid!"); - } - } - // acceptedTypes checks - if (json["acceptedTypes"] == null) { - console.error("EUS acceptedTypes array does not exist!"); - performShutdownAfterValidation = true; - } else { - if (json["acceptedTypes"].length < 1) console.warn("EUS acceptedTypes array has no extentions in it, users will not be able to upload images!"); - } - // uploadKey checks - if (json["uploadKey"] == null) { - console.error("EUS uploadKey property does not exist!"); - performShutdownAfterValidation = true; - } else { - if (json["uploadKey"] == "") useUploadKey = false; - } + let performShutdownAfterValidation = false; + // URL Tests + if (json["baseURL"] == null) { + console.error("EUS baseURL property does not exist!"); + performShutdownAfterValidation = true; + } else { + if (json["baseURL"] == "") console.warn("EUS baseURL property is blank"); + const bURL = `${json["baseURL"]}`.split(""); + if (bURL.length > 1) { + if (bURL[bURL.length-1] != "/") console.warn("EUS baseURL property doesn't have a / at the end, this can lead to unpredictable results!"); + } + else { + if (json["baseURL"] != "http://" || json["baseURL"] != "https://") console.warn("EUS baseURL property is possibly invalid!"); + } + } + // acceptedTypes checks + if (json["acceptedTypes"] == null) { + console.error("EUS acceptedTypes array does not exist!"); + performShutdownAfterValidation = true; + } else { + if (json["acceptedTypes"].length < 1) console.warn("EUS acceptedTypes array has no extentions in it, users will not be able to upload images!"); + } + // uploadKey checks + if (json["uploadKey"] == null) { + console.error("EUS uploadKey property does not exist!"); + performShutdownAfterValidation = true; + } else { + if (json["uploadKey"] == "") useUploadKey = false; + } - // Check if server needs to be shutdown - if (performShutdownAfterValidation) { - console.error("EUS config properties are missing, refer to docs for more details (https://docs.ethanus.ml)"); - process.exit(1); - } - else return true; + // Check if server needs to be shutdown + if (performShutdownAfterValidation) { + console.error("EUS config properties are missing, refer to docs for more details (https://docs.ethanus.ml)"); + process.exit(1); + } + else return true; +} + +function cleanURL(url = "") { + return url.split("%20").join(" ").split("%22").join("\"").split("%23").join("#").split("%24").join("$").split("%25").join("%").split("%26").join("&").split("%27").join("'").split("%2B").join("+").split("%2C").join(",").split("%2F").join("/") + .split("%3A").join(":").split("%3B").join(";").split("%3C").join("<").split("%3D").join("=").split("%3E").join(">").split("%3F").join("?") + .split("%40").join("@") + .split("%5B").join("[").split("%5C").join("\\").split("%5D").join("]").split("%5E").join("^") + .split("%60").join("`") + .split("%7B").join("{").split("%7C").join("|").split("%7D").join("}").split("%7E").join("~"); } module.exports = { - extras:function() { - // Setup express to use busboy - global.app.use(busboy()); - cacheFilesAndSpace(); - }, - get:function(req, res) { - /* - req - Request from client - res - Response from server - */ - - // Set some headers - res.set("Strict-Transport-Security", "max-age=31536000; includeSubDomains"); - res.set("X-XSS-Protection", "1; mode=block"); - res.set("Feature-Policy", "fullscreen 'none'"); - res.set("Permissions-Policy", "microphone=(), geolocation=(), magnetometer=(), camera=(), payment=(), usb=(), accelerometer=(), gyroscope=()"); - res.set("Referrer-Policy", "strict-origin-when-cross-origin"); - res.set("Content-Security-Policy", "block-all-mixed-content;frame-ancestors 'self'"); - res.set("X-Frame-Options", "SAMEORIGIN"); - res.set("X-Content-Type-Options", "nosniff"); + extras:async function() { + // Setup express to use busboy + global.app.use(busboy()); + cacheJSON = JSON.stringify(await cacheFilesAndSpace()); + cacheIsReady = true; + }, + get:async function(req, res) { + /* + req - Request from client + res - Response from server + */ + + // Set some headers + res.set("Strict-Transport-Security", "max-age=31536000; includeSubDomains"); + res.set("X-XSS-Protection", "1; mode=block"); + res.set("Feature-Policy", "fullscreen 'none'"); + res.set("Permissions-Policy", "microphone=(), geolocation=(), magnetometer=(), camera=(), payment=(), usb=(), accelerometer=(), gyroscope=()"); + res.set("Referrer-Policy", "strict-origin-when-cross-origin"); + res.set("Content-Security-Policy", "block-all-mixed-content;frame-ancestors 'self'"); + res.set("X-Frame-Options", "SAMEORIGIN"); + res.set("X-Content-Type-Options", "nosniff"); - req.url = cleanURL(req.url); + req.url = cleanURL(req.url); - // Check if returned value is true. - if (req.url.includes("/api/")) return handleAPI(req, res); + // Check if returned value is true. + if (req.url.includes("/api/")) return handleAPI(req, res); - // Register the time at the start of the request - d = new Date(); - startTime = d.getTime(); - // Get the requested image - let urs = ""+req.url; urs = urs.split("/")[1]; - // Get the file type of the image from image_json and make sure it exists - fs.access(__dirname + BASE_PATH + "/i/"+urs+image_json[urs], error => { - if (error) { - // Doesn't exist, handle request normaly - if (req.url === "/") { urs = "/index.html" } else { urs = req.url } - fs.access(__dirname + BASE_PATH + "/files"+urs, error => { - if (error) { - // Doesn't exist, send a 404 to the client. - res.status(404).end("404!"); - d = new Date(); - endTime = d.getTime(); - global.modules.consoleHelper.printInfo(emoji.cross, `${req.method}: ${chalk.red("[404]")} ${req.url} ${endTime - startTime}ms`); - } else { - // File does exist, send it back to the client. - res.sendFile(__dirname + BASE_PATH + "/files"+req.url); - d = new Date(); - endTime = d.getTime(); - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); - } - }); - } else { - // Image does exist, send it back. - res.sendFile(__dirname + BASE_PATH + "/i/"+urs+image_json[urs]); - d = new Date(); - endTime = d.getTime(); - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); - } - }); - }, - post:function(req, res) { - /* - req - Request from client - res - Response from server - */ + // Register the time at the start of the request + d = new Date(); + startTime = d.getTime(); - // Make sure the endpoint is /upload - // If it isn't upload send an empty response - if (req.url != "/upload") return res.end(""); + // Get the requested image + let urs = `${req.url}`; urs = urs.split("/")[1]; + // Get the file type of the image from image_json and make sure it exists + fs.access(`${__dirname}${BASE_PATH}/i/${urs}${image_json[urs]}`, (error) => { + if (error) { + // Doesn't exist, handle request normaly + if (req.url === "/") { urs = "/index.html" } else { urs = req.url } + fs.access(`${__dirname}${BASE_PATH}/files${urs}`, (error) => { + if (error) { + // Doesn't exist, send a 404 to the client. + res.status(404).send("404!
EUS"); + d = new Date(); + endTime = d.getTime(); + global.modules.consoleHelper.printInfo(emoji.cross, `${req.method}: ${chalk.red("[404]")} ${req.url} ${endTime - startTime}ms`); + } else { + // File does exist, send it back to the client. + res.sendFile(__dirname + BASE_PATH + "/files"+req.url); + d = new Date(); + endTime = d.getTime(); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); + } + }); + } else { + // Image does exist, send it back. + res.sendFile(`${__dirname}${BASE_PATH}/i/${urs}${image_json[urs]}`); + d = new Date(); + endTime = d.getTime(); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); + } + }); + }, + post:async function(req, res) { + /* + req - Request from client + res - Response from server + */ - // Get time at the start of upload + // Make sure the endpoint is /upload + // If it isn't upload send an empty response + if (req.url != "/upload") return res.end(""); - if (useUploadKey && eusConfig["uploadKey"] != req.header("key")) return res.end("Incorrect key provided for upload"); + // Get time at the start of upload - d = new Date(); startTime = d.getTime(); - var fstream; - var thefe; - // Pipe the request to busboy - req.pipe(req.busboy); - req.busboy.on('file', function (fieldname, file, filename) { - // Get the image-type json - image_json = require(`${__dirname}${BASE_PATH}/image-type.json`); - // Make a new file name - fileOutName = randomstring.generate(14); - global.modules.consoleHelper.printInfo(emoji.fast_up, `${req.method}: Upload of ${fileOutName} started.`); - // Check the file is within the accepted file types - if (eusConfig.acceptedTypes.includes(`.${filename.split(".")[filename.split(".").length-1]}`)) { - // File is accepted, set the extention of the file in thefe for later use. - thefe = `.${filename.split(".")[filename.split(".").length-1]}`; - } else { - // File isn't accepted, send response back to client stating so. - res.status(403).end("This file type isn't accepted currently."); - return; - } - // Create a write stream for the file - fstream = fs.createWriteStream(__dirname + BASE_PATH + "/i/" + fileOutName + thefe); - file.pipe(fstream); - fstream.on('close', function () { - // Get the time at the end of the upload - d = new Date(); endTime = d.getTime(); - // Add image file type to the image_json array - image_json[fileOutName] = `.${filename.split(".")[filename.split(".").length-1]}`; - // Save image_json array to the file - fs.writeFile(`${__dirname}${BASE_PATH}/image-type.json`, JSON.stringify(image_json), function(err) { - if (err) throw err; - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: Upload of ${fileOutName} finished. Took ${endTime - startTime}ms`); - // Send URL of the uploaded image to the client - res.end(eusConfig.baseURL+""+fileOutName); + if (useUploadKey && eusConfig["uploadKey"] != req.header("key")) return res.end("Incorrect key provided for upload"); - // Update cached files & space - cacheFilesAndSpace(); - }); - }); - }); - } + d = new Date(); startTime = d.getTime(); + var fstream; + var thefe; + // Pipe the request to busboy + req.pipe(req.busboy); + req.busboy.on('file', function (fieldname, file, filename) { + // Get the image-type json + image_json = require(`${__dirname}${BASE_PATH}/image-type.json`); + // Make a new file name + fileOutName = randomstring.generate(14); + global.modules.consoleHelper.printInfo(emoji.fast_up, `${req.method}: Upload of ${fileOutName} started.`); + // Check the file is within the accepted file types + if (eusConfig.acceptedTypes.includes(`.${filename.split(".")[filename.split(".").length-1]}`)) { + // File is accepted, set the extention of the file in thefe for later use. + thefe = `.${filename.split(".")[filename.split(".").length-1]}`; + } else { + // File isn't accepted, send response back to client stating so. + res.status(403).end("This file type isn't accepted currently."); + return; + } + // Create a write stream for the file + fstream = fs.createWriteStream(__dirname + BASE_PATH + "/i/" + fileOutName + thefe); + file.pipe(fstream); + fstream.on('close', function () { + // Get the time at the end of the upload + d = new Date(); endTime = d.getTime(); + // Add image file type to the image_json array + image_json[fileOutName] = `.${filename.split(".")[filename.split(".").length-1]}`; + // Save image_json array to the file + fs.writeFile(`${__dirname}${BASE_PATH}/image-type.json`, JSON.stringify(image_json), async (err) => { + if (err) reject(err); + else { + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: Upload of ${fileOutName} finished. Took ${endTime - startTime}ms`); + // Send URL of the uploaded image to the client + res.end(eusConfig.baseURL+""+fileOutName); + + // Update cached files & space + cacheJSON = JSON.stringify(await cacheFilesAndSpace()); + } + }); + }); + }); + } } -function handleAPI(req, res) { - d = new Date(); startTime = d.getTime(); - let jsonaa = {}, filesaa = 0, spaceaa = 0; - switch (req.url.split("?")[0]) { - // Status check to see the onlint status of EUS - // Used by ESL to make sure EUS is online - case "/api/get-server-status": - d = new Date(); endTime = d.getTime(); - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); - return res.end('{ "status":1, "version":"'+global.internals.version+'" }'); - - /* Stats api endpoint - Query inputs - f : Values [0,1] - s : Values [0,1] - */ - case "/api/get-stats": - filesaa = req.query["f"]; - spaceaa = req.query["s"]; - if (!cacheIsReady) return res.end("Cache is not ready"); - jsonaa = JSON.parse(cacheJSON); - // If total files is asked for - if (filesaa == 1) { - // If getting the space used on the server isn't required send the json - if (spaceaa != 1) { - d = new Date(); endTime = d.getTime(); - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); - delete jsonaa["space"]; - return res.end(JSON.stringify(jsonaa)); - } - } - // Getting space is required - if (spaceaa == 1) { - d = new Date(); endTime = d.getTime(); - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); - if (filesaa != 1) delete jsonaa["files"]; - return res.end(JSON.stringify(jsonaa)); - } +async function handleAPI(req, res) { + d = new Date(); startTime = d.getTime(); + let jsonaa = {}, filesaa = 0, spaceaa = 0; + switch (req.url.split("?")[0]) { + // Status check to see the onlint status of EUS + // Used by ESL to make sure EUS is online + case "/api/get-server-status": + d = new Date(); endTime = d.getTime(); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); + return res.end('{ "status":1, "version":"'+global.internals.version+'" }'); + + /* Stats api endpoint + Query inputs + f : Values [0,1] + s : Values [0,1] + */ + case "/api/get-stats": + filesaa = req.query["f"]; + spaceaa = req.query["s"]; + if (!cacheIsReady) return res.end("Cache is not ready"); + jsonaa = JSON.parse(cacheJSON); + // If total files is asked for + if (filesaa == 1) { + // If getting the space used on the server isn't required send the json + if (spaceaa != 1) { + d = new Date(); endTime = d.getTime(); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); + delete jsonaa["space"]; + return res.end(JSON.stringify(jsonaa)); + } + } + // Getting space is required + if (spaceaa == 1) { + d = new Date(); endTime = d.getTime(); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); + if (filesaa != 1) delete jsonaa["files"]; + return res.end(JSON.stringify(jsonaa)); + } - if (filesaa != 1 && spaceaa != 1) { - d = new Date(); endTime = d.getTime(); - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); - return res.end("Please add f and or s to your queries to get the files and space"); - } - break; + if (filesaa != 1 && spaceaa != 1) { + d = new Date(); endTime = d.getTime(); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); + return res.end("Please add f and or s to your queries to get the files and space"); + } + break; - // Information API - case "/api/get-info": - d = new Date(); endTime = d.getTime(); - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); - return res.end(JSON.stringify({ - version: global.internals.version, - instance: config["server"]["instance_type"] - })); + // Information API + case "/api/get-info": + d = new Date(); endTime = d.getTime(); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); + return res.end(JSON.stringify({ + version: global.internals.version, + instance: config["server"]["instance_type"] + })); - default: - d = new Date(); endTime = d.getTime(); - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); - return res.send(` -

All currently avaliable api endpoints

- /api/get-server-status - /api/get-stats - /api/get-info - `); - } + default: + d = new Date(); endTime = d.getTime(); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); + return res.send(` +

All currently avaliable api endpoints

+ /api/get-server-status + /api/get-stats + /api/get-info + `); + } } const spaceValues = ["B", "KB", "MB", "GB", "TB", "PB", "EB"]; // Futureproofing:tm: -function spaceToLowest(spaceValue, includeStringValue) { - // Converts space values to lower values e.g MB, GB, TB etc depending on the size of the number - let i1 = 1; - // Loop through until value is at it's lowest - for (let i = 0; i < i1; i++) { - if (spaceValue >= 1024) { - spaceValue = spaceValue / 1024; - } +async function spaceToLowest(spaceValue, includeStringValue) { + return new Promise((resolve, reject) => { + // Converts space values to lower values e.g MB, GB, TB etc depending on the size of the number + let i1 = 1; + // Loop through until value is at it's lowest + for (let i = 0; i < i1; i++) { + if (spaceValue >= 1024) { + spaceValue = spaceValue / 1024; + } - if (spaceValue >= 1024) i1++; - } + if (spaceValue >= 1024) i1++; + } - if (includeStringValue) return `${spaceValue.toFixed(2)} ${spaceValues[i1]}`; - else return spaceValue; + if (includeStringValue) resolve(`${spaceValue.toFixed(2)} ${spaceValues[i1]}`); + else resolve(spaceValue); + }); } module.exports.MOD_FUNC = MODULE_FUNCTION; \ No newline at end of file -- 2.47.0 From 85fa9e82ce0aa7414a19e3e281bbb883ca927610 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 7 May 2021 03:00:48 +0100 Subject: [PATCH 05/24] https --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d9e49af..1adac0a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- +

EUS is my public screenshot server built using Revolution
-- 2.47.0 From e3d5a84756ff498029a0d71c6a010f15036498aa Mon Sep 17 00:00:00 2001 From: Holly Date: Sat, 21 Aug 2021 09:42:08 +0100 Subject: [PATCH 06/24] Delete quick-setup.sh --- quick-setup.sh | 8 -------- 1 file changed, 8 deletions(-) delete mode 100755 quick-setup.sh diff --git a/quick-setup.sh b/quick-setup.sh deleted file mode 100755 index 9c4c87a..0000000 --- a/quick-setup.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -git clone https://github.com/tgpethan/Revolution.git -cd Revolution -npm i -wget https://raw.githubusercontent.com/tgpethan/EUS/master/EUS.js -P modules/ -rm modules/example_request_handler.js -cp config/config.example.json config/config.json -npm i connect-busboy randomstring diskusage get-folder-size -- 2.47.0 From 87453e510946cecf792275355f05ea9a92fd51c3 Mon Sep 17 00:00:00 2001 From: Holly Date: Thu, 23 Sep 2021 23:03:14 +0100 Subject: [PATCH 07/24] aaaaa --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index ca96115..f95419e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2021 Ethan Stubbs +Copyright (c) 2021 Holly Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal -- 2.47.0 From bfe6676867b79c47726f2337cadf9eff1676afc6 Mon Sep 17 00:00:00 2001 From: Holly Date: Sat, 11 Dec 2021 06:24:35 +0000 Subject: [PATCH 08/24] database shite --- EUS.js | 366 +++++++++++++++++++++++++++++++----------------------- README.md | 3 +- 2 files changed, 211 insertions(+), 158 deletions(-) diff --git a/EUS.js b/EUS.js index 7853547..889a233 100644 --- a/EUS.js +++ b/EUS.js @@ -3,8 +3,9 @@ const fs = require("fs"), chalk = require("chalk"), busboy = require("connect-busboy"), randomstring = require("randomstring"), - getSize = require("get-folder-size"), - diskUsage = require("diskusage-ng"), + diskUsage = require("diskusage"), + streamMeter = require("stream-meter"), + mysql = require("mysql"), emoji = require("../misc/emoji_list.json"); // Defines the function of this module @@ -13,13 +14,70 @@ const MODULE_FUNCTION = "handle_requests", // Base path for module folder creation and navigation BASE_PATH = "/EUS"; +console.log("[EUS] Loading EUS..."); + +// This will never change +const diskRunningOnSize = diskUsage.checkSync(__dirname).total; + let eusConfig = {}, - image_json = {}, - d = new Date(), - startTime, - endTime, useUploadKey = true, - cacheJSON = ""; + cacheJSON = "", + startupFinished = false; + +class Database { + constructor(databaseAddress, databasePort = 3306, databaseUsername, databasePassword, databaseName, connectedCallback) { + this.connectionPool = mysql.createPool({ + connectionLimit: 128, + host: databaseAddress, + port: databasePort, + user: databaseUsername, + password: databasePassword, + database: databaseName + }); + + this.dbActive = false; + if (connectedCallback == null) { + this.dbActive = true; + } else { + const connectionCheckInterval = setInterval(() => { + this.query("SELECT id FROM images LIMIT 1") + .then(data => { + if (startupFinished) global.modules.consoleHelper.printInfo(emoji.globe_europe, `Connected to database`); + else console.log("[EUS] Connected to database"); + this.dbActive = true; + clearInterval(connectionCheckInterval); + + connectedCallback(); + }) + .catch(err => {}); + }, 167); // Roughly 6 times per sec + } + } + + async query(sqlQuery) { + return new Promise((resolve, reject) => { + this.connectionPool.getConnection((err, connection) => { + if (err) { + reject(err); + try { + connection.release(); + } catch (e) {} + } else { + connection.query(sqlQuery, (err, data) => { + if (err) { + reject(err); + connection.release(); + } else { + if (sqlQuery.includes("LIMIT 1")) resolve(data[0]); + else resolve(data); + connection.release(); + } + }); + } + }); + }); + } +} // Only ran on startup so using sync functions is fine // Makes the folder for files of the module @@ -37,24 +95,11 @@ if (!fs.existsSync(__dirname + BASE_PATH + "/i")) { fs.mkdirSync(__dirname + BASE_PATH + "/i"); console.log(`[EUS] Made EUS images folder`); } -// Makes the image-type file -if (!fs.existsSync(__dirname + BASE_PATH + "/image-type.json")) { - // Doesn't exist, create it. - fs.writeFileSync(`${__dirname}${BASE_PATH}/image-type.json`, '{}'); - console.log("[EUS] Made EUS image-type File!"); - // File has been created, load it. - image_json = require(`${__dirname}${BASE_PATH}/image-type.json`); -} else { - // File already exists, load it. - const ijLoadStartTime = new Date().getTime(); - image_json = require(`${__dirname}${BASE_PATH}/image-type.json`); - console.log(`[EUS] Loaded image-type file, took ${new Date().getTime() - ijLoadStartTime}ms`); -} // Makes the config file if (!fs.existsSync(__dirname + BASE_PATH + "/config.json")) { // Config doesn't exist, make it. - fs.writeFileSync(`${__dirname}${BASE_PATH}/config.json`, '{\n\t"baseURL":"http://example.com/",\n\t"acceptedTypes": [\n\t\t".png",\n\t\t".jpg",\n\t\t".jpeg",\n\t\t".gif"\n\t],\n\t"uploadKey": ""\n}'); + fs.writeFileSync(`${__dirname}${BASE_PATH}/config.json`, '{\n\t"baseURL":"http://example.com/",\n\t"acceptedTypes": [\n\t\t".png",\n\t\t".jpg",\n\t\t".jpeg",\n\t\t".gif"\n\t],\n\t"uploadKey": "",\n\t"database": {\n\t\t"databaseAddress": "127.0.0.1",\n\t\t"databasePort": 3306,\n\t\t"databaseUsername": "root",\n\t\t"databasePassword": "password",\n\t\t"databaseName": "EUS"\n\t}\n}'); console.log("[EUS] Made EUS config File!"); console.log("[EUS] Please edit the EUS Config file before restarting."); // Config has been made, close framework. @@ -64,75 +109,46 @@ if (!fs.existsSync(__dirname + BASE_PATH + "/config.json")) { if (validateConfig(eusConfig)) console.log("[EUS] EUS config passed all checks"); } +const dbConnection = new Database(eusConfig["database"]["databaseAddress"], eusConfig["database"]["databasePort"], eusConfig["database"]["databaseUsername"], eusConfig["database"]["databasePassword"], eusConfig["database"]["databaseName"], async () => { + cacheJSON = JSON.stringify(await cacheFilesAndSpace()); + cacheIsReady = true; +}); + // Cache for the file count and space usage, this takes a while to do so it's best to cache the result let cacheIsReady = false; -async function cacheFilesAndSpace() { - return new Promise((resolve, reject) => { - const startCacheTime = new Date().getTime(); +function cacheFilesAndSpace() { + return new Promise(async (resolve, reject) => { + const startCacheTime = Date.now(); + cacheIsReady = false; let cachedFilesAndSpace = { - files: {} - }, + fileCounts: {}, + }; - // Cache file totals - total = 0; - // Add each accepted file type to the json - for (var i2 = 0; i2 < eusConfig.acceptedTypes.length; i2++) { - cachedFilesAndSpace["files"][`${eusConfig.acceptedTypes[i2]}`.replace(".", "")] = 0; - } - // Read all files from the images directory - fs.readdir(__dirname + BASE_PATH + "/i", async (err, files) => { - if (err) reject(err); - else { - // Loop through all files - for (var i = 0; i < files.length; i++) { - // Loop through all accepted file types to check for a match - for (var i1 = 0; i1 < eusConfig.acceptedTypes.length; i1++) { - const readFileName = files[i].split("."); - if (`.${readFileName[readFileName.length-1]}` == eusConfig.acceptedTypes[i1]) { - // There is a match! Add it to the json - cachedFilesAndSpace["files"][eusConfig.acceptedTypes[i1].replace(".", "")]++; - // Also increase the total - total++; - } - } - } - // Set the total in the json to the calculated total value - cachedFilesAndSpace["files"]["total"] = total; - - // Cache usage - cachedFilesAndSpace["space"] = { - usage: {} - }; - // Get the space used on the disk - getSize(__dirname + BASE_PATH + "/i", async (err, size) => { - if (err) reject(err); - else { - // Calculate in different units the space taken up on disk - let sizeOfFolder = (size / 2048); - cachedFilesAndSpace["space"]["usage"]["mb"] = sizeOfFolder; - sizeOfFolder = (size / 3072); - cachedFilesAndSpace["space"]["usage"]["gb"] = sizeOfFolder; - cachedFilesAndSpace["space"]["usage"]["string"] = await spaceToLowest(size, true); - // Get total disk space - diskUsage(__dirname, async (err, data) => { - if (err) reject(err); - else { - cachedFilesAndSpace["space"]["total"] = { - value: await spaceToLowest(data["total"], false), - mbvalue: (data["total"] / 2048), - gbvalue: (data["total"] / 3072), - stringValue: (await spaceToLowest(data["total"], true)).split(" ")[1].toLowerCase(), - string: await spaceToLowest(data["total"], true) - }; - - resolve(cachedFilesAndSpace); - global.modules.consoleHelper.printInfo(emoji.folder, `Stats api cache took ${new Date().getTime() - startCacheTime}ms`); - } - }); - } - }); - } + const dbData = await dbConnection.query(`SELECT imageType, COUNT(imageType) AS "count" FROM images GROUP BY imageType`); + let totalFiles = 0; + dbData.forEach(fileType => { + cachedFilesAndSpace["fileCounts"][fileType.imageType] = fileType.count; + totalFiles += fileType.count; }); + cachedFilesAndSpace["filesTotal"] = totalFiles; + + cachedFilesAndSpace["space"] = {usage: {}, total:{}}; + + const dbSize = (await dbConnection.query(`SELECT SUM(imageSize) FROM images LIMIT 1`))["SUM(imageSize)"]; + const totalSizeBytes = dbSize == null ? 0 : dbSize; + const mbSize = totalSizeBytes / 1024 / 1024; + cachedFilesAndSpace["space"]["usage"]["mb"] = parseFloat(mbSize.toFixed(4)); + cachedFilesAndSpace["space"]["usage"]["gb"] = parseFloat((mbSize / 1024).toFixed(4)); + cachedFilesAndSpace["space"]["usage"]["string"] = await spaceToLowest(totalSizeBytes, true); + + //totalDiskSize + const totalMBSize = diskRunningOnSize / 1024 / 1024; + cachedFilesAndSpace["space"]["total"]["mb"] = parseFloat(totalMBSize.toFixed(4)); + cachedFilesAndSpace["space"]["total"]["gb"] = parseFloat((totalMBSize / 1024).toFixed(4)); + cachedFilesAndSpace["space"]["total"]["string"] = await spaceToLowest(diskRunningOnSize, true); + + resolve(cachedFilesAndSpace); + global.modules.consoleHelper.printInfo(emoji.folder, `Stats api cache took ${Date.now() - startCacheTime}ms`); }); } @@ -154,7 +170,7 @@ function validateConfig(json) { } // acceptedTypes checks if (json["acceptedTypes"] == null) { - console.error("EUS acceptedTypes array does not exist!"); + console.error("EUS acceptedTypes list does not exist!"); performShutdownAfterValidation = true; } else { if (json["acceptedTypes"].length < 1) console.warn("EUS acceptedTypes array has no extentions in it, users will not be able to upload images!"); @@ -166,10 +182,41 @@ function validateConfig(json) { } else { if (json["uploadKey"] == "") useUploadKey = false; } + // database checks + if (json["database"] == null) { + console.error("EUS database properties do not exist!"); + performShutdownAfterValidation = true; + } else { + // databaseAddress + if (json["database"]["databaseAddress"] == null) { + console.error("EUS database.databaseAddress property does not exist!"); + performShutdownAfterValidation = true; + } + // databasePort + if (json["database"]["databasePort"] == null) { + console.error("EUS database.databasePort property does not exist!"); + performShutdownAfterValidation = true; + } + // databaseUsername + if (json["database"]["databaseUsername"] == null) { + console.error("EUS database.databaseUsername property does not exist!"); + performShutdownAfterValidation = true; + } + // databasePassword + if (json["database"]["databasePassword"] == null) { + console.error("EUS database.databasePassword property does not exist!"); + performShutdownAfterValidation = true; + } + // databaseName + if (json["database"]["databaseName"] == null) { + console.error("EUS database.databaseName property does not exist!"); + performShutdownAfterValidation = true; + } + } // Check if server needs to be shutdown if (performShutdownAfterValidation) { - console.error("EUS config properties are missing, refer to docs for more details (https://docs.ethanus.ml)"); + console.error("EUS config properties are missing, refer to docs for more details (https://wiki.eusv.ml)"); process.exit(1); } else return true; @@ -184,12 +231,32 @@ function cleanURL(url = "") { .split("%7B").join("{").split("%7C").join("|").split("%7D").join("}").split("%7E").join("~"); } +function regularFile(req, res, urs = "", startTime = 0) { + if (req.url === "/") { urs = "/index.html" } else { urs = req.url } + fs.access(`${__dirname}${BASE_PATH}/files${urs}`, (error) => { + if (error) { + // Doesn't exist, send a 404 to the client. + error404Page(res); + global.modules.consoleHelper.printInfo(emoji.cross, `${req.method}: ${chalk.red("[404]")} ${req.url} ${Date.now() - startTime}ms`); + } else { + // File does exist, send it back to the client. + res.sendFile(__dirname + BASE_PATH + "/files"+req.url); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${Date.now() - startTime}ms`); + } + }); +} + +function error404Page(res) { + res.status(404).send("404!

EUS"); +} + module.exports = { extras:async function() { // Setup express to use busboy global.app.use(busboy()); - cacheJSON = JSON.stringify(await cacheFilesAndSpace()); - cacheIsReady = true; + startupFinished = true; + //cacheJSON = JSON.stringify(await cacheFilesAndSpace()); + //cacheIsReady = true; }, get:async function(req, res) { /* @@ -213,39 +280,31 @@ module.exports = { if (req.url.includes("/api/")) return handleAPI(req, res); // Register the time at the start of the request - d = new Date(); - startTime = d.getTime(); + const startTime = Date.now(); // Get the requested image let urs = `${req.url}`; urs = urs.split("/")[1]; - // Get the file type of the image from image_json and make sure it exists - fs.access(`${__dirname}${BASE_PATH}/i/${urs}${image_json[urs]}`, (error) => { - if (error) { - // Doesn't exist, handle request normaly - if (req.url === "/") { urs = "/index.html" } else { urs = req.url } - fs.access(`${__dirname}${BASE_PATH}/files${urs}`, (error) => { - if (error) { - // Doesn't exist, send a 404 to the client. - res.status(404).send("404!
EUS"); - d = new Date(); - endTime = d.getTime(); - global.modules.consoleHelper.printInfo(emoji.cross, `${req.method}: ${chalk.red("[404]")} ${req.url} ${endTime - startTime}ms`); - } else { - // File does exist, send it back to the client. - res.sendFile(__dirname + BASE_PATH + "/files"+req.url); - d = new Date(); - endTime = d.getTime(); - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); - } - }); - } else { - // Image does exist, send it back. - res.sendFile(`${__dirname}${BASE_PATH}/i/${urs}${image_json[urs]}`); - d = new Date(); - endTime = d.getTime(); - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); + // Check if we even need to query the DB + const dbAble = (!/[^0-9A-Za-z]/.test(urs)) && urs != "" && urs != "index" && (req.url.split("/").length == 2); + + if (dbAble) { + if (dbConnection.dbActive) { + // Try to get what we think is an image's details from the DB + const dbEntry = await dbConnection.query(`SELECT imageType FROM images WHERE imageId = "${urs}" LIMIT 1`); + + // There's an entry in the DB for this, send the file back. + if (dbEntry != null) { + res.sendFile(`${__dirname}${BASE_PATH}/i/${urs}.${dbEntry.imageType}`); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} (ImageReq) ${req.url} ${Date.now() - startTime}ms`); + } + // There's no entry, so treat this as a regular file. + else regularFile(req, res, urs, startTime); } - }); + else res.status(400).end("EUS is restarting, please try again in a few secs."); + } + // We can still serve files if they are not dbable + // since we don't need to check the db + else regularFile(req, res, urs, startTime); }, post:async function(req, res) { /* @@ -261,61 +320,55 @@ module.exports = { if (useUploadKey && eusConfig["uploadKey"] != req.header("key")) return res.end("Incorrect key provided for upload"); - d = new Date(); startTime = d.getTime(); + const startTime = Date.now(); var fstream; var thefe; // Pipe the request to busboy req.pipe(req.busboy); req.busboy.on('file', function (fieldname, file, filename) { - // Get the image-type json - image_json = require(`${__dirname}${BASE_PATH}/image-type.json`); // Make a new file name fileOutName = randomstring.generate(14); global.modules.consoleHelper.printInfo(emoji.fast_up, `${req.method}: Upload of ${fileOutName} started.`); // Check the file is within the accepted file types if (eusConfig.acceptedTypes.includes(`.${filename.split(".")[filename.split(".").length-1]}`)) { // File is accepted, set the extention of the file in thefe for later use. - thefe = `.${filename.split(".")[filename.split(".").length-1]}`; + thefe = `${filename.split(".")[filename.split(".").length-1]}`; } else { // File isn't accepted, send response back to client stating so. res.status(403).end("This file type isn't accepted currently."); return; } // Create a write stream for the file - fstream = fs.createWriteStream(__dirname + BASE_PATH + "/i/" + fileOutName + thefe); - file.pipe(fstream); - fstream.on('close', function () { - // Get the time at the end of the upload - d = new Date(); endTime = d.getTime(); - // Add image file type to the image_json array - image_json[fileOutName] = `.${filename.split(".")[filename.split(".").length-1]}`; - // Save image_json array to the file - fs.writeFile(`${__dirname}${BASE_PATH}/image-type.json`, JSON.stringify(image_json), async (err) => { - if (err) reject(err); - else { - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: Upload of ${fileOutName} finished. Took ${endTime - startTime}ms`); - // Send URL of the uploaded image to the client - res.end(eusConfig.baseURL+""+fileOutName); + fstream = fs.createWriteStream(__dirname + BASE_PATH + "/i/" + fileOutName + "." + thefe); + // Create meter for tracking the size of the file + const meter = streamMeter(); + file.pipe(meter).pipe(fstream); + fstream.on('close', async () => { + // Add this image to the database + await dbConnection.query(`INSERT INTO images (id, imageId, imageType, imageSize) VALUES (NULL, "${fileOutName}", "${thefe}", ${meter.bytes})`); + + // Send URL of the uploaded image to the client + res.end(eusConfig.baseURL + fileOutName); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: Upload of ${fileOutName} finished. Took ${Date.now() - startTime}ms`); - // Update cached files & space - cacheJSON = JSON.stringify(await cacheFilesAndSpace()); - } - }); + // Update cached files & space + cacheJSON = JSON.stringify(await cacheFilesAndSpace()); + cacheIsReady = true; }); }); } } async function handleAPI(req, res) { - d = new Date(); startTime = d.getTime(); - let jsonaa = {}, filesaa = 0, spaceaa = 0; + const startTime = Date.now(); + let jsonaa = {}, filesaa = 0, spaceaa = 0, endTime = 0; switch (req.url.split("?")[0]) { - // Status check to see the onlint status of EUS + // Status check to see the online status of EUS // Used by ESL to make sure EUS is online case "/api/get-server-status": - d = new Date(); endTime = d.getTime(); + endTime = Date.now(); global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); - return res.end('{ "status":1, "version":"'+global.internals.version+'" }'); + return res.end('{"status":1,"version":"'+global.internals.version+'"}'); /* Stats api endpoint Query inputs @@ -331,7 +384,7 @@ async function handleAPI(req, res) { if (filesaa == 1) { // If getting the space used on the server isn't required send the json if (spaceaa != 1) { - d = new Date(); endTime = d.getTime(); + endTime = Date.now(); global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); delete jsonaa["space"]; return res.end(JSON.stringify(jsonaa)); @@ -339,14 +392,14 @@ async function handleAPI(req, res) { } // Getting space is required if (spaceaa == 1) { - d = new Date(); endTime = d.getTime(); + endTime = Date.now(); global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); if (filesaa != 1) delete jsonaa["files"]; return res.end(JSON.stringify(jsonaa)); } if (filesaa != 1 && spaceaa != 1) { - d = new Date(); endTime = d.getTime(); + endTime = Date.now(); global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); return res.end("Please add f and or s to your queries to get the files and space"); } @@ -354,7 +407,7 @@ async function handleAPI(req, res) { // Information API case "/api/get-info": - d = new Date(); endTime = d.getTime(); + endTime = Date.now(); global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); return res.end(JSON.stringify({ version: global.internals.version, @@ -362,7 +415,7 @@ async function handleAPI(req, res) { })); default: - d = new Date(); endTime = d.getTime(); + endTime = Date.now(); global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); return res.send(`

All currently avaliable api endpoints

@@ -373,18 +426,15 @@ async function handleAPI(req, res) { } } -const spaceValues = ["B", "KB", "MB", "GB", "TB", "PB", "EB"]; // Futureproofing:tm: +const spaceValues = ["B", "KB", "MB", "GB", "TB", "PB", "EB"]; // Futureproofing™ async function spaceToLowest(spaceValue, includeStringValue) { return new Promise((resolve, reject) => { // Converts space values to lower values e.g MB, GB, TB etc depending on the size of the number let i1 = 1; // Loop through until value is at it's lowest - for (let i = 0; i < i1; i++) { - if (spaceValue >= 1024) { - spaceValue = spaceValue / 1024; - } - + while (spaceValue >= 1024) { + spaceValue = spaceValue / 1024; if (spaceValue >= 1024) i1++; } @@ -393,4 +443,6 @@ async function spaceToLowest(spaceValue, includeStringValue) { }); } -module.exports.MOD_FUNC = MODULE_FUNCTION; \ No newline at end of file +module.exports.MOD_FUNC = MODULE_FUNCTION; + +console.log("[EUS] Finished loading"); \ No newline at end of file diff --git a/README.md b/README.md index 1adac0a..67e3731 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,9 @@ EUS has extra dependencies other than those of [Revolution](https://github.com/tgpethan/Revolution), the server EUS is made on, of which include: - [connect-busboy](https://www.npmjs.com/package/connect-busboy) - [randomstring](https://www.npmjs.com/package/randomstring) - - [get-folder-size](https://www.npmjs.com/package/get-folder-size) - [diskusage](https://www.npmjs.com/package/diskusage) + - [stream-meter](https://www.npmjs.com/package/stream-meter) + - [mysql](https://www.npmjs.com/package/mysql) Install the dependencies and then simply drop the EUS.js into a Revolution instance's modules folder **(If you still have [example_request_handler.js](https://github.com/tgpethan/Revolution/blob/master/modules/example_request_handler.js) be sure to delete it!)** -- 2.47.0 From 3a3fee6283a0423865b9c14ca2859162de61a553 Mon Sep 17 00:00:00 2001 From: Holly Date: Fri, 7 Jan 2022 04:54:39 +0000 Subject: [PATCH 09/24] Add "(APIReq)" to API requests to make them easier to see in logs --- EUS.js | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/EUS.js b/EUS.js index 889a233..942df2d 100644 --- a/EUS.js +++ b/EUS.js @@ -361,13 +361,12 @@ module.exports = { async function handleAPI(req, res) { const startTime = Date.now(); - let jsonaa = {}, filesaa = 0, spaceaa = 0, endTime = 0; + let jsonaa = {}, filesaa = 0, spaceaa = 0; switch (req.url.split("?")[0]) { // Status check to see the online status of EUS // Used by ESL to make sure EUS is online case "/api/get-server-status": - endTime = Date.now(); - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`); return res.end('{"status":1,"version":"'+global.internals.version+'"}'); /* Stats api endpoint @@ -384,39 +383,34 @@ async function handleAPI(req, res) { if (filesaa == 1) { // If getting the space used on the server isn't required send the json if (spaceaa != 1) { - endTime = Date.now(); - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`); delete jsonaa["space"]; return res.end(JSON.stringify(jsonaa)); } } // Getting space is required if (spaceaa == 1) { - endTime = Date.now(); - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`); if (filesaa != 1) delete jsonaa["files"]; return res.end(JSON.stringify(jsonaa)); } if (filesaa != 1 && spaceaa != 1) { - endTime = Date.now(); - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`); return res.end("Please add f and or s to your queries to get the files and space"); } break; // Information API case "/api/get-info": - endTime = Date.now(); - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`); return res.end(JSON.stringify({ version: global.internals.version, instance: config["server"]["instance_type"] })); default: - endTime = Date.now(); - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${endTime - startTime}ms`); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`); return res.send(`

All currently avaliable api endpoints

/api/get-server-status -- 2.47.0 From 20d789f2e6a7e446cace76668dfe890458485170 Mon Sep 17 00:00:00 2001 From: Holly Date: Fri, 7 Jan 2022 06:17:20 +0000 Subject: [PATCH 10/24] Fix filename fetching in line with busboy changes --- EUS.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/EUS.js b/EUS.js index 942df2d..9502ece 100644 --- a/EUS.js +++ b/EUS.js @@ -325,14 +325,15 @@ module.exports = { var thefe; // Pipe the request to busboy req.pipe(req.busboy); - req.busboy.on('file', function (fieldname, file, filename) { + req.busboy.on('file', function (fieldname, file, info) { // Make a new file name fileOutName = randomstring.generate(14); global.modules.consoleHelper.printInfo(emoji.fast_up, `${req.method}: Upload of ${fileOutName} started.`); - // Check the file is within the accepted file types - if (eusConfig.acceptedTypes.includes(`.${filename.split(".")[filename.split(".").length-1]}`)) { + // Check the file is within the accepted file types + const fileType = info.filename.split(".").slice(-1); + if (eusConfig.acceptedTypes.includes(`.${fileType}`)) { // File is accepted, set the extention of the file in thefe for later use. - thefe = `${filename.split(".")[filename.split(".").length-1]}`; + thefe = fileType; } else { // File isn't accepted, send response back to client stating so. res.status(403).end("This file type isn't accepted currently."); -- 2.47.0 From e12c9c6ac768d8942f2b7eed9e4c378f7047e039 Mon Sep 17 00:00:00 2001 From: Holly Date: Fri, 7 Jan 2022 06:40:58 +0000 Subject: [PATCH 11/24] add (overdue) .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ba9e398 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +EUS/ \ No newline at end of file -- 2.47.0 From 58aa2f6b47bb81e00b238973119d7bee0fb8c21a Mon Sep 17 00:00:00 2001 From: Holly Date: Fri, 7 Jan 2022 13:25:25 +0000 Subject: [PATCH 12/24] update EUS to support the new Revolution node module checks and auto installer --- EUS.js | 132 +++++++++++++++++++++++++++++++-------------------------- 1 file changed, 72 insertions(+), 60 deletions(-) diff --git a/EUS.js b/EUS.js index 9502ece..d99306e 100644 --- a/EUS.js +++ b/EUS.js @@ -1,12 +1,4 @@ -const fs = require("fs"), - config = require("../config/config.json"), - chalk = require("chalk"), - busboy = require("connect-busboy"), - randomstring = require("randomstring"), - diskUsage = require("diskusage"), - streamMeter = require("stream-meter"), - mysql = require("mysql"), - emoji = require("../misc/emoji_list.json"); +const fs = require("fs"), config = require("../config/config.json"), emoji = require("../misc/emoji_list.json"); // Defines the function of this module const MODULE_FUNCTION = "handle_requests", @@ -14,19 +6,17 @@ const MODULE_FUNCTION = "handle_requests", // Base path for module folder creation and navigation BASE_PATH = "/EUS"; -console.log("[EUS] Loading EUS..."); - -// This will never change -const diskRunningOnSize = diskUsage.checkSync(__dirname).total; +let node_modules = {}; let eusConfig = {}, useUploadKey = true, cacheJSON = "", - startupFinished = false; + startupFinished = false, + diskRunningOnSize = 0; class Database { constructor(databaseAddress, databasePort = 3306, databaseUsername, databasePassword, databaseName, connectedCallback) { - this.connectionPool = mysql.createPool({ + this.connectionPool = node_modules.mysql.createPool({ connectionLimit: 128, host: databaseAddress, port: databasePort, @@ -79,40 +69,58 @@ class Database { } } -// Only ran on startup so using sync functions is fine -// Makes the folder for files of the module -if (!fs.existsSync(__dirname + BASE_PATH)) { - fs.mkdirSync(__dirname + BASE_PATH); - console.log(`[EUS] Made EUS module folder`); -} -// Makes the folder for frontend files -if (!fs.existsSync(__dirname + BASE_PATH + "/files")) { - fs.mkdirSync(__dirname + BASE_PATH + "/files"); - console.log(`[EUS] Made EUS web files folder`); -} -// Makes the folder for images -if (!fs.existsSync(__dirname + BASE_PATH + "/i")) { - fs.mkdirSync(__dirname + BASE_PATH + "/i"); - console.log(`[EUS] Made EUS images folder`); -} +function init() { + // Require node modules + node_modules["chalk"] = require("chalk"); + node_modules["busboy"] = require("connect-busboy"); + node_modules["randomstring"] = require("randomstring"); + node_modules["diskUsage"] = require("diskusage"); + node_modules["streamMeter"] = require("stream-meter"); + node_modules["mysql"] = require("mysql"); -// Makes the config file -if (!fs.existsSync(__dirname + BASE_PATH + "/config.json")) { - // Config doesn't exist, make it. - fs.writeFileSync(`${__dirname}${BASE_PATH}/config.json`, '{\n\t"baseURL":"http://example.com/",\n\t"acceptedTypes": [\n\t\t".png",\n\t\t".jpg",\n\t\t".jpeg",\n\t\t".gif"\n\t],\n\t"uploadKey": "",\n\t"database": {\n\t\t"databaseAddress": "127.0.0.1",\n\t\t"databasePort": 3306,\n\t\t"databaseUsername": "root",\n\t\t"databasePassword": "password",\n\t\t"databaseName": "EUS"\n\t}\n}'); - console.log("[EUS] Made EUS config File!"); - console.log("[EUS] Please edit the EUS Config file before restarting."); - // Config has been made, close framework. - process.exit(0); -} else { - eusConfig = require(`${__dirname}${BASE_PATH}/config.json`); - if (validateConfig(eusConfig)) console.log("[EUS] EUS config passed all checks"); -} + // Only ran on startup so using sync functions is fine -const dbConnection = new Database(eusConfig["database"]["databaseAddress"], eusConfig["database"]["databasePort"], eusConfig["database"]["databaseUsername"], eusConfig["database"]["databasePassword"], eusConfig["database"]["databaseName"], async () => { - cacheJSON = JSON.stringify(await cacheFilesAndSpace()); - cacheIsReady = true; -}); + // Fetch total size of disk on startup, this will never change during runtime + // if it does something seriously wrong has happened. + diskRunningOnSize = node_modules.diskUsage.checkSync(__dirname).total; + + // Makes the folder for files of the module + if (!fs.existsSync(__dirname + BASE_PATH)) { + fs.mkdirSync(__dirname + BASE_PATH); + console.log(`[EUS] Made EUS module folder`); + } + // Makes the folder for frontend files + if (!fs.existsSync(__dirname + BASE_PATH + "/files")) { + fs.mkdirSync(__dirname + BASE_PATH + "/files"); + console.log(`[EUS] Made EUS web files folder`); + } + // Makes the folder for images + if (!fs.existsSync(__dirname + BASE_PATH + "/i")) { + fs.mkdirSync(__dirname + BASE_PATH + "/i"); + console.log(`[EUS] Made EUS images folder`); + } + + // Makes the config file + if (!fs.existsSync(__dirname + BASE_PATH + "/config.json")) { + // Config doesn't exist, make it. + fs.writeFileSync(`${__dirname}${BASE_PATH}/config.json`, '{\n\t"baseURL":"http://example.com/",\n\t"acceptedTypes": [\n\t\t".png",\n\t\t".jpg",\n\t\t".jpeg",\n\t\t".gif"\n\t],\n\t"uploadKey": "",\n\t"database": {\n\t\t"databaseAddress": "127.0.0.1",\n\t\t"databasePort": 3306,\n\t\t"databaseUsername": "root",\n\t\t"databasePassword": "password",\n\t\t"databaseName": "EUS"\n\t}\n}'); + console.log("[EUS] Made EUS config File!"); + console.log("[EUS] Please edit the EUS Config file before restarting."); + // Config has been made, close framework. + process.exit(0); + } else { + eusConfig = require(`${__dirname}${BASE_PATH}/config.json`); + if (validateConfig(eusConfig)) console.log("[EUS] EUS config passed all checks"); + } + + // This is using a callback but that's fine, the server will just react properly to the db not being ready yet. + const dbConnection = new Database(eusConfig["database"]["databaseAddress"], eusConfig["database"]["databasePort"], eusConfig["database"]["databaseUsername"], eusConfig["database"]["databasePassword"], eusConfig["database"]["databaseName"], async () => { + cacheJSON = JSON.stringify(await cacheFilesAndSpace()); + cacheIsReady = true; + }); + + console.log("[EUS] Finished loading."); +} // Cache for the file count and space usage, this takes a while to do so it's best to cache the result let cacheIsReady = false; @@ -237,11 +245,11 @@ function regularFile(req, res, urs = "", startTime = 0) { if (error) { // Doesn't exist, send a 404 to the client. error404Page(res); - global.modules.consoleHelper.printInfo(emoji.cross, `${req.method}: ${chalk.red("[404]")} ${req.url} ${Date.now() - startTime}ms`); + global.modules.consoleHelper.printInfo(emoji.cross, `${req.method}: ${node_modules.chalk.red("[404]")} ${req.url} ${Date.now() - startTime}ms`); } else { // File does exist, send it back to the client. res.sendFile(__dirname + BASE_PATH + "/files"+req.url); - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} ${req.url} ${Date.now() - startTime}ms`); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${node_modules.chalk.green("[200]")} ${req.url} ${Date.now() - startTime}ms`); } }); } @@ -251,9 +259,10 @@ function error404Page(res) { } module.exports = { + init: init, extras:async function() { // Setup express to use busboy - global.app.use(busboy()); + global.app.use(node_modules.busboy()); startupFinished = true; //cacheJSON = JSON.stringify(await cacheFilesAndSpace()); //cacheIsReady = true; @@ -295,7 +304,7 @@ module.exports = { // There's an entry in the DB for this, send the file back. if (dbEntry != null) { res.sendFile(`${__dirname}${BASE_PATH}/i/${urs}.${dbEntry.imageType}`); - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} (ImageReq) ${req.url} ${Date.now() - startTime}ms`); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${node_modules.chalk.green("[200]")} (ImageReq) ${req.url} ${Date.now() - startTime}ms`); } // There's no entry, so treat this as a regular file. else regularFile(req, res, urs, startTime); @@ -327,7 +336,7 @@ module.exports = { req.pipe(req.busboy); req.busboy.on('file', function (fieldname, file, info) { // Make a new file name - fileOutName = randomstring.generate(14); + fileOutName = node_modules.randomstring.generate(14); global.modules.consoleHelper.printInfo(emoji.fast_up, `${req.method}: Upload of ${fileOutName} started.`); // Check the file is within the accepted file types const fileType = info.filename.split(".").slice(-1); @@ -342,7 +351,7 @@ module.exports = { // Create a write stream for the file fstream = fs.createWriteStream(__dirname + BASE_PATH + "/i/" + fileOutName + "." + thefe); // Create meter for tracking the size of the file - const meter = streamMeter(); + const meter = node_modules.streamMeter(); file.pipe(meter).pipe(fstream); fstream.on('close', async () => { // Add this image to the database @@ -367,7 +376,7 @@ async function handleAPI(req, res) { // Status check to see the online status of EUS // Used by ESL to make sure EUS is online case "/api/get-server-status": - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${node_modules.chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`); return res.end('{"status":1,"version":"'+global.internals.version+'"}'); /* Stats api endpoint @@ -384,34 +393,34 @@ async function handleAPI(req, res) { if (filesaa == 1) { // If getting the space used on the server isn't required send the json if (spaceaa != 1) { - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${node_modules.chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`); delete jsonaa["space"]; return res.end(JSON.stringify(jsonaa)); } } // Getting space is required if (spaceaa == 1) { - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${node_modules.chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`); if (filesaa != 1) delete jsonaa["files"]; return res.end(JSON.stringify(jsonaa)); } if (filesaa != 1 && spaceaa != 1) { - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${node_modules.chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`); return res.end("Please add f and or s to your queries to get the files and space"); } break; // Information API case "/api/get-info": - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${node_modules.chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`); return res.end(JSON.stringify({ version: global.internals.version, instance: config["server"]["instance_type"] })); default: - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${node_modules.chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`); return res.send(`

All currently avaliable api endpoints

/api/get-server-status @@ -440,4 +449,7 @@ async function spaceToLowest(spaceValue, includeStringValue) { module.exports.MOD_FUNC = MODULE_FUNCTION; -console.log("[EUS] Finished loading"); \ No newline at end of file +module.exports.REQUIRED_NODE_MODULES = [ + "chalk", "connect-busboy", "randomstring", + "diskusage", "stream-meter", "mysql" +]; \ No newline at end of file -- 2.47.0 From 7f8ea1f843f1dec4fa28945ff34c29e4177d6cf9 Mon Sep 17 00:00:00 2001 From: Holly Date: Sat, 23 Apr 2022 12:11:31 +0100 Subject: [PATCH 13/24] whoops --- EUS.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/EUS.js b/EUS.js index d99306e..dc24474 100644 --- a/EUS.js +++ b/EUS.js @@ -69,6 +69,7 @@ class Database { } } +let dbConnection; function init() { // Require node modules node_modules["chalk"] = require("chalk"); @@ -114,7 +115,7 @@ function init() { } // This is using a callback but that's fine, the server will just react properly to the db not being ready yet. - const dbConnection = new Database(eusConfig["database"]["databaseAddress"], eusConfig["database"]["databasePort"], eusConfig["database"]["databaseUsername"], eusConfig["database"]["databasePassword"], eusConfig["database"]["databaseName"], async () => { + dbConnection = new Database(eusConfig["database"]["databaseAddress"], eusConfig["database"]["databasePort"], eusConfig["database"]["databaseUsername"], eusConfig["database"]["databasePassword"], eusConfig["database"]["databaseName"], async () => { cacheJSON = JSON.stringify(await cacheFilesAndSpace()); cacheIsReady = true; }); -- 2.47.0 From aa15a0bd95d9862e501e0b72eaa0fc308d8b8e25 Mon Sep 17 00:00:00 2001 From: Holly Stubbs Date: Mon, 14 Nov 2022 10:32:06 +0000 Subject: [PATCH 14/24] Various changes to improve speed, reliability and code quality --- EUS.js | 170 +++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 124 insertions(+), 46 deletions(-) diff --git a/EUS.js b/EUS.js index dc24474..881ee7e 100644 --- a/EUS.js +++ b/EUS.js @@ -1,4 +1,4 @@ -const fs = require("fs"), config = require("../config/config.json"), emoji = require("../misc/emoji_list.json"); +const config = require("../config/config.json"), crypto = require("crypto"), emoji = require("../misc/emoji_list.json"), fs = require("fs"); // Defines the function of this module const MODULE_FUNCTION = "handle_requests", @@ -12,11 +12,12 @@ let eusConfig = {}, useUploadKey = true, cacheJSON = "", startupFinished = false, - diskRunningOnSize = 0; + diskRunningOnSize = 0, + timeSinceLastCache = Date.now(); class Database { constructor(databaseAddress, databasePort = 3306, databaseUsername, databasePassword, databaseName, connectedCallback) { - this.connectionPool = node_modules.mysql.createPool({ + this.connectionPool = node_modules["mysql2"].createPool({ connectionLimit: 128, host: databaseAddress, port: databasePort, @@ -25,6 +26,7 @@ class Database { database: databaseName }); + const classCreationTime = Date.now(); this.dbActive = false; if (connectedCallback == null) { this.dbActive = true; @@ -32,37 +34,60 @@ class Database { const connectionCheckInterval = setInterval(() => { this.query("SELECT id FROM images LIMIT 1") .then(data => { - if (startupFinished) global.modules.consoleHelper.printInfo(emoji.globe_europe, `Connected to database`); - else console.log("[EUS] Connected to database"); + global.modules.consoleHelper.printInfo(emoji.globe_europe, `Connected to database. Took ${Date.now() - classCreationTime}ms`); this.dbActive = true; clearInterval(connectionCheckInterval); connectedCallback(); }) - .catch(err => {}); + .catch(err => { + console.error(err); + }); }, 167); // Roughly 6 times per sec } } - async query(sqlQuery) { + dataReceived(resolveCallback, data, limited = false) { + if (limited) resolveCallback(data[0]); + else resolveCallback(data); + } + + query(query = "", data) { + const limited = query.includes("LIMIT 1"); + return new Promise((resolve, reject) => { this.connectionPool.getConnection((err, connection) => { if (err) { reject(err); - try { - connection.release(); - } catch (e) {} + try { connection.release();} + catch (e) { + console.error("Failed to release mysql connection", err); + } } else { - connection.query(sqlQuery, (err, data) => { - if (err) { - reject(err); - connection.release(); - } else { - if (sqlQuery.includes("LIMIT 1")) resolve(data[0]); - else resolve(data); - connection.release(); - } - }); + // Use old query + if (data == null) { + connection.query(query, (err, data) => { + if (err) { + reject(err); + connection.release(); + } else { + this.dataReceived(resolve, data, limited); + connection.release(); + } + }); + } + // Use new prepared statements w/ placeholders + else { + connection.execute(query, data, (err, data) => { + if (err) { + reject(err); + connection.release(); + } else { + this.dataReceived(resolve, data, limited); + connection.release(); + } + }); + } } }); }); @@ -77,7 +102,7 @@ function init() { node_modules["randomstring"] = require("randomstring"); node_modules["diskUsage"] = require("diskusage"); node_modules["streamMeter"] = require("stream-meter"); - node_modules["mysql"] = require("mysql"); + node_modules["mysql2"] = require("mysql2"); // Only ran on startup so using sync functions is fine @@ -237,9 +262,12 @@ function cleanURL(url = "") { .split("%40").join("@") .split("%5B").join("[").split("%5C").join("\\").split("%5D").join("]").split("%5E").join("^") .split("%60").join("`") - .split("%7B").join("{").split("%7C").join("|").split("%7D").join("}").split("%7E").join("~"); + .split("%7B").join("{").split("%7C").join("|").split("%7D").join("}").split("%7E").join("~") + .split("%CE%A9").join("Ω"); } +let existanceCache = {}; + function regularFile(req, res, urs = "", startTime = 0) { if (req.url === "/") { urs = "/index.html" } else { urs = req.url } fs.access(`${__dirname}${BASE_PATH}/files${urs}`, (error) => { @@ -255,6 +283,11 @@ function regularFile(req, res, urs = "", startTime = 0) { }); } +function imageFile(req, res, file, startTime = 0) { + res.sendFile(`${__dirname}${BASE_PATH}/i/${file}`); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${node_modules.chalk.green("[200]")} (ImageReq) ${req.url} ${Date.now() - startTime}ms`); +} + function error404Page(res) { res.status(404).send("404!
EUS"); } @@ -286,6 +319,12 @@ module.exports = { req.url = cleanURL(req.url); + // The Funny Response + if (req.url.includes(".php") || req.url.includes("/wp-")) { + global.modules.consoleHelper.printWarn(emoji.globe_europe, `${req.method}: SUSSY ${req.headers["cf-connecting-ip"]} ${req.url}`); + return res.status(418).send("Unfortunately for you this server does not use PHP or WordPress.
EUS"); + } + // Check if returned value is true. if (req.url.includes("/api/")) return handleAPI(req, res); @@ -293,24 +332,28 @@ module.exports = { const startTime = Date.now(); // Get the requested image - let urs = `${req.url}`; urs = urs.split("/")[1]; + const urs = req.url.split("/")[1]; // Check if we even need to query the DB const dbAble = (!/[^0-9A-Za-z]/.test(urs)) && urs != "" && urs != "index" && (req.url.split("/").length == 2); if (dbAble) { - if (dbConnection.dbActive) { - // Try to get what we think is an image's details from the DB - const dbEntry = await dbConnection.query(`SELECT imageType FROM images WHERE imageId = "${urs}" LIMIT 1`); - - // There's an entry in the DB for this, send the file back. - if (dbEntry != null) { - res.sendFile(`${__dirname}${BASE_PATH}/i/${urs}.${dbEntry.imageType}`); - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${node_modules.chalk.green("[200]")} (ImageReq) ${req.url} ${Date.now() - startTime}ms`); + if (urs in existanceCache) { + imageFile(req, res, `${urs}.${existanceCache[urs]}`, startTime); + } else { + if (dbConnection.dbActive) { + // Try to get what we think is an image's details from the DB + const dbEntry = await dbConnection.query(`SELECT imageType FROM images WHERE imageId = ? LIMIT 1`, [urs]); + + // There's an entry in the DB for this, send the file back. + if (dbEntry != null) { + existanceCache[urs] = dbEntry.imageType; + imageFile(req, res, `${urs}.${dbEntry.imageType}`, startTime); + } + // There's no entry, so treat this as a regular file. + else regularFile(req, res, urs, startTime); } - // There's no entry, so treat this as a regular file. - else regularFile(req, res, urs, startTime); + else res.status(400).end("EUS is restarting, please try again in a few secs."); } - else res.status(400).end("EUS is restarting, please try again in a few secs."); } // We can still serve files if they are not dbable // since we don't need to check the db @@ -328,11 +371,11 @@ module.exports = { // Get time at the start of upload + res.header("Access-Control-Allow-Origin", "*"); + if (useUploadKey && eusConfig["uploadKey"] != req.header("key")) return res.end("Incorrect key provided for upload"); const startTime = Date.now(); - var fstream; - var thefe; // Pipe the request to busboy req.pipe(req.busboy); req.busboy.on('file', function (fieldname, file, info) { @@ -340,7 +383,11 @@ module.exports = { fileOutName = node_modules.randomstring.generate(14); global.modules.consoleHelper.printInfo(emoji.fast_up, `${req.method}: Upload of ${fileOutName} started.`); // Check the file is within the accepted file types - const fileType = info.filename.split(".").slice(-1); + let fileType = info.filename.split(".").slice(-1); + if (info.filename === "blob") { + fileType = info.mimeType.split("/")[1]; + } + var thefe = ""; if (eusConfig.acceptedTypes.includes(`.${fileType}`)) { // File is accepted, set the extention of the file in thefe for later use. thefe = fileType; @@ -350,14 +397,32 @@ module.exports = { return; } // Create a write stream for the file - fstream = fs.createWriteStream(__dirname + BASE_PATH + "/i/" + fileOutName + "." + thefe); - // Create meter for tracking the size of the file - const meter = node_modules.streamMeter(); - file.pipe(meter).pipe(fstream); + let fstream = fs.createWriteStream(__dirname + BASE_PATH + "/i/" + fileOutName + "." + thefe); + file.pipe(fstream); + + // Get all file data for the file MD5 + let fileData = []; + file.on("data", (chunk) => { + fileData.push(chunk); + }); + fstream.on('close', async () => { - // Add this image to the database - await dbConnection.query(`INSERT INTO images (id, imageId, imageType, imageSize) VALUES (NULL, "${fileOutName}", "${thefe}", ${meter.bytes})`); - + let md5Buffer = Buffer.concat(fileData); + // bye + fileData = null; + + // Create MD5 hash of file + const hash = crypto.createHash("md5"); + hash.setEncoding("hex"); + hash.write(md5Buffer); + hash.end(); + + // Add to the existance cache + existanceCache[fileOutName] = thefe[0]; + + // Store image data in db + await dbConnection.query(`INSERT INTO images (id, imageId, imageType, hash, imageSize) VALUES (NULL, ?, ?, ?, ?)`, [fileOutName, thefe[0], hash.read(), md5Buffer.length]); + // Send URL of the uploaded image to the client res.end(eusConfig.baseURL + fileOutName); global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: Upload of ${fileOutName} finished. Took ${Date.now() - startTime}ms`); @@ -366,6 +431,19 @@ module.exports = { cacheJSON = JSON.stringify(await cacheFilesAndSpace()); cacheIsReady = true; }); + + /*fstream.on('close', async () => { + // Add this image to the database + await dbConnection.query(`INSERT INTO images (id, imageId, imageType, hash, imageSize) VALUES (NULL, "${fileOutName}", "${thefe}", ${meter.bytes})`); + + // Send URL of the uploaded image to the client + res.end(eusConfig.baseURL + fileOutName); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: Upload of ${fileOutName} finished. Took ${Date.now() - startTime}ms`); + + // Update cached files & space + cacheJSON = JSON.stringify(await cacheFilesAndSpace()); + cacheIsReady = true; + });*/ }); } } @@ -452,5 +530,5 @@ module.exports.MOD_FUNC = MODULE_FUNCTION; module.exports.REQUIRED_NODE_MODULES = [ "chalk", "connect-busboy", "randomstring", - "diskusage", "stream-meter", "mysql" -]; \ No newline at end of file + "diskusage", "stream-meter", "mysql2" +]; -- 2.47.0 From cea5c5ec272aa9c36d03161d34220a484b33ac60 Mon Sep 17 00:00:00 2001 From: Holly Stubbs Date: Mon, 14 Nov 2022 10:33:41 +0000 Subject: [PATCH 15/24] update LICENSE --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index f95419e..3cd11ea 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2021 Holly +Copyright (c) 2022 Holly Stubbs Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal -- 2.47.0 From f32396565bed35dbda6f36ce8db65bc54b6ddc7d Mon Sep 17 00:00:00 2001 From: Holly Stubbs Date: Mon, 14 Nov 2022 10:35:29 +0000 Subject: [PATCH 16/24] (finally) remove Feature-Policy --- EUS.js | 1 - 1 file changed, 1 deletion(-) diff --git a/EUS.js b/EUS.js index 881ee7e..d6d59b2 100644 --- a/EUS.js +++ b/EUS.js @@ -310,7 +310,6 @@ module.exports = { // Set some headers res.set("Strict-Transport-Security", "max-age=31536000; includeSubDomains"); res.set("X-XSS-Protection", "1; mode=block"); - res.set("Feature-Policy", "fullscreen 'none'"); res.set("Permissions-Policy", "microphone=(), geolocation=(), magnetometer=(), camera=(), payment=(), usb=(), accelerometer=(), gyroscope=()"); res.set("Referrer-Policy", "strict-origin-when-cross-origin"); res.set("Content-Security-Policy", "block-all-mixed-content;frame-ancestors 'self'"); -- 2.47.0 From d73b5ddc84c6cb76359b1239079fc7c36225e6b2 Mon Sep 17 00:00:00 2001 From: Holly Date: Tue, 21 Feb 2023 09:01:56 +0000 Subject: [PATCH 17/24] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 67e3731..f84363a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- +

EUS is my public screenshot server built using Revolution
-- 2.47.0 From c0fbde74614aa9693b8df58f56fbbcdc004e7926 Mon Sep 17 00:00:00 2001 From: Holly Date: Tue, 4 Apr 2023 12:55:35 +0100 Subject: [PATCH 18/24] Update LICENSE --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 3cd11ea..3498e1d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022 Holly Stubbs +Copyright (c) 2023 Holly Stubbs (tgpholly) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal -- 2.47.0 From f4157911930c4c503366050cfc812c27eb54200a Mon Sep 17 00:00:00 2001 From: Holly Date: Tue, 4 Apr 2023 12:56:15 +0100 Subject: [PATCH 19/24] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f84363a..fee8bed 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,10 @@

- EUS is my public screenshot server built using Revolution
+ EUS is my public screenshot server built using Revolution

- CodeFactor + CodeFactor

-- 2.47.0 From 1a610afb14ec52440878e8bf548e139d2d0e566b Mon Sep 17 00:00:00 2001 From: Holly Date: Sat, 13 May 2023 12:08:23 +0100 Subject: [PATCH 20/24] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fee8bed..d84ec8c 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ EUS has extra dependencies other than those of [Revolution](https://github.com/t - [stream-meter](https://www.npmjs.com/package/stream-meter) - [mysql](https://www.npmjs.com/package/mysql) -Install the dependencies and then simply drop the EUS.js into a Revolution instance's modules folder **(If you still have [example_request_handler.js](https://github.com/tgpethan/Revolution/blob/master/modules/example_request_handler.js) be sure to delete it!)** +Simply drop the EUS.js into a Revolution instance's modules folder **(If you still have [example_request_handler.js](https://github.com/tgpethan/Revolution/blob/master/modules/example_request_handler.js) be sure to delete it!)** and the extra required modules should be automatically installed. ## Config On first startup EUS will create a new config file in the **modules/EUS/** folder, some of these values may need to be changed depending on your use case. -- 2.47.0 From b6455e914decbff2b908e9dbdbf5de19e1fb879a Mon Sep 17 00:00:00 2001 From: Holly Date: Sat, 13 May 2023 12:09:47 +0100 Subject: [PATCH 21/24] Update README.md --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d84ec8c..bd3905f 100644 --- a/README.md +++ b/README.md @@ -30,9 +30,7 @@ If you want to expand the files that the server allows to be sent to it this can The value of **uploadKey** is used to restrict who can upload to your server, set this to something and the server will restrict who can upload depending on if they provided the key or not. If this field is left blank EUS will asume you don't want an upload key and uploads to it will be unrestricted ## API -EUS has 3 apis, they are located at **[/api/get-stats](https://eusv.ml/api/get-stats)**, **[/api/get-info](https://eusv.ml/api/get-info)** and **[/api/get-server-status](https://eusv.ml/api/get-server-status)** - -[These are better documented on EUS Docs](https://docs.eusv.ml) +EUS has 3 api endpoints, they are **[/api/get-stats](https://eusv.ml/api/get-stats)**, **[/api/get-info](https://eusv.ml/api/get-info)** and **[/api/get-server-status](https://eusv.ml/api/get-server-status)** ## Websites that use EUS -[EUS](https://eusv.ml) +[EUS](https://eusv.net) -- 2.47.0 From c8b96b101b0b8c3962d85409a6cf7fd2fa5c1000 Mon Sep 17 00:00:00 2001 From: Holly Date: Sat, 13 May 2023 12:10:44 +0100 Subject: [PATCH 22/24] Update all references of .ml to .net --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bd3905f..66330af 100644 --- a/README.md +++ b/README.md @@ -23,14 +23,14 @@ Simply drop the EUS.js into a Revolution instance's modules folder **(If you sti ## Config On first startup EUS will create a new config file in the **modules/EUS/** folder, some of these values may need to be changed depending on your use case. -The value of **baseURL** will need to be changed to what you access the server from, for example if the server's ip is 192.168.1.100 and you are not planning to use EUS at a url you would change the value to **http://192.168.1.100/**. **baseURL** is used to construct the response url for file uploads, for example the value of **baseURL** on my instance of EUS is **https://eusv.ml/**. +The value of **baseURL** will need to be changed to what you access the server from, for example if the server's ip is 192.168.1.100 and you are not planning to use EUS at a url you would change the value to **http://192.168.1.100/**. **baseURL** is used to construct the response url for file uploads, for example the value of **baseURL** on my instance of EUS is **https://eusv.net/**. If you want to expand the files that the server allows to be sent to it this can be done in the **allowedTypes** array. By default the array contains **png, jpg and gif**. The value of **uploadKey** is used to restrict who can upload to your server, set this to something and the server will restrict who can upload depending on if they provided the key or not. If this field is left blank EUS will asume you don't want an upload key and uploads to it will be unrestricted ## API -EUS has 3 api endpoints, they are **[/api/get-stats](https://eusv.ml/api/get-stats)**, **[/api/get-info](https://eusv.ml/api/get-info)** and **[/api/get-server-status](https://eusv.ml/api/get-server-status)** +EUS has 3 api endpoints, they are **[/api/get-stats](https://eusv.net/api/get-stats)**, **[/api/get-info](https://eusv.net/api/get-info)** and **[/api/get-server-status](https://eusv.net/api/get-server-status)** ## Websites that use EUS [EUS](https://eusv.net) -- 2.47.0 From 520b2ca3ad33656345180387cdd76841172fb7c1 Mon Sep 17 00:00:00 2001 From: Holly Date: Sun, 23 Jul 2023 00:10:39 +0100 Subject: [PATCH 23/24] Cache Database requests & decode uri better --- EUS.js | 103 ++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 61 insertions(+), 42 deletions(-) diff --git a/EUS.js b/EUS.js index d6d59b2..cdb4762 100644 --- a/EUS.js +++ b/EUS.js @@ -4,7 +4,8 @@ const config = require("../config/config.json"), crypto = require("crypto"), emo const MODULE_FUNCTION = "handle_requests", // Base path for module folder creation and navigation - BASE_PATH = "/EUS"; + BASE_PATH = "/EUS", + API_CACHE_LIFESPAN = 3600000; let node_modules = {}; @@ -12,7 +13,6 @@ let eusConfig = {}, useUploadKey = true, cacheJSON = "", startupFinished = false, - diskRunningOnSize = 0, timeSinceLastCache = Date.now(); class Database { @@ -100,16 +100,11 @@ function init() { node_modules["chalk"] = require("chalk"); node_modules["busboy"] = require("connect-busboy"); node_modules["randomstring"] = require("randomstring"); - node_modules["diskUsage"] = require("diskusage"); node_modules["streamMeter"] = require("stream-meter"); node_modules["mysql2"] = require("mysql2"); // Only ran on startup so using sync functions is fine - // Fetch total size of disk on startup, this will never change during runtime - // if it does something seriously wrong has happened. - diskRunningOnSize = node_modules.diskUsage.checkSync(__dirname).total; - // Makes the folder for files of the module if (!fs.existsSync(__dirname + BASE_PATH)) { fs.mkdirSync(__dirname + BASE_PATH); @@ -126,6 +121,11 @@ function init() { console.log(`[EUS] Made EUS images folder`); } + if (!fs.existsSync("/tmp/EUS_UPLOADS")) { + fs.mkdirSync("/tmp/EUS_UPLOADS"); + console.log("[EUS] Made EUS temp upload folder"); + } + // Makes the config file if (!fs.existsSync(__dirname + BASE_PATH + "/config.json")) { // Config doesn't exist, make it. @@ -151,6 +151,7 @@ function init() { // Cache for the file count and space usage, this takes a while to do so it's best to cache the result let cacheIsReady = false; function cacheFilesAndSpace() { + timeSinceLastCache = Date.now(); return new Promise(async (resolve, reject) => { const startCacheTime = Date.now(); cacheIsReady = false; @@ -166,20 +167,14 @@ function cacheFilesAndSpace() { }); cachedFilesAndSpace["filesTotal"] = totalFiles; - cachedFilesAndSpace["space"] = {usage: {}, total:{}}; + cachedFilesAndSpace["size"] = {}; - const dbSize = (await dbConnection.query(`SELECT SUM(imageSize) FROM images LIMIT 1`))["SUM(imageSize)"]; + const dbSize = (await dbConnection.query(`SELECT SUM(fileSize) FROM images LIMIT 1`))["SUM(fileSize)"]; const totalSizeBytes = dbSize == null ? 0 : dbSize; const mbSize = totalSizeBytes / 1024 / 1024; - cachedFilesAndSpace["space"]["usage"]["mb"] = parseFloat(mbSize.toFixed(4)); - cachedFilesAndSpace["space"]["usage"]["gb"] = parseFloat((mbSize / 1024).toFixed(4)); - cachedFilesAndSpace["space"]["usage"]["string"] = await spaceToLowest(totalSizeBytes, true); - - //totalDiskSize - const totalMBSize = diskRunningOnSize / 1024 / 1024; - cachedFilesAndSpace["space"]["total"]["mb"] = parseFloat(totalMBSize.toFixed(4)); - cachedFilesAndSpace["space"]["total"]["gb"] = parseFloat((totalMBSize / 1024).toFixed(4)); - cachedFilesAndSpace["space"]["total"]["string"] = await spaceToLowest(diskRunningOnSize, true); + cachedFilesAndSpace["size"]["mb"] = parseFloat(mbSize.toFixed(4)); + cachedFilesAndSpace["size"]["gb"] = parseFloat((mbSize / 1024).toFixed(4)); + cachedFilesAndSpace["size"]["string"] = await spaceToLowest(totalSizeBytes, true); resolve(cachedFilesAndSpace); global.modules.consoleHelper.printInfo(emoji.folder, `Stats api cache took ${Date.now() - startCacheTime}ms`); @@ -250,7 +245,7 @@ function validateConfig(json) { // Check if server needs to be shutdown if (performShutdownAfterValidation) { - console.error("EUS config properties are missing, refer to docs for more details (https://wiki.eusv.ml)"); + console.error("EUS config properties are missing, refer to example config on GitHub (https://github.com/tgpholly/EUS)"); process.exit(1); } else return true; @@ -277,7 +272,7 @@ function regularFile(req, res, urs = "", startTime = 0) { global.modules.consoleHelper.printInfo(emoji.cross, `${req.method}: ${node_modules.chalk.red("[404]")} ${req.url} ${Date.now() - startTime}ms`); } else { // File does exist, send it back to the client. - res.sendFile(__dirname + BASE_PATH + "/files"+req.url); + res.sendFile(`${__dirname}${BASE_PATH}/files${req.url}`); global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${node_modules.chalk.green("[200]")} ${req.url} ${Date.now() - startTime}ms`); } }); @@ -298,8 +293,6 @@ module.exports = { // Setup express to use busboy global.app.use(node_modules.busboy()); startupFinished = true; - //cacheJSON = JSON.stringify(await cacheFilesAndSpace()); - //cacheIsReady = true; }, get:async function(req, res) { /* @@ -316,7 +309,7 @@ module.exports = { res.set("X-Frame-Options", "SAMEORIGIN"); res.set("X-Content-Type-Options", "nosniff"); - req.url = cleanURL(req.url); + req.url = decodeURIComponent(req.url.split("?")[0]); // The Funny Response if (req.url.includes(".php") || req.url.includes("/wp-")) { @@ -337,16 +330,20 @@ module.exports = { if (dbAble) { if (urs in existanceCache) { - imageFile(req, res, `${urs}.${existanceCache[urs]}`, startTime); + const cachedFile = existanceCache[urs]; + imageFile(req, res, `${cachedFile.fileHash}.${cachedFile.fileType}`, startTime); } else { if (dbConnection.dbActive) { // Try to get what we think is an image's details from the DB - const dbEntry = await dbConnection.query(`SELECT imageType FROM images WHERE imageId = ? LIMIT 1`, [urs]); + const dbEntry = await dbConnection.query(`SELECT hash, imageType FROM images WHERE imageId = ? LIMIT 1`, [urs]); // There's an entry in the DB for this, send the file back. if (dbEntry != null) { - existanceCache[urs] = dbEntry.imageType; - imageFile(req, res, `${urs}.${dbEntry.imageType}`, startTime); + existanceCache[urs] = { + fileHash: dbEntry.hash, + fileType: dbEntry.imageType + }; + imageFile(req, res, `${dbEntry.hash}.${dbEntry.imageType}`, startTime); } // There's no entry, so treat this as a regular file. else regularFile(req, res, urs, startTime); @@ -396,7 +393,7 @@ module.exports = { return; } // Create a write stream for the file - let fstream = fs.createWriteStream(__dirname + BASE_PATH + "/i/" + fileOutName + "." + thefe); + let fstream = fs.createWriteStream("/tmp/EUS_UPLOADS/" + fileOutName); file.pipe(fstream); // Get all file data for the file MD5 @@ -416,19 +413,41 @@ module.exports = { hash.write(md5Buffer); hash.end(); - // Add to the existance cache - existanceCache[fileOutName] = thefe[0]; - - // Store image data in db - await dbConnection.query(`INSERT INTO images (id, imageId, imageType, hash, imageSize) VALUES (NULL, ?, ?, ?, ?)`, [fileOutName, thefe[0], hash.read(), md5Buffer.length]); - - // Send URL of the uploaded image to the client - res.end(eusConfig.baseURL + fileOutName); - global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: Upload of ${fileOutName} finished. Took ${Date.now() - startTime}ms`); - - // Update cached files & space - cacheJSON = JSON.stringify(await cacheFilesAndSpace()); - cacheIsReady = true; + const fileHash = hash.read(); + const dataOnHash = await dbConnection.query("SELECT imageId FROM images WHERE hash = ? LIMIT 1", [fileHash]); + if (dataOnHash !== undefined) + { + fs.unlink(`/tmp/EUS_UPLOADS/${fileOutName}`, () => {}); + res.end(`${eusConfig.baseURL}${dataOnHash.imageId}`); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: Hash matched! Sending ${dataOnHash.imageId} instead. Took ${Date.now() - startTime}ms`); + return; + } else { + fs.rename(`/tmp/EUS_UPLOADS/${fileOutName}`, `${__dirname}${BASE_PATH}/i/${fileHash}.${thefe[0]}`, async () => { + // Add to the existance cache + existanceCache[fileOutName] = { + fileHash: fileHash, + fileType: thefe[0] + }; + + // Store image data in db + await dbConnection.query(`INSERT INTO images (id, imageId, imageType, hash, fileSize) VALUES (NULL, ?, ?, ?, ?)`, [fileOutName, thefe[0], fileHash, md5Buffer.length]); + + // Send URL of the uploaded image to the client + res.end(eusConfig.baseURL + fileOutName); + global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: Upload of ${fileOutName} finished. Took ${Date.now() - startTime}ms`); + + // Update cached files & space + if ((Date.now() - timeSinceLastCache) >= API_CACHE_LIFESPAN) { + cacheJSON = JSON.stringify(await cacheFilesAndSpace()); + } else { + const tempJson = JSON.parse(cacheJSON); + tempJson.fileCounts[thefe[0]]++; + cacheJSON = JSON.stringify(tempJson); + global.modules.consoleHelper.printInfo(emoji.folder, `Skiped api cache`); + } + cacheIsReady = true; + }); + } }); /*fstream.on('close', async () => { @@ -529,5 +548,5 @@ module.exports.MOD_FUNC = MODULE_FUNCTION; module.exports.REQUIRED_NODE_MODULES = [ "chalk", "connect-busboy", "randomstring", - "diskusage", "stream-meter", "mysql2" + "stream-meter", "mysql2" ]; -- 2.47.0 From c6b2d60821a7a992d2d8f2cb27e5056d66cec5f4 Mon Sep 17 00:00:00 2001 From: Holly Date: Sun, 23 Jul 2023 00:14:05 +0100 Subject: [PATCH 24/24] add example config --- config.example.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100755 config.example.json diff --git a/config.example.json b/config.example.json new file mode 100755 index 0000000..c63b560 --- /dev/null +++ b/config.example.json @@ -0,0 +1,18 @@ +{ + "baseURL":"https://example.com/", + "acceptedTypes": [ + ".png", + ".jpg", + ".jpeg", + ".gif", + ".mp4" + ], + "uploadKey":"", + "database": { + "databaseAddress": "127.0.0.1", + "databasePort": 3306, + "databaseUsername": "root", + "databasePassword": "password", + "databaseName": "EUS" + } +} -- 2.47.0