EUS/EUS.js

551 lines
19 KiB
JavaScript

const config = require("../config/config.json"), crypto = require("crypto"), emoji = require("../misc/emoji_list.json"), fs = require("fs");
// Defines the function of this module
const MODULE_FUNCTION = "handle_requests",
// Base path for module folder creation and navigation
BASE_PATH = "/EUS",
API_CACHE_LIFESPAN = 3600000;
let node_modules = {};
let eusConfig = {},
useUploadKey = true,
cacheJSON = "",
timeSinceLastCache = Date.now();
class Database {
constructor(databaseAddress, databasePort = 3306, databaseUsername, databasePassword, databaseName) {
this.connectionPool = node_modules["mysql2"].createPool({
connectionLimit: 128,
host: databaseAddress,
port: databasePort,
user: databaseUsername,
password: databasePassword,
database: databaseName
});
}
dataReceived(resolveCallback, data, limited = false) {
if (limited) resolveCallback(data[0]);
else resolveCallback(data);
}
query(query = "", data) {
const limited = query.includes("LIMIT 1");
return new Promise((resolve, reject) => {
this.connectionPool.getConnection((err, connection) => {
if (err) {
reject(err);
try { connection.release();}
catch (e) {
console.error("Failed to release mysql connection", err);
}
} else {
// Use old query
if (data == null) {
connection.query(query, (err, data) => {
if (err) {
reject(err);
connection.release();
} else {
this.dataReceived(resolve, data, limited);
connection.release();
}
});
}
// Use new prepared statements w/ placeholders
else {
connection.execute(query, data, (err, data) => {
if (err) {
reject(err);
connection.release();
} else {
this.dataReceived(resolve, data, limited);
connection.release();
}
});
}
}
});
});
}
}
let dbConnection;
function init() {
// Require node modules
node_modules["chalk"] = require("chalk");
node_modules["busboy"] = require("connect-busboy");
node_modules["randomstring"] = require("randomstring");
node_modules["streamMeter"] = require("stream-meter");
node_modules["mysql2"] = require("mysql2");
// Only ran on startup so using sync functions is fine
// Makes the folder for files of the module
if (!fs.existsSync(__dirname + BASE_PATH)) {
fs.mkdirSync(__dirname + BASE_PATH);
console.log(`[EUS] Made EUS module folder`);
}
// Makes the folder for frontend files
if (!fs.existsSync(__dirname + BASE_PATH + "/files")) {
fs.mkdirSync(__dirname + BASE_PATH + "/files");
console.log(`[EUS] Made EUS web files folder`);
}
// Makes the folder for images
if (!fs.existsSync(__dirname + BASE_PATH + "/i")) {
fs.mkdirSync(__dirname + BASE_PATH + "/i");
console.log(`[EUS] Made EUS images folder`);
}
if (!fs.existsSync("/tmp/EUS_UPLOADS")) {
fs.mkdirSync("/tmp/EUS_UPLOADS");
console.log("[EUS] Made EUS temp upload folder");
}
// Makes the config file
if (!fs.existsSync(__dirname + BASE_PATH + "/config.json")) {
// Config doesn't exist, make it.
fs.writeFileSync(`${__dirname}${BASE_PATH}/config.json`, '{\n\t"baseURL":"http://example.com/",\n\t"acceptedTypes": [\n\t\t".png",\n\t\t".jpg",\n\t\t".jpeg",\n\t\t".gif"\n\t],\n\t"uploadKey": "",\n\t"database": {\n\t\t"databaseAddress": "127.0.0.1",\n\t\t"databasePort": 3306,\n\t\t"databaseUsername": "root",\n\t\t"databasePassword": "password",\n\t\t"databaseName": "EUS"\n\t}\n}');
console.log("[EUS] Made EUS config File!");
console.log("[EUS] Please edit the EUS Config file before restarting.");
// Config has been made, close framework.
process.exit(0);
} else {
eusConfig = require(`${__dirname}${BASE_PATH}/config.json`);
if (validateConfig(eusConfig)) console.log("[EUS] EUS config passed all checks");
}
if (!eusConfig["noUpload"]) {
dbConnection = new Database(eusConfig["database"]["databaseAddress"], eusConfig["database"]["databasePort"], eusConfig["database"]["databaseUsername"], eusConfig["database"]["databasePassword"], eusConfig["database"]["databaseName"]);
}
console.log("[EUS] Finished loading.");
}
// Cache for the file count and space usage, this takes a while to do so it's best to cache the result
let cacheIsReady = false;
function cacheFilesAndSpace() {
timeSinceLastCache = Date.now();
return new Promise(async (resolve, reject) => {
const startCacheTime = Date.now();
cacheIsReady = false;
let cachedFilesAndSpace = {
fileCounts: {},
};
const dbData = await dbConnection.query(`SELECT imageType, COUNT(imageType) AS "count" FROM images GROUP BY imageType`);
let totalFiles = 0;
dbData.forEach(fileType => {
cachedFilesAndSpace["fileCounts"][fileType.imageType] = fileType.count;
totalFiles += fileType.count;
});
cachedFilesAndSpace["filesTotal"] = totalFiles;
cachedFilesAndSpace["size"] = {};
const dbSize = (await dbConnection.query(`SELECT SUM(fileSize) FROM images LIMIT 1`))["SUM(fileSize)"];
const totalSizeBytes = dbSize == null ? 0 : dbSize;
const mbSize = totalSizeBytes / 1024 / 1024;
cachedFilesAndSpace["size"]["mb"] = parseFloat(mbSize.toFixed(4));
cachedFilesAndSpace["size"]["gb"] = parseFloat((mbSize / 1024).toFixed(4));
cachedFilesAndSpace["size"]["string"] = await spaceToLowest(totalSizeBytes, true);
resolve(cachedFilesAndSpace);
global.modules.consoleHelper.printInfo(emoji.folder, `Stats api cache took ${Date.now() - startCacheTime}ms`);
});
}
function validateConfig(json) {
let performShutdownAfterValidation = false;
// URL Tests
if (!json["noUpload"]) {
if (json["baseURL"] == null) {
console.error("EUS baseURL property does not exist!");
performShutdownAfterValidation = true;
} else {
if (json["baseURL"] == "") console.warn("EUS baseURL property is blank");
const bURL = `${json["baseURL"]}`.split("");
if (bURL.length > 1) {
if (bURL[bURL.length-1] != "/") console.warn("EUS baseURL property doesn't have a / at the end, this can lead to unpredictable results!");
}
else {
if (json["baseURL"] != "http://" || json["baseURL"] != "https://") console.warn("EUS baseURL property is possibly invalid!");
}
}
}
// acceptedTypes checks
if (!json["noUpload"]) {
if (json["acceptedTypes"] == null) {
console.error("EUS acceptedTypes list does not exist!");
performShutdownAfterValidation = true;
} else {
if (json["acceptedTypes"].length < 1) console.warn("EUS acceptedTypes array has no extentions in it, users will not be able to upload images!");
}
}
// uploadKey checks
if (!json["noUpload"]) {
if (json["uploadKey"] == null) {
console.error("EUS uploadKey property does not exist!");
performShutdownAfterValidation = true;
} else {
if (json["uploadKey"] == "") useUploadKey = false;
}
}
// database checks
if (!json["noUpload"]) {
if (json["database"] == null) {
console.error("EUS database properties do not exist!");
performShutdownAfterValidation = true;
} else {
// databaseAddress
if (json["database"]["databaseAddress"] == null) {
console.error("EUS database.databaseAddress property does not exist!");
performShutdownAfterValidation = true;
}
// databasePort
if (json["database"]["databasePort"] == null) {
console.error("EUS database.databasePort property does not exist!");
performShutdownAfterValidation = true;
}
// databaseUsername
if (json["database"]["databaseUsername"] == null) {
console.error("EUS database.databaseUsername property does not exist!");
performShutdownAfterValidation = true;
}
// databasePassword
if (json["database"]["databasePassword"] == null) {
console.error("EUS database.databasePassword property does not exist!");
performShutdownAfterValidation = true;
}
// databaseName
if (json["database"]["databaseName"] == null) {
console.error("EUS database.databaseName property does not exist!");
performShutdownAfterValidation = true;
}
}
}
// Check if server needs to be shutdown
if (performShutdownAfterValidation) {
console.error("EUS config properties are missing, refer to example config on GitHub (https://github.com/tgpholly/EUS)");
process.exit(1);
}
else return true;
}
let existanceCache = {};
function fileExists(file) {
return new Promise((resolve) => {
fs.access(file, (error) => {
resolve(!error);
});
});
}
function sendFile(req, res, path, startTime) {
// File does exist, send it back to the client.
res.sendFile(path);
global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${node_modules.chalk.green("[200]")} ${req.url} ${Date.now() - startTime}ms`);
}
async function regularFile(req, res, urs = "", startTime) {
if (req.url === "/") { urs = "/index.html" } else if (!req.url.includes(".") && !req.url.endsWith("/")) { urs = `${req.url}/index.html` } else { urs = req.url }
if (await fileExists(`${__dirname}${BASE_PATH}/files${urs}`)) {
sendFile(req, res, `${__dirname}${BASE_PATH}/files${decodeURIComponent(urs)}`, startTime);
} else {
let isSuccess = false;
if (!urs.endsWith(".html")) {
if (await fileExists(`${__dirname}${BASE_PATH}/files${decodeURIComponent(urs)}.html`)) {
isSuccess = true;
sendFile(req, res, `${__dirname}${BASE_PATH}/files${decodeURIComponent(urs)}.html`, startTime);
}
} else if (urs.endsWith("/index.html")) {
const path = `${__dirname}${BASE_PATH}/files${decodeURIComponent(req.url)}.html`;
if (await fileExists(path)) {
isSuccess = true;
sendFile(req, res, path, startTime);
}
}
if (!isSuccess) {
// Doesn't exist, send a 404 to the client.
await error404Page(res);
global.modules.consoleHelper.printInfo(emoji.cross, `${req.method}: ${node_modules.chalk.red("[404]")} ${req.url} ${Date.now() - startTime}ms`);
}
}
}
function imageFile(req, res, file, startTime = 0) {
res.sendFile(`${__dirname}${BASE_PATH}/i/${file}`);
global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${node_modules.chalk.green("[200]")} (ImageReq) ${req.url} ${Date.now() - startTime}ms`);
}
async function doImageLookup(req, res, urs, startTime) {
// Check if we even need to query the DB
const dbAble = (!/[^0-9A-Za-z]/.test(urs)) && urs != "" && urs != "index" && (req.url.split("/").length == 2);
if (dbAble) {
if (urs in existanceCache) {
const cachedFile = existanceCache[urs];
imageFile(req, res, `${cachedFile.fileHash}.${cachedFile.fileType}`, startTime);
} else {
// Try to get what we think is an image's details from the DB
const dbEntry = await dbConnection.query(`SELECT hash, imageType FROM images WHERE imageId = ? LIMIT 1`, [urs]);
// There's an entry in the DB for this, send the file back.
if (dbEntry != null) {
existanceCache[urs] = {
fileHash: dbEntry.hash,
fileType: dbEntry.imageType
};
imageFile(req, res, `${dbEntry.hash}.${dbEntry.imageType}`, startTime);
}
// There's no entry, so treat this as a regular file.
else regularFile(req, res, urs, startTime);
}
}
// We can still serve files if they are not dbable
// since we don't need to check the db
else regularFile(req, res, urs, startTime);
}
const PATH_404 = `${__dirname}${BASE_PATH}/files/404.html`;
async function error404Page(res) {
if (await fileExists(PATH_404)) {
res.status(404).sendFile(PATH_404);
} else {
res.status(404).send("404!<hr>EUS");
}
}
let getHandler = (req, res, urs, startTime) => res.send("EUS is starting up, please try again in a few seconds.");
module.exports = {
init: init,
extras:async function() {
if (!eusConfig["noUpload"]) {
// Setup express to use busboy
global.app.use(node_modules.busboy());
getHandler = doImageLookup;
cacheJSON = JSON.stringify(await cacheFilesAndSpace());
cacheIsReady = true;
} else {
getHandler = regularFile;
}
},
get:async function(req, res) {
/*
req - Request from client
res - Response from server
*/
// Set some headers
res.set("Strict-Transport-Security", "max-age=31536000; includeSubDomains");
res.set("X-XSS-Protection", "1; mode=block");
res.set("Permissions-Policy", "microphone=(), geolocation=(), magnetometer=(), camera=(), payment=(), usb=(), accelerometer=(), gyroscope=()");
res.set("Referrer-Policy", "strict-origin-when-cross-origin");
res.set("Content-Security-Policy", "block-all-mixed-content;frame-ancestors 'self'");
res.set("X-Frame-Options", "SAMEORIGIN");
res.set("X-Content-Type-Options", "nosniff");
req.url = decodeURIComponent(req.url.split("?")[0]);
// Auto 404 any attempted php / wp access and log it.
// TODO: IP ban based on mass access. Very clearly a scan.
if (req.url.includes(".php") || req.url.includes("/wp-")) {
global.modules.consoleHelper.printWarn(emoji.globe_europe, `${req.method}: SUSSY ${req.headers["cf-connecting-ip"]} ${req.url}`);
return await error404Page(res);
}
if (req.url.includes("/api/")) {
return handleAPI(req, res);
}
const startTime = Date.now();
const urs = req.url.split("/")[1];
getHandler(req, res, urs, startTime);
},
post:async function(req, res) {
/*
req - Request from client
res - Response from server
*/
if (eusConfig["noUpload"]) return res.end("");
if (req.url != "/upload") return res.end("");
res.header("Access-Control-Allow-Origin", "*");
if (useUploadKey && eusConfig["uploadKey"] != req.header("key")) return res.end("Incorrect key provided for upload");
const startTime = Date.now();
req.pipe(req.busboy);
req.busboy.on('file', function (fieldname, file, info) {
fileOutName = node_modules.randomstring.generate(14);
global.modules.consoleHelper.printInfo(emoji.fast_up, `${req.method}: Upload of ${fileOutName} started.`);
// Check the file is within the accepted file types
let fileType = info.filename.split(".").slice(-1);
if (info.filename === "blob") {
fileType = info.mimeType.split("/")[1];
}
var thefe = "";
if (eusConfig.acceptedTypes.includes(`.${fileType}`)) {
// File is accepted, set the extention of the file in thefe for later use.
thefe = fileType;
} else {
// File isn't accepted, send response back to client stating so.
res.status(403).end("This file type isn't accepted.");
return;
}
// Create a write stream for the file
let fstream = fs.createWriteStream("/tmp/EUS_UPLOADS/" + fileOutName);
file.pipe(fstream);
// Get all file data for the file MD5
let fileData = [];
file.on("data", (chunk) => {
fileData.push(chunk);
});
fstream.on('close', async () => {
let md5Buffer = Buffer.concat(fileData);
// bye
fileData = null;
// Create MD5 hash of file
const hash = crypto.createHash("md5");
hash.setEncoding("hex");
hash.write(md5Buffer);
hash.end();
const fileHash = hash.read();
const dataOnHash = await dbConnection.query("SELECT imageId FROM images WHERE hash = ? LIMIT 1", [fileHash]);
if (dataOnHash !== undefined)
{
fs.unlink(`/tmp/EUS_UPLOADS/${fileOutName}`, () => {});
res.end(`${eusConfig.baseURL}${dataOnHash.imageId}`);
global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: Hash matched! Sending ${dataOnHash.imageId} instead. Took ${Date.now() - startTime}ms`);
return;
} else {
fs.rename(`/tmp/EUS_UPLOADS/${fileOutName}`, `${__dirname}${BASE_PATH}/i/${fileHash}.${thefe[0]}`, async () => {
// Add to the existance cache
existanceCache[fileOutName] = {
fileHash: fileHash,
fileType: thefe[0]
};
// Store image data in db
await dbConnection.query(`INSERT INTO images (id, imageId, imageType, hash, fileSize) VALUES (NULL, ?, ?, ?, ?)`, [fileOutName, thefe[0], fileHash, md5Buffer.length]);
// Send URL of the uploaded image to the client
res.end(eusConfig.baseURL + fileOutName);
global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: Upload of ${fileOutName} finished. Took ${Date.now() - startTime}ms`);
// Update cached files & space
if ((Date.now() - timeSinceLastCache) >= API_CACHE_LIFESPAN) {
cacheJSON = JSON.stringify(await cacheFilesAndSpace());
} else {
const tempJson = JSON.parse(cacheJSON);
tempJson.fileCounts[thefe[0]]++;
cacheJSON = JSON.stringify(tempJson);
global.modules.consoleHelper.printInfo(emoji.folder, `Skiped api cache`);
}
cacheIsReady = true;
});
}
});
});
}
}
async function handleAPI(req, res) {
const startTime = Date.now();
let jsonaa = {}, filesaa = 0, spaceaa = 0;
switch (req.url.split("?")[0]) {
// Status check to see the online status of EUS
// Used by ESL to make sure EUS is online
case "/api/get-server-status":
global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${node_modules.chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`);
return res.end('{"status":1,"version":"'+global.internals.version+'"}');
/* Stats api endpoint
Query inputs
f : Values [0,1]
s : Values [0,1]
*/
case "/api/get-stats":
filesaa = req.query["f"];
spaceaa = req.query["s"];
if (!cacheIsReady) return res.end("Cache is not ready");
jsonaa = JSON.parse(cacheJSON);
// If total files is asked for
if (filesaa == 1) {
// If getting the space used on the server isn't required send the json
if (spaceaa != 1) {
global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${node_modules.chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`);
delete jsonaa["space"];
return res.end(JSON.stringify(jsonaa));
}
}
// Getting space is required
if (spaceaa == 1) {
global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${node_modules.chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`);
if (filesaa != 1) delete jsonaa["files"];
return res.end(JSON.stringify(jsonaa));
}
if (filesaa != 1 && spaceaa != 1) {
global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${node_modules.chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`);
return res.end("Please add f and or s to your queries to get the files and space");
}
break;
// Information API
case "/api/get-info":
global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${node_modules.chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`);
return res.end(JSON.stringify({
version: global.internals.version,
instance: config["server"]["instance_type"]
}));
default:
global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${node_modules.chalk.green("[200]")} (APIReq) ${req.url} ${Date.now() - startTime}ms`);
return res.send(`
<h2>All currently avaliable api endpoints</h2>
<a href="/api/get-server-status">/api/get-server-status</a>
<a href="/api/get-stats">/api/get-stats</a>
<a href="/api/get-info">/api/get-info</a>
`);
}
}
const spaceValues = ["B", "KB", "MB", "GB", "TB", "PB", "EB"]; // Futureproofing™
async function spaceToLowest(spaceValue, includeStringValue) {
return new Promise((resolve, reject) => {
// Converts space values to lower values e.g MB, GB, TB etc depending on the size of the number
let i1 = 1;
// Loop through until value is at it's lowest
while (spaceValue >= 1024) {
spaceValue = spaceValue / 1024;
if (spaceValue >= 1024) i1++;
}
if (includeStringValue) resolve(`${spaceValue.toFixed(2)} ${spaceValues[i1]}`);
else resolve(spaceValue);
});
}
module.exports.MOD_FUNC = MODULE_FUNCTION;
module.exports.REQUIRED_NODE_MODULES = [
"chalk", "connect-busboy", "randomstring",
"stream-meter", "mysql2"
];