add new env vars MM_MODULES_DIR and MM_CUSTOMCSS_FILE … (#3530)

… for setting these things from outside (and overriding corresponding
config.js properties `config.foreignModulesDir` and `customCss`)

- remove elements from index.html when loading script or stylesheet
files fails
- removed `config.paths.vendor` (could never work because `vendor` is
hardcoded in `index.html`) and renamed `config.paths.modules` to
`config.foreignModulesDir`. The `config.paths. ...` properties were
implemented in the initial commit in `js/defaults.js` but were never
functional.
- fixes `app.js` which didn't respect `config.paths.modules` before
- as `modules/defaults` is directly set in many places in the source
code restrict `config.paths.modules` to foreign modules (it has never
worked for default modules), now renamed to `config.foreignModulesDir`
- adds new `/env` section in `server.js` for getting the new env vars in
the browser
- fixes TODO in `server.js` so test directories are now only published
when running tests

These changes allow changing some main paths from outside mm with the
new env vars. You now **can** put all user stuff into one directory,
e.g. the `config` dir:
- `config.js` as before
- `custom.css`
- foreign modules

This would simplify other setups e.g. the docker setup. At the moment we
have to deal with 3 directories where 2 of them (`modules` and `css`)
contains mixed stuff, which means mm owned files and user files. This
can now simplified and leads to cleaner setups (if wanted).

CHANGELOG.md

@@ -12,6 +12,8 @@ _This release is scheduled to be released on 2024-10-01._
 ### Added
 
 - [core] Add spelling check (cspell): `npm run test:spelling` and handle spelling issues
+- [core] removed `config.paths.vendor` (could not work because `vendor` is hardcoded in `index.html`), renamed `config.paths.modules` to `config.foreignModulesDir`, added variable `MM_CUSTOMCSS_FILE` which - if set - overrides `config.customCss`, added variable `MM_MODULES_DIR` which - if set - overrides `config.foreignModulesDir`
+- [core] elements are now removed from index.html when loading script or stylesheet files fails
 
 ### Removed
 

js/app.js

@@ -9,6 +9,7 @@ const Log = require("logger");
 const Server = require(`${__dirname}/server`);
 const Utils = require(`${__dirname}/utils`);
 const defaultModules = require(`${__dirname}/../modules/default/defaultmodules`);
+const { getEnvVarsAsObj } = require(`${__dirname}/server_functions`);
 
 // Get version number.
 global.version = require(`${__dirname}/../package.json`).version;
@@ -159,7 +160,8 @@ function App () {
 	function loadModule (module) {
 		const elements = module.split("/");
 		const moduleName = elements[elements.length - 1];
-		let moduleFolder = `${__dirname}/../modules/${module}`;
+		const env = getEnvVarsAsObj();
+		let moduleFolder = `${__dirname}/../${env.modulesDir}/${module}`;
 
 		if (defaultModules.includes(moduleName)) {
 			moduleFolder = `${__dirname}/../modules/default/${module}`;

js/defaults.js

@@ -19,6 +19,7 @@ const defaults = {
 	units: "metric",
 	zoom: 1,
 	customCss: "css/custom.css",
+	foreignModulesDir: "modules",
 	// httpHeaders used by helmet, see https://helmetjs.github.io/. You can add other/more object values by overriding this in config.js,
 	// e.g. you need to add `frameguard: false` for embedding MagicMirror in another website, see https://github.com/MagicMirrorOrg/MagicMirror/issues/2847
 	httpHeaders: { contentSecurityPolicy: false, crossOriginOpenerPolicy: false, crossOriginEmbedderPolicy: false, crossOriginResourcePolicy: false, originAgentCluster: false },
@@ -72,12 +73,7 @@ const defaults = {
 				text: "www.michaelteeuw.nl"
 			}
 		}
-	],
-
-	paths: {
-		modules: "modules",
-		vendor: "vendor"
-	}
+	]
 };
 
 /*************** DO NOT EDIT THE LINE BELOW ***************/

js/loader.js

@@ -10,6 +10,15 @@ const Loader = (function () {
 
 	/* Private Methods */
 
+	/**
+	 * Retrieve object of env variables.
+	 * @returns {object} with key: values as assembled in js/server_functions.js
+	 */
+	const getEnvVars = async function () {
+		const res = await fetch(`${location.protocol}//${location.host}/env`);
+		return JSON.parse(await res.text());
+	};
+
 	/**
 	 * Loops through all modules and requests start for every module.
 	 */
@@ -58,19 +67,20 @@ const Loader = (function () {
 	 * Generate array with module information including module paths.
 	 * @returns {object[]} Module information.
 	 */
-	const getModuleData = function () {
+	const getModuleData = async function () {
 		const modules = getAllModules();
 		const moduleFiles = [];
+		const envVars = await getEnvVars();
 
 		modules.forEach(function (moduleData, index) {
 			const module = moduleData.module;
 
 			const elements = module.split("/");
 			const moduleName = elements[elements.length - 1];
-			let moduleFolder = `${config.paths.modules}/${module}`;
+			let moduleFolder = `${envVars.modulesDir}/${module}`;
 
 			if (defaultModules.indexOf(moduleName) !== -1) {
-				moduleFolder = `${config.paths.modules}/default/${module}`;
+				moduleFolder = `modules/default/${module}`;
 			}
 
 			if (moduleData.disabled === true) {
@@ -166,6 +176,7 @@ const Loader = (function () {
 					};
 					script.onerror = function () {
 						Log.error("Error on loading script:", fileName);
+						script.remove();
 						resolve();
 					};
 					document.getElementsByTagName("body")[0].appendChild(script);
@@ -183,6 +194,7 @@ const Loader = (function () {
 					};
 					stylesheet.onerror = function () {
 						Log.error("Error on loading stylesheet:", fileName);
+						stylesheet.remove();
 						resolve();
 					};
 					document.getElementsByTagName("head")[0].appendChild(stylesheet);
@@ -197,7 +209,9 @@ const Loader = (function () {
 		 * Load all modules as defined in the config.
 		 */
 		async loadModules () {
-			let moduleData = getModuleData();
+			let moduleData = await getModuleData();
+			const envVars = await getEnvVars();
+			const customCss = envVars.customCss;
 
 			/**
 			 * @returns {Promise<void>} when all modules are loaded
@@ -212,7 +226,7 @@ const Loader = (function () {
 					// All modules loaded. Load custom.css
 					// This is done after all the modules so we can
 					// overwrite all the defined styles.
-					await loadFile(config.customCss);
+					await loadFile(customCss);
 					// custom.css loaded. Start all modules.
 					await startModules();
 				}
@@ -244,7 +258,7 @@ const Loader = (function () {
 				// This file is available in the vendor folder.
 				// Load it from this vendor folder.
 				loadedFiles.push(fileName.toLowerCase());
-				return loadFile(`${config.paths.vendor}/${vendor[fileName]}`);
+				return loadFile(`vendor/${vendor[fileName]}`);
 			}
 
 			// File not loaded yet.

js/server.js

@@ -8,8 +8,7 @@ const helmet = require("helmet");
 const socketio = require("socket.io");
 
 const Log = require("logger");
-const Utils = require("./utils");
-const { cors, getConfig, getHtml, getVersion, getStartup } = require("./server_functions");
+const { cors, getConfig, getHtml, getVersion, getStartup, getEnvVars } = require("./server_functions");
 
 /**
  * Server
@@ -73,8 +72,11 @@ function Server (config) {
 			app.use(helmet(config.httpHeaders));
 			app.use("/js", express.static(__dirname));
 
-			// TODO add tests directory only when running tests?
-			const directories = ["/config", "/css", "/fonts", "/modules", "/vendor", "/translations", "/tests/configs", "/tests/mocks"];
+			let directories = ["/config", "/css", "/fonts", "/modules", "/vendor", "/translations"];
+			if (process.env.JEST_WORKER_ID !== undefined) {
+				// add tests directories only when running tests
+				directories.push("/tests/configs", "/tests/mocks");
+			}
 			for (const directory of directories) {
 				app.use(directory, express.static(path.resolve(global.root_path + directory)));
 			}
@@ -87,6 +89,8 @@ function Server (config) {
 
 			app.get("/startup", (req, res) => getStartup(req, res));
 
+			app.get("/env", (req, res) => getEnvVars(req, res));
+
 			app.get("/", (req, res) => getHtml(req, res));
 
 			server.on("listening", () => {

js/server_functions.js

@@ -128,4 +128,30 @@ function getVersion (req, res) {
 	res.send(global.version);
 }
 
-module.exports = { cors, getConfig, getHtml, getVersion, getStartup };
+/**
+ * Gets environment variables needed in the browser.
+ * @returns {object} environment variables key: values
+ */
+function getEnvVarsAsObj () {
+	const obj = { modulesDir: `${config.foreignModulesDir}`, customCss: `${config.customCss}` };
+	if (process.env.MM_MODULES_DIR) {
+		obj.modulesDir = process.env.MM_MODULES_DIR.replace(`${global.root_path}/`, "");
+	}
+	if (process.env.MM_CUSTOMCSS_FILE) {
+		obj.customCss = process.env.MM_CUSTOMCSS_FILE.replace(`${global.root_path}/`, "");
+	}
+
+	return obj;
+}
+
+/**
+ * Gets environment variables needed in the browser.
+ * @param {Request} req - the request
+ * @param {Response} res - the result
+ */
+function getEnvVars (req, res) {
+	const obj = getEnvVarsAsObj();
+	res.send(obj);
+}
+
+module.exports = { cors, getConfig, getHtml, getVersion, getStartup, getEnvVars, getEnvVarsAsObj };