Make the e2e tests wait for the app to start and close before running next test (#2952)

When trying to debug why the tests broke for
https://github.com/MichMich/MagicMirror/pull/2946 I found that the tests
does not wait for the app to start and close. So if the startup isn't
blocking that would fail.

So I added a callback for `close()` too and converted them to promises
for the `startApplication()` and `stopApplication()` and updated all the
e2e tests to await both. Will try to refactor all these callbacks to
promises in a later PR.

CHANGELOG.md

@@ -41,6 +41,7 @@ Special thanks to: @rejas, @sdetweil
 - Handle node_helper errors during startup (#2944)
 - Possibility to change FontAwesome class in calendar, so icons like `fab fa-facebook-square` works.
 - Fix cors problems with newsfeed articles (as far as possible), allow disabling cors per feed with option `useCorsProxy: false` (#2840)
+- Tests not waiting for the application to start and stop before starting the next test
 
 ## [2.21.0] - 2022-10-01
 

js/app.js

@@ -222,9 +222,9 @@ function App() {
 				}
 			}
 
-			loadModules(modules, function () {
+			loadModules(modules, async function () {
 				httpServer = new Server(config);
-				const { app, io } = httpServer.open();
+				const { app, io } = await httpServer.open();
 				Log.log("Server started ...");
 
 				const nodePromises = [];
@@ -262,14 +262,16 @@ function App() {
 	 * exists.
 	 *
 	 * Added to fix #1056
+	 *
+	 * @param {Function} callback Function to be called after the app has stopped
 	 */
-	this.stop = function () {
+	this.stop = function (callback) {
 		for (const nodeHelper of nodeHelpers) {
 			if (typeof nodeHelper.stop === "function") {
 				nodeHelper.stop();
 			}
 		}
-		httpServer.close();
+		httpServer.close().then(callback);
 	};
 
 	/**

js/server.js

@@ -26,7 +26,13 @@ function Server(config) {
 	const serverSockets = new Set();
 	let server = null;
 
+	/**
+	 * Opens the server for incoming connections
+	 *
+	 * @returns {Promise} A promise that is resolved when the server listens to connections
+	 */
 	this.open = function () {
+		return new Promise((resolve) => {
 			if (config.useHttps) {
 				const options = {
 					key: fs.readFileSync(config.httpsPrivateKey),
@@ -126,17 +132,27 @@ function Server(config) {
 				res.send(html);
 			});
 
-		return {
+			server.on("listening", () => {
+				resolve({
 					app,
 					io
-		};
+				});
+			});
+		});
 	};
 
+	/**
+	 * Closes the server and destroys all lingering connections to it.
+	 *
+	 * @returns {Promise} A promise that resolves when server has successfully shut down
+	 */
 	this.close = function () {
+		return new Promise((resolve) => {
 			for (const socket of serverSockets.values()) {
 				socket.destroy();
 			}
-		server.close();
+			server.close(resolve);
+		});
 	};
 }
 

tests/e2e/env_spec.js

@@ -2,7 +2,7 @@ const helpers = require("./helpers/global-setup");
 
 describe("App environment", () => {
 	beforeAll(async () => {
-		helpers.startApplication("tests/configs/default.js");
+		await helpers.startApplication("tests/configs/default.js");
 		await helpers.getDocument();
 	});
 	afterAll(async () => {

tests/e2e/fonts_spec.js

@@ -13,8 +13,8 @@ describe("All font files from roboto.css should be downloadable", () => {
 		match = regex.exec(fileContent);
 	}
 
-	beforeAll(() => {
+	beforeAll(async () => {
-		helpers.startApplication("tests/configs/without_modules.js");
+		await helpers.startApplication("tests/configs/without_modules.js");
 	});
 	afterAll(async () => {
 		await helpers.stopApplication();

tests/e2e/helpers/global-setup.js

@@ -1,9 +1,10 @@
 const jsdom = require("jsdom");
-const corefetch = require("fetch");
 
-exports.startApplication = (configFilename, exec) => {
+exports.startApplication = async (configFilename, exec) => {
 	jest.resetModules();
-	this.stopApplication();
+	if (global.app) {
+		await this.stopApplication();
+	}
 	// Set config sample for use in test
 	if (configFilename === "") {
 		process.env.MM_CONFIG_FILE = "config/config.js";
@@ -12,14 +13,20 @@ exports.startApplication = (configFilename, exec) => {
 	}
 	if (exec) exec;
 	global.app = require("app.js");
-	global.app.start();
+
+	return new Promise((resolve) => {
+		global.app.start(resolve);
+	});
 };
 
 exports.stopApplication = async () => {
 	if (global.app) {
-		global.app.stop();
+		return new Promise((resolve) => {
+			global.app.stop(resolve);
+			delete global.app;
+		});
 	}
-	await new Promise((resolve) => setTimeout(resolve, 100));
+	return Promise.resolve();
 };
 
 exports.getDocument = () => {
@@ -75,13 +82,8 @@ exports.waitForAllElements = (selector) => {
 	});
 };
 
-exports.fetch = (url) => {
+// When native fetch is used keep-alive is set which causes issues with tests that should not share the connection, fall back to use the older one for now...
-	return new Promise((resolve) => {
+exports.fetch = require("node-fetch");
-		corefetch(url).then((res) => {
-			resolve(res);
-		});
-	});
-};
 
 exports.testMatch = async (element, regex) => {
 	const elem = await this.waitForElement(element);

tests/e2e/helpers/weather-functions.js

@@ -24,6 +24,6 @@ exports.startApp = async (configFile, additionalMockData) => {
 	let content = fs.readFileSync(path.resolve(__dirname + "../../../../" + configFile)).toString();
 	content = content.replace("#####WEATHERDATA#####", mockWeather);
 	fs.writeFileSync(path.resolve(__dirname + "../../../../config/config.js"), content);
-	helpers.startApplication("");
+	await helpers.startApplication("");
 	await helpers.getDocument();
 };

tests/e2e/ipWhitelist_spec.js

@@ -2,8 +2,8 @@ const helpers = require("./helpers/global-setup");
 
 describe("ipWhitelist directive configuration", () => {
 	describe("Set ipWhitelist without access", () => {
-		beforeAll(() => {
+		beforeAll(async () => {
-			helpers.startApplication("tests/configs/noIpWhiteList.js");
+			await helpers.startApplication("tests/configs/noIpWhiteList.js");
 		});
 		afterAll(async () => {
 			await helpers.stopApplication();
@@ -16,8 +16,8 @@ describe("ipWhitelist directive configuration", () => {
 	});
 
 	describe("Set ipWhitelist []", () => {
-		beforeAll(() => {
+		beforeAll(async () => {
-			helpers.startApplication("tests/configs/empty_ipWhiteList.js");
+			await helpers.startApplication("tests/configs/empty_ipWhiteList.js");
 		});
 		afterAll(async () => {
 			await helpers.stopApplication();

tests/e2e/modules/alert_spec.js

@@ -2,7 +2,7 @@ const helpers = require("../helpers/global-setup");
 
 describe("Alert module", () => {
 	beforeAll(async () => {
-		helpers.startApplication("tests/configs/modules/alert/default.js");
+		await helpers.startApplication("tests/configs/modules/alert/default.js");
 		await helpers.getDocument();
 	});
 	afterAll(async () => {

tests/e2e/modules/calendar_spec.js

@@ -29,7 +29,7 @@ describe("Calendar module", () => {
 
 	describe("Default configuration", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/calendar/default.js");
+			await helpers.startApplication("tests/configs/modules/calendar/default.js");
 			await helpers.getDocument();
 		});
 
@@ -44,7 +44,7 @@ describe("Calendar module", () => {
 
 	describe("Custom configuration", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/calendar/custom.js");
+			await helpers.startApplication("tests/configs/modules/calendar/custom.js");
 			await helpers.getDocument();
 		});
 
@@ -67,7 +67,7 @@ describe("Calendar module", () => {
 
 	describe("Recurring event", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/calendar/recurring.js");
+			await helpers.startApplication("tests/configs/modules/calendar/recurring.js");
 			await helpers.getDocument();
 		});
 
@@ -83,7 +83,7 @@ describe("Calendar module", () => {
 				Date.prototype.getTimezoneOffset = () => {
 					return i * 60;
 				};
-				helpers.startApplication("tests/configs/modules/calendar/recurring.js");
+				await helpers.startApplication("tests/configs/modules/calendar/recurring.js");
 				await helpers.getDocument();
 			});
 
@@ -95,7 +95,7 @@ describe("Calendar module", () => {
 
 	describe("Changed port", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/calendar/changed-port.js");
+			await helpers.startApplication("tests/configs/modules/calendar/changed-port.js");
 			serverBasicAuth.listen(8010);
 			await helpers.getDocument();
 		});
@@ -111,7 +111,7 @@ describe("Calendar module", () => {
 
 	describe("Basic auth", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/calendar/basic-auth.js");
+			await helpers.startApplication("tests/configs/modules/calendar/basic-auth.js");
 			await helpers.getDocument();
 		});
 
@@ -122,7 +122,7 @@ describe("Calendar module", () => {
 
 	describe("Basic auth by default", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/calendar/auth-default.js");
+			await helpers.startApplication("tests/configs/modules/calendar/auth-default.js");
 			await helpers.getDocument();
 		});
 
@@ -133,7 +133,7 @@ describe("Calendar module", () => {
 
 	describe("Basic auth backward compatibility configuration: DEPRECATED", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/calendar/old-basic-auth.js");
+			await helpers.startApplication("tests/configs/modules/calendar/old-basic-auth.js");
 			await helpers.getDocument();
 		});
 
@@ -144,7 +144,7 @@ describe("Calendar module", () => {
 
 	describe("Fail Basic auth", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/calendar/fail-basic-auth.js");
+			await helpers.startApplication("tests/configs/modules/calendar/fail-basic-auth.js");
 			serverBasicAuth.listen(8020);
 			await helpers.getDocument();
 		});

tests/e2e/modules/clock_es_spec.js

@@ -7,7 +7,7 @@ describe("Clock set to spanish language module", () => {
 
 	describe("with default 24hr clock config", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/clock/es/clock_24hr.js");
+			await helpers.startApplication("tests/configs/modules/clock/es/clock_24hr.js");
 			await helpers.getDocument();
 		});
 
@@ -24,7 +24,7 @@ describe("Clock set to spanish language module", () => {
 
 	describe("with default 12hr clock config", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/clock/es/clock_12hr.js");
+			await helpers.startApplication("tests/configs/modules/clock/es/clock_12hr.js");
 			await helpers.getDocument();
 		});
 
@@ -41,7 +41,7 @@ describe("Clock set to spanish language module", () => {
 
 	describe("with showPeriodUpper config enabled", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/clock/es/clock_showPeriodUpper.js");
+			await helpers.startApplication("tests/configs/modules/clock/es/clock_showPeriodUpper.js");
 			await helpers.getDocument();
 		});
 
@@ -53,7 +53,7 @@ describe("Clock set to spanish language module", () => {
 
 	describe("with showWeek config enabled", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/clock/es/clock_showWeek.js");
+			await helpers.startApplication("tests/configs/modules/clock/es/clock_showWeek.js");
 			await helpers.getDocument();
 		});
 

tests/e2e/modules/clock_spec.js

@@ -8,7 +8,7 @@ describe("Clock module", () => {
 
 	describe("with default 24hr clock config", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/clock/clock_24hr.js");
+			await helpers.startApplication("tests/configs/modules/clock/clock_24hr.js");
 			await helpers.getDocument();
 		});
 
@@ -25,7 +25,7 @@ describe("Clock module", () => {
 
 	describe("with default 12hr clock config", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/clock/clock_12hr.js");
+			await helpers.startApplication("tests/configs/modules/clock/clock_12hr.js");
 			await helpers.getDocument();
 		});
 
@@ -42,7 +42,7 @@ describe("Clock module", () => {
 
 	describe("with showPeriodUpper config enabled", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/clock/clock_showPeriodUpper.js");
+			await helpers.startApplication("tests/configs/modules/clock/clock_showPeriodUpper.js");
 			await helpers.getDocument();
 		});
 
@@ -54,7 +54,7 @@ describe("Clock module", () => {
 
 	describe("with displaySeconds config disabled", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/clock/clock_displaySeconds_false.js");
+			await helpers.startApplication("tests/configs/modules/clock/clock_displaySeconds_false.js");
 			await helpers.getDocument();
 		});
 
@@ -66,7 +66,7 @@ describe("Clock module", () => {
 
 	describe("with showTime config disabled", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/clock/clock_showTime.js");
+			await helpers.startApplication("tests/configs/modules/clock/clock_showTime.js");
 			await helpers.getDocument();
 		});
 
@@ -78,7 +78,7 @@ describe("Clock module", () => {
 
 	describe("with showWeek config enabled", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/clock/clock_showWeek.js");
+			await helpers.startApplication("tests/configs/modules/clock/clock_showWeek.js");
 			await helpers.getDocument();
 		});
 
@@ -98,7 +98,7 @@ describe("Clock module", () => {
 
 	describe("with analog clock face enabled", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/clock/clock_analog.js");
+			await helpers.startApplication("tests/configs/modules/clock/clock_analog.js");
 			await helpers.getDocument();
 		});
 

tests/e2e/modules/compliments_spec.js

@@ -21,7 +21,7 @@ describe("Compliments module", () => {
 	describe("Feature anytime in compliments module", () => {
 		describe("Set anytime and empty compliments for morning, evening and afternoon ", () => {
 			beforeAll(async () => {
-				helpers.startApplication("tests/configs/modules/compliments/compliments_anytime.js");
+				await helpers.startApplication("tests/configs/modules/compliments/compliments_anytime.js");
 				await helpers.getDocument();
 			});
 
@@ -32,7 +32,7 @@ describe("Compliments module", () => {
 
 		describe("Only anytime present in configuration compliments", () => {
 			beforeAll(async () => {
-				helpers.startApplication("tests/configs/modules/compliments/compliments_only_anytime.js");
+				await helpers.startApplication("tests/configs/modules/compliments/compliments_only_anytime.js");
 				await helpers.getDocument();
 			});
 
@@ -44,7 +44,7 @@ describe("Compliments module", () => {
 
 	describe("remoteFile option", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/compliments/compliments_remote.js");
+			await helpers.startApplication("tests/configs/modules/compliments/compliments_remote.js");
 			await helpers.getDocument();
 		});
 

tests/e2e/modules/helloworld_spec.js

@@ -7,7 +7,7 @@ describe("Test helloworld module", () => {
 
 	describe("helloworld set config text", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/helloworld/helloworld.js");
+			await helpers.startApplication("tests/configs/modules/helloworld/helloworld.js");
 			await helpers.getDocument();
 		});
 
@@ -20,7 +20,7 @@ describe("Test helloworld module", () => {
 
 	describe("helloworld default config text", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/helloworld/helloworld_default.js");
+			await helpers.startApplication("tests/configs/modules/helloworld/helloworld_default.js");
 			await helpers.getDocument();
 		});
 

tests/e2e/modules/newsfeed_spec.js

@@ -7,7 +7,7 @@ describe("Newsfeed module", () => {
 
 	describe("Default configuration", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/newsfeed/default.js");
+			await helpers.startApplication("tests/configs/modules/newsfeed/default.js");
 			await helpers.getDocument();
 		});
 
@@ -32,7 +32,7 @@ describe("Newsfeed module", () => {
 
 	describe("Custom configuration", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/newsfeed/prohibited_words.js");
+			await helpers.startApplication("tests/configs/modules/newsfeed/prohibited_words.js");
 			await helpers.getDocument();
 		});
 
@@ -51,7 +51,7 @@ describe("Newsfeed module", () => {
 
 	describe("Invalid configuration", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/newsfeed/incorrect_url.js");
+			await helpers.startApplication("tests/configs/modules/newsfeed/incorrect_url.js");
 			await helpers.getDocument();
 		});
 
@@ -64,7 +64,7 @@ describe("Newsfeed module", () => {
 
 	describe("Ignore items", () => {
 		beforeAll(async () => {
-			helpers.startApplication("tests/configs/modules/newsfeed/ignore_items.js");
+			await helpers.startApplication("tests/configs/modules/newsfeed/ignore_items.js");
 			await helpers.getDocument();
 		});
 

tests/e2e/modules_display_spec.js

@@ -2,7 +2,7 @@ const helpers = require("./helpers/global-setup");
 
 describe("Display of modules", () => {
 	beforeAll(async () => {
-		helpers.startApplication("tests/configs/modules/display.js");
+		await helpers.startApplication("tests/configs/modules/display.js");
 		await helpers.getDocument();
 	});
 	afterAll(async () => {

tests/e2e/modules_empty_spec.js

@@ -2,7 +2,7 @@ const helpers = require("./helpers/global-setup");
 
 describe("Check configuration without modules", () => {
 	beforeAll(async () => {
-		helpers.startApplication("tests/configs/without_modules.js");
+		await helpers.startApplication("tests/configs/without_modules.js");
 		await helpers.getDocument();
 	});
 	afterAll(async () => {

tests/e2e/modules_position_spec.js

@@ -2,7 +2,7 @@ const helpers = require("./helpers/global-setup");
 
 describe("Position of modules", () => {
 	beforeAll(async () => {
-		helpers.startApplication("tests/configs/modules/positions.js");
+		await helpers.startApplication("tests/configs/modules/positions.js");
 		await helpers.getDocument();
 	});
 	afterAll(async () => {

tests/e2e/port_spec.js

@@ -2,8 +2,8 @@ const helpers = require("./helpers/global-setup");
 
 describe("port directive configuration", () => {
 	describe("Set port 8090", () => {
-		beforeAll(() => {
+		beforeAll(async () => {
-			helpers.startApplication("tests/configs/port_8090.js");
+			await helpers.startApplication("tests/configs/port_8090.js");
 		});
 		afterAll(async () => {
 			await helpers.stopApplication();
@@ -16,8 +16,8 @@ describe("port directive configuration", () => {
 	});
 
 	describe("Set port 8100 on environment variable MM_PORT", () => {
-		beforeAll(() => {
+		beforeAll(async () => {
-			helpers.startApplication("tests/configs/port_8090.js", (process.env.MM_PORT = 8100));
+			await helpers.startApplication("tests/configs/port_8090.js", (process.env.MM_PORT = 8100));
 		});
 		afterAll(async () => {
 			await helpers.stopApplication();

tests/e2e/vendor_spec.js

@@ -1,8 +1,8 @@
 const helpers = require("./helpers/global-setup");
 
 describe("Vendors", () => {
-	beforeAll(() => {
+	beforeAll(async () => {
-		helpers.startApplication("tests/configs/default.js");
+		await helpers.startApplication("tests/configs/default.js");
 	});
 	afterAll(async () => {
 		await helpers.stopApplication();