Support HTTP headers with CORS-method (#2957)

Adds support for sending and receiving HTTP-headers when using the
CORS-method.

This change is required for the Yr weather-provider introduced in
https://github.com/MichMich/MagicMirror/pull/2948.

To make it easier to add unit tests I moved the server-functions into a
separate file.

CHANGELOG.md

@@ -9,7 +9,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
 
 _This release is scheduled to be released on 2023-01-01._
 
-Special thanks to: @rejas, @sdetweil
+Special thanks to: @rejas, @sdetweil, @MagMar94
 
 ### Added
 
@@ -35,6 +35,7 @@ Special thanks to: @rejas, @sdetweil
   - Reworked how weatherproviders handle units (#2849)
   - Use unix() method for parsing times, fix suntimes on the way (#2950)
   - Refactor conversion functions into utils class (#2958)
+- The `cors`-method in `server.js` now supports sending and recieving HTTP headers.
 
 ### Fixed
 

js/server.js

@@ -9,10 +9,10 @@ const path = require("path");
 const ipfilter = require("express-ipfilter").IpFilter;
 const fs = require("fs");
 const helmet = require("helmet");
-const fetch = require("fetch");
 
 const Log = require("logger");
 const Utils = require("./utils.js");
+const { cors, getConfig, getHtml, getVersion } = require("./server_functions.js");
 
 /**
  * Server
@@ -84,53 +84,13 @@ function Server(config) {
 				app.use(directory, express.static(path.resolve(global.root_path + directory)));
 			}
 
-			app.get("/cors", async function (req, res) {
-				// example: http://localhost:8080/cors?url=https://google.de
+			app.get("/cors", async (req, res) => await cors(req, res));
 
-				try {
-					const reg = "^/cors.+url=(.*)";
-					let url = "";
+			app.get("/version", (req, res) => getVersion(req, res));
 
-					let match = new RegExp(reg, "g").exec(req.url);
-					if (!match) {
-						url = "invalid url: " + req.url;
-						Log.error(url);
-						res.send(url);
-					} else {
-						url = match[1];
-						Log.log("cors url: " + url);
-						const response = await fetch(url, { headers: { "User-Agent": "Mozilla/5.0 MagicMirror/" + global.version } });
-						const header = response.headers.get("Content-Type");
-						const data = await response.text();
-						if (header) res.set("Content-Type", header);
-						res.send(data);
-					}
-				} catch (error) {
-					Log.error(error);
-					res.send(error);
-				}
-			});
+			app.get("/config", (req, res) => getConfig(req, res));
 
-			app.get("/version", function (req, res) {
-				res.send(global.version);
-			});
-
-			app.get("/config", function (req, res) {
-				res.send(config);
-			});
-
-			app.get("/", function (req, res) {
-				let html = fs.readFileSync(path.resolve(`${global.root_path}/index.html`), { encoding: "utf8" });
-				html = html.replace("#VERSION#", global.version);
-
-				let configFile = "config/config.js";
-				if (typeof global.configuration_file !== "undefined") {
-					configFile = global.configuration_file;
-				}
-				html = html.replace("#CONFIG_FILE#", configFile);
-
-				res.send(html);
-			});
+			app.get("/", (req, res) => getHtml(req, res));
 
 			server.on("listening", () => {
 				resolve({

js/server_functions.js

@@ -0,0 +1,127 @@
+const fetch = require("./fetch");
+const fs = require("fs");
+const path = require("path");
+const Log = require("logger");
+
+/**
+ * Gets the config.
+ *
+ * @param {Request} req - the request
+ * @param {Response} res - the result
+ */
+function getConfig(req, res) {
+	res.send(config);
+}
+
+/**
+ * A method that forewards HTTP Get-methods to the internet to avoid CORS-errors.
+ *
+ * Example input request url: /cors?sendheaders=header1:value1,header2:value2&expectedheaders=header1,header2&url=http://www.test.com/path?param1=value1
+ *
+ * Only the url-param of the input request url is required. It must be the last parameter.
+ *
+ * @param {Request} req - the request
+ * @param {Response} res - the result
+ */
+async function cors(req, res) {
+	try {
+		const urlRegEx = "url=(.+?)$";
+		let url = "";
+
+		const match = new RegExp(urlRegEx, "g").exec(req.url);
+		if (!match) {
+			url = "invalid url: " + req.url;
+			Log.error(url);
+			res.send(url);
+		} else {
+			url = match[1];
+
+			const headersToSend = getHeadersToSend(req.url);
+			const expectedRecievedHeaders = geExpectedRecievedHeaders(req.url);
+
+			Log.log("cors url: " + url);
+			const response = await fetch(url, { headers: headersToSend });
+
+			for (const header of expectedRecievedHeaders) {
+				const headerValue = response.headers.get(header);
+				if (header) res.set(header, headerValue);
+			}
+			const data = await response.text();
+			res.send(data);
+		}
+	} catch (error) {
+		Log.error(error);
+		res.send(error);
+	}
+}
+
+/**
+ * Gets headers and values to attatch to the web request.
+ *
+ * @param {string} url - The url containing the headers and values to send.
+ * @returns {object} An object specifying name and value of the headers.
+ */
+function getHeadersToSend(url) {
+	const headersToSend = { "User-Agent": "Mozilla/5.0 MagicMirror/" + global.version };
+	const headersToSendMatch = new RegExp("sendheaders=(.+?)(&|$)", "g").exec(url);
+	if (headersToSendMatch) {
+		const headers = headersToSendMatch[1].split(",");
+		for (const header of headers) {
+			const keyValue = header.split(":");
+			if (keyValue.length !== 2) {
+				throw new Error(`Invalid format for header ${header}`);
+			}
+			headersToSend[keyValue[0]] = decodeURIComponent(keyValue[1]);
+		}
+	}
+	return headersToSend;
+}
+
+/**
+ * Gets the headers expected from the response.
+ *
+ * @param {string} url - The url containing the expected headers from the response.
+ * @returns {string[]} headers - The name of the expected headers.
+ */
+function geExpectedRecievedHeaders(url) {
+	const expectedRecievedHeaders = ["Content-Type"];
+	const expectedRecievedHeadersMatch = new RegExp("expectedheaders=(.+?)(&|$)", "g").exec(url);
+	if (expectedRecievedHeadersMatch) {
+		const headers = expectedRecievedHeadersMatch[1].split(",");
+		for (const header of headers) {
+			expectedRecievedHeaders.push(header);
+		}
+	}
+	return expectedRecievedHeaders;
+}
+
+/**
+ * Gets the HTML to display the magic mirror.
+ *
+ * @param {Request} req - the request
+ * @param {Response} res - the result
+ */
+function getHtml(req, res) {
+	let html = fs.readFileSync(path.resolve(`${global.root_path}/index.html`), { encoding: "utf8" });
+	html = html.replace("#VERSION#", global.version);
+
+	let configFile = "config/config.js";
+	if (typeof global.configuration_file !== "undefined") {
+		configFile = global.configuration_file;
+	}
+	html = html.replace("#CONFIG_FILE#", configFile);
+
+	res.send(html);
+}
+
+/**
+ * Gets the MagicMirror version.
+ *
+ * @param {Request} req - the request
+ * @param {Response} res - the result
+ */
+function getVersion(req, res) {
+	res.send(global.version);
+}
+
+module.exports = { cors, getConfig, getHtml, getVersion };

tests/unit/functions/server_functions_spec.js

@@ -0,0 +1,149 @@
+const { cors } = require("../../../js/server_functions");
+
+describe("server_functions tests", () => {
+	describe("The cors method", () => {
+		let fetchResponse;
+		let fetchResponseHeadersGet;
+		let fetchResponseHeadersText;
+		let corsResponse;
+		let request;
+
+		jest.mock("node-fetch");
+		let nodefetch = require("node-fetch");
+		let fetchMock;
+
+		beforeEach(() => {
+			nodefetch.mockReset();
+
+			fetchResponseHeadersGet = jest.fn(() => {});
+			fetchResponseHeadersText = jest.fn(() => {});
+			fetchResponse = {
+				headers: {
+					get: fetchResponseHeadersGet
+				},
+				text: fetchResponseHeadersText
+			};
+			jest.mock("node-fetch", () => jest.fn());
+			nodefetch.mockImplementation(() => fetchResponse);
+
+			fetchMock = nodefetch;
+
+			corsResponse = {
+				set: jest.fn(() => {}),
+				send: jest.fn(() => {})
+			};
+
+			request = {
+				url: `/cors?url=www.test.com`
+			};
+		});
+
+		test("Calls correct URL once", async () => {
+			const urlToCall = "http://www.test.com/path?param1=value1";
+			request.url = `/cors?url=${urlToCall}`;
+
+			await cors(request, corsResponse);
+
+			expect(fetchMock.mock.calls.length).toBe(1);
+			expect(fetchMock.mock.calls[0][0]).toBe(urlToCall);
+		});
+
+		test("Forewards Content-Type if json", async () => {
+			fetchResponseHeadersGet.mockImplementation(() => "json");
+
+			await cors(request, corsResponse);
+
+			expect(fetchResponseHeadersGet.mock.calls.length).toBe(1);
+			expect(fetchResponseHeadersGet.mock.calls[0][0]).toBe("Content-Type");
+
+			expect(corsResponse.set.mock.calls.length).toBe(1);
+			expect(corsResponse.set.mock.calls[0][0]).toBe("Content-Type");
+			expect(corsResponse.set.mock.calls[0][1]).toBe("json");
+		});
+
+		test("Forewards Content-Type if xml", async () => {
+			fetchResponseHeadersGet.mockImplementation(() => "xml");
+
+			await cors(request, corsResponse);
+
+			expect(fetchResponseHeadersGet.mock.calls.length).toBe(1);
+			expect(fetchResponseHeadersGet.mock.calls[0][0]).toBe("Content-Type");
+
+			expect(corsResponse.set.mock.calls.length).toBe(1);
+			expect(corsResponse.set.mock.calls[0][0]).toBe("Content-Type");
+			expect(corsResponse.set.mock.calls[0][1]).toBe("xml");
+		});
+
+		test("Sends correct data from response", async () => {
+			const responseData = "some data";
+			fetchResponseHeadersText.mockImplementation(() => responseData);
+
+			let sentData;
+			corsResponse.send = jest.fn((input) => {
+				sentData = input;
+			});
+
+			await cors(request, corsResponse);
+
+			expect(fetchResponseHeadersText.mock.calls.length).toBe(1);
+			expect(sentData).toBe(responseData);
+		});
+
+		test("Sends error data from response", async () => {
+			const error = new Error("error data");
+			fetchResponseHeadersText.mockImplementation(() => {
+				throw error;
+			});
+
+			let sentData;
+			corsResponse.send = jest.fn((input) => {
+				sentData = input;
+			});
+
+			await cors(request, corsResponse);
+
+			expect(fetchResponseHeadersText.mock.calls.length).toBe(1);
+			expect(sentData).toBe(error);
+		});
+
+		test("Fetches with user agent by default", async () => {
+			await cors(request, corsResponse);
+
+			expect(fetchMock.mock.calls.length).toBe(1);
+			expect(fetchMock.mock.calls[0][1]).toHaveProperty("headers");
+			expect(fetchMock.mock.calls[0][1].headers).toHaveProperty("User-Agent");
+		});
+
+		test("Fetches with specified headers", async () => {
+			const headersParam = "sendheaders=header1:value1,header2:value2";
+			const urlParam = "http://www.test.com/path?param1=value1";
+			request.url = `/cors?${headersParam}&url=${urlParam}`;
+
+			await cors(request, corsResponse);
+
+			expect(fetchMock.mock.calls.length).toBe(1);
+			expect(fetchMock.mock.calls[0][1]).toHaveProperty("headers");
+			expect(fetchMock.mock.calls[0][1].headers).toHaveProperty("header1", "value1");
+			expect(fetchMock.mock.calls[0][1].headers).toHaveProperty("header2", "value2");
+		});
+
+		test("Sends specified headers", async () => {
+			fetchResponseHeadersGet.mockImplementation((input) => input.replace("header", "value"));
+
+			const expectedheaders = "expectedheaders=header1,header2";
+			const urlParam = "http://www.test.com/path?param1=value1";
+			request.url = `/cors?${expectedheaders}&url=${urlParam}`;
+
+			await cors(request, corsResponse);
+
+			expect(fetchMock.mock.calls.length).toBe(1);
+			expect(fetchMock.mock.calls[0][1]).toHaveProperty("headers");
+			expect(corsResponse.set.mock.calls.length).toBe(3);
+			expect(corsResponse.set.mock.calls[0][0]).toBe("Content-Type");
+			expect(corsResponse.set.mock.calls[1][0]).toBe("header1");
+			expect(corsResponse.set.mock.calls[1][1]).toBe("value1");
+			expect(corsResponse.set.mock.calls[2][0]).toBe("header2");
+			expect(corsResponse.set.mock.calls[2][1]).toBe("value2");
+		});
+	});
+});