From 7cfc7b9d746a762a9f7fb1388a6927d19476aeea Mon Sep 17 00:00:00 2001 From: Karsten Hassel Date: Mon, 24 Jan 2022 23:45:17 +0100 Subject: [PATCH 1/5] first cors approach --- js/server.js | 21 +++++++++++++++++++ .../default/weather/providers/envcanada.js | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/js/server.js b/js/server.js index 3e48323d..b4d71ca5 100644 --- a/js/server.js +++ b/js/server.js @@ -10,6 +10,7 @@ const path = require("path"); const ipfilter = require("express-ipfilter").IpFilter; const fs = require("fs"); const helmet = require("helmet"); +const fetch = require("node-fetch"); const Log = require("logger"); const Utils = require("./utils.js"); @@ -61,6 +62,7 @@ function Server(config, callback) { app.use(function (req, res, next) { ipfilter(config.ipWhitelist, { mode: config.ipWhitelist.length === 0 ? "deny" : "allow", log: false })(req, res, function (err) { if (err === undefined) { + res.header("Access-Control-Allow-Origin", "*"); return next(); } Log.log(err.message); @@ -76,6 +78,25 @@ function Server(config, callback) { app.use(directory, express.static(path.resolve(global.root_path + directory))); } + app.get("/cors", function (req, res) { + // http://localhost:8080/cors?url=https://google.de + console.dir(req.query.url); + + fetch(req.query.url, { headers: { "User-Agent": "Mozilla/5.0 MagicMirror/" + global.version } }) + .then((response) => { + global.mmCorsHeader = response.headers.get("Content-Type"); + if (! global.mmCorsHeader) {global.mmCorsHeader = "application/text"} + return response.text(); + }) + .then((responseData) => { + res.set("Content-Type", global.mmCorsHeader); + res.send(responseData); + }) + .catch((error) => { + Log.error(error); + }); + }); + app.get("/version", function (req, res) { res.send(global.version); }); diff --git a/modules/default/weather/providers/envcanada.js b/modules/default/weather/providers/envcanada.js index 868ee6e2..9ea65e2f 100644 --- a/modules/default/weather/providers/envcanada.js +++ b/modules/default/weather/providers/envcanada.js @@ -164,7 +164,7 @@ WeatherProvider.register("envcanada", { // CORS errors when accessing EC // getUrl() { - return "https://thingproxy.freeboard.io/fetch/https://dd.weather.gc.ca/citypage_weather/xml/" + this.config.provCode + "/" + this.config.siteCode + "_e.xml"; + return "http://localhost:8080/cors?url=https://dd.weather.gc.ca/citypage_weather/xml/" + this.config.provCode + "/" + this.config.siteCode + "_e.xml"; }, // From c622db918ba3bfec3655e1876b653e0196f9575e Mon Sep 17 00:00:00 2001 From: Karsten Hassel Date: Tue, 25 Jan 2022 22:30:16 +0100 Subject: [PATCH 2/5] working version, use corsUrl in weather providers --- js/server.js | 40 +++++++++++-------- modules/default/weather/providers/darksky.js | 4 +- .../default/weather/providers/envcanada.js | 2 +- .../weather/providers/openweathermap.js | 2 +- modules/default/weather/providers/smhi.js | 2 +- .../default/weather/providers/ukmetoffice.js | 2 +- .../weather/providers/ukmetofficedatahub.js | 2 +- .../default/weather/providers/weatherbit.js | 2 +- .../default/weather/providers/weathergov.js | 2 +- modules/default/weather/weatherprovider.js | 9 +++++ 10 files changed, 42 insertions(+), 25 deletions(-) diff --git a/js/server.js b/js/server.js index b4d71ca5..fcd096ad 100644 --- a/js/server.js +++ b/js/server.js @@ -78,23 +78,31 @@ function Server(config, callback) { app.use(directory, express.static(path.resolve(global.root_path + directory))); } - app.get("/cors", function (req, res) { - // http://localhost:8080/cors?url=https://google.de - console.dir(req.query.url); + app.get("/cors", async function (req, res) { + // example: http://localhost:8080/cors?url=https://google.de - fetch(req.query.url, { headers: { "User-Agent": "Mozilla/5.0 MagicMirror/" + global.version } }) - .then((response) => { - global.mmCorsHeader = response.headers.get("Content-Type"); - if (! global.mmCorsHeader) {global.mmCorsHeader = "application/text"} - return response.text(); - }) - .then((responseData) => { - res.set("Content-Type", global.mmCorsHeader); - res.send(responseData); - }) - .catch((error) => { - Log.error(error); - }); + try { + const reg = "^/cors.+url=(.*)"; + let url = ""; + + let match = new RegExp(reg, "g").exec(req.url); + if (!match) { + url = "invalid url: " + req.url; + Log.error(url); + res.send(url); + } else { + url = match[1]; + Log.log("cors url: " + url); + const response = await fetch(url, { headers: { "User-Agent": "Mozilla/5.0 MagicMirror/" + global.version } }); + const header = response.headers.get("Content-Type"); + const data = await response.text(); + if (header) res.set("Content-Type", header); + res.send(data); + } + } catch (error) { + Log.error(error); + res.send(error); + } }); app.get("/version", function (req, res) { diff --git a/modules/default/weather/providers/darksky.js b/modules/default/weather/providers/darksky.js index 451ac98b..01bd79cd 100755 --- a/modules/default/weather/providers/darksky.js +++ b/modules/default/weather/providers/darksky.js @@ -18,7 +18,7 @@ WeatherProvider.register("darksky", { // Set the default config properties that is specific to this provider defaults: { - apiBase: "https://cors-anywhere.herokuapp.com/https://api.darksky.net", + apiBase: "https://api.darksky.net", weatherEndpoint: "/forecast", apiKey: "", lat: 0, @@ -67,7 +67,7 @@ WeatherProvider.register("darksky", { // Create a URL from the config and base URL. getUrl() { const units = this.units[this.config.units] || "auto"; - return `${this.config.apiBase}${this.config.weatherEndpoint}/${this.config.apiKey}/${this.config.lat},${this.config.lon}?units=${units}&lang=${this.config.lang}`; + return this.getCorsUrl() + `${this.config.apiBase}${this.config.weatherEndpoint}/${this.config.apiKey}/${this.config.lat},${this.config.lon}?units=${units}&lang=${this.config.lang}`; }, // Implement WeatherDay generator. diff --git a/modules/default/weather/providers/envcanada.js b/modules/default/weather/providers/envcanada.js index 9ea65e2f..8cbf313c 100644 --- a/modules/default/weather/providers/envcanada.js +++ b/modules/default/weather/providers/envcanada.js @@ -164,7 +164,7 @@ WeatherProvider.register("envcanada", { // CORS errors when accessing EC // getUrl() { - return "http://localhost:8080/cors?url=https://dd.weather.gc.ca/citypage_weather/xml/" + this.config.provCode + "/" + this.config.siteCode + "_e.xml"; + return this.getCorsUrl() + "https://dd.weather.gc.ca/citypage_weather/xml/" + this.config.provCode + "/" + this.config.siteCode + "_e.xml"; }, // diff --git a/modules/default/weather/providers/openweathermap.js b/modules/default/weather/providers/openweathermap.js index c1258d3c..4d3238f6 100755 --- a/modules/default/weather/providers/openweathermap.js +++ b/modules/default/weather/providers/openweathermap.js @@ -116,7 +116,7 @@ WeatherProvider.register("openweathermap", { * Gets the complete url for the request */ getUrl() { - return this.config.apiBase + this.config.apiVersion + this.config.weatherEndpoint + this.getParams(); + return this.getCorsUrl() + this.config.apiBase + this.config.apiVersion + this.config.weatherEndpoint + this.getParams(); }, /* diff --git a/modules/default/weather/providers/smhi.js b/modules/default/weather/providers/smhi.js index 603aa814..adc5f07a 100644 --- a/modules/default/weather/providers/smhi.js +++ b/modules/default/weather/providers/smhi.js @@ -91,7 +91,7 @@ WeatherProvider.register("smhi", { getURL() { let lon = this.config.lon; let lat = this.config.lat; - return `https://opendata-download-metfcst.smhi.se/api/category/pmp3g/version/2/geotype/point/lon/${lon}/lat/${lat}/data.json`; + return this.getCorsUrl() + `https://opendata-download-metfcst.smhi.se/api/category/pmp3g/version/2/geotype/point/lon/${lon}/lat/${lat}/data.json`; }, /** diff --git a/modules/default/weather/providers/ukmetoffice.js b/modules/default/weather/providers/ukmetoffice.js index 158592bd..195e9d2a 100755 --- a/modules/default/weather/providers/ukmetoffice.js +++ b/modules/default/weather/providers/ukmetoffice.js @@ -73,7 +73,7 @@ WeatherProvider.register("ukmetoffice", { * Gets the complete url for the request */ getUrl(forecastType) { - return this.config.apiBase + this.config.locationID + this.getParams(forecastType); + return this.getCorsUrl() + this.config.apiBase + this.config.locationID + this.getParams(forecastType); }, /* diff --git a/modules/default/weather/providers/ukmetofficedatahub.js b/modules/default/weather/providers/ukmetofficedatahub.js index bab1220d..86ede528 100644 --- a/modules/default/weather/providers/ukmetofficedatahub.js +++ b/modules/default/weather/providers/ukmetofficedatahub.js @@ -63,7 +63,7 @@ WeatherProvider.register("ukmetofficedatahub", { queryStrings += "&includeLocationName=" + true; // Return URL, making sure there is a trailing "/" in the base URL. - return this.config.apiBase + (this.config.apiBase.endsWith("/") ? "" : "/") + forecastType + queryStrings; + return this.getCorsUrl() + this.config.apiBase + (this.config.apiBase.endsWith("/") ? "" : "/") + forecastType + queryStrings; }, // Build the list of headers for the request diff --git a/modules/default/weather/providers/weatherbit.js b/modules/default/weather/providers/weatherbit.js index 17f27812..9023f742 100644 --- a/modules/default/weather/providers/weatherbit.js +++ b/modules/default/weather/providers/weatherbit.js @@ -72,7 +72,7 @@ WeatherProvider.register("weatherbit", { // Create a URL from the config and base URL. getUrl() { const units = this.units[this.config.units] || "auto"; - return `${this.config.apiBase}${this.config.weatherEndpoint}?lat=${this.config.lat}&lon=${this.config.lon}&units=${units}&key=${this.config.apiKey}`; + return this.getCorsUrl() + `${this.config.apiBase}${this.config.weatherEndpoint}?lat=${this.config.lat}&lon=${this.config.lon}&units=${units}&key=${this.config.apiKey}`; }, // Implement WeatherDay generator. diff --git a/modules/default/weather/providers/weathergov.js b/modules/default/weather/providers/weathergov.js index 31934784..cf7912c3 100755 --- a/modules/default/weather/providers/weathergov.js +++ b/modules/default/weather/providers/weathergov.js @@ -40,7 +40,7 @@ WeatherProvider.register("weathergov", { // Called to set the config, this config is the same as the weather module's config. setConfig: function (config) { this.config = config; - (this.config.apiBase = "https://api.weather.gov"), this.fetchWxGovURLs(this.config); + (this.config.apiBase = this.getCorsUrl() + "https://api.weather.gov"), this.fetchWxGovURLs(this.config); }, // Called when the weather provider is about to start. diff --git a/modules/default/weather/weatherprovider.js b/modules/default/weather/weatherprovider.js index 5721dad6..e2a9d32f 100644 --- a/modules/default/weather/weatherprovider.js +++ b/modules/default/weather/weatherprovider.js @@ -111,6 +111,15 @@ const WeatherProvider = Class.extend({ this.delegate.updateAvailable(this); }, + getCorsUrl: function () { + const url = window.config.address + ":" + window.config.port + "/cors?url="; + if (window.config.useHttps) { + return "https://" + url; + } else { + return "http://" + url; + } + }, + // A convenience function to make requests. It returns a promise. fetchData: function (url, method = "GET", data = null) { const getData = function (mockData) { From 59bc2318f8042216ca6a857302713919fddadba3 Mon Sep 17 00:00:00 2001 From: Karsten Hassel Date: Wed, 26 Jan 2022 00:22:32 +0100 Subject: [PATCH 3/5] fix weather tests and add CHANGELOG --- CHANGELOG.md | 1 + modules/default/weather/weatherprovider.js | 12 ++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c631831c..c478e379 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ _This release is scheduled to be released on 2022-04-01._ - Added a config option under the weather module, absoluteDates, providing an option to format weather forecast date output with either absolute or relative dates. - Added test for new weather forecast absoluteDates porperty. - The modules get a class hidden added/removed if they get hidden/shown +- Added internal cors proxy to get weather providers working without public proxies (fixes #2714). The new url `http(s)://address:port/cors?url=https://whatever-to-proxy` can be used in other modules too. ### Updated diff --git a/modules/default/weather/weatherprovider.js b/modules/default/weather/weatherprovider.js index e2a9d32f..4acfe47e 100644 --- a/modules/default/weather/weatherprovider.js +++ b/modules/default/weather/weatherprovider.js @@ -112,11 +112,15 @@ const WeatherProvider = Class.extend({ }, getCorsUrl: function () { - const url = window.config.address + ":" + window.config.port + "/cors?url="; - if (window.config.useHttps) { - return "https://" + url; + if (this.config.mockData) { + return ""; } else { - return "http://" + url; + const url = window.config.address + ":" + window.config.port + "/cors?url="; + if (window.config.useHttps) { + return "https://" + url; + } else { + return "http://" + url; + } } }, From 9f9c17ea8a0090159dfb97fe3d0302768966ca49 Mon Sep 17 00:00:00 2001 From: Karsten Hassel Date: Wed, 26 Jan 2022 00:38:34 +0100 Subject: [PATCH 4/5] increase test timeout to 20 sec. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b3cae096..249e3d7c 100644 --- a/package.json +++ b/package.json @@ -94,7 +94,7 @@ }, "jest": { "verbose": true, - "testTimeout": 15000, + "testTimeout": 20000, "testSequencer": "/tests/configs/test_sequencer.js", "projects": [ { From f04dd6b6cd4fdb6a44f71bbbee536ad91f784371 Mon Sep 17 00:00:00 2001 From: Karsten Hassel Date: Wed, 26 Jan 2022 19:23:08 +0100 Subject: [PATCH 5/5] introduce useCorsProxy, per default only enabled for darksky and envcanada --- modules/default/weather/providers/darksky.js | 3 +- .../default/weather/providers/envcanada.js | 32 +++---------------- .../weather/providers/openweathermap.js | 2 +- modules/default/weather/providers/smhi.js | 2 +- .../default/weather/providers/ukmetoffice.js | 2 +- .../weather/providers/ukmetofficedatahub.js | 2 +- .../default/weather/providers/weatherbit.js | 2 +- .../default/weather/providers/weathergov.js | 2 +- modules/default/weather/weatherprovider.js | 18 +++++------ 9 files changed, 22 insertions(+), 43 deletions(-) diff --git a/modules/default/weather/providers/darksky.js b/modules/default/weather/providers/darksky.js index 01bd79cd..04595d1a 100755 --- a/modules/default/weather/providers/darksky.js +++ b/modules/default/weather/providers/darksky.js @@ -18,6 +18,7 @@ WeatherProvider.register("darksky", { // Set the default config properties that is specific to this provider defaults: { + useCorsProxy: true, apiBase: "https://api.darksky.net", weatherEndpoint: "/forecast", apiKey: "", @@ -67,7 +68,7 @@ WeatherProvider.register("darksky", { // Create a URL from the config and base URL. getUrl() { const units = this.units[this.config.units] || "auto"; - return this.getCorsUrl() + `${this.config.apiBase}${this.config.weatherEndpoint}/${this.config.apiKey}/${this.config.lat},${this.config.lon}?units=${units}&lang=${this.config.lang}`; + return `${this.config.apiBase}${this.config.weatherEndpoint}/${this.config.apiKey}/${this.config.lat},${this.config.lon}?units=${units}&lang=${this.config.lang}`; }, // Implement WeatherDay generator. diff --git a/modules/default/weather/providers/envcanada.js b/modules/default/weather/providers/envcanada.js index 8cbf313c..e293edb4 100644 --- a/modules/default/weather/providers/envcanada.js +++ b/modules/default/weather/providers/envcanada.js @@ -40,6 +40,7 @@ WeatherProvider.register("envcanada", { // Set the default config properties that is specific to this provider defaults: { + useCorsProxy: true, siteCode: "s1234567", provCode: "ON" }, @@ -73,7 +74,7 @@ WeatherProvider.register("envcanada", { // Override the fetchCurrentWeather method to query EC and construct a Current weather object // fetchCurrentWeather() { - this.fetchData(this.getUrl(), "GET") + this.fetchData(this.getUrl(), "GET", "xml") .then((data) => { if (!data) { // Did not receive usable new data. @@ -93,7 +94,7 @@ WeatherProvider.register("envcanada", { // Override the fetchWeatherForecast method to query EC and construct Forecast weather objects // fetchWeatherForecast() { - this.fetchData(this.getUrl(), "GET") + this.fetchData(this.getUrl(), "GET", "xml") .then((data) => { if (!data) { // Did not receive usable new data. @@ -113,7 +114,7 @@ WeatherProvider.register("envcanada", { // Override the fetchWeatherHourly method to query EC and construct Forecast weather objects // fetchWeatherHourly() { - this.fetchData(this.getUrl(), "GET") + this.fetchData(this.getUrl(), "GET", "xml") .then((data) => { if (!data) { // Did not receive usable new data. @@ -129,26 +130,6 @@ WeatherProvider.register("envcanada", { .finally(() => this.updateAvailable()); }, - // - // Override fetchData function to handle XML document (base function assumes JSON) - // - fetchData: function (url, method = "GET", data = null) { - return new Promise(function (resolve, reject) { - const request = new XMLHttpRequest(); - request.open(method, url, true); - request.onreadystatechange = function () { - if (this.readyState === 4) { - if (this.status === 200) { - resolve(this.responseXML); - } else { - reject(request); - } - } - }; - request.send(); - }); - }, - ////////////////////////////////////////////////////////////////////////////////// // // Environment Canada methods - not part of the standard Provider methods @@ -160,11 +141,8 @@ WeatherProvider.register("envcanada", { // URL defaults to the Englsih version simply because there is no language dependancy in the data // being accessed. This is only pertinent when using the EC data elements that contain a textual forecast. // - // Also note that access is supported through a proxy service (thingproxy.freeboard.io) to mitigate - // CORS errors when accessing EC - // getUrl() { - return this.getCorsUrl() + "https://dd.weather.gc.ca/citypage_weather/xml/" + this.config.provCode + "/" + this.config.siteCode + "_e.xml"; + return "https://dd.weather.gc.ca/citypage_weather/xml/" + this.config.provCode + "/" + this.config.siteCode + "_e.xml"; }, // diff --git a/modules/default/weather/providers/openweathermap.js b/modules/default/weather/providers/openweathermap.js index 4d3238f6..c1258d3c 100755 --- a/modules/default/weather/providers/openweathermap.js +++ b/modules/default/weather/providers/openweathermap.js @@ -116,7 +116,7 @@ WeatherProvider.register("openweathermap", { * Gets the complete url for the request */ getUrl() { - return this.getCorsUrl() + this.config.apiBase + this.config.apiVersion + this.config.weatherEndpoint + this.getParams(); + return this.config.apiBase + this.config.apiVersion + this.config.weatherEndpoint + this.getParams(); }, /* diff --git a/modules/default/weather/providers/smhi.js b/modules/default/weather/providers/smhi.js index adc5f07a..603aa814 100644 --- a/modules/default/weather/providers/smhi.js +++ b/modules/default/weather/providers/smhi.js @@ -91,7 +91,7 @@ WeatherProvider.register("smhi", { getURL() { let lon = this.config.lon; let lat = this.config.lat; - return this.getCorsUrl() + `https://opendata-download-metfcst.smhi.se/api/category/pmp3g/version/2/geotype/point/lon/${lon}/lat/${lat}/data.json`; + return `https://opendata-download-metfcst.smhi.se/api/category/pmp3g/version/2/geotype/point/lon/${lon}/lat/${lat}/data.json`; }, /** diff --git a/modules/default/weather/providers/ukmetoffice.js b/modules/default/weather/providers/ukmetoffice.js index 195e9d2a..158592bd 100755 --- a/modules/default/weather/providers/ukmetoffice.js +++ b/modules/default/weather/providers/ukmetoffice.js @@ -73,7 +73,7 @@ WeatherProvider.register("ukmetoffice", { * Gets the complete url for the request */ getUrl(forecastType) { - return this.getCorsUrl() + this.config.apiBase + this.config.locationID + this.getParams(forecastType); + return this.config.apiBase + this.config.locationID + this.getParams(forecastType); }, /* diff --git a/modules/default/weather/providers/ukmetofficedatahub.js b/modules/default/weather/providers/ukmetofficedatahub.js index 86ede528..bab1220d 100644 --- a/modules/default/weather/providers/ukmetofficedatahub.js +++ b/modules/default/weather/providers/ukmetofficedatahub.js @@ -63,7 +63,7 @@ WeatherProvider.register("ukmetofficedatahub", { queryStrings += "&includeLocationName=" + true; // Return URL, making sure there is a trailing "/" in the base URL. - return this.getCorsUrl() + this.config.apiBase + (this.config.apiBase.endsWith("/") ? "" : "/") + forecastType + queryStrings; + return this.config.apiBase + (this.config.apiBase.endsWith("/") ? "" : "/") + forecastType + queryStrings; }, // Build the list of headers for the request diff --git a/modules/default/weather/providers/weatherbit.js b/modules/default/weather/providers/weatherbit.js index 9023f742..17f27812 100644 --- a/modules/default/weather/providers/weatherbit.js +++ b/modules/default/weather/providers/weatherbit.js @@ -72,7 +72,7 @@ WeatherProvider.register("weatherbit", { // Create a URL from the config and base URL. getUrl() { const units = this.units[this.config.units] || "auto"; - return this.getCorsUrl() + `${this.config.apiBase}${this.config.weatherEndpoint}?lat=${this.config.lat}&lon=${this.config.lon}&units=${units}&key=${this.config.apiKey}`; + return `${this.config.apiBase}${this.config.weatherEndpoint}?lat=${this.config.lat}&lon=${this.config.lon}&units=${units}&key=${this.config.apiKey}`; }, // Implement WeatherDay generator. diff --git a/modules/default/weather/providers/weathergov.js b/modules/default/weather/providers/weathergov.js index cf7912c3..31934784 100755 --- a/modules/default/weather/providers/weathergov.js +++ b/modules/default/weather/providers/weathergov.js @@ -40,7 +40,7 @@ WeatherProvider.register("weathergov", { // Called to set the config, this config is the same as the weather module's config. setConfig: function (config) { this.config = config; - (this.config.apiBase = this.getCorsUrl() + "https://api.weather.gov"), this.fetchWxGovURLs(this.config); + (this.config.apiBase = "https://api.weather.gov"), this.fetchWxGovURLs(this.config); }, // Called when the weather provider is about to start. diff --git a/modules/default/weather/weatherprovider.js b/modules/default/weather/weatherprovider.js index 4acfe47e..6533b472 100644 --- a/modules/default/weather/weatherprovider.js +++ b/modules/default/weather/weatherprovider.js @@ -112,20 +112,16 @@ const WeatherProvider = Class.extend({ }, getCorsUrl: function () { - if (this.config.mockData) { + if (this.config.mockData || typeof this.config.useCorsProxy === "undefined" || !this.config.useCorsProxy) { return ""; } else { - const url = window.config.address + ":" + window.config.port + "/cors?url="; - if (window.config.useHttps) { - return "https://" + url; - } else { - return "http://" + url; - } + return location.protocol + "//" + location.host + "/cors?url="; } }, // A convenience function to make requests. It returns a promise. - fetchData: function (url, method = "GET", data = null) { + fetchData: function (url, method = "GET", type = "json") { + url = this.getCorsUrl() + url; const getData = function (mockData) { return new Promise(function (resolve, reject) { if (mockData) { @@ -138,7 +134,11 @@ const WeatherProvider = Class.extend({ request.onreadystatechange = function () { if (this.readyState === 4) { if (this.status === 200) { - resolve(JSON.parse(this.response)); + if (type === "xml") { + resolve(this.responseXML); + } else { + resolve(JSON.parse(this.response)); + } } else { reject(request); }