From 53e300bd31db919d94f9c019ac8a88f1831ca9d5 Mon Sep 17 00:00:00 2001 From: Karsten Hassel Date: Tue, 2 Mar 2021 00:17:13 +0100 Subject: [PATCH 1/7] snapshot --- modules/default/calendar/calendarfetcher.js | 45 +++++++++++---------- modules/default/newsfeed/newsfeedfetcher.js | 23 +++++------ package.json | 2 +- 3 files changed, 36 insertions(+), 34 deletions(-) diff --git a/modules/default/calendar/calendarfetcher.js b/modules/default/calendar/calendarfetcher.js index 73f92b1e..b0849da7 100644 --- a/modules/default/calendar/calendarfetcher.js +++ b/modules/default/calendar/calendarfetcher.js @@ -6,7 +6,8 @@ */ const Log = require("logger"); const ical = require("node-ical"); -const request = require("request"); +const fetch = require("node-fetch"); +const https = require('https'); /** * Moment date @@ -43,22 +44,22 @@ const CalendarFetcher = function (url, reloadInterval, excludedEvents, maximumEn const fetchCalendar = function () { clearTimeout(reloadTimer); reloadTimer = null; + httpsAgent = null; const nodeVersion = Number(process.version.match(/^v(\d+\.\d+)/)[1]); - const opts = { - headers: { - "User-Agent": "Mozilla/5.0 (Node.js " + nodeVersion + ") MagicMirror/" + global.version + " (https://github.com/MichMich/MagicMirror/)" - }, - gzip: true + const headers = { + "User-Agent": "Mozilla/5.0 (Node.js " + nodeVersion + ") MagicMirror/" + global.version + " (https://github.com/MichMich/MagicMirror/)" }; if (selfSignedCert) { - var agentOptions = { - rejectUnauthorized: false - }; - opts.agentOptions = agentOptions; + httpsAgent = new https.Agent({ + rejectUnauthorized: false, + }); } +// todo: auth, +// https://github.com/devfans/digest-fetch +// https://hackersandslackers.com/making-api-requests-with-nodejs/ if (auth) { if (auth.method === "bearer") { opts.auth = { @@ -73,25 +74,27 @@ const CalendarFetcher = function (url, reloadInterval, excludedEvents, maximumEn } } - request(url, opts, function (err, r, requestData) { - if (err) { - fetchFailedCallback(self, err); - scheduleTimer(); - return; - } else if (r.statusCode !== 200) { - fetchFailedCallback(self, r.statusCode + ": " + r.statusMessage); - scheduleTimer(); - return; + fetch(url, { headers: headers, httpsAgent: httpsAgent }) + .catch(error => { + fetchFailedCallback(self, error); + scheduleTimer(); + }) + .then(response => { + if (response.status !== 200) { + throw new Error(response.status + ": " + response.statusText); } + return response; + }) + .then(response => response.text()) + .then(responseData => { let data = []; try { - data = ical.parseICS(requestData); + data = ical.parseICS(responseData); } catch (error) { fetchFailedCallback(self, error.message); scheduleTimer(); - return; } Log.debug(" parsed data=" + JSON.stringify(data)); diff --git a/modules/default/newsfeed/newsfeedfetcher.js b/modules/default/newsfeed/newsfeedfetcher.js index 68ac75ac..48c4cbbd 100644 --- a/modules/default/newsfeed/newsfeedfetcher.js +++ b/modules/default/newsfeed/newsfeedfetcher.js @@ -6,7 +6,7 @@ */ const Log = require("logger"); const FeedMe = require("feedme"); -const request = require("request"); +const fetch = require("node-fetch"); const iconv = require("iconv-lite"); /** @@ -79,22 +79,21 @@ const NewsfeedFetcher = function (url, reloadInterval, encoding, logFeedWarnings }); const nodeVersion = Number(process.version.match(/^v(\d+\.\d+)/)[1]); - const opts = { - headers: { - "User-Agent": "Mozilla/5.0 (Node.js " + nodeVersion + ") MagicMirror/" + global.version + " (https://github.com/MichMich/MagicMirror/)", - "Cache-Control": "max-age=0, no-cache, no-store, must-revalidate", - Pragma: "no-cache" - }, - encoding: null + const headers = { + "User-Agent": "Mozilla/5.0 (Node.js " + nodeVersion + ") MagicMirror/" + global.version + " (https://github.com/MichMich/MagicMirror/)", + "Cache-Control": "max-age=0, no-cache, no-store, must-revalidate", + Pragma: "no-cache" }; - request(url, opts) - .on("error", function (error) { + fetch(url, { headers: headers }) + .catch(error => { fetchFailedCallback(self, error); scheduleTimer(); }) - .pipe(iconv.decodeStream(encoding)) - .pipe(parser); + .then(res => { + res.body.pipe(iconv.decodeStream(encoding)).pipe(parser); + }); + }; /** diff --git a/package.json b/package.json index 8ca6cd30..97e52710 100644 --- a/package.json +++ b/package.json @@ -80,8 +80,8 @@ "iconv-lite": "^0.6.2", "module-alias": "^2.2.2", "moment": "^2.29.1", + "node-fetch": "^2.6.1", "node-ical": "^0.12.7", - "request": "^2.88.2", "rrule": "^2.6.6", "rrule-alt": "^2.2.8", "simple-git": "^2.31.0", From 92a35692f2526d9cc898026f9bdb35b346aceaa6 Mon Sep 17 00:00:00 2001 From: Karsten Hassel Date: Tue, 2 Mar 2021 00:40:55 +0100 Subject: [PATCH 2/7] snapshot --- modules/default/calendar/calendarfetcher.js | 21 +++++++++++---------- package.json | 1 + 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/modules/default/calendar/calendarfetcher.js b/modules/default/calendar/calendarfetcher.js index b0849da7..e4748cf7 100644 --- a/modules/default/calendar/calendarfetcher.js +++ b/modules/default/calendar/calendarfetcher.js @@ -6,7 +6,7 @@ */ const Log = require("logger"); const ical = require("node-ical"); -const fetch = require("node-fetch"); +const fetch = require("digest-fetch"); const https = require('https'); /** @@ -45,6 +45,8 @@ const CalendarFetcher = function (url, reloadInterval, excludedEvents, maximumEn clearTimeout(reloadTimer); reloadTimer = null; httpsAgent = null; + user = ""; + password = ""; const nodeVersion = Number(process.version.match(/^v(\d+\.\d+)/)[1]); const headers = { @@ -59,22 +61,21 @@ const CalendarFetcher = function (url, reloadInterval, excludedEvents, maximumEn // todo: auth, // https://github.com/devfans/digest-fetch +// https://github.com/pablopunk/auth-fetch/blob/master/index.js // https://hackersandslackers.com/making-api-requests-with-nodejs/ if (auth) { if (auth.method === "bearer") { - opts.auth = { - bearer: auth.pass - }; + // opts.auth = { + // bearer: auth.pass + // }; } else { - opts.auth = { - user: auth.user, - pass: auth.pass, - sendImmediately: auth.method !== "digest" - }; + user = auth.user; + password = auth.pass; } } - fetch(url, { headers: headers, httpsAgent: httpsAgent }) + const client = new fetch(user, password) ; + client.fetch(url, { headers: headers, httpsAgent: httpsAgent }) .catch(error => { fetchFailedCallback(self, error); scheduleTimer(); diff --git a/package.json b/package.json index 97e52710..43add524 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,7 @@ "dependencies": { "colors": "^1.4.0", "console-stamp": "^3.0.0-rc4.2", + "digest-fetch": "^1.1.6", "eslint": "^7.19.0", "express": "^4.17.1", "express-ipfilter": "^1.1.2", From f09c54184aedba98669b34382f21f13ad002fc33 Mon Sep 17 00:00:00 2001 From: Karsten Hassel Date: Tue, 2 Mar 2021 21:26:25 +0100 Subject: [PATCH 3/7] moved tests to fetch, run prettier --- modules/default/calendar/calendarfetcher.js | 636 ++++++++++---------- modules/default/newsfeed/newsfeedfetcher.js | 5 +- tests/e2e/env_spec.js | 10 +- tests/e2e/fonts.js | 6 +- tests/e2e/ipWhistlist_spec.js | 10 +- tests/e2e/port_config.js | 10 +- tests/e2e/vendor_spec.js | 10 +- 7 files changed, 343 insertions(+), 344 deletions(-) diff --git a/modules/default/calendar/calendarfetcher.js b/modules/default/calendar/calendarfetcher.js index e4748cf7..673a4880 100644 --- a/modules/default/calendar/calendarfetcher.js +++ b/modules/default/calendar/calendarfetcher.js @@ -7,7 +7,7 @@ const Log = require("logger"); const ical = require("node-ical"); const fetch = require("digest-fetch"); -const https = require('https'); +const https = require("https"); /** * Moment date @@ -55,14 +55,14 @@ const CalendarFetcher = function (url, reloadInterval, excludedEvents, maximumEn if (selfSignedCert) { httpsAgent = new https.Agent({ - rejectUnauthorized: false, + rejectUnauthorized: false }); } -// todo: auth, -// https://github.com/devfans/digest-fetch -// https://github.com/pablopunk/auth-fetch/blob/master/index.js -// https://hackersandslackers.com/making-api-requests-with-nodejs/ + // todo: auth, + // https://github.com/devfans/digest-fetch + // https://github.com/pablopunk/auth-fetch/blob/master/index.js + // https://hackersandslackers.com/making-api-requests-with-nodejs/ if (auth) { if (auth.method === "bearer") { // opts.auth = { @@ -74,358 +74,358 @@ const CalendarFetcher = function (url, reloadInterval, excludedEvents, maximumEn } } - const client = new fetch(user, password) ; - client.fetch(url, { headers: headers, httpsAgent: httpsAgent }) - .catch(error => { - fetchFailedCallback(self, error); - scheduleTimer(); - }) - .then(response => { - if (response.status !== 200) { - throw new Error(response.status + ": " + response.statusText); - } - return response; - }) - .then(response => response.text()) - .then(responseData => { - - let data = []; - - try { - data = ical.parseICS(responseData); - } catch (error) { - fetchFailedCallback(self, error.message); + const client = new fetch(user, password); + client + .fetch(url, { headers: headers, httpsAgent: httpsAgent }) + .catch((error) => { + fetchFailedCallback(self, error); scheduleTimer(); - } + }) + .then((response) => { + if (response.status !== 200) { + throw new Error(response.status + ": " + response.statusText); + } + return response; + }) + .then((response) => response.text()) + .then((responseData) => { + let data = []; - Log.debug(" parsed data=" + JSON.stringify(data)); - - const newEvents = []; - - // limitFunction doesn't do much limiting, see comment re: the dates array in rrule section below as to why we need to do the filtering ourselves - const limitFunction = function (date, i) { - return true; - }; - - const eventDate = function (event, time) { - return isFullDayEvent(event) ? moment(event[time], "YYYYMMDD") : moment(new Date(event[time])); - }; - Log.debug("there are " + Object.entries(data).length + " calendar entries"); - Object.entries(data).forEach(([key, event]) => { - const now = new Date(); - const today = moment().startOf("day").toDate(); - const future = moment().startOf("day").add(maximumNumberOfDays, "days").subtract(1, "seconds").toDate(); // Subtract 1 second so that events that start on the middle of the night will not repeat. - let past = today; - Log.debug("have entries "); - if (includePastEvents) { - past = moment().startOf("day").subtract(maximumNumberOfDays, "days").toDate(); + try { + data = ical.parseICS(responseData); + } catch (error) { + fetchFailedCallback(self, error.message); + scheduleTimer(); } - // FIXME: Ugly fix to solve the facebook birthday issue. - // Otherwise, the recurring events only show the birthday for next year. - let isFacebookBirthday = false; - if (typeof event.uid !== "undefined") { - if (event.uid.indexOf("@facebook.com") !== -1) { - isFacebookBirthday = true; + Log.debug(" parsed data=" + JSON.stringify(data)); + + const newEvents = []; + + // limitFunction doesn't do much limiting, see comment re: the dates array in rrule section below as to why we need to do the filtering ourselves + const limitFunction = function (date, i) { + return true; + }; + + const eventDate = function (event, time) { + return isFullDayEvent(event) ? moment(event[time], "YYYYMMDD") : moment(new Date(event[time])); + }; + Log.debug("there are " + Object.entries(data).length + " calendar entries"); + Object.entries(data).forEach(([key, event]) => { + const now = new Date(); + const today = moment().startOf("day").toDate(); + const future = moment().startOf("day").add(maximumNumberOfDays, "days").subtract(1, "seconds").toDate(); // Subtract 1 second so that events that start on the middle of the night will not repeat. + let past = today; + Log.debug("have entries "); + if (includePastEvents) { + past = moment().startOf("day").subtract(maximumNumberOfDays, "days").toDate(); } - } - if (event.type === "VEVENT") { - let startDate = eventDate(event, "start"); - let endDate; + // FIXME: Ugly fix to solve the facebook birthday issue. + // Otherwise, the recurring events only show the birthday for next year. + let isFacebookBirthday = false; + if (typeof event.uid !== "undefined") { + if (event.uid.indexOf("@facebook.com") !== -1) { + isFacebookBirthday = true; + } + } - Log.debug("\nevent=" + JSON.stringify(event)); - if (typeof event.end !== "undefined") { - endDate = eventDate(event, "end"); - } else if (typeof event.duration !== "undefined") { - endDate = startDate.clone().add(moment.duration(event.duration)); - } else { - if (!isFacebookBirthday) { - // make copy of start date, separate storage area - endDate = moment(startDate.format("x"), "x"); + if (event.type === "VEVENT") { + let startDate = eventDate(event, "start"); + let endDate; + + Log.debug("\nevent=" + JSON.stringify(event)); + if (typeof event.end !== "undefined") { + endDate = eventDate(event, "end"); + } else if (typeof event.duration !== "undefined") { + endDate = startDate.clone().add(moment.duration(event.duration)); } else { - endDate = moment(startDate).add(1, "days"); - } - } - - Log.debug(" start=" + startDate.toDate() + " end=" + endDate.toDate()); - - // calculate the duration of the event for use with recurring events. - let duration = parseInt(endDate.format("x")) - parseInt(startDate.format("x")); - - if (event.start.length === 8) { - startDate = startDate.startOf("day"); - } - - const title = getTitleFromEvent(event); - - let excluded = false, - dateFilter = null; - - for (let f in excludedEvents) { - let filter = excludedEvents[f], - testTitle = title.toLowerCase(), - until = null, - useRegex = false, - regexFlags = "g"; - - if (filter instanceof Object) { - if (typeof filter.until !== "undefined") { - until = filter.until; - } - - if (typeof filter.regex !== "undefined") { - useRegex = filter.regex; - } - - // If additional advanced filtering is added in, this section - // must remain last as we overwrite the filter object with the - // filterBy string - if (filter.caseSensitive) { - filter = filter.filterBy; - testTitle = title; - } else if (useRegex) { - filter = filter.filterBy; - testTitle = title; - regexFlags += "i"; + if (!isFacebookBirthday) { + // make copy of start date, separate storage area + endDate = moment(startDate.format("x"), "x"); } else { - filter = filter.filterBy.toLowerCase(); + endDate = moment(startDate).add(1, "days"); } - } else { - filter = filter.toLowerCase(); } - if (testTitleByFilter(testTitle, filter, useRegex, regexFlags)) { - if (until) { - dateFilter = until; - } else { - excluded = true; - } - break; - } - } + Log.debug(" start=" + startDate.toDate() + " end=" + endDate.toDate()); - if (excluded) { - return; - } + // calculate the duration of the event for use with recurring events. + let duration = parseInt(endDate.format("x")) - parseInt(startDate.format("x")); - const location = event.location || false; - const geo = event.geo || false; - const description = event.description || false; - - if (typeof event.rrule !== "undefined" && event.rrule !== null && !isFacebookBirthday) { - const rule = event.rrule; - let addedEvents = 0; - - const pastMoment = moment(past); - const futureMoment = moment(future); - - // can cause problems with e.g. birthdays before 1900 - if ((rule.options && rule.origOptions && rule.origOptions.dtstart && rule.origOptions.dtstart.getFullYear() < 1900) || (rule.options && rule.options.dtstart && rule.options.dtstart.getFullYear() < 1900)) { - rule.origOptions.dtstart.setYear(1900); - rule.options.dtstart.setYear(1900); + if (event.start.length === 8) { + startDate = startDate.startOf("day"); } - // For recurring events, get the set of start dates that fall within the range - // of dates we're looking for. - // kblankenship1989 - to fix issue #1798, converting all dates to locale time first, then converting back to UTC time - let pastLocal = 0; - let futureLocal = 0; - if (isFullDayEvent(event)) { - // if full day event, only use the date part of the ranges - pastLocal = pastMoment.toDate(); - futureLocal = futureMoment.toDate(); - } else { - // if we want past events - if (includePastEvents) { - // use the calculated past time for the between from - pastLocal = pastMoment.toDate(); - } else { - // otherwise use NOW.. cause we shouldnt use any before now - pastLocal = moment().toDate(); //now - } - futureLocal = futureMoment.toDate(); // future - } - Log.debug(" between=" + pastLocal + " to " + futureLocal); - const dates = rule.between(pastLocal, futureLocal, true, limitFunction); - Log.debug("title=" + event.summary + " dates=" + JSON.stringify(dates)); - // The "dates" array contains the set of dates within our desired date range range that are valid - // for the recurrence rule. *However*, it's possible for us to have a specific recurrence that - // had its date changed from outside the range to inside the range. For the time being, - // we'll handle this by adding *all* recurrence entries into the set of dates that we check, - // because the logic below will filter out any recurrences that don't actually belong within - // our display range. - // Would be great if there was a better way to handle this. - if (event.recurrences !== undefined) { - for (let r in event.recurrences) { - // Only add dates that weren't already in the range we added from the rrule so that - // we don"t double-add those events. - if (moment(new Date(r)).isBetween(pastMoment, futureMoment) !== true) { - dates.push(new Date(r)); + const title = getTitleFromEvent(event); + + let excluded = false, + dateFilter = null; + + for (let f in excludedEvents) { + let filter = excludedEvents[f], + testTitle = title.toLowerCase(), + until = null, + useRegex = false, + regexFlags = "g"; + + if (filter instanceof Object) { + if (typeof filter.until !== "undefined") { + until = filter.until; } + + if (typeof filter.regex !== "undefined") { + useRegex = filter.regex; + } + + // If additional advanced filtering is added in, this section + // must remain last as we overwrite the filter object with the + // filterBy string + if (filter.caseSensitive) { + filter = filter.filterBy; + testTitle = title; + } else if (useRegex) { + filter = filter.filterBy; + testTitle = title; + regexFlags += "i"; + } else { + filter = filter.filterBy.toLowerCase(); + } + } else { + filter = filter.toLowerCase(); + } + + if (testTitleByFilter(testTitle, filter, useRegex, regexFlags)) { + if (until) { + dateFilter = until; + } else { + excluded = true; + } + break; } } - // Loop through the set of date entries to see which recurrences should be added to our event list. - for (let d in dates) { - let date = dates[d]; - // ical.js started returning recurrences and exdates as ISOStrings without time information. - // .toISOString().substring(0,10) is the method they use to calculate keys, so we'll do the same - // (see https://github.com/peterbraden/ical.js/pull/84 ) - const dateKey = date.toISOString().substring(0, 10); - let curEvent = event; - let showRecurrence = true; - // for full day events, the time might be off from RRULE/Luxon problem + if (excluded) { + return; + } + + const location = event.location || false; + const geo = event.geo || false; + const description = event.description || false; + + if (typeof event.rrule !== "undefined" && event.rrule !== null && !isFacebookBirthday) { + const rule = event.rrule; + let addedEvents = 0; + + const pastMoment = moment(past); + const futureMoment = moment(future); + + // can cause problems with e.g. birthdays before 1900 + if ((rule.options && rule.origOptions && rule.origOptions.dtstart && rule.origOptions.dtstart.getFullYear() < 1900) || (rule.options && rule.options.dtstart && rule.options.dtstart.getFullYear() < 1900)) { + rule.origOptions.dtstart.setYear(1900); + rule.options.dtstart.setYear(1900); + } + + // For recurring events, get the set of start dates that fall within the range + // of dates we're looking for. + // kblankenship1989 - to fix issue #1798, converting all dates to locale time first, then converting back to UTC time + let pastLocal = 0; + let futureLocal = 0; if (isFullDayEvent(event)) { - Log.debug("fullday"); - // if the offset is negative, east of GMT where the problem is - if (date.getTimezoneOffset() < 0) { - // get the offset of today where we are processing - // this will be the correction we need to apply - let nowOffset = new Date().getTimezoneOffset(); - Log.debug("now offset is " + nowOffset); - // reduce the time by the offset - Log.debug(" recurring date is " + date + " offset is " + date.getTimezoneOffset()); - // apply the correction to the date/time to get it UTC relative - date = new Date(date.getTime() - Math.abs(nowOffset) * 60000); - // the duration was calculated way back at the top before we could correct the start time.. - // fix it for this event entry - duration = 24 * 60 * 60 * 1000; - Log.debug("new recurring date is " + date); + // if full day event, only use the date part of the ranges + pastLocal = pastMoment.toDate(); + futureLocal = futureMoment.toDate(); + } else { + // if we want past events + if (includePastEvents) { + // use the calculated past time for the between from + pastLocal = pastMoment.toDate(); + } else { + // otherwise use NOW.. cause we shouldnt use any before now + pastLocal = moment().toDate(); //now + } + futureLocal = futureMoment.toDate(); // future + } + Log.debug(" between=" + pastLocal + " to " + futureLocal); + const dates = rule.between(pastLocal, futureLocal, true, limitFunction); + Log.debug("title=" + event.summary + " dates=" + JSON.stringify(dates)); + // The "dates" array contains the set of dates within our desired date range range that are valid + // for the recurrence rule. *However*, it's possible for us to have a specific recurrence that + // had its date changed from outside the range to inside the range. For the time being, + // we'll handle this by adding *all* recurrence entries into the set of dates that we check, + // because the logic below will filter out any recurrences that don't actually belong within + // our display range. + // Would be great if there was a better way to handle this. + if (event.recurrences !== undefined) { + for (let r in event.recurrences) { + // Only add dates that weren't already in the range we added from the rrule so that + // we don"t double-add those events. + if (moment(new Date(r)).isBetween(pastMoment, futureMoment) !== true) { + dates.push(new Date(r)); + } } } - startDate = moment(date); + // Loop through the set of date entries to see which recurrences should be added to our event list. + for (let d in dates) { + let date = dates[d]; + // ical.js started returning recurrences and exdates as ISOStrings without time information. + // .toISOString().substring(0,10) is the method they use to calculate keys, so we'll do the same + // (see https://github.com/peterbraden/ical.js/pull/84 ) + const dateKey = date.toISOString().substring(0, 10); + let curEvent = event; + let showRecurrence = true; - let adjustDays = getCorrection(event, date); + // for full day events, the time might be off from RRULE/Luxon problem + if (isFullDayEvent(event)) { + Log.debug("fullday"); + // if the offset is negative, east of GMT where the problem is + if (date.getTimezoneOffset() < 0) { + // get the offset of today where we are processing + // this will be the correction we need to apply + let nowOffset = new Date().getTimezoneOffset(); + Log.debug("now offset is " + nowOffset); + // reduce the time by the offset + Log.debug(" recurring date is " + date + " offset is " + date.getTimezoneOffset()); + // apply the correction to the date/time to get it UTC relative + date = new Date(date.getTime() - Math.abs(nowOffset) * 60000); + // the duration was calculated way back at the top before we could correct the start time.. + // fix it for this event entry + duration = 24 * 60 * 60 * 1000; + Log.debug("new recurring date is " + date); + } + } + startDate = moment(date); - // For each date that we're checking, it's possible that there is a recurrence override for that one day. - if (curEvent.recurrences !== undefined && curEvent.recurrences[dateKey] !== undefined) { - // We found an override, so for this recurrence, use a potentially different title, start date, and duration. - curEvent = curEvent.recurrences[dateKey]; - startDate = moment(curEvent.start); - duration = parseInt(moment(curEvent.end).format("x")) - parseInt(startDate.format("x")); + let adjustDays = getCorrection(event, date); + + // For each date that we're checking, it's possible that there is a recurrence override for that one day. + if (curEvent.recurrences !== undefined && curEvent.recurrences[dateKey] !== undefined) { + // We found an override, so for this recurrence, use a potentially different title, start date, and duration. + curEvent = curEvent.recurrences[dateKey]; + startDate = moment(curEvent.start); + duration = parseInt(moment(curEvent.end).format("x")) - parseInt(startDate.format("x")); + } + // If there's no recurrence override, check for an exception date. Exception dates represent exceptions to the rule. + else if (curEvent.exdate !== undefined && curEvent.exdate[dateKey] !== undefined) { + // This date is an exception date, which means we should skip it in the recurrence pattern. + showRecurrence = false; + } + Log.debug("duration=" + duration); + + endDate = moment(parseInt(startDate.format("x")) + duration, "x"); + if (startDate.format("x") === endDate.format("x")) { + endDate = endDate.endOf("day"); + } + + const recurrenceTitle = getTitleFromEvent(curEvent); + + // If this recurrence ends before the start of the date range, or starts after the end of the date range, don"t add + // it to the event list. + if (endDate.isBefore(past) || startDate.isAfter(future)) { + showRecurrence = false; + } + + if (timeFilterApplies(now, endDate, dateFilter)) { + showRecurrence = false; + } + + if (showRecurrence === true) { + Log.debug("saving event =" + description); + addedEvents++; + newEvents.push({ + title: recurrenceTitle, + startDate: (adjustDays ? (adjustDays > 0 ? startDate.add(adjustDays, "hours") : startDate.subtract(Math.abs(adjustDays), "hours")) : startDate).format("x"), + endDate: (adjustDays ? (adjustDays > 0 ? endDate.add(adjustDays, "hours") : endDate.subtract(Math.abs(adjustDays), "hours")) : endDate).format("x"), + fullDayEvent: isFullDayEvent(event), + recurringEvent: true, + class: event.class, + firstYear: event.start.getFullYear(), + location: location, + geo: geo, + description: description + }); + } } - // If there's no recurrence override, check for an exception date. Exception dates represent exceptions to the rule. - else if (curEvent.exdate !== undefined && curEvent.exdate[dateKey] !== undefined) { - // This date is an exception date, which means we should skip it in the recurrence pattern. - showRecurrence = false; - } - Log.debug("duration=" + duration); + // end recurring event parsing + } else { + // Single event. + const fullDayEvent = isFacebookBirthday ? true : isFullDayEvent(event); + // Log.debug("full day event") - endDate = moment(parseInt(startDate.format("x")) + duration, "x"); - if (startDate.format("x") === endDate.format("x")) { - endDate = endDate.endOf("day"); + if (includePastEvents) { + // Past event is too far in the past, so skip. + if (endDate < past) { + return; + } + } else { + // It's not a fullday event, and it is in the past, so skip. + if (!fullDayEvent && endDate < new Date()) { + return; + } + + // It's a fullday event, and it is before today, So skip. + if (fullDayEvent && endDate <= today) { + return; + } } - const recurrenceTitle = getTitleFromEvent(curEvent); - - // If this recurrence ends before the start of the date range, or starts after the end of the date range, don"t add - // it to the event list. - if (endDate.isBefore(past) || startDate.isAfter(future)) { - showRecurrence = false; + // It exceeds the maximumNumberOfDays limit, so skip. + if (startDate > future) { + return; } if (timeFilterApplies(now, endDate, dateFilter)) { - showRecurrence = false; - } - - if (showRecurrence === true) { - Log.debug("saving event =" + description); - addedEvents++; - newEvents.push({ - title: recurrenceTitle, - startDate: (adjustDays ? (adjustDays > 0 ? startDate.add(adjustDays, "hours") : startDate.subtract(Math.abs(adjustDays), "hours")) : startDate).format("x"), - endDate: (adjustDays ? (adjustDays > 0 ? endDate.add(adjustDays, "hours") : endDate.subtract(Math.abs(adjustDays), "hours")) : endDate).format("x"), - fullDayEvent: isFullDayEvent(event), - recurringEvent: true, - class: event.class, - firstYear: event.start.getFullYear(), - location: location, - geo: geo, - description: description - }); - } - } - // end recurring event parsing - } else { - // Single event. - const fullDayEvent = isFacebookBirthday ? true : isFullDayEvent(event); - // Log.debug("full day event") - - if (includePastEvents) { - // Past event is too far in the past, so skip. - if (endDate < past) { - return; - } - } else { - // It's not a fullday event, and it is in the past, so skip. - if (!fullDayEvent && endDate < new Date()) { return; } - // It's a fullday event, and it is before today, So skip. - if (fullDayEvent && endDate <= today) { - return; + // Adjust start date so multiple day events will be displayed as happening today even though they started some days ago already + if (fullDayEvent && startDate <= today) { + startDate = moment(today); } + // if the start and end are the same, then make end the 'end of day' value (start is at 00:00:00) + if (fullDayEvent && startDate.format("x") === endDate.format("x")) { + endDate = endDate.endOf("day"); + } + // get correction for date saving and dst change between now and then + let adjustDays = getCorrection(event, startDate.toDate()); + // Every thing is good. Add it to the list. + newEvents.push({ + title: title, + startDate: (adjustDays ? (adjustDays > 0 ? startDate.add(adjustDays, "hours") : startDate.subtract(Math.abs(adjustDays), "hours")) : startDate).format("x"), + endDate: (adjustDays ? (adjustDays > 0 ? endDate.add(adjustDays, "hours") : endDate.subtract(Math.abs(adjustDays), "hours")) : endDate).format("x"), + fullDayEvent: fullDayEvent, + class: event.class, + location: location, + geo: geo, + description: description + }); } - - // It exceeds the maximumNumberOfDays limit, so skip. - if (startDate > future) { - return; - } - - if (timeFilterApplies(now, endDate, dateFilter)) { - return; - } - - // Adjust start date so multiple day events will be displayed as happening today even though they started some days ago already - if (fullDayEvent && startDate <= today) { - startDate = moment(today); - } - // if the start and end are the same, then make end the 'end of day' value (start is at 00:00:00) - if (fullDayEvent && startDate.format("x") === endDate.format("x")) { - endDate = endDate.endOf("day"); - } - // get correction for date saving and dst change between now and then - let adjustDays = getCorrection(event, startDate.toDate()); - // Every thing is good. Add it to the list. - newEvents.push({ - title: title, - startDate: (adjustDays ? (adjustDays > 0 ? startDate.add(adjustDays, "hours") : startDate.subtract(Math.abs(adjustDays), "hours")) : startDate).format("x"), - endDate: (adjustDays ? (adjustDays > 0 ? endDate.add(adjustDays, "hours") : endDate.subtract(Math.abs(adjustDays), "hours")) : endDate).format("x"), - fullDayEvent: fullDayEvent, - class: event.class, - location: location, - geo: geo, - description: description - }); } + }); + + newEvents.sort(function (a, b) { + return a.startDate - b.startDate; + }); + + // include up to maximumEntries current or upcoming events + // If past events should be included, include all past events + const now = moment(); + var entries = 0; + events = []; + for (let ne of newEvents) { + if (moment(ne.endDate, "x").isBefore(now)) { + if (includePastEvents) events.push(ne); + continue; + } + entries++; + // If max events has been saved, skip the rest + if (entries > maximumEntries) break; + events.push(ne); } + + self.broadcastEvents(); + scheduleTimer(); }); - - newEvents.sort(function (a, b) { - return a.startDate - b.startDate; - }); - - // include up to maximumEntries current or upcoming events - // If past events should be included, include all past events - const now = moment(); - var entries = 0; - events = []; - for (let ne of newEvents) { - if (moment(ne.endDate, "x").isBefore(now)) { - if (includePastEvents) events.push(ne); - continue; - } - entries++; - // If max events has been saved, skip the rest - if (entries > maximumEntries) break; - events.push(ne); - } - - self.broadcastEvents(); - scheduleTimer(); - }); }; /* diff --git a/modules/default/newsfeed/newsfeedfetcher.js b/modules/default/newsfeed/newsfeedfetcher.js index 48c4cbbd..8fc18a9e 100644 --- a/modules/default/newsfeed/newsfeedfetcher.js +++ b/modules/default/newsfeed/newsfeedfetcher.js @@ -86,14 +86,13 @@ const NewsfeedFetcher = function (url, reloadInterval, encoding, logFeedWarnings }; fetch(url, { headers: headers }) - .catch(error => { + .catch((error) => { fetchFailedCallback(self, error); scheduleTimer(); }) - .then(res => { + .then((res) => { res.body.pipe(iconv.decodeStream(encoding)).pipe(parser); }); - }; /** diff --git a/tests/e2e/env_spec.js b/tests/e2e/env_spec.js index 09c4f9cb..9c827055 100644 --- a/tests/e2e/env_spec.js +++ b/tests/e2e/env_spec.js @@ -1,5 +1,5 @@ const helpers = require("./global-setup"); -const request = require("request"); +const fetch = require("node-fetch"); const expect = require("chai").expect; const describe = global.describe; @@ -46,15 +46,15 @@ describe("Electron app environment", function () { }); it("get request from http://localhost:8080 should return 200", function (done) { - request.get("http://localhost:8080", function (err, res, body) { - expect(res.statusCode).to.equal(200); + fetch("http://localhost:8080").then((res) => { + expect(res.status).to.equal(200); done(); }); }); it("get request from http://localhost:8080/nothing should return 404", function (done) { - request.get("http://localhost:8080/nothing", function (err, res, body) { - expect(res.statusCode).to.equal(404); + fetch("http://localhost:8080/nothing").then((res) => { + expect(res.status).to.equal(404); done(); }); }); diff --git a/tests/e2e/fonts.js b/tests/e2e/fonts.js index d6c69e50..c8c2a063 100644 --- a/tests/e2e/fonts.js +++ b/tests/e2e/fonts.js @@ -1,5 +1,5 @@ const helpers = require("./global-setup"); -const request = require("request"); +const fetch = require("node-fetch"); const expect = require("chai").expect; const forEach = require("mocha-each"); @@ -40,8 +40,8 @@ describe("All font files from roboto.css should be downloadable", function () { forEach(fontFiles).it("should return 200 HTTP code for file '%s'", (fontFile, done) => { var fontUrl = "http://localhost:8080/fonts/" + fontFile; - request.get(fontUrl, function (err, res, body) { - expect(res.statusCode).to.equal(200); + fetch(fontUrl).then((res) => { + expect(res.status).to.equal(200); done(); }); }); diff --git a/tests/e2e/ipWhistlist_spec.js b/tests/e2e/ipWhistlist_spec.js index e22231b9..203bc711 100644 --- a/tests/e2e/ipWhistlist_spec.js +++ b/tests/e2e/ipWhistlist_spec.js @@ -1,5 +1,5 @@ const helpers = require("./global-setup"); -const request = require("request"); +const fetch = require("node-fetch"); const expect = require("chai").expect; const describe = global.describe; @@ -32,8 +32,8 @@ describe("ipWhitelist directive configuration", function () { process.env.MM_CONFIG_FILE = "tests/configs/noIpWhiteList.js"; }); it("should return 403", function (done) { - request.get("http://localhost:8080", function (err, res, body) { - expect(res.statusCode).to.equal(403); + fetch("http://localhost:8080").then((res) => { + expect(res.status).to.equal(403); done(); }); }); @@ -45,8 +45,8 @@ describe("ipWhitelist directive configuration", function () { process.env.MM_CONFIG_FILE = "tests/configs/empty_ipWhiteList.js"; }); it("should return 200", function (done) { - request.get("http://localhost:8080", function (err, res, body) { - expect(res.statusCode).to.equal(200); + fetch("http://localhost:8080").then((res) => { + expect(res.status).to.equal(200); done(); }); }); diff --git a/tests/e2e/port_config.js b/tests/e2e/port_config.js index 77889748..9f45f486 100644 --- a/tests/e2e/port_config.js +++ b/tests/e2e/port_config.js @@ -1,5 +1,5 @@ const helpers = require("./global-setup"); -const request = require("request"); +const fetch = require("node-fetch"); const expect = require("chai").expect; const describe = global.describe; @@ -33,8 +33,8 @@ describe("port directive configuration", function () { }); it("should return 200", function (done) { - request.get("http://localhost:8090", function (err, res, body) { - expect(res.statusCode).to.equal(200); + fetch("http://localhost:8090").then((res) => { + expect(res.status).to.equal(200); done(); }); }); @@ -52,8 +52,8 @@ describe("port directive configuration", function () { }); it("should return 200", function (done) { - request.get("http://localhost:8100", function (err, res, body) { - expect(res.statusCode).to.equal(200); + fetch("http://localhost:8100").then((res) => { + expect(res.status).to.equal(200); done(); }); }); diff --git a/tests/e2e/vendor_spec.js b/tests/e2e/vendor_spec.js index bc318b2e..d31a2493 100644 --- a/tests/e2e/vendor_spec.js +++ b/tests/e2e/vendor_spec.js @@ -1,5 +1,5 @@ const helpers = require("./global-setup"); -const request = require("request"); +const fetch = require("node-fetch"); const expect = require("chai").expect; const describe = global.describe; @@ -32,8 +32,8 @@ describe("Vendors", function () { Object.keys(vendors).forEach((vendor) => { it(`should return 200 HTTP code for vendor "${vendor}"`, function () { var urlVendor = "http://localhost:8080/vendor/" + vendors[vendor]; - request.get(urlVendor, function (err, res, body) { - expect(res.statusCode).to.equal(200); + fetch(urlVendor).then((res) => { + expect(res.status).to.equal(200); }); }); }); @@ -41,8 +41,8 @@ describe("Vendors", function () { Object.keys(vendors).forEach((vendor) => { it(`should return 404 HTTP code for vendor https://localhost/"${vendor}"`, function () { var urlVendor = "http://localhost:8080/" + vendors[vendor]; - request.get(urlVendor, function (err, res, body) { - expect(res.statusCode).to.equal(404); + fetch(urlVendor).then((res) => { + expect(res.status).to.equal(404); }); }); }); From 1f9109f8e40dfe43c892e908286d152292aeb22d Mon Sep 17 00:00:00 2001 From: Karsten Hassel Date: Wed, 3 Mar 2021 00:09:32 +0100 Subject: [PATCH 4/7] snapshot --- modules/default/calendar/calendarfetcher.js | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/modules/default/calendar/calendarfetcher.js b/modules/default/calendar/calendarfetcher.js index 673a4880..cec74a31 100644 --- a/modules/default/calendar/calendarfetcher.js +++ b/modules/default/calendar/calendarfetcher.js @@ -44,9 +44,10 @@ const CalendarFetcher = function (url, reloadInterval, excludedEvents, maximumEn const fetchCalendar = function () { clearTimeout(reloadTimer); reloadTimer = null; - httpsAgent = null; - user = ""; - password = ""; + let httpsAgent = null; + let user = ""; + let password = ""; + let opts = {}; const nodeVersion = Number(process.version.match(/^v(\d+\.\d+)/)[1]); const headers = { @@ -59,7 +60,7 @@ const CalendarFetcher = function (url, reloadInterval, excludedEvents, maximumEn }); } - // todo: auth, + // todo: bearer // https://github.com/devfans/digest-fetch // https://github.com/pablopunk/auth-fetch/blob/master/index.js // https://hackersandslackers.com/making-api-requests-with-nodejs/ @@ -71,11 +72,13 @@ const CalendarFetcher = function (url, reloadInterval, excludedEvents, maximumEn } else { user = auth.user; password = auth.pass; + if (auth.method === "basic") { + opts = { basic: true }; + }; } } - const client = new fetch(user, password); - client + new fetch(user, password, opts) .fetch(url, { headers: headers, httpsAgent: httpsAgent }) .catch((error) => { fetchFailedCallback(self, error); From c80aeff9453e0b9381cb46e8d34d489a6160f60a Mon Sep 17 00:00:00 2001 From: Karsten Hassel Date: Thu, 4 Mar 2021 21:12:53 +0100 Subject: [PATCH 5/7] fixes calendarfetcher --- modules/default/calendar/calendarfetcher.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/default/calendar/calendarfetcher.js b/modules/default/calendar/calendarfetcher.js index cec74a31..c439380a 100644 --- a/modules/default/calendar/calendarfetcher.js +++ b/modules/default/calendar/calendarfetcher.js @@ -72,7 +72,7 @@ const CalendarFetcher = function (url, reloadInterval, excludedEvents, maximumEn } else { user = auth.user; password = auth.pass; - if (auth.method === "basic") { + if (auth.method !== "digest") { opts = { basic: true }; }; } @@ -86,7 +86,8 @@ const CalendarFetcher = function (url, reloadInterval, excludedEvents, maximumEn }) .then((response) => { if (response.status !== 200) { - throw new Error(response.status + ": " + response.statusText); + fetchFailedCallback(self, response.statusText); + scheduleTimer(); } return response; }) From 504e7cadd7a3bcefaec306d9609ab7b07c4b0516 Mon Sep 17 00:00:00 2001 From: Karsten Hassel Date: Fri, 5 Mar 2021 00:13:32 +0100 Subject: [PATCH 6/7] getFetcher --- modules/default/calendar/calendarfetcher.js | 63 +++++++++------------ 1 file changed, 27 insertions(+), 36 deletions(-) diff --git a/modules/default/calendar/calendarfetcher.js b/modules/default/calendar/calendarfetcher.js index c439380a..97229089 100644 --- a/modules/default/calendar/calendarfetcher.js +++ b/modules/default/calendar/calendarfetcher.js @@ -6,8 +6,10 @@ */ const Log = require("logger"); const ical = require("node-ical"); -const fetch = require("digest-fetch"); +const fetch = require("node-fetch"); +const digest = require("digest-fetch"); const https = require("https"); +const base64 = require("base-64"); /** * Moment date @@ -42,44 +44,33 @@ const CalendarFetcher = function (url, reloadInterval, excludedEvents, maximumEn * Initiates calendar fetch. */ const fetchCalendar = function () { + function getFetcher(url, auth) { + const nodeVersion = Number(process.version.match(/^v(\d+\.\d+)/)[1]); + let headers = { + "User-Agent": "Mozilla/5.0 (Node.js " + nodeVersion + ") MagicMirror/" + global.version + " (https://github.com/MichMich/MagicMirror/)" + }; + let httpsAgent = null; + + if (selfSignedCert) { + httpsAgent = new https.Agent({ + rejectUnauthorized: false + }); + } + if (auth) { + if (auth.method === "bearer") { + headers.Authorization = "Bearer " + auth.pass; + } else if (auth.method === "digest") { + return new digest(auth.user, auth.pass).fetch(url, { headers: headers, httpsAgent: httpsAgent }); + } else { + headers.Authorization = "Basic " + base64.encode(auth.user + ":" + auth.pass); + } + } + return fetch(url, { headers: headers, httpsAgent: httpsAgent }); + } clearTimeout(reloadTimer); reloadTimer = null; - let httpsAgent = null; - let user = ""; - let password = ""; - let opts = {}; - const nodeVersion = Number(process.version.match(/^v(\d+\.\d+)/)[1]); - const headers = { - "User-Agent": "Mozilla/5.0 (Node.js " + nodeVersion + ") MagicMirror/" + global.version + " (https://github.com/MichMich/MagicMirror/)" - }; - - if (selfSignedCert) { - httpsAgent = new https.Agent({ - rejectUnauthorized: false - }); - } - - // todo: bearer - // https://github.com/devfans/digest-fetch - // https://github.com/pablopunk/auth-fetch/blob/master/index.js - // https://hackersandslackers.com/making-api-requests-with-nodejs/ - if (auth) { - if (auth.method === "bearer") { - // opts.auth = { - // bearer: auth.pass - // }; - } else { - user = auth.user; - password = auth.pass; - if (auth.method !== "digest") { - opts = { basic: true }; - }; - } - } - - new fetch(user, password, opts) - .fetch(url, { headers: headers, httpsAgent: httpsAgent }) + getFetcher(url, auth) .catch((error) => { fetchFailedCallback(self, error); scheduleTimer(); From 514b9453f8bcc295801e1e980c7d2252035815fd Mon Sep 17 00:00:00 2001 From: Karsten Hassel Date: Sat, 6 Mar 2021 21:19:14 +0100 Subject: [PATCH 7/7] removed getFetcher function --- modules/default/calendar/calendarfetcher.js | 49 +++++++++++---------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/modules/default/calendar/calendarfetcher.js b/modules/default/calendar/calendarfetcher.js index 97229089..31ba5022 100644 --- a/modules/default/calendar/calendarfetcher.js +++ b/modules/default/calendar/calendarfetcher.js @@ -44,33 +44,34 @@ const CalendarFetcher = function (url, reloadInterval, excludedEvents, maximumEn * Initiates calendar fetch. */ const fetchCalendar = function () { - function getFetcher(url, auth) { - const nodeVersion = Number(process.version.match(/^v(\d+\.\d+)/)[1]); - let headers = { - "User-Agent": "Mozilla/5.0 (Node.js " + nodeVersion + ") MagicMirror/" + global.version + " (https://github.com/MichMich/MagicMirror/)" - }; - let httpsAgent = null; - - if (selfSignedCert) { - httpsAgent = new https.Agent({ - rejectUnauthorized: false - }); - } - if (auth) { - if (auth.method === "bearer") { - headers.Authorization = "Bearer " + auth.pass; - } else if (auth.method === "digest") { - return new digest(auth.user, auth.pass).fetch(url, { headers: headers, httpsAgent: httpsAgent }); - } else { - headers.Authorization = "Basic " + base64.encode(auth.user + ":" + auth.pass); - } - } - return fetch(url, { headers: headers, httpsAgent: httpsAgent }); - } clearTimeout(reloadTimer); reloadTimer = null; + const nodeVersion = Number(process.version.match(/^v(\d+\.\d+)/)[1]); + let fetcher = null; + let httpsAgent = null; + let headers = { + "User-Agent": "Mozilla/5.0 (Node.js " + nodeVersion + ") MagicMirror/" + global.version + " (https://github.com/MichMich/MagicMirror/)" + }; - getFetcher(url, auth) + if (selfSignedCert) { + httpsAgent = new https.Agent({ + rejectUnauthorized: false + }); + } + if (auth) { + if (auth.method === "bearer") { + headers.Authorization = "Bearer " + auth.pass; + } else if (auth.method === "digest") { + fetcher = new digest(auth.user, auth.pass).fetch(url, { headers: headers, httpsAgent: httpsAgent }); + } else { + headers.Authorization = "Basic " + base64.encode(auth.user + ":" + auth.pass); + } + } + if (fetcher === null) { + fetcher = fetch(url, { headers: headers, httpsAgent: httpsAgent }); + } + + fetcher .catch((error) => { fetchFailedCallback(self, error); scheduleTimer();