diff --git a/docs/developer-orientation.md b/docs/developer-orientation.md index bff64c2..0f5e0c0 100644 --- a/docs/developer-orientation.md +++ b/docs/developer-orientation.md @@ -113,6 +113,7 @@ Total transitive production dependencies: 144 * (0) ansi-colors: Helps with interactive prompting for the initial setup, and it's already pulled in by enquirer. * (1) chunk-text: It does what I want. * (0) cloudstorm: Discord gateway library with bring-your-own-caching that I trust. +* (0) denque: Much faster than using arrays as a queue. * (0) discord-api-types: Bitfields needed at runtime and types needed for development. * (0) domino: DOM implementation that's already pulled in by turndown. * (2) enquirer: Interactive prompting for the initial setup rather than forcing users to edit YAML non-interactively. diff --git a/src/d2m/actions/create-space.js b/src/d2m/actions/create-space.js index 7a751e2..403fdb2 100644 --- a/src/d2m/actions/create-space.js +++ b/src/d2m/actions/create-space.js @@ -8,6 +8,8 @@ const {reg} = require("../../matrix/read-registration") const passthrough = require("../../passthrough") const {discord, sync, db, select} = passthrough +/** @type {import("../../matrix/mreq")} */ +const mreq = sync.require("../../matrix/mreq") /** @type {import("../../matrix/api")} */ const api = sync.require("../../matrix/api") /** @type {import("../../matrix/file")} */ @@ -237,8 +239,12 @@ async function syncSpaceExpressions(data, checkBeforeSync) { try { existing = await api.getStateEvent(spaceID, "im.ponies.room_emotes", eventKey) } catch (e) { - // State event not found. This space doesn't have any existing emojis. We create a dummy empty event for comparison's sake. - existing = fn([], guild) + if (e instanceof mreq.MatrixServerError && e.httpStatus < 400) { + // State event not found. This space doesn't have any existing emojis. We create a dummy empty event for comparison's sake. + existing = fn([], guild) + } else { + throw e + } } if (isDeepStrictEqual(existing, content)) return } diff --git a/src/d2m/discord-packets.js b/src/d2m/discord-packets.js index afea9ea..9bab1de 100644 --- a/src/d2m/discord-packets.js +++ b/src/d2m/discord-packets.js @@ -1,173 +1,194 @@ // @ts-check -// Discord library internals type beat - -const DiscordTypes = require("discord-api-types/v10") +const assert = require("assert") const passthrough = require("../passthrough") -const {sync, db} = passthrough +const {sync} = passthrough -const utils = { - /** - * @param {import("./discord-client")} client - * @param {import("cloudstorm").IGatewayMessage} message - * @param {string} listen "full", "half", "no" - whether to set up the event listeners for OOYE to operate - */ - async onPacket(client, message, listen) { - // requiring this later so that the client is already constructed by the time event-dispatcher is loaded - /** @type {typeof import("./event-dispatcher")} */ - const eventDispatcher = sync.require("./event-dispatcher") - /** @type {import("../discord/register-interactions")} */ - const interactions = sync.require("../discord/register-interactions") +/** @type {import("../matrix/homeserver-status")} */ +const homeserverStatus = sync.require("../matrix/homeserver-status") - // Client internals, keep track of the state we need - if (message.t === "READY") { - if (client.ready) return - client.ready = true - client.user = message.d.user - client.application = message.d.application - console.log(`Discord logged in as ${client.user.username}#${client.user.discriminator} (${client.user.id})`) - interactions.registerInteractions() +/** + * @param {import("./discord-client")} client + * @param {import("cloudstorm").IGatewayMessage} message + * @param {string} listen "full", "half", "no" - whether to set up the event listeners for OOYE to operate + */ +async function onPacket(client, message, listen) { + // requiring this later so that the client is already constructed by the time event-dispatcher is loaded + /** @type {typeof import("./event-dispatcher")} */ + const eventDispatcher = sync.require("./event-dispatcher") + /** @type {import("../discord/register-interactions")} */ + const interactions = sync.require("../discord/register-interactions") - } else if (message.t === "GUILD_CREATE") { - message.d.members = message.d.members.filter(m => m.user.id === client.user.id) // only keep the bot's own member - it's needed to determine private channels on web - client.guilds.set(message.d.id, message.d) - const arr = [] - client.guildChannelMap.set(message.d.id, arr) - for (const channel of message.d.channels || []) { - // @ts-ignore - channel.guild_id = message.d.id - arr.push(channel.id) - client.channels.set(channel.id, channel) - } - for (const thread of message.d.threads || []) { - // @ts-ignore - thread.guild_id = message.d.id - arr.push(thread.id) - client.channels.set(thread.id, thread) + // Client internals, keep track of the state we need + if (message.t === "READY") { + if (client.ready) return + client.ready = true + client.user = message.d.user + client.application = message.d.application + console.log(`Discord logged in as ${client.user.username}#${client.user.discriminator} (${client.user.id})`) + interactions.registerInteractions() + + } else if (message.t === "GUILD_CREATE") { + message.d.members = message.d.members.filter(m => m.user.id === client.user.id) // only keep the bot's own member - it's needed to determine private channels on web + client.guilds.set(message.d.id, message.d) + const arr = [] + client.guildChannelMap.set(message.d.id, arr) + for (const channel of message.d.channels || []) { + // @ts-ignore + channel.guild_id = message.d.id + arr.push(channel.id) + client.channels.set(channel.id, channel) + } + for (const thread of message.d.threads || []) { + // @ts-ignore + thread.guild_id = message.d.id + arr.push(thread.id) + client.channels.set(thread.id, thread) + } + + if (listen === "full") { + try { + await eventDispatcher.checkMissedExpressions(message.d) + await eventDispatcher.checkMissedMessages(client, message.d) + await eventDispatcher.checkMissedPins(client, message.d) + await eventDispatcher.checkMissedLeaves(client, message.d) + } catch (e) { + console.error("Failed to sync missed events. To retry, please fix this error and restart OOYE:") + console.error(e) } + } - if (listen === "full") { - try { - await eventDispatcher.checkMissedExpressions(message.d) - await eventDispatcher.checkMissedMessages(client, message.d) - await eventDispatcher.checkMissedPins(client, message.d) - await eventDispatcher.checkMissedLeaves(client, message.d) - } catch (e) { - console.error("Failed to sync missed events. To retry, please fix this error and restart OOYE:") - console.error(e) - } - } - - } else if (message.t === "GUILD_UPDATE") { - const guild = client.guilds.get(message.d.id) - if (guild) { - for (const prop of Object.keys(message.d)) { - if (!["channels", "threads"].includes(prop)) { - guild[prop] = message.d[prop] - } - } - } - - } else if (message.t === "GUILD_EMOJIS_UPDATE") { - const guild = client.guilds.get(message.d.guild_id) - if (guild) { - guild.emojis = message.d.emojis - } - - } else if (message.t === "GUILD_STICKERS_UPDATE") { - const guild = client.guilds.get(message.d.guild_id) - if (guild) { - guild.stickers = message.d.stickers - } - - } else if (message.t === "GUILD_ROLE_CREATE" || message.t === "GUILD_ROLE_UPDATE" || message.t === "GUILD_ROLE_DELETE") { - const guild = client.guilds.get(message.d.guild_id) - /** Delete this in case of UPDATE or DELETE */ - const targetID = "role_id" in message.d ? message.d.role_id : message.d.role.id - /** Add this in case of CREATE or UPDATE */ - const newRoles = [] - if ("role" in message.d) newRoles.push(message.d.role) - if (guild) { - const targetIndex = guild.roles.findIndex(r => r.id === targetID) - if (targetIndex !== -1) { - // Role already exists. Delete it and maybe replace it. - guild.roles.splice(targetIndex, 1, ...newRoles) - } else { - // Role doesn't already exist. - guild.roles.push(...newRoles) - } - } - - } else if (message.t === "GUILD_MEMBER_UPDATE") { - const guild = client.guilds.get(message.d.guild_id) - const member = guild?.members.find(m => m.user.id === message.d.user.id) - if (member) { // only update existing members (i.e. the bot's own member) - don't want to inflate the cache with new irrelevant ones - Object.assign(member, message.d) - } - - } else if (message.t === "THREAD_CREATE") { - client.channels.set(message.d.id, message.d) - if (message.d["guild_id"]) { - const channels = client.guildChannelMap.get(message.d["guild_id"]) - if (channels && !channels.includes(message.d.id)) channels.push(message.d.id) - } - - } else if (message.t === "CHANNEL_UPDATE" || message.t === "THREAD_UPDATE") { - client.channels.set(message.d.id, message.d) - - - } else if (message.t === "CHANNEL_PINS_UPDATE") { - const channel = client.channels.get(message.d.channel_id) - if (channel) { - channel["last_pin_timestamp"] = message.d.last_pin_timestamp - } - - - } else if (message.t === "GUILD_DELETE") { - client.guilds.delete(message.d.id) - const channels = client.guildChannelMap.get(message.d.id) - if (channels) { - for (const id of channels) client.channels.delete(id) - } - client.guildChannelMap.delete(message.d.id) - - - } else if (message.t === "CHANNEL_CREATE") { - client.channels.set(message.d.id, message.d) - if (message.d["guild_id"]) { // obj[prop] notation can be used to access a property without typescript complaining that it doesn't exist on all values something can have - const channels = client.guildChannelMap.get(message.d["guild_id"]) - if (channels && !channels.includes(message.d.id)) channels.push(message.d.id) - } - - } else if (message.t === "CHANNEL_DELETE") { - client.channels.delete(message.d.id) - if (message.d["guild_id"]) { - const channels = client.guildChannelMap.get(message.d["guild_id"]) - if (channels) { - const previous = channels.indexOf(message.d.id) - if (previous !== -1) channels.splice(previous, 1) + } else if (message.t === "GUILD_UPDATE") { + const guild = client.guilds.get(message.d.id) + if (guild) { + for (const prop of Object.keys(message.d)) { + if (!["channels", "threads"].includes(prop)) { + guild[prop] = message.d[prop] } } } - // Event dispatcher for OOYE bridge operations - if (listen === "full" && message.t) { - try { - if (message.t === "MESSAGE_REACTION_REMOVE" || message.t === "MESSAGE_REACTION_REMOVE_EMOJI" || message.t === "MESSAGE_REACTION_REMOVE_ALL") { - await eventDispatcher.onSomeReactionsRemoved(client, message.d) + } else if (message.t === "GUILD_EMOJIS_UPDATE") { + const guild = client.guilds.get(message.d.guild_id) + if (guild) { + guild.emojis = message.d.emojis + } - } else if (message.t === "INTERACTION_CREATE") { - await interactions.dispatchInteraction(message.d) + } else if (message.t === "GUILD_STICKERS_UPDATE") { + const guild = client.guilds.get(message.d.guild_id) + if (guild) { + guild.stickers = message.d.stickers + } - } else if (message.t in eventDispatcher) { - await eventDispatcher[message.t](client, message.d) - } - } catch (e) { - // Let OOYE try to handle errors too - await eventDispatcher.onError(client, e, message) + } else if (message.t === "GUILD_ROLE_CREATE" || message.t === "GUILD_ROLE_UPDATE" || message.t === "GUILD_ROLE_DELETE") { + const guild = client.guilds.get(message.d.guild_id) + /** Delete this in case of UPDATE or DELETE */ + const targetID = "role_id" in message.d ? message.d.role_id : message.d.role.id + /** Add this in case of CREATE or UPDATE */ + const newRoles = [] + if ("role" in message.d) newRoles.push(message.d.role) + if (guild) { + const targetIndex = guild.roles.findIndex(r => r.id === targetID) + if (targetIndex !== -1) { + // Role already exists. Delete it and maybe replace it. + guild.roles.splice(targetIndex, 1, ...newRoles) + } else { + // Role doesn't already exist. + guild.roles.push(...newRoles) } } + + } else if (message.t === "GUILD_MEMBER_UPDATE") { + const guild = client.guilds.get(message.d.guild_id) + const member = guild?.members.find(m => m.user.id === message.d.user.id) + if (member) { // only update existing members (i.e. the bot's own member) - don't want to inflate the cache with new irrelevant ones + Object.assign(member, message.d) + } + + } else if (message.t === "THREAD_CREATE") { + client.channels.set(message.d.id, message.d) + if (message.d["guild_id"]) { + const channels = client.guildChannelMap.get(message.d["guild_id"]) + if (channels && !channels.includes(message.d.id)) channels.push(message.d.id) + } + + } else if (message.t === "CHANNEL_UPDATE" || message.t === "THREAD_UPDATE") { + client.channels.set(message.d.id, message.d) + + + } else if (message.t === "CHANNEL_PINS_UPDATE") { + const channel = client.channels.get(message.d.channel_id) + if (channel) { + channel["last_pin_timestamp"] = message.d.last_pin_timestamp + } + + + } else if (message.t === "GUILD_DELETE") { + client.guilds.delete(message.d.id) + const channels = client.guildChannelMap.get(message.d.id) + if (channels) { + for (const id of channels) client.channels.delete(id) + } + client.guildChannelMap.delete(message.d.id) + + + } else if (message.t === "CHANNEL_CREATE") { + client.channels.set(message.d.id, message.d) + if (message.d["guild_id"]) { // obj[prop] notation can be used to access a property without typescript complaining that it doesn't exist on all values something can have + const channels = client.guildChannelMap.get(message.d["guild_id"]) + if (channels && !channels.includes(message.d.id)) channels.push(message.d.id) + } + + } else if (message.t === "CHANNEL_DELETE") { + client.channels.delete(message.d.id) + if (message.d["guild_id"]) { + const channels = client.guildChannelMap.get(message.d["guild_id"]) + if (channels) { + const previous = channels.indexOf(message.d.id) + if (previous !== -1) channels.splice(previous, 1) + } + } + } + + // Event dispatcher for OOYE bridge operations + if (listen === "full" && message.t) { + const alwaysRealTimeEvents = ["PRESENCE_UPDATE"] + if (alwaysRealTimeEvents.includes(message.t) || homeserverStatus.homeserverStatus.isRealTime()) { + dispatchPacketToBridge(client, message) + } else { + homeserverStatus.homeserverStatus.queuePacket(message) + } } } -module.exports = utils +/** + * @param {import("./discord-client")} client + * @param {import("cloudstorm").IGatewayMessage} message + */ +async function dispatchPacketToBridge(client, message) { + // requiring this later so that the client is already constructed by the time event-dispatcher is loaded + /** @type {typeof import("./event-dispatcher")} */ + const eventDispatcher = sync.require("./event-dispatcher") + /** @type {import("../discord/register-interactions")} */ + const interactions = sync.require("../discord/register-interactions") + + assert(message.t) // checked above + + try { + if (message.t === "MESSAGE_REACTION_REMOVE" || message.t === "MESSAGE_REACTION_REMOVE_EMOJI" || message.t === "MESSAGE_REACTION_REMOVE_ALL") { + await eventDispatcher.onSomeReactionsRemoved(client, message.d) + + } else if (message.t === "INTERACTION_CREATE") { + await interactions.dispatchInteraction(message.d) + + } else if (message.t in eventDispatcher) { + await eventDispatcher[message.t](client, message.d) + } + } catch (e) { + // Let OOYE try to handle errors too + await eventDispatcher.onError(client, e, message) + } +} + +module.exports.onPacket = onPacket +module.exports.dispatchPacketToBridge = dispatchPacketToBridge diff --git a/src/m2d/event-dispatcher.js b/src/m2d/event-dispatcher.js index 3580d1b..0615743 100644 --- a/src/m2d/event-dispatcher.js +++ b/src/m2d/event-dispatcher.js @@ -24,6 +24,8 @@ const vote = sync.require("./actions/vote") const matrixCommandHandler = sync.require("../matrix/matrix-command-handler") /** @type {import("../matrix/utils")} */ const utils = sync.require("../matrix/utils") +/** @type {import("../matrix/mreq")}) */ +const mreq = sync.require("../matrix/mreq") /** @type {import("../matrix/api")}) */ const api = sync.require("../matrix/api") /** @type {import("../d2m/actions/create-room")} */ @@ -32,6 +34,8 @@ const createRoom = sync.require("../d2m/actions/create-room") const roomUpgrade = require("../matrix/room-upgrade") /** @type {import("../d2m/actions/retrigger")} */ const retrigger = sync.require("../d2m/actions/retrigger") +/** @type {import("../matrix/homeserver-status")} */ +const homeserverStatus = sync.require("../matrix/homeserver-status") const {reg} = require("../matrix/read-registration") let lastReportedEvent = 0 @@ -166,7 +170,12 @@ async function sendError(roomID, source, type, e, payload) { key: "🔁" } }) - } catch (e) {} + } catch (e) { + if (e instanceof mreq.MatrixServerError && [502, 503].includes(e.httpStatus)) { + // Matrix homeserver is down (reverse proxy indicated failure; synapse doesn't generate 502/503 when posting an event to a room) + homeserverStatus.homeserverStatus.setErrorWithPacket(payload) + } + } } function guard(type, fn) { diff --git a/src/matrix/api.js b/src/matrix/api.js index f68380f..47fc565 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -452,7 +452,8 @@ async function ping() { headers: { Authorization: `Bearer ${reg.as_token}` }, - body: "{}" + body: "{}", + signal: AbortSignal.timeout(15e3) }) const root = await res.json() return { diff --git a/src/matrix/homeserver-status.js b/src/matrix/homeserver-status.js new file mode 100644 index 0000000..0c5966f --- /dev/null +++ b/src/matrix/homeserver-status.js @@ -0,0 +1,94 @@ +// @ts-check + +const assert = require("assert") +const Denque = require("denque") +const StateMachine = require("snowtransfer").StateMachine + +const passthrough = require("../passthrough") +const {sync} = passthrough +/** @type {import("../d2m/discord-packets")} */ +const discordPackets = sync.require("../d2m/discord-packets") +/** @type {import("../matrix/api")} */ +const api = sync.require("../matrix/api") + +const DEBUG_HOMESERVER_STATUS = true + +function debugHomeserverStatus(message) { + if (DEBUG_HOMESERVER_STATUS) { + console.log(message) + } +} + +const homeserverStatus = new class HomeserverStatus { + constructor() { + /** @private */ + this.queue = new Denque() + + /** @private */ + this.pingInterval = undefined + + /** @private */ + this.sm = new StateMachine("online") + .defineState("online") + + .defineState("offline", { + onEnter: [() => { + this.pingInterval = setInterval(async () => { + const pingResult = await api.ping().catch(e => ({ok: false, status: "net", root: e.message})) + if (pingResult.ok) { + this.sm.doTransition("ping ok") + } + }, 15e3) + }], + onLeave: [() => { + clearInterval(this.pingInterval) + }], + transitions: new Map() + }) + + .defineState("recovering", { + onEnter: [async () => { // Drain queue. + while (!this.queue.isEmpty()) { + const packet = this.queue.peekFront() // same position as .shift() + debugHomeserverStatus(`homeserver status: ${new Date().toISOString()} dq packet ${packet.t} ${packet.d?.content}`) + await discordPackets.dispatchPacketToBridge(passthrough.discord, packet) + if (this.sm.currentStateName !== "recovering") return // got kicked out due to another error + this.queue.shift() + } + this.sm.doTransition("recovered") + }], + onLeave: [], + transitions: new Map() + }) + + .defineUniversalTransition("error", "offline") + .defineTransition("offline", "ping ok", "recovering") + .defineTransition("recovering", "recovered", "online") + + this.sm.on("enter", st => debugHomeserverStatus(`homeserver status: ${st}`)) + + this.sm.freeze() + } + + isRealTime() { + return this.sm.currentStateName === "online" + } + + /** + * When offline or recovering, call this for incoming packets to queue them to be sent in order later. + */ + queuePacket(packet) { + assert(["offline", "recovering"].includes(this.sm.currentStateName)) + this.queue.push(packet) + } + + setErrorWithPacket(packet) { + const wasRecovering = this.sm.currentStateName === "recovering" + this.sm.doTransition("error") + if (!wasRecovering) { // if was recovering then packet is already in the right place in queue + this.queuePacket(packet) + } + } +} + +module.exports.homeserverStatus = homeserverStatus