Queue events for later if homeserver is down

docs/developer-orientation.md

@@ -113,6 +113,7 @@ Total transitive production dependencies: 144
 * (0) ansi-colors: Helps with interactive prompting for the initial setup, and it's already pulled in by enquirer.
 * (1) chunk-text: It does what I want.
 * (0) cloudstorm: Discord gateway library with bring-your-own-caching that I trust.
+* (0) denque: Much faster than using arrays as a queue.
 * (0) discord-api-types: Bitfields needed at runtime and types needed for development.
 * (0) domino: DOM implementation that's already pulled in by turndown.
 * (2) enquirer: Interactive prompting for the initial setup rather than forcing users to edit YAML non-interactively.

src/d2m/actions/create-space.js

@@ -8,6 +8,8 @@ const {reg} = require("../../matrix/read-registration")
 
 const passthrough = require("../../passthrough")
 const {discord, sync, db, select} = passthrough
+/** @type {import("../../matrix/mreq")} */
+const mreq = sync.require("../../matrix/mreq")
 /** @type {import("../../matrix/api")} */
 const api = sync.require("../../matrix/api")
 /** @type {import("../../matrix/file")} */
@@ -237,8 +239,12 @@ async function syncSpaceExpressions(data, checkBeforeSync) {
 			try {
 				existing = await api.getStateEvent(spaceID, "im.ponies.room_emotes", eventKey)
 			} catch (e) {
-				// State event not found. This space doesn't have any existing emojis. We create a dummy empty event for comparison's sake.
+				if (e instanceof mreq.MatrixServerError && e.httpStatus < 400) {
-				existing = fn([], guild)
+					// State event not found. This space doesn't have any existing emojis. We create a dummy empty event for comparison's sake.
+					existing = fn([], guild)
+				} else {
+					throw e
+				}
 			}
 			if (isDeepStrictEqual(existing, content)) return
 		}

src/d2m/discord-packets.js

@@ -1,173 +1,194 @@
 // @ts-check
 
-// Discord library internals type beat
+const assert = require("assert")
-
-const DiscordTypes = require("discord-api-types/v10")
 const passthrough = require("../passthrough")
-const {sync, db} = passthrough
+const {sync} = passthrough
 
-const utils = {
+/** @type {import("../matrix/homeserver-status")} */
-	/**
+const homeserverStatus = sync.require("../matrix/homeserver-status")
-	 * @param {import("./discord-client")} client
-	 * @param {import("cloudstorm").IGatewayMessage} message
-	 * @param {string} listen "full", "half", "no" - whether to set up the event listeners for OOYE to operate
-	 */
-	async onPacket(client, message, listen) {
-		// requiring this later so that the client is already constructed by the time event-dispatcher is loaded
-		/** @type {typeof import("./event-dispatcher")} */
-		const eventDispatcher = sync.require("./event-dispatcher")
-		/** @type {import("../discord/register-interactions")} */
-		const interactions = sync.require("../discord/register-interactions")
 
-		// Client internals, keep track of the state we need
+/**
-		if (message.t === "READY") {
+ * @param {import("./discord-client")} client
-			if (client.ready) return
+ * @param {import("cloudstorm").IGatewayMessage} message
-			client.ready = true
+ * @param {string} listen "full", "half", "no" - whether to set up the event listeners for OOYE to operate
-			client.user = message.d.user
+ */
-			client.application = message.d.application
+async function onPacket(client, message, listen) {
-			console.log(`Discord logged in as ${client.user.username}#${client.user.discriminator} (${client.user.id})`)
+	// requiring this later so that the client is already constructed by the time event-dispatcher is loaded
-			interactions.registerInteractions()
+	/** @type {typeof import("./event-dispatcher")} */
+	const eventDispatcher = sync.require("./event-dispatcher")
+	/** @type {import("../discord/register-interactions")} */
+	const interactions = sync.require("../discord/register-interactions")
 
-		} else if (message.t === "GUILD_CREATE") {
+	// Client internals, keep track of the state we need
-			message.d.members = message.d.members.filter(m => m.user.id === client.user.id) // only keep the bot's own member - it's needed to determine private channels on web
+	if (message.t === "READY") {
-			client.guilds.set(message.d.id, message.d)
+		if (client.ready) return
-			const arr = []
+		client.ready = true
-			client.guildChannelMap.set(message.d.id, arr)
+		client.user = message.d.user
-			for (const channel of message.d.channels || []) {
+		client.application = message.d.application
-				// @ts-ignore
+		console.log(`Discord logged in as ${client.user.username}#${client.user.discriminator} (${client.user.id})`)
-				channel.guild_id = message.d.id
+		interactions.registerInteractions()
-				arr.push(channel.id)
+
-				client.channels.set(channel.id, channel)
+	} else if (message.t === "GUILD_CREATE") {
-			}
+		message.d.members = message.d.members.filter(m => m.user.id === client.user.id) // only keep the bot's own member - it's needed to determine private channels on web
-			for (const thread of message.d.threads || []) {
+		client.guilds.set(message.d.id, message.d)
-				// @ts-ignore
+		const arr = []
-				thread.guild_id = message.d.id
+		client.guildChannelMap.set(message.d.id, arr)
-				arr.push(thread.id)
+		for (const channel of message.d.channels || []) {
-				client.channels.set(thread.id, thread)
+			// @ts-ignore
+			channel.guild_id = message.d.id
+			arr.push(channel.id)
+			client.channels.set(channel.id, channel)
+		}
+		for (const thread of message.d.threads || []) {
+			// @ts-ignore
+			thread.guild_id = message.d.id
+			arr.push(thread.id)
+			client.channels.set(thread.id, thread)
+		}
+
+		if (listen === "full") {
+			try {
+				await eventDispatcher.checkMissedExpressions(message.d)
+				await eventDispatcher.checkMissedMessages(client, message.d)
+				await eventDispatcher.checkMissedPins(client, message.d)
+				await eventDispatcher.checkMissedLeaves(client, message.d)
+			} catch (e) {
+				console.error("Failed to sync missed events. To retry, please fix this error and restart OOYE:")
+				console.error(e)
 			}
+		}
 
-			if (listen === "full") {
+	} else if (message.t === "GUILD_UPDATE") {
-				try {
+		const guild = client.guilds.get(message.d.id)
-					await eventDispatcher.checkMissedExpressions(message.d)
+		if (guild) {
-					await eventDispatcher.checkMissedMessages(client, message.d)
+			for (const prop of Object.keys(message.d)) {
-					await eventDispatcher.checkMissedPins(client, message.d)
+				if (!["channels", "threads"].includes(prop)) {
-					await eventDispatcher.checkMissedLeaves(client, message.d)
+					guild[prop] = message.d[prop]
-				} catch (e) {
-					console.error("Failed to sync missed events. To retry, please fix this error and restart OOYE:")
-					console.error(e)
-				}
-			}
-
-		} else if (message.t === "GUILD_UPDATE") {
-			const guild = client.guilds.get(message.d.id)
-			if (guild) {
-				for (const prop of Object.keys(message.d)) {
-					if (!["channels", "threads"].includes(prop)) {
-						guild[prop] = message.d[prop]
-					}
-				}
-			}
-
-		} else if (message.t === "GUILD_EMOJIS_UPDATE") {
-			const guild = client.guilds.get(message.d.guild_id)
-			if (guild) {
-				guild.emojis = message.d.emojis
-			}
-
-		} else if (message.t === "GUILD_STICKERS_UPDATE") {
-			const guild = client.guilds.get(message.d.guild_id)
-			if (guild) {
-				guild.stickers = message.d.stickers
-			}
-
-		} else if (message.t === "GUILD_ROLE_CREATE" || message.t === "GUILD_ROLE_UPDATE" || message.t === "GUILD_ROLE_DELETE") {
-			const guild = client.guilds.get(message.d.guild_id)
-			/** Delete this in case of UPDATE or DELETE */
-			const targetID = "role_id" in message.d ? message.d.role_id : message.d.role.id
-			/** Add this in case of CREATE or UPDATE */
-			const newRoles = []
-			if ("role" in message.d) newRoles.push(message.d.role)
-			if (guild) {
-				const targetIndex = guild.roles.findIndex(r => r.id === targetID)
-				if (targetIndex !== -1) {
-					// Role already exists. Delete it and maybe replace it.
-					guild.roles.splice(targetIndex, 1, ...newRoles)
-				} else {
-					// Role doesn't already exist.
-					guild.roles.push(...newRoles)
-				}
-			}
-
-		} else if (message.t === "GUILD_MEMBER_UPDATE") {
-			const guild = client.guilds.get(message.d.guild_id)
-			const member = guild?.members.find(m => m.user.id === message.d.user.id)
-			if (member) { // only update existing members (i.e. the bot's own member) - don't want to inflate the cache with new irrelevant ones
-				Object.assign(member, message.d)
-			}
-
-		} else if (message.t === "THREAD_CREATE") {
-			client.channels.set(message.d.id, message.d)
-			if (message.d["guild_id"]) {
-				const channels = client.guildChannelMap.get(message.d["guild_id"])
-				if (channels && !channels.includes(message.d.id)) channels.push(message.d.id)
-			}
-
-		} else if (message.t === "CHANNEL_UPDATE" || message.t === "THREAD_UPDATE") {
-			client.channels.set(message.d.id, message.d)
-
-
-		} else if (message.t === "CHANNEL_PINS_UPDATE") {
-			const channel = client.channels.get(message.d.channel_id)
-			if (channel) {
-				channel["last_pin_timestamp"] = message.d.last_pin_timestamp
-			}
-
-
-		} else if (message.t === "GUILD_DELETE") {
-			client.guilds.delete(message.d.id)
-			const channels = client.guildChannelMap.get(message.d.id)
-			if (channels) {
-				for (const id of channels) client.channels.delete(id)
-			}
-			client.guildChannelMap.delete(message.d.id)
-
-
-		} else if (message.t === "CHANNEL_CREATE") {
-			client.channels.set(message.d.id, message.d)
-			if (message.d["guild_id"]) { // obj[prop] notation can be used to access a property without typescript complaining that it doesn't exist on all values something can have
-				const channels = client.guildChannelMap.get(message.d["guild_id"])
-				if (channels && !channels.includes(message.d.id)) channels.push(message.d.id)
-			}
-
-		} else if (message.t === "CHANNEL_DELETE") {
-			client.channels.delete(message.d.id)
-			if (message.d["guild_id"]) {
-				const channels = client.guildChannelMap.get(message.d["guild_id"])
-				if (channels) {
-					const previous = channels.indexOf(message.d.id)
-					if (previous !== -1) channels.splice(previous, 1)
 				}
 			}
 		}
 
-		// Event dispatcher for OOYE bridge operations
+	} else if (message.t === "GUILD_EMOJIS_UPDATE") {
-		if (listen === "full" && message.t) {
+		const guild = client.guilds.get(message.d.guild_id)
-			try {
+		if (guild) {
-				if (message.t === "MESSAGE_REACTION_REMOVE" || message.t === "MESSAGE_REACTION_REMOVE_EMOJI" || message.t === "MESSAGE_REACTION_REMOVE_ALL") {
+			guild.emojis = message.d.emojis
-					await eventDispatcher.onSomeReactionsRemoved(client, message.d)
+		}
 
-				} else if (message.t === "INTERACTION_CREATE") {
+	} else if (message.t === "GUILD_STICKERS_UPDATE") {
-					await interactions.dispatchInteraction(message.d)
+		const guild = client.guilds.get(message.d.guild_id)
+		if (guild) {
+			guild.stickers = message.d.stickers
+		}
 
-				} else if (message.t in eventDispatcher) {
+	} else if (message.t === "GUILD_ROLE_CREATE" || message.t === "GUILD_ROLE_UPDATE" || message.t === "GUILD_ROLE_DELETE") {
-					await eventDispatcher[message.t](client, message.d)
+		const guild = client.guilds.get(message.d.guild_id)
-				}
+		/** Delete this in case of UPDATE or DELETE */
-			} catch (e) {
+		const targetID = "role_id" in message.d ? message.d.role_id : message.d.role.id
-				// Let OOYE try to handle errors too
+		/** Add this in case of CREATE or UPDATE */
-				await eventDispatcher.onError(client, e, message)
+		const newRoles = []
+		if ("role" in message.d) newRoles.push(message.d.role)
+		if (guild) {
+			const targetIndex = guild.roles.findIndex(r => r.id === targetID)
+			if (targetIndex !== -1) {
+				// Role already exists. Delete it and maybe replace it.
+				guild.roles.splice(targetIndex, 1, ...newRoles)
+			} else {
+				// Role doesn't already exist.
+				guild.roles.push(...newRoles)
 			}
 		}
+
+	} else if (message.t === "GUILD_MEMBER_UPDATE") {
+		const guild = client.guilds.get(message.d.guild_id)
+		const member = guild?.members.find(m => m.user.id === message.d.user.id)
+		if (member) { // only update existing members (i.e. the bot's own member) - don't want to inflate the cache with new irrelevant ones
+			Object.assign(member, message.d)
+		}
+
+	} else if (message.t === "THREAD_CREATE") {
+		client.channels.set(message.d.id, message.d)
+		if (message.d["guild_id"]) {
+			const channels = client.guildChannelMap.get(message.d["guild_id"])
+			if (channels && !channels.includes(message.d.id)) channels.push(message.d.id)
+		}
+
+	} else if (message.t === "CHANNEL_UPDATE" || message.t === "THREAD_UPDATE") {
+		client.channels.set(message.d.id, message.d)
+
+
+	} else if (message.t === "CHANNEL_PINS_UPDATE") {
+		const channel = client.channels.get(message.d.channel_id)
+		if (channel) {
+			channel["last_pin_timestamp"] = message.d.last_pin_timestamp
+		}
+
+
+	} else if (message.t === "GUILD_DELETE") {
+		client.guilds.delete(message.d.id)
+		const channels = client.guildChannelMap.get(message.d.id)
+		if (channels) {
+			for (const id of channels) client.channels.delete(id)
+		}
+		client.guildChannelMap.delete(message.d.id)
+
+
+	} else if (message.t === "CHANNEL_CREATE") {
+		client.channels.set(message.d.id, message.d)
+		if (message.d["guild_id"]) { // obj[prop] notation can be used to access a property without typescript complaining that it doesn't exist on all values something can have
+			const channels = client.guildChannelMap.get(message.d["guild_id"])
+			if (channels && !channels.includes(message.d.id)) channels.push(message.d.id)
+		}
+
+	} else if (message.t === "CHANNEL_DELETE") {
+		client.channels.delete(message.d.id)
+		if (message.d["guild_id"]) {
+			const channels = client.guildChannelMap.get(message.d["guild_id"])
+			if (channels) {
+				const previous = channels.indexOf(message.d.id)
+				if (previous !== -1) channels.splice(previous, 1)
+			}
+		}
+	}
+
+	// Event dispatcher for OOYE bridge operations
+	if (listen === "full" && message.t) {
+		const alwaysRealTimeEvents = ["PRESENCE_UPDATE"]
+		if (alwaysRealTimeEvents.includes(message.t) || homeserverStatus.homeserverStatus.isRealTime()) {
+			dispatchPacketToBridge(client, message)
+		} else {
+			homeserverStatus.homeserverStatus.queuePacket(message)
+		}
 	}
 }
 
-module.exports = utils
+/**
+ * @param {import("./discord-client")} client
+ * @param {import("cloudstorm").IGatewayMessage} message
+ */
+async function dispatchPacketToBridge(client, message) {
+	// requiring this later so that the client is already constructed by the time event-dispatcher is loaded
+	/** @type {typeof import("./event-dispatcher")} */
+	const eventDispatcher = sync.require("./event-dispatcher")
+	/** @type {import("../discord/register-interactions")} */
+	const interactions = sync.require("../discord/register-interactions")
+
+	assert(message.t) // checked above
+
+	try {
+		if (message.t === "MESSAGE_REACTION_REMOVE" || message.t === "MESSAGE_REACTION_REMOVE_EMOJI" || message.t === "MESSAGE_REACTION_REMOVE_ALL") {
+			await eventDispatcher.onSomeReactionsRemoved(client, message.d)
+
+		} else if (message.t === "INTERACTION_CREATE") {
+			await interactions.dispatchInteraction(message.d)
+
+		} else if (message.t in eventDispatcher) {
+			await eventDispatcher[message.t](client, message.d)
+		}
+	} catch (e) {
+		// Let OOYE try to handle errors too
+		await eventDispatcher.onError(client, e, message)
+	}
+}
+
+module.exports.onPacket = onPacket
+module.exports.dispatchPacketToBridge = dispatchPacketToBridge

src/m2d/event-dispatcher.js

@@ -24,6 +24,8 @@ const vote = sync.require("./actions/vote")
 const matrixCommandHandler = sync.require("../matrix/matrix-command-handler")
 /** @type {import("../matrix/utils")} */
 const utils = sync.require("../matrix/utils")
+/** @type {import("../matrix/mreq")}) */
+const mreq = sync.require("../matrix/mreq")
 /** @type {import("../matrix/api")}) */
 const api = sync.require("../matrix/api")
 /** @type {import("../d2m/actions/create-room")} */
@@ -32,6 +34,8 @@ const createRoom = sync.require("../d2m/actions/create-room")
 const roomUpgrade = require("../matrix/room-upgrade")
 /** @type {import("../d2m/actions/retrigger")} */
 const retrigger = sync.require("../d2m/actions/retrigger")
+/** @type {import("../matrix/homeserver-status")} */
+const homeserverStatus = sync.require("../matrix/homeserver-status")
 const {reg} = require("../matrix/read-registration")
 
 let lastReportedEvent = 0
@@ -166,7 +170,12 @@ async function sendError(roomID, source, type, e, payload) {
 				key: "🔁"
 			}
 		})
-	} catch (e) {}
+	} catch (e) {
+		if (e instanceof mreq.MatrixServerError && [502, 503].includes(e.httpStatus)) {
+			// Matrix homeserver is down (reverse proxy indicated failure; synapse doesn't generate 502/503 when posting an event to a room)
+			homeserverStatus.homeserverStatus.setErrorWithPacket(payload)
+		}
+	}
 }
 
 function guard(type, fn) {

src/matrix/api.js

@@ -452,7 +452,8 @@ async function ping() {
 		headers: {
 			Authorization: `Bearer ${reg.as_token}`
 		},
-		body: "{}"
+		body: "{}",
+		signal: AbortSignal.timeout(15e3)
 	})
 	const root = await res.json()
 	return {

src/matrix/homeserver-status.js

@@ -0,0 +1,94 @@
+// @ts-check
+
+const assert = require("assert")
+const Denque = require("denque")
+const StateMachine = require("snowtransfer").StateMachine
+
+const passthrough = require("../passthrough")
+const {sync} = passthrough
+/** @type {import("../d2m/discord-packets")} */
+const discordPackets = sync.require("../d2m/discord-packets")
+/** @type {import("../matrix/api")} */
+const api = sync.require("../matrix/api")
+
+const DEBUG_HOMESERVER_STATUS = true
+
+function debugHomeserverStatus(message) {
+	if (DEBUG_HOMESERVER_STATUS) {
+		console.log(message)
+	}
+}
+
+const homeserverStatus = new class HomeserverStatus {
+	constructor() {
+		/** @private */
+		this.queue = new Denque()
+
+		/** @private */
+		this.pingInterval = undefined
+
+		/** @private */
+		this.sm = new StateMachine("online")
+			.defineState("online")
+
+			.defineState("offline", {
+				onEnter: [() => {
+					this.pingInterval = setInterval(async () => {
+						const pingResult = await api.ping().catch(e => ({ok: false, status: "net", root: e.message}))
+						if (pingResult.ok) {
+							this.sm.doTransition("ping ok")
+						}
+					}, 15e3)
+				}],
+				onLeave: [() => {
+					clearInterval(this.pingInterval)
+				}],
+				transitions: new Map()
+			})
+
+			.defineState("recovering", {
+				onEnter: [async () => { // Drain queue.
+					while (!this.queue.isEmpty()) {
+						const packet = this.queue.peekFront() // same position as .shift()
+						debugHomeserverStatus(`homeserver status: ${new Date().toISOString()} dq packet ${packet.t} ${packet.d?.content}`)
+						await discordPackets.dispatchPacketToBridge(passthrough.discord, packet)
+						if (this.sm.currentStateName !== "recovering") return // got kicked out due to another error
+						this.queue.shift()
+					}
+					this.sm.doTransition("recovered")
+				}],
+				onLeave: [],
+				transitions: new Map()
+			})
+
+			.defineUniversalTransition("error", "offline")
+			.defineTransition("offline", "ping ok", "recovering")
+			.defineTransition("recovering", "recovered", "online")
+
+		this.sm.on("enter", st => debugHomeserverStatus(`homeserver status: ${st}`))
+
+		this.sm.freeze()
+	}
+
+	isRealTime() {
+		return this.sm.currentStateName === "online"
+	}
+
+	/**
+	 * When offline or recovering, call this for incoming packets to queue them to be sent in order later.
+	 */
+	queuePacket(packet) {
+		assert(["offline", "recovering"].includes(this.sm.currentStateName))
+		this.queue.push(packet)
+	}
+
+	setErrorWithPacket(packet) {
+		const wasRecovering = this.sm.currentStateName === "recovering"
+		this.sm.doTransition("error")
+		if (!wasRecovering) { // if was recovering then packet is already in the right place in queue
+			this.queuePacket(packet)
+		}
+	}
+}
+
+module.exports.homeserverStatus = homeserverStatus