Queue events for later if homeserver is down

This commit is contained in:
Cadence Ember
2026-07-12 02:12:53 +12:00
parent 8779b8b1b0
commit 414da4caf4
6 changed files with 290 additions and 158 deletions
+1
View File
@@ -113,6 +113,7 @@ Total transitive production dependencies: 144
* (0) ansi-colors: Helps with interactive prompting for the initial setup, and it's already pulled in by enquirer.
* (1) chunk-text: It does what I want.
* (0) cloudstorm: Discord gateway library with bring-your-own-caching that I trust.
* (0) denque: Much faster than using arrays as a queue.
* (0) discord-api-types: Bitfields needed at runtime and types needed for development.
* (0) domino: DOM implementation that's already pulled in by turndown.
* (2) enquirer: Interactive prompting for the initial setup rather than forcing users to edit YAML non-interactively.
+8 -2
View File
@@ -8,6 +8,8 @@ const {reg} = require("../../matrix/read-registration")
const passthrough = require("../../passthrough")
const {discord, sync, db, select} = passthrough
/** @type {import("../../matrix/mreq")} */
const mreq = sync.require("../../matrix/mreq")
/** @type {import("../../matrix/api")} */
const api = sync.require("../../matrix/api")
/** @type {import("../../matrix/file")} */
@@ -237,8 +239,12 @@ async function syncSpaceExpressions(data, checkBeforeSync) {
try {
existing = await api.getStateEvent(spaceID, "im.ponies.room_emotes", eventKey)
} catch (e) {
// State event not found. This space doesn't have any existing emojis. We create a dummy empty event for comparison's sake.
existing = fn([], guild)
if (e instanceof mreq.MatrixServerError && e.httpStatus < 400) {
// State event not found. This space doesn't have any existing emojis. We create a dummy empty event for comparison's sake.
existing = fn([], guild)
} else {
throw e
}
}
if (isDeepStrictEqual(existing, content)) return
}
+175 -154
View File
@@ -1,173 +1,194 @@
// @ts-check
// Discord library internals type beat
const DiscordTypes = require("discord-api-types/v10")
const assert = require("assert")
const passthrough = require("../passthrough")
const {sync, db} = passthrough
const {sync} = passthrough
const utils = {
/**
* @param {import("./discord-client")} client
* @param {import("cloudstorm").IGatewayMessage} message
* @param {string} listen "full", "half", "no" - whether to set up the event listeners for OOYE to operate
*/
async onPacket(client, message, listen) {
// requiring this later so that the client is already constructed by the time event-dispatcher is loaded
/** @type {typeof import("./event-dispatcher")} */
const eventDispatcher = sync.require("./event-dispatcher")
/** @type {import("../discord/register-interactions")} */
const interactions = sync.require("../discord/register-interactions")
/** @type {import("../matrix/homeserver-status")} */
const homeserverStatus = sync.require("../matrix/homeserver-status")
// Client internals, keep track of the state we need
if (message.t === "READY") {
if (client.ready) return
client.ready = true
client.user = message.d.user
client.application = message.d.application
console.log(`Discord logged in as ${client.user.username}#${client.user.discriminator} (${client.user.id})`)
interactions.registerInteractions()
/**
* @param {import("./discord-client")} client
* @param {import("cloudstorm").IGatewayMessage} message
* @param {string} listen "full", "half", "no" - whether to set up the event listeners for OOYE to operate
*/
async function onPacket(client, message, listen) {
// requiring this later so that the client is already constructed by the time event-dispatcher is loaded
/** @type {typeof import("./event-dispatcher")} */
const eventDispatcher = sync.require("./event-dispatcher")
/** @type {import("../discord/register-interactions")} */
const interactions = sync.require("../discord/register-interactions")
} else if (message.t === "GUILD_CREATE") {
message.d.members = message.d.members.filter(m => m.user.id === client.user.id) // only keep the bot's own member - it's needed to determine private channels on web
client.guilds.set(message.d.id, message.d)
const arr = []
client.guildChannelMap.set(message.d.id, arr)
for (const channel of message.d.channels || []) {
// @ts-ignore
channel.guild_id = message.d.id
arr.push(channel.id)
client.channels.set(channel.id, channel)
}
for (const thread of message.d.threads || []) {
// @ts-ignore
thread.guild_id = message.d.id
arr.push(thread.id)
client.channels.set(thread.id, thread)
// Client internals, keep track of the state we need
if (message.t === "READY") {
if (client.ready) return
client.ready = true
client.user = message.d.user
client.application = message.d.application
console.log(`Discord logged in as ${client.user.username}#${client.user.discriminator} (${client.user.id})`)
interactions.registerInteractions()
} else if (message.t === "GUILD_CREATE") {
message.d.members = message.d.members.filter(m => m.user.id === client.user.id) // only keep the bot's own member - it's needed to determine private channels on web
client.guilds.set(message.d.id, message.d)
const arr = []
client.guildChannelMap.set(message.d.id, arr)
for (const channel of message.d.channels || []) {
// @ts-ignore
channel.guild_id = message.d.id
arr.push(channel.id)
client.channels.set(channel.id, channel)
}
for (const thread of message.d.threads || []) {
// @ts-ignore
thread.guild_id = message.d.id
arr.push(thread.id)
client.channels.set(thread.id, thread)
}
if (listen === "full") {
try {
await eventDispatcher.checkMissedExpressions(message.d)
await eventDispatcher.checkMissedMessages(client, message.d)
await eventDispatcher.checkMissedPins(client, message.d)
await eventDispatcher.checkMissedLeaves(client, message.d)
} catch (e) {
console.error("Failed to sync missed events. To retry, please fix this error and restart OOYE:")
console.error(e)
}
}
if (listen === "full") {
try {
await eventDispatcher.checkMissedExpressions(message.d)
await eventDispatcher.checkMissedMessages(client, message.d)
await eventDispatcher.checkMissedPins(client, message.d)
await eventDispatcher.checkMissedLeaves(client, message.d)
} catch (e) {
console.error("Failed to sync missed events. To retry, please fix this error and restart OOYE:")
console.error(e)
}
}
} else if (message.t === "GUILD_UPDATE") {
const guild = client.guilds.get(message.d.id)
if (guild) {
for (const prop of Object.keys(message.d)) {
if (!["channels", "threads"].includes(prop)) {
guild[prop] = message.d[prop]
}
}
}
} else if (message.t === "GUILD_EMOJIS_UPDATE") {
const guild = client.guilds.get(message.d.guild_id)
if (guild) {
guild.emojis = message.d.emojis
}
} else if (message.t === "GUILD_STICKERS_UPDATE") {
const guild = client.guilds.get(message.d.guild_id)
if (guild) {
guild.stickers = message.d.stickers
}
} else if (message.t === "GUILD_ROLE_CREATE" || message.t === "GUILD_ROLE_UPDATE" || message.t === "GUILD_ROLE_DELETE") {
const guild = client.guilds.get(message.d.guild_id)
/** Delete this in case of UPDATE or DELETE */
const targetID = "role_id" in message.d ? message.d.role_id : message.d.role.id
/** Add this in case of CREATE or UPDATE */
const newRoles = []
if ("role" in message.d) newRoles.push(message.d.role)
if (guild) {
const targetIndex = guild.roles.findIndex(r => r.id === targetID)
if (targetIndex !== -1) {
// Role already exists. Delete it and maybe replace it.
guild.roles.splice(targetIndex, 1, ...newRoles)
} else {
// Role doesn't already exist.
guild.roles.push(...newRoles)
}
}
} else if (message.t === "GUILD_MEMBER_UPDATE") {
const guild = client.guilds.get(message.d.guild_id)
const member = guild?.members.find(m => m.user.id === message.d.user.id)
if (member) { // only update existing members (i.e. the bot's own member) - don't want to inflate the cache with new irrelevant ones
Object.assign(member, message.d)
}
} else if (message.t === "THREAD_CREATE") {
client.channels.set(message.d.id, message.d)
if (message.d["guild_id"]) {
const channels = client.guildChannelMap.get(message.d["guild_id"])
if (channels && !channels.includes(message.d.id)) channels.push(message.d.id)
}
} else if (message.t === "CHANNEL_UPDATE" || message.t === "THREAD_UPDATE") {
client.channels.set(message.d.id, message.d)
} else if (message.t === "CHANNEL_PINS_UPDATE") {
const channel = client.channels.get(message.d.channel_id)
if (channel) {
channel["last_pin_timestamp"] = message.d.last_pin_timestamp
}
} else if (message.t === "GUILD_DELETE") {
client.guilds.delete(message.d.id)
const channels = client.guildChannelMap.get(message.d.id)
if (channels) {
for (const id of channels) client.channels.delete(id)
}
client.guildChannelMap.delete(message.d.id)
} else if (message.t === "CHANNEL_CREATE") {
client.channels.set(message.d.id, message.d)
if (message.d["guild_id"]) { // obj[prop] notation can be used to access a property without typescript complaining that it doesn't exist on all values something can have
const channels = client.guildChannelMap.get(message.d["guild_id"])
if (channels && !channels.includes(message.d.id)) channels.push(message.d.id)
}
} else if (message.t === "CHANNEL_DELETE") {
client.channels.delete(message.d.id)
if (message.d["guild_id"]) {
const channels = client.guildChannelMap.get(message.d["guild_id"])
if (channels) {
const previous = channels.indexOf(message.d.id)
if (previous !== -1) channels.splice(previous, 1)
} else if (message.t === "GUILD_UPDATE") {
const guild = client.guilds.get(message.d.id)
if (guild) {
for (const prop of Object.keys(message.d)) {
if (!["channels", "threads"].includes(prop)) {
guild[prop] = message.d[prop]
}
}
}
// Event dispatcher for OOYE bridge operations
if (listen === "full" && message.t) {
try {
if (message.t === "MESSAGE_REACTION_REMOVE" || message.t === "MESSAGE_REACTION_REMOVE_EMOJI" || message.t === "MESSAGE_REACTION_REMOVE_ALL") {
await eventDispatcher.onSomeReactionsRemoved(client, message.d)
} else if (message.t === "GUILD_EMOJIS_UPDATE") {
const guild = client.guilds.get(message.d.guild_id)
if (guild) {
guild.emojis = message.d.emojis
}
} else if (message.t === "INTERACTION_CREATE") {
await interactions.dispatchInteraction(message.d)
} else if (message.t === "GUILD_STICKERS_UPDATE") {
const guild = client.guilds.get(message.d.guild_id)
if (guild) {
guild.stickers = message.d.stickers
}
} else if (message.t in eventDispatcher) {
await eventDispatcher[message.t](client, message.d)
}
} catch (e) {
// Let OOYE try to handle errors too
await eventDispatcher.onError(client, e, message)
} else if (message.t === "GUILD_ROLE_CREATE" || message.t === "GUILD_ROLE_UPDATE" || message.t === "GUILD_ROLE_DELETE") {
const guild = client.guilds.get(message.d.guild_id)
/** Delete this in case of UPDATE or DELETE */
const targetID = "role_id" in message.d ? message.d.role_id : message.d.role.id
/** Add this in case of CREATE or UPDATE */
const newRoles = []
if ("role" in message.d) newRoles.push(message.d.role)
if (guild) {
const targetIndex = guild.roles.findIndex(r => r.id === targetID)
if (targetIndex !== -1) {
// Role already exists. Delete it and maybe replace it.
guild.roles.splice(targetIndex, 1, ...newRoles)
} else {
// Role doesn't already exist.
guild.roles.push(...newRoles)
}
}
} else if (message.t === "GUILD_MEMBER_UPDATE") {
const guild = client.guilds.get(message.d.guild_id)
const member = guild?.members.find(m => m.user.id === message.d.user.id)
if (member) { // only update existing members (i.e. the bot's own member) - don't want to inflate the cache with new irrelevant ones
Object.assign(member, message.d)
}
} else if (message.t === "THREAD_CREATE") {
client.channels.set(message.d.id, message.d)
if (message.d["guild_id"]) {
const channels = client.guildChannelMap.get(message.d["guild_id"])
if (channels && !channels.includes(message.d.id)) channels.push(message.d.id)
}
} else if (message.t === "CHANNEL_UPDATE" || message.t === "THREAD_UPDATE") {
client.channels.set(message.d.id, message.d)
} else if (message.t === "CHANNEL_PINS_UPDATE") {
const channel = client.channels.get(message.d.channel_id)
if (channel) {
channel["last_pin_timestamp"] = message.d.last_pin_timestamp
}
} else if (message.t === "GUILD_DELETE") {
client.guilds.delete(message.d.id)
const channels = client.guildChannelMap.get(message.d.id)
if (channels) {
for (const id of channels) client.channels.delete(id)
}
client.guildChannelMap.delete(message.d.id)
} else if (message.t === "CHANNEL_CREATE") {
client.channels.set(message.d.id, message.d)
if (message.d["guild_id"]) { // obj[prop] notation can be used to access a property without typescript complaining that it doesn't exist on all values something can have
const channels = client.guildChannelMap.get(message.d["guild_id"])
if (channels && !channels.includes(message.d.id)) channels.push(message.d.id)
}
} else if (message.t === "CHANNEL_DELETE") {
client.channels.delete(message.d.id)
if (message.d["guild_id"]) {
const channels = client.guildChannelMap.get(message.d["guild_id"])
if (channels) {
const previous = channels.indexOf(message.d.id)
if (previous !== -1) channels.splice(previous, 1)
}
}
}
// Event dispatcher for OOYE bridge operations
if (listen === "full" && message.t) {
const alwaysRealTimeEvents = ["PRESENCE_UPDATE"]
if (alwaysRealTimeEvents.includes(message.t) || homeserverStatus.homeserverStatus.isRealTime()) {
dispatchPacketToBridge(client, message)
} else {
homeserverStatus.homeserverStatus.queuePacket(message)
}
}
}
module.exports = utils
/**
* @param {import("./discord-client")} client
* @param {import("cloudstorm").IGatewayMessage} message
*/
async function dispatchPacketToBridge(client, message) {
// requiring this later so that the client is already constructed by the time event-dispatcher is loaded
/** @type {typeof import("./event-dispatcher")} */
const eventDispatcher = sync.require("./event-dispatcher")
/** @type {import("../discord/register-interactions")} */
const interactions = sync.require("../discord/register-interactions")
assert(message.t) // checked above
try {
if (message.t === "MESSAGE_REACTION_REMOVE" || message.t === "MESSAGE_REACTION_REMOVE_EMOJI" || message.t === "MESSAGE_REACTION_REMOVE_ALL") {
await eventDispatcher.onSomeReactionsRemoved(client, message.d)
} else if (message.t === "INTERACTION_CREATE") {
await interactions.dispatchInteraction(message.d)
} else if (message.t in eventDispatcher) {
await eventDispatcher[message.t](client, message.d)
}
} catch (e) {
// Let OOYE try to handle errors too
await eventDispatcher.onError(client, e, message)
}
}
module.exports.onPacket = onPacket
module.exports.dispatchPacketToBridge = dispatchPacketToBridge
+10 -1
View File
@@ -24,6 +24,8 @@ const vote = sync.require("./actions/vote")
const matrixCommandHandler = sync.require("../matrix/matrix-command-handler")
/** @type {import("../matrix/utils")} */
const utils = sync.require("../matrix/utils")
/** @type {import("../matrix/mreq")}) */
const mreq = sync.require("../matrix/mreq")
/** @type {import("../matrix/api")}) */
const api = sync.require("../matrix/api")
/** @type {import("../d2m/actions/create-room")} */
@@ -32,6 +34,8 @@ const createRoom = sync.require("../d2m/actions/create-room")
const roomUpgrade = require("../matrix/room-upgrade")
/** @type {import("../d2m/actions/retrigger")} */
const retrigger = sync.require("../d2m/actions/retrigger")
/** @type {import("../matrix/homeserver-status")} */
const homeserverStatus = sync.require("../matrix/homeserver-status")
const {reg} = require("../matrix/read-registration")
let lastReportedEvent = 0
@@ -166,7 +170,12 @@ async function sendError(roomID, source, type, e, payload) {
key: "🔁"
}
})
} catch (e) {}
} catch (e) {
if (e instanceof mreq.MatrixServerError && [502, 503].includes(e.httpStatus)) {
// Matrix homeserver is down (reverse proxy indicated failure; synapse doesn't generate 502/503 when posting an event to a room)
homeserverStatus.homeserverStatus.setErrorWithPacket(payload)
}
}
}
function guard(type, fn) {
+2 -1
View File
@@ -452,7 +452,8 @@ async function ping() {
headers: {
Authorization: `Bearer ${reg.as_token}`
},
body: "{}"
body: "{}",
signal: AbortSignal.timeout(15e3)
})
const root = await res.json()
return {
+94
View File
@@ -0,0 +1,94 @@
// @ts-check
const assert = require("assert")
const Denque = require("denque")
const StateMachine = require("snowtransfer").StateMachine
const passthrough = require("../passthrough")
const {sync} = passthrough
/** @type {import("../d2m/discord-packets")} */
const discordPackets = sync.require("../d2m/discord-packets")
/** @type {import("../matrix/api")} */
const api = sync.require("../matrix/api")
const DEBUG_HOMESERVER_STATUS = true
function debugHomeserverStatus(message) {
if (DEBUG_HOMESERVER_STATUS) {
console.log(message)
}
}
const homeserverStatus = new class HomeserverStatus {
constructor() {
/** @private */
this.queue = new Denque()
/** @private */
this.pingInterval = undefined
/** @private */
this.sm = new StateMachine("online")
.defineState("online")
.defineState("offline", {
onEnter: [() => {
this.pingInterval = setInterval(async () => {
const pingResult = await api.ping().catch(e => ({ok: false, status: "net", root: e.message}))
if (pingResult.ok) {
this.sm.doTransition("ping ok")
}
}, 15e3)
}],
onLeave: [() => {
clearInterval(this.pingInterval)
}],
transitions: new Map()
})
.defineState("recovering", {
onEnter: [async () => { // Drain queue.
while (!this.queue.isEmpty()) {
const packet = this.queue.peekFront() // same position as .shift()
debugHomeserverStatus(`homeserver status: ${new Date().toISOString()} dq packet ${packet.t} ${packet.d?.content}`)
await discordPackets.dispatchPacketToBridge(passthrough.discord, packet)
if (this.sm.currentStateName !== "recovering") return // got kicked out due to another error
this.queue.shift()
}
this.sm.doTransition("recovered")
}],
onLeave: [],
transitions: new Map()
})
.defineUniversalTransition("error", "offline")
.defineTransition("offline", "ping ok", "recovering")
.defineTransition("recovering", "recovered", "online")
this.sm.on("enter", st => debugHomeserverStatus(`homeserver status: ${st}`))
this.sm.freeze()
}
isRealTime() {
return this.sm.currentStateName === "online"
}
/**
* When offline or recovering, call this for incoming packets to queue them to be sent in order later.
*/
queuePacket(packet) {
assert(["offline", "recovering"].includes(this.sm.currentStateName))
this.queue.push(packet)
}
setErrorWithPacket(packet) {
const wasRecovering = this.sm.currentStateName === "recovering"
this.sm.doTransition("error")
if (!wasRecovering) { // if was recovering then packet is already in the right place in queue
this.queuePacket(packet)
}
}
}
module.exports.homeserverStatus = homeserverStatus