- Add board_channel_links table to db.js with CRUD helpers
- Add mm_file_id column to images table for dedup tracking - mm-pull fetches posts from MM API, downloads media files, uploads with LOD to MinIO - Register bridge routes in server.js under /api/boards
This commit is contained in:
+55
-4
@@ -83,6 +83,18 @@ db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_boards_collection ON boards(collection_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_boards_created_by ON boards(created_by);
|
||||
CREATE INDEX IF NOT EXISTS idx_images_board ON images(board_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS board_channel_links (
|
||||
id TEXT PRIMARY KEY,
|
||||
board_id TEXT NOT NULL,
|
||||
channel_id TEXT NOT NULL,
|
||||
channel_name TEXT,
|
||||
created_by TEXT NOT NULL,
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
UNIQUE(board_id, channel_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_board_channel_links_board ON board_channel_links(board_id);
|
||||
`);
|
||||
|
||||
// Migrations — add columns to existing tables
|
||||
@@ -106,6 +118,11 @@ try {
|
||||
} catch {
|
||||
db.exec("ALTER TABLE images ADD COLUMN media_type TEXT DEFAULT 'image'");
|
||||
}
|
||||
try {
|
||||
db.prepare("SELECT mm_file_id FROM images LIMIT 0").get();
|
||||
} catch {
|
||||
db.exec("ALTER TABLE images ADD COLUMN mm_file_id TEXT");
|
||||
}
|
||||
|
||||
// ---------------------
|
||||
// User helpers
|
||||
@@ -330,11 +347,11 @@ function saveBoardCanvas(boardId, canvasState, thumbnail) {
|
||||
// ---------------------
|
||||
// Images
|
||||
// ---------------------
|
||||
function createImage({ id, boardId, filename, mimeType, fileSize, width, height, minioPath, publicUrl, uploadedBy, assetKey, mediaType }) {
|
||||
function createImage({ id, boardId, filename, mimeType, fileSize, width, height, minioPath, publicUrl, uploadedBy, assetKey, mediaType, mmFileId }) {
|
||||
db.prepare(`
|
||||
INSERT INTO images (id, board_id, filename, mime_type, file_size, width, height, minio_path, public_url, uploaded_by, asset_key, media_type)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(id, boardId, filename, mimeType, fileSize, width || null, height || null, minioPath, publicUrl || null, uploadedBy, assetKey || null, mediaType || 'image');
|
||||
INSERT INTO images (id, board_id, filename, mime_type, file_size, width, height, minio_path, public_url, uploaded_by, asset_key, media_type, mm_file_id)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(id, boardId, filename, mimeType, fileSize, width || null, height || null, minioPath, publicUrl || null, uploadedBy, assetKey || null, mediaType || 'image', mmFileId || null);
|
||||
return db.prepare('SELECT * FROM images WHERE id = ?').get(id);
|
||||
}
|
||||
|
||||
@@ -358,6 +375,37 @@ function deleteBoardImageRecords(boardId) {
|
||||
return images;
|
||||
}
|
||||
|
||||
// ---------------------
|
||||
// Board–Channel Links
|
||||
// ---------------------
|
||||
function createBoardChannelLink({ id, boardId, channelId, channelName, createdBy }) {
|
||||
db.prepare(`
|
||||
INSERT INTO board_channel_links (id, board_id, channel_id, channel_name, created_by)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`).run(id, boardId, channelId, channelName || null, createdBy);
|
||||
return db.prepare('SELECT * FROM board_channel_links WHERE id = ?').get(id);
|
||||
}
|
||||
|
||||
function getBoardChannelLinks(boardId) {
|
||||
return db.prepare('SELECT * FROM board_channel_links WHERE board_id = ? ORDER BY created_at DESC').all(boardId);
|
||||
}
|
||||
|
||||
function getBoardChannelLink(linkId) {
|
||||
return db.prepare('SELECT * FROM board_channel_links WHERE id = ?').get(linkId);
|
||||
}
|
||||
|
||||
function deleteBoardChannelLink(linkId) {
|
||||
db.prepare('DELETE FROM board_channel_links WHERE id = ?').run(linkId);
|
||||
}
|
||||
|
||||
function getAllBoardChannelLinks() {
|
||||
return db.prepare('SELECT * FROM board_channel_links ORDER BY created_at DESC').all();
|
||||
}
|
||||
|
||||
function getImageByMmFileId(boardId, mmFileId) {
|
||||
return db.prepare('SELECT * FROM images WHERE board_id = ? AND mm_file_id = ?').get(boardId, mmFileId);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
db,
|
||||
// Users
|
||||
@@ -372,4 +420,7 @@ module.exports = {
|
||||
getCollectionBoards, getBoard, createBoard, updateBoard, deleteBoard, saveBoardCanvas,
|
||||
// Images
|
||||
createImage, getBoardImages, getImage, deleteImage, deleteBoardImageRecords,
|
||||
// Board–Channel Links
|
||||
createBoardChannelLink, getBoardChannelLinks, getBoardChannelLink, deleteBoardChannelLink,
|
||||
getAllBoardChannelLinks, getImageByMmFileId,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,347 @@
|
||||
const { Router } = require('express');
|
||||
const { v4: uuidv4 } = require('uuid');
|
||||
const { authMiddleware } = require('../auth');
|
||||
const {
|
||||
getBoard, getCollectionMember,
|
||||
createBoardChannelLink, getBoardChannelLinks, getBoardChannelLink, deleteBoardChannelLink,
|
||||
createImage, getImageByMmFileId,
|
||||
} = require('../db');
|
||||
const { putBuffer, getImageUrl, MIME_TO_EXT } = require('../minio');
|
||||
const { generateLOD } = require('../services/lod-generator');
|
||||
const sharp = require('sharp');
|
||||
|
||||
const router = Router();
|
||||
|
||||
const MM_URL = process.env.MM_URL || 'http://mattermost:8065';
|
||||
const MM_BOT_TOKEN = process.env.MM_BOT_TOKEN || '';
|
||||
|
||||
const IMAGE_MIME_TYPES = [
|
||||
'image/png', 'image/jpeg', 'image/gif', 'image/webp', 'image/svg+xml',
|
||||
];
|
||||
const VIDEO_MIME_TYPES = [
|
||||
'video/mp4', 'video/webm', 'video/quicktime',
|
||||
];
|
||||
const ALLOWED_MIME_TYPES = [...IMAGE_MIME_TYPES, ...VIDEO_MIME_TYPES];
|
||||
|
||||
// All routes require auth
|
||||
router.use(authMiddleware);
|
||||
|
||||
// ---------------------
|
||||
// Helpers
|
||||
// ---------------------
|
||||
|
||||
/**
|
||||
* Check board access for editor+ role.
|
||||
* Returns board or null (sends error response).
|
||||
*/
|
||||
function checkEditorAccess(req, res) {
|
||||
const board = getBoard(req.params.boardId);
|
||||
if (!board) {
|
||||
res.status(404).json({ error: 'Board not found' });
|
||||
return null;
|
||||
}
|
||||
const member = getCollectionMember(board.collection_id, req.user.id);
|
||||
const hierarchy = { owner: 3, editor: 2, viewer: 1 };
|
||||
if (!member || (hierarchy[member.role] || 0) < 2) {
|
||||
res.status(403).json({ error: 'Editor or owner access required' });
|
||||
return null;
|
||||
}
|
||||
return board;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check board access for any authenticated member (viewer+).
|
||||
*/
|
||||
function checkViewerAccess(req, res) {
|
||||
const board = getBoard(req.params.boardId);
|
||||
if (!board) {
|
||||
res.status(404).json({ error: 'Board not found' });
|
||||
return null;
|
||||
}
|
||||
const member = getCollectionMember(board.collection_id, req.user.id);
|
||||
if (!member) {
|
||||
res.status(403).json({ error: 'Access denied' });
|
||||
return null;
|
||||
}
|
||||
return board;
|
||||
}
|
||||
|
||||
/**
|
||||
* Make authenticated request to Mattermost API.
|
||||
*/
|
||||
async function mmFetch(path, options = {}) {
|
||||
const url = `${MM_URL}/api/v4${path}`;
|
||||
const resp = await fetch(url, {
|
||||
...options,
|
||||
headers: {
|
||||
'Authorization': `Bearer ${MM_BOT_TOKEN}`,
|
||||
...options.headers,
|
||||
},
|
||||
});
|
||||
if (!resp.ok) {
|
||||
const body = await resp.text().catch(() => '');
|
||||
throw new Error(`Mattermost API error ${resp.status}: ${body}`);
|
||||
}
|
||||
return resp;
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify MIME type as image or video.
|
||||
*/
|
||||
function classifyMedia(mimeType) {
|
||||
if (VIDEO_MIME_TYPES.includes(mimeType)) return 'video';
|
||||
return 'image';
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload image buffer with LOD tiers to MinIO.
|
||||
*/
|
||||
async function uploadImageWithLOD(boardId, imageId, buffer, mimetype) {
|
||||
const assetKey = `boards/${boardId}/${imageId}`;
|
||||
const originalExt = MIME_TO_EXT[mimetype] || '.bin';
|
||||
|
||||
if (mimetype === 'image/svg+xml' || mimetype === 'image/gif') {
|
||||
const fullPath = `${assetKey}/full${originalExt}`;
|
||||
await putBuffer(fullPath, buffer, mimetype);
|
||||
let width = null, height = null;
|
||||
try {
|
||||
const metadata = await sharp(buffer).metadata();
|
||||
width = metadata.width || null;
|
||||
height = metadata.height || null;
|
||||
} catch {}
|
||||
return { assetKey, minioPath: fullPath, width, height };
|
||||
}
|
||||
|
||||
const lod = await generateLOD(buffer, originalExt);
|
||||
await Promise.all([
|
||||
putBuffer(`${assetKey}/thumb${lod.thumb.ext}`, lod.thumb.buffer, lod.thumb.ext === '.webp' ? 'image/webp' : mimetype),
|
||||
putBuffer(`${assetKey}/medium${lod.medium.ext}`, lod.medium.buffer, lod.medium.ext === '.webp' ? 'image/webp' : mimetype),
|
||||
putBuffer(`${assetKey}/full${lod.full.ext}`, lod.full.buffer, mimetype),
|
||||
]);
|
||||
|
||||
const minioPath = `${assetKey}/full${lod.full.ext}`;
|
||||
return { assetKey, minioPath, width: lod.full.width, height: lod.full.height };
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload video buffer to MinIO (no LOD).
|
||||
*/
|
||||
async function uploadVideo(boardId, imageId, buffer, mimetype) {
|
||||
const assetKey = `boards/${boardId}/${imageId}`;
|
||||
const ext = MIME_TO_EXT[mimetype] || '.bin';
|
||||
const fullPath = `${assetKey}/full${ext}`;
|
||||
await putBuffer(fullPath, buffer, mimetype);
|
||||
return { assetKey, minioPath: fullPath, width: null, height: null };
|
||||
}
|
||||
|
||||
// ---------------------
|
||||
// Channel Link Routes
|
||||
// ---------------------
|
||||
|
||||
/**
|
||||
* POST /api/boards/:boardId/mm-link
|
||||
* Link a Mattermost channel to a board.
|
||||
*/
|
||||
router.post('/:boardId/mm-link', (req, res) => {
|
||||
try {
|
||||
const board = checkEditorAccess(req, res);
|
||||
if (!board) return;
|
||||
|
||||
const { channelId, channelName } = req.body;
|
||||
if (!channelId) {
|
||||
return res.status(400).json({ error: 'channelId is required' });
|
||||
}
|
||||
|
||||
const link = createBoardChannelLink({
|
||||
id: uuidv4(),
|
||||
boardId: board.id,
|
||||
channelId,
|
||||
channelName: channelName || null,
|
||||
createdBy: req.user.id,
|
||||
});
|
||||
|
||||
return res.status(201).json(link);
|
||||
} catch (err) {
|
||||
if (err.message && err.message.includes('UNIQUE constraint failed')) {
|
||||
return res.status(409).json({ error: 'Channel already linked to this board' });
|
||||
}
|
||||
console.error('[mm-bridge] link error:', err);
|
||||
return res.status(500).json({ error: 'Internal server error' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/boards/:boardId/mm-link
|
||||
* List linked channels for a board.
|
||||
*/
|
||||
router.get('/:boardId/mm-link', (req, res) => {
|
||||
try {
|
||||
const board = checkViewerAccess(req, res);
|
||||
if (!board) return;
|
||||
|
||||
const links = getBoardChannelLinks(board.id);
|
||||
return res.json({ links });
|
||||
} catch (err) {
|
||||
console.error('[mm-bridge] list links error:', err);
|
||||
return res.status(500).json({ error: 'Internal server error' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* DELETE /api/boards/:boardId/mm-link/:linkId
|
||||
* Unlink a channel from a board.
|
||||
*/
|
||||
router.delete('/:boardId/mm-link/:linkId', (req, res) => {
|
||||
try {
|
||||
const board = checkEditorAccess(req, res);
|
||||
if (!board) return;
|
||||
|
||||
const link = getBoardChannelLink(req.params.linkId);
|
||||
if (!link || link.board_id !== board.id) {
|
||||
return res.status(404).json({ error: 'Link not found' });
|
||||
}
|
||||
|
||||
deleteBoardChannelLink(link.id);
|
||||
return res.json({ ok: true });
|
||||
} catch (err) {
|
||||
console.error('[mm-bridge] unlink error:', err);
|
||||
return res.status(500).json({ error: 'Internal server error' });
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------
|
||||
// Media Pull Route
|
||||
// ---------------------
|
||||
|
||||
/**
|
||||
* POST /api/boards/:boardId/mm-pull
|
||||
* Manual pull media from a Mattermost channel or thread.
|
||||
*/
|
||||
router.post('/:boardId/mm-pull', async (req, res) => {
|
||||
try {
|
||||
const board = checkEditorAccess(req, res);
|
||||
if (!board) return;
|
||||
|
||||
if (!MM_BOT_TOKEN) {
|
||||
return res.status(503).json({ error: 'Mattermost bot token not configured' });
|
||||
}
|
||||
|
||||
const { channelId, threadId } = req.body;
|
||||
if (!channelId && !threadId) {
|
||||
return res.status(400).json({ error: 'channelId or threadId is required' });
|
||||
}
|
||||
|
||||
// 1. Fetch posts from Mattermost
|
||||
let postsData;
|
||||
if (threadId) {
|
||||
const resp = await mmFetch(`/posts/${threadId}/thread`);
|
||||
postsData = await resp.json();
|
||||
} else {
|
||||
const resp = await mmFetch(`/channels/${channelId}/posts`);
|
||||
postsData = await resp.json();
|
||||
}
|
||||
|
||||
// postsData.order is array of post IDs, postsData.posts is { id: post }
|
||||
const posts = postsData.posts || {};
|
||||
const order = postsData.order || Object.keys(posts);
|
||||
|
||||
// 2. Collect all file_ids from posts
|
||||
const fileIds = [];
|
||||
for (const postId of order) {
|
||||
const post = posts[postId];
|
||||
if (post && post.file_ids && post.file_ids.length > 0) {
|
||||
fileIds.push(...post.file_ids);
|
||||
}
|
||||
}
|
||||
|
||||
if (fileIds.length === 0) {
|
||||
return res.json({ assets: [], message: 'No files found in posts' });
|
||||
}
|
||||
|
||||
// 3. Process each file
|
||||
const assets = [];
|
||||
const errors = [];
|
||||
|
||||
for (const fileId of fileIds) {
|
||||
try {
|
||||
// Skip if already imported (dedup by mm_file_id)
|
||||
const existing = getImageByMmFileId(board.id, fileId);
|
||||
if (existing) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Get file info
|
||||
const infoResp = await mmFetch(`/files/${fileId}/info`);
|
||||
const fileInfo = await infoResp.json();
|
||||
|
||||
const mimeType = fileInfo.mime_type || '';
|
||||
if (!ALLOWED_MIME_TYPES.includes(mimeType)) {
|
||||
continue; // skip non-media files
|
||||
}
|
||||
|
||||
// Download file
|
||||
const fileResp = await mmFetch(`/files/${fileId}`);
|
||||
const arrayBuf = await fileResp.arrayBuffer();
|
||||
const buffer = Buffer.from(arrayBuf);
|
||||
|
||||
const imageId = uuidv4();
|
||||
const mediaType = classifyMedia(mimeType);
|
||||
|
||||
let assetKey, minioPath, width, height;
|
||||
|
||||
if (mediaType === 'video') {
|
||||
({ assetKey, minioPath, width, height } = await uploadVideo(board.id, imageId, buffer, mimeType));
|
||||
} else {
|
||||
({ assetKey, minioPath, width, height } = await uploadImageWithLOD(board.id, imageId, buffer, mimeType));
|
||||
}
|
||||
|
||||
const publicUrl = getImageUrl(minioPath);
|
||||
|
||||
const image = createImage({
|
||||
id: imageId,
|
||||
boardId: board.id,
|
||||
filename: fileInfo.name || `mm-${fileId}`,
|
||||
mimeType,
|
||||
fileSize: buffer.length,
|
||||
width,
|
||||
height,
|
||||
minioPath,
|
||||
publicUrl,
|
||||
uploadedBy: req.user.id,
|
||||
assetKey,
|
||||
mediaType,
|
||||
mmFileId: fileId,
|
||||
});
|
||||
|
||||
assets.push({
|
||||
id: image.id,
|
||||
url: publicUrl,
|
||||
public_url: publicUrl,
|
||||
width: image.width,
|
||||
height: image.height,
|
||||
file_size: image.file_size,
|
||||
mime_type: image.mime_type,
|
||||
asset_key: image.asset_key,
|
||||
media_type: image.media_type,
|
||||
mm_file_id: fileId,
|
||||
});
|
||||
} catch (fileErr) {
|
||||
console.error(`[mm-bridge] failed to process file ${fileId}:`, fileErr.message);
|
||||
errors.push({ fileId, error: fileErr.message });
|
||||
}
|
||||
}
|
||||
|
||||
return res.json({
|
||||
assets,
|
||||
total_files: fileIds.length,
|
||||
imported: assets.length,
|
||||
skipped: fileIds.length - assets.length - errors.length,
|
||||
errors: errors.length > 0 ? errors : undefined,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('[mm-bridge] pull error:', err);
|
||||
return res.status(500).json({ error: 'Failed to pull media from Mattermost' });
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -71,12 +71,14 @@ const collectionRoutes = require('./routes/collections');
|
||||
const boardRoutes = require('./routes/boards');
|
||||
const uploadRoutes = require('./routes/upload');
|
||||
const adminRoutes = require('./routes/admin');
|
||||
const mmBridgeRoutes = require('./routes/mattermost-bridge');
|
||||
|
||||
app.use('/api/auth', authRoutes);
|
||||
app.use('/api/collections', collectionRoutes);
|
||||
app.use('/api/boards', boardRoutes);
|
||||
app.use('/api/upload', uploadRoutes);
|
||||
app.use('/api/admin', adminRoutes);
|
||||
app.use('/api/boards', mmBridgeRoutes);
|
||||
|
||||
// Public shared collection route (no auth required)
|
||||
app.get('/api/c/:shareToken', (req, res) => {
|
||||
@@ -136,6 +138,14 @@ async function start() {
|
||||
console.error('[server] Image uploads will not work until MinIO is available');
|
||||
}
|
||||
|
||||
// Start Mattermost auto-sync watcher (no-op if env vars missing)
|
||||
try {
|
||||
const { startWatcher } = require('./services/mm-watcher');
|
||||
startWatcher(io);
|
||||
} catch (err) {
|
||||
console.error('[server] MM watcher failed to start:', err.message);
|
||||
}
|
||||
|
||||
server.listen(PORT, '0.0.0.0', () => {
|
||||
console.log(`[server] RefBoard backend listening on port ${PORT}`);
|
||||
});
|
||||
@@ -145,6 +155,11 @@ async function start() {
|
||||
function shutdown(signal) {
|
||||
console.log(`[server] Received ${signal}, shutting down gracefully...`);
|
||||
|
||||
try {
|
||||
const { stopWatcher } = require('./services/mm-watcher');
|
||||
stopWatcher();
|
||||
} catch {}
|
||||
|
||||
io.close(() => {
|
||||
console.log('[server] Socket.IO closed');
|
||||
});
|
||||
|
||||
@@ -0,0 +1,354 @@
|
||||
/**
|
||||
* Mattermost Auto-Sync Watcher
|
||||
*
|
||||
* Polls linked Mattermost channels for new file attachments and
|
||||
* automatically imports them into the corresponding RefBoard boards.
|
||||
*
|
||||
* Requires MM_URL and MM_BOT_TOKEN env vars. Silently skips if not set.
|
||||
*/
|
||||
const https = require('https');
|
||||
const http = require('http');
|
||||
const { URL } = require('url');
|
||||
const { v4: uuidv4 } = require('uuid');
|
||||
const sharp = require('sharp');
|
||||
const { getAllBoardChannelLinks, getImageByMmFileId, createImage, getBoard } = require('../db');
|
||||
const { putBuffer, getImageUrl, MIME_TO_EXT } = require('../minio');
|
||||
const { generateLOD } = require('./lod-generator');
|
||||
|
||||
const MM_URL = process.env.MM_URL;
|
||||
const MM_BOT_TOKEN = process.env.MM_BOT_TOKEN;
|
||||
const POLL_INTERVAL_MS = parseInt(process.env.MM_WATCHER_INTERVAL || '30000', 10);
|
||||
|
||||
const IMAGE_MIME_TYPES = [
|
||||
'image/png', 'image/jpeg', 'image/gif', 'image/webp', 'image/svg+xml',
|
||||
];
|
||||
const VIDEO_MIME_TYPES = [
|
||||
'video/mp4', 'video/webm', 'video/quicktime',
|
||||
];
|
||||
const ALLOWED_MIME_TYPES = [...IMAGE_MIME_TYPES, ...VIDEO_MIME_TYPES];
|
||||
|
||||
// Track last-check timestamp per channel link (in-memory)
|
||||
const lastCheckMap = new Map();
|
||||
|
||||
/**
|
||||
* Make an authenticated GET request to the Mattermost API.
|
||||
* Returns parsed JSON.
|
||||
*/
|
||||
function mmGet(apiPath) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const url = new URL(apiPath, MM_URL);
|
||||
const client = url.protocol === 'https:' ? https : http;
|
||||
|
||||
const req = client.get(url.toString(), {
|
||||
headers: { Authorization: `Bearer ${MM_BOT_TOKEN}` },
|
||||
timeout: 15000,
|
||||
}, (res) => {
|
||||
if (res.statusCode !== 200) {
|
||||
// Drain and reject
|
||||
res.resume();
|
||||
return reject(new Error(`MM API ${apiPath} returned ${res.statusCode}`));
|
||||
}
|
||||
const chunks = [];
|
||||
res.on('data', (chunk) => chunks.push(chunk));
|
||||
res.on('end', () => {
|
||||
try {
|
||||
resolve(JSON.parse(Buffer.concat(chunks).toString()));
|
||||
} catch (e) {
|
||||
reject(new Error(`MM API ${apiPath}: invalid JSON`));
|
||||
}
|
||||
});
|
||||
res.on('error', reject);
|
||||
});
|
||||
req.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a file from Mattermost by file ID.
|
||||
* Returns { buffer, mimeType, filename }.
|
||||
*/
|
||||
function mmDownloadFile(fileId) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const url = new URL(`/api/v4/files/${fileId}`, MM_URL);
|
||||
const client = url.protocol === 'https:' ? https : http;
|
||||
|
||||
const req = client.get(url.toString(), {
|
||||
headers: { Authorization: `Bearer ${MM_BOT_TOKEN}` },
|
||||
timeout: 30000,
|
||||
}, (res) => {
|
||||
if ([301, 302, 303, 307, 308].includes(res.statusCode) && res.headers.location) {
|
||||
// Follow redirect
|
||||
return mmDownloadFileUrl(res.headers.location).then(resolve).catch(reject);
|
||||
}
|
||||
if (res.statusCode !== 200) {
|
||||
res.resume();
|
||||
return reject(new Error(`MM file download ${fileId} returned ${res.statusCode}`));
|
||||
}
|
||||
const contentType = (res.headers['content-type'] || 'application/octet-stream').split(';')[0].trim();
|
||||
const chunks = [];
|
||||
let totalSize = 0;
|
||||
const MAX_SIZE = 50 * 1024 * 1024;
|
||||
|
||||
res.on('data', (chunk) => {
|
||||
totalSize += chunk.length;
|
||||
if (totalSize > MAX_SIZE) {
|
||||
res.destroy();
|
||||
return reject(new Error(`File ${fileId} too large`));
|
||||
}
|
||||
chunks.push(chunk);
|
||||
});
|
||||
res.on('end', () => {
|
||||
resolve({ buffer: Buffer.concat(chunks), mimeType: contentType });
|
||||
});
|
||||
res.on('error', reject);
|
||||
});
|
||||
req.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Follow a redirect URL for file download.
|
||||
*/
|
||||
function mmDownloadFileUrl(downloadUrl) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const parsed = new URL(downloadUrl);
|
||||
const client = parsed.protocol === 'https:' ? https : http;
|
||||
|
||||
const req = client.get(downloadUrl, { timeout: 30000 }, (res) => {
|
||||
if (res.statusCode !== 200) {
|
||||
res.resume();
|
||||
return reject(new Error(`File redirect download returned ${res.statusCode}`));
|
||||
}
|
||||
const contentType = (res.headers['content-type'] || 'application/octet-stream').split(';')[0].trim();
|
||||
const chunks = [];
|
||||
res.on('data', (chunk) => chunks.push(chunk));
|
||||
res.on('end', () => {
|
||||
resolve({ buffer: Buffer.concat(chunks), mimeType: contentType });
|
||||
});
|
||||
res.on('error', reject);
|
||||
});
|
||||
req.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get file metadata from Mattermost.
|
||||
*/
|
||||
async function mmGetFileInfo(fileId) {
|
||||
return mmGet(`/api/v4/files/${fileId}/info`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify MIME type as image or video.
|
||||
*/
|
||||
function classifyMedia(mimeType) {
|
||||
if (VIDEO_MIME_TYPES.includes(mimeType)) return 'video';
|
||||
return 'image';
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload image with LOD tiers (mirrors upload.js logic).
|
||||
*/
|
||||
async function uploadImageWithLOD(boardId, imageId, buffer, mimetype) {
|
||||
const assetKey = `boards/${boardId}/${imageId}`;
|
||||
const originalExt = MIME_TO_EXT[mimetype] || '.bin';
|
||||
|
||||
if (mimetype === 'image/svg+xml' || mimetype === 'image/gif') {
|
||||
const fullPath = `${assetKey}/full${originalExt}`;
|
||||
await putBuffer(fullPath, buffer, mimetype);
|
||||
let width = null, height = null;
|
||||
if (mimetype !== 'image/svg+xml') {
|
||||
try {
|
||||
const meta = await sharp(buffer).metadata();
|
||||
width = meta.width || null;
|
||||
height = meta.height || null;
|
||||
} catch {}
|
||||
}
|
||||
return { assetKey, minioPath: fullPath, width, height };
|
||||
}
|
||||
|
||||
const lod = await generateLOD(buffer, originalExt);
|
||||
|
||||
await Promise.all([
|
||||
putBuffer(`${assetKey}/thumb${lod.thumb.ext}`, lod.thumb.buffer, lod.thumb.ext === '.webp' ? 'image/webp' : mimetype),
|
||||
putBuffer(`${assetKey}/medium${lod.medium.ext}`, lod.medium.buffer, lod.medium.ext === '.webp' ? 'image/webp' : mimetype),
|
||||
putBuffer(`${assetKey}/full${lod.full.ext}`, lod.full.buffer, mimetype),
|
||||
]);
|
||||
|
||||
const minioPath = `${assetKey}/full${lod.full.ext}`;
|
||||
return { assetKey, minioPath, width: lod.full.width, height: lod.full.height };
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a video file (single file, no LOD).
|
||||
*/
|
||||
async function uploadVideo(boardId, imageId, buffer, mimetype) {
|
||||
const assetKey = `boards/${boardId}/${imageId}`;
|
||||
const ext = MIME_TO_EXT[mimetype] || '.bin';
|
||||
const fullPath = `${assetKey}/full${ext}`;
|
||||
await putBuffer(fullPath, buffer, mimetype);
|
||||
return { assetKey, minioPath: fullPath, width: null, height: null };
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single channel link: fetch new posts, import new files.
|
||||
* Returns array of newly imported assets for socket notification.
|
||||
*/
|
||||
async function processLink(link) {
|
||||
const { board_id: boardId, channel_id: channelId, id: linkId } = link;
|
||||
|
||||
// Verify board still exists
|
||||
const board = getBoard(boardId);
|
||||
if (!board) return [];
|
||||
|
||||
const since = lastCheckMap.get(linkId) || Date.now() - POLL_INTERVAL_MS;
|
||||
lastCheckMap.set(linkId, Date.now());
|
||||
|
||||
let postsData;
|
||||
try {
|
||||
postsData = await mmGet(`/api/v4/channels/${channelId}/posts?since=${since}`);
|
||||
} catch (err) {
|
||||
console.error(`[mm-watcher] Failed to fetch posts for channel ${channelId}:`, err.message);
|
||||
return [];
|
||||
}
|
||||
|
||||
if (!postsData || !postsData.order || !postsData.posts) return [];
|
||||
|
||||
const newAssets = [];
|
||||
|
||||
for (const postId of postsData.order) {
|
||||
const post = postsData.posts[postId];
|
||||
if (!post || !post.file_ids || post.file_ids.length === 0) continue;
|
||||
|
||||
for (const fileId of post.file_ids) {
|
||||
try {
|
||||
// Deduplication check
|
||||
const existing = getImageByMmFileId(boardId, fileId);
|
||||
if (existing) continue;
|
||||
|
||||
// Get file info to check MIME type
|
||||
const fileInfo = await mmGetFileInfo(fileId);
|
||||
const mimeType = fileInfo.mime_type || 'application/octet-stream';
|
||||
|
||||
if (!ALLOWED_MIME_TYPES.includes(mimeType)) {
|
||||
continue; // Skip non-image/video files
|
||||
}
|
||||
|
||||
// Download the file
|
||||
const { buffer } = await mmDownloadFile(fileId);
|
||||
const imageId = uuidv4();
|
||||
const mediaType = classifyMedia(mimeType);
|
||||
|
||||
let assetKey, minioPath, width, height;
|
||||
|
||||
if (mediaType === 'video') {
|
||||
({ assetKey, minioPath, width, height } = await uploadVideo(boardId, imageId, buffer, mimeType));
|
||||
} else {
|
||||
({ assetKey, minioPath, width, height } = await uploadImageWithLOD(boardId, imageId, buffer, mimeType));
|
||||
}
|
||||
|
||||
const publicUrl = getImageUrl(minioPath);
|
||||
|
||||
// Use the link creator as the uploader
|
||||
const image = createImage({
|
||||
id: imageId,
|
||||
boardId,
|
||||
filename: fileInfo.name || `mm-${fileId}`,
|
||||
mimeType,
|
||||
fileSize: buffer.length,
|
||||
width,
|
||||
height,
|
||||
minioPath,
|
||||
publicUrl,
|
||||
uploadedBy: link.created_by,
|
||||
assetKey,
|
||||
mediaType,
|
||||
mmFileId: fileId,
|
||||
});
|
||||
|
||||
newAssets.push({
|
||||
assetKey: image.asset_key,
|
||||
name: image.filename,
|
||||
width: image.width,
|
||||
height: image.height,
|
||||
mediaType: image.media_type,
|
||||
});
|
||||
|
||||
console.log(`[mm-watcher] Imported file ${fileInfo.name || fileId} → board ${boardId}`);
|
||||
} catch (err) {
|
||||
console.error(`[mm-watcher] Failed to import file ${fileId}:`, err.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return newAssets;
|
||||
}
|
||||
|
||||
/**
|
||||
* Single poll cycle: process all links.
|
||||
*/
|
||||
async function pollOnce(io) {
|
||||
let links;
|
||||
try {
|
||||
links = getAllBoardChannelLinks();
|
||||
} catch (err) {
|
||||
console.error('[mm-watcher] Failed to query links:', err.message);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!links || links.length === 0) return;
|
||||
|
||||
for (const link of links) {
|
||||
try {
|
||||
const newAssets = await processLink(link);
|
||||
if (newAssets.length > 0 && io) {
|
||||
io.to(`board:${link.board_id}`).emit('board:media-arrived', {
|
||||
boardId: link.board_id,
|
||||
assets: newAssets,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[mm-watcher] Error processing link ${link.id}:`, err.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let pollTimer = null;
|
||||
|
||||
/**
|
||||
* Start the Mattermost watcher. Requires Socket.IO server instance.
|
||||
* Silently does nothing if MM_URL or MM_BOT_TOKEN are not set.
|
||||
*/
|
||||
function startWatcher(io) {
|
||||
if (!MM_URL || !MM_BOT_TOKEN) {
|
||||
console.log('[mm-watcher] MM_URL or MM_BOT_TOKEN not set — watcher disabled');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[mm-watcher] Starting watcher (interval: ${POLL_INTERVAL_MS}ms)`);
|
||||
|
||||
// Run first poll after a short delay to let the server finish starting
|
||||
setTimeout(() => {
|
||||
pollOnce(io).catch(err => console.error('[mm-watcher] Poll error:', err.message));
|
||||
}, 5000);
|
||||
|
||||
pollTimer = setInterval(() => {
|
||||
pollOnce(io).catch(err => console.error('[mm-watcher] Poll error:', err.message));
|
||||
}, POLL_INTERVAL_MS);
|
||||
|
||||
// Don't prevent process exit
|
||||
if (pollTimer.unref) pollTimer.unref();
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the watcher (for graceful shutdown).
|
||||
*/
|
||||
function stopWatcher() {
|
||||
if (pollTimer) {
|
||||
clearInterval(pollTimer);
|
||||
pollTimer = null;
|
||||
console.log('[mm-watcher] Watcher stopped');
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { startWatcher, stopWatcher };
|
||||
Reference in New Issue
Block a user