feat(mcp): implement MCP infrastructure
- Add `mcp_settings` table to database schema - Enable internal MCP API endpoint via middleware header authentication - Introduce runtime log directory and capture utilities for diagnostics - Update build pipeline to capture build logs
This commit is contained in:
1 parent
ae9b554189
commit
b2669e2ad5
19 files changed
+3280
-20
No files matched your search
@@ -0,0 +1,371 @@
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const MAX_LOG_FILE_BYTES = 2 * 1024 * 1024; // 2 MB per service file (+ .1 backup)
|
||||
const DEFAULT_TAIL_LINES = 120;
|
||||
const MAX_TAIL_LINES = 500;
|
||||
const MAX_SCAN_TAIL_BYTES = 512 * 1024; // Bound in-memory file slice reads
|
||||
|
||||
const SECRET_SCRUB_PATTERNS = [
|
||||
[/\b(Bearer\s+)[A-Za-z0-9\-._~+/]+=*/gi, '$1[REDACTED]'],
|
||||
[/\b(dsmcp_)[A-Za-z0-9_-]{8,}\b/gi, '$1[REDACTED]'],
|
||||
[/\b(eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.)[A-Za-z0-9_-]+\b/g, '$1[REDACTED]'],
|
||||
[/("(?:token|secret|password|jwtSecret|s3SecretAccessKey|webhookSecret|prometheusMetricsToken|coolifyToken|mcpToken|x-datastack-internal-secret)"\s*:\s*")([^"]+)(")/gi, '$1[REDACTED]$3'],
|
||||
];
|
||||
|
||||
function scrubSensitiveLogText(rawText) {
|
||||
let clean = String(rawText ?? '');
|
||||
for (const [pattern, replacement] of SECRET_SCRUB_PATTERNS) {
|
||||
clean = clean.replace(pattern, replacement);
|
||||
}
|
||||
return clean;
|
||||
}
|
||||
|
||||
function resolveRuntimeLogsDir(env = process.env) {
|
||||
const explicitDir = typeof env.DATASTACK_RUNTIME_LOGS_DIR === 'string'
|
||||
? env.DATASTACK_RUNTIME_LOGS_DIR.trim()
|
||||
: '';
|
||||
if (explicitDir) {
|
||||
return path.resolve(explicitDir);
|
||||
}
|
||||
|
||||
const storageDir = typeof env.DATASTACK_STORAGE_DIR === 'string'
|
||||
? env.DATASTACK_STORAGE_DIR.trim()
|
||||
: '';
|
||||
if (storageDir) {
|
||||
const resolved = path.resolve(storageDir);
|
||||
if (path.basename(resolved).toLowerCase() === 'data') {
|
||||
return path.join(path.dirname(resolved), '.runtime-logs');
|
||||
}
|
||||
return path.join(resolved, '.runtime-logs');
|
||||
}
|
||||
|
||||
return path.resolve(process.cwd(), 'storage', '.runtime-logs');
|
||||
}
|
||||
|
||||
function normalizeServiceName(service) {
|
||||
const s = String(service || '').trim().toLowerCase();
|
||||
if (s === 'web' || s === 'nextjs') return 'web';
|
||||
if (s === 'ingestion' || s === 'server' || s === 'socket') return 'ingestion';
|
||||
return 'web';
|
||||
}
|
||||
|
||||
function getServiceLogFilePath(service, logsDir = resolveRuntimeLogsDir()) {
|
||||
const safeService = normalizeServiceName(service);
|
||||
return path.join(logsDir, `${safeService}.log`);
|
||||
}
|
||||
|
||||
function rotateLogFileIfNeeded(filePath) {
|
||||
try {
|
||||
const stat = fs.statSync(filePath);
|
||||
if (stat.size >= MAX_LOG_FILE_BYTES) {
|
||||
const backupPath = `${filePath}.1`;
|
||||
try {
|
||||
if (fs.existsSync(backupPath)) {
|
||||
fs.unlinkSync(backupPath);
|
||||
}
|
||||
} catch {
|
||||
// ignore backup cleanup error
|
||||
}
|
||||
fs.renameSync(filePath, backupPath);
|
||||
}
|
||||
} catch {
|
||||
// file does not exist yet
|
||||
}
|
||||
}
|
||||
|
||||
function appendRuntimeLogLine(service, streamName, rawChunk, options = {}) {
|
||||
const logsDir = options.logsDir || resolveRuntimeLogsDir(options.env);
|
||||
const safeService = normalizeServiceName(service);
|
||||
const text = scrubSensitiveLogText(rawChunk);
|
||||
if (!text) return 0;
|
||||
|
||||
const lines = text.split(/\r?\n/).filter((line) => line.trim().length > 0);
|
||||
if (lines.length === 0) return 0;
|
||||
|
||||
try {
|
||||
fs.mkdirSync(logsDir, { recursive: true });
|
||||
const filePath = getServiceLogFilePath(safeService, logsDir);
|
||||
rotateLogFileIfNeeded(filePath);
|
||||
const level = streamName === 'stderr' ? 'error' : 'info';
|
||||
const ts = new Date().toISOString();
|
||||
const formatted =
|
||||
lines
|
||||
.map((line) => JSON.stringify({ ts, service: safeService, level, message: line }))
|
||||
.join('\n') + '\n';
|
||||
fs.appendFileSync(filePath, formatted, 'utf8');
|
||||
return lines.length;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
function installRuntimeLogHook(service, options = {}) {
|
||||
const safeService = normalizeServiceName(service);
|
||||
if (process.env.DATASTACK_LOG_CAPTURED_BY_RUNNER === '1' && !options.force) {
|
||||
return { installed: false, reason: 'captured_by_runner' };
|
||||
}
|
||||
const guardKey = `__dataStackRuntimeLogHook_${safeService}`;
|
||||
if (globalThis[guardKey]) {
|
||||
return { installed: false, reason: 'already_installed' };
|
||||
}
|
||||
globalThis[guardKey] = true;
|
||||
|
||||
let isWriting = false;
|
||||
const wrapStream = (stream, streamName) => {
|
||||
const originalWrite = stream.write.bind(stream);
|
||||
stream.write = function patchedWrite(chunk, encoding, callback) {
|
||||
const result = originalWrite(chunk, encoding, callback);
|
||||
if (!isWriting) {
|
||||
isWriting = true;
|
||||
try {
|
||||
const raw =
|
||||
typeof chunk === 'string'
|
||||
? chunk
|
||||
: Buffer.isBuffer(chunk)
|
||||
? chunk.toString(typeof encoding === 'string' ? encoding : 'utf8')
|
||||
: String(chunk ?? '');
|
||||
appendRuntimeLogLine(safeService, streamName, raw, options);
|
||||
} catch {
|
||||
// Never interrupt stdout/stderr
|
||||
} finally {
|
||||
isWriting = false;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
};
|
||||
};
|
||||
|
||||
wrapStream(process.stdout, 'stdout');
|
||||
wrapStream(process.stderr, 'stderr');
|
||||
return { installed: true, service: safeService };
|
||||
}
|
||||
|
||||
function readTailLinesFromFile(filePath, maxBytes = MAX_SCAN_TAIL_BYTES) {
|
||||
if (!fs.existsSync(filePath)) return [];
|
||||
const stat = fs.statSync(filePath);
|
||||
if (!stat.isFile() || stat.size === 0) return [];
|
||||
|
||||
const readSize = Math.min(stat.size, maxBytes);
|
||||
const startPos = Math.max(0, stat.size - readSize);
|
||||
const fd = fs.openSync(filePath, 'r');
|
||||
try {
|
||||
const buf = Buffer.allocUnsafe(readSize);
|
||||
const bytesRead = fs.readSync(fd, buf, 0, readSize, startPos);
|
||||
let content = buf.subarray(0, bytesRead).toString('utf8');
|
||||
if (startPos > 0) {
|
||||
const firstNewline = content.indexOf('\n');
|
||||
content = firstNewline >= 0 ? content.slice(firstNewline + 1) : '';
|
||||
}
|
||||
return content.split(/\r?\n/).filter(Boolean);
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
}
|
||||
|
||||
function parseStoredLogLine(rawLine, fallbackService) {
|
||||
try {
|
||||
const parsed = JSON.parse(rawLine);
|
||||
if (parsed && typeof parsed === 'object' && typeof parsed.message === 'string') {
|
||||
return {
|
||||
ts: typeof parsed.ts === 'string' ? parsed.ts : '',
|
||||
service: typeof parsed.service === 'string' ? parsed.service : fallbackService,
|
||||
level: typeof parsed.level === 'string' ? parsed.level : 'info',
|
||||
message: parsed.message,
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
// Plain-text fallback
|
||||
}
|
||||
return {
|
||||
ts: '',
|
||||
service: fallbackService,
|
||||
level: 'info',
|
||||
message: rawLine,
|
||||
};
|
||||
}
|
||||
|
||||
function readRuntimeLogs(options = {}) {
|
||||
const logsDir = options.logsDir || resolveRuntimeLogsDir(options.env);
|
||||
const serviceFilter = String(options.service || 'all').trim().toLowerCase();
|
||||
const requestedLines = Number(options.lines);
|
||||
const limit = Number.isFinite(requestedLines)
|
||||
? Math.min(MAX_TAIL_LINES, Math.max(1, Math.floor(requestedLines)))
|
||||
: DEFAULT_TAIL_LINES;
|
||||
const search = typeof options.search === 'string' ? options.search.trim().toLowerCase() : '';
|
||||
const levelFilter = typeof options.level === 'string' ? options.level.trim().toLowerCase() : '';
|
||||
|
||||
const services =
|
||||
serviceFilter === 'web'
|
||||
? ['web']
|
||||
: serviceFilter === 'ingestion'
|
||||
? ['ingestion']
|
||||
: ['web', 'ingestion'];
|
||||
|
||||
const collected = [];
|
||||
const filesChecked = [];
|
||||
for (const svc of services) {
|
||||
const mainFile = getServiceLogFilePath(svc, logsDir);
|
||||
const backupFile = `${mainFile}.1`;
|
||||
for (const candidate of [backupFile, mainFile]) {
|
||||
if (fs.existsSync(candidate)) {
|
||||
filesChecked.push(path.basename(candidate));
|
||||
const rawLines = readTailLinesFromFile(candidate);
|
||||
for (const rawLine of rawLines) {
|
||||
const entry = parseStoredLogLine(rawLine, svc);
|
||||
if (levelFilter && levelFilter !== 'all' && entry.level !== levelFilter) {
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
search &&
|
||||
!entry.message.toLowerCase().includes(search) &&
|
||||
!entry.service.toLowerCase().includes(search)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
collected.push(entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (filesChecked.length === 0) {
|
||||
return {
|
||||
entries: [],
|
||||
totalMatched: 0,
|
||||
filesChecked: [],
|
||||
logsDir,
|
||||
exists: false,
|
||||
code: 'LOG_CAPTURE_UNAVAILABLE',
|
||||
warning: 'No runtime logs captured yet in .runtime-logs/. Start services or emit stdout logs first.',
|
||||
};
|
||||
}
|
||||
|
||||
collected.sort((a, b) => {
|
||||
if (a.ts && b.ts && a.ts !== b.ts) {
|
||||
return a.ts.localeCompare(b.ts);
|
||||
}
|
||||
return 0;
|
||||
});
|
||||
|
||||
const entries = collected.slice(-limit);
|
||||
return {
|
||||
entries,
|
||||
totalMatched: collected.length,
|
||||
filesChecked,
|
||||
logsDir,
|
||||
exists: true,
|
||||
};
|
||||
}
|
||||
|
||||
function readBuildLogs(options = {}) {
|
||||
const logsDir = options.logsDir || resolveRuntimeLogsDir(options.env);
|
||||
const cwd = options.cwd || process.cwd();
|
||||
const requestedLines = Number(options.lines);
|
||||
const limit = Number.isFinite(requestedLines)
|
||||
? Math.min(MAX_TAIL_LINES, Math.max(1, Math.floor(requestedLines)))
|
||||
: 200;
|
||||
const search = typeof options.search === 'string' ? options.search.trim().toLowerCase() : '';
|
||||
|
||||
const metaCandidates = [
|
||||
path.join(logsDir, 'last-build-meta.json'),
|
||||
path.join(cwd, 'build-meta.json'),
|
||||
path.join(cwd, '.next', 'standalone', 'build-meta.json'),
|
||||
];
|
||||
const logCandidates = [
|
||||
path.join(logsDir, 'last-build.log'),
|
||||
path.join(cwd, 'build.log'),
|
||||
path.join(cwd, '.next', 'standalone', 'build.log'),
|
||||
];
|
||||
|
||||
let meta = null;
|
||||
for (const metaPath of metaCandidates) {
|
||||
try {
|
||||
if (fs.existsSync(metaPath)) {
|
||||
meta = JSON.parse(fs.readFileSync(metaPath, 'utf8'));
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
// try next candidate
|
||||
}
|
||||
}
|
||||
|
||||
let logPathUsed = null;
|
||||
let allLines = [];
|
||||
for (const logPath of logCandidates) {
|
||||
try {
|
||||
if (fs.existsSync(logPath)) {
|
||||
logPathUsed = path.basename(logPath);
|
||||
allLines = readTailLinesFromFile(logPath);
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
// try next candidate
|
||||
}
|
||||
}
|
||||
|
||||
if (!meta && !logPathUsed) {
|
||||
return {
|
||||
meta: null,
|
||||
lines: [],
|
||||
totalLines: 0,
|
||||
exists: false,
|
||||
code: 'LOG_CAPTURE_UNAVAILABLE',
|
||||
warning: 'No captured build log or build-meta.json found yet. Run npm run build first.',
|
||||
};
|
||||
}
|
||||
|
||||
const filtered = search
|
||||
? allLines.filter((line) => line.toLowerCase().includes(search))
|
||||
: allLines;
|
||||
|
||||
return {
|
||||
meta,
|
||||
logFile: logPathUsed,
|
||||
lines: filtered.slice(-limit),
|
||||
totalLines: filtered.length,
|
||||
exists: true,
|
||||
};
|
||||
}
|
||||
|
||||
function syncStandaloneBuildArtifactsToLogsDir(options = {}) {
|
||||
const logsDir = options.logsDir || resolveRuntimeLogsDir(options.env);
|
||||
const cwd = options.cwd || process.cwd();
|
||||
const buildLogSrc = [
|
||||
path.join(cwd, 'build.log'),
|
||||
path.join(cwd, '.next', 'standalone', 'build.log'),
|
||||
].find((candidate) => fs.existsSync(candidate));
|
||||
const buildMetaSrc = [
|
||||
path.join(cwd, 'build-meta.json'),
|
||||
path.join(cwd, '.next', 'standalone', 'build-meta.json'),
|
||||
].find((candidate) => fs.existsSync(candidate));
|
||||
|
||||
if (!buildLogSrc && !buildMetaSrc) {
|
||||
return { synced: false };
|
||||
}
|
||||
|
||||
try {
|
||||
fs.mkdirSync(logsDir, { recursive: true });
|
||||
if (buildLogSrc) {
|
||||
fs.copyFileSync(buildLogSrc, path.join(logsDir, 'last-build.log'));
|
||||
}
|
||||
if (buildMetaSrc) {
|
||||
fs.copyFileSync(buildMetaSrc, path.join(logsDir, 'last-build-meta.json'));
|
||||
}
|
||||
return { synced: true, logsDir };
|
||||
} catch {
|
||||
return { synced: false };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
MAX_LOG_FILE_BYTES,
|
||||
scrubSensitiveLogText,
|
||||
resolveRuntimeLogsDir,
|
||||
getServiceLogFilePath,
|
||||
appendRuntimeLogLine,
|
||||
installRuntimeLogHook,
|
||||
readRuntimeLogs,
|
||||
readBuildLogs,
|
||||
syncStandaloneBuildArtifactsToLogsDir,
|
||||
};
|
||||
Reference in new issue
Block a user