docs: update AI Studio design token audits
Refresh design token inventory files to reflect the latest UI sizing and layout configurations from the sandbox prototype.
This commit is contained in:
1 parent
50449e96ad
commit
d39d9f92f9
139 files changed
+10982
-6
No files matched your search
@@ -0,0 +1,58 @@
|
||||
const express = require('express');
|
||||
const http = require('http');
|
||||
const cors = require('cors');
|
||||
const { Server } = require('socket.io');
|
||||
const { setupSocketHandlers } = require('./socket/handlers');
|
||||
const { CsvWriterWorker } = require('./workers/csvWriter');
|
||||
|
||||
const PORT = process.env.PORT_SOCKET || 4000;
|
||||
const app = express();
|
||||
app.use(cors({ origin: '*' }));
|
||||
app.use(express.json());
|
||||
|
||||
const bufferQueue = [];
|
||||
const writer = new CsvWriterWorker();
|
||||
|
||||
// Periodic Batch Flush: every 2000ms or when buffer hits 500
|
||||
const FLUSH_INTERVAL_MS = 2000;
|
||||
const FLUSH_THRESHOLD = 500;
|
||||
|
||||
async function flushBuffer() {
|
||||
if (bufferQueue.length === 0) return;
|
||||
const batch = bufferQueue.splice(0, bufferQueue.length);
|
||||
try {
|
||||
const result = await writer.writeBatch(batch);
|
||||
if (result.written > 0) {
|
||||
console.log(`[CSV Worker] Flushed ${result.written} events to ${result.files.length} file(s)`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[CSV Worker] Flush error:', err);
|
||||
bufferQueue.unshift(...batch); // Retry next cycle
|
||||
}
|
||||
}
|
||||
|
||||
setInterval(flushBuffer, FLUSH_INTERVAL_MS);
|
||||
|
||||
// Health check endpoint
|
||||
app.get('/health', (_req, res) => {
|
||||
res.json({
|
||||
status: 'online',
|
||||
service: 'DataStack Express Ingestion',
|
||||
port: PORT,
|
||||
bufferSize: bufferQueue.length,
|
||||
uptime: process.uptime(),
|
||||
});
|
||||
});
|
||||
|
||||
const server = http.createServer(app);
|
||||
const io = new Server(server, {
|
||||
cors: { origin: '*', methods: ['GET', 'POST'] },
|
||||
});
|
||||
|
||||
setupSocketHandlers(io, bufferQueue, writer);
|
||||
|
||||
server.listen(PORT, () => {
|
||||
console.log(`[DataStack Ingestion Engine] Express + Socket.IO listening on port ${PORT}`);
|
||||
});
|
||||
|
||||
module.exports = { app, server, io, bufferQueue };
|
||||
@@ -0,0 +1,17 @@
|
||||
# Server Manifest
|
||||
|
||||
## Purpose
|
||||
Independent Express.js and Socket.IO ingestion service operating on Port 4000.
|
||||
|
||||
## Architecture
|
||||
- **Port**: 4000 (Decoupled from Next.js on port 3000)
|
||||
- **Role**: High-throughput WebSocket handshake, JWT verify, event intake
|
||||
- **Worker**: `workers/csvWriter.js` for async batch appending to `/storage/data/{clientId}/{YYYY-MM-DD}.csv`
|
||||
- **Socket Handlers**: `socket/handlers.js` with client room partitioning and rate limiting.
|
||||
|
||||
## Dependencies
|
||||
- `express`
|
||||
- `socket.io`
|
||||
- `cors`
|
||||
- `jsonwebtoken`
|
||||
- `json2csv`
|
||||
@@ -0,0 +1,102 @@
|
||||
const jwt = require('jsonwebtoken');
|
||||
|
||||
function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
const jwtSecret = options.jwtSecret || process.env.DATASTACK_JWT_SECRET || 'datastack-universal-secret-key-2026';
|
||||
const idempotencyMap = new Map();
|
||||
const rateLimitMap = new Map();
|
||||
|
||||
// Middleware: Handshake Auth
|
||||
io.use((socket, next) => {
|
||||
const authHeader =
|
||||
socket.handshake.auth?.token ||
|
||||
socket.handshake.headers?.authorization ||
|
||||
socket.handshake.query?.token;
|
||||
|
||||
if (!authHeader) {
|
||||
return next(new Error('AUTH_REQUIRED: Handshake missing JWT'));
|
||||
}
|
||||
|
||||
try {
|
||||
const cleanToken = authHeader.startsWith('Bearer ') ? authHeader.slice(7) : authHeader;
|
||||
const decoded = jwt.verify(cleanToken, jwtSecret);
|
||||
socket.data.clientId = decoded.clientId;
|
||||
socket.data.rateLimit = decoded.rateLimit || 100;
|
||||
next();
|
||||
} catch (err) {
|
||||
next(new Error(`AUTH_INVALID: ${err.message}`));
|
||||
}
|
||||
});
|
||||
|
||||
io.on('connection', (socket) => {
|
||||
const clientId = socket.data.clientId;
|
||||
const rateLimit = socket.data.rateLimit;
|
||||
socket.join(`client:${clientId}`);
|
||||
|
||||
// Single event
|
||||
socket.on('sync_data', (payload, callback) => {
|
||||
try {
|
||||
const eventId = payload.eventId || `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||
const timestamp = payload.timestamp || new Date().toISOString();
|
||||
|
||||
// Idempotency
|
||||
const key = `${clientId}:${eventId}`;
|
||||
if (idempotencyMap.has(key)) {
|
||||
if (callback) callback({ success: true, eventId, note: 'DUPLICATE_DROPPED' });
|
||||
return;
|
||||
}
|
||||
idempotencyMap.set(key, Date.now());
|
||||
|
||||
// Rate limit
|
||||
const now = Date.now();
|
||||
const rl = rateLimitMap.get(clientId);
|
||||
if (!rl || now - rl.start >= 1000) {
|
||||
rateLimitMap.set(clientId, { count: 1, start: now });
|
||||
} else if (rl.count >= rateLimit) {
|
||||
if (callback) callback({ success: false, error: 'RATE_LIMIT_EXCEEDED' });
|
||||
return;
|
||||
} else {
|
||||
rl.count += 1;
|
||||
}
|
||||
|
||||
const event = {
|
||||
clientId,
|
||||
eventId,
|
||||
timestamp,
|
||||
payload: typeof payload.data === 'string' ? payload.data : JSON.stringify(payload.data),
|
||||
};
|
||||
|
||||
bufferQueue.push(event);
|
||||
|
||||
if (callback) callback({ success: true, eventId });
|
||||
} catch (err) {
|
||||
if (callback) callback({ success: false, error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Batch event
|
||||
socket.on('batch_sync', (batch, callback) => {
|
||||
if (!Array.isArray(batch)) {
|
||||
if (callback) callback({ success: false, error: 'Expected array' });
|
||||
return;
|
||||
}
|
||||
|
||||
let count = 0;
|
||||
for (const item of batch) {
|
||||
const eventId = item.eventId || `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||
const timestamp = item.timestamp || new Date().toISOString();
|
||||
|
||||
bufferQueue.push({
|
||||
clientId,
|
||||
eventId,
|
||||
timestamp,
|
||||
payload: typeof item.data === 'string' ? item.data : JSON.stringify(item.data),
|
||||
});
|
||||
count++;
|
||||
}
|
||||
|
||||
if (callback) callback({ success: true, processed: count });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { setupSocketHandlers };
|
||||
@@ -0,0 +1,95 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { Parser } = require('json2csv');
|
||||
|
||||
/**
|
||||
* Background CSV Writer Worker
|
||||
* Flushes batches of real-time events to partitioned CSV files.
|
||||
* Path convention: /storage/data/{clientId}/{YYYY-MM-DD}.csv
|
||||
*/
|
||||
class CsvWriterWorker {
|
||||
constructor(baseDir = path.resolve(__dirname, '../../storage/data'), maxFileSize = 100 * 1024 * 1024) {
|
||||
this.baseDir = baseDir;
|
||||
this.maxFileSize = maxFileSize;
|
||||
this.ensureBaseDir();
|
||||
}
|
||||
|
||||
ensureBaseDir() {
|
||||
if (!fs.existsSync(this.baseDir)) {
|
||||
fs.mkdirSync(this.baseDir, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
getTargetFilePath(clientId, dateStr) {
|
||||
const safeClientId = clientId.replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
const clientDir = path.join(this.baseDir, safeClientId);
|
||||
if (!fs.existsSync(clientDir)) {
|
||||
fs.mkdirSync(clientDir, { recursive: true });
|
||||
}
|
||||
|
||||
const date = dateStr || new Date().toISOString().split('T')[0];
|
||||
let baseFileName = `${date}.csv`;
|
||||
let targetPath = path.join(clientDir, baseFileName);
|
||||
|
||||
let partIndex = 1;
|
||||
while (fs.existsSync(targetPath)) {
|
||||
try {
|
||||
const stats = fs.statSync(targetPath);
|
||||
if (stats.size >= this.maxFileSize) {
|
||||
partIndex += 1;
|
||||
baseFileName = `${date}-part${partIndex}.csv`;
|
||||
targetPath = path.join(clientDir, baseFileName);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return targetPath;
|
||||
}
|
||||
|
||||
async writeBatch(events) {
|
||||
if (!events || events.length === 0) return { written: 0, files: [] };
|
||||
|
||||
const grouped = new Map();
|
||||
for (const evt of events) {
|
||||
const date = (evt.timestamp || new Date().toISOString()).split('T')[0];
|
||||
const key = `${evt.clientId}:::${date}`;
|
||||
if (!grouped.has(key)) grouped.set(key, []);
|
||||
grouped.get(key).push(evt);
|
||||
}
|
||||
|
||||
const files = [];
|
||||
const fields = ['clientId', 'eventId', 'timestamp', 'payload'];
|
||||
|
||||
for (const [key, batch] of grouped.entries()) {
|
||||
const [clientId, date] = key.split(':::');
|
||||
const targetPath = this.getTargetFilePath(clientId, date);
|
||||
const fileExists = fs.existsSync(targetPath);
|
||||
|
||||
const cleanBatch = batch.map((item) => ({
|
||||
clientId: item.clientId,
|
||||
eventId: item.eventId,
|
||||
timestamp: item.timestamp,
|
||||
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
|
||||
}));
|
||||
|
||||
const parser = new Parser({ fields, header: !fileExists, eol: '\n' });
|
||||
const csv = parser.parse(cleanBatch);
|
||||
|
||||
if (fileExists) {
|
||||
fs.appendFileSync(targetPath, '\n' + csv, 'utf8');
|
||||
} else {
|
||||
fs.writeFileSync(targetPath, csv, 'utf8');
|
||||
}
|
||||
|
||||
files.push(targetPath);
|
||||
}
|
||||
|
||||
return { written: events.length, files };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { CsvWriterWorker };
|
||||
Reference in new issue
Block a user