2026-11-21 00:06:55 +06:00
import assert from "node:assert/strict" ;
import fs from "node:fs" ;
import http from "node:http" ;
import os from "node:os" ;
import path from "node:path" ;
import test from "node:test" ;
import zlib from "node:zlib" ;
import { NextRequest } from "next/server" ;
import { GET as dataGet } from "../app/api/data/route" ;
import { GET as exportGet } from "../app/api/export/route" ;
import { SettingsManager } from "../core/adminSettings/settingsManager" ;
import {
createAdminSessionToken ,
getAdminSessionSecret ,
} from "../core/auth/adminSession" ;
import {
createInitialAdmin ,
resetAdminStateForTests ,
} from "../core/auth/adminUsers" ;
import { DataBufferEngine } from "../core/data/buffer" ;
import { SDKManager } from "../core/sdk/sdkManager" ;
import { CSVStorageManager } from "../core/Storage/csvStorage" ;
const { CsvWriterWorker } = require ( "../server/workers/csvWriter.js" );
const ORIGINAL_STORAGE_DIR = CSVStorageManager . baseStorageDir ;
const ORIGINAL_DB_URL = process . env . DATABASE_URL ;
const ORIGINAL_SESSION_SECRET = process . env . DATASTACK_ADMIN_SESSION_SECRET ;
const ORIGINAL_JWT_SECRET = process . env . DATASTACK_JWT_SECRET ;
function makeTempDir ( prefix : string ) : string {
return fs . mkdtempSync ( path . join ( os . tmpdir (), prefix ));
}
function writeGzipCsv ( filePath : string , content : string ) : void {
fs . mkdirSync ( path . dirname ( filePath ), { recursive : true });
fs . writeFileSync ( filePath , zlib . gzipSync ( Buffer . from ( content , "utf8" )));
}
async function createOwnerCookie () : Promise < string > {
delete process . env . DATABASE_URL ;
const sessionSecret = getAdminSessionSecret ();
resetAdminStateForTests ();
const ownerEmail = "owner-r3p2@datastack.local" ;
await createInitialAdmin ( ownerEmail , "StrongPassword123!" );
const token = await createAdminSessionToken (
ownerEmail ,
sessionSecret ,
Date . now () + 3600 _000 ,
1 ,
"owner" ,
"owner-1" ,
);
return `datastack_admin_session= ${ token } ` ;
}
test . after ( async () => {
SettingsManager . stopMaintenanceSchedulerForTests ();
SDKManager . stopActivityFlushTimerForTests ();
await DataBufferEngine . shutdown ();
CSVStorageManager . baseStorageDir = ORIGINAL_STORAGE_DIR ;
CSVStorageManager . clearRowCountCache ();
CSVStorageManager . clearQuotaCaches ();
if ( ORIGINAL_DB_URL === undefined ) delete process . env . DATABASE_URL ;
else process . env . DATABASE_URL = ORIGINAL_DB_URL ;
if ( ORIGINAL_SESSION_SECRET === undefined ) {
delete process . env . DATASTACK_ADMIN_SESSION_SECRET ;
} else {
process . env . DATASTACK_ADMIN_SESSION_SECRET = ORIGINAL_SESSION_SECRET ;
}
if ( ORIGINAL_JWT_SECRET === undefined ) delete process . env . DATASTACK_JWT_SECRET ;
else process . env . DATASTACK_JWT_SECRET = ORIGINAL_JWT_SECRET ;
resetAdminStateForTests ();
});
test ( "R3-Phase-02 BUG-09: Port 3000 and Port 4000 extractDateFromFilename parse -gzN and -partN-gzN identically" , () => {
const tmpDir = makeTempDir ( "datastack-r3p2-date-" );
const worker = new CsvWriterWorker ( tmpDir );
try {
const cases : Array < [ string , string ] > = [
[ "2026-10-01.csv" , "2026-10-01" ],
[ "2026-10-01.csv.gz" , "2026-10-01" ],
[ "2026-10-01-part2.csv" , "2026-10-01" ],
[ "2026-10-01-part12.csv.gz" , "2026-10-01" ],
[ "2026-10-01-gz2.csv.gz" , "2026-10-01" ],
[ "2026-10-01-part2-gz3.csv.gz" , "2026-10-01" ],
[ "2026-10-01-gz2-part3.csv.gz" , "2026-10-01" ],
[ "not-a-date.csv.gz" , "not-a-date" ],
];
for ( const [ filename , expected ] of cases ) {
assert . equal (
CSVStorageManager . extractDateFromFilename ( filename ),
expected ,
`Port 3000 mismatch for ${ filename } ` ,
);
assert . equal (
worker . extractDateFromFilename ( filename ),
expected ,
`Port 4000 mismatch for ${ filename } ` ,
);
}
} finally {
fs . rmSync ( tmpDir , { recursive : true , force : true });
}
});
test ( "R3-Phase-02 BUG-09: -gzN collision files work in /api/data (to filter + fallback timestamp), /api/export (to filter), and Port 3000/4000 pruning" , async () => {
const tmpDir = makeTempDir ( "datastack-r3p2-gzn-" );
const cookie = await createOwnerCookie ();
SettingsManager . updateSettings ({
autoPruneDays : 0 ,
compressAfterDays : 0 ,
enableS3Backup : false ,
});
CSVStorageManager . baseStorageDir = tmpDir ;
CSVStorageManager . clearRowCountCache ();
CSVStorageManager . clearQuotaCaches ();
try {
const clientDir = path . join ( tmpDir , "gzn_client" );
const targetDate = "2025-02-15" ;
const gz2Path = path . join ( clientDir , ` ${ targetDate } -gz2.csv.gz` );
const partGzPath = path . join ( clientDir , ` ${ targetDate } -part2-gz3.csv.gz` );
// Row in -gz2 intentionally omits timestamp to verify valid ISO fallback (`2025-02-15T00:00:00.000Z`)
writeGzipCsv (
gz2Path ,
`clientId,eventId,timestamp,payload \ ngzn_client,evt_gz2_1,,"{""src"":""gz2""}" \ n` ,
);
writeGzipCsv (
partGzPath ,
`clientId,eventId,timestamp,payload \ ngzn_client,evt_part_gz3_1, ${ targetDate } T12:30:00.000Z,"{""src"":""part-gz3""}" \ n` ,
);
// 1. /api/data with exact to=2025-02-15 MUST return BOTH events with valid ISO timestamps
const dataRes = await dataGet (
new NextRequest (
`http://localhost:3000/api/data?clientId=gzn_client&from= ${ targetDate } &to= ${ targetDate } ` ,
{ headers : { cookie } },
),
);
assert . equal ( dataRes . status , 200 );
const dataJson = ( await dataRes . json ()) as {
total : number ;
events : Array < { eventId : string ; timestamp : string } > ;
};
assert . equal ( dataJson . total , 2 );
const gz2Evt = dataJson . events . find (( e ) => e . eventId === "evt_gz2_1" );
assert . ok ( gz2Evt , "evt_gz2_1 must not be excluded by to= filter" );
assert . equal ( gz2Evt . timestamp , ` ${ targetDate } T00:00:00.000Z` );
assert . ok (
dataJson . events . some (( e ) => e . eventId === "evt_part_gz3_1" ),
"evt_part_gz3_1 must not be excluded by to= filter" ,
);
// 2. /api/export with exact to=2025-02-15 MUST include BOTH -gzN files
const exportRes = await exportGet (
new NextRequest (
`http://localhost:3000/api/export?clientId=gzn_client&from= ${ targetDate } &to= ${ targetDate } &format=json&flush=false` ,
{ headers : { cookie } },
),
);
assert . equal ( exportRes . status , 200 );
const exported = JSON . parse ( await exportRes . text ()) as Array < {
eventId : string ;
} > ;
assert . equal ( exported . length , 2 );
assert . ok ( exported . some (( e ) => e . eventId === "evt_gz2_1" ));
assert . ok ( exported . some (( e ) => e . eventId === "evt_part_gz3_1" ));
// 3. Port 3000 pruneExpiredPartitionsAsync deletes -gzN collision files older than cutoff
const pruned3000 = await CSVStorageManager . pruneExpiredPartitionsAsync ( 7 );
assert . equal ( pruned3000 , 2 );
assert . equal ( fs . existsSync ( gz2Path ), false );
assert . equal ( fs . existsSync ( partGzPath ), false );
// 4. Port 4000 CsvWriterWorker.pruneExpiredPartitionsAsync also deletes -gzN collision files
writeGzipCsv (
gz2Path ,
`clientId,eventId,timestamp,payload \ ngzn_client,evt_gz2_2, ${ targetDate } T00:00:00.000Z,"{}" \ n` ,
);
const worker = new CsvWriterWorker ( tmpDir );
const pruned4000 = await worker . pruneExpiredPartitionsAsync ( 7 );
assert . equal ( pruned4000 , 1 );
assert . equal ( fs . existsSync ( gz2Path ), false );
} finally {
fs . rmSync ( tmpDir , { recursive : true , force : true });
}
});
test ( "R3-Phase-02 BUG-05: Singleton periodic scheduler automatically archives expired files (including -gzN) to S3 and prunes locally, with overlap and unhydrated-DB guards" , async () => {
const tmpDir = makeTempDir ( "datastack-r3p2-s3sched-" );
delete process . env . DATABASE_URL ;
process . env . DATASTACK_ADMIN_SESSION_SECRET =
"r3-phase02-test-session-secret-0123456789abcdef" ;
CSVStorageManager . baseStorageDir = tmpDir ;
CSVStorageManager . clearRowCountCache ();
CSVStorageManager . clearQuotaCaches ();
SettingsManager . stopMaintenanceSchedulerForTests ();
const uploadedPaths : string [] = [];
let failS3 = false ;
const mockS3 = http . createServer (( req , res ) => {
const chunks : Buffer [] = [];
req . on ( "data" , ( chunk : Buffer ) => chunks . push ( chunk ));
req . on ( "end" , () => {
if ( failS3 ) {
res . statusCode = 500 ;
res . end ( "S3 temporary error" );
return ;
}
if ( req . method === "PUT" ) {
uploadedPaths . push ( req . url || "" );
res . statusCode = 200 ;
res . end ();
return ;
}
res . statusCode = 200 ;
res . end ();
});
});
await new Promise < void >(( resolve ) => mockS3 . listen ( 0 , "127.0.0.1" , resolve ));
const addr = mockS3 . address ();
assert . ok ( addr && typeof addr === "object" );
2026-11-21 22:15:32 +06:00
const prevAllowPrivate = process . env . DATASTACK_ALLOW_PRIVATE_URLS ;
process . env . DATASTACK_ALLOW_PRIVATE_URLS = "true" ;
2026-11-21 00:06:55 +06:00
try {
// Configure S3 backup + autoPruneDays (cooldown will be reset so scheduled timer runs)
await SettingsManager . updateSettingsAsync ({
enableS3Backup : true ,
autoPruneDays : 7 ,
compressAfterDays : 0 ,
s3Endpoint : `http://127.0.0.1: ${ addr . port } ` ,
s3Region : "auto" ,
s3Bucket : "sched-bucket" ,
s3Prefix : "auto-archive/" ,
s3AccessKeyId : "AKIA_SCHED_TEST" ,
s3SecretAccessKey : "super_secret_sched_key_9999" ,
});
assert . equal (
SettingsManager . isMaintenanceSchedulerActive (),
true ,
"Scheduler must auto-bootstrap on settings access/update" ,
);
// Confirm singleton guard returns false when called again with no interval change
assert . equal ( SettingsManager . ensureMaintenanceScheduler (), false );
// Plant expired files AFTER initial save so ONLY the scheduled timer can archive+prune them
const clientDir = path . join ( tmpDir , "sched_client" );
fs . mkdirSync ( clientDir , { recursive : true });
const oldCsv = path . join ( clientDir , "2024-11-01.csv" );
const oldGz2 = path . join ( clientDir , "2024-11-01-gz2.csv.gz" );
const todayDate = new Date (). toISOString (). slice ( 0 , 10 );
const todayCsv = path . join ( clientDir , ` ${ todayDate } .csv` );
fs . writeFileSync (
oldCsv ,
`clientId,eventId,timestamp,payload \ nsched_client,evt_old_csv,2024-11-01T00:00:00.000Z,"{}" \ n` ,
);
writeGzipCsv (
oldGz2 ,
`clientId,eventId,timestamp,payload \ nsched_client,evt_old_gz2,2024-11-01T01:00:00.000Z,"{}" \ n` ,
);
fs . writeFileSync (
todayCsv ,
`clientId,eventId,timestamp,payload \ nsched_client,evt_today, ${ todayDate } T00:00:00.000Z,"{}" \ n` ,
);
// Expire prune cooldown and start a fast 30ms test interval to verify real timer firing
const globals = globalThis as typeof globalThis & {
__dataStackLastPruneMeta ?: { days : number ; ranAtMs : number };
__dataStackMaintenanceRunning? : boolean ;
__dataStackSettingsHydrated? : boolean ;
};
globals . __dataStackLastPruneMeta = {
days : 7 ,
ranAtMs : Date.now () - 2 * 60 * 60 * 1000 ,
};
SettingsManager . ensureMaintenanceScheduler ( 30 );
// Wait up to 1.5s for scheduled timer tick to upload to S3 and prune both expired files
const deadline = Date . now () + 1500 ;
while (
Date . now () < deadline &&
( fs . existsSync ( oldCsv ) || fs . existsSync ( oldGz2 ))
) {
await new Promise (( r ) => setTimeout ( r , 25 ));
}
SettingsManager . stopMaintenanceSchedulerForTests ();
assert . equal ( fs . existsSync ( oldCsv ), false , "Expired .csv must be pruned after S3 upload" );
assert . equal ( fs . existsSync ( oldGz2 ), false , "Expired -gz2.csv.gz must be pruned after S3 upload" );
assert . equal ( fs . existsSync ( todayCsv ), true , "Today's partition must remain intact" );
assert . ok (
uploadedPaths . some (( u ) => u . includes ( "/sched-bucket/auto-archive/sched_client/2024-11-01.csv" )),
`Expected 2024-11-01.csv S3 upload, got: ${ uploadedPaths . join ( ", " ) } ` ,
);
assert . ok (
uploadedPaths . some (( u ) => u . includes ( "/sched-bucket/auto-archive/sched_client/2024-11-01-gz2.csv.gz" )),
`Expected 2024-11-01-gz2.csv.gz S3 upload, got: ${ uploadedPaths . join ( ", " ) } ` ,
);
// Verify fail-safe when S3 returns 500: local expired file MUST NOT be deleted
failS3 = true ;
const failCsv = path . join ( clientDir , "2024-10-15.csv" );
fs . writeFileSync (
failCsv ,
`clientId,eventId,timestamp,payload \ nsched_client,evt_fail_s3,2024-10-15T00:00:00.000Z,"{}" \ n` ,
);
await SettingsManager . runScheduledMaintenanceTick ({ forcePrune : true });
assert . equal (
fs . existsSync ( failCsv ),
true ,
"Fail-safe: local file must NEVER be deleted when S3 upload fails" ,
);
// Verify overlap guard blocks concurrent execution
globals . __dataStackMaintenanceRunning = true ;
const overlapResult = await SettingsManager . runScheduledMaintenanceTick ({
forcePrune : true ,
});
globals . __dataStackMaintenanceRunning = false ;
assert . equal ( overlapResult . ran , false );
assert . equal ( overlapResult . skippedReason , "ALREADY_RUNNING" );
// Verify unhydrated-DB guard skips maintenance when DATABASE_URL is configured but unhydrated
process . env . DATABASE_URL = "postgresql://user:pass@127.0.0.1:1/unreachable" ;
globals . __dataStackSettingsHydrated = false ;
const unhydratedResult = await SettingsManager . runScheduledMaintenanceTick ({
forcePrune : true ,
skipDbSync : true ,
});
delete process . env . DATABASE_URL ;
assert . equal ( unhydratedResult . ran , false );
assert . equal ( unhydratedResult . skippedReason , "DB_UNHYDRATED" );
} finally {
2026-11-21 22:15:32 +06:00
if ( prevAllowPrivate === undefined ) delete process . env . DATASTACK_ALLOW_PRIVATE_URLS ;
else process . env . DATASTACK_ALLOW_PRIVATE_URLS = prevAllowPrivate ;
2026-11-21 00:06:55 +06:00
SettingsManager . stopMaintenanceSchedulerForTests ();
await new Promise < void >(( resolve ) => mockS3 . close (() => resolve ()));
fs . rmSync ( tmpDir , { recursive : true , force : true });
}
});