Compare commits

..

3 Commits

Author SHA1 Message Date
hypercross 5a1627c6f1 test: improve reverse reference resolution handling 2026-04-19 14:19:45 +08:00
hypercross be8eb277d4 test(csv-loader): add test for multiple comment lines at the beginning
with reverse references
2026-04-19 14:15:44 +08:00
hypercross 9dd4f60c2d feat(csv-loader): allow configurable comment character
Allow `parseCsv` to accept a custom comment character for parsing
reverse reference declarations, instead of being hardcoded to `#`.
2026-04-19 14:06:11 +08:00
2 changed files with 75 additions and 26 deletions

View File

@ -1164,6 +1164,52 @@ describe("parseCsv - reverse reference resolution", () => {
expect(result.reverseReferences).toHaveLength(1);
expect(result.data[0]).toHaveProperty("orders");
});
it("should handle multiple comment lines including plain comments and reverse references", () => {
const orderCsvPath = path.join(fixturesDir, "order.csv");
const orderContent = [
"id,user,total",
"string,string,number",
"o01,u01,100",
"o02,u01,50",
"o03,u02,75",
].join("\n");
fs.writeFileSync(orderCsvPath, orderContent);
try {
const csv = [
"# id: id of user",
"# orders: list of related orders",
"# orders := ~order(user)",
"id,name",
"string,string",
"u01,Alice",
"u02,Bob",
].join("\n");
const result = parseCsv(csv, {
emitTypes: false,
currentFilePath: path.join(fixturesDir, "test.csv"),
});
expect(result.reverseReferences).toHaveLength(1);
expect(result.reverseReferences[0].fieldName).toBe("orders");
expect(result.data).toHaveLength(2);
expect(result.data[0].id).toBe("u01");
expect(result.data[1].id).toBe("u02");
// User u01 (Alice) should have 2 orders
const aliceOrders = result.data[0].orders as Record<string, unknown>[];
expect(aliceOrders).toHaveLength(2);
expect(aliceOrders[0]).toEqual({ id: "o01", user: "u01", total: 100 });
expect(aliceOrders[1]).toEqual({ id: "o02", user: "u01", total: 50 });
// User u02 (Bob) should have 1 order
const bobOrders = result.data[1].orders as Record<string, unknown>[];
expect(bobOrders).toHaveLength(1);
expect(bobOrders[0].id).toBe("o03");
} finally {
fs.unlinkSync(orderCsvPath);
}
});
});
describe("parseCsv - reverse reference with resolveReferences: false", () => {

View File

@ -538,12 +538,13 @@ export interface ReverseReferenceDeclaration {
*/
function parseReverseReferenceDeclaration(
line: string,
commentChar: string = "#",
): ReverseReferenceDeclaration | null {
const trimmed = line.trim();
// Must start with # (comment)
if (!trimmed.startsWith("#")) return null;
// Must start with the comment character
if (!trimmed.startsWith(commentChar)) return null;
const content = trimmed.slice(1).trim();
const content = trimmed.slice(commentChar.length).trim();
// Match pattern: fieldName := ~tableName(foreignKey)
const match = content.match(/^(\w+)\s*:=\s*~(\w+)\((\w+)\)(\?)?$/);
@ -793,18 +794,36 @@ export function parseCsv(
quote,
escape,
bom,
comment: undefined, // Don't let csv-parse skip comments; we need to parse them for reverse references
// Don't let csv-parse skip comments; we need to parse them for reverse references.
// Comment lines are filtered out manually below using the configured comment character.
comment: undefined,
trim,
skip_empty_lines: true,
relax_column_count: true,
});
if (records.length < 2) {
// Filter out comment lines from all records, collecting reverse reference declarations
const reverseReferences: ReverseReferenceDeclaration[] = [];
const filteredRecords: string[][] = [];
for (const row of records) {
const firstCell = (row[0] ?? "").trim();
if (comment && firstCell.startsWith(comment)) {
const decl = parseReverseReferenceDeclaration(firstCell, comment);
if (decl) {
reverseReferences.push(decl);
}
// Skip comment lines (whether or not they're reverse ref declarations)
continue;
}
filteredRecords.push(row);
}
if (filteredRecords.length < 2) {
throw new Error("CSV must have at least 2 rows: headers and schemas");
}
const headers = records[0];
const schemas = records[1];
const headers = filteredRecords[0];
const schemas = filteredRecords[1];
if (headers.length !== schemas.length) {
throw new Error(
@ -812,30 +831,14 @@ export function parseCsv(
);
}
// Parse reverse reference declarations from comment lines between schema row and data rows
const reverseReferences: ReverseReferenceDeclaration[] = [];
const dataRows: string[][] = [];
for (let i = 2; i < records.length; i++) {
const row = records[i];
// Check if this is a single-column row starting with # (comment with reverse ref declaration)
const firstCell = (row[0] ?? "").trim();
if (firstCell.startsWith("#")) {
const decl = parseReverseReferenceDeclaration(firstCell);
if (decl) {
reverseReferences.push(decl);
}
// Skip comment lines (whether or not they're reverse ref declarations)
continue;
}
dataRows.push(row);
}
const dataRows = filteredRecords.slice(2);
// Also check schema row cells for comment-prefixed reverse reference declarations
// (in case they appear as schema cells rather than separate rows)
for (let col = 0; col < schemas.length; col++) {
const cell = (schemas[col] ?? "").trim();
if (cell.startsWith("#")) {
const decl = parseReverseReferenceDeclaration(cell);
if (comment && cell.startsWith(comment)) {
const decl = parseReverseReferenceDeclaration(cell, comment);
if (decl) {
reverseReferences.push(decl);
}