From daac7badbbda7eed35fe2f7aff52b7f6c584cf0d Mon Sep 17 00:00:00 2001 From: hypercross Date: Sat, 11 Apr 2026 22:56:01 +0800 Subject: [PATCH] feat: parse references --- src/csv-loader/esbuild.ts | 7 + src/csv-loader/loader.ts | 271 ++++++++++++++++++++++++++++++++++++-- src/csv-loader/rollup.ts | 7 + src/csv-loader/webpack.ts | 10 +- src/index.ts | 4 +- src/parser.ts | 54 +++++++- src/types.ts | 10 +- src/validator.ts | 72 +++++++++- 8 files changed, 408 insertions(+), 27 deletions(-) diff --git a/src/csv-loader/esbuild.ts b/src/csv-loader/esbuild.ts index 9c6fc88..585c61e 100644 --- a/src/csv-loader/esbuild.ts +++ b/src/csv-loader/esbuild.ts @@ -9,6 +9,10 @@ export interface CsvEsbuildOptions extends CsvLoaderOptions { include?: RegExp | string | Array; /** Exclude pattern for CSV files */ exclude?: RegExp | string | Array; + /** Base directory for resolving referenced CSV files (default: directory of current file) */ + refBaseDir?: string; + /** Primary key field name for referenced tables (default: 'id') */ + defaultPrimaryKey?: string; } function createFilter( @@ -79,6 +83,9 @@ export function csvLoader(options: CsvEsbuildOptions = {}): Plugin { ...parseOptions, emitTypes, resourceName, + currentFilePath: args.path, + refBaseDir: options.refBaseDir, + defaultPrimaryKey: options.defaultPrimaryKey, }); // Emit type definition file if enabled diff --git a/src/csv-loader/loader.ts b/src/csv-loader/loader.ts index 4df1a63..cf05b5a 100644 --- a/src/csv-loader/loader.ts +++ b/src/csv-loader/loader.ts @@ -1,6 +1,8 @@ import { parse } from 'csv-parse/sync'; import { parseSchema, createValidator, parseValue } from '../index.js'; -import type { Schema } from '../types.js'; +import type { Schema, ReferenceSchema } from '../types.js'; +import * as fs from 'fs'; +import * as path from 'path'; export interface CsvLoaderOptions { delimiter?: string; @@ -15,6 +17,12 @@ export interface CsvLoaderOptions { typesOutputDir?: string; /** Write .d.ts files to disk (useful for dev server) */ writeToDisk?: boolean; + /** Base directory for resolving referenced CSV files (default: directory of current file) */ + refBaseDir?: string; + /** Primary key field name for referenced tables (default: 'id') */ + defaultPrimaryKey?: string; + /** Current file path (used to resolve relative references) */ + currentFilePath?: string; } export interface CsvParseResult { @@ -24,6 +32,8 @@ export interface CsvParseResult { typeDefinition?: string; /** Property configurations for the CSV columns */ propertyConfigs: PropertyConfig[]; + /** Referenced table names */ + references: Set; } interface PropertyConfig { @@ -31,12 +41,186 @@ interface PropertyConfig { schema: any; validator: (value: unknown) => boolean; parser: (valueString: string) => unknown; + /** Whether this property is a reference to another table */ + isReference?: boolean; + /** Referenced table name (if isReference is true) */ + referenceTableName?: string; + /** Whether it's an array reference */ + referenceIsArray?: boolean; +} + +/** Cache for loaded referenced tables */ +const referenceTableCache = new Map[]>(); + +/** + * Parse and resolve a reference value. + * Loads the referenced table and replaces IDs with actual objects. + */ +function parseReferenceValue( + schema: ReferenceSchema, + valueString: string, + refBaseDir: string | undefined, + defaultPrimaryKey: string, + currentFilePath: string | undefined +): unknown { + // Determine the directory to search for referenced files + const baseDir = refBaseDir || (currentFilePath ? path.dirname(currentFilePath) : process.cwd()); + + // Build the referenced file path + const fileName = `${schema.tableName}.csv`; + const refFilePath = path.isAbsolute(fileName) + ? fileName + : path.join(baseDir, fileName); + + // Load the referenced table (use cache if already loaded) + let refTable: Record[]; + if (referenceTableCache.has(refFilePath)) { + refTable = referenceTableCache.get(refFilePath)!; + } else { + try { + const refContent = fs.readFileSync(refFilePath, 'utf-8'); + const refResult = parseCsv(refContent, { + currentFilePath: refFilePath, + emitTypes: false, + }); + refTable = refResult.data; + referenceTableCache.set(refFilePath, refTable); + } catch (error) { + throw new Error( + `Failed to load referenced table "${schema.tableName}" from ${refFilePath}: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + // Build a lookup map by primary key + const primaryKeyMap = new Map>(); + refTable.forEach(row => { + const pkValue = row[defaultPrimaryKey]; + if (pkValue !== undefined) { + primaryKeyMap.set(String(pkValue), row); + } + }); + + // Parse the value string to extract IDs + const valueParser = new ReferenceValueParser(valueString.trim()); + const ids = valueParser.parseIds(schema.isArray); + + // Resolve IDs to actual objects + if (schema.isArray) { + return ids.map(id => { + const obj = primaryKeyMap.get(id); + if (!obj) { + throw new Error( + `Reference to "${schema.tableName}" with ${defaultPrimaryKey}="${id}" not found` + ); + } + return obj; + }); + } else { + // Single reference (first ID if array provided) + const id = ids[0]; + const obj = primaryKeyMap.get(id); + if (!obj) { + throw new Error( + `Reference to "${schema.tableName}" with ${defaultPrimaryKey}="${id}" not found` + ); + } + return obj; + } +} + +/** + * Parser for reference values (extracts IDs from value string) + */ +class ReferenceValueParser { + private input: string; + private pos: number = 0; + + constructor(input: string) { + this.input = input; + } + + private peek(): string { + return this.input[this.pos] || ''; + } + + private consume(): string { + return this.input[this.pos++] || ''; + } + + private skipWhitespace(): void { + while (this.pos < this.input.length && /\s/.test(this.input[this.pos])) { + this.pos++; + } + } + + private consumeStr(str: string): boolean { + if (this.input.slice(this.pos, this.pos + str.length) === str) { + this.pos += str.length; + return true; + } + return false; + } + + parseIds(isArray: boolean): string[] { + this.skipWhitespace(); + + if (isArray) { + // Parse array format: [id1; id2; id3] + if (this.peek() === '[') { + this.consume(); + } + + this.skipWhitespace(); + + if (this.peek() === ']') { + this.consume(); + return []; + } + + const ids: string[] = []; + while (true) { + this.skipWhitespace(); + let id = ''; + while (this.pos < this.input.length && this.peek() !== ';' && this.peek() !== ']') { + id += this.consume(); + } + const trimmedId = id.trim(); + if (trimmedId) { + ids.push(trimmedId); + } + this.skipWhitespace(); + + if (!this.consumeStr(';')) { + break; + } + } + + this.skipWhitespace(); + if (this.peek() === ']') { + this.consume(); + } + + return ids; + } else { + // Parse single ID + let id = ''; + while (this.pos < this.input.length) { + const char = this.peek(); + if (char === ';' || char === ']' || char === ',') { + break; + } + id += this.consume(); + } + return [id.trim()]; + } + } } /** * Convert a schema to TypeScript type string */ -function schemaToTypeString(schema: Schema): string { +function schemaToTypeString(schema: Schema, resourceNames?: Map): string { switch (schema.type) { case 'string': return 'string'; @@ -46,18 +230,24 @@ function schemaToTypeString(schema: Schema): string { return 'number'; case 'boolean': return 'boolean'; + case 'reference': { + // Use the resource name mapping if provided, otherwise capitalize table name + const typeName = resourceNames?.get(schema.tableName) || + schema.tableName.charAt(0).toUpperCase() + schema.tableName.slice(1); + return schema.isArray ? `readonly ${typeName}[]` : typeName; + } case 'array': if (schema.element.type === 'tuple') { const tupleElements = schema.element.elements.map((el) => { - const typeStr = schemaToTypeString(el.schema); + const typeStr = schemaToTypeString(el.schema, resourceNames); return el.name ? `readonly ${el.name}: ${typeStr}` : typeStr; }); return `readonly [${tupleElements.join(', ')}]`; } - return `readonly ${schemaToTypeString(schema.element)}[]`; + return `readonly ${schemaToTypeString(schema.element, resourceNames)}[]`; case 'tuple': const tupleElements = schema.elements.map((el) => { - const typeStr = schemaToTypeString(el.schema); + const typeStr = schemaToTypeString(el.schema, resourceNames); return el.name ? `readonly ${el.name}: ${typeStr}` : typeStr; }); return `readonly [${tupleElements.join(', ')}]`; @@ -71,14 +261,40 @@ function schemaToTypeString(schema: Schema): string { */ function generateTypeDefinition( resourceName: string, - propertyConfigs: PropertyConfig[] + propertyConfigs: PropertyConfig[], + references: Set, + currentFilePath?: string ): string { const typeName = resourceName ? `${resourceName}Table` : 'Table'; + + // Generate import statements for referenced tables + const imports: string[] = []; + const resourceNames = new Map(); + + references.forEach(tableName => { + // Convert table name to type name (parts -> Part, recipes -> Recipe) + // Remove trailing 's' to get singular form, then capitalize + let singularName = tableName; + if (singularName.endsWith('s') && singularName.length > 1) { + singularName = singularName.slice(0, -1); + } + const typeBase = singularName.charAt(0).toUpperCase() + singularName.slice(1); + resourceNames.set(tableName, typeBase); + + // Import from relative path + const importPath = currentFilePath + ? `./${tableName}.csv` + : `../${tableName}.csv`; + imports.push(`import type { ${typeBase} } from '${importPath}';`); + }); + + const importSection = imports.length > 0 ? imports.join('\n') + '\n\n' : ''; + const properties = propertyConfigs - .map((config) => ` readonly ${config.name}: ${schemaToTypeString(config.schema)};`) + .map((config) => ` readonly ${config.name}: ${schemaToTypeString(config.schema, resourceNames)};`) .join('\n'); - return `type ${typeName} = readonly { + return `${importSection}type ${typeName} = readonly { ${properties} }[]; @@ -90,7 +306,7 @@ export default data; /** * Parse CSV content string into structured data with schema validation. * This is a standalone function that doesn't depend on webpack/rspack LoaderContext. - * + * * @param content - CSV content string (must have at least headers + schema row + 1 data row) * @param options - Parsing options * @returns CsvParseResult containing parsed data and optional type definitions @@ -106,6 +322,8 @@ export function parseCsv( const comment = options.comment === false ? undefined : (options.comment ?? '#'); const trim = options.trim ?? true; const emitTypes = options.emitTypes ?? true; + const refBaseDir = options.refBaseDir; + const defaultPrimaryKey = options.defaultPrimaryKey ?? 'id'; const records = parse(content, { delimiter, @@ -134,12 +352,34 @@ export function parseCsv( const propertyConfigs: PropertyConfig[] = headers.map((header: string, index: number) => { const schemaString = schemas[index]; const schema = parseSchema(schemaString); - return { + + const config: PropertyConfig = { name: header, schema, validator: createValidator(schema), parser: (valueString: string) => parseValue(schema, valueString), }; + + // Check if it's a reference type + if (schema.type === 'reference') { + config.isReference = true; + config.referenceTableName = schema.tableName; + config.referenceIsArray = schema.isArray; + // Override parser for reference fields + config.parser = (valueString: string) => { + return parseReferenceValue(schema, valueString, refBaseDir, defaultPrimaryKey, options.currentFilePath); + }; + } + + return config; + }); + + // Collect all referenced tables + const references = new Set(); + propertyConfigs.forEach(config => { + if (config.isReference && config.referenceTableName) { + references.add(config.referenceTableName); + } }); const dataRows = records.slice(2); @@ -149,7 +389,8 @@ export function parseCsv( const rawValue = row[colIndex] ?? ''; try { const parsed = config.parser(rawValue); - if (!config.validator(parsed)) { + // Skip validation for reference fields (validation happens during reference resolution) + if (!config.isReference && !config.validator(parsed)) { throw new Error( `Validation failed for property "${config.name}" at row ${rowIndex + 3}: ${rawValue}` ); @@ -170,10 +411,16 @@ export function parseCsv( const result: CsvParseResult = { data: objects, propertyConfigs, + references, }; if (emitTypes) { - result.typeDefinition = generateTypeDefinition(options.resourceName || '', propertyConfigs); + result.typeDefinition = generateTypeDefinition( + options.resourceName || '', + propertyConfigs, + references, + options.currentFilePath + ); } return result; diff --git a/src/csv-loader/rollup.ts b/src/csv-loader/rollup.ts index 930088c..c7a0e73 100644 --- a/src/csv-loader/rollup.ts +++ b/src/csv-loader/rollup.ts @@ -8,6 +8,10 @@ export interface CsvRollupOptions extends CsvLoaderOptions { include?: RegExp | string | Array; /** Exclude pattern for CSV files */ exclude?: RegExp | string | Array; + /** Base directory for resolving referenced CSV files (default: directory of current file) */ + refBaseDir?: string; + /** Primary key field name for referenced tables (default: 'id') */ + defaultPrimaryKey?: string; } function matchesPattern( @@ -83,6 +87,9 @@ export function csvLoader(options: CsvRollupOptions = {}): RollupPlugin { ...parseOptions, emitTypes, resourceName, + currentFilePath: id, + refBaseDir: options.refBaseDir, + defaultPrimaryKey: options.defaultPrimaryKey, }); // Emit type definition file if enabled diff --git a/src/csv-loader/webpack.ts b/src/csv-loader/webpack.ts index 56a212a..ed675d4 100644 --- a/src/csv-loader/webpack.ts +++ b/src/csv-loader/webpack.ts @@ -9,6 +9,10 @@ export interface CsvWebpackLoaderOptions extends CsvLoaderOptions { typesOutputDir?: string; /** Write .d.ts files to disk (useful for dev server) */ writeToDisk?: boolean; + /** Base directory for resolving referenced CSV files (default: directory of current file) */ + refBaseDir?: string; + /** Primary key field name for referenced tables (default: 'id') */ + defaultPrimaryKey?: string; } export default function csvLoader( @@ -26,7 +30,11 @@ export default function csvLoader( .replace(/[-_\s]+(.)?/g, (_, char) => char ? char.toUpperCase() : '') .replace(/^(.)/, (_, char) => char.toUpperCase()); - const result = parseCsv(content, { ...options, resourceName }); + const result = parseCsv(content, { + ...options, + resourceName, + currentFilePath: this.resourcePath, + }); // Emit type definition file if enabled if (emitTypes && result.typeDefinition) { diff --git a/src/index.ts b/src/index.ts index 0778430..0aff876 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,6 @@ import { parseSchema } from './parser'; import { parseValue, createValidator } from './validator'; -import type { Schema, PrimitiveSchema, TupleSchema, ArraySchema, ParsedSchema } from './types'; +import type { Schema, PrimitiveSchema, TupleSchema, ArraySchema, ReferenceSchema, ParsedSchema } from './types'; import { ParseError } from './parser'; export function defineSchema(schemaString: string): ParsedSchema { @@ -15,4 +15,4 @@ export function defineSchema(schemaString: string): ParsedSchema { } export { parseSchema, parseValue, createValidator, ParseError }; -export type { Schema, PrimitiveSchema, TupleSchema, ArraySchema, ParsedSchema }; +export type { Schema, PrimitiveSchema, TupleSchema, ArraySchema, ReferenceSchema, ParsedSchema }; diff --git a/src/parser.ts b/src/parser.ts index da26b2b..50daf03 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -1,4 +1,4 @@ -import type { Schema, PrimitiveSchema, TupleSchema, ArraySchema, NamedSchema } from './types'; +import type { Schema, PrimitiveSchema, TupleSchema, ArraySchema, NamedSchema, ReferenceSchema } from './types'; export class ParseError extends Error { constructor(message: string, public position?: number) { @@ -7,6 +7,13 @@ export class ParseError extends Error { } } +export interface ReferenceInfo { + /** Referenced table name (e.g., 'parts' from '@parts[]') */ + tableName: string; + /** Whether it's an array reference */ + isArray: boolean; +} + class Parser { private input: string; private pos: number = 0; @@ -52,6 +59,11 @@ class Parser { parseSchema(): Schema { this.skipWhitespace(); + // Check for reference syntax: @tablename[] + if (this.consumeStr('@')) { + return this.parseReferenceSchema(); + } + if (this.consumeStr('string')) { if (this.consumeStr('[')) { this.skipWhitespace(); @@ -177,20 +189,20 @@ class Parser { private parseNamedSchema(): NamedSchema { this.skipWhitespace(); - + const startpos = this.pos; let identifier = ''; - + while (this.pos < this.input.length && /[a-zA-Z0-9\-_]/.test(this.peek())) { identifier += this.consume(); } - + if (identifier.length === 0) { throw new ParseError('Expected schema or named schema', this.pos); } - + this.skipWhitespace(); - + if (this.consumeStr(':')) { this.skipWhitespace(); const name = identifier; @@ -202,6 +214,36 @@ class Parser { return { schema }; } } + + private parseReferenceSchema(): Schema { + // Parse table name + let tableName = ''; + while (this.pos < this.input.length && /[a-zA-Z0-9\-_]/.test(this.peek())) { + tableName += this.consume(); + } + + if (tableName.length === 0) { + throw new ParseError('Expected table name after @', this.pos); + } + + this.skipWhitespace(); + + // Check for array syntax + if (this.consumeStr('[]')) { + return { + type: 'reference', + tableName, + isArray: true, + }; + } + + // Single reference (non-array) + return { + type: 'reference', + tableName, + isArray: false, + }; + } } export function parseSchema(schemaString: string): Schema { diff --git a/src/types.ts b/src/types.ts index 9f8f07b..56053bb 100644 --- a/src/types.ts +++ b/src/types.ts @@ -19,7 +19,15 @@ export interface ArraySchema { element: Schema; } -export type Schema = PrimitiveSchema | TupleSchema | ArraySchema; +export interface ReferenceSchema { + type: 'reference'; + /** Referenced table name (e.g., 'parts' from '@parts[]') */ + tableName: string; + /** Whether it's an array reference */ + isArray: boolean; +} + +export type Schema = PrimitiveSchema | TupleSchema | ArraySchema | ReferenceSchema; export interface ParsedSchema { schema: Schema; diff --git a/src/validator.ts b/src/validator.ts index 2ca2799..a47ca55 100644 --- a/src/validator.ts +++ b/src/validator.ts @@ -1,4 +1,4 @@ -import type { Schema, PrimitiveSchema, TupleSchema, ArraySchema, NamedSchema } from './types'; +import type { Schema, PrimitiveSchema, TupleSchema, ArraySchema, NamedSchema, ReferenceSchema } from './types'; import { ParseError } from './parser'; class ValueParser { @@ -49,6 +49,9 @@ class ValueParser { return this.parseTupleValue(schema, allowOmitBrackets); case 'array': return this.parseArrayValue(schema, allowOmitBrackets); + case 'reference': + // Reference values are parsed as strings (IDs) initially, resolved later + return this.parseReferenceValue(schema); default: throw new ParseError(`Unknown schema type: ${(schema as { type: string }).type}`, this.pos); } @@ -175,7 +178,7 @@ class ValueParser { private parseArrayValue(schema: ArraySchema, allowOmitBrackets: boolean): unknown[] { let hasOpenBracket = false; const elementIsTupleOrArray = schema.element.type === 'tuple' || schema.element.type === 'array'; - + if (this.peek() === '[') { if (!elementIsTupleOrArray) { this.consume(); @@ -185,13 +188,13 @@ class ValueParser { hasOpenBracket = true; } } - + if (!hasOpenBracket && !allowOmitBrackets && !elementIsTupleOrArray) { throw new ParseError('Expected [', this.pos); } this.skipWhitespace(); - + if (this.peek() === ']' && hasOpenBracket) { this.consume(); return []; @@ -209,7 +212,7 @@ class ValueParser { } this.skipWhitespace(); - + if (hasOpenBracket) { if (!this.consumeStr(']')) { throw new ParseError('Expected ]', this.pos); @@ -219,6 +222,59 @@ class ValueParser { return result; } + private parseReferenceValue(schema: ReferenceSchema): string | string[] { + if (schema.isArray) { + // Parse array of IDs: [id1; id2; id3] + let hasOpenBracket = false; + if (this.peek() === '[') { + this.consume(); + hasOpenBracket = true; + } + + this.skipWhitespace(); + + if (this.peek() === ']' && hasOpenBracket) { + this.consume(); + return []; + } + + const ids: string[] = []; + while (true) { + this.skipWhitespace(); + // Parse each ID as a string + let id = ''; + while (this.pos < this.input.length && this.peek() !== ';' && this.peek() !== ']') { + id += this.consume(); + } + ids.push(id.trim()); + this.skipWhitespace(); + + if (!this.consumeStr(';')) { + break; + } + } + + if (hasOpenBracket) { + if (!this.consumeStr(']')) { + throw new ParseError('Expected ]', this.pos); + } + } + + return ids; + } else { + // Parse single ID as string + let id = ''; + while (this.pos < this.input.length) { + const char = this.peek(); + if (char === ';' || char === ']' || char === ',') { + break; + } + id += this.consume(); + } + return id.trim(); + } + } + getPosition(): number { return this.pos; } @@ -262,6 +318,12 @@ export function createValidator(schema: Schema): (value: unknown) => boolean { case 'array': if (!Array.isArray(value)) return false; return value.every((item) => createValidator(schema.element)(item)); + case 'reference': + // Reference can be a string (single ID) or array of strings (IDs) + if (schema.isArray) { + return Array.isArray(value) && value.every((id) => typeof id === 'string'); + } + return typeof value === 'string' || (Array.isArray(value) && value.every((id) => typeof id === 'string')); default: return false; }