diff --git a/src/lib/data/export-metadata/export-per-type/postgresql.ts b/src/lib/data/export-metadata/export-per-type/postgresql.ts index f9634fc5..5d111db2 100644 --- a/src/lib/data/export-metadata/export-per-type/postgresql.ts +++ b/src/lib/data/export-metadata/export-per-type/postgresql.ts @@ -10,7 +10,7 @@ import type { DBField } from '@/lib/domain/db-field'; import type { DBRelationship } from '@/lib/domain/db-relationship'; function parsePostgresDefault(field: DBField): string { - if (!field.default) { + if (!field.default || typeof field.default !== 'string') { return ''; } @@ -165,6 +165,21 @@ export function exportPostgreSQL(diagram: Diagram): string { // Handle PostgreSQL specific type formatting let typeWithSize = typeName; + let serialType = null; + + if (field.increment && !field.nullable) { + if ( + typeName.toLowerCase() === 'integer' || + typeName.toLowerCase() === 'int' + ) { + serialType = 'SERIAL'; + } else if (typeName.toLowerCase() === 'bigint') { + serialType = 'BIGSERIAL'; + } else if (typeName.toLowerCase() === 'smallint') { + serialType = 'SMALLSERIAL'; + } + } + if (field.characterMaximumLength) { if ( typeName.toLowerCase() === 'varchar' || @@ -221,7 +236,7 @@ export function exportPostgreSQL(diagram: Diagram): string { : ''; // Do not add PRIMARY KEY as a column constraint - will add as table constraint - return `${exportFieldComment(field.comments ?? '')} ${fieldName} ${typeWithSize}${notNull}${identity}${unique}${defaultValue}`; + return `${exportFieldComment(field.comments ?? '')} ${fieldName} ${serialType || typeWithSize}${serialType ? '' : notNull}${identity}${unique}${defaultValue}`; }) .join(',\n')}${ primaryKeyFields.length > 0 diff --git a/src/lib/data/sql-import/common.ts b/src/lib/data/sql-import/common.ts index 38f7e882..57ca489b 100644 --- a/src/lib/data/sql-import/common.ts +++ b/src/lib/data/sql-import/common.ts @@ -548,6 +548,7 @@ export function convertToChartDBDiagram( unique: column.unique, default: column.default || '', createdAt: Date.now(), + increment: column.increment, }; // Add type arguments if present diff --git a/src/lib/data/sql-import/dialect-importers/postgresql/postgresql.ts b/src/lib/data/sql-import/dialect-importers/postgresql/postgresql.ts index fa13cee2..215133a0 100644 --- a/src/lib/data/sql-import/dialect-importers/postgresql/postgresql.ts +++ b/src/lib/data/sql-import/dialect-importers/postgresql/postgresql.ts @@ -26,6 +26,275 @@ import { getTableIdWithSchemaSupport, } from './postgresql-common'; +/** + * Uses regular expressions to find foreign key relationships in PostgreSQL SQL content. + * This is a fallback method to catch relationships that might be missed by the parser. + */ +function findForeignKeysUsingRegex( + sqlContent: string, + tableMap: Record, + relationships: SQLForeignKey[] +): void { + // Track already added relationships to avoid duplicates + const addedRelationships = new Set(); + + // Build a set of existing relationships to avoid duplicates + relationships.forEach((rel) => { + const relationshipKey = `${rel.sourceTable}.${rel.sourceColumn}-${rel.targetTable}.${rel.targetColumn}`; + addedRelationships.add(relationshipKey); + }); + + // Normalize SQL content: replace multiple whitespaces and newlines with single space + // This helps handle DDL with unusual formatting like linebreaks in column definitions + const normalizedSQL = sqlContent + .replace(/\s+/g, ' ') + // Replace common bracket/brace formatting issues + .replace(/\[\s*(\d+)\s*\]/g, '[$1]') + .replace(/\{\s*(\d+)\s*\}/g, '{$1}'); + + // First extract all table names to ensure they're in the tableMap + const tableNamePattern = + /CREATE\s+TABLE(?:\s+IF\s+NOT\s+EXISTS)?(?:\s+ONLY)?\s+(?:"?([^"\s.]+)"?\.)?["'`]?([^"'`\s.(]+)["'`]?/gi; + let match; + + tableNamePattern.lastIndex = 0; + while ((match = tableNamePattern.exec(normalizedSQL)) !== null) { + const schemaName = match[1] || 'public'; + const tableName = match[2]; + + // Skip invalid table names + if (!tableName || tableName.toUpperCase() === 'CREATE') continue; + + // Ensure the table is in our tableMap + const tableKey = `${schemaName}.${tableName}`; + if (!tableMap[tableKey]) { + const tableId = generateId(); + tableMap[tableKey] = tableId; + } + } + + // Extract original column names from CREATE TABLE statements + const createTablePattern = + /CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?(?:"?([^"\s.]+)"?\.)?["'`]?([^"'`\s.(]+)["'`]?\s*\((.*?)(?:,\s*(?:CONSTRAINT|PRIMARY|UNIQUE|CHECK|FOREIGN|INDEX|EXCLUDE)\s|,\s*\);|\);)/gis; + + // Map to store column names by table + const tableColumns: Record = {}; + + createTablePattern.lastIndex = 0; + while ((match = createTablePattern.exec(normalizedSQL)) !== null) { + const schemaName = match[1] || 'public'; + const tableName = match[2]; + const columnDefinitions = match[3]; + + if (!tableName || !columnDefinitions) continue; + + const tableKey = `${schemaName}.${tableName}`; + + // Extract column names from definitions + const columns: string[] = []; + const columnPattern = /["'`]?(\w+)["'`]?\s+\w+/g; + let columnMatch; + + while ((columnMatch = columnPattern.exec(columnDefinitions)) !== null) { + if ( + columnMatch[1] && + !columnMatch[1].match( + /^(CONSTRAINT|PRIMARY|UNIQUE|CHECK|FOREIGN|KEY|INDEX|EXCLUDE)$/i + ) + ) { + columns.push(columnMatch[1]); + } + } + + tableColumns[tableKey] = columns; + } + + // Define patterns for finding foreign keys in PostgreSQL DDL + const foreignKeyPatterns = [ + // In-line column references pattern - more flexible for odd formatting + /CREATE\s+TABLE(?:\s+IF\s+NOT\s+EXISTS)?(?:\s+ONLY)?\s+(?:"?([^"\s.]+)"?\.)?["'`]?([^"'`\s.(]+)["'`]?.*?["'`]?(\w+)["'`]?\s+\w+(?:\([^)]*\))?\s+(?:NOT\s+NULL\s+)?REFERENCES\s+(?:"?([^"\s.]+)"?\.)?["'`]?([^"'`\s.(]+)["'`]?\s*\(\s*["'`]?(\w+)["'`]?\s*\)/gi, + + // Multi-line foreign key declarations with better support for varied formatting + /CREATE\s+TABLE(?:\s+IF\s+NOT\s+EXISTS)?(?:\s+ONLY)?\s+(?:"?([^"\s.]+)"?\.)?["'`]?([^"'`\s.(]+)["'`]?.*?FOREIGN\s+KEY\s*\(\s*["'`]?(\w+)["'`]?\s*\)\s+REFERENCES\s+(?:"?([^"\s.]+)"?\.)?["'`]?([^"'`\s.(]+)["'`]?\s*\(\s*["'`]?(\w+)["'`]?\s*\)/gi, + + // ALTER TABLE pattern with improved matching + /ALTER\s+TABLE(?:\s+ONLY)?\s+(?:"?([^"\s.]+)"?\.)?["'`]?([^"'`\s.(]+)["'`]?\s+ADD\s+(?:CONSTRAINT\s+\w+\s+)?FOREIGN\s+KEY\s*\(\s*["'`]?(\w+)["'`]?\s*\)\s+REFERENCES\s+(?:"?([^"\s.]+)"?\.)?["'`]?([^"'`\s.(]+)["'`]?\s*\(\s*["'`]?(\w+)["'`]?\s*\)/gi, + ]; + + // Process each pattern + for (const pattern of foreignKeyPatterns) { + pattern.lastIndex = 0; + while ((match = pattern.exec(normalizedSQL)) !== null) { + const sourceSchema = match[1] || 'public'; + const sourceTable = match[2]; + const sourceColumn = match[3]; + const targetSchema = match[4] || 'public'; + const targetTable = match[5]; + const targetColumn = match[6]; + + // Skip if any part is invalid + if (!sourceTable || !sourceColumn || !targetTable || !targetColumn) + continue; + + // Create a unique key to track this relationship + const relationshipKey = `${sourceTable}.${sourceColumn}-${targetTable}.${targetColumn}`; + + // Skip if we've already added this relationship + if (addedRelationships.has(relationshipKey)) continue; + addedRelationships.add(relationshipKey); + + // Get table IDs + const sourceTableKey = `${sourceSchema}.${sourceTable}`; + const targetTableKey = `${targetSchema}.${targetTable}`; + + const sourceTableId = tableMap[sourceTableKey]; + const targetTableId = tableMap[targetTableKey]; + + // Skip if either table ID is missing + if (!sourceTableId || !targetTableId) continue; + + // Add the relationship + relationships.push({ + name: `FK_${sourceTable}_${sourceColumn}_${targetTable}`, + sourceTable, + sourceSchema, + sourceColumn, + targetTable, + targetSchema, + targetColumn, + sourceTableId, + targetTableId, + }); + } + } + + // Special handling for CHECK constraints with REFERENCES pattern + // This captures the cases where column definitions have CHECK constraints + // that might interfere with FK detection + const checkWithReferencesPattern = + /CREATE\s+TABLE.*?["'`]?([^"'`\s.(]+)["'`]?.*?CHECK\s*\(\s*(\w+)\s+(?:IN|=|REFERENCES)\s+(?:"?([^"\s.]+)"?\.)?["'`]?([^"'`\s.(]+)["'`]?\s*\(\s*["'`]?(\w+)["'`]?\s*\)/gi; + + checkWithReferencesPattern.lastIndex = 0; + while ((match = checkWithReferencesPattern.exec(normalizedSQL)) !== null) { + // Extract potential FK information from CHECK constraints + // This is a best-effort approach for particularly complex DDL + // Only continue processing if it looks like a valid relationship + if (match.length >= 5 && match[1] && match[2] && match[4] && match[5]) { + // Confirm it's a potential relationship by checking the column exists + const sourceTable = match[1]; + const sourceColumn = match[2]; + const targetSchema = match[3] || 'public'; + const targetTable = match[4]; + const targetColumn = match[5]; + + const sourceTableKey = `public.${sourceTable}`; + const tableColumnList = tableColumns[sourceTableKey] || []; + + // Only if the column actually exists in the table + if (tableColumnList.includes(sourceColumn)) { + // Create a unique key to track this relationship + const relationshipKey = `${sourceTable}.${sourceColumn}-${targetTable}.${targetColumn}`; + + // Skip if we've already added this relationship + if (addedRelationships.has(relationshipKey)) continue; + addedRelationships.add(relationshipKey); + + // Get table IDs + const sourceTableId = tableMap[sourceTableKey]; + const targetTableKey = `${targetSchema}.${targetTable}`; + const targetTableId = tableMap[targetTableKey]; + + // Skip if either table ID is missing + if (!sourceTableId || !targetTableId) continue; + + // Add the relationship + relationships.push({ + name: `FK_${sourceTable}_${sourceColumn}_${targetTable}`, + sourceTable, + sourceSchema: 'public', + sourceColumn, + targetTable, + targetSchema, + targetColumn, + sourceTableId, + targetTableId, + }); + } + } + } +} + +function getDefaultValueString( + columnDef: ColumnDefinition, + columnName: string +): string | undefined { + let defVal = columnDef.default_val; + + // Unwrap {type: 'default', value: ...} + if ( + defVal && + typeof defVal === 'object' && + defVal.type === 'default' && + 'value' in defVal + ) { + defVal = defVal.value; + } + + if (defVal === undefined || defVal === null) return undefined; + + let value: string | undefined; + console.log(`AST for column '${columnName}':`, defVal); + + switch (typeof defVal) { + case 'string': + value = defVal; + break; + case 'number': + value = String(defVal); + break; + case 'boolean': + value = defVal ? 'TRUE' : 'FALSE'; + break; + case 'object': + if ('value' in defVal && typeof defVal.value === 'string') { + value = defVal.value; + } else if ('raw' in defVal && typeof defVal.raw === 'string') { + value = defVal.raw; + } else if (defVal.type === 'bool') { + value = defVal.value ? 'TRUE' : 'FALSE'; + } else if (defVal.type === 'function' && defVal.name) { + // Handle nested structure: { name: { name: [{ value: ... }] } } + const fnName = defVal.name; + if ( + fnName && + typeof fnName === 'object' && + Array.isArray(fnName.name) && + fnName.name.length > 0 && + fnName.name[0].value + ) { + value = fnName.name[0].value.toUpperCase(); + } else if (typeof fnName === 'string') { + value = fnName.toUpperCase(); + } else { + value = 'UNKNOWN_FUNCTION'; + } + } else { + const built = buildSQLFromAST(defVal); + console.log( + `buildSQLFromAST for column '${columnName}':`, + built + ); + value = + typeof built === 'string' ? built : JSON.stringify(built); + } + break; + default: + value = undefined; + } + + return value; +} + // PostgreSQL-specific parsing logic export async function fromPostgres( sqlContent: string @@ -114,8 +383,22 @@ export async function fromPostgres( const columnName = extractColumnName( columnDef.column ); - const dataType = - columnDef.definition?.dataType || ''; + const rawDataType = + columnDef.definition?.dataType?.toUpperCase() || + ''; + let finalDataType = rawDataType; + let isSerialType = false; + + if (rawDataType === 'SERIAL') { + finalDataType = 'INTEGER'; + isSerialType = true; + } else if (rawDataType === 'BIGSERIAL') { + finalDataType = 'BIGINT'; + isSerialType = true; + } else if (rawDataType === 'SMALLSERIAL') { + finalDataType = 'SMALLINT'; + isSerialType = true; + } // Handle the column definition and add to columns array if (columnName) { @@ -128,23 +411,27 @@ export async function fromPostgres( columns.push({ name: columnName, - type: dataType, - nullable: - columnDef.nullable?.type !== - 'not null', - primaryKey: isPrimaryKey, + type: finalDataType, + nullable: isSerialType + ? false + : columnDef.nullable?.type !== + 'not null', + primaryKey: + isPrimaryKey || isSerialType, unique: columnDef.unique === 'unique', typeArgs: getTypeArgs( columnDef.definition ), - default: columnDef.default_val - ? buildSQLFromAST( - columnDef.default_val - ) - : undefined, + default: isSerialType + ? undefined + : getDefaultValueString( + columnDef, + columnName + ), increment: + isSerialType || columnDef.auto_increment === - 'auto_increment', + 'auto_increment', }); } } else if (def.resource === 'constraint') { @@ -867,6 +1154,9 @@ export async function fromPostgres( } }); + // Use regex as fallback to find additional foreign keys that the parser may have missed + findForeignKeysUsingRegex(sqlContent, tableMap, relationships); + // Filter out relationships with missing source table IDs or target table IDs const validRelationships = relationships.filter( (rel) => rel.sourceTableId && rel.targetTableId diff --git a/src/lib/domain/db-field.ts b/src/lib/domain/db-field.ts index 620baf6e..4d793702 100644 --- a/src/lib/domain/db-field.ts +++ b/src/lib/domain/db-field.ts @@ -14,6 +14,7 @@ export interface DBField { primaryKey: boolean; unique: boolean; nullable: boolean; + increment?: boolean; createdAt: number; characterMaximumLength?: string; precision?: number; @@ -30,6 +31,7 @@ export const dbFieldSchema: z.ZodType = z.object({ primaryKey: z.boolean(), unique: z.boolean(), nullable: z.boolean(), + increment: z.boolean().optional(), createdAt: z.number(), characterMaximumLength: z.string().optional(), precision: z.number().optional(),