mirror of
https://github.com/stablyai/orca.git
synced 2026-09-29 00:02:56 +00:00
fix: sniff csv delimiters outside quotes (#4055)
This commit is contained in:
@@ -83,4 +83,14 @@ describe('detectCsvDelimiter', () => {
|
||||
it('strips a leading BOM before sniffing', () => {
|
||||
expect(detectCsvDelimiter('x.csv', '\uFEFFa\tb\tc')).toBe('\t')
|
||||
})
|
||||
|
||||
it('ignores delimiters inside quoted fields when sniffing', () => {
|
||||
const content = '"Doe, Jane"\tAge\n"Roe, John"\t42\n'
|
||||
|
||||
expect(detectCsvDelimiter('contacts.csv', content)).toBe('\t')
|
||||
expect(parseCsv(content, detectCsvDelimiter('contacts.csv', content)).rows).toEqual([
|
||||
['Doe, Jane', 'Age'],
|
||||
['Roe, John', '42']
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -122,7 +122,27 @@ export function detectCsvDelimiter(filePath: string, content: string): string {
|
||||
break
|
||||
}
|
||||
}
|
||||
const tabs = (firstLine.match(/\t/g) ?? []).length
|
||||
const commas = (firstLine.match(/,/g) ?? []).length
|
||||
const tabs = countDelimiterOutsideQuotes(firstLine, '\t')
|
||||
const commas = countDelimiterOutsideQuotes(firstLine, ',')
|
||||
return tabs > commas ? '\t' : ','
|
||||
}
|
||||
|
||||
function countDelimiterOutsideQuotes(line: string, delimiter: string): number {
|
||||
let count = 0
|
||||
let inQuotes = false
|
||||
for (let i = 0; i < line.length; i += 1) {
|
||||
const ch = line[i]
|
||||
if (ch === '"') {
|
||||
if (inQuotes && line[i + 1] === '"') {
|
||||
i += 1
|
||||
} else {
|
||||
inQuotes = !inQuotes
|
||||
}
|
||||
continue
|
||||
}
|
||||
if (!inQuotes && ch === delimiter) {
|
||||
count += 1
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user