Skip to content

Commit be504ea

Browse files
limzykennethlimzykenneth
authored
Merge pull request #7391 from limzykenneth/2.0-table
[p5.js 2.0] Fix CSV parsing
2 parents d4809cc + f2f4c0b commit be504ea

File tree

7 files changed

+248
-59
lines changed

7 files changed

+248
-59
lines changed

‎package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -90,4 +90,4 @@
9090
"test"
9191
]
9292
}
93-
}
93+
}

‎src/io/csv.js

+211
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
/*
2+
The MIT License (MIT)
3+
4+
Copyright (c) 2019 Evan Plaice <[email protected]>
5+
6+
Permission is hereby granted, free of charge, to any person obtaining
7+
a copy of this software and associated documentation files (the
8+
'Software'), to deal in the Software without restriction, including
9+
without limitation the rights to use, copy, modify, merge, publish,
10+
distribute, sublicense, and/or sell copies of the Software, and to
11+
permit persons to whom the Software is furnished to do so, subject to
12+
the following conditions:
13+
14+
The above copyright notice and this permission notice shall be
15+
included in all copies or substantial portions of the Software.
16+
17+
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
18+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
21+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
22+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
23+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24+
*/
25+
export function parse (csv, options, reviver = v => v) {
26+
const ctx = Object.create(null)
27+
ctx.options = options || {}
28+
ctx.reviver = reviver
29+
ctx.value = ''
30+
ctx.entry = []
31+
ctx.output = []
32+
ctx.col = 1
33+
ctx.row = 1
34+
35+
ctx.options.delimiter = ctx.options.delimiter === undefined ? '"' : options.delimiter;
36+
if(ctx.options.delimiter.length > 1 || ctx.options.delimiter.length === 0)
37+
throw Error(`CSVError: delimiter must be one character [${ctx.options.separator}]`)
38+
39+
ctx.options.separator = ctx.options.separator === undefined ? ',' : options.separator;
40+
if(ctx.options.separator.length > 1 || ctx.options.separator.length === 0)
41+
throw Error(`CSVError: separator must be one character [${ctx.options.separator}]`)
42+
43+
const lexer = new RegExp(`${escapeRegExp(ctx.options.delimiter)}|${escapeRegExp(ctx.options.separator)}|\r\n|\n|\r|[^${escapeRegExp(ctx.options.delimiter)}${escapeRegExp(ctx.options.separator)}\r\n]+`, 'y')
44+
const isNewline = /^(\r\n|\n|\r)$/
45+
46+
let matches = []
47+
let match = ''
48+
let state = 0
49+
50+
while ((matches = lexer.exec(csv)) !== null) {
51+
match = matches[0]
52+
53+
switch (state) {
54+
case 0: // start of entry
55+
switch (true) {
56+
case match === ctx.options.delimiter:
57+
state = 3
58+
break
59+
case match === ctx.options.separator:
60+
state = 0
61+
valueEnd(ctx)
62+
break
63+
case isNewline.test(match):
64+
state = 0
65+
valueEnd(ctx)
66+
entryEnd(ctx)
67+
break
68+
default:
69+
ctx.value += match
70+
state = 2
71+
break
72+
}
73+
break
74+
case 2: // un-delimited input
75+
switch (true) {
76+
case match === ctx.options.separator:
77+
state = 0
78+
valueEnd(ctx)
79+
break
80+
case isNewline.test(match):
81+
state = 0
82+
valueEnd(ctx)
83+
entryEnd(ctx)
84+
break
85+
default:
86+
state = 4
87+
throw Error(`CSVError: Illegal state [row:${ctx.row}, col:${ctx.col}]`)
88+
}
89+
break
90+
case 3: // delimited input
91+
switch (true) {
92+
case match === ctx.options.delimiter:
93+
state = 4
94+
break
95+
default:
96+
state = 3
97+
ctx.value += match
98+
break
99+
}
100+
break
101+
case 4: // escaped or closing delimiter
102+
switch (true) {
103+
case match === ctx.options.delimiter:
104+
state = 3
105+
ctx.value += match
106+
break
107+
case match === ctx.options.separator:
108+
state = 0
109+
valueEnd(ctx)
110+
break
111+
case isNewline.test(match):
112+
state = 0
113+
valueEnd(ctx)
114+
entryEnd(ctx)
115+
break
116+
default:
117+
throw Error(`CSVError: Illegal state [row:${ctx.row}, col:${ctx.col}]`)
118+
}
119+
break
120+
}
121+
}
122+
123+
// flush the last value
124+
if (ctx.entry.length !== 0) {
125+
valueEnd(ctx)
126+
entryEnd(ctx)
127+
}
128+
129+
return ctx.output
130+
}
131+
132+
export function stringify (array, options = {}, replacer = v => v) {
133+
const ctx = Object.create(null)
134+
ctx.options = options
135+
ctx.options.eof = ctx.options.eof !== undefined ? ctx.options.eof : true
136+
ctx.row = 1
137+
ctx.col = 1
138+
ctx.output = ''
139+
140+
ctx.options.delimiter = ctx.options.delimiter === undefined ? '"' : options.delimiter;
141+
if(ctx.options.delimiter.length > 1 || ctx.options.delimiter.length === 0)
142+
throw Error(`CSVError: delimiter must be one character [${ctx.options.separator}]`)
143+
144+
ctx.options.separator = ctx.options.separator === undefined ? ',' : options.separator;
145+
if(ctx.options.separator.length > 1 || ctx.options.separator.length === 0)
146+
throw Error(`CSVError: separator must be one character [${ctx.options.separator}]`)
147+
148+
const needsDelimiters = new RegExp(`${escapeRegExp(ctx.options.delimiter)}|${escapeRegExp(ctx.options.separator)}|\r\n|\n|\r`)
149+
150+
array.forEach((row, rIdx) => {
151+
let entry = ''
152+
ctx.col = 1
153+
row.forEach((col, cIdx) => {
154+
if (typeof col === 'string') {
155+
col = col.replace(new RegExp(ctx.options.delimiter, 'g'), `${ctx.options.delimiter}${ctx.options.delimiter}`)
156+
col = needsDelimiters.test(col) ? `${ctx.options.delimiter}${col}${ctx.options.delimiter}` : col
157+
}
158+
entry += replacer(col, ctx.row, ctx.col)
159+
if (cIdx !== row.length - 1) {
160+
entry += ctx.options.separator
161+
}
162+
ctx.col++
163+
})
164+
switch (true) {
165+
case ctx.options.eof:
166+
case !ctx.options.eof && rIdx !== array.length - 1:
167+
ctx.output += `${entry}\n`
168+
break
169+
default:
170+
ctx.output += `${entry}`
171+
break
172+
}
173+
ctx.row++
174+
})
175+
176+
return ctx.output
177+
}
178+
179+
function valueEnd (ctx) {
180+
const value = ctx.options.typed ? inferType(ctx.value) : ctx.value
181+
ctx.entry.push(ctx.reviver(value, ctx.row, ctx.col))
182+
ctx.value = ''
183+
ctx.col++
184+
}
185+
186+
function entryEnd (ctx) {
187+
ctx.output.push(ctx.entry)
188+
ctx.entry = []
189+
ctx.row++
190+
ctx.col = 1
191+
}
192+
193+
function inferType (value) {
194+
const isNumber = /.\./
195+
196+
switch (true) {
197+
case value === 'true':
198+
case value === 'false':
199+
return value === 'true'
200+
case isNumber.test(value):
201+
return parseFloat(value)
202+
case isFinite(value):
203+
return parseInt(value)
204+
default:
205+
return value
206+
}
207+
}
208+
209+
function escapeRegExp(str) {
210+
return str.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&");
211+
}

‎src/io/files.js

+9-38
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import * as fileSaver from 'file-saver';
99
import { Renderer } from '../core/p5.Renderer';
1010
import { Graphics } from '../core/p5.Graphics';
11+
import { parse } from './csv';
1112

1213
class HTTPError extends Error {
1314
status;
@@ -530,18 +531,20 @@ function files(p5, fn){
530531

531532
try{
532533
let { data } = await request(path, 'text');
533-
data = data.split(/\r?\n/);
534534

535535
let ret = new p5.Table();
536+
data = parse(data, {
537+
separator
538+
});
536539

537540
if(header){
538-
ret.columns = data.shift().split(separator);
541+
ret.columns = data.shift();
539542
}else{
540-
ret.columns = data[0].split(separator).map(() => null);
543+
ret.columns = Array(data[0].length).fill(null);
541544
}
542545

543546
data.forEach((line) => {
544-
const row = new p5.TableRow(line, separator);
547+
const row = new p5.TableRow(line);
545548
ret.addRow(row);
546549
});
547550

@@ -2032,40 +2035,8 @@ function files(p5, fn){
20322035
sep = '\t';
20332036
}
20342037
if (ext !== 'html') {
2035-
// make header if it has values
2036-
if (header[0] !== '0') {
2037-
for (let h = 0; h < header.length; h++) {
2038-
if (h < header.length - 1) {
2039-
pWriter.write(header[h] + sep);
2040-
} else {
2041-
pWriter.write(header[h]);
2042-
}
2043-
}
2044-
pWriter.write('\n');
2045-
}
2046-
2047-
// make rows
2048-
for (let i = 0; i < table.rows.length; i++) {
2049-
let j;
2050-
for (j = 0; j < table.rows[i].arr.length; j++) {
2051-
if (j < table.rows[i].arr.length - 1) {
2052-
//double quotes should be inserted in csv only if contains comma separated single value
2053-
if (ext === 'csv' && String(table.rows[i].arr[j]).includes(',')) {
2054-
pWriter.write('"' + table.rows[i].arr[j] + '"' + sep);
2055-
} else {
2056-
pWriter.write(table.rows[i].arr[j] + sep);
2057-
}
2058-
} else {
2059-
//double quotes should be inserted in csv only if contains comma separated single value
2060-
if (ext === 'csv' && String(table.rows[i].arr[j]).includes(',')) {
2061-
pWriter.write('"' + table.rows[i].arr[j] + '"');
2062-
} else {
2063-
pWriter.write(table.rows[i].arr[j]);
2064-
}
2065-
}
2066-
}
2067-
pWriter.write('\n');
2068-
}
2038+
const output = table.toString(sep);
2039+
pWriter.write(output);
20692040
} else {
20702041
// otherwise, make HTML
20712042
pWriter.print('<html>');

‎src/io/p5.Table.js

+14
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
* @requires core
55
*/
66

7+
import { stringify } from './csv';
8+
79
function table(p5, fn){
810
/**
911
* Table Options
@@ -43,6 +45,18 @@ function table(p5, fn){
4345
this.rows = [];
4446
}
4547

48+
toString(separator=',') {
49+
let rows = this.rows.map((row) => row.arr);
50+
51+
if(!this.columns.some((column) => column === null)){
52+
rows = [this.columns, ...rows,]
53+
}
54+
55+
return stringify(rows, {
56+
separator
57+
});
58+
}
59+
4660
/**
4761
* Use <a href="#/p5/addRow">addRow()</a> to add a new row of data to a <a href="#/p5.Table">p5.Table</a> object. By default,
4862
* an empty row is created. Typically, you would store a reference to

‎src/io/p5.TableRow.js

+4-10
Original file line numberDiff line numberDiff line change
@@ -14,18 +14,12 @@ function tableRow(p5, fn){
1414
*
1515
* @class p5.TableRow
1616
* @constructor
17-
* @param {String} [str] optional: populate the row with a
18-
* string of values, separated by the
19-
* separator
20-
* @param {String} [separator] comma separated values (csv) by default
17+
* @param {any[]} row optional: populate the row with an
18+
* array of values
2119
*/
2220
p5.TableRow = class {
23-
constructor(str, separator){
24-
let arr = [];
25-
if (str) {
26-
separator = separator || ',';
27-
arr = str.split(separator);
28-
}
21+
constructor(row=[]){
22+
let arr = row;
2923

3024
this.arr = arr;
3125
this.obj = Object.fromEntries(arr.entries());

0 commit comments

Comments
 (0)