Merge pull request #7391 from limzykenneth/2.0-table

[p5.js 2.0] Fix CSV parsing

package.json

@@ -90,4 +90,4 @@
       "test"
     ]
   }
-}
+}

src/io/csv.js

@@ -0,0 +1,211 @@
+/*
+The MIT License (MIT)
+
+Copyright (c) 2019 Evan Plaice <[email protected]>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+export function parse (csv, options, reviver = v => v) {
+  const ctx = Object.create(null)
+  ctx.options = options || {}
+  ctx.reviver = reviver
+  ctx.value = ''
+  ctx.entry = []
+  ctx.output = []
+  ctx.col = 1
+  ctx.row = 1
+
+  ctx.options.delimiter = ctx.options.delimiter === undefined ? '"' : options.delimiter;
+  if(ctx.options.delimiter.length > 1 || ctx.options.delimiter.length === 0)
+    throw Error(`CSVError: delimiter must be one character [${ctx.options.separator}]`)
+
+  ctx.options.separator = ctx.options.separator === undefined ? ',' : options.separator;
+  if(ctx.options.separator.length > 1 || ctx.options.separator.length === 0)
+    throw Error(`CSVError: separator must be one character [${ctx.options.separator}]`)
+
+  const lexer = new RegExp(`${escapeRegExp(ctx.options.delimiter)}|${escapeRegExp(ctx.options.separator)}|\r\n|\n|\r|[^${escapeRegExp(ctx.options.delimiter)}${escapeRegExp(ctx.options.separator)}\r\n]+`, 'y')
+  const isNewline = /^(\r\n|\n|\r)$/
+
+  let matches = []
+  let match = ''
+  let state = 0
+
+  while ((matches = lexer.exec(csv)) !== null) {
+    match = matches[0]
+
+    switch (state) {
+      case 0: // start of entry
+        switch (true) {
+          case match === ctx.options.delimiter:
+            state = 3
+            break
+          case match === ctx.options.separator:
+            state = 0
+            valueEnd(ctx)
+            break
+          case isNewline.test(match):
+            state = 0
+            valueEnd(ctx)
+            entryEnd(ctx)
+            break
+          default:
+            ctx.value += match
+            state = 2
+            break
+        }
+        break
+      case 2: // un-delimited input
+        switch (true) {
+          case match === ctx.options.separator:
+            state = 0
+            valueEnd(ctx)
+            break
+          case isNewline.test(match):
+            state = 0
+            valueEnd(ctx)
+            entryEnd(ctx)
+            break
+          default:
+            state = 4
+            throw Error(`CSVError: Illegal state [row:${ctx.row}, col:${ctx.col}]`)
+        }
+        break
+      case 3: // delimited input
+        switch (true) {
+          case match === ctx.options.delimiter:
+            state = 4
+            break
+          default:
+            state = 3
+            ctx.value += match
+            break
+        }
+        break
+      case 4: // escaped or closing delimiter
+        switch (true) {
+          case match === ctx.options.delimiter:
+            state = 3
+            ctx.value += match
+            break
+          case match === ctx.options.separator:
+            state = 0
+            valueEnd(ctx)
+            break
+          case isNewline.test(match):
+            state = 0
+            valueEnd(ctx)
+            entryEnd(ctx)
+            break
+          default:
+            throw Error(`CSVError: Illegal state [row:${ctx.row}, col:${ctx.col}]`)
+        }
+        break
+    }
+  }
+
+  // flush the last value
+  if (ctx.entry.length !== 0) {
+    valueEnd(ctx)
+    entryEnd(ctx)
+  }
+
+  return ctx.output
+}
+
+export function stringify (array, options = {}, replacer = v => v) {
+  const ctx = Object.create(null)
+  ctx.options = options
+  ctx.options.eof = ctx.options.eof !== undefined ? ctx.options.eof : true
+  ctx.row = 1
+  ctx.col = 1
+  ctx.output = ''
+
+  ctx.options.delimiter = ctx.options.delimiter === undefined ? '"' : options.delimiter;
+  if(ctx.options.delimiter.length > 1 || ctx.options.delimiter.length === 0)
+    throw Error(`CSVError: delimiter must be one character [${ctx.options.separator}]`)
+
+  ctx.options.separator = ctx.options.separator === undefined ? ',' : options.separator;
+  if(ctx.options.separator.length > 1 || ctx.options.separator.length === 0)
+    throw Error(`CSVError: separator must be one character [${ctx.options.separator}]`)
+
+  const needsDelimiters = new RegExp(`${escapeRegExp(ctx.options.delimiter)}|${escapeRegExp(ctx.options.separator)}|\r\n|\n|\r`)
+
+  array.forEach((row, rIdx) => {
+    let entry = ''
+    ctx.col = 1
+    row.forEach((col, cIdx) => {
+      if (typeof col === 'string') {
+        col = col.replace(new RegExp(ctx.options.delimiter, 'g'), `${ctx.options.delimiter}${ctx.options.delimiter}`)
+        col = needsDelimiters.test(col) ? `${ctx.options.delimiter}${col}${ctx.options.delimiter}` : col
+      }
+      entry += replacer(col, ctx.row, ctx.col)
+      if (cIdx !== row.length - 1) {
+        entry += ctx.options.separator
+      }
+      ctx.col++
+    })
+    switch (true) {
+      case ctx.options.eof:
+      case !ctx.options.eof && rIdx !== array.length - 1:
+        ctx.output += `${entry}\n`
+        break
+      default:
+        ctx.output += `${entry}`
+        break
+    }
+    ctx.row++
+  })
+
+  return ctx.output
+}
+
+function valueEnd (ctx) {
+  const value = ctx.options.typed ? inferType(ctx.value) : ctx.value
+  ctx.entry.push(ctx.reviver(value, ctx.row, ctx.col))
+  ctx.value = ''
+  ctx.col++
+}
+
+function entryEnd (ctx) {
+  ctx.output.push(ctx.entry)
+  ctx.entry = []
+  ctx.row++
+  ctx.col = 1
+}
+
+function inferType (value) {
+  const isNumber = /.\./
+
+  switch (true) {
+    case value === 'true':
+    case value === 'false':
+      return value === 'true'
+    case isNumber.test(value):
+      return parseFloat(value)
+    case isFinite(value):
+      return parseInt(value)
+    default:
+      return value
+  }
+}
+
+function escapeRegExp(str) {
+  return str.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&");
+}

src/io/files.js

@@ -8,6 +8,7 @@
 import * as fileSaver from 'file-saver';
 import { Renderer } from '../core/p5.Renderer';
 import { Graphics } from '../core/p5.Graphics';
+import { parse } from './csv';
 
 class HTTPError extends Error {
   status;
@@ -530,18 +531,20 @@ function files(p5, fn){
 
     try{
       let { data } = await request(path, 'text');
-      data = data.split(/\r?\n/);
 
       let ret = new p5.Table();
+      data = parse(data, {
+        separator
+      });
 
       if(header){
-        ret.columns = data.shift().split(separator);
+        ret.columns = data.shift();
       }else{
-        ret.columns = data[0].split(separator).map(() => null);
+        ret.columns = Array(data[0].length).fill(null);
       }
 
       data.forEach((line) => {
-        const row = new p5.TableRow(line, separator);
+        const row = new p5.TableRow(line);
         ret.addRow(row);
       });
 
@@ -2032,40 +2035,8 @@ function files(p5, fn){
       sep = '\t';
     }
     if (ext !== 'html') {
-      // make header if it has values
-      if (header[0] !== '0') {
-        for (let h = 0; h < header.length; h++) {
-          if (h < header.length - 1) {
-            pWriter.write(header[h] + sep);
-          } else {
-            pWriter.write(header[h]);
-          }
-        }
-        pWriter.write('\n');
-      }
-
-      // make rows
-      for (let i = 0; i < table.rows.length; i++) {
-        let j;
-        for (j = 0; j < table.rows[i].arr.length; j++) {
-          if (j < table.rows[i].arr.length - 1) {
-            //double quotes should be inserted in csv only if contains comma separated single value
-            if (ext === 'csv' && String(table.rows[i].arr[j]).includes(',')) {
-              pWriter.write('"' + table.rows[i].arr[j] + '"' + sep);
-            } else {
-              pWriter.write(table.rows[i].arr[j] + sep);
-            }
-          } else {
-            //double quotes should be inserted in csv only if contains comma separated single value
-            if (ext === 'csv' && String(table.rows[i].arr[j]).includes(',')) {
-              pWriter.write('"' + table.rows[i].arr[j] + '"');
-            } else {
-              pWriter.write(table.rows[i].arr[j]);
-            }
-          }
-        }
-        pWriter.write('\n');
-      }
+      const output = table.toString(sep);
+      pWriter.write(output);
     } else {
       // otherwise, make HTML
       pWriter.print('<html>');

src/io/p5.Table.js

@@ -4,6 +4,8 @@
  * @requires core
  */
 
+import { stringify } from './csv';
+
 function table(p5, fn){
   /**
    *  Table Options
@@ -43,6 +45,18 @@ function table(p5, fn){
       this.rows = [];
     }
 
+    toString(separator=',') {
+      let rows = this.rows.map((row) => row.arr);
+
+      if(!this.columns.some((column) => column === null)){
+        rows = [this.columns, ...rows,]
+      }
+
+      return stringify(rows, {
+        separator
+      });
+    }
+
     /**
      *  Use <a href="#/p5/addRow">addRow()</a> to add a new row of data to a <a href="#/p5.Table">p5.Table</a> object. By default,
      *  an empty row is created. Typically, you would store a reference to

src/io/p5.TableRow.js

@@ -14,18 +14,12 @@ function tableRow(p5, fn){
    *
    *  @class p5.TableRow
    *  @constructor
-   *  @param {String} [str]       optional: populate the row with a
-   *                              string of values, separated by the
-   *                              separator
-   *  @param {String} [separator] comma separated values (csv) by default
+   *  @param {any[]} row         optional: populate the row with an
+   *                              array of values
    */
   p5.TableRow = class {
-    constructor(str, separator){
-      let arr = [];
-      if (str) {
-        separator = separator || ',';
-        arr = str.split(separator);
-      }
+    constructor(row=[]){
+      let arr = row;
 
       this.arr = arr;
       this.obj = Object.fromEntries(arr.entries());