cleanup and guess all columns

2019-03-21 21:52:58 -07:00
parent 50ebc768c8
commit 7498de044c
7 changed files with 228 additions and 83 deletions
@@ -1,4 +1,6 @@
-import { parseCSV, toTableData } from './processTableData';
+import { parseCSV, toTableData, guessColumnTypes, guessColumnTypeFromValue } from './processTableData';
+import { ColumnType } from '../types/data';
+import moment from 'moment';

 describe('processTableData', () => {
  describe('basic processing', () => {
@@ -20,23 +22,23 @@ describe('processTableData', () => {
 });

 describe('toTableData', () => {
-  it('converts timeseries to table skipping nulls', () => {
+  it('converts timeseries to table ', () => {
    const input1 = {
      target: 'Field Name',
      datapoints: [[100, 1], [200, 2]],
    };
+    let table = toTableData(input1);
+    expect(table.columns[0].text).toBe(input1.target);
+    expect(table.rows).toBe(input1.datapoints);
+
+    // Should fill a default name if target is empty
    const input2 = {
      // without target
      target: '',
      datapoints: [[100, 1], [200, 2]],
    };
-    const data = toTableData([null, input1, input2, null, null]);
-    expect(data.length).toBe(2);
-    expect(data[0].columns[0].text).toBe(input1.target);
-    expect(data[0].rows).toBe(input1.datapoints);
-
-    // Default name
-    expect(data[1].columns[0].text).toEqual('Value');
+    table = toTableData(input2);
+    expect(table.columns[0].text).toEqual('Value');
  });

  it('keeps tableData unchanged', () => {
@@ -44,15 +46,42 @@ describe('toTableData', () => {
      columns: [{ text: 'A' }, { text: 'B' }, { text: 'C' }],
      rows: [[100, 'A', 1], [200, 'B', 2], [300, 'C', 3]],
    };
-    const data = toTableData([null, input, null, null]);
-    expect(data.length).toBe(1);
-    expect(data[0]).toBe(input);
+    const table = toTableData(input);
+    expect(table).toBe(input);
  });

-  it('supports null values OK', () => {
-    expect(toTableData([null, null, null, null])).toEqual([]);
-    expect(toTableData(undefined)).toEqual([]);
-    expect(toTableData((null as unknown) as any[])).toEqual([]);
-    expect(toTableData([])).toEqual([]);
+  it('Guess Colum Types from value', () => {
+    expect(guessColumnTypeFromValue(1)).toBe(ColumnType.number);
+    expect(guessColumnTypeFromValue(1.234)).toBe(ColumnType.number);
+    expect(guessColumnTypeFromValue(3.125e7)).toBe(ColumnType.number);
+    expect(guessColumnTypeFromValue('1')).toBe(ColumnType.string);
+    expect(guessColumnTypeFromValue('1.234')).toBe(ColumnType.string);
+    expect(guessColumnTypeFromValue('3.125e7')).toBe(ColumnType.string);
+    expect(guessColumnTypeFromValue(true)).toBe(ColumnType.boolean);
+    expect(guessColumnTypeFromValue(false)).toBe(ColumnType.boolean);
+    expect(guessColumnTypeFromValue(new Date())).toBe(ColumnType.time);
+    expect(guessColumnTypeFromValue(moment())).toBe(ColumnType.time);
+  });
+
+  it('Guess Colum Types from strings', () => {
+    expect(guessColumnTypeFromValue('1', true)).toBe(ColumnType.number);
+    expect(guessColumnTypeFromValue('1.234', true)).toBe(ColumnType.number);
+    expect(guessColumnTypeFromValue('3.125e7', true)).toBe(ColumnType.number);
+    expect(guessColumnTypeFromValue('True', true)).toBe(ColumnType.boolean);
+    expect(guessColumnTypeFromValue('FALSE', true)).toBe(ColumnType.boolean);
+    expect(guessColumnTypeFromValue('true', true)).toBe(ColumnType.boolean);
+    expect(guessColumnTypeFromValue('xxxx', true)).toBe(ColumnType.string);
+  });
+
+  it('Guess Colum Types from table', () => {
+    const table = {
+      columns: [{ text: 'A (number)' }, { text: 'B (strings)' }, { text: 'C (nulls)' }, { text: 'Time' }],
+      rows: [[123, null, null, '2000'], [null, '123', null, 'XXX']],
+    };
+    const norm = guessColumnTypes(table);
+    expect(norm.columns[0].type).toBe(ColumnType.number);
+    expect(norm.columns[1].type).toBe(ColumnType.string);
+    expect(norm.columns[2].type).toBeUndefined();
+    expect(norm.columns[3].type).toBe(ColumnType.time); // based on name
  });
 });
@@ -1,6 +1,8 @@
 // Libraries
 import isNumber from 'lodash/isNumber';
 import isString from 'lodash/isString';
+import isBoolean from 'lodash/isBoolean';
+import moment from 'moment';

 import Papa, { ParseError, ParseMeta } from 'papaparse';

@@ -159,77 +161,112 @@ export const getFirstTimeColumn = (table: TableData): number => {
  return -1;
 };

+// PapaParse Dynamic Typing regex:
+// https://github.com/mholt/PapaParse/blob/master/papaparse.js#L998
+const NUMBER = /^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i;
+
 /**
- * @returns a table Returns a copy of the table with the best guess for each column type
+ * Given a value this will guess the best column type
+ *
+ * TODO: better Date/Time support!  Look for standard date strings?
 */
-export const guessColumnTypes = (table: TableData): TableData => {
-  let changed = false;
-  const columns = table.columns.map((column, index) => {
-    if (!column.type) {
-      // 1. Use the column name to guess
-      if (column.text) {
-        const name = column.text.toLowerCase();
-        if (name === 'date' || name === 'time') {
-          changed = true;
-          return {
-            ...column,
-            type: ColumnType.time,
-          };
+export function guessColumnTypeFromValue(v: any, parseString?: boolean): ColumnType {
+  if (isNumber(v)) {
+    return ColumnType.number;
+  }
+
+  if (isString(v)) {
+    if (parseString) {
+      const c0 = v[0].toLowerCase();
+      if (c0 === 't' || c0 === 'f') {
+        if (v === 'true' || v === 'TRUE' || v === 'True' || v === 'false' || v === 'FALSE' || v === 'False') {
+          return ColumnType.boolean;
        }
      }

-      // 2. Check the first non-null value
-      for (let i = 0; i < table.rows.length; i++) {
-        const v = table.rows[i][index];
-        if (v !== null) {
-          let type: ColumnType | undefined;
-          if (isNumber(v)) {
-            type = ColumnType.number;
-          } else if (isString(v)) {
-            type = ColumnType.string;
-          }
-          if (type) {
-            changed = true;
-            return {
-              ...column,
-              type,
-            };
-          }
-          break;
-        }
+      if (NUMBER.test(v)) {
+        return ColumnType.number;
      }
    }
-    return column;
-  });
-  if (changed) {
-    return {
-      ...table,
-      columns,
-    };
+    return ColumnType.string;
  }
+
+  if (isBoolean(v)) {
+    return ColumnType.boolean;
+  }
+
+  if (v instanceof Date || v instanceof moment) {
+    return ColumnType.time;
+  }
+
+  return ColumnType.other;
+}
+
+/**
+ * Looks at the data to guess the column type.  This ignores any existing setting
+ */
+function guessColumnTypeFromTable(table: TableData, index: number, parseString?: boolean): ColumnType | undefined {
+  const column = table.columns[index];
+
+  // 1. Use the column name to guess
+  if (column.text) {
+    const name = column.text.toLowerCase();
+    if (name === 'date' || name === 'time') {
+      return ColumnType.time;
+    }
+  }
+
+  // 2. Check the first non-null value
+  for (let i = 0; i < table.rows.length; i++) {
+    const v = table.rows[i][index];
+    if (v !== null) {
+      return guessColumnTypeFromValue(v, parseString);
+    }
+  }
+
+  // Could not find anything
+  return undefined;
+}
+
+/**
+ * @returns a table Returns a copy of the table with the best guess for each column type
+ * If the table already has column types defined, they will be used
+ */
+export const guessColumnTypes = (table: TableData): TableData => {
+  for (let i = 0; i < table.columns.length; i++) {
+    if (!table.columns[i].type) {
+      // Somethign is missing a type return a modified copy
+      return {
+        ...table,
+        columns: table.columns.map((column, index) => {
+          if (column.type) {
+            return column;
+          }
+          // Replace it with a calculated version
+          return {
+            ...column,
+            type: guessColumnTypeFromTable(table, index),
+          };
+        }),
+      };
+    }
+  }
+  // No changes necessary
  return table;
 };

 export const isTableData = (data: any): data is TableData => data && data.hasOwnProperty('columns');

-export const toTableData = (results?: any[]): TableData[] => {
-  if (!results) {
-    return [];
+export const toTableData = (data: any): TableData => {
+  if (data.hasOwnProperty('columns')) {
+    return data as TableData;
  }
-
-  return results
-    .filter(d => !!d)
-    .map(data => {
-      if (data.hasOwnProperty('columns')) {
-        return data as TableData;
-      }
-      if (data.hasOwnProperty('datapoints')) {
-        return convertTimeSeriesToTableData(data);
-      }
-      // TODO, try to convert JSON to table?
-      console.warn('Can not convert', data);
-      throw new Error('Unsupported data format');
-    });
+  if (data.hasOwnProperty('datapoints')) {
+    return convertTimeSeriesToTableData(data);
+  }
+  // TODO, try to convert JSON/Array to table?
+  console.warn('Can not convert', data);
+  throw new Error('Unsupported data format');
 };

 export function sortTableData(data: TableData, sortIndex?: number, reverse = false): TableData {