Transformations: Convert field types to time string number or boolean (#38517)

* outline string to time add stringToTime transformer start to add format add type and dateformat rename stringToTime to fieldConversion add more type support and use FieldNamePicker add field conversion transformation * adjust for performance feedback rename and adjust labels and widths shorten labels and null values rename to convertFieldType update test * make updates
2021-08-27 13:49:31 -07:00
parent 3c72f1678f
commit a54a139176
10 changed files with 574 additions and 11 deletions
@@ -182,4 +182,14 @@ export interface FieldNamePickerConfigSettings {
   * information, including validation etc
   */
  info?: ComponentType<FieldNamePickerInfoProps> | null;
+
+  /**
+   * Sets the width to a pixel value.
+   */
+  width?: number;
+
+  /**
+   * Placeholder text to display when nothing is selected.
+   */
+  placeholderText?: string;
 }
@@ -13,3 +13,4 @@ export { RegexpOrNamesMatcherOptions, ByNamesMatcherOptions, ByNamesMatcherMode
 export { RenameByRegexTransformerOptions } from './transformers/renameByRegex';
 export { outerJoinDataFrames } from './transformers/joinDataFrames';
 export * from './transformers/histogram';
+export { ensureTimeField } from './transformers/convertFieldType';
@@ -18,6 +18,7 @@ import { mergeTransformer } from './transformers/merge';
 import { renameByRegexTransformer } from './transformers/renameByRegex';
 import { filterByValueTransformer } from './transformers/filterByValue';
 import { histogramTransformer } from './transformers/histogram';
+import { convertFieldTypeTransformer } from './transformers/convertFieldType';

 export const standardTransformers = {
  noopTransformer,
@@ -41,4 +42,5 @@ export const standardTransformers = {
  mergeTransformer,
  renameByRegexTransformer,
  histogramTransformer,
+  convertFieldTypeTransformer,
 };
@@ -0,0 +1,235 @@
+import { toDataFrame } from '../../dataframe/processDataFrame';
+import { FieldType } from '../../types/dataFrame';
+import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
+import { ArrayVector } from '../../vector';
+import { ensureTimeField, convertFieldType, convertFieldTypes, convertFieldTypeTransformer } from './convertFieldType';
+
+describe('field convert type', () => {
+  it('will parse properly formatted strings to time', () => {
+    const options = { targetField: 'proper dates', destinationType: FieldType.time };
+
+    const stringTime = {
+      name: 'proper dates',
+      type: FieldType.string,
+      values: new ArrayVector([
+        '2021-07-19 00:00:00.000',
+        '2021-07-23 00:00:00.000',
+        '2021-07-25 00:00:00.000',
+        '2021-08-01 00:00:00.000',
+        '2021-08-02 00:00:00.000',
+      ]),
+      config: {},
+    };
+
+    const timefield = convertFieldType(stringTime, options);
+    expect(timefield).toEqual({
+      name: 'proper dates',
+      type: FieldType.time,
+      values: new ArrayVector([1626674400000, 1627020000000, 1627192800000, 1627797600000, 1627884000000]),
+      config: {},
+    });
+  });
+  it('will parse string time to specified format in time', () => {
+    const options = { targetField: 'format to year', destinationType: FieldType.time, dateFormat: 'YYYY' };
+
+    const yearFormat = {
+      name: 'format to year',
+      type: FieldType.string,
+      values: new ArrayVector([
+        '2017-07-19 00:00:00.000',
+        '2018-07-23 00:00:00.000',
+        '2019-07-25 00:00:00.000',
+        '2020-08-01 00:00:00.000',
+        '2021-08-02 00:00:00.000',
+      ]),
+      config: {},
+    };
+
+    const timefield = convertFieldType(yearFormat, options);
+    expect(timefield).toEqual({
+      name: 'format to year',
+      type: FieldType.time,
+      values: new ArrayVector([1483246800000, 1514782800000, 1546318800000, 1577854800000, 1609477200000]),
+      config: {},
+    });
+  });
+
+  it('will not parse improperly formatted date strings', () => {
+    const options = { targetField: 'misformatted dates', destinationType: FieldType.time };
+
+    const misformattedStrings = {
+      name: 'misformatted dates',
+      type: FieldType.string,
+      values: new ArrayVector(['2021/08-01 00:00.00:000', '2021/08/01 00.00-000', '2021/08-01 00:00.00:000']),
+      config: { unit: 'time' },
+    };
+
+    const timefield = convertFieldType(misformattedStrings, options);
+    expect(timefield).toEqual({
+      name: 'misformatted dates',
+      type: FieldType.time,
+      values: new ArrayVector([null, null, null]),
+      config: { unit: 'time' },
+    });
+  });
+
+  it('can convert strings to numbers', () => {
+    const options = { targetField: 'stringy nums', destinationType: FieldType.number };
+
+    const stringyNumbers = {
+      name: 'stringy nums',
+      type: FieldType.string,
+      values: new ArrayVector(['10', '12', '30', '14', '10']),
+      config: {},
+    };
+
+    const numbers = convertFieldType(stringyNumbers, options);
+
+    expect(numbers).toEqual({
+      name: 'stringy nums',
+      type: FieldType.number,
+      values: new ArrayVector([10, 12, 30, 14, 10]),
+      config: {},
+    });
+  });
+});
+
+describe('field convert types transformer', () => {
+  beforeAll(() => {
+    mockTransformationsRegistry([convertFieldTypeTransformer]);
+  });
+  it('can convert multiple fields', () => {
+    const options = {
+      conversions: [
+        { targetField: 'stringy nums', destinationType: FieldType.number },
+        { targetField: 'proper dates', destinationType: FieldType.time },
+      ],
+    };
+
+    const stringyNumbers = toDataFrame({
+      fields: [
+        { name: 'A', type: FieldType.number, values: [1, 2, 3, 4, 5] },
+        {
+          name: 'proper dates',
+          type: FieldType.string,
+          values: [
+            '2021-07-19 00:00:00.000',
+            '2021-07-23 00:00:00.000',
+            '2021-07-25 00:00:00.000',
+            '2021-08-01 00:00:00.000',
+            '2021-08-02 00:00:00.000',
+          ],
+        },
+        { name: 'stringy nums', type: FieldType.string, values: ['10', '12', '30', '14', '10'] },
+      ],
+    });
+
+    const numbers = convertFieldTypes(options, [stringyNumbers]);
+    expect(
+      numbers[0].fields.map((f) => ({
+        type: f.type,
+        values: f.values.toArray(),
+      }))
+    ).toEqual([
+      { type: FieldType.number, values: [1, 2, 3, 4, 5] },
+      {
+        type: FieldType.time,
+        values: [1626674400000, 1627020000000, 1627192800000, 1627797600000, 1627884000000],
+      },
+      {
+        type: FieldType.number,
+        values: [10, 12, 30, 14, 10],
+      },
+    ]);
+  });
+
+  it('will convert field to booleans', () => {
+    const options = {
+      conversions: [
+        { targetField: 'numbers', destinationType: FieldType.boolean },
+        { targetField: 'strings', destinationType: FieldType.boolean },
+      ],
+    };
+
+    const comboTypes = toDataFrame({
+      fields: [
+        { name: 'numbers', type: FieldType.number, values: [-100, 0, 1, null, NaN] },
+        {
+          name: 'strings',
+          type: FieldType.string,
+          values: ['true', 'false', '0', '99', '2021-08-02 00:00:00.000'],
+        },
+      ],
+    });
+
+    const booleans = convertFieldTypes(options, [comboTypes]);
+    expect(
+      booleans[0].fields.map((f) => ({
+        type: f.type,
+        values: f.values.toArray(),
+      }))
+    ).toEqual([
+      {
+        type: FieldType.boolean,
+        values: [true, false, true, false, false],
+      },
+      { type: FieldType.boolean, values: [true, true, true, true, true] },
+    ]);
+  });
+
+  it('will convert field to strings', () => {
+    const options = {
+      conversions: [{ targetField: 'numbers', destinationType: FieldType.string }],
+    };
+
+    const comboTypes = toDataFrame({
+      fields: [
+        { name: 'numbers', type: FieldType.number, values: [-100, 0, 1, null, NaN] },
+        {
+          name: 'strings',
+          type: FieldType.string,
+          values: ['true', 'false', '0', '99', '2021-08-02 00:00:00.000'],
+        },
+      ],
+    });
+
+    const stringified = convertFieldTypes(options, [comboTypes]);
+    expect(
+      stringified[0].fields.map((f) => ({
+        type: f.type,
+        values: f.values.toArray(),
+      }))
+    ).toEqual([
+      {
+        type: FieldType.string,
+        values: ['-100', '0', '1', 'null', 'NaN'],
+      },
+      {
+        type: FieldType.string,
+        values: ['true', 'false', '0', '99', '2021-08-02 00:00:00.000'],
+      },
+    ]);
+  });
+});
+
+describe('ensureTimeField', () => {
+  it('will make the field have a type of time if already a number', () => {
+    const stringTime = toDataFrame({
+      fields: [
+        {
+          name: 'proper dates',
+          type: FieldType.number,
+          values: [1626674400000, 1627020000000, 1627192800000, 1627797600000, 1627884000000],
+        },
+        { name: 'A', type: FieldType.number, values: [1, 2, 3, 4, 5] },
+      ],
+    });
+
+    expect(ensureTimeField(stringTime.fields[0])).toEqual({
+      config: {},
+      name: 'proper dates',
+      type: FieldType.time,
+      values: new ArrayVector([1626674400000, 1627020000000, 1627192800000, 1627797600000, 1627884000000]),
+    });
+  });
+});
@@ -0,0 +1,170 @@
+import { SynchronousDataTransformerInfo } from '../../types';
+import { map } from 'rxjs/operators';
+
+import { DataTransformerID } from './ids';
+import { DataFrame, Field, FieldType } from '../../types/dataFrame';
+import { dateTimeParse } from '../../datetime';
+import { ArrayVector } from '../../vector';
+
+export interface ConvertFieldTypeTransformerOptions {
+  conversions: ConvertFieldTypeOptions[];
+}
+
+export interface ConvertFieldTypeOptions {
+  targetField?: string;
+  destinationType?: FieldType;
+  dateFormat?: string;
+}
+
+/**
+ * @alpha
+ */
+export const convertFieldTypeTransformer: SynchronousDataTransformerInfo<ConvertFieldTypeTransformerOptions> = {
+  id: DataTransformerID.convertFieldType,
+  name: 'Convert field type',
+  description: 'Convert a field to a specified field type',
+  defaultOptions: {
+    fields: {},
+    conversions: [{ targetField: undefined, destinationType: undefined, dateFormat: undefined }],
+  },
+
+  operator: (options) => (source) => source.pipe(map((data) => convertFieldTypeTransformer.transformer(options)(data))),
+
+  transformer: (options: ConvertFieldTypeTransformerOptions) => (data: DataFrame[]) => {
+    if (!Array.isArray(data) || data.length === 0) {
+      return data;
+    }
+    const timeParsed = convertFieldTypes(options, data);
+    if (!timeParsed) {
+      return [];
+    }
+    return timeParsed;
+  },
+};
+
+/**
+ * @alpha
+ */
+export function convertFieldTypes(options: ConvertFieldTypeTransformerOptions, frames: DataFrame[]): DataFrame[] {
+  if (!options.conversions.length) {
+    return frames;
+  }
+
+  const frameCopy: DataFrame[] = [];
+
+  frames.forEach((frame) => {
+    for (let fieldIdx = 0; fieldIdx < frame.fields.length; fieldIdx++) {
+      let field = frame.fields[fieldIdx];
+      for (let cIdx = 0; cIdx < options.conversions.length; cIdx++) {
+        if (field.name === options.conversions[cIdx].targetField) {
+          //check in about matchers with Ryan
+          const conversion = options.conversions[cIdx];
+          frame.fields[fieldIdx] = convertFieldType(field, conversion);
+          break;
+        }
+      }
+    }
+    frameCopy.push(frame);
+  });
+  return frameCopy;
+}
+
+export function convertFieldType(field: Field, opts: ConvertFieldTypeOptions): Field {
+  switch (opts.destinationType) {
+    case FieldType.time:
+      return ensureTimeField(field, opts.dateFormat);
+    case FieldType.number:
+      return fieldToNumberField(field);
+    case FieldType.string:
+      return fieldToStringField(field);
+    case FieldType.boolean:
+      return fieldToBooleanField(field);
+    default:
+      return field;
+  }
+}
+
+export function fieldToTimeField(field: Field, dateFormat?: string): Field {
+  let opts = dateFormat ? { format: dateFormat } : undefined;
+
+  const timeValues = field.values.toArray().slice();
+
+  for (let t = 0; t < timeValues.length; t++) {
+    if (timeValues[t]) {
+      let parsed = dateTimeParse(timeValues[t], opts).valueOf();
+      timeValues[t] = Number.isFinite(parsed) ? parsed : null;
+    } else {
+      timeValues[t] = null;
+    }
+  }
+
+  return {
+    ...field,
+    type: FieldType.time,
+    values: new ArrayVector(timeValues),
+  };
+}
+
+function fieldToNumberField(field: Field): Field {
+  const numValues = field.values.toArray().slice();
+
+  for (let n = 0; n < numValues.length; n++) {
+    if (numValues[n]) {
+      let number = +numValues[n];
+      numValues[n] = Number.isFinite(number) ? number : null;
+    } else {
+      numValues[n] = null;
+    }
+  }
+
+  return {
+    ...field,
+    type: FieldType.number,
+    values: new ArrayVector(numValues),
+  };
+}
+
+function fieldToBooleanField(field: Field): Field {
+  const booleanValues = field.values.toArray().slice();
+
+  for (let b = 0; b < booleanValues.length; b++) {
+    booleanValues[b] = Boolean(booleanValues[b]);
+  }
+
+  return {
+    ...field,
+    type: FieldType.boolean,
+    values: new ArrayVector(booleanValues),
+  };
+}
+
+function fieldToStringField(field: Field): Field {
+  const stringValues = field.values.toArray().slice();
+
+  for (let s = 0; s < stringValues.length; s++) {
+    stringValues[s] = `${stringValues[s]}`;
+  }
+
+  return {
+    ...field,
+    type: FieldType.string,
+    values: new ArrayVector(stringValues),
+  };
+}
+
+/**
+ * @alpha
+ */
+export function ensureTimeField(field: Field, dateFormat?: string): Field {
+  const firstValueTypeIsNumber = typeof field.values.get(0) === 'number';
+  if (field.type === FieldType.time && firstValueTypeIsNumber) {
+    return field; //already time
+  }
+  if (firstValueTypeIsNumber) {
+    return {
+      ...field,
+      type: FieldType.time, //assumes it should be time
+    };
+  }
+  return fieldToTimeField(field, dateFormat);
+}
@@ -26,4 +26,5 @@ export enum DataTransformerID {
  configFromData = 'configFromData',
  rowsToFields = 'rowsToFields',
  prepareTimeSeries = 'prepareTimeSeries',
+  convertFieldType = 'convertFieldType',
 }