transform: changes to support sdk v0.2.0 (#20426)

and update vendor
This commit is contained in:
Kyle Brandt
2019-11-18 11:40:40 -05:00
committed by GitHub
parent 7a8c6a9b54
commit c9309edbb0
142 changed files with 4305 additions and 928 deletions
+194 -71
View File
@@ -74,21 +74,20 @@ func buildArrowFields(f *Frame) ([]arrow.Field, error) {
arrowFields := make([]arrow.Field, len(f.Fields))
for i, field := range f.Fields {
t, err := fieldToArrow(field.Type)
t, nullable, err := fieldToArrow(field)
if err != nil {
return nil, err
}
fieldMeta := map[string]string{
"name": field.Name,
"type": field.Type.String(),
}
arrowFields[i] = arrow.Field{
Name: field.Name,
Type: t,
Metadata: arrow.MetadataFrom(fieldMeta),
Nullable: true,
Nullable: nullable,
}
}
@@ -101,56 +100,45 @@ func buildArrowColumns(f *Frame, arrowFields []arrow.Field) ([]array.Column, err
columns := make([]array.Column, len(f.Fields))
for fieldIdx, field := range f.Fields {
switch field.Type {
case FieldTypeNumber:
columns[fieldIdx] = *buildFloatColumn(pool, arrowFields[fieldIdx], field.Vector.(*floatVector))
case FieldTypeTime:
columns[fieldIdx] = *buildTimeColumn(pool, arrowFields[fieldIdx], field.Vector.(*timeVector))
switch v := field.Vector.(type) {
case *intVector:
columns[fieldIdx] = *buildIntColumn(pool, arrowFields[fieldIdx], v)
case *nullableIntVector:
columns[fieldIdx] = *buildNullableIntColumn(pool, arrowFields[fieldIdx], v)
case *uintVector:
columns[fieldIdx] = *buildUIntColumn(pool, arrowFields[fieldIdx], v)
case *nullableUintVector:
columns[fieldIdx] = *buildNullableUIntColumn(pool, arrowFields[fieldIdx], v)
case *stringVector:
columns[fieldIdx] = *buildStringColumn(pool, arrowFields[fieldIdx], v)
case *nullableStringVector:
columns[fieldIdx] = *buildNullableStringColumn(pool, arrowFields[fieldIdx], v)
case *floatVector:
columns[fieldIdx] = *buildFloatColumn(pool, arrowFields[fieldIdx], v)
case *nullableFloatVector:
columns[fieldIdx] = *buildNullableFloatColumn(pool, arrowFields[fieldIdx], v)
case *boolVector:
columns[fieldIdx] = *buildBoolColumn(pool, arrowFields[fieldIdx], v)
case *nullableBoolVector:
columns[fieldIdx] = *buildNullableBoolColumn(pool, arrowFields[fieldIdx], v)
case *timeVector:
columns[fieldIdx] = *buildTimeColumn(pool, arrowFields[fieldIdx], v)
case *nullableTimeVector:
columns[fieldIdx] = *buildNullableTimeColumn(pool, arrowFields[fieldIdx], v)
default:
return nil, fmt.Errorf("unsupported field type: %s", field.Type)
return nil, fmt.Errorf("unsupported field vector type for conversion to arrow: %T", v)
}
}
return columns, nil
}
func buildFloatColumn(pool memory.Allocator, field arrow.Field, vec *floatVector) *array.Column {
builder := array.NewFloat64Builder(pool)
defer builder.Release()
for _, v := range *vec {
if v == nil {
builder.AppendNull()
continue
}
builder.Append(*v)
}
chunked := array.NewChunked(field.Type, []array.Interface{builder.NewArray()})
defer chunked.Release()
return array.NewColumn(field, chunked)
}
func buildTimeColumn(pool memory.Allocator, field arrow.Field, vec *timeVector) *array.Column {
builder := array.NewTimestampBuilder(pool, &arrow.TimestampType{
Unit: arrow.Nanosecond,
})
defer builder.Release()
for _, v := range *vec {
if v == nil {
builder.AppendNull()
continue
}
builder.Append(arrow.Timestamp((*v).UnixNano()))
}
chunked := array.NewChunked(field.Type, []array.Interface{builder.NewArray()})
defer chunked.Release()
return array.NewColumn(field, chunked)
}
// buildArrowSchema builds an Arrow schema for a DataFrame.
func buildArrowSchema(f *Frame, fs []arrow.Field) (*arrow.Schema, error) {
tableMetaMap := map[string]string{
@@ -167,29 +155,56 @@ func buildArrowSchema(f *Frame, fs []arrow.Field) (*arrow.Schema, error) {
return arrow.NewSchema(fs, &tableMeta), nil
}
// fieldToArrow returns the corresponding Arrow primitive type to the fields'
// fieldToArrow returns the corresponding Arrow primitive type and nullable property to the fields'
// Vector primitives.
func fieldToArrow(f FieldType) (arrow.DataType, error) {
switch f {
case FieldTypeString:
return &arrow.StringType{}, nil
case FieldTypeNumber:
return &arrow.Float64Type{}, nil
case FieldTypeTime:
return &arrow.TimestampType{}, nil
func fieldToArrow(f *Field) (arrow.DataType, bool, error) {
switch f.Vector.(type) {
case *stringVector:
return &arrow.StringType{}, false, nil
case *nullableStringVector:
return &arrow.StringType{}, true, nil
case *intVector:
return &arrow.Int64Type{}, false, nil
case *nullableIntVector:
return &arrow.Int64Type{}, true, nil
case *uintVector:
return &arrow.Uint64Type{}, false, nil
case *nullableUintVector:
return &arrow.Uint64Type{}, true, nil
case *floatVector:
return &arrow.Float64Type{}, false, nil
case *nullableFloatVector:
return &arrow.Float64Type{}, true, nil
case *boolVector:
return &arrow.BooleanType{}, false, nil
case *nullableBoolVector:
return &arrow.BooleanType{}, true, nil
case *timeVector:
return &arrow.TimestampType{}, false, nil
case *nullableTimeVector:
return &arrow.TimestampType{}, true, nil
default:
return nil, fmt.Errorf("unsupported type: %s", f)
return nil, false, fmt.Errorf("unsupported type for conversion to arrow: %T", f.Vector)
}
}
// UnMarshalArrow converts a byte representation of an arrow table to a Frame
// TODO: Break up this function.
func UnMarshalArrow(b []byte) (*Frame, error) {
fB := filebuffer.New(b)
fR, err := ipc.NewFileReader(fB)
defer fR.Close()
if err != nil {
return nil, err
}
defer fR.Close()
schema := fR.Schema()
metaData := schema.Metadata()
frame := &Frame{}
@@ -208,24 +223,55 @@ func UnMarshalArrow(b []byte) (*Frame, error) {
return nil, err
}
}
for _, field := range schema.Fields() {
nullable := make([]bool, len(schema.Fields()))
for idx, field := range schema.Fields() {
sdkField := &Field{
Name: field.Name,
}
nullable[idx] = field.Nullable
switch field.Type.ID() {
case arrow.STRING:
if nullable[idx] {
sdkField.Vector = newNullableStringVector(0)
break
}
sdkField.Vector = newStringVector(0)
case arrow.INT64:
if nullable[idx] {
sdkField.Vector = newNullableIntVector(0)
break
}
sdkField.Vector = newIntVector(0)
case arrow.UINT64:
if nullable[idx] {
sdkField.Vector = newNullableUintVector(0)
break
}
sdkField.Vector = newUintVector(0)
case arrow.FLOAT64:
sdkField.Type = FieldTypeNumber
sdkField.Vector = newVector(FieldTypeNumber, 0)
if nullable[idx] {
sdkField.Vector = newNullableFloatVector(0)
break
}
sdkField.Vector = newFloatVector(0)
case arrow.BOOL:
if nullable[idx] {
sdkField.Vector = newNullableBoolVector(0)
break
}
sdkField.Vector = newBoolVector(0)
case arrow.TIMESTAMP:
sdkField.Type = FieldTypeTime
sdkField.Vector = newVector(FieldTypeTime, 0)
if nullable[idx] {
sdkField.Vector = newNullableTimeVector(0)
break
}
sdkField.Vector = newTimeVector(0)
default:
return nil, fmt.Errorf("unsupported arrow type %s for conversion", field.Type)
return nil, fmt.Errorf("unsupported conversion from arrow to sdk type for arrow type %v", field.Type.ID().String())
}
frame.Fields = append(frame.Fields, sdkField)
}
rIdx := 0
for {
record, err := fR.Read()
if err == io.EOF {
@@ -237,23 +283,100 @@ func UnMarshalArrow(b []byte) (*Frame, error) {
for i := 0; i < len(frame.Fields); i++ {
col := record.Column(i)
switch col.DataType().ID() {
case arrow.STRING:
v := array.NewStringData(col.Data())
for rIdx := 0; rIdx < col.Len(); rIdx++ {
if nullable[i] {
if v.IsNull(rIdx) {
var ns *string
frame.Fields[i].Vector.Append(ns)
continue
}
rv := v.Value(rIdx)
frame.Fields[i].Vector.Append(&rv)
continue
}
frame.Fields[i].Vector.Append(v.Value(rIdx))
}
case arrow.INT64:
v := array.NewInt64Data(col.Data())
for rIdx := 0; rIdx < col.Len(); rIdx++ {
if nullable[i] {
if v.IsNull(rIdx) {
var ns *int64
frame.Fields[i].Vector.Append(ns)
continue
}
rv := v.Value(rIdx)
frame.Fields[i].Vector.Append(&rv)
continue
}
frame.Fields[i].Vector.Append(v.Value(rIdx))
}
case arrow.UINT64:
v := array.NewUint64Data(col.Data())
for rIdx := 0; rIdx < col.Len(); rIdx++ {
if nullable[i] {
if v.IsNull(rIdx) {
var ns *uint64
frame.Fields[i].Vector.Append(ns)
continue
}
rv := v.Value(rIdx)
frame.Fields[i].Vector.Append(&rv)
continue
}
frame.Fields[i].Vector.Append(v.Value(rIdx))
}
case arrow.FLOAT64:
v := array.NewFloat64Data(col.Data())
for _, f := range v.Float64Values() {
vF := f
frame.Fields[i].Vector.Append(&vF)
for vIdx, f := range v.Float64Values() {
if nullable[i] {
if v.IsNull(vIdx) {
var nf *float64
frame.Fields[i].Vector.Append(nf)
continue
}
vF := f
frame.Fields[i].Vector.Append(&vF)
continue
}
frame.Fields[i].Vector.Append(f)
}
case arrow.BOOL:
v := array.NewBooleanData(col.Data())
for sIdx := 0; sIdx < col.Len(); sIdx++ {
if nullable[i] {
if v.IsNull(sIdx) {
var ns *bool
frame.Fields[i].Vector.Append(ns)
continue
}
vB := v.Value(sIdx)
frame.Fields[i].Vector.Append(&vB)
continue
}
frame.Fields[i].Vector.Append(v.Value(sIdx))
}
case arrow.TIMESTAMP:
v := array.NewTimestampData(col.Data())
for _, ts := range v.TimestampValues() {
for vIdx, ts := range v.TimestampValues() {
t := time.Unix(0, int64(ts)) // nanosecond assumption
frame.Fields[i].Vector.Append(&t)
if nullable[i] {
if v.IsNull(vIdx) {
var nt *time.Time
frame.Fields[i].Vector.Append(nt)
continue
}
frame.Fields[i].Vector.Append(&t)
continue
}
frame.Fields[i].Vector.Append(t)
}
default:
return nil, fmt.Errorf("unsupported arrow type %s for conversion", col.DataType().ID())
}
}
rIdx++
}
return frame, nil
@@ -0,0 +1,203 @@
package dataframe
import (
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/array"
"github.com/apache/arrow/go/arrow/memory"
)
func buildStringColumn(pool memory.Allocator, field arrow.Field, vec *stringVector) *array.Column {
builder := array.NewStringBuilder(pool)
defer builder.Release()
for _, v := range *vec {
builder.Append(v)
}
chunked := array.NewChunked(field.Type, []array.Interface{builder.NewArray()})
defer chunked.Release()
return array.NewColumn(field, chunked)
}
func buildNullableStringColumn(pool memory.Allocator, field arrow.Field, vec *nullableStringVector) *array.Column {
builder := array.NewStringBuilder(pool)
defer builder.Release()
for _, v := range *vec {
if v == nil {
builder.AppendNull()
continue
}
builder.Append(*v)
}
chunked := array.NewChunked(field.Type, []array.Interface{builder.NewArray()})
defer chunked.Release()
return array.NewColumn(field, chunked)
}
func buildIntColumn(pool memory.Allocator, field arrow.Field, vec *intVector) *array.Column {
builder := array.NewInt64Builder(pool)
defer builder.Release()
for _, v := range *vec {
builder.Append(v)
}
chunked := array.NewChunked(field.Type, []array.Interface{builder.NewArray()})
defer chunked.Release()
return array.NewColumn(field, chunked)
}
func buildNullableIntColumn(pool memory.Allocator, field arrow.Field, vec *nullableIntVector) *array.Column {
builder := array.NewInt64Builder(pool)
defer builder.Release()
for _, v := range *vec {
if v == nil {
builder.AppendNull()
continue
}
builder.Append(*v)
}
chunked := array.NewChunked(field.Type, []array.Interface{builder.NewArray()})
defer chunked.Release()
return array.NewColumn(field, chunked)
}
func buildUIntColumn(pool memory.Allocator, field arrow.Field, vec *uintVector) *array.Column {
builder := array.NewUint64Builder(pool)
defer builder.Release()
for _, v := range *vec {
builder.Append(v)
}
chunked := array.NewChunked(field.Type, []array.Interface{builder.NewArray()})
defer chunked.Release()
return array.NewColumn(field, chunked)
}
func buildNullableUIntColumn(pool memory.Allocator, field arrow.Field, vec *nullableUintVector) *array.Column {
builder := array.NewUint64Builder(pool)
defer builder.Release()
for _, v := range *vec {
if v == nil {
builder.AppendNull()
continue
}
builder.Append(*v)
}
chunked := array.NewChunked(field.Type, []array.Interface{builder.NewArray()})
defer chunked.Release()
return array.NewColumn(field, chunked)
}
func buildFloatColumn(pool memory.Allocator, field arrow.Field, vec *floatVector) *array.Column {
builder := array.NewFloat64Builder(pool)
defer builder.Release()
for _, v := range *vec {
builder.Append(v)
}
chunked := array.NewChunked(field.Type, []array.Interface{builder.NewArray()})
defer chunked.Release()
return array.NewColumn(field, chunked)
}
func buildNullableFloatColumn(pool memory.Allocator, field arrow.Field, vec *nullableFloatVector) *array.Column {
builder := array.NewFloat64Builder(pool)
defer builder.Release()
for _, v := range *vec {
if v == nil {
builder.AppendNull()
continue
}
builder.Append(*v)
}
chunked := array.NewChunked(field.Type, []array.Interface{builder.NewArray()})
defer chunked.Release()
return array.NewColumn(field, chunked)
}
func buildBoolColumn(pool memory.Allocator, field arrow.Field, vec *boolVector) *array.Column {
builder := array.NewBooleanBuilder(pool)
defer builder.Release()
for _, v := range *vec {
builder.Append(v)
}
chunked := array.NewChunked(field.Type, []array.Interface{builder.NewArray()})
defer chunked.Release()
return array.NewColumn(field, chunked)
}
func buildNullableBoolColumn(pool memory.Allocator, field arrow.Field, vec *nullableBoolVector) *array.Column {
builder := array.NewBooleanBuilder(pool)
defer builder.Release()
for _, v := range *vec {
if v == nil {
builder.AppendNull()
continue
}
builder.Append(*v)
}
chunked := array.NewChunked(field.Type, []array.Interface{builder.NewArray()})
defer chunked.Release()
return array.NewColumn(field, chunked)
}
func buildTimeColumn(pool memory.Allocator, field arrow.Field, vec *timeVector) *array.Column {
builder := array.NewTimestampBuilder(pool, &arrow.TimestampType{
Unit: arrow.Nanosecond,
})
defer builder.Release()
for _, v := range *vec {
builder.Append(arrow.Timestamp((v).UnixNano()))
}
chunked := array.NewChunked(field.Type, []array.Interface{builder.NewArray()})
defer chunked.Release()
return array.NewColumn(field, chunked)
}
func buildNullableTimeColumn(pool memory.Allocator, field arrow.Field, vec *nullableTimeVector) *array.Column {
builder := array.NewTimestampBuilder(pool, &arrow.TimestampType{
Unit: arrow.Nanosecond,
})
defer builder.Release()
for _, v := range *vec {
if v == nil {
builder.AppendNull()
continue
}
builder.Append(arrow.Timestamp((*v).UnixNano()))
}
chunked := array.NewChunked(field.Type, []array.Interface{builder.NewArray()})
defer chunked.Release()
return array.NewColumn(field, chunked)
}
+40 -75
View File
@@ -7,120 +7,85 @@ import (
"time"
)
// FieldType is used to describe the type of data in a field.
type FieldType int
// All valid field types.
const (
FieldTypeOther FieldType = iota
FieldTypeTime
FieldTypeNumber
FieldTypeString
FieldTypeBoolean
)
func (f FieldType) String() string {
switch f {
case FieldTypeOther:
return "other"
case FieldTypeNumber:
return "number"
case FieldTypeString:
return "string"
case FieldTypeBoolean:
return "boolean"
case FieldTypeTime:
return "time"
default:
return "unknown"
}
}
// Field represents a column of data with a specific type.
type Field struct {
Name string
Type FieldType
Vector Vector
}
// Fields is a slice of Field pointers.
type Fields []*Field
func assertFieldType(got, want FieldType) {
if got != want {
panic(fmt.Sprintf("values doesn't match the specified field type"))
}
}
// NewField returns a new instance of Field.
func NewField(name string, fieldType FieldType, values interface{}) *Field {
func NewField(name string, values interface{}) *Field {
var vec Vector
switch v := values.(type) {
case []*float64:
assertFieldType(fieldType, FieldTypeNumber)
vec = newVector(fieldType, len(v))
case []int64:
vec = newVector(v, len(v))
for i := 0; i < len(v); i++ {
vec.Set(i, v[i])
}
case []*int64:
assertFieldType(fieldType, FieldTypeNumber)
vec = newVector(fieldType, len(v))
vec = newVector(v, len(v))
for i := 0; i < len(v); i++ {
vec.Set(i, v[i])
}
case []*time.Time:
assertFieldType(fieldType, FieldTypeTime)
vec = newVector(fieldType, len(v))
case []uint64:
vec = newVector(v, len(v))
for i := 0; i < len(v); i++ {
vec.Set(i, v[i])
}
case []*string:
assertFieldType(fieldType, FieldTypeString)
vec = newVector(fieldType, len(v))
for i := 0; i < len(v); i++ {
vec.Set(i, v[i])
}
case []*bool:
assertFieldType(fieldType, FieldTypeBoolean)
vec = newVector(fieldType, len(v))
case []*uint64:
vec = newVector(v, len(v))
for i := 0; i < len(v); i++ {
vec.Set(i, v[i])
}
case []float64:
assertFieldType(fieldType, FieldTypeNumber)
vec = newVector(fieldType, len(v))
vec = newVector(v, len(v))
for i := 0; i < len(v); i++ {
vec.Set(i, &v[i])
vec.Set(i, v[i])
}
case []int64:
assertFieldType(fieldType, FieldTypeNumber)
vec = newVector(fieldType, len(v))
case []*float64:
vec = newVector(v, len(v))
for i := 0; i < len(v); i++ {
vec.Set(i, &v[i])
}
case []time.Time:
assertFieldType(fieldType, FieldTypeTime)
vec = newVector(fieldType, len(v))
for i := 0; i < len(v); i++ {
vec.Set(i, &v[i])
vec.Set(i, v[i])
}
case []string:
assertFieldType(fieldType, FieldTypeString)
vec = newVector(fieldType, len(v))
vec = newVector(v, len(v))
for i := 0; i < len(v); i++ {
vec.Set(i, &v[i])
vec.Set(i, v[i])
}
case []*string:
vec = newVector(v, len(v))
for i := 0; i < len(v); i++ {
vec.Set(i, v[i])
}
case []bool:
assertFieldType(fieldType, FieldTypeBoolean)
vec = newVector(fieldType, len(v))
vec = newVector(v, len(v))
for i := 0; i < len(v); i++ {
vec.Set(i, &v[i])
vec.Set(i, v[i])
}
case []*bool:
vec = newVector(v, len(v))
for i := 0; i < len(v); i++ {
vec.Set(i, v[i])
}
case []time.Time:
vec = newVector(v, len(v))
for i := 0; i < len(v); i++ {
vec.Set(i, v[i])
}
case []*time.Time:
vec = newVector(v, len(v))
for i := 0; i < len(v); i++ {
vec.Set(i, v[i])
}
default:
panic(fmt.Errorf("unsupported field type %T", v))
}
return &Field{
Name: name,
Type: fieldType,
Vector: vec,
}
}
+33 -11
View File
@@ -1,5 +1,10 @@
package dataframe
import (
"fmt"
"time"
)
// Vector represents a collection of Elements.
type Vector interface {
Set(idx int, i interface{})
@@ -8,17 +13,34 @@ type Vector interface {
Len() int
}
func newVector(t FieldType, n int) Vector {
switch t {
case FieldTypeNumber:
return newFloatVector(n)
case FieldTypeTime:
return newTimeVector(n)
case FieldTypeString:
return newStringVector(n)
case FieldTypeBoolean:
return newBoolVector(n)
func newVector(t interface{}, n int) (v Vector) {
switch t.(type) {
case []int64:
v = newIntVector(n)
case []*int64:
v = newNullableIntVector(n)
case []uint64:
v = newUintVector(n)
case []*uint64:
v = newNullableUintVector(n)
case []float64:
v = newFloatVector(n)
case []*float64:
v = newNullableFloatVector(n)
case []string:
v = newStringVector(n)
case []*string:
v = newNullableStringVector(n)
case []bool:
v = newBoolVector(n)
case []*bool:
v = newNullableBoolVector(n)
case []time.Time:
v = newTimeVector(n)
case []*time.Time:
v = newNullableTimeVector(n)
default:
return nil
panic(fmt.Sprintf("unsupported vector type of %T", t))
}
return
}
@@ -1,6 +1,6 @@
package dataframe
type boolVector []*bool
type boolVector []bool
func newBoolVector(l int) *boolVector {
v := make(boolVector, l)
@@ -8,11 +8,11 @@ func newBoolVector(l int) *boolVector {
}
func (v *boolVector) Set(i int, val interface{}) {
(*v)[i] = val.(*bool)
(*v)[i] = val.(bool)
}
func (v *boolVector) Append(val interface{}) {
*v = append(*v, val.(*bool))
*v = append(*v, val.(bool))
}
func (v *boolVector) At(i int) interface{} {
@@ -0,0 +1,24 @@
package dataframe
type nullableBoolVector []*bool
func newNullableBoolVector(l int) *nullableBoolVector {
v := make(nullableBoolVector, l)
return &v
}
func (v *nullableBoolVector) Set(i int, val interface{}) {
(*v)[i] = val.(*bool)
}
func (v *nullableBoolVector) Append(val interface{}) {
*v = append(*v, val.(*bool))
}
func (v *nullableBoolVector) At(i int) interface{} {
return (*v)[i]
}
func (v *nullableBoolVector) Len() int {
return len(*v)
}
@@ -1,6 +1,6 @@
package dataframe
type floatVector []*float64
type floatVector []float64
func newFloatVector(l int) *floatVector {
v := make(floatVector, l)
@@ -8,13 +8,17 @@ func newFloatVector(l int) *floatVector {
}
func (v *floatVector) Set(i int, val interface{}) {
(*v)[i] = val.(*float64)
(*v)[i] = val.(float64)
}
func (v *floatVector) Append(val interface{}) {
*v = append(*v, val.(*float64))
*v = append(*v, val.(float64))
}
func (v *floatVector) At(i int) interface{} { return (*v)[i] }
func (v *floatVector) At(i int) interface{} {
return (*v)[i]
}
func (v *floatVector) Len() int { return len(*v) }
func (v *floatVector) Len() int {
return len(*v)
}
@@ -0,0 +1,20 @@
package dataframe
type nullableFloatVector []*float64
func newNullableFloatVector(l int) *nullableFloatVector {
v := make(nullableFloatVector, l)
return &v
}
func (v *nullableFloatVector) Set(i int, val interface{}) {
(*v)[i] = val.(*float64)
}
func (v *nullableFloatVector) Append(val interface{}) {
*v = append(*v, val.(*float64))
}
func (v *nullableFloatVector) At(i int) interface{} { return (*v)[i] }
func (v *nullableFloatVector) Len() int { return len(*v) }
@@ -0,0 +1,24 @@
package dataframe
type intVector []int64
func newIntVector(l int) *intVector {
v := make(intVector, l)
return &v
}
func (v *intVector) Set(i int, val interface{}) {
(*v)[i] = val.(int64)
}
func (v *intVector) Append(val interface{}) {
*v = append(*v, val.(int64))
}
func (v *intVector) At(i int) interface{} {
return (*v)[i]
}
func (v *intVector) Len() int {
return len(*v)
}
@@ -0,0 +1,20 @@
package dataframe
type nullableIntVector []*int64
func newNullableIntVector(l int) *nullableIntVector {
v := make(nullableIntVector, l)
return &v
}
func (v *nullableIntVector) Set(i int, val interface{}) {
(*v)[i] = val.(*int64)
}
func (v *nullableIntVector) Append(val interface{}) {
*v = append(*v, val.(*int64))
}
func (v *nullableIntVector) At(i int) interface{} { return (*v)[i] }
func (v *nullableIntVector) Len() int { return len(*v) }
@@ -1,6 +1,6 @@
package dataframe
type stringVector []*string
type stringVector []string
func newStringVector(l int) *stringVector {
v := make(stringVector, l)
@@ -8,11 +8,11 @@ func newStringVector(l int) *stringVector {
}
func (v *stringVector) Set(i int, val interface{}) {
(*v)[i] = val.(*string)
(*v)[i] = val.(string)
}
func (v *stringVector) Append(val interface{}) {
*v = append(*v, val.(*string))
*v = append(*v, val.(string))
}
func (v *stringVector) At(i int) interface{} {
@@ -21,4 +21,4 @@ func (v *stringVector) At(i int) interface{} {
func (v *stringVector) Len() int {
return len(*v)
}
}
@@ -0,0 +1,24 @@
package dataframe
type nullableStringVector []*string
func newNullableStringVector(l int) *nullableStringVector {
v := make(nullableStringVector, l)
return &v
}
func (v *nullableStringVector) Set(i int, val interface{}) {
(*v)[i] = val.(*string)
}
func (v *nullableStringVector) Append(val interface{}) {
*v = append(*v, val.(*string))
}
func (v *nullableStringVector) At(i int) interface{} {
return (*v)[i]
}
func (v *nullableStringVector) Len() int {
return len(*v)
}
@@ -4,7 +4,7 @@ import (
"time"
)
type timeVector []*time.Time
type timeVector []time.Time
func newTimeVector(l int) *timeVector {
v := make(timeVector, l)
@@ -12,11 +12,11 @@ func newTimeVector(l int) *timeVector {
}
func (v *timeVector) Set(i int, val interface{}) {
(*v)[i] = val.(*time.Time)
(*v)[i] = val.(time.Time)
}
func (v *timeVector) Append(val interface{}) {
*v = append(*v, val.(*time.Time))
*v = append(*v, val.(time.Time))
}
func (v *timeVector) At(i int) interface{} {
@@ -26,3 +26,4 @@ func (v *timeVector) At(i int) interface{} {
func (v *timeVector) Len() int {
return len(*v)
}
@@ -0,0 +1,28 @@
package dataframe
import (
"time"
)
type nullableTimeVector []*time.Time
func newNullableTimeVector(l int) *nullableTimeVector {
v := make(nullableTimeVector, l)
return &v
}
func (v *nullableTimeVector) Set(i int, val interface{}) {
(*v)[i] = val.(*time.Time)
}
func (v *nullableTimeVector) Append(val interface{}) {
*v = append(*v, val.(*time.Time))
}
func (v *nullableTimeVector) At(i int) interface{} {
return (*v)[i]
}
func (v *nullableTimeVector) Len() int {
return len(*v)
}
@@ -0,0 +1,24 @@
package dataframe
type uintVector []uint64
func newUintVector(l int) *uintVector {
v := make(uintVector, l)
return &v
}
func (v *uintVector) Set(i int, val interface{}) {
(*v)[i] = val.(uint64)
}
func (v *uintVector) Append(val interface{}) {
*v = append(*v, val.(uint64))
}
func (v *uintVector) At(i int) interface{} {
return (*v)[i]
}
func (v *uintVector) Len() int {
return len(*v)
}
@@ -0,0 +1,20 @@
package dataframe
type nullableUintVector []*uint64
func newNullableUintVector(l int) *nullableUintVector {
v := make(nullableUintVector, l)
return &v
}
func (v *nullableUintVector) Set(i int, val interface{}) {
(*v)[i] = val.(*uint64)
}
func (v *nullableUintVector) Append(val interface{}) {
*v = append(*v, val.(*uint64))
}
func (v *nullableUintVector) At(i int) interface{} { return (*v)[i] }
func (v *nullableUintVector) Len() int { return len(*v) }
+3 -3
View File
@@ -9,7 +9,7 @@ import (
_ "github.com/mattn/go-isatty"
)
// NewColorable return new instance of Writer which handle escape sequence.
// NewColorable returns new instance of Writer which handles escape sequence.
func NewColorable(file *os.File) io.Writer {
if file == nil {
panic("nil passed instead of *os.File to NewColorable()")
@@ -18,12 +18,12 @@ func NewColorable(file *os.File) io.Writer {
return file
}
// NewColorableStdout return new instance of Writer which handle escape sequence for stdout.
// NewColorableStdout returns new instance of Writer which handles escape sequence for stdout.
func NewColorableStdout() io.Writer {
return os.Stdout
}
// NewColorableStderr return new instance of Writer which handle escape sequence for stderr.
// NewColorableStderr returns new instance of Writer which handles escape sequence for stderr.
func NewColorableStderr() io.Writer {
return os.Stderr
}
+3 -3
View File
@@ -10,7 +10,7 @@ import (
_ "github.com/mattn/go-isatty"
)
// NewColorable return new instance of Writer which handle escape sequence.
// NewColorable returns new instance of Writer which handles escape sequence.
func NewColorable(file *os.File) io.Writer {
if file == nil {
panic("nil passed instead of *os.File to NewColorable()")
@@ -19,12 +19,12 @@ func NewColorable(file *os.File) io.Writer {
return file
}
// NewColorableStdout return new instance of Writer which handle escape sequence for stdout.
// NewColorableStdout returns new instance of Writer which handles escape sequence for stdout.
func NewColorableStdout() io.Writer {
return os.Stdout
}
// NewColorableStderr return new instance of Writer which handle escape sequence for stderr.
// NewColorableStderr returns new instance of Writer which handles escape sequence for stderr.
func NewColorableStderr() io.Writer {
return os.Stderr
}
+34 -9
View File
@@ -81,7 +81,7 @@ var (
procCreateConsoleScreenBuffer = kernel32.NewProc("CreateConsoleScreenBuffer")
)
// Writer provide colorable Writer to the console
// Writer provides colorable Writer to the console
type Writer struct {
out io.Writer
handle syscall.Handle
@@ -91,7 +91,7 @@ type Writer struct {
rest bytes.Buffer
}
// NewColorable return new instance of Writer which handle escape sequence from File.
// NewColorable returns new instance of Writer which handles escape sequence from File.
func NewColorable(file *os.File) io.Writer {
if file == nil {
panic("nil passed instead of *os.File to NewColorable()")
@@ -106,12 +106,12 @@ func NewColorable(file *os.File) io.Writer {
return file
}
// NewColorableStdout return new instance of Writer which handle escape sequence for stdout.
// NewColorableStdout returns new instance of Writer which handles escape sequence for stdout.
func NewColorableStdout() io.Writer {
return NewColorable(os.Stdout)
}
// NewColorableStderr return new instance of Writer which handle escape sequence for stderr.
// NewColorableStderr returns new instance of Writer which handles escape sequence for stderr.
func NewColorableStderr() io.Writer {
return NewColorable(os.Stderr)
}
@@ -414,7 +414,15 @@ func doTitleSequence(er *bytes.Reader) error {
return nil
}
// Write write data on console
// returns Atoi(s) unless s == "" in which case it returns def
func atoiWithDefault(s string, def int) (int, error) {
if s == "" {
return def, nil
}
return strconv.Atoi(s)
}
// Write writes data on console
func (w *Writer) Write(data []byte) (n int, err error) {
var csbi consoleScreenBufferInfo
procGetConsoleScreenBufferInfo.Call(uintptr(w.handle), uintptr(unsafe.Pointer(&csbi)))
@@ -500,7 +508,7 @@ loop:
switch m {
case 'A':
n, err = strconv.Atoi(buf.String())
n, err = atoiWithDefault(buf.String(), 1)
if err != nil {
continue
}
@@ -508,7 +516,7 @@ loop:
csbi.cursorPosition.y -= short(n)
procSetConsoleCursorPosition.Call(uintptr(handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition)))
case 'B':
n, err = strconv.Atoi(buf.String())
n, err = atoiWithDefault(buf.String(), 1)
if err != nil {
continue
}
@@ -516,7 +524,7 @@ loop:
csbi.cursorPosition.y += short(n)
procSetConsoleCursorPosition.Call(uintptr(handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition)))
case 'C':
n, err = strconv.Atoi(buf.String())
n, err = atoiWithDefault(buf.String(), 1)
if err != nil {
continue
}
@@ -524,7 +532,7 @@ loop:
csbi.cursorPosition.x += short(n)
procSetConsoleCursorPosition.Call(uintptr(handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition)))
case 'D':
n, err = strconv.Atoi(buf.String())
n, err = atoiWithDefault(buf.String(), 1)
if err != nil {
continue
}
@@ -557,6 +565,9 @@ loop:
if err != nil {
continue
}
if n < 1 {
n = 1
}
procGetConsoleScreenBufferInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&csbi)))
csbi.cursorPosition.x = short(n - 1)
procSetConsoleCursorPosition.Call(uintptr(handle), *(*uintptr)(unsafe.Pointer(&csbi.cursorPosition)))
@@ -635,6 +646,20 @@ loop:
}
procFillConsoleOutputCharacter.Call(uintptr(handle), uintptr(' '), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&written)))
procFillConsoleOutputAttribute.Call(uintptr(handle), uintptr(csbi.attributes), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&written)))
case 'X':
n := 0
if buf.Len() > 0 {
n, err = strconv.Atoi(buf.String())
if err != nil {
continue
}
}
procGetConsoleScreenBufferInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&csbi)))
var cursor coord
var written dword
cursor = coord{x: csbi.cursorPosition.x, y: csbi.cursorPosition.y}
procFillConsoleOutputCharacter.Call(uintptr(handle), uintptr(' '), uintptr(n), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&written)))
procFillConsoleOutputAttribute.Call(uintptr(handle), uintptr(csbi.attributes), uintptr(n), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&written)))
case 'm':
procGetConsoleScreenBufferInfo.Call(uintptr(handle), uintptr(unsafe.Pointer(&csbi)))
attr := csbi.attributes
+1 -1
View File
@@ -1,3 +1,3 @@
module github.com/mattn/go-colorable
require github.com/mattn/go-isatty v0.0.5
require github.com/mattn/go-isatty v0.0.8
+3 -3
View File
@@ -5,17 +5,17 @@ import (
"io"
)
// NonColorable hold writer but remove escape sequence.
// NonColorable holds writer but removes escape sequence.
type NonColorable struct {
out io.Writer
}
// NewNonColorable return new instance of Writer which remove escape sequence from Writer.
// NewNonColorable returns new instance of Writer which removes escape sequence from Writer.
func NewNonColorable(w io.Writer) io.Writer {
return &NonColorable{out: w}
}
// Write write data on console
// Write writes data on console
func (w *NonColorable) Write(data []byte) (n int, err error) {
er := bytes.NewReader(data)
var bw [1]byte
+3 -1
View File
@@ -1,3 +1,5 @@
module github.com/mattn/go-isatty
require golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223
require golang.org/x/sys v0.0.0-20191008105621-543471e840be
go 1.14
+4 -2
View File
@@ -1,2 +1,4 @@
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU=
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+1 -1
View File
@@ -1,4 +1,4 @@
// +build appengine js
// +build appengine js nacl
package isatty
+22
View File
@@ -0,0 +1,22 @@
// +build plan9
package isatty
import (
"syscall"
)
// IsTerminal returns true if the given file descriptor is a terminal.
func IsTerminal(fd uintptr) bool {
path, err := syscall.Fd2path(fd)
if err != nil {
return false
}
return path == "/dev/cons" || path == "/mnt/term/dev/cons"
}
// IsCygwinTerminal return true if the file descriptor is a cygwin or msys2
// terminal. This is also always false on this environment.
func IsCygwinTerminal(fd uintptr) bool {
return false
}
@@ -1,4 +1,4 @@
// +build linux
// +build linux aix
// +build !appengine
// +build !android
+35 -4
View File
@@ -4,6 +4,7 @@
package isatty
import (
"errors"
"strings"
"syscall"
"unicode/utf16"
@@ -11,15 +12,18 @@ import (
)
const (
fileNameInfo uintptr = 2
fileTypePipe = 3
objectNameInfo uintptr = 1
fileNameInfo = 2
fileTypePipe = 3
)
var (
kernel32 = syscall.NewLazyDLL("kernel32.dll")
ntdll = syscall.NewLazyDLL("ntdll.dll")
procGetConsoleMode = kernel32.NewProc("GetConsoleMode")
procGetFileInformationByHandleEx = kernel32.NewProc("GetFileInformationByHandleEx")
procGetFileType = kernel32.NewProc("GetFileType")
procNtQueryObject = ntdll.NewProc("NtQueryObject")
)
func init() {
@@ -45,7 +49,10 @@ func isCygwinPipeName(name string) bool {
return false
}
if token[0] != `\msys` && token[0] != `\cygwin` {
if token[0] != `\msys` &&
token[0] != `\cygwin` &&
token[0] != `\Device\NamedPipe\msys` &&
token[0] != `\Device\NamedPipe\cygwin` {
return false
}
@@ -68,11 +75,35 @@ func isCygwinPipeName(name string) bool {
return true
}
// getFileNameByHandle use the undocomented ntdll NtQueryObject to get file full name from file handler
// since GetFileInformationByHandleEx is not avilable under windows Vista and still some old fashion
// guys are using Windows XP, this is a workaround for those guys, it will also work on system from
// Windows vista to 10
// see https://stackoverflow.com/a/18792477 for details
func getFileNameByHandle(fd uintptr) (string, error) {
if procNtQueryObject == nil {
return "", errors.New("ntdll.dll: NtQueryObject not supported")
}
var buf [4 + syscall.MAX_PATH]uint16
var result int
r, _, e := syscall.Syscall6(procNtQueryObject.Addr(), 5,
fd, objectNameInfo, uintptr(unsafe.Pointer(&buf)), uintptr(2*len(buf)), uintptr(unsafe.Pointer(&result)), 0)
if r != 0 {
return "", e
}
return string(utf16.Decode(buf[4 : 4+buf[0]/2])), nil
}
// IsCygwinTerminal() return true if the file descriptor is a cygwin or msys2
// terminal.
func IsCygwinTerminal(fd uintptr) bool {
if procGetFileInformationByHandleEx == nil {
return false
name, err := getFileNameByHandle(fd)
if err != nil {
return false
}
return isCygwinPipeName(name)
}
// Cygwin/msys's pty is a pipe.