migrate from govendor to dep

This commit is contained in:
bergquist
2018-01-19 09:48:15 +01:00
parent 5bb22b836d
commit e023f79c5a
7349 changed files with 3153302 additions and 64302 deletions
+158 -51
View File
@@ -8,43 +8,81 @@ import (
type sbuf []string
func (s *sbuf) Write(b []byte) (int, error) {
*s = append(*s, string(b))
return len(b), nil
func (p *sbuf) Printf(format string, a ...interface{}) {
s := fmt.Sprintf(format, a...)
*p = append(*p, s)
}
// Diff returns a slice where each element describes
// a difference between a and b.
func Diff(a, b interface{}) (desc []string) {
Fdiff((*sbuf)(&desc), a, b)
Pdiff((*sbuf)(&desc), a, b)
return desc
}
// wprintfer calls Fprintf on w for each Printf call
// with a trailing newline.
type wprintfer struct{ w io.Writer }
func (p *wprintfer) Printf(format string, a ...interface{}) {
fmt.Fprintf(p.w, format+"\n", a...)
}
// Fdiff writes to w a description of the differences between a and b.
func Fdiff(w io.Writer, a, b interface{}) {
diffWriter{w: w}.diff(reflect.ValueOf(a), reflect.ValueOf(b))
Pdiff(&wprintfer{w}, a, b)
}
type diffWriter struct {
w io.Writer
type Printfer interface {
Printf(format string, a ...interface{})
}
// Pdiff prints to p a description of the differences between a and b.
// It calls Printf once for each difference, with no trailing newline.
// The standard library log.Logger is a Printfer.
func Pdiff(p Printfer, a, b interface{}) {
diffPrinter{w: p}.diff(reflect.ValueOf(a), reflect.ValueOf(b))
}
type Logfer interface {
Logf(format string, a ...interface{})
}
// logprintfer calls Fprintf on w for each Printf call
// with a trailing newline.
type logprintfer struct{ l Logfer }
func (p *logprintfer) Printf(format string, a ...interface{}) {
p.l.Logf(format, a...)
}
// Ldiff prints to l a description of the differences between a and b.
// It calls Logf once for each difference, with no trailing newline.
// The standard library testing.T and testing.B are Logfers.
func Ldiff(l Logfer, a, b interface{}) {
Pdiff(&logprintfer{l}, a, b)
}
type diffPrinter struct {
w Printfer
l string // label
}
func (w diffWriter) printf(f string, a ...interface{}) {
func (w diffPrinter) printf(f string, a ...interface{}) {
var l string
if w.l != "" {
l = w.l + ": "
}
fmt.Fprintf(w.w, l+f, a...)
w.w.Printf(l+f, a...)
}
func (w diffWriter) diff(av, bv reflect.Value) {
func (w diffPrinter) diff(av, bv reflect.Value) {
if !av.IsValid() && bv.IsValid() {
w.printf("nil != %#v", bv.Interface())
w.printf("nil != %# v", formatter{v: bv, quote: true})
return
}
if av.IsValid() && !bv.IsValid() {
w.printf("%#v != nil", av.Interface())
w.printf("%# v != nil", formatter{v: av, quote: true})
return
}
if !av.IsValid() && !bv.IsValid() {
@@ -58,34 +96,61 @@ func (w diffWriter) diff(av, bv reflect.Value) {
return
}
// numeric types, including bool
if at.Kind() < reflect.Array {
a, b := av.Interface(), bv.Interface()
if a != b {
w.printf("%#v != %#v", a, b)
switch kind := at.Kind(); kind {
case reflect.Bool:
if a, b := av.Bool(), bv.Bool(); a != b {
w.printf("%v != %v", a, b)
}
return
}
switch at.Kind() {
case reflect.String:
a, b := av.Interface(), bv.Interface()
if a != b {
w.printf("%q != %q", a, b)
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
if a, b := av.Int(), bv.Int(); a != b {
w.printf("%d != %d", a, b)
}
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
if a, b := av.Uint(), bv.Uint(); a != b {
w.printf("%d != %d", a, b)
}
case reflect.Float32, reflect.Float64:
if a, b := av.Float(), bv.Float(); a != b {
w.printf("%v != %v", a, b)
}
case reflect.Complex64, reflect.Complex128:
if a, b := av.Complex(), bv.Complex(); a != b {
w.printf("%v != %v", a, b)
}
case reflect.Array:
n := av.Len()
for i := 0; i < n; i++ {
w.relabel(fmt.Sprintf("[%d]", i)).diff(av.Index(i), bv.Index(i))
}
case reflect.Chan, reflect.Func, reflect.UnsafePointer:
if a, b := av.Pointer(), bv.Pointer(); a != b {
w.printf("%#x != %#x", a, b)
}
case reflect.Interface:
w.diff(av.Elem(), bv.Elem())
case reflect.Map:
ak, both, bk := keyDiff(av.MapKeys(), bv.MapKeys())
for _, k := range ak {
w := w.relabel(fmt.Sprintf("[%#v]", k))
w.printf("%q != (missing)", av.MapIndex(k))
}
for _, k := range both {
w := w.relabel(fmt.Sprintf("[%#v]", k))
w.diff(av.MapIndex(k), bv.MapIndex(k))
}
for _, k := range bk {
w := w.relabel(fmt.Sprintf("[%#v]", k))
w.printf("(missing) != %q", bv.MapIndex(k))
}
case reflect.Ptr:
switch {
case av.IsNil() && !bv.IsNil():
w.printf("nil != %v", bv.Interface())
w.printf("nil != %# v", formatter{v: bv, quote: true})
case !av.IsNil() && bv.IsNil():
w.printf("%v != nil", av.Interface())
w.printf("%# v != nil", formatter{v: av, quote: true})
case !av.IsNil() && !bv.IsNil():
w.diff(av.Elem(), bv.Elem())
}
case reflect.Struct:
for i := 0; i < av.NumField(); i++ {
w.relabel(at.Field(i).Name).diff(av.Field(i), bv.Field(i))
}
case reflect.Slice:
lenA := av.Len()
lenB := bv.Len()
@@ -96,30 +161,20 @@ func (w diffWriter) diff(av, bv reflect.Value) {
for i := 0; i < lenA; i++ {
w.relabel(fmt.Sprintf("[%d]", i)).diff(av.Index(i), bv.Index(i))
}
case reflect.Map:
ak, both, bk := keyDiff(av.MapKeys(), bv.MapKeys())
for _, k := range ak {
w := w.relabel(fmt.Sprintf("[%#v]", k.Interface()))
w.printf("%q != (missing)", av.MapIndex(k))
case reflect.String:
if a, b := av.String(), bv.String(); a != b {
w.printf("%q != %q", a, b)
}
for _, k := range both {
w := w.relabel(fmt.Sprintf("[%#v]", k.Interface()))
w.diff(av.MapIndex(k), bv.MapIndex(k))
case reflect.Struct:
for i := 0; i < av.NumField(); i++ {
w.relabel(at.Field(i).Name).diff(av.Field(i), bv.Field(i))
}
for _, k := range bk {
w := w.relabel(fmt.Sprintf("[%#v]", k.Interface()))
w.printf("(missing) != %q", bv.MapIndex(k))
}
case reflect.Interface:
w.diff(reflect.ValueOf(av.Interface()), reflect.ValueOf(bv.Interface()))
default:
if !reflect.DeepEqual(av.Interface(), bv.Interface()) {
w.printf("%# v != %# v", Formatter(av.Interface()), Formatter(bv.Interface()))
}
panic("unknown reflect Kind: " + kind.String())
}
}
func (d diffWriter) relabel(name string) (d1 diffWriter) {
func (d diffPrinter) relabel(name string) (d1 diffPrinter) {
d1 = d
if d.l != "" && name[0] != '[' {
d1.l += "."
@@ -128,11 +183,63 @@ func (d diffWriter) relabel(name string) (d1 diffWriter) {
return d1
}
// keyEqual compares a and b for equality.
// Both a and b must be valid map keys.
func keyEqual(av, bv reflect.Value) bool {
if !av.IsValid() && !bv.IsValid() {
return true
}
if !av.IsValid() || !bv.IsValid() || av.Type() != bv.Type() {
return false
}
switch kind := av.Kind(); kind {
case reflect.Bool:
a, b := av.Bool(), bv.Bool()
return a == b
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
a, b := av.Int(), bv.Int()
return a == b
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
a, b := av.Uint(), bv.Uint()
return a == b
case reflect.Float32, reflect.Float64:
a, b := av.Float(), bv.Float()
return a == b
case reflect.Complex64, reflect.Complex128:
a, b := av.Complex(), bv.Complex()
return a == b
case reflect.Array:
for i := 0; i < av.Len(); i++ {
if !keyEqual(av.Index(i), bv.Index(i)) {
return false
}
}
return true
case reflect.Chan, reflect.UnsafePointer, reflect.Ptr:
a, b := av.Pointer(), bv.Pointer()
return a == b
case reflect.Interface:
return keyEqual(av.Elem(), bv.Elem())
case reflect.String:
a, b := av.String(), bv.String()
return a == b
case reflect.Struct:
for i := 0; i < av.NumField(); i++ {
if !keyEqual(av.Field(i), bv.Field(i)) {
return false
}
}
return true
default:
panic("invalid map key type " + av.Type().String())
}
}
func keyDiff(a, b []reflect.Value) (ak, both, bk []reflect.Value) {
for _, av := range a {
inBoth := false
for _, bv := range b {
if reflect.DeepEqual(av.Interface(), bv.Interface()) {
if keyEqual(av, bv) {
inBoth = true
both = append(both, av)
break
@@ -145,7 +252,7 @@ func keyDiff(a, b []reflect.Value) (ak, both, bk []reflect.Value) {
for _, bv := range b {
inBoth := false
for _, av := range a {
if reflect.DeepEqual(av.Interface(), bv.Interface()) {
if keyEqual(av, bv) {
inBoth = true
break
}
+141 -2
View File
@@ -1,7 +1,18 @@
package pretty
import (
"bytes"
"fmt"
"log"
"reflect"
"testing"
"unsafe"
)
var (
_ Logfer = (*testing.T)(nil)
_ Logfer = (*testing.B)(nil)
_ Printfer = (*log.Logger)(nil)
)
type difftest struct {
@@ -17,6 +28,20 @@ type S struct {
C []int
}
type (
N struct{ N int }
E interface{}
)
var (
c0 = make(chan int)
c1 = make(chan int)
f0 = func() {}
f1 = func() {}
i0 = 0
i1 = 1
)
var diffs = []difftest{
{a: nil, b: nil},
{a: S{A: 1}, b: S{A: 1}},
@@ -28,12 +53,79 @@ var diffs = []difftest{
{S{}, S{A: 1}, []string{`A: 0 != 1`}},
{new(S), &S{A: 1}, []string{`A: 0 != 1`}},
{S{S: new(S)}, S{S: &S{A: 1}}, []string{`S.A: 0 != 1`}},
{S{}, S{I: 0}, []string{`I: nil != 0`}},
{S{}, S{I: 0}, []string{`I: nil != int(0)`}},
{S{I: 1}, S{I: "x"}, []string{`I: int != string`}},
{S{}, S{C: []int{1}}, []string{`C: []int[0] != []int[1]`}},
{S{C: []int{}}, S{C: []int{1}}, []string{`C: []int[0] != []int[1]`}},
{S{C: []int{1, 2, 3}}, S{C: []int{1, 2, 4}}, []string{`C[2]: 3 != 4`}},
{S{}, S{A: 1, S: new(S)}, []string{`A: 0 != 1`, `S: nil != &{0 <nil> <nil> []}`}},
{S{}, S{A: 1, S: new(S)}, []string{`A: 0 != 1`, `S: nil != &pretty.S{}`}},
// unexported fields of every reflect.Kind (both equal and unequal)
{struct{ x bool }{false}, struct{ x bool }{false}, nil},
{struct{ x bool }{false}, struct{ x bool }{true}, []string{`x: false != true`}},
{struct{ x int }{0}, struct{ x int }{0}, nil},
{struct{ x int }{0}, struct{ x int }{1}, []string{`x: 0 != 1`}},
{struct{ x int8 }{0}, struct{ x int8 }{0}, nil},
{struct{ x int8 }{0}, struct{ x int8 }{1}, []string{`x: 0 != 1`}},
{struct{ x int16 }{0}, struct{ x int16 }{0}, nil},
{struct{ x int16 }{0}, struct{ x int16 }{1}, []string{`x: 0 != 1`}},
{struct{ x int32 }{0}, struct{ x int32 }{0}, nil},
{struct{ x int32 }{0}, struct{ x int32 }{1}, []string{`x: 0 != 1`}},
{struct{ x int64 }{0}, struct{ x int64 }{0}, nil},
{struct{ x int64 }{0}, struct{ x int64 }{1}, []string{`x: 0 != 1`}},
{struct{ x uint }{0}, struct{ x uint }{0}, nil},
{struct{ x uint }{0}, struct{ x uint }{1}, []string{`x: 0 != 1`}},
{struct{ x uint8 }{0}, struct{ x uint8 }{0}, nil},
{struct{ x uint8 }{0}, struct{ x uint8 }{1}, []string{`x: 0 != 1`}},
{struct{ x uint16 }{0}, struct{ x uint16 }{0}, nil},
{struct{ x uint16 }{0}, struct{ x uint16 }{1}, []string{`x: 0 != 1`}},
{struct{ x uint32 }{0}, struct{ x uint32 }{0}, nil},
{struct{ x uint32 }{0}, struct{ x uint32 }{1}, []string{`x: 0 != 1`}},
{struct{ x uint64 }{0}, struct{ x uint64 }{0}, nil},
{struct{ x uint64 }{0}, struct{ x uint64 }{1}, []string{`x: 0 != 1`}},
{struct{ x uintptr }{0}, struct{ x uintptr }{0}, nil},
{struct{ x uintptr }{0}, struct{ x uintptr }{1}, []string{`x: 0 != 1`}},
{struct{ x float32 }{0}, struct{ x float32 }{0}, nil},
{struct{ x float32 }{0}, struct{ x float32 }{1}, []string{`x: 0 != 1`}},
{struct{ x float64 }{0}, struct{ x float64 }{0}, nil},
{struct{ x float64 }{0}, struct{ x float64 }{1}, []string{`x: 0 != 1`}},
{struct{ x complex64 }{0}, struct{ x complex64 }{0}, nil},
{struct{ x complex64 }{0}, struct{ x complex64 }{1}, []string{`x: (0+0i) != (1+0i)`}},
{struct{ x complex128 }{0}, struct{ x complex128 }{0}, nil},
{struct{ x complex128 }{0}, struct{ x complex128 }{1}, []string{`x: (0+0i) != (1+0i)`}},
{struct{ x [1]int }{[1]int{0}}, struct{ x [1]int }{[1]int{0}}, nil},
{struct{ x [1]int }{[1]int{0}}, struct{ x [1]int }{[1]int{1}}, []string{`x[0]: 0 != 1`}},
{struct{ x chan int }{c0}, struct{ x chan int }{c0}, nil},
{struct{ x chan int }{c0}, struct{ x chan int }{c1}, []string{fmt.Sprintf("x: %p != %p", c0, c1)}},
{struct{ x func() }{f0}, struct{ x func() }{f0}, nil},
{struct{ x func() }{f0}, struct{ x func() }{f1}, []string{fmt.Sprintf("x: %p != %p", f0, f1)}},
{struct{ x interface{} }{0}, struct{ x interface{} }{0}, nil},
{struct{ x interface{} }{0}, struct{ x interface{} }{1}, []string{`x: 0 != 1`}},
{struct{ x interface{} }{0}, struct{ x interface{} }{""}, []string{`x: int != string`}},
{struct{ x interface{} }{0}, struct{ x interface{} }{nil}, []string{`x: int(0) != nil`}},
{struct{ x interface{} }{nil}, struct{ x interface{} }{0}, []string{`x: nil != int(0)`}},
{struct{ x map[int]int }{map[int]int{0: 0}}, struct{ x map[int]int }{map[int]int{0: 0}}, nil},
{struct{ x map[int]int }{map[int]int{0: 0}}, struct{ x map[int]int }{map[int]int{0: 1}}, []string{`x[0]: 0 != 1`}},
{struct{ x *int }{new(int)}, struct{ x *int }{new(int)}, nil},
{struct{ x *int }{&i0}, struct{ x *int }{&i1}, []string{`x: 0 != 1`}},
{struct{ x *int }{nil}, struct{ x *int }{&i0}, []string{`x: nil != &int(0)`}},
{struct{ x *int }{&i0}, struct{ x *int }{nil}, []string{`x: &int(0) != nil`}},
{struct{ x []int }{[]int{0}}, struct{ x []int }{[]int{0}}, nil},
{struct{ x []int }{[]int{0}}, struct{ x []int }{[]int{1}}, []string{`x[0]: 0 != 1`}},
{struct{ x string }{"a"}, struct{ x string }{"a"}, nil},
{struct{ x string }{"a"}, struct{ x string }{"b"}, []string{`x: "a" != "b"`}},
{struct{ x N }{N{0}}, struct{ x N }{N{0}}, nil},
{struct{ x N }{N{0}}, struct{ x N }{N{1}}, []string{`x.N: 0 != 1`}},
{
struct{ x unsafe.Pointer }{unsafe.Pointer(uintptr(0))},
struct{ x unsafe.Pointer }{unsafe.Pointer(uintptr(0))},
nil,
},
{
struct{ x unsafe.Pointer }{unsafe.Pointer(uintptr(0))},
struct{ x unsafe.Pointer }{unsafe.Pointer(uintptr(1))},
[]string{`x: 0x0 != 0x1`},
},
}
func TestDiff(t *testing.T) {
@@ -54,6 +146,53 @@ func TestDiff(t *testing.T) {
}
}
func TestKeyEqual(t *testing.T) {
var emptyInterfaceZero interface{} = 0
cases := []interface{}{
new(bool),
new(int),
new(int8),
new(int16),
new(int32),
new(int64),
new(uint),
new(uint8),
new(uint16),
new(uint32),
new(uint64),
new(uintptr),
new(float32),
new(float64),
new(complex64),
new(complex128),
new([1]int),
new(chan int),
new(unsafe.Pointer),
new(interface{}),
&emptyInterfaceZero,
new(*int),
new(string),
new(struct{ int }),
}
for _, test := range cases {
rv := reflect.ValueOf(test).Elem()
if !keyEqual(rv, rv) {
t.Errorf("keyEqual(%s, %s) = false want true", rv.Type(), rv.Type())
}
}
}
func TestFdiff(t *testing.T) {
var buf bytes.Buffer
Fdiff(&buf, 0, 1)
want := "0 != 1\n"
if got := buf.String(); got != want {
t.Errorf("Fdiff(0, 1) = %q want %q", got, want)
}
}
func diffdiff(t *testing.T, got, exp []string) {
minus(t, "unexpected:", got, exp)
minus(t, "missing:", exp, got)
+5 -14
View File
@@ -10,12 +10,8 @@ import (
"github.com/kr/text"
)
const (
limit = 50
)
type formatter struct {
x interface{}
v reflect.Value
force bool
quote bool
}
@@ -30,11 +26,11 @@ type formatter struct {
// format x according to the usual rules of package fmt.
// In particular, if x satisfies fmt.Formatter, then x.Format will be called.
func Formatter(x interface{}) (f fmt.Formatter) {
return formatter{x: x, quote: true}
return formatter{v: reflect.ValueOf(x), quote: true}
}
func (fo formatter) String() string {
return fmt.Sprint(fo.x) // unwrap it
return fmt.Sprint(fo.v.Interface()) // unwrap it
}
func (fo formatter) passThrough(f fmt.State, c rune) {
@@ -51,14 +47,14 @@ func (fo formatter) passThrough(f fmt.State, c rune) {
s += fmt.Sprintf(".%d", p)
}
s += string(c)
fmt.Fprintf(f, s, fo.x)
fmt.Fprintf(f, s, fo.v.Interface())
}
func (fo formatter) Format(f fmt.State, c rune) {
if fo.force || c == 'v' && f.Flag('#') && f.Flag(' ') {
w := tabwriter.NewWriter(f, 4, 4, 1, ' ', 0)
p := &printer{tw: w, Writer: w, visited: make(map[visit]int)}
p.printValue(reflect.ValueOf(fo.x), true, fo.quote)
p.printValue(fo.v, true, fo.quote)
w.Flush()
return
}
@@ -319,11 +315,6 @@ func (p *printer) fmtString(s string, quote bool) {
io.WriteString(p, s)
}
func tryDeepEqual(a, b interface{}) bool {
defer func() { recover() }()
return reflect.DeepEqual(a, b)
}
func writeByte(w io.Writer, b byte) {
w.Write([]byte{b})
}
+27
View File
@@ -13,6 +13,11 @@ type test struct {
s string
}
type passtest struct {
v interface{}
f, s string
}
type LongStructTypeName struct {
longFieldName interface{}
otherLongFieldName interface{}
@@ -33,8 +38,30 @@ func (f F) Format(s fmt.State, c rune) {
fmt.Fprintf(s, "F(%d)", int(f))
}
type Stringer struct { i int }
func (s *Stringer) String() string { return "foo" }
var long = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
var passthrough = []passtest{
{1, "%d", "1"},
{"a", "%s", "a"},
{&Stringer{}, "%s", "foo"},
}
func TestPassthrough(t *testing.T) {
for _, tt := range passthrough {
s := fmt.Sprintf(tt.f, Formatter(tt.v))
if tt.s != s {
t.Errorf("expected %q", tt.s)
t.Errorf("got %q", s)
t.Errorf("expraw\n%s", tt.s)
t.Errorf("gotraw\n%s", s)
}
}
}
var gosyntax = []test{
{nil, `nil`},
{"", `""`},
+11 -1
View File
@@ -11,6 +11,7 @@ import (
"fmt"
"io"
"log"
"reflect"
)
// Errorf is a convenience wrapper for fmt.Errorf.
@@ -81,6 +82,15 @@ func Println(a ...interface{}) (n int, errno error) {
return fmt.Println(wrap(a, true)...)
}
// Sprint is a convenience wrapper for fmt.Sprintf.
//
// Calling Sprint(x, y) is equivalent to
// fmt.Sprint(Formatter(x), Formatter(y)), but each operand is
// formatted with "%# v".
func Sprint(a ...interface{}) string {
return fmt.Sprint(wrap(a, true)...)
}
// Sprintf is a convenience wrapper for fmt.Sprintf.
//
// Calling Sprintf(f, x, y) is equivalent to
@@ -92,7 +102,7 @@ func Sprintf(format string, a ...interface{}) string {
func wrap(a []interface{}, force bool) []interface{} {
w := make([]interface{}, len(a))
for i, x := range a {
w[i] = formatter{x: x, force: force}
w[i] = formatter{v: reflect.ValueOf(x), force: force}
}
return w
}
-19
View File
@@ -1,19 +0,0 @@
Copyright (c) 2012 Keith Rarick.
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
-4
View File
@@ -1,4 +0,0 @@
Package s3 signs HTTP requests for use with Amazon’s S3 API.
Documentation:
http://godoc.org/github.com/kr/s3
-4
View File
@@ -1,4 +0,0 @@
Package s3util provides streaming transfers to and from Amazon S3.
Full documentation:
http://godoc.org/github.com/kr/s3/s3util
-27
View File
@@ -1,27 +0,0 @@
// Package s3util provides streaming transfers to and from Amazon S3.
//
// To use it, open or create an S3 object, read or write data,
// and close the object.
//
// You must assign valid credentials to DefaultConfig.Keys before using
// DefaultConfig. Be sure to close an io.WriteCloser returned by this package,
// to flush buffers and complete the multipart upload process.
package s3util
// TODO(kr): parse error responses; return structured data
import (
"github.com/kr/s3"
"net/http"
)
var DefaultConfig = &Config{
Service: s3.DefaultService,
Keys: new(s3.Keys),
}
type Config struct {
*s3.Service
*s3.Keys
*http.Client // if nil, uses http.DefaultClient
}
-29
View File
@@ -1,29 +0,0 @@
package s3util
import (
"bytes"
"fmt"
"io"
"net/http"
)
type respError struct {
r *http.Response
b bytes.Buffer
}
func newRespError(r *http.Response) *respError {
e := new(respError)
e.r = r
io.Copy(&e.b, r.Body)
r.Body.Close()
return e
}
func (e *respError) Error() string {
return fmt.Sprintf(
"unwanted http status %d: %q",
e.r.StatusCode,
e.b.String(),
)
}
-33
View File
@@ -1,33 +0,0 @@
package s3util
import (
"io"
"net/http"
"time"
)
// Open requests the S3 object at url. An HTTP status other than 200 is
// considered an error.
//
// If c is nil, Open uses DefaultConfig.
func Open(url string, c *Config) (io.ReadCloser, error) {
if c == nil {
c = DefaultConfig
}
// TODO(kr): maybe parallel range fetching
r, _ := http.NewRequest("GET", url, nil)
r.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat))
c.Sign(r, *c.Keys)
client := c.Client
if client == nil {
client = http.DefaultClient
}
resp, err := client.Do(r)
if err != nil {
return nil, err
}
if resp.StatusCode != 200 {
return nil, newRespError(resp)
}
return resp.Body, nil
}
-208
View File
@@ -1,208 +0,0 @@
package s3util
import (
"bytes"
"encoding/xml"
"errors"
"io"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"time"
)
// File represents an S3 object or directory.
type File struct {
url string
prefix string
config *Config
result *listObjectsResult
}
type fileInfo struct {
name string
size int64
dir bool
modTime time.Time
sys *Stat
}
// Stat contains information about an S3 object or directory.
// It is the "underlying data source" returned by method Sys
// for each FileInfo produced by this package.
// fi.Sys().(*s3util.Stat)
// For the meaning of these fields, see
// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html.
type Stat struct {
Key string
LastModified string
ETag string // ETag value, without double quotes.
Size string
StorageClass string
OwnerID string `xml:"Owner>ID"`
OwnerName string `xml:"Owner>DisplayName"`
}
type listObjectsResult struct {
IsTruncated bool
Contents []Stat
Directories []string `xml:"CommonPrefixes>Prefix"` // Suffix "/" trimmed
}
func (f *fileInfo) Name() string { return f.name }
func (f *fileInfo) Size() int64 { return f.size }
func (f *fileInfo) Mode() os.FileMode {
if f.dir {
return 0755 | os.ModeDir
}
return 0644
}
func (f *fileInfo) ModTime() time.Time {
if f.modTime.IsZero() && f.sys != nil {
// we return the zero value if a parse error ever happens.
f.modTime, _ = time.Parse(time.RFC3339Nano, f.sys.LastModified)
}
return f.modTime
}
func (f *fileInfo) IsDir() bool { return f.dir }
func (f *fileInfo) Sys() interface{} { return f.sys }
// NewFile returns a new File with the given URL and config.
//
// Set rawurl to a directory on S3, such as
// https://mybucket.s3.amazonaws.com/myfolder.
// The URL cannot have query parameters or a fragment.
// If c is nil, DefaultConfig will be used.
func NewFile(rawurl string, c *Config) (*File, error) {
u, err := url.Parse(rawurl)
if err != nil {
return nil, err
}
if u.RawQuery != "" {
return nil, errors.New("url cannot have raw query parameters.")
}
if u.Fragment != "" {
return nil, errors.New("url cannot have a fragment.")
}
prefix := strings.TrimLeft(u.Path, "/")
if prefix != "" && !strings.HasSuffix(prefix, "/") {
prefix += "/"
}
u.Path = ""
return &File{u.String(), prefix, c, nil}, nil
}
// Readdir requests a list of entries in the S3 directory
// represented by f and returns a slice of up to n FileInfo
// values, in alphabetical order. Subsequent calls
// on the same File will yield further FileInfos.
// Only direct children are returned, not deeper descendants.
func (f *File) Readdir(n int) ([]os.FileInfo, error) {
if f.result != nil && !f.result.IsTruncated {
return make([]os.FileInfo, 0), io.EOF
}
reader, err := f.sendRequest(n)
if err != nil {
return nil, err
}
defer reader.Close()
return f.parseResponse(reader)
}
func (f *File) sendRequest(count int) (io.ReadCloser, error) {
c := f.config
if c == nil {
c = DefaultConfig
}
var buf bytes.Buffer
buf.WriteString(f.url)
buf.WriteString("?delimiter=%2F")
if f.prefix != "" {
buf.WriteString("&prefix=")
buf.WriteString(url.QueryEscape(f.prefix))
}
if count > 0 {
buf.WriteString("&max-keys=")
buf.WriteString(strconv.Itoa(count))
}
if f.result != nil && f.result.IsTruncated {
var lastDir, lastKey, marker string
if len(f.result.Directories) > 0 {
lastDir = f.result.Directories[len(f.result.Directories)-1]
}
if len(f.result.Contents) > 0 {
lastKey = f.result.Contents[len(f.result.Contents)-1].Key
}
if lastKey > lastDir {
marker = lastKey
} else {
marker = lastDir
}
if marker != "" {
buf.WriteString("&marker=")
buf.WriteString(url.QueryEscape(marker))
}
}
u := buf.String()
r, _ := http.NewRequest("GET", u, nil)
r.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat))
c.Sign(r, *c.Keys)
resp, err := http.DefaultClient.Do(r)
if err != nil {
return nil, err
}
if resp.StatusCode != 200 {
return nil, newRespError(resp)
}
return resp.Body, nil
}
func (f *File) parseResponse(reader io.Reader) ([]os.FileInfo, error) {
decoder := xml.NewDecoder(reader)
result := listObjectsResult{}
var err error
err = decoder.Decode(&result)
if err != nil {
return nil, err
}
infos := make([]os.FileInfo, len(result.Contents)+len(result.Directories))
var size int64
var name string
var is_dir bool
for i, content := range result.Contents {
c := content
c.ETag = strings.Trim(c.ETag, `"`)
size, _ = strconv.ParseInt(c.Size, 10, 0)
if size == 0 && strings.HasSuffix(c.Key, "/") {
name = strings.TrimRight(c.Key, "/")
is_dir = true
} else {
name = c.Key
is_dir = false
}
infos[i] = &fileInfo{
name: name,
size: size,
dir: is_dir,
sys: &c,
}
}
for i, dir := range result.Directories {
infos[len(result.Contents)+i] = &fileInfo{
name: strings.TrimRight(dir, "/"),
size: 0,
dir: true,
}
}
f.result = &result
return infos, nil
}
-267
View File
@@ -1,267 +0,0 @@
package s3util
import (
"bytes"
"encoding/xml"
"github.com/kr/s3"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"sync"
"syscall"
"time"
)
// defined by amazon
const (
minPartSize = 5 * 1024 * 1024
maxPartSize = 1<<31 - 1 // for 32-bit use; amz max is 5GiB
maxObjSize = 5 * 1024 * 1024 * 1024 * 1024
maxNPart = 10000
)
const (
concurrency = 5
nTry = 2
)
type part struct {
r io.ReadSeeker
len int64
// read by xml encoder
PartNumber int
ETag string
}
type uploader struct {
s3 s3.Service
keys s3.Keys
url string
client *http.Client
UploadId string // written by xml decoder
bufsz int64
buf []byte
off int
ch chan *part
part int
closed bool
err error
wg sync.WaitGroup
xml struct {
XMLName string `xml:"CompleteMultipartUpload"`
Part []*part
}
}
// Create creates an S3 object at url and sends multipart upload requests as
// data is written.
//
// If h is not nil, each of its entries is added to the HTTP request header.
// If c is nil, Create uses DefaultConfig.
func Create(url string, h http.Header, c *Config) (io.WriteCloser, error) {
if c == nil {
c = DefaultConfig
}
return newUploader(url, h, c)
}
// Sends an S3 multipart upload initiation request.
// See http://docs.amazonwebservices.com/AmazonS3/latest/dev/mpuoverview.html.
// This initial request returns an UploadId that we use to identify
// subsequent PUT requests.
func newUploader(url string, h http.Header, c *Config) (u *uploader, err error) {
u = new(uploader)
u.s3 = *c.Service
u.url = url
u.keys = *c.Keys
u.client = c.Client
if u.client == nil {
u.client = http.DefaultClient
}
u.bufsz = minPartSize
r, err := http.NewRequest("POST", url+"?uploads", nil)
if err != nil {
return nil, err
}
r.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat))
for k := range h {
for _, v := range h[k] {
r.Header.Add(k, v)
}
}
u.s3.Sign(r, u.keys)
resp, err := u.client.Do(r)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return nil, newRespError(resp)
}
err = xml.NewDecoder(resp.Body).Decode(u)
if err != nil {
return nil, err
}
u.ch = make(chan *part)
for i := 0; i < concurrency; i++ {
go u.worker()
}
return u, nil
}
func (u *uploader) Write(p []byte) (n int, err error) {
if u.closed {
return 0, syscall.EINVAL
}
if u.err != nil {
return 0, u.err
}
for n < len(p) {
if cap(u.buf) == 0 {
u.buf = make([]byte, int(u.bufsz))
// Increase part size (1.001x).
// This lets us reach the max object size (5TiB) while
// still doing minimal buffering for small objects.
u.bufsz = min(u.bufsz+u.bufsz/1000, maxPartSize)
}
r := copy(u.buf[u.off:], p[n:])
u.off += r
n += r
if u.off == len(u.buf) {
u.flush()
}
}
return n, nil
}
func (u *uploader) flush() {
u.wg.Add(1)
u.part++
p := &part{bytes.NewReader(u.buf[:u.off]), int64(u.off), u.part, ""}
u.xml.Part = append(u.xml.Part, p)
u.ch <- p
u.buf, u.off = nil, 0
}
func (u *uploader) worker() {
for p := range u.ch {
u.retryUploadPart(p)
}
}
// Calls putPart up to nTry times to recover from transient errors.
func (u *uploader) retryUploadPart(p *part) {
defer u.wg.Done()
defer func() { p.r = nil }() // free the large buffer
var err error
for i := 0; i < nTry; i++ {
p.r.Seek(0, 0)
err = u.putPart(p)
if err == nil {
return
}
}
u.err = err
}
// Uploads part p, reading its contents from p.r.
// Stores the ETag in p.ETag.
func (u *uploader) putPart(p *part) error {
v := url.Values{}
v.Set("partNumber", strconv.Itoa(p.PartNumber))
v.Set("uploadId", u.UploadId)
req, err := http.NewRequest("PUT", u.url+"?"+v.Encode(), p.r)
if err != nil {
return err
}
req.ContentLength = p.len
req.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat))
u.s3.Sign(req, u.keys)
resp, err := u.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return newRespError(resp)
}
s := resp.Header.Get("etag") // includes quote chars for some reason
if len(s) < 2 {
return fmt.Errorf("received invalid etag %q", s)
}
p.ETag = s[1 : len(s)-1]
return nil
}
func (u *uploader) Close() error {
if u.closed {
return syscall.EINVAL
}
if cap(u.buf) > 0 {
u.flush()
}
u.wg.Wait()
close(u.ch)
u.closed = true
if u.err != nil {
u.abort()
return u.err
}
body, err := xml.Marshal(u.xml)
if err != nil {
return err
}
b := bytes.NewBuffer(body)
v := url.Values{}
v.Set("uploadId", u.UploadId)
req, err := http.NewRequest("POST", u.url+"?"+v.Encode(), b)
if err != nil {
return err
}
req.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat))
u.s3.Sign(req, u.keys)
resp, err := u.client.Do(req)
if err != nil {
return err
}
if resp.StatusCode != 200 {
return newRespError(resp)
}
resp.Body.Close()
return nil
}
func (u *uploader) abort() {
// TODO(kr): devise a reasonable way to report an error here in addition
// to the error that caused the abort.
v := url.Values{}
v.Set("uploadId", u.UploadId)
s := u.url + "?" + v.Encode()
req, err := http.NewRequest("DELETE", s, nil)
if err != nil {
return
}
req.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat))
u.s3.Sign(req, u.keys)
resp, err := u.client.Do(req)
if err != nil {
return
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return
}
}
func min(a, b int64) int64 {
if a < b {
return a
}
return b
}
-200
View File
@@ -1,200 +0,0 @@
// Package s3 signs HTTP requests for Amazon S3 and compatible services.
package s3
// See
// http://docs.amazonwebservices.com/AmazonS3/2006-03-01/dev/RESTAuthentication.html.
import (
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"io"
"net/http"
"sort"
"strings"
)
var signParams = map[string]bool{
"acl": true,
"delete": true,
"lifecycle": true,
"location": true,
"logging": true,
"notification": true,
"partNumber": true,
"policy": true,
"requestPayment": true,
"response-cache-control": true,
"response-content-disposition": true,
"response-content-encoding": true,
"response-content-language": true,
"response-content-type": true,
"response-expires": true,
"restore": true,
"torrent": true,
"uploadId": true,
"uploads": true,
"versionId": true,
"versioning": true,
"versions": true,
"website": true,
}
// Keys holds a set of Amazon Security Credentials.
type Keys struct {
AccessKey string
SecretKey string
// SecurityToken is used for temporary security credentials.
// If set, it will be added to header field X-Amz-Security-Token
// before signing a request.
SecurityToken string
// See http://docs.aws.amazon.com/AmazonS3/latest/dev/MakingRequests.html#TypesofSecurityCredentials
}
// IdentityBucket returns subdomain.
// It is designed to be used with S3-compatible services that
// treat the entire subdomain as the bucket name, for example
// storage.io.
func IdentityBucket(subdomain string) string {
return subdomain
}
// AmazonBucket returns everything up to the last '.' in subdomain.
// It is designed to be used with the Amazon service.
// "johnsmith.s3" becomes "johnsmith"
// "johnsmith.s3-eu-west-1" becomes "johnsmith"
// "www.example.com.s3" becomes "www.example.com"
func AmazonBucket(subdomain string) string {
if i := strings.LastIndex(subdomain, "."); i != -1 {
return subdomain[:i]
}
return ""
}
// DefaultService is the default Service used by Sign.
var DefaultService = &Service{Domain: "amazonaws.com"}
// Sign signs an HTTP request with the given S3 keys.
//
// This function is a wrapper around DefaultService.Sign.
func Sign(r *http.Request, k Keys) {
DefaultService.Sign(r, k)
}
// Service represents an S3-compatible service.
type Service struct {
// Domain is the service's root domain. It is used to extract
// the subdomain from an http.Request before passing the
// subdomain to Bucket.
Domain string
// Bucket derives the bucket name from a subdomain.
// If nil, AmazonBucket is used.
Bucket func(subdomain string) string
}
// Sign signs an HTTP request with the given S3 keys for use on service s.
func (s *Service) Sign(r *http.Request, k Keys) {
if k.SecurityToken != "" {
r.Header.Set("X-Amz-Security-Token", k.SecurityToken)
}
h := hmac.New(sha1.New, []byte(k.SecretKey))
s.writeSigData(h, r)
sig := make([]byte, base64.StdEncoding.EncodedLen(h.Size()))
base64.StdEncoding.Encode(sig, h.Sum(nil))
r.Header.Set("Authorization", "AWS "+k.AccessKey+":"+string(sig))
}
func (s *Service) writeSigData(w io.Writer, r *http.Request) {
w.Write([]byte(r.Method))
w.Write([]byte{'\n'})
w.Write([]byte(r.Header.Get("content-md5")))
w.Write([]byte{'\n'})
w.Write([]byte(r.Header.Get("content-type")))
w.Write([]byte{'\n'})
if _, ok := r.Header["X-Amz-Date"]; !ok {
w.Write([]byte(r.Header.Get("date")))
}
w.Write([]byte{'\n'})
writeAmzHeaders(w, r)
s.writeResource(w, r)
}
func (s *Service) writeResource(w io.Writer, r *http.Request) {
s.writeVhostBucket(w, strings.ToLower(r.Host))
path := r.URL.RequestURI()
if r.URL.RawQuery != "" {
path = path[:len(path)-len(r.URL.RawQuery)-1]
}
w.Write([]byte(path))
s.writeSubResource(w, r)
}
func (s *Service) writeVhostBucket(w io.Writer, host string) {
if i := strings.Index(host, ":"); i != -1 {
host = host[:i]
}
if host == s.Domain {
// no vhost - do nothing
} else if strings.HasSuffix(host, "."+s.Domain) {
// vhost - bucket may be in prefix
b := s.Bucket
if b == nil {
b = AmazonBucket
}
bucket := b(host[:len(host)-len(s.Domain)-1])
if bucket != "" {
w.Write([]byte{'/'})
w.Write([]byte(bucket))
}
} else {
// cname - bucket is host
w.Write([]byte{'/'})
w.Write([]byte(host))
}
}
func (s *Service) writeSubResource(w io.Writer, r *http.Request) {
var a []string
for k, vs := range r.URL.Query() {
if signParams[k] {
for _, v := range vs {
if v == "" {
a = append(a, k)
} else {
a = append(a, k+"="+v)
}
}
}
}
sort.Strings(a)
var p byte = '?'
for _, s := range a {
w.Write([]byte{p})
w.Write([]byte(s))
p = '&'
}
}
func writeAmzHeaders(w io.Writer, r *http.Request) {
var keys []string
for k, _ := range r.Header {
if strings.HasPrefix(strings.ToLower(k), "x-amz-") {
keys = append(keys, k)
}
}
sort.Strings(keys)
var a []string
for _, k := range keys {
v := r.Header[k]
a = append(a, strings.ToLower(k)+":"+strings.Join(v, ","))
}
for _, h := range a {
w.Write([]byte(h))
w.Write([]byte{'\n'})
}
}
+73
View File
@@ -0,0 +1,73 @@
/*
Agg computes aggregate values over tabular text.
It behaves somewhat like the SQL “GROUP BY” clause.
Usage:
agg [function...]
It reads input from stdin as a sequence of records, one per line.
It treats each line as a set of fields separated by white space.
One field (the first, by default) is designated as the key.
Successive lines with equal keys are grouped into a group,
and agg produces one line of output for each group.
(Note that only contiguous input lines can form a group.
If you need to make sure that all records for a given key
are grouped together, sort the input first.)
For each remaining field,
agg applies a function to all the values in the group,
producing a single output value.
The command line arguments specify which functions to use,
one per field in the input table.
Functions
The available functions are:
key group by this field (default for field 1)
first value from first line of group (default for rest)
last value from last line of group
sample value from any line of group, uniformly at random
prefix longest common string prefix
join:sep concatenate strings with given sep
smin lexically least string
smax lexically greatest string
min numerically least value
max numerically greatest value
sum numeric sum
mean arithmetic mean
count number of records (ignores input value)
const:val print val, ignoring input
drop omit the column entirely
The numeric functions skip items that don't parse as numbers.
Examples
Using the following input:
$ cat >input
-rwx alice 100 /home/alice/bin/crdt
-rw- alice 210002 /home/alice/thesis.tex
-rw- bob 10051 /home/bob/expenses.tab
-rwx kr 862060 /home/kr/bin/blog
-rwx kr 304608 /home/kr/bin/agg
Disk usage for each user, plus where that disk usage occurs
(longest common prefix of filesystem paths):
$ agg <input drop key sum prefix
alice 210153 /home/alice/
bob 10051 /home/bob/expenses.tab
kr 1166668 /home/kr/
Disk usage for executable vs non-executable files:
$ sort input | agg key drop sum join:,
-rw- 220053 /home/alice/thesis.tex,/home/bob/expenses.tab
-rwx 1166768 /home/alice/bin/crdt,/home/kr/bin/agg,/home/kr/bin/blog
*/
package main
+112
View File
@@ -0,0 +1,112 @@
package main
// TODO(kr): tests
import (
"bufio"
"fmt"
"log"
"math/rand"
"os"
"strings"
"time"
)
type agg interface {
merge(string)
String() string
}
var (
key = 0
funcmap = make(map[int]func(init, arg string) agg)
argmap = make(map[int]string)
symtab = map[string]func(init, arg string) agg{
"first": first,
"last": last,
"prefix": prefix,
"sample": sample,
"join": join,
"smin": smin,
"smax": smax,
"min": min,
"max": max,
"sum": sum,
"mean": mean,
"count": count,
"const": constf,
"drop": nil,
}
)
func main() {
log.SetPrefix("agg: ")
log.SetFlags(0)
rand.Seed(time.Now().UnixNano())
for i, sym := range os.Args[1:] {
if p := strings.IndexByte(sym, ':'); p >= 0 {
sym, argmap[i] = sym[:p], sym[p+1:]
}
if sym == "key" {
key, sym = i, "first"
}
f, ok := symtab[sym]
if !ok {
log.Fatalf("bad function: %q", sym)
}
funcmap[i] = f
}
sc := bufio.NewScanner(os.Stdin)
var g *group
for sc.Scan() {
ss := strings.Fields(sc.Text())
if !matches(g, ss) {
emit(g)
g = &group{key: ss[key]}
}
mergeLine(g, ss)
}
emit(g)
}
type group struct {
key string
agg []agg
}
func matches(g *group, ss []string) bool {
return g != nil && g.key == ss[key]
}
func emit(g *group) {
if g == nil {
return
}
rest := false
for i, a := range g.agg {
if f, ok := funcmap[i]; ok && f == nil {
continue
}
if rest {
fmt.Print("\t")
}
rest = true
fmt.Print(a)
}
fmt.Println()
}
func mergeLine(g *group, ss []string) {
for i, s := range ss {
if i >= len(g.agg) {
f := funcmap[i]
if f == nil {
f = first
}
g.agg = append(g.agg, f(s, argmap[i]))
} else {
g.agg[i].merge(s)
}
}
}
+99
View File
@@ -0,0 +1,99 @@
package main
import (
"math/big"
"strconv"
)
func min(s, arg string) agg { return newBinop(s, opmin) }
func max(s, arg string) agg { return newBinop(s, opmax) }
func sum(s, arg string) agg { return newBinop(s, opsum) }
type binop struct {
v *big.Float
f func(a, b *big.Float) *big.Float
}
func newBinop(s string, f func(a, b *big.Float) *big.Float) *binop {
v, _ := parseFloat(s)
return &binop{v, f}
}
func (o *binop) String() string {
if o.v == nil {
return "NaN"
}
return o.v.Text('f', -1)
}
func (o *binop) merge(s string) {
v, ok := parseFloat(s)
if !ok {
return
}
o.v = o.f(o.v, v)
}
func opmin(a, b *big.Float) *big.Float {
if a != nil && (b == nil || a.Cmp(b) <= 0) {
return a
}
return b
}
func opmax(a, b *big.Float) *big.Float {
if a != nil && (b == nil || a.Cmp(b) >= 0) {
return a
}
return b
}
func opsum(a, b *big.Float) *big.Float {
if a == nil {
return b
} else if b == nil {
return a
}
return a.Add(a, b)
}
type meanagg struct {
v *big.Float
d float64 // actually an integer
}
func mean(s, arg string) agg {
v, ok := parseFloat(s)
if !ok {
return &meanagg{new(big.Float), 0}
}
return &meanagg{v, 1}
}
func (m *meanagg) String() string {
if m.d == 0 {
return "NaN"
}
v := new(big.Float).Quo(m.v, big.NewFloat(m.d))
return v.Text('f', -1)
}
func (m *meanagg) merge(s string) {
v, ok := parseFloat(s)
if !ok {
return
}
m.v.Add(m.v, v)
m.d++
}
func parseFloat(s string) (*big.Float, bool) {
v, _, err := big.ParseFloat(s, 0, 1000, big.ToNearestEven)
return v, err == nil
}
type counter int
func count(init, arg string) agg { return new(counter) }
func (c *counter) String() string { return strconv.Itoa(int(*c) + 1) }
func (c *counter) merge(string) { *c++ }
+74
View File
@@ -0,0 +1,74 @@
package main
import (
"math/rand"
"strings"
)
func first(s, arg string) agg { return &sbinop{s, opfirst} }
func last(s, arg string) agg { return &sbinop{s, oplast} }
func prefix(s, arg string) agg { return &sbinop{s, opprefix} }
func join(s, arg string) agg { return &sbinop{s, opjoin(arg)} }
func smin(s, arg string) agg { return &sbinop{s, opsmin} }
func smax(s, arg string) agg { return &sbinop{s, opsmax} }
type sbinop struct {
s string
f func(a, b string) string
}
func (o *sbinop) String() string { return o.s }
func (o *sbinop) merge(s string) { o.s = o.f(o.s, s) }
func opfirst(a, b string) string { return a }
func oplast(a, b string) string { return b }
func opprefix(a, b string) string {
for i := range a {
if i >= len(b) || a[i] != b[i] {
return a[:i]
}
}
return a
}
func opjoin(sep string) func(a, b string) string {
return func(a, b string) string {
return a + sep + b // TODO(kr): too slow? maybe strings.Join?
}
}
func opsmin(a, b string) string {
if strings.Compare(a, b) <= 0 {
return a
}
return b
}
func opsmax(a, b string) string {
if strings.Compare(a, b) >= 0 {
return a
}
return b
}
type sampler struct {
n int
s string
}
func sample(s, arg string) agg { return &sampler{1, s} }
func (p *sampler) String() string { return p.s }
func (p *sampler) merge(s string) {
p.n++
if rand.Intn(p.n) == 0 {
p.s = s
}
}
type constant string
func constf(init, arg string) agg { return constant(arg) }
func (c constant) String() string { return string(c) }
func (c constant) merge(string) {}