migrate from govendor to dep
This commit is contained in:
+158
-51
@@ -8,43 +8,81 @@ import (
|
||||
|
||||
type sbuf []string
|
||||
|
||||
func (s *sbuf) Write(b []byte) (int, error) {
|
||||
*s = append(*s, string(b))
|
||||
return len(b), nil
|
||||
func (p *sbuf) Printf(format string, a ...interface{}) {
|
||||
s := fmt.Sprintf(format, a...)
|
||||
*p = append(*p, s)
|
||||
}
|
||||
|
||||
// Diff returns a slice where each element describes
|
||||
// a difference between a and b.
|
||||
func Diff(a, b interface{}) (desc []string) {
|
||||
Fdiff((*sbuf)(&desc), a, b)
|
||||
Pdiff((*sbuf)(&desc), a, b)
|
||||
return desc
|
||||
}
|
||||
|
||||
// wprintfer calls Fprintf on w for each Printf call
|
||||
// with a trailing newline.
|
||||
type wprintfer struct{ w io.Writer }
|
||||
|
||||
func (p *wprintfer) Printf(format string, a ...interface{}) {
|
||||
fmt.Fprintf(p.w, format+"\n", a...)
|
||||
}
|
||||
|
||||
// Fdiff writes to w a description of the differences between a and b.
|
||||
func Fdiff(w io.Writer, a, b interface{}) {
|
||||
diffWriter{w: w}.diff(reflect.ValueOf(a), reflect.ValueOf(b))
|
||||
Pdiff(&wprintfer{w}, a, b)
|
||||
}
|
||||
|
||||
type diffWriter struct {
|
||||
w io.Writer
|
||||
type Printfer interface {
|
||||
Printf(format string, a ...interface{})
|
||||
}
|
||||
|
||||
// Pdiff prints to p a description of the differences between a and b.
|
||||
// It calls Printf once for each difference, with no trailing newline.
|
||||
// The standard library log.Logger is a Printfer.
|
||||
func Pdiff(p Printfer, a, b interface{}) {
|
||||
diffPrinter{w: p}.diff(reflect.ValueOf(a), reflect.ValueOf(b))
|
||||
}
|
||||
|
||||
type Logfer interface {
|
||||
Logf(format string, a ...interface{})
|
||||
}
|
||||
|
||||
// logprintfer calls Fprintf on w for each Printf call
|
||||
// with a trailing newline.
|
||||
type logprintfer struct{ l Logfer }
|
||||
|
||||
func (p *logprintfer) Printf(format string, a ...interface{}) {
|
||||
p.l.Logf(format, a...)
|
||||
}
|
||||
|
||||
// Ldiff prints to l a description of the differences between a and b.
|
||||
// It calls Logf once for each difference, with no trailing newline.
|
||||
// The standard library testing.T and testing.B are Logfers.
|
||||
func Ldiff(l Logfer, a, b interface{}) {
|
||||
Pdiff(&logprintfer{l}, a, b)
|
||||
}
|
||||
|
||||
type diffPrinter struct {
|
||||
w Printfer
|
||||
l string // label
|
||||
}
|
||||
|
||||
func (w diffWriter) printf(f string, a ...interface{}) {
|
||||
func (w diffPrinter) printf(f string, a ...interface{}) {
|
||||
var l string
|
||||
if w.l != "" {
|
||||
l = w.l + ": "
|
||||
}
|
||||
fmt.Fprintf(w.w, l+f, a...)
|
||||
w.w.Printf(l+f, a...)
|
||||
}
|
||||
|
||||
func (w diffWriter) diff(av, bv reflect.Value) {
|
||||
func (w diffPrinter) diff(av, bv reflect.Value) {
|
||||
if !av.IsValid() && bv.IsValid() {
|
||||
w.printf("nil != %#v", bv.Interface())
|
||||
w.printf("nil != %# v", formatter{v: bv, quote: true})
|
||||
return
|
||||
}
|
||||
if av.IsValid() && !bv.IsValid() {
|
||||
w.printf("%#v != nil", av.Interface())
|
||||
w.printf("%# v != nil", formatter{v: av, quote: true})
|
||||
return
|
||||
}
|
||||
if !av.IsValid() && !bv.IsValid() {
|
||||
@@ -58,34 +96,61 @@ func (w diffWriter) diff(av, bv reflect.Value) {
|
||||
return
|
||||
}
|
||||
|
||||
// numeric types, including bool
|
||||
if at.Kind() < reflect.Array {
|
||||
a, b := av.Interface(), bv.Interface()
|
||||
if a != b {
|
||||
w.printf("%#v != %#v", a, b)
|
||||
switch kind := at.Kind(); kind {
|
||||
case reflect.Bool:
|
||||
if a, b := av.Bool(), bv.Bool(); a != b {
|
||||
w.printf("%v != %v", a, b)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
switch at.Kind() {
|
||||
case reflect.String:
|
||||
a, b := av.Interface(), bv.Interface()
|
||||
if a != b {
|
||||
w.printf("%q != %q", a, b)
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
if a, b := av.Int(), bv.Int(); a != b {
|
||||
w.printf("%d != %d", a, b)
|
||||
}
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
|
||||
if a, b := av.Uint(), bv.Uint(); a != b {
|
||||
w.printf("%d != %d", a, b)
|
||||
}
|
||||
case reflect.Float32, reflect.Float64:
|
||||
if a, b := av.Float(), bv.Float(); a != b {
|
||||
w.printf("%v != %v", a, b)
|
||||
}
|
||||
case reflect.Complex64, reflect.Complex128:
|
||||
if a, b := av.Complex(), bv.Complex(); a != b {
|
||||
w.printf("%v != %v", a, b)
|
||||
}
|
||||
case reflect.Array:
|
||||
n := av.Len()
|
||||
for i := 0; i < n; i++ {
|
||||
w.relabel(fmt.Sprintf("[%d]", i)).diff(av.Index(i), bv.Index(i))
|
||||
}
|
||||
case reflect.Chan, reflect.Func, reflect.UnsafePointer:
|
||||
if a, b := av.Pointer(), bv.Pointer(); a != b {
|
||||
w.printf("%#x != %#x", a, b)
|
||||
}
|
||||
case reflect.Interface:
|
||||
w.diff(av.Elem(), bv.Elem())
|
||||
case reflect.Map:
|
||||
ak, both, bk := keyDiff(av.MapKeys(), bv.MapKeys())
|
||||
for _, k := range ak {
|
||||
w := w.relabel(fmt.Sprintf("[%#v]", k))
|
||||
w.printf("%q != (missing)", av.MapIndex(k))
|
||||
}
|
||||
for _, k := range both {
|
||||
w := w.relabel(fmt.Sprintf("[%#v]", k))
|
||||
w.diff(av.MapIndex(k), bv.MapIndex(k))
|
||||
}
|
||||
for _, k := range bk {
|
||||
w := w.relabel(fmt.Sprintf("[%#v]", k))
|
||||
w.printf("(missing) != %q", bv.MapIndex(k))
|
||||
}
|
||||
case reflect.Ptr:
|
||||
switch {
|
||||
case av.IsNil() && !bv.IsNil():
|
||||
w.printf("nil != %v", bv.Interface())
|
||||
w.printf("nil != %# v", formatter{v: bv, quote: true})
|
||||
case !av.IsNil() && bv.IsNil():
|
||||
w.printf("%v != nil", av.Interface())
|
||||
w.printf("%# v != nil", formatter{v: av, quote: true})
|
||||
case !av.IsNil() && !bv.IsNil():
|
||||
w.diff(av.Elem(), bv.Elem())
|
||||
}
|
||||
case reflect.Struct:
|
||||
for i := 0; i < av.NumField(); i++ {
|
||||
w.relabel(at.Field(i).Name).diff(av.Field(i), bv.Field(i))
|
||||
}
|
||||
case reflect.Slice:
|
||||
lenA := av.Len()
|
||||
lenB := bv.Len()
|
||||
@@ -96,30 +161,20 @@ func (w diffWriter) diff(av, bv reflect.Value) {
|
||||
for i := 0; i < lenA; i++ {
|
||||
w.relabel(fmt.Sprintf("[%d]", i)).diff(av.Index(i), bv.Index(i))
|
||||
}
|
||||
case reflect.Map:
|
||||
ak, both, bk := keyDiff(av.MapKeys(), bv.MapKeys())
|
||||
for _, k := range ak {
|
||||
w := w.relabel(fmt.Sprintf("[%#v]", k.Interface()))
|
||||
w.printf("%q != (missing)", av.MapIndex(k))
|
||||
case reflect.String:
|
||||
if a, b := av.String(), bv.String(); a != b {
|
||||
w.printf("%q != %q", a, b)
|
||||
}
|
||||
for _, k := range both {
|
||||
w := w.relabel(fmt.Sprintf("[%#v]", k.Interface()))
|
||||
w.diff(av.MapIndex(k), bv.MapIndex(k))
|
||||
case reflect.Struct:
|
||||
for i := 0; i < av.NumField(); i++ {
|
||||
w.relabel(at.Field(i).Name).diff(av.Field(i), bv.Field(i))
|
||||
}
|
||||
for _, k := range bk {
|
||||
w := w.relabel(fmt.Sprintf("[%#v]", k.Interface()))
|
||||
w.printf("(missing) != %q", bv.MapIndex(k))
|
||||
}
|
||||
case reflect.Interface:
|
||||
w.diff(reflect.ValueOf(av.Interface()), reflect.ValueOf(bv.Interface()))
|
||||
default:
|
||||
if !reflect.DeepEqual(av.Interface(), bv.Interface()) {
|
||||
w.printf("%# v != %# v", Formatter(av.Interface()), Formatter(bv.Interface()))
|
||||
}
|
||||
panic("unknown reflect Kind: " + kind.String())
|
||||
}
|
||||
}
|
||||
|
||||
func (d diffWriter) relabel(name string) (d1 diffWriter) {
|
||||
func (d diffPrinter) relabel(name string) (d1 diffPrinter) {
|
||||
d1 = d
|
||||
if d.l != "" && name[0] != '[' {
|
||||
d1.l += "."
|
||||
@@ -128,11 +183,63 @@ func (d diffWriter) relabel(name string) (d1 diffWriter) {
|
||||
return d1
|
||||
}
|
||||
|
||||
// keyEqual compares a and b for equality.
|
||||
// Both a and b must be valid map keys.
|
||||
func keyEqual(av, bv reflect.Value) bool {
|
||||
if !av.IsValid() && !bv.IsValid() {
|
||||
return true
|
||||
}
|
||||
if !av.IsValid() || !bv.IsValid() || av.Type() != bv.Type() {
|
||||
return false
|
||||
}
|
||||
switch kind := av.Kind(); kind {
|
||||
case reflect.Bool:
|
||||
a, b := av.Bool(), bv.Bool()
|
||||
return a == b
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
a, b := av.Int(), bv.Int()
|
||||
return a == b
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
|
||||
a, b := av.Uint(), bv.Uint()
|
||||
return a == b
|
||||
case reflect.Float32, reflect.Float64:
|
||||
a, b := av.Float(), bv.Float()
|
||||
return a == b
|
||||
case reflect.Complex64, reflect.Complex128:
|
||||
a, b := av.Complex(), bv.Complex()
|
||||
return a == b
|
||||
case reflect.Array:
|
||||
for i := 0; i < av.Len(); i++ {
|
||||
if !keyEqual(av.Index(i), bv.Index(i)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
case reflect.Chan, reflect.UnsafePointer, reflect.Ptr:
|
||||
a, b := av.Pointer(), bv.Pointer()
|
||||
return a == b
|
||||
case reflect.Interface:
|
||||
return keyEqual(av.Elem(), bv.Elem())
|
||||
case reflect.String:
|
||||
a, b := av.String(), bv.String()
|
||||
return a == b
|
||||
case reflect.Struct:
|
||||
for i := 0; i < av.NumField(); i++ {
|
||||
if !keyEqual(av.Field(i), bv.Field(i)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
default:
|
||||
panic("invalid map key type " + av.Type().String())
|
||||
}
|
||||
}
|
||||
|
||||
func keyDiff(a, b []reflect.Value) (ak, both, bk []reflect.Value) {
|
||||
for _, av := range a {
|
||||
inBoth := false
|
||||
for _, bv := range b {
|
||||
if reflect.DeepEqual(av.Interface(), bv.Interface()) {
|
||||
if keyEqual(av, bv) {
|
||||
inBoth = true
|
||||
both = append(both, av)
|
||||
break
|
||||
@@ -145,7 +252,7 @@ func keyDiff(a, b []reflect.Value) (ak, both, bk []reflect.Value) {
|
||||
for _, bv := range b {
|
||||
inBoth := false
|
||||
for _, av := range a {
|
||||
if reflect.DeepEqual(av.Interface(), bv.Interface()) {
|
||||
if keyEqual(av, bv) {
|
||||
inBoth = true
|
||||
break
|
||||
}
|
||||
|
||||
+141
-2
@@ -1,7 +1,18 @@
|
||||
package pretty
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"log"
|
||||
"reflect"
|
||||
"testing"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
var (
|
||||
_ Logfer = (*testing.T)(nil)
|
||||
_ Logfer = (*testing.B)(nil)
|
||||
_ Printfer = (*log.Logger)(nil)
|
||||
)
|
||||
|
||||
type difftest struct {
|
||||
@@ -17,6 +28,20 @@ type S struct {
|
||||
C []int
|
||||
}
|
||||
|
||||
type (
|
||||
N struct{ N int }
|
||||
E interface{}
|
||||
)
|
||||
|
||||
var (
|
||||
c0 = make(chan int)
|
||||
c1 = make(chan int)
|
||||
f0 = func() {}
|
||||
f1 = func() {}
|
||||
i0 = 0
|
||||
i1 = 1
|
||||
)
|
||||
|
||||
var diffs = []difftest{
|
||||
{a: nil, b: nil},
|
||||
{a: S{A: 1}, b: S{A: 1}},
|
||||
@@ -28,12 +53,79 @@ var diffs = []difftest{
|
||||
{S{}, S{A: 1}, []string{`A: 0 != 1`}},
|
||||
{new(S), &S{A: 1}, []string{`A: 0 != 1`}},
|
||||
{S{S: new(S)}, S{S: &S{A: 1}}, []string{`S.A: 0 != 1`}},
|
||||
{S{}, S{I: 0}, []string{`I: nil != 0`}},
|
||||
{S{}, S{I: 0}, []string{`I: nil != int(0)`}},
|
||||
{S{I: 1}, S{I: "x"}, []string{`I: int != string`}},
|
||||
{S{}, S{C: []int{1}}, []string{`C: []int[0] != []int[1]`}},
|
||||
{S{C: []int{}}, S{C: []int{1}}, []string{`C: []int[0] != []int[1]`}},
|
||||
{S{C: []int{1, 2, 3}}, S{C: []int{1, 2, 4}}, []string{`C[2]: 3 != 4`}},
|
||||
{S{}, S{A: 1, S: new(S)}, []string{`A: 0 != 1`, `S: nil != &{0 <nil> <nil> []}`}},
|
||||
{S{}, S{A: 1, S: new(S)}, []string{`A: 0 != 1`, `S: nil != &pretty.S{}`}},
|
||||
|
||||
// unexported fields of every reflect.Kind (both equal and unequal)
|
||||
{struct{ x bool }{false}, struct{ x bool }{false}, nil},
|
||||
{struct{ x bool }{false}, struct{ x bool }{true}, []string{`x: false != true`}},
|
||||
{struct{ x int }{0}, struct{ x int }{0}, nil},
|
||||
{struct{ x int }{0}, struct{ x int }{1}, []string{`x: 0 != 1`}},
|
||||
{struct{ x int8 }{0}, struct{ x int8 }{0}, nil},
|
||||
{struct{ x int8 }{0}, struct{ x int8 }{1}, []string{`x: 0 != 1`}},
|
||||
{struct{ x int16 }{0}, struct{ x int16 }{0}, nil},
|
||||
{struct{ x int16 }{0}, struct{ x int16 }{1}, []string{`x: 0 != 1`}},
|
||||
{struct{ x int32 }{0}, struct{ x int32 }{0}, nil},
|
||||
{struct{ x int32 }{0}, struct{ x int32 }{1}, []string{`x: 0 != 1`}},
|
||||
{struct{ x int64 }{0}, struct{ x int64 }{0}, nil},
|
||||
{struct{ x int64 }{0}, struct{ x int64 }{1}, []string{`x: 0 != 1`}},
|
||||
{struct{ x uint }{0}, struct{ x uint }{0}, nil},
|
||||
{struct{ x uint }{0}, struct{ x uint }{1}, []string{`x: 0 != 1`}},
|
||||
{struct{ x uint8 }{0}, struct{ x uint8 }{0}, nil},
|
||||
{struct{ x uint8 }{0}, struct{ x uint8 }{1}, []string{`x: 0 != 1`}},
|
||||
{struct{ x uint16 }{0}, struct{ x uint16 }{0}, nil},
|
||||
{struct{ x uint16 }{0}, struct{ x uint16 }{1}, []string{`x: 0 != 1`}},
|
||||
{struct{ x uint32 }{0}, struct{ x uint32 }{0}, nil},
|
||||
{struct{ x uint32 }{0}, struct{ x uint32 }{1}, []string{`x: 0 != 1`}},
|
||||
{struct{ x uint64 }{0}, struct{ x uint64 }{0}, nil},
|
||||
{struct{ x uint64 }{0}, struct{ x uint64 }{1}, []string{`x: 0 != 1`}},
|
||||
{struct{ x uintptr }{0}, struct{ x uintptr }{0}, nil},
|
||||
{struct{ x uintptr }{0}, struct{ x uintptr }{1}, []string{`x: 0 != 1`}},
|
||||
{struct{ x float32 }{0}, struct{ x float32 }{0}, nil},
|
||||
{struct{ x float32 }{0}, struct{ x float32 }{1}, []string{`x: 0 != 1`}},
|
||||
{struct{ x float64 }{0}, struct{ x float64 }{0}, nil},
|
||||
{struct{ x float64 }{0}, struct{ x float64 }{1}, []string{`x: 0 != 1`}},
|
||||
{struct{ x complex64 }{0}, struct{ x complex64 }{0}, nil},
|
||||
{struct{ x complex64 }{0}, struct{ x complex64 }{1}, []string{`x: (0+0i) != (1+0i)`}},
|
||||
{struct{ x complex128 }{0}, struct{ x complex128 }{0}, nil},
|
||||
{struct{ x complex128 }{0}, struct{ x complex128 }{1}, []string{`x: (0+0i) != (1+0i)`}},
|
||||
{struct{ x [1]int }{[1]int{0}}, struct{ x [1]int }{[1]int{0}}, nil},
|
||||
{struct{ x [1]int }{[1]int{0}}, struct{ x [1]int }{[1]int{1}}, []string{`x[0]: 0 != 1`}},
|
||||
{struct{ x chan int }{c0}, struct{ x chan int }{c0}, nil},
|
||||
{struct{ x chan int }{c0}, struct{ x chan int }{c1}, []string{fmt.Sprintf("x: %p != %p", c0, c1)}},
|
||||
{struct{ x func() }{f0}, struct{ x func() }{f0}, nil},
|
||||
{struct{ x func() }{f0}, struct{ x func() }{f1}, []string{fmt.Sprintf("x: %p != %p", f0, f1)}},
|
||||
{struct{ x interface{} }{0}, struct{ x interface{} }{0}, nil},
|
||||
{struct{ x interface{} }{0}, struct{ x interface{} }{1}, []string{`x: 0 != 1`}},
|
||||
{struct{ x interface{} }{0}, struct{ x interface{} }{""}, []string{`x: int != string`}},
|
||||
{struct{ x interface{} }{0}, struct{ x interface{} }{nil}, []string{`x: int(0) != nil`}},
|
||||
{struct{ x interface{} }{nil}, struct{ x interface{} }{0}, []string{`x: nil != int(0)`}},
|
||||
{struct{ x map[int]int }{map[int]int{0: 0}}, struct{ x map[int]int }{map[int]int{0: 0}}, nil},
|
||||
{struct{ x map[int]int }{map[int]int{0: 0}}, struct{ x map[int]int }{map[int]int{0: 1}}, []string{`x[0]: 0 != 1`}},
|
||||
{struct{ x *int }{new(int)}, struct{ x *int }{new(int)}, nil},
|
||||
{struct{ x *int }{&i0}, struct{ x *int }{&i1}, []string{`x: 0 != 1`}},
|
||||
{struct{ x *int }{nil}, struct{ x *int }{&i0}, []string{`x: nil != &int(0)`}},
|
||||
{struct{ x *int }{&i0}, struct{ x *int }{nil}, []string{`x: &int(0) != nil`}},
|
||||
{struct{ x []int }{[]int{0}}, struct{ x []int }{[]int{0}}, nil},
|
||||
{struct{ x []int }{[]int{0}}, struct{ x []int }{[]int{1}}, []string{`x[0]: 0 != 1`}},
|
||||
{struct{ x string }{"a"}, struct{ x string }{"a"}, nil},
|
||||
{struct{ x string }{"a"}, struct{ x string }{"b"}, []string{`x: "a" != "b"`}},
|
||||
{struct{ x N }{N{0}}, struct{ x N }{N{0}}, nil},
|
||||
{struct{ x N }{N{0}}, struct{ x N }{N{1}}, []string{`x.N: 0 != 1`}},
|
||||
{
|
||||
struct{ x unsafe.Pointer }{unsafe.Pointer(uintptr(0))},
|
||||
struct{ x unsafe.Pointer }{unsafe.Pointer(uintptr(0))},
|
||||
nil,
|
||||
},
|
||||
{
|
||||
struct{ x unsafe.Pointer }{unsafe.Pointer(uintptr(0))},
|
||||
struct{ x unsafe.Pointer }{unsafe.Pointer(uintptr(1))},
|
||||
[]string{`x: 0x0 != 0x1`},
|
||||
},
|
||||
}
|
||||
|
||||
func TestDiff(t *testing.T) {
|
||||
@@ -54,6 +146,53 @@ func TestDiff(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeyEqual(t *testing.T) {
|
||||
var emptyInterfaceZero interface{} = 0
|
||||
|
||||
cases := []interface{}{
|
||||
new(bool),
|
||||
new(int),
|
||||
new(int8),
|
||||
new(int16),
|
||||
new(int32),
|
||||
new(int64),
|
||||
new(uint),
|
||||
new(uint8),
|
||||
new(uint16),
|
||||
new(uint32),
|
||||
new(uint64),
|
||||
new(uintptr),
|
||||
new(float32),
|
||||
new(float64),
|
||||
new(complex64),
|
||||
new(complex128),
|
||||
new([1]int),
|
||||
new(chan int),
|
||||
new(unsafe.Pointer),
|
||||
new(interface{}),
|
||||
&emptyInterfaceZero,
|
||||
new(*int),
|
||||
new(string),
|
||||
new(struct{ int }),
|
||||
}
|
||||
|
||||
for _, test := range cases {
|
||||
rv := reflect.ValueOf(test).Elem()
|
||||
if !keyEqual(rv, rv) {
|
||||
t.Errorf("keyEqual(%s, %s) = false want true", rv.Type(), rv.Type())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFdiff(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
Fdiff(&buf, 0, 1)
|
||||
want := "0 != 1\n"
|
||||
if got := buf.String(); got != want {
|
||||
t.Errorf("Fdiff(0, 1) = %q want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func diffdiff(t *testing.T, got, exp []string) {
|
||||
minus(t, "unexpected:", got, exp)
|
||||
minus(t, "missing:", exp, got)
|
||||
|
||||
+5
-14
@@ -10,12 +10,8 @@ import (
|
||||
"github.com/kr/text"
|
||||
)
|
||||
|
||||
const (
|
||||
limit = 50
|
||||
)
|
||||
|
||||
type formatter struct {
|
||||
x interface{}
|
||||
v reflect.Value
|
||||
force bool
|
||||
quote bool
|
||||
}
|
||||
@@ -30,11 +26,11 @@ type formatter struct {
|
||||
// format x according to the usual rules of package fmt.
|
||||
// In particular, if x satisfies fmt.Formatter, then x.Format will be called.
|
||||
func Formatter(x interface{}) (f fmt.Formatter) {
|
||||
return formatter{x: x, quote: true}
|
||||
return formatter{v: reflect.ValueOf(x), quote: true}
|
||||
}
|
||||
|
||||
func (fo formatter) String() string {
|
||||
return fmt.Sprint(fo.x) // unwrap it
|
||||
return fmt.Sprint(fo.v.Interface()) // unwrap it
|
||||
}
|
||||
|
||||
func (fo formatter) passThrough(f fmt.State, c rune) {
|
||||
@@ -51,14 +47,14 @@ func (fo formatter) passThrough(f fmt.State, c rune) {
|
||||
s += fmt.Sprintf(".%d", p)
|
||||
}
|
||||
s += string(c)
|
||||
fmt.Fprintf(f, s, fo.x)
|
||||
fmt.Fprintf(f, s, fo.v.Interface())
|
||||
}
|
||||
|
||||
func (fo formatter) Format(f fmt.State, c rune) {
|
||||
if fo.force || c == 'v' && f.Flag('#') && f.Flag(' ') {
|
||||
w := tabwriter.NewWriter(f, 4, 4, 1, ' ', 0)
|
||||
p := &printer{tw: w, Writer: w, visited: make(map[visit]int)}
|
||||
p.printValue(reflect.ValueOf(fo.x), true, fo.quote)
|
||||
p.printValue(fo.v, true, fo.quote)
|
||||
w.Flush()
|
||||
return
|
||||
}
|
||||
@@ -319,11 +315,6 @@ func (p *printer) fmtString(s string, quote bool) {
|
||||
io.WriteString(p, s)
|
||||
}
|
||||
|
||||
func tryDeepEqual(a, b interface{}) bool {
|
||||
defer func() { recover() }()
|
||||
return reflect.DeepEqual(a, b)
|
||||
}
|
||||
|
||||
func writeByte(w io.Writer, b byte) {
|
||||
w.Write([]byte{b})
|
||||
}
|
||||
|
||||
+27
@@ -13,6 +13,11 @@ type test struct {
|
||||
s string
|
||||
}
|
||||
|
||||
type passtest struct {
|
||||
v interface{}
|
||||
f, s string
|
||||
}
|
||||
|
||||
type LongStructTypeName struct {
|
||||
longFieldName interface{}
|
||||
otherLongFieldName interface{}
|
||||
@@ -33,8 +38,30 @@ func (f F) Format(s fmt.State, c rune) {
|
||||
fmt.Fprintf(s, "F(%d)", int(f))
|
||||
}
|
||||
|
||||
type Stringer struct { i int }
|
||||
|
||||
func (s *Stringer) String() string { return "foo" }
|
||||
|
||||
var long = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
||||
|
||||
var passthrough = []passtest{
|
||||
{1, "%d", "1"},
|
||||
{"a", "%s", "a"},
|
||||
{&Stringer{}, "%s", "foo"},
|
||||
}
|
||||
|
||||
func TestPassthrough(t *testing.T) {
|
||||
for _, tt := range passthrough {
|
||||
s := fmt.Sprintf(tt.f, Formatter(tt.v))
|
||||
if tt.s != s {
|
||||
t.Errorf("expected %q", tt.s)
|
||||
t.Errorf("got %q", s)
|
||||
t.Errorf("expraw\n%s", tt.s)
|
||||
t.Errorf("gotraw\n%s", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var gosyntax = []test{
|
||||
{nil, `nil`},
|
||||
{"", `""`},
|
||||
|
||||
+11
-1
@@ -11,6 +11,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
// Errorf is a convenience wrapper for fmt.Errorf.
|
||||
@@ -81,6 +82,15 @@ func Println(a ...interface{}) (n int, errno error) {
|
||||
return fmt.Println(wrap(a, true)...)
|
||||
}
|
||||
|
||||
// Sprint is a convenience wrapper for fmt.Sprintf.
|
||||
//
|
||||
// Calling Sprint(x, y) is equivalent to
|
||||
// fmt.Sprint(Formatter(x), Formatter(y)), but each operand is
|
||||
// formatted with "%# v".
|
||||
func Sprint(a ...interface{}) string {
|
||||
return fmt.Sprint(wrap(a, true)...)
|
||||
}
|
||||
|
||||
// Sprintf is a convenience wrapper for fmt.Sprintf.
|
||||
//
|
||||
// Calling Sprintf(f, x, y) is equivalent to
|
||||
@@ -92,7 +102,7 @@ func Sprintf(format string, a ...interface{}) string {
|
||||
func wrap(a []interface{}, force bool) []interface{} {
|
||||
w := make([]interface{}, len(a))
|
||||
for i, x := range a {
|
||||
w[i] = formatter{x: x, force: force}
|
||||
w[i] = formatter{v: reflect.ValueOf(x), force: force}
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
-19
@@ -1,19 +0,0 @@
|
||||
Copyright (c) 2012 Keith Rarick.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
this software and associated documentation files (the "Software"), to deal in
|
||||
the Software without restriction, including without limitation the rights to
|
||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||
of the Software, and to permit persons to whom the Software is furnished to do
|
||||
so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
-4
@@ -1,4 +0,0 @@
|
||||
Package s3 signs HTTP requests for use with Amazon’s S3 API.
|
||||
|
||||
Documentation:
|
||||
http://godoc.org/github.com/kr/s3
|
||||
-4
@@ -1,4 +0,0 @@
|
||||
Package s3util provides streaming transfers to and from Amazon S3.
|
||||
|
||||
Full documentation:
|
||||
http://godoc.org/github.com/kr/s3/s3util
|
||||
-27
@@ -1,27 +0,0 @@
|
||||
// Package s3util provides streaming transfers to and from Amazon S3.
|
||||
//
|
||||
// To use it, open or create an S3 object, read or write data,
|
||||
// and close the object.
|
||||
//
|
||||
// You must assign valid credentials to DefaultConfig.Keys before using
|
||||
// DefaultConfig. Be sure to close an io.WriteCloser returned by this package,
|
||||
// to flush buffers and complete the multipart upload process.
|
||||
package s3util
|
||||
|
||||
// TODO(kr): parse error responses; return structured data
|
||||
|
||||
import (
|
||||
"github.com/kr/s3"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
var DefaultConfig = &Config{
|
||||
Service: s3.DefaultService,
|
||||
Keys: new(s3.Keys),
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
*s3.Service
|
||||
*s3.Keys
|
||||
*http.Client // if nil, uses http.DefaultClient
|
||||
}
|
||||
-29
@@ -1,29 +0,0 @@
|
||||
package s3util
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type respError struct {
|
||||
r *http.Response
|
||||
b bytes.Buffer
|
||||
}
|
||||
|
||||
func newRespError(r *http.Response) *respError {
|
||||
e := new(respError)
|
||||
e.r = r
|
||||
io.Copy(&e.b, r.Body)
|
||||
r.Body.Close()
|
||||
return e
|
||||
}
|
||||
|
||||
func (e *respError) Error() string {
|
||||
return fmt.Sprintf(
|
||||
"unwanted http status %d: %q",
|
||||
e.r.StatusCode,
|
||||
e.b.String(),
|
||||
)
|
||||
}
|
||||
-33
@@ -1,33 +0,0 @@
|
||||
package s3util
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Open requests the S3 object at url. An HTTP status other than 200 is
|
||||
// considered an error.
|
||||
//
|
||||
// If c is nil, Open uses DefaultConfig.
|
||||
func Open(url string, c *Config) (io.ReadCloser, error) {
|
||||
if c == nil {
|
||||
c = DefaultConfig
|
||||
}
|
||||
// TODO(kr): maybe parallel range fetching
|
||||
r, _ := http.NewRequest("GET", url, nil)
|
||||
r.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat))
|
||||
c.Sign(r, *c.Keys)
|
||||
client := c.Client
|
||||
if client == nil {
|
||||
client = http.DefaultClient
|
||||
}
|
||||
resp, err := client.Do(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
return nil, newRespError(resp)
|
||||
}
|
||||
return resp.Body, nil
|
||||
}
|
||||
-208
@@ -1,208 +0,0 @@
|
||||
package s3util
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// File represents an S3 object or directory.
|
||||
type File struct {
|
||||
url string
|
||||
prefix string
|
||||
config *Config
|
||||
result *listObjectsResult
|
||||
}
|
||||
|
||||
type fileInfo struct {
|
||||
name string
|
||||
size int64
|
||||
dir bool
|
||||
modTime time.Time
|
||||
sys *Stat
|
||||
}
|
||||
|
||||
// Stat contains information about an S3 object or directory.
|
||||
// It is the "underlying data source" returned by method Sys
|
||||
// for each FileInfo produced by this package.
|
||||
// fi.Sys().(*s3util.Stat)
|
||||
// For the meaning of these fields, see
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html.
|
||||
type Stat struct {
|
||||
Key string
|
||||
LastModified string
|
||||
ETag string // ETag value, without double quotes.
|
||||
Size string
|
||||
StorageClass string
|
||||
OwnerID string `xml:"Owner>ID"`
|
||||
OwnerName string `xml:"Owner>DisplayName"`
|
||||
}
|
||||
|
||||
type listObjectsResult struct {
|
||||
IsTruncated bool
|
||||
Contents []Stat
|
||||
Directories []string `xml:"CommonPrefixes>Prefix"` // Suffix "/" trimmed
|
||||
}
|
||||
|
||||
func (f *fileInfo) Name() string { return f.name }
|
||||
func (f *fileInfo) Size() int64 { return f.size }
|
||||
func (f *fileInfo) Mode() os.FileMode {
|
||||
if f.dir {
|
||||
return 0755 | os.ModeDir
|
||||
}
|
||||
return 0644
|
||||
}
|
||||
func (f *fileInfo) ModTime() time.Time {
|
||||
if f.modTime.IsZero() && f.sys != nil {
|
||||
// we return the zero value if a parse error ever happens.
|
||||
f.modTime, _ = time.Parse(time.RFC3339Nano, f.sys.LastModified)
|
||||
}
|
||||
return f.modTime
|
||||
}
|
||||
func (f *fileInfo) IsDir() bool { return f.dir }
|
||||
func (f *fileInfo) Sys() interface{} { return f.sys }
|
||||
|
||||
// NewFile returns a new File with the given URL and config.
|
||||
//
|
||||
// Set rawurl to a directory on S3, such as
|
||||
// https://mybucket.s3.amazonaws.com/myfolder.
|
||||
// The URL cannot have query parameters or a fragment.
|
||||
// If c is nil, DefaultConfig will be used.
|
||||
func NewFile(rawurl string, c *Config) (*File, error) {
|
||||
u, err := url.Parse(rawurl)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if u.RawQuery != "" {
|
||||
return nil, errors.New("url cannot have raw query parameters.")
|
||||
}
|
||||
if u.Fragment != "" {
|
||||
return nil, errors.New("url cannot have a fragment.")
|
||||
}
|
||||
|
||||
prefix := strings.TrimLeft(u.Path, "/")
|
||||
if prefix != "" && !strings.HasSuffix(prefix, "/") {
|
||||
prefix += "/"
|
||||
}
|
||||
u.Path = ""
|
||||
return &File{u.String(), prefix, c, nil}, nil
|
||||
}
|
||||
|
||||
// Readdir requests a list of entries in the S3 directory
|
||||
// represented by f and returns a slice of up to n FileInfo
|
||||
// values, in alphabetical order. Subsequent calls
|
||||
// on the same File will yield further FileInfos.
|
||||
// Only direct children are returned, not deeper descendants.
|
||||
func (f *File) Readdir(n int) ([]os.FileInfo, error) {
|
||||
if f.result != nil && !f.result.IsTruncated {
|
||||
return make([]os.FileInfo, 0), io.EOF
|
||||
}
|
||||
|
||||
reader, err := f.sendRequest(n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
return f.parseResponse(reader)
|
||||
}
|
||||
|
||||
func (f *File) sendRequest(count int) (io.ReadCloser, error) {
|
||||
c := f.config
|
||||
if c == nil {
|
||||
c = DefaultConfig
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
buf.WriteString(f.url)
|
||||
buf.WriteString("?delimiter=%2F")
|
||||
if f.prefix != "" {
|
||||
buf.WriteString("&prefix=")
|
||||
buf.WriteString(url.QueryEscape(f.prefix))
|
||||
}
|
||||
if count > 0 {
|
||||
buf.WriteString("&max-keys=")
|
||||
buf.WriteString(strconv.Itoa(count))
|
||||
}
|
||||
if f.result != nil && f.result.IsTruncated {
|
||||
var lastDir, lastKey, marker string
|
||||
if len(f.result.Directories) > 0 {
|
||||
lastDir = f.result.Directories[len(f.result.Directories)-1]
|
||||
}
|
||||
if len(f.result.Contents) > 0 {
|
||||
lastKey = f.result.Contents[len(f.result.Contents)-1].Key
|
||||
}
|
||||
|
||||
if lastKey > lastDir {
|
||||
marker = lastKey
|
||||
} else {
|
||||
marker = lastDir
|
||||
}
|
||||
|
||||
if marker != "" {
|
||||
buf.WriteString("&marker=")
|
||||
buf.WriteString(url.QueryEscape(marker))
|
||||
}
|
||||
}
|
||||
u := buf.String()
|
||||
r, _ := http.NewRequest("GET", u, nil)
|
||||
r.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat))
|
||||
c.Sign(r, *c.Keys)
|
||||
resp, err := http.DefaultClient.Do(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
return nil, newRespError(resp)
|
||||
}
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
func (f *File) parseResponse(reader io.Reader) ([]os.FileInfo, error) {
|
||||
decoder := xml.NewDecoder(reader)
|
||||
result := listObjectsResult{}
|
||||
var err error
|
||||
err = decoder.Decode(&result)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
infos := make([]os.FileInfo, len(result.Contents)+len(result.Directories))
|
||||
var size int64
|
||||
var name string
|
||||
var is_dir bool
|
||||
for i, content := range result.Contents {
|
||||
c := content
|
||||
c.ETag = strings.Trim(c.ETag, `"`)
|
||||
size, _ = strconv.ParseInt(c.Size, 10, 0)
|
||||
if size == 0 && strings.HasSuffix(c.Key, "/") {
|
||||
name = strings.TrimRight(c.Key, "/")
|
||||
is_dir = true
|
||||
} else {
|
||||
name = c.Key
|
||||
is_dir = false
|
||||
}
|
||||
infos[i] = &fileInfo{
|
||||
name: name,
|
||||
size: size,
|
||||
dir: is_dir,
|
||||
sys: &c,
|
||||
}
|
||||
}
|
||||
for i, dir := range result.Directories {
|
||||
infos[len(result.Contents)+i] = &fileInfo{
|
||||
name: strings.TrimRight(dir, "/"),
|
||||
size: 0,
|
||||
dir: true,
|
||||
}
|
||||
}
|
||||
f.result = &result
|
||||
|
||||
return infos, nil
|
||||
}
|
||||
-267
@@ -1,267 +0,0 @@
|
||||
package s3util
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"github.com/kr/s3"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// defined by amazon
|
||||
const (
|
||||
minPartSize = 5 * 1024 * 1024
|
||||
maxPartSize = 1<<31 - 1 // for 32-bit use; amz max is 5GiB
|
||||
maxObjSize = 5 * 1024 * 1024 * 1024 * 1024
|
||||
maxNPart = 10000
|
||||
)
|
||||
|
||||
const (
|
||||
concurrency = 5
|
||||
nTry = 2
|
||||
)
|
||||
|
||||
type part struct {
|
||||
r io.ReadSeeker
|
||||
len int64
|
||||
|
||||
// read by xml encoder
|
||||
PartNumber int
|
||||
ETag string
|
||||
}
|
||||
|
||||
type uploader struct {
|
||||
s3 s3.Service
|
||||
keys s3.Keys
|
||||
url string
|
||||
client *http.Client
|
||||
UploadId string // written by xml decoder
|
||||
|
||||
bufsz int64
|
||||
buf []byte
|
||||
off int
|
||||
ch chan *part
|
||||
part int
|
||||
closed bool
|
||||
err error
|
||||
wg sync.WaitGroup
|
||||
|
||||
xml struct {
|
||||
XMLName string `xml:"CompleteMultipartUpload"`
|
||||
Part []*part
|
||||
}
|
||||
}
|
||||
|
||||
// Create creates an S3 object at url and sends multipart upload requests as
|
||||
// data is written.
|
||||
//
|
||||
// If h is not nil, each of its entries is added to the HTTP request header.
|
||||
// If c is nil, Create uses DefaultConfig.
|
||||
func Create(url string, h http.Header, c *Config) (io.WriteCloser, error) {
|
||||
if c == nil {
|
||||
c = DefaultConfig
|
||||
}
|
||||
return newUploader(url, h, c)
|
||||
}
|
||||
|
||||
// Sends an S3 multipart upload initiation request.
|
||||
// See http://docs.amazonwebservices.com/AmazonS3/latest/dev/mpuoverview.html.
|
||||
// This initial request returns an UploadId that we use to identify
|
||||
// subsequent PUT requests.
|
||||
func newUploader(url string, h http.Header, c *Config) (u *uploader, err error) {
|
||||
u = new(uploader)
|
||||
u.s3 = *c.Service
|
||||
u.url = url
|
||||
u.keys = *c.Keys
|
||||
u.client = c.Client
|
||||
if u.client == nil {
|
||||
u.client = http.DefaultClient
|
||||
}
|
||||
u.bufsz = minPartSize
|
||||
r, err := http.NewRequest("POST", url+"?uploads", nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat))
|
||||
for k := range h {
|
||||
for _, v := range h[k] {
|
||||
r.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
u.s3.Sign(r, u.keys)
|
||||
resp, err := u.client.Do(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
return nil, newRespError(resp)
|
||||
}
|
||||
err = xml.NewDecoder(resp.Body).Decode(u)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
u.ch = make(chan *part)
|
||||
for i := 0; i < concurrency; i++ {
|
||||
go u.worker()
|
||||
}
|
||||
return u, nil
|
||||
}
|
||||
|
||||
func (u *uploader) Write(p []byte) (n int, err error) {
|
||||
if u.closed {
|
||||
return 0, syscall.EINVAL
|
||||
}
|
||||
if u.err != nil {
|
||||
return 0, u.err
|
||||
}
|
||||
for n < len(p) {
|
||||
if cap(u.buf) == 0 {
|
||||
u.buf = make([]byte, int(u.bufsz))
|
||||
// Increase part size (1.001x).
|
||||
// This lets us reach the max object size (5TiB) while
|
||||
// still doing minimal buffering for small objects.
|
||||
u.bufsz = min(u.bufsz+u.bufsz/1000, maxPartSize)
|
||||
}
|
||||
r := copy(u.buf[u.off:], p[n:])
|
||||
u.off += r
|
||||
n += r
|
||||
if u.off == len(u.buf) {
|
||||
u.flush()
|
||||
}
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (u *uploader) flush() {
|
||||
u.wg.Add(1)
|
||||
u.part++
|
||||
p := &part{bytes.NewReader(u.buf[:u.off]), int64(u.off), u.part, ""}
|
||||
u.xml.Part = append(u.xml.Part, p)
|
||||
u.ch <- p
|
||||
u.buf, u.off = nil, 0
|
||||
}
|
||||
|
||||
func (u *uploader) worker() {
|
||||
for p := range u.ch {
|
||||
u.retryUploadPart(p)
|
||||
}
|
||||
}
|
||||
|
||||
// Calls putPart up to nTry times to recover from transient errors.
|
||||
func (u *uploader) retryUploadPart(p *part) {
|
||||
defer u.wg.Done()
|
||||
defer func() { p.r = nil }() // free the large buffer
|
||||
var err error
|
||||
for i := 0; i < nTry; i++ {
|
||||
p.r.Seek(0, 0)
|
||||
err = u.putPart(p)
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
u.err = err
|
||||
}
|
||||
|
||||
// Uploads part p, reading its contents from p.r.
|
||||
// Stores the ETag in p.ETag.
|
||||
func (u *uploader) putPart(p *part) error {
|
||||
v := url.Values{}
|
||||
v.Set("partNumber", strconv.Itoa(p.PartNumber))
|
||||
v.Set("uploadId", u.UploadId)
|
||||
req, err := http.NewRequest("PUT", u.url+"?"+v.Encode(), p.r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.ContentLength = p.len
|
||||
req.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat))
|
||||
u.s3.Sign(req, u.keys)
|
||||
resp, err := u.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
return newRespError(resp)
|
||||
}
|
||||
s := resp.Header.Get("etag") // includes quote chars for some reason
|
||||
if len(s) < 2 {
|
||||
return fmt.Errorf("received invalid etag %q", s)
|
||||
}
|
||||
p.ETag = s[1 : len(s)-1]
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u *uploader) Close() error {
|
||||
if u.closed {
|
||||
return syscall.EINVAL
|
||||
}
|
||||
if cap(u.buf) > 0 {
|
||||
u.flush()
|
||||
}
|
||||
u.wg.Wait()
|
||||
close(u.ch)
|
||||
u.closed = true
|
||||
if u.err != nil {
|
||||
u.abort()
|
||||
return u.err
|
||||
}
|
||||
|
||||
body, err := xml.Marshal(u.xml)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b := bytes.NewBuffer(body)
|
||||
v := url.Values{}
|
||||
v.Set("uploadId", u.UploadId)
|
||||
req, err := http.NewRequest("POST", u.url+"?"+v.Encode(), b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat))
|
||||
u.s3.Sign(req, u.keys)
|
||||
resp, err := u.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
return newRespError(resp)
|
||||
}
|
||||
resp.Body.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u *uploader) abort() {
|
||||
// TODO(kr): devise a reasonable way to report an error here in addition
|
||||
// to the error that caused the abort.
|
||||
v := url.Values{}
|
||||
v.Set("uploadId", u.UploadId)
|
||||
s := u.url + "?" + v.Encode()
|
||||
req, err := http.NewRequest("DELETE", s, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
req.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat))
|
||||
u.s3.Sign(req, u.keys)
|
||||
resp, err := u.client.Do(req)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func min(a, b int64) int64 {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
-200
@@ -1,200 +0,0 @@
|
||||
// Package s3 signs HTTP requests for Amazon S3 and compatible services.
|
||||
package s3
|
||||
|
||||
// See
|
||||
// http://docs.amazonwebservices.com/AmazonS3/2006-03-01/dev/RESTAuthentication.html.
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var signParams = map[string]bool{
|
||||
"acl": true,
|
||||
"delete": true,
|
||||
"lifecycle": true,
|
||||
"location": true,
|
||||
"logging": true,
|
||||
"notification": true,
|
||||
"partNumber": true,
|
||||
"policy": true,
|
||||
"requestPayment": true,
|
||||
"response-cache-control": true,
|
||||
"response-content-disposition": true,
|
||||
"response-content-encoding": true,
|
||||
"response-content-language": true,
|
||||
"response-content-type": true,
|
||||
"response-expires": true,
|
||||
"restore": true,
|
||||
"torrent": true,
|
||||
"uploadId": true,
|
||||
"uploads": true,
|
||||
"versionId": true,
|
||||
"versioning": true,
|
||||
"versions": true,
|
||||
"website": true,
|
||||
}
|
||||
|
||||
// Keys holds a set of Amazon Security Credentials.
|
||||
type Keys struct {
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
|
||||
// SecurityToken is used for temporary security credentials.
|
||||
// If set, it will be added to header field X-Amz-Security-Token
|
||||
// before signing a request.
|
||||
SecurityToken string
|
||||
// See http://docs.aws.amazon.com/AmazonS3/latest/dev/MakingRequests.html#TypesofSecurityCredentials
|
||||
}
|
||||
|
||||
// IdentityBucket returns subdomain.
|
||||
// It is designed to be used with S3-compatible services that
|
||||
// treat the entire subdomain as the bucket name, for example
|
||||
// storage.io.
|
||||
func IdentityBucket(subdomain string) string {
|
||||
return subdomain
|
||||
}
|
||||
|
||||
// AmazonBucket returns everything up to the last '.' in subdomain.
|
||||
// It is designed to be used with the Amazon service.
|
||||
// "johnsmith.s3" becomes "johnsmith"
|
||||
// "johnsmith.s3-eu-west-1" becomes "johnsmith"
|
||||
// "www.example.com.s3" becomes "www.example.com"
|
||||
func AmazonBucket(subdomain string) string {
|
||||
if i := strings.LastIndex(subdomain, "."); i != -1 {
|
||||
return subdomain[:i]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// DefaultService is the default Service used by Sign.
|
||||
var DefaultService = &Service{Domain: "amazonaws.com"}
|
||||
|
||||
// Sign signs an HTTP request with the given S3 keys.
|
||||
//
|
||||
// This function is a wrapper around DefaultService.Sign.
|
||||
func Sign(r *http.Request, k Keys) {
|
||||
DefaultService.Sign(r, k)
|
||||
}
|
||||
|
||||
// Service represents an S3-compatible service.
|
||||
type Service struct {
|
||||
// Domain is the service's root domain. It is used to extract
|
||||
// the subdomain from an http.Request before passing the
|
||||
// subdomain to Bucket.
|
||||
Domain string
|
||||
|
||||
// Bucket derives the bucket name from a subdomain.
|
||||
// If nil, AmazonBucket is used.
|
||||
Bucket func(subdomain string) string
|
||||
}
|
||||
|
||||
// Sign signs an HTTP request with the given S3 keys for use on service s.
|
||||
func (s *Service) Sign(r *http.Request, k Keys) {
|
||||
if k.SecurityToken != "" {
|
||||
r.Header.Set("X-Amz-Security-Token", k.SecurityToken)
|
||||
}
|
||||
h := hmac.New(sha1.New, []byte(k.SecretKey))
|
||||
s.writeSigData(h, r)
|
||||
sig := make([]byte, base64.StdEncoding.EncodedLen(h.Size()))
|
||||
base64.StdEncoding.Encode(sig, h.Sum(nil))
|
||||
r.Header.Set("Authorization", "AWS "+k.AccessKey+":"+string(sig))
|
||||
}
|
||||
|
||||
func (s *Service) writeSigData(w io.Writer, r *http.Request) {
|
||||
w.Write([]byte(r.Method))
|
||||
w.Write([]byte{'\n'})
|
||||
w.Write([]byte(r.Header.Get("content-md5")))
|
||||
w.Write([]byte{'\n'})
|
||||
w.Write([]byte(r.Header.Get("content-type")))
|
||||
w.Write([]byte{'\n'})
|
||||
if _, ok := r.Header["X-Amz-Date"]; !ok {
|
||||
w.Write([]byte(r.Header.Get("date")))
|
||||
}
|
||||
w.Write([]byte{'\n'})
|
||||
writeAmzHeaders(w, r)
|
||||
s.writeResource(w, r)
|
||||
}
|
||||
|
||||
func (s *Service) writeResource(w io.Writer, r *http.Request) {
|
||||
s.writeVhostBucket(w, strings.ToLower(r.Host))
|
||||
path := r.URL.RequestURI()
|
||||
if r.URL.RawQuery != "" {
|
||||
path = path[:len(path)-len(r.URL.RawQuery)-1]
|
||||
}
|
||||
w.Write([]byte(path))
|
||||
s.writeSubResource(w, r)
|
||||
}
|
||||
|
||||
func (s *Service) writeVhostBucket(w io.Writer, host string) {
|
||||
if i := strings.Index(host, ":"); i != -1 {
|
||||
host = host[:i]
|
||||
}
|
||||
|
||||
if host == s.Domain {
|
||||
// no vhost - do nothing
|
||||
} else if strings.HasSuffix(host, "."+s.Domain) {
|
||||
// vhost - bucket may be in prefix
|
||||
b := s.Bucket
|
||||
if b == nil {
|
||||
b = AmazonBucket
|
||||
}
|
||||
bucket := b(host[:len(host)-len(s.Domain)-1])
|
||||
|
||||
if bucket != "" {
|
||||
w.Write([]byte{'/'})
|
||||
w.Write([]byte(bucket))
|
||||
}
|
||||
} else {
|
||||
// cname - bucket is host
|
||||
w.Write([]byte{'/'})
|
||||
w.Write([]byte(host))
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) writeSubResource(w io.Writer, r *http.Request) {
|
||||
var a []string
|
||||
for k, vs := range r.URL.Query() {
|
||||
if signParams[k] {
|
||||
for _, v := range vs {
|
||||
if v == "" {
|
||||
a = append(a, k)
|
||||
} else {
|
||||
a = append(a, k+"="+v)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(a)
|
||||
var p byte = '?'
|
||||
for _, s := range a {
|
||||
w.Write([]byte{p})
|
||||
w.Write([]byte(s))
|
||||
p = '&'
|
||||
}
|
||||
}
|
||||
|
||||
func writeAmzHeaders(w io.Writer, r *http.Request) {
|
||||
var keys []string
|
||||
for k, _ := range r.Header {
|
||||
if strings.HasPrefix(strings.ToLower(k), "x-amz-") {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
}
|
||||
|
||||
sort.Strings(keys)
|
||||
var a []string
|
||||
for _, k := range keys {
|
||||
v := r.Header[k]
|
||||
a = append(a, strings.ToLower(k)+":"+strings.Join(v, ","))
|
||||
}
|
||||
for _, h := range a {
|
||||
w.Write([]byte(h))
|
||||
w.Write([]byte{'\n'})
|
||||
}
|
||||
}
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
/*
|
||||
|
||||
Agg computes aggregate values over tabular text.
|
||||
It behaves somewhat like the SQL “GROUP BY” clause.
|
||||
|
||||
Usage:
|
||||
|
||||
agg [function...]
|
||||
|
||||
It reads input from stdin as a sequence of records, one per line.
|
||||
It treats each line as a set of fields separated by white space.
|
||||
One field (the first, by default) is designated as the key.
|
||||
Successive lines with equal keys are grouped into a group,
|
||||
and agg produces one line of output for each group.
|
||||
(Note that only contiguous input lines can form a group.
|
||||
If you need to make sure that all records for a given key
|
||||
are grouped together, sort the input first.)
|
||||
|
||||
For each remaining field,
|
||||
agg applies a function to all the values in the group,
|
||||
producing a single output value.
|
||||
The command line arguments specify which functions to use,
|
||||
one per field in the input table.
|
||||
|
||||
Functions
|
||||
|
||||
The available functions are:
|
||||
|
||||
key group by this field (default for field 1)
|
||||
first value from first line of group (default for rest)
|
||||
last value from last line of group
|
||||
sample value from any line of group, uniformly at random
|
||||
prefix longest common string prefix
|
||||
join:sep concatenate strings with given sep
|
||||
smin lexically least string
|
||||
smax lexically greatest string
|
||||
min numerically least value
|
||||
max numerically greatest value
|
||||
sum numeric sum
|
||||
mean arithmetic mean
|
||||
count number of records (ignores input value)
|
||||
const:val print val, ignoring input
|
||||
drop omit the column entirely
|
||||
|
||||
The numeric functions skip items that don't parse as numbers.
|
||||
|
||||
Examples
|
||||
|
||||
Using the following input:
|
||||
|
||||
$ cat >input
|
||||
-rwx alice 100 /home/alice/bin/crdt
|
||||
-rw- alice 210002 /home/alice/thesis.tex
|
||||
-rw- bob 10051 /home/bob/expenses.tab
|
||||
-rwx kr 862060 /home/kr/bin/blog
|
||||
-rwx kr 304608 /home/kr/bin/agg
|
||||
|
||||
Disk usage for each user, plus where that disk usage occurs
|
||||
(longest common prefix of filesystem paths):
|
||||
|
||||
$ agg <input drop key sum prefix
|
||||
alice 210153 /home/alice/
|
||||
bob 10051 /home/bob/expenses.tab
|
||||
kr 1166668 /home/kr/
|
||||
|
||||
Disk usage for executable vs non-executable files:
|
||||
|
||||
$ sort input | agg key drop sum join:,
|
||||
-rw- 220053 /home/alice/thesis.tex,/home/bob/expenses.tab
|
||||
-rwx 1166768 /home/alice/bin/crdt,/home/kr/bin/agg,/home/kr/bin/blog
|
||||
|
||||
*/
|
||||
package main
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
package main
|
||||
|
||||
// TODO(kr): tests
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/rand"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type agg interface {
|
||||
merge(string)
|
||||
String() string
|
||||
}
|
||||
|
||||
var (
|
||||
key = 0
|
||||
funcmap = make(map[int]func(init, arg string) agg)
|
||||
argmap = make(map[int]string)
|
||||
symtab = map[string]func(init, arg string) agg{
|
||||
"first": first,
|
||||
"last": last,
|
||||
"prefix": prefix,
|
||||
"sample": sample,
|
||||
"join": join,
|
||||
"smin": smin,
|
||||
"smax": smax,
|
||||
"min": min,
|
||||
"max": max,
|
||||
"sum": sum,
|
||||
"mean": mean,
|
||||
"count": count,
|
||||
"const": constf,
|
||||
"drop": nil,
|
||||
}
|
||||
)
|
||||
|
||||
func main() {
|
||||
log.SetPrefix("agg: ")
|
||||
log.SetFlags(0)
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
for i, sym := range os.Args[1:] {
|
||||
if p := strings.IndexByte(sym, ':'); p >= 0 {
|
||||
sym, argmap[i] = sym[:p], sym[p+1:]
|
||||
}
|
||||
if sym == "key" {
|
||||
key, sym = i, "first"
|
||||
}
|
||||
f, ok := symtab[sym]
|
||||
if !ok {
|
||||
log.Fatalf("bad function: %q", sym)
|
||||
}
|
||||
funcmap[i] = f
|
||||
}
|
||||
|
||||
sc := bufio.NewScanner(os.Stdin)
|
||||
var g *group
|
||||
for sc.Scan() {
|
||||
ss := strings.Fields(sc.Text())
|
||||
if !matches(g, ss) {
|
||||
emit(g)
|
||||
g = &group{key: ss[key]}
|
||||
}
|
||||
mergeLine(g, ss)
|
||||
}
|
||||
emit(g)
|
||||
}
|
||||
|
||||
type group struct {
|
||||
key string
|
||||
agg []agg
|
||||
}
|
||||
|
||||
func matches(g *group, ss []string) bool {
|
||||
return g != nil && g.key == ss[key]
|
||||
}
|
||||
|
||||
func emit(g *group) {
|
||||
if g == nil {
|
||||
return
|
||||
}
|
||||
rest := false
|
||||
for i, a := range g.agg {
|
||||
if f, ok := funcmap[i]; ok && f == nil {
|
||||
continue
|
||||
}
|
||||
if rest {
|
||||
fmt.Print("\t")
|
||||
}
|
||||
rest = true
|
||||
fmt.Print(a)
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
func mergeLine(g *group, ss []string) {
|
||||
for i, s := range ss {
|
||||
if i >= len(g.agg) {
|
||||
f := funcmap[i]
|
||||
if f == nil {
|
||||
f = first
|
||||
}
|
||||
g.agg = append(g.agg, f(s, argmap[i]))
|
||||
} else {
|
||||
g.agg[i].merge(s)
|
||||
}
|
||||
}
|
||||
}
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"math/big"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
func min(s, arg string) agg { return newBinop(s, opmin) }
|
||||
func max(s, arg string) agg { return newBinop(s, opmax) }
|
||||
func sum(s, arg string) agg { return newBinop(s, opsum) }
|
||||
|
||||
type binop struct {
|
||||
v *big.Float
|
||||
f func(a, b *big.Float) *big.Float
|
||||
}
|
||||
|
||||
func newBinop(s string, f func(a, b *big.Float) *big.Float) *binop {
|
||||
v, _ := parseFloat(s)
|
||||
return &binop{v, f}
|
||||
}
|
||||
|
||||
func (o *binop) String() string {
|
||||
if o.v == nil {
|
||||
return "NaN"
|
||||
}
|
||||
return o.v.Text('f', -1)
|
||||
}
|
||||
|
||||
func (o *binop) merge(s string) {
|
||||
v, ok := parseFloat(s)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
o.v = o.f(o.v, v)
|
||||
}
|
||||
|
||||
func opmin(a, b *big.Float) *big.Float {
|
||||
if a != nil && (b == nil || a.Cmp(b) <= 0) {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func opmax(a, b *big.Float) *big.Float {
|
||||
if a != nil && (b == nil || a.Cmp(b) >= 0) {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func opsum(a, b *big.Float) *big.Float {
|
||||
if a == nil {
|
||||
return b
|
||||
} else if b == nil {
|
||||
return a
|
||||
}
|
||||
return a.Add(a, b)
|
||||
}
|
||||
|
||||
type meanagg struct {
|
||||
v *big.Float
|
||||
d float64 // actually an integer
|
||||
}
|
||||
|
||||
func mean(s, arg string) agg {
|
||||
v, ok := parseFloat(s)
|
||||
if !ok {
|
||||
return &meanagg{new(big.Float), 0}
|
||||
}
|
||||
return &meanagg{v, 1}
|
||||
}
|
||||
|
||||
func (m *meanagg) String() string {
|
||||
if m.d == 0 {
|
||||
return "NaN"
|
||||
}
|
||||
v := new(big.Float).Quo(m.v, big.NewFloat(m.d))
|
||||
return v.Text('f', -1)
|
||||
}
|
||||
|
||||
func (m *meanagg) merge(s string) {
|
||||
v, ok := parseFloat(s)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
m.v.Add(m.v, v)
|
||||
m.d++
|
||||
}
|
||||
|
||||
func parseFloat(s string) (*big.Float, bool) {
|
||||
v, _, err := big.ParseFloat(s, 0, 1000, big.ToNearestEven)
|
||||
return v, err == nil
|
||||
}
|
||||
|
||||
type counter int
|
||||
|
||||
func count(init, arg string) agg { return new(counter) }
|
||||
func (c *counter) String() string { return strconv.Itoa(int(*c) + 1) }
|
||||
func (c *counter) merge(string) { *c++ }
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func first(s, arg string) agg { return &sbinop{s, opfirst} }
|
||||
func last(s, arg string) agg { return &sbinop{s, oplast} }
|
||||
func prefix(s, arg string) agg { return &sbinop{s, opprefix} }
|
||||
func join(s, arg string) agg { return &sbinop{s, opjoin(arg)} }
|
||||
func smin(s, arg string) agg { return &sbinop{s, opsmin} }
|
||||
func smax(s, arg string) agg { return &sbinop{s, opsmax} }
|
||||
|
||||
type sbinop struct {
|
||||
s string
|
||||
f func(a, b string) string
|
||||
}
|
||||
|
||||
func (o *sbinop) String() string { return o.s }
|
||||
|
||||
func (o *sbinop) merge(s string) { o.s = o.f(o.s, s) }
|
||||
|
||||
func opfirst(a, b string) string { return a }
|
||||
func oplast(a, b string) string { return b }
|
||||
|
||||
func opprefix(a, b string) string {
|
||||
for i := range a {
|
||||
if i >= len(b) || a[i] != b[i] {
|
||||
return a[:i]
|
||||
}
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
func opjoin(sep string) func(a, b string) string {
|
||||
return func(a, b string) string {
|
||||
return a + sep + b // TODO(kr): too slow? maybe strings.Join?
|
||||
}
|
||||
}
|
||||
|
||||
func opsmin(a, b string) string {
|
||||
if strings.Compare(a, b) <= 0 {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func opsmax(a, b string) string {
|
||||
if strings.Compare(a, b) >= 0 {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
type sampler struct {
|
||||
n int
|
||||
s string
|
||||
}
|
||||
|
||||
func sample(s, arg string) agg { return &sampler{1, s} }
|
||||
func (p *sampler) String() string { return p.s }
|
||||
func (p *sampler) merge(s string) {
|
||||
p.n++
|
||||
if rand.Intn(p.n) == 0 {
|
||||
p.s = s
|
||||
}
|
||||
}
|
||||
|
||||
type constant string
|
||||
|
||||
func constf(init, arg string) agg { return constant(arg) }
|
||||
func (c constant) String() string { return string(c) }
|
||||
func (c constant) merge(string) {}
|
||||
Reference in New Issue
Block a user