dep ensure

This commit is contained in:
Mitsuhiro Tanda
2018-07-09 13:11:50 +09:00
parent 664944980a
commit 077cf9a343
41 changed files with 3183 additions and 84 deletions
+2 -2
View File
@@ -91,6 +91,6 @@ func (c *Client) AddDebugHandlers() {
return
}
c.Handlers.Send.PushFrontNamed(request.NamedHandler{Name: "awssdk.client.LogRequest", Fn: logRequest})
c.Handlers.Send.PushBackNamed(request.NamedHandler{Name: "awssdk.client.LogResponse", Fn: logResponse})
c.Handlers.Send.PushFrontNamed(LogHTTPRequestHandler)
c.Handlers.Send.PushBackNamed(LogHTTPResponseHandler)
}
+87 -15
View File
@@ -44,12 +44,22 @@ func (reader *teeReaderCloser) Close() error {
return reader.Source.Close()
}
// LogHTTPRequestHandler is a SDK request handler to log the HTTP request sent
// to a service. Will include the HTTP request body if the LogLevel of the
// request matches LogDebugWithHTTPBody.
var LogHTTPRequestHandler = request.NamedHandler{
Name: "awssdk.client.LogRequest",
Fn: logRequest,
}
func logRequest(r *request.Request) {
logBody := r.Config.LogLevel.Matches(aws.LogDebugWithHTTPBody)
bodySeekable := aws.IsReaderSeekable(r.Body)
dumpedBody, err := httputil.DumpRequestOut(r.HTTPRequest, logBody)
b, err := httputil.DumpRequestOut(r.HTTPRequest, logBody)
if err != nil {
r.Config.Logger.Log(fmt.Sprintf(logReqErrMsg, r.ClientInfo.ServiceName, r.Operation.Name, err))
r.Config.Logger.Log(fmt.Sprintf(logReqErrMsg,
r.ClientInfo.ServiceName, r.Operation.Name, err))
return
}
@@ -63,7 +73,28 @@ func logRequest(r *request.Request) {
r.ResetBody()
}
r.Config.Logger.Log(fmt.Sprintf(logReqMsg, r.ClientInfo.ServiceName, r.Operation.Name, string(dumpedBody)))
r.Config.Logger.Log(fmt.Sprintf(logReqMsg,
r.ClientInfo.ServiceName, r.Operation.Name, string(b)))
}
// LogHTTPRequestHeaderHandler is a SDK request handler to log the HTTP request sent
// to a service. Will only log the HTTP request's headers. The request payload
// will not be read.
var LogHTTPRequestHeaderHandler = request.NamedHandler{
Name: "awssdk.client.LogRequestHeader",
Fn: logRequestHeader,
}
func logRequestHeader(r *request.Request) {
b, err := httputil.DumpRequestOut(r.HTTPRequest, false)
if err != nil {
r.Config.Logger.Log(fmt.Sprintf(logReqErrMsg,
r.ClientInfo.ServiceName, r.Operation.Name, err))
return
}
r.Config.Logger.Log(fmt.Sprintf(logReqMsg,
r.ClientInfo.ServiceName, r.Operation.Name, string(b)))
}
const logRespMsg = `DEBUG: Response %s/%s Details:
@@ -76,27 +107,44 @@ const logRespErrMsg = `DEBUG ERROR: Response %s/%s:
%s
-----------------------------------------------------`
// LogHTTPResponseHandler is a SDK request handler to log the HTTP response
// received from a service. Will include the HTTP response body if the LogLevel
// of the request matches LogDebugWithHTTPBody.
var LogHTTPResponseHandler = request.NamedHandler{
Name: "awssdk.client.LogResponse",
Fn: logResponse,
}
func logResponse(r *request.Request) {
lw := &logWriter{r.Config.Logger, bytes.NewBuffer(nil)}
r.HTTPResponse.Body = &teeReaderCloser{
Reader: io.TeeReader(r.HTTPResponse.Body, lw),
Source: r.HTTPResponse.Body,
logBody := r.Config.LogLevel.Matches(aws.LogDebugWithHTTPBody)
if logBody {
r.HTTPResponse.Body = &teeReaderCloser{
Reader: io.TeeReader(r.HTTPResponse.Body, lw),
Source: r.HTTPResponse.Body,
}
}
handlerFn := func(req *request.Request) {
body, err := httputil.DumpResponse(req.HTTPResponse, false)
b, err := httputil.DumpResponse(req.HTTPResponse, false)
if err != nil {
lw.Logger.Log(fmt.Sprintf(logRespErrMsg, req.ClientInfo.ServiceName, req.Operation.Name, err))
lw.Logger.Log(fmt.Sprintf(logRespErrMsg,
req.ClientInfo.ServiceName, req.Operation.Name, err))
return
}
b, err := ioutil.ReadAll(lw.buf)
if err != nil {
lw.Logger.Log(fmt.Sprintf(logRespErrMsg, req.ClientInfo.ServiceName, req.Operation.Name, err))
return
}
lw.Logger.Log(fmt.Sprintf(logRespMsg, req.ClientInfo.ServiceName, req.Operation.Name, string(body)))
if req.Config.LogLevel.Matches(aws.LogDebugWithHTTPBody) {
lw.Logger.Log(fmt.Sprintf(logRespMsg,
req.ClientInfo.ServiceName, req.Operation.Name, string(b)))
if logBody {
b, err := ioutil.ReadAll(lw.buf)
if err != nil {
lw.Logger.Log(fmt.Sprintf(logRespErrMsg,
req.ClientInfo.ServiceName, req.Operation.Name, err))
return
}
lw.Logger.Log(string(b))
}
}
@@ -110,3 +158,27 @@ func logResponse(r *request.Request) {
Name: handlerName, Fn: handlerFn,
})
}
// LogHTTPResponseHeaderHandler is a SDK request handler to log the HTTP
// response received from a service. Will only log the HTTP response's headers.
// The response payload will not be read.
var LogHTTPResponseHeaderHandler = request.NamedHandler{
Name: "awssdk.client.LogResponseHeader",
Fn: logResponseHeader,
}
func logResponseHeader(r *request.Request) {
if r.Config.Logger == nil {
return
}
b, err := httputil.DumpResponse(r.HTTPResponse, false)
if err != nil {
r.Config.Logger.Log(fmt.Sprintf(logRespErrMsg,
r.ClientInfo.ServiceName, r.Operation.Name, err))
return
}
r.Config.Logger.Log(fmt.Sprintf(logRespMsg,
r.ClientInfo.ServiceName, r.Operation.Name, string(b)))
}
+1
View File
@@ -3,6 +3,7 @@ package metadata
// ClientInfo wraps immutable data from the client.Client structure.
type ClientInfo struct {
ServiceName string
ServiceID string
APIVersion string
Endpoint string
SigningName string
+15 -3
View File
@@ -178,7 +178,8 @@ func (e *Expiry) IsExpired() bool {
type Credentials struct {
creds Value
forceRefresh bool
m sync.Mutex
m sync.RWMutex
provider Provider
}
@@ -201,6 +202,17 @@ func NewCredentials(provider Provider) *Credentials {
// If Credentials.Expire() was called the credentials Value will be force
// expired, and the next call to Get() will cause them to be refreshed.
func (c *Credentials) Get() (Value, error) {
// Check the cached credentials first with just the read lock.
c.m.RLock()
if !c.isExpired() {
creds := c.creds
c.m.RUnlock()
return creds, nil
}
c.m.RUnlock()
// Credentials are expired need to retrieve the credentials taking the full
// lock.
c.m.Lock()
defer c.m.Unlock()
@@ -234,8 +246,8 @@ func (c *Credentials) Expire() {
// If the Credentials were forced to be expired with Expire() this will
// reflect that override.
func (c *Credentials) IsExpired() bool {
c.m.Lock()
defer c.m.Unlock()
c.m.RLock()
defer c.m.RUnlock()
return c.isExpired()
}
+46
View File
@@ -0,0 +1,46 @@
// Package csm provides Client Side Monitoring (CSM) which enables sending metrics
// via UDP connection. Using the Start function will enable the reporting of
// metrics on a given port. If Start is called, with different parameters, again,
// a panic will occur.
//
// Pause can be called to pause any metrics publishing on a given port. Sessions
// that have had their handlers modified via InjectHandlers may still be used.
// However, the handlers will act as a no-op meaning no metrics will be published.
//
// Example:
// r, err := csm.Start("clientID", ":31000")
// if err != nil {
// panic(fmt.Errorf("failed starting CSM: %v", err))
// }
//
// sess, err := session.NewSession(&aws.Config{})
// if err != nil {
// panic(fmt.Errorf("failed loading session: %v", err))
// }
//
// r.InjectHandlers(&sess.Handlers)
//
// client := s3.New(sess)
// resp, err := client.GetObject(&s3.GetObjectInput{
// Bucket: aws.String("bucket"),
// Key: aws.String("key"),
// })
//
// // Will pause monitoring
// r.Pause()
// resp, err = client.GetObject(&s3.GetObjectInput{
// Bucket: aws.String("bucket"),
// Key: aws.String("key"),
// })
//
// // Resume monitoring
// r.Continue()
//
// Start returns a Reporter that is used to enable or disable monitoring. If
// access to the Reporter is required later, calling Get will return the Reporter
// singleton.
//
// Example:
// r := csm.Get()
// r.Continue()
package csm
+67
View File
@@ -0,0 +1,67 @@
package csm
import (
"fmt"
"sync"
)
var (
lock sync.Mutex
)
// Client side metric handler names
const (
APICallMetricHandlerName = "awscsm.SendAPICallMetric"
APICallAttemptMetricHandlerName = "awscsm.SendAPICallAttemptMetric"
)
// Start will start the a long running go routine to capture
// client side metrics. Calling start multiple time will only
// start the metric listener once and will panic if a different
// client ID or port is passed in.
//
// Example:
// r, err := csm.Start("clientID", "127.0.0.1:8094")
// if err != nil {
// panic(fmt.Errorf("expected no error, but received %v", err))
// }
// sess := session.NewSession()
// r.InjectHandlers(sess.Handlers)
//
// svc := s3.New(sess)
// out, err := svc.GetObject(&s3.GetObjectInput{
// Bucket: aws.String("bucket"),
// Key: aws.String("key"),
// })
func Start(clientID string, url string) (*Reporter, error) {
lock.Lock()
defer lock.Unlock()
if sender == nil {
sender = newReporter(clientID, url)
} else {
if sender.clientID != clientID {
panic(fmt.Errorf("inconsistent client IDs. %q was expected, but received %q", sender.clientID, clientID))
}
if sender.url != url {
panic(fmt.Errorf("inconsistent URLs. %q was expected, but received %q", sender.url, url))
}
}
if err := connect(url); err != nil {
sender = nil
return nil, err
}
return sender, nil
}
// Get will return a reporter if one exists, if one does not exist, nil will
// be returned.
func Get() *Reporter {
lock.Lock()
defer lock.Unlock()
return sender
}
+51
View File
@@ -0,0 +1,51 @@
package csm
import (
"strconv"
"time"
)
type metricTime time.Time
func (t metricTime) MarshalJSON() ([]byte, error) {
ns := time.Duration(time.Time(t).UnixNano())
return []byte(strconv.FormatInt(int64(ns/time.Millisecond), 10)), nil
}
type metric struct {
ClientID *string `json:"ClientId,omitempty"`
API *string `json:"Api,omitempty"`
Service *string `json:"Service,omitempty"`
Timestamp *metricTime `json:"Timestamp,omitempty"`
Type *string `json:"Type,omitempty"`
Version *int `json:"Version,omitempty"`
AttemptCount *int `json:"AttemptCount,omitempty"`
Latency *int `json:"Latency,omitempty"`
Fqdn *string `json:"Fqdn,omitempty"`
UserAgent *string `json:"UserAgent,omitempty"`
AttemptLatency *int `json:"AttemptLatency,omitempty"`
SessionToken *string `json:"SessionToken,omitempty"`
Region *string `json:"Region,omitempty"`
AccessKey *string `json:"AccessKey,omitempty"`
HTTPStatusCode *int `json:"HttpStatusCode,omitempty"`
XAmzID2 *string `json:"XAmzId2,omitempty"`
XAmzRequestID *string `json:"XAmznRequestId,omitempty"`
AWSException *string `json:"AwsException,omitempty"`
AWSExceptionMessage *string `json:"AwsExceptionMessage,omitempty"`
SDKException *string `json:"SdkException,omitempty"`
SDKExceptionMessage *string `json:"SdkExceptionMessage,omitempty"`
DestinationIP *string `json:"DestinationIp,omitempty"`
ConnectionReused *int `json:"ConnectionReused,omitempty"`
AcquireConnectionLatency *int `json:"AcquireConnectionLatency,omitempty"`
ConnectLatency *int `json:"ConnectLatency,omitempty"`
RequestLatency *int `json:"RequestLatency,omitempty"`
DNSLatency *int `json:"DnsLatency,omitempty"`
TCPLatency *int `json:"TcpLatency,omitempty"`
SSLLatency *int `json:"SslLatency,omitempty"`
}
+54
View File
@@ -0,0 +1,54 @@
package csm
import (
"sync/atomic"
)
const (
runningEnum = iota
pausedEnum
)
var (
// MetricsChannelSize of metrics to hold in the channel
MetricsChannelSize = 100
)
type metricChan struct {
ch chan metric
paused int64
}
func newMetricChan(size int) metricChan {
return metricChan{
ch: make(chan metric, size),
}
}
func (ch *metricChan) Pause() {
atomic.StoreInt64(&ch.paused, pausedEnum)
}
func (ch *metricChan) Continue() {
atomic.StoreInt64(&ch.paused, runningEnum)
}
func (ch *metricChan) IsPaused() bool {
v := atomic.LoadInt64(&ch.paused)
return v == pausedEnum
}
// Push will push metrics to the metric channel if the channel
// is not paused
func (ch *metricChan) Push(m metric) bool {
if ch.IsPaused() {
return false
}
select {
case ch.ch <- m:
return true
default:
return false
}
}
+230
View File
@@ -0,0 +1,230 @@
package csm
import (
"encoding/json"
"net"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
)
const (
// DefaultPort is used when no port is specified
DefaultPort = "31000"
)
// Reporter will gather metrics of API requests made and
// send those metrics to the CSM endpoint.
type Reporter struct {
clientID string
url string
conn net.Conn
metricsCh metricChan
done chan struct{}
}
var (
sender *Reporter
)
func connect(url string) error {
const network = "udp"
if err := sender.connect(network, url); err != nil {
return err
}
if sender.done == nil {
sender.done = make(chan struct{})
go sender.start()
}
return nil
}
func newReporter(clientID, url string) *Reporter {
return &Reporter{
clientID: clientID,
url: url,
metricsCh: newMetricChan(MetricsChannelSize),
}
}
func (rep *Reporter) sendAPICallAttemptMetric(r *request.Request) {
if rep == nil {
return
}
now := time.Now()
creds, _ := r.Config.Credentials.Get()
m := metric{
ClientID: aws.String(rep.clientID),
API: aws.String(r.Operation.Name),
Service: aws.String(r.ClientInfo.ServiceID),
Timestamp: (*metricTime)(&now),
UserAgent: aws.String(r.HTTPRequest.Header.Get("User-Agent")),
Region: r.Config.Region,
Type: aws.String("ApiCallAttempt"),
Version: aws.Int(1),
XAmzRequestID: aws.String(r.RequestID),
AttemptCount: aws.Int(r.RetryCount + 1),
AttemptLatency: aws.Int(int(now.Sub(r.AttemptTime).Nanoseconds() / int64(time.Millisecond))),
AccessKey: aws.String(creds.AccessKeyID),
}
if r.HTTPResponse != nil {
m.HTTPStatusCode = aws.Int(r.HTTPResponse.StatusCode)
}
if r.Error != nil {
if awserr, ok := r.Error.(awserr.Error); ok {
setError(&m, awserr)
}
}
rep.metricsCh.Push(m)
}
func setError(m *metric, err awserr.Error) {
msg := err.Message()
code := err.Code()
switch code {
case "RequestError",
"SerializationError",
request.CanceledErrorCode:
m.SDKException = &code
m.SDKExceptionMessage = &msg
default:
m.AWSException = &code
m.AWSExceptionMessage = &msg
}
}
func (rep *Reporter) sendAPICallMetric(r *request.Request) {
if rep == nil {
return
}
now := time.Now()
m := metric{
ClientID: aws.String(rep.clientID),
API: aws.String(r.Operation.Name),
Service: aws.String(r.ClientInfo.ServiceID),
Timestamp: (*metricTime)(&now),
Type: aws.String("ApiCall"),
AttemptCount: aws.Int(r.RetryCount + 1),
Latency: aws.Int(int(time.Now().Sub(r.Time) / time.Millisecond)),
XAmzRequestID: aws.String(r.RequestID),
}
// TODO: Probably want to figure something out for logging dropped
// metrics
rep.metricsCh.Push(m)
}
func (rep *Reporter) connect(network, url string) error {
if rep.conn != nil {
rep.conn.Close()
}
conn, err := net.Dial(network, url)
if err != nil {
return awserr.New("UDPError", "Could not connect", err)
}
rep.conn = conn
return nil
}
func (rep *Reporter) close() {
if rep.done != nil {
close(rep.done)
}
rep.metricsCh.Pause()
}
func (rep *Reporter) start() {
defer func() {
rep.metricsCh.Pause()
}()
for {
select {
case <-rep.done:
rep.done = nil
return
case m := <-rep.metricsCh.ch:
// TODO: What to do with this error? Probably should just log
b, err := json.Marshal(m)
if err != nil {
continue
}
rep.conn.Write(b)
}
}
}
// Pause will pause the metric channel preventing any new metrics from
// being added.
func (rep *Reporter) Pause() {
lock.Lock()
defer lock.Unlock()
if rep == nil {
return
}
rep.close()
}
// Continue will reopen the metric channel and allow for monitoring
// to be resumed.
func (rep *Reporter) Continue() {
lock.Lock()
defer lock.Unlock()
if rep == nil {
return
}
if !rep.metricsCh.IsPaused() {
return
}
rep.metricsCh.Continue()
}
// InjectHandlers will will enable client side metrics and inject the proper
// handlers to handle how metrics are sent.
//
// Example:
// // Start must be called in order to inject the correct handlers
// r, err := csm.Start("clientID", "127.0.0.1:8094")
// if err != nil {
// panic(fmt.Errorf("expected no error, but received %v", err))
// }
//
// sess := session.NewSession()
// r.InjectHandlers(&sess.Handlers)
//
// // create a new service client with our client side metric session
// svc := s3.New(sess)
func (rep *Reporter) InjectHandlers(handlers *request.Handlers) {
if rep == nil {
return
}
apiCallHandler := request.NamedHandler{Name: APICallMetricHandlerName, Fn: rep.sendAPICallMetric}
handlers.Complete.PushFrontNamed(apiCallHandler)
apiCallAttemptHandler := request.NamedHandler{Name: APICallAttemptMetricHandlerName, Fn: rep.sendAPICallAttemptMetric}
handlers.AfterRetry.PushFrontNamed(apiCallAttemptHandler)
}
+65 -10
View File
@@ -48,6 +48,7 @@ const (
A4bServiceID = "a4b" // A4b.
AcmServiceID = "acm" // Acm.
AcmPcaServiceID = "acm-pca" // AcmPca.
ApiMediatailorServiceID = "api.mediatailor" // ApiMediatailor.
ApiPricingServiceID = "api.pricing" // ApiPricing.
ApigatewayServiceID = "apigateway" // Apigateway.
ApplicationAutoscalingServiceID = "application-autoscaling" // ApplicationAutoscaling.
@@ -130,6 +131,7 @@ const (
ModelsLexServiceID = "models.lex" // ModelsLex.
MonitoringServiceID = "monitoring" // Monitoring.
MturkRequesterServiceID = "mturk-requester" // MturkRequester.
NeptuneServiceID = "neptune" // Neptune.
OpsworksServiceID = "opsworks" // Opsworks.
OpsworksCmServiceID = "opsworks-cm" // OpsworksCm.
OrganizationsServiceID = "organizations" // Organizations.
@@ -307,6 +309,16 @@ var awsPartition = partition{
"us-west-2": endpoint{},
},
},
"api.mediatailor": service{
Endpoints: endpoints{
"ap-northeast-1": endpoint{},
"ap-southeast-1": endpoint{},
"ap-southeast-2": endpoint{},
"eu-west-1": endpoint{},
"us-east-1": endpoint{},
},
},
"api.pricing": service{
Defaults: endpoint{
CredentialScope: credentialScope{
@@ -434,6 +446,7 @@ var awsPartition = partition{
Endpoints: endpoints{
"ap-northeast-1": endpoint{},
"ap-northeast-2": endpoint{},
"ap-south-1": endpoint{},
"ap-southeast-1": endpoint{},
"ap-southeast-2": endpoint{},
"ca-central-1": endpoint{},
@@ -1046,6 +1059,7 @@ var awsPartition = partition{
"elasticfilesystem": service{
Endpoints: endpoints{
"ap-northeast-2": endpoint{},
"ap-southeast-2": endpoint{},
"eu-central-1": endpoint{},
"eu-west-1": endpoint{},
@@ -1242,11 +1256,13 @@ var awsPartition = partition{
Endpoints: endpoints{
"ap-northeast-1": endpoint{},
"ap-northeast-2": endpoint{},
"ap-south-1": endpoint{},
"ap-southeast-1": endpoint{},
"ap-southeast-2": endpoint{},
"eu-central-1": endpoint{},
"eu-west-1": endpoint{},
"eu-west-2": endpoint{},
"us-east-1": endpoint{},
"us-east-2": endpoint{},
"us-west-2": endpoint{},
@@ -1509,8 +1525,10 @@ var awsPartition = partition{
Endpoints: endpoints{
"ap-northeast-1": endpoint{},
"ap-northeast-2": endpoint{},
"ap-southeast-1": endpoint{},
"ap-southeast-2": endpoint{},
"eu-central-1": endpoint{},
"eu-west-1": endpoint{},
"us-east-1": endpoint{},
"us-west-2": endpoint{},
@@ -1622,6 +1640,35 @@ var awsPartition = partition{
"us-east-1": endpoint{},
},
},
"neptune": service{
Endpoints: endpoints{
"eu-west-1": endpoint{
Hostname: "rds.eu-west-1.amazonaws.com",
CredentialScope: credentialScope{
Region: "eu-west-1",
},
},
"us-east-1": endpoint{
Hostname: "rds.us-east-1.amazonaws.com",
CredentialScope: credentialScope{
Region: "us-east-1",
},
},
"us-east-2": endpoint{
Hostname: "rds.us-east-2.amazonaws.com",
CredentialScope: credentialScope{
Region: "us-east-2",
},
},
"us-west-2": endpoint{
Hostname: "rds.us-west-2.amazonaws.com",
CredentialScope: credentialScope{
Region: "us-west-2",
},
},
},
},
"opsworks": service{
Endpoints: endpoints{
@@ -1805,10 +1852,11 @@ var awsPartition = partition{
"runtime.sagemaker": service{
Endpoints: endpoints{
"eu-west-1": endpoint{},
"us-east-1": endpoint{},
"us-east-2": endpoint{},
"us-west-2": endpoint{},
"ap-northeast-1": endpoint{},
"eu-west-1": endpoint{},
"us-east-1": endpoint{},
"us-east-2": endpoint{},
"us-west-2": endpoint{},
},
},
"s3": service{
@@ -1873,10 +1921,11 @@ var awsPartition = partition{
"sagemaker": service{
Endpoints: endpoints{
"eu-west-1": endpoint{},
"us-east-1": endpoint{},
"us-east-2": endpoint{},
"us-west-2": endpoint{},
"ap-northeast-1": endpoint{},
"eu-west-1": endpoint{},
"us-east-1": endpoint{},
"us-east-2": endpoint{},
"us-west-2": endpoint{},
},
},
"sdb": service{
@@ -2081,6 +2130,10 @@ var awsPartition = partition{
"eu-west-1": endpoint{},
"eu-west-2": endpoint{},
"eu-west-3": endpoint{},
"fips-us-east-1": endpoint{},
"fips-us-east-2": endpoint{},
"fips-us-west-1": endpoint{},
"fips-us-west-2": endpoint{},
"sa-east-1": endpoint{},
"us-east-1": endpoint{
SSLCommonName: "queue.{dnsSuffix}",
@@ -2507,13 +2560,15 @@ var awscnPartition = partition{
"ecr": service{
Endpoints: endpoints{
"cn-north-1": endpoint{},
"cn-north-1": endpoint{},
"cn-northwest-1": endpoint{},
},
},
"ecs": service{
Endpoints: endpoints{
"cn-north-1": endpoint{},
"cn-north-1": endpoint{},
"cn-northwest-1": endpoint{},
},
},
"elasticache": service{
+6
View File
@@ -71,6 +71,12 @@ const (
// LogDebugWithRequestErrors states the SDK should log when service requests fail
// to build, send, validate, or unmarshal.
LogDebugWithRequestErrors
// LogDebugWithEventStreamBody states the SDK should log EventStream
// request and response bodys. This should be used to log the EventStream
// wire unmarshaled message content of requests and responses made while
// using the SDK Will also enable LogDebug.
LogDebugWithEventStreamBody
)
// A Logger is a minimalistic interface for the SDK to log messages to. Should
+18
View File
@@ -14,6 +14,7 @@ type Handlers struct {
Send HandlerList
ValidateResponse HandlerList
Unmarshal HandlerList
UnmarshalStream HandlerList
UnmarshalMeta HandlerList
UnmarshalError HandlerList
Retry HandlerList
@@ -30,6 +31,7 @@ func (h *Handlers) Copy() Handlers {
Send: h.Send.copy(),
ValidateResponse: h.ValidateResponse.copy(),
Unmarshal: h.Unmarshal.copy(),
UnmarshalStream: h.UnmarshalStream.copy(),
UnmarshalError: h.UnmarshalError.copy(),
UnmarshalMeta: h.UnmarshalMeta.copy(),
Retry: h.Retry.copy(),
@@ -45,6 +47,7 @@ func (h *Handlers) Clear() {
h.Send.Clear()
h.Sign.Clear()
h.Unmarshal.Clear()
h.UnmarshalStream.Clear()
h.UnmarshalMeta.Clear()
h.UnmarshalError.Clear()
h.ValidateResponse.Clear()
@@ -172,6 +175,21 @@ func (l *HandlerList) SwapNamed(n NamedHandler) (swapped bool) {
return swapped
}
// Swap will swap out all handlers matching the name passed in. The matched
// handlers will be swapped in. True is returned if the handlers were swapped.
func (l *HandlerList) Swap(name string, replace NamedHandler) bool {
var swapped bool
for i := 0; i < len(l.list); i++ {
if l.list[i].Name == name {
l.list[i] = replace
swapped = true
}
}
return swapped
}
// SetBackNamed will replace the named handler if it exists in the handler list.
// If the handler does not exist the handler will be added to the end of the list.
func (l *HandlerList) SetBackNamed(n NamedHandler) {
+6 -3
View File
@@ -46,6 +46,7 @@ type Request struct {
Handlers Handlers
Retryer
AttemptTime time.Time
Time time.Time
Operation *Operation
HTTPRequest *http.Request
@@ -121,6 +122,7 @@ func New(cfg aws.Config, clientInfo metadata.ClientInfo, handlers Handlers,
Handlers: handlers.Copy(),
Retryer: retryer,
AttemptTime: time.Now(),
Time: time.Now(),
ExpireTime: 0,
Operation: operation,
@@ -368,9 +370,9 @@ func (r *Request) Build() error {
return r.Error
}
// Sign will sign the request returning error if errors are encountered.
// Sign will sign the request, returning error if errors are encountered.
//
// Send will build the request prior to signing. All Sign Handlers will
// Sign will build the request prior to signing. All Sign Handlers will
// be executed in the order they were set.
func (r *Request) Sign() error {
r.Build()
@@ -440,7 +442,7 @@ func (r *Request) GetBody() io.ReadSeeker {
return r.safeBody
}
// Send will send the request returning error if errors are encountered.
// Send will send the request, returning error if errors are encountered.
//
// Send will sign the request prior to sending. All Send Handlers will
// be executed in the order they were set.
@@ -461,6 +463,7 @@ func (r *Request) Send() error {
}()
for {
r.AttemptTime = time.Now()
if aws.BoolValue(r.Retryable) {
if r.Config.LogLevel.Matches(aws.LogDebugWithRequestRetries) {
r.Config.Logger.Log(fmt.Sprintf("DEBUG: Retrying Request %s/%s, attempt %d",
+1 -1
View File
@@ -21,7 +21,7 @@ func (noBody) WriteTo(io.Writer) (int64, error) { return 0, nil }
var NoBody = noBody{}
// ResetBody rewinds the request body back to its starting position, and
// set's the HTTP Request body reference. When the body is read prior
// sets the HTTP Request body reference. When the body is read prior
// to being sent in the HTTP request it will need to be rewound.
//
// ResetBody will automatically be called by the SDK's build handler, but if
+1 -1
View File
@@ -11,7 +11,7 @@ import (
var NoBody = http.NoBody
// ResetBody rewinds the request body back to its starting position, and
// set's the HTTP Request body reference. When the body is read prior
// sets the HTTP Request body reference. When the body is read prior
// to being sent in the HTTP request it will need to be rewound.
//
// ResetBody will automatically be called by the SDK's build handler, but if
+14 -1
View File
@@ -35,8 +35,12 @@ type Pagination struct {
// NewRequest should always be built from the same API operations. It is
// undefined if different API operations are returned on subsequent calls.
NewRequest func() (*Request, error)
// EndPageOnSameToken, when enabled, will allow the paginator to stop on
// token that are the same as its previous tokens.
EndPageOnSameToken bool
started bool
prevTokens []interface{}
nextTokens []interface{}
err error
@@ -49,7 +53,15 @@ type Pagination struct {
//
// Will always return true if Next has not been called yet.
func (p *Pagination) HasNextPage() bool {
return !(p.started && len(p.nextTokens) == 0)
if !p.started {
return true
}
hasNextPage := len(p.nextTokens) != 0
if p.EndPageOnSameToken {
return hasNextPage && !awsutil.DeepEqual(p.nextTokens, p.prevTokens)
}
return hasNextPage
}
// Err returns the error Pagination encountered when retrieving the next page.
@@ -96,6 +108,7 @@ func (p *Pagination) Next() bool {
return false
}
p.prevTokens = p.nextTokens
p.nextTokens = req.nextPageTokens()
p.curPage = req.Data
+20
View File
@@ -96,9 +96,23 @@ type envConfig struct {
//
// AWS_CA_BUNDLE=$HOME/my_custom_ca_bundle
CustomCABundle string
csmEnabled string
CSMEnabled bool
CSMPort string
CSMClientID string
}
var (
csmEnabledEnvKey = []string{
"AWS_CSM_ENABLED",
}
csmPortEnvKey = []string{
"AWS_CSM_PORT",
}
csmClientIDEnvKey = []string{
"AWS_CSM_CLIENT_ID",
}
credAccessEnvKey = []string{
"AWS_ACCESS_KEY_ID",
"AWS_ACCESS_KEY",
@@ -157,6 +171,12 @@ func envConfigLoad(enableSharedConfig bool) envConfig {
setFromEnvVal(&cfg.Creds.SecretAccessKey, credSecretEnvKey)
setFromEnvVal(&cfg.Creds.SessionToken, credSessionEnvKey)
// CSM environment variables
setFromEnvVal(&cfg.csmEnabled, csmEnabledEnvKey)
setFromEnvVal(&cfg.CSMPort, csmPortEnvKey)
setFromEnvVal(&cfg.CSMClientID, csmClientIDEnvKey)
cfg.CSMEnabled = len(cfg.csmEnabled) > 0
// Require logical grouping of credentials
if len(cfg.Creds.AccessKeyID) == 0 || len(cfg.Creds.SecretAccessKey) == 0 {
cfg.Creds = credentials.Value{}
+24 -2
View File
@@ -15,6 +15,7 @@ import (
"github.com/aws/aws-sdk-go/aws/corehandlers"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
"github.com/aws/aws-sdk-go/aws/csm"
"github.com/aws/aws-sdk-go/aws/defaults"
"github.com/aws/aws-sdk-go/aws/endpoints"
"github.com/aws/aws-sdk-go/aws/request"
@@ -81,10 +82,16 @@ func New(cfgs ...*aws.Config) *Session {
r.Error = err
})
}
return s
}
return deprecatedNewSession(cfgs...)
s := deprecatedNewSession(cfgs...)
if envCfg.CSMEnabled {
enableCSM(&s.Handlers, envCfg.CSMClientID, envCfg.CSMPort, s.Config.Logger)
}
return s
}
// NewSession returns a new Session created from SDK defaults, config files,
@@ -300,10 +307,22 @@ func deprecatedNewSession(cfgs ...*aws.Config) *Session {
}
initHandlers(s)
return s
}
func enableCSM(handlers *request.Handlers, clientID string, port string, logger aws.Logger) {
logger.Log("Enabling CSM")
if len(port) == 0 {
port = csm.DefaultPort
}
r, err := csm.Start(clientID, "127.0.0.1:"+port)
if err != nil {
return
}
r.InjectHandlers(handlers)
}
func newSession(opts Options, envCfg envConfig, cfgs ...*aws.Config) (*Session, error) {
cfg := defaults.Config()
handlers := defaults.Handlers()
@@ -343,6 +362,9 @@ func newSession(opts Options, envCfg envConfig, cfgs ...*aws.Config) (*Session,
}
initHandlers(s)
if envCfg.CSMEnabled {
enableCSM(&s.Handlers, envCfg.CSMClientID, envCfg.CSMPort, s.Config.Logger)
}
// Setup HTTP client with custom cert bundle if enabled
if opts.CustomCABundle != nil {
+11 -2
View File
@@ -135,6 +135,7 @@ var requiredSignedHeaders = rules{
"X-Amz-Server-Side-Encryption-Customer-Key-Md5": struct{}{},
"X-Amz-Storage-Class": struct{}{},
"X-Amz-Website-Redirect-Location": struct{}{},
"X-Amz-Content-Sha256": struct{}{},
},
},
patterns{"X-Amz-Meta-"},
@@ -671,8 +672,15 @@ func (ctx *signingCtx) buildSignature() {
func (ctx *signingCtx) buildBodyDigest() error {
hash := ctx.Request.Header.Get("X-Amz-Content-Sha256")
if hash == "" {
if ctx.unsignedPayload || (ctx.isPresign && ctx.ServiceName == "s3") {
includeSHA256Header := ctx.unsignedPayload ||
ctx.ServiceName == "s3" ||
ctx.ServiceName == "glacier"
s3Presign := ctx.isPresign && ctx.ServiceName == "s3"
if ctx.unsignedPayload || s3Presign {
hash = "UNSIGNED-PAYLOAD"
includeSHA256Header = !s3Presign
} else if ctx.Body == nil {
hash = emptyStringSHA256
} else {
@@ -681,7 +689,8 @@ func (ctx *signingCtx) buildBodyDigest() error {
}
hash = hex.EncodeToString(makeSha256Reader(ctx.Body))
}
if ctx.unsignedPayload || ctx.ServiceName == "s3" || ctx.ServiceName == "glacier" {
if includeSHA256Header {
ctx.Request.Header.Set("X-Amz-Content-Sha256", hash)
}
}
+1 -1
View File
@@ -5,4 +5,4 @@ package aws
const SDKName = "aws-sdk-go"
// SDKVersion is the version of this SDK
const SDKVersion = "1.13.49"
const SDKVersion = "1.14.12"
+144
View File
@@ -0,0 +1,144 @@
package eventstream
import (
"bytes"
"encoding/base64"
"encoding/json"
"fmt"
"strconv"
)
type decodedMessage struct {
rawMessage
Headers decodedHeaders `json:"headers"`
}
type jsonMessage struct {
Length json.Number `json:"total_length"`
HeadersLen json.Number `json:"headers_length"`
PreludeCRC json.Number `json:"prelude_crc"`
Headers decodedHeaders `json:"headers"`
Payload []byte `json:"payload"`
CRC json.Number `json:"message_crc"`
}
func (d *decodedMessage) UnmarshalJSON(b []byte) (err error) {
var jsonMsg jsonMessage
if err = json.Unmarshal(b, &jsonMsg); err != nil {
return err
}
d.Length, err = numAsUint32(jsonMsg.Length)
if err != nil {
return err
}
d.HeadersLen, err = numAsUint32(jsonMsg.HeadersLen)
if err != nil {
return err
}
d.PreludeCRC, err = numAsUint32(jsonMsg.PreludeCRC)
if err != nil {
return err
}
d.Headers = jsonMsg.Headers
d.Payload = jsonMsg.Payload
d.CRC, err = numAsUint32(jsonMsg.CRC)
if err != nil {
return err
}
return nil
}
func (d *decodedMessage) MarshalJSON() ([]byte, error) {
jsonMsg := jsonMessage{
Length: json.Number(strconv.Itoa(int(d.Length))),
HeadersLen: json.Number(strconv.Itoa(int(d.HeadersLen))),
PreludeCRC: json.Number(strconv.Itoa(int(d.PreludeCRC))),
Headers: d.Headers,
Payload: d.Payload,
CRC: json.Number(strconv.Itoa(int(d.CRC))),
}
return json.Marshal(jsonMsg)
}
func numAsUint32(n json.Number) (uint32, error) {
v, err := n.Int64()
if err != nil {
return 0, fmt.Errorf("failed to get int64 json number, %v", err)
}
return uint32(v), nil
}
func (d decodedMessage) Message() Message {
return Message{
Headers: Headers(d.Headers),
Payload: d.Payload,
}
}
type decodedHeaders Headers
func (hs *decodedHeaders) UnmarshalJSON(b []byte) error {
var jsonHeaders []struct {
Name string `json:"name"`
Type valueType `json:"type"`
Value interface{} `json:"value"`
}
decoder := json.NewDecoder(bytes.NewReader(b))
decoder.UseNumber()
if err := decoder.Decode(&jsonHeaders); err != nil {
return err
}
var headers Headers
for _, h := range jsonHeaders {
value, err := valueFromType(h.Type, h.Value)
if err != nil {
return err
}
headers.Set(h.Name, value)
}
(*hs) = decodedHeaders(headers)
return nil
}
func valueFromType(typ valueType, val interface{}) (Value, error) {
switch typ {
case trueValueType:
return BoolValue(true), nil
case falseValueType:
return BoolValue(false), nil
case int8ValueType:
v, err := val.(json.Number).Int64()
return Int8Value(int8(v)), err
case int16ValueType:
v, err := val.(json.Number).Int64()
return Int16Value(int16(v)), err
case int32ValueType:
v, err := val.(json.Number).Int64()
return Int32Value(int32(v)), err
case int64ValueType:
v, err := val.(json.Number).Int64()
return Int64Value(v), err
case bytesValueType:
v, err := base64.StdEncoding.DecodeString(val.(string))
return BytesValue(v), err
case stringValueType:
v, err := base64.StdEncoding.DecodeString(val.(string))
return StringValue(string(v)), err
case timestampValueType:
v, err := val.(json.Number).Int64()
return TimestampValue(timeFromEpochMilli(v)), err
case uuidValueType:
v, err := base64.StdEncoding.DecodeString(val.(string))
var tv UUIDValue
copy(tv[:], v)
return tv, err
default:
panic(fmt.Sprintf("unknown type, %s, %T", typ.String(), val))
}
}
+199
View File
@@ -0,0 +1,199 @@
package eventstream
import (
"bytes"
"encoding/binary"
"encoding/hex"
"encoding/json"
"fmt"
"hash"
"hash/crc32"
"io"
"github.com/aws/aws-sdk-go/aws"
)
// Decoder provides decoding of an Event Stream messages.
type Decoder struct {
r io.Reader
logger aws.Logger
}
// NewDecoder initializes and returns a Decoder for decoding event
// stream messages from the reader provided.
func NewDecoder(r io.Reader) *Decoder {
return &Decoder{
r: r,
}
}
// Decode attempts to decode a single message from the event stream reader.
// Will return the event stream message, or error if Decode fails to read
// the message from the stream.
func (d *Decoder) Decode(payloadBuf []byte) (m Message, err error) {
reader := d.r
if d.logger != nil {
debugMsgBuf := bytes.NewBuffer(nil)
reader = io.TeeReader(reader, debugMsgBuf)
defer func() {
logMessageDecode(d.logger, debugMsgBuf, m, err)
}()
}
crc := crc32.New(crc32IEEETable)
hashReader := io.TeeReader(reader, crc)
prelude, err := decodePrelude(hashReader, crc)
if err != nil {
return Message{}, err
}
if prelude.HeadersLen > 0 {
lr := io.LimitReader(hashReader, int64(prelude.HeadersLen))
m.Headers, err = decodeHeaders(lr)
if err != nil {
return Message{}, err
}
}
if payloadLen := prelude.PayloadLen(); payloadLen > 0 {
buf, err := decodePayload(payloadBuf, io.LimitReader(hashReader, int64(payloadLen)))
if err != nil {
return Message{}, err
}
m.Payload = buf
}
msgCRC := crc.Sum32()
if err := validateCRC(reader, msgCRC); err != nil {
return Message{}, err
}
return m, nil
}
// UseLogger specifies the Logger that that the decoder should use to log the
// message decode to.
func (d *Decoder) UseLogger(logger aws.Logger) {
d.logger = logger
}
func logMessageDecode(logger aws.Logger, msgBuf *bytes.Buffer, msg Message, decodeErr error) {
w := bytes.NewBuffer(nil)
defer func() { logger.Log(w.String()) }()
fmt.Fprintf(w, "Raw message:\n%s\n",
hex.Dump(msgBuf.Bytes()))
if decodeErr != nil {
fmt.Fprintf(w, "Decode error: %v\n", decodeErr)
return
}
rawMsg, err := msg.rawMessage()
if err != nil {
fmt.Fprintf(w, "failed to create raw message, %v\n", err)
return
}
decodedMsg := decodedMessage{
rawMessage: rawMsg,
Headers: decodedHeaders(msg.Headers),
}
fmt.Fprintf(w, "Decoded message:\n")
encoder := json.NewEncoder(w)
if err := encoder.Encode(decodedMsg); err != nil {
fmt.Fprintf(w, "failed to generate decoded message, %v\n", err)
}
}
func decodePrelude(r io.Reader, crc hash.Hash32) (messagePrelude, error) {
var p messagePrelude
var err error
p.Length, err = decodeUint32(r)
if err != nil {
return messagePrelude{}, err
}
p.HeadersLen, err = decodeUint32(r)
if err != nil {
return messagePrelude{}, err
}
if err := p.ValidateLens(); err != nil {
return messagePrelude{}, err
}
preludeCRC := crc.Sum32()
if err := validateCRC(r, preludeCRC); err != nil {
return messagePrelude{}, err
}
p.PreludeCRC = preludeCRC
return p, nil
}
func decodePayload(buf []byte, r io.Reader) ([]byte, error) {
w := bytes.NewBuffer(buf[0:0])
_, err := io.Copy(w, r)
return w.Bytes(), err
}
func decodeUint8(r io.Reader) (uint8, error) {
type byteReader interface {
ReadByte() (byte, error)
}
if br, ok := r.(byteReader); ok {
v, err := br.ReadByte()
return uint8(v), err
}
var b [1]byte
_, err := io.ReadFull(r, b[:])
return uint8(b[0]), err
}
func decodeUint16(r io.Reader) (uint16, error) {
var b [2]byte
bs := b[:]
_, err := io.ReadFull(r, bs)
if err != nil {
return 0, err
}
return binary.BigEndian.Uint16(bs), nil
}
func decodeUint32(r io.Reader) (uint32, error) {
var b [4]byte
bs := b[:]
_, err := io.ReadFull(r, bs)
if err != nil {
return 0, err
}
return binary.BigEndian.Uint32(bs), nil
}
func decodeUint64(r io.Reader) (uint64, error) {
var b [8]byte
bs := b[:]
_, err := io.ReadFull(r, bs)
if err != nil {
return 0, err
}
return binary.BigEndian.Uint64(bs), nil
}
func validateCRC(r io.Reader, expect uint32) error {
msgCRC, err := decodeUint32(r)
if err != nil {
return err
}
if msgCRC != expect {
return ChecksumError{}
}
return nil
}
+114
View File
@@ -0,0 +1,114 @@
package eventstream
import (
"bytes"
"encoding/binary"
"hash"
"hash/crc32"
"io"
)
// Encoder provides EventStream message encoding.
type Encoder struct {
w io.Writer
headersBuf *bytes.Buffer
}
// NewEncoder initializes and returns an Encoder to encode Event Stream
// messages to an io.Writer.
func NewEncoder(w io.Writer) *Encoder {
return &Encoder{
w: w,
headersBuf: bytes.NewBuffer(nil),
}
}
// Encode encodes a single EventStream message to the io.Writer the Encoder
// was created with. An error is returned if writing the message fails.
func (e *Encoder) Encode(msg Message) error {
e.headersBuf.Reset()
err := encodeHeaders(e.headersBuf, msg.Headers)
if err != nil {
return err
}
crc := crc32.New(crc32IEEETable)
hashWriter := io.MultiWriter(e.w, crc)
headersLen := uint32(e.headersBuf.Len())
payloadLen := uint32(len(msg.Payload))
if err := encodePrelude(hashWriter, crc, headersLen, payloadLen); err != nil {
return err
}
if headersLen > 0 {
if _, err := io.Copy(hashWriter, e.headersBuf); err != nil {
return err
}
}
if payloadLen > 0 {
if _, err := hashWriter.Write(msg.Payload); err != nil {
return err
}
}
msgCRC := crc.Sum32()
return binary.Write(e.w, binary.BigEndian, msgCRC)
}
func encodePrelude(w io.Writer, crc hash.Hash32, headersLen, payloadLen uint32) error {
p := messagePrelude{
Length: minMsgLen + headersLen + payloadLen,
HeadersLen: headersLen,
}
if err := p.ValidateLens(); err != nil {
return err
}
err := binaryWriteFields(w, binary.BigEndian,
p.Length,
p.HeadersLen,
)
if err != nil {
return err
}
p.PreludeCRC = crc.Sum32()
err = binary.Write(w, binary.BigEndian, p.PreludeCRC)
if err != nil {
return err
}
return nil
}
func encodeHeaders(w io.Writer, headers Headers) error {
for _, h := range headers {
hn := headerName{
Len: uint8(len(h.Name)),
}
copy(hn.Name[:hn.Len], h.Name)
if err := hn.encode(w); err != nil {
return err
}
if err := h.Value.encode(w); err != nil {
return err
}
}
return nil
}
func binaryWriteFields(w io.Writer, order binary.ByteOrder, vs ...interface{}) error {
for _, v := range vs {
if err := binary.Write(w, order, v); err != nil {
return err
}
}
return nil
}
+23
View File
@@ -0,0 +1,23 @@
package eventstream
import "fmt"
// LengthError provides the error for items being larger than a maximum length.
type LengthError struct {
Part string
Want int
Have int
Value interface{}
}
func (e LengthError) Error() string {
return fmt.Sprintf("%s length invalid, %d/%d, %v",
e.Part, e.Want, e.Have, e.Value)
}
// ChecksumError provides the error for message checksum invalidation errors.
type ChecksumError struct{}
func (e ChecksumError) Error() string {
return "message checksum mismatch"
}
@@ -0,0 +1,160 @@
package eventstreamapi
import (
"fmt"
"io"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/private/protocol"
"github.com/aws/aws-sdk-go/private/protocol/eventstream"
)
// Unmarshaler provides the interface for unmarshaling a EventStream
// message into a SDK type.
type Unmarshaler interface {
UnmarshalEvent(protocol.PayloadUnmarshaler, eventstream.Message) error
}
// EventStream headers with specific meaning to async API functionality.
const (
MessageTypeHeader = `:message-type` // Identifies type of message.
EventMessageType = `event`
ErrorMessageType = `error`
ExceptionMessageType = `exception`
// Message Events
EventTypeHeader = `:event-type` // Identifies message event type e.g. "Stats".
// Message Error
ErrorCodeHeader = `:error-code`
ErrorMessageHeader = `:error-message`
// Message Exception
ExceptionTypeHeader = `:exception-type`
)
// EventReader provides reading from the EventStream of an reader.
type EventReader struct {
reader io.ReadCloser
decoder *eventstream.Decoder
unmarshalerForEventType func(string) (Unmarshaler, error)
payloadUnmarshaler protocol.PayloadUnmarshaler
payloadBuf []byte
}
// NewEventReader returns a EventReader built from the reader and unmarshaler
// provided. Use ReadStream method to start reading from the EventStream.
func NewEventReader(
reader io.ReadCloser,
payloadUnmarshaler protocol.PayloadUnmarshaler,
unmarshalerForEventType func(string) (Unmarshaler, error),
) *EventReader {
return &EventReader{
reader: reader,
decoder: eventstream.NewDecoder(reader),
payloadUnmarshaler: payloadUnmarshaler,
unmarshalerForEventType: unmarshalerForEventType,
payloadBuf: make([]byte, 10*1024),
}
}
// UseLogger instructs the EventReader to use the logger and log level
// specified.
func (r *EventReader) UseLogger(logger aws.Logger, logLevel aws.LogLevelType) {
if logger != nil && logLevel.Matches(aws.LogDebugWithEventStreamBody) {
r.decoder.UseLogger(logger)
}
}
// ReadEvent attempts to read a message from the EventStream and return the
// unmarshaled event value that the message is for.
//
// For EventStream API errors check if the returned error satisfies the
// awserr.Error interface to get the error's Code and Message components.
//
// EventUnmarshalers called with EventStream messages must take copies of the
// message's Payload. The payload will is reused between events read.
func (r *EventReader) ReadEvent() (event interface{}, err error) {
msg, err := r.decoder.Decode(r.payloadBuf)
if err != nil {
return nil, err
}
defer func() {
// Reclaim payload buffer for next message read.
r.payloadBuf = msg.Payload[0:0]
}()
typ, err := GetHeaderString(msg, MessageTypeHeader)
if err != nil {
return nil, err
}
switch typ {
case EventMessageType:
return r.unmarshalEventMessage(msg)
case ErrorMessageType:
return nil, r.unmarshalErrorMessage(msg)
default:
return nil, fmt.Errorf("unknown eventstream message type, %v", typ)
}
}
func (r *EventReader) unmarshalEventMessage(
msg eventstream.Message,
) (event interface{}, err error) {
eventType, err := GetHeaderString(msg, EventTypeHeader)
if err != nil {
return nil, err
}
ev, err := r.unmarshalerForEventType(eventType)
if err != nil {
return nil, err
}
err = ev.UnmarshalEvent(r.payloadUnmarshaler, msg)
if err != nil {
return nil, err
}
return ev, nil
}
func (r *EventReader) unmarshalErrorMessage(msg eventstream.Message) (err error) {
var msgErr messageError
msgErr.code, err = GetHeaderString(msg, ErrorCodeHeader)
if err != nil {
return err
}
msgErr.msg, err = GetHeaderString(msg, ErrorMessageHeader)
if err != nil {
return err
}
return msgErr
}
// Close closes the EventReader's EventStream reader.
func (r *EventReader) Close() error {
return r.reader.Close()
}
// GetHeaderString returns the value of the header as a string. If the header
// is not set or the value is not a string an error will be returned.
func GetHeaderString(msg eventstream.Message, headerName string) (string, error) {
headerVal := msg.Headers.Get(headerName)
if headerVal == nil {
return "", fmt.Errorf("error header %s not present", headerName)
}
v, ok := headerVal.Get().(string)
if !ok {
return "", fmt.Errorf("error header value is not a string, %T", headerVal)
}
return v, nil
}
@@ -0,0 +1,24 @@
package eventstreamapi
import "fmt"
type messageError struct {
code string
msg string
}
func (e messageError) Code() string {
return e.code
}
func (e messageError) Message() string {
return e.msg
}
func (e messageError) Error() string {
return fmt.Sprintf("%s: %s", e.code, e.msg)
}
func (e messageError) OrigErr() error {
return nil
}
+166
View File
@@ -0,0 +1,166 @@
package eventstream
import (
"encoding/binary"
"fmt"
"io"
)
// Headers are a collection of EventStream header values.
type Headers []Header
// Header is a single EventStream Key Value header pair.
type Header struct {
Name string
Value Value
}
// Set associates the name with a value. If the header name already exists in
// the Headers the value will be replaced with the new one.
func (hs *Headers) Set(name string, value Value) {
var i int
for ; i < len(*hs); i++ {
if (*hs)[i].Name == name {
(*hs)[i].Value = value
return
}
}
*hs = append(*hs, Header{
Name: name, Value: value,
})
}
// Get returns the Value associated with the header. Nil is returned if the
// value does not exist.
func (hs Headers) Get(name string) Value {
for i := 0; i < len(hs); i++ {
if h := hs[i]; h.Name == name {
return h.Value
}
}
return nil
}
// Del deletes the value in the Headers if it exists.
func (hs *Headers) Del(name string) {
for i := 0; i < len(*hs); i++ {
if (*hs)[i].Name == name {
copy((*hs)[i:], (*hs)[i+1:])
(*hs) = (*hs)[:len(*hs)-1]
}
}
}
func decodeHeaders(r io.Reader) (Headers, error) {
hs := Headers{}
for {
name, err := decodeHeaderName(r)
if err != nil {
if err == io.EOF {
// EOF while getting header name means no more headers
break
}
return nil, err
}
value, err := decodeHeaderValue(r)
if err != nil {
return nil, err
}
hs.Set(name, value)
}
return hs, nil
}
func decodeHeaderName(r io.Reader) (string, error) {
var n headerName
var err error
n.Len, err = decodeUint8(r)
if err != nil {
return "", err
}
name := n.Name[:n.Len]
if _, err := io.ReadFull(r, name); err != nil {
return "", err
}
return string(name), nil
}
func decodeHeaderValue(r io.Reader) (Value, error) {
var raw rawValue
typ, err := decodeUint8(r)
if err != nil {
return nil, err
}
raw.Type = valueType(typ)
var v Value
switch raw.Type {
case trueValueType:
v = BoolValue(true)
case falseValueType:
v = BoolValue(false)
case int8ValueType:
var tv Int8Value
err = tv.decode(r)
v = tv
case int16ValueType:
var tv Int16Value
err = tv.decode(r)
v = tv
case int32ValueType:
var tv Int32Value
err = tv.decode(r)
v = tv
case int64ValueType:
var tv Int64Value
err = tv.decode(r)
v = tv
case bytesValueType:
var tv BytesValue
err = tv.decode(r)
v = tv
case stringValueType:
var tv StringValue
err = tv.decode(r)
v = tv
case timestampValueType:
var tv TimestampValue
err = tv.decode(r)
v = tv
case uuidValueType:
var tv UUIDValue
err = tv.decode(r)
v = tv
default:
panic(fmt.Sprintf("unknown value type %d", raw.Type))
}
// Error could be EOF, let caller deal with it
return v, err
}
const maxHeaderNameLen = 255
type headerName struct {
Len uint8
Name [maxHeaderNameLen]byte
}
func (v headerName) encode(w io.Writer) error {
if err := binary.Write(w, binary.BigEndian, v.Len); err != nil {
return err
}
_, err := w.Write(v.Name[:v.Len])
return err
}
@@ -0,0 +1,501 @@
package eventstream
import (
"encoding/base64"
"encoding/binary"
"fmt"
"io"
"strconv"
"time"
)
const maxHeaderValueLen = 1<<15 - 1 // 2^15-1 or 32KB - 1
// valueType is the EventStream header value type.
type valueType uint8
// Header value types
const (
trueValueType valueType = iota
falseValueType
int8ValueType // Byte
int16ValueType // Short
int32ValueType // Integer
int64ValueType // Long
bytesValueType
stringValueType
timestampValueType
uuidValueType
)
func (t valueType) String() string {
switch t {
case trueValueType:
return "bool"
case falseValueType:
return "bool"
case int8ValueType:
return "int8"
case int16ValueType:
return "int16"
case int32ValueType:
return "int32"
case int64ValueType:
return "int64"
case bytesValueType:
return "byte_array"
case stringValueType:
return "string"
case timestampValueType:
return "timestamp"
case uuidValueType:
return "uuid"
default:
return fmt.Sprintf("unknown value type %d", uint8(t))
}
}
type rawValue struct {
Type valueType
Len uint16 // Only set for variable length slices
Value []byte // byte representation of value, BigEndian encoding.
}
func (r rawValue) encodeScalar(w io.Writer, v interface{}) error {
return binaryWriteFields(w, binary.BigEndian,
r.Type,
v,
)
}
func (r rawValue) encodeFixedSlice(w io.Writer, v []byte) error {
binary.Write(w, binary.BigEndian, r.Type)
_, err := w.Write(v)
return err
}
func (r rawValue) encodeBytes(w io.Writer, v []byte) error {
if len(v) > maxHeaderValueLen {
return LengthError{
Part: "header value",
Want: maxHeaderValueLen, Have: len(v),
Value: v,
}
}
r.Len = uint16(len(v))
err := binaryWriteFields(w, binary.BigEndian,
r.Type,
r.Len,
)
if err != nil {
return err
}
_, err = w.Write(v)
return err
}
func (r rawValue) encodeString(w io.Writer, v string) error {
if len(v) > maxHeaderValueLen {
return LengthError{
Part: "header value",
Want: maxHeaderValueLen, Have: len(v),
Value: v,
}
}
r.Len = uint16(len(v))
type stringWriter interface {
WriteString(string) (int, error)
}
err := binaryWriteFields(w, binary.BigEndian,
r.Type,
r.Len,
)
if err != nil {
return err
}
if sw, ok := w.(stringWriter); ok {
_, err = sw.WriteString(v)
} else {
_, err = w.Write([]byte(v))
}
return err
}
func decodeFixedBytesValue(r io.Reader, buf []byte) error {
_, err := io.ReadFull(r, buf)
return err
}
func decodeBytesValue(r io.Reader) ([]byte, error) {
var raw rawValue
var err error
raw.Len, err = decodeUint16(r)
if err != nil {
return nil, err
}
buf := make([]byte, raw.Len)
_, err = io.ReadFull(r, buf)
if err != nil {
return nil, err
}
return buf, nil
}
func decodeStringValue(r io.Reader) (string, error) {
v, err := decodeBytesValue(r)
return string(v), err
}
// Value represents the abstract header value.
type Value interface {
Get() interface{}
String() string
valueType() valueType
encode(io.Writer) error
}
// An BoolValue provides eventstream encoding, and representation
// of a Go bool value.
type BoolValue bool
// Get returns the underlying type
func (v BoolValue) Get() interface{} {
return bool(v)
}
// valueType returns the EventStream header value type value.
func (v BoolValue) valueType() valueType {
if v {
return trueValueType
}
return falseValueType
}
func (v BoolValue) String() string {
return strconv.FormatBool(bool(v))
}
// encode encodes the BoolValue into an eventstream binary value
// representation.
func (v BoolValue) encode(w io.Writer) error {
return binary.Write(w, binary.BigEndian, v.valueType())
}
// An Int8Value provides eventstream encoding, and representation of a Go
// int8 value.
type Int8Value int8
// Get returns the underlying value.
func (v Int8Value) Get() interface{} {
return int8(v)
}
// valueType returns the EventStream header value type value.
func (Int8Value) valueType() valueType {
return int8ValueType
}
func (v Int8Value) String() string {
return fmt.Sprintf("0x%02x", int8(v))
}
// encode encodes the Int8Value into an eventstream binary value
// representation.
func (v Int8Value) encode(w io.Writer) error {
raw := rawValue{
Type: v.valueType(),
}
return raw.encodeScalar(w, v)
}
func (v *Int8Value) decode(r io.Reader) error {
n, err := decodeUint8(r)
if err != nil {
return err
}
*v = Int8Value(n)
return nil
}
// An Int16Value provides eventstream encoding, and representation of a Go
// int16 value.
type Int16Value int16
// Get returns the underlying value.
func (v Int16Value) Get() interface{} {
return int16(v)
}
// valueType returns the EventStream header value type value.
func (Int16Value) valueType() valueType {
return int16ValueType
}
func (v Int16Value) String() string {
return fmt.Sprintf("0x%04x", int16(v))
}
// encode encodes the Int16Value into an eventstream binary value
// representation.
func (v Int16Value) encode(w io.Writer) error {
raw := rawValue{
Type: v.valueType(),
}
return raw.encodeScalar(w, v)
}
func (v *Int16Value) decode(r io.Reader) error {
n, err := decodeUint16(r)
if err != nil {
return err
}
*v = Int16Value(n)
return nil
}
// An Int32Value provides eventstream encoding, and representation of a Go
// int32 value.
type Int32Value int32
// Get returns the underlying value.
func (v Int32Value) Get() interface{} {
return int32(v)
}
// valueType returns the EventStream header value type value.
func (Int32Value) valueType() valueType {
return int32ValueType
}
func (v Int32Value) String() string {
return fmt.Sprintf("0x%08x", int32(v))
}
// encode encodes the Int32Value into an eventstream binary value
// representation.
func (v Int32Value) encode(w io.Writer) error {
raw := rawValue{
Type: v.valueType(),
}
return raw.encodeScalar(w, v)
}
func (v *Int32Value) decode(r io.Reader) error {
n, err := decodeUint32(r)
if err != nil {
return err
}
*v = Int32Value(n)
return nil
}
// An Int64Value provides eventstream encoding, and representation of a Go
// int64 value.
type Int64Value int64
// Get returns the underlying value.
func (v Int64Value) Get() interface{} {
return int64(v)
}
// valueType returns the EventStream header value type value.
func (Int64Value) valueType() valueType {
return int64ValueType
}
func (v Int64Value) String() string {
return fmt.Sprintf("0x%016x", int64(v))
}
// encode encodes the Int64Value into an eventstream binary value
// representation.
func (v Int64Value) encode(w io.Writer) error {
raw := rawValue{
Type: v.valueType(),
}
return raw.encodeScalar(w, v)
}
func (v *Int64Value) decode(r io.Reader) error {
n, err := decodeUint64(r)
if err != nil {
return err
}
*v = Int64Value(n)
return nil
}
// An BytesValue provides eventstream encoding, and representation of a Go
// byte slice.
type BytesValue []byte
// Get returns the underlying value.
func (v BytesValue) Get() interface{} {
return []byte(v)
}
// valueType returns the EventStream header value type value.
func (BytesValue) valueType() valueType {
return bytesValueType
}
func (v BytesValue) String() string {
return base64.StdEncoding.EncodeToString([]byte(v))
}
// encode encodes the BytesValue into an eventstream binary value
// representation.
func (v BytesValue) encode(w io.Writer) error {
raw := rawValue{
Type: v.valueType(),
}
return raw.encodeBytes(w, []byte(v))
}
func (v *BytesValue) decode(r io.Reader) error {
buf, err := decodeBytesValue(r)
if err != nil {
return err
}
*v = BytesValue(buf)
return nil
}
// An StringValue provides eventstream encoding, and representation of a Go
// string.
type StringValue string
// Get returns the underlying value.
func (v StringValue) Get() interface{} {
return string(v)
}
// valueType returns the EventStream header value type value.
func (StringValue) valueType() valueType {
return stringValueType
}
func (v StringValue) String() string {
return string(v)
}
// encode encodes the StringValue into an eventstream binary value
// representation.
func (v StringValue) encode(w io.Writer) error {
raw := rawValue{
Type: v.valueType(),
}
return raw.encodeString(w, string(v))
}
func (v *StringValue) decode(r io.Reader) error {
s, err := decodeStringValue(r)
if err != nil {
return err
}
*v = StringValue(s)
return nil
}
// An TimestampValue provides eventstream encoding, and representation of a Go
// timestamp.
type TimestampValue time.Time
// Get returns the underlying value.
func (v TimestampValue) Get() interface{} {
return time.Time(v)
}
// valueType returns the EventStream header value type value.
func (TimestampValue) valueType() valueType {
return timestampValueType
}
func (v TimestampValue) epochMilli() int64 {
nano := time.Time(v).UnixNano()
msec := nano / int64(time.Millisecond)
return msec
}
func (v TimestampValue) String() string {
msec := v.epochMilli()
return strconv.FormatInt(msec, 10)
}
// encode encodes the TimestampValue into an eventstream binary value
// representation.
func (v TimestampValue) encode(w io.Writer) error {
raw := rawValue{
Type: v.valueType(),
}
msec := v.epochMilli()
return raw.encodeScalar(w, msec)
}
func (v *TimestampValue) decode(r io.Reader) error {
n, err := decodeUint64(r)
if err != nil {
return err
}
*v = TimestampValue(timeFromEpochMilli(int64(n)))
return nil
}
func timeFromEpochMilli(t int64) time.Time {
secs := t / 1e3
msec := t % 1e3
return time.Unix(secs, msec*int64(time.Millisecond))
}
// An UUIDValue provides eventstream encoding, and representation of a UUID
// value.
type UUIDValue [16]byte
// Get returns the underlying value.
func (v UUIDValue) Get() interface{} {
return v[:]
}
// valueType returns the EventStream header value type value.
func (UUIDValue) valueType() valueType {
return uuidValueType
}
func (v UUIDValue) String() string {
return fmt.Sprintf(`%X-%X-%X-%X-%X`, v[0:4], v[4:6], v[6:8], v[8:10], v[10:])
}
// encode encodes the UUIDValue into an eventstream binary value
// representation.
func (v UUIDValue) encode(w io.Writer) error {
raw := rawValue{
Type: v.valueType(),
}
return raw.encodeFixedSlice(w, v[:])
}
func (v *UUIDValue) decode(r io.Reader) error {
tv := (*v)[:]
return decodeFixedBytesValue(r, tv)
}
@@ -0,0 +1,103 @@
package eventstream
import (
"bytes"
"encoding/binary"
"hash/crc32"
)
const preludeLen = 8
const preludeCRCLen = 4
const msgCRCLen = 4
const minMsgLen = preludeLen + preludeCRCLen + msgCRCLen
const maxPayloadLen = 1024 * 1024 * 16 // 16MB
const maxHeadersLen = 1024 * 128 // 128KB
const maxMsgLen = minMsgLen + maxHeadersLen + maxPayloadLen
var crc32IEEETable = crc32.MakeTable(crc32.IEEE)
// A Message provides the eventstream message representation.
type Message struct {
Headers Headers
Payload []byte
}
func (m *Message) rawMessage() (rawMessage, error) {
var raw rawMessage
if len(m.Headers) > 0 {
var headers bytes.Buffer
if err := encodeHeaders(&headers, m.Headers); err != nil {
return rawMessage{}, err
}
raw.Headers = headers.Bytes()
raw.HeadersLen = uint32(len(raw.Headers))
}
raw.Length = raw.HeadersLen + uint32(len(m.Payload)) + minMsgLen
hash := crc32.New(crc32IEEETable)
binaryWriteFields(hash, binary.BigEndian, raw.Length, raw.HeadersLen)
raw.PreludeCRC = hash.Sum32()
binaryWriteFields(hash, binary.BigEndian, raw.PreludeCRC)
if raw.HeadersLen > 0 {
hash.Write(raw.Headers)
}
// Read payload bytes and update hash for it as well.
if len(m.Payload) > 0 {
raw.Payload = m.Payload
hash.Write(raw.Payload)
}
raw.CRC = hash.Sum32()
return raw, nil
}
type messagePrelude struct {
Length uint32
HeadersLen uint32
PreludeCRC uint32
}
func (p messagePrelude) PayloadLen() uint32 {
return p.Length - p.HeadersLen - minMsgLen
}
func (p messagePrelude) ValidateLens() error {
if p.Length == 0 || p.Length > maxMsgLen {
return LengthError{
Part: "message prelude",
Want: maxMsgLen,
Have: int(p.Length),
}
}
if p.HeadersLen > maxHeadersLen {
return LengthError{
Part: "message headers",
Want: maxHeadersLen,
Have: int(p.HeadersLen),
}
}
if payloadLen := p.PayloadLen(); payloadLen > maxPayloadLen {
return LengthError{
Part: "message payload",
Want: maxPayloadLen,
Have: int(payloadLen),
}
}
return nil
}
type rawMessage struct {
messagePrelude
Headers []byte
Payload []byte
CRC uint32
}
+81
View File
@@ -0,0 +1,81 @@
package protocol
import (
"io"
"io/ioutil"
"net/http"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/client/metadata"
"github.com/aws/aws-sdk-go/aws/request"
)
// PayloadUnmarshaler provides the interface for unmarshaling a payload's
// reader into a SDK shape.
type PayloadUnmarshaler interface {
UnmarshalPayload(io.Reader, interface{}) error
}
// HandlerPayloadUnmarshal implements the PayloadUnmarshaler from a
// HandlerList. This provides the support for unmarshaling a payload reader to
// a shape without needing a SDK request first.
type HandlerPayloadUnmarshal struct {
Unmarshalers request.HandlerList
}
// UnmarshalPayload unmarshals the io.Reader payload into the SDK shape using
// the Unmarshalers HandlerList provided. Returns an error if unable
// unmarshaling fails.
func (h HandlerPayloadUnmarshal) UnmarshalPayload(r io.Reader, v interface{}) error {
req := &request.Request{
HTTPRequest: &http.Request{},
HTTPResponse: &http.Response{
StatusCode: 200,
Header: http.Header{},
Body: ioutil.NopCloser(r),
},
Data: v,
}
h.Unmarshalers.Run(req)
return req.Error
}
// PayloadMarshaler provides the interface for marshaling a SDK shape into and
// io.Writer.
type PayloadMarshaler interface {
MarshalPayload(io.Writer, interface{}) error
}
// HandlerPayloadMarshal implements the PayloadMarshaler from a HandlerList.
// This provides support for marshaling a SDK shape into an io.Writer without
// needing a SDK request first.
type HandlerPayloadMarshal struct {
Marshalers request.HandlerList
}
// MarshalPayload marshals the SDK shape into the io.Writer using the
// Marshalers HandlerList provided. Returns an error if unable if marshal
// fails.
func (h HandlerPayloadMarshal) MarshalPayload(w io.Writer, v interface{}) error {
req := request.New(
aws.Config{},
metadata.ClientInfo{},
request.Handlers{},
nil,
&request.Operation{HTTPMethod: "GET"},
v,
nil,
)
h.Marshalers.Run(req)
if req.Error != nil {
return req.Error
}
io.Copy(w, req.GetBody())
return nil
}
+5 -3
View File
@@ -20,8 +20,10 @@ import (
"github.com/aws/aws-sdk-go/private/protocol"
)
// RFC822 returns an RFC822 formatted timestamp for AWS protocols
const RFC822 = "Mon, 2 Jan 2006 15:04:05 GMT"
// RFC1123GMT is a RFC1123 (RFC822) formated timestame. This format is not
// using the standard library's time.RFC1123 due to the desire to always use
// GMT as the timezone.
const RFC1123GMT = "Mon, 2 Jan 2006 15:04:05 GMT"
// Whether the byte value can be sent without escaping in AWS URLs
var noEscape [256]bool
@@ -270,7 +272,7 @@ func convertType(v reflect.Value, tag reflect.StructTag) (str string, err error)
case float64:
str = strconv.FormatFloat(value, 'f', -1, 64)
case time.Time:
str = value.UTC().Format(RFC822)
str = value.UTC().Format(RFC1123GMT)
case aws.JSONValue:
if len(value) == 0 {
return "", errValueNotSet
+1 -1
View File
@@ -198,7 +198,7 @@ func unmarshalHeader(v reflect.Value, header string, tag reflect.StructTag) erro
}
v.Set(reflect.ValueOf(&f))
case *time.Time:
t, err := time.Parse(RFC822, header)
t, err := time.Parse(time.RFC1123, header)
if err != nil {
return err
}
+4 -2
View File
@@ -29,8 +29,9 @@ var initRequest func(*request.Request)
// Service information constants
const (
ServiceName = "monitoring" // Service endpoint prefix API calls made to.
EndpointsID = ServiceName // Service ID for Regions and Endpoints metadata.
ServiceName = "monitoring" // Name of service.
EndpointsID = ServiceName // ID to lookup a service endpoint with.
ServiceID = "CloudWatch" // ServiceID is a unique identifer of a specific service.
)
// New creates a new instance of the CloudWatch client with a session.
@@ -55,6 +56,7 @@ func newClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegio
cfg,
metadata.ClientInfo{
ServiceName: ServiceName,
ServiceID: ServiceID,
SigningName: signingName,
SigningRegion: signingRegion,
Endpoint: endpoint,
+66 -28
View File
@@ -2268,11 +2268,7 @@ func (c *EC2) CancelSpotInstanceRequestsRequest(input *CancelSpotInstanceRequest
// CancelSpotInstanceRequests API operation for Amazon Elastic Compute Cloud.
//
// Cancels one or more Spot Instance requests. Spot Instances are instances
// that Amazon EC2 starts on your behalf when the maximum price that you specify
// exceeds the current Spot price. For more information, see Spot Instance Requests
// (http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-requests.html) in
// the Amazon EC2 User Guide for Linux Instances.
// Cancels one or more Spot Instance requests.
//
// Canceling a Spot Instance request does not terminate running Spot Instances
// associated with the request.
@@ -4179,8 +4175,8 @@ func (c *EC2) CreateNetworkInterfacePermissionRequest(input *CreateNetworkInterf
// CreateNetworkInterfacePermission API operation for Amazon Elastic Compute Cloud.
//
// Grants an AWS authorized partner account permission to attach the specified
// network interface to an instance in their account.
// Grants an AWS-authorized account permission to attach the specified network
// interface to an instance in their account.
//
// You can grant permission to a single AWS account only, and only one account
// at a time.
@@ -13675,11 +13671,7 @@ func (c *EC2) DescribeSpotInstanceRequestsRequest(input *DescribeSpotInstanceReq
// DescribeSpotInstanceRequests API operation for Amazon Elastic Compute Cloud.
//
// Describes the Spot Instance requests that belong to your account. Spot Instances
// are instances that Amazon EC2 launches when the Spot price that you specify
// exceeds the current Spot price. For more information, see Spot Instance Requests
// (http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-requests.html) in
// the Amazon EC2 User Guide for Linux Instances.
// Describes the specified Spot Instance requests.
//
// You can use DescribeSpotInstanceRequests to find a running Spot Instance
// by examining the response. If the status of the Spot Instance is fulfilled,
@@ -21367,9 +21359,9 @@ func (c *EC2) RequestSpotInstancesRequest(input *RequestSpotInstancesInput) (req
// RequestSpotInstances API operation for Amazon Elastic Compute Cloud.
//
// Creates a Spot Instance request. Spot Instances are instances that Amazon
// EC2 launches when the maximum price that you specify exceeds the current
// Spot price. For more information, see Spot Instance Requests (http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-requests.html)
// Creates a Spot Instance request.
//
// For more information, see Spot Instance Requests (http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-requests.html)
// in the Amazon EC2 User Guide for Linux Instances.
//
// Returns awserr.Error for service API and SDK errors. Use runtime type assertions
@@ -37615,7 +37607,7 @@ type DescribeInstancesInput struct {
// The maximum number of results to return in a single call. To retrieve the
// remaining results, make another call with the returned NextToken value. This
// value can be between 5 and 1000. You cannot specify this parameter and the
// instance IDs parameter or tag filters in the same call.
// instance IDs parameter in the same call.
MaxResults *int64 `locationName:"maxResults" type:"integer"`
// The token to request the next page of results.
@@ -66458,19 +66450,23 @@ type StateReason struct {
// The message for the state change.
//
// * Server.InsufficientInstanceCapacity: There was insufficient instance
// capacity to satisfy the launch request.
// * Server.InsufficientInstanceCapacity: There was insufficient capacity
// available to satisfy the launch request.
//
// * Server.InternalError: An internal error occurred during instance launch,
// resulting in termination.
// * Server.InternalError: An internal error caused the instance to terminate
// during launch.
//
// * Server.ScheduledStop: The instance was stopped due to a scheduled retirement.
//
// * Server.SpotInstanceTermination: A Spot Instance was terminated due to
// an increase in the Spot price.
// * Server.SpotInstanceShutdown: The instance was stopped because the number
// of Spot requests with a maximum price equal to or higher than the Spot
// price exceeded available capacity or because of an increase in the Spot
// price.
//
// * Client.InternalError: A client error caused the instance to terminate
// on launch.
// * Server.SpotInstanceTermination: The instance was terminated because
// the number of Spot requests with a maximum price equal to or higher than
// the Spot price exceeded available capacity or because of an increase in
// the Spot price.
//
// * Client.InstanceInitiatedShutdown: The instance was shut down using the
// shutdown -h command from the instance.
@@ -66478,14 +66474,17 @@ type StateReason struct {
// * Client.InstanceTerminated: The instance was terminated or rebooted during
// AMI creation.
//
// * Client.InternalError: A client error caused the instance to terminate
// during launch.
//
// * Client.InvalidSnapshot.NotFound: The specified snapshot was not found.
//
// * Client.UserInitiatedShutdown: The instance was shut down using the Amazon
// EC2 API.
//
// * Client.VolumeLimitExceeded: The limit on the number of EBS volumes or
// total storage was exceeded. Decrease usage or request an increase in your
// limits.
//
// * Client.InvalidSnapshot.NotFound: The specified snapshot was not found.
// account limits.
Message *string `locationName:"message" type:"string"`
}
@@ -66969,7 +66968,7 @@ type TagSpecification struct {
_ struct{} `type:"structure"`
// The type of resource to tag. Currently, the resource types that support tagging
// on creation are instance and volume.
// on creation are instance, snapshot, and volume.
ResourceType *string `locationName:"resourceType" type:"string" enum:"ResourceType"`
// The tags to apply to the resource.
@@ -70694,6 +70693,9 @@ const (
// InstanceTypeI316xlarge is a InstanceType enum value
InstanceTypeI316xlarge = "i3.16xlarge"
// InstanceTypeI3Metal is a InstanceType enum value
InstanceTypeI3Metal = "i3.metal"
// InstanceTypeHi14xlarge is a InstanceType enum value
InstanceTypeHi14xlarge = "hi1.4xlarge"
@@ -70754,6 +70756,24 @@ const (
// InstanceTypeC518xlarge is a InstanceType enum value
InstanceTypeC518xlarge = "c5.18xlarge"
// InstanceTypeC5dLarge is a InstanceType enum value
InstanceTypeC5dLarge = "c5d.large"
// InstanceTypeC5dXlarge is a InstanceType enum value
InstanceTypeC5dXlarge = "c5d.xlarge"
// InstanceTypeC5d2xlarge is a InstanceType enum value
InstanceTypeC5d2xlarge = "c5d.2xlarge"
// InstanceTypeC5d4xlarge is a InstanceType enum value
InstanceTypeC5d4xlarge = "c5d.4xlarge"
// InstanceTypeC5d9xlarge is a InstanceType enum value
InstanceTypeC5d9xlarge = "c5d.9xlarge"
// InstanceTypeC5d18xlarge is a InstanceType enum value
InstanceTypeC5d18xlarge = "c5d.18xlarge"
// InstanceTypeCc14xlarge is a InstanceType enum value
InstanceTypeCc14xlarge = "cc1.4xlarge"
@@ -70832,6 +70852,24 @@ const (
// InstanceTypeM524xlarge is a InstanceType enum value
InstanceTypeM524xlarge = "m5.24xlarge"
// InstanceTypeM5dLarge is a InstanceType enum value
InstanceTypeM5dLarge = "m5d.large"
// InstanceTypeM5dXlarge is a InstanceType enum value
InstanceTypeM5dXlarge = "m5d.xlarge"
// InstanceTypeM5d2xlarge is a InstanceType enum value
InstanceTypeM5d2xlarge = "m5d.2xlarge"
// InstanceTypeM5d4xlarge is a InstanceType enum value
InstanceTypeM5d4xlarge = "m5d.4xlarge"
// InstanceTypeM5d12xlarge is a InstanceType enum value
InstanceTypeM5d12xlarge = "m5d.12xlarge"
// InstanceTypeM5d24xlarge is a InstanceType enum value
InstanceTypeM5d24xlarge = "m5d.24xlarge"
// InstanceTypeH12xlarge is a InstanceType enum value
InstanceTypeH12xlarge = "h1.2xlarge"
+4 -2
View File
@@ -29,8 +29,9 @@ var initRequest func(*request.Request)
// Service information constants
const (
ServiceName = "ec2" // Service endpoint prefix API calls made to.
EndpointsID = ServiceName // Service ID for Regions and Endpoints metadata.
ServiceName = "ec2" // Name of service.
EndpointsID = ServiceName // ID to lookup a service endpoint with.
ServiceID = "EC2" // ServiceID is a unique identifer of a specific service.
)
// New creates a new instance of the EC2 client with a session.
@@ -55,6 +56,7 @@ func newClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegio
cfg,
metadata.ClientInfo{
ServiceName: ServiceName,
ServiceID: ServiceID,
SigningName: signingName,
SigningRegion: signingRegion,
Endpoint: endpoint,
+796
View File
@@ -3,14 +3,21 @@
package s3
import (
"bytes"
"fmt"
"io"
"sync"
"sync/atomic"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awsutil"
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/private/protocol"
"github.com/aws/aws-sdk-go/private/protocol/eventstream"
"github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi"
"github.com/aws/aws-sdk-go/private/protocol/rest"
"github.com/aws/aws-sdk-go/private/protocol/restxml"
)
@@ -6017,6 +6024,88 @@ func (c *S3) RestoreObjectWithContext(ctx aws.Context, input *RestoreObjectInput
return out, req.Send()
}
const opSelectObjectContent = "SelectObjectContent"
// SelectObjectContentRequest generates a "aws/request.Request" representing the
// client's request for the SelectObjectContent operation. The "output" return
// value will be populated with the request's response once the request completes
// successfuly.
//
// Use "Send" method on the returned Request to send the API call to the service.
// the "output" return value is not valid until after Send returns without error.
//
// See SelectObjectContent for more information on using the SelectObjectContent
// API call, and error handling.
//
// This method is useful when you want to inject custom logic or configuration
// into the SDK's request lifecycle. Such as custom headers, or retry logic.
//
//
// // Example sending a request using the SelectObjectContentRequest method.
// req, resp := client.SelectObjectContentRequest(params)
//
// err := req.Send()
// if err == nil { // resp is now filled
// fmt.Println(resp)
// }
//
// See also, https://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/SelectObjectContent
func (c *S3) SelectObjectContentRequest(input *SelectObjectContentInput) (req *request.Request, output *SelectObjectContentOutput) {
op := &request.Operation{
Name: opSelectObjectContent,
HTTPMethod: "POST",
HTTPPath: "/{Bucket}/{Key+}?select&select-type=2",
}
if input == nil {
input = &SelectObjectContentInput{}
}
output = &SelectObjectContentOutput{}
req = c.newRequest(op, input, output)
req.Handlers.Send.Swap(client.LogHTTPResponseHandler.Name, client.LogHTTPResponseHeaderHandler)
req.Handlers.Unmarshal.Swap(restxml.UnmarshalHandler.Name, rest.UnmarshalHandler)
req.Handlers.Unmarshal.PushBack(output.runEventStreamLoop)
return
}
// SelectObjectContent API operation for Amazon Simple Storage Service.
//
// This operation filters the contents of an Amazon S3 object based on a simple
// Structured Query Language (SQL) statement. In the request, along with the
// SQL expression, you must also specify a data serialization format (JSON or
// CSV) of the object. Amazon S3 uses this to parse object data into records,
// and returns only records that match the specified SQL expression. You must
// also specify the data serialization format for the response.
//
// Returns awserr.Error for service API and SDK errors. Use runtime type assertions
// with awserr.Error's Code and Message methods to get detailed information about
// the error.
//
// See the AWS API reference guide for Amazon Simple Storage Service's
// API operation SelectObjectContent for usage and error information.
// See also, https://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/SelectObjectContent
func (c *S3) SelectObjectContent(input *SelectObjectContentInput) (*SelectObjectContentOutput, error) {
req, out := c.SelectObjectContentRequest(input)
return out, req.Send()
}
// SelectObjectContentWithContext is the same as SelectObjectContent with the addition of
// the ability to pass a context and additional request options.
//
// See SelectObjectContent for details on how to use this API operation.
//
// The context must be non-nil and will be used for request cancellation. If
// the context is nil a panic will occur. In the future the SDK may create
// sub-contexts for http.Requests. See https://golang.org/pkg/context/
// for more information on using Contexts.
func (c *S3) SelectObjectContentWithContext(ctx aws.Context, input *SelectObjectContentInput, opts ...request.Option) (*SelectObjectContentOutput, error) {
req, out := c.SelectObjectContentRequest(input)
req.SetContext(ctx)
req.ApplyOptions(opts...)
return out, req.Send()
}
const opUploadPart = "UploadPart"
// UploadPartRequest generates a "aws/request.Request" representing the
@@ -7474,6 +7563,32 @@ func (s *Condition) SetKeyPrefixEquals(v string) *Condition {
return s
}
type ContinuationEvent struct {
_ struct{} `type:"structure"`
}
// String returns the string representation
func (s ContinuationEvent) String() string {
return awsutil.Prettify(s)
}
// GoString returns the string representation
func (s ContinuationEvent) GoString() string {
return s.String()
}
// The ContinuationEvent is and event in the SelectObjectContentEventStream group of events.
func (s *ContinuationEvent) eventSelectObjectContentEventStream() {}
// UnmarshalEvent unmarshals the EventStream Message into the ContinuationEvent value.
// This method is only used internally within the SDK's EventStream handling.
func (s *ContinuationEvent) UnmarshalEvent(
payloadUnmarshaler protocol.PayloadUnmarshaler,
msg eventstream.Message,
) error {
return nil
}
type CopyObjectInput struct {
_ struct{} `type:"structure"`
@@ -9919,6 +10034,32 @@ func (s *EncryptionConfiguration) SetReplicaKmsKeyID(v string) *EncryptionConfig
return s
}
type EndEvent struct {
_ struct{} `type:"structure"`
}
// String returns the string representation
func (s EndEvent) String() string {
return awsutil.Prettify(s)
}
// GoString returns the string representation
func (s EndEvent) GoString() string {
return s.String()
}
// The EndEvent is and event in the SelectObjectContentEventStream group of events.
func (s *EndEvent) eventSelectObjectContentEventStream() {}
// UnmarshalEvent unmarshals the EventStream Message into the EndEvent value.
// This method is only used internally within the SDK's EventStream handling.
func (s *EndEvent) UnmarshalEvent(
payloadUnmarshaler protocol.PayloadUnmarshaler,
msg eventstream.Message,
) error {
return nil
}
type Error struct {
_ struct{} `type:"structure"`
@@ -16380,6 +16521,87 @@ func (s *Part) SetSize(v int64) *Part {
return s
}
type Progress struct {
_ struct{} `type:"structure"`
// Current number of uncompressed object bytes processed.
BytesProcessed *int64 `type:"long"`
// Current number of bytes of records payload data returned.
BytesReturned *int64 `type:"long"`
// Current number of object bytes scanned.
BytesScanned *int64 `type:"long"`
}
// String returns the string representation
func (s Progress) String() string {
return awsutil.Prettify(s)
}
// GoString returns the string representation
func (s Progress) GoString() string {
return s.String()
}
// SetBytesProcessed sets the BytesProcessed field's value.
func (s *Progress) SetBytesProcessed(v int64) *Progress {
s.BytesProcessed = &v
return s
}
// SetBytesReturned sets the BytesReturned field's value.
func (s *Progress) SetBytesReturned(v int64) *Progress {
s.BytesReturned = &v
return s
}
// SetBytesScanned sets the BytesScanned field's value.
func (s *Progress) SetBytesScanned(v int64) *Progress {
s.BytesScanned = &v
return s
}
type ProgressEvent struct {
_ struct{} `type:"structure" payload:"Details"`
// The Progress event details.
Details *Progress `locationName:"Details" type:"structure"`
}
// String returns the string representation
func (s ProgressEvent) String() string {
return awsutil.Prettify(s)
}
// GoString returns the string representation
func (s ProgressEvent) GoString() string {
return s.String()
}
// SetDetails sets the Details field's value.
func (s *ProgressEvent) SetDetails(v *Progress) *ProgressEvent {
s.Details = v
return s
}
// The ProgressEvent is and event in the SelectObjectContentEventStream group of events.
func (s *ProgressEvent) eventSelectObjectContentEventStream() {}
// UnmarshalEvent unmarshals the EventStream Message into the ProgressEvent value.
// This method is only used internally within the SDK's EventStream handling.
func (s *ProgressEvent) UnmarshalEvent(
payloadUnmarshaler protocol.PayloadUnmarshaler,
msg eventstream.Message,
) error {
if err := payloadUnmarshaler.UnmarshalPayload(
bytes.NewReader(msg.Payload), s,
); err != nil {
return fmt.Errorf("failed to unmarshal payload, %v", err)
}
return nil
}
type PutBucketAccelerateConfigurationInput struct {
_ struct{} `type:"structure" payload:"AccelerateConfiguration"`
@@ -18622,6 +18844,45 @@ func (s *QueueConfigurationDeprecated) SetQueue(v string) *QueueConfigurationDep
return s
}
type RecordsEvent struct {
_ struct{} `type:"structure" payload:"Payload"`
// The byte array of partial, one or more result records.
//
// Payload is automatically base64 encoded/decoded by the SDK.
Payload []byte `type:"blob"`
}
// String returns the string representation
func (s RecordsEvent) String() string {
return awsutil.Prettify(s)
}
// GoString returns the string representation
func (s RecordsEvent) GoString() string {
return s.String()
}
// SetPayload sets the Payload field's value.
func (s *RecordsEvent) SetPayload(v []byte) *RecordsEvent {
s.Payload = v
return s
}
// The RecordsEvent is and event in the SelectObjectContentEventStream group of events.
func (s *RecordsEvent) eventSelectObjectContentEventStream() {}
// UnmarshalEvent unmarshals the EventStream Message into the RecordsEvent value.
// This method is only used internally within the SDK's EventStream handling.
func (s *RecordsEvent) UnmarshalEvent(
payloadUnmarshaler protocol.PayloadUnmarshaler,
msg eventstream.Message,
) error {
s.Payload = make([]byte, len(msg.Payload))
copy(s.Payload, msg.Payload)
return nil
}
type Redirect struct {
_ struct{} `type:"structure"`
@@ -18939,6 +19200,30 @@ func (s *RequestPaymentConfiguration) SetPayer(v string) *RequestPaymentConfigur
return s
}
type RequestProgress struct {
_ struct{} `type:"structure"`
// Specifies whether periodic QueryProgress frames should be sent. Valid values:
// TRUE, FALSE. Default value: FALSE.
Enabled *bool `type:"boolean"`
}
// String returns the string representation
func (s RequestProgress) String() string {
return awsutil.Prettify(s)
}
// GoString returns the string representation
func (s RequestProgress) GoString() string {
return s.String()
}
// SetEnabled sets the Enabled field's value.
func (s *RequestProgress) SetEnabled(v bool) *RequestProgress {
s.Enabled = &v
return s
}
type RestoreObjectInput struct {
_ struct{} `type:"structure" payload:"RestoreRequest"`
@@ -19392,6 +19677,436 @@ func (s SSES3) GoString() string {
return s.String()
}
// SelectObjectContentEventStream provides handling of EventStreams for
// the SelectObjectContent API.
//
// Use this type to receive SelectObjectContentEventStream events. The events
// can be read from the Events channel member.
//
// The events that can be received are:
//
// * ContinuationEvent
// * EndEvent
// * ProgressEvent
// * RecordsEvent
// * StatsEvent
type SelectObjectContentEventStream struct {
// Reader is the EventStream reader for the SelectObjectContentEventStream
// events. This value is automatically set by the SDK when the API call is made
// Use this member when unit testing your code with the SDK to mock out the
// EventStream Reader.
//
// Must not be nil.
Reader SelectObjectContentEventStreamReader
// StreamCloser is the io.Closer for the EventStream connection. For HTTP
// EventStream this is the response Body. The stream will be closed when
// the Close method of the EventStream is called.
StreamCloser io.Closer
}
// Close closes the EventStream. This will also cause the Events channel to be
// closed. You can use the closing of the Events channel to terminate your
// application's read from the API's EventStream.
//
// Will close the underlying EventStream reader. For EventStream over HTTP
// connection this will also close the HTTP connection.
//
// Close must be called when done using the EventStream API. Not calling Close
// may result in resource leaks.
func (es *SelectObjectContentEventStream) Close() (err error) {
es.Reader.Close()
return es.Err()
}
// Err returns any error that occurred while reading EventStream Events from
// the service API's response. Returns nil if there were no errors.
func (es *SelectObjectContentEventStream) Err() error {
if err := es.Reader.Err(); err != nil {
return err
}
es.StreamCloser.Close()
return nil
}
// Events returns a channel to read EventStream Events from the
// SelectObjectContent API.
//
// These events are:
//
// * ContinuationEvent
// * EndEvent
// * ProgressEvent
// * RecordsEvent
// * StatsEvent
func (es *SelectObjectContentEventStream) Events() <-chan SelectObjectContentEventStreamEvent {
return es.Reader.Events()
}
// SelectObjectContentEventStreamEvent groups together all EventStream
// events read from the SelectObjectContent API.
//
// These events are:
//
// * ContinuationEvent
// * EndEvent
// * ProgressEvent
// * RecordsEvent
// * StatsEvent
type SelectObjectContentEventStreamEvent interface {
eventSelectObjectContentEventStream()
}
// SelectObjectContentEventStreamReader provides the interface for reading EventStream
// Events from the SelectObjectContent API. The
// default implementation for this interface will be SelectObjectContentEventStream.
//
// The reader's Close method must allow multiple concurrent calls.
//
// These events are:
//
// * ContinuationEvent
// * EndEvent
// * ProgressEvent
// * RecordsEvent
// * StatsEvent
type SelectObjectContentEventStreamReader interface {
// Returns a channel of events as they are read from the event stream.
Events() <-chan SelectObjectContentEventStreamEvent
// Close will close the underlying event stream reader. For event stream over
// HTTP this will also close the HTTP connection.
Close() error
// Returns any error that has occured while reading from the event stream.
Err() error
}
type readSelectObjectContentEventStream struct {
eventReader *eventstreamapi.EventReader
stream chan SelectObjectContentEventStreamEvent
errVal atomic.Value
done chan struct{}
closeOnce sync.Once
}
func newReadSelectObjectContentEventStream(
reader io.ReadCloser,
unmarshalers request.HandlerList,
logger aws.Logger,
logLevel aws.LogLevelType,
) *readSelectObjectContentEventStream {
r := &readSelectObjectContentEventStream{
stream: make(chan SelectObjectContentEventStreamEvent),
done: make(chan struct{}),
}
r.eventReader = eventstreamapi.NewEventReader(
reader,
protocol.HandlerPayloadUnmarshal{
Unmarshalers: unmarshalers,
},
r.unmarshalerForEventType,
)
r.eventReader.UseLogger(logger, logLevel)
return r
}
// Close will close the underlying event stream reader. For EventStream over
// HTTP this will also close the HTTP connection.
func (r *readSelectObjectContentEventStream) Close() error {
r.closeOnce.Do(r.safeClose)
return r.Err()
}
func (r *readSelectObjectContentEventStream) safeClose() {
close(r.done)
err := r.eventReader.Close()
if err != nil {
r.errVal.Store(err)
}
}
func (r *readSelectObjectContentEventStream) Err() error {
if v := r.errVal.Load(); v != nil {
return v.(error)
}
return nil
}
func (r *readSelectObjectContentEventStream) Events() <-chan SelectObjectContentEventStreamEvent {
return r.stream
}
func (r *readSelectObjectContentEventStream) readEventStream() {
defer close(r.stream)
for {
event, err := r.eventReader.ReadEvent()
if err != nil {
if err == io.EOF {
return
}
select {
case <-r.done:
// If closed already ignore the error
return
default:
}
r.errVal.Store(err)
return
}
select {
case r.stream <- event.(SelectObjectContentEventStreamEvent):
case <-r.done:
return
}
}
}
func (r *readSelectObjectContentEventStream) unmarshalerForEventType(
eventType string,
) (eventstreamapi.Unmarshaler, error) {
switch eventType {
case "Cont":
return &ContinuationEvent{}, nil
case "End":
return &EndEvent{}, nil
case "Progress":
return &ProgressEvent{}, nil
case "Records":
return &RecordsEvent{}, nil
case "Stats":
return &StatsEvent{}, nil
default:
return nil, fmt.Errorf(
"unknown event type name, %s, for SelectObjectContentEventStream", eventType)
}
}
// Request to filter the contents of an Amazon S3 object based on a simple Structured
// Query Language (SQL) statement. In the request, along with the SQL expression,
// you must also specify a data serialization format (JSON or CSV) of the object.
// Amazon S3 uses this to parse object data into records, and returns only records
// that match the specified SQL expression. You must also specify the data serialization
// format for the response. For more information, go to S3Select API Documentation
// (https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html)
type SelectObjectContentInput struct {
_ struct{} `locationName:"SelectObjectContentRequest" type:"structure" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"`
// The S3 Bucket.
//
// Bucket is a required field
Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
// The expression that is used to query the object.
//
// Expression is a required field
Expression *string `type:"string" required:"true"`
// The type of the provided expression (e.g., SQL).
//
// ExpressionType is a required field
ExpressionType *string `type:"string" required:"true" enum:"ExpressionType"`
// Describes the format of the data in the object that is being queried.
//
// InputSerialization is a required field
InputSerialization *InputSerialization `type:"structure" required:"true"`
// The Object Key.
//
// Key is a required field
Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"`
// Describes the format of the data that you want Amazon S3 to return in response.
//
// OutputSerialization is a required field
OutputSerialization *OutputSerialization `type:"structure" required:"true"`
// Specifies if periodic request progress information should be enabled.
RequestProgress *RequestProgress `type:"structure"`
// The SSE Algorithm used to encrypt the object. For more information, go to
// Server-Side Encryption (Using Customer-Provided Encryption Keys (https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html)
SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"`
// The SSE Customer Key. For more information, go to Server-Side Encryption
// (Using Customer-Provided Encryption Keys (https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html)
SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"`
// The SSE Customer Key MD5. For more information, go to Server-Side Encryption
// (Using Customer-Provided Encryption Keys (https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html)
SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"`
}
// String returns the string representation
func (s SelectObjectContentInput) String() string {
return awsutil.Prettify(s)
}
// GoString returns the string representation
func (s SelectObjectContentInput) GoString() string {
return s.String()
}
// Validate inspects the fields of the type to determine if they are valid.
func (s *SelectObjectContentInput) Validate() error {
invalidParams := request.ErrInvalidParams{Context: "SelectObjectContentInput"}
if s.Bucket == nil {
invalidParams.Add(request.NewErrParamRequired("Bucket"))
}
if s.Expression == nil {
invalidParams.Add(request.NewErrParamRequired("Expression"))
}
if s.ExpressionType == nil {
invalidParams.Add(request.NewErrParamRequired("ExpressionType"))
}
if s.InputSerialization == nil {
invalidParams.Add(request.NewErrParamRequired("InputSerialization"))
}
if s.Key == nil {
invalidParams.Add(request.NewErrParamRequired("Key"))
}
if s.Key != nil && len(*s.Key) < 1 {
invalidParams.Add(request.NewErrParamMinLen("Key", 1))
}
if s.OutputSerialization == nil {
invalidParams.Add(request.NewErrParamRequired("OutputSerialization"))
}
if invalidParams.Len() > 0 {
return invalidParams
}
return nil
}
// SetBucket sets the Bucket field's value.
func (s *SelectObjectContentInput) SetBucket(v string) *SelectObjectContentInput {
s.Bucket = &v
return s
}
func (s *SelectObjectContentInput) getBucket() (v string) {
if s.Bucket == nil {
return v
}
return *s.Bucket
}
// SetExpression sets the Expression field's value.
func (s *SelectObjectContentInput) SetExpression(v string) *SelectObjectContentInput {
s.Expression = &v
return s
}
// SetExpressionType sets the ExpressionType field's value.
func (s *SelectObjectContentInput) SetExpressionType(v string) *SelectObjectContentInput {
s.ExpressionType = &v
return s
}
// SetInputSerialization sets the InputSerialization field's value.
func (s *SelectObjectContentInput) SetInputSerialization(v *InputSerialization) *SelectObjectContentInput {
s.InputSerialization = v
return s
}
// SetKey sets the Key field's value.
func (s *SelectObjectContentInput) SetKey(v string) *SelectObjectContentInput {
s.Key = &v
return s
}
// SetOutputSerialization sets the OutputSerialization field's value.
func (s *SelectObjectContentInput) SetOutputSerialization(v *OutputSerialization) *SelectObjectContentInput {
s.OutputSerialization = v
return s
}
// SetRequestProgress sets the RequestProgress field's value.
func (s *SelectObjectContentInput) SetRequestProgress(v *RequestProgress) *SelectObjectContentInput {
s.RequestProgress = v
return s
}
// SetSSECustomerAlgorithm sets the SSECustomerAlgorithm field's value.
func (s *SelectObjectContentInput) SetSSECustomerAlgorithm(v string) *SelectObjectContentInput {
s.SSECustomerAlgorithm = &v
return s
}
// SetSSECustomerKey sets the SSECustomerKey field's value.
func (s *SelectObjectContentInput) SetSSECustomerKey(v string) *SelectObjectContentInput {
s.SSECustomerKey = &v
return s
}
func (s *SelectObjectContentInput) getSSECustomerKey() (v string) {
if s.SSECustomerKey == nil {
return v
}
return *s.SSECustomerKey
}
// SetSSECustomerKeyMD5 sets the SSECustomerKeyMD5 field's value.
func (s *SelectObjectContentInput) SetSSECustomerKeyMD5(v string) *SelectObjectContentInput {
s.SSECustomerKeyMD5 = &v
return s
}
type SelectObjectContentOutput struct {
_ struct{} `type:"structure" payload:"Payload"`
// Use EventStream to use the API's stream.
EventStream *SelectObjectContentEventStream `type:"structure"`
}
// String returns the string representation
func (s SelectObjectContentOutput) String() string {
return awsutil.Prettify(s)
}
// GoString returns the string representation
func (s SelectObjectContentOutput) GoString() string {
return s.String()
}
// SetEventStream sets the EventStream field's value.
func (s *SelectObjectContentOutput) SetEventStream(v *SelectObjectContentEventStream) *SelectObjectContentOutput {
s.EventStream = v
return s
}
func (s *SelectObjectContentOutput) runEventStreamLoop(r *request.Request) {
if r.Error != nil {
return
}
reader := newReadSelectObjectContentEventStream(
r.HTTPResponse.Body,
r.Handlers.UnmarshalStream,
r.Config.Logger,
r.Config.LogLevel.Value(),
)
go reader.readEventStream()
eventStream := &SelectObjectContentEventStream{
StreamCloser: r.HTTPResponse.Body,
Reader: reader,
}
s.EventStream = eventStream
}
// Describes the parameters for Select job types.
type SelectParameters struct {
_ struct{} `type:"structure"`
@@ -19696,6 +20411,87 @@ func (s *SseKmsEncryptedObjects) SetStatus(v string) *SseKmsEncryptedObjects {
return s
}
type Stats struct {
_ struct{} `type:"structure"`
// Total number of uncompressed object bytes processed.
BytesProcessed *int64 `type:"long"`
// Total number of bytes of records payload data returned.
BytesReturned *int64 `type:"long"`
// Total number of object bytes scanned.
BytesScanned *int64 `type:"long"`
}
// String returns the string representation
func (s Stats) String() string {
return awsutil.Prettify(s)
}
// GoString returns the string representation
func (s Stats) GoString() string {
return s.String()
}
// SetBytesProcessed sets the BytesProcessed field's value.
func (s *Stats) SetBytesProcessed(v int64) *Stats {
s.BytesProcessed = &v
return s
}
// SetBytesReturned sets the BytesReturned field's value.
func (s *Stats) SetBytesReturned(v int64) *Stats {
s.BytesReturned = &v
return s
}
// SetBytesScanned sets the BytesScanned field's value.
func (s *Stats) SetBytesScanned(v int64) *Stats {
s.BytesScanned = &v
return s
}
type StatsEvent struct {
_ struct{} `type:"structure" payload:"Details"`
// The Stats event details.
Details *Stats `locationName:"Details" type:"structure"`
}
// String returns the string representation
func (s StatsEvent) String() string {
return awsutil.Prettify(s)
}
// GoString returns the string representation
func (s StatsEvent) GoString() string {
return s.String()
}
// SetDetails sets the Details field's value.
func (s *StatsEvent) SetDetails(v *Stats) *StatsEvent {
s.Details = v
return s
}
// The StatsEvent is and event in the SelectObjectContentEventStream group of events.
func (s *StatsEvent) eventSelectObjectContentEventStream() {}
// UnmarshalEvent unmarshals the EventStream Message into the StatsEvent value.
// This method is only used internally within the SDK's EventStream handling.
func (s *StatsEvent) UnmarshalEvent(
payloadUnmarshaler protocol.PayloadUnmarshaler,
msg eventstream.Message,
) error {
if err := payloadUnmarshaler.UnmarshalPayload(
bytes.NewReader(msg.Payload), s,
); err != nil {
return fmt.Errorf("failed to unmarshal payload, %v", err)
}
return nil
}
type StorageClassAnalysis struct {
_ struct{} `type:"structure"`
+6 -2
View File
@@ -29,8 +29,9 @@ var initRequest func(*request.Request)
// Service information constants
const (
ServiceName = "s3" // Service endpoint prefix API calls made to.
EndpointsID = ServiceName // Service ID for Regions and Endpoints metadata.
ServiceName = "s3" // Name of service.
EndpointsID = ServiceName // ID to lookup a service endpoint with.
ServiceID = "S3" // ServiceID is a unique identifer of a specific service.
)
// New creates a new instance of the S3 client with a session.
@@ -55,6 +56,7 @@ func newClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegio
cfg,
metadata.ClientInfo{
ServiceName: ServiceName,
ServiceID: ServiceID,
SigningName: signingName,
SigningRegion: signingRegion,
Endpoint: endpoint,
@@ -71,6 +73,8 @@ func newClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegio
svc.Handlers.UnmarshalMeta.PushBackNamed(restxml.UnmarshalMetaHandler)
svc.Handlers.UnmarshalError.PushBackNamed(restxml.UnmarshalErrorHandler)
svc.Handlers.UnmarshalStream.PushBackNamed(restxml.UnmarshalHandler)
// Run custom client initialization if present
if initClient != nil {
initClient(svc.Client)
+4 -2
View File
@@ -29,8 +29,9 @@ var initRequest func(*request.Request)
// Service information constants
const (
ServiceName = "sts" // Service endpoint prefix API calls made to.
EndpointsID = ServiceName // Service ID for Regions and Endpoints metadata.
ServiceName = "sts" // Name of service.
EndpointsID = ServiceName // ID to lookup a service endpoint with.
ServiceID = "STS" // ServiceID is a unique identifer of a specific service.
)
// New creates a new instance of the STS client with a session.
@@ -55,6 +56,7 @@ func newClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegio
cfg,
metadata.ClientInfo{
ServiceName: ServiceName,
ServiceID: ServiceID,
SigningName: signingName,
SigningRegion: signingRegion,
Endpoint: endpoint,