S3 IAM support (#7058)

* add s3 dependencly

* rewrite image uploader by aws-sdk-go

* improve backward compatibility of s3 image uploading

* set default for expires option

* remove acl and expires

pkg/components/imguploader/imguploader.go

@@ -2,6 +2,7 @@ package imguploader
 
 import (
 	"fmt"
+	"regexp"
 
 	"github.com/grafana/grafana/pkg/setting"
 )
@@ -30,19 +31,21 @@ func NewImageUploader() (ImageUploader, error) {
 		accessKey := s3sec.Key("access_key").MustString("")
 		secretKey := s3sec.Key("secret_key").MustString("")
 
-		if bucket == "" {
+		region := ""
+		rBucket := regexp.MustCompile(`https?:\/\/(.*)\.s3(-([^.]+))?\.amazonaws\.com\/?`)
+		matches := rBucket.FindStringSubmatch(bucket)
+		if len(matches) == 0 {
 			return nil, fmt.Errorf("Could not find bucket setting for image.uploader.s3")
+		} else {
+			bucket = matches[1]
+			if matches[3] != "" {
+				region = matches[3]
+			} else {
+				region = "us-east-1"
+			}
 		}
 
-		if accessKey == "" {
-			return nil, fmt.Errorf("Could not find accessKey setting for image.uploader.s3")
-		}
-
-		if secretKey == "" {
-			return nil, fmt.Errorf("Could not find secretKey setting for image.uploader.s3")
-		}
-
-		return NewS3Uploader(bucket, accessKey, secretKey), nil
+		return NewS3Uploader(region, bucket, "public-read", accessKey, secretKey), nil
 	case "webdav":
 		webdavSec, err := setting.Cfg.GetSection("external_image_storage.webdav")
 		if err != nil {

pkg/components/imguploader/imguploader_test.go

@@ -19,7 +19,7 @@ func TestImageUploaderFactory(t *testing.T) {
 			setting.ImageUploadProvider = "s3"
 
 			s3sec, err := setting.Cfg.GetSection("external_image_storage.s3")
-			s3sec.NewKey("bucket_url", "bucket_url")
+			s3sec.NewKey("bucket_url", "https://foo.bar.baz.s3-us-east-2.amazonaws.com")
 			s3sec.NewKey("access_key", "access_key")
 			s3sec.NewKey("secret_key", "secret_key")
 
@@ -29,9 +29,10 @@ func TestImageUploaderFactory(t *testing.T) {
 			original, ok := uploader.(*S3Uploader)
 
 			So(ok, ShouldBeTrue)
+			So(original.region, ShouldEqual, "us-east-2")
+			So(original.bucket, ShouldEqual, "foo.bar.baz")
 			So(original.accessKey, ShouldEqual, "access_key")
 			So(original.secretKey, ShouldEqual, "secret_key")
-			So(original.bucket, ShouldEqual, "bucket_url")
 		})
 
 		Convey("Webdav uploader", func() {

pkg/components/imguploader/s3uploader.go

@@ -1,26 +1,33 @@
 package imguploader
 
 import (
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"path"
+	"os"
+	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
+	"github.com/aws/aws-sdk-go/aws/ec2metadata"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/grafana/grafana/pkg/log"
 	"github.com/grafana/grafana/pkg/util"
-	"github.com/kr/s3/s3util"
 )
 
 type S3Uploader struct {
+	region    string
 	bucket    string
+	acl       string
 	secretKey string
 	accessKey string
 	log       log.Logger
 }
 
-func NewS3Uploader(bucket, accessKey, secretKey string) *S3Uploader {
+func NewS3Uploader(region, bucket, acl, accessKey, secretKey string) *S3Uploader {
 	return &S3Uploader{
+		region:    region,
 		bucket:    bucket,
+		acl:       acl,
 		accessKey: accessKey,
 		secretKey: secretKey,
 		log:       log.New("s3uploader"),
@@ -28,42 +35,41 @@ func NewS3Uploader(bucket, accessKey, secretKey string) *S3Uploader {
 }
 
 func (u *S3Uploader) Upload(imageDiskPath string) (string, error) {
-
-	s3util.DefaultConfig.AccessKey = u.accessKey
-	s3util.DefaultConfig.SecretKey = u.secretKey
-
-	header := make(http.Header)
-	header.Add("x-amz-acl", "public-read")
-	header.Add("Content-Type", "image/png")
-
-	var imageUrl *url.URL
-	var err error
-
-	if imageUrl, err = url.Parse(u.bucket); err != nil {
-		return "", err
+	sess := session.New()
+	creds := credentials.NewChainCredentials(
+		[]credentials.Provider{
+			&credentials.StaticProvider{Value: credentials.Value{
+				AccessKeyID:     u.accessKey,
+				SecretAccessKey: u.secretKey,
+			}},
+			&credentials.EnvProvider{},
+			&ec2rolecreds.EC2RoleProvider{Client: ec2metadata.New(sess), ExpiryWindow: 5 * time.Minute},
+		})
+	cfg := &aws.Config{
+		Region:      aws.String(u.region),
+		Credentials: creds,
 	}
 
-	// add image to url
-	imageUrl.Path = path.Join(imageUrl.Path, util.GetRandomString(20)+".png")
-	imageUrlString := imageUrl.String()
-	log.Debug("Uploading image to s3", "url", imageUrlString)
+	key := util.GetRandomString(20) + ".png"
+	log.Debug("Uploading image to s3", "bucket = ", u.bucket, ", key = ", key)
 
-	writer, err := s3util.Create(imageUrlString, header, nil)
+	file, err := os.Open(imageDiskPath)
 	if err != nil {
 		return "", err
 	}
 
-	defer writer.Close()
-
-	imgData, err := ioutil.ReadFile(imageDiskPath)
+	svc := s3.New(session.New(cfg), cfg)
+	params := &s3.PutObjectInput{
+		Bucket:      aws.String(u.bucket),
+		Key:         aws.String(key),
+		ACL:         aws.String(u.acl),
+		Body:        file,
+		ContentType: aws.String("image/png"),
+	}
+	_, err = svc.PutObject(params)
 	if err != nil {
 		return "", err
 	}
 
-	_, err = writer.Write(imgData)
-	if err != nil {
-		return "", err
-	}
-
-	return imageUrlString, nil
+	return "https://" + u.bucket + ".s3.amazonaws.com/" + key, nil
 }