Plugins: Refactor plugin download/installation (#43046)

* installer -> repo

* add semver format checking

* add plugin callbacks in test

* remove newline

* post install only scans new directories

* remove unused stuff

* everything in own package

* add missing cli params

* make grafana version part of the API

* resolve conflicts

* tidy up logger

* fix cli and tidy log statements

* rename log package

* update struct name

* fix linter issue

* fs -> filestore

* reorder imports

* alias import

* fix test

* fix test

* inline var

* revert jsonc file

* make repo dep of manager

* actually inject the thing

* accept all args for compatability checks

* accept compat from store

* pass os + arch vals

* don't inject fs

* tidy up

* tidy up

* merge with main and tidy fs storage

* fix test

* fix packages

* fix comment + field name

* update fs naming

* fixed wire

* remove unused func

* fix mocks

* fix storage test

* renaming

* fix log line

* fix test

* re-order field

* tidying

* add test for update with same version

* fix wire for CLI

* remove use of ioutil

* don't pass field

* small tidy

* ignore code scanning warn

* fix testdata link

* update lgtm code
This commit is contained in:
Will Browne
2022-08-23 11:50:50 +02:00
committed by GitHub
parent cc78486535
commit 26dfdd5af3
36 changed files with 1399 additions and 998 deletions
+249
View File
@@ -0,0 +1,249 @@
package repo
import (
"archive/zip"
"bufio"
"context"
"crypto/sha256"
"crypto/tls"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"time"
"github.com/grafana/grafana/pkg/plugins/logger"
)
type Client struct {
httpClient http.Client
httpClientNoTimeout http.Client
retryCount int
log logger.Logger
}
func newClient(skipTLSVerify bool, logger logger.Logger) *Client {
return &Client{
httpClient: makeHttpClient(skipTLSVerify, 10*time.Second),
httpClientNoTimeout: makeHttpClient(skipTLSVerify, 0),
log: logger,
}
}
func (c *Client) download(_ context.Context, pluginZipURL, checksum string, compatOpts CompatOpts) (*PluginArchive, error) {
// Create temp file for downloading zip file
tmpFile, err := os.CreateTemp("", "*.zip")
if err != nil {
return nil, fmt.Errorf("%v: %w", "failed to create temporary file", err)
}
defer func() {
if err := os.Remove(tmpFile.Name()); err != nil {
c.log.Warn("Failed to remove temporary file", "file", tmpFile.Name(), "err", err)
}
}()
c.log.Debugf("Installing plugin from %s", pluginZipURL)
err = c.downloadFile(tmpFile, pluginZipURL, checksum, compatOpts)
if err != nil {
if err := tmpFile.Close(); err != nil {
c.log.Warn("Failed to close file", "err", err)
}
return nil, fmt.Errorf("%v: %w", "failed to download plugin archive", err)
}
rc, err := zip.OpenReader(tmpFile.Name())
if err != nil {
return nil, err
}
return &PluginArchive{
File: rc,
}, nil
}
func (c *Client) downloadFile(tmpFile *os.File, pluginURL, checksum string, compatOpts CompatOpts) (err error) {
// Try handling URL as a local file path first
if _, err := os.Stat(pluginURL); err == nil {
// TODO re-verify
// We can ignore this gosec G304 warning since `pluginURL` stems from command line flag "pluginUrl". If the
// user shouldn't be able to read the file, it should be handled through filesystem permissions.
// nolint:gosec
f, err := os.Open(pluginURL)
if err != nil {
return fmt.Errorf("%v: %w", "Failed to read plugin archive", err)
}
defer func() {
if err := f.Close(); err != nil {
c.log.Warn("Failed to close file", "err", err)
}
}()
_, err = io.Copy(tmpFile, f)
if err != nil {
return fmt.Errorf("%v: %w", "Failed to copy plugin archive", err)
}
return nil
}
c.retryCount = 0
defer func() {
if r := recover(); r != nil {
c.retryCount++
if c.retryCount < 3 {
c.log.Debug("Failed downloading. Will retry once.")
err = tmpFile.Truncate(0)
if err != nil {
return
}
_, err = tmpFile.Seek(0, 0)
if err != nil {
return
}
err = c.downloadFile(tmpFile, pluginURL, checksum, compatOpts)
} else {
c.retryCount = 0
failure := fmt.Sprintf("%v", r)
if failure == "runtime error: makeslice: len out of range" {
err = fmt.Errorf("corrupt HTTP response from source, please try again")
} else {
panic(r)
}
}
}
}()
u, err := url.Parse(pluginURL)
if err != nil {
return err
}
// Using no timeout here as some plugins can be bigger and smaller timeout would prevent to download a plugin on
// slow network. As this is CLI operation hanging is not a big of an issue as user can just abort.
bodyReader, err := c.sendReqNoTimeout(u, compatOpts)
if err != nil {
return err
}
defer func() {
if err := bodyReader.Close(); err != nil {
c.log.Warn("Failed to close body", "err", err)
}
}()
w := bufio.NewWriter(tmpFile)
h := sha256.New()
if _, err = io.Copy(w, io.TeeReader(bodyReader, h)); err != nil {
return fmt.Errorf("%v: %w", "failed to compute SHA256 checksum", err)
}
if err := w.Flush(); err != nil {
return fmt.Errorf("failed to write to %q: %w", tmpFile.Name(), err)
}
if len(checksum) > 0 && checksum != fmt.Sprintf("%x", h.Sum(nil)) {
return fmt.Errorf("expected SHA256 checksum does not match the downloaded archive - please contact security@grafana.com")
}
return nil
}
func (c *Client) sendReq(url *url.URL, compatOpts CompatOpts) ([]byte, error) {
req, err := c.createReq(url, compatOpts)
if err != nil {
return nil, err
}
res, err := c.httpClient.Do(req)
if err != nil {
return nil, err
}
bodyReader, err := c.handleResp(res, compatOpts)
if err != nil {
return nil, err
}
defer func() {
if err := bodyReader.Close(); err != nil {
c.log.Warn("Failed to close stream", "err", err)
}
}()
return io.ReadAll(bodyReader)
}
func (c *Client) sendReqNoTimeout(url *url.URL, compatOpts CompatOpts) (io.ReadCloser, error) {
req, err := c.createReq(url, compatOpts)
if err != nil {
return nil, err
}
res, err := c.httpClientNoTimeout.Do(req)
if err != nil {
return nil, err
}
return c.handleResp(res, compatOpts)
}
func (c *Client) createReq(url *url.URL, compatOpts CompatOpts) (*http.Request, error) {
req, err := http.NewRequest(http.MethodGet, url.String(), nil)
if err != nil {
return nil, err
}
req.Header.Set("grafana-version", compatOpts.GrafanaVersion)
req.Header.Set("grafana-os", compatOpts.OS)
req.Header.Set("grafana-arch", compatOpts.Arch)
req.Header.Set("User-Agent", "grafana "+compatOpts.GrafanaVersion)
return req, err
}
func (c *Client) handleResp(res *http.Response, compatOpts CompatOpts) (io.ReadCloser, error) {
if res.StatusCode/100 == 4 {
body, err := io.ReadAll(res.Body)
defer func() {
if err := res.Body.Close(); err != nil {
c.log.Warn("Failed to close response body", "err", err)
}
}()
if err != nil || len(body) == 0 {
return nil, Response4xxError{StatusCode: res.StatusCode}
}
var message string
var jsonBody map[string]string
err = json.Unmarshal(body, &jsonBody)
if err != nil || len(jsonBody["message"]) == 0 {
message = string(body)
} else {
message = jsonBody["message"]
}
return nil, Response4xxError{StatusCode: res.StatusCode, Message: message, SystemInfo: compatOpts.String()}
}
if res.StatusCode/100 != 2 {
return nil, fmt.Errorf("API returned invalid status: %s", res.Status)
}
return res.Body, nil
}
func makeHttpClient(skipTLSVerify bool, timeout time.Duration) http.Client {
tr := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: skipTLSVerify,
},
}
return http.Client{
Timeout: timeout,
Transport: tr,
}
}
+39
View File
@@ -0,0 +1,39 @@
package repo
import (
"context"
"fmt"
"strings"
)
// Service is responsible for retrieving plugin information from a repository.
type Service interface {
// GetPluginArchive fetches the requested plugin archive.
GetPluginArchive(ctx context.Context, pluginID, version string, opts CompatOpts) (*PluginArchive, error)
// GetPluginArchiveByURL fetches the requested plugin from the specified URL.
GetPluginArchiveByURL(ctx context.Context, archiveURL string, opts CompatOpts) (*PluginArchive, error)
// GetPluginDownloadOptions fetches information for downloading the requested plugin.
GetPluginDownloadOptions(ctx context.Context, pluginID, version string, opts CompatOpts) (*PluginDownloadOptions, error)
}
type CompatOpts struct {
GrafanaVersion string
OS string
Arch string
}
func NewCompatOpts(grafanaVersion, os, arch string) CompatOpts {
return CompatOpts{
GrafanaVersion: grafanaVersion,
OS: os,
Arch: arch,
}
}
func (co CompatOpts) OSAndArch() string {
return fmt.Sprintf("%s-%s", strings.ToLower(co.OS), co.Arch)
}
func (co CompatOpts) String() string {
return fmt.Sprintf("Grafana v%s %s", co.GrafanaVersion, co.OSAndArch())
}
+74
View File
@@ -0,0 +1,74 @@
package repo
import (
"archive/zip"
"fmt"
)
type PluginArchive struct {
File *zip.ReadCloser
}
type PluginDownloadOptions struct {
PluginZipURL string
Version string
Checksum string
}
type Plugin struct {
ID string `json:"id"`
Category string `json:"category"`
Versions []Version `json:"versions"`
}
type Version struct {
Commit string `json:"commit"`
URL string `json:"repoURL"`
Version string `json:"version"`
Arch map[string]ArchMeta `json:"arch"`
}
type ArchMeta struct {
SHA256 string `json:"sha256"`
}
type PluginRepo struct {
Plugins []Plugin `json:"plugins"`
Version string `json:"version"`
}
type Response4xxError struct {
Message string
StatusCode int
SystemInfo string
}
func (e Response4xxError) Error() string {
if len(e.Message) > 0 {
if len(e.SystemInfo) > 0 {
return fmt.Sprintf("%s (%s)", e.Message, e.SystemInfo)
}
return fmt.Sprintf("%d: %s", e.StatusCode, e.Message)
}
return fmt.Sprintf("%d", e.StatusCode)
}
type ErrVersionUnsupported struct {
PluginID string
RequestedVersion string
SystemInfo string
}
func (e ErrVersionUnsupported) Error() string {
return fmt.Sprintf("%s v%s is not supported on your system (%s)", e.PluginID, e.RequestedVersion, e.SystemInfo)
}
type ErrVersionNotFound struct {
PluginID string
RequestedVersion string
SystemInfo string
}
func (e ErrVersionNotFound) Error() string {
return fmt.Sprintf("%s v%s either does not exist or is not supported on your system (%s)", e.PluginID, e.RequestedVersion, e.SystemInfo)
}
+183
View File
@@ -0,0 +1,183 @@
package repo
import (
"context"
"encoding/json"
"fmt"
"net/url"
"path"
"strings"
"github.com/grafana/grafana/pkg/plugins/logger"
)
type Manager struct {
client *Client
baseURL string
log logger.Logger
}
func ProvideService() *Manager {
defaultBaseURL := "https://grafana.com/api/plugins"
return New(false, defaultBaseURL, logger.NewLogger("plugin.repository"))
}
func New(skipTLSVerify bool, baseURL string, logger logger.Logger) *Manager {
return &Manager{
client: newClient(skipTLSVerify, logger),
baseURL: baseURL,
log: logger,
}
}
// GetPluginArchive fetches the requested plugin archive
func (m *Manager) GetPluginArchive(ctx context.Context, pluginID, version string, compatOpts CompatOpts) (*PluginArchive, error) {
dlOpts, err := m.GetPluginDownloadOptions(ctx, pluginID, version, compatOpts)
if err != nil {
return nil, err
}
return m.client.download(ctx, dlOpts.PluginZipURL, dlOpts.Checksum, compatOpts)
}
// GetPluginArchiveByURL fetches the requested plugin archive from the provided `pluginZipURL`
func (m *Manager) GetPluginArchiveByURL(ctx context.Context, pluginZipURL string, compatOpts CompatOpts) (*PluginArchive, error) {
return m.client.download(ctx, pluginZipURL, "", compatOpts)
}
// GetPluginDownloadOptions returns the options for downloading the requested plugin (with optional `version`)
func (m *Manager) GetPluginDownloadOptions(_ context.Context, pluginID, version string, compatOpts CompatOpts) (*PluginDownloadOptions, error) {
plugin, err := m.pluginMetadata(pluginID, compatOpts)
if err != nil {
return nil, err
}
v, err := m.selectVersion(&plugin, version, compatOpts)
if err != nil {
return nil, err
}
// Plugins which are downloaded just as sourcecode zipball from GitHub do not have checksum
var checksum string
if v.Arch != nil {
archMeta, exists := v.Arch[compatOpts.OSAndArch()]
if !exists {
archMeta = v.Arch["any"]
}
checksum = archMeta.SHA256
}
return &PluginDownloadOptions{
Version: v.Version,
Checksum: checksum,
PluginZipURL: fmt.Sprintf("%s/%s/versions/%s/download", m.baseURL, pluginID, v.Version),
}, nil
}
func (m *Manager) pluginMetadata(pluginID string, compatOpts CompatOpts) (Plugin, error) {
m.log.Debugf("Fetching metadata for plugin \"%s\" from repo %s", pluginID, m.baseURL)
u, err := url.Parse(m.baseURL)
if err != nil {
return Plugin{}, err
}
u.Path = path.Join(u.Path, "repo", pluginID)
body, err := m.client.sendReq(u, compatOpts)
if err != nil {
return Plugin{}, err
}
var data Plugin
err = json.Unmarshal(body, &data)
if err != nil {
m.log.Error("Failed to unmarshal plugin repo response error", err)
return Plugin{}, err
}
return data, nil
}
// selectVersion selects the most appropriate plugin version
// returns the specified version if supported.
// returns the latest version if no specific version is specified.
// returns error if the supplied version does not exist.
// returns error if supplied version exists but is not supported.
// NOTE: It expects plugin.Versions to be sorted so the newest version is first.
func (m *Manager) selectVersion(plugin *Plugin, version string, compatOpts CompatOpts) (*Version, error) {
version = normalizeVersion(version)
var ver Version
latestForArch := latestSupportedVersion(plugin, compatOpts)
if latestForArch == nil {
return nil, ErrVersionUnsupported{
PluginID: plugin.ID,
RequestedVersion: version,
SystemInfo: compatOpts.String(),
}
}
if version == "" {
return latestForArch, nil
}
for _, v := range plugin.Versions {
if v.Version == version {
ver = v
break
}
}
if len(ver.Version) == 0 {
m.log.Debugf("Requested plugin version %s v%s not found but potential fallback version '%s' was found",
plugin.ID, version, latestForArch.Version)
return nil, ErrVersionNotFound{
PluginID: plugin.ID,
RequestedVersion: version,
SystemInfo: compatOpts.String(),
}
}
if !supportsCurrentArch(&ver, compatOpts) {
m.log.Debugf("Requested plugin version %s v%s is not supported on your system but potential fallback version '%s' was found",
plugin.ID, version, latestForArch.Version)
return nil, ErrVersionUnsupported{
PluginID: plugin.ID,
RequestedVersion: version,
SystemInfo: compatOpts.String(),
}
}
return &ver, nil
}
func supportsCurrentArch(version *Version, compatOpts CompatOpts) bool {
if version.Arch == nil {
return true
}
for arch := range version.Arch {
if arch == compatOpts.OSAndArch() || arch == "any" {
return true
}
}
return false
}
func latestSupportedVersion(plugin *Plugin, compatOpts CompatOpts) *Version {
for _, v := range plugin.Versions {
ver := v
if supportsCurrentArch(&ver, compatOpts) {
return &ver
}
}
return nil
}
func normalizeVersion(version string) string {
normalized := strings.ReplaceAll(version, " ", "")
if strings.HasPrefix(normalized, "^") || strings.HasPrefix(normalized, "v") {
return normalized[1:]
}
return normalized
}
+94
View File
@@ -0,0 +1,94 @@
package repo
import (
"fmt"
"testing"
"github.com/stretchr/testify/require"
)
func TestSelectVersion(t *testing.T) {
i := &Manager{log: &fakeLogger{}}
t.Run("Should return error when requested version does not exist", func(t *testing.T) {
_, err := i.selectVersion(createPlugin(versionArg{version: "version"}), "1.1.1", CompatOpts{})
require.Error(t, err)
})
t.Run("Should return error when no version supports current arch", func(t *testing.T) {
_, err := i.selectVersion(createPlugin(versionArg{version: "version", arch: []string{"non-existent"}}), "", CompatOpts{})
require.Error(t, err)
})
t.Run("Should return error when requested version does not support current arch", func(t *testing.T) {
_, err := i.selectVersion(createPlugin(
versionArg{version: "2.0.0"},
versionArg{version: "1.1.1", arch: []string{"non-existent"}},
), "1.1.1", CompatOpts{})
require.Error(t, err)
})
t.Run("Should return latest available for arch when no version specified", func(t *testing.T) {
ver, err := i.selectVersion(createPlugin(
versionArg{version: "2.0.0", arch: []string{"non-existent"}},
versionArg{version: "1.0.0"},
), "", CompatOpts{})
require.NoError(t, err)
require.Equal(t, "1.0.0", ver.Version)
})
t.Run("Should return latest version when no version specified", func(t *testing.T) {
ver, err := i.selectVersion(createPlugin(versionArg{version: "2.0.0"}, versionArg{version: "1.0.0"}), "", CompatOpts{})
require.NoError(t, err)
require.Equal(t, "2.0.0", ver.Version)
})
t.Run("Should return requested version", func(t *testing.T) {
ver, err := i.selectVersion(createPlugin(versionArg{version: "2.0.0"}, versionArg{version: "1.0.0"}), "1.0.0", CompatOpts{})
require.NoError(t, err)
require.Equal(t, "1.0.0", ver.Version)
})
}
type versionArg struct {
version string
arch []string
}
func createPlugin(versions ...versionArg) *Plugin {
p := &Plugin{
Versions: []Version{},
}
for _, version := range versions {
ver := Version{
Version: version.version,
Commit: fmt.Sprintf("commit_%s", version.version),
URL: fmt.Sprintf("url_%s", version.version),
}
if version.arch != nil {
ver.Arch = map[string]ArchMeta{}
for _, arch := range version.arch {
ver.Arch[arch] = ArchMeta{
SHA256: fmt.Sprintf("sha256_%s", arch),
}
}
}
p.Versions = append(p.Versions, ver)
}
return p
}
type fakeLogger struct{}
func (f *fakeLogger) Successf(_ string, _ ...interface{}) {}
func (f *fakeLogger) Failuref(_ string, _ ...interface{}) {}
func (f *fakeLogger) Info(_ ...interface{}) {}
func (f *fakeLogger) Infof(_ string, _ ...interface{}) {}
func (f *fakeLogger) Debug(_ ...interface{}) {}
func (f *fakeLogger) Debugf(_ string, _ ...interface{}) {}
func (f *fakeLogger) Warn(_ ...interface{}) {}
func (f *fakeLogger) Warnf(_ string, _ ...interface{}) {}
func (f *fakeLogger) Error(_ ...interface{}) {}
func (f *fakeLogger) Errorf(_ string, _ ...interface{}) {}