CI: move grabpl build-plugins from grabpl to grafana (#53071)

* add grabpl command to build internal plugins * grabpl build-plugins -> ./bin/build build-plugins
2022-08-04 11:39:36 -05:00
parent bf3fa4a445
commit 138f03aad1
11 changed files with 609 additions and 12 deletions
@@ -0,0 +1,66 @@
+package plugins
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os/exec"
+	"path/filepath"
+
+	"github.com/grafana/grafana/pkg/build/config"
+	"github.com/grafana/grafana/pkg/build/errutil"
+	"github.com/grafana/grafana/pkg/build/syncutil"
+	"github.com/grafana/grafana/pkg/infra/fs"
+)
+
+type PluginSigningMode = int
+
+// BuildPlugins builds internal plugins.
+// The built plugins are placed in plugins-bundled/dist/.
+func Build(ctx context.Context, grafanaDir string, p syncutil.WorkerPool, g *errutil.Group, verMode *config.Version) error {
+	log.Printf("Building plugins in %q...", grafanaDir)
+
+	root := filepath.Join(grafanaDir, "plugins-bundled", "internal")
+	fis, err := ioutil.ReadDir(root)
+	if err != nil {
+		return err
+	}
+
+	for i := range fis {
+		fi := fis[i]
+		if !fi.IsDir() {
+			continue
+		}
+
+		dpath := filepath.Join(root, fi.Name())
+
+		p.Schedule(g.Wrap(func() error {
+			log.Printf("Building plugin %q...", dpath)
+
+			cmd := exec.Command("yarn", "build")
+			cmd.Dir = dpath
+			if output, err := cmd.CombinedOutput(); err != nil {
+				return fmt.Errorf("yarn build failed: %s", output)
+			}
+
+			dstPath := filepath.Join("plugins-bundled", "dist", fi.Name())
+			if err := fs.CopyRecursive(filepath.Join(dpath, "dist"), dstPath); err != nil {
+				return err
+			}
+			if !verMode.PluginSignature.Sign {
+				return nil
+			}
+
+			return BuildManifest(ctx, dstPath, verMode.PluginSignature.AdminSign)
+		}))
+	}
+
+	if err := g.Wait(); err != nil {
+		return err
+	}
+
+	log.Printf("Built all plug-ins successfully!")
+
+	return nil
+}
@@ -0,0 +1,118 @@
+package plugins
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/grafana/grafana/pkg/build/errutil"
+	"github.com/grafana/grafana/pkg/build/syncutil"
+)
+
+// logCloseError executes the closeFunc; if it returns an error, it is logged by the log package.
+func logCloseError(closeFunc func() error) {
+	if err := closeFunc(); err != nil {
+		log.Println(err)
+	}
+}
+
+// logCloseError executes the closeFunc; if it returns an error, it is logged by the log package.
+func logError(err error) {
+	if err != nil {
+		log.Println(err)
+	}
+}
+
+// pluginManifest has details of an external plugin package.
+type pluginManifest struct {
+	Name     string `json:"name"`
+	Version  string `json:"version"`
+	Checksum string `json:"checksum"`
+}
+
+// pluginsManifest represents a manifest of Grafana's external plugins.
+type pluginsManifest struct {
+	Plugins []pluginManifest `json:"plugins"`
+}
+
+// downloadPlugins downloads Grafana plugins that should be bundled into packages.
+//
+// The plugin archives are downloaded into <grafanaDir>/plugins-bundled.
+func Download(ctx context.Context, grafanaDir string, p syncutil.WorkerPool) error {
+	g, _ := errutil.GroupWithContext(ctx)
+
+	log.Println("Downloading external plugins...")
+
+	var m pluginsManifest
+	manifestPath := filepath.Join(grafanaDir, "plugins-bundled", "external.json")
+	//nolint:gosec
+	manifestB, err := ioutil.ReadFile(manifestPath)
+	if err != nil {
+		return fmt.Errorf("failed to open plugins manifest %q: %w", manifestPath, err)
+	}
+	if err := json.Unmarshal(manifestB, &m); err != nil {
+		return err
+	}
+
+	for i := range m.Plugins {
+		pm := m.Plugins[i]
+		p.Schedule(g.Wrap(func() error {
+			tgt := filepath.Join(grafanaDir, "plugins-bundled", fmt.Sprintf("%s-%s.zip", pm.Name, pm.Version))
+			out, err := os.Create(tgt)
+			if err != nil {
+				return err
+			}
+			defer logCloseError(out.Close)
+
+			u := fmt.Sprintf("http://storage.googleapis.com/plugins-ci/plugins/%s/%s-%s.zip", pm.Name, pm.Name,
+				pm.Version)
+			log.Printf("Downloading plugin %q to %q...", u, tgt)
+			// nolint:gosec
+			resp, err := http.Get(u)
+			if err != nil {
+				return fmt.Errorf("downloading %q failed: %w", u, err)
+			}
+			defer logError(resp.Body.Close())
+
+			if resp.StatusCode != http.StatusOK {
+				return fmt.Errorf("failed to download %q, status code %d", u, resp.StatusCode)
+			}
+
+			if _, err := io.Copy(out, resp.Body); err != nil {
+				return fmt.Errorf("downloading %q failed: %w", u, err)
+			}
+			if err := out.Close(); err != nil {
+				return fmt.Errorf("downloading %q failed: %w", u, err)
+			}
+
+			//nolint:gosec
+			fd, err := os.Open(tgt)
+			if err != nil {
+				return err
+			}
+			defer logCloseError(fd.Close)
+
+			h := sha256.New()
+			if _, err := io.Copy(h, fd); err != nil {
+				return err
+			}
+
+			chksum := hex.EncodeToString(h.Sum(nil))
+			if chksum != pm.Checksum {
+				return fmt.Errorf("plugin %q has bad checksum: %s (expected %s)", u, chksum, pm.Checksum)
+			}
+
+			return Unzip(tgt, filepath.Join(grafanaDir, "plugins-bundled"))
+		}))
+	}
+
+	return g.Wait()
+}
@@ -0,0 +1,196 @@
+package plugins
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+)
+
+type manifest struct {
+	Plugin  string            `json:"plugin"`
+	Version string            `json:"version"`
+	Files   map[string]string `json:"files"`
+}
+
+func getManifest(dpath string, chksums map[string]string) (manifest, error) {
+	m := manifest{}
+
+	type pluginInfo struct {
+		Version string `json:"version"`
+	}
+
+	type plugin struct {
+		ID   string     `json:"id"`
+		Info pluginInfo `json:"info"`
+	}
+
+	//nolint:gosec
+	f, err := os.Open(filepath.Join(dpath, "plugin.json"))
+	if err != nil {
+		return m, err
+	}
+	decoder := json.NewDecoder(f)
+	var p plugin
+	if err := decoder.Decode(&p); err != nil {
+		return m, err
+	}
+
+	if p.ID == "" {
+		return m, fmt.Errorf("plugin.json doesn't define id")
+	}
+	if p.Info.Version == "" {
+		return m, fmt.Errorf("plugin.json doesn't define info.version")
+	}
+
+	return manifest{
+		Plugin:  p.ID,
+		Version: p.Info.Version,
+		Files:   chksums,
+	}, nil
+}
+
+// BuildManifest requests a plugin's signed manifest file fromt he Grafana API.
+// If signingAdmin is true, the manifest signing admin endpoint (without plugin ID) will be used, and requires
+// an admin API key.
+func BuildManifest(ctx context.Context, dpath string, signingAdmin bool) error {
+	log.Printf("Building manifest for plug-in at %q", dpath)
+
+	apiKey := os.Getenv("GRAFANA_API_KEY")
+	if apiKey == "" {
+		return fmt.Errorf("GRAFANA_API_KEY must be set")
+	}
+
+	manifestPath := filepath.Join(dpath, "MANIFEST.txt")
+	chksums, err := getChksums(dpath, manifestPath)
+	if err != nil {
+		return err
+	}
+	m, err := getManifest(dpath, chksums)
+	if err != nil {
+		return err
+	}
+
+	b := bytes.NewBuffer(nil)
+	encoder := json.NewEncoder(b)
+	if err := encoder.Encode(&m); err != nil {
+		return err
+	}
+	jsonB := b.Bytes()
+	u := "https://grafana.com/api/plugins/ci/sign"
+	if !signingAdmin {
+		u = fmt.Sprintf("https://grafana.com/api/plugins/%s/ci/sign", m.Plugin)
+	}
+	log.Printf("Requesting signed manifest from Grafana API...")
+	req, err := http.NewRequestWithContext(ctx, "POST", u, bytes.NewReader(jsonB))
+	if err != nil {
+		return err
+	}
+	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", apiKey))
+	req.Header.Add("Content-Type", "application/json")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to get signed manifest from Grafana API: %w", err)
+	}
+	defer logError(resp.Body.Close())
+	if resp.StatusCode != 200 {
+		msg, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			log.Printf("Failed to read response body: %s", err)
+			msg = []byte("")
+		}
+		return fmt.Errorf("request for signed manifest failed with status code %d: %s", resp.StatusCode, string(msg))
+	}
+
+	log.Printf("Successfully signed manifest via Grafana API, writing to %q", manifestPath)
+	f, err := os.Create(manifestPath)
+	if err != nil {
+		return fmt.Errorf("failed to create %s: %w", manifestPath, err)
+	}
+	defer logCloseError(f.Close)
+	if _, err := io.Copy(f, resp.Body); err != nil {
+		return fmt.Errorf("failed to write %s: %w", manifestPath, err)
+	}
+	if err := f.Close(); err != nil {
+		return fmt.Errorf("failed to write %s: %w", manifestPath, err)
+	}
+
+	return nil
+}
+
+func getChksums(dpath, manifestPath string) (map[string]string, error) {
+	manifestPath = filepath.Clean(manifestPath)
+
+	chksums := map[string]string{}
+	if err := filepath.Walk(dpath, func(path string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if fi.IsDir() {
+			return nil
+		}
+
+		path = filepath.Clean(path)
+
+		// Handle symbolic links
+		if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
+			finalPath, err := filepath.EvalSymlinks(path)
+			if err != nil {
+				return err
+			}
+
+			log.Printf("Handling symlink %q, pointing to %q", path, finalPath)
+
+			info, err := os.Stat(finalPath)
+			if err != nil {
+				return err
+			}
+			if info.IsDir() {
+				return nil
+			}
+
+			if _, err := filepath.Rel(dpath, finalPath); err != nil {
+				return fmt.Errorf("symbolic link %q targets a file outside of the plugin directory: %q", path, finalPath)
+			}
+
+			if finalPath == manifestPath {
+				return nil
+			}
+		}
+
+		if path == manifestPath {
+			return nil
+		}
+
+		h := sha256.New()
+		//nolint:gosec
+		f, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer logCloseError(f.Close)
+		if _, err := io.Copy(h, f); err != nil {
+			return err
+		}
+
+		relPath, err := filepath.Rel(dpath, path)
+		if err != nil {
+			return err
+		}
+		chksums[relPath] = fmt.Sprintf("%x", h.Sum(nil))
+
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return chksums, nil
+}
@@ -0,0 +1,64 @@
+package plugins
+
+import (
+	"archive/zip"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+)
+
+// Unzip unzips a plugin.
+func Unzip(fpath, tgtDir string) error {
+	log.Printf("Unzipping plugin %q into %q...", fpath, tgtDir)
+
+	r, err := zip.OpenReader(fpath)
+	if err != nil {
+		return err
+	}
+	defer logCloseError(r.Close)
+
+	// Closure to address file descriptors issue with all the deferred .Close() methods
+	extractAndWriteFile := func(f *zip.File) error {
+		log.Printf("Extracting zip member %q...", f.Name)
+
+		rc, err := f.Open()
+		if err != nil {
+			return err
+		}
+		defer logCloseError(rc.Close)
+
+		//nolint:gosec
+		dstPath := filepath.Join(tgtDir, f.Name)
+
+		if f.FileInfo().IsDir() {
+			return os.MkdirAll(dstPath, f.Mode())
+		}
+
+		if err := os.MkdirAll(filepath.Dir(dstPath), f.Mode()); err != nil {
+			return err
+		}
+
+		//nolint:gosec
+		fd, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
+		if err != nil {
+			return err
+		}
+		defer logCloseError(fd.Close)
+
+		// nolint:gosec
+		if _, err := io.Copy(fd, rc); err != nil {
+			return err
+		}
+
+		return fd.Close()
+	}
+
+	for _, f := range r.File {
+		if err := extractAndWriteFile(f); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}