grafana/toolkit: remove aws-sdk and upload to grafana.com API endpoint (#20372)

* remove aws-sdk and upload directly

* remove unused imports

* put the plugin file in the root directory

packages/grafana-toolkit/package.json

@@ -42,7 +42,6 @@
     "@types/semver": "^6.0.0",
     "@types/tmp": "^0.1.0",
     "@types/webpack": "4.4.34",
-    "aws-sdk": "^2.495.0",
     "axios": "0.19.0",
     "babel-jest": "24.8.0",
     "babel-loader": "8.0.6",

packages/grafana-toolkit/src/cli/tasks/plugin.ci.ts

@@ -1,7 +1,6 @@
 import { Task, TaskRunner } from './task';
 import { pluginBuildRunner } from './plugin.build';
 import { restoreCwd } from '../utils/cwd';
-import { S3Client } from '../../plugins/aws';
 import { getPluginJson } from '../../config/utils/pluginValidation';
 import { getPluginId } from '../../config/utils/getPluginId';
 import { PluginMeta } from '@grafana/data';
@@ -10,28 +9,18 @@ import { PluginMeta } from '@grafana/data';
 import execa = require('execa');
 import path = require('path');
 import fs from 'fs';
-import { getPackageDetails, findImagesInFolder, appendPluginHistory, getGrafanaVersions } from '../../plugins/utils';
+import { getPackageDetails, findImagesInFolder, getGrafanaVersions } from '../../plugins/utils';
 import {
   job,
   getJobFolder,
   writeJobStats,
   getCiFolder,
   getPluginBuildInfo,
-  getBuildNumber,
   getPullRequestNumber,
   getCircleDownloadBaseURL,
 } from '../../plugins/env';
 import { agregateWorkflowInfo, agregateCoverageInfo, agregateTestInfo } from '../../plugins/workflow';
-import {
+import { PluginPackageDetails, PluginBuildReport, TestResultsInfo } from '../../plugins/types';
-  PluginPackageDetails,
-  PluginBuildReport,
-  PluginHistory,
-  defaultPluginHistory,
-  TestResultsInfo,
-  PluginDevInfo,
-  PluginDevSummary,
-  DevSummary,
-} from '../../plugins/types';
 import { runEndToEndTests } from '../../plugins/e2e/launcher';
 import { getEndToEndSettings } from '../../plugins/index';
 
@@ -185,6 +174,9 @@ const packagePluginRunner: TaskRunner<PluginCIOptions> = async () => {
     throw new Error('Invalid zip file: ' + zipFile);
   }
 
+  // Make a copy so it is easy for report to read
+  await execa('cp', [pluginJsonFile, distDir]);
+
   const info: PluginPackageDetails = {
     plugin: await getPackageDetails(zipFile, distDir),
   };
@@ -346,88 +338,19 @@ const pluginReportRunner: TaskRunner<PluginCIOptions> = async ({ upload }) => {
     }
   });
 
-  console.log('Initalizing S3 Client');
+  const GRAFANA_API_KEY = process.env.GRAFANA_API_KEY;
-  const s3 = new S3Client();
+  if (!GRAFANA_API_KEY) {
-
+    console.log('Enter a GRAFANA_API_KEY to upload the plugin report');
-  const build = pluginMeta.info.build;
+    return;
-  if (!build) {
-    throw new Error('Metadata missing build info');
   }
+  const url = `https://grafana.com/api/plugins/${report.plugin.id}/ci`;
 
-  const version = pluginMeta.info.version || 'unknown';
+  console.log('Sending report to:', url);
-  const branch = build.branch || 'unknown';
+  const axios = require('axios');
-  const buildNumber = getBuildNumber();
+  const info = await axios.post(url, report, {
-  const root = `dev/${pluginMeta.id}`;
+    headers: { Authorization: 'bearer ' + GRAFANA_API_KEY },
-  const dirKey = pr ? `${root}/pr/${pr}/${buildNumber}` : `${root}/branch/${branch}/${buildNumber}`;
-
-  const jobKey = `${dirKey}/index.json`;
-  if (await s3.exists(jobKey)) {
-    throw new Error('Job already registered: ' + jobKey);
-  }
-
-  console.log('Write Job', jobKey);
-  await s3.writeJSON(jobKey, report, {
-    Tagging: `version=${version}&type=${pluginMeta.type}`,
   });
-
+  console.log('RESULT: ', info);
-  // Upload logo
-  const logo = await s3.uploadLogo(report.plugin.info, {
-    local: path.resolve(ciDir, 'dist'),
-    remote: root,
-  });
-
-  const latest: PluginDevInfo = {
-    pluginId: pluginMeta.id,
-    name: pluginMeta.name,
-    logo,
-    build: pluginMeta.info.build!,
-    version,
-  };
-
-  let base = `${root}/branch/${branch}/`;
-  latest.build.number = buildNumber;
-  if (pr) {
-    latest.build.pr = pr;
-    base = `${root}/pr/${pr}/`;
-  }
-
-  const historyKey = base + `history.json`;
-  console.log('Read', historyKey);
-  const history: PluginHistory = await s3.readJSON(historyKey, defaultPluginHistory);
-  appendPluginHistory(report, latest, history);
-
-  await s3.writeJSON(historyKey, history);
-  console.log('wrote history');
-
-  // Private things may want to upload
-  if (upload) {
-    s3.uploadPackages(packageInfo, {
-      local: packageDir,
-      remote: dirKey + '/packages',
-    });
-
-    s3.uploadTestFiles(report.tests, {
-      local: ciDir,
-      remote: dirKey,
-    });
-  }
-
-  console.log('Update Directory Indexes');
-
-  let indexKey = `${root}/index.json`;
-  const index: PluginDevSummary = await s3.readJSON(indexKey, { branch: {}, pr: {} });
-  if (pr) {
-    index.pr[pr] = latest;
-  } else {
-    index.branch[branch] = latest;
-  }
-  await s3.writeJSON(indexKey, index);
-
-  indexKey = `dev/index.json`;
-  const pluginIndex: DevSummary = await s3.readJSON(indexKey, {});
-  pluginIndex[pluginMeta.id] = latest;
-  await s3.writeJSON(indexKey, pluginIndex);
-  console.log('wrote index');
 };
 
 export const ciPluginReportTask = new Task<PluginCIOptions>('Generate Plugin Report', pluginReportRunner);

packages/grafana-toolkit/src/plugins/aws.ts

@@ -1,183 +0,0 @@
-import AWS from 'aws-sdk';
-import path from 'path';
-import fs from 'fs';
-
-import { PluginPackageDetails, ZipFileInfo, TestResultsInfo } from './types';
-import defaults from 'lodash/defaults';
-import clone from 'lodash/clone';
-import { PluginMetaInfo } from '@grafana/data';
-
-interface UploadArgs {
-  local: string;
-  remote: string;
-}
-
-export class S3Client {
-  readonly bucket: string;
-  readonly prefix: string;
-  readonly s3: AWS.S3;
-
-  constructor(bucket?: string) {
-    this.bucket = bucket || 'grafana-experiments';
-    this.prefix = 'plugins/';
-
-    this.s3 = new AWS.S3({ apiVersion: '2006-03-01' });
-    this.s3.headBucket({ Bucket: this.bucket }, (err, data) => {
-      if (err) {
-        throw new Error('Unable to read: ' + this.bucket);
-      } else {
-        console.log('s3: ' + data);
-      }
-    });
-  }
-
-  private async uploadPackage(file: ZipFileInfo, folder: UploadArgs): Promise<string> {
-    const fpath = path.resolve(process.cwd(), folder.local, file.name);
-    return await this.uploadFile(fpath, folder.remote + '/' + file.name, file.md5);
-  }
-
-  async uploadPackages(packageInfo: PluginPackageDetails, folder: UploadArgs) {
-    await this.uploadPackage(packageInfo.plugin, folder);
-    if (packageInfo.docs) {
-      await this.uploadPackage(packageInfo.docs, folder);
-    }
-  }
-
-  async uploadTestFiles(tests: TestResultsInfo[], folder: UploadArgs) {
-    for (const test of tests) {
-      for (const s of test.screenshots) {
-        const img = path.resolve(folder.local, 'jobs', test.job, s);
-        await this.uploadFile(img, folder.remote + `/jobs/${test.job}/${s}`);
-      }
-    }
-  }
-
-  async uploadLogo(meta: PluginMetaInfo, folder: UploadArgs): Promise<string | undefined> {
-    const { logos } = meta;
-    if (logos && logos.large) {
-      const img = folder.local + '/' + logos.large;
-      const idx = img.lastIndexOf('.');
-      const name = 'logo' + img.substring(idx);
-      const key = folder.remote + '/' + name;
-      await this.uploadFile(img, key);
-      return name;
-    }
-    return undefined;
-  }
-
-  async uploadFile(fpath: string, path: string, md5?: string): Promise<string> {
-    if (!fs.existsSync(fpath)) {
-      return Promise.reject('File not found: ' + fpath);
-    }
-    console.log('Uploading: ' + fpath);
-    const stream = fs.createReadStream(fpath);
-    return new Promise((resolve, reject) => {
-      this.s3.putObject(
-        {
-          Key: this.prefix + path,
-          Bucket: this.bucket,
-          Body: stream,
-          ContentType: getContentTypeForFile(path),
-        },
-        (err, data) => {
-          if (err) {
-            reject(err);
-          } else {
-            if (md5 && md5 !== data.ETag && `"${md5}"` !== data.ETag) {
-              reject(`Upload ETag does not match MD5 (${md5} !== ${data.ETag})`);
-            } else {
-              resolve(data.ETag);
-            }
-          }
-        }
-      );
-    });
-  }
-
-  async exists(key: string): Promise<boolean> {
-    return new Promise((resolve, reject) => {
-      this.s3.getObject(
-        {
-          Bucket: this.bucket,
-          Key: this.prefix + key,
-        },
-        (err, data) => {
-          if (err) {
-            resolve(false);
-          } else {
-            resolve(true);
-          }
-        }
-      );
-    });
-  }
-
-  async readJSON<T>(key: string, defaultValue: T): Promise<T> {
-    return new Promise((resolve, reject) => {
-      this.s3.getObject(
-        {
-          Bucket: this.bucket,
-          Key: this.prefix + key,
-        },
-        (err, data) => {
-          if (err) {
-            resolve(clone(defaultValue));
-          } else {
-            try {
-              const v = JSON.parse(data.Body as string);
-              resolve(defaults(v, defaultValue));
-            } catch (e) {
-              console.log('ERROR', e);
-              reject('Error reading response');
-            }
-          }
-        }
-      );
-    });
-  }
-
-  async writeJSON(
-    key: string,
-    obj: {},
-    params?: Partial<AWS.S3.Types.PutObjectRequest>
-  ): Promise<AWS.S3.Types.PutObjectOutput> {
-    return new Promise((resolve, reject) => {
-      this.s3.putObject(
-        {
-          ...params,
-          Key: this.prefix + key,
-          Bucket: this.bucket,
-          Body: JSON.stringify(obj, null, 2), // Pretty print
-          ContentType: 'application/json',
-        },
-        (err, data) => {
-          if (err) {
-            reject(err);
-          } else {
-            resolve(data);
-          }
-        }
-      );
-    });
-  }
-}
-
-function getContentTypeForFile(name: string): string | undefined {
-  const idx = name.lastIndexOf('.');
-  if (idx > 0) {
-    const ext = name.substring(idx + 1).toLowerCase();
-    if (ext === 'zip') {
-      return 'application/zip';
-    }
-    if (ext === 'json') {
-      return 'application/json';
-    }
-    if (ext === 'svg') {
-      return 'image/svg+xml';
-    }
-    if (ext === 'png') {
-      return 'image/png';
-    }
-  }
-  return undefined;
-}

packages/grafana-toolkit/src/plugins/index.ts

@@ -1,4 +1,3 @@
-export * from './aws';
 export * from './env';
 export * from './utils';
 export * from './workflow';

yarn.lock

@@ -4782,21 +4782,6 @@ autoprefixer@^9.4.9:
     postcss "^7.0.18"
     postcss-value-parser "^4.0.2"
 
-aws-sdk@^2.495.0:
-  version "2.553.0"
-  resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.553.0.tgz#a82c611015138db8f720e0079fe929a65b359a8e"
-  integrity sha512-tcITF/3ijBumvP13Qrq/VB1eRWW6szKF0xVwZ/ch0MGkaEiTQ9n3zNRPtQc1drllsVEm5u5aXp3inoi5zmq0xg==
-  dependencies:
-    buffer "4.9.1"
-    events "1.1.1"
-    ieee754 "1.1.13"
-    jmespath "0.15.0"
-    querystring "0.2.0"
-    sax "1.2.1"
-    url "0.10.3"
-    uuid "3.3.2"
-    xml2js "0.4.19"
-
 aws-sign2@~0.7.0:
   version "0.7.0"
   resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8"
@@ -5638,7 +5623,7 @@ buffer-xor@^1.0.3:
   resolved "https://registry.yarnpkg.com/buffer-xor/-/buffer-xor-1.0.3.tgz#26e61ed1422fb70dd42e6e36729ed51d855fe8d9"
   integrity sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk=
 
-buffer@4.9.1, buffer@^4.3.0:
+buffer@^4.3.0:
   version "4.9.1"
   resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.1.tgz#6d1bb601b07a4efced97094132093027c95bc298"
   integrity sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=
@@ -8886,11 +8871,6 @@ eventemitter3@^4.0.0:
   resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.0.tgz#d65176163887ee59f386d64c82610b696a4a74eb"
   integrity sha512-qerSRB0p+UDEssxTtm6EDKcE7W4OaoisfIMl4CngyEhjpYglocpNg6UEqCvemdGhosAsg4sO2dXJOdyBifPGCg==
 
-events@1.1.1:
-  version "1.1.1"
-  resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924"
-  integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=
-
 events@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/events/-/events-3.0.0.tgz#9a0a0dfaf62893d92b875b8f2698ca4114973e88"
@@ -10943,7 +10923,7 @@ icss-utils@^4.0.0, icss-utils@^4.1.0, icss-utils@^4.1.1:
   dependencies:
     postcss "^7.0.14"
 
-ieee754@1.1.13, ieee754@^1.1.4:
+ieee754@^1.1.4:
   version "1.1.13"
   resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84"
   integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==
@@ -12270,11 +12250,6 @@ jest@24.8.0:
     import-local "^2.0.0"
     jest-cli "^24.8.0"
 
-jmespath@0.15.0:
-  version "0.15.0"
-  resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217"
-  integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=
-
 jquery@3.4.1:
   version "3.4.1"
   resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.4.1.tgz#714f1f8d9dde4bdfa55764ba37ef214630d80ef2"
@@ -18497,12 +18472,7 @@ sass-loader@7.1.0:
     pify "^3.0.0"
     semver "^5.5.0"
 
-sax@1.2.1:
+sax@^1.2.4, sax@~1.2.4:
-  version "1.2.1"
-  resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a"
-  integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o=
-
-sax@>=0.6.0, sax@^1.2.4, sax@~1.2.4:
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
   integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==
@@ -20833,14 +20803,6 @@ url-parse@^1.4.3:
     querystringify "^2.1.1"
     requires-port "^1.0.0"
 
-url@0.10.3:
-  version "0.10.3"
-  resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64"
-  integrity sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=
-  dependencies:
-    punycode "1.3.2"
-    querystring "0.2.0"
-
 url@^0.11.0:
   version "0.11.0"
   resolved "https://registry.yarnpkg.com/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1"
@@ -20928,11 +20890,6 @@ utils-merge@1.0.1:
   resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713"
   integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=
 
-uuid@3.3.2:
-  version "3.3.2"
-  resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131"
-  integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==
-
 uuid@^3.0.1, uuid@^3.1.0, uuid@^3.3.2:
   version "3.3.3"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.3.tgz#4568f0216e78760ee1dbf3a4d2cf53e224112866"
@@ -21590,24 +21547,11 @@ xml-name-validator@^3.0.0:
   resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a"
   integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==
 
-xml2js@0.4.19:
-  version "0.4.19"
-  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7"
-  integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==
-  dependencies:
-    sax ">=0.6.0"
-    xmlbuilder "~9.0.1"
-
 xml@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5"
   integrity sha1-eLpyAgApxbyHuKgaPPzXS0ovweU=
 
-xmlbuilder@~9.0.1:
-  version "9.0.7"
-  resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"
-  integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=
-
 xmlhttprequest@1:
   version "1.8.0"
   resolved "https://registry.yarnpkg.com/xmlhttprequest/-/xmlhttprequest-1.8.0.tgz#67fe075c5c24fef39f9d65f5f7b7fe75171968fc"