grafana/toolkit: remove aws-sdk and upload to grafana.com API endpoint (#20372)

* remove aws-sdk and upload directly

* remove unused imports

* put the plugin file in the root directory

packages/grafana-toolkit/package.json

@@ -42,7 +42,6 @@
     "@types/semver": "^6.0.0",
     "@types/tmp": "^0.1.0",
     "@types/webpack": "4.4.34",
-    "aws-sdk": "^2.495.0",
     "axios": "0.19.0",
     "babel-jest": "24.8.0",
     "babel-loader": "8.0.6",

packages/grafana-toolkit/src/cli/tasks/plugin.ci.ts

@@ -1,7 +1,6 @@
 import { Task, TaskRunner } from './task';
 import { pluginBuildRunner } from './plugin.build';
 import { restoreCwd } from '../utils/cwd';
-import { S3Client } from '../../plugins/aws';
 import { getPluginJson } from '../../config/utils/pluginValidation';
 import { getPluginId } from '../../config/utils/getPluginId';
 import { PluginMeta } from '@grafana/data';
@@ -10,28 +9,18 @@ import { PluginMeta } from '@grafana/data';
 import execa = require('execa');
 import path = require('path');
 import fs from 'fs';
-import { getPackageDetails, findImagesInFolder, appendPluginHistory, getGrafanaVersions } from '../../plugins/utils';
+import { getPackageDetails, findImagesInFolder, getGrafanaVersions } from '../../plugins/utils';
 import {
   job,
   getJobFolder,
   writeJobStats,
   getCiFolder,
   getPluginBuildInfo,
-  getBuildNumber,
   getPullRequestNumber,
   getCircleDownloadBaseURL,
 } from '../../plugins/env';
 import { agregateWorkflowInfo, agregateCoverageInfo, agregateTestInfo } from '../../plugins/workflow';
-import {
-  PluginPackageDetails,
-  PluginBuildReport,
-  PluginHistory,
-  defaultPluginHistory,
-  TestResultsInfo,
-  PluginDevInfo,
-  PluginDevSummary,
-  DevSummary,
-} from '../../plugins/types';
+import { PluginPackageDetails, PluginBuildReport, TestResultsInfo } from '../../plugins/types';
 import { runEndToEndTests } from '../../plugins/e2e/launcher';
 import { getEndToEndSettings } from '../../plugins/index';
 
@@ -185,6 +174,9 @@ const packagePluginRunner: TaskRunner<PluginCIOptions> = async () => {
     throw new Error('Invalid zip file: ' + zipFile);
   }
 
+  // Make a copy so it is easy for report to read
+  await execa('cp', [pluginJsonFile, distDir]);
+
   const info: PluginPackageDetails = {
     plugin: await getPackageDetails(zipFile, distDir),
   };
@@ -346,88 +338,19 @@ const pluginReportRunner: TaskRunner<PluginCIOptions> = async ({ upload }) => {
     }
   });
 
-  console.log('Initalizing S3 Client');
-  const s3 = new S3Client();
-
-  const build = pluginMeta.info.build;
-  if (!build) {
-    throw new Error('Metadata missing build info');
+  const GRAFANA_API_KEY = process.env.GRAFANA_API_KEY;
+  if (!GRAFANA_API_KEY) {
+    console.log('Enter a GRAFANA_API_KEY to upload the plugin report');
+    return;
   }
+  const url = `https://grafana.com/api/plugins/${report.plugin.id}/ci`;
 
-  const version = pluginMeta.info.version || 'unknown';
-  const branch = build.branch || 'unknown';
-  const buildNumber = getBuildNumber();
-  const root = `dev/${pluginMeta.id}`;
-  const dirKey = pr ? `${root}/pr/${pr}/${buildNumber}` : `${root}/branch/${branch}/${buildNumber}`;
-
-  const jobKey = `${dirKey}/index.json`;
-  if (await s3.exists(jobKey)) {
-    throw new Error('Job already registered: ' + jobKey);
-  }
-
-  console.log('Write Job', jobKey);
-  await s3.writeJSON(jobKey, report, {
-    Tagging: `version=${version}&type=${pluginMeta.type}`,
+  console.log('Sending report to:', url);
+  const axios = require('axios');
+  const info = await axios.post(url, report, {
+    headers: { Authorization: 'bearer ' + GRAFANA_API_KEY },
   });
-
-  // Upload logo
-  const logo = await s3.uploadLogo(report.plugin.info, {
-    local: path.resolve(ciDir, 'dist'),
-    remote: root,
-  });
-
-  const latest: PluginDevInfo = {
-    pluginId: pluginMeta.id,
-    name: pluginMeta.name,
-    logo,
-    build: pluginMeta.info.build!,
-    version,
-  };
-
-  let base = `${root}/branch/${branch}/`;
-  latest.build.number = buildNumber;
-  if (pr) {
-    latest.build.pr = pr;
-    base = `${root}/pr/${pr}/`;
-  }
-
-  const historyKey = base + `history.json`;
-  console.log('Read', historyKey);
-  const history: PluginHistory = await s3.readJSON(historyKey, defaultPluginHistory);
-  appendPluginHistory(report, latest, history);
-
-  await s3.writeJSON(historyKey, history);
-  console.log('wrote history');
-
-  // Private things may want to upload
-  if (upload) {
-    s3.uploadPackages(packageInfo, {
-      local: packageDir,
-      remote: dirKey + '/packages',
-    });
-
-    s3.uploadTestFiles(report.tests, {
-      local: ciDir,
-      remote: dirKey,
-    });
-  }
-
-  console.log('Update Directory Indexes');
-
-  let indexKey = `${root}/index.json`;
-  const index: PluginDevSummary = await s3.readJSON(indexKey, { branch: {}, pr: {} });
-  if (pr) {
-    index.pr[pr] = latest;
-  } else {
-    index.branch[branch] = latest;
-  }
-  await s3.writeJSON(indexKey, index);
-
-  indexKey = `dev/index.json`;
-  const pluginIndex: DevSummary = await s3.readJSON(indexKey, {});
-  pluginIndex[pluginMeta.id] = latest;
-  await s3.writeJSON(indexKey, pluginIndex);
-  console.log('wrote index');
+  console.log('RESULT: ', info);
 };
 
 export const ciPluginReportTask = new Task<PluginCIOptions>('Generate Plugin Report', pluginReportRunner);

packages/grafana-toolkit/src/plugins/aws.ts

@@ -1,183 +0,0 @@
-import AWS from 'aws-sdk';
-import path from 'path';
-import fs from 'fs';
-
-import { PluginPackageDetails, ZipFileInfo, TestResultsInfo } from './types';
-import defaults from 'lodash/defaults';
-import clone from 'lodash/clone';
-import { PluginMetaInfo } from '@grafana/data';
-
-interface UploadArgs {
-  local: string;
-  remote: string;
-}
-
-export class S3Client {
-  readonly bucket: string;
-  readonly prefix: string;
-  readonly s3: AWS.S3;
-
-  constructor(bucket?: string) {
-    this.bucket = bucket || 'grafana-experiments';
-    this.prefix = 'plugins/';
-
-    this.s3 = new AWS.S3({ apiVersion: '2006-03-01' });
-    this.s3.headBucket({ Bucket: this.bucket }, (err, data) => {
-      if (err) {
-        throw new Error('Unable to read: ' + this.bucket);
-      } else {
-        console.log('s3: ' + data);
-      }
-    });
-  }
-
-  private async uploadPackage(file: ZipFileInfo, folder: UploadArgs): Promise<string> {
-    const fpath = path.resolve(process.cwd(), folder.local, file.name);
-    return await this.uploadFile(fpath, folder.remote + '/' + file.name, file.md5);
-  }
-
-  async uploadPackages(packageInfo: PluginPackageDetails, folder: UploadArgs) {
-    await this.uploadPackage(packageInfo.plugin, folder);
-    if (packageInfo.docs) {
-      await this.uploadPackage(packageInfo.docs, folder);
-    }
-  }
-
-  async uploadTestFiles(tests: TestResultsInfo[], folder: UploadArgs) {
-    for (const test of tests) {
-      for (const s of test.screenshots) {
-        const img = path.resolve(folder.local, 'jobs', test.job, s);
-        await this.uploadFile(img, folder.remote + `/jobs/${test.job}/${s}`);
-      }
-    }
-  }
-
-  async uploadLogo(meta: PluginMetaInfo, folder: UploadArgs): Promise<string | undefined> {
-    const { logos } = meta;
-    if (logos && logos.large) {
-      const img = folder.local + '/' + logos.large;
-      const idx = img.lastIndexOf('.');
-      const name = 'logo' + img.substring(idx);
-      const key = folder.remote + '/' + name;
-      await this.uploadFile(img, key);
-      return name;
-    }
-    return undefined;
-  }
-
-  async uploadFile(fpath: string, path: string, md5?: string): Promise<string> {
-    if (!fs.existsSync(fpath)) {
-      return Promise.reject('File not found: ' + fpath);
-    }
-    console.log('Uploading: ' + fpath);
-    const stream = fs.createReadStream(fpath);
-    return new Promise((resolve, reject) => {
-      this.s3.putObject(
-        {
-          Key: this.prefix + path,
-          Bucket: this.bucket,
-          Body: stream,
-          ContentType: getContentTypeForFile(path),
-        },
-        (err, data) => {
-          if (err) {
-            reject(err);
-          } else {
-            if (md5 && md5 !== data.ETag && `"${md5}"` !== data.ETag) {
-              reject(`Upload ETag does not match MD5 (${md5} !== ${data.ETag})`);
-            } else {
-              resolve(data.ETag);
-            }
-          }
-        }
-      );
-    });
-  }
-
-  async exists(key: string): Promise<boolean> {
-    return new Promise((resolve, reject) => {
-      this.s3.getObject(
-        {
-          Bucket: this.bucket,
-          Key: this.prefix + key,
-        },
-        (err, data) => {
-          if (err) {
-            resolve(false);
-          } else {
-            resolve(true);
-          }
-        }
-      );
-    });
-  }
-
-  async readJSON<T>(key: string, defaultValue: T): Promise<T> {
-    return new Promise((resolve, reject) => {
-      this.s3.getObject(
-        {
-          Bucket: this.bucket,
-          Key: this.prefix + key,
-        },
-        (err, data) => {
-          if (err) {
-            resolve(clone(defaultValue));
-          } else {
-            try {
-              const v = JSON.parse(data.Body as string);
-              resolve(defaults(v, defaultValue));
-            } catch (e) {
-              console.log('ERROR', e);
-              reject('Error reading response');
-            }
-          }
-        }
-      );
-    });
-  }
-
-  async writeJSON(
-    key: string,
-    obj: {},
-    params?: Partial<AWS.S3.Types.PutObjectRequest>
-  ): Promise<AWS.S3.Types.PutObjectOutput> {
-    return new Promise((resolve, reject) => {
-      this.s3.putObject(
-        {
-          ...params,
-          Key: this.prefix + key,
-          Bucket: this.bucket,
-          Body: JSON.stringify(obj, null, 2), // Pretty print
-          ContentType: 'application/json',
-        },
-        (err, data) => {
-          if (err) {
-            reject(err);
-          } else {
-            resolve(data);
-          }
-        }
-      );
-    });
-  }
-}
-
-function getContentTypeForFile(name: string): string | undefined {
-  const idx = name.lastIndexOf('.');
-  if (idx > 0) {
-    const ext = name.substring(idx + 1).toLowerCase();
-    if (ext === 'zip') {
-      return 'application/zip';
-    }
-    if (ext === 'json') {
-      return 'application/json';
-    }
-    if (ext === 'svg') {
-      return 'image/svg+xml';
-    }
-    if (ext === 'png') {
-      return 'image/png';
-    }
-  }
-  return undefined;
-}

packages/grafana-toolkit/src/plugins/index.ts

@@ -1,4 +1,3 @@
-export * from './aws';
 export * from './env';
 export * from './utils';
 export * from './workflow';