API clients: Extract into a package (#111810)

Co-authored-by: Tom Ratcliffe <tom.ratcliffe@grafana.com>
Co-authored-by: Alex Khomenko <Clarity-89@users.noreply.github.com>
This commit is contained in:
Tom Ratcliffe
2025-10-23 13:57:51 +00:00
committed by GitHub
co-authored by Alex Khomenko
parent 91b7ff2ece
commit 811ee99dac
103 changed files with 1335 additions and 727 deletions
-116
View File
@@ -1,116 +0,0 @@
// Generates Redux Toolkit API slices for certain APIs from the OpenAPI spec
import type { ConfigFile } from '@rtk-query/codegen-openapi';
const config: ConfigFile = {
schemaFile: '', // leave this empty, and instead populate the outputFiles object below
apiFile: '', // leave this empty, and instead populate the outputFiles object below
exportName: 'generatedAPI',
outputFiles: {
'../public/app/features/migrate-to-cloud/api/endpoints.gen.ts': {
schemaFile: '../public/openapi3.json',
apiFile: '../public/app/features/migrate-to-cloud/api/baseAPI.ts',
apiImport: 'baseAPI',
hooks: true,
filterEndpoints: [
'getSessionList',
'getSession',
'deleteSession',
'createSession',
'getShapshotList',
'getSnapshot',
'uploadSnapshot',
'createSnapshot',
'cancelSnapshot',
'createCloudMigrationToken',
'deleteCloudMigrationToken',
'getCloudMigrationToken',
'getDashboardByUid',
'getLibraryElementByUid',
'getResourceDependencies',
],
},
'../public/app/features/preferences/api/user/endpoints.gen.ts': {
schemaFile: '../public/openapi3.json',
hooks: true,
apiFile: '../public/app/features/preferences/api/user/baseAPI.ts',
apiImport: 'baseAPI',
filterEndpoints: ['getUserPreferences', 'updateUserPreferences', 'patchUserPreferences'],
},
'../public/app/api/clients/iam/v0alpha1/endpoints.gen.ts': {
schemaFile: '../data/openapi/iam.grafana.app-v0alpha1.json',
apiFile: '../public/app/api/clients/iam/v0alpha1/baseAPI.ts',
filterEndpoints: ['getDisplayMapping'],
tag: true,
},
'../public/app/api/clients/provisioning/v0alpha1/endpoints.gen.ts': {
apiFile: '../public/app/api/clients/provisioning/v0alpha1/baseAPI.ts',
schemaFile: '../data/openapi/provisioning.grafana.app-v0alpha1.json',
filterEndpoints,
tag: true,
hooks: true,
},
'../public/app/api/clients/folder/v1beta1/endpoints.gen.ts': {
apiFile: '../public/app/api/clients/folder/v1beta1/baseAPI.ts',
schemaFile: '../data/openapi/folder.grafana.app-v1beta1.json',
tag: true,
},
'../public/app/api/clients/advisor/v0alpha1/endpoints.gen.ts': {
apiFile: '../public/app/api/clients/advisor/v0alpha1/baseAPI.ts',
schemaFile: '../data/openapi/advisor.grafana.app-v0alpha1.json',
filterEndpoints: [
'createCheck',
'getCheck',
'listCheck',
'deleteCheck',
'updateCheck',
'listCheckType',
'updateCheckType',
],
tag: true,
},
'../public/app/api/clients/playlist/v0alpha1/endpoints.gen.ts': {
apiFile: '../public/app/api/clients/playlist/v0alpha1/baseAPI.ts',
schemaFile: '../data/openapi/playlist.grafana.app-v0alpha1.json',
filterEndpoints: ['listPlaylist', 'getPlaylist', 'createPlaylist', 'deletePlaylist', 'replacePlaylist'],
tag: true,
},
'../public/app/api/clients/dashboard/v0alpha1/endpoints.gen.ts': {
apiFile: '../public/app/api/clients/dashboard/v0alpha1/baseAPI.ts',
schemaFile: '../data/openapi/dashboard.grafana.app-v0alpha1.json',
filterEndpoints: [
// Do not use any other endpoints from this version
// If other endpoints are required, they must be used from a newer version of the dashboard API
'getSearch',
],
tag: true,
},
'../public/app/api/clients/shorturl/v1alpha1/endpoints.gen.ts': {
apiFile: '../public/app/api/clients/shorturl/v1alpha1/baseAPI.ts',
schemaFile: '../data/openapi/shorturl.grafana.app-v1alpha1.json',
tag: true,
},
'../public/app/api/clients/correlations/v0alpha1/endpoints.gen.ts': {
apiFile: '../public/app/api/clients/correlations/v0alpha1/baseAPI.ts',
schemaFile: '../data/openapi/correlations.grafana.app-v0alpha1.json',
tag: true,
},
'../public/app/api/clients/preferences/v1alpha1/endpoints.gen.ts': {
apiFile: '../public/app/api/clients/preferences/v1alpha1/baseAPI.ts',
schemaFile: '../data/openapi/preferences.grafana.app-v1alpha1.json',
tag: true,
hooks: true,
},
// PLOP_INJECT_API_CLIENT - Used by the API client generator
},
};
function filterEndpoints(name: string) {
return !name.toLowerCase().includes('getapiresources') && !name.toLowerCase().includes('update');
}
export default config;
-183
View File
@@ -1,183 +0,0 @@
import fs from 'fs';
import { OpenAPIV3 } from 'openapi-types';
import path from 'path';
/**
* Process an OpenAPI spec to remove k8s metadata from names and paths:
* - Remove paths containing "/watch/" as they're deprecated.
* - Remove 'ForAllNamespaces' endpoints
* - Remove the prefix: "/apis/<group>/<version>/namespaces/{namespace}" from paths.
* - Filter out `namespace` from path parameters.
* - Update all $ref fields to remove k8s metadata from schema names.
* - Simplify schema names in "components.schemas".
*/
function processOpenAPISpec(spec: OpenAPIV3.Document) {
// Create a deep copy of the spec to avoid mutating the original
const newSpec = JSON.parse(JSON.stringify(spec));
// Process 'paths' property
const newPaths: Record<string, unknown> = {};
for (const [path, pathItem] of Object.entries<OpenAPIV3.PathItemObject>(newSpec.paths)) {
// Remove empty path items
if (!pathItem) {
continue;
}
// Remove the specified part from the path key
const newPathKey = path.replace(/^\/apis\/[^\/]+\/[^\/]+\/namespaces\/\{namespace}/, '');
// Process each method in the path (e.g., get, post)
const newPathItem: Record<string, unknown> = {};
// Filter out namespace parameter at path level
if (Array.isArray(pathItem.parameters)) {
pathItem.parameters = filterNamespaceParameters(pathItem.parameters);
}
for (const method of Object.keys(pathItem)) {
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
const operation = pathItem[method as keyof OpenAPIV3.PathItemObject];
if (
typeof operation === 'object' &&
operation !== null &&
'operationId' in operation &&
operation.operationId?.includes('ForAllNamespaces')
) {
continue;
}
// Filter out namespace parameter at operation level
if (
operation &&
typeof operation === 'object' &&
'parameters' in operation &&
Array.isArray(operation.parameters)
) {
operation.parameters = filterNamespaceParameters(operation.parameters);
}
updateRefs(operation);
newPathItem[method] = operation;
}
newPaths[newPathKey] = newPathItem;
}
newSpec.paths = newPaths;
// Process 'components.schemas', i.e., type definitions
const newSchemas: Record<string, unknown> = {};
for (const schemaKey of Object.keys(newSpec.components.schemas)) {
const newKey = simplifySchemaName(schemaKey);
if (newSchemas[newKey]) {
// This can happen when invalid specs are used, although ignoring the error will work
// it is better to fix the spec to avoid confusion.
throw new Error(`Duplicate schema key found: ${newKey}. from: ${schemaKey}`);
}
const schemaObject = newSpec.components.schemas[schemaKey];
updateRefs(schemaObject);
newSchemas[newKey] = schemaObject;
}
newSpec.components.schemas = newSchemas;
return newSpec;
}
/**
* Filter out namespace parameters from an array of parameters
*/
function filterNamespaceParameters(parameters: Array<OpenAPIV3.ReferenceObject | OpenAPIV3.ParameterObject>) {
return parameters.filter((param) => 'name' in param && param.name !== 'namespace');
}
/**
* Recursively update all $ref fields to remove k8s metadata from names
*/
function updateRefs(obj: unknown) {
if (Array.isArray(obj)) {
for (const item of obj) {
updateRefs(item);
}
} else if (typeof obj === 'object' && obj !== null) {
if ('$ref' in obj && typeof obj.$ref === 'string') {
const refParts = obj.$ref.split('/');
const lastRefPart = refParts[refParts.length - 1];
const newRefName = simplifySchemaName(lastRefPart);
obj.$ref = `#/components/schemas/${newRefName}`;
}
for (const key in obj) {
if (key !== '$ref') {
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
updateRefs(obj[key as keyof typeof obj]);
}
}
}
}
/**
* Simplify a schema name by removing the version prefix if present.
* For example, 'io.k8s.apimachinery.pkg.apis.meta.v1.Time' becomes 'Time'.
*/
function simplifySchemaName(schemaName: string) {
const parts = schemaName.split('.');
// Regex to match version segments like 'v1', 'v1beta1', 'v0alpha1', etc.
const versionRegex = /^v\d+[a-zA-Z0-9]*$/;
const versionIndex = parts.findIndex((part) => versionRegex.test(part));
if (versionIndex !== -1 && versionIndex + 1 < parts.length) {
return parts.slice(versionIndex + 1).join('.');
} else {
return schemaName;
}
}
/**
* Process all files in a source directory and write results to output directory
*/
function processDirectory(sourceDir: string, outputDir: string) {
// Skip if source directory doesn't exist
if (!fs.existsSync(sourceDir)) {
return;
}
// Create the output directory if it doesn't exist
if (!fs.existsSync(outputDir)) {
fs.mkdirSync(outputDir, { recursive: true });
}
const files = fs.readdirSync(sourceDir).filter((file: string) => file.endsWith('.json'));
for (const file of files) {
const inputPath = path.join(sourceDir, file);
const outputPath = path.join(outputDir, file);
console.log(`Processing file "${file}"...`);
const fileContent = fs.readFileSync(inputPath, 'utf-8');
let inputSpec;
try {
inputSpec = JSON.parse(fileContent);
} catch (err) {
console.error(`Invalid JSON file "${file}". Skipping this file.`);
continue;
}
const outputSpec = processOpenAPISpec(inputSpec);
fs.writeFileSync(outputPath, JSON.stringify(outputSpec, null, 2), 'utf-8');
console.log(`Processing completed for file "${file}".`);
}
}
const sourceDirs = [
path.resolve(import.meta.dirname, '../pkg/tests/apis/openapi_snapshots'),
path.resolve(import.meta.dirname, '../pkg/extensions/apiserver/tests/openapi_snapshots'),
];
const outputDir = path.resolve(import.meta.dirname, '../data/openapi');
for (const sourceDir of sourceDirs) {
processDirectory(sourceDir, outputDir);
}
-52
View File
@@ -1,52 +0,0 @@
# RTK Query API Client Generator
This generator automates the process of creating RTK Query API clients for Grafana's API groups. It replaces the manual steps outlined in the [main API documentation](../../public/app/api/README.md).
## Usage
```bash
yarn generate:api-client
```
The CLI will prompt for:
1. **Enterprise or OSS API** - Whether this is an Enterprise or OSS API. This affects paths and build commands.
2. **API group name** - The basic name for the API (e.g., `dashboard`)
3. **API group** - The full API group name (defaults to `<group-name>.grafana.app`)
4. **API version** - The API version (e.g., `v0alpha1`)
5. **Reducer path** - The Redux reducer path (defaults to `<group-name>API`). This will also be used as the API's named export.
6. **Endpoints** - Optional comma-separated list of endpoints to include (e.g., `createDashboard,updateDashboard`). If not provided, all endpoints will be included.
## What It Does
The generator automates the following:
1. Creates the `baseAPI.ts` file for the API group
2. Updates the appropriate generate script to include the API client
- `scripts/generate-rtk-apis.ts` for OSS APIs
- `local/generate-enterprise-apis.ts` for Enterprise APIs
3. Creates the `index.ts` file with proper exports
4. For OSS APIs only: Registers Redux reducers and middleware in the store. For Enterprise this needs to be done manually
5. Formats all generated files using Prettier and ESLint
6. Automatically runs the appropriate command to generate endpoints from the OpenAPI schema
## Limitations
- The generator is optimized for Kubernetes-style APIs, as it requires Kubernetes resource details. For legacy APIs, manual adjustments may be needed.
- It expects processed OpenAPI specifications to exist in the `openapi_snapshots` directory
## Troubleshooting
### Missing OpenAPI Schema
If an error about a missing OpenAPI schema appears, check that:
1. The API group and version exist in the backend
2. The `TestIntegrationOpenAPIs` test has been run to generate the schema (step 1 in the [main API documentation](../../public/app/api/README.md)).
3. The schema file exists at `data/openapi/<group>-<version>.json`
### Validation Errors
- API group must include `.grafana.app`
- Version must be in format `v0alpha1`, `v1beta2`, etc.
- Reducer path must end with `API`
-115
View File
@@ -1,115 +0,0 @@
import { execSync } from 'child_process';
import path from 'path';
type PlopActionFunction = (
answers: Record<string, unknown>,
config?: Record<string, unknown>
) => string | Promise<string>;
// Helper to remove quotes from operation IDs
export const removeQuotes = (str: string | unknown) => {
if (typeof str !== 'string') {
return str;
}
return str.replace(/^['"](.*)['"]$/, '$1');
};
export const formatEndpoints = () => (endpointsInput: string | string[]) => {
if (Array.isArray(endpointsInput)) {
return endpointsInput.map((op) => `'${removeQuotes(op)}'`).join(', ');
}
// Handle string input (comma-separated)
if (typeof endpointsInput === 'string') {
const endpointsArray = endpointsInput
.split(',')
.map((id) => id.trim())
.filter(Boolean);
return endpointsArray.map((op) => `'${removeQuotes(op)}'`).join(', ');
}
return '';
};
// List of created or modified files
export const getFilesToFormat = (groupName: string, version: string, isEnterprise = false) => {
const apiClientBasePath = isEnterprise ? 'public/app/extensions/api/clients' : 'public/app/api/clients';
const generateScriptPath = isEnterprise ? 'local/generate-enterprise-apis.ts' : 'scripts/generate-rtk-apis.ts';
return [
`${apiClientBasePath}/${groupName}/${version}/baseAPI.ts`,
`${apiClientBasePath}/${groupName}/${version}/index.ts`,
generateScriptPath,
...(isEnterprise ? [] : [`public/app/core/reducers/root.ts`, `public/app/store/configureStore.ts`]),
];
};
export const runGenerateApis =
(basePath: string): PlopActionFunction =>
(answers, config) => {
try {
const isEnterprise = answers.isEnterprise || (config && config.isEnterprise);
let command;
if (isEnterprise) {
command = 'yarn process-specs && npx rtk-query-codegen-openapi ./local/generate-enterprise-apis.ts';
} else {
command = 'yarn generate-apis';
}
console.log(`⏳ Running ${command} to generate endpoints...`);
execSync(command, { stdio: 'inherit', cwd: basePath });
return '✅ API endpoints generated successfully!';
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
console.error('❌ Failed to generate API endpoints:', errorMessage);
return '❌ Failed to generate API endpoints. See error above.';
}
};
export const formatFiles =
(basePath: string): PlopActionFunction =>
(_, config) => {
if (!config || !Array.isArray(config.files)) {
console.error('Invalid config passed to formatFiles action');
return '❌ Formatting failed: Invalid configuration';
}
const filesToFormat = config.files.map((file: string) => path.join(basePath, file));
try {
const filesList = filesToFormat.map((file: string) => `"${file}"`).join(' ');
console.log('🧹 Running ESLint on generated/modified files...');
try {
execSync(`yarn eslint --fix ${filesList}`, { cwd: basePath });
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
console.warn(`⚠️ Warning: ESLint encountered issues: ${errorMessage}`);
}
console.log('🧹 Running Prettier on generated/modified files...');
try {
// '--ignore-path' is necessary so the gitignored files ('local/' folder) can still be formatted
execSync(`yarn prettier --write ${filesList} --ignore-path=./.prettierignore`, { cwd: basePath });
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
console.warn(`⚠️ Warning: Prettier encountered issues: ${errorMessage}`);
}
return '✅ Files linted and formatted successfully!';
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
console.error('⚠️ Warning: Formatting operations failed:', errorMessage);
return '⚠️ Warning: Formatting operations failed.';
}
};
export const validateGroup = (group: string) => {
return group && group.includes('.grafana.app') ? true : 'Group should be in format: name.grafana.app';
};
export const validateVersion = (version: string) => {
return version && /^v\d+[a-z]*\d+$/.test(version) ? true : 'Version should be in format: v0alpha1, v1beta2, etc.';
};
-167
View File
@@ -1,167 +0,0 @@
import path from 'path';
import type { NodePlopAPI, PlopGeneratorConfig } from 'plop';
import {
formatEndpoints,
validateGroup,
validateVersion,
getFilesToFormat,
runGenerateApis,
formatFiles,
// The file extension is necessary to make the imports
// work with the '--experimental-strip-types' flag
// @ts-ignore
} from './helpers.ts';
// @ts-ignore
import { type ActionConfig, type PlopData, isPlopData } from './types.ts';
export default function plopGenerator(plop: NodePlopAPI) {
// Grafana root path
const basePath = path.resolve(import.meta.dirname, '../..');
// Register custom action types
plop.setActionType('runGenerateApis', runGenerateApis(basePath));
plop.setActionType('formatFiles', formatFiles(basePath));
// Used in templates to format endpoints
plop.setHelper('formatEndpoints', formatEndpoints());
const generateRtkApiActions = (data: PlopData) => {
const { reducerPath, groupName, version, isEnterprise } = data;
const apiClientBasePath = isEnterprise ? 'public/app/extensions/api/clients' : 'public/app/api/clients';
const generateScriptPath = isEnterprise ? 'local/generate-enterprise-apis.ts' : 'scripts/generate-rtk-apis.ts';
// Using app path, so the imports work on any file level
const clientImportPath = isEnterprise ? '../extensions/api/clients' : 'app/api/clients';
const apiPathPrefix = isEnterprise ? '../public/app/extensions/api/clients' : '../public/app/api/clients';
const templateData = {
...data,
apiPathPrefix,
};
// Base actions that are always added
const actions: ActionConfig[] = [
{
type: 'add',
path: path.join(basePath, `${apiClientBasePath}/${groupName}/${version}/baseAPI.ts`),
templateFile: './templates/baseAPI.ts.hbs',
},
{
type: 'modify',
path: path.join(basePath, generateScriptPath),
pattern: '// PLOP_INJECT_API_CLIENT - Used by the API client generator',
templateFile: './templates/config-entry.hbs',
data: templateData,
},
{
type: 'add',
path: path.join(basePath, `${apiClientBasePath}/${groupName}/${version}/index.ts`),
templateFile: './templates/index.ts.hbs',
},
];
// Only add redux reducer and middleware for OSS clients
if (!isEnterprise) {
actions.push(
{
type: 'modify',
path: path.join(basePath, 'public/app/core/reducers/root.ts'),
pattern: '// PLOP_INJECT_IMPORT',
template: `import { ${reducerPath} } from '${clientImportPath}/${groupName}/${version}';\n// PLOP_INJECT_IMPORT`,
},
{
type: 'modify',
path: path.join(basePath, 'public/app/core/reducers/root.ts'),
pattern: '// PLOP_INJECT_REDUCER',
template: `[${reducerPath}.reducerPath]: ${reducerPath}.reducer,\n // PLOP_INJECT_REDUCER`,
},
{
type: 'modify',
path: path.join(basePath, 'public/app/store/configureStore.ts'),
pattern: '// PLOP_INJECT_IMPORT',
template: `import { ${reducerPath} } from '${clientImportPath}/${groupName}/${version}';\n// PLOP_INJECT_IMPORT`,
},
{
type: 'modify',
path: path.join(basePath, 'public/app/store/configureStore.ts'),
pattern: '// PLOP_INJECT_MIDDLEWARE',
template: `${reducerPath}.middleware,\n // PLOP_INJECT_MIDDLEWARE`,
}
);
}
// Add formatting and generation actions
actions.push(
{
type: 'formatFiles',
files: getFilesToFormat(groupName, version, isEnterprise),
},
{
type: 'runGenerateApis',
isEnterprise,
}
);
return actions;
};
const generator: PlopGeneratorConfig = {
description: 'Generate RTK Query API client for a Grafana API group',
prompts: [
{
type: 'confirm',
name: 'isEnterprise',
message: 'Is this a Grafana Enterprise API?',
default: false,
},
{
type: 'input',
name: 'groupName',
message: 'API group name (e.g. dashboard):',
validate: (input: string) => (input?.trim() ? true : 'Group name is required'),
},
{
type: 'input',
name: 'group',
message: 'API group (e.g. dashboard.grafana.app):',
default: (answers: { groupName?: string }) => `${answers.groupName}.grafana.app`,
validate: validateGroup,
},
{
type: 'input',
name: 'version',
message: 'API version (e.g. v0alpha1):',
default: 'v0alpha1',
validate: validateVersion,
},
{
type: 'input',
name: 'reducerPath',
message: 'Reducer path (e.g. dashboardAPIv0alpha1):',
default: (answers: { groupName?: string; version?: string }) => `${answers.groupName}API${answers.version}`,
validate: (input: string) =>
input?.endsWith('API') || input?.match(/API[a-z]\d+[a-z]*\d*$/)
? true
: 'Reducer path should end with "API" or "API<version>" (e.g. dashboardAPI, dashboardAPIv0alpha1)',
},
{
type: 'input',
name: 'endpoints',
message: 'Endpoints to include (comma-separated, optional):',
validate: () => true,
},
],
actions: function (data) {
if (!isPlopData(data)) {
throw new Error('Invalid data format received from prompts');
}
return generateRtkApiActions(data);
},
};
plop.setGenerator('rtk-api-client', generator);
}
@@ -1,14 +0,0 @@
import { createApi } from '@reduxjs/toolkit/query/react';
import { createBaseQuery } from 'app/api/createBaseQuery';
import { getAPIBaseURL } from 'app/api/utils';
export const BASE_URL = getAPIBaseURL('{{group}}', '{{version}}');
export const api = createApi({
reducerPath: '{{reducerPath}}',
baseQuery: createBaseQuery({
baseURL: BASE_URL,
}),
endpoints: () => ({}),
});
@@ -1,9 +0,0 @@
'{{apiPathPrefix}}/{{groupName}}/{{version}}/endpoints.gen.ts': {
apiFile: '{{apiPathPrefix}}/{{groupName}}/{{version}}/baseAPI.ts',
schemaFile: '../data/openapi/{{group}}-{{version}}.json',
{{#if endpoints}}
filterEndpoints: [{{{formatEndpoints endpoints}}}],
{{/if}}
tag: true,
},
// PLOP_INJECT_API_CLIENT - Used by the API client generator
@@ -1,3 +0,0 @@
import { generatedAPI } from './endpoints.gen';
export const {{reducerPath}} = generatedAPI.enhanceEndpoints({});
-27
View File
@@ -1,27 +0,0 @@
import type { AddActionConfig, ModifyActionConfig } from 'plop';
export interface FormatFilesActionConfig {
type: 'formatFiles';
files: string[];
}
export interface RunGenerateApisActionConfig {
type: 'runGenerateApis';
isEnterprise: boolean;
}
// Union type of all possible action configs
export type ActionConfig = AddActionConfig | ModifyActionConfig | FormatFilesActionConfig | RunGenerateApisActionConfig;
export interface PlopData {
groupName: string;
group: string;
version: string;
reducerPath: string;
endpoints: string;
isEnterprise: boolean;
}
export function isPlopData(data: unknown): data is PlopData {
return typeof data === 'object' && data !== null;
}