diff --git a/.drone.yml b/.drone.yml index 6f896b5526f..9f6ba848ad2 100644 --- a/.drone.yml +++ b/.drone.yml @@ -243,11 +243,11 @@ steps: - commands: - apk add --update git - |- - yarn run i18n:extract || (echo " + make i18n-extract || (echo " Extraction failed. Make sure that you have no dynamic translation phrases, such as 't(\`preferences.theme.\$${themeID}\`, themeName)' and that no translation key is used twice. Search the output for '[warning]' to find the offending file." && false) - "\n file_diff=$(git diff --dirstat public/locales)\n if [ -n \"$file_diff\" ]; then\n echo $file_diff\n echo - \"\nTranslation extraction has not been committed. Please run 'yarn i18n:extract', + \"\nTranslation extraction has not been committed. Please run 'make i18n-extract', commit the changes and push again.\"\n exit 1\n fi\n \ " depends_on: @@ -1627,11 +1627,11 @@ steps: - commands: - apk add --update git - |- - yarn run i18n:extract || (echo " + make i18n-extract || (echo " Extraction failed. Make sure that you have no dynamic translation phrases, such as 't(\`preferences.theme.\$${themeID}\`, themeName)' and that no translation key is used twice. Search the output for '[warning]' to find the offending file." && false) - "\n file_diff=$(git diff --dirstat public/locales)\n if [ -n \"$file_diff\" ]; then\n echo $file_diff\n echo - \"\nTranslation extraction has not been committed. Please run 'yarn i18n:extract', + \"\nTranslation extraction has not been committed. Please run 'make i18n-extract', commit the changes and push again.\"\n exit 1\n fi\n \ " depends_on: @@ -4925,6 +4925,6 @@ kind: secret name: gcr_credentials --- kind: signature -hmac: fbd59890dac44eb6fb34562f2b2b2db0fc0a8a50d451f9887f815ee35757e0f6 +hmac: 958ed40ca0620498c01fa867b05a1d6c3bcd908067fbb34be691923e95cfc77b ... diff --git a/.prettierignore b/.prettierignore index 160926fd718..76bf2adf696 100644 --- a/.prettierignore +++ b/.prettierignore @@ -19,10 +19,6 @@ vendor # TS generate from cue by cuetsy **/*.gen.ts -# Auto-generated internationalization files -public/locales/_build/ -public/locales/**/*.js - # Auto-generated theme files theme.light.generated.json theme.dark.generated.json diff --git a/Makefile b/Makefile index bd6bc808bb4..0d47a3d0215 100644 --- a/Makefile +++ b/Makefile @@ -110,6 +110,25 @@ OAPI_SPEC_TARGET = public/openapi3.json openapi3-gen: swagger-gen ## Generates OpenApi 3 specs from the Swagger 2 already generated $(GO) run scripts/openapi3/openapi3conv.go $(MERGED_SPEC_TARGET) $(OAPI_SPEC_TARGET) +##@ Internationalisation +.PHONY: i18n-extract-enterprise +ENTERPRISE_FE_EXT_FILE = public/app/extensions/index.ts +ifeq ("$(wildcard $(ENTERPRISE_FE_EXT_FILE))","") ## if enterprise is not enabled +i18n-extract-enterprise: + @echo "Skipping i18n extract for Enterprise: not enabled" +else +i18n-extract-enterprise: $(SWAGGER) ## Generate API Swagger specification + @echo "Extracting i18n strings for Enterprise" + yarn run i18next --config public/locales/i18next-parser-enterprise.config.cjs + node ./public/locales/pseudo.mjs --mode enterprise +endif + +.PHONY: i18n-extract +i18n-extract: i18n-extract-enterprise + @echo "Extracting i18n strings for OSS" + yarn run i18next --config public/locales/i18next-parser.config.cjs + node ./public/locales/pseudo.mjs --mode oss + ##@ Building .PHONY: gen-cue gen-cue: ## Do all CUE/Thema code generation diff --git a/contribute/internationalization.md b/contribute/internationalization.md index 7a0ac2a249f..577d0188eb6 100644 --- a/contribute/internationalization.md +++ b/contribute/internationalization.md @@ -9,7 +9,7 @@ Grafana uses the [i18next](https://www.i18next.com/) framework for managing tran - Use `Go to {{ pageTitle }}` in code to add a translatable phrase - Translations are stored in JSON files in `public/locales/{locale}/grafana.json` - If a particular phrase is not available in the a language then it will fall back to English -- To update phrases in English, edit the default phrase in the component's source and then run `yarn i18n:extract`. +- To update phrases in English, edit the default phrase in the component's source and then run `make i18n-extract`. - The single source of truth for en-US (fallback language) is in grafana/grafana, the single source of truth for any translated language is Crowdin - To update phrases in any translated language, edit the phrase in Crowdin. Do not edit the `{locale}/grafana.json` @@ -41,7 +41,7 @@ const ErrorMessage = ({ id, message }) => There 2. Upon reload, the default English phrase will appear on the page. -3. Before submitting your PR, run the `yarn i18n:extract` command to extract the messages you added into the `public/locales/en-US/grafana.json` file and make them available for translation. +3. Before submitting your PR, run the `make i18n-extract` command to extract the messages you added into the `public/locales/en-US/grafana.json` file and make them available for translation. **Note:** All other languages will receive their translations when they are ready to be downloaded from Crowdin. ### Plain JS usage @@ -79,6 +79,7 @@ While the `t` function can technically be used outside of React functions (e.g, 1. Add a new constant for the new language 2. Add the new constant to the `LOCALES` array 3. Create a PR with the changes and merge when you are ready to release the new language (probably wait until we have translations for it) +4. In the Enterprise repo, update `src/public/locales/localeExtensions.ts` ## How translations work in Grafana @@ -186,7 +187,7 @@ import { t } from 'app/core/internationalization'; const translatedString = t('inbox.heading', 'You got {{count}} message', { count: messages.length }); ``` -Once extracted with `yarn i18n:extract` you will need to manually edit the [English grafana.json message catalogue](../public/locales/en-US/grafana.json) to correct the plural forms. See the [react-i18next docs](https://react.i18next.com/latest/trans-component#plural) for more details. +Once extracted with `make i18n-extract` you will need to manually edit the [English grafana.json message catalogue](../public/locales/en-US/grafana.json) to correct the plural forms. See the [react-i18next docs](https://react.i18next.com/latest/trans-component#plural) for more details. ```json { diff --git a/package.json b/package.json index 5a182619e3a..4c9bc730654 100644 --- a/package.json +++ b/package.json @@ -48,9 +48,6 @@ "plugins:build-bundled": "find plugins-bundled -name package.json -not -path '*/node_modules/*' -execdir yarn build \\;", "watch": "yarn start -d watch,start core:start --watchTheme", "ci:test-frontend": "yarn run test:ci", - "i18n:clean": "rimraf public/locales/en-US/grafana.json", - "i18n:extract": "yarn run i18next -c public/locales/i18next-parser.config.cjs 'public/**/*.{tsx,ts}' 'packages/grafana-ui/**/*.{tsx,ts}' && yarn i18n:pseudo", - "i18n:pseudo": "node ./public/locales/pseudo.js", "i18n:stats": "node ./scripts/cli/reportI18nStats.mjs", "betterer": "betterer", "betterer:json": "ts-node --transpile-only --project ./scripts/cli/tsconfig.json ./scripts/cli/bettererResultsToJson.ts", diff --git a/public/app/core/internationalization/constants.ts b/public/app/core/internationalization/constants.ts index b367bb57a44..a241519fed4 100644 --- a/public/app/core/internationalization/constants.ts +++ b/public/app/core/internationalization/constants.ts @@ -1,4 +1,5 @@ import { ResourceKey } from 'i18next'; +import { uniq } from 'lodash'; export const ENGLISH_US = 'en-US'; export const FRENCH_FRANCE = 'fr-FR'; @@ -10,7 +11,9 @@ export const PSEUDO_LOCALE = 'pseudo-LOCALE'; export const DEFAULT_LANGUAGE = ENGLISH_US; -interface LanguageDefinitions { +export type LocaleFileLoader = () => Promise; + +export interface LanguageDefinition { /** IETF language tag for the language e.g. en-US */ code: string; @@ -18,53 +21,90 @@ interface LanguageDefinitions { name: string; /** Function to load translations */ - loader: () => Promise; + loader: Record; } -export const LANGUAGES: LanguageDefinitions[] = [ +export const LANGUAGES: LanguageDefinition[] = [ { code: ENGLISH_US, name: 'English', - loader: () => import('../../../locales/en-US/grafana.json'), + loader: { + grafana: () => import('../../../locales/en-US/grafana.json'), + }, }, { code: FRENCH_FRANCE, name: 'Français', - loader: () => import('../../../locales/fr-FR/grafana.json'), + loader: { + grafana: () => import('../../../locales/fr-FR/grafana.json'), + }, }, { code: SPANISH_SPAIN, name: 'Español', - loader: () => import('../../../locales/es-ES/grafana.json'), + loader: { + grafana: () => import('../../../locales/es-ES/grafana.json'), + }, }, { code: GERMAN_GERMANY, name: 'Deutsch', - loader: () => import('../../../locales/de-DE/grafana.json'), + loader: { + grafana: () => import('../../../locales/de-DE/grafana.json'), + }, }, { code: CHINESE_SIMPLIFIED, name: '中文（简体）', - loader: () => import('../../../locales/zh-Hans/grafana.json'), + loader: { + grafana: () => import('../../../locales/zh-Hans/grafana.json'), + }, }, { code: BRAZILIAN_PORTUGUESE, name: 'Português Brasileiro', - loader: () => import('../../../locales/pt-BR/grafana.json'), + loader: { + grafana: () => import('../../../locales/pt-BR/grafana.json'), + }, }, -]; +] satisfies Array>; if (process.env.NODE_ENV === 'development') { LANGUAGES.push({ code: PSEUDO_LOCALE, name: 'Pseudo-locale', - loader: () => import('../../../locales/pseudo-LOCALE/grafana.json'), + loader: { + grafana: () => import('../../../locales/pseudo-LOCALE/grafana.json'), + }, }); } +// Optionally load enterprise locale extensions, if they are present. +// It is important that this happens before NAMESPACES is defined so it has the correct value +// +// require.context doesn't work in jest, so we don't even attempt to load enterprise translations... +if (process.env.NODE_ENV !== 'test') { + const extensionRequireContext = require.context('../../', true, /app\/extensions\/locales\/localeExtensions/); + if (extensionRequireContext.keys().includes('app/extensions/locales/localeExtensions')) { + const { LOCALE_EXTENSIONS, ENTERPRISE_I18N_NAMESPACE } = extensionRequireContext( + 'app/extensions/locales/localeExtensions' + ); + + for (const language of LANGUAGES) { + const localeLoader = LOCALE_EXTENSIONS[language.code]; + + if (localeLoader) { + language.loader[ENTERPRISE_I18N_NAMESPACE] = localeLoader; + } + } + } +} + export const VALID_LANGUAGES = LANGUAGES.map((v) => v.code); + +export const NAMESPACES = uniq(LANGUAGES.flatMap((v) => Object.keys(v.loader))); diff --git a/public/app/core/internationalization/index.tsx b/public/app/core/internationalization/index.tsx index 4d3e4bf047d..812fc4f83b7 100644 --- a/public/app/core/internationalization/index.tsx +++ b/public/app/core/internationalization/index.tsx @@ -3,12 +3,12 @@ import LanguageDetector, { DetectorOptions } from 'i18next-browser-languagedetec import React from 'react'; import { Trans as I18NextTrans, initReactI18next } from 'react-i18next'; // eslint-disable-line no-restricted-imports -import { DEFAULT_LANGUAGE, VALID_LANGUAGES } from './constants'; +import { DEFAULT_LANGUAGE, NAMESPACES, VALID_LANGUAGES } from './constants'; import { loadTranslations } from './loadTranslations'; let tFunc: TFunction | undefined; -export function initializeI18n(language: string): Promise<{ language: string | undefined }> { +export async function initializeI18n(language: string): Promise<{ language: string | undefined }> { // This is a placeholder so we can put a 'comment' in the message json files. // Starts with an underscore so it's sorted to the top of the file. Even though it is in a comment the following line is still extracted // t('_comment', 'The code is the source of truth for English phrases. They should be updated in the components directly, and additional plurals specified in this file.'); @@ -24,7 +24,10 @@ export function initializeI18n(language: string): Promise<{ language: string | u // Required to ensure that `resolvedLanguage` is set property when an invalid language is passed (such as through 'detect') supportedLngs: VALID_LANGUAGES, fallbackLng: DEFAULT_LANGUAGE, + + ns: NAMESPACES, }; + let i18nInstance = i18n; if (language === 'detect') { i18nInstance = i18nInstance.use(LanguageDetector); @@ -39,13 +42,13 @@ export function initializeI18n(language: string): Promise<{ language: string | u .use(initReactI18next) // passes i18n down to react-i18next .init(options); - tFunc = i18n.t; + await loadPromise; - return loadPromise.then(() => { - return { - language: i18nInstance.resolvedLanguage, - }; - }); + tFunc = i18n.getFixedT(null, NAMESPACES); + + return { + language: i18nInstance.resolvedLanguage, + }; } export function changeLanguage(locale: string) { @@ -54,7 +57,7 @@ export function changeLanguage(locale: string) { } export const Trans: typeof I18NextTrans = (props) => { - return ; + return ; }; // Wrap t() to provide default namespaces and enforce a consistent API diff --git a/public/app/core/internationalization/loadTranslations.ts b/public/app/core/internationalization/loadTranslations.ts index 3e618027ee4..301261fca6d 100644 --- a/public/app/core/internationalization/loadTranslations.ts +++ b/public/app/core/internationalization/loadTranslations.ts @@ -12,10 +12,17 @@ export const loadTranslations: BackendModule = { if (!localeDef) { localeDef = LANGUAGES.find((v) => getLanguagePartFromCode(v.code) === getLanguagePartFromCode(language)); } + if (!localeDef) { - return callback(new Error('No message loader available for ' + language), null); + return callback(new Error(`No message loader available for ${language}`), null); } - const messages = await localeDef.loader(); + + const namespaceLoader = localeDef.loader[namespace]; + if (!namespaceLoader) { + return callback(new Error(`No message loader available for ${language} with namespace ${namespace}`), null); + } + + const messages = await namespaceLoader(); callback(null, messages); }, }; diff --git a/public/locales/i18next-parser-enterprise.config.cjs b/public/locales/i18next-parser-enterprise.config.cjs new file mode 100644 index 00000000000..66e8cd9cca2 --- /dev/null +++ b/public/locales/i18next-parser-enterprise.config.cjs @@ -0,0 +1,8 @@ +const baseConfig = require('./i18next-parser.config.cjs'); + +module.exports = { + ...baseConfig, + defaultNamespace: 'grafana-enterprise', + input: ['../../public/app/extensions/**/*.{tsx,ts}'], + output: './public/app/extensions/locales/$LOCALE/$NAMESPACE.json', +}; diff --git a/public/locales/i18next-parser.config.cjs b/public/locales/i18next-parser.config.cjs index 2aeca9d3745..f6c47aa7cc1 100644 --- a/public/locales/i18next-parser.config.cjs +++ b/public/locales/i18next-parser.config.cjs @@ -1,6 +1,6 @@ module.exports = { - // Base config - locales: ['en-US'], // Only en-US is updated - Crowdin will PR with other languages + // Base config - same for both OSS and Enterprise + locales: ['en-US'], // Only en-US is updated - Crowdin will PR with other languages sort: true, createOldCatalogs: false, failOnWarnings: true, @@ -9,6 +9,10 @@ module.exports = { // OSS-specific config defaultNamespace: 'grafana', - input: ['../../public/**/*.{tsx,ts}', '!../../public/app/extensions/**/*', '../../packages/grafana-ui/**/*.{tsx,ts}'], + input: [ + '../../public/**/*.{tsx,ts}', + '!../../public/app/extensions/**/*', // Don't extract from Enterprise + '../../packages/grafana-ui/**/*.{tsx,ts}', + ], output: './public/locales/$LOCALE/$NAMESPACE.json', }; diff --git a/public/locales/pseudo.js b/public/locales/pseudo.js deleted file mode 100644 index 4bcfef5df05..00000000000 --- a/public/locales/pseudo.js +++ /dev/null @@ -1,24 +0,0 @@ -const fs = require('fs/promises'); -const pseudoizer = require('pseudoizer'); -const prettier = require('prettier'); - -function pseudoizeJsonReplacer(key, value) { - if (typeof value === 'string') { - // Split string on brace-enclosed segments. Odd indices will be {{variables}} - const phraseParts = value.split(/(\{\{[^}]+}\})/g); - const translatedParts = phraseParts.map((str, index) => index % 2 ? str : pseudoizer.pseudoize(str)) - return translatedParts.join("") - } - - return value; -} - -fs.readFile('./public/locales/en-US/grafana.json').then(async (enJson) => { - const enMessages = JSON.parse(enJson); - // Add newline to make prettier happy - const pseudoJson = await prettier.format(JSON.stringify(enMessages, pseudoizeJsonReplacer, 2), { - parser: 'json', - }); - - return fs.writeFile('./public/locales/pseudo-LOCALE/grafana.json', pseudoJson); -}); diff --git a/public/locales/pseudo.mjs b/public/locales/pseudo.mjs new file mode 100644 index 00000000000..05f3f89626d --- /dev/null +++ b/public/locales/pseudo.mjs @@ -0,0 +1,63 @@ +// @ts-check +import { readFile, writeFile } from 'fs/promises'; +import { format } from 'prettier'; +import { pseudoize } from 'pseudoizer'; +import { hideBin } from 'yargs/helpers'; +import yargs from 'yargs/yargs'; + +const argv = await yargs(hideBin(process.argv)) + .option('mode', { + demandOption: true, + describe: 'Path to a template to use for each issue. See source bettererIssueTemplate.md for an example', + type: 'string', + choices: ['oss', 'enterprise', 'both'], + }) + .version(false).argv; + +const extractOSS = ['oss', 'both'].includes(argv.mode); +const extractEnterprise = ['enterprise', 'both'].includes(argv.mode); + +/** + * @param {string} key + * @param {unknown} value + */ +function pseudoizeJsonReplacer(key, value) { + if (typeof value === 'string') { + // Split string on brace-enclosed segments. Odd indices will be {{variables}} + const phraseParts = value.split(/(\{\{[^}]+}\})/g); + const translatedParts = phraseParts.map((str, index) => (index % 2 ? str : pseudoize(str))); + return translatedParts.join(''); + } + + return value; +} +/** + * @param {string} inputPath + * @param {string} outputPath + */ +async function pseudoizeJson(inputPath, outputPath) { + const baseJson = await readFile(inputPath, 'utf-8'); + const enMessages = JSON.parse(baseJson); + const pseudoJson = JSON.stringify(enMessages, pseudoizeJsonReplacer, 2); + const prettyPseudoJson = await format(pseudoJson, { + parser: 'json', + }); + + await writeFile(outputPath, prettyPseudoJson); + console.log('Wrote', outputPath); +} + +// +// OSS translations +if (extractOSS) { + await pseudoizeJson('./public/locales/en-US/grafana.json', './public/locales/pseudo-LOCALE/grafana.json'); +} + +// +// Enterprise translations +if (extractEnterprise) { + await pseudoizeJson( + './public/app/extensions/locales/en-US/grafana-enterprise.json', + './public/app/extensions/locales/pseudo-LOCALE/grafana-enterprise.json' + ); +} diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index 85d435f7f7c..26070962065 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -641,7 +641,7 @@ def lint_frontend_step(): def verify_i18n_step(): extract_error_message = "\nExtraction failed. Make sure that you have no dynamic translation phrases, such as 't(\\`preferences.theme.\\$${themeID}\\`, themeName)' and that no translation key is used twice. Search the output for '[warning]' to find the offending file." - uncommited_error_message = "\nTranslation extraction has not been committed. Please run 'yarn i18n:extract', commit the changes and push again." + uncommited_error_message = "\nTranslation extraction has not been committed. Please run 'make i18n-extract', commit the changes and push again." return { "name": "verify-i18n", "image": images["node"], @@ -651,7 +651,7 @@ def verify_i18n_step(): "failure": "ignore", "commands": [ "apk add --update git", - "yarn run i18n:extract || (echo \"{}\" && false)".format(extract_error_message), + "make i18n-extract || (echo \"{}\" && false)".format(extract_error_message), # Verify that translation extraction has been committed ''' file_diff=$(git diff --dirstat public/locales) diff --git a/yarn.lock b/yarn.lock index 700b3a5c9bb..39e99715e8d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -18365,8 +18365,8 @@ __metadata: linkType: hard "glob-stream@npm:^8.0.0": - version: 8.0.0 - resolution: "glob-stream@npm:8.0.0" + version: 8.0.2 + resolution: "glob-stream@npm:8.0.2" dependencies: "@gulpjs/to-absolute-glob": "npm:^4.0.0" anymatch: "npm:^3.1.3" @@ -18376,7 +18376,7 @@ __metadata: is-negated-glob: "npm:^1.0.0" normalize-path: "npm:^3.0.0" streamx: "npm:^2.12.5" - checksum: 10/b1d18b6fd49086ff02e031f03e3debac747047d304b349a6dced3b7944c665344ef63496363f483acc7c6afcd6ebfb11af1652824f2c370d83c0f3905d5c67e0 + checksum: 10/cda46c02b6313d4a5cd0a3e67c7a2bd477d5f708904dc761c0d6364611f188a303051ec4e0cd405597522c7f7ffbba530f147754b4bf5af9f18e970c024734d8 languageName: node linkType: hard