Do codegen and check no-diff of all (non-blacklisted) CUE->TS codegen during CI (#39922)

* Add file blacklist to `grafana-cli cue gen-ts` cmd

* Add CI step checking all cuetsification is done

* Add dummy command to make the next one fail

* Generate drone bits

* Check diff output failure

* Echo list of untracked files, for failure locality

* Move git cleanness checking into script

* Blacklist of cue files is complete and correct

* Remove news panel plugin from cuetsify blacklist

* Dummy commit, check that untracked gen still fail

* Tie off remaining errors

* Re-add barchart to blacklist
* Remove file left around by earlier pipeline
* Commit generated news models.gen.ts

* Include eslint as part of cuetsified output gen

* Update pkg/cmd/grafana-cli/commands/cuetsify_command.go

Co-authored-by: Ashley Harrison <ashley.harrison@grafana.com>

* Update scripts/drone/steps/lib.star

Co-authored-by: Maria Alexandra <239999+axelavargas@users.noreply.github.com>

* Update drone.yml

* Last fix on .drone.yml

Co-authored-by: Ashley Harrison <ashley.harrison@grafana.com>
Co-authored-by: Maria Alexandra <239999+axelavargas@users.noreply.github.com>

scripts/clean-git-or-error.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -e
+
+# Checks that a git working tree is clean. Exit 0 if it is, or exit 1 with a list of
+# dirty files - including modifications, removals, and untracked+unignored files.
+
+# Primarily intended for ensuring in CI that codegen operations are a no-op/have
+# already been performed.
+
+STAT="$(git status --porcelain 2>/dev/null)"
+if [ -z "$STAT" ]
+then
+    exit 0
+else
+    echo "$STAT"
+    exit 1
+fi
+
+# For safekeeping, alternative commands that meet a similar goal (in case the
+# above approach ends up being problematic)
+# 
+# List modified/removed files and exit nonzero if any exist:
+# git diff --stat --exit-code
+#
+# List untracked files, and exit nonzero if any exist:
+# git ls-files --others --exclude-standard',
+# u="$(git ls-files --others --exclude-standard)" && test -z "$u"

scripts/drone/pipelines/main.star

@@ -32,6 +32,7 @@ load(
     'publish_packages_step',
     'upload_cdn_step',
     'validate_scuemata_step',
+    'ensure_cuetsified_step',
     'test_a11y_frontend_step'
 )
 
@@ -64,6 +65,7 @@ def get_steps(edition, is_downstream=False):
         build_frontend_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
         build_plugins_step(edition=edition, sign=True),
         validate_scuemata_step(),
+        ensure_cuetsified_step(),
     ]
 
     # Have to insert Enterprise2 steps before they're depended on (in the gen-version step)

scripts/drone/pipelines/pr.star

@@ -24,6 +24,7 @@ load(
     'memcached_integration_tests_step',
     'benchmark_ldap_step',
     'validate_scuemata_step',
+    'ensure_cuetsified_step',
     'test_a11y_frontend_step_pr',
 )
 
@@ -55,6 +56,7 @@ def pr_pipelines(edition):
         build_frontend_step(edition=edition, ver_mode=ver_mode),
         build_plugins_step(edition=edition),
         validate_scuemata_step(),
+        ensure_cuetsified_step(),
     ]
 
     # Have to insert Enterprise2 steps before they're depended on (in the gen-version step)

scripts/drone/pipelines/release.star

@@ -30,7 +30,8 @@ load(
     'upload_packages_step',
     'publish_packages_step',
     'upload_cdn_step',
-    'validate_scuemata_step'
+    'validate_scuemata_step',
+    'ensure_cuetsified_step'
 )
 
 load(
@@ -91,6 +92,7 @@ def get_steps(edition, ver_mode):
         build_frontend_step(edition=edition, ver_mode=ver_mode),
         build_plugins_step(edition=edition, sign=True),
         validate_scuemata_step(),
+        ensure_cuetsified_step(),
     ]
 
     # Have to insert Enterprise2 steps before they're depended on (in the gen-version step)

scripts/drone/steps/lib.star

@@ -480,6 +480,7 @@ def codespell_step():
             # Important: all words have to be in lowercase, and separated by "\n".
             'echo -e "unknwon\nreferer\nerrorstring\neror\niam\nwan" > words_to_ignore.txt',
             'codespell -I words_to_ignore.txt docs/',
+            'rm words_to_ignore.txt',
         ],
     }
 
@@ -1016,3 +1017,21 @@ def validate_scuemata_step():
             './bin/linux-amd64/grafana-cli cue validate-schema --grafana-root .',
         ],
     }
+
+def ensure_cuetsified_step():
+    return {
+        'name': 'ensure-cuetsified',
+        'image': build_image,
+        'depends_on': [
+            'validate-scuemata',
+        ],
+        'commands': [
+            './bin/linux-amd64/grafana-cli cue gen-ts --grafana-root .',
+            '# The above command generates Typescript files (*.gen.ts) from all appropriate .cue files.',
+            '# It is required that the generated Typescript be in sync with the input CUE files.',
+            '# ...Modulo eslint auto-fixes...:',
+            './node_modules/.bin/eslint . --ext .gen.ts --fix',
+            '# If any filenames are emitted by the below script, run the generator command `grafana-cli cue gen-ts` locally and commit the result.',
+            './scripts/clean-git-or-error.sh',
+        ],
+    }