Improve handling of old/slow-to-start remote renderer (#40492)

* Assume the remote renderer is old if it returns 404 to the version endpoint

* Retry fetch of remote image renderer version on failure

Co-authored-by: Agnès Toulet <35176601+AgnesToulet@users.noreply.github.com>

pkg/services/rendering/rendering.go

@@ -10,6 +10,7 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -43,6 +44,7 @@ type RenderingService struct {
 	domain          string
 	inProgressCount int32
 	version         string
+	versionMutex    sync.RWMutex
 
 	Cfg                   *setting.Cfg
 	RemoteCacheService    *remotecache.RemoteCache
@@ -93,13 +95,18 @@ func (rs *RenderingService) Run(ctx context.Context) error {
 	if rs.remoteAvailable() {
 		rs.log = rs.log.New("renderer", "http")
 
-		version, err := rs.getRemotePluginVersion()
-		if err != nil {
-			rs.log.Info("Couldn't get remote renderer version", "err", err)
-		}
+		rs.getRemotePluginVersionWithRetry(func(version string, err error) {
+			if err != nil {
+				rs.log.Info("Couldn't get remote renderer version", "err", err)
+			}
 
-		rs.log.Info("Backend rendering via external http server", "version", version)
-		rs.version = version
+			rs.log.Info("Backend rendering via external http server", "version", version)
+
+			rs.versionMutex.Lock()
+			defer rs.versionMutex.Unlock()
+
+			rs.version = version
+		})
 		rs.renderAction = rs.renderViaHTTP
 		rs.renderCSVAction = rs.renderCSVViaHTTP
 		<-ctx.Done()
@@ -153,6 +160,9 @@ func (rs *RenderingService) IsAvailable() bool {
 }
 
 func (rs *RenderingService) Version() string {
+	rs.versionMutex.RLock()
+	defer rs.versionMutex.RUnlock()
+
 	return rs.version
 }