Unistore Chore: Add database-level observability (#92266)

* add testing harness

* fix mockery and linters

* WIP

* wip

* fix transactions

* fix transaction tracing; add tracing by default

* rename package

* move WithTx to simplify logic of DB implementations

* fix potential issue with context deadline

* add db instrumentation to dbutil

* add otel tests

* improve naming

* minor fix in semantics and add comprehensive OTel testing

* fix naming

* instrument resourceVersionAtomicInc

* provide a default testing tracer

* fix docs

* fix typo in docs

* add semconv for k8s

pkg/storage/unified/sql/db/dbimpl/dbimpl.go

@@ -7,9 +7,9 @@ import (
 	"sync"
 
 	"github.com/dlmiddlecote/sqlstats"
-	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.opentelemetry.io/otel/trace"
+	"go.opentelemetry.io/otel/trace/noop"
 	"xorm.io/xorm"
 
 	infraDB "github.com/grafana/grafana/pkg/infra/db"
@@ -17,6 +17,7 @@ import (
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/storage/unified/sql/db"
 	"github.com/grafana/grafana/pkg/storage/unified/sql/db/migrations"
+	"github.com/grafana/grafana/pkg/storage/unified/sql/db/otel"
 )
 
 const (
@@ -33,7 +34,10 @@ var errGrafanaDBInstrumentedNotSupported = errors.New("the Resource API is " +
 	grafanaDBInstrumentQueriesKey + "` is enabled in [database], and that" +
 	" setup is currently unsupported. Please, consider disabling that flag")
 
-func ProvideResourceDB(grafanaDB infraDB.DB, cfg *setting.Cfg, tracer tracing.Tracer) (db.DBProvider, error) {
+func ProvideResourceDB(grafanaDB infraDB.DB, cfg *setting.Cfg, tracer trace.Tracer) (db.DBProvider, error) {
+	if tracer == nil {
+		tracer = noop.NewTracerProvider().Tracer("test-tracer")
+	}
 	p, err := newResourceDBProvider(grafanaDB, cfg, tracer)
 	if err != nil {
 		return nil, fmt.Errorf("provide Resource DB: %w", err)
@@ -148,6 +152,7 @@ func (p *resourceDBProvider) init(ctx context.Context) (db.DB, error) {
 	}
 
 	d := NewDB(p.engine.DB().DB, p.engine.Dialect().DriverName())
+	d = otel.NewInstrumentedDB(d, p.tracer)
 
 	return d, nil
 }