Chore: Logging improvements (#44925) (#44996)

Fixing a couple bugs, adds some tests and hopefully decrease
lock contention when logging.
Switching from using sync.RWMutex to go-kit SwapLogger.
Fixes bug when creating a new logger from an existing one that
screwed up the keyvals and/or lost the logger name.

Ref #44681

(cherry picked from commit 05ea825c76)

Co-authored-by: Marcus Efraimsson <marcus.efraimsson@gmail.com>

pkg/api/frontend_logging_test.go

@@ -42,11 +42,11 @@ func logSentryEventScenario(t *testing.T, desc string, event frontendlogging.Fro
 		}))
 
 		origHandler := frontendLogger.GetLogger()
-		frontendLogger.SetLogger(level.NewFilter(newfrontendLogger, level.AllowInfo()))
+		frontendLogger.Swap(level.NewFilter(newfrontendLogger, level.AllowInfo()))
 		sourceMapReads := []SourceMapReadRecord{}
 
 		t.Cleanup(func() {
-			frontendLogger.SetLogger(origHandler)
+			frontendLogger.Swap(origHandler)
 		})
 
 		sc := setupScenarioContext(t, "/log")