diff --git a/packages/grafana-data/src/panel/PanelPlugin.ts b/packages/grafana-data/src/panel/PanelPlugin.ts
index dfbb009efdc..b4e56fb52d6 100644
--- a/packages/grafana-data/src/panel/PanelPlugin.ts
+++ b/packages/grafana-data/src/panel/PanelPlugin.ts
@@ -375,4 +375,8 @@ export class PanelPlugin<
   getSuggestionsSupplier(): VisualizationSuggestionsSupplier | undefined {
     return this.suggestionsSupplier;
   }
+
+  hasPluginId(pluginId: string) {
+    return this.meta.id === pluginId;
+  }
 }
diff --git a/packages/grafana-runtime/src/components/PanelRenderer.tsx b/packages/grafana-runtime/src/components/PanelRenderer.tsx
index 374ac9bb5fe..8c0619ba3ba 100644
--- a/packages/grafana-runtime/src/components/PanelRenderer.tsx
+++ b/packages/grafana-runtime/src/components/PanelRenderer.tsx
@@ -16,6 +16,7 @@ export interface PanelRendererProps<P extends object = any, F extends object = a
   title: string;
   options?: Partial<P>;
   onOptionsChange?: (options: P) => void;
+  onFieldConfigChange?: (config: FieldConfigSource<F>) => void;
   onChangeTimeRange?: (timeRange: AbsoluteTimeRange) => void;
   fieldConfig?: FieldConfigSource<Partial<F>>;
   timeZone?: string;
diff --git a/public/app/features/panel/components/PanelRenderer.tsx b/public/app/features/panel/components/PanelRenderer.tsx
index 98a86b3a209..2c0a9a55aee 100644
--- a/public/app/features/panel/components/PanelRenderer.tsx
+++ b/public/app/features/panel/components/PanelRenderer.tsx
@@ -1,5 +1,4 @@
 import React, { useState, useMemo, useEffect, useRef } from 'react';
-import { useAsync } from 'react-use';
 
 import { applyFieldOverrides, FieldConfigSource, getTimeZone, PanelData, PanelPlugin } from '@grafana/data';
 import { PanelRendererProps } from '@grafana/runtime';
@@ -7,7 +6,7 @@ import { ErrorBoundaryAlert, useTheme2 } from '@grafana/ui';
 import { appEvents } from 'app/core/core';
 
 import { getPanelOptionsWithDefaults, OptionDefaults } from '../../dashboard/state/getPanelOptionsWithDefaults';
-import { importPanelPlugin } from '../../plugins/importPanelPlugin';
+import { importPanelPlugin, syncGetPanelPlugin } from '../../plugins/importPanelPlugin';
 
 const defaultFieldConfig = { defaults: {}, overrides: [] };
 
@@ -22,27 +21,38 @@ export function PanelRenderer<P extends object = any, F extends object = any>(pr
     title,
     onOptionsChange = () => {},
     onChangeTimeRange = () => {},
-    fieldConfig: externalFieldConfig = defaultFieldConfig,
+    onFieldConfigChange = () => {},
+    fieldConfig = defaultFieldConfig,
   } = props;
 
-  const [localFieldConfig, setFieldConfig] = useState(externalFieldConfig);
-  const { value: plugin, error, loading } = useAsync(() => importPanelPlugin(pluginId), [pluginId]);
-  const optionsWithDefaults = useOptionDefaults(plugin, options, localFieldConfig);
+  const [plugin, setPlugin] = useState(syncGetPanelPlugin(pluginId));
+  const [error, setError] = useState<string | undefined>();
+  const optionsWithDefaults = useOptionDefaults(plugin, options, fieldConfig);
   const dataWithOverrides = useFieldOverrides(plugin, optionsWithDefaults, data, timeZone);
 
   useEffect(() => {
-    setFieldConfig((lfc) => ({ ...lfc, ...externalFieldConfig }));
-  }, [externalFieldConfig]);
+    // If we already have a plugin and it's correct one do nothing
+    if (plugin && plugin.hasPluginId(pluginId)) {
+      return;
+    }
+
+    // Async load the plugin
+    importPanelPlugin(pluginId)
+      .then((result) => setPlugin(result))
+      .catch((err: Error) => {
+        setError(err.message);
+      });
+  }, [pluginId, plugin]);
 
   if (error) {
-    return <div>Failed to load plugin: {error.message}</div>;
+    return <div>Failed to load plugin: {error}</div>;
   }
 
-  if (pluginIsLoading(loading, plugin, pluginId)) {
+  if (!plugin || !plugin.hasPluginId(pluginId)) {
     return <div>Loading plugin panel...</div>;
   }
 
-  if (!plugin || !plugin.panel) {
+  if (!plugin.panel) {
     return <div>Seems like the plugin you are trying to load does not have a panel component.</div>;
   }
 
@@ -61,14 +71,14 @@ export function PanelRenderer<P extends object = any, F extends object = any>(pr
         timeRange={dataWithOverrides.timeRange}
         timeZone={timeZone}
         options={optionsWithDefaults!.options}
-        fieldConfig={localFieldConfig}
+        fieldConfig={fieldConfig}
         transparent={false}
         width={width}
         height={height}
         renderCounter={0}
         replaceVariables={(str: string) => str}
         onOptionsChange={onOptionsChange}
-        onFieldConfigChange={setFieldConfig}
+        onFieldConfigChange={onFieldConfigChange}
         onChangeTimeRange={onChangeTimeRange}
         eventBus={appEvents}
       />
@@ -127,7 +137,3 @@ function useFieldOverrides(
     };
   }, [fieldConfigRegistry, fieldConfig, data, series, timeZone, theme]);
 }
-
-function pluginIsLoading(loading: boolean, plugin: PanelPlugin<any, any> | undefined, pluginId: string) {
-  return loading || plugin?.meta.id !== pluginId;
-}
diff --git a/public/app/features/plugins/importPanelPlugin.ts b/public/app/features/plugins/importPanelPlugin.ts
index 042db64bae1..b6994de5fb7 100644
--- a/public/app/features/plugins/importPanelPlugin.ts
+++ b/public/app/features/plugins/importPanelPlugin.ts
@@ -1,16 +1,15 @@
-import * as grafanaData from '@grafana/data';
+import { PanelPlugin, PanelPluginMeta } from '@grafana/data';
 import config from 'app/core/config';
 
 import { getPanelPluginLoadError } from '../panel/components/PanelPluginError';
 
 import { importPluginModule } from './plugin_loader';
-interface PanelCache {
-  [key: string]: Promise<grafanaData.PanelPlugin>;
-}
-const panelCache: PanelCache = {};
 
-export function importPanelPlugin(id: string): Promise<grafanaData.PanelPlugin> {
-  const loaded = panelCache[id];
+const promiseCache: Record<string, Promise<PanelPlugin>> = {};
+const panelPluginCache: Record<string, PanelPlugin> = {};
+
+export function importPanelPlugin(id: string): Promise<PanelPlugin> {
+  const loaded = promiseCache[id];
   if (loaded) {
     return loaded;
   }
@@ -21,22 +20,26 @@ export function importPanelPlugin(id: string): Promise<grafanaData.PanelPlugin>
     throw new Error(`Plugin ${id} not found`);
   }
 
-  panelCache[id] = getPanelPlugin(meta);
+  promiseCache[id] = getPanelPlugin(meta);
 
-  return panelCache[id];
+  return promiseCache[id];
 }
 
-export function importPanelPluginFromMeta(meta: grafanaData.PanelPluginMeta): Promise<grafanaData.PanelPlugin> {
+export function importPanelPluginFromMeta(meta: PanelPluginMeta): Promise<PanelPlugin> {
   return getPanelPlugin(meta);
 }
 
-function getPanelPlugin(meta: grafanaData.PanelPluginMeta): Promise<grafanaData.PanelPlugin> {
+export function syncGetPanelPlugin(id: string): PanelPlugin | undefined {
+  return panelPluginCache[id];
+}
+
+function getPanelPlugin(meta: PanelPluginMeta): Promise<PanelPlugin> {
   return importPluginModule(meta.module, meta.info?.version)
     .then((pluginExports) => {
       if (pluginExports.plugin) {
-        return pluginExports.plugin as grafanaData.PanelPlugin;
+        return pluginExports.plugin as PanelPlugin;
       } else if (pluginExports.PanelCtrl) {
-        const plugin = new grafanaData.PanelPlugin(null);
+        const plugin = new PanelPlugin(null);
         plugin.angularPanelCtrl = pluginExports.PanelCtrl;
         return plugin;
       }
@@ -44,6 +47,7 @@ function getPanelPlugin(meta: grafanaData.PanelPluginMeta): Promise<grafanaData.
     })
     .then((plugin) => {
       plugin.meta = meta;
+      panelPluginCache[meta.id] = plugin;
       return plugin;
     })
     .catch((err) => {