diff --git a/public/app/plugins/panel/timeseries/plugins/ContextMenuPlugin.tsx b/public/app/plugins/panel/timeseries/plugins/ContextMenuPlugin.tsx
index ee6573ff7cf..d3e06eaf2d9 100644
--- a/public/app/plugins/panel/timeseries/plugins/ContextMenuPlugin.tsx
+++ b/public/app/plugins/panel/timeseries/plugins/ContextMenuPlugin.tsx
@@ -1,4 +1,4 @@
-import React, { useCallback, useLayoutEffect, useMemo, useRef, useState } from 'react';
+import React, { useLayoutEffect, useMemo, useRef, useState } from 'react';
 import { useClickAway } from 'react-use';
 
 import { CartesianCoords2D, DataFrame, getFieldDisplayName, InterpolateFunction, TimeZone } from '@grafana/data';
@@ -11,7 +11,6 @@ import {
   MenuItem,
   UPlotConfigBuilder,
 } from '@grafana/ui';
-import { pluginLog } from '@grafana/ui/src/components/uPlot/utils';
 
 type ContextMenuSelectionCoords = { viewport: CartesianCoords2D; plotCanvas: CartesianCoords2D };
 type ContextMenuSelectionPoint = { seriesIdx: number | null; dataIdx: number | null };
@@ -39,108 +38,37 @@ export const ContextMenuPlugin = ({
   replaceVariables,
   ...otherProps
 }: ContextMenuPluginProps) => {
-  const plotCanvas = useRef<HTMLDivElement>();
   const [coords, setCoords] = useState<ContextMenuSelectionCoords | null>(null);
   const [point, setPoint] = useState<ContextMenuSelectionPoint | null>(null);
   const [isOpen, setIsOpen] = useState(false);
 
-  const openMenu = useCallback(() => {
-    setIsOpen(true);
-  }, [setIsOpen]);
-
-  const closeMenu = useCallback(() => {
-    setIsOpen(false);
-  }, [setIsOpen]);
-
-  const clearSelection = useCallback(() => {
-    pluginLog('ContextMenuPlugin', false, 'clearing click selection');
-    setPoint(null);
-  }, [setPoint]);
-
-  // Add uPlot hooks to the config, or re-add when the config changed
   useLayoutEffect(() => {
-    let bbox: DOMRect | undefined = undefined;
-
-    const onMouseCapture = (e: MouseEvent) => {
-      let update = {
-        viewport: {
-          x: e.clientX,
-          y: e.clientY,
-        },
-        plotCanvas: {
-          x: 0,
-          y: 0,
-        },
-      };
-      if (bbox) {
-        update = {
-          ...update,
-          plotCanvas: {
-            x: e.clientX - bbox.left,
-            y: e.clientY - bbox.top,
-          },
-        };
-      }
-      setCoords(update);
-    };
-
-    // cache uPlot plotting area bounding box
-    config.addHook('syncRect', (u, rect) => {
-      bbox = rect;
-    });
+    let seriesIdx: number | null = null;
 
     config.addHook('init', (u) => {
-      const canvas = u.over;
-      plotCanvas.current = canvas || undefined;
-      plotCanvas.current?.addEventListener('mousedown', onMouseCapture);
-
-      pluginLog('ContextMenuPlugin', false, 'init');
-      // for naive click&drag check
-      let isClick = false;
-
-      // REF: https://github.com/leeoniya/uPlot/issues/239
-      let pts = Array.from(u.root.querySelectorAll<HTMLDivElement>('.u-cursor-pt'));
-
-      plotCanvas.current?.addEventListener('mousedown', () => {
-        isClick = true;
-      });
-
-      plotCanvas.current?.addEventListener('mousemove', () => {
-        isClick = false;
-      });
-
-      // TODO: remove listeners on unmount
-      plotCanvas.current?.addEventListener('mouseup', (e: MouseEvent) => {
-        // ignore cmd+click, this is handled by annotation editor
-        if (!isClick || e.metaKey || e.ctrlKey) {
-          setPoint(null);
-          return;
-        }
-        isClick = true;
-
-        if (e.target instanceof HTMLElement) {
-          if (!e.target.classList.contains('u-cursor-pt')) {
-            pluginLog('ContextMenuPlugin', false, 'canvas click');
-            setPoint({ seriesIdx: null, dataIdx: null });
-          }
-        }
-
-        openMenu();
-      });
-
-      if (pts.length > 0) {
-        pts.forEach((pt, i) => {
-          // TODO: remove listeners on unmount
-          pt.addEventListener('click', () => {
-            const seriesIdx = i + 1;
-            const dataIdx = u.cursor.idx;
-            pluginLog('ContextMenuPlugin', false, seriesIdx, dataIdx);
-            setPoint({ seriesIdx, dataIdx: dataIdx ?? null });
+      u.over.addEventListener('click', (e) => {
+        // only open when have a focused point, and not for explicit annotations, zooms, etc.
+        if (seriesIdx != null && !e.metaKey && !e.ctrlKey && !e.shiftKey) {
+          setCoords({
+            viewport: {
+              x: e.clientX,
+              y: e.clientY,
+            },
+            plotCanvas: {
+              x: e.clientX - u.rect.left,
+              y: e.clientY - u.rect.top,
+            },
           });
-        });
-      }
+          setPoint({ seriesIdx, dataIdx: u.cursor.idxs![seriesIdx] });
+          setIsOpen(true);
+        }
+      });
     });
-  }, [config, openMenu, setCoords, setPoint]);
+
+    config.addHook('setSeries', (u, _seriesIdx) => {
+      seriesIdx = _seriesIdx;
+    });
+  }, [config]);
 
   const defaultItems = useMemo(() => {
     return otherProps.defaultItems
@@ -175,8 +103,8 @@ export const ContextMenuPlugin = ({
           selection={{ point, coords }}
           replaceVariables={replaceVariables}
           onClose={() => {
-            clearSelection();
-            closeMenu();
+            setPoint(null);
+            setIsOpen(false);
             if (onClose) {
               onClose();
             }
diff --git a/public/app/plugins/panel/trend/TrendPanel.tsx b/public/app/plugins/panel/trend/TrendPanel.tsx
index 1241485acc1..371b0679a65 100644
--- a/public/app/plugins/panel/trend/TrendPanel.tsx
+++ b/public/app/plugins/panel/trend/TrendPanel.tsx
@@ -14,7 +14,6 @@ import {
 import { XYFieldMatchers } from '@grafana/ui/src/components/GraphNG/types';
 import { findFieldIndex } from 'app/features/dimensions';
 
-import { ContextMenuPlugin } from '../timeseries/plugins/ContextMenuPlugin';
 import { prepareGraphableFields, regenerateLinksSupplier } from '../timeseries/utils';
 
 import { Options } from './panelcfg.gen';
@@ -137,15 +136,6 @@ export const TrendPanel = ({
                 timeZone={timeZone}
               />
             )}
-
-            <ContextMenuPlugin
-              data={alignedDataFrame}
-              frames={info.frames!}
-              config={config}
-              timeZone={timeZone}
-              replaceVariables={replaceVariables}
-              defaultItems={[]}
-            />
           </>
         );
       }}