diff --git a/public/app/features/dashboard/dashgrid/SeriesVisibilityConfigFactory.ts b/public/app/features/dashboard/dashgrid/SeriesVisibilityConfigFactory.ts
index 190fcf6fd51..a03cc64bb5a 100644
--- a/public/app/features/dashboard/dashgrid/SeriesVisibilityConfigFactory.ts
+++ b/public/app/features/dashboard/dashgrid/SeriesVisibilityConfigFactory.ts
@@ -1,9 +1,11 @@
 import {
   ByNamesMatcherMode,
+  ConfigOverrideRule,
   DataFrame,
   DynamicConfigValue,
   FieldConfigSource,
   FieldMatcherID,
+  fieldMatchers,
   FieldType,
   getFieldDisplayName,
   isSystemOverrideWithRef,
@@ -27,7 +29,7 @@ export function seriesVisibilityConfigFactory(
 
   if (currentIndex < 0) {
     if (mode === SeriesVisibilityChangeMode.ToggleSelection) {
-      const override = createOverride([displayName]);
+      const override = createOverride([displayName, ...getNamesOfHiddenFields(overrides, data)]);
 
       return {
         ...fieldConfig,
@@ -48,7 +50,12 @@ export function seriesVisibilityConfigFactory(
   const [current] = overridesCopy.splice(currentIndex, 1) as SystemConfigOverrideRule[];
 
   if (mode === SeriesVisibilityChangeMode.ToggleSelection) {
-    const existing = getExistingDisplayNames(current);
+    let existing = getExistingDisplayNames(current);
+    const nameOfHiddenFields = getNamesOfHiddenFields(overridesCopy, data);
+
+    if (nameOfHiddenFields.length > 0) {
+      existing = existing.filter((el) => nameOfHiddenFields.indexOf(el) < 0);
+    }
 
     if (existing[0] === displayName && existing.length === 1) {
       return {
@@ -57,7 +64,7 @@ export function seriesVisibilityConfigFactory(
       };
     }
 
-    const override = createOverride([displayName]);
+    const override = createOverride([displayName, ...nameOfHiddenFields]);
 
     return {
       ...fieldConfig,
@@ -169,3 +176,32 @@ const getDisplayNames = (data: DataFrame[], excludeName?: string): string[] => {
 
   return Array.from(unique);
 };
+
+const getNamesOfHiddenFields = (overrides: ConfigOverrideRule[], data: DataFrame[]): string[] => {
+  let names: string[] = [];
+
+  for (const override of overrides) {
+    const property = override.properties.find((p) => p.id === 'custom.hideFrom');
+
+    if (property !== undefined && property.value?.legend === true) {
+      const info = fieldMatchers.get(override.matcher.id);
+      const matcher = info.get(override.matcher.options);
+
+      for (const frame of data) {
+        for (const field of frame.fields) {
+          if (field.type !== FieldType.number) {
+            continue;
+          }
+
+          const name = getFieldDisplayName(field, frame, data);
+
+          if (matcher(field, frame, data)) {
+            names.push(name);
+          }
+        }
+      }
+    }
+  }
+
+  return names;
+};