diff --git a/tmk_core/common/action.c b/tmk_core/common/action.c
index acc6d11ea..4457d16d8 100644
--- a/tmk_core/common/action.c
+++ b/tmk_core/common/action.c
@@ -55,7 +55,7 @@ void action_exec(keyevent_t event)
 
 #if !defined(NO_ACTION_LAYER) && defined(PREVENT_STUCK_MODIFIERS)
 bool disable_action_cache = false;
-int8_t pressed_actions_cache[MATRIX_ROWS][MATRIX_COLS];
+uint8_t source_layers_cache[5][((MATRIX_ROWS * MATRIX_COLS) / 8) ? ((MATRIX_ROWS * MATRIX_COLS) / 8) : 1];
 
 void process_action_nocache(keyrecord_t *record)
 {
@@ -82,11 +82,22 @@ action_t store_or_get_action(bool pressed, keypos_t key)
     if (disable_action_cache) {
         return layer_switch_get_action(key);
     }
-
+    uint8_t key_number = (key.col + (key.row * MATRIX_COLS));
+    uint8_t storage_row = key_number / 8;
+    uint8_t storage_bit = key_number % 8;
+    uint8_t layer;
     if (pressed) {
-        pressed_actions_cache[key.row][key.col] = layer_switch_get_layer(key);
+        layer = layer_switch_get_layer(key);
+        for (uint8_t bit_number = 0; bit_number <= 4; bit_number++) {
+            source_layers_cache[bit_number][storage_row] ^= (-(!!(layer & (1 << bit_number)) ^ source_layers_cache[bit_number][storage_row])) & (1 << storage_bit);
+        }
+    } else {
+        layer = 0;
+        for (uint8_t bit_number = 0; bit_number <= 4; bit_number++) {
+            layer |= (!!(source_layers_cache[bit_number][storage_row] & (1 << storage_bit))) << bit_number;
+        }
     }
-    return action_for_key(pressed_actions_cache[key.row][key.col], key);
+    return action_for_key(layer, key);
 #else
     return layer_switch_get_action(key);
 #endif