diff --git a/history-substring-search.zsh b/history-substring-search.zsh
index 2759139..b5b5706 100644
--- a/history-substring-search.zsh
+++ b/history-substring-search.zsh
@@ -137,6 +137,12 @@ if [[ $+functions[_zsh_highlight] -eq 0 ]]; then
         accept-and-menu-complete)
           eval "$event() { builtin zle .$event && _zsh_highlight } ; zle -N $event"
           ;;
+
+        # The following widgets should NOT remove any previously
+        # applied highlighting.  Therefore we do not remap them.
+        .forward-char|.backward-char|.up-line-or-history|.down-line-or-history)
+          ;;
+
         .*)
           clean_event=$event[2,${#event}] # Remove the leading dot in the event name
           case ${widgets[$clean_event]-} in