diff --git a/zsh-history-substring-search.zsh b/zsh-history-substring-search.zsh
index 333c966..1a9a13f 100644
--- a/zsh-history-substring-search.zsh
+++ b/zsh-history-substring-search.zsh
@@ -474,8 +474,9 @@ function _history-substring-search-up-search() {
   #
   # When HIST_FIND_NO_DUPS is set, meaning that only unique command lines from
   # history should be matched, make sure the new and old results are different.
+  # But when HIST_IGNORE_ALL_DUPS is set, ZSH already ensures a unique history.
   #
-  if [[ -o HIST_FIND_NO_DUPS && $BUFFER == $_history_substring_search_result ]]; then
+  if [[ ! -o HIST_IGNORE_ALL_DUPS && -o HIST_FIND_NO_DUPS && $BUFFER == $_history_substring_search_result ]]; then
     #
     # Repeat the current search so that a different (unique) match is found.
     #
@@ -570,8 +571,9 @@ function _history-substring-search-down-search() {
   #
   # When HIST_FIND_NO_DUPS is set, meaning that only unique command lines from
   # history should be matched, make sure the new and old results are different.
+  # But when HIST_IGNORE_ALL_DUPS is set, ZSH already ensures a unique history.
   #
-  if [[ -o HIST_FIND_NO_DUPS && $BUFFER == $_history_substring_search_result ]]; then
+  if [[ ! -o HIST_IGNORE_ALL_DUPS && -o HIST_FIND_NO_DUPS && $BUFFER == $_history_substring_search_result ]]; then
     #
     # Repeat the current search so that a different (unique) match is found.
     #