Use lazy enumeration to keep response time fast.

With HISTORY_SUBSTRING_SEARCH_ENSURE_UNIQUE, it can take a while to compute
all unique search results, which would normally keep the shell unresponsive
during this computation.

This commit separates the retrieval of all history matches from the
computation of the matches that need to be presented to the user (unique
matches only if HISTORY_SUBSTRING_SEARCH_ENSURE_UNIQUE is set). The
computation of the matches needing to be displayed is then done lazily,
and only when the user requests a new previously undisplayed result.
This keeps the response time fast while still enabling us to filter for
unique search results.

README.md

@@ -102,13 +102,19 @@ default values only after having loaded this script into your ZSH session.
   Flags" section in the zshexpn(1) man page to learn about the kinds of
   values you may assign to this variable.
 
-To always receive _unique_ search results, use `setopt HIST_IGNORE_ALL_DUPS`.
+* HISTORY_SUBSTRING_SEARCH_ENSURE_UNIQUE is a global variable that defines
-Alternatively, use `setopt HIST_FIND_NO_DUPS` which makes this plugin skip
+  whether all search results returned are _unique_. If set to a non-empty
-duplicate _adjacent_ search results as you cycle through them---however, this
+  value, then only unique search results are presented. It is unset by
-does not guarantee that search results are unique: if your search results were
+  default. An alternative way to ensure that search results are unique is
-"Dog", "Dog", "HotDog", "Dog", then cycling them gives "Dog", "HotDog", "Dog".
+  to use `setopt HIST_IGNORE_ALL_DUPS`. If both this configuration variable
-Notice that the "Dog" search result appeared twice as you cycled through them!
+  and `setopt HIST_IGNORE_ALL_DUPS` are unset, then `setopt HIST_FIND_NO_DUPS`
-If you wish to avoid this limitation, then use `setopt HIST_IGNORE_ALL_DUPS`.
+  is still respected, and makes this plugin skip duplicate _adjacent_ search
+  results as you cycle through them, but this does not guarantee that search
+  results are unique: if your search results were "Dog", "Dog", "HotDog",
+  "Dog", then cycling them gives "Dog", "HotDog", "Dog". Notice that the "Dog"
+  search result appeared twice as you cycled through them. If you wish to
+  receive globally unique search results only once, then set this
+  configuration variable, or use `setopt HIST_IGNORE_ALL_DUPS`.
 
 ------------------------------------------------------------------------------
 History

zsh-history-substring-search.zsh

@@ -6,6 +6,7 @@
 # Copyright (c) 2011 Suraj N. Kurapati
 # Copyright (c) 2011 Sorin Ionescu
 # Copyright (c) 2011 Vincent Guerci
+# Copyright (c) 2016 Geza Lore
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -196,7 +197,7 @@ _history-substring-search-begin() {
     # speed things up a little.
     #
     _history_substring_search_query=
-    _history_substring_search_matches=()
+    _history_substring_search_raw_matches=()
 
   else
     #
@@ -219,65 +220,36 @@ _history-substring-search-begin() {
     # (R) returns values in reverse older, so the index of the youngest
     # matching history entry is at the head of the list.
     #
-    _history_substring_search_matches=(${(k)history[(R)(#$HISTORY_SUBSTRING_SEARCH_GLOBBING_FLAGS)*${escaped_query}*]})
+    _history_substring_search_raw_matches=(${(k)history[(R)(#$HISTORY_SUBSTRING_SEARCH_GLOBBING_FLAGS)*${escaped_query}*]})
-
-    #
-    # If HISTORY_SUBSTRING_SEARCH_ENSURE_UNIQUE is set to a non-empty value,
-    # then ensure that only unique matches are presented to the user.
-    # When HIST_IGNORE_ALL_DUPS is set, ZSH already ensures a unique history,
-    # so in this case we do not need to do anything.
-    #
-    if [[ ! -o HIST_IGNORE_ALL_DUPS && -n "$HISTORY_SUBSTRING_SEARCH_ENSURE_UNIQUE" ]]; then
-      #
-      # We use an associative array (filter) as a 'set' data structure to
-      # collect only unique matches. If an entry (key) is already in the
-      # set (non-empty value), then we have already seen it before and will
-      # ignore it. Otherwise we add the entry both to the set and the list
-      # of unique matches
-      #
-      local -A filter
-      local -a unique_matches
-
-      #
-      # Iterate through all matching history indices in younger to older
-      # order. This ensures that younger entries are always presented before
-      # older entries.
-      #
-      for index in $_history_substring_search_matches; do
-        #
-        # Get that actual history entry at this index
-        #
-        local entry=${history[$index]}
-
-        if [[ -z ${filter[$entry]} ]]; then
-          #
-          # This is a new unique entry. Add it to the filter and the list
-          # of unique matches.
-          #
-          filter[$entry]=1
-          unique_matches+=($index)
-        fi
-      done
-
-      #
-      # Use the unique entries only
-      #
-      _history_substring_search_matches=($unique_matches)
-    fi
   fi
 
   #
-  # Define the range of values that $_history_substring_search_match_index
+  # In order to stay as responsive as possible, we will process the raw
-  # can take: [0, $_history_substring_search_matches_count + 1].
+  # matches lazily (when the user requests the next match) to choose items
+  # that need to be displayed to the user.
+  # _history_substring_search_raw_match_index holds the index of the last
+  # unprocessed entry in _history_substring_search_raw_matches. Any items
+  # that need to be displayed will be added to
+  # _history_substring_search_matches.
   #
-  _history_substring_search_matches_count=$#_history_substring_search_matches
+  # We use an associative array (_history_substring_search_unique_filter) as
+  # a 'set' data structure to ensure uniqueness of the results if desired.
+  # If an entry (key) is in the set (non-empty value), then we have already
+  # added that entry to _history_substring_search_matches.
+  #
+  _history_substring_search_raw_match_index=0
+  _history_substring_search_matches=()
+  unset _history_substring_search_unique_filter
+  typeset -A -g _history_substring_search_unique_filter
 
   #
   # If $_history_substring_search_match_index is equal to
-  # $_history_substring_search_matches_count + 1, this indicates that we
+  # $#_history_substring_search_matches + 1, this indicates that we
-  # are beyond the end of $_history_substring_search_matches.
+  # are beyond the end of $_history_substring_search_matches and that we
+  # have also processed all entries in
+  # _history_substring_search_raw_matches.
   #
-  # If $_history_substring_search_match_index is equal to 0, this indicates
+  # If $#_history_substring_search_match_index is equal to 0, this indicates
   # that we are beyond the beginning of $_history_substring_search_matches.
   #
   # If we have initially pressed "up" we have to initialize
@@ -434,6 +406,119 @@ _history-substring-search-down-history() {
   return 1
 }
 
+_history_substring_search_process_raw_matches() {
+  #
+  # Process more outstanding raw matches and append any matches that need to
+  # be displayed to the user to _history_substring_search_matches.
+  # Return whether there were any more results appended.
+  #
+
+  #
+  # While we have more raw matches. Process them to see if there are any more
+  # matches that need to be displayed to the user.
+  #
+  while [[ $_history_substring_search_raw_match_index -lt $#_history_substring_search_raw_matches ]]; do
+    #
+    # Move on to the next raw entry and get its history index.
+    #
+    (( _history_substring_search_raw_match_index++ ))
+    local index=${_history_substring_search_raw_matches[$_history_substring_search_raw_match_index]}
+
+    #
+    # If HISTORY_SUBSTRING_SEARCH_ENSURE_UNIQUE is set to a non-empty value,
+    # then ensure that only unique matches are presented to the user.
+    # When HIST_IGNORE_ALL_DUPS is set, ZSH already ensures a unique history,
+    # so in this case we do not need to do anything.
+    #
+    if [[ ! -o HIST_IGNORE_ALL_DUPS && -n $HISTORY_SUBSTRING_SEARCH_ENSURE_UNIQUE ]]; then
+      #
+      # Get the actual history entry at the new index, and check if we have
+      # already added it to _history_substring_search_matches.
+      #
+      local entry=${history[$index]}
+
+      if [[ -z ${_history_substring_search_unique_filter[$entry]} ]]; then
+        #
+        # This is a new unique entry. Add it to the filter and append the
+        # index to _history_substring_search_matches.
+        #
+        _history_substring_search_unique_filter[$entry]=1
+        _history_substring_search_matches+=($index)
+
+        #
+        # Indicate that we did find a match.
+        #
+        return 0
+      fi
+
+    else
+      #
+      # Just append the new history index to the processed matches.
+      #
+      _history_substring_search_matches+=($index)
+
+      #
+      # Indicate that we did find a match.
+      #
+      return 0
+    fi
+
+  done
+
+  #
+  # We are beyond the end of the list of raw matches. Indicate that no
+  # more matches are available.
+  #
+  return 1
+}
+
+_history-substring-search-has-next() {
+  #
+  # Predicate function that returns whether any more older matches are
+  # available.
+  #
+
+  if  [[ $_history_substring_search_match_index -lt $#_history_substring_search_matches ]]; then
+    #
+    # We did not reach the end of the processed list, so we do have further
+    # matches.
+    #
+    return 0
+
+  else
+    #
+    # We are at the end of the processed list. Try to process further
+    # unprocessed matches. _history_substring_search_process_raw_matches
+    # returns whether any more matches were available, so just return
+    # that result.
+    #
+    _history_substring_search_process_raw_matches
+    return $?
+  fi
+}
+
+_history-substring-search-has-prev() {
+  #
+  # Predicate function that returns whether any more younger matches are
+  # available.
+  #
+
+  if [[ $_history_substring_search_match_index -gt 1 ]]; then
+    #
+    # We did not reach the beginning of the processed list, so we do have
+    # further matches.
+    #
+    return 0
+
+  else
+    #
+    # We are at the beginning of the processed list. We do not have any more
+    # matches.
+    #
+    return 1
+  fi
+}
+
 _history-substring-search-found() {
   #
   # A match is available. The index of the match is held in
@@ -466,66 +551,84 @@ _history-substring-search-up-search() {
   _history_substring_search_refresh_display=1
 
   #
-  # Highlight matches during history-substring-up-search:
+  # Select history entry during history-substring-down-search:
   #
-  # The following constants have been initialized in
+  # The following variables have been initialized in
   # _history-substring-search-up/down-search():
   #
-  # $_history_substring_search_matches is the current list of matches
+  # $_history_substring_search_matches is the current list of matches that
-  # $_history_substring_search_matches_count is the current number of matches
+  # need to be displayed to the user.
   # $_history_substring_search_match_index is the index of the current match
+  # that is being displayed to the user.
   #
   # The range of values that $_history_substring_search_match_index can take
-  # is: [0, $_history_substring_search_matches_count + 1].  A value of 0
+  # is: [0, $#_history_substring_search_matches + 1].  A value of 0
   # indicates that we are beyond the beginning of
   # $_history_substring_search_matches. A value of
-  # $_history_substring_search_matches_count + 1 indicates that we are beyond
+  # $#_history_substring_search_matches + 1 indicates that we are beyond
-  # the end of $_history_substring_search_matches.
+  # the end of $_history_substring_search_matches and that we have also
+  # processed all entries in _history_substring_search_raw_matches.
+  #
+  # If $_history_substring_search_match_index equals
+  # $#_history_substring_search_matches and
+  # $_history_substring_search_raw_match_index is not greater than
+  # $#_history_substring_search_raw_matches, then we need to further process
+  # $_history_substring_search_raw_matches to see if there are any more
+  # entries that need to be displayed to the user.
   #
   # In _history-substring-search-up-search() the initial value of
   # $_history_substring_search_match_index is 0. This value is set in
   # _history-substring-search-begin(). _history-substring-search-up-search()
   # will initially increment it to 1.
   #
-  if [[ $_history_substring_search_match_index -lt $_history_substring_search_matches_count ]]; then
-    #
-    # Pick the next (older) match:
-    #
-    # 1. Increment the value of $_history_substring_search_match_index.
-    #
-    # 2. Update display (search found)
-    #
-    (( _history_substring_search_match_index++ ))
-    _history-substring-search-found
 
-  elif [[ $_history_substring_search_match_index -eq $_history_substring_search_matches_count ]]; then
+  if [[ $_history_substring_search_match_index -gt $#_history_substring_search_matches ]]; then
     #
-    # We will move beyond the end of $_history_substring_search_matches:
+    # We are beyond the end of $_history_substring_search_matches. This
+    # can only happen if we have also exhausted the unprocessed matches in
+    # _history_substring_search_raw_matches.
     #
-    # 1. Increment the value of $_history_substring_search_match_index.
+    # 1. Update display to indicate search not found.
-    #
-    # 2. Update display (search not found)
-    #
-    (( _history_substring_search_match_index++ ))
-    _history-substring-search-not-found
-    return
-
-  else
-    #
-    # We are beyond the end of $_history_substring_search_matches.
-    #
-    # 1. Update display (search not found)
     #
     _history-substring-search-not-found
     return
   fi
 
+  if _history-substring-search-has-next; then
+    #
+    # We do have older matches.
+    #
+    # 1. Move index to point to the next match.
+    # 2. Update display to indicate search found.
+    #
+    (( _history_substring_search_match_index++ ))
+    _history-substring-search-found
+
+  else
+    #
+    # We do not have older matches.
+    #
+    # 1. Move the index beyond the end of
+    #    _history_substring_search_matches.
+    # 2. Update display to indicate search not found.
+    #
+    (( _history_substring_search_match_index++ ))
+    _history-substring-search-not-found
+  fi
+
   #
   # When HIST_FIND_NO_DUPS is set, meaning that only unique command lines from
   # history should be matched, make sure the new and old results are different.
-  # But when HIST_IGNORE_ALL_DUPS is set, ZSH already ensures a unique history.
   #
-  if [[ ! -o HIST_IGNORE_ALL_DUPS && -o HIST_FIND_NO_DUPS && $BUFFER == $_history_substring_search_result ]]; then
+  # However, if the HIST_IGNORE_ALL_DUPS shell option, or
+  # HISTORY_SUBSTRING_SEARCH_ENSURE_UNIQUE is set, then we already have a
+  # unique history, so in this case we do not need to do anything.
+  #
+  if [[ -o HIST_IGNORE_ALL_DUPS || -n $HISTORY_SUBSTRING_SEARCH_ENSURE_UNIQUE ]]; then
+    return
+  fi
+
+  if [[ -o HIST_FIND_NO_DUPS && $BUFFER == $_history_substring_search_result ]]; then
     #
     # Repeat the current search so that a different (unique) match is found.
     #
@@ -537,66 +640,75 @@ _history-substring-search-down-search() {
   _history_substring_search_refresh_display=1
 
   #
-  # Highlight matches during history-substring-down-search:
+  # Select history entry during history-substring-down-search:
   #
-  # The following constants have been initialized in
+  # The following variables have been initialized in
   # _history-substring-search-up/down-search():
   #
-  # $_history_substring_search_matches is the current list of matches
+  # $_history_substring_search_matches is the current list of matches that
-  # $_history_substring_search_matches_count is the current number of matches
+  # need to be displayed to the user.
   # $_history_substring_search_match_index is the index of the current match
+  # that is being displayed to the user.
   #
   # The range of values that $_history_substring_search_match_index can take
-  # is: [0, $_history_substring_search_matches_count + 1].  A value of 0
+  # is: [0, $#_history_substring_search_matches + 1].  A value of 0
   # indicates that we are beyond the beginning of
   # $_history_substring_search_matches. A value of
-  # $_history_substring_search_matches_count + 1 indicates that we are beyond
+  # $#_history_substring_search_matches + 1 indicates that we are beyond
-  # the end of $_history_substring_search_matches.
+  # the end of $_history_substring_search_matches and that we have also
+  # processed all entries in _history_substring_search_raw_matches.
   #
   # In _history-substring-search-down-search() the initial value of
   # $_history_substring_search_match_index is 1. This value is set in
   # _history-substring-search-begin(). _history-substring-search-down-search()
   # will initially decrement it to 0.
   #
-  if [[ $_history_substring_search_match_index -gt 1 ]]; then
-    #
-    # Highlight the previous (younger) match:
-    #
-    # 1. Decrement $_history_substring_search_match_index.
-    #
-    # 2. Update display (search found)
-    #
-    (( _history_substring_search_match_index-- ))
-    _history-substring-search-found
 
-  elif [[ $_history_substring_search_match_index -eq 1 ]]; then
+  if [[ $_history_substring_search_match_index -lt 1 ]]; then
-    #
-    # We will move beyond the beginning of $_history_substring_search_matches:
-    #
-    # 1. Decrement $_history_substring_search_match_index.
-    #
-    # 2. Update display (search not found)
-    #
-    (( _history_substring_search_match_index-- ))
-    _history-substring-search-not-found
-    return
-
-  else
     #
     # We are beyond the beginning of $_history_substring_search_matches.
     #
-    # 1. Update display (search not found)
+    # 1. Update display to indicate search not found.
     #
     _history-substring-search-not-found
     return
   fi
 
+  if _history-substring-search-has-prev; then
+    #
+    # We do have younger matches.
+    #
+    # 1. Move index to point to the previous match.
+    # 2. Update display to indicate search found.
+    #
+    (( _history_substring_search_match_index-- ))
+    _history-substring-search-found
+
+  else
+    #
+    # We do not have younger matches.
+    #
+    # 1. Move the index beyond the beginning of
+    #    _history_substring_search_matches.
+    # 2. Update display to indicate search not found.
+    #
+    (( _history_substring_search_match_index-- ))
+    _history-substring-search-not-found
+  fi
+
   #
   # When HIST_FIND_NO_DUPS is set, meaning that only unique command lines from
   # history should be matched, make sure the new and old results are different.
-  # But when HIST_IGNORE_ALL_DUPS is set, ZSH already ensures a unique history.
   #
-  if [[ ! -o HIST_IGNORE_ALL_DUPS && -o HIST_FIND_NO_DUPS && $BUFFER == $_history_substring_search_result ]]; then
+  # However, if the HIST_IGNORE_ALL_DUPS shell option, or
+  # HISTORY_SUBSTRING_SEARCH_ENSURE_UNIQUE is set, then we already have a
+  # unique history, so in this case we do not need to do anything.
+  #
+  if [[ -o HIST_IGNORE_ALL_DUPS || -n $HISTORY_SUBSTRING_SEARCH_ENSURE_UNIQUE ]]; then
+    return
+  fi
+
+  if [[ -o HIST_FIND_NO_DUPS && $BUFFER == $_history_substring_search_result ]]; then
     #
     # Repeat the current search so that a different (unique) match is found.
     #