parameters-multi-word-machinery-v2

highlighters/main/main-highlighter.zsh

@@ -77,6 +77,16 @@ _zsh_highlight_main_add_region_highlight() {
     [[ $1 == unknown-token ]] && alias_style=unknown-token
     return
   fi
+  if (( in_param )); then
+    if [[ $1 == unknown-token ]]; then
+      param_style=unknown-token
+    fi
+    if [[ -n $param_style ]]; then
+      return
+    fi
+    param_style=$1
+    return
+  fi
 
   # The calculation was relative to $buf but region_highlight is relative to $BUFFER.
   (( start += buf_offset ))
@@ -406,11 +416,13 @@ _zsh_highlight_main_highlighter_highlight_list()
   # alias_style is the style to apply to an alias once in_alias=0
   #     Usually 'alias' but set to 'unknown-token' if any word expanded from
   #     the alias would be highlighted as unknown-token
-  local alias_style arg buf=$4 highlight_glob=true style
+  # param_style is analogous for parameter expansions
+  local alias_style param_style arg buf=$4 highlight_glob=true style
   local in_array_assignment=false # true between 'a=(' and the matching ')'
   # in_alias is equal to the number of shifts needed until arg=args[1] pops an
   #     arg from BUFFER and not added by an alias.
-  integer in_alias=0 len=$#buf
+  # in_param is analogous for parameter expansions
+  integer in_alias=0 in_param=0 len=$#buf
   local -a match mbegin mend list_highlights
   # seen_alias is a map of aliases already seen to avoid loops like alias a=b b=a
   local -A seen_alias
@@ -489,6 +501,14 @@ _zsh_highlight_main_highlighter_highlight_list()
         _zsh_highlight_main_add_region_highlight $start_pos $end_pos $alias_style
       fi
     fi
+    if (( in_param )); then
+      (( in_param-- ))
+      if (( in_param == 0 )); then
+        # start_pos and end_pos are of the alias (previous $arg) here
+        _zsh_highlight_main_add_region_highlight $start_pos $end_pos ${param_style:-"unknown-token"}
+        param_style=""
+      fi
+    fi
 
     # Initialize this_word and next_word.
     if (( in_redirection == 0 )); then
@@ -513,7 +533,7 @@ _zsh_highlight_main_highlighter_highlight_list()
       fi
     fi
 
-    if (( in_alias == 0 )); then
+    if (( in_alias == 0 && in_param == 0 )); then
       # Compute the new $start_pos and $end_pos, skipping over whitespace in $buf.
       [[ "$proc_buf" = (#b)(#s)(([ $'\t']|\\$'\n')#)* ]]
       # The first, outer parenthesis
@@ -626,6 +646,7 @@ _zsh_highlight_main_highlighter_highlight_list()
       local -a match mbegin mend
       local MATCH; integer MBEGIN MEND
       local parameter_name
+      local -a words
       if [[ $arg[1] == '$' ]] && [[ ${arg[2]} == '{' ]] && [[ ${arg[-1]} == '}' ]]; then
         parameter_name=${${arg:2}%?}
       elif [[ $arg[1] == '$' ]]; then
@@ -638,14 +659,16 @@ _zsh_highlight_main_highlighter_highlight_list()
         # Set $arg.
         case ${(tP)MATCH} in
           (*array*|*assoc*)
-            local -a words; words=( ${(P)MATCH} )
-            arg=${words[1]}
+            words=( ${(P)MATCH} )
             ;;
           (*)
             # scalar, presumably
-            arg=${(P)MATCH}
+            words=( ${(P)MATCH} )
             ;;
         esac
+        (( in_param = 1 + $#words ))
+        args=( $words $args )
+        arg=$args[1]
         _zsh_highlight_main__type "$arg" 0
         res=$REPLY
       fi
@@ -817,7 +840,7 @@ _zsh_highlight_main_highlighter_highlight_list()
         function)       style=function;;
         command)        style=command;;
         hashed)         style=hashed-command;;
-        none)           if _zsh_highlight_main_highlighter_check_assign; then
+        none)           if (( ! in_param )) && _zsh_highlight_main_highlighter_check_assign; then
                           _zsh_highlight_main_add_region_highlight $start_pos $end_pos assign
                           local i=$(( arg[(i)=] + 1 ))
                           if [[ $arg[i] == '(' ]]; then
@@ -838,11 +861,14 @@ _zsh_highlight_main_highlighter_highlight_list()
                             fi
                           fi
                           continue
-                        elif [[ $arg[0,1] = $histchars[0,1] ]] && (( $#arg[0,2] == 2 )); then
+                        elif (( ! in_param )) &&
+                             [[ $arg[0,1] = $histchars[0,1] ]] && (( $#arg[0,2] == 2 )); then
                           style=history-expansion
-                        elif [[ $arg[0,1] == $histchars[2,2] ]]; then
+                        elif (( ! in_param )) &&
+                             [[ $arg[0,1] == $histchars[2,2] ]]; then
                           style=history-expansion
-                        elif [[ $arg[1,2] == '((' ]]; then
+                        elif (( ! in_param )) &&
+                             [[ $arg[1,2] == '((' ]]; then
                           # Arithmetic evaluation.
                           #
                           # Note: prior to zsh-5.1.1-52-g4bed2cf (workers/36669), the ${(z)...}
@@ -857,14 +883,17 @@ _zsh_highlight_main_highlighter_highlight_list()
                             _zsh_highlight_main_add_region_highlight $((end_pos - 2)) $end_pos reserved-word
                           fi
                           continue
-                        elif [[ $arg == '()' ]]; then
+                        elif (( ! in_param )) &&
+                             [[ $arg == '()' ]]; then
                           # anonymous function
                           style=reserved-word
-                        elif [[ $arg == $'\x28' ]]; then
+                        elif (( ! in_param )) &&
+                             [[ $arg == $'\x28' ]]; then
                           # subshell
                           style=reserved-word
                           braces_stack='R'"$braces_stack"
-                        elif [[ $arg == $'\x29' ]]; then
+                        elif (( ! in_param )) &&
+                             [[ $arg == $'\x29' ]]; then
                           # end of subshell or command substitution
                           if _zsh_highlight_main__stack_pop 'S'; then
                             REPLY=$start_pos
@@ -888,6 +917,7 @@ _zsh_highlight_main_highlighter_highlight_list()
      if [[ -n ${(M)ZSH_HIGHLIGHT_TOKENS_CONTROL_FLOW:#"$arg"} ]]; then
       next_word=':start::start_of_pipeline:'
      fi
+     : ${param_style:=$style}
    else # $arg is a non-command word
       case $arg in
         $'\x29') # subshell or end of array assignment
@@ -945,7 +975,9 @@ _zsh_highlight_main_highlighter_highlight_list()
     fi
     _zsh_highlight_main_add_region_highlight $start_pos $end_pos $style
   done
+  : ${param_style:=$style}
   (( in_alias == 1 )) && in_alias=0 _zsh_highlight_main_add_region_highlight $start_pos $end_pos $alias_style
+  (( in_param == 1 )) && in_param=0 _zsh_highlight_main_add_region_highlight $start_pos $end_pos $param_style
   [[ "$proc_buf" = (#b)(#s)(([[:space:]]|\\$'\n')#) ]]
   REPLY=$(( end_pos + ${#match[1]} - 1 ))
   reply=($list_highlights)

highlighters/main/test-data/param-precommand-option-argument1.zsh

@@ -36,7 +36,7 @@ BUFFER='$sudo_u phy1729 echo foo'
 
 expected_region_highlight=(
   '1 7 precommand' # $sudo_u
-  '9 15 default "issue #674"' # phy1729
+  '9 15 default' # phy1729
   '18 20 command "issue #540"' # echo (not builtin)
   '22 24 default' # foo
 )

highlighters/main/test-data/param-precommand-option-argument3.zsh

@@ -36,7 +36,7 @@ BUFFER='$sudo_u phy1729 ls foo'
 
 expected_region_highlight=(
   '1 7 precommand' # sudo_u
-  '9 15 default "issue #674"' # phy1729
-  '17 18 command "issue #674"' # ls
+  '9 15 default' # phy1729
+  '17 18 command' # ls
   '20 22 default' # foo
 )

highlighters/main/test-data/parameter-expansion-untokenized1.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+local x="()"
+
+BUFFER=$'$x ls'
+
+expected_region_highlight=(
+  '1 2 unknown-token' # $x
+  '4 5 command' # ls
+)

highlighters/main/test-data/parameter-expansion-untokenized2.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+local x="^foo^bar"
+
+BUFFER=$'$x ls'
+
+expected_region_highlight=(
+  '1 2 unknown-token' # $x
+  '4 5 default' # ls
+)

highlighters/main/test-data/parameter-value-contains-command-position1.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+local foobar='x=$(ls)'
+
+BUFFER=$'$foobar'
+
+expected_region_highlight=(
+  # Used to highlight the "ba" as 'command' because the 'ls' showed through; issues #670 and #674
+  '1 7 unknown-token' # $foobar (not an assignment)
+)

highlighters/main/test-data/parameter-value-contains-command-position2.zsh

@@ -0,0 +1,38 @@
+#!/usr/bin/env zsh
+# -------------------------------------------------------------------------------------------------
+# Copyright (c) 2020 zsh-syntax-highlighting contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this list of conditions
+#    and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright notice, this list of
+#    conditions and the following disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of the zsh-syntax-highlighting contributors nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -------------------------------------------------------------------------------------------------
+# -*- mode: zsh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*-
+# vim: ft=zsh sw=2 ts=2 et
+# -------------------------------------------------------------------------------------------------
+
+local y='x=$(ls)'
+
+BUFFER=$'$y'
+
+expected_region_highlight=(
+  # Used to trigger a "BUG" message on stderr - issues #670 and #674
+  '1 2 unknown-token' # $y (not an assignment)
+)

tests/tap-filter

@@ -41,5 +41,7 @@ use strict;
 undef $/; # slurp mode
 print for
   grep { /^ok.*# TODO/m or /^not ok(?!.*# TODO)/m or /^Bail out!/m }
-    split /^(?=#)/m,
+    # Split on plan lines and remove them from the output.  (To keep them,
+    # use the lookahead syntax, «(?=…)», to make the match zero-length.)
+    split /^\d+\.\.\d+$/m,
       <STDIN>;

tests/test-highlighting.zsh

@@ -93,9 +93,13 @@ ZSH_HIGHLIGHT_HIGHLIGHTERS=($1)
 
 # In zsh<5.3, 'typeset -p arrayvar' emits two lines, so we use this wrapper instead.
 typeset_p() {
-	for 1 ; do
-		print -r -- "$1=( ${(@q-P)1} )"
-	done
+  for 1 ; do
+    if [[ ${(tP)1} == *array* ]]; then
+      print -r -- "$1=( ${(@q-P)1} )"
+    else
+      print -r -- "$1=${(q-P)1}"
+    fi
+  done
 }
 
 # Escape # as ♯ and newline as ↵ they are illegal in the 'description' part of TAP output
@@ -113,8 +117,6 @@ run_test_internal() {
   local srcdir="$PWD"
   builtin cd -q -- "$tests_tempdir" || { echo >&2 "Bail out! On ${(qq)1}: cd failed: $?"; return 1 }
 
-  echo "# ${1:t:r}"
-
   # Load the data and prepare checking it.
   local BUFFER CURSOR MARK PENDING PREBUFFER REGION_ACTIVE WIDGET REPLY skip_test unsorted=0
   local expected_mismatch
@@ -139,7 +141,12 @@ run_test_internal() {
     expected_region_highlight=("${(@n)expected_region_highlight}")
   fi
 
+  # Print the plan line, and some comments for human readers
   echo "1..$(( $#expected_region_highlight + 1))"
+  echo "## ${1:t:r}"
+  [[ -n $PREBUFFER ]] && printf '# %s\n' "$(typeset_p PREBUFFER)"
+  [[ -n $BUFFER ]] && printf '# %s\n' "$(typeset_p BUFFER)"
+
   local i
   for ((i=1; i<=$#expected_region_highlight; i++)); do
     local -a expected_highlight_zone; expected_highlight_zone=( ${(z)expected_region_highlight[i]} )
@@ -202,7 +209,7 @@ run_test() {
       local ret=$pipestatus[1] stderr=$pipestatus[2]
       if (( ! stderr )); then
         # stdout will become stderr
-	echo "Bail out! On ${(qq)1}: output on stderr"; return 1
+        echo "Bail out! On ${(qq)1}: output on stderr"; return 1
       else
         return $ret
       fi

zsh-syntax-highlighting.zsh

@@ -161,12 +161,12 @@ _zsh_highlight()
       (( REGION_ACTIVE )) || return
       integer min max
       if (( MARK > CURSOR )) ; then
-	min=$CURSOR max=$MARK
+        min=$CURSOR max=$MARK
       else
-	min=$MARK max=$CURSOR
+        min=$MARK max=$CURSOR
       fi
       if (( REGION_ACTIVE == 1 )); then
-	[[ $KEYMAP = vicmd ]] && (( max++ ))
+        [[ $KEYMAP = vicmd ]] && (( max++ ))
       elif (( REGION_ACTIVE == 2 )); then
         local needle=$'\n'
         # CURSOR and MARK are 0 indexed between letters like region_highlight