Imported Upstream version 5.2.2upstream/5.2.2

3 files changed, 340 insertions, 114 deletions

diff --git a/ext/tokenizer/tests/002.phpt b/ext/tokenizer/tests/002.phpt
index fad8f2ae7..2a40ffe29 100644
--- a/ext/tokenizer/tests/002.phpt
+++ b/ext/tokenizer/tests/002.phpt
@@ -21,768 +21,962 @@ foreach ($strings as $s) {
 
 echo "Done\n";
 ?>
---EXPECTF--	
+--EXPECTF--
 array(49) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "<?"
+    [2]=>
+    int(1)
   }
   [1]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [2]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(4) "echo"
+    [2]=>
+    int(1)
   }
   [3]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [4]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "1"
+    [2]=>
+    int(1)
   }
   [5]=>
   string(1) ";"
   [6]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [7]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "if"
+    [2]=>
+    int(1)
   }
   [8]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [9]=>
   string(1) "("
   [10]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "isset"
+    [2]=>
+    int(1)
   }
   [11]=>
   string(1) "("
   [12]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [13]=>
   string(1) ")"
   [14]=>
   string(1) ")"
   [15]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [16]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "print"
+    [2]=>
+    int(1)
   }
   [17]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [18]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [19]=>
   string(1) "+"
   [20]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "1"
+    [2]=>
+    int(1)
   }
   [21]=>
   string(1) ";"
   [22]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [23]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [24]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "++"
+    [2]=>
+    int(1)
   }
   [25]=>
   string(1) ";"
   [26]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [27]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [28]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "--"
+    [2]=>
+    int(1)
   }
   [29]=>
   string(1) ";"
   [30]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [31]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [32]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [33]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "=="
+    [2]=>
+    int(1)
   }
   [34]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [35]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [36]=>
   string(1) ";"
   [37]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [38]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [39]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [40]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(3) "==="
+    [2]=>
+    int(1)
   }
   [41]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [42]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [43]=>
   string(1) ";"
   [44]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [45]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "endif"
+    [2]=>
+    int(1)
   }
   [46]=>
   string(1) ";"
   [47]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [48]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "?>"
+    [2]=>
+    int(1)
   }
 }
 array(37) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(6) "<?php "
+    [2]=>
+    int(1)
   }
   [1]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(6) "switch"
+    [2]=>
+    int(1)
   }
   [2]=>
   string(1) "("
   [3]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [4]=>
   string(1) ")"
   [5]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [6]=>
   string(1) "{"
   [7]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [8]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(4) "case"
+    [2]=>
+    int(1)
   }
   [9]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [10]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "1"
+    [2]=>
+    int(1)
   }
   [11]=>
   string(1) ":"
   [12]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [13]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "break"
+    [2]=>
+    int(1)
   }
   [14]=>
   string(1) ";"
   [15]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [16]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(7) "default"
+    [2]=>
+    int(1)
   }
   [17]=>
   string(1) ":"
   [18]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [19]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "break"
+    [2]=>
+    int(1)
   }
   [20]=>
   string(1) ";"
   [21]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [22]=>
   string(1) "}"
   [23]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [24]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "while"
+    [2]=>
+    int(1)
   }
   [25]=>
   string(1) "("
   [26]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [27]=>
   string(1) ")"
   [28]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [29]=>
   string(1) "{"
   [30]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [31]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(4) "exit"
+    [2]=>
+    int(1)
   }
   [32]=>
   string(1) ";"
   [33]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [34]=>
   string(1) "}"
   [35]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [36]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "?>"
+    [2]=>
+    int(1)
   }
 }
 array(48) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "<?"
+    [2]=>
+    int(1)
   }
   [1]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [2]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(13) "/* comment */"
+    [2]=>
+    int(1)
   }
   [3]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [4]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "if"
+    [2]=>
+    int(1)
   }
   [5]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [6]=>
   string(1) "("
   [7]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "1"
+    [2]=>
+    int(1)
   }
   [8]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [9]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "||"
+    [2]=>
+    int(1)
   }
   [10]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [11]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [12]=>
   string(1) ")"
   [13]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [14]=>
   string(1) "{"
   [15]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [16]=>
   string(1) "}"
   [17]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [18]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [19]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [20]=>
   string(1) "="
   [21]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [22]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [23]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [24]=>
   string(1) "|"
   [25]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [26]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "1"
+    [2]=>
+    int(1)
   }
   [27]=>
   string(1) ";"
   [28]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [29]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$b"
+    [2]=>
+    int(1)
   }
   [30]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [31]=>
   string(1) "="
   [32]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [33]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "3"
+    [2]=>
+    int(1)
   }
   [34]=>
   string(1) "^"
   [35]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [36]=>
   string(1) ";"
   [37]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [38]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$c"
+    [2]=>
+    int(1)
   }
   [39]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [40]=>
   string(1) "="
   [41]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [42]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "4"
+    [2]=>
+    int(1)
   }
   [43]=>
   string(1) "&"
   [44]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [45]=>
   string(1) ";"
   [46]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [47]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "?>"
+    [2]=>
+    int(1)
   }
 }
 array(1) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(17) "wrong syntax here"
+    [2]=>
+    int(1)
   }
 }
 Done
diff --git a/ext/tokenizer/tests/003.phpt b/ext/tokenizer/tests/003.phpt
index 66d088eec..fdcf7748a 100644
--- a/ext/tokenizer/tests/003.phpt
+++ b/ext/tokenizer/tests/003.phpt
@@ -23,20 +23,24 @@ array(0) {
 }
 array(1) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "0"
+    [2]=>
+    int(1)
   }
 }
 array(1) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "-1"
+    [2]=>
+    int(1)
   }
 }
 Done
diff --git a/ext/tokenizer/tests/bug26463.phpt b/ext/tokenizer/tests/bug26463.phpt
index c5d2148ea..d07476b20 100644
--- a/ext/tokenizer/tests/bug26463.phpt
+++ b/ext/tokenizer/tests/bug26463.phpt
@@ -18,119 +18,147 @@ var_dump(token_get_all($str));
 --EXPECTF--
 array(19) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(6) "<?php
 "
+    [2]=>
+    int(1)
   }
   [1]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$x"
+    [2]=>
+    int(2)
   }
   [2]=>
   string(1) "="
   [3]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(6) "<<<DD
 "
+    [2]=>
+    int(2)
   }
   [4]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(13) "jhdsjkfhjdsh
 "
+    [2]=>
+    int(3)
   }
   [5]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "DD"
+    [2]=>
+    int(4)
   }
   [6]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "
 "
+    [2]=>
+    int(4)
   }
   [7]=>
   string(1) "."
   [8]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) """"
+    [2]=>
+    int(5)
   }
   [9]=>
   string(1) ";"
   [10]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "
 "
+    [2]=>
+    int(5)
   }
   [11]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(6)
   }
   [12]=>
   string(1) "="
   [13]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(8) "<<<DDDD
 "
+    [2]=>
+    int(6)
   }
   [14]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(13) "jhdsjkfhjdsh
 "
+    [2]=>
+    int(7)
   }
   [15]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(4) "DDDD"
+    [2]=>
+    int(8)
   }
   [16]=>
   string(1) ";"
   [17]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "
 "
+    [2]=>
+    int(8)
   }
   [18]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "?>"
+    [2]=>
+    int(9)
   }
 }