Imported Upstream version 5.2.2upstream/5.2.2

6 files changed, 349 insertions, 154 deletions

diff --git a/ext/tokenizer/package.xml b/ext/tokenizer/package.xml
index 16757db7d..f6f754c3d 100644
--- a/ext/tokenizer/package.xml
+++ b/ext/tokenizer/package.xml
@@ -23,7 +23,7 @@ to deal with the language specification at the lexical level.
   <version>5.0.0rc1</version>
   <date>2004-03-19</date>
   <notes>
-package.xml added to support intallation using pear installer
+package.xml added to support installation using pear installer
   </notes>
   <filelist>
    <file role="doc" name="CREDITS"/>
diff --git a/ext/tokenizer/php_tokenizer.h b/ext/tokenizer/php_tokenizer.h
index 3cc5ee704..94c614a92 100644
--- a/ext/tokenizer/php_tokenizer.h
+++ b/ext/tokenizer/php_tokenizer.h
@@ -2,7 +2,7 @@
    +----------------------------------------------------------------------+
    | PHP Version 5                                                        |
    +----------------------------------------------------------------------+
-   | Copyright (c) 1997-2006 The PHP Group                                |
+   | Copyright (c) 1997-2007 The PHP Group                                |
    +----------------------------------------------------------------------+
    | This source file is subject to version 3.01 of the PHP license,      |
    | that is bundled with this package in the file LICENSE, and is        |
@@ -16,7 +16,7 @@
    +----------------------------------------------------------------------+
  */
 
-/* $Id: php_tokenizer.h,v 1.9.2.1.2.1 2006/06/20 22:39:15 iliaa Exp $ */
+/* $Id: php_tokenizer.h,v 1.9.2.1.2.2 2007/01/01 09:36:09 sebastian Exp $ */
 
 #ifndef PHP_TOKENIZER_H
 #define PHP_TOKENIZER_H
diff --git a/ext/tokenizer/tests/002.phpt b/ext/tokenizer/tests/002.phpt
index fad8f2ae7..2a40ffe29 100644
--- a/ext/tokenizer/tests/002.phpt
+++ b/ext/tokenizer/tests/002.phpt
@@ -21,768 +21,962 @@ foreach ($strings as $s) {
 
 echo "Done\n";
 ?>
---EXPECTF--	
+--EXPECTF--
 array(49) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "<?"
+    [2]=>
+    int(1)
   }
   [1]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [2]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(4) "echo"
+    [2]=>
+    int(1)
   }
   [3]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [4]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "1"
+    [2]=>
+    int(1)
   }
   [5]=>
   string(1) ";"
   [6]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [7]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "if"
+    [2]=>
+    int(1)
   }
   [8]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [9]=>
   string(1) "("
   [10]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "isset"
+    [2]=>
+    int(1)
   }
   [11]=>
   string(1) "("
   [12]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [13]=>
   string(1) ")"
   [14]=>
   string(1) ")"
   [15]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [16]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "print"
+    [2]=>
+    int(1)
   }
   [17]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [18]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [19]=>
   string(1) "+"
   [20]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "1"
+    [2]=>
+    int(1)
   }
   [21]=>
   string(1) ";"
   [22]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [23]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [24]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "++"
+    [2]=>
+    int(1)
   }
   [25]=>
   string(1) ";"
   [26]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [27]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [28]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "--"
+    [2]=>
+    int(1)
   }
   [29]=>
   string(1) ";"
   [30]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [31]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [32]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [33]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "=="
+    [2]=>
+    int(1)
   }
   [34]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [35]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [36]=>
   string(1) ";"
   [37]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [38]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [39]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [40]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(3) "==="
+    [2]=>
+    int(1)
   }
   [41]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [42]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [43]=>
   string(1) ";"
   [44]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [45]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "endif"
+    [2]=>
+    int(1)
   }
   [46]=>
   string(1) ";"
   [47]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [48]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "?>"
+    [2]=>
+    int(1)
   }
 }
 array(37) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(6) "<?php "
+    [2]=>
+    int(1)
   }
   [1]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(6) "switch"
+    [2]=>
+    int(1)
   }
   [2]=>
   string(1) "("
   [3]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [4]=>
   string(1) ")"
   [5]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [6]=>
   string(1) "{"
   [7]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [8]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(4) "case"
+    [2]=>
+    int(1)
   }
   [9]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [10]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "1"
+    [2]=>
+    int(1)
   }
   [11]=>
   string(1) ":"
   [12]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [13]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "break"
+    [2]=>
+    int(1)
   }
   [14]=>
   string(1) ";"
   [15]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [16]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(7) "default"
+    [2]=>
+    int(1)
   }
   [17]=>
   string(1) ":"
   [18]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [19]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "break"
+    [2]=>
+    int(1)
   }
   [20]=>
   string(1) ";"
   [21]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [22]=>
   string(1) "}"
   [23]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [24]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(5) "while"
+    [2]=>
+    int(1)
   }
   [25]=>
   string(1) "("
   [26]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [27]=>
   string(1) ")"
   [28]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [29]=>
   string(1) "{"
   [30]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [31]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(4) "exit"
+    [2]=>
+    int(1)
   }
   [32]=>
   string(1) ";"
   [33]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [34]=>
   string(1) "}"
   [35]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [36]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "?>"
+    [2]=>
+    int(1)
   }
 }
 array(48) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "<?"
+    [2]=>
+    int(1)
   }
   [1]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [2]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(13) "/* comment */"
+    [2]=>
+    int(1)
   }
   [3]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [4]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "if"
+    [2]=>
+    int(1)
   }
   [5]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [6]=>
   string(1) "("
   [7]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "1"
+    [2]=>
+    int(1)
   }
   [8]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [9]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "||"
+    [2]=>
+    int(1)
   }
   [10]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [11]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [12]=>
   string(1) ")"
   [13]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [14]=>
   string(1) "{"
   [15]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [16]=>
   string(1) "}"
   [17]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [18]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(1)
   }
   [19]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [20]=>
   string(1) "="
   [21]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [22]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [23]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [24]=>
   string(1) "|"
   [25]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [26]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "1"
+    [2]=>
+    int(1)
   }
   [27]=>
   string(1) ";"
   [28]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [29]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$b"
+    [2]=>
+    int(1)
   }
   [30]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [31]=>
   string(1) "="
   [32]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [33]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "3"
+    [2]=>
+    int(1)
   }
   [34]=>
   string(1) "^"
   [35]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [36]=>
   string(1) ";"
   [37]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [38]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$c"
+    [2]=>
+    int(1)
   }
   [39]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [40]=>
   string(1) "="
   [41]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [42]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "4"
+    [2]=>
+    int(1)
   }
   [43]=>
   string(1) "&"
   [44]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "2"
+    [2]=>
+    int(1)
   }
   [45]=>
   string(1) ";"
   [46]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) " "
+    [2]=>
+    int(1)
   }
   [47]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "?>"
+    [2]=>
+    int(1)
   }
 }
 array(1) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(17) "wrong syntax here"
+    [2]=>
+    int(1)
   }
 }
 Done
diff --git a/ext/tokenizer/tests/003.phpt b/ext/tokenizer/tests/003.phpt
index 66d088eec..fdcf7748a 100644
--- a/ext/tokenizer/tests/003.phpt
+++ b/ext/tokenizer/tests/003.phpt
@@ -23,20 +23,24 @@ array(0) {
 }
 array(1) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "0"
+    [2]=>
+    int(1)
   }
 }
 array(1) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "-1"
+    [2]=>
+    int(1)
   }
 }
 Done
diff --git a/ext/tokenizer/tests/bug26463.phpt b/ext/tokenizer/tests/bug26463.phpt
index c5d2148ea..d07476b20 100644
--- a/ext/tokenizer/tests/bug26463.phpt
+++ b/ext/tokenizer/tests/bug26463.phpt
@@ -18,119 +18,147 @@ var_dump(token_get_all($str));
 --EXPECTF--
 array(19) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(6) "<?php
 "
+    [2]=>
+    int(1)
   }
   [1]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$x"
+    [2]=>
+    int(2)
   }
   [2]=>
   string(1) "="
   [3]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(6) "<<<DD
 "
+    [2]=>
+    int(2)
   }
   [4]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(13) "jhdsjkfhjdsh
 "
+    [2]=>
+    int(3)
   }
   [5]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "DD"
+    [2]=>
+    int(4)
   }
   [6]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "
 "
+    [2]=>
+    int(4)
   }
   [7]=>
   string(1) "."
   [8]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) """"
+    [2]=>
+    int(5)
   }
   [9]=>
   string(1) ";"
   [10]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "
 "
+    [2]=>
+    int(5)
   }
   [11]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(6)
   }
   [12]=>
   string(1) "="
   [13]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(8) "<<<DDDD
 "
+    [2]=>
+    int(6)
   }
   [14]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(13) "jhdsjkfhjdsh
 "
+    [2]=>
+    int(7)
   }
   [15]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(4) "DDDD"
+    [2]=>
+    int(8)
   }
   [16]=>
   string(1) ";"
   [17]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "
 "
+    [2]=>
+    int(8)
   }
   [18]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "?>"
+    [2]=>
+    int(9)
   }
 }
diff --git a/ext/tokenizer/tokenizer.c b/ext/tokenizer/tokenizer.c
index 4d450c0d1..dc5f1eba6 100644
--- a/ext/tokenizer/tokenizer.c
+++ b/ext/tokenizer/tokenizer.c
@@ -2,7 +2,7 @@
    +----------------------------------------------------------------------+
    | PHP Version 5                                                        |
    +----------------------------------------------------------------------+
-   | Copyright (c) 1997-2006 The PHP Group                                |
+   | Copyright (c) 1997-2007 The PHP Group                                |
    +----------------------------------------------------------------------+
    | This source file is subject to version 3.01 of the PHP license,      |
    | that is bundled with this package in the file LICENSE, and is        |
@@ -16,7 +16,7 @@
    +----------------------------------------------------------------------+
 */
 
-/* $Id: tokenizer.c,v 1.31.2.5.2.2 2006/06/20 22:39:15 iliaa Exp $ */
+/* $Id: tokenizer.c,v 1.31.2.5.2.5 2007/04/08 00:18:38 johannes Exp $ */
 
 #ifdef HAVE_CONFIG_H
 #include "config.h"
@@ -92,12 +92,6 @@ struct yy_buffer_state
 
 #define zendtext LANG_SCNG(yy_text)
 #define zendleng LANG_SCNG(yy_leng)
-/* If you declare any globals in php_tokenizer.h uncomment this:
-ZEND_DECLARE_MODULE_GLOBALS(tokenizer)
-*/
-
-/* True global resources - no need for thread safety here */
-/* static int le_tokenizer; */
 
 /* {{{ tokenizer_functions[]
  *
@@ -134,35 +128,10 @@ zend_module_entry tokenizer_module_entry = {
 ZEND_GET_MODULE(tokenizer)
 #endif
 
-/* {{{ PHP_INI
- */
-/* Remove comments and fill if you need to have entries in php.ini
-PHP_INI_BEGIN()
-	STD_PHP_INI_ENTRY("tokenizer.global_value",      "42", PHP_INI_ALL, OnUpdateLong, global_value, zend_tokenizer_globals, tokenizer_globals)
-	STD_PHP_INI_ENTRY("tokenizer.global_string", "foobar", PHP_INI_ALL, OnUpdateString, global_string, zend_tokenizer_globals, tokenizer_globals)
-PHP_INI_END()
-*/
-/* }}} */
-
-/* {{{ PHP_GINIT_FUNCTION
- */
-/* Uncomment this function if you have INI entries
-static PHP_GINIT_FUNCTION(tokenizer)
-{
-	tokenizer_globals->global_value = 0;
-	tokenizer_globals->global_string = NULL;
-}
-*/
-/* }}} */
-
 /* {{{ PHP_MINIT_FUNCTION
  */
 PHP_MINIT_FUNCTION(tokenizer)
 {
-	/* If you have INI entries, uncomment these lines 
-	REGISTER_INI_ENTRIES();
-	*/
-
 	REGISTER_LONG_CONSTANT("T_INCLUDE", T_INCLUDE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_INCLUDE_ONCE", T_INCLUDE_ONCE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_EVAL", T_EVAL, CONST_CS | CONST_PERSISTENT);
@@ -294,10 +263,6 @@ PHP_MINFO_FUNCTION(tokenizer)
 	php_info_print_table_start();
 	php_info_print_table_row(2, "Tokenizer Support", "enabled");
 	php_info_print_table_end();
-
-	/* Remove comments if you have entries in php.ini
-	DISPLAY_INI_ENTRIES();
-	*/
 }
 /* }}} */
 
@@ -307,6 +272,7 @@ static void tokenize(zval *return_value TSRMLS_DC)
 	zval *keyword;
 	int token_type;
 	zend_bool destroy;
+	int token_line = 1;
 
 	array_init(return_value);
 
@@ -336,6 +302,7 @@ static void tokenize(zval *return_value TSRMLS_DC)
 			} else {
 				add_next_index_stringl(keyword, zendtext, zendleng, 1);
 			}
+			add_next_index_long(keyword, token_line);
 			add_next_index_zval(return_value, keyword);
 		} else {
 			add_next_index_stringl(return_value, zendtext, zendleng, 1);
@@ -344,6 +311,8 @@ static void tokenize(zval *return_value TSRMLS_DC)
 			zval_dtor(&token);
 		}
 		ZVAL_NULL(&token);
+
+		token_line = CG(zend_lineno);
 	}
 }