Imported Upstream version 5.1.2upstream/5.1.2

6 files changed, 1143 insertions, 32 deletions

diff --git a/ext/tokenizer/php_tokenizer.h b/ext/tokenizer/php_tokenizer.h
index 4f069c3e2..3964c91b9 100644
--- a/ext/tokenizer/php_tokenizer.h
+++ b/ext/tokenizer/php_tokenizer.h
@@ -2,12 +2,12 @@
    +----------------------------------------------------------------------+
    | PHP Version 5                                                        |
    +----------------------------------------------------------------------+
-   | Copyright (c) 1997-2005 The PHP Group                                |
+   | Copyright (c) 1997-2006 The PHP Group                                |
    +----------------------------------------------------------------------+
-   | This source file is subject to version 3.0 of the PHP license,       |
+   | This source file is subject to version 3.01 of the PHP license,      |
    | that is bundled with this package in the file LICENSE, and is        |
    | available through the world-wide-web at the following url:           |
-   | http://www.php.net/license/3_0.txt.                                  |
+   | http://www.php.net/license/3_01.txt                                  |
    | If you did not receive a copy of the PHP license and are unable to   |
    | obtain it through the world-wide-web, please send a note to          |
    | license@php.net so we can mail you a copy immediately.               |
@@ -16,7 +16,7 @@
    +----------------------------------------------------------------------+
  */
 
-/* $Id: php_tokenizer.h,v 1.9 2005/08/03 14:08:19 sniper Exp $ */
+/* $Id: php_tokenizer.h,v 1.9.2.1 2006/01/01 12:50:16 sniper Exp $ */
 
 #ifndef PHP_TOKENIZER_H
 #define PHP_TOKENIZER_H
diff --git a/ext/tokenizer/tests/001.phpt b/ext/tokenizer/tests/001.phpt
new file mode 100644
index 000000000..36ed2696f
--- /dev/null
+++ b/ext/tokenizer/tests/001.phpt
@@ -0,0 +1,263 @@
+--TEST--
+token_name()
+--SKIPIF--
+<?php if (!extension_loaded("tokenizer")) print "skip"; ?>
+--FILE--
+<?php
+
+echo token_name(T_INCLUDE), "\n";
+echo token_name(T_INCLUDE_ONCE), "\n";
+echo token_name(T_EVAL), "\n";
+echo token_name(T_REQUIRE), "\n";
+echo token_name(T_REQUIRE_ONCE), "\n";
+echo token_name(T_LOGICAL_OR), "\n";
+echo token_name(T_LOGICAL_XOR), "\n";
+echo token_name(T_LOGICAL_AND), "\n";
+echo token_name(T_PRINT), "\n";
+echo token_name(T_PLUS_EQUAL), "\n";
+echo token_name(T_MINUS_EQUAL), "\n";
+echo token_name(T_MUL_EQUAL), "\n";
+echo token_name(T_DIV_EQUAL), "\n";
+echo token_name(T_CONCAT_EQUAL), "\n";
+echo token_name(T_MOD_EQUAL), "\n";
+echo token_name(T_AND_EQUAL), "\n";
+echo token_name(T_OR_EQUAL), "\n";
+echo token_name(T_XOR_EQUAL), "\n";
+echo token_name(T_SL_EQUAL), "\n";
+echo token_name(T_SR_EQUAL), "\n";
+echo token_name(T_BOOLEAN_OR), "\n";
+echo token_name(T_BOOLEAN_AND), "\n";
+echo token_name(T_IS_EQUAL), "\n";
+echo token_name(T_IS_NOT_EQUAL), "\n";
+echo token_name(T_IS_IDENTICAL), "\n";
+echo token_name(T_IS_NOT_IDENTICAL), "\n";
+echo token_name(T_IS_SMALLER_OR_EQUAL), "\n";
+echo token_name(T_IS_GREATER_OR_EQUAL), "\n";
+echo token_name(T_SL), "\n";
+echo token_name(T_SR), "\n";
+echo token_name(T_INC), "\n";
+echo token_name(T_DEC), "\n";
+echo token_name(T_INT_CAST), "\n";
+echo token_name(T_DOUBLE_CAST), "\n";
+echo token_name(T_STRING_CAST), "\n";
+echo token_name(T_ARRAY_CAST), "\n";
+echo token_name(T_OBJECT_CAST), "\n";
+echo token_name(T_BOOL_CAST), "\n";
+echo token_name(T_UNSET_CAST), "\n";
+echo token_name(T_NEW), "\n";
+echo token_name(T_EXIT), "\n";
+echo token_name(T_IF), "\n";
+echo token_name(T_ELSEIF), "\n";
+echo token_name(T_ELSE), "\n";
+echo token_name(T_ENDIF), "\n";
+echo token_name(T_LNUMBER), "\n";
+echo token_name(T_DNUMBER), "\n";
+echo token_name(T_STRING), "\n";
+echo token_name(T_STRING_VARNAME), "\n";
+echo token_name(T_VARIABLE), "\n";
+echo token_name(T_NUM_STRING), "\n";
+echo token_name(T_INLINE_HTML), "\n";
+echo token_name(T_CHARACTER), "\n";
+echo token_name(T_BAD_CHARACTER), "\n";
+echo token_name(T_ENCAPSED_AND_WHITESPACE), "\n";
+echo token_name(T_CONSTANT_ENCAPSED_STRING), "\n";
+echo token_name(T_ECHO), "\n";
+echo token_name(T_DO), "\n";
+echo token_name(T_WHILE), "\n";
+echo token_name(T_ENDWHILE), "\n";
+echo token_name(T_FOR), "\n";
+echo token_name(T_ENDFOR), "\n";
+echo token_name(T_FOREACH), "\n";
+echo token_name(T_ENDFOREACH), "\n";
+echo token_name(T_DECLARE), "\n";
+echo token_name(T_ENDDECLARE), "\n";
+echo token_name(T_AS), "\n";
+echo token_name(T_SWITCH), "\n";
+echo token_name(T_ENDSWITCH), "\n";
+echo token_name(T_CASE), "\n";
+echo token_name(T_DEFAULT), "\n";
+echo token_name(T_BREAK), "\n";
+echo token_name(T_CONTINUE), "\n";
+echo token_name(T_FUNCTION), "\n";
+echo token_name(T_CONST), "\n";
+echo token_name(T_RETURN), "\n";
+echo token_name(T_USE), "\n";
+echo token_name(T_GLOBAL), "\n";
+echo token_name(T_STATIC), "\n";
+echo token_name(T_VAR), "\n";
+echo token_name(T_UNSET), "\n";
+echo token_name(T_ISSET), "\n";
+echo token_name(T_EMPTY), "\n";
+echo token_name(T_CLASS), "\n";
+echo token_name(T_EXTENDS), "\n";
+echo token_name(T_INTERFACE), "\n";
+echo token_name(T_IMPLEMENTS), "\n";
+echo token_name(T_OBJECT_OPERATOR), "\n";
+echo token_name(T_DOUBLE_ARROW), "\n";
+echo token_name(T_LIST), "\n";
+echo token_name(T_ARRAY), "\n";
+echo token_name(T_CLASS_C), "\n";
+echo token_name(T_FUNC_C), "\n";
+echo token_name(T_METHOD_C), "\n";
+echo token_name(T_LINE), "\n";
+echo token_name(T_FILE), "\n";
+echo token_name(T_COMMENT), "\n";
+echo token_name(T_DOC_COMMENT), "\n";
+echo token_name(T_OPEN_TAG), "\n";
+echo token_name(T_OPEN_TAG_WITH_ECHO), "\n";
+echo token_name(T_CLOSE_TAG), "\n";
+echo token_name(T_WHITESPACE), "\n";
+echo token_name(T_START_HEREDOC), "\n";
+echo token_name(T_END_HEREDOC), "\n";
+echo token_name(T_DOLLAR_OPEN_CURLY_BRACES), "\n";
+echo token_name(T_CURLY_OPEN), "\n";
+echo token_name(T_PAAMAYIM_NEKUDOTAYIM), "\n";
+echo token_name(T_PAAMAYIM_NEKUDOTAYIM), "\n";
+echo token_name(T_ABSTRACT), "\n";
+echo token_name(T_CATCH), "\n";
+echo token_name(T_FINAL), "\n";
+echo token_name(T_INSTANCEOF), "\n";
+echo token_name(T_PRIVATE), "\n";
+echo token_name(T_PROTECTED), "\n";
+echo token_name(T_PUBLIC), "\n";
+echo token_name(T_THROW), "\n";
+echo token_name(T_TRY), "\n";
+echo token_name(T_CLONE), "\n";
+echo token_name(T_HALT_COMPILER), "\n";
+
+echo token_name(-1), "\n";
+echo token_name(0x8000000F), "\n";
+echo token_name("string"), "\n";
+echo token_name(array()), "\n";
+
+echo "Done\n";
+?>
+--EXPECTF--	
+T_INCLUDE
+T_INCLUDE_ONCE
+T_EVAL
+T_REQUIRE
+T_REQUIRE_ONCE
+T_LOGICAL_OR
+T_LOGICAL_XOR
+T_LOGICAL_AND
+T_PRINT
+T_PLUS_EQUAL
+T_MINUS_EQUAL
+T_MUL_EQUAL
+T_DIV_EQUAL
+T_CONCAT_EQUAL
+T_MOD_EQUAL
+T_AND_EQUAL
+T_OR_EQUAL
+T_XOR_EQUAL
+T_SL_EQUAL
+T_SR_EQUAL
+T_BOOLEAN_OR
+T_BOOLEAN_AND
+T_IS_EQUAL
+T_IS_NOT_EQUAL
+T_IS_IDENTICAL
+T_IS_NOT_IDENTICAL
+T_IS_SMALLER_OR_EQUAL
+T_IS_GREATER_OR_EQUAL
+T_SL
+T_SR
+T_INC
+T_DEC
+T_INT_CAST
+T_DOUBLE_CAST
+T_STRING_CAST
+T_ARRAY_CAST
+T_OBJECT_CAST
+T_BOOL_CAST
+T_UNSET_CAST
+T_NEW
+T_EXIT
+T_IF
+T_ELSEIF
+T_ELSE
+T_ENDIF
+T_LNUMBER
+T_DNUMBER
+T_STRING
+T_STRING_VARNAME
+T_VARIABLE
+T_NUM_STRING
+T_INLINE_HTML
+T_CHARACTER
+T_BAD_CHARACTER
+T_ENCAPSED_AND_WHITESPACE
+T_CONSTANT_ENCAPSED_STRING
+T_ECHO
+T_DO
+T_WHILE
+T_ENDWHILE
+T_FOR
+T_ENDFOR
+T_FOREACH
+T_ENDFOREACH
+T_DECLARE
+T_ENDDECLARE
+T_AS
+T_SWITCH
+T_ENDSWITCH
+T_CASE
+T_DEFAULT
+T_BREAK
+T_CONTINUE
+T_FUNCTION
+T_CONST
+T_RETURN
+T_USE
+T_GLOBAL
+T_STATIC
+T_VAR
+T_UNSET
+T_ISSET
+T_EMPTY
+T_CLASS
+T_EXTENDS
+T_INTERFACE
+T_IMPLEMENTS
+T_OBJECT_OPERATOR
+T_DOUBLE_ARROW
+T_LIST
+T_ARRAY
+T_CLASS_C
+T_FUNC_C
+T_METHOD_C
+T_LINE
+T_FILE
+T_COMMENT
+T_DOC_COMMENT
+T_OPEN_TAG
+T_OPEN_TAG_WITH_ECHO
+T_CLOSE_TAG
+T_WHITESPACE
+T_START_HEREDOC
+T_END_HEREDOC
+T_DOLLAR_OPEN_CURLY_BRACES
+T_CURLY_OPEN
+T_DOUBLE_COLON
+T_DOUBLE_COLON
+T_ABSTRACT
+T_CATCH
+T_FINAL
+T_INSTANCEOF
+T_PRIVATE
+T_PROTECTED
+T_PUBLIC
+T_THROW
+T_TRY
+T_CLONE
+T_HALT_COMPILER
+UNKNOWN
+UNKNOWN
+
+Warning: token_name() expects parameter 1 to be long, string given in %s on line %d
+
+
+Warning: token_name() expects parameter 1 to be long, array given in %s on line %d
+
+Done
diff --git a/ext/tokenizer/tests/002.phpt b/ext/tokenizer/tests/002.phpt
new file mode 100644
index 000000000..c9a822b74
--- /dev/null
+++ b/ext/tokenizer/tests/002.phpt
@@ -0,0 +1,788 @@
+--TEST--
+token_get_all()
+--SKIPIF--
+<?php if (!extension_loaded("tokenizer")) print "skip"; ?>
+--INI--
+short_open_tag=1
+--FILE--
+<?php
+
+$strings = array(
+	'<? echo 1; if (isset($a)) print $a+1; $a++; $a--; $a == 2; $a === 2; endif; ?>',
+	'<?php switch($a) { case 1: break; default: break; } while($a) { exit; } ?>',
+	'<? /* comment */ if (1 || 2) { } $a = 2 | 1; $b = 3^2; $c = 4&2; ?>',
+	/* feel free to add more yourself */
+	'wrong syntax here'
+);
+
+foreach ($strings as $s) {
+	var_dump(token_get_all($s));
+}
+
+echo "Done\n";
+?>
+--EXPECT--	
+array(49) {
+  [0]=>
+  array(2) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(2) "<?"
+  }
+  [1]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [2]=>
+  array(2) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+  }
+  [3]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [4]=>
+  array(2) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "1"
+  }
+  [5]=>
+  string(1) ";"
+  [6]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [7]=>
+  array(2) {
+    [0]=>
+    int(301)
+    [1]=>
+    string(2) "if"
+  }
+  [8]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [9]=>
+  string(1) "("
+  [10]=>
+  array(2) {
+    [0]=>
+    int(349)
+    [1]=>
+    string(5) "isset"
+  }
+  [11]=>
+  string(1) "("
+  [12]=>
+  array(2) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+  }
+  [13]=>
+  string(1) ")"
+  [14]=>
+  string(1) ")"
+  [15]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [16]=>
+  array(2) {
+    [0]=>
+    int(266)
+    [1]=>
+    string(5) "print"
+  }
+  [17]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [18]=>
+  array(2) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+  }
+  [19]=>
+  string(1) "+"
+  [20]=>
+  array(2) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "1"
+  }
+  [21]=>
+  string(1) ";"
+  [22]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [23]=>
+  array(2) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+  }
+  [24]=>
+  array(2) {
+    [0]=>
+    int(297)
+    [1]=>
+    string(2) "++"
+  }
+  [25]=>
+  string(1) ";"
+  [26]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [27]=>
+  array(2) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+  }
+  [28]=>
+  array(2) {
+    [0]=>
+    int(296)
+    [1]=>
+    string(2) "--"
+  }
+  [29]=>
+  string(1) ";"
+  [30]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [31]=>
+  array(2) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+  }
+  [32]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [33]=>
+  array(2) {
+    [0]=>
+    int(283)
+    [1]=>
+    string(2) "=="
+  }
+  [34]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [35]=>
+  array(2) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "2"
+  }
+  [36]=>
+  string(1) ";"
+  [37]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [38]=>
+  array(2) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+  }
+  [39]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [40]=>
+  array(2) {
+    [0]=>
+    int(281)
+    [1]=>
+    string(3) "==="
+  }
+  [41]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [42]=>
+  array(2) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "2"
+  }
+  [43]=>
+  string(1) ";"
+  [44]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [45]=>
+  array(2) {
+    [0]=>
+    int(304)
+    [1]=>
+    string(5) "endif"
+  }
+  [46]=>
+  string(1) ";"
+  [47]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [48]=>
+  array(2) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+  }
+}
+array(37) {
+  [0]=>
+  array(2) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+  }
+  [1]=>
+  array(2) {
+    [0]=>
+    int(327)
+    [1]=>
+    string(6) "switch"
+  }
+  [2]=>
+  string(1) "("
+  [3]=>
+  array(2) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+  }
+  [4]=>
+  string(1) ")"
+  [5]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [6]=>
+  string(1) "{"
+  [7]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [8]=>
+  array(2) {
+    [0]=>
+    int(329)
+    [1]=>
+    string(4) "case"
+  }
+  [9]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [10]=>
+  array(2) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "1"
+  }
+  [11]=>
+  string(1) ":"
+  [12]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [13]=>
+  array(2) {
+    [0]=>
+    int(331)
+    [1]=>
+    string(5) "break"
+  }
+  [14]=>
+  string(1) ";"
+  [15]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [16]=>
+  array(2) {
+    [0]=>
+    int(330)
+    [1]=>
+    string(7) "default"
+  }
+  [17]=>
+  string(1) ":"
+  [18]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [19]=>
+  array(2) {
+    [0]=>
+    int(331)
+    [1]=>
+    string(5) "break"
+  }
+  [20]=>
+  string(1) ";"
+  [21]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [22]=>
+  string(1) "}"
+  [23]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [24]=>
+  array(2) {
+    [0]=>
+    int(318)
+    [1]=>
+    string(5) "while"
+  }
+  [25]=>
+  string(1) "("
+  [26]=>
+  array(2) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+  }
+  [27]=>
+  string(1) ")"
+  [28]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [29]=>
+  string(1) "{"
+  [30]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [31]=>
+  array(2) {
+    [0]=>
+    int(300)
+    [1]=>
+    string(4) "exit"
+  }
+  [32]=>
+  string(1) ";"
+  [33]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [34]=>
+  string(1) "}"
+  [35]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [36]=>
+  array(2) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+  }
+}
+array(48) {
+  [0]=>
+  array(2) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(2) "<?"
+  }
+  [1]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [2]=>
+  array(2) {
+    [0]=>
+    int(365)
+    [1]=>
+    string(13) "/* comment */"
+  }
+  [3]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [4]=>
+  array(2) {
+    [0]=>
+    int(301)
+    [1]=>
+    string(2) "if"
+  }
+  [5]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [6]=>
+  string(1) "("
+  [7]=>
+  array(2) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "1"
+  }
+  [8]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [9]=>
+  array(2) {
+    [0]=>
+    int(278)
+    [1]=>
+    string(2) "||"
+  }
+  [10]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [11]=>
+  array(2) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "2"
+  }
+  [12]=>
+  string(1) ")"
+  [13]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [14]=>
+  string(1) "{"
+  [15]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [16]=>
+  string(1) "}"
+  [17]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [18]=>
+  array(2) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+  }
+  [19]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [20]=>
+  string(1) "="
+  [21]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [22]=>
+  array(2) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "2"
+  }
+  [23]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [24]=>
+  string(1) "|"
+  [25]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [26]=>
+  array(2) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "1"
+  }
+  [27]=>
+  string(1) ";"
+  [28]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [29]=>
+  array(2) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+  }
+  [30]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [31]=>
+  string(1) "="
+  [32]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [33]=>
+  array(2) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "3"
+  }
+  [34]=>
+  string(1) "^"
+  [35]=>
+  array(2) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "2"
+  }
+  [36]=>
+  string(1) ";"
+  [37]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [38]=>
+  array(2) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$c"
+  }
+  [39]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [40]=>
+  string(1) "="
+  [41]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [42]=>
+  array(2) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "4"
+  }
+  [43]=>
+  string(1) "&"
+  [44]=>
+  array(2) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "2"
+  }
+  [45]=>
+  string(1) ";"
+  [46]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+  }
+  [47]=>
+  array(2) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+  }
+}
+array(1) {
+  [0]=>
+  array(2) {
+    [0]=>
+    int(311)
+    [1]=>
+    string(17) "wrong syntax here"
+  }
+}
+Done
diff --git a/ext/tokenizer/tests/003.phpt b/ext/tokenizer/tests/003.phpt
new file mode 100644
index 000000000..802728b97
--- /dev/null
+++ b/ext/tokenizer/tests/003.phpt
@@ -0,0 +1,42 @@
+--TEST--
+token_get_all() and wrong parameters
+--SKIPIF--
+<?php if (!extension_loaded("tokenizer")) print "skip"; ?>
+--FILE--
+<?php
+
+var_dump(token_get_all(array()));
+var_dump(token_get_all(new stdClass));
+var_dump(token_get_all(""));
+var_dump(token_get_all(0));
+var_dump(token_get_all(-1));
+
+echo "Done\n";
+?>
+--EXPECTF--	
+Warning: token_get_all() expects parameter 1 to be string, array given in %s on line %d
+NULL
+
+Warning: token_get_all() expects parameter 1 to be string, object given in %s on line %d
+NULL
+array(0) {
+}
+array(1) {
+  [0]=>
+  array(2) {
+    [0]=>
+    int(311)
+    [1]=>
+    string(1) "0"
+  }
+}
+array(1) {
+  [0]=>
+  array(2) {
+    [0]=>
+    int(311)
+    [1]=>
+    string(2) "-1"
+  }
+}
+Done
diff --git a/ext/tokenizer/tests/bug26463.phpt b/ext/tokenizer/tests/bug26463.phpt
index 7480aa2f5..8739c2a2b 100644
--- a/ext/tokenizer/tests/bug26463.phpt
+++ b/ext/tokenizer/tests/bug26463.phpt
@@ -1,5 +1,7 @@
 --TEST--
 Bug #26463 (token_get_all() does not correctly handle semicolons after T_END_HEREDOC)
+--SKIPIF--
+<?php if (!extension_loaded("tokenizer")) print "skip"; ?>
 --FILE--
 <?php
 $str = '<?php
@@ -14,11 +16,11 @@ DDDD;
 var_dump(token_get_all($str));
 ?>
 --EXPECTF--
-array(17) {
+array(19) {
   [0]=>
   array(2) {
     [0]=>
-    int(%d)
+    int(367)
     [1]=>
     string(6) "<?php
 "
@@ -26,7 +28,7 @@ array(17) {
   [1]=>
   array(2) {
     [0]=>
-    int(%d)
+    int(309)
     [1]=>
     string(2) "$x"
   }
@@ -35,7 +37,7 @@ array(17) {
   [3]=>
   array(2) {
     [0]=>
-    int(%d)
+    int(371)
     [1]=>
     string(6) "<<<DD
 "
@@ -43,7 +45,7 @@ array(17) {
   [4]=>
   array(2) {
     [0]=>
-    int(%d)
+    int(307)
     [1]=>
     string(13) "jhdsjkfhjdsh
 "
@@ -51,67 +53,83 @@ array(17) {
   [5]=>
   array(2) {
     [0]=>
-    int(%d)
+    int(372)
     [1]=>
     string(2) "DD"
   }
   [6]=>
-  string(1) "."
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+  }
   [7]=>
+  string(1) "."
+  [8]=>
   array(2) {
     [0]=>
-    int(%d)
+    int(315)
     [1]=>
     string(2) """"
   }
-  [8]=>
-  string(1) ";"
   [9]=>
+  string(1) ";"
+  [10]=>
   array(2) {
     [0]=>
-    int(%d)
+    int(370)
     [1]=>
     string(1) "
 "
   }
-  [10]=>
+  [11]=>
   array(2) {
     [0]=>
-    int(%d)
+    int(309)
     [1]=>
     string(2) "$a"
   }
-  [11]=>
-  string(1) "="
   [12]=>
+  string(1) "="
+  [13]=>
   array(2) {
     [0]=>
-    int(%d)
+    int(371)
     [1]=>
     string(8) "<<<DDDD
 "
   }
-  [13]=>
+  [14]=>
   array(2) {
     [0]=>
-    int(%d)
+    int(307)
     [1]=>
     string(13) "jhdsjkfhjdsh
 "
   }
-  [14]=>
+  [15]=>
   array(2) {
     [0]=>
-    int(%d)
+    int(372)
     [1]=>
     string(4) "DDDD"
   }
-  [15]=>
-  string(1) ";"
   [16]=>
+  string(1) ";"
+  [17]=>
+  array(2) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+  }
+  [18]=>
   array(2) {
     [0]=>
-    int(%d)
+    int(369)
     [1]=>
     string(2) "?>"
   }
diff --git a/ext/tokenizer/tokenizer.c b/ext/tokenizer/tokenizer.c
index e05bef24a..67b8c1a52 100644
--- a/ext/tokenizer/tokenizer.c
+++ b/ext/tokenizer/tokenizer.c
@@ -2,12 +2,12 @@
    +----------------------------------------------------------------------+
    | PHP Version 5                                                        |
    +----------------------------------------------------------------------+
-   | Copyright (c) 1997-2005 The PHP Group                                |
+   | Copyright (c) 1997-2006 The PHP Group                                |
    +----------------------------------------------------------------------+
-   | This source file is subject to version 3.0 of the PHP license,       |
+   | This source file is subject to version 3.01 of the PHP license,      |
    | that is bundled with this package in the file LICENSE, and is        |
    | available through the world-wide-web at the following url:           |
-   | http://www.php.net/license/3_0.txt.                                  |
+   | http://www.php.net/license/3_01.txt                                  |
    | If you did not receive a copy of the PHP license and are unable to   |
    | obtain it through the world-wide-web, please send a note to          |
    | license@php.net so we can mail you a copy immediately.               |
@@ -16,7 +16,7 @@
    +----------------------------------------------------------------------+
 */
 
-/* $Id: tokenizer.c,v 1.31.2.3 2005/11/27 06:41:31 iliaa Exp $ */
+/* $Id: tokenizer.c,v 1.31.2.5 2006/01/01 12:50:16 sniper Exp $ */
 
 #ifdef HAVE_CONFIG_H
 #include "config.h"
@@ -103,7 +103,7 @@ ZEND_DECLARE_MODULE_GLOBALS(tokenizer)
  *
  * Every user visible function must have an entry in tokenizer_functions[].
  */
-function_entry tokenizer_functions[] = {
+zend_function_entry tokenizer_functions[] = {
 	PHP_FE(token_get_all,	NULL)
 	PHP_FE(token_name,	NULL)
 	{NULL, NULL, NULL}	/* Must be the last line in tokenizer_functions[] */