Source for file Tokenizer.php
Documentation is available at Tokenizer.php
/* vim: set expandtab tabstop=4 shiftwidth=4: */
// +----------------------------------------------------------------------+
// +----------------------------------------------------------------------+
// | Copyright (c) 1997-2002 The PHP Group |
// +----------------------------------------------------------------------+
// | This source file is subject to version 2.02 of the PHP license, |
// | that is bundled with this package in the file LICENSE, and is |
// | available at through the world-wide-web at |
// | http://www.php.net/license/2_02.txt. |
// | If you did not receive a copy of the PHP license and are unable to |
// | obtain it through the world-wide-web, please send a note to |
// | license@php.net so we can mail you a copy immediately. |
// +----------------------------------------------------------------------+
// | Authors: nobody <nobody@localhost> |
// +----------------------------------------------------------------------+
// $Id: Tokenizer.php 216607 2006-07-18 00:53:10Z cellog $
require_once dirname (__FILE__ ) . '/PHP_Parser.php';
* The tokenizer wrapper for parser - implements the 'standard?' yylex interface
* <li>constructor, which takes the data to parse
* calls php's internal tokenizer, then tidies up the array
* a little (key=>value) rather than mixed type.</li>
* <li>advance, which returns true while tokens are available
* <li>parseError, which returns a string to appear on parser error messages.
* (could also display some of the code that has an error)
* - {@link $line} - current line number
* - {@link $pos} - current token id
* - {@link $N} - total no. of tokens
* @version $Id: Tokenizer.php 216607 2006-07-18 00:53:10Z cellog $
class PHP_Parser_Tokenizer {
* Tokens - array of all the tokens.
* Total Number of tokens.
* Current token position.
* The current token (either a ord(';') or token numer - see php tokenizer.
* The value associated with a token - eg. for T_STRING it's the string
* The value associated with a token plus preceding whitespace, if any
* This is only filled if whitespace attachment is turned on, for performance reasons
var $valueWithWhitespace;
* ID of the last Comment Token
* ID of the last Comment Token
* The string of the last Comment Token
* String of global variable to search for
* phpDocumentor-specific usage, extracted from
* documentation's @global tag
var $_globalSearch = false;
private $_trackingWhitespace = 0;
* Load the tokenizer - with a string to tokenize.
* tidies up array, sets vars pos, line, N and tokens
* @param string PHP code to serialize
function __construct ($data, $options = array ())
$this->_options['documentationParser'] =
$this->_options['documentationLexer'] =
$this->_options['publishAllDocumentation'] =
$this->_options['documentationContainer'] =
$this->_options['publisher'] =
$this->_options['publishMethod'] =
$this->_options['publishMessageClass'] =
$this->_options['publishDocumentation'] =
$this->_options['publishDocumentationMessage'] =
$this->_options = array_merge($this->_options, $options);
if (!class_exists($this->_options['documentationContainer'])) {
$this->_options['documentationContainer'] = false;
if (!is_object($this->_options['documentationParser'])) {
$this->_options['documentationParser'] = false;
$this->_options['documentationLexer'] = false;
$this->_options['documentationParser'] = &$options['documentationParser'];
$this->_options['documentationLexer'] = &$options['documentationLexer'];
// make sure it's an exact match
if (!is_object($this->_options['publisher'])) {
$this->_options['publisher'] = false;
$this->_options['publishAllDocumentation'] = false;
if (!method_exists($this->_options['publisher'], $this->_options['publishMethod'])) {
$this->_options['publishMethod'] = false;
$this->_options['publisher'] = false;
$this->_options['publishAllDocumentation'] = false;
$this->_options['publishMethod'] = 'publish';
if (!class_exists($this->_options['publishMessageClass'])) {
$this->_options['publishMessageClass'] = false;
$this->N = count($this->tokens);
for ($i=0; $i< $this->N; $i++ ) {
$this->tokens[$i] = array (ord($this->tokens[$i]),$this->tokens[$i]);
* Return the whitespace (if any) that preceded the current token
return $this->_whitespace;
function handleMessage ($msg)
if ($msg->getMsg () == 'find global') {
$this->_setGlobalSearch ($msg->getData ());
function _setGlobalSearch ($var)
$this->_globalSearch = $var;
function trackWhitespace ()
$this->_trackingWhitespace++;
function stopTrackingWhitespace ()
$this->_trackingWhitespace--;
* Compare global variable to search value, to see if we've
* found a variable that must be documented
* @param string global variable found in source code
function globalSearch ($var)
if ($this->_globalSearch) {
$ret = $var == $this->_globalSearch;
$this->_globalSearch = false;
* get the last comment block (and reset it)
* @return array ($commmentstring and $tokenPosition)
function getLastComment ()
$com = $this->lastComment;
$tok = $this->lastCommentToken;
$line = $this->lastCommentLine;
$this->lastCommentToken = -1;
$this->lastCommentLine = -1;
return array ($com, $tok, $line);
* Helper function for advance(), parses and publishes doc
function _handleDocumentation ()
$this->lastComment = $this->tokens[$this->pos][1 ];
$this->lastCommentLine = $this->line;
$this->lastCommentToken = $this->pos;
if ($this->_options['documentationParser']) {
$parser = &$this->_options['documentationParser'];
$err = $parser->parse (array ('comment' => $this->lastComment,
'commentline' => $this->lastCommentLine,
'commenttoken' => $this->lastCommentToken,
'lexer' => $this->_options['documentationLexer'],
if (PEAR ::isError ($err)) {
$this->lastComment = false;
$this->lastCommentLine = -1;
$this->lastCommentToken = -1;
$this->lastComment = $err;
if ($this->_options['publishAllDocumentation']) {
$publish = $this->_options['publishMethod'];
$message = 'documentation';
if ($this->_options['publishDocumentationMessage']) {
$message = $this->_options['publishDocumentationMessage'];
if ($this->_options['publishMessageClass']) {
$pc = $this->_options['publishMessageClass'];
$publisher = $this->_options['publisher'];
$message = new $pc($message, $this->lastComment);
$publisher->$publish($pc);
$publisher = $this->_options['publisher'];
$publisher->$publish($message, $this->lastComment);
* The main advance call required by the parser
* return true if a token is available, false if no more are available.
* skips stuff that is not a valid token
* stores lastcomment, lastcommenttoken
* @return boolean - true = have tokens
while ($this->pos < $this->N) {
echo token_name($this->tokens[$this->pos][0 ]). '(' .
(PHPyyParser ::tokenName (PHPyyParser ::$transTable[$this->tokens[$this->pos[0 ]]])) .
')' ." : {$this->tokens[$this->pos][1 ]}\n";
static $T_DOC_COMMENT = false;
$T_DOC_COMMENT = defined ('T_DOC_COMMENT') ? constant ('T_DOC_COMMENT') : 10000;
switch ($this->tokens [$this->pos ][0 ]) {
case T_OPEN_TAG_WITH_ECHO:
// comments - store for phpdoc
$this->lastCommentToken = -1;
$this->lastCommentLine = -1;
if (substr($this->tokens[$this->pos][1 ],0 ,2 ) == '/*') {
$this->_handleDocumentation ();
$this->line += substr_count ($this->tokens[$this->pos][1 ], "\n");
$this->_handleDocumentation ();
// ... continues into m/l skipeed tags..
case T_ENCAPSED_AND_WHITESPACE:
if ($this->tokens[$this->pos][0 ] == T_WHITESPACE ) {
$this->_whitespace = $this->tokens[$this->pos][1 ];
$this->line += substr_count ($this->tokens[$this->pos][1 ], "\n");
//--- begin returnable values--
// end statement - clear any comment details.
$this->lastCommentToken = -1;
$this->line += substr_count ($this->tokens[$this->pos][1 ], "\n");
$this->token = $this->tokens[$this->pos][0 ];
$this->value = $this->tokens[$this->pos][1 ];
$this->token = PHPyyParser ::$transTable[$this->token];
$this->valueWithWhitespace = $this->_whitespace . $this->value;
if ($this->_trackingWhitespace ) {
return $this->valueWithWhitespace;
* return something useful, when a parse error occurs.
* used to build error messages if the parser fails, and needs to know the line number..
return " Error at line {$this->line }";
Documentation generated on Mon, 11 Mar 2019 15:41:13 -0400 by phpDocumentor 1.4.4. PEAR Logo Copyright © PHP Group 2004.
|