function Tokenizer(text) {
        this._state = TEXT;
        this._buffer = text;
        this._sectionStart = 0;
        this._sectionStartPos = {line: 0, ch: 0};
        this._index = 0;
        this._indexPos = {line: 0, ch: 0};
        this._special = 0; // 1 for script, 2 for style
        this._token = null;
        this._nextToken = null;
    }
function Tokenizer(text) {
        this._state = TEXT;
        this._buffer = text;
        this._sectionStart = 0;
        this._sectionStartPos = {line: 0, ch: 0};
        this._index = 0;
        this._indexPos = {line: 0, ch: 0};
        this._special = 0; // 1 for script, 2 for style
        this._token = null;
        this._nextToken = null;
    }
function Tokenizer ( text ) { this . _state = TEXT ; this . _buffer = text ; this . _sectionStart = 0 ; this . _sectionStartPos = { line : 0 , ch : 0 } ; this . _index = 0 ; this . _indexPos = { line : 0 , ch : 0 } ; this . _special = 0 ; // 1 for script, 2 for style this . _token = null ; this . _nextToken = null ; }
A simple HTML tokenizer. See the description of nextToken() for usage details.
@constructor
@param {string} text The HTML document to tokenize.
A simple HTML tokenizer . See the description of nextToken () for usage details .
A tokenizer that can be used to parse a text