Server : Apache/2 System : Linux server-15-235-50-60 5.15.0-164-generic #174-Ubuntu SMP Fri Nov 14 20:25:16 UTC 2025 x86_64 User : gositeme ( 1004) PHP Version : 8.2.29 Disable Function : exec,system,passthru,shell_exec,proc_close,proc_open,dl,popen,show_source,posix_kill,posix_mkfifo,posix_getpwuid,posix_setpgid,posix_setsid,posix_setuid,posix_setgid,posix_seteuid,posix_setegid,posix_uname Directory :  /home/gositeme/domains/lavocat.quebec/private_html/node_modules/lop/test/

Upload File : current_dir [ Writeable ] document_root [ Writeable ]

var Token = require("../lib/Token");
var StringSource = require("../lib/StringSource");

var Tokeniser = module.exports = function(options) {
    var keywords = options.keywords;
    var tokenise = function(input) {
        var source = new StringSource(input);
        
        var createToken = function(startIndex, endIndex) {
            var value = input.substring(startIndex, endIndex);
            var tokenType = keywords.indexOf(value) === -1 ? "identifier" : "keyword";
            return new Token(
                tokenType,
                value,
                source.range(startIndex, endIndex)
            );
        };
        
        var position = 0;
        var tokens = [];
        var done = input === "";
        
        while (!done) {
            var nextWhitespace = indexOfRegex(input, /\s/, position);
            if (nextWhitespace === -1) {
                done = true;
                tokens.push(createToken(position, input.length));
            } else {
                tokens.push(createToken(position, nextWhitespace));
                position = indexOfRegex(input, /\S/, nextWhitespace);
            }
        }
        tokens.push(new Token("end", null, source.range(input.length, input.length)));
        return tokens;
    };
    
    var indexOfRegex = function(string, regex, startIndex) {
        startIndex = startIndex || 0;
        var index = string.substring(startIndex).search(regex);
        if (index === -1) {
            return -1;
        } else {
            return index + startIndex;
        }
    };
    
    return {
        tokenise: tokenise
    };
};