698 lines
20 KiB
JavaScript
Executable File
698 lines
20 KiB
JavaScript
Executable File
require("amd-loader");
|
|
|
|
var fs = require("fs");
|
|
var util = require("util");
|
|
var lib = require("./lib");
|
|
var pathlib = require("path");
|
|
var parseLanguage = lib.parsePlist;
|
|
|
|
var tk = require("./regexp_tokenizer");
|
|
var tokenize = tk.tokenize;
|
|
var toStr = tk.toStr;
|
|
|
|
function last(array) {return array[array.length - 1]}
|
|
|
|
function convertHexEscape(tokens) {
|
|
var inChClass = false;
|
|
tokens.forEach(function(t) {
|
|
if (t.type == "charclass")
|
|
inChClass = true;
|
|
else if (t.type == "charclass.end")
|
|
inChClass = false;
|
|
else if (t.type == "charType"){
|
|
if (t.value == "\\h") {
|
|
t.type = "text";
|
|
t.value = inChClass ? "\\da-fA-F" : "[\\da-fA-F]";
|
|
}
|
|
else if (t.value == "\\H") {
|
|
if (inChClass) {
|
|
console.warn("can't convert \\H in charclass");
|
|
return;
|
|
}
|
|
t.type = "text";
|
|
t.value = "[^\\da-fA-F]";
|
|
}
|
|
}
|
|
});
|
|
return tokens;
|
|
}
|
|
|
|
function convertNewLinesTo$(str) {
|
|
var tokens = tokenize(str);
|
|
for (var i = 0; i < tokens.length; i++) {
|
|
var t= tokens[i];
|
|
if (t.type == "char" && t.value == "\\n") {
|
|
var p = tokens[i + 1] || {};
|
|
if (p.type != "quantifier") {
|
|
t.value = "$";
|
|
while (p.value == "\\n" || p.type == "quantifier") {
|
|
p.value = "";
|
|
p = tokens[++i + 1] || {};
|
|
}
|
|
} else if (/\?|\*|{,|{0,/.test(p.value)) {
|
|
t.value = p.value = "";
|
|
} else
|
|
p.value = "";
|
|
}
|
|
}
|
|
return toStr(tokens).replace(/[$]+/g, "$");
|
|
}
|
|
|
|
function convertCharacterTypes(str) {
|
|
var tokens = tokenize(str);
|
|
tokens = convertHexEscape(tokens);
|
|
|
|
var warn = false;
|
|
tokens.forEach(function(t){
|
|
if (t.type == "quantifier") {
|
|
var val = t.value;
|
|
if (val.slice(-1) == "+" && val.length > 1) {
|
|
t.value = val.slice(0, -1);
|
|
warn = val;
|
|
}
|
|
}
|
|
});
|
|
if (warn)
|
|
console.log("converted possesive quantifier " + warn + " to *");
|
|
return toStr(tokens);
|
|
}
|
|
|
|
function removeInlineFlags(str, rule) {
|
|
var tokens = tokenize(str);
|
|
var caseInsensitive = false;
|
|
tokens.forEach(function(t, i) {
|
|
if (t.type == "group.start" && /[imsx]/.test(t.value)) {
|
|
if (/i/.test(t.value))
|
|
caseInsensitive = true;
|
|
t.value = t.value.replace(/[imsx\-]/g, "");
|
|
var next = tokens[i + 1];
|
|
if (next && next.type == "group.end") {
|
|
t.value = next.value = "";
|
|
}
|
|
}
|
|
});
|
|
if (caseInsensitive && rule)
|
|
rule.caseInsensitive = true;
|
|
return toStr(tokens);
|
|
}
|
|
|
|
function convertToNonCapturingGroups(str) {
|
|
var tokens = tokenize(str);
|
|
tokens.forEach(function(t, i) {
|
|
if (t.type == "group.start" && t.value == "(")
|
|
t.value += "?:";
|
|
});
|
|
return toStr(tokens);
|
|
}
|
|
|
|
function simplifyNonCapturingGroups(str) {
|
|
var tokens = tokenize(str);
|
|
var t = tokens[0] || {};
|
|
if (t.type == "group.start" && t.value == "(?:"
|
|
&& t.end == last(tokens)) {
|
|
t.value = t.end.value = "";
|
|
}
|
|
var i = 0;
|
|
function iter(f) {
|
|
for (i = 0; i < tokens.length; i++)
|
|
f(tokens[i]);
|
|
}
|
|
function iterGroup(end, f) {
|
|
for (var i1 = i + 1; i1 < tokens.length; i1++) {
|
|
var t = tokens[i1];
|
|
if (t == end)
|
|
break;
|
|
var index = f && f(t);
|
|
if (index > i1)
|
|
i1 = index;
|
|
}
|
|
return i1;
|
|
}
|
|
|
|
iter(function (t) {
|
|
if (t.type == "group.start" && t.value == "(?:") {
|
|
if (!t.end)
|
|
return console.error("malformed regex: " + str);
|
|
|
|
var canRemove = true;
|
|
var next = tokens[tokens.indexOf(t.end, i) + 1];
|
|
if (next && next.type == "quantifier")
|
|
return;
|
|
iterGroup(t.end, function(t) {
|
|
if (t.type == "alternation")
|
|
canRemove = false;
|
|
else if (t.type == "group.start" && t.end)
|
|
return iterGroup(t.end);
|
|
});
|
|
if (canRemove)
|
|
t.value = t.end.value = "";
|
|
}
|
|
});
|
|
|
|
return toStr(tokens);
|
|
}
|
|
|
|
function removeLookBehinds(str) {
|
|
var tokens = tokenize(str);
|
|
var toRemove = null;
|
|
tokens.forEach(function(t, i) {
|
|
if (!toRemove && t.type == "group.start" && /</.test(t.value)) {
|
|
toRemove = t.end;
|
|
toRemove.content = [];
|
|
}
|
|
if (toRemove) {
|
|
toRemove.content.push(t.value);
|
|
t.value = "";
|
|
}
|
|
if (t == toRemove) {
|
|
var c = toRemove.content.slice(1, -1).join("");
|
|
if (/\^/.test(c))
|
|
toRemove.value = "(?:" + c +")";
|
|
|
|
toRemove = null;
|
|
}
|
|
});
|
|
return toStr(tokens);
|
|
}
|
|
|
|
function convertBeginEndBackrefs(rule) {
|
|
if (!/\\\d/.test(rule.end))
|
|
return;
|
|
var startTokens = tokenize(rule.begin);
|
|
var endTokens = tokenize(rule.end);
|
|
|
|
|
|
var groups = {};
|
|
startTokens.forEach(function(t, i) {
|
|
if (t.number && t.end && t.type == "group.start") {
|
|
var endIndex = startTokens.indexOf(t.end, i + 1);
|
|
var content = startTokens.slice(i+1, endIndex);
|
|
groups[t.number] = toStr(content);
|
|
}
|
|
});
|
|
|
|
endTokens.forEach(function(t) {
|
|
if (t.type == "backRef") {
|
|
var num = t.value.substr(1);
|
|
if (groups[num])
|
|
t.value = "(?:" + groups[num] + ")";
|
|
}
|
|
});
|
|
|
|
rule.end = toStr(endTokens);
|
|
|
|
console.warn("Begin-End-Backreference is detected", rule);
|
|
}
|
|
|
|
function checkForNamedCaptures(str) {
|
|
var tokens = tokenize(str);
|
|
tokens.forEach(function(t) {
|
|
if (t.type == "group.start" && t.name)
|
|
console.warn("named capture not implemented", str);
|
|
if (t.type == "backRef")
|
|
console.warn("backRef not implemented ", str);
|
|
});
|
|
}
|
|
|
|
function fixGroups(captures, defaultName, regex) {
|
|
var tokens = tokenize(regex);
|
|
|
|
var opened = [], isStart = true, i = 0;
|
|
function open() {
|
|
var t = {value: "(", type: "group.start", isGroup: true};
|
|
opened.push(t);
|
|
tokens.splice(i++, 0, t);
|
|
}
|
|
function close() {
|
|
var t = {value: ")", type: "group.start"};
|
|
t.start = opened.pop();
|
|
t.start.end = t;
|
|
tokens.splice(i++, 0, t);
|
|
}
|
|
function tryOpen(){if (isStart) {open(); isStart = false}}
|
|
function tryClose(){if (opened.length) close()}
|
|
function skip(t) {
|
|
var i1 = tokens.indexOf(t.end, i);
|
|
if (i1 > i)
|
|
i = i1;
|
|
}
|
|
function lst(t) {return t[t.length - 1]}
|
|
function iter(f) {
|
|
for (i = 0; i < tokens.length; i++)
|
|
f(tokens[i]);
|
|
}
|
|
function iterGroup(end, f) {
|
|
for (var i1 = i + 1; i1 < tokens.length; i1++) {
|
|
var t = tokens[i1];
|
|
if (t == end)
|
|
break;
|
|
f(t);
|
|
}
|
|
}
|
|
function peek() { return tokens[i + 1] || {}}
|
|
|
|
// groupify
|
|
iter(function(t){
|
|
if (t.type == "group.start") {
|
|
tryClose();
|
|
isStart = true;
|
|
if (!t.hasChildren || t.isSpecial)
|
|
skip(t);
|
|
} else if (t.type == "group.end") {
|
|
isStart = true;
|
|
tryClose();
|
|
} else if (t.type == "alternation") {
|
|
isStart = true;
|
|
tryClose();
|
|
} else if (t.type != "anchor" && t.type != "quantifier"){
|
|
tryOpen();
|
|
}
|
|
});
|
|
tryClose();
|
|
|
|
// remove redundand groups
|
|
var names = [defaultName];
|
|
iter(function(t){
|
|
if (t.type == "group.start" && !t.isSpecial) {
|
|
var captureName = captures[t.number];
|
|
|
|
if (!t.hasChildren) {
|
|
t.tokenName = captureName || lst(names);
|
|
skip(t);
|
|
} else {
|
|
var hasCapture = false;
|
|
iterGroup(t.end, function(t1) {
|
|
if (t1.type == "group.start" && captures[t1.number])
|
|
hasCapture = true;
|
|
});
|
|
if (hasCapture) {
|
|
t.value = "(?:";
|
|
if (captureName) {
|
|
names.push(captureName);
|
|
t.isTokenGroup = true;
|
|
}
|
|
} else {
|
|
t.tokenName = captureName || lst(names);
|
|
iterGroup(t.end, function(t1) {
|
|
if (t1.value == "(")
|
|
t1.value = "(?:";
|
|
});
|
|
}
|
|
}
|
|
} else if (t.type == "group.end") {
|
|
if (t.start.isTokenGroup)
|
|
names.pop();
|
|
}
|
|
});
|
|
|
|
// wrap capturing groups with quantifier
|
|
iter(function(t){
|
|
if (t.type == "group.end" && t.start.value == "(" && peek().type == "quantifier") {
|
|
peek().value += ")";
|
|
t.start.value += "(?:";
|
|
}
|
|
});
|
|
|
|
names = [];
|
|
tokens.forEach(function(t) {
|
|
if (t.value == "(" || t.value == "((?:" )
|
|
t.tokenName && names.push(t.tokenName);
|
|
});
|
|
return {
|
|
names: names,
|
|
regex: toStr(tokens)
|
|
};
|
|
}
|
|
|
|
/***** converter */
|
|
|
|
function logDebug(string, obj) {
|
|
console.log(string, obj);
|
|
}
|
|
|
|
|
|
// tmLanguage processor
|
|
|
|
// for tracking token states
|
|
var states = {start: []};
|
|
|
|
function processRules(rules){
|
|
if (rules.patterns)
|
|
states.start = processPatterns(rules.patterns);
|
|
if (rules.repository)
|
|
processRepository(rules.repository);
|
|
return states;
|
|
}
|
|
function processRepository(r) {
|
|
for (var key in r) {
|
|
var p = r[key];
|
|
if (p.begin)
|
|
var stateObj = [processPattern(r[key])];
|
|
else if (p.patterns && !p.repository)
|
|
var stateObj = processPatterns(p.patterns);
|
|
else
|
|
var stateObj = [processPattern(r[key])];
|
|
|
|
if (stateObj)
|
|
states["#" + key] = stateObj;
|
|
}
|
|
}
|
|
function processPatterns(pl) {
|
|
return pl.map(processPattern);
|
|
}
|
|
function processPattern(p) {
|
|
if (p.end == "(?!\\G)" && p.patterns && p.patterns.length == 1) {
|
|
var rule = processPattern(p.patterns[0]);
|
|
}
|
|
else if (p.begin != null && p.end != null) {
|
|
convertBeginEndBackrefs(p);
|
|
|
|
var rule = simpleRule(p.begin, p.name, p.beginCaptures || p.captures);
|
|
|
|
var next = processPatterns(p.patterns || []);
|
|
var endRule = simpleRule(p.end, p.name, p.endCaptures || p.captures);
|
|
endRule.next = "pop";
|
|
if (p.applyEndPatternLast)
|
|
next.push(endRule);
|
|
else
|
|
next.unshift(endRule);
|
|
|
|
if (p.name || p.contentName)
|
|
next.push({defaultToken: p.name || p.contentName});
|
|
|
|
rule.push = next;
|
|
|
|
rule = removeIncludeSelf(rule);
|
|
}
|
|
else if (p.match) {
|
|
var rule = simpleRule(p.match, p.name, p.captures);
|
|
}
|
|
else if (p.include) {
|
|
var rule = {include: p.include};
|
|
}
|
|
else {
|
|
var rule = {todo: p};
|
|
}
|
|
|
|
if (p.comment)
|
|
rule.comment = (rule.comment || "") + p.comment;
|
|
|
|
if (p.repository)
|
|
processRepository(p.repository);
|
|
return rule;
|
|
}
|
|
function simpleRule(regex, name, captures) {
|
|
name = name || "text";
|
|
var rule = {token: "", regex: ""};
|
|
|
|
var origRegex = regex;
|
|
regex = transformRegExp(origRegex, rule);
|
|
if (captures) {
|
|
var tokenArray = [];
|
|
Object.keys(captures).forEach(function(x){
|
|
tokenArray[x] = captures[x] && captures[x].name;
|
|
});
|
|
|
|
if (tokenArray.length == 1) {
|
|
name = tokenArray[0];
|
|
} else {
|
|
var fixed = fixGroups(tokenArray, name, regex);
|
|
name = fixed.names;
|
|
regex = fixed.regex;
|
|
if (name.length == 1)
|
|
name = name[0];
|
|
}
|
|
}
|
|
|
|
if (typeof name == "string")
|
|
regex = convertToNonCapturingGroups(regex);
|
|
|
|
regex = simplifyNonCapturingGroups(regex);
|
|
|
|
try {new RegExp(regex);} catch(e) {
|
|
rule.TODO = "FIXME: regexp doesn't have js equivalent";
|
|
rule.originalRegex = origRegex;
|
|
|
|
// lookbehinds are mostly used to force ordering
|
|
// regex = removeLookBehinds(regex);
|
|
}
|
|
rule.token = name;
|
|
rule.regex = regex;
|
|
return rule;
|
|
}
|
|
|
|
function removeIncludeSelf(rule) {
|
|
if (!rule.push)
|
|
return rule;
|
|
var hasSelfInclude = false;
|
|
var escapeRule = null;
|
|
var complexSelfInclude = false;
|
|
rule.push.forEach(function(sub) {
|
|
if (sub.include == "$self") {
|
|
hasSelfInclude = true;
|
|
} else if (sub.defaultToken) {
|
|
return;
|
|
} else if (sub.next == "pop") {
|
|
escapeRule = sub;
|
|
} else
|
|
complexSelfInclude = true;
|
|
});
|
|
|
|
if (hasSelfInclude) {
|
|
console.warn("can't convert include $self");
|
|
return {todo: rule};
|
|
|
|
if (complexSelfInclude) {
|
|
console.warn("can't convert include $self");
|
|
rule.toDo = "include $self not fully supported";
|
|
return rule;
|
|
}
|
|
console.warn("include $self not fully supported");
|
|
delete rule.push;
|
|
delete escapeRule.next;
|
|
rule.includeSelf = true;
|
|
escapeRule.includeSelf = true;
|
|
return [rule, escapeRule];
|
|
}
|
|
return rule;
|
|
}
|
|
|
|
// regex transformation
|
|
|
|
function removeXFlag(str) {
|
|
var tokens = tokenize(str);
|
|
return toStr(tokens);
|
|
}
|
|
|
|
function transformRegExp(str, rule) {
|
|
str = convertNewLinesTo$(str);
|
|
|
|
str = removeInlineFlags(str, rule);
|
|
|
|
str = str.replace(/(\\[xu]){([a-fA-F\d]+)}/g, '$1$2');
|
|
|
|
str = convertCharacterTypes(str);
|
|
|
|
checkForNamedCaptures(str);
|
|
|
|
return str;
|
|
}
|
|
|
|
//
|
|
function extractPatterns(tmRules) {
|
|
return processRules(tmRules);
|
|
}
|
|
|
|
|
|
function detectLoops(states) {
|
|
var data = {};
|
|
var keys = Object.keys(states);
|
|
var flattenedStates = {};
|
|
function addRef(item, name) {
|
|
if (item.refs.indexOf(name) == -1)
|
|
item.refs.push(name);
|
|
}
|
|
function anonStateId(name, next) {
|
|
var i = 0, old = name;
|
|
while (flattenedStates[name] || states[name]) {
|
|
name = old + "_" + i++;
|
|
}
|
|
// console.log(old, name)
|
|
return name;
|
|
}
|
|
function addState(key, rules) {
|
|
if (rules && !flattenedStates[key])
|
|
flattenedStates[key] = rules;
|
|
return rules || flattenedStates[key];
|
|
}
|
|
|
|
|
|
for (var i = 0; i < keys.length; i++) {
|
|
var key = keys[i];
|
|
var state = addState(key, states[key]);
|
|
|
|
var item = data[key] || (data[key] = {/* name: key, */ refs: []});
|
|
state.forEach(function(rule) {
|
|
var next = rule.push || rule.next;
|
|
if (next == "pop") {
|
|
// nothing
|
|
} else if (typeof next == "string") {
|
|
addRef(item, next);
|
|
} else if (next) {
|
|
var anonId = anonStateId(key, next);
|
|
addState(anonId, next);
|
|
if (rule.push)
|
|
addRef(item, anonId);
|
|
keys.push(anonId);
|
|
} else if (rule.include) {
|
|
addRef(item, rule.include);
|
|
}
|
|
});
|
|
}
|
|
|
|
|
|
var cycles = [];
|
|
function addPath(start, path) {
|
|
var node = data[start];
|
|
path.push(start);
|
|
if (!node || !node.refs)
|
|
console.log(start);
|
|
var i = path.indexOf(start);
|
|
if (i > -1 && i != path.length - 1 || start == "$self" || start == "$base") {
|
|
if (i != -1)
|
|
path = path.slice(i);
|
|
for (var j = 0; j < cycles.length; j++) {
|
|
if (cycles[j] + "" == path + "")
|
|
return;
|
|
}
|
|
return cycles.push(path);
|
|
}
|
|
|
|
if (!node || !node.refs || !node.refs.length || path.length>30)
|
|
return;
|
|
node.refs.forEach(function(x) {
|
|
addPath(x, path.concat());
|
|
});
|
|
}
|
|
addPath("start", []);
|
|
|
|
console.error(cycles.join("\n"));
|
|
}
|
|
|
|
|
|
function test(fileName) {
|
|
console.log("testing highlighter");
|
|
try {
|
|
var module = require(fileName);
|
|
var Mode = module[Object.keys(module)[0]];
|
|
var mode = new Mode();
|
|
mode.getTokenizer().getLineTokens("hello world");
|
|
} catch(e) {
|
|
console.log(e);
|
|
}
|
|
}
|
|
|
|
function guessComment(patterns) {
|
|
var comment = {};
|
|
for (var i in patterns) {
|
|
var state = patterns[i];
|
|
state.forEach(function(r) {
|
|
if (typeof r.token == "string") {
|
|
if (/\bcomment\b/.test(r.token)) {
|
|
comment.line = r.regex;
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
return comment;
|
|
}
|
|
|
|
// cli stuff
|
|
var modeTemplate = fs.readFileSync(__dirname + "/templates/mode.js", "utf8");
|
|
var modeHighlightTemplate = fs.readFileSync(__dirname + "/templates/highlight_rules.js", "utf8");
|
|
|
|
function fetchAndConvert(name) {
|
|
console.log("Converting " + name);
|
|
if (/^http/.test(name)) {
|
|
if (/:\/\/github.com/.test(name)) {
|
|
name = name.replace(/\/blob\//, "/").replace("github.com", "raw.github.com");
|
|
}
|
|
return lib.download(name, function(data) {
|
|
convertTmLanguage(name, data);
|
|
});
|
|
}
|
|
var path = /^(\/|\w:)/.test(name) ? name : process.cwd() + "/" + name;
|
|
var langStr = fs.readFileSync(path, "utf8");
|
|
convertTmLanguage(name, langStr);
|
|
}
|
|
|
|
|
|
function convertTmLanguage(name, langStr) {
|
|
parseLanguage(langStr, function(language) {
|
|
var highlighterFilename = lib.snakeCase(language.name).replace(/[^\w]/g, "");
|
|
var languageNameSanitized = lib.camelCase(language.name).replace(/[^\w]/g, "");
|
|
|
|
require("./add_mode")(languageNameSanitized, (language.fileTypes || []).join("|"));
|
|
|
|
var highlighterFile = pathlib.normalize(lib.AceLib + "ace/mode/" + highlighterFilename + "_highlight_rules.js");
|
|
var modeFile = pathlib.normalize(lib.AceLib + "ace/mode/" + highlighterFilename + ".js");
|
|
|
|
if (devMode) {
|
|
console.log(util.inspect(language.patterns, false, 4));
|
|
console.log(util.inspect(language.repository, false, 4));
|
|
}
|
|
|
|
var patterns = extractPatterns(language);
|
|
detectLoops(patterns);
|
|
|
|
// var uuid = language.uuid
|
|
delete language.uuid;
|
|
delete language.patterns;
|
|
delete language.repository;
|
|
|
|
var comment = guessComment(patterns);
|
|
var languageMode = lib.fillTemplate(modeTemplate, {
|
|
language: languageNameSanitized,
|
|
languageHighlightFilename: highlighterFilename,
|
|
lineCommentStart: JSON.stringify(comment.line || "//"),
|
|
blockCommentStart: JSON.stringify(comment.start || "/*"),
|
|
blockCommentEnd: JSON.stringify(comment.end || "*/")
|
|
});
|
|
|
|
var languageHighlightRules = lib.fillTemplate(modeHighlightTemplate, {
|
|
language: languageNameSanitized,
|
|
languageTokens: lib.formatJS(patterns, " ").trim(),
|
|
uuid: language.uuid,
|
|
name: name,
|
|
metaData: lib.formatJS(language, "").trim()
|
|
});
|
|
|
|
if (devMode) {
|
|
console.log(languageMode);
|
|
console.log(languageHighlightRules);
|
|
console.log("Not writing, 'cause we're in dev mode, baby.");
|
|
}
|
|
else {
|
|
fs.writeFileSync(highlighterFile, languageHighlightRules);
|
|
fs.writeFileSync(modeFile, languageMode);
|
|
console.log("created file " + highlighterFile);
|
|
test(modeFile);
|
|
}
|
|
});
|
|
}
|
|
|
|
if (!module.parent) {
|
|
var args = process.argv.splice(2);
|
|
var devMode = args[0] == "--dev";
|
|
if (devMode)
|
|
args.shift();
|
|
if (args.length < 1) {
|
|
console.error("Usage: node tmlanguage.js [--dev] path/or/url/to/syntax.file ...");
|
|
process.exit(1);
|
|
}
|
|
args.forEach(fetchAndConvert);
|
|
} else {
|
|
exports.fetchAndConvert = fetchAndConvert;
|
|
}
|