From 4cd50f0545da947522408471a0cbceb35fa0f053 Mon Sep 17 00:00:00 2001 From: Timothy Farrell Date: Sat, 15 Dec 2012 21:28:17 -0600 Subject: [PATCH 1/2] Consolidate variable declarations and make JSHint happy. There are no logic changes to this commit. --- mode/python/python.js | 169 ++++++++++++++++++++++-------------------- 1 file changed, 88 insertions(+), 81 deletions(-) diff --git a/mode/python/python.js b/mode/python/python.js index a12e326dc8..ee7c14d2c4 100644 --- a/mode/python/python.js +++ b/mode/python/python.js @@ -1,24 +1,20 @@ +/*global CodeMirror:false */ + CodeMirror.defineMode("python", function(conf, parserConf) { - var ERRORCLASS = 'error'; - - function wordRegexp(words) { - return new RegExp("^((" + words.join(")|(") + "))\\b"); - } - - var singleOperators = new RegExp("^[\\+\\-\\*/%&|\\^~<>!]"); - var singleDelimiters = new RegExp('^[\\(\\)\\[\\]\\{\\}@,:`=;\\.]'); - var doubleOperators = new RegExp("^((==)|(!=)|(<=)|(>=)|(<>)|(<<)|(>>)|(//)|(\\*\\*))"); - var doubleDelimiters = new RegExp("^((\\+=)|(\\-=)|(\\*=)|(%=)|(/=)|(&=)|(\\|=)|(\\^=))"); - var tripleDelimiters = new RegExp("^((//=)|(>>=)|(<<=)|(\\*\\*=))"); - var identifiers = new RegExp("^[_A-Za-z][_A-Za-z0-9]*"); - - var wordOperators = wordRegexp(['and', 'or', 'not', 'is', 'in']); - var commonkeywords = ['as', 'assert', 'break', 'class', 'continue', + var ERRORCLASS = 'error', + singleOperators = new RegExp("^[\\+\\-\\*/%&|\\^~<>!]"), + singleDelimiters = new RegExp('^[\\(\\)\\[\\]\\{\\}@,:`=;\\.]'), + doubleOperators = new RegExp("^((==)|(!=)|(<=)|(>=)|(<>)|(<<)|(>>)|(//)|(\\*\\*))"), + doubleDelimiters = new RegExp("^((\\+=)|(\\-=)|(\\*=)|(%=)|(/=)|(&=)|(\\|=)|(\\^=))"), + tripleDelimiters = new RegExp("^((//=)|(>>=)|(<<=)|(\\*\\*=))"), + identifiers = new RegExp("^[_A-Za-z][_A-Za-z0-9]*"), + wordOperators = wordRegexp(['and', 'or', 'not', 'is', 'in']), + commonkeywords = ['as', 'assert', 'break', 'class', 'continue', 'def', 'del', 'elif', 'else', 'except', 'finally', 'for', 'from', 'global', 'if', 'import', 'lambda', 'pass', 'raise', 'return', - 'try', 'while', 'with', 'yield']; - var commonBuiltins = ['abs', 'all', 'any', 'bin', 'bool', 'bytearray', 'callable', 'chr', + 'try', 'while', 'with', 'yield'], + commonBuiltins = ['abs', 'all', 'any', 'bin', 'bool', 'bytearray', 'callable', 'chr', 'classmethod', 'compile', 'complex', 'delattr', 'dict', 'dir', 'divmod', 'enumerate', 'eval', 'filter', 'float', 'format', 'frozenset', 'getattr', 'globals', 'hasattr', 'hash', 'help', 'hex', 'id', @@ -28,35 +24,40 @@ CodeMirror.defineMode("python", function(conf, parserConf) { 'repr', 'reversed', 'round', 'set', 'setattr', 'slice', 'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', 'type', 'vars', 'zip', '__import__', 'NotImplemented', - 'Ellipsis', '__debug__']; - var py2 = {'builtins': ['apply', 'basestring', 'buffer', 'cmp', 'coerce', 'execfile', + 'Ellipsis', '__debug__'], + py2 = {'builtins': ['apply', 'basestring', 'buffer', 'cmp', 'coerce', 'execfile', 'file', 'intern', 'long', 'raw_input', 'reduce', 'reload', 'unichr', 'unicode', 'xrange', 'False', 'True', 'None'], - 'keywords': ['exec', 'print']}; - var py3 = {'builtins': ['ascii', 'bytes', 'exec', 'print'], - 'keywords': ['nonlocal', 'False', 'True', 'None']}; + 'keywords': ['exec', 'print']}, + py3 = {'builtins': ['ascii', 'bytes', 'exec', 'print'], + 'keywords': ['nonlocal', 'False', 'True', 'None']}, + stringPrefixes = new RegExp("^(([rub]|(ur)|(br))?('{3}|\"{3}|['\"]))", "i"), + indentInfo = null, + keywords, builtins; + + function wordRegexp(words) { + return new RegExp("^((" + words.join(")|(") + "))\\b"); + } if (!!parserConf.version && parseInt(parserConf.version, 10) === 3) { commonkeywords = commonkeywords.concat(py3.keywords); commonBuiltins = commonBuiltins.concat(py3.builtins); - var stringPrefixes = new RegExp("^(([rb]|(br))?('{3}|\"{3}|['\"]))", "i"); + stringPrefixes = new RegExp("^(([rb]|(br))?('{3}|\"{3}|['\"]))", "i"); } else { commonkeywords = commonkeywords.concat(py2.keywords); commonBuiltins = commonBuiltins.concat(py2.builtins); - var stringPrefixes = new RegExp("^(([rub]|(ur)|(br))?('{3}|\"{3}|['\"]))", "i"); } - var keywords = wordRegexp(commonkeywords); - var builtins = wordRegexp(commonBuiltins); - - var indentInfo = null; + keywords = wordRegexp(commonkeywords); + builtins = wordRegexp(commonBuiltins); // tokenizers function tokenBase(stream, state) { + var scopeOffset, lineOffset, ch, floatLiteral, intLiteral; // Handle scope changes if (stream.sol()) { - var scopeOffset = state.scopes[0].offset; + scopeOffset = state.scopes[0].offset; if (stream.eatSpace()) { - var lineOffset = stream.indentation(); + lineOffset = stream.indentation(); if (lineOffset > scopeOffset) { indentInfo = 'indent'; } else if (lineOffset < scopeOffset) { @@ -72,18 +73,18 @@ CodeMirror.defineMode("python", function(conf, parserConf) { if (stream.eatSpace()) { return null; } - - var ch = stream.peek(); - + + ch = stream.peek(); + // Handle Comments if (ch === '#') { stream.skipToEnd(); return 'comment'; } - + // Handle Number Literals if (stream.match(/^[0-9\.]/, false)) { - var floatLiteral = false; + floatLiteral = false; // Floats if (stream.match(/^\d*\.\d+(e[\+\-]?\d+)?/i)) { floatLiteral = true; } if (stream.match(/^\d+\.\d*/)) { floatLiteral = true; } @@ -94,7 +95,7 @@ CodeMirror.defineMode("python", function(conf, parserConf) { return 'number'; } // Integers - var intLiteral = false; + intLiteral = false; // Hex if (stream.match(/^0x[0-9a-f]+/i)) { intLiteral = true; } // Binary @@ -116,50 +117,52 @@ CodeMirror.defineMode("python", function(conf, parserConf) { return 'number'; } } - + // Handle Strings if (stream.match(stringPrefixes)) { state.tokenize = tokenStringFactory(stream.current()); return state.tokenize(stream, state); } - + // Handle operators and Delimiters if (stream.match(tripleDelimiters) || stream.match(doubleDelimiters)) { return null; } - if (stream.match(doubleOperators) - || stream.match(singleOperators) - || stream.match(wordOperators)) { + + if (stream.match(doubleOperators) || stream.match(singleOperators) || stream.match(wordOperators)) { return 'operator'; } + if (stream.match(singleDelimiters)) { return null; } - + if (stream.match(keywords)) { return 'keyword'; } - + if (stream.match(builtins)) { return 'builtin'; } - + if (stream.match(identifiers)) { return 'variable'; } - + // Handle non-detected items stream.next(); return ERRORCLASS; } - + function tokenStringFactory(delimiter) { + var OUTCLASS = 'string', + singleline = 1; + while ('rub'.indexOf(delimiter.charAt(0).toLowerCase()) >= 0) { delimiter = delimiter.substr(1); } - var singleline = delimiter.length == 1; - var OUTCLASS = 'string'; - + singleline = delimiter.length === 1; + function tokenString(stream, state) { while (!stream.eol()) { stream.eatWhile(/[^'"\\]/); @@ -187,16 +190,17 @@ CodeMirror.defineMode("python", function(conf, parserConf) { tokenString.isString = true; return tokenString; } - + function indent(stream, state, type) { + var indentUnit = 0, + i; type = type || 'py'; - var indentUnit = 0; if (type === 'py') { if (state.scopes[0].type !== 'py') { state.scopes[0].offset = stream.indentation(); return; } - for (var i = 0; i < state.scopes.length; ++i) { + for (i = 0; i < state.scopes.length; ++i) { if (state.scopes[i].type === 'py') { indentUnit = state.scopes[i].offset + conf.indentUnit; break; @@ -210,14 +214,16 @@ CodeMirror.defineMode("python", function(conf, parserConf) { type: type }); } - + function dedent(stream, state, type) { + var _indent, _indent_index, i; + type = type || 'py'; - if (state.scopes.length == 1) return; + if (state.scopes.length === 1) { return; } if (state.scopes[0].type === 'py') { - var _indent = stream.indentation(); - var _indent_index = -1; - for (var i = 0; i < state.scopes.length; ++i) { + _indent = stream.indentation(); + _indent_index = -1; + for (i = 0; i < state.scopes.length; ++i) { if (_indent === state.scopes[i].offset) { _indent_index = i; break; @@ -235,7 +241,7 @@ CodeMirror.defineMode("python", function(conf, parserConf) { state.scopes[0].offset = stream.indentation(); return false; } else { - if (state.scopes[0].type != type) { + if (state.scopes[0].type !== type) { return true; } state.scopes.shift(); @@ -245,9 +251,11 @@ CodeMirror.defineMode("python", function(conf, parserConf) { } function tokenLexer(stream, state) { + var style, current, delimiter_index; + indentInfo = null; - var style = state.tokenize(stream, state); - var current = stream.current(); + style = state.tokenize(stream, state); + current = stream.current(); // Handle '.' connected identifiers if (current === '.') { @@ -259,27 +267,25 @@ CodeMirror.defineMode("python", function(conf, parserConf) { } return style; } - + // Handle decorators if (current === '@') { return stream.match(identifiers, false) ? 'meta' : ERRORCLASS; } - if ((style === 'variable' || style === 'builtin') - && state.lastToken === 'meta') { + if ((style === 'variable' || style === 'builtin') && state.lastToken === 'meta') { style = 'meta'; } - + // Handle scope changes. if (current === 'pass' || current === 'return') { state.dedent += 1; } - if (current === 'lambda') state.lambda = true; - if ((current === ':' && !state.lambda && state.scopes[0].type == 'py') - || indentInfo === 'indent') { + if (current === 'lambda') { state.lambda = true; } + if ((current === ':' && !state.lambda && state.scopes[0].type === 'py') || indentInfo === 'indent') { indent(stream, state); } - var delimiter_index = '[({'.indexOf(current); + delimiter_index = '[({'.indexOf(current); if (delimiter_index !== -1) { indent(stream, state, '])}'.slice(delimiter_index, delimiter_index+1)); } @@ -294,15 +300,17 @@ CodeMirror.defineMode("python", function(conf, parserConf) { return ERRORCLASS; } } - if (state.dedent > 0 && stream.eol() && state.scopes[0].type == 'py') { - if (state.scopes.length > 1) state.scopes.shift(); + if (state.dedent > 0 && stream.eol() && state.scopes[0].type === 'py') { + if (state.scopes.length > 1) { + state.scopes.shift(); + } state.dedent -= 1; } - + return style; } - var external = { + return { startState: function(basecolumn) { return { tokenize: tokenBase, @@ -312,29 +320,28 @@ CodeMirror.defineMode("python", function(conf, parserConf) { dedent: 0 }; }, - + token: function(stream, state) { var style = tokenLexer(stream, state); - + state.lastToken = style; - + if (stream.eol() && stream.lambda) { state.lambda = false; } - + return style; }, - + indent: function(state) { - if (state.tokenize != tokenBase) { + if (state.tokenize !== tokenBase) { return state.tokenize.isString ? CodeMirror.Pass : 0; } - + return state.scopes[0].offset; } - + }; - return external; }); CodeMirror.defineMIME("text/x-python", "python"); From b3c64bc1e7e3ca0b1e0107305d4d6c923c172393 Mon Sep 17 00:00:00 2001 From: Timothy Farrell Date: Thu, 20 Dec 2012 14:04:53 -0600 Subject: [PATCH 2/2] Start rewriting the Python parser to handle more things. And it's in coffeescript. --- mode/python/index.html | 1108 +++++++++++++++++++++++++++++++++---- mode/python/python.coffee | 401 ++++++++++++++ mode/python/python.js | 347 ------------ 3 files changed, 1407 insertions(+), 449 deletions(-) create mode 100644 mode/python/python.coffee delete mode 100644 mode/python/python.js diff --git a/mode/python/index.html b/mode/python/index.html index 7a26d278db..7525599b29 100644 --- a/mode/python/index.html +++ b/mode/python/index.html @@ -6,112 +6,1016 @@ - + - + +

CodeMirror: Python mode

- -
-

Configuration Options: