Skip to content

Commit 64749a3

Browse files
hdimahdima
committed
Added support for non-ASCII identifiers
1 parent 69dcbc3 commit 64749a3

File tree

4 files changed

+61
-57
lines changed

4 files changed

+61
-57
lines changed

‎CHANGES.txt‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
1-
Revision 3.0.0 (2008-09-?):
1+
Revision 3.0.0 (2008-12-07):
22

3-
- Removed trailing 'L' support for all numbers;
3+
- Added support for non-ASCII identifiers;
4+
- Added support for new text strings and binary data (bytes);
5+
- Updated support for numeric literals;
46
- Updated support for str.format;
57
- Added new builtins introduced in Python 2.6: "ascii", "exec",
68
"memoryview", "print";

‎TODO.txt‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,6 @@
11
Now
22
===
33

4-
- (Python 3.0) non-ASCII identifiers. Also str.format should be updated;
5-
6-
- (Python 3.0) support for b"..." syntax and remove u"..." syntax. Also all
7-
escapes need to be updated;
8-
94
Later
105
=====
116

‎python3.0.vim‎

Lines changed: 49 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@
22
" Language: Python
33
" Maintainer: Dmitry Vasiliev <dima@hlabs.spb.ru>
44
" URL: http://www.hlabs.spb.ru/vim/python.vim
5-
" Last Change: 2008-09-21
5+
" Last Change: 2008-12-07
66
" Filenames: *.py
7-
" Version: 2.6.1
7+
" Version: 3.0.0
88
"
99
" Based on python.vim (from Vim 6.1 distribution)
1010
" by Neil Schemenauer <nas@python.ca>
@@ -103,7 +103,7 @@ syn keyword pythonStatement lambda yield
103103
syn keyword pythonStatement with nonlocal
104104
syn keyword pythonStatement False None True
105105
syn keyword pythonStatement def class nextgroup=pythonFunction skipwhite
106-
syn match pythonFunction "[a-zA-Z_][a-zA-Z0-9_]*" display contained
106+
syn match pythonFunction "\%([^[:cntrl:][:space:][:punct:][:digit:]]\|_\)\%([^[:cntrl:][:punct:][:space:]]\|_\)*" display contained
107107
syn keyword pythonRepeat for while
108108
syn keyword pythonConditional if elif else
109109
syn keyword pythonImport import from
@@ -116,7 +116,7 @@ syn match pythonDecorator "@" display nextgroup=pythonFunction skipwhite
116116
" Comments
117117
syn match pythonComment "#.*$" display contains=pythonTodo,@Spell
118118
syn match pythonRun "\%^#!.*$"
119-
syn match pythonCoding "\%^.*\(\n.*\)\?#.*coding[:=]\s*[0-9A-Za-z-_.]\+.*$"
119+
syn match pythonCoding "\%^.*\%(\n.*\)\?#.*coding[:=]\s*[0-9A-Za-z-_.]\+.*$"
120120
syn keyword pythonTodo TODO FIXME XXX contained
121121

122122
" Errors
@@ -128,7 +128,7 @@ syn match pythonError "[=]\{3,}" display
128128
" TODO: Mixing spaces and tabs also may be used for pretty formatting multiline
129129
" statements. For now I don't know how to work around this.
130130
if exists("python_highlight_indent_errors") && python_highlight_indent_errors != 0
131-
syn match pythonIndentError "^\s*\( \t\|\t \)\s*\S"me=e-1 display
131+
syn match pythonIndentError "^\s*\%( \t\|\t \)\s*\S"me=e-1 display
132132
endif
133133

134134
" Trailing space errors
@@ -142,25 +142,18 @@ syn region pythonString start=+"+ skip=+\\\\\|\\"\|\\$+ excludenl end=+"+ end=+
142142
syn region pythonString start=+"""+ end=+"""+ keepend contains=pythonEscape,pythonEscapeError,pythonDocTest2,pythonSpaceError,@Spell
143143
syn region pythonString start=+'''+ end=+'''+ keepend contains=pythonEscape,pythonEscapeError,pythonDocTest,pythonSpaceError,@Spell
144144

145-
syn match pythonEscape +\\[abfnrtv'"\\]+ display contained
146-
syn match pythonEscape "\\\o\o\=\o\=" display contained
145+
syn match pythonEscape +\\[abfnrtv'"\\]+ display contained
146+
syn match pythonEscape "\\\o\o\=\o\=" display contained
147147
syn match pythonEscapeError "\\\o\{,2}[89]" display contained
148-
syn match pythonEscape "\\x\x\{2}" display contained
148+
syn match pythonEscape "\\x\x\{2}" display contained
149149
syn match pythonEscapeError "\\x\x\=\X" display contained
150-
syn match pythonEscape "\\$"
151-
152-
" Unicode strings
153-
syn region pythonUniString start=+[uU]'+ skip=+\\\\\|\\'\|\\$+ excludenl end=+'+ end=+$+ keepend contains=pythonEscape,pythonUniEscape,pythonEscapeError,pythonUniEscapeError,@Spell
154-
syn region pythonUniString start=+[uU]"+ skip=+\\\\\|\\"\|\\$+ excludenl end=+"+ end=+$+ keepend contains=pythonEscape,pythonUniEscape,pythonEscapeError,pythonUniEscapeError,@Spell
155-
syn region pythonUniString start=+[uU]"""+ end=+"""+ keepend contains=pythonEscape,pythonUniEscape,pythonEscapeError,pythonUniEscapeError,pythonDocTest2,pythonSpaceError,@Spell
156-
syn region pythonUniString start=+[uU]'''+ end=+'''+ keepend contains=pythonEscape,pythonUniEscape,pythonEscapeError,pythonUniEscapeError,pythonDocTest,pythonSpaceError,@Spell
157-
158-
syn match pythonUniEscape "\\u\x\{4}" display contained
159-
syn match pythonUniEscapeError "\\u\x\{,3}\X" display contained
160-
syn match pythonUniEscape "\\U\x\{8}" display contained
161-
syn match pythonUniEscapeError "\\U\x\{,7}\X" display contained
162-
syn match pythonUniEscape "\\N{[A-Z ]\+}" display contained
163-
syn match pythonUniEscapeError "\\N{[^A-Z ]\+}" display contained
150+
syn match pythonEscape "\\$"
151+
syn match pythonEscape "\\u\x\{4}" display contained
152+
syn match pythonEscapeError "\\u\x\{,3}\X" display contained
153+
syn match pythonEscape "\\U\x\{8}" display contained
154+
syn match pythonEscapeError "\\U\x\{,7}\X" display contained
155+
syn match pythonEscape "\\N{[A-Z ]\+}" display contained
156+
syn match pythonEscapeError "\\N{[^A-Z ]\+}" display contained
164157

165158
" Raw strings
166159
syn region pythonRawString start=+[rR]'+ skip=+\\\\\|\\'\|\\$+ excludenl end=+'+ end=+$+ keepend contains=pythonRawEscape,@Spell
@@ -170,32 +163,39 @@ syn region pythonRawString start=+[rR]'''+ end=+'''+ keepend contains=pythonDocT
170163

171164
syn match pythonRawEscape +\\['"]+ display transparent contained
172165

173-
" Unicode raw strings
174-
syn region pythonUniRawString start=+[uU][rR]'+ skip=+\\\\\|\\'\|\\$+ excludenl end=+'+ end=+$+ keepend contains=pythonRawEscape,pythonUniRawEscape,pythonUniRawEscapeError,@Spell
175-
syn region pythonUniRawString start=+[uU][rR]"+ skip=+\\\\\|\\"\|\\$+ excludenl end=+"+ end=+$+ keepend contains=pythonRawEscape,pythonUniRawEscape,pythonUniRawEscapeError,@Spell
176-
syn region pythonUniRawString start=+[uU][rR]"""+ end=+"""+ keepend contains=pythonUniRawEscape,pythonUniRawEscapeError,pythonDocTest2,pythonSpaceError,@Spell
177-
syn region pythonUniRawString start=+[uU][rR]'''+ end=+'''+ keepend contains=pythonUniRawEscape,pythonUniRawEscapeError,pythonDocTest,pythonSpaceError,@Spell
166+
" Bytes
167+
syn region pythonBytes start=+[bB]'+ skip=+\\\\\|\\'\|\\$+ excludenl end=+'+ end=+$+ keepend contains=pythonBytesContent,pythonBytesError,pythonBytesEscape,pythonBytesEscapeError,@Spell
168+
syn region pythonBytes start=+[bB]"+ skip=+\\\\\|\\"\|\\$+ excludenl end=+"+ end=+$+ keepend contains=pythonBytesContent,pythonBytesError,pythonBytesEscape,pythonBytesEscapeError,@Spell
169+
syn region pythonBytes start=+[bB]"""+ end=+"""+ keepend contains=pythonBytesContent,pythonBytesError,pythonBytesEscape,pythonBytesEscapeError,pythonDocTest2,pythonSpaceError,@Spell
170+
syn region pythonBytes start=+[bB]'''+ end=+'''+ keepend contains=pythonBytesContent,pythonBytesError,pythonBytesEscape,pythonBytesEscapeError,pythonDocTest,pythonSpaceError,@Spell
171+
172+
syn match pythonBytesContent "[\u0001-\u007f]\+" display contained
173+
syn match pythonBytesError "[^\u0001-\u007f]\+" display contained
178174

179-
syn match pythonUniRawEscape "\([^\\]\(\\\\\)*\)\@<=\\u\x\{4}" display contained
180-
syn match pythonUniRawEscapeError "\([^\\]\(\\\\\)*\)\@<=\\u\x\{,3}\X" display contained
175+
syn match pythonBytesEscape +\\[abfnrtv'"\\]+ display contained
176+
syn match pythonBytesEscape "\\\o\o\=\o\=" display contained
177+
syn match pythonBytesEscapeError "\\\o\{,2}[89]" display contained
178+
syn match pythonBytesEscape "\\x\x\{2}" display contained
179+
syn match pythonBytesEscapeError "\\x\x\=\X" display contained
180+
syn match pythonBytesEscape "\\$"
181181

182182
if exists("python_highlight_string_formatting") && python_highlight_string_formatting != 0
183183
" String formatting
184-
syn match pythonStrFormatting "%\(([^)]\+)\)\=[-#0 +]*\d*\(\.\d\+\)\=[hlL]\=[diouxXeEfFgGcrs%]" contained containedin=pythonString,pythonUniString,pythonRawString,pythonUniRawString
185-
syn match pythonStrFormatting "%[-#0 +]*\(\*\|\d\+\)\=\(\.\(\*\|\d\+\)\)\=[hlL]\=[diouxXeEfFgGcrs%]" contained containedin=pythonString,pythonUniString,pythonRawString,pythonUniRawString
184+
syn match pythonStrFormatting "%\%(([^)]\+)\)\=[-#0 +]*\d*\%(\.\d\+\)\=[hlL]\=[diouxXeEfFgGcrs%]" contained containedin=pythonString,pythonRawString
185+
syn match pythonStrFormatting "%[-#0 +]*\%(\*\|\d\+\)\=\%(\.\%(\*\|\d\+\)\)\=[hlL]\=[diouxXeEfFgGcrs%]" contained containedin=pythonString,pythonRawString
186186
endif
187187

188188
if exists("python_highlight_string_format") && python_highlight_string_format != 0
189189
" str.format syntax
190-
syn match pythonStrFormat "{{\|}}" contained containedin=pythonString,pythonUniString,pythonRawString,pythonUniRawString
191-
syn match pythonStrFormat "{\([a-zA-Z_][a-zA-Z0-9_]*\|\d\+\)\(\.[a-zA-Z_][a-zA-Z0-9_]*\|\[\(\d\+\|[^!:\}]\+\)\]\)*\(![rsa]\)\=\(:\({\([a-zA-Z_][a-zA-Z0-9_]*\|\d\+\)}\|\([^}]\=[<>=^]\)\=[ +-]\=#\=0\=\d*\(\.\d\+\)\=[bcdeEfFgGnoxX%]\=\)\=\)\=}" contained containedin=pythonString,pythonUniString,pythonRawString,pythonUniRawString
190+
syn match pythonStrFormat "{{\|}}" contained containedin=pythonString,pythonRawString
191+
syn match pythonStrFormat "{\%(\%([^[:cntrl:][:space:][:punct:][:digit:]]\|_\)\%([^[:cntrl:][:punct:][:space:]]\|_\)*\|\d\+\)\%(\.\%([^[:cntrl:][:space:][:punct:][:digit:]]\|_\)\%([^[:cntrl:][:punct:][:space:]]\|_*\)\|\[\%(\d\+\|[^!:\}]\+\)\]\)*\%(![rsa]\)\=\%(:\%({\%(\%([^[:cntrl:][:space:][:punct:][:digit:]]\|_\)\%([^[:cntrl:][:punct:][:space:]]\|_\)*\|\d\+\)}\|\%([^}]\=[<>=^]\)\=[ +-]\=#\=0\=\d*\%(\.\d\+\)\=[bcdeEfFgGnoxX%]\=\)\=\)\=}" contained containedin=pythonString,pythonRawString
192192
endif
193193

194194
if exists("python_highlight_string_templates") && python_highlight_string_templates != 0
195195
" String templates
196-
syn match pythonStrTemplate "\$\$" contained containedin=pythonString,pythonUniString,pythonRawString,pythonUniRawString
197-
syn match pythonStrTemplate "\${[a-zA-Z_][a-zA-Z0-9_]*}" contained containedin=pythonString,pythonUniString,pythonRawString,pythonUniRawString
198-
syn match pythonStrTemplate "\$[a-zA-Z_][a-zA-Z0-9_]*" contained containedin=pythonString,pythonUniString,pythonRawString,pythonUniRawString
196+
syn match pythonStrTemplate "\$\$" contained containedin=pythonString,pythonRawString
197+
syn match pythonStrTemplate "\${[a-zA-Z_][a-zA-Z0-9_]*}" contained containedin=pythonString,pythonRawString
198+
syn match pythonStrTemplate "\$[a-zA-Z_][a-zA-Z0-9_]*" contained containedin=pythonString,pythonRawString
199199
endif
200200

201201
if exists("python_highlight_doctests") && python_highlight_doctests != 0
@@ -211,11 +211,14 @@ syn match pythonHexNumber "\<0[xX]\x\+\>" display
211211
syn match pythonOctNumber "\<0[oO]\o\+\>" display
212212
syn match pythonBinNumber "\<0[bB][01]\+\>" display
213213

214-
syn match pythonNumber "\<\d\+[jJ]\=\>" display
214+
syn match pythonNumber "\<0\>" display
215+
syn match pythonNumber "\<[1-9]\d\+\>" display
216+
syn match pythonNumber "\<\d\+[jJ]\>" display
217+
syn match pythonNumberError "\<0\d\+\>" display
215218

216-
syn match pythonFloat "\.\d\+\([eE][+-]\=\d\+\)\=[jJ]\=\>" display
219+
syn match pythonFloat "\.\d\+\%([eE][+-]\=\d\+\)\=[jJ]\=\>" display
217220
syn match pythonFloat "\<\d\+[eE][+-]\=\d\+[jJ]\=\>" display
218-
syn match pythonFloat "\<\d\+\.\d*\([eE][+-]\=\d\+\)\=[jJ]\=" display
221+
syn match pythonFloat "\<\d\+\.\d*\%([eE][+-]\=\d\+\)\=[jJ]\=" display
219222

220223
syn match pythonOctError "\<0[oO]\=\o*[8-9]\d*\>" display
221224
syn match pythonBinError "\<0[bB][01]*[2-9]\d*\>" display
@@ -303,16 +306,15 @@ if version >= 508 || !exists("did_python_syn_inits")
303306
HiLink pythonSpaceError Error
304307

305308
HiLink pythonString String
306-
HiLink pythonUniString String
307309
HiLink pythonRawString String
308-
HiLink pythonUniRawString String
309-
310310
HiLink pythonEscape Special
311311
HiLink pythonEscapeError Error
312-
HiLink pythonUniEscape Special
313-
HiLink pythonUniEscapeError Error
314-
HiLink pythonUniRawEscape Special
315-
HiLink pythonUniRawEscapeError Error
312+
313+
HiLink pythonBytes String
314+
HiLink pythonBytesContent String
315+
HiLink pythonBytesError Error
316+
HiLink pythonBytesEscape Special
317+
HiLink pythonBytesEscapeError Error
316318

317319
HiLink pythonStrFormatting Special
318320
HiLink pythonStrFormat Special
@@ -326,6 +328,7 @@ if version >= 508 || !exists("did_python_syn_inits")
326328
HiLink pythonOctNumber Number
327329
HiLink pythonBinNumber Number
328330
HiLink pythonFloat Float
331+
HiLink pythonNumberError Error
329332
HiLink pythonOctError Error
330333
HiLink pythonHexError Error
331334
HiLink pythonBinError Error

‎test.py‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@
1111
with break continue del exec return pass print raise global assert lambda yield
1212
for while if elif else import from as try except finally and in is not or
1313
def functionname
14-
class classname
14+
class Classname
15+
def функция
16+
class Класс
1517

1618
# Builtin objects.
1719

@@ -48,11 +50,11 @@ class classname
4850

4951
# Numbers
5052

51-
0 0x1f 077 .3 12.34 100L 0j 0j 34.2E-3 0b10 0o77 0xfffffffL 0L
53+
0 0x1f .3 12.34 0j 0j 34.2E-3 0b10 0o77
5254

5355
# Erroneous numbers
5456

55-
08 0xk 0x 0b102 0o78
57+
077 100L 0xfffffffL 0L 08 0xk 0x 0b102 0o78
5658

5759
# Strings
5860

@@ -67,10 +69,12 @@ class classname
6769
" \a\b\c\"\'\n\r \x34\077 \08 \xag"
6870
r" \" \' "
6971

70-
u"test"
72+
"testтест"
7173

7274
b"test"
7375

76+
b"тестtest"
77+
7478
# Formattings
7579

7680
" %f "

0 commit comments

Comments
 (0)