diff --git a/nml/parser.py b/nml/parser.py index 088b830eb..8ed3a0a84 100644 --- a/nml/parser.py +++ b/nml/parser.py @@ -13,6 +13,7 @@ with NML; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.""" +import codecs import ply.yacc as yacc from nml import expression, generic, nmlop, tokens, unit @@ -114,12 +115,43 @@ def p_main_script(self, t): def p_script(self, t): """script : - | script main_block""" + | script main_block + | script include""" if len(t) == 1: t[0] = [] else: t[0] = t[1] + [t[2]] + def p_include(self, t): + """include : INCLUDE LPAREN STRING_LITERAL RPAREN SEMICOLON + | INCLUDE LPAREN STRING_LITERAL COMMA include_define_list RPAREN SEMICOLON""" + fname = t[3].value + try: + with codecs.open(generic.find_file(fname), "r", "utf-8") as input: + script = input.read() + except UnicodeDecodeError as ex: + raise generic.ScriptError("Input file is not utf-8 encoded: {}".format(ex)) + # Strip a possible BOM + script = script.lstrip(str(codecs.BOM_UTF8, "utf-8")) + self.lexer.push_state(t.lineno(1), t[5] if len(t) == 8 else None) + for i in self.lexer.includes: + if generic.find_file(i.filename) == generic.find_file(fname): + raise generic.ScriptError("Include loop detected: {}".format(fname), t.lineno(1)) + t[0] = self.parse(script, fname) + self.lexer.pop_state() + + def p_include_define_list(self, t): + """include_define_list : include_define + | include_define_list COMMA include_define""" + if len(t) == 2: + t[0] = t[1] + else: + t[0] = t[1] | t[3] + + def p_include_define(self, t): + "include_define : identifier EQ expression" + t[0] = {t[1].value: t[3]} + def p_main_block(self, t): """main_block : switch | random_switch @@ -158,6 +190,14 @@ def p_main_block(self, t): | constant""" t[0] = t[1] + def p_identifier(self, t): + """identifier : ID + | identifier CONCAT identifier""" + if len(t) == 4: + t[0] = expression.Identifier(t[1].value + t[3].value) + else: + t[0] = t[1] + # # Expressions # @@ -166,7 +206,7 @@ def p_expression(self, t): | FLOAT | param | variable - | ID + | identifier | STRING_LITERAL | string""" t[0] = t[1] @@ -252,7 +292,7 @@ def p_variable(self, t): t[0].pos = t.lineno(1) def p_function(self, t): - "expression : ID LPAREN expression_list RPAREN" + "expression : identifier LPAREN expression_list RPAREN" t[0] = expression.FunctionCall(t[1], t[3], t[1].pos) def p_array(self, t): @@ -273,7 +313,7 @@ def p_assignment_list(self, t): t[0] = t[1] + [t[2]] def p_assignment(self, t): - "assignment : ID COLON expression SEMICOLON" + "assignment : identifier COLON expression SEMICOLON" t[0] = assignment.Assignment(t[1], t[3], t[1].pos) def p_param_desc(self, t): @@ -293,7 +333,7 @@ def p_setting_list(self, t): t[0] = t[1] + [t[2]] def p_setting(self, t): - "setting : ID LBRACE setting_value_list RBRACE" + "setting : identifier LBRACE setting_value_list RBRACE" t[0] = grf.ParameterSetting(t[1], t[3]) def p_setting_value_list(self, t): @@ -309,7 +349,7 @@ def p_setting_value(self, t): t[0] = t[1] def p_names_setting_value(self, t): - "setting_value : ID COLON LBRACE name_string_list RBRACE SEMICOLON" + "setting_value : identifier COLON LBRACE name_string_list RBRACE SEMICOLON" t[0] = assignment.Assignment(t[1], t[4], t[1].pos) def p_name_string_list(self, t): @@ -343,8 +383,8 @@ def p_expression_list(self, t): t[0] = [] if len(t) == 1 else t[1] def p_non_empty_id_list(self, t): - """non_empty_id_list : ID - | non_empty_id_list COMMA ID""" + """non_empty_id_list : identifier + | non_empty_id_list COMMA identifier""" if len(t) == 2: t[0] = [t[1]] else: @@ -396,8 +436,8 @@ def p_property_list(self, t): t[0] = t[1] + [t[2]] def p_property_assignment(self, t): - """property_assignment : ID COLON expression SEMICOLON - | ID COLON expression UNIT SEMICOLON + """property_assignment : identifier COLON expression SEMICOLON + | identifier COLON expression UNIT SEMICOLON | NUMBER COLON expression SEMICOLON | NUMBER COLON expression UNIT SEMICOLON""" name = t[1] @@ -525,9 +565,9 @@ def p_produce_cargo_list(self, t): t[0] = t[2] def p_produce(self, t): - """produce : PRODUCE LPAREN ID COMMA expression_list RPAREN SEMICOLON - | PRODUCE LPAREN ID COMMA produce_cargo_list COMMA produce_cargo_list COMMA expression RPAREN - | PRODUCE LPAREN ID COMMA produce_cargo_list COMMA produce_cargo_list RPAREN""" + """produce : PRODUCE LPAREN identifier COMMA expression_list RPAREN SEMICOLON + | PRODUCE LPAREN identifier COMMA produce_cargo_list COMMA produce_cargo_list COMMA expression RPAREN + | PRODUCE LPAREN identifier COMMA produce_cargo_list COMMA produce_cargo_list RPAREN""" if len(t) == 8: t[0] = produce.ProduceOld([t[3]] + t[5], t.lineno(1)) elif len(t) == 11: @@ -540,7 +580,7 @@ def p_produce(self, t): # def p_real_sprite(self, t): """real_sprite : LBRACKET expression_list RBRACKET - | ID COLON LBRACKET expression_list RBRACKET""" + | identifier COLON LBRACKET expression_list RBRACKET""" if len(t) == 4: t[0] = real_sprite.RealSprite(param_list=t[2], poslist=[t.lineno(1)]) else: @@ -565,19 +605,19 @@ def p_recolour_assignment_3(self, t): def p_recolour_sprite(self, t): """real_sprite : RECOLOUR_SPRITE LBRACE recolour_assignment_list RBRACE - | ID COLON RECOLOUR_SPRITE LBRACE recolour_assignment_list RBRACE""" + | identifier COLON RECOLOUR_SPRITE LBRACE recolour_assignment_list RBRACE""" if len(t) == 5: t[0] = real_sprite.RecolourSprite(mapping=t[3], poslist=[t.lineno(1)]) else: t[0] = real_sprite.RecolourSprite(mapping=t[5], label=t[1], poslist=[t.lineno(1)]) def p_template_declaration(self, t): - "template_declaration : TEMPLATE ID LPAREN id_list RPAREN LBRACE spriteset_contents RBRACE" + "template_declaration : TEMPLATE identifier LPAREN id_list RPAREN LBRACE spriteset_contents RBRACE" t[0] = spriteblock.TemplateDeclaration(t[2], t[4], t[7], t.lineno(1)) def p_template_usage(self, t): - """template_usage : ID LPAREN expression_list RPAREN - | ID COLON ID LPAREN expression_list RPAREN""" + """template_usage : identifier LPAREN expression_list RPAREN + | identifier COLON identifier LPAREN expression_list RPAREN""" if len(t) == 5: t[0] = real_sprite.TemplateUsage(t[1], t[3], None, t.lineno(1)) else: @@ -595,7 +635,7 @@ def p_spriteset_contents(self, t): def p_replace(self, t): """replace : REPLACESPRITE LPAREN expression_list RPAREN LBRACE spriteset_contents RBRACE - | REPLACESPRITE ID LPAREN expression_list RPAREN LBRACE spriteset_contents RBRACE""" + | REPLACESPRITE identifier LPAREN expression_list RPAREN LBRACE spriteset_contents RBRACE""" if len(t) == 9: t[0] = replace.ReplaceSprite(t[4], t[7], t[2], t.lineno(1)) else: @@ -603,7 +643,7 @@ def p_replace(self, t): def p_replace_new(self, t): """replace_new : REPLACENEWSPRITE LPAREN expression_list RPAREN LBRACE spriteset_contents RBRACE - | REPLACENEWSPRITE ID LPAREN expression_list RPAREN LBRACE spriteset_contents RBRACE""" + | REPLACENEWSPRITE identifier LPAREN expression_list RPAREN LBRACE spriteset_contents RBRACE""" if len(t) == 9: t[0] = replace.ReplaceNewSprite(t[4], t[7], t[2], t.lineno(1)) else: @@ -611,7 +651,7 @@ def p_replace_new(self, t): def p_base_graphics(self, t): """base_graphics : BASE_GRAPHICS LPAREN expression_list RPAREN LBRACE spriteset_contents RBRACE - | BASE_GRAPHICS ID LPAREN expression_list RPAREN LBRACE spriteset_contents RBRACE""" + | BASE_GRAPHICS identifier LPAREN expression_list RPAREN LBRACE spriteset_contents RBRACE""" if len(t) == 9: t[0] = base_graphics.BaseGraphics(t[4], t[7], t[2], t.lineno(1)) else: @@ -619,7 +659,7 @@ def p_base_graphics(self, t): def p_font_glyph(self, t): """font_glyph : FONTGLYPH LPAREN expression_list RPAREN LBRACE spriteset_contents RBRACE - | FONTGLYPH ID LPAREN expression_list RPAREN LBRACE spriteset_contents RBRACE""" + | FONTGLYPH identifier LPAREN expression_list RPAREN LBRACE spriteset_contents RBRACE""" if len(t) == 9: t[0] = font.FontGlyphBlock(t[4], t[7], t[2], t.lineno(1)) else: @@ -638,12 +678,12 @@ def p_spriteset(self, t): t[0] = spriteblock.SpriteSet(t[3], t[6], t.lineno(1)) def p_spritegroup_normal(self, t): - "spritegroup : SPRITEGROUP ID LBRACE spriteview_list RBRACE" + "spritegroup : SPRITEGROUP identifier LBRACE spriteview_list RBRACE" t[0] = spriteblock.SpriteGroup(t[2], t[4], t.lineno(1)) def p_spritelayout(self, t): - """spritelayout : SPRITELAYOUT ID LBRACE layout_sprite_list RBRACE - | SPRITELAYOUT ID LPAREN id_list RPAREN LBRACE layout_sprite_list RBRACE""" + """spritelayout : SPRITELAYOUT identifier LBRACE layout_sprite_list RBRACE + | SPRITELAYOUT identifier LPAREN id_list RPAREN LBRACE layout_sprite_list RBRACE""" if len(t) == 6: t[0] = spriteblock.SpriteLayout(t[2], [], t[4], t.lineno(1)) else: @@ -658,8 +698,8 @@ def p_spriteview_list(self, t): t[0] = t[1] + [t[2]] def p_spriteview(self, t): - """spriteview : ID COLON LBRACKET expression_list RBRACKET SEMICOLON - | ID COLON expression SEMICOLON""" + """spriteview : identifier COLON LBRACKET expression_list RBRACKET SEMICOLON + | identifier COLON expression SEMICOLON""" if len(t) == 7: t[0] = spriteblock.SpriteView(t[1], t[4], t.lineno(1)) else: @@ -674,7 +714,7 @@ def p_layout_sprite_list(self, t): t[0] = t[1] + [t[2]] def p_layout_sprite(self, t): - "layout_sprite : ID LBRACE layout_param_list RBRACE" + "layout_sprite : identifier LBRACE layout_param_list RBRACE" t[0] = spriteblock.LayoutSprite(t[1], t[3], t.lineno(1)) def p_layout_param_list(self, t): @@ -705,7 +745,7 @@ def p_town_names_param_list(self, t): t[0] = t[1] + [t[2]] def p_town_names_param(self, t): - """town_names_param : ID COLON string SEMICOLON + """town_names_param : identifier COLON string SEMICOLON | LBRACE town_names_part_list RBRACE | LBRACE town_names_part_list COMMA RBRACE""" if t[1] != "{": @@ -723,7 +763,7 @@ def p_town_names_part_list(self, t): def p_town_names_part(self, t): """town_names_part : TOWN_NAMES LPAREN expression COMMA expression RPAREN - | ID LPAREN STRING_LITERAL COMMA expression RPAREN""" + | identifier LPAREN STRING_LITERAL COMMA expression RPAREN""" if t[1] == "town_names": t[0] = townnames.TownNamesEntryDefinition(t[3], t[5], t.lineno(1)) else: @@ -733,7 +773,7 @@ def p_town_names_part(self, t): # Snow line # def p_snowline(self, t): - "snowline : SNOWLINE LPAREN ID RPAREN LBRACE snowline_assignment_list RBRACE" + "snowline : SNOWLINE LPAREN identifier RPAREN LBRACE snowline_assignment_list RBRACE" t[0] = snowline.Snowline(t[3], t[6], t.lineno(1)) # @@ -757,9 +797,9 @@ def p_cargotable(self, t): t[0] = cargotable.CargoTable(t[3], t.lineno(1)) def p_cargotable_list(self, t): - """cargotable_list : ID + """cargotable_list : identifier | STRING_LITERAL - | cargotable_list COMMA ID + | cargotable_list COMMA identifier | cargotable_list COMMA STRING_LITERAL""" if len(t) == 2: t[0] = [t[1]] @@ -790,9 +830,9 @@ def p_tracktypetable_list(self, t): t[0] = t[1] + [t[3]] def p_tracktypetable_item(self, t): - """tracktypetable_item : ID + """tracktypetable_item : identifier | STRING_LITERAL - | ID COLON LBRACKET expression_list RBRACKET""" + | identifier COLON LBRACKET expression_list RBRACKET""" if len(t) == 2: t[0] = t[1] else: @@ -823,7 +863,7 @@ def p_sort_vehicles(self, t): t[0] = sort_vehicles.SortVehicles(t[3], t.lineno(1)) def p_tilelayout(self, t): - "tilelayout : TILELAYOUT ID LBRACE tilelayout_list RBRACE" + "tilelayout : TILELAYOUT identifier LBRACE tilelayout_list RBRACE" t[0] = tilelayout.TileLayout(t[2], t[4], t.lineno(1)) def p_tilelayout_list(self, t): diff --git a/nml/tokens.py b/nml/tokens.py index 821e2dd2d..6fe1a7d39 100644 --- a/nml/tokens.py +++ b/nml/tokens.py @@ -61,7 +61,8 @@ "recolour_sprite": "RECOLOUR_SPRITE", "engine_override": "ENGINE_OVERRIDE", "sort": "SORT_VEHICLES", - "const": "CONST" + "const": "CONST", + "include": "INCLUDE" } # fmt: on @@ -121,6 +122,7 @@ class NMLLexer: "NUMBER", "FLOAT", "UNIT", + "CONCAT", ] t_PLUS = r"\+" @@ -156,6 +158,7 @@ class NMLLexer: t_TERNARY_OPEN = r"\?" t_COLON = r":" t_SEMICOLON = r";" + t_CONCAT = r"\#\#" def t_FLOAT(self, t): r"\d+\.\d+" @@ -179,6 +182,9 @@ def t_ID(self, t): r"[a-zA-Z_][a-zA-Z0-9_]*" if t.value in reserved: # Check for reserved words t.type = reserved[t.value] + elif t.value in self.defines: + t.value = self.defines[t.value] + t.value.pos = t.lineno else: t.type = "ID" t.value = expression.Identifier(t.value, t.lineno) @@ -257,6 +263,11 @@ def t_error(self, t): ) sys.exit(1) + def t_eof(self, t): + if self.lexer.lexpos != self.lexer.lexlen: + return self.lexer.token() + return None + def build(self, rebuild=False): """ Initial construction of the scanner. @@ -270,6 +281,9 @@ def build(self, rebuild=False): # Tried to remove a non existing file pass self.lexer = lex.lex(module=self, optimize=1, lextab="nml.generated.lextab") + self.includes = [] + self.states = [] + self.defines = {} def setup(self, text, fname): """ @@ -281,11 +295,20 @@ def setup(self, text, fname): @param fname: Filename associated with the input text (main input file). @type fname: C{str} """ - self.includes = [] self.text = text self.set_position(fname, 1) self.lexer.input(text) + def push_state(self, pos, defines): + self.states.append((self.text, self.lexer.clone(), self.defines.copy())) + self.includes.append(pos) + if defines: + self.defines |= defines + + def pop_state(self): + self.text, self.lexer, self.defines = self.states.pop() + self.includes.pop() + def set_position(self, fname, line): """ @note: The lexer.lineno contains a Position object. diff --git a/regression/042_include.inc b/regression/042_include.inc new file mode 100644 index 000000000..e63825b41 --- /dev/null +++ b/regression/042_include.inc @@ -0,0 +1,3 @@ +param[0] = param[0] + VALUE; +const _ ## NAME ## x ## y = VALUE; +param[0] = param[0] + _ ## NAME ## x ## y; diff --git a/regression/042_include.nml b/regression/042_include.nml new file mode 100644 index 000000000..e4f0e2c7a --- /dev/null +++ b/regression/042_include.nml @@ -0,0 +1,4 @@ +param[0] = 0; +include("042_include.inc", VALUE = 1); +include("042_include.inc", VALUE = 2, NAME = test); +include("042_include.inc", VALUE = 3, NAME = test2); diff --git a/regression/expected/042_include.grf b/regression/expected/042_include.grf new file mode 100644 index 000000000..3d2ff2d49 Binary files /dev/null and b/regression/expected/042_include.grf differ diff --git a/regression/expected/042_include.nfo b/regression/expected/042_include.nfo new file mode 100644 index 000000000..04ca11bcf --- /dev/null +++ b/regression/expected/042_include.nfo @@ -0,0 +1,28 @@ +// Automatically generated by GRFCODEC. Do not modify! +// (Info version 32) +// Escapes: 2+ 2- 2< 2> 2u< 2u> 2/ 2% 2u/ 2u% 2* 2& 2| 2^ 2sto = 2s 2rst = 2r 2psto 2ror = 2rot 2cmp 2ucmp 2<< 2u>> 2>> +// Escapes: 71 70 7= 7! 7< 7> 7G 7g 7gG 7GG 7gg 7c 7C +// Escapes: D= = DR D+ = DF D- = DC Du* = DM D* = DnF Du<< = DnC D<< = DO D& D| Du/ D/ Du% D% +// Format: spritenum imagefile depth xpos ypos xsize ysize xrel yrel zoom flags + +// param[0] = 0 +0 * 9 0D 00 \D= FF 00 \dx00000000 + +// param[0] = (param[0] + 1) +1 * 9 0D 00 \D+ 00 FF \dx00000001 + +// param[0] = (param[0] + 1) +2 * 9 0D 00 \D+ 00 FF \dx00000001 + +// param[0] = (param[0] + 2) +3 * 9 0D 00 \D+ 00 FF \dx00000002 + +// param[0] = (param[0] + 2) +4 * 9 0D 00 \D+ 00 FF \dx00000002 + +// param[0] = (param[0] + 3) +5 * 9 0D 00 \D+ 00 FF \dx00000003 + +// param[0] = (param[0] + 3) +6 * 9 0D 00 \D+ 00 FF \dx00000003 +