From 911b863d03d5a12a704c3f5f45116cd39684fe94 Mon Sep 17 00:00:00 2001 From: Hayk Martiros Date: Thu, 2 Jun 2022 12:12:43 +0200 Subject: [PATCH 01/18] Generalize matrix symbol formatting by codegen config --- symforce/codegen/codegen_config.py | 6 ++++++ symforce/codegen/codegen_util.py | 11 +++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/symforce/codegen/codegen_config.py b/symforce/codegen/codegen_config.py index 9e03b3218..8a70df949 100644 --- a/symforce/codegen/codegen_config.py +++ b/symforce/codegen/codegen_config.py @@ -20,6 +20,7 @@ class CodegenConfig: use_eigen_types: Use eigen_lcm types for vectors instead of lists autoformat: Run a code formatter on the generated code cse_optimizations: Optimizations argument to pass to sm.cse + matrix_is_1d: Whether geo.Matrix symbols get formatted as 1D """ doc_comment_line_prefix: str @@ -29,6 +30,8 @@ class CodegenConfig: cse_optimizations: T.Optional[ T.Union[T.Literal["basic"], T.Sequence[T.Tuple[T.Callable, T.Callable]]] ] = None + # TODO(hayk): Remove this parameter (by making everything 2D?) + matrix_is_1d: bool = False @dataclass @@ -78,9 +81,12 @@ class PythonConfig(CodegenConfig): on the first call and some overhead on subsequent calls, so it should not be used for small functions or functions that are only called a handfull of times. + matrix_is_1D: geo.Matrix symbols get formatted as a 1D array """ doc_comment_line_prefix: str = "" line_length: int = 100 use_eigen_types: bool = True use_numba: bool = False + matrix_is_1d: bool = True + diff --git a/symforce/codegen/codegen_util.py b/symforce/codegen/codegen_util.py index db3410fce..8ba1726bf 100644 --- a/symforce/codegen/codegen_util.py +++ b/symforce/codegen/codegen_util.py @@ -342,20 +342,19 @@ def get_formatted_list( formatted_symbols = [sm.Symbol(key)] flattened_value = [value] elif issubclass(arg_cls, geo.Matrix): - if isinstance(config, codegen_config.PythonConfig): - # TODO(nathan): Not sure this works for 2D matrices + if config.matrix_is_1d: + # TODO(nathan): Not sure this works for 2D matrices. Get rid of this. formatted_symbols = [sm.Symbol(f"{key}[{j}]") for j in range(storage_dim)] - elif isinstance(config, codegen_config.CppConfig): - formatted_symbols = [] + else: # NOTE(brad): The order of the symbols must match the storage order of geo.Matrix # (as returned by geo.Matrix.to_storage). Hence, if there storage order were # changed to, say, row major, the below for loops would have to be swapped to # reflect that. + formatted_symbols = [] for j in range(value.shape[1]): for i in range(value.shape[0]): formatted_symbols.append(sm.Symbol(f"{key}({i}, {j})")) - else: - raise NotImplementedError() + flattened_value = ops.StorageOps.to_storage(value) elif issubclass(arg_cls, Values): From bd1ca81d3e4279feca1ab01a739178ffedae2525 Mon Sep 17 00:00:00 2001 From: Hayk Martiros Date: Thu, 2 Jun 2022 12:35:13 +0200 Subject: [PATCH 02/18] Pull out printer to config --- symforce/codegen/codegen_config.py | 17 +++++++++++++- symforce/codegen/codegen_util.py | 22 +------------------ .../symforce_cpp_code_printer_codegen_test.py | 2 +- 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/symforce/codegen/codegen_config.py b/symforce/codegen/codegen_config.py index 8a70df949..c20807bed 100644 --- a/symforce/codegen/codegen_config.py +++ b/symforce/codegen/codegen_config.py @@ -2,7 +2,7 @@ # SymForce - Copyright 2022, Skydio, Inc. # This source code is under the Apache 2.0 license found in the LICENSE file. # ---------------------------------------------------------------------------- - +from __future__ import annotations from dataclasses import dataclass from symforce import typing as T @@ -33,6 +33,11 @@ class CodegenConfig: # TODO(hayk): Remove this parameter (by making everything 2D?) matrix_is_1d: bool = False + def printer(self) -> "sm.CodePrinter": + """ + Return the code printer to use for this language. + """ + raise NotImplementedError() @dataclass class CppConfig(CodegenConfig): @@ -64,6 +69,12 @@ class CppConfig(CodegenConfig): zero_initialization_sparsity_threshold: float = 0.5 explicit_template_instantiation_types: T.Optional[T.Sequence[str]] = None + def printer(self) -> "sm.CodePrinter": + from symforce.codegen.printers import cpp_code_printer + if self.support_complex: + return cpp_code_printer.ComplexCppCodePrinter() + else: + return cpp_code_printer.CppCodePrinter() @dataclass class PythonConfig(CodegenConfig): @@ -90,3 +101,7 @@ class PythonConfig(CodegenConfig): use_numba: bool = False matrix_is_1d: bool = True + def printer(self) -> "sm.CodePrinter": + from symforce.codegen.printers import python_code_printer + return python_code_printer.PythonCodePrinter() + diff --git a/symforce/codegen/codegen_util.py b/symforce/codegen/codegen_util.py index 8ba1726bf..a926007bd 100644 --- a/symforce/codegen/codegen_util.py +++ b/symforce/codegen/codegen_util.py @@ -179,7 +179,7 @@ def count_ops(expr: sm.Expr) -> int: ) # Get printer - printer = get_code_printer(config) + printer = config.printer() # Print code intermediate_terms = [(str(var), printer.doprint(t)) for var, t in temps_formatted] @@ -494,26 +494,6 @@ def get_formatted_sparse_list(sparse_outputs: Values) -> T.List[T.List[T.Scalar] return symbolic_args -def get_code_printer(config: codegen_config.CodegenConfig) -> "sm.CodePrinter": - """ - Pick a code printer for the given mode. - """ - # TODO(hayk): Consider symengine printer if this becomes slow. - - if isinstance(config, codegen_config.PythonConfig): - printer: sm.printing.codeprinter.CodePrinter = printers.PythonCodePrinter() - - elif isinstance(config, codegen_config.CppConfig): - if config.support_complex: - printer = printers.ComplexCppCodePrinter() - else: - printer = printers.CppCodePrinter() - else: - raise NotImplementedError(f"Unknown config type: {config}") - - return printer - - def _load_generated_package_internal(name: str, path: Path) -> T.Tuple[T.Any, T.List[str]]: """ Dynamically load generated package (or module). diff --git a/test/symforce_cpp_code_printer_codegen_test.py b/test/symforce_cpp_code_printer_codegen_test.py index cb7266626..e95302781 100644 --- a/test/symforce_cpp_code_printer_codegen_test.py +++ b/test/symforce_cpp_code_printer_codegen_test.py @@ -27,7 +27,7 @@ class SymforceCppCodePrinterTest(TestCase): """ def test_max_min(self) -> None: - printer = codegen_util.get_code_printer(codegen.CppConfig()) + printer = codegen.CppConfig().printer() a = sm.Symbol("a") b = sm.Symbol("b") From f25a5cbe34ec50fa36ac668353ca6960d6500eaf Mon Sep 17 00:00:00 2001 From: Hayk Martiros Date: Thu, 2 Jun 2022 12:36:13 +0200 Subject: [PATCH 03/18] rename to out_function_dir --- symforce/codegen/codegen.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/symforce/codegen/codegen.py b/symforce/codegen/codegen.py index ffdfbdf64..0c8a69110 100644 --- a/symforce/codegen/codegen.py +++ b/symforce/codegen/codegen.py @@ -397,48 +397,46 @@ def generate_function( # Generate the function if isinstance(self.config, codegen_config.PythonConfig): if skip_directory_nesting: - python_function_dir = output_dir + out_function_dir = output_dir else: - python_function_dir = output_dir / "python" / "symforce" / namespace + out_function_dir = output_dir / "python" / "symforce" / namespace - logger.info(f'Creating python function "{self.name}" at "{python_function_dir}"') + logger.info(f'Creating python function "{self.name}" at "{out_function_dir}"') templates.add( Path(template_util.PYTHON_TEMPLATE_DIR) / "function" / "FUNCTION.py.jinja", - python_function_dir / f"{generated_file_name}.py", + out_function_dir / f"{generated_file_name}.py", template_data, ) templates.add( Path(template_util.PYTHON_TEMPLATE_DIR) / "function" / "__init__.py.jinja", - python_function_dir / "__init__.py", + out_function_dir / "__init__.py", template_data, ) - - out_function_dir = python_function_dir elif isinstance(self.config, codegen_config.CppConfig): if skip_directory_nesting: - cpp_function_dir = output_dir + out_function_dir = output_dir else: - cpp_function_dir = output_dir / "cpp" / "symforce" / namespace + out_function_dir = output_dir / "cpp" / "symforce" / namespace logger.info( - f'Creating C++ function "{python_util.snakecase_to_camelcase(self.name)}" at "{cpp_function_dir}"' + f'Creating C++ function "{python_util.snakecase_to_camelcase(self.name)}" at "{out_function_dir}"' ) templates.add( Path(template_util.CPP_TEMPLATE_DIR) / "function" / "FUNCTION.h.jinja", - cpp_function_dir / f"{generated_file_name}.h", + out_function_dir / f"{generated_file_name}.h", template_data, ) if self.config.explicit_template_instantiation_types is not None: templates.add( Path(template_util.CPP_TEMPLATE_DIR) / "function" / "FUNCTION.cc.jinja", - cpp_function_dir / f"{generated_file_name}.cc", + out_function_dir / f"{generated_file_name}.cc", template_data, ) - out_function_dir = cpp_function_dir + else: raise NotImplementedError(f'Unknown config type: "{self.config}"') From 5aadb99c371016879a06f0b30e6c060e7666fd8b Mon Sep 17 00:00:00 2001 From: Hayk Martiros Date: Thu, 2 Jun 2022 15:09:36 +0200 Subject: [PATCH 04/18] format data accessor pulled out --- symforce/codegen/codegen_config.py | 13 +++++++++++++ symforce/codegen/codegen_util.py | 10 ++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/symforce/codegen/codegen_config.py b/symforce/codegen/codegen_config.py index c20807bed..f7bb152d4 100644 --- a/symforce/codegen/codegen_config.py +++ b/symforce/codegen/codegen_config.py @@ -39,6 +39,15 @@ def printer(self) -> "sm.CodePrinter": """ raise NotImplementedError() + # TODO(hayk): Move this into code printer. + @staticmethod + def format_data_accessor(prefix: str, index: int) -> str: + """ + Format data for accessing a data array in code. + """ + return f"{prefix}.data[{index}]" + + @dataclass class CppConfig(CodegenConfig): """ @@ -76,6 +85,10 @@ def printer(self) -> "sm.CodePrinter": else: return cpp_code_printer.CppCodePrinter() + @staticmethod + def format_data_accessor(prefix: str, index: int) -> str: + return f"{prefix}.Data()[{index}]" + @dataclass class PythonConfig(CodegenConfig): """ diff --git a/symforce/codegen/codegen_util.py b/symforce/codegen/codegen_util.py index a926007bd..0302106a1 100644 --- a/symforce/codegen/codegen_util.py +++ b/symforce/codegen/codegen_util.py @@ -466,12 +466,10 @@ def _get_scalar_keys_recursive( vec.extend(sm.Symbol(f"{prefix}[{i}]") for i in range(index_value.storage_dim)) else: # We have a geo/cam or other object that uses "data" to store a flat vector of scalars. - if isinstance(config, codegen_config.PythonConfig): - vec.extend(sm.Symbol(f"{prefix}.data[{i}]") for i in range(index_value.storage_dim)) - elif isinstance(config, codegen_config.CppConfig): - vec.extend(sm.Symbol(f"{prefix}.Data()[{i}]") for i in range(index_value.storage_dim)) - else: - raise NotImplementedError() + vec.extend( + sm.Symbol(config.format_data_accessor(prefix=prefix, index=i)) + for i in range(index_value.storage_dim) + ) assert len(vec) == len(set(vec)), "Non-unique keys:\n{}".format( [symbol for symbol in vec if vec.count(symbol) > 1] From db6a005b1491a3fa226a225f9110d0dca183165a Mon Sep 17 00:00:00 2001 From: Hayk Martiros Date: Thu, 2 Jun 2022 15:25:18 +0200 Subject: [PATCH 05/18] move prefix out of preamble --- symforce/codegen/template_util.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/symforce/codegen/template_util.py b/symforce/codegen/template_util.py index 3e947d322..f487282bf 100644 --- a/symforce/codegen/template_util.py +++ b/symforce/codegen/template_util.py @@ -19,6 +19,8 @@ from symforce.codegen import format_util CURRENT_DIR = os.path.dirname(__file__) + +# TODO(hayk): Move up to language-specific config or printer. (tag=centralize-language-diffs) CPP_TEMPLATE_DIR = os.path.join(CURRENT_DIR, "cpp_templates") PYTHON_TEMPLATE_DIR = os.path.join(CURRENT_DIR, "python_templates") LCM_TEMPLATE_DIR = os.path.join(CURRENT_DIR, "lcm_templates") @@ -73,12 +75,7 @@ def join_path(self, template: T.Union[jinja2.Template, str], parent: str) -> str return os.path.normpath(os.path.join(os.path.dirname(parent), str(template))) -def add_preamble(source: str, name: Path, filetype: FileType) -> str: - prefix = ( - "//" - if filetype in (FileType.CPP, FileType.CUDA, FileType.LCM, FileType.TYPESCRIPT) - else "#" - ) +def add_preamble(source: str, name: Path, prefix: str) -> str: dashes = "-" * 77 return ( textwrap.dedent( @@ -142,10 +139,19 @@ def render_template( filetype = FileType.from_template_path(Path(template_name)) + # TODO(hayk): Move up to language-specific config or printer. (tag=centralize-language-diffs) + prefix = ( + "//" + if filetype + in (FileType.CPP, FileType.CUDA, FileType.LCM, FileType.JAVASCRIPT, FileType.TYPESCRIPT) + else "#" + ) + template = jinja_env(template_dir).get_template(os.fspath(template_name)) - rendered_str = add_preamble(str(template.render(**data)), template_name, filetype) + rendered_str = add_preamble(str(template.render(**data)), template_name, prefix=prefix) if autoformat: + # TODO(hayk): Move up to language-specific config or printer. (tag=centralize-language-diffs) if filetype in (FileType.CPP, FileType.CUDA): # Come up with a fake filename to give to the formatter just for formatting purposes, even # if this isn't being written to disk From 99601ebe7a03e920df5e147c4b6de7c0bc7161bd Mon Sep 17 00:00:00 2001 From: Hayk Martiros Date: Thu, 2 Jun 2022 16:22:14 +0200 Subject: [PATCH 06/18] Refactor template directories into codegen config --- symforce/codegen/cam_package_codegen.py | 9 ++++---- symforce/codegen/codegen.py | 13 ++++++------ symforce/codegen/codegen_config.py | 21 +++++++++++++++++++ symforce/codegen/geo_package_codegen.py | 20 +++++++++--------- symforce/codegen/sym_util_package_codegen.py | 22 ++++++++------------ symforce/codegen/template_util.py | 13 ++++-------- symforce/codegen/types_package_codegen.py | 2 +- symforce/codegen/values_codegen.py | 5 +++-- test/symforce_gen_codegen_test.py | 11 +++++----- test/symforce_lcm_codegen_test.py | 2 +- 10 files changed, 66 insertions(+), 52 deletions(-) diff --git a/symforce/codegen/cam_package_codegen.py b/symforce/codegen/cam_package_codegen.py index b2c94cda3..2c7c24b1c 100644 --- a/symforce/codegen/cam_package_codegen.py +++ b/symforce/codegen/cam_package_codegen.py @@ -257,7 +257,7 @@ def generate(config: CodegenConfig, output_dir: str = None) -> str: if isinstance(config, PythonConfig): logger.info(f'Creating Python package at: "{cam_package_dir}"') - template_dir = pathlib.Path(template_util.PYTHON_TEMPLATE_DIR) + template_dir = config.template_dir() # First generate the geo package as it's a dependency of the cam package from symforce.codegen import geo_package_codegen @@ -310,7 +310,7 @@ def generate(config: CodegenConfig, output_dir: str = None) -> str: elif isinstance(config, CppConfig): logger.info(f'Creating C++ cam package at: "{cam_package_dir}"') - template_dir = pathlib.Path(template_util.CPP_TEMPLATE_DIR, "cam_package") + template_dir = config.template_dir() / "cam_package" # First generate the geo package as it's a dependency of the cam package from symforce.codegen import geo_package_codegen @@ -331,9 +331,8 @@ def generate(config: CodegenConfig, output_dir: str = None) -> str: (".", "ops/CLASS/lie_group_ops.h"), (".", "ops/CLASS/lie_group_ops.cc"), ): - template_path = pathlib.Path( - template_util.CPP_TEMPLATE_DIR, base_dir, relative_path + ".jinja" - ) + template_path = config.template_dir() / base_dir / relative_path + ".jinja" + output_path = cam_package_dir / relative_path.replace( "CLASS", python_util.camelcase_to_snakecase(cls.__name__) ) diff --git a/symforce/codegen/codegen.py b/symforce/codegen/codegen.py index 0c8a69110..2189cc01d 100644 --- a/symforce/codegen/codegen.py +++ b/symforce/codegen/codegen.py @@ -29,7 +29,7 @@ from symforce.codegen import types_package_codegen from symforce.type_helpers import symbolic_inputs -CURRENT_DIR = os.path.dirname(__file__) +CURRENT_DIR = Path(__file__).parent class LinearizationMode(enum.Enum): @@ -256,7 +256,7 @@ def common_data() -> T.Dict[str, T.Any]: data["DataBuffer"] = sm.DataBuffer data["Values"] = Values data["pathlib"] = pathlib - data["path_to_codegen"] = CURRENT_DIR + data["path_to_codegen"] = str(CURRENT_DIR) data["scalar_types"] = ("double", "float") data["camelcase_to_snakecase"] = python_util.camelcase_to_snakecase data["python_util"] = python_util @@ -393,6 +393,7 @@ def generate_function( self.namespace = namespace template_data = dict(self.common_data(), spec=self) + template_dir = self.config.template_dir() # Generate the function if isinstance(self.config, codegen_config.PythonConfig): @@ -404,12 +405,12 @@ def generate_function( logger.info(f'Creating python function "{self.name}" at "{out_function_dir}"') templates.add( - Path(template_util.PYTHON_TEMPLATE_DIR) / "function" / "FUNCTION.py.jinja", + template_dir / "function" / "FUNCTION.py.jinja", out_function_dir / f"{generated_file_name}.py", template_data, ) templates.add( - Path(template_util.PYTHON_TEMPLATE_DIR) / "function" / "__init__.py.jinja", + template_dir / "function" / "__init__.py.jinja", out_function_dir / "__init__.py", template_data, ) @@ -424,14 +425,14 @@ def generate_function( ) templates.add( - Path(template_util.CPP_TEMPLATE_DIR) / "function" / "FUNCTION.h.jinja", + template_dir / "function" / "FUNCTION.h.jinja", out_function_dir / f"{generated_file_name}.h", template_data, ) if self.config.explicit_template_instantiation_types is not None: templates.add( - Path(template_util.CPP_TEMPLATE_DIR) / "function" / "FUNCTION.cc.jinja", + template_dir / "function" / "FUNCTION.cc.jinja", out_function_dir / f"{generated_file_name}.cc", template_data, ) diff --git a/symforce/codegen/codegen_config.py b/symforce/codegen/codegen_config.py index f7bb152d4..2d755a9de 100644 --- a/symforce/codegen/codegen_config.py +++ b/symforce/codegen/codegen_config.py @@ -4,9 +4,12 @@ # ---------------------------------------------------------------------------- from __future__ import annotations from dataclasses import dataclass +from pathlib import Path from symforce import typing as T +CURRENT_DIR = Path(__file__).parent + @dataclass class CodegenConfig: @@ -39,6 +42,13 @@ def printer(self) -> "sm.CodePrinter": """ raise NotImplementedError() + @classmethod + def template_dir(cls) -> Path: + """ + Directory for jinja templates. + """ + raise NotImplementedError() + # TODO(hayk): Move this into code printer. @staticmethod def format_data_accessor(prefix: str, index: int) -> str: @@ -80,15 +90,21 @@ class CppConfig(CodegenConfig): def printer(self) -> "sm.CodePrinter": from symforce.codegen.printers import cpp_code_printer + if self.support_complex: return cpp_code_printer.ComplexCppCodePrinter() else: return cpp_code_printer.CppCodePrinter() + @classmethod + def template_dir(cls) -> Path: + return CURRENT_DIR / "cpp_templates" + @staticmethod def format_data_accessor(prefix: str, index: int) -> str: return f"{prefix}.Data()[{index}]" + @dataclass class PythonConfig(CodegenConfig): """ @@ -116,5 +132,10 @@ class PythonConfig(CodegenConfig): def printer(self) -> "sm.CodePrinter": from symforce.codegen.printers import python_code_printer + return python_code_printer.PythonCodePrinter() + @classmethod + def template_dir(cls) -> Path: + return CURRENT_DIR / "python_templates" + diff --git a/symforce/codegen/geo_package_codegen.py b/symforce/codegen/geo_package_codegen.py index 65abd8f5e..b3ba41ea6 100644 --- a/symforce/codegen/geo_package_codegen.py +++ b/symforce/codegen/geo_package_codegen.py @@ -145,7 +145,7 @@ def generate(config: CodegenConfig, output_dir: str = None) -> str: if isinstance(config, PythonConfig): logger.info(f'Creating Python package at: "{package_dir}"') - template_dir = Path(template_util.PYTHON_TEMPLATE_DIR) + template_dir = config.template_dir() # Build up templates for each type @@ -192,7 +192,7 @@ def generate(config: CodegenConfig, output_dir: str = None) -> str: sym_util_package_codegen.generate(config, output_dir=output_dir) logger.info(f'Creating C++ package at: "{package_dir}"') - template_dir = Path(template_util.CPP_TEMPLATE_DIR, "geo_package") + template_dir = config.template_dir() # Build up templates for each type for cls in DEFAULT_GEO_TYPES: @@ -210,14 +210,14 @@ def generate(config: CodegenConfig, output_dir: str = None) -> str: (".", "ops/CLASS/lie_group_ops.h"), (".", "ops/CLASS/lie_group_ops.cc"), ): - template_path = Path( - template_util.CPP_TEMPLATE_DIR, base_dir, relative_path + ".jinja" - ) + template_path = template_dir / base_dir / f"{relative_path}.jinja" output_path = package_dir / relative_path.replace("CLASS", cls.__name__.lower()) templates.add(template_path, output_path, data) # Render non geo type specific templates - for template_name in python_util.files_in_dir(str(template_dir / "ops"), relative=True): + for template_name in python_util.files_in_dir( + template_dir / "geo_package" / "ops", relative=True + ): if "CLASS" in template_name: continue @@ -225,15 +225,15 @@ def generate(config: CodegenConfig, output_dir: str = None) -> str: continue templates.add( - template_dir / "ops" / template_name, - package_dir / "ops" / template_name[: -len(".jinja")], + template_dir / "geo_package" / "ops" / template_name, + package_dir / "geo_package" / "ops" / template_name[: -len(".jinja")], dict(Codegen.common_data()), ) # Test example for name in ("geo_package_cpp_test.cc",): templates.add( - template_dir / ".." / "tests" / (name + ".jinja"), + template_dir / "tests" / (name + ".jinja"), Path(output_dir, "tests", name), dict( Codegen.common_data(), @@ -256,7 +256,7 @@ def generate(config: CodegenConfig, output_dir: str = None) -> str: # LCM type_t templates.add( - Path(template_util.LCM_TEMPLATE_DIR, "symforce_types.lcm.jinja"), + template_util.LCM_TEMPLATE_DIR / "symforce_types.lcm.jinja", package_dir / ".." / "lcmtypes" / "lcmtypes" / "symforce_types.lcm", lcm_types_codegen.lcm_symforce_types_data(), ) diff --git a/symforce/codegen/sym_util_package_codegen.py b/symforce/codegen/sym_util_package_codegen.py index 185f36282..552ab52ae 100644 --- a/symforce/codegen/sym_util_package_codegen.py +++ b/symforce/codegen/sym_util_package_codegen.py @@ -29,14 +29,15 @@ def generate(config: codegen.CodegenConfig, output_dir: str = None) -> str: templates = template_util.TemplateList() if isinstance(config, codegen.CppConfig): + template_dir = config.template_dir() templates.add( - template_path=os.path.join(template_util.CPP_TEMPLATE_DIR, "typedefs.h.jinja"), + template_path=template_dir / "typedefs.h.jinja", output_path=os.path.join(package_dir, "typedefs.h"), data={}, ) templates.add( - template_path=os.path.join(template_util.CPP_TEMPLATE_DIR, "type_ops.h.jinja"), + template_path=template_dir / "type_ops.h.jinja", output_path=os.path.join(package_dir, "type_ops.h"), data=dict( python_util=python_util, @@ -44,17 +45,12 @@ def generate(config: codegen.CodegenConfig, output_dir: str = None) -> str: ), ) - templates.add( - template_path=os.path.join(template_util.CPP_TEMPLATE_DIR, "epsilon.h.jinja"), - output_path=os.path.join(package_dir, "epsilon.h"), - data={}, - ) - - templates.add( - template_path=os.path.join(template_util.CPP_TEMPLATE_DIR, "epsilon.cc.jinja"), - output_path=os.path.join(package_dir, "epsilon.cc"), - data={}, - ) + for filename in ("epsilon.h", "epsilon.cc"): + templates.add( + template_path=template_dir / f"{filename}.jinja", + output_path=os.path.join(package_dir, filename), + data={}, + ) else: # sym/util is currently C++ only pass diff --git a/symforce/codegen/template_util.py b/symforce/codegen/template_util.py index f487282bf..3b1af6f2d 100644 --- a/symforce/codegen/template_util.py +++ b/symforce/codegen/template_util.py @@ -18,13 +18,8 @@ from symforce import typing as T from symforce.codegen import format_util -CURRENT_DIR = os.path.dirname(__file__) - -# TODO(hayk): Move up to language-specific config or printer. (tag=centralize-language-diffs) -CPP_TEMPLATE_DIR = os.path.join(CURRENT_DIR, "cpp_templates") -PYTHON_TEMPLATE_DIR = os.path.join(CURRENT_DIR, "python_templates") -LCM_TEMPLATE_DIR = os.path.join(CURRENT_DIR, "lcm_templates") - +CURRENT_DIR = Path(__file__).parent +LCM_TEMPLATE_DIR = CURRENT_DIR / "lcm_templates" class FileType(enum.Enum): CPP = enum.auto() @@ -143,7 +138,7 @@ def render_template( prefix = ( "//" if filetype - in (FileType.CPP, FileType.CUDA, FileType.LCM, FileType.JAVASCRIPT, FileType.TYPESCRIPT) + in (FileType.CPP, FileType.CUDA, FileType.LCM, FileType.TYPESCRIPT) else "#" ) @@ -161,7 +156,7 @@ def render_template( format_cpp_filename = os.fspath(template_name).replace(".jinja", "") rendered_str = format_util.format_cpp( - rendered_str, filename=os.path.join(CURRENT_DIR, format_cpp_filename) + rendered_str, filename=str(CURRENT_DIR / format_cpp_filename) ) elif filetype == FileType.PYTHON: rendered_str = format_util.format_py(rendered_str) diff --git a/symforce/codegen/types_package_codegen.py b/symforce/codegen/types_package_codegen.py index 7f723a93a..e9a12c860 100644 --- a/symforce/codegen/types_package_codegen.py +++ b/symforce/codegen/types_package_codegen.py @@ -94,7 +94,7 @@ def generate_types( lcm_files = [] if len(types_to_generate) > 0: logger.info(f'Creating LCM type at: "{lcm_type_dir}"') - lcm_template = os.path.join(template_util.LCM_TEMPLATE_DIR, "types.lcm.jinja") + lcm_template = template_util.LCM_TEMPLATE_DIR / "types.lcm.jinja" # Type definition lcm_file_name = f"{file_name}.lcm" diff --git a/symforce/codegen/values_codegen.py b/symforce/codegen/values_codegen.py index 8fecf7c05..caa94b0a9 100644 --- a/symforce/codegen/values_codegen.py +++ b/symforce/codegen/values_codegen.py @@ -7,7 +7,7 @@ import re from symforce import typing as T -from symforce.codegen import Codegen, template_util +from symforce.codegen import Codegen, CppConfig, template_util from symforce.values import generated_key_selection from symforce.values.values import Values @@ -33,6 +33,7 @@ def generate_values_keys( skip_directory_nesting: Generate the output file directly into output_dir instead of adding the usual directory structure inside output_dir """ + config = CppConfig() if not isinstance(output_dir, Path): output_dir = Path(output_dir) @@ -50,7 +51,7 @@ def generate_values_keys( cpp_function_dir = output_dir / "cpp" / "symforce" / namespace template_util.render_template( - template_path=Path(template_util.CPP_TEMPLATE_DIR) / "keys.h.jinja", + template_path=config.template_dir() / "keys.h.jinja", data=dict(Codegen.common_data(), namespace=namespace, vars=vars_to_generate), output_path=cpp_function_dir / generated_file_name, ) diff --git a/test/symforce_gen_codegen_test.py b/test/symforce_gen_codegen_test.py index 06f21822b..6aaa586d9 100644 --- a/test/symforce_gen_codegen_test.py +++ b/test/symforce_gen_codegen_test.py @@ -23,7 +23,6 @@ from symforce.codegen import sym_util_package_codegen from symforce.codegen import template_util from symforce.test_util import TestCase, symengine_only -from symforce import path_util SYMFORCE_DIR = os.path.dirname(os.path.dirname(__file__)) TEST_DATA_DIR = os.path.join( @@ -87,9 +86,10 @@ def test_gen_package_codegen_python(self) -> None: """ output_dir = self.make_output_dir("sf_gen_codegen_test_") - cam_package_codegen.generate(config=codegen.PythonConfig(), output_dir=output_dir) + config = codegen.PythonConfig() + cam_package_codegen.generate(config=config, output_dir=output_dir) template_util.render_template( - template_path=os.path.join(template_util.PYTHON_TEMPLATE_DIR, "setup.py.jinja"), + template_path=config.template_dir() / "setup.py.jinja", output_path=os.path.join(output_dir, "setup.py"), data=dict( package_name="symforce-sym", @@ -140,6 +140,7 @@ def test_gen_package_codegen_cpp(self) -> None: """ Test C++ code generation """ + config = codegen.CppConfig() output_dir = self.make_output_dir("sf_gen_codegen_test_") # Prior factors, between factors, and SLAM factors for C++. @@ -147,11 +148,11 @@ def test_gen_package_codegen_cpp(self) -> None: slam_factors_codegen.generate(os.path.join(output_dir, "sym")) # Generate typedefs.h - sym_util_package_codegen.generate(config=codegen.CppConfig(), output_dir=output_dir) + sym_util_package_codegen.generate(config=config, output_dir=output_dir) # Generate cam package, geo package, and tests # This calls geo_package_codegen.generate internally - cam_package_codegen.generate(config=codegen.CppConfig(), output_dir=output_dir) + cam_package_codegen.generate(config=config, output_dir=output_dir) # Check against existing generated package (only on SymEngine) self.compare_or_update_directory( diff --git a/test/symforce_lcm_codegen_test.py b/test/symforce_lcm_codegen_test.py index f3bd969fb..69e902b78 100644 --- a/test/symforce_lcm_codegen_test.py +++ b/test/symforce_lcm_codegen_test.py @@ -17,7 +17,7 @@ def test_generate_lcm(self) -> None: output_dir = self.make_output_dir("sf_lcm_codegen_test") template_util.render_template( - os.path.join(template_util.LCM_TEMPLATE_DIR, "symforce_types.lcm.jinja"), + template_util.LCM_TEMPLATE_DIR / "symforce_types.lcm.jinja", data=lcm_symforce_types_data(), output_path=os.path.join(output_dir, "symforce_types.lcm"), ) From 423a73e8d050646a6ca2d87af7b2c73ba3356762 Mon Sep 17 00:00:00 2001 From: Hayk Martiros Date: Thu, 2 Jun 2022 19:03:45 +0200 Subject: [PATCH 07/18] Clear out printer enumeratino --- symforce/codegen/codegen_util.py | 2 +- symforce/codegen/printers/__init__.py | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/symforce/codegen/codegen_util.py b/symforce/codegen/codegen_util.py index 0302106a1..89ec4698c 100644 --- a/symforce/codegen/codegen_util.py +++ b/symforce/codegen/codegen_util.py @@ -24,7 +24,7 @@ from symforce.values import Values, IndexEntry from symforce import sympy as sm from symforce import typing as T -from symforce.codegen import printers, format_util +from symforce.codegen import format_util from symforce.codegen import codegen_config from symforce import python_util from symforce import _sympy_count_ops diff --git a/symforce/codegen/printers/__init__.py b/symforce/codegen/printers/__init__.py index 8140e53a1..e69de29bb 100644 --- a/symforce/codegen/printers/__init__.py +++ b/symforce/codegen/printers/__init__.py @@ -1,7 +0,0 @@ -# ---------------------------------------------------------------------------- -# SymForce - Copyright 2022, Skydio, Inc. -# This source code is under the Apache 2.0 license found in the LICENSE file. -# ---------------------------------------------------------------------------- - -from .cpp_code_printer import CppCodePrinter, ComplexCppCodePrinter -from .python_code_printer import PythonCodePrinter From c65ae3af77c74a3588f7fd19cc260ff05a3c6382 Mon Sep 17 00:00:00 2001 From: Hayk Martiros Date: Thu, 2 Jun 2022 16:41:13 +0200 Subject: [PATCH 08/18] Centralize templates and printers into backend / language format --- .../ops/__init__.py.jinja => backends/__init__.py} | 0 symforce/codegen/backends/cpp/README.md | 0 symforce/codegen/backends/cpp/__init__.py | 0 .../{printers => backends/cpp}/cpp_code_printer.py | 0 .../cpp/templates}/cam_package/CLASS.cc.jinja | 0 .../cpp/templates}/cam_package/CLASS.h.jinja | 0 .../cpp/templates}/cam_package/camera.h.jinja | 0 .../cpp/templates}/cam_package/posed_camera.h.jinja | 0 .../cpp/templates}/epsilon.cc.jinja | 0 .../cpp/templates}/epsilon.h.jinja | 0 .../cpp/templates}/function/FUNCTION.cc.jinja | 0 .../cpp/templates}/function/FUNCTION.h.jinja | 0 .../cpp/templates}/geo_package/CLASS.cc.jinja | 0 .../cpp/templates}/geo_package/CLASS.h.jinja | 0 .../templates}/geo_package/custom_methods/pose2.h.jinja | 0 .../templates}/geo_package/custom_methods/pose3.h.jinja | 0 .../templates}/geo_package/custom_methods/rot2.h.jinja | 0 .../templates}/geo_package/custom_methods/rot3.h.jinja | 0 .../cpp/templates}/geo_package/ops/group_ops.h.jinja | 0 .../cpp/templates}/geo_package/ops/lie_group_ops.h.jinja | 0 .../templates}/geo_package/ops/matrix/group_ops.cc.jinja | 0 .../templates}/geo_package/ops/matrix/group_ops.h.jinja | 0 .../geo_package/ops/matrix/lie_group_ops.cc.jinja | 0 .../geo_package/ops/matrix/lie_group_ops.h.jinja | 0 .../geo_package/ops/matrix/storage_ops.cc.jinja | 0 .../templates}/geo_package/ops/matrix/storage_ops.h.jinja | 0 .../templates}/geo_package/ops/scalar/group_ops.cc.jinja | 0 .../templates}/geo_package/ops/scalar/group_ops.h.jinja | 0 .../geo_package/ops/scalar/lie_group_ops.cc.jinja | 0 .../geo_package/ops/scalar/lie_group_ops.h.jinja | 0 .../geo_package/ops/scalar/storage_ops.cc.jinja | 0 .../templates}/geo_package/ops/scalar/storage_ops.h.jinja | 0 .../cpp/templates}/geo_package/ops/storage_ops.h.jinja | 0 .../cpp/templates}/keys.h.jinja | 0 .../cpp/templates}/ops/CLASS/group_ops.cc.jinja | 0 .../cpp/templates}/ops/CLASS/group_ops.h.jinja | 0 .../cpp/templates}/ops/CLASS/lie_group_ops.cc.jinja | 0 .../cpp/templates}/ops/CLASS/lie_group_ops.h.jinja | 0 .../cpp/templates}/ops/CLASS/storage_ops.cc.jinja | 0 .../cpp/templates}/ops/CLASS/storage_ops.h.jinja | 0 .../tests/cam_function_codegen_cpp_test.cc.jinja | 0 .../cpp/templates}/tests/cam_package_cpp_test.cc.jinja | 0 .../cpp/templates}/tests/geo_package_cpp_test.cc.jinja | 0 .../cpp/templates}/type_ops.h.jinja | 0 .../cpp/templates}/typedefs.h.jinja | 0 .../cpp/templates}/util/util.jinja | 0 symforce/codegen/backends/python/README.md | 0 symforce/codegen/backends/python/__init__.py | 0 .../{printers => backends/python}/python_code_printer.py | 0 .../python/templates}/cam_package/CLASS.py.jinja | 0 .../templates}/cam_package/ops/CLASS/__init__.py.jinja | 0 .../templates}/cam_package/ops/CLASS/camera_ops.py.jinja | 0 .../python/templates}/function/FUNCTION.py.jinja | 0 .../python/templates}/function/__init__.py.jinja | 0 .../python/templates}/geo_package/CLASS.py.jinja | 0 .../python/templates}/geo_package/__init__.py.jinja | 0 .../templates}/geo_package/custom_methods/pose2.py.jinja | 0 .../templates}/geo_package/custom_methods/pose3.py.jinja | 0 .../templates}/geo_package/custom_methods/rot2.py.jinja | 0 .../templates}/geo_package/custom_methods/rot3.py.jinja | 0 .../python/templates}/ops/CLASS/__init__.py.jinja | 0 .../python/templates}/ops/CLASS/group_ops.py.jinja | 0 .../python/templates}/ops/CLASS/lie_group_ops.py.jinja | 0 .../backends/python/templates/ops/__init__.py.jinja | 0 .../python/templates}/setup.py.jinja | 0 .../templates}/tests/cam_package_python_test.py.jinja | 0 .../templates}/tests/geo_package_python_test.py.jinja | 0 .../python/templates}/util/util.jinja | 0 symforce/codegen/codegen_config.py | 8 ++++---- test/symforce_cpp_code_printer_codegen_test.py | 6 +----- 70 files changed, 5 insertions(+), 9 deletions(-) rename symforce/codegen/{python_templates/ops/__init__.py.jinja => backends/__init__.py} (100%) create mode 100644 symforce/codegen/backends/cpp/README.md create mode 100644 symforce/codegen/backends/cpp/__init__.py rename symforce/codegen/{printers => backends/cpp}/cpp_code_printer.py (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/cam_package/CLASS.cc.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/cam_package/CLASS.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/cam_package/camera.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/cam_package/posed_camera.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/epsilon.cc.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/epsilon.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/function/FUNCTION.cc.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/function/FUNCTION.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/geo_package/CLASS.cc.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/geo_package/CLASS.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/geo_package/custom_methods/pose2.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/geo_package/custom_methods/pose3.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/geo_package/custom_methods/rot2.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/geo_package/custom_methods/rot3.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/geo_package/ops/group_ops.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/geo_package/ops/lie_group_ops.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/geo_package/ops/matrix/group_ops.cc.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/geo_package/ops/matrix/group_ops.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/geo_package/ops/matrix/lie_group_ops.cc.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/geo_package/ops/matrix/lie_group_ops.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/geo_package/ops/matrix/storage_ops.cc.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/geo_package/ops/matrix/storage_ops.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/geo_package/ops/scalar/group_ops.cc.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/geo_package/ops/scalar/group_ops.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/geo_package/ops/scalar/lie_group_ops.cc.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/geo_package/ops/scalar/lie_group_ops.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/geo_package/ops/scalar/storage_ops.cc.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/geo_package/ops/scalar/storage_ops.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/geo_package/ops/storage_ops.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/keys.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/ops/CLASS/group_ops.cc.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/ops/CLASS/group_ops.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/ops/CLASS/lie_group_ops.cc.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/ops/CLASS/lie_group_ops.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/ops/CLASS/storage_ops.cc.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/ops/CLASS/storage_ops.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/tests/cam_function_codegen_cpp_test.cc.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/tests/cam_package_cpp_test.cc.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/tests/geo_package_cpp_test.cc.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/type_ops.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/typedefs.h.jinja (100%) rename symforce/codegen/{cpp_templates => backends/cpp/templates}/util/util.jinja (100%) create mode 100644 symforce/codegen/backends/python/README.md create mode 100644 symforce/codegen/backends/python/__init__.py rename symforce/codegen/{printers => backends/python}/python_code_printer.py (100%) rename symforce/codegen/{python_templates => backends/python/templates}/cam_package/CLASS.py.jinja (100%) rename symforce/codegen/{python_templates => backends/python/templates}/cam_package/ops/CLASS/__init__.py.jinja (100%) rename symforce/codegen/{python_templates => backends/python/templates}/cam_package/ops/CLASS/camera_ops.py.jinja (100%) rename symforce/codegen/{python_templates => backends/python/templates}/function/FUNCTION.py.jinja (100%) rename symforce/codegen/{python_templates => backends/python/templates}/function/__init__.py.jinja (100%) rename symforce/codegen/{python_templates => backends/python/templates}/geo_package/CLASS.py.jinja (100%) rename symforce/codegen/{python_templates => backends/python/templates}/geo_package/__init__.py.jinja (100%) rename symforce/codegen/{python_templates => backends/python/templates}/geo_package/custom_methods/pose2.py.jinja (100%) rename symforce/codegen/{python_templates => backends/python/templates}/geo_package/custom_methods/pose3.py.jinja (100%) rename symforce/codegen/{python_templates => backends/python/templates}/geo_package/custom_methods/rot2.py.jinja (100%) rename symforce/codegen/{python_templates => backends/python/templates}/geo_package/custom_methods/rot3.py.jinja (100%) rename symforce/codegen/{python_templates => backends/python/templates}/ops/CLASS/__init__.py.jinja (100%) rename symforce/codegen/{python_templates => backends/python/templates}/ops/CLASS/group_ops.py.jinja (100%) rename symforce/codegen/{python_templates => backends/python/templates}/ops/CLASS/lie_group_ops.py.jinja (100%) create mode 100644 symforce/codegen/backends/python/templates/ops/__init__.py.jinja rename symforce/codegen/{python_templates => backends/python/templates}/setup.py.jinja (100%) rename symforce/codegen/{python_templates => backends/python/templates}/tests/cam_package_python_test.py.jinja (100%) rename symforce/codegen/{python_templates => backends/python/templates}/tests/geo_package_python_test.py.jinja (100%) rename symforce/codegen/{python_templates => backends/python/templates}/util/util.jinja (100%) diff --git a/symforce/codegen/python_templates/ops/__init__.py.jinja b/symforce/codegen/backends/__init__.py similarity index 100% rename from symforce/codegen/python_templates/ops/__init__.py.jinja rename to symforce/codegen/backends/__init__.py diff --git a/symforce/codegen/backends/cpp/README.md b/symforce/codegen/backends/cpp/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/symforce/codegen/backends/cpp/__init__.py b/symforce/codegen/backends/cpp/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/symforce/codegen/printers/cpp_code_printer.py b/symforce/codegen/backends/cpp/cpp_code_printer.py similarity index 100% rename from symforce/codegen/printers/cpp_code_printer.py rename to symforce/codegen/backends/cpp/cpp_code_printer.py diff --git a/symforce/codegen/cpp_templates/cam_package/CLASS.cc.jinja b/symforce/codegen/backends/cpp/templates/cam_package/CLASS.cc.jinja similarity index 100% rename from symforce/codegen/cpp_templates/cam_package/CLASS.cc.jinja rename to symforce/codegen/backends/cpp/templates/cam_package/CLASS.cc.jinja diff --git a/symforce/codegen/cpp_templates/cam_package/CLASS.h.jinja b/symforce/codegen/backends/cpp/templates/cam_package/CLASS.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/cam_package/CLASS.h.jinja rename to symforce/codegen/backends/cpp/templates/cam_package/CLASS.h.jinja diff --git a/symforce/codegen/cpp_templates/cam_package/camera.h.jinja b/symforce/codegen/backends/cpp/templates/cam_package/camera.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/cam_package/camera.h.jinja rename to symforce/codegen/backends/cpp/templates/cam_package/camera.h.jinja diff --git a/symforce/codegen/cpp_templates/cam_package/posed_camera.h.jinja b/symforce/codegen/backends/cpp/templates/cam_package/posed_camera.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/cam_package/posed_camera.h.jinja rename to symforce/codegen/backends/cpp/templates/cam_package/posed_camera.h.jinja diff --git a/symforce/codegen/cpp_templates/epsilon.cc.jinja b/symforce/codegen/backends/cpp/templates/epsilon.cc.jinja similarity index 100% rename from symforce/codegen/cpp_templates/epsilon.cc.jinja rename to symforce/codegen/backends/cpp/templates/epsilon.cc.jinja diff --git a/symforce/codegen/cpp_templates/epsilon.h.jinja b/symforce/codegen/backends/cpp/templates/epsilon.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/epsilon.h.jinja rename to symforce/codegen/backends/cpp/templates/epsilon.h.jinja diff --git a/symforce/codegen/cpp_templates/function/FUNCTION.cc.jinja b/symforce/codegen/backends/cpp/templates/function/FUNCTION.cc.jinja similarity index 100% rename from symforce/codegen/cpp_templates/function/FUNCTION.cc.jinja rename to symforce/codegen/backends/cpp/templates/function/FUNCTION.cc.jinja diff --git a/symforce/codegen/cpp_templates/function/FUNCTION.h.jinja b/symforce/codegen/backends/cpp/templates/function/FUNCTION.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/function/FUNCTION.h.jinja rename to symforce/codegen/backends/cpp/templates/function/FUNCTION.h.jinja diff --git a/symforce/codegen/cpp_templates/geo_package/CLASS.cc.jinja b/symforce/codegen/backends/cpp/templates/geo_package/CLASS.cc.jinja similarity index 100% rename from symforce/codegen/cpp_templates/geo_package/CLASS.cc.jinja rename to symforce/codegen/backends/cpp/templates/geo_package/CLASS.cc.jinja diff --git a/symforce/codegen/cpp_templates/geo_package/CLASS.h.jinja b/symforce/codegen/backends/cpp/templates/geo_package/CLASS.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/geo_package/CLASS.h.jinja rename to symforce/codegen/backends/cpp/templates/geo_package/CLASS.h.jinja diff --git a/symforce/codegen/cpp_templates/geo_package/custom_methods/pose2.h.jinja b/symforce/codegen/backends/cpp/templates/geo_package/custom_methods/pose2.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/geo_package/custom_methods/pose2.h.jinja rename to symforce/codegen/backends/cpp/templates/geo_package/custom_methods/pose2.h.jinja diff --git a/symforce/codegen/cpp_templates/geo_package/custom_methods/pose3.h.jinja b/symforce/codegen/backends/cpp/templates/geo_package/custom_methods/pose3.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/geo_package/custom_methods/pose3.h.jinja rename to symforce/codegen/backends/cpp/templates/geo_package/custom_methods/pose3.h.jinja diff --git a/symforce/codegen/cpp_templates/geo_package/custom_methods/rot2.h.jinja b/symforce/codegen/backends/cpp/templates/geo_package/custom_methods/rot2.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/geo_package/custom_methods/rot2.h.jinja rename to symforce/codegen/backends/cpp/templates/geo_package/custom_methods/rot2.h.jinja diff --git a/symforce/codegen/cpp_templates/geo_package/custom_methods/rot3.h.jinja b/symforce/codegen/backends/cpp/templates/geo_package/custom_methods/rot3.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/geo_package/custom_methods/rot3.h.jinja rename to symforce/codegen/backends/cpp/templates/geo_package/custom_methods/rot3.h.jinja diff --git a/symforce/codegen/cpp_templates/geo_package/ops/group_ops.h.jinja b/symforce/codegen/backends/cpp/templates/geo_package/ops/group_ops.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/geo_package/ops/group_ops.h.jinja rename to symforce/codegen/backends/cpp/templates/geo_package/ops/group_ops.h.jinja diff --git a/symforce/codegen/cpp_templates/geo_package/ops/lie_group_ops.h.jinja b/symforce/codegen/backends/cpp/templates/geo_package/ops/lie_group_ops.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/geo_package/ops/lie_group_ops.h.jinja rename to symforce/codegen/backends/cpp/templates/geo_package/ops/lie_group_ops.h.jinja diff --git a/symforce/codegen/cpp_templates/geo_package/ops/matrix/group_ops.cc.jinja b/symforce/codegen/backends/cpp/templates/geo_package/ops/matrix/group_ops.cc.jinja similarity index 100% rename from symforce/codegen/cpp_templates/geo_package/ops/matrix/group_ops.cc.jinja rename to symforce/codegen/backends/cpp/templates/geo_package/ops/matrix/group_ops.cc.jinja diff --git a/symforce/codegen/cpp_templates/geo_package/ops/matrix/group_ops.h.jinja b/symforce/codegen/backends/cpp/templates/geo_package/ops/matrix/group_ops.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/geo_package/ops/matrix/group_ops.h.jinja rename to symforce/codegen/backends/cpp/templates/geo_package/ops/matrix/group_ops.h.jinja diff --git a/symforce/codegen/cpp_templates/geo_package/ops/matrix/lie_group_ops.cc.jinja b/symforce/codegen/backends/cpp/templates/geo_package/ops/matrix/lie_group_ops.cc.jinja similarity index 100% rename from symforce/codegen/cpp_templates/geo_package/ops/matrix/lie_group_ops.cc.jinja rename to symforce/codegen/backends/cpp/templates/geo_package/ops/matrix/lie_group_ops.cc.jinja diff --git a/symforce/codegen/cpp_templates/geo_package/ops/matrix/lie_group_ops.h.jinja b/symforce/codegen/backends/cpp/templates/geo_package/ops/matrix/lie_group_ops.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/geo_package/ops/matrix/lie_group_ops.h.jinja rename to symforce/codegen/backends/cpp/templates/geo_package/ops/matrix/lie_group_ops.h.jinja diff --git a/symforce/codegen/cpp_templates/geo_package/ops/matrix/storage_ops.cc.jinja b/symforce/codegen/backends/cpp/templates/geo_package/ops/matrix/storage_ops.cc.jinja similarity index 100% rename from symforce/codegen/cpp_templates/geo_package/ops/matrix/storage_ops.cc.jinja rename to symforce/codegen/backends/cpp/templates/geo_package/ops/matrix/storage_ops.cc.jinja diff --git a/symforce/codegen/cpp_templates/geo_package/ops/matrix/storage_ops.h.jinja b/symforce/codegen/backends/cpp/templates/geo_package/ops/matrix/storage_ops.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/geo_package/ops/matrix/storage_ops.h.jinja rename to symforce/codegen/backends/cpp/templates/geo_package/ops/matrix/storage_ops.h.jinja diff --git a/symforce/codegen/cpp_templates/geo_package/ops/scalar/group_ops.cc.jinja b/symforce/codegen/backends/cpp/templates/geo_package/ops/scalar/group_ops.cc.jinja similarity index 100% rename from symforce/codegen/cpp_templates/geo_package/ops/scalar/group_ops.cc.jinja rename to symforce/codegen/backends/cpp/templates/geo_package/ops/scalar/group_ops.cc.jinja diff --git a/symforce/codegen/cpp_templates/geo_package/ops/scalar/group_ops.h.jinja b/symforce/codegen/backends/cpp/templates/geo_package/ops/scalar/group_ops.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/geo_package/ops/scalar/group_ops.h.jinja rename to symforce/codegen/backends/cpp/templates/geo_package/ops/scalar/group_ops.h.jinja diff --git a/symforce/codegen/cpp_templates/geo_package/ops/scalar/lie_group_ops.cc.jinja b/symforce/codegen/backends/cpp/templates/geo_package/ops/scalar/lie_group_ops.cc.jinja similarity index 100% rename from symforce/codegen/cpp_templates/geo_package/ops/scalar/lie_group_ops.cc.jinja rename to symforce/codegen/backends/cpp/templates/geo_package/ops/scalar/lie_group_ops.cc.jinja diff --git a/symforce/codegen/cpp_templates/geo_package/ops/scalar/lie_group_ops.h.jinja b/symforce/codegen/backends/cpp/templates/geo_package/ops/scalar/lie_group_ops.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/geo_package/ops/scalar/lie_group_ops.h.jinja rename to symforce/codegen/backends/cpp/templates/geo_package/ops/scalar/lie_group_ops.h.jinja diff --git a/symforce/codegen/cpp_templates/geo_package/ops/scalar/storage_ops.cc.jinja b/symforce/codegen/backends/cpp/templates/geo_package/ops/scalar/storage_ops.cc.jinja similarity index 100% rename from symforce/codegen/cpp_templates/geo_package/ops/scalar/storage_ops.cc.jinja rename to symforce/codegen/backends/cpp/templates/geo_package/ops/scalar/storage_ops.cc.jinja diff --git a/symforce/codegen/cpp_templates/geo_package/ops/scalar/storage_ops.h.jinja b/symforce/codegen/backends/cpp/templates/geo_package/ops/scalar/storage_ops.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/geo_package/ops/scalar/storage_ops.h.jinja rename to symforce/codegen/backends/cpp/templates/geo_package/ops/scalar/storage_ops.h.jinja diff --git a/symforce/codegen/cpp_templates/geo_package/ops/storage_ops.h.jinja b/symforce/codegen/backends/cpp/templates/geo_package/ops/storage_ops.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/geo_package/ops/storage_ops.h.jinja rename to symforce/codegen/backends/cpp/templates/geo_package/ops/storage_ops.h.jinja diff --git a/symforce/codegen/cpp_templates/keys.h.jinja b/symforce/codegen/backends/cpp/templates/keys.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/keys.h.jinja rename to symforce/codegen/backends/cpp/templates/keys.h.jinja diff --git a/symforce/codegen/cpp_templates/ops/CLASS/group_ops.cc.jinja b/symforce/codegen/backends/cpp/templates/ops/CLASS/group_ops.cc.jinja similarity index 100% rename from symforce/codegen/cpp_templates/ops/CLASS/group_ops.cc.jinja rename to symforce/codegen/backends/cpp/templates/ops/CLASS/group_ops.cc.jinja diff --git a/symforce/codegen/cpp_templates/ops/CLASS/group_ops.h.jinja b/symforce/codegen/backends/cpp/templates/ops/CLASS/group_ops.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/ops/CLASS/group_ops.h.jinja rename to symforce/codegen/backends/cpp/templates/ops/CLASS/group_ops.h.jinja diff --git a/symforce/codegen/cpp_templates/ops/CLASS/lie_group_ops.cc.jinja b/symforce/codegen/backends/cpp/templates/ops/CLASS/lie_group_ops.cc.jinja similarity index 100% rename from symforce/codegen/cpp_templates/ops/CLASS/lie_group_ops.cc.jinja rename to symforce/codegen/backends/cpp/templates/ops/CLASS/lie_group_ops.cc.jinja diff --git a/symforce/codegen/cpp_templates/ops/CLASS/lie_group_ops.h.jinja b/symforce/codegen/backends/cpp/templates/ops/CLASS/lie_group_ops.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/ops/CLASS/lie_group_ops.h.jinja rename to symforce/codegen/backends/cpp/templates/ops/CLASS/lie_group_ops.h.jinja diff --git a/symforce/codegen/cpp_templates/ops/CLASS/storage_ops.cc.jinja b/symforce/codegen/backends/cpp/templates/ops/CLASS/storage_ops.cc.jinja similarity index 100% rename from symforce/codegen/cpp_templates/ops/CLASS/storage_ops.cc.jinja rename to symforce/codegen/backends/cpp/templates/ops/CLASS/storage_ops.cc.jinja diff --git a/symforce/codegen/cpp_templates/ops/CLASS/storage_ops.h.jinja b/symforce/codegen/backends/cpp/templates/ops/CLASS/storage_ops.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/ops/CLASS/storage_ops.h.jinja rename to symforce/codegen/backends/cpp/templates/ops/CLASS/storage_ops.h.jinja diff --git a/symforce/codegen/cpp_templates/tests/cam_function_codegen_cpp_test.cc.jinja b/symforce/codegen/backends/cpp/templates/tests/cam_function_codegen_cpp_test.cc.jinja similarity index 100% rename from symforce/codegen/cpp_templates/tests/cam_function_codegen_cpp_test.cc.jinja rename to symforce/codegen/backends/cpp/templates/tests/cam_function_codegen_cpp_test.cc.jinja diff --git a/symforce/codegen/cpp_templates/tests/cam_package_cpp_test.cc.jinja b/symforce/codegen/backends/cpp/templates/tests/cam_package_cpp_test.cc.jinja similarity index 100% rename from symforce/codegen/cpp_templates/tests/cam_package_cpp_test.cc.jinja rename to symforce/codegen/backends/cpp/templates/tests/cam_package_cpp_test.cc.jinja diff --git a/symforce/codegen/cpp_templates/tests/geo_package_cpp_test.cc.jinja b/symforce/codegen/backends/cpp/templates/tests/geo_package_cpp_test.cc.jinja similarity index 100% rename from symforce/codegen/cpp_templates/tests/geo_package_cpp_test.cc.jinja rename to symforce/codegen/backends/cpp/templates/tests/geo_package_cpp_test.cc.jinja diff --git a/symforce/codegen/cpp_templates/type_ops.h.jinja b/symforce/codegen/backends/cpp/templates/type_ops.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/type_ops.h.jinja rename to symforce/codegen/backends/cpp/templates/type_ops.h.jinja diff --git a/symforce/codegen/cpp_templates/typedefs.h.jinja b/symforce/codegen/backends/cpp/templates/typedefs.h.jinja similarity index 100% rename from symforce/codegen/cpp_templates/typedefs.h.jinja rename to symforce/codegen/backends/cpp/templates/typedefs.h.jinja diff --git a/symforce/codegen/cpp_templates/util/util.jinja b/symforce/codegen/backends/cpp/templates/util/util.jinja similarity index 100% rename from symforce/codegen/cpp_templates/util/util.jinja rename to symforce/codegen/backends/cpp/templates/util/util.jinja diff --git a/symforce/codegen/backends/python/README.md b/symforce/codegen/backends/python/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/symforce/codegen/backends/python/__init__.py b/symforce/codegen/backends/python/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/symforce/codegen/printers/python_code_printer.py b/symforce/codegen/backends/python/python_code_printer.py similarity index 100% rename from symforce/codegen/printers/python_code_printer.py rename to symforce/codegen/backends/python/python_code_printer.py diff --git a/symforce/codegen/python_templates/cam_package/CLASS.py.jinja b/symforce/codegen/backends/python/templates/cam_package/CLASS.py.jinja similarity index 100% rename from symforce/codegen/python_templates/cam_package/CLASS.py.jinja rename to symforce/codegen/backends/python/templates/cam_package/CLASS.py.jinja diff --git a/symforce/codegen/python_templates/cam_package/ops/CLASS/__init__.py.jinja b/symforce/codegen/backends/python/templates/cam_package/ops/CLASS/__init__.py.jinja similarity index 100% rename from symforce/codegen/python_templates/cam_package/ops/CLASS/__init__.py.jinja rename to symforce/codegen/backends/python/templates/cam_package/ops/CLASS/__init__.py.jinja diff --git a/symforce/codegen/python_templates/cam_package/ops/CLASS/camera_ops.py.jinja b/symforce/codegen/backends/python/templates/cam_package/ops/CLASS/camera_ops.py.jinja similarity index 100% rename from symforce/codegen/python_templates/cam_package/ops/CLASS/camera_ops.py.jinja rename to symforce/codegen/backends/python/templates/cam_package/ops/CLASS/camera_ops.py.jinja diff --git a/symforce/codegen/python_templates/function/FUNCTION.py.jinja b/symforce/codegen/backends/python/templates/function/FUNCTION.py.jinja similarity index 100% rename from symforce/codegen/python_templates/function/FUNCTION.py.jinja rename to symforce/codegen/backends/python/templates/function/FUNCTION.py.jinja diff --git a/symforce/codegen/python_templates/function/__init__.py.jinja b/symforce/codegen/backends/python/templates/function/__init__.py.jinja similarity index 100% rename from symforce/codegen/python_templates/function/__init__.py.jinja rename to symforce/codegen/backends/python/templates/function/__init__.py.jinja diff --git a/symforce/codegen/python_templates/geo_package/CLASS.py.jinja b/symforce/codegen/backends/python/templates/geo_package/CLASS.py.jinja similarity index 100% rename from symforce/codegen/python_templates/geo_package/CLASS.py.jinja rename to symforce/codegen/backends/python/templates/geo_package/CLASS.py.jinja diff --git a/symforce/codegen/python_templates/geo_package/__init__.py.jinja b/symforce/codegen/backends/python/templates/geo_package/__init__.py.jinja similarity index 100% rename from symforce/codegen/python_templates/geo_package/__init__.py.jinja rename to symforce/codegen/backends/python/templates/geo_package/__init__.py.jinja diff --git a/symforce/codegen/python_templates/geo_package/custom_methods/pose2.py.jinja b/symforce/codegen/backends/python/templates/geo_package/custom_methods/pose2.py.jinja similarity index 100% rename from symforce/codegen/python_templates/geo_package/custom_methods/pose2.py.jinja rename to symforce/codegen/backends/python/templates/geo_package/custom_methods/pose2.py.jinja diff --git a/symforce/codegen/python_templates/geo_package/custom_methods/pose3.py.jinja b/symforce/codegen/backends/python/templates/geo_package/custom_methods/pose3.py.jinja similarity index 100% rename from symforce/codegen/python_templates/geo_package/custom_methods/pose3.py.jinja rename to symforce/codegen/backends/python/templates/geo_package/custom_methods/pose3.py.jinja diff --git a/symforce/codegen/python_templates/geo_package/custom_methods/rot2.py.jinja b/symforce/codegen/backends/python/templates/geo_package/custom_methods/rot2.py.jinja similarity index 100% rename from symforce/codegen/python_templates/geo_package/custom_methods/rot2.py.jinja rename to symforce/codegen/backends/python/templates/geo_package/custom_methods/rot2.py.jinja diff --git a/symforce/codegen/python_templates/geo_package/custom_methods/rot3.py.jinja b/symforce/codegen/backends/python/templates/geo_package/custom_methods/rot3.py.jinja similarity index 100% rename from symforce/codegen/python_templates/geo_package/custom_methods/rot3.py.jinja rename to symforce/codegen/backends/python/templates/geo_package/custom_methods/rot3.py.jinja diff --git a/symforce/codegen/python_templates/ops/CLASS/__init__.py.jinja b/symforce/codegen/backends/python/templates/ops/CLASS/__init__.py.jinja similarity index 100% rename from symforce/codegen/python_templates/ops/CLASS/__init__.py.jinja rename to symforce/codegen/backends/python/templates/ops/CLASS/__init__.py.jinja diff --git a/symforce/codegen/python_templates/ops/CLASS/group_ops.py.jinja b/symforce/codegen/backends/python/templates/ops/CLASS/group_ops.py.jinja similarity index 100% rename from symforce/codegen/python_templates/ops/CLASS/group_ops.py.jinja rename to symforce/codegen/backends/python/templates/ops/CLASS/group_ops.py.jinja diff --git a/symforce/codegen/python_templates/ops/CLASS/lie_group_ops.py.jinja b/symforce/codegen/backends/python/templates/ops/CLASS/lie_group_ops.py.jinja similarity index 100% rename from symforce/codegen/python_templates/ops/CLASS/lie_group_ops.py.jinja rename to symforce/codegen/backends/python/templates/ops/CLASS/lie_group_ops.py.jinja diff --git a/symforce/codegen/backends/python/templates/ops/__init__.py.jinja b/symforce/codegen/backends/python/templates/ops/__init__.py.jinja new file mode 100644 index 000000000..e69de29bb diff --git a/symforce/codegen/python_templates/setup.py.jinja b/symforce/codegen/backends/python/templates/setup.py.jinja similarity index 100% rename from symforce/codegen/python_templates/setup.py.jinja rename to symforce/codegen/backends/python/templates/setup.py.jinja diff --git a/symforce/codegen/python_templates/tests/cam_package_python_test.py.jinja b/symforce/codegen/backends/python/templates/tests/cam_package_python_test.py.jinja similarity index 100% rename from symforce/codegen/python_templates/tests/cam_package_python_test.py.jinja rename to symforce/codegen/backends/python/templates/tests/cam_package_python_test.py.jinja diff --git a/symforce/codegen/python_templates/tests/geo_package_python_test.py.jinja b/symforce/codegen/backends/python/templates/tests/geo_package_python_test.py.jinja similarity index 100% rename from symforce/codegen/python_templates/tests/geo_package_python_test.py.jinja rename to symforce/codegen/backends/python/templates/tests/geo_package_python_test.py.jinja diff --git a/symforce/codegen/python_templates/util/util.jinja b/symforce/codegen/backends/python/templates/util/util.jinja similarity index 100% rename from symforce/codegen/python_templates/util/util.jinja rename to symforce/codegen/backends/python/templates/util/util.jinja diff --git a/symforce/codegen/codegen_config.py b/symforce/codegen/codegen_config.py index 2d755a9de..03549fc78 100644 --- a/symforce/codegen/codegen_config.py +++ b/symforce/codegen/codegen_config.py @@ -89,7 +89,7 @@ class CppConfig(CodegenConfig): explicit_template_instantiation_types: T.Optional[T.Sequence[str]] = None def printer(self) -> "sm.CodePrinter": - from symforce.codegen.printers import cpp_code_printer + from symforce.codegen.backends.cpp import cpp_code_printer if self.support_complex: return cpp_code_printer.ComplexCppCodePrinter() @@ -98,7 +98,7 @@ def printer(self) -> "sm.CodePrinter": @classmethod def template_dir(cls) -> Path: - return CURRENT_DIR / "cpp_templates" + return CURRENT_DIR / "backends" / "cpp" / "templates" @staticmethod def format_data_accessor(prefix: str, index: int) -> str: @@ -131,11 +131,11 @@ class PythonConfig(CodegenConfig): matrix_is_1d: bool = True def printer(self) -> "sm.CodePrinter": - from symforce.codegen.printers import python_code_printer + from symforce.codegen.backends.python import python_code_printer return python_code_printer.PythonCodePrinter() @classmethod def template_dir(cls) -> Path: - return CURRENT_DIR / "python_templates" + return CURRENT_DIR / "backends" / "python" / "templates" diff --git a/test/symforce_cpp_code_printer_codegen_test.py b/test/symforce_cpp_code_printer_codegen_test.py index e95302781..851a31f44 100644 --- a/test/symforce_cpp_code_printer_codegen_test.py +++ b/test/symforce_cpp_code_printer_codegen_test.py @@ -7,13 +7,9 @@ import symforce -from symforce import typing as T -from symforce.test_util import TestCase, sympy_only -from symforce.values import Values - from symforce import codegen -from symforce.codegen import codegen_util from symforce import sympy as sm +from symforce.test_util import TestCase, sympy_only SYMFORCE_DIR = os.path.dirname(os.path.dirname(__file__)) TEST_DATA_DIR = os.path.join( From 5a9e0202431cf6dcc2769dbcac83056c546f560f Mon Sep 17 00:00:00 2001 From: Hayk Martiros Date: Thu, 2 Jun 2022 19:06:24 +0200 Subject: [PATCH 09/18] Move codegen configs to backend directories --- symforce/codegen/__init__.py | 6 +- symforce/codegen/backends/cpp/cpp_config.py | 60 +++++++++++++ .../codegen/backends/python/python_config.py | 47 ++++++++++ symforce/codegen/codegen.py | 5 +- symforce/codegen/codegen_config.py | 85 +------------------ 5 files changed, 116 insertions(+), 87 deletions(-) create mode 100644 symforce/codegen/backends/cpp/cpp_config.py create mode 100644 symforce/codegen/backends/python/python_config.py diff --git a/symforce/codegen/__init__.py b/symforce/codegen/__init__.py index 526e7f6c7..955730c0a 100644 --- a/symforce/codegen/__init__.py +++ b/symforce/codegen/__init__.py @@ -7,5 +7,9 @@ Package for executable code generation from symbolic expressions. """ -from .codegen_config import CodegenConfig, CppConfig, PythonConfig from .codegen import Codegen, LinearizationMode, GeneratedPaths +from .codegen_config import CodegenConfig + +# TODO(hayk): Do we want to explicitly enumerate all configs here? (tag=centralize-language-diffs) +from .backends.cpp.cpp_config import CppConfig +from .backends.python.python_config import PythonConfig diff --git a/symforce/codegen/backends/cpp/cpp_config.py b/symforce/codegen/backends/cpp/cpp_config.py new file mode 100644 index 000000000..ff77a6eb5 --- /dev/null +++ b/symforce/codegen/backends/cpp/cpp_config.py @@ -0,0 +1,60 @@ +# ---------------------------------------------------------------------------- +# SymForce - Copyright 2022, Skydio, Inc. +# This source code is under the Apache 2.0 license found in the LICENSE file. +# ---------------------------------------------------------------------------- +from __future__ import annotations +from dataclasses import dataclass +from pathlib import Path + +from symforce import typing as T +from symforce.codegen.codegen_config import CodegenConfig + +CURRENT_DIR = Path(__file__).parent + + +@dataclass +class CppConfig(CodegenConfig): + """ + Code generation config for the C++ backend. + + Args: + doc_comment_line_prefix: Prefix applied to each line in a docstring + line_length: Maximum allowed line length in docstrings; used for formatting docstrings. + use_eigen_types: Use eigen_lcm types for vectors instead of lists + autoformat: Run a code formatter on the generated code + cse_optimizations: Optimizations argument to pass to sm.cse + support_complex: Generate code that can work with std::complex or with regular float types + force_no_inline: Mark generated functions as `__attribute__((noinline))` + zero_initialization_sparsity_threshold: Threshold between 0 and 1 for the sparsity below + which we'll initialize an output matrix to 0, so we + don't have to generate a line to set each zero + element to 0 individually + explicit_template_instantiation_types: Explicity instantiates templated functions in a `.cc` + file for each given type. This allows the generated function to be compiled in its own + translation unit. Useful for large functions which take a long time to compile. + """ + + doc_comment_line_prefix: str = " * " + line_length: int = 100 + use_eigen_types: bool = True + support_complex: bool = False + force_no_inline: bool = False + zero_initialization_sparsity_threshold: float = 0.5 + explicit_template_instantiation_types: T.Optional[T.Sequence[str]] = None + + def printer(self) -> "sm.CodePrinter": + # NOTE(hayk): Is there any benefit to this being lazy? + from symforce.codegen.backends.cpp import cpp_code_printer + + if self.support_complex: + return cpp_code_printer.ComplexCppCodePrinter() + else: + return cpp_code_printer.CppCodePrinter() + + @classmethod + def template_dir(cls) -> Path: + return CURRENT_DIR / "templates" + + @staticmethod + def format_data_accessor(prefix: str, index: int) -> str: + return f"{prefix}.Data()[{index}]" diff --git a/symforce/codegen/backends/python/python_config.py b/symforce/codegen/backends/python/python_config.py new file mode 100644 index 000000000..c430a1fe9 --- /dev/null +++ b/symforce/codegen/backends/python/python_config.py @@ -0,0 +1,47 @@ +# ---------------------------------------------------------------------------- +# SymForce - Copyright 2022, Skydio, Inc. +# This source code is under the Apache 2.0 license found in the LICENSE file. +# ---------------------------------------------------------------------------- +from __future__ import annotations +from dataclasses import dataclass +from pathlib import Path + +from symforce.codegen.codegen_config import CodegenConfig + + +CURRENT_DIR = Path(__file__).parent + + +@dataclass +class PythonConfig(CodegenConfig): + """ + Code generation config for the Python backend. + + Args: + doc_comment_line_prefix: Prefix applied to each line in a docstring + line_length: Maximum allowed line length in docstrings; used for formatting docstrings. + use_eigen_types: Use eigen_lcm types for vectors instead of lists + autoformat: Run a code formatter on the generated code + cse_optimizations: Optimizations argument to pass to sm.cse + use_numba: Add the `@numba.njit` decorator to generated functions. This will greatly + speed up functions by compiling them to machine code, but has large overhead + on the first call and some overhead on subsequent calls, so it should not be + used for small functions or functions that are only called a handfull of + times. + matrix_is_1D: geo.Matrix symbols get formatted as a 1D array + """ + + doc_comment_line_prefix: str = "" + line_length: int = 100 + use_eigen_types: bool = True + use_numba: bool = False + matrix_is_1d: bool = True + + def printer(self) -> "sm.CodePrinter": + from symforce.codegen.backends.python import python_code_printer + + return python_code_printer.PythonCodePrinter() + + @classmethod + def template_dir(cls) -> Path: + return CURRENT_DIR / "templates" diff --git a/symforce/codegen/codegen.py b/symforce/codegen/codegen.py index 2189cc01d..0795b61e9 100644 --- a/symforce/codegen/codegen.py +++ b/symforce/codegen/codegen.py @@ -23,6 +23,7 @@ from symforce import python_util from symforce import typing as T from symforce.values import Values +from symforce import codegen from symforce.codegen import template_util from symforce.codegen import codegen_util from symforce.codegen import codegen_config @@ -396,7 +397,7 @@ def generate_function( template_dir = self.config.template_dir() # Generate the function - if isinstance(self.config, codegen_config.PythonConfig): + if isinstance(self.config, codegen.PythonConfig): if skip_directory_nesting: out_function_dir = output_dir else: @@ -414,7 +415,7 @@ def generate_function( out_function_dir / "__init__.py", template_data, ) - elif isinstance(self.config, codegen_config.CppConfig): + elif isinstance(self.config, codegen.CppConfig): if skip_directory_nesting: out_function_dir = output_dir else: diff --git a/symforce/codegen/codegen_config.py b/symforce/codegen/codegen_config.py index 03549fc78..f4c5e828e 100644 --- a/symforce/codegen/codegen_config.py +++ b/symforce/codegen/codegen_config.py @@ -14,7 +14,7 @@ @dataclass class CodegenConfig: """ - Base class for language-specific arguments for code generation + Base class for backend-specific arguments for code generation. Args: doc_comment_line_prefix: Prefix applied to each line in a docstring, e.g. " * " for C++ @@ -56,86 +56,3 @@ def format_data_accessor(prefix: str, index: int) -> str: Format data for accessing a data array in code. """ return f"{prefix}.data[{index}]" - - -@dataclass -class CppConfig(CodegenConfig): - """ - C++ Codegen configuration - - Args: - doc_comment_line_prefix: Prefix applied to each line in a docstring - line_length: Maximum allowed line length in docstrings; used for formatting docstrings. - use_eigen_types: Use eigen_lcm types for vectors instead of lists - autoformat: Run a code formatter on the generated code - cse_optimizations: Optimizations argument to pass to sm.cse - support_complex: Generate code that can work with std::complex or with regular float types - force_no_inline: Mark generated functions as `__attribute__((noinline))` - zero_initialization_sparsity_threshold: Threshold between 0 and 1 for the sparsity below - which we'll initialize an output matrix to 0, so we - don't have to generate a line to set each zero - element to 0 individually - explicit_template_instantiation_types: Explicity instantiates templated functions in a `.cc` - file for each given type. This allows the generated function to be compiled in its own - translation unit. Useful for large functions which take a long time to compile. - """ - - doc_comment_line_prefix: str = " * " - line_length: int = 100 - use_eigen_types: bool = True - support_complex: bool = False - force_no_inline: bool = False - zero_initialization_sparsity_threshold: float = 0.5 - explicit_template_instantiation_types: T.Optional[T.Sequence[str]] = None - - def printer(self) -> "sm.CodePrinter": - from symforce.codegen.backends.cpp import cpp_code_printer - - if self.support_complex: - return cpp_code_printer.ComplexCppCodePrinter() - else: - return cpp_code_printer.CppCodePrinter() - - @classmethod - def template_dir(cls) -> Path: - return CURRENT_DIR / "backends" / "cpp" / "templates" - - @staticmethod - def format_data_accessor(prefix: str, index: int) -> str: - return f"{prefix}.Data()[{index}]" - - -@dataclass -class PythonConfig(CodegenConfig): - """ - Python Codegen configuration - - Args: - doc_comment_line_prefix: Prefix applied to each line in a docstring - line_length: Maximum allowed line length in docstrings; used for formatting docstrings. - use_eigen_types: Use eigen_lcm types for vectors instead of lists - autoformat: Run a code formatter on the generated code - cse_optimizations: Optimizations argument to pass to sm.cse - use_numba: Add the `@numba.njit` decorator to generated functions. This will greatly - speed up functions by compiling them to machine code, but has large overhead - on the first call and some overhead on subsequent calls, so it should not be - used for small functions or functions that are only called a handfull of - times. - matrix_is_1D: geo.Matrix symbols get formatted as a 1D array - """ - - doc_comment_line_prefix: str = "" - line_length: int = 100 - use_eigen_types: bool = True - use_numba: bool = False - matrix_is_1d: bool = True - - def printer(self) -> "sm.CodePrinter": - from symforce.codegen.backends.python import python_code_printer - - return python_code_printer.PythonCodePrinter() - - @classmethod - def template_dir(cls) -> Path: - return CURRENT_DIR / "backends" / "python" / "templates" - From 1da2e7dce702d53f974aba5bf8ade6ca61ec74d3 Mon Sep 17 00:00:00 2001 From: Hayk Martiros Date: Thu, 2 Jun 2022 19:21:40 +0200 Subject: [PATCH 10/18] generalize comment prefix a bit --- symforce/codegen/template_util.py | 37 ++++++++++++++++++------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/symforce/codegen/template_util.py b/symforce/codegen/template_util.py index 3b1af6f2d..28fd61cae 100644 --- a/symforce/codegen/template_util.py +++ b/symforce/codegen/template_util.py @@ -21,6 +21,7 @@ CURRENT_DIR = Path(__file__).parent LCM_TEMPLATE_DIR = CURRENT_DIR / "lcm_templates" + class FileType(enum.Enum): CPP = enum.auto() PYTHON = enum.auto() @@ -58,6 +59,18 @@ def from_template_path(template_path: Path) -> FileType: ) return FileType.from_extension(parts[-2]) + # TODO(hayk): Move up to language-specific config or printer. (tag=centralize-language-diffs) + def comment_prefix(self) -> str: + """ + Return the comment prefix for this file type. + """ + if self in (FileType.CPP, FileType.CUDA, FileType.LCM): + return "//" + elif self in (FileType.PYTHON, FileType.PYTHON_INTERFACE): + return "#" + else: + raise NotImplementedError(f"Unknown comment prefix for {self}") + class RelEnvironment(jinja2.Environment): """ @@ -70,16 +83,16 @@ def join_path(self, template: T.Union[jinja2.Template, str], parent: str) -> str return os.path.normpath(os.path.join(os.path.dirname(parent), str(template))) -def add_preamble(source: str, name: Path, prefix: str) -> str: +def add_preamble(source: str, name: Path, comment_prefix: str) -> str: dashes = "-" * 77 return ( textwrap.dedent( f""" - {prefix} {dashes} - {prefix} This file was autogenerated by symforce from template: - {prefix} {name} - {prefix} Do NOT modify by hand. - {prefix} {dashes} + {comment_prefix} {dashes} + {comment_prefix} This file was autogenerated by symforce from template: + {comment_prefix} {name} + {comment_prefix} Do NOT modify by hand. + {comment_prefix} {dashes} """ ).lstrip() @@ -134,16 +147,10 @@ def render_template( filetype = FileType.from_template_path(Path(template_name)) - # TODO(hayk): Move up to language-specific config or printer. (tag=centralize-language-diffs) - prefix = ( - "//" - if filetype - in (FileType.CPP, FileType.CUDA, FileType.LCM, FileType.TYPESCRIPT) - else "#" - ) - template = jinja_env(template_dir).get_template(os.fspath(template_name)) - rendered_str = add_preamble(str(template.render(**data)), template_name, prefix=prefix) + rendered_str = add_preamble( + str(template.render(**data)), template_name, comment_prefix=filetype.comment_prefix() + ) if autoformat: # TODO(hayk): Move up to language-specific config or printer. (tag=centralize-language-diffs) From 06c982b682682dff940c2da6202f9ed78ef85c7d Mon Sep 17 00:00:00 2001 From: Hayk Martiros Date: Thu, 2 Jun 2022 19:34:23 +0200 Subject: [PATCH 11/18] Autoformat slightly generalized --- symforce/codegen/template_util.py | 48 ++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/symforce/codegen/template_util.py b/symforce/codegen/template_util.py index 28fd61cae..e417cd51d 100644 --- a/symforce/codegen/template_util.py +++ b/symforce/codegen/template_util.py @@ -59,7 +59,6 @@ def from_template_path(template_path: Path) -> FileType: ) return FileType.from_extension(parts[-2]) - # TODO(hayk): Move up to language-specific config or printer. (tag=centralize-language-diffs) def comment_prefix(self) -> str: """ Return the comment prefix for this file type. @@ -71,6 +70,34 @@ def comment_prefix(self) -> str: else: raise NotImplementedError(f"Unknown comment prefix for {self}") + def autoformat( + self, file_contents: str, template_name: T.Openable, output_path: T.Openable = None + ) -> str: + """ + Format code of this file type. + """ + # TODO(hayk): Move up to language-specific config or printer. This is quite an awkward + # place for auto-format logic, but I thought it was better centralized here than down below + # hidden in a function. We might want to somehow pass the config through to render a + # template so we can move things into the backend code. (tag=centralize-language-diffs) + if self in (FileType.CPP, FileType.CUDA): + # Come up with a fake filename to give to the formatter just for formatting purposes, + # even if this isn't being written to disk + if output_path is not None: + format_cpp_filename = os.path.basename(output_path) + else: + format_cpp_filename = os.fspath(template_name).replace(".jinja", "") + + return format_util.format_cpp( + file_contents, filename=str(CURRENT_DIR / format_cpp_filename) + ) + elif self == FileType.PYTHON: + return format_util.format_py(file_contents) + elif self == FileType.PYTHON_INTERFACE: + return format_util.format_pyi(file_contents) + else: + raise NotImplementedError(f"Unknown autoformatter for {self}") + class RelEnvironment(jinja2.Environment): """ @@ -153,22 +180,9 @@ def render_template( ) if autoformat: - # TODO(hayk): Move up to language-specific config or printer. (tag=centralize-language-diffs) - if filetype in (FileType.CPP, FileType.CUDA): - # Come up with a fake filename to give to the formatter just for formatting purposes, even - # if this isn't being written to disk - if output_path is not None: - format_cpp_filename = os.path.basename(output_path) - else: - format_cpp_filename = os.fspath(template_name).replace(".jinja", "") - - rendered_str = format_util.format_cpp( - rendered_str, filename=str(CURRENT_DIR / format_cpp_filename) - ) - elif filetype == FileType.PYTHON: - rendered_str = format_util.format_py(rendered_str) - elif filetype == FileType.PYTHON_INTERFACE: - rendered_str = format_util.format_pyi(rendered_str) + rendered_str = filetype.autoformat( + file_contents=rendered_str, template_name=template_name, output_path=output_path + ) if output_path: directory = os.path.dirname(output_path) From 460d36cdec5098e3a9f5521365ca213c27063d7e Mon Sep 17 00:00:00 2001 From: Hayk Martiros Date: Thu, 2 Jun 2022 20:32:35 +0200 Subject: [PATCH 12/18] Get rid of backend-specific work in codegen.py --- symforce/codegen/backends/cpp/cpp_config.py | 22 ++++++-- .../codegen/backends/python/python_config.py | 19 +++++-- symforce/codegen/codegen.py | 54 ++++--------------- symforce/codegen/codegen_config.py | 15 +++++- 4 files changed, 57 insertions(+), 53 deletions(-) diff --git a/symforce/codegen/backends/cpp/cpp_config.py b/symforce/codegen/backends/cpp/cpp_config.py index ff77a6eb5..65da31bfa 100644 --- a/symforce/codegen/backends/cpp/cpp_config.py +++ b/symforce/codegen/backends/cpp/cpp_config.py @@ -42,6 +42,24 @@ class CppConfig(CodegenConfig): zero_initialization_sparsity_threshold: float = 0.5 explicit_template_instantiation_types: T.Optional[T.Sequence[str]] = None + @classmethod + def backend_name(cls) -> str: + return "cpp" + + @classmethod + def template_dir(cls) -> Path: + return CURRENT_DIR / "templates" + + def templates_to_render(self, generated_file_name: str) -> T.List[T.Tuple[str, str]]: + # Generate code into a header (since the code is templated) + templates = [("function/FUNCTION.h.jinja", f"{generated_file_name}.h")] + + # Generate a cc file only if we need explicit instantiation. + if self.explicit_template_instantiation_types is not None: + templates.append(("function/FUNCTION.cc.jinja", f"{generated_file_name}.cc")) + + return templates + def printer(self) -> "sm.CodePrinter": # NOTE(hayk): Is there any benefit to this being lazy? from symforce.codegen.backends.cpp import cpp_code_printer @@ -51,10 +69,6 @@ def printer(self) -> "sm.CodePrinter": else: return cpp_code_printer.CppCodePrinter() - @classmethod - def template_dir(cls) -> Path: - return CURRENT_DIR / "templates" - @staticmethod def format_data_accessor(prefix: str, index: int) -> str: return f"{prefix}.Data()[{index}]" diff --git a/symforce/codegen/backends/python/python_config.py b/symforce/codegen/backends/python/python_config.py index c430a1fe9..c28e92388 100644 --- a/symforce/codegen/backends/python/python_config.py +++ b/symforce/codegen/backends/python/python_config.py @@ -6,6 +6,7 @@ from dataclasses import dataclass from pathlib import Path +from symforce import typing as T from symforce.codegen.codegen_config import CodegenConfig @@ -37,11 +38,21 @@ class PythonConfig(CodegenConfig): use_numba: bool = False matrix_is_1d: bool = True - def printer(self) -> "sm.CodePrinter": - from symforce.codegen.backends.python import python_code_printer - - return python_code_printer.PythonCodePrinter() + @classmethod + def backend_name(cls) -> str: + return "python" @classmethod def template_dir(cls) -> Path: return CURRENT_DIR / "templates" + + def templates_to_render(self, generated_file_name: str) -> T.List[T.Tuple[str, str]]: + return [ + ("function/FUNCTION.py.jinja", f"{generated_file_name}.py"), + ("function/__init__.py.jinja", "__init__.py"), + ] + + def printer(self) -> "sm.CodePrinter": + from symforce.codegen.backends.python import python_code_printer + + return python_code_printer.PythonCodePrinter() diff --git a/symforce/codegen/codegen.py b/symforce/codegen/codegen.py index 0795b61e9..f642271d7 100644 --- a/symforce/codegen/codegen.py +++ b/symforce/codegen/codegen.py @@ -396,53 +396,21 @@ def generate_function( template_data = dict(self.common_data(), spec=self) template_dir = self.config.template_dir() - # Generate the function - if isinstance(self.config, codegen.PythonConfig): - if skip_directory_nesting: - out_function_dir = output_dir - else: - out_function_dir = output_dir / "python" / "symforce" / namespace - - logger.info(f'Creating python function "{self.name}" at "{out_function_dir}"') - - templates.add( - template_dir / "function" / "FUNCTION.py.jinja", - out_function_dir / f"{generated_file_name}.py", - template_data, - ) - templates.add( - template_dir / "function" / "__init__.py.jinja", - out_function_dir / "__init__.py", - template_data, - ) - elif isinstance(self.config, codegen.CppConfig): - if skip_directory_nesting: - out_function_dir = output_dir - else: - out_function_dir = output_dir / "cpp" / "symforce" / namespace - - logger.info( - f'Creating C++ function "{python_util.snakecase_to_camelcase(self.name)}" at "{out_function_dir}"' - ) - - templates.add( - template_dir / "function" / "FUNCTION.h.jinja", - out_function_dir / f"{generated_file_name}.h", - template_data, - ) - - if self.config.explicit_template_instantiation_types is not None: - templates.add( - template_dir / "function" / "FUNCTION.cc.jinja", - out_function_dir / f"{generated_file_name}.cc", - template_data, - ) + backend_name = self.config.backend_name() + if skip_directory_nesting: + out_function_dir = output_dir + else: + out_function_dir = output_dir / backend_name / "symforce" / namespace + logger.info(f'Creating {backend_name} function from "{self.name}" at "{out_function_dir}"') - else: - raise NotImplementedError(f'Unknown config type: "{self.config}"') + # Get templates to render + for source, dest in self.config.templates_to_render(generated_file_name): + templates.add(template_dir / source, out_function_dir / dest, template_data) + # Render templates.render(autoformat=self.config.autoformat) + lcm_data = codegen_util.generate_lcm_types( lcm_type_dir=types_codegen_data["lcm_type_dir"], lcm_files=types_codegen_data["lcm_files"], diff --git a/symforce/codegen/codegen_config.py b/symforce/codegen/codegen_config.py index f4c5e828e..fdb488e63 100644 --- a/symforce/codegen/codegen_config.py +++ b/symforce/codegen/codegen_config.py @@ -36,9 +36,11 @@ class CodegenConfig: # TODO(hayk): Remove this parameter (by making everything 2D?) matrix_is_1d: bool = False - def printer(self) -> "sm.CodePrinter": + @classmethod + def backend_name(cls) -> str: """ - Return the code printer to use for this language. + String name for the backend. This should match the directory name in codegen/backends + and will be used to namespace by backend in generated code. """ raise NotImplementedError() @@ -49,6 +51,15 @@ def template_dir(cls) -> Path: """ raise NotImplementedError() + def templates_to_render(self, generated_file_name: str) -> T.List[T.Tuple[str, str]]: + raise NotImplementedError() + + def printer(self) -> "sm.CodePrinter": + """ + Return the code printer to use for this language. + """ + raise NotImplementedError() + # TODO(hayk): Move this into code printer. @staticmethod def format_data_accessor(prefix: str, index: int) -> str: From 3af09b8b86f5ac1d4875bc4ed763ef90b0bde7f3 Mon Sep 17 00:00:00 2001 From: Hayk Martiros Date: Thu, 2 Jun 2022 20:33:16 +0200 Subject: [PATCH 13/18] Add backend README --- symforce/codegen/backends/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 symforce/codegen/backends/README.md diff --git a/symforce/codegen/backends/README.md b/symforce/codegen/backends/README.md new file mode 100644 index 000000000..6417b7650 --- /dev/null +++ b/symforce/codegen/backends/README.md @@ -0,0 +1,14 @@ +# Code Generation Backends + +SymForce takes symbolic functions and generates runtime functions for multiple target backends. It aims to make it very straightforward to add new backends. + +The minimal steps to support a new backend are: + + 1. Choose a name for your backend (for example 'julia') and create a corresponding package in `symforce/codegen/backends`. + 2. Implement a subclass of SymPy `CodePrinter` that emits backend math code while traversing symbolic expressions. Sometimes SymPy already contains the backend and the best pattern is to inherit from it and customize as needed. The best way to do this is by looking at existing backends as examples. + 3. Implement a subclass of `symforce.codegen.codegen_config.CodegenConfig`. This is the spec that users pass to the `Codegen` object to use your backend. Again, see existing examples. Optionally import your config in `symforce/codegen/__init__.py`. + 4. Create a `templates` directory containing jinja templates that are used to generate the actual output files. They specify the high level structure and APIs around the math code. Your codegen config has a `templates_to_render` method that should match your templates. A typical start is just one function template. + 5. Add your backend's extensions to `FileType` in `symforce/codegen/template_util.py`, filling out relevant methods there. + 6. Add tests to `test/symforce_codegen_test.py`. + +This will result in being able to generate functions for your backend that deal with scalars and arrays, but the `sym` geometry and camera classes. To implement those, follow the C++ and Python examples. From 12b1cdb3639c53520c8caf525964cfc2f7ae85b8 Mon Sep 17 00:00:00 2001 From: Hayk Martiros Date: Thu, 2 Jun 2022 19:09:49 +0200 Subject: [PATCH 14/18] Add JavaScript backend --- symforce/codegen/__init__.py | 4 +- .../codegen/backends/javascript/__init__.py | 0 .../backends/javascript/javascript_config.py | 52 +++++ .../templates/function/FUNCTION.js.jinja | 23 +++ .../javascript/templates/util/util.jinja | 177 ++++++++++++++++++ .../printers/javascript_code_printer.py | 17 ++ symforce/codegen/template_util.py | 8 +- 7 files changed, 277 insertions(+), 4 deletions(-) create mode 100644 symforce/codegen/backends/javascript/__init__.py create mode 100644 symforce/codegen/backends/javascript/javascript_config.py create mode 100644 symforce/codegen/backends/javascript/templates/function/FUNCTION.js.jinja create mode 100644 symforce/codegen/backends/javascript/templates/util/util.jinja create mode 100644 symforce/codegen/printers/javascript_code_printer.py diff --git a/symforce/codegen/__init__.py b/symforce/codegen/__init__.py index 955730c0a..c38efe878 100644 --- a/symforce/codegen/__init__.py +++ b/symforce/codegen/__init__.py @@ -10,6 +10,8 @@ from .codegen import Codegen, LinearizationMode, GeneratedPaths from .codegen_config import CodegenConfig -# TODO(hayk): Do we want to explicitly enumerate all configs here? (tag=centralize-language-diffs) + +# TODO(hayk): Do we want to explicitly expose all configs here? (tag=centralize-language-diffs) from .backends.cpp.cpp_config import CppConfig +from .backends.javascript.javascript_config import JavascriptConfig from .backends.python.python_config import PythonConfig diff --git a/symforce/codegen/backends/javascript/__init__.py b/symforce/codegen/backends/javascript/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/symforce/codegen/backends/javascript/javascript_config.py b/symforce/codegen/backends/javascript/javascript_config.py new file mode 100644 index 000000000..16f62a583 --- /dev/null +++ b/symforce/codegen/backends/javascript/javascript_config.py @@ -0,0 +1,52 @@ +# ---------------------------------------------------------------------------- +# SymForce - Copyright 2022, Skydio, Inc. +# This source code is under the Apache 2.0 license found in the LICENSE file. +# ---------------------------------------------------------------------------- +from __future__ import annotations +from dataclasses import dataclass +from pathlib import Path + +from symforce import typing as T +from symforce.codegen.codegen_config import CodegenConfig + + +CURRENT_DIR = Path(__file__).parent + + +@dataclass +class JavascriptConfig(CodegenConfig): + """ + Code generation config for the javascript backend. + + Args: + doc_comment_line_prefix: Prefix applied to each line in a docstring + line_length: Maximum allowed line length in docstrings; used for formatting docstrings. + use_eigen_types: Use eigen_lcm types for vectors instead of lists + autoformat: Run a code formatter on the generated code + matrix_is_1D: geo.Matrix symbols get formatted as a 1D array + """ + + doc_comment_line_prefix: str = " * " + line_length: int = 100 + use_eigen_types: bool = True + # NOTE(hayk): Add JS autoformatter + autoformat: bool = False + matrix_is_1d: bool = True + + @classmethod + def backend_name(cls) -> str: + return "javascript" + + @classmethod + def template_dir(cls) -> Path: + return CURRENT_DIR / "templates" + + def templates_to_render(self, generated_file_name: str) -> T.List[T.Tuple[str, str]]: + return [ + ("function/FUNCTION.js.jinja", f"{generated_file_name}.js"), + ] + + def printer(self) -> "sm.CodePrinter": + from symforce.codegen.printers import javascript_code_printer + + return javascript_code_printer.JavascriptCodePrinter() diff --git a/symforce/codegen/backends/javascript/templates/function/FUNCTION.js.jinja b/symforce/codegen/backends/javascript/templates/function/FUNCTION.js.jinja new file mode 100644 index 000000000..ea65591a0 --- /dev/null +++ b/symforce/codegen/backends/javascript/templates/function/FUNCTION.js.jinja @@ -0,0 +1,23 @@ +{# ------------------------------------------------------------------------- #} +{# Function codegen template for Javascript #} +{# ------------------------------------------------------------------------- #} +{%- import "../util/util.jinja" as util with context -%} + +{% for typename in ('Rot3', 'Rot2', 'Pose3', 'Pose2') %} +{% if typename in spec.types_included %} +{# #include #} +{% endif %} +{% endfor %} + +{%- for name in spec.inputs.keys() | list + spec.outputs.keys() | list %} +{% if name in spec.typenames_dict %} +{# #include #} +{% endif %} +{% endfor %} + +{% if spec.docstring %} +{{ util.print_docstring(spec.docstring) }} +{% endif %} +{{ util.function_declaration(spec) -}} { +{{ util.expr_code(spec) }} +} diff --git a/symforce/codegen/backends/javascript/templates/util/util.jinja b/symforce/codegen/backends/javascript/templates/util/util.jinja new file mode 100644 index 000000000..f8784ab2b --- /dev/null +++ b/symforce/codegen/backends/javascript/templates/util/util.jinja @@ -0,0 +1,177 @@ +{# ------------------------------------------------------------------------- #} +{# Utilities for Javascript code generation templates. #} +{# ------------------------------------------------------------------------- #} + +{# Vector type of a given dimension + # + # Args: + # dim (int): + #} +{%- macro vector_type(dim) -%} +Eigen::Matrix +{%- endmacro -%} + +{# Matrix type of a given dimension + # + # Args: + # rows (int): + # cols (int): + # is_input (bool): Is this an input argument or return value? + #} +{%- macro matrix_type(rows, cols, is_input) -%} + {%- if cols == 1 -%} + T.{%- if is_input -%}Sequence{%- else -%}List{%- endif -%}[float] + {%- else -%} + numpy.ndarray + {%- endif -%} +{%- endmacro -%} + +{# ------------------------------------------------------------------------- #} + +{# Convert a class to the emitted string + # + # Args: + # T_or_value (type or Element): + # name (str): Name in case type is a generated struct + # is_input (bool): Is this an input argument or return value? + #} +{%- macro format_typename(T_or_value, name, is_input) %} + {%- set T = python_util.get_type(T_or_value) -%} + {%- if T.__name__ == 'Symbol' or is_symbolic(T_or_value) -%} + float + {%- elif T.__name__ == 'NoneType' -%} + None + {%- elif issubclass(T, Matrix) -%} + {{ matrix_type(T_or_value.shape[0], T_or_value.shape[1], is_input) }} + {%- elif issubclass(T, Values) -%} + {#- TODO(aaron): We don't currently know where to import lcmtypes from or what they should be + # called, at some point we should fix this and do something like + # {{ spec.namespaces_dict[name] }}.{{ spec.typenames_dict[name] }} + -#} + T.Any + {%- elif is_sequence(T_or_value) -%} + {%- if is_input -%} + T.Sequence[{{ format_typename(T_or_value[0], name, is_input) }}] + {%- else -%} + T.List[float] + {%- endif -%} + {%- else -%} + {%- if "geo" in T.__module__ or "cam" in T.__module__ -%} + sym. + {%- endif -%} + {{- T.__name__ -}} + {%- endif -%} +{% endmacro -%} + +{# ------------------------------------------------------------------------- #} + +{# Get the type of the object in the ouput Values with key given by spec.return_key + # + # Args: + # spec (Codegen): + #} +{%- macro get_return_type(spec) %} + {%- if spec.outputs.keys() | length == 1 -%} + {%- set name, type = spec.outputs.items() | first -%} + {{ format_typename(type, name, is_input=False) }} + {%- elif spec.outputs -%} + T.Tuple[ + {%- for name, type in spec.outputs.items() -%} + {{ format_typename(type, name, is_input=False) }}{% if not loop.last %}, {% endif %} + {%- endfor -%}] + {%- else -%} + None + {%- endif -%} +{% endmacro -%} + +{# ------------------------------------------------------------------------- #} + + {# Format function docstring + # + # Args: + # docstring (str): + #} +{% macro print_docstring(docstring) %} +{%- if docstring %} + +/* +{%- for line in docstring.split('\n') %} +*{{ ' {}'.format(line).rstrip() }} +{% endfor -%} +*/ +{%- endif -%} +{% endmacro %} + +{# ------------------------------------------------------------------------- #} + +{# Generate function declaration + # + # Args: + # spec (Codegen): + #} +{%- macro function_declaration(spec) -%} +function {{ camelcase_to_snakecase(spec.name) }}( + {%- for name in spec.inputs.keys() -%} + {{ name }}{% if not loop.last %}, {% endif %} + {%- endfor -%}) +{% endmacro -%} + +{# ------------------------------------------------------------------------- #} + +{# Generate inner code for computing the given expression. + # + # Args: + # spec (Codegen): + #} +{% macro expr_code(spec) %} + // Total ops: {{ spec.print_code_results.total_ops }} + + // Input arrays + {% for name, type in spec.inputs.items() %} + {% set T = python_util.get_type(type) %} + {% if not issubclass(T, Values) and not issubclass(T, Matrix) and not is_symbolic(type) and not is_sequence(type) %} + _{{ name }} = {{ name }}.data + {% endif %} + {% endfor %} + + // Intermediate terms ({{ spec.print_code_results.intermediate_terms | length }}) + {% for lhs, rhs in spec.print_code_results.intermediate_terms %} + const {{ lhs }} = {{ rhs }}; + {% endfor %} + + // Output terms ({{ spec.outputs.items() | length }}) + {% for name, type, terms in spec.print_code_results.dense_terms %} + {%- set T = python_util.get_type(type) -%} + {% if issubclass(T, Matrix) and type.shape[1] > 1 %} + {% set rows = type.shape[0] %} + {% set cols = type.shape[1] %} + _{{ name }} = numpy.zeros(({{ rows }}, {{ cols }})) + {% set ns = namespace(iter=0) %} + {% for i in range(rows) %} + {% for j in range(cols) %} + _{{ name }}[{{ i }}, {{ j }}] = {{ terms[ns.iter][1] }} + {% set ns.iter = ns.iter + 1 %} + {% endfor %} + {% endfor %} + {% elif not is_symbolic(type) %} + {% set dims = ops.StorageOps.storage_dim(type) %} + let _{{name}} = Array({{ dims }}); + {% for i in range(dims) %} + _{{ name }}[{{ i }}] = {{ terms[i][1] }}; + {% endfor %} + + {% else %} + const _{{name}} = {{ terms[0][1] }}; + {% endif %} + {% endfor %} + return + {%- for name, type in spec.outputs.items() %} + {% set T = python_util.get_type(type) %} + {% if issubclass(T, (Matrix, Values)) or is_sequence(type) or is_symbolic(type) %} + _{{name}} + {%- else %} + sym.{{T.__name__}}.from_storage(_{{name}}) + {%- endif %} + {%- if not loop.last %}, {% endif %} + {%- endfor -%} +{% endmacro %} diff --git a/symforce/codegen/printers/javascript_code_printer.py b/symforce/codegen/printers/javascript_code_printer.py new file mode 100644 index 000000000..66d4b031d --- /dev/null +++ b/symforce/codegen/printers/javascript_code_printer.py @@ -0,0 +1,17 @@ +from sympy.printing.jscode import JavascriptCodePrinter as SympyJsCodePrinter + +from symforce import typing as T + +# Everything in this file is SymPy, not SymEngine (even when SymForce is on the SymEngine backend) +import sympy + + +class JavascriptCodePrinter(SympyJsCodePrinter): + """ + Symforce customized code printer for Javascript. Modifies the Sympy printing + behavior for codegen compatibility and efficiency. + """ + + def __init__(self, settings: T.Dict[str, T.Any] = None) -> None: + settings = dict(settings or {},) + super().__init__(settings) diff --git a/symforce/codegen/template_util.py b/symforce/codegen/template_util.py index e417cd51d..b3b6abfab 100644 --- a/symforce/codegen/template_util.py +++ b/symforce/codegen/template_util.py @@ -29,10 +29,12 @@ class FileType(enum.Enum): CUDA = enum.auto() LCM = enum.auto() MAKEFILE = enum.auto() + JAVASCRIPT = enum.auto() TYPESCRIPT = enum.auto() @staticmethod def from_extension(extension: str) -> FileType: + # TODO(hayk): Move up to language-specific directory. (tag=centralize-language-diffs) if extension in ("c", "cpp", "cxx", "cc", "tcc", "h", "hpp", "hxx", "hh"): return FileType.CPP elif extension in ("cu", "cuh"): @@ -45,8 +47,8 @@ def from_extension(extension: str) -> FileType: return FileType.LCM elif extension == "Makefile": return FileType.MAKEFILE - elif extension == "ts": - return FileType.TYPESCRIPT + elif extension == "js": + return FileType.JAVASCRIPT else: raise ValueError(f"Could not get FileType from extension {extension}") @@ -63,7 +65,7 @@ def comment_prefix(self) -> str: """ Return the comment prefix for this file type. """ - if self in (FileType.CPP, FileType.CUDA, FileType.LCM): + if self in (FileType.CPP, FileType.CUDA, FileType.LCM, FileType.JAVASCRIPT): return "//" elif self in (FileType.PYTHON, FileType.PYTHON_INTERFACE): return "#" From e89cad0aba6038117d64f72551d62e474ce48e9e Mon Sep 17 00:00:00 2001 From: Hayk Martiros Date: Sun, 5 Jun 2022 05:46:06 -0700 Subject: [PATCH 15/18] Address comments --- symforce/codegen/__init__.py | 1 - symforce/codegen/backends/README.md | 6 +++--- symforce/codegen/backends/cpp/cpp_config.py | 2 +- .../codegen/backends/python/python_config.py | 2 +- symforce/codegen/codegen_config.py | 21 +++++++++++++------ 5 files changed, 20 insertions(+), 12 deletions(-) diff --git a/symforce/codegen/__init__.py b/symforce/codegen/__init__.py index 955730c0a..3c5b22286 100644 --- a/symforce/codegen/__init__.py +++ b/symforce/codegen/__init__.py @@ -10,6 +10,5 @@ from .codegen import Codegen, LinearizationMode, GeneratedPaths from .codegen_config import CodegenConfig -# TODO(hayk): Do we want to explicitly enumerate all configs here? (tag=centralize-language-diffs) from .backends.cpp.cpp_config import CppConfig from .backends.python.python_config import PythonConfig diff --git a/symforce/codegen/backends/README.md b/symforce/codegen/backends/README.md index 6417b7650..7efa4bbf1 100644 --- a/symforce/codegen/backends/README.md +++ b/symforce/codegen/backends/README.md @@ -1,14 +1,14 @@ # Code Generation Backends -SymForce takes symbolic functions and generates runtime functions for multiple target backends. It aims to make it very straightforward to add new backends. +SymForce takes symbolic functions and generates runtime functions for multiple target backends. It is quite straightforward to add new backends. Before you do this, you should be familiar with [SymPy printing](https://docs.sympy.org/latest/modules/printing.html) for getting code from symbolic expressions, and with [Jinja templating](https://realpython.com/primer-on-jinja-templating/) for rendering output files. The minimal steps to support a new backend are: 1. Choose a name for your backend (for example 'julia') and create a corresponding package in `symforce/codegen/backends`. - 2. Implement a subclass of SymPy `CodePrinter` that emits backend math code while traversing symbolic expressions. Sometimes SymPy already contains the backend and the best pattern is to inherit from it and customize as needed. The best way to do this is by looking at existing backends as examples. + 2. Implement a subclass of `sympy.CodePrinter` that emits backend math code while traversing symbolic expressions. Sometimes SymPy already contains the backend and the best pattern is to inherit from it and customize as needed. The best way to do this is by looking at existing backends as examples. 3. Implement a subclass of `symforce.codegen.codegen_config.CodegenConfig`. This is the spec that users pass to the `Codegen` object to use your backend. Again, see existing examples. Optionally import your config in `symforce/codegen/__init__.py`. 4. Create a `templates` directory containing jinja templates that are used to generate the actual output files. They specify the high level structure and APIs around the math code. Your codegen config has a `templates_to_render` method that should match your templates. A typical start is just one function template. 5. Add your backend's extensions to `FileType` in `symforce/codegen/template_util.py`, filling out relevant methods there. 6. Add tests to `test/symforce_codegen_test.py`. -This will result in being able to generate functions for your backend that deal with scalars and arrays, but the `sym` geometry and camera classes. To implement those, follow the C++ and Python examples. +This will result in being able to generate functions for your backend that deal with scalars and arrays, but the `sym` geometry and camera classes. To implement those, follow the examples in `geo_package_codegen` and `cam_package_codegen`. diff --git a/symforce/codegen/backends/cpp/cpp_config.py b/symforce/codegen/backends/cpp/cpp_config.py index 65da31bfa..0f25a20ed 100644 --- a/symforce/codegen/backends/cpp/cpp_config.py +++ b/symforce/codegen/backends/cpp/cpp_config.py @@ -60,7 +60,7 @@ def templates_to_render(self, generated_file_name: str) -> T.List[T.Tuple[str, s return templates - def printer(self) -> "sm.CodePrinter": + def printer(self) -> "sympy.CodePrinter": # NOTE(hayk): Is there any benefit to this being lazy? from symforce.codegen.backends.cpp import cpp_code_printer diff --git a/symforce/codegen/backends/python/python_config.py b/symforce/codegen/backends/python/python_config.py index c28e92388..27e10df98 100644 --- a/symforce/codegen/backends/python/python_config.py +++ b/symforce/codegen/backends/python/python_config.py @@ -52,7 +52,7 @@ def templates_to_render(self, generated_file_name: str) -> T.List[T.Tuple[str, s ("function/__init__.py.jinja", "__init__.py"), ] - def printer(self) -> "sm.CodePrinter": + def printer(self) -> "sympy.CodePrinter": from symforce.codegen.backends.python import python_code_printer return python_code_printer.PythonCodePrinter() diff --git a/symforce/codegen/codegen_config.py b/symforce/codegen/codegen_config.py index fdb488e63..6ac5ac32a 100644 --- a/symforce/codegen/codegen_config.py +++ b/symforce/codegen/codegen_config.py @@ -3,6 +3,7 @@ # This source code is under the Apache 2.0 license found in the LICENSE file. # ---------------------------------------------------------------------------- from __future__ import annotations +from abc import abstractmethod from dataclasses import dataclass from pathlib import Path @@ -37,28 +38,36 @@ class CodegenConfig: matrix_is_1d: bool = False @classmethod + @abstractmethod def backend_name(cls) -> str: """ String name for the backend. This should match the directory name in codegen/backends and will be used to namespace by backend in generated code. """ - raise NotImplementedError() + pass @classmethod + @abstractmethod def template_dir(cls) -> Path: """ Directory for jinja templates. """ - raise NotImplementedError() + pass + @abstractmethod def templates_to_render(self, generated_file_name: str) -> T.List[T.Tuple[str, str]]: - raise NotImplementedError() + """ + Given a single symbolic function's filename, provide one or more Jinja templates to + render and the relative output paths where they should go. + """ + pass - def printer(self) -> "sm.CodePrinter": + @abstractmethod + def printer(self) -> "sympy.CodePrinter": """ - Return the code printer to use for this language. + Return an instance of the code printer to use for this language. """ - raise NotImplementedError() + pass # TODO(hayk): Move this into code printer. @staticmethod From dc803873f27b5256159fe30777b27be1a418a3f6 Mon Sep 17 00:00:00 2001 From: Hayk Martiros Date: Sun, 5 Jun 2022 07:09:24 -0700 Subject: [PATCH 16/18] Remove matrix_is_1d config and add a format_matrix_accessor method This fixes Python not being able to properly generate functions that accept 1D and 2D matrix arguments. --- symforce/codegen/backends/cpp/cpp_config.py | 6 ++++++ symforce/codegen/backends/python/python_config.py | 7 ++++++- symforce/codegen/codegen_config.py | 12 +++++++++--- symforce/codegen/codegen_util.py | 13 +++++++++---- 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/symforce/codegen/backends/cpp/cpp_config.py b/symforce/codegen/backends/cpp/cpp_config.py index 0f25a20ed..917b0ff9a 100644 --- a/symforce/codegen/backends/cpp/cpp_config.py +++ b/symforce/codegen/backends/cpp/cpp_config.py @@ -72,3 +72,9 @@ def printer(self) -> "sympy.CodePrinter": @staticmethod def format_data_accessor(prefix: str, index: int) -> str: return f"{prefix}.Data()[{index}]" + + @staticmethod + def format_matrix_accessor(key: str, i: int, j: int = None) -> str: + if j is None: + return f"{key}({i}, {0})" + return f"{key}({i}, {j})" diff --git a/symforce/codegen/backends/python/python_config.py b/symforce/codegen/backends/python/python_config.py index 27e10df98..d92891b00 100644 --- a/symforce/codegen/backends/python/python_config.py +++ b/symforce/codegen/backends/python/python_config.py @@ -36,7 +36,6 @@ class PythonConfig(CodegenConfig): line_length: int = 100 use_eigen_types: bool = True use_numba: bool = False - matrix_is_1d: bool = True @classmethod def backend_name(cls) -> str: @@ -56,3 +55,9 @@ def printer(self) -> "sympy.CodePrinter": from symforce.codegen.backends.python import python_code_printer return python_code_printer.PythonCodePrinter() + + @staticmethod + def format_matrix_accessor(key: str, i: int, j: int = None) -> str: + if j is None: + return f"{key}[{i}]" + return f"{key}[{i}, {j}]" diff --git a/symforce/codegen/codegen_config.py b/symforce/codegen/codegen_config.py index 6ac5ac32a..ae233225e 100644 --- a/symforce/codegen/codegen_config.py +++ b/symforce/codegen/codegen_config.py @@ -24,7 +24,6 @@ class CodegenConfig: use_eigen_types: Use eigen_lcm types for vectors instead of lists autoformat: Run a code formatter on the generated code cse_optimizations: Optimizations argument to pass to sm.cse - matrix_is_1d: Whether geo.Matrix symbols get formatted as 1D """ doc_comment_line_prefix: str @@ -34,8 +33,6 @@ class CodegenConfig: cse_optimizations: T.Optional[ T.Union[T.Literal["basic"], T.Sequence[T.Tuple[T.Callable, T.Callable]]] ] = None - # TODO(hayk): Remove this parameter (by making everything 2D?) - matrix_is_1d: bool = False @classmethod @abstractmethod @@ -76,3 +73,12 @@ def format_data_accessor(prefix: str, index: int) -> str: Format data for accessing a data array in code. """ return f"{prefix}.data[{index}]" + + @staticmethod + @abstractmethod + def format_matrix_accessor(key: str, i: int, j: int = None) -> str: + """ + Format accessor for 2D matrices. If j is None, it is a 1D vector type, which for some + languages is accessed with 2D indices and in some with 1D. + """ + pass diff --git a/symforce/codegen/codegen_util.py b/symforce/codegen/codegen_util.py index 2cdebfae2..08caa3f85 100644 --- a/symforce/codegen/codegen_util.py +++ b/symforce/codegen/codegen_util.py @@ -341,9 +341,12 @@ def get_formatted_list( formatted_symbols = [sm.Symbol(key)] flattened_value = [value] elif issubclass(arg_cls, geo.Matrix): - if config.matrix_is_1d: - # TODO(nathan): Not sure this works for 2D matrices. Get rid of this. - formatted_symbols = [sm.Symbol(f"{key}[{j}]") for j in range(storage_dim)] + if value.shape[1] == 1: + # Pass in None as the second index for 1D matrices, so the per-backend config can + # decide whether to use 1D or 2D indexing, depending on the language. + formatted_symbols = [] + for i in range(value.shape[0]): + formatted_symbols.append(sm.Symbol(config.format_matrix_accessor(key, i, None))) else: # NOTE(brad): The order of the symbols must match the storage order of geo.Matrix # (as returned by geo.Matrix.to_storage). Hence, if there storage order were @@ -352,7 +355,9 @@ def get_formatted_list( formatted_symbols = [] for j in range(value.shape[1]): for i in range(value.shape[0]): - formatted_symbols.append(sm.Symbol(f"{key}({i}, {j})")) + formatted_symbols.append( + sm.Symbol(config.format_matrix_accessor(key, i, j)) + ) flattened_value = ops.StorageOps.to_storage(value) From b9590257eca6e6b062dc4c6ccefcb643afb00863 Mon Sep 17 00:00:00 2001 From: Hayk Martiros Date: Sun, 5 Jun 2022 07:26:57 -0700 Subject: [PATCH 17/18] Handle matrices in javascript as arrays and create a simple test --- .../backends/javascript/javascript_config.py | 8 +- .../templates/function/FUNCTION.js.jinja | 12 -- .../javascript/templates/util/util.jinja | 106 +++--------------- .../printers/javascript_code_printer.py | 8 +- test/symforce_javascript_codegen_test.py | 52 +++++++++ test/test_data/javascript_codegen_example.js | 52 +++++++++ 6 files changed, 129 insertions(+), 109 deletions(-) create mode 100644 test/symforce_javascript_codegen_test.py create mode 100644 test/test_data/javascript_codegen_example.js diff --git a/symforce/codegen/backends/javascript/javascript_config.py b/symforce/codegen/backends/javascript/javascript_config.py index 16f62a583..890bb9d41 100644 --- a/symforce/codegen/backends/javascript/javascript_config.py +++ b/symforce/codegen/backends/javascript/javascript_config.py @@ -2,6 +2,7 @@ # SymForce - Copyright 2022, Skydio, Inc. # This source code is under the Apache 2.0 license found in the LICENSE file. # ---------------------------------------------------------------------------- + from __future__ import annotations from dataclasses import dataclass from pathlib import Path @@ -31,7 +32,6 @@ class JavascriptConfig(CodegenConfig): use_eigen_types: bool = True # NOTE(hayk): Add JS autoformatter autoformat: bool = False - matrix_is_1d: bool = True @classmethod def backend_name(cls) -> str: @@ -50,3 +50,9 @@ def printer(self) -> "sm.CodePrinter": from symforce.codegen.printers import javascript_code_printer return javascript_code_printer.JavascriptCodePrinter() + + @staticmethod + def format_matrix_accessor(key: str, i: int, j: int = None) -> str: + if j is None: + return f"{key}[{i}]" + return f"{key}[{i}][{j}]" diff --git a/symforce/codegen/backends/javascript/templates/function/FUNCTION.js.jinja b/symforce/codegen/backends/javascript/templates/function/FUNCTION.js.jinja index ea65591a0..75db3630c 100644 --- a/symforce/codegen/backends/javascript/templates/function/FUNCTION.js.jinja +++ b/symforce/codegen/backends/javascript/templates/function/FUNCTION.js.jinja @@ -3,18 +3,6 @@ {# ------------------------------------------------------------------------- #} {%- import "../util/util.jinja" as util with context -%} -{% for typename in ('Rot3', 'Rot2', 'Pose3', 'Pose2') %} -{% if typename in spec.types_included %} -{# #include #} -{% endif %} -{% endfor %} - -{%- for name in spec.inputs.keys() | list + spec.outputs.keys() | list %} -{% if name in spec.typenames_dict %} -{# #include #} -{% endif %} -{% endfor %} - {% if spec.docstring %} {{ util.print_docstring(spec.docstring) }} {% endif %} diff --git a/symforce/codegen/backends/javascript/templates/util/util.jinja b/symforce/codegen/backends/javascript/templates/util/util.jinja index f8784ab2b..4ae07e23e 100644 --- a/symforce/codegen/backends/javascript/templates/util/util.jinja +++ b/symforce/codegen/backends/javascript/templates/util/util.jinja @@ -2,88 +2,6 @@ {# Utilities for Javascript code generation templates. #} {# ------------------------------------------------------------------------- #} -{# Vector type of a given dimension - # - # Args: - # dim (int): - #} -{%- macro vector_type(dim) -%} -Eigen::Matrix -{%- endmacro -%} - -{# Matrix type of a given dimension - # - # Args: - # rows (int): - # cols (int): - # is_input (bool): Is this an input argument or return value? - #} -{%- macro matrix_type(rows, cols, is_input) -%} - {%- if cols == 1 -%} - T.{%- if is_input -%}Sequence{%- else -%}List{%- endif -%}[float] - {%- else -%} - numpy.ndarray - {%- endif -%} -{%- endmacro -%} - -{# ------------------------------------------------------------------------- #} - -{# Convert a class to the emitted string - # - # Args: - # T_or_value (type or Element): - # name (str): Name in case type is a generated struct - # is_input (bool): Is this an input argument or return value? - #} -{%- macro format_typename(T_or_value, name, is_input) %} - {%- set T = python_util.get_type(T_or_value) -%} - {%- if T.__name__ == 'Symbol' or is_symbolic(T_or_value) -%} - float - {%- elif T.__name__ == 'NoneType' -%} - None - {%- elif issubclass(T, Matrix) -%} - {{ matrix_type(T_or_value.shape[0], T_or_value.shape[1], is_input) }} - {%- elif issubclass(T, Values) -%} - {#- TODO(aaron): We don't currently know where to import lcmtypes from or what they should be - # called, at some point we should fix this and do something like - # {{ spec.namespaces_dict[name] }}.{{ spec.typenames_dict[name] }} - -#} - T.Any - {%- elif is_sequence(T_or_value) -%} - {%- if is_input -%} - T.Sequence[{{ format_typename(T_or_value[0], name, is_input) }}] - {%- else -%} - T.List[float] - {%- endif -%} - {%- else -%} - {%- if "geo" in T.__module__ or "cam" in T.__module__ -%} - sym. - {%- endif -%} - {{- T.__name__ -}} - {%- endif -%} -{% endmacro -%} - -{# ------------------------------------------------------------------------- #} - -{# Get the type of the object in the ouput Values with key given by spec.return_key - # - # Args: - # spec (Codegen): - #} -{%- macro get_return_type(spec) %} - {%- if spec.outputs.keys() | length == 1 -%} - {%- set name, type = spec.outputs.items() | first -%} - {{ format_typename(type, name, is_input=False) }} - {%- elif spec.outputs -%} - T.Tuple[ - {%- for name, type in spec.outputs.items() -%} - {{ format_typename(type, name, is_input=False) }}{% if not loop.last %}, {% endif %} - {%- endfor -%}] - {%- else -%} - None - {%- endif -%} -{% endmacro -%} - {# ------------------------------------------------------------------------- #} {# Format function docstring @@ -145,33 +63,35 @@ function {{ camelcase_to_snakecase(spec.name) }}( {% if issubclass(T, Matrix) and type.shape[1] > 1 %} {% set rows = type.shape[0] %} {% set cols = type.shape[1] %} - _{{ name }} = numpy.zeros(({{ rows }}, {{ cols }})) + let _{{ name }} = [...Array({{ rows }})].map(e => Array({{ cols }})); {% set ns = namespace(iter=0) %} {% for i in range(rows) %} {% for j in range(cols) %} - _{{ name }}[{{ i }}, {{ j }}] = {{ terms[ns.iter][1] }} + _{{ name }}[{{ i }}][{{ j }}] = {{ terms[ns.iter][1] }}; {% set ns.iter = ns.iter + 1 %} {% endfor %} {% endfor %} {% elif not is_symbolic(type) %} {% set dims = ops.StorageOps.storage_dim(type) %} - let _{{name}} = Array({{ dims }}); + let _{{name}} = new Array({{ dims }}); {% for i in range(dims) %} _{{ name }}[{{ i }}] = {{ terms[i][1] }}; {% endfor %} - {% else %} const _{{name}} = {{ terms[0][1] }}; {% endif %} + {% endfor %} - return - {%- for name, type in spec.outputs.items() %} + return { + {% for name, type in spec.outputs.items() %} {% set T = python_util.get_type(type) %} {% if issubclass(T, (Matrix, Values)) or is_sequence(type) or is_symbolic(type) %} - _{{name}} + {{ name }}: _{{name}} {%- else %} - sym.{{T.__name__}}.from_storage(_{{name}}) - {%- endif %} - {%- if not loop.last %}, {% endif %} - {%- endfor -%} + {{ name }}: sym.{{T.__name__}}.from_storage(_{{name}}) + {% endif %} + {% if not loop.last %}, {% endif %} + + {% endfor %} + }; {% endmacro %} diff --git a/symforce/codegen/printers/javascript_code_printer.py b/symforce/codegen/printers/javascript_code_printer.py index 66d4b031d..0a41842fa 100644 --- a/symforce/codegen/printers/javascript_code_printer.py +++ b/symforce/codegen/printers/javascript_code_printer.py @@ -1,10 +1,12 @@ +# ---------------------------------------------------------------------------- +# SymForce - Copyright 2022, Skydio, Inc. +# This source code is under the Apache 2.0 license found in the LICENSE file. +# ---------------------------------------------------------------------------- + from sympy.printing.jscode import JavascriptCodePrinter as SympyJsCodePrinter from symforce import typing as T -# Everything in this file is SymPy, not SymEngine (even when SymForce is on the SymEngine backend) -import sympy - class JavascriptCodePrinter(SympyJsCodePrinter): """ diff --git a/test/symforce_javascript_codegen_test.py b/test/symforce_javascript_codegen_test.py new file mode 100644 index 000000000..ea3aa3dce --- /dev/null +++ b/test/symforce_javascript_codegen_test.py @@ -0,0 +1,52 @@ +# ---------------------------------------------------------------------------- +# SymForce - Copyright 2022, Skydio, Inc. +# This source code is under the Apache 2.0 license found in the LICENSE file. +# ---------------------------------------------------------------------------- +from pathlib import Path + +from symforce import codegen +from symforce import geo +from symforce import logger +from symforce import path_util +from symforce import sympy as sm +from symforce import typing as T +from symforce.test_util import TestCase + + +class SymforceJavascriptCodegenTest(TestCase): + """ + Simple test for the Javascript codegen backend. + """ + + @staticmethod + def javascript_codegen_example( + a: T.Scalar, b: geo.V2, c: geo.M22, epsilon: T.Scalar + ) -> T.Tuple[geo.V3, geo.M22, T.Scalar]: + return ( + geo.V3(a + c[0], sm.sin(b[0]) ** a, b[1] ** 2 / (a - b[0] - c[1] - epsilon)), + geo.M22( + [[-sm.atan2(b[1], a), (a + b[0]) / c[1, :].norm(epsilon=epsilon)], [1, c[1, 0]]] + ), + a ** 2, + ) + + def test_javascript_codegen(self) -> None: + for config in (codegen.PythonConfig(), codegen.CppConfig(), codegen.JavascriptConfig()): + cg = codegen.Codegen.function( + func=self.javascript_codegen_example, + config=config, + output_names=["d", "e", "f"], + ) + out_path = cg.generate_function().generated_files[0] + + logger.debug(Path(out_path).read_text()) + + if config.backend_name() == "javascript": + self.compare_or_update_file( + path=path_util.symforce_dir() / "test" / "test_data" / out_path.name, + new_file=out_path, + ) + + +if __name__ == "__main__": + TestCase.main() diff --git a/test/test_data/javascript_codegen_example.js b/test/test_data/javascript_codegen_example.js new file mode 100644 index 000000000..3b6460a5b --- /dev/null +++ b/test/test_data/javascript_codegen_example.js @@ -0,0 +1,52 @@ +// ----------------------------------------------------------------------------- +// This file was autogenerated by symforce from template: +// backends/javascript/templates/function/FUNCTION.js.jinja +// Do NOT modify by hand. +// ----------------------------------------------------------------------------- + + +/** +* This function was autogenerated from a symbolic function. Do not modify by hand. +* +* Symbolic function: javascript_codegen_example +* +* Args: +* a: Scalar +* b: Matrix21 +* c: Matrix22 +* epsilon: Scalar +* +* Outputs: +* d: Matrix31 +* e: Matrix22 +* f: Scalar +*/ +function javascript_codegen_example(a, b, c, epsilon) +{ + // Total ops: 18 + + // Input arrays + + // Intermediate terms (0) + + // Output terms (3) + let _d = new Array(3); + _d[0] = a + c[0][0]; + _d[1] = Math.pow(Math.sin(b[0]), a); + _d[2] = Math.pow(b[1], 2)/(a - b[0] - c[0][1] - epsilon); + + let _e = [...Array(2)].map(e => Array(2)); + _e[0][0] = -Math.atan2(b[1], a); + _e[0][1] = 1; + _e[1][0] = (a + b[0])/Math.sqrt(Math.pow(c[1][0], 2) + Math.pow(c[1][1], 2) + epsilon); + _e[1][1] = c[1][0]; + + const _f = Math.pow(a, 2); + + return { + d: _d, + e: _e, + f: _f + }; + +} From b4476a052cbee3b0e411300ae4d1d915d989e862 Mon Sep 17 00:00:00 2001 From: Hayk Martiros Date: Fri, 1 Jul 2022 19:19:46 -0400 Subject: [PATCH 18/18] Minor comments --- symforce/codegen/codegen.py | 1 - symforce/codegen/codegen_util.py | 6 +++--- symforce/codegen/geo_package_codegen.py | 2 +- symforce/codegen/template_util.py | 2 ++ 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/symforce/codegen/codegen.py b/symforce/codegen/codegen.py index 803c6f11f..a7905ff3c 100644 --- a/symforce/codegen/codegen.py +++ b/symforce/codegen/codegen.py @@ -22,7 +22,6 @@ from symforce import python_util from symforce import typing as T from symforce.values import Values -from symforce import codegen from symforce.codegen import template_util from symforce.codegen import codegen_util from symforce.codegen import codegen_config diff --git a/symforce/codegen/codegen_util.py b/symforce/codegen/codegen_util.py index 2ee782904..ac078679c 100644 --- a/symforce/codegen/codegen_util.py +++ b/symforce/codegen/codegen_util.py @@ -345,10 +345,10 @@ def get_formatted_list( # decide whether to use 1D or 2D indexing, depending on the language. formatted_symbols = [] for i in range(value.shape[0]): - formatted_symbols.append(sm.Symbol(config.format_matrix_accessor(key, i, None))) + formatted_symbols.append(sf.Symbol(config.format_matrix_accessor(key, i, None))) else: - # NOTE(brad): The order of the symbols must match the storage order of geo.Matrix - # (as returned by geo.Matrix.to_storage). Hence, if there storage order were + # NOTE(brad): The order of the symbols must match the storage order of sf.Matrix + # (as returned by sf.Matrix.to_storage). Hence, if there storage order were # changed to, say, row major, the below for loops would have to be swapped to # reflect that. formatted_symbols = [] diff --git a/symforce/codegen/geo_package_codegen.py b/symforce/codegen/geo_package_codegen.py index cd974809e..aa541ae35 100644 --- a/symforce/codegen/geo_package_codegen.py +++ b/symforce/codegen/geo_package_codegen.py @@ -225,7 +225,7 @@ def generate(config: CodegenConfig, output_dir: str = None) -> str: templates.add( template_dir / "geo_package" / "ops" / template_name, - package_dir / "geo_package" / "ops" / template_name[: -len(".jinja")], + package_dir / "ops" / template_name[: -len(".jinja")], dict(Codegen.common_data()), ) diff --git a/symforce/codegen/template_util.py b/symforce/codegen/template_util.py index 674e66cd0..ea73912ba 100644 --- a/symforce/codegen/template_util.py +++ b/symforce/codegen/template_util.py @@ -47,6 +47,8 @@ def from_extension(extension: str) -> FileType: return FileType.LCM elif extension == "Makefile": return FileType.MAKEFILE + elif extension == "ts": + return FileType.TYPESCRIPT elif extension == "js": return FileType.JAVASCRIPT else: