From 2e74b145a643cc598fc5c12b633347de9855ee7c Mon Sep 17 00:00:00 2001 From: Ali Caglayan Date: Wed, 28 Jan 2026 19:39:13 +0100 Subject: [PATCH 1/6] test(pkg): add test for version variable of absent packages Signed-off-by: Ali Caglayan --- .../pkg/opam-var/absent-pkg-version.t | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 test/blackbox-tests/test-cases/pkg/opam-var/absent-pkg-version.t diff --git a/test/blackbox-tests/test-cases/pkg/opam-var/absent-pkg-version.t b/test/blackbox-tests/test-cases/pkg/opam-var/absent-pkg-version.t new file mode 100644 index 00000000000..0eac83ba84c --- /dev/null +++ b/test/blackbox-tests/test-cases/pkg/opam-var/absent-pkg-version.t @@ -0,0 +1,53 @@ +Test the "version" variable for packages not in the solution in different +contexts: string interpolation vs truthy/filter context. + + $ mkrepo + +First, test the variable in string interpolation context (command argument): + + $ mkpkg "string-context" <<'EOF' + > build: [ + > [ "echo" "version=%{not-in-lock:version}%" ] + > ] + > EOF + + $ solve string-context + Solution for dune.lock: + - string-context.0.0.1 + +The variable resolves to empty string at solve time: + + $ cat dune.lock/string-context.0.0.1.pkg + (version 0.0.1) + + (build + (all_platforms ((action (run echo version=%{pkg:not-in-lock:version}))))) + + +Now test the variable in truthy/filter context (conditional on command): + + $ mkpkg "truthy-context" <<'EOF' + > build: [ + > [ "echo" "has version" ] {not-in-lock:version} + > [ "echo" "no version" ] {!not-in-lock:version} + > ] + > EOF + + $ solve truthy-context + Solution for dune.lock: + - truthy-context.0.0.1 + +In truthy context, the variable is left for build time evaluation: + + $ cat dune.lock/truthy-context.0.0.1.pkg + (version 0.0.1) + + (build + (all_platforms + ((action + (progn + (when %{pkg:not-in-lock:version} (run echo "has version")) + (when (not %{pkg:not-in-lock:version}) (run echo "no version"))))))) + + + From 6d25a0d25cc284a7c42963dbbd052ef9434cd44f Mon Sep 17 00:00:00 2001 From: Ali Caglayan Date: Wed, 28 Jan 2026 18:44:58 +0100 Subject: [PATCH 2/6] test(pkg): add tests for absent package variables in different contexts Signed-off-by: Ali Caglayan --- .../pkg/opam-var/absent-pkg-installed.t | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 test/blackbox-tests/test-cases/pkg/opam-var/absent-pkg-installed.t diff --git a/test/blackbox-tests/test-cases/pkg/opam-var/absent-pkg-installed.t b/test/blackbox-tests/test-cases/pkg/opam-var/absent-pkg-installed.t new file mode 100644 index 00000000000..414264fdc2e --- /dev/null +++ b/test/blackbox-tests/test-cases/pkg/opam-var/absent-pkg-installed.t @@ -0,0 +1,79 @@ +Test the "installed" variable for packages not in the solution in different +contexts: string interpolation vs truthy/filter context. + + $ mkrepo + +First, test the variable in string interpolation context (command argument): + + $ mkpkg "string-context" <<'EOF' + > build: [ + > [ "echo" "%{not-in-lock:installed}%" ] + > ] + > EOF + + $ solve string-context + Solution for dune.lock: + - string-context.0.0.1 + +Currently the variable is left as a pform. It should resolve to "false" at +solve time: + + $ cat dune.lock/string-context.0.0.1.pkg + (version 0.0.1) + + (build + (all_platforms ((action (run echo %{pkg:not-in-lock:installed}))))) + +Now test the variable in truthy/filter context (conditional on command): + + $ mkpkg "truthy-context" <<'EOF' + > build: [ + > [ "echo" "yes" ] {not-in-lock:installed} + > [ "echo" "no" ] {!not-in-lock:installed} + > ] + > EOF + + $ solve truthy-context + Solution for dune.lock: + - truthy-context.0.0.1 + +In truthy context, the variable is left for build time evaluation: + + $ cat dune.lock/truthy-context.0.0.1.pkg + (version 0.0.1) + + (build + (all_platforms + ((action + (progn + (when %{pkg:not-in-lock:installed} (run echo yes)) + (when (not %{pkg:not-in-lock:installed}) (run echo no))))))) + +The "enable" variable is desugared to "installed?enable:disable". The +"?then:else" syntax uses catch_undefined_var to handle undefined: + + $ mkpkg "enable-context" <<'EOF' + > build: [ + > [ "echo" "%{not-in-lock:enable}%" ] + > ] + > EOF + + $ solve enable-context + Solution for dune.lock: + - enable-context.0.0.1 + +The conditional handles undefined at build time (evaluates to "disable"): + + $ cat dune.lock/enable-context.0.0.1.pkg + (version 0.0.1) + + (build + (all_platforms + ((action + (run + echo + (if + (catch_undefined_var %{pkg:not-in-lock:installed} false) + enable + disable)))))) + From 4b1c5efcd6e9852e1e5b2c6a7d1e8fa36fe0cfe7 Mon Sep 17 00:00:00 2001 From: Ali Caglayan Date: Mon, 16 Feb 2026 16:41:35 +0100 Subject: [PATCH 3/6] fix(pkg): evaluate absent package 'installed' variable at solve time When a package is absent from the solution, its 'installed' variable is known to be false. This change evaluates it at solve time in both filter contexts and string interpolation contexts, enabling further simplification. Signed-off-by: Ali Caglayan --- src/dune_pkg/lock_pkg.ml | 39 ++++++++++++++--- .../test-cases/pkg/convert-opam-commands.t | 43 +++---------------- .../pkg/opam-var/absent-pkg-installed.t | 28 ++++-------- 3 files changed, 46 insertions(+), 64 deletions(-) diff --git a/src/dune_pkg/lock_pkg.ml b/src/dune_pkg/lock_pkg.ml index f46369c4a0f..3b47e94459e 100644 --- a/src/dune_pkg/lock_pkg.ml +++ b/src/dune_pkg/lock_pkg.ml @@ -12,7 +12,7 @@ let add_self_to_filter_env package env variable = else env variable ;; -let simplify_filter get_solver_var = +let simplify_filter ~packages_in_solution get_solver_var = OpamFilter.partial_eval (fun var -> match OpamVariable.Full.scope var with | Global -> @@ -25,7 +25,18 @@ let simplify_filter get_solver_var = *) Some (B false) else get_solver_var name |> Option.map ~f:Variable_value.to_opam_variable_contents - | _ -> None) + | Self -> None + | Package pkg_name -> + let name = Package_name.of_opam_package_name pkg_name in + if Package_name.Map.mem packages_in_solution name + then None + else ( + (* Package is absent from solution - substitute known boolean values. + We only substitute 'installed' because it's a boolean variable. + String variables like 'version' can't be converted to boolean + by OpamFilter.eval_to_bool when used in filter contexts. *) + let var_name = OpamVariable.Full.variable var |> Package_variable_name.of_opam in + if Package_variable_name.(equal var_name installed) then Some (B false) else None)) ;; let partial_eval_filter = function @@ -76,6 +87,8 @@ let opam_variable_to_slang = Package_variable_name.absent_package_value variable_name |> Option.value ~default:"" |> Slang.text + else if Package_variable_name.(equal variable_name installed) + then Slang.bool false else opam_var_to_pform variable_name (Package pkg_name) | None -> opam_var_to_pform variable_name Self) in @@ -140,10 +153,18 @@ let opam_fident_to_slang ~loc ~packages_in_solution ~for_string_interp fident = an undefined variable. The catch_undefined_var operator is used to convert expressions that throw undefined variable exceptions into false. *) - let condition = - Blang.Expr (Slang.catch_undefined_var slang ~fallback:(Slang.bool false)) + let is_known_false = + match slang with + | Form (_, Blang (Blang.Const false)) -> true + | _ -> false in - Slang.if_ condition ~then_:(Slang.text then_) ~else_:(Slang.text else_) + if is_known_false + then Slang.text else_ + else ( + let condition = + Blang.Expr (Slang.catch_undefined_var slang ~fallback:(Slang.bool false)) + in + Slang.if_ condition ~then_:(Slang.text then_) ~else_:(Slang.text else_)) ;; let opam_raw_fident_to_slang ~loc ~packages_in_solution ~for_string_interp raw_ident = @@ -293,13 +314,17 @@ let opam_commands_to_actions (commands : OpamTypes.command list) = List.filter_map commands ~f:(fun (args, filter) -> - let filter = Option.map filter ~f:(simplify_filter get_solver_var) in + let filter = + Option.map filter ~f:(simplify_filter ~packages_in_solution get_solver_var) + in match partial_eval_filter filter with | `Skip -> None | `Filter filter -> let terms = List.filter_map args ~f:(fun ((simple_arg : OpamTypes.simple_arg), filter) -> - let filter = Option.map filter ~f:(simplify_filter get_solver_var) in + let filter = + Option.map filter ~f:(simplify_filter ~packages_in_solution get_solver_var) + in match partial_eval_filter filter with | `Skip -> None | `Filter filter -> diff --git a/test/blackbox-tests/test-cases/pkg/convert-opam-commands.t b/test/blackbox-tests/test-cases/pkg/convert-opam-commands.t index cc08697a62b..048366f98ed 100644 --- a/test/blackbox-tests/test-cases/pkg/convert-opam-commands.t +++ b/test/blackbox-tests/test-cases/pkg/convert-opam-commands.t @@ -191,11 +191,7 @@ Package which has boolean where string was expected. This should be caught while (when (< %{pkg-self:version} 1.0) (run echo g)) (run echo i) (run echo j) - (when %{pkg:foo:installed} (run echo k)) - (when (< %{pkg:foo:version} 0.4) (run echo l)) - (when - (and %{pkg:foo:installed} %{pkg:bar:installed} %{pkg:baz:installed}) - (run echo m))))))) + (when (< %{pkg:foo:version} 0.4) (run echo l))))))) Test that if opam filter translation is disabled the output doesn't contain any translated filters: $ solve exercise-filters @@ -235,11 +231,7 @@ Test that if opam filter translation is disabled the output doesn't contain any (when (< %{pkg-self:version} 1.0) (run echo g)) (run echo i) (run echo j) - (when %{pkg:foo:installed} (run echo k)) - (when (< %{pkg:foo:version} 0.4) (run echo l)) - (when - (and %{pkg:foo:installed} %{pkg:bar:installed} %{pkg:baz:installed}) - (run echo m))))))) + (when (< %{pkg:foo:version} 0.4) (run echo l))))))) $ solve exercise-term-filters Solution for dune.lock: @@ -318,23 +310,8 @@ preserved between opam and dune. "c " (if (catch_undefined_var %{pkg-self:installed} false) x y) " d")) - (run - echo - (concat - "e " - (if (catch_undefined_var %{pkg:foo:installed} false) x y) - " f")) - (run - echo - (concat - "g " - (if - (catch_undefined_var - (and %{pkg:foo:installed} %{pkg:bar:installed} %{pkg-self:installed}) - false) - x - y) - " h")) + (run echo "e y f") + (run echo "g y h") (run echo --%{pkg-self:enable}-feature) (run echo @@ -342,17 +319,7 @@ preserved between opam and dune. -- (if (catch_undefined_var %{pkg-self:installed} false) enable disable) -feature)) - (run - echo - (concat - -- - (if - (catch_undefined_var - (and %{pkg:foo:installed} %{pkg:bar:installed}) - false) - enable - disable) - -feature)) + (run echo --disable-feature) (run echo (concat diff --git a/test/blackbox-tests/test-cases/pkg/opam-var/absent-pkg-installed.t b/test/blackbox-tests/test-cases/pkg/opam-var/absent-pkg-installed.t index 1af9dba582c..b97f644334c 100644 --- a/test/blackbox-tests/test-cases/pkg/opam-var/absent-pkg-installed.t +++ b/test/blackbox-tests/test-cases/pkg/opam-var/absent-pkg-installed.t @@ -36,21 +36,19 @@ Now test the variable in truthy/filter context (conditional on command): Solution for dune.lock: - truthy-context.0.0.1 -In truthy context, the variable is left for build time evaluation where it will -be undefined and thus falsey: +In truthy context, the variable evaluates to false at solve time. Commands with +always-false conditions are removed, and commands with always-true conditions +have their filters removed: $ cat dune.lock/truthy-context.0.0.1.pkg (version 0.0.1) (build - (all_platforms - ((action - (progn - (when %{pkg:not-in-lock:installed} (run echo yes)) - (when (not %{pkg:not-in-lock:installed}) (run echo no))))))) + (all_platforms ((action (run echo no))))) -The "enable" variable is desugared to "installed?enable:disable". The -"?then:else" syntax uses catch_undefined_var to handle undefined: +The "enable" variable is desugared to "installed?enable:disable". Since +"installed" is known to be false for absent packages, the conditional is +evaluated at solve time: $ mkpkg "enable-context" <<'EOF' > build: [ @@ -62,19 +60,11 @@ The "enable" variable is desugared to "installed?enable:disable". The Solution for dune.lock: - enable-context.0.0.1 -The conditional evaluates to "disable" at build time when the variable is -undefined: +The conditional evaluates to "disable" at solve time since "installed" is false: $ cat dune.lock/enable-context.0.0.1.pkg (version 0.0.1) (build - (all_platforms - ((action - (run - echo - (if - (catch_undefined_var %{pkg:not-in-lock:installed} false) - enable - disable)))))) + (all_platforms ((action (run echo disable))))) From be66cda147b3c00be410983ad9deeb0f02a8f7b6 Mon Sep 17 00:00:00 2001 From: Ali Caglayan Date: Mon, 16 Feb 2026 16:43:32 +0100 Subject: [PATCH 4/6] fix(pkg): evaluate absent package 'pinned' and 'enable' variables at solve time Extends the absent package variable evaluation to also handle 'pinned' and 'enable' variables, which are also false for absent packages. Also skips wrapping constant filters with catch_undefined_var since they cannot throw undefined variable exceptions. Signed-off-by: Ali Caglayan --- src/dune_lang/package_variable_name.ml | 4 ++ src/dune_lang/package_variable_name.mli | 2 + src/dune_pkg/lock_pkg.ml | 38 ++++++++++++------- .../test-cases/pkg/convert-opam-commands.t | 10 +---- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/src/dune_lang/package_variable_name.ml b/src/dune_lang/package_variable_name.ml index 2eedc08f828..a805d2201b9 100644 --- a/src/dune_lang/package_variable_name.ml +++ b/src/dune_lang/package_variable_name.ml @@ -46,6 +46,8 @@ let build = of_string "build" let post = of_string "post" let dev = of_string "dev" let installed = of_string "installed" +let pinned = of_string "pinned" +let enable = of_string "enable" let one_of t xs = List.mem xs ~equal t let absent_package_value t = @@ -75,6 +77,8 @@ let all_known = ; post ; dev ; installed + ; pinned + ; enable ] ;; diff --git a/src/dune_lang/package_variable_name.mli b/src/dune_lang/package_variable_name.mli index a8070be04e7..cee4b3baa44 100644 --- a/src/dune_lang/package_variable_name.mli +++ b/src/dune_lang/package_variable_name.mli @@ -33,6 +33,8 @@ val post : t val build : t val dev : t val installed : t +val pinned : t +val enable : t val one_of : t -> t list -> bool (** Returns the value of a package variable when the package is not in the diff --git a/src/dune_pkg/lock_pkg.ml b/src/dune_pkg/lock_pkg.ml index 3b47e94459e..2bd47a85b1e 100644 --- a/src/dune_pkg/lock_pkg.ml +++ b/src/dune_pkg/lock_pkg.ml @@ -31,12 +31,14 @@ let simplify_filter ~packages_in_solution get_solver_var = if Package_name.Map.mem packages_in_solution name then None else ( - (* Package is absent from solution - substitute known boolean values. - We only substitute 'installed' because it's a boolean variable. - String variables like 'version' can't be converted to boolean - by OpamFilter.eval_to_bool when used in filter contexts. *) + (* Package is absent from solution - substitute known values. + All these are false/falsey for absent packages. *) let var_name = OpamVariable.Full.variable var |> Package_variable_name.of_opam in - if Package_variable_name.(equal var_name installed) then Some (B false) else None)) + if + Package_variable_name.( + equal var_name installed || equal var_name pinned || equal var_name enable) + then Some (B false) + else None)) ;; let partial_eval_filter = function @@ -87,7 +89,11 @@ let opam_variable_to_slang = Package_variable_name.absent_package_value variable_name |> Option.value ~default:"" |> Slang.text - else if Package_variable_name.(equal variable_name installed) + else if + Package_variable_name.( + equal variable_name installed + || equal variable_name pinned + || equal variable_name enable) then Slang.bool false else opam_var_to_pform variable_name (Package pkg_name) | None -> opam_var_to_pform variable_name Self) @@ -352,14 +358,18 @@ let opam_commands_to_actions in let filter_blang_handling_undefined = (* Wrap the blang filter so that if any undefined - variables are expanded while evaluating the filter, - the filter will return false. *) - let slang = - Slang.catch_undefined_var - (Slang.blang filter_blang) - ~fallback:(Slang.bool false) - in - Blang.Expr slang + variables are expanded while evaluating the filter, + the filter will return false. Skip wrapping if the + filter is already a constant. *) + match filter_blang with + | Const _ -> filter_blang + | _ -> + let slang = + Slang.catch_undefined_var + (Slang.blang filter_blang) + ~fallback:(Slang.bool false) + in + Blang.Expr slang in Slang.when_ filter_blang_handling_undefined slang in diff --git a/test/blackbox-tests/test-cases/pkg/convert-opam-commands.t b/test/blackbox-tests/test-cases/pkg/convert-opam-commands.t index 048366f98ed..03f05a33b97 100644 --- a/test/blackbox-tests/test-cases/pkg/convert-opam-commands.t +++ b/test/blackbox-tests/test-cases/pkg/convert-opam-commands.t @@ -320,12 +320,4 @@ preserved between opam and dune. (if (catch_undefined_var %{pkg-self:installed} false) enable disable) -feature)) (run echo --disable-feature) - (run - echo - (concat - -- - (if - (catch_undefined_var (and %{pkg:foo:enable} %{pkg:bar:enable}) false) - x - y) - -feature))))))) + (run echo --y-feature)))))) From 752a6d66b81a12149ac7e94500b5414ecc81ac56 Mon Sep 17 00:00:00 2001 From: Ali Caglayan Date: Mon, 23 Feb 2026 10:47:04 +0100 Subject: [PATCH 5/6] refactor(pkg): introduce is_falsey_for_absent_package Signed-off-by: Ali Caglayan --- src/dune_lang/package_variable_name.ml | 4 ++++ src/dune_lang/package_variable_name.mli | 4 ++++ src/dune_pkg/lock_pkg.ml | 10 ++-------- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/dune_lang/package_variable_name.ml b/src/dune_lang/package_variable_name.ml index a805d2201b9..97be6e360ca 100644 --- a/src/dune_lang/package_variable_name.ml +++ b/src/dune_lang/package_variable_name.ml @@ -54,6 +54,10 @@ let absent_package_value t = if equal t installed then Some "false" else if equal t version then Some "" else None ;; +let is_falsey_for_absent_package t = + equal t installed || equal t pinned || equal t enable +;; + let platform_specific = Set.of_list [ arch; os; os_version; os_distribution; os_family; sys_ocaml_version ] ;; diff --git a/src/dune_lang/package_variable_name.mli b/src/dune_lang/package_variable_name.mli index cee4b3baa44..6b9241dcf05 100644 --- a/src/dune_lang/package_variable_name.mli +++ b/src/dune_lang/package_variable_name.mli @@ -42,6 +42,10 @@ val one_of : t -> t list -> bool (e.g., "installed" is "false"), while others are undefined and return None. *) val absent_package_value : t -> string option +(** Returns true if the variable is known to be false/falsey for absent packages. + This includes "installed", "pinned", and "enable". *) +val is_falsey_for_absent_package : t -> bool + (** The set of variable names whose values are expected to differ depending on the current platform. *) val platform_specific : Set.t diff --git a/src/dune_pkg/lock_pkg.ml b/src/dune_pkg/lock_pkg.ml index 2bd47a85b1e..66175f00eb5 100644 --- a/src/dune_pkg/lock_pkg.ml +++ b/src/dune_pkg/lock_pkg.ml @@ -34,9 +34,7 @@ let simplify_filter ~packages_in_solution get_solver_var = (* Package is absent from solution - substitute known values. All these are false/falsey for absent packages. *) let var_name = OpamVariable.Full.variable var |> Package_variable_name.of_opam in - if - Package_variable_name.( - equal var_name installed || equal var_name pinned || equal var_name enable) + if Package_variable_name.is_falsey_for_absent_package var_name then Some (B false) else None)) ;; @@ -89,11 +87,7 @@ let opam_variable_to_slang = Package_variable_name.absent_package_value variable_name |> Option.value ~default:"" |> Slang.text - else if - Package_variable_name.( - equal variable_name installed - || equal variable_name pinned - || equal variable_name enable) + else if Package_variable_name.is_falsey_for_absent_package variable_name then Slang.bool false else opam_var_to_pform variable_name (Package pkg_name) | None -> opam_var_to_pform variable_name Self) From e389f75286f29145b64821743b0d0e0500bcfd8e Mon Sep 17 00:00:00 2001 From: Ali Caglayan Date: Thu, 1 Jan 2026 20:24:31 +0000 Subject: [PATCH 6/6] feat(pkg): relocatable compiler support Signed-off-by: Ali Caglayan --- src/dune_rules/pkg_rules.ml | 68 +++++--- src/dune_rules/pkg_toolchain.ml | 2 +- src/dune_rules/pkg_toolchain.mli | 2 +- .../test-cases/pkg/relocatable-compiler.t | 149 ++++++++++++++++++ 4 files changed, 198 insertions(+), 23 deletions(-) create mode 100644 test/blackbox-tests/test-cases/pkg/relocatable-compiler.t diff --git a/src/dune_rules/pkg_rules.ml b/src/dune_rules/pkg_rules.ml index 6e362f85640..432fc795e2a 100644 --- a/src/dune_rules/pkg_rules.ml +++ b/src/dune_rules/pkg_rules.ml @@ -527,17 +527,6 @@ module Pkg = struct |> List.fold_left ~init:Dep.Set.empty ~f:(fun acc t -> dep t |> Dep.Set.add acc) ;; - let install_roots t = - let default_install_roots = Paths.install_roots t.paths in - match Pkg_toolchain.is_compiler_and_toolchains_enabled t.info.name with - | false -> default_install_roots - | true -> - (* Compiler packages store their libraries in a subdirectory named "ocaml". *) - { default_install_roots with - lib_root = Path.relative default_install_roots.lib_root "ocaml" - } - ;; - (* Given a list of packages, construct an env containing variables set by each package. Variables containing delimited lists of paths (e.g. PATH) which appear in multiple package's envs are @@ -548,7 +537,7 @@ module Pkg = struct let build_env_of_deps ts = List.fold_left ts ~init:Env.Map.empty ~f:(fun env t -> let env = - let roots = install_roots t in + let roots = Paths.install_roots t.paths in let init = Value_list_env.add_path env Env_path.var roots.bin in let vars = Install.Roots.to_env_without_path roots ~relative:Path.relative in List.fold_left vars ~init ~f:(fun acc (var, path) -> @@ -1356,12 +1345,42 @@ module DB = struct { id : Id.t ; pkg_digest_table : Pkg_table.t ; system_provided : Package.Name.Set.t + ; is_relocatable_compiler_context : bool } let equal x y = Id.equal x.id y.id - let create ~pkg_digest_table ~system_provided = - { id = Id.gen (); pkg_digest_table; system_provided } + let create = + (* In dra27's relocatable repository, a compiler is relocatable if it + depends on this meta-package. *) + let[@inline] is_relocatable_compiler_meta_package name = + Package.Name.equal name (Package.Name.of_string "relocatable-compiler") + in + (* OCaml 5.5.0+ is natively relocatable. *) + let[@inline] is_ocaml_5_5_or_above name version = + Dune_pkg.Dev_tool.is_compiler_package name + && OpamPackage.Version.compare + (Dune_pkg.Package_version.to_opam_package_version version) + (OpamPackage.Version.of_string "5.5.0") + |> Ordering.of_int + |> function + | Gt | Eq -> true + | Lt -> false + in + fun ~pkg_digest_table ~system_provided -> + { id = Id.gen () + ; pkg_digest_table + ; system_provided + (* To know if a given package table has the relocatable compiler, we + need to find either the "relocatable-compiler" meta-package or + assert that the version of the OCaml compiler is at least 5.5.0. *) + ; is_relocatable_compiler_context = + Pkg_digest.Map.existsi + pkg_digest_table + ~f:(fun _ { Pkg_table.pkg = { info = { name; version; _ }; _ }; _ } -> + is_relocatable_compiler_meta_package name + || is_ocaml_5_5_or_above name version) + } ;; let pkg_digest_of_name lock_dir platform pkg_name ~system_provided = @@ -1590,9 +1609,11 @@ end = struct let build_command = Option.map build_command ~f:relocate_build in let paths = let paths = Paths.map_path write_paths ~f:Path.build in - match Pkg_toolchain.is_compiler_and_toolchains_enabled info.name with - | false -> paths - | true -> + if + db.is_relocatable_compiler_context + || not (Pkg_toolchain.is_compiler_package_with_toolchains_enabled info.name) + then paths + else ( (* Modify the environment as well as build and install commands for the compiler package. The specific changes are: - setting the prefix in the build environment to inside the user's @@ -1603,16 +1624,21 @@ end = struct toolchain directory - if a matching version of the compiler is already installed in the user's toolchain directory then the - build and install commands are replaced with no-ops *) + build and install commands are replaced with no-ops + - compiler packages store their libraries in a subdirectory + named "ocaml" *) let prefix = Pkg_toolchain.installation_prefix pkg in let install_roots = - Pkg_toolchain.install_roots ~prefix - |> Install.Roots.map ~f:Path.outside_build_dir + let roots = + Pkg_toolchain.install_roots ~prefix + |> Install.Roots.map ~f:Path.outside_build_dir + in + { roots with lib_root = Path.relative roots.lib_root "ocaml" } in { paths with prefix = Path.outside_build_dir prefix ; install_roots = Lazy.from_val install_roots - } + }) in let t = { Pkg.id diff --git a/src/dune_rules/pkg_toolchain.ml b/src/dune_rules/pkg_toolchain.ml index b98b734a794..68286c802aa 100644 --- a/src/dune_rules/pkg_toolchain.ml +++ b/src/dune_rules/pkg_toolchain.ml @@ -47,7 +47,7 @@ let installation_prefix pkg = Path.Outside_build_dir.relative pkg_dir "target" ;; -let is_compiler_and_toolchains_enabled name = +let is_compiler_package_with_toolchains_enabled name = match Config.get Compile_time.toolchains with | `Enabled -> Dune_pkg.Dev_tool.is_compiler_package name | `Disabled -> false diff --git a/src/dune_rules/pkg_toolchain.mli b/src/dune_rules/pkg_toolchain.mli index d6b91169e5c..651568fa9f1 100644 --- a/src/dune_rules/pkg_toolchain.mli +++ b/src/dune_rules/pkg_toolchain.mli @@ -18,7 +18,7 @@ val base_dir : unit -> Path.Outside_build_dir.t manage their compiler installation with opam or a system package manager, as compilers packages that would be installed by dune will not work correctly. *) -val is_compiler_and_toolchains_enabled : Package.Name.t -> bool +val is_compiler_package_with_toolchains_enabled : Package.Name.t -> bool (** Returns the path to the directory containing the given package within the toolchain directory. This will be something like diff --git a/test/blackbox-tests/test-cases/pkg/relocatable-compiler.t b/test/blackbox-tests/test-cases/pkg/relocatable-compiler.t new file mode 100644 index 00000000000..880137f17ce --- /dev/null +++ b/test/blackbox-tests/test-cases/pkg/relocatable-compiler.t @@ -0,0 +1,149 @@ +Test that relocatable compilers bypass toolchain cache. This includes: +- OCaml < 5.5 with the relocatable-compiler package +- OCaml >= 5.5 which is natively relocatable + +This mirrors the package layout from +https://github.com/dra27/opam-repository/tree/relocatable + +The relocatable repo is separate from the main opam repo and should take priority. + + $ mkrepo + $ mkdir -p relocatable-repo/packages + +Create the relocatable compiler packages in separate repo: + + $ mkdir -p relocatable-repo/packages/compiler-cloning/compiler-cloning.0.0.1 + $ cat > relocatable-repo/packages/compiler-cloning/compiler-cloning.0.0.1/opam << 'EOF' + > opam-version: "2.0" + > EOF + + $ mkdir -p relocatable-repo/packages/relocatable-compiler/relocatable-compiler.5.4.0 + $ cat > relocatable-repo/packages/relocatable-compiler/relocatable-compiler.5.4.0/opam << 'EOF' + > opam-version: "2.0" + > depends: [ "compiler-cloning" ] + > build: [ "sh" "-c" "echo %{_:build-id}% > build-id.txt" ] + > install: [ "cp" "build-id.txt" "%{lib}%/relocatable-compiler/" ] + > EOF + + $ mkdir -p relocatable-repo/packages/ocaml-base-compiler/ocaml-base-compiler.5.4.0 + $ cat > relocatable-repo/packages/ocaml-base-compiler/ocaml-base-compiler.5.4.0/opam << 'EOF' + > opam-version: "2.0" + > depends: [ "relocatable-compiler" {= "5.4.0"} ] + > build: [ + > [ "sh" "-c" "echo 'ocamlc binary' > ocamlc.opt" ] + > [ "ln" "-s" "ocamlc.opt" "ocamlc" ] + > ] + > install: [ + > [ "mkdir" "-p" "%{bin}%" ] + > [ "cp" "ocamlc.opt" "%{bin}%/" ] + > [ "cp" "-P" "ocamlc" "%{bin}%/" ] + > ] + > EOF + + $ mkdir -p relocatable-repo/packages/ocaml/ocaml.5.4.0 + $ cat > relocatable-repo/packages/ocaml/ocaml.5.4.0/opam << 'EOF' + > opam-version: "2.0" + > depends: [ "ocaml-base-compiler" {= "5.4.0"} ] + > EOF + +Also create a standard ocaml-base-compiler 5.4 in mock repo. This tests that +the relocatable repo takes priority: + + $ mkpkg ocaml-base-compiler 5.4.0 + + $ mkpkg ocaml 5.4.0 << 'EOF' + > depends: [ "ocaml-base-compiler" {= "5.4.0"} ] + > EOF + + $ cat > dune-project << 'EOF' + > (lang dune 3.22) + > (package + > (name test) + > (allow_empty) + > (depends ocaml)) + > EOF + + $ cat > dune-workspace << EOF + > (lang dune 3.20) + > (lock_dir + > (repositories relocatable-repo mock)) ; Order here is important here + > (repository + > (name relocatable-repo) + > (url "file://$PWD/relocatable-repo")) + > (repository + > (name mock) + > (url "file://$PWD/mock-opam-repository")) + > EOF + +We enable caching to verify that packages are built normally and get hardlinked +into the cache. A hardlink count > 1 proves the file was cached. + + $ export DUNE_CACHE=enabled + $ export DUNE_CACHE_ROOT=$PWD/dune-cache + +Solving for OCaml 5.4 should pick up the relocatable compiler. + + $ dune_pkg_lock_normalized + Solution for dune.lock: + - compiler-cloning.0.0.1 + - ocaml.5.4.0 + - ocaml-base-compiler.5.4.0 + - relocatable-compiler.5.4.0 + + $ build_pkg ocaml-base-compiler + +Toolchain cache should be empty: + + $ test -d dune-cache/toolchains + [1] + +Check the installed files from the compiler are hardlinked (cached as regular +package): + + $ dune_cmd stat hardlinks $(get_build_pkg_dir ocaml-base-compiler)/target/bin/ocamlc.opt + 3 + +For OCaml >= 5.5, relocatable is built-in, so no special packages needed. These +go in the main opam repo. + + $ mkpkg ocaml-base-compiler 5.5.0 << 'EOF' + > flags: compiler + > build: [ "sh" "-c" "echo 'native relocatable' > marker.txt" ] + > install: [ + > [ "mkdir" "-p" "%{lib}%/ocaml" ] + > [ "cp" "marker.txt" "%{lib}%/ocaml/" ] + > ] + > EOF + + $ mkpkg ocaml 5.5.0 << 'EOF' + > depends: [ "ocaml-base-compiler" {= "5.5.0"} ] + > EOF + + $ cat > dune-project << 'EOF' + > (lang dune 3.16) + > (package + > (name test) + > (allow_empty) + > (depends (ocaml (= 5.5.0)))) + > EOF + +The relocatable repo will not have OCaml >= 5.5 so we do not need to bother +updating the repositories in the workspace. + + $ dune_pkg_lock_normalized + Solution for dune.lock: + - ocaml.5.5.0 + - ocaml-base-compiler.5.5.0 + + $ build_pkg ocaml-base-compiler + +Toolchain cache should be empty: + + $ test -d dune-cache/toolchains + [1] + +Check the installed files from the compiler are hardlinked (cached as regular +package): + + $ dune_cmd stat hardlinks $(get_build_pkg_dir ocaml-base-compiler)/target/lib/ocaml/marker.txt + 2