From d00f978a20524b1f7ec4cae00e67c3b4f7bccc15 Mon Sep 17 00:00:00 2001 From: Iain Buclaw Date: Sun, 11 Feb 2024 22:12:34 +0100 Subject: [PATCH 1/5] Fix macOS ld: multiple errors: symbol count from symbol table and dynamic symbol table differ Encode macosx_version_min or build_version into the object file, originally authored by @jacob-carlborg in #10476. This has been simplified to omit parsing the SDK version. Based on what I see GCC is doing (`-platform_version macos $version_min 0.0`), this information is not required in order for things to work. Co-authored-by: Jacob Carlborg --- compiler/src/build.d | 3 + compiler/src/dmd/backend/mach.d | 65 +++++++++++ compiler/src/dmd/backend/machobj.d | 175 ++++++++++++++++++++++++++++- 3 files changed, 241 insertions(+), 2 deletions(-) diff --git a/compiler/src/build.d b/compiler/src/build.d index 8e9ccd4dfa85..fe708cbc2ca2 100755 --- a/compiler/src/build.d +++ b/compiler/src/build.d @@ -1002,6 +1002,9 @@ alias toolchainInfo = makeRule!((builder, rule) => builder app.put("==== Toolchain Information ====\n"); + version (OSX) + show("OS", ["sw_vers"]); + version (Windows) show("SYSTEM", ["systeminfo"]); else diff --git a/compiler/src/dmd/backend/mach.d b/compiler/src/dmd/backend/mach.d index 1686537942de..a7078e2766f7 100644 --- a/compiler/src/dmd/backend/mach.d +++ b/compiler/src/dmd/backend/mach.d @@ -94,6 +94,12 @@ enum LC_SYMTAB = 2, LC_DYSYMTAB = 11, LC_SEGMENT_64 = 0x19, + + /// Build for MacOSX min OS version. + LC_VERSION_MIN_MACOSX = 0x24, + + /// Build for platform min OS version. + LC_BUILD_VERSION = 0x32, } struct load_command @@ -407,3 +413,62 @@ struct scattered_relocation_info int r_value; } + +/** + * The version_min_command contains the min OS version on which this binary was + * built to run. + */ +struct version_min_command +{ + /// + uint cmd = LC_VERSION_MIN_MACOSX; + + /// + uint cmdsize = typeof(this).sizeof; + + /// X.Y.Z is encoded in nibbles xxxx.yy.zz + uint version_; + + /// X.Y.Z is encoded in nibbles xxxx.yy.zz + uint sdk = 0; +} + +/** + * The `build_version_command` contains the min OS version on which this binary + * was built to run for its platform. + */ +struct build_version_command +{ + /// + uint cmd = LC_BUILD_VERSION; + + /// + uint cmdsize = typeof(this).sizeof; + + /// Platform + uint platform = PLATFORM_MACOS; + + /// X.Y.Z is encoded in nibbles xxxx.yy.zz + uint minos; + + /// X.Y.Z is encoded in nibbles xxxx.yy.zz + uint sdk = 0; + + /// Number of tool entries following this + uint ntools = 0; +} + +/// Known values for the platform field in `build_version_command` +enum +{ + PLATFORM_MACOS = 1, + PLATFORM_IOS = 2, + PLATFORM_TVOS = 3, + PLATFORM_WATCHOS = 4, + PLATFORM_BRIDGEOS = 5, + PLATFORM_MACCATALYST = 6, + PLATFORM_IOSSIMULATOR = 7, + PLATFORM_TVOSSIMULATOR = 8, + PLATFORM_WATCHOSSIMULATOR = 9, + PLATFORM_DRIVERKIT = 10 +} diff --git a/compiler/src/dmd/backend/machobj.d b/compiler/src/dmd/backend/machobj.d index 305c91991ff1..27443fbf2076 100644 --- a/compiler/src/dmd/backend/machobj.d +++ b/compiler/src/dmd/backend/machobj.d @@ -699,6 +699,7 @@ void MachObj_term(const(char)* objfilename) * { sections } * symtab_command * dysymtab_command + * build_version_command/version_min_command * { segment contents } * { relocations } * symbol table @@ -706,6 +707,7 @@ void MachObj_term(const(char)* objfilename) * indirect symbol table */ + auto versionCommand = VersionCommand(operatingSystemVersion); uint foffset; uint headersize; uint sizeofcmds; @@ -719,7 +721,7 @@ void MachObj_term(const(char)* objfilename) header.cputype = CPU_TYPE_X86_64; header.cpusubtype = CPU_SUBTYPE_I386_ALL; header.filetype = MH_OBJECT; - header.ncmds = 3; + header.ncmds = 4; header.sizeofcmds = cast(uint)(segment_command_64.sizeof + (section_cnt - 1) * section_64.sizeof + symtab_command.sizeof + @@ -747,7 +749,8 @@ void MachObj_term(const(char)* objfilename) header.sizeofcmds = cast(uint)(segment_command.sizeof + (section_cnt - 1) * section.sizeof + symtab_command.sizeof + - dysymtab_command.sizeof); + dysymtab_command.sizeof + + versionCommand.size); header.flags = MH_SUBSECTIONS_VIA_SYMBOLS; fobjbuf.write(&header, header.sizeof); foffset = header.sizeof; // start after header @@ -1475,6 +1478,7 @@ void MachObj_term(const(char)* objfilename) } fobjbuf.write(&symtab_cmd, symtab_cmd.sizeof); fobjbuf.write(&dysymtab_cmd, dysymtab_cmd.sizeof); + fobjbuf.write(versionCommand.data, versionCommand.size); fobjbuf.position(foffset, 0); } @@ -2956,3 +2960,170 @@ int dwarf_eh_frame_fixup(int dfseg, targ_size_t offset, Symbol *s, targ_size_t v return I64 ? 8 : 4; } + + +private: + +/** + * Encapsulates the build_version_command/version_min_command load commands. + * + * For the 10.14 and later SDK, the `build_version_command` load command is used. + * For earlier versions, the `version_min_command` load command is used. + */ +const struct VersionCommand +{ + pure: + nothrow: + @nogc: + @safe: + + private + { + /** + * This is the absolute minimum supported version of macOS (64 bit) for DMD. + * + * Earlier versions did not support thread local storage. + */ + enum fallbackOSVersion = Version(10, 7).encode; + + /// The first minor version that uses the `build_version_command`. + enum firstMinorUsingBuildVersionCommand = 14; + + /// `true` if the `build_version_command` load command should be used. + bool useBuild; + + /// The `build_version_command` load command. + build_version_command buildVersionCommand; + + /// The `version_min_command` load command. + version_min_command versionMinCommand; + } + + /** + * Initializes the VersionCommand. + * + * Params: + * os = the version of the operating system + */ + this(Version os) + { + useBuild = os.minor >= firstMinorUsingBuildVersionCommand; + + const encodedOs = os.isValid ? os.encode : fallbackOSVersion; + + const build_version_command buildVersionCommand = { minos: encodedOs }; + const version_min_command versionMinCommand = { version_: encodedOs }; + + this.buildVersionCommand = buildVersionCommand; + this.versionMinCommand = versionMinCommand; + } + + /// Returns: the size of the load command. + size_t size() + { + return useBuild ? build_version_command.sizeof : version_min_command.sizeof; + } + + /// Returns: the data for the load command. + const(void)* data() return + { + return useBuild ? cast(const(void)*) &buildVersionCommand : cast(const(void)*) &versionMinCommand; + } +} + +/// Holds an operating system version or a SDK version. +immutable struct Version +{ + /// + int major; + + /// + int minor; + + /// + int build; + + /// Returns: `true` if the version is valid + bool isValid() pure nothrow @nogc @safe + { + return major >= 10 && major < 100 && + minor >= 0 && minor < 100 && + build >= 0 && build < 100; + } +} + +/** + * Returns the given version encoded as a single integer. + * + * Params: + * version_ = the version to encode. Needs to be a valid version + * (`version_.isValid`) + * + * Returns: the encoded version + */ +int encode(Version version_) pure @nogc @safe +in +{ + assert(version_.isValid); +} +do +{ + with (version_) + return major * 2^^16 + minor * 2^^8 + build * 2^^0; +} + +unittest +{ + assert(Version(10, 14, 0).encode == 658944); + assert(Version(10, 14, 1).encode == 658945); + assert(Version(10, 14, 6).encode == 658950); + assert(Version(10, 14, 99).encode == 659043); + + assert(Version(10, 15, 6).encode == 659206); + + assert(Version(10, 16, 0).encode == 659456); + assert(Version(10, 16, 6).encode == 659462); + + assert(Version(10, 17, 0).encode == 659712); +} + +/// Returns: the version of the currently running operating system. +@trusted +Version operatingSystemVersion() +{ + if (const deploymentTarget = getenv("MACOSX_DEPLOYMENT_TARGET")) + { + const version_ = toVersion(deploymentTarget); + + if (version_.isValid) + return version_; + + error(null, 0, 0, "invalid version number in 'MACOSX_DEPLOYMENT_TARGET=%s'", deploymentTarget); + } + return Version(); +} + +/** + * Converts the given string to a `Version`. + * + * Params: + * str = the string to convert. Should have the format `XX.YY(.ZZ)`. Needs to + * be `\0` terminated. + * + * Returns: the converted `Version`. + */ +@trusted +Version toVersion(const char* str) @nogc +{ + import core.stdc.stdio : sscanf; + + if (!str) + return Version(); + + Version version_; + + with (version_) + str.sscanf("%d.%d.%d", &major, &minor, &build); + + return version_; +} From c170b3831fa34320f014e702168c03e11ced253e Mon Sep 17 00:00:00 2001 From: Iain Buclaw Date: Sun, 11 Feb 2024 22:52:25 +0100 Subject: [PATCH 2/5] Revert "Fix macOS ld: multiple errors: symbol count from symbol table and dynamic symbol table differ" This reverts commit c01f8cc77e03be9b9fde4d8cb1e3b4ba886615da. --- compiler/src/build.d | 3 - compiler/src/dmd/backend/mach.d | 65 ----------- compiler/src/dmd/backend/machobj.d | 175 +---------------------------- 3 files changed, 2 insertions(+), 241 deletions(-) diff --git a/compiler/src/build.d b/compiler/src/build.d index fe708cbc2ca2..8e9ccd4dfa85 100755 --- a/compiler/src/build.d +++ b/compiler/src/build.d @@ -1002,9 +1002,6 @@ alias toolchainInfo = makeRule!((builder, rule) => builder app.put("==== Toolchain Information ====\n"); - version (OSX) - show("OS", ["sw_vers"]); - version (Windows) show("SYSTEM", ["systeminfo"]); else diff --git a/compiler/src/dmd/backend/mach.d b/compiler/src/dmd/backend/mach.d index a7078e2766f7..1686537942de 100644 --- a/compiler/src/dmd/backend/mach.d +++ b/compiler/src/dmd/backend/mach.d @@ -94,12 +94,6 @@ enum LC_SYMTAB = 2, LC_DYSYMTAB = 11, LC_SEGMENT_64 = 0x19, - - /// Build for MacOSX min OS version. - LC_VERSION_MIN_MACOSX = 0x24, - - /// Build for platform min OS version. - LC_BUILD_VERSION = 0x32, } struct load_command @@ -413,62 +407,3 @@ struct scattered_relocation_info int r_value; } - -/** - * The version_min_command contains the min OS version on which this binary was - * built to run. - */ -struct version_min_command -{ - /// - uint cmd = LC_VERSION_MIN_MACOSX; - - /// - uint cmdsize = typeof(this).sizeof; - - /// X.Y.Z is encoded in nibbles xxxx.yy.zz - uint version_; - - /// X.Y.Z is encoded in nibbles xxxx.yy.zz - uint sdk = 0; -} - -/** - * The `build_version_command` contains the min OS version on which this binary - * was built to run for its platform. - */ -struct build_version_command -{ - /// - uint cmd = LC_BUILD_VERSION; - - /// - uint cmdsize = typeof(this).sizeof; - - /// Platform - uint platform = PLATFORM_MACOS; - - /// X.Y.Z is encoded in nibbles xxxx.yy.zz - uint minos; - - /// X.Y.Z is encoded in nibbles xxxx.yy.zz - uint sdk = 0; - - /// Number of tool entries following this - uint ntools = 0; -} - -/// Known values for the platform field in `build_version_command` -enum -{ - PLATFORM_MACOS = 1, - PLATFORM_IOS = 2, - PLATFORM_TVOS = 3, - PLATFORM_WATCHOS = 4, - PLATFORM_BRIDGEOS = 5, - PLATFORM_MACCATALYST = 6, - PLATFORM_IOSSIMULATOR = 7, - PLATFORM_TVOSSIMULATOR = 8, - PLATFORM_WATCHOSSIMULATOR = 9, - PLATFORM_DRIVERKIT = 10 -} diff --git a/compiler/src/dmd/backend/machobj.d b/compiler/src/dmd/backend/machobj.d index 27443fbf2076..305c91991ff1 100644 --- a/compiler/src/dmd/backend/machobj.d +++ b/compiler/src/dmd/backend/machobj.d @@ -699,7 +699,6 @@ void MachObj_term(const(char)* objfilename) * { sections } * symtab_command * dysymtab_command - * build_version_command/version_min_command * { segment contents } * { relocations } * symbol table @@ -707,7 +706,6 @@ void MachObj_term(const(char)* objfilename) * indirect symbol table */ - auto versionCommand = VersionCommand(operatingSystemVersion); uint foffset; uint headersize; uint sizeofcmds; @@ -721,7 +719,7 @@ void MachObj_term(const(char)* objfilename) header.cputype = CPU_TYPE_X86_64; header.cpusubtype = CPU_SUBTYPE_I386_ALL; header.filetype = MH_OBJECT; - header.ncmds = 4; + header.ncmds = 3; header.sizeofcmds = cast(uint)(segment_command_64.sizeof + (section_cnt - 1) * section_64.sizeof + symtab_command.sizeof + @@ -749,8 +747,7 @@ void MachObj_term(const(char)* objfilename) header.sizeofcmds = cast(uint)(segment_command.sizeof + (section_cnt - 1) * section.sizeof + symtab_command.sizeof + - dysymtab_command.sizeof + - versionCommand.size); + dysymtab_command.sizeof); header.flags = MH_SUBSECTIONS_VIA_SYMBOLS; fobjbuf.write(&header, header.sizeof); foffset = header.sizeof; // start after header @@ -1478,7 +1475,6 @@ void MachObj_term(const(char)* objfilename) } fobjbuf.write(&symtab_cmd, symtab_cmd.sizeof); fobjbuf.write(&dysymtab_cmd, dysymtab_cmd.sizeof); - fobjbuf.write(versionCommand.data, versionCommand.size); fobjbuf.position(foffset, 0); } @@ -2960,170 +2956,3 @@ int dwarf_eh_frame_fixup(int dfseg, targ_size_t offset, Symbol *s, targ_size_t v return I64 ? 8 : 4; } - - -private: - -/** - * Encapsulates the build_version_command/version_min_command load commands. - * - * For the 10.14 and later SDK, the `build_version_command` load command is used. - * For earlier versions, the `version_min_command` load command is used. - */ -const struct VersionCommand -{ - pure: - nothrow: - @nogc: - @safe: - - private - { - /** - * This is the absolute minimum supported version of macOS (64 bit) for DMD. - * - * Earlier versions did not support thread local storage. - */ - enum fallbackOSVersion = Version(10, 7).encode; - - /// The first minor version that uses the `build_version_command`. - enum firstMinorUsingBuildVersionCommand = 14; - - /// `true` if the `build_version_command` load command should be used. - bool useBuild; - - /// The `build_version_command` load command. - build_version_command buildVersionCommand; - - /// The `version_min_command` load command. - version_min_command versionMinCommand; - } - - /** - * Initializes the VersionCommand. - * - * Params: - * os = the version of the operating system - */ - this(Version os) - { - useBuild = os.minor >= firstMinorUsingBuildVersionCommand; - - const encodedOs = os.isValid ? os.encode : fallbackOSVersion; - - const build_version_command buildVersionCommand = { minos: encodedOs }; - const version_min_command versionMinCommand = { version_: encodedOs }; - - this.buildVersionCommand = buildVersionCommand; - this.versionMinCommand = versionMinCommand; - } - - /// Returns: the size of the load command. - size_t size() - { - return useBuild ? build_version_command.sizeof : version_min_command.sizeof; - } - - /// Returns: the data for the load command. - const(void)* data() return - { - return useBuild ? cast(const(void)*) &buildVersionCommand : cast(const(void)*) &versionMinCommand; - } -} - -/// Holds an operating system version or a SDK version. -immutable struct Version -{ - /// - int major; - - /// - int minor; - - /// - int build; - - /// Returns: `true` if the version is valid - bool isValid() pure nothrow @nogc @safe - { - return major >= 10 && major < 100 && - minor >= 0 && minor < 100 && - build >= 0 && build < 100; - } -} - -/** - * Returns the given version encoded as a single integer. - * - * Params: - * version_ = the version to encode. Needs to be a valid version - * (`version_.isValid`) - * - * Returns: the encoded version - */ -int encode(Version version_) pure @nogc @safe -in -{ - assert(version_.isValid); -} -do -{ - with (version_) - return major * 2^^16 + minor * 2^^8 + build * 2^^0; -} - -unittest -{ - assert(Version(10, 14, 0).encode == 658944); - assert(Version(10, 14, 1).encode == 658945); - assert(Version(10, 14, 6).encode == 658950); - assert(Version(10, 14, 99).encode == 659043); - - assert(Version(10, 15, 6).encode == 659206); - - assert(Version(10, 16, 0).encode == 659456); - assert(Version(10, 16, 6).encode == 659462); - - assert(Version(10, 17, 0).encode == 659712); -} - -/// Returns: the version of the currently running operating system. -@trusted -Version operatingSystemVersion() -{ - if (const deploymentTarget = getenv("MACOSX_DEPLOYMENT_TARGET")) - { - const version_ = toVersion(deploymentTarget); - - if (version_.isValid) - return version_; - - error(null, 0, 0, "invalid version number in 'MACOSX_DEPLOYMENT_TARGET=%s'", deploymentTarget); - } - return Version(); -} - -/** - * Converts the given string to a `Version`. - * - * Params: - * str = the string to convert. Should have the format `XX.YY(.ZZ)`. Needs to - * be `\0` terminated. - * - * Returns: the converted `Version`. - */ -@trusted -Version toVersion(const char* str) @nogc -{ - import core.stdc.stdio : sscanf; - - if (!str) - return Version(); - - Version version_; - - with (version_) - str.sscanf("%d.%d.%d", &major, &minor, &build); - - return version_; -} From 0c4ed265931867d50abe3401365e5aff2f3651b8 Mon Sep 17 00:00:00 2001 From: Iain Buclaw Date: Sun, 11 Feb 2024 22:53:29 +0100 Subject: [PATCH 3/5] osx: Build build tools with -platform_version --- ci/run.sh | 7 ++++++- compiler/src/build.d | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ci/run.sh b/ci/run.sh index d130df03ae18..e8d85431d29c 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -58,10 +58,15 @@ clone() { # build dmd (incl. building and running the unittests), druntime, phobos build() { + BUILD_DFLAGS= if [ "$OS_NAME" != "windows" ]; then source ~/dlang/*/activate # activate host compiler, incl. setting `DMD` fi - $DMD compiler/src/build.d -ofgenerated/build + if [ "$OS_NAME" == "osx" ]; then + BUILD_DFLAGS="-L-platform_version -Lmacos -L${MACOSX_DEPLOYMENT_TARGET+10.9} -L0.0" + CI_DFLAGS="$CI_DFLAGS ${BUILD_DFLAGS}" + fi + $DMD compiler/src/build.d -ofgenerated/build $BUILD_DFLAGS generated/build -j$N MODEL=$MODEL HOST_DMD=$DMD DFLAGS="$CI_DFLAGS" BUILD=debug unittest generated/build -j$N MODEL=$MODEL HOST_DMD=$DMD DFLAGS="$CI_DFLAGS" ENABLE_RELEASE=1 dmd make -j$N -C druntime MODEL=$MODEL diff --git a/compiler/src/build.d b/compiler/src/build.d index 8e9ccd4dfa85..fe708cbc2ca2 100755 --- a/compiler/src/build.d +++ b/compiler/src/build.d @@ -1002,6 +1002,9 @@ alias toolchainInfo = makeRule!((builder, rule) => builder app.put("==== Toolchain Information ====\n"); + version (OSX) + show("OS", ["sw_vers"]); + version (Windows) show("SYSTEM", ["systeminfo"]); else From 765ae6cb25c218921a5220fcef11243a23687f8f Mon Sep 17 00:00:00 2001 From: Iain Buclaw Date: Thu, 15 Feb 2024 01:59:21 +0100 Subject: [PATCH 4/5] Update ci/run.sh --- ci/run.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/run.sh b/ci/run.sh index e8d85431d29c..c30c77fdffaa 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -58,13 +58,13 @@ clone() { # build dmd (incl. building and running the unittests), druntime, phobos build() { - BUILD_DFLAGS= if [ "$OS_NAME" != "windows" ]; then source ~/dlang/*/activate # activate host compiler, incl. setting `DMD` fi + BUILD_DFLAGS= if [ "$OS_NAME" == "osx" ]; then - BUILD_DFLAGS="-L-platform_version -Lmacos -L${MACOSX_DEPLOYMENT_TARGET+10.9} -L0.0" - CI_DFLAGS="$CI_DFLAGS ${BUILD_DFLAGS}" + BUILD_DFLAGS="-L-ld_classic" + CI_DFLAGS="$CI_DFLAGS $BUILD_DFLAGS" fi $DMD compiler/src/build.d -ofgenerated/build $BUILD_DFLAGS generated/build -j$N MODEL=$MODEL HOST_DMD=$DMD DFLAGS="$CI_DFLAGS" BUILD=debug unittest From 9a12c7dfdaa9beb50750f9193827de8b86ca3519 Mon Sep 17 00:00:00 2001 From: Iain Buclaw Date: Sun, 18 Feb 2024 23:04:56 +0100 Subject: [PATCH 5/5] Update compiler/src/dmd/backend/machobj.d --- compiler/src/dmd/backend/machobj.d | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/src/dmd/backend/machobj.d b/compiler/src/dmd/backend/machobj.d index 305c91991ff1..8308adccad85 100644 --- a/compiler/src/dmd/backend/machobj.d +++ b/compiler/src/dmd/backend/machobj.d @@ -1426,6 +1426,7 @@ void MachObj_term(const(char)* objfilename) sym32.n_sect = sym.n_sect; fobjbuf.write(&sym32, sym32.sizeof); } + dysymtab_cmd.nundefsym++; symtab_cmd.nsyms++; } foffset += symtab_cmd.nsyms * (I64 ? nlist_64.sizeof : nlist.sizeof);