From 5fb47fbdaa3debb8d62cfabb8c34ec8a4c715142 Mon Sep 17 00:00:00 2001 From: Xinhao Yuan Date: Tue, 5 Oct 2021 23:58:00 -0400 Subject: [PATCH 1/2] Refactor the inline Python commands into a Python binary. This is needed when `python3` does not exist in the runtime environment. --- fuzzing/engines/BUILD | 10 ++++++++++ fuzzing/engines/honggfuzz_launcher.sh | 2 +- fuzzing/engines/jazzer_launcher.sh | 2 +- fuzzing/engines/libfuzzer_launcher.sh | 2 +- fuzzing/engines/replay_launcher.sh | 2 +- fuzzing/private/engine.bzl | 9 ++++++--- fuzzing/private/java_utils.bzl | 22 ++++++++++++++++------ fuzzing/tools/BUILD | 7 +++++++ fuzzing/tools/realpath.py | 27 +++++++++++++++++++++++++++ 9 files changed, 70 insertions(+), 13 deletions(-) create mode 100644 fuzzing/tools/realpath.py diff --git a/fuzzing/engines/BUILD b/fuzzing/engines/BUILD index ab38bc44..9d368d5e 100644 --- a/fuzzing/engines/BUILD +++ b/fuzzing/engines/BUILD @@ -23,6 +23,9 @@ cc_fuzzing_engine( name = "libfuzzer", display_name = "libFuzzer", launcher = "libfuzzer_launcher.sh", + launcher_data = { + "//fuzzing/tools:realpath": "REALPATH_PATH", + }, library = ":libfuzzer_stub", visibility = ["//visibility:public"], ) @@ -46,6 +49,7 @@ cc_fuzzing_engine( launcher = "honggfuzz_launcher.sh", launcher_data = { "@honggfuzz//:honggfuzz": "HONGGFUZZ_PATH", + "//fuzzing/tools:realpath": "REALPATH_PATH", }, library = "@honggfuzz//:honggfuzz_engine", visibility = ["//visibility:public"], @@ -58,6 +62,9 @@ cc_fuzzing_engine( name = "replay", display_name = "Replay", launcher = "replay_launcher.sh", + launcher_data = { + "//fuzzing/tools:realpath": "REALPATH_PATH", + }, library = "//fuzzing/replay:replay_main", visibility = ["//visibility:public"], ) @@ -69,6 +76,9 @@ java_fuzzing_engine( name = "jazzer", display_name = "Jazzer", launcher = "jazzer_launcher.sh", + launcher_data = { + "//fuzzing/tools:realpath": "REALPATH_PATH", + }, library = "@jazzer//agent:jazzer_api_compile_only", visibility = ["//visibility:public"], ) diff --git a/fuzzing/engines/honggfuzz_launcher.sh b/fuzzing/engines/honggfuzz_launcher.sh index 9c0bf621..25faf062 100755 --- a/fuzzing/engines/honggfuzz_launcher.sh +++ b/fuzzing/engines/honggfuzz_launcher.sh @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -command_line="$(python3 -c 'import os, sys; print(os.path.realpath(sys.argv[1]))' ${HONGGFUZZ_PATH})" +command_line="$("${REALPATH_PATH}" ${HONGGFUZZ_PATH})" command_line+=("--workspace=${FUZZER_OUTPUT_ROOT}") if [[ -n "${FUZZER_SEED_CORPUS_DIR}" ]]; then diff --git a/fuzzing/engines/jazzer_launcher.sh b/fuzzing/engines/jazzer_launcher.sh index c1601e54..81821f3f 100755 --- a/fuzzing/engines/jazzer_launcher.sh +++ b/fuzzing/engines/jazzer_launcher.sh @@ -16,7 +16,7 @@ # engine. The launch configuration is supplied by the launcher script through # environment variables. -command_line=("$(python3 -c 'import os, sys; print(os.path.realpath(sys.argv[1]))' ${FUZZER_BINARY})") +command_line=("$("${REALPATH_PATH}" ${FUZZER_BINARY})") # libFuzzer flags (compatible with Jazzer). diff --git a/fuzzing/engines/libfuzzer_launcher.sh b/fuzzing/engines/libfuzzer_launcher.sh index 6346db8a..7a59e049 100755 --- a/fuzzing/engines/libfuzzer_launcher.sh +++ b/fuzzing/engines/libfuzzer_launcher.sh @@ -16,7 +16,7 @@ # libFuzzer engine. The launch configuration is supplied by the launcher # script through environment variables. -command_line=("$(python3 -c 'import os, sys; print(os.path.realpath(sys.argv[1]))' ${FUZZER_BINARY})") +command_line=("$("${REALPATH_PATH}" ${FUZZER_BINARY})") # libFuzzer flags. diff --git a/fuzzing/engines/replay_launcher.sh b/fuzzing/engines/replay_launcher.sh index 4ebf2936..570b42ba 100755 --- a/fuzzing/engines/replay_launcher.sh +++ b/fuzzing/engines/replay_launcher.sh @@ -16,7 +16,7 @@ if (( ! FUZZER_IS_REGRESSION )); then echo "NOTE: Non-regression mode is not supported by the replay engine." fi -command_line=("$(python3 -c 'import os, sys; print(os.path.realpath(sys.argv[1]))' ${FUZZER_BINARY})") +command_line=("$("${REALPATH_PATH}" ${FUZZER_BINARY})") if [[ -n "${FUZZER_SEED_CORPUS_DIR}" ]]; then command_line+=("${FUZZER_SEED_CORPUS_DIR}") fi diff --git a/fuzzing/private/engine.bzl b/fuzzing/private/engine.bzl index 9d8656d1..39bab2c5 100644 --- a/fuzzing/private/engine.bzl +++ b/fuzzing/private/engine.bzl @@ -37,9 +37,12 @@ def _make_fuzzing_engine_info(ctx): if data_env_var: if data_env_var in env_vars: fail("Multiple data dependencies map to variable '%s'." % data_env_var) - if len(data_files) != 1: - fail("Data dependency for variable '%s' doesn't map to exactly one file." % data_env_var) - env_vars[data_env_var] = data_files[0] + if len(data_files) == 1: + env_vars[data_env_var] = data_files[0] + elif data_label.files_to_run.executable: + env_vars[data_env_var] = data_label.files_to_run.executable + else: + fail("Data dependency for variable '%s' is not a single file or executable." % data_env_var) launcher_runfiles = launcher_runfiles.merge(ctx.runfiles(files = data_files)) launcher_runfiles = launcher_runfiles.merge(data_label[DefaultInfo].default_runfiles) diff --git a/fuzzing/private/java_utils.bzl b/fuzzing/private/java_utils.bzl index a0a03aca..e63b1522 100644 --- a/fuzzing/private/java_utils.bzl +++ b/fuzzing/private/java_utils.bzl @@ -106,18 +106,18 @@ source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/ # Export the env variables required for subprocesses to find their runfiles. runfiles_export_envvars +""" + script_format_part = """ # Determine the path to load libjvm.so from, either relative to the location of # the java binary or to $JAVA_HOME, if set. On OSS-Fuzz, the path is provided in # JVM_LD_LIBRARY_PATH. -JAVA_BIN=$(python3 -c 'import os, sys; print(os.path.realpath(sys.argv[1]))' "$(which java)") -JAVA_HOME=${JAVA_HOME:-${JAVA_BIN%/bin/java}} +JAVA_BIN=$("$(rlocation {realpath})" "$(which java)") +JAVA_HOME=${{JAVA_HOME:-${{JAVA_BIN%/bin/java}}}} # The location of libjvm.so relative to the JDK differs between JDK <= 8 and 9+. -JVM_LD_LIBRARY_PATH=${JVM_LD_LIBRARY_PATH:-"$JAVA_HOME/lib/server:$JAVA_HOME/lib/amd64/server"} -export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$JVM_LD_LIBRARY_PATH -""" +JVM_LD_LIBRARY_PATH=${{JVM_LD_LIBRARY_PATH:-"$JAVA_HOME/lib/server:$JAVA_HOME/lib/amd64/server"}} +export LD_LIBRARY_PATH=${{LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}}$JVM_LD_LIBRARY_PATH - script_format_part = """ source "$(rlocation {sanitizer_options})" exec "$(rlocation {driver})" \ --agent_path="$(rlocation {agent})" \ @@ -136,6 +136,7 @@ exec "$(rlocation {driver})" \ deploy_jar = runfile_path(ctx, ctx.file.target_deploy_jar), driver = runfile_path(ctx, driver), native_dirs = ":".join(native_dirs), + realpath = runfile_path(ctx, ctx.executable._realpath), sanitizer_options = runfile_path(ctx, ctx.file.sanitizer_options), ) ctx.actions.write(script, script_content, is_executable = True) @@ -232,6 +233,9 @@ def _jazzer_fuzz_binary_impl(ctx): runfiles = runfiles.merge(ctx.runfiles([ctx.file.sanitizer_options])) + runfiles = runfiles.merge(ctx.runfiles([ctx.executable._realpath])) + runfiles = runfiles.merge(ctx.attr._realpath[DefaultInfo].default_runfiles) + script = _jazzer_fuzz_binary_script(ctx, native_libs, driver) return [DefaultInfo(executable = script, runfiles = runfiles)] @@ -283,6 +287,12 @@ Rule that creates a binary that invokes Jazzer on the specified target. allow_single_file = [".jar"], mandatory = True, ), + "_realpath": attr.label( + doc = "The realpath util needed by the binary script.", + default = "//fuzzing/tools:realpath", + executable = True, + cfg = "target", + ), "_allowlist_function_transition": attr.label( default = "@bazel_tools//tools/allowlists/function_transition_allowlist", ), diff --git a/fuzzing/tools/BUILD b/fuzzing/tools/BUILD index 21691a6f..59ad6d2d 100644 --- a/fuzzing/tools/BUILD +++ b/fuzzing/tools/BUILD @@ -36,6 +36,13 @@ py_binary( deps = [requirement("absl-py")], ) +py_binary( + name = "realpath", + srcs = ["realpath.py"], + python_version = "PY3", + visibility = ["//visibility:public"], +) + py_binary( name = "validate_dict", srcs = ["validate_dict.py"], diff --git a/fuzzing/tools/realpath.py b/fuzzing/tools/realpath.py new file mode 100644 index 00000000..cfd28352 --- /dev/null +++ b/fuzzing/tools/realpath.py @@ -0,0 +1,27 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +""" +Gets the realpath of a input path. + +This is a portable replacement of `readlink -f`/`realpath`. +""" + +import os, sys + +if len(sys.argv) < 2: + print("Need an argument for the input path.", file=sys.stderr) + sys.exit(1) +print(os.path.realpath(sys.argv[1])) From 788de08060b1fcdfdca24be550b19b809bcc6b56 Mon Sep 17 00:00:00 2001 From: Xinhao Yuan Date: Wed, 6 Oct 2021 11:16:06 -0400 Subject: [PATCH 2/2] Keep the python3 usage in the Jazzer binary script, since packaging py_binary to OSS-fuzz seems not working. --- fuzzing/private/java_utils.bzl | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/fuzzing/private/java_utils.bzl b/fuzzing/private/java_utils.bzl index e63b1522..a0a03aca 100644 --- a/fuzzing/private/java_utils.bzl +++ b/fuzzing/private/java_utils.bzl @@ -106,18 +106,18 @@ source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/ # Export the env variables required for subprocesses to find their runfiles. runfiles_export_envvars -""" - script_format_part = """ # Determine the path to load libjvm.so from, either relative to the location of # the java binary or to $JAVA_HOME, if set. On OSS-Fuzz, the path is provided in # JVM_LD_LIBRARY_PATH. -JAVA_BIN=$("$(rlocation {realpath})" "$(which java)") -JAVA_HOME=${{JAVA_HOME:-${{JAVA_BIN%/bin/java}}}} +JAVA_BIN=$(python3 -c 'import os, sys; print(os.path.realpath(sys.argv[1]))' "$(which java)") +JAVA_HOME=${JAVA_HOME:-${JAVA_BIN%/bin/java}} # The location of libjvm.so relative to the JDK differs between JDK <= 8 and 9+. -JVM_LD_LIBRARY_PATH=${{JVM_LD_LIBRARY_PATH:-"$JAVA_HOME/lib/server:$JAVA_HOME/lib/amd64/server"}} -export LD_LIBRARY_PATH=${{LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}}$JVM_LD_LIBRARY_PATH +JVM_LD_LIBRARY_PATH=${JVM_LD_LIBRARY_PATH:-"$JAVA_HOME/lib/server:$JAVA_HOME/lib/amd64/server"} +export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$JVM_LD_LIBRARY_PATH +""" + script_format_part = """ source "$(rlocation {sanitizer_options})" exec "$(rlocation {driver})" \ --agent_path="$(rlocation {agent})" \ @@ -136,7 +136,6 @@ exec "$(rlocation {driver})" \ deploy_jar = runfile_path(ctx, ctx.file.target_deploy_jar), driver = runfile_path(ctx, driver), native_dirs = ":".join(native_dirs), - realpath = runfile_path(ctx, ctx.executable._realpath), sanitizer_options = runfile_path(ctx, ctx.file.sanitizer_options), ) ctx.actions.write(script, script_content, is_executable = True) @@ -233,9 +232,6 @@ def _jazzer_fuzz_binary_impl(ctx): runfiles = runfiles.merge(ctx.runfiles([ctx.file.sanitizer_options])) - runfiles = runfiles.merge(ctx.runfiles([ctx.executable._realpath])) - runfiles = runfiles.merge(ctx.attr._realpath[DefaultInfo].default_runfiles) - script = _jazzer_fuzz_binary_script(ctx, native_libs, driver) return [DefaultInfo(executable = script, runfiles = runfiles)] @@ -287,12 +283,6 @@ Rule that creates a binary that invokes Jazzer on the specified target. allow_single_file = [".jar"], mandatory = True, ), - "_realpath": attr.label( - doc = "The realpath util needed by the binary script.", - default = "//fuzzing/tools:realpath", - executable = True, - cfg = "target", - ), "_allowlist_function_transition": attr.label( default = "@bazel_tools//tools/allowlists/function_transition_allowlist", ),